Cleanup of full connectivity helpers

DistTestCore/Helpers/FullConnectivityHelper.cs

@@ -0,0 +1,211 @@
+using DistTestCore.Codex;
+using NUnit.Framework;
+using Utils;
+
+namespace DistTestCore.Helpers
+{
+    public interface IFullConnectivityImplementation
+    {
+        string Description();
+        string ValidateEntry(FullConnectivityHelper.Entry entry, FullConnectivityHelper.Entry[] allEntries);
+        FullConnectivityHelper.PeerConnectionState Check(FullConnectivityHelper.Entry from, FullConnectivityHelper.Entry to);
+    }
+
+    public class FullConnectivityHelper
+    {
+        private static string Nl = Environment.NewLine;
+        private readonly DistTest test;
+        private readonly IFullConnectivityImplementation implementation;
+
+        public FullConnectivityHelper(DistTest test, IFullConnectivityImplementation implementation)
+        {
+            this.test = test;
+            this.implementation = implementation;
+        }
+
+        public void AssertFullyConnected(IEnumerable<IOnlineCodexNode> nodes)
+        {
+            AssertFullyConnected(nodes.ToArray());
+        }
+
+        private void AssertFullyConnected(IOnlineCodexNode[] nodes)
+        {
+            test.Log($"Asserting '{implementation.Description()}' for nodes: '{string.Join(",", nodes.Select(n => n.GetName()))}'...");
+            var entries = CreateEntries(nodes);
+            var pairs = CreatePairs(entries);
+
+            RetryWhilePairs(pairs, () =>
+            {
+                CheckAndRemoveSuccessful(pairs);
+            });
+
+            if (pairs.Any())
+            {
+                var pairDetails = string.Join(Nl, pairs.SelectMany(p => p.GetResultMessages()));
+
+                test.Log($"Connections failed:{Nl}{pairDetails}");
+
+                Assert.Fail(string.Join(Nl, pairs.SelectMany(p => p.GetResultMessages())));
+            }
+            else
+            {
+                test.Log($"'{implementation.Description()}' = Success! for nodes: {string.Join(",", nodes.Select(n => n.GetName()))}");
+            }
+        }
+
+        private static void RetryWhilePairs(List<Pair> pairs, Action action)
+        {
+            var timeout = DateTime.UtcNow + TimeSpan.FromMinutes(5);
+            while (pairs.Any(p => p.Inconclusive) && timeout > DateTime.UtcNow)
+            {
+                action();
+
+                Time.Sleep(TimeSpan.FromSeconds(2));
+            }
+        }
+
+        private void CheckAndRemoveSuccessful(List<Pair> pairs)
+        {
+            // For large sets, don't try and do all of them at once.
+            var selectedPair = pairs.Take(20).ToArray();
+            var pairDetails = new List<string>();
+
+            foreach (var pair in selectedPair)
+            {
+                test.ScopedTestFiles(pair.Check);
+
+                if (pair.Success)
+                {
+                    pairDetails.AddRange(pair.GetResultMessages());
+                    pairs.Remove(pair);
+                }
+            }
+
+            test.Log($"Connections successful:{Nl}{string.Join(Nl, pairDetails)}");
+        }
+
+        private Entry[] CreateEntries(IOnlineCodexNode[] nodes)
+        {
+            var entries = nodes.Select(n => new Entry(n)).ToArray();
+
+            var errors = entries
+                            .Select(e => implementation.ValidateEntry(e, entries))
+                            .Where(s => !string.IsNullOrEmpty(s))
+                            .ToArray();
+
+            if (errors.Any())
+            {
+                Assert.Fail("Some node entries failed to validate: " + string.Join(Nl, errors));
+            }
+
+            return entries;
+        }
+
+        private List<Pair> CreatePairs(Entry[] entries)
+        {
+            return CreatePairsIterator(entries).ToList();
+        }
+
+        private IEnumerable<Pair> CreatePairsIterator(Entry[] entries)
+        {
+            for (var x = 0; x < entries.Length; x++)
+            {
+                for (var y = x + 1; y < entries.Length; y++)
+                {
+                    yield return new Pair(implementation, entries[x], entries[y]);
+                }
+            }
+        }
+
+        public class Entry
+        {
+            public Entry(IOnlineCodexNode node)
+            {
+                Node = node;
+                Response = node.GetDebugInfo();
+            }
+
+            public IOnlineCodexNode Node { get; }
+            public CodexDebugResponse Response { get; }
+
+            public override string ToString()
+            {
+                if (Response == null || string.IsNullOrEmpty(Response.id)) return "UNKNOWN";
+                return Response.id;
+            }
+        }
+
+        public enum PeerConnectionState
+        {
+            Unknown,
+            Connection,
+            NoConnection,
+        }
+
+        public class Pair
+        {
+            private TimeSpan aToBTime = TimeSpan.FromSeconds(0);
+            private TimeSpan bToATime = TimeSpan.FromSeconds(0);
+            private readonly IFullConnectivityImplementation implementation;
+
+            public Pair(IFullConnectivityImplementation implementation, Entry a, Entry b)
+            {
+                this.implementation = implementation;
+                A = a;
+                B = b;
+            }
+
+            public Entry A { get; }
+            public Entry B { get; }
+            public PeerConnectionState AKnowsB { get; private set; }
+            public PeerConnectionState BKnowsA { get; private set; }
+            public bool Success { get { return AKnowsB == PeerConnectionState.Connection && BKnowsA == PeerConnectionState.Connection; } }
+            public bool Inconclusive { get { return AKnowsB == PeerConnectionState.Unknown || BKnowsA == PeerConnectionState.Unknown; } }
+
+            public void Check()
+            {
+                aToBTime = Measure(() => AKnowsB = Check(A, B));
+                bToATime = Measure(() => BKnowsA = Check(B, A));
+            }
+
+            public override string ToString()
+            {
+                return $"[{string.Join(",", GetResultMessages())}]";
+            }
+
+            public string[] GetResultMessages()
+            {
+                var aName = A.ToString();
+                var bName = B.ToString();
+
+                return new[]
+                {
+                    $"[{aName} --> {bName}] = {AKnowsB} ({aToBTime.TotalSeconds} seconds)",
+                    $"[{aName} <-- {bName}] = {BKnowsA} ({bToATime.TotalSeconds} seconds)"
+                };
+            }
+
+            private static TimeSpan Measure(Action action)
+            {
+                var start = DateTime.UtcNow;
+                action();
+                return DateTime.UtcNow - start;
+            }
+
+            private PeerConnectionState Check(Entry from, Entry to)
+            {
+                Thread.Sleep(10);
+
+                try
+                {
+                    return implementation.Check(from, to);
+                }
+                catch
+                {
+                    // Didn't get a conclusive answer. Try again later.
+                    return PeerConnectionState.Unknown;
+                }
+            }
+        }
+    }
+}

DistTestCore/Helpers/PeerConnectionTestHelpers.cs

@@ -1,155 +1,55 @@
 using DistTestCore.Codex;
-using NUnit.Framework;
+using static DistTestCore.Helpers.FullConnectivityHelper;
-using Utils;
 
 namespace DistTestCore.Helpers
 {
-    public class PeerConnectionTestHelpers
+    public class PeerConnectionTestHelpers : IFullConnectivityImplementation
     {
-        private static string Nl = Environment.NewLine;
+        private readonly FullConnectivityHelper helper;
-        private readonly Random random = new Random();
-        private readonly DistTest test;
 
         public PeerConnectionTestHelpers(DistTest test)
         {
-            this.test = test;
+            helper = new FullConnectivityHelper(test, this);
         }
 
         public void AssertFullyConnected(IEnumerable<IOnlineCodexNode> nodes)
         {
-            var n = nodes.ToArray();
+            helper.AssertFullyConnected(nodes);
-
-            AssertFullyConnected(n);
-
-            for (int i = 0; i < 5; i++)
-            {
-                Time.Sleep(TimeSpan.FromSeconds(30));
-                AssertFullyConnected(n);
-            }
         }
 
-        private void AssertFullyConnected(IOnlineCodexNode[] nodes)
+        public string Description()
         {
-            test.Log($"Asserting peers are fully-connected for nodes: '{string.Join(",", nodes.Select(n => n.GetName()))}'...");
+            return "Peer Discovery";
-            var entries = CreateEntries(nodes);
-            var pairs = CreatePairs(entries);
-
-            RetryWhilePairs(pairs, () =>
-            {
-                CheckAndRemoveSuccessful(pairs);
-            });
-
-            if (pairs.Any())
-            {
-                var pairDetails = string.Join(Nl, pairs.SelectMany(p => p.GetResultMessages()));
-
-                test.Log($"Connections failed:{Nl}{pairDetails}");
-
-                Assert.Fail(string.Join(Nl, pairs.SelectMany(p => p.GetResultMessages())));
-            }
-            else
-            {
-                test.Log($"Success! Peers are fully-connected: {string.Join(",", nodes.Select(n => n.GetName()))}");
-            }
         }
 
-        private static void RetryWhilePairs(List<Pair> pairs, Action action)
+        public string ValidateEntry(Entry entry, Entry[] allEntries)
         {
-            var timeout = DateTime.UtcNow + TimeSpan.FromMinutes(5);
+            var result = string.Empty;
-            while (pairs.Any(p => p.Inconclusive) && timeout > DateTime.UtcNow)
+            foreach (var peer in entry.Response.table.nodes)
-            {
-                action();
-
-                Time.Sleep(TimeSpan.FromSeconds(2));
-            }
-        }
-
-        private void CheckAndRemoveSuccessful(List<Pair> pairs)
-        {
-            // For large sets, don't try and do all of them at once.
-            var checkTasks = pairs.Take(20).Select(p => Task.Run(() =>
-            {
-                ApplyRandomDelay();
-                p.Check();
-            })).ToArray();
-
-            Task.WaitAll(checkTasks);
-
-            var pairDetails = new List<string>();
-            foreach (var pair in pairs.ToArray())
-            {
-                if (pair.Success)
-                {
-                    pairDetails.AddRange(pair.GetResultMessages());
-                    pairs.Remove(pair);
-                }
-            }
-            test.Log($"Connections successful:{Nl}{string.Join(Nl, pairDetails)}");
-        }
-
-        private static Entry[] CreateEntries(IOnlineCodexNode[] nodes)
-        {
-            var entries = nodes.Select(n => new Entry(n)).ToArray();
-            var incorrectDiscoveryEndpoints = entries.SelectMany(e => e.GetInCorrectDiscoveryEndpoints(entries)).ToArray();
-
-            if (incorrectDiscoveryEndpoints.Any())
-            {
-                Assert.Fail("Some nodes contain peer records with incorrect discovery ip/port information: " +
-                    string.Join(Nl, incorrectDiscoveryEndpoints));
-            }
-
-            return entries;
-        }
-
-        private static List<Pair> CreatePairs(Entry[] entries)
-        {
-            return CreatePairsIterator(entries).ToList();
-        }
-
-        private static IEnumerable<Pair> CreatePairsIterator(Entry[] entries)
-        {
-            for (var x = 0; x < entries.Length; x++)
-            {
-                for (var y = x + 1; y < entries.Length; y++)
-                {
-                    yield return new Pair(entries[x], entries[y]);
-                }
-            }
-        }
-
-        private void ApplyRandomDelay()
-        {
-            // Calling all the nodes all at the same time is not exactly nice.
-            Time.Sleep(TimeSpan.FromMicroseconds(random.Next(10, 1000)));
-        }
-
-        public class Entry
-        {
-            public Entry(IOnlineCodexNode node)
-            {
-                Node = node;
-                Response = node.GetDebugInfo();
-            }
-
-            public IOnlineCodexNode Node { get; }
-            public CodexDebugResponse Response { get; }
-
-            public IEnumerable<string> GetInCorrectDiscoveryEndpoints(Entry[] allEntries)
-            {
-                foreach (var peer in Response.table.nodes)
             {
                 var expected = GetExpectedDiscoveryEndpoint(allEntries, peer);
                 if (expected != peer.address)
                 {
-                        yield return $"Node:{Node.GetName()} has incorrect peer table entry. Was: '{peer.address}', expected: '{expected}'";
+                    result += $"Node:{entry.Node.GetName()} has incorrect peer table entry. Was: '{peer.address}', expected: '{expected}'. ";
                 }
             }
+            return result;
         }
 
-            public override string ToString()
+        public PeerConnectionState Check(Entry from, Entry to)
         {
-                if (Response == null || string.IsNullOrEmpty(Response.id)) return "UNKNOWN";
+            var peerId = to.Response.id;
-                return Response.id;
+
+            var response = from.Node.GetDebugPeer(peerId);
+            if (!response.IsPeerFound)
+            {
+                return PeerConnectionState.NoConnection;
+            }
+            if (!string.IsNullOrEmpty(response.peerId) && response.addresses.Any())
+            {
+                return PeerConnectionState.Connection;
+            }
+            return PeerConnectionState.Unknown;
         }
 
         private static string GetExpectedDiscoveryEndpoint(Entry[] allEntries, CodexDebugTableNodeResponse node)
@@ -163,89 +63,4 @@ namespace DistTestCore.Helpers
             return $"{ip}:{discPort.Number}";
         }
     }
-
-        public enum PeerConnectionState
-        {
-            Unknown,
-            Connection,
-            NoConnection,
-        }
-
-        public class Pair
-        {
-            private TimeSpan aToBTime = TimeSpan.FromSeconds(0);
-            private TimeSpan bToATime = TimeSpan.FromSeconds(0);
-
-            public Pair(Entry a, Entry b)
-            {
-                A = a;
-                B = b;
-            }
-
-            public Entry A { get; }
-            public Entry B { get; }
-            public PeerConnectionState AKnowsB { get; private set; }
-            public PeerConnectionState BKnowsA { get; private set; }
-            public bool Success { get { return AKnowsB == PeerConnectionState.Connection && BKnowsA == PeerConnectionState.Connection; } }
-            public bool Inconclusive { get { return AKnowsB == PeerConnectionState.Unknown || BKnowsA == PeerConnectionState.Unknown; } }
-
-            public void Check()
-            {
-                aToBTime = Measure(() => AKnowsB = Knows(A, B));
-                bToATime = Measure(() => BKnowsA = Knows(B, A));
-            }
-
-            public override string ToString()
-            {
-                return $"[{string.Join(",", GetResultMessages())}]";
-            }
-
-            public string[] GetResultMessages()
-            {
-                var aName = A.ToString();
-                var bName = B.ToString();
-
-                return new[]
-                {
-                    $"[{aName} --> {bName}] = {AKnowsB} ({aToBTime.TotalSeconds} seconds)",
-                    $"[{aName} <-- {bName}] = {BKnowsA} ({bToATime.TotalSeconds} seconds)"
-                };
-            }
-
-            private static TimeSpan Measure(Action action)
-            {
-                var start = DateTime.UtcNow;
-                action();
-                return DateTime.UtcNow - start;
-            }
-
-            private PeerConnectionState Knows(Entry a, Entry b)
-            {
-                lock (a)
-                {
-                    Thread.Sleep(10);
-                    var peerId = b.Response.id;
-
-                    try
-                    {
-                        var response = a.Node.GetDebugPeer(peerId);
-                        if (!response.IsPeerFound)
-                        {
-                            return PeerConnectionState.NoConnection;
-                        }
-                        if (!string.IsNullOrEmpty(response.peerId) && response.addresses.Any())
-                        {
-                            return PeerConnectionState.Connection;
-                        }
-                    }
-                    catch
-                    {
-                    }
-
-                    // Didn't get a conclusive answer. Try again later.
-                    return PeerConnectionState.Unknown;
-                }
-            }
-        }
-    }
 }

DistTestCore/Helpers/PeerDownloadTestHelpers.cs

@@ -1,74 +1,63 @@
-using DistTestCore.Codex;
+using static DistTestCore.Helpers.FullConnectivityHelper;
-using NUnit.Framework;
 
 namespace DistTestCore.Helpers
 {
-    public class PeerDownloadTestHelpers
+    public class PeerDownloadTestHelpers : IFullConnectivityImplementation
     {
+        private readonly FullConnectivityHelper helper;
         private readonly DistTest test;
+        private ByteSize testFileSize;
 
         public PeerDownloadTestHelpers(DistTest test)
         {
+            helper = new FullConnectivityHelper(test, this);
+            testFileSize = 1.MB();
             this.test = test;
         }
 
         public void AssertFullDownloadInterconnectivity(IEnumerable<IOnlineCodexNode> nodes, ByteSize testFileSize)
         {
-            test.Log($"Asserting full download interconnectivity for nodes: '{string.Join(",", nodes.Select(n => n.GetName()))}'...");
+            this.testFileSize = testFileSize;
-            var start = DateTime.UtcNow;
+            helper.AssertFullyConnected(nodes);
-
-            foreach (var node in nodes)
-            {
-                var uploader = node;
-                var downloaders = nodes.Where(n => n != uploader).ToArray();
-
-                test.ScopedTestFiles(() =>
-                {
-                    PerformTest(uploader, downloaders, testFileSize);
-                });
         }
 
-            test.Log($"Success! Full download interconnectivity for nodes: {string.Join(",", nodes.Select(n => n.GetName()))}");
+        public string Description()
-            var timeTaken = DateTime.UtcNow - start;
+        {
-
+            return "Download Connectivity";
-            AssertTimePerMB(timeTaken, nodes.Count(), testFileSize);
         }
 
-        private void AssertTimePerMB(TimeSpan timeTaken, int numberOfNodes, ByteSize size)
+        public string ValidateEntry(Entry entry, Entry[] allEntries)
         {
-            var numberOfDownloads = numberOfNodes * (numberOfNodes - 1);
+            return string.Empty;
-            var timePerDownload = timeTaken / numberOfDownloads;
-            float sizeInMB = size.ToMB();
-            var timePerMB = timePerDownload / sizeInMB;
-
-            test.Log($"Performed {numberOfDownloads} downloads of {size} in {timeTaken.TotalSeconds} seconds, for an average of {timePerMB.TotalSeconds} seconds per MB.");
-
-            Assert.That(timePerMB, Is.LessThan(CodexContainerRecipe.MaxDownloadTimePerMegabyte), "MaxDownloadTimePerMegabyte performance threshold breached.");
         }
 
-        private void PerformTest(IOnlineCodexNode uploader, IOnlineCodexNode[] downloaders, ByteSize testFileSize)
+        public PeerConnectionState Check(Entry from, Entry to)
         {
-            // Generate 1 test file per downloader.
+            var expectedFile = GenerateTestFile(from.Node, to.Node);
-            var files = downloaders.Select(d => GenerateTestFile(uploader, d, testFileSize)).ToArray();
 
-            // Upload all the test files to the uploader.
+            var contentId = from.Node.UploadFile(expectedFile);
-            var contentIds = files.Select(uploader.UploadFile).ToArray();
 
-            // Each downloader should retrieve its own test file.
+            try
-            for (var i = 0; i < downloaders.Length; i++)
             {
-                var expectedFile = files[i];
+                var downloadedFile = to.Node.DownloadContent(contentId, expectedFile.Label + "_downloaded");
-                var downloadedFile = downloaders[i].DownloadContent(contentIds[i], $"{expectedFile.Label}DOWNLOADED");
-
                 expectedFile.AssertIsEqual(downloadedFile);
+                return PeerConnectionState.Connection;
             }
+            catch
+            {
+                // Should an exception occur during the download or file-content assertion,
+                // We consider that as no-connection for the purpose of this test.
+                return PeerConnectionState.NoConnection;
             }
 
-        private TestFile GenerateTestFile(IOnlineCodexNode uploader, IOnlineCodexNode downloader, ByteSize testFileSize)
+            // Should an exception occur during upload, then this try is inconclusive and we try again next loop.
+        }
+
+        private TestFile GenerateTestFile(IOnlineCodexNode uploader, IOnlineCodexNode downloader)
         {
             var up = uploader.GetName().Replace("<", "").Replace(">", "");
             var down = downloader.GetName().Replace("<", "").Replace(">", "");
-            var label = $"FROM{up}TO{down}";
+            var label = $"~from:{up}-to:{down}~";
             return test.GenerateTestFile(testFileSize, label);
         }
     }