Adds peer connectivity check after deployment

CodexNetDeployer/CodexNodeStarter.cs

@@ -21,7 +21,7 @@ namespace CodexNetDeployer
             validatorsLeft = numberOfValidators;
         }
 
-        public RunningContainer? Start(int i)
+        public CodexNodeStartResult? Start(int i)
         {
             Console.Write($" - {i} = ");
             var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
@@ -60,7 +60,7 @@ namespace CodexNetDeployer
 
                         if (string.IsNullOrEmpty(bootstrapSpr)) bootstrapSpr = debugInfo.spr;
                         validatorsLeft--;
-                        return container;
+                        return new CodexNodeStartResult(container, codexAccess);
                     }
                 }
             }
@@ -102,4 +102,16 @@ namespace CodexNetDeployer
             return codexStart;
         }
     }
+
+    public class CodexNodeStartResult
+    {
+        public CodexNodeStartResult(RunningContainer container, CodexAccess access)
+        {
+            Container = container;
+            Access = access;
+        }
+
+        public RunningContainer Container { get; }
+        public CodexAccess Access { get; }
+    }
 }

CodexNetDeployer/Configuration.cs

@@ -57,6 +57,9 @@ namespace CodexNetDeployer
         [Uniform("teststype-podlabel", "ttpl", "TESTSTYPE-PODLABEL", false, "Each kubernetes pod will be created with a label 'teststype' with value 'continuous'. " +
             "set this option to override the label value.")]
         public string TestsTypePodLabel { get; set; } = "continuous-tests";
+
+        [Uniform("check-connect", "cc", "CHECKCONNECT", false, "If true, deployer check ensure peer-connectivity between all deployed nodes after deployment.")]
+        public bool CheckPeerConnection { get; set; } = false;
        
         public List<string> Validate()
         {

CodexNetDeployer/Deployer.cs

@@ -8,14 +8,14 @@ namespace CodexNetDeployer
     public class Deployer
     {
         private readonly Configuration config;
-        private readonly NullLog log;
         private readonly DefaultTimeSet timeset;
+        private readonly PeerConnectivityChecker peerConnectivityChecker;
 
         public Deployer(Configuration config)
         {
             this.config = config;
-            log = new NullLog();
             timeset = new DefaultTimeSet();
+            peerConnectivityChecker = new PeerConnectivityChecker();
         }
 
         public CodexDeployment Deploy()
@@ -45,16 +45,18 @@ namespace CodexNetDeployer
 
             // Each node must have its own IP, so it needs it own pod. Start them 1 at a time.
             var codexStarter = new CodexNodeStarter(config, lifecycle, gethResults, config.NumberOfValidators!.Value);
-            var codexContainers = new List<RunningContainer>();
+            var startResults = new List<CodexNodeStartResult>();
             for (var i = 0; i < config.NumberOfCodexNodes; i++)
             {
-                var container = codexStarter.Start(i);
+                var result = codexStarter.Start(i);
-                if (container != null) codexContainers.Add(container);
+                if (result != null) startResults.Add(result);
             }
 
-            var (prometheusContainer, grafanaStartInfo) = StartMetricsService(lifecycle, setup, codexContainers);
+            var (prometheusContainer, grafanaStartInfo) = StartMetricsService(lifecycle, setup, startResults.Select(r => r.Container));
 
-            return new CodexDeployment(gethResults, codexContainers.ToArray(), prometheusContainer, grafanaStartInfo, CreateMetadata());
+            CheckPeerConnectivity(startResults);
+
+            return new CodexDeployment(gethResults, startResults.Select(r => r.Container).ToArray(), prometheusContainer, grafanaStartInfo, CreateMetadata());
         }
 
         private TestLifecycle CreateTestLifecycle()
@@ -71,10 +73,10 @@ namespace CodexNetDeployer
                 k8sNamespacePrefix: config.KubeNamespace
             );
 
-            return new TestLifecycle(log, lifecycleConfig, timeset, config.TestsTypePodLabel, string.Empty);
+            return new TestLifecycle(new NullLog(), lifecycleConfig, timeset, config.TestsTypePodLabel, string.Empty);
         }
 
-        private (RunningContainer?, GrafanaStartInfo?) StartMetricsService(TestLifecycle lifecycle, CodexSetup setup, List<RunningContainer> codexContainers)
+        private (RunningContainer?, GrafanaStartInfo?) StartMetricsService(TestLifecycle lifecycle, CodexSetup setup, IEnumerable<RunningContainer> codexContainers)
         {
             if (setup.MetricsMode == DistTestCore.Metrics.MetricsMode.None) return (null, null);
 
@@ -95,6 +97,15 @@ namespace CodexNetDeployer
             return kubeConfigFile;
         }
 
+        private void CheckPeerConnectivity(List<CodexNodeStartResult> codexContainers)
+        {
+            if (!config.CheckPeerConnection) return;
+
+            Log("Starting peer-connectivity check for deployed nodes...");
+            peerConnectivityChecker.CheckConnectivity(codexContainers);
+            Log("Check passed.");
+        }
+
         private DeploymentMetadata CreateMetadata()
         {
             return new DeploymentMetadata(

CodexNetDeployer/PeerConnectivityChecker.cs

@@ -0,0 +1,34 @@
+using DistTestCore.Helpers;
+using Logging;
+
+namespace CodexNetDeployer
+{
+    public class PeerConnectivityChecker
+    {
+        public void CheckConnectivity(List<CodexNodeStartResult> startResults)
+        {
+            var checker = new PeerConnectionTestHelpers(new ConsoleLogger());
+
+            var access = startResults.Select(r => r.Access);
+
+            checker.AssertFullyConnected(access);
+        }
+    }
+
+    public class ConsoleLogger : BaseLog
+    {
+        public ConsoleLogger() : base(false)
+        {
+        }
+
+        protected override string GetFullName()
+        {
+            return "CONSOLE";
+        }
+
+        public override void Log(string message)
+        {
+            Console.WriteLine(message);
+        }
+    }
+}

DistTestCore/Codex/CodexAccess.cs

@@ -88,6 +88,11 @@ namespace DistTestCore.Codex
             return Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}");
         }
 
+        public string GetName()
+        {
+            return Container.Name;
+        }
+
         private Http Http()
         {
             CheckContainerCrashed();

DistTestCore/Helpers/FullConnectivityHelper.cs

@@ -24,12 +24,12 @@ namespace DistTestCore.Helpers
             this.implementation = implementation;
         }
 
-        public void AssertFullyConnected(IEnumerable<IOnlineCodexNode> nodes)
+        public void AssertFullyConnected(IEnumerable<CodexAccess> nodes)
         {
             AssertFullyConnected(nodes.ToArray());
         }
 
-        private void AssertFullyConnected(IOnlineCodexNode[] nodes)
+        private void AssertFullyConnected(CodexAccess[] nodes)
         {
             Log($"Asserting '{implementation.Description()}' for nodes: '{string.Join(",", nodes.Select(n => n.GetName()))}'...");
             var entries = CreateEntries(nodes);
@@ -85,7 +85,7 @@ namespace DistTestCore.Helpers
             Log($"Connections successful:{Nl}{string.Join(Nl, pairDetails)}");
         }
 
-        private Entry[] CreateEntries(IOnlineCodexNode[] nodes)
+        private Entry[] CreateEntries(CodexAccess[] nodes)
         {
             var entries = nodes.Select(n => new Entry(n)).ToArray();
 
@@ -125,13 +125,13 @@ namespace DistTestCore.Helpers
 
         public class Entry
         {
-            public Entry(IOnlineCodexNode node)
+            public Entry(CodexAccess node)
             {
                 Node = node;
                 Response = node.GetDebugInfo();
             }
 
-            public IOnlineCodexNode Node { get; }
+            public CodexAccess Node { get; }
             public CodexDebugResponse Response { get; }
 
             public override string ToString()

DistTestCore/Helpers/PeerConnectionTestHelpers.cs

@@ -14,6 +14,11 @@ namespace DistTestCore.Helpers
         }
 
         public void AssertFullyConnected(IEnumerable<IOnlineCodexNode> nodes)
+        {
+            AssertFullyConnected(nodes.Select(n => ((OnlineCodexNode)n).CodexAccess));
+        }
+
+        public void AssertFullyConnected(IEnumerable<CodexAccess> nodes)
         {
             helper.AssertFullyConnected(nodes);
         }
@@ -58,9 +63,8 @@ namespace DistTestCore.Helpers
             var peer = allEntries.SingleOrDefault(e => e.Response.table.localNode.peerId == node.peerId);
             if (peer == null) return $"peerId: {node.peerId} is not known.";
 
-            var n = (OnlineCodexNode)peer.Node;
+            var ip = peer.Node.Container.Pod.PodInfo.Ip;
-            var ip = n.CodexAccess.Container.Pod.PodInfo.Ip;
+            var discPort = peer.Node.Container.Recipe.GetPortByTag(CodexContainerRecipe.DiscoveryPortTag);
-            var discPort = n.CodexAccess.Container.Recipe.GetPortByTag(CodexContainerRecipe.DiscoveryPortTag);
             return $"{ip}:{discPort.Number}";
         }
     }

DistTestCore/Helpers/PeerDownloadTestHelpers.cs

@@ -1,4 +1,5 @@
-using Logging;
+using DistTestCore.Codex;
+using Logging;
 using static DistTestCore.Helpers.FullConnectivityHelper;
 
 namespace DistTestCore.Helpers
@@ -17,6 +18,11 @@ namespace DistTestCore.Helpers
         }
 
         public void AssertFullDownloadInterconnectivity(IEnumerable<IOnlineCodexNode> nodes, ByteSize testFileSize)
+        {
+            AssertFullDownloadInterconnectivity(nodes.Select(n => ((OnlineCodexNode)n).CodexAccess), testFileSize);
+        }
+
+        public void AssertFullDownloadInterconnectivity(IEnumerable<CodexAccess> nodes, ByteSize testFileSize)
         {
             this.testFileSize = testFileSize;
             helper.AssertFullyConnected(nodes);
@@ -37,11 +43,16 @@ namespace DistTestCore.Helpers
             fileManager.PushFileSet();
             var expectedFile = GenerateTestFile(from.Node, to.Node);
 
-            var contentId = from.Node.UploadFile(expectedFile);
+            using var uploadStream = File.OpenRead(expectedFile.Filename);
+            var contentId = from.Node.UploadFile(uploadStream);
 
             try
             {
-                var downloadedFile = to.Node.DownloadContent(contentId, expectedFile.Label + "_downloaded");
+                var downloadedFile = fileManager.CreateEmptyTestFile(expectedFile.Label + "_downloaded");
+                using var downloadStream = File.OpenWrite(downloadedFile.Filename);
+                using var stream = to.Node.DownloadFile(contentId);
+                stream.CopyTo(downloadStream);
+
                 expectedFile.AssertIsEqual(downloadedFile);
                 return PeerConnectionState.Connection;
             }
@@ -59,7 +70,7 @@ namespace DistTestCore.Helpers
             // Should an exception occur during upload, then this try is inconclusive and we try again next loop.
         }
 
-        private TestFile GenerateTestFile(IOnlineCodexNode uploader, IOnlineCodexNode downloader)
+        private TestFile GenerateTestFile(CodexAccess uploader, CodexAccess downloader)
         {
             var up = uploader.GetName().Replace("<", "").Replace(">", "");
             var down = downloader.GetName().Replace("<", "").Replace(">", "");