diff --git a/CodexNetDeployer/CodexNodeStarter.cs b/CodexNetDeployer/CodexNodeStarter.cs index 39bc1f9..79435c9 100644 --- a/CodexNetDeployer/CodexNodeStarter.cs +++ b/CodexNetDeployer/CodexNodeStarter.cs @@ -21,7 +21,7 @@ namespace CodexNetDeployer validatorsLeft = numberOfValidators; } - public RunningContainer? Start(int i) + public CodexNodeStartResult? Start(int i) { Console.Write($" - {i} = "); var workflow = lifecycle.WorkflowCreator.CreateWorkflow(); @@ -60,7 +60,7 @@ namespace CodexNetDeployer if (string.IsNullOrEmpty(bootstrapSpr)) bootstrapSpr = debugInfo.spr; validatorsLeft--; - return container; + return new CodexNodeStartResult(container, codexAccess); } } } @@ -102,4 +102,16 @@ namespace CodexNetDeployer return codexStart; } } + + public class CodexNodeStartResult + { + public CodexNodeStartResult(RunningContainer container, CodexAccess access) + { + Container = container; + Access = access; + } + + public RunningContainer Container { get; } + public CodexAccess Access { get; } + } } diff --git a/CodexNetDeployer/Configuration.cs b/CodexNetDeployer/Configuration.cs index 4df18be..7ad241a 100644 --- a/CodexNetDeployer/Configuration.cs +++ b/CodexNetDeployer/Configuration.cs @@ -57,6 +57,9 @@ namespace CodexNetDeployer [Uniform("teststype-podlabel", "ttpl", "TESTSTYPE-PODLABEL", false, "Each kubernetes pod will be created with a label 'teststype' with value 'continuous'. " + "set this option to override the label value.")] public string TestsTypePodLabel { get; set; } = "continuous-tests"; + + [Uniform("check-connect", "cc", "CHECKCONNECT", false, "If true, deployer check ensure peer-connectivity between all deployed nodes after deployment.")] + public bool CheckPeerConnection { get; set; } = false; public List Validate() { diff --git a/CodexNetDeployer/Deployer.cs b/CodexNetDeployer/Deployer.cs index 55ad4fe..f589813 100644 --- a/CodexNetDeployer/Deployer.cs +++ b/CodexNetDeployer/Deployer.cs @@ -8,14 +8,14 @@ namespace CodexNetDeployer public class Deployer { private readonly Configuration config; - private readonly NullLog log; private readonly DefaultTimeSet timeset; + private readonly PeerConnectivityChecker peerConnectivityChecker; public Deployer(Configuration config) { this.config = config; - log = new NullLog(); timeset = new DefaultTimeSet(); + peerConnectivityChecker = new PeerConnectivityChecker(); } public CodexDeployment Deploy() @@ -45,16 +45,18 @@ namespace CodexNetDeployer // Each node must have its own IP, so it needs it own pod. Start them 1 at a time. var codexStarter = new CodexNodeStarter(config, lifecycle, gethResults, config.NumberOfValidators!.Value); - var codexContainers = new List(); + var startResults = new List(); for (var i = 0; i < config.NumberOfCodexNodes; i++) { - var container = codexStarter.Start(i); - if (container != null) codexContainers.Add(container); + var result = codexStarter.Start(i); + if (result != null) startResults.Add(result); } - var (prometheusContainer, grafanaStartInfo) = StartMetricsService(lifecycle, setup, codexContainers); + var (prometheusContainer, grafanaStartInfo) = StartMetricsService(lifecycle, setup, startResults.Select(r => r.Container)); - return new CodexDeployment(gethResults, codexContainers.ToArray(), prometheusContainer, grafanaStartInfo, CreateMetadata()); + CheckPeerConnectivity(startResults); + + return new CodexDeployment(gethResults, startResults.Select(r => r.Container).ToArray(), prometheusContainer, grafanaStartInfo, CreateMetadata()); } private TestLifecycle CreateTestLifecycle() @@ -71,10 +73,10 @@ namespace CodexNetDeployer k8sNamespacePrefix: config.KubeNamespace ); - return new TestLifecycle(log, lifecycleConfig, timeset, config.TestsTypePodLabel, string.Empty); + return new TestLifecycle(new NullLog(), lifecycleConfig, timeset, config.TestsTypePodLabel, string.Empty); } - private (RunningContainer?, GrafanaStartInfo?) StartMetricsService(TestLifecycle lifecycle, CodexSetup setup, List codexContainers) + private (RunningContainer?, GrafanaStartInfo?) StartMetricsService(TestLifecycle lifecycle, CodexSetup setup, IEnumerable codexContainers) { if (setup.MetricsMode == DistTestCore.Metrics.MetricsMode.None) return (null, null); @@ -95,6 +97,15 @@ namespace CodexNetDeployer return kubeConfigFile; } + private void CheckPeerConnectivity(List codexContainers) + { + if (!config.CheckPeerConnection) return; + + Log("Starting peer-connectivity check for deployed nodes..."); + peerConnectivityChecker.CheckConnectivity(codexContainers); + Log("Check passed."); + } + private DeploymentMetadata CreateMetadata() { return new DeploymentMetadata( diff --git a/CodexNetDeployer/PeerConnectivityChecker.cs b/CodexNetDeployer/PeerConnectivityChecker.cs new file mode 100644 index 0000000..a3baa17 --- /dev/null +++ b/CodexNetDeployer/PeerConnectivityChecker.cs @@ -0,0 +1,34 @@ +using DistTestCore.Helpers; +using Logging; + +namespace CodexNetDeployer +{ + public class PeerConnectivityChecker + { + public void CheckConnectivity(List startResults) + { + var checker = new PeerConnectionTestHelpers(new ConsoleLogger()); + + var access = startResults.Select(r => r.Access); + + checker.AssertFullyConnected(access); + } + } + + public class ConsoleLogger : BaseLog + { + public ConsoleLogger() : base(false) + { + } + + protected override string GetFullName() + { + return "CONSOLE"; + } + + public override void Log(string message) + { + Console.WriteLine(message); + } + } +} diff --git a/DistTestCore/Codex/CodexAccess.cs b/DistTestCore/Codex/CodexAccess.cs index 076a335..57f7689 100644 --- a/DistTestCore/Codex/CodexAccess.cs +++ b/DistTestCore/Codex/CodexAccess.cs @@ -88,6 +88,11 @@ namespace DistTestCore.Codex return Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}"); } + public string GetName() + { + return Container.Name; + } + private Http Http() { CheckContainerCrashed(); diff --git a/DistTestCore/Helpers/FullConnectivityHelper.cs b/DistTestCore/Helpers/FullConnectivityHelper.cs index 0ac6f82..228beeb 100644 --- a/DistTestCore/Helpers/FullConnectivityHelper.cs +++ b/DistTestCore/Helpers/FullConnectivityHelper.cs @@ -24,12 +24,12 @@ namespace DistTestCore.Helpers this.implementation = implementation; } - public void AssertFullyConnected(IEnumerable nodes) + public void AssertFullyConnected(IEnumerable nodes) { AssertFullyConnected(nodes.ToArray()); } - private void AssertFullyConnected(IOnlineCodexNode[] nodes) + private void AssertFullyConnected(CodexAccess[] nodes) { Log($"Asserting '{implementation.Description()}' for nodes: '{string.Join(",", nodes.Select(n => n.GetName()))}'..."); var entries = CreateEntries(nodes); @@ -85,7 +85,7 @@ namespace DistTestCore.Helpers Log($"Connections successful:{Nl}{string.Join(Nl, pairDetails)}"); } - private Entry[] CreateEntries(IOnlineCodexNode[] nodes) + private Entry[] CreateEntries(CodexAccess[] nodes) { var entries = nodes.Select(n => new Entry(n)).ToArray(); @@ -125,13 +125,13 @@ namespace DistTestCore.Helpers public class Entry { - public Entry(IOnlineCodexNode node) + public Entry(CodexAccess node) { Node = node; Response = node.GetDebugInfo(); } - public IOnlineCodexNode Node { get; } + public CodexAccess Node { get; } public CodexDebugResponse Response { get; } public override string ToString() diff --git a/DistTestCore/Helpers/PeerConnectionTestHelpers.cs b/DistTestCore/Helpers/PeerConnectionTestHelpers.cs index 6b1a673..65bb4ce 100644 --- a/DistTestCore/Helpers/PeerConnectionTestHelpers.cs +++ b/DistTestCore/Helpers/PeerConnectionTestHelpers.cs @@ -14,6 +14,11 @@ namespace DistTestCore.Helpers } public void AssertFullyConnected(IEnumerable nodes) + { + AssertFullyConnected(nodes.Select(n => ((OnlineCodexNode)n).CodexAccess)); + } + + public void AssertFullyConnected(IEnumerable nodes) { helper.AssertFullyConnected(nodes); } @@ -58,9 +63,8 @@ namespace DistTestCore.Helpers var peer = allEntries.SingleOrDefault(e => e.Response.table.localNode.peerId == node.peerId); if (peer == null) return $"peerId: {node.peerId} is not known."; - var n = (OnlineCodexNode)peer.Node; - var ip = n.CodexAccess.Container.Pod.PodInfo.Ip; - var discPort = n.CodexAccess.Container.Recipe.GetPortByTag(CodexContainerRecipe.DiscoveryPortTag); + var ip = peer.Node.Container.Pod.PodInfo.Ip; + var discPort = peer.Node.Container.Recipe.GetPortByTag(CodexContainerRecipe.DiscoveryPortTag); return $"{ip}:{discPort.Number}"; } } diff --git a/DistTestCore/Helpers/PeerDownloadTestHelpers.cs b/DistTestCore/Helpers/PeerDownloadTestHelpers.cs index 4ba22a5..b43b7ab 100644 --- a/DistTestCore/Helpers/PeerDownloadTestHelpers.cs +++ b/DistTestCore/Helpers/PeerDownloadTestHelpers.cs @@ -1,4 +1,5 @@ -using Logging; +using DistTestCore.Codex; +using Logging; using static DistTestCore.Helpers.FullConnectivityHelper; namespace DistTestCore.Helpers @@ -17,6 +18,11 @@ namespace DistTestCore.Helpers } public void AssertFullDownloadInterconnectivity(IEnumerable nodes, ByteSize testFileSize) + { + AssertFullDownloadInterconnectivity(nodes.Select(n => ((OnlineCodexNode)n).CodexAccess), testFileSize); + } + + public void AssertFullDownloadInterconnectivity(IEnumerable nodes, ByteSize testFileSize) { this.testFileSize = testFileSize; helper.AssertFullyConnected(nodes); @@ -37,11 +43,16 @@ namespace DistTestCore.Helpers fileManager.PushFileSet(); var expectedFile = GenerateTestFile(from.Node, to.Node); - var contentId = from.Node.UploadFile(expectedFile); + using var uploadStream = File.OpenRead(expectedFile.Filename); + var contentId = from.Node.UploadFile(uploadStream); try { - var downloadedFile = to.Node.DownloadContent(contentId, expectedFile.Label + "_downloaded"); + var downloadedFile = fileManager.CreateEmptyTestFile(expectedFile.Label + "_downloaded"); + using var downloadStream = File.OpenWrite(downloadedFile.Filename); + using var stream = to.Node.DownloadFile(contentId); + stream.CopyTo(downloadStream); + expectedFile.AssertIsEqual(downloadedFile); return PeerConnectionState.Connection; } @@ -59,7 +70,7 @@ namespace DistTestCore.Helpers // Should an exception occur during upload, then this try is inconclusive and we try again next loop. } - private TestFile GenerateTestFile(IOnlineCodexNode uploader, IOnlineCodexNode downloader) + private TestFile GenerateTestFile(CodexAccess uploader, CodexAccess downloader) { var up = uploader.GetName().Replace("<", "").Replace(">", ""); var down = downloader.GetName().Replace("<", "").Replace(">", "");