diff --git a/CodexNetDeployer/CodexNodeStarter.cs b/CodexNetDeployer/CodexNodeStarter.cs index 39bc1f9d..1bc4af94 100644 --- a/CodexNetDeployer/CodexNodeStarter.cs +++ b/CodexNetDeployer/CodexNodeStarter.cs @@ -21,7 +21,7 @@ namespace CodexNetDeployer validatorsLeft = numberOfValidators; } - public RunningContainer? Start(int i) + public CodexNodeStartResult? Start(int i) { Console.Write($" - {i} = "); var workflow = lifecycle.WorkflowCreator.CreateWorkflow(); @@ -60,7 +60,7 @@ namespace CodexNetDeployer if (string.IsNullOrEmpty(bootstrapSpr)) bootstrapSpr = debugInfo.spr; validatorsLeft--; - return container; + return new CodexNodeStartResult(workflow, container, codexAccess); } } } @@ -102,4 +102,18 @@ namespace CodexNetDeployer return codexStart; } } + + public class CodexNodeStartResult + { + public CodexNodeStartResult(StartupWorkflow workflow, RunningContainer container, CodexAccess access) + { + Workflow = workflow; + Container = container; + Access = access; + } + + public StartupWorkflow Workflow { get; } + public RunningContainer Container { get; } + public CodexAccess Access { get; } + } } diff --git a/CodexNetDeployer/Configuration.cs b/CodexNetDeployer/Configuration.cs index 4df18be6..7ad241a3 100644 --- a/CodexNetDeployer/Configuration.cs +++ b/CodexNetDeployer/Configuration.cs @@ -57,6 +57,9 @@ namespace CodexNetDeployer [Uniform("teststype-podlabel", "ttpl", "TESTSTYPE-PODLABEL", false, "Each kubernetes pod will be created with a label 'teststype' with value 'continuous'. " + "set this option to override the label value.")] public string TestsTypePodLabel { get; set; } = "continuous-tests"; + + [Uniform("check-connect", "cc", "CHECKCONNECT", false, "If true, deployer check ensure peer-connectivity between all deployed nodes after deployment.")] + public bool CheckPeerConnection { get; set; } = false; public List Validate() { diff --git a/CodexNetDeployer/Deployer.cs b/CodexNetDeployer/Deployer.cs index 55ad4fe5..feaff514 100644 --- a/CodexNetDeployer/Deployer.cs +++ b/CodexNetDeployer/Deployer.cs @@ -8,14 +8,14 @@ namespace CodexNetDeployer public class Deployer { private readonly Configuration config; - private readonly NullLog log; private readonly DefaultTimeSet timeset; + private readonly PeerConnectivityChecker peerConnectivityChecker; public Deployer(Configuration config) { this.config = config; - log = new NullLog(); timeset = new DefaultTimeSet(); + peerConnectivityChecker = new PeerConnectivityChecker(); } public CodexDeployment Deploy() @@ -45,16 +45,19 @@ namespace CodexNetDeployer // Each node must have its own IP, so it needs it own pod. Start them 1 at a time. var codexStarter = new CodexNodeStarter(config, lifecycle, gethResults, config.NumberOfValidators!.Value); - var codexContainers = new List(); + var startResults = new List(); for (var i = 0; i < config.NumberOfCodexNodes; i++) { - var container = codexStarter.Start(i); - if (container != null) codexContainers.Add(container); + var result = codexStarter.Start(i); + if (result != null) startResults.Add(result); } - var (prometheusContainer, grafanaStartInfo) = StartMetricsService(lifecycle, setup, codexContainers); + var (prometheusContainer, grafanaStartInfo) = StartMetricsService(lifecycle, setup, startResults.Select(r => r.Container)); - return new CodexDeployment(gethResults, codexContainers.ToArray(), prometheusContainer, grafanaStartInfo, CreateMetadata()); + CheckPeerConnectivity(startResults); + CheckContainerRestarts(startResults); + + return new CodexDeployment(gethResults, startResults.Select(r => r.Container).ToArray(), prometheusContainer, grafanaStartInfo, CreateMetadata()); } private TestLifecycle CreateTestLifecycle() @@ -71,10 +74,10 @@ namespace CodexNetDeployer k8sNamespacePrefix: config.KubeNamespace ); - return new TestLifecycle(log, lifecycleConfig, timeset, config.TestsTypePodLabel, string.Empty); + return new TestLifecycle(new NullLog(), lifecycleConfig, timeset, config.TestsTypePodLabel, string.Empty); } - private (RunningContainer?, GrafanaStartInfo?) StartMetricsService(TestLifecycle lifecycle, CodexSetup setup, List codexContainers) + private (RunningContainer?, GrafanaStartInfo?) StartMetricsService(TestLifecycle lifecycle, CodexSetup setup, IEnumerable codexContainers) { if (setup.MetricsMode == DistTestCore.Metrics.MetricsMode.None) return (null, null); @@ -95,6 +98,35 @@ namespace CodexNetDeployer return kubeConfigFile; } + private void CheckPeerConnectivity(List codexContainers) + { + if (!config.CheckPeerConnection) return; + + Log("Starting peer-connectivity check for deployed nodes..."); + peerConnectivityChecker.CheckConnectivity(codexContainers); + Log("Check passed."); + } + + private void CheckContainerRestarts(List startResults) + { + var crashes = new List(); + foreach (var startResult in startResults) + { + var watcher = startResult.Workflow.CreateCrashWatcher(startResult.Container); + if (watcher.HasContainerCrashed()) crashes.Add(startResult.Container); + } + + if (!crashes.Any()) + { + Log("Container restart check passed."); + } + else + { + Log($"Deployment failed. The following containers have crashed: {string.Join(",", crashes.Select(c => c.Name))}"); + throw new Exception("Deployment failed: One or more containers crashed."); + } + } + private DeploymentMetadata CreateMetadata() { return new DeploymentMetadata( diff --git a/CodexNetDeployer/PeerConnectivityChecker.cs b/CodexNetDeployer/PeerConnectivityChecker.cs new file mode 100644 index 00000000..6e7eddb7 --- /dev/null +++ b/CodexNetDeployer/PeerConnectivityChecker.cs @@ -0,0 +1,34 @@ +using DistTestCore.Helpers; +using Logging; + +namespace CodexNetDeployer +{ + public class PeerConnectivityChecker + { + public void CheckConnectivity(List startResults) + { + var log = new ConsoleLog(); + var checker = new PeerConnectionTestHelpers(log); + var access = startResults.Select(r => r.Access); + + checker.AssertFullyConnected(access); + } + } + + public class ConsoleLog : BaseLog + { + public ConsoleLog() : base(false) + { + } + + protected override string GetFullName() + { + return "CONSOLE"; + } + + public override void Log(string message) + { + Console.WriteLine(message); + } + } +} diff --git a/CodexNetDeployer/Program.cs b/CodexNetDeployer/Program.cs index da18e01c..8ddf527b 100644 --- a/CodexNetDeployer/Program.cs +++ b/CodexNetDeployer/Program.cs @@ -56,8 +56,5 @@ public class Program Console.WriteLine("CodexNetDeployer allows you to easily deploy multiple Codex nodes in a Kubernetes cluster. " + "The deployer will set up the required supporting services, deploy the Codex on-chain contracts, start and bootstrap the Codex instances. " + "All Kubernetes objects will be created in the namespace provided, allowing you to easily find, modify, and delete them afterwards." + nl); - - Console.WriteLine("CodexNetDeployer assumes you are running this tool from *inside* the Kubernetes cluster you want to deploy to. " + - "If you are not running this from a container inside the cluster, add the argument '--external'." + nl); } } diff --git a/CodexNetDeployer/deploy-continuous-testnet.sh b/CodexNetDeployer/deploy-continuous-testnet.sh index 25a67c7b..afcfc448 100644 --- a/CodexNetDeployer/deploy-continuous-testnet.sh +++ b/CodexNetDeployer/deploy-continuous-testnet.sh @@ -12,4 +12,5 @@ dotnet run \ --block-ttl=180 \ --block-mi=120 \ --block-mn=10000 \ - --metrics=Dashboard + --metrics=Dashboard \ + --check-connect=1 diff --git a/ContinuousTests/Tests/HoldMyBeerTest.cs b/ContinuousTests/Tests/HoldMyBeerTest.cs index f54108e6..18e72f15 100644 --- a/ContinuousTests/Tests/HoldMyBeerTest.cs +++ b/ContinuousTests/Tests/HoldMyBeerTest.cs @@ -1,5 +1,4 @@ using DistTestCore; -using Logging; using NUnit.Framework; namespace ContinuousTests.Tests @@ -26,7 +25,6 @@ namespace ContinuousTests.Tests var filesize = filesizePerUploadMb.MB(); double codexDefaultBlockSize = 31 * 64 * 33; var numberOfBlocks = Convert.ToInt64(Math.Ceiling(filesize.SizeInBytes / codexDefaultBlockSize)); - var sizeInBytes = filesize.SizeInBytes; Assert.That(numberOfBlocks, Is.EqualTo(1282)); file = FileManager.GenerateTestFile(filesize); @@ -34,75 +32,8 @@ namespace ContinuousTests.Tests cid = UploadFile(Nodes[0], file); Assert.That(cid, Is.Not.Null); - var cidTag = cid!.Id.Substring(cid.Id.Length - 6); - Stopwatch.Measure(Log, "upload-log-asserts", () => - { - var uploadLog = DownloadContainerLog(Nodes[0].Container, 50000); - - var storeLines = uploadLog.FindLinesThatContain("Stored data", "topics=\"codex node\""); - uploadLog.DeleteFile(); - - var storeLine = GetLineForCidTag(storeLines, cidTag); - AssertStoreLineContains(storeLine, numberOfBlocks, sizeInBytes); - }); - var dl = DownloadFile(Nodes[0], cid!); file.AssertIsEqual(dl); - - Stopwatch.Measure(Log, "download-log-asserts", () => - { - var downloadLog = DownloadContainerLog(Nodes[0].Container, 50000); - - var sentLines = downloadLog.FindLinesThatContain("Sent bytes", "topics=\"codex restapi\""); - downloadLog.DeleteFile(); - - var sentLine = GetLineForCidTag(sentLines, cidTag); - AssertSentLineContains(sentLine, sizeInBytes); - }); - } - - private void AssertSentLineContains(string sentLine, long sizeInBytes) - { - var tag = "bytes="; - var token = sentLine.Substring(sentLine.IndexOf(tag) + tag.Length); - var bytes = Convert.ToInt64(token); - Assert.AreEqual(sizeInBytes, bytes, $"Sent bytes: Number of bytes incorrect. Line: '{sentLine}'"); - } - - private void AssertStoreLineContains(string storeLine, long numberOfBlocks, long sizeInBytes) - { - var tokens = storeLine.Split(" "); - - var blocksToken = GetToken(tokens, "blocks="); - var sizeToken = GetToken(tokens, "size="); - if (blocksToken == null) Assert.Fail("blockToken not found in " + storeLine); - if (sizeToken == null) Assert.Fail("sizeToken not found in " + storeLine); - - var blocks = Convert.ToInt64(blocksToken); - var size = Convert.ToInt64(sizeToken?.Replace("'NByte", "")); - - var lineLog = $" Line: '{storeLine}'"; - Assert.AreEqual(numberOfBlocks, blocks, "Stored data: Number of blocks incorrect." + lineLog); - Assert.AreEqual(sizeInBytes, size, "Stored data: Number of blocks incorrect." + lineLog); - } - - private string GetLineForCidTag(string[] lines, string cidTag) - { - var result = lines.SingleOrDefault(l => l.Contains(cidTag)); - if (result == null) - { - Assert.Fail($"Failed to find '{cidTag}' in lines: '{string.Join(",", lines)}'"); - throw new Exception(); - } - - return result; - } - - private string? GetToken(string[] tokens, string tag) - { - var token = tokens.SingleOrDefault(t => t.StartsWith(tag)); - if (token == null) return null; - return token.Substring(tag.Length); } } } diff --git a/DistTestCore/Codex/CodexAccess.cs b/DistTestCore/Codex/CodexAccess.cs index 076a3358..cf6ac199 100644 --- a/DistTestCore/Codex/CodexAccess.cs +++ b/DistTestCore/Codex/CodexAccess.cs @@ -88,13 +88,17 @@ namespace DistTestCore.Codex return Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}"); } - private Http Http() + public string GetName() { - CheckContainerCrashed(); - return new Http(log, timeSet, Address, baseUrl: "/api/codex/v1", Container.Name); + return Container.Name; } - private void CheckContainerCrashed() + private Http Http() + { + return new Http(log, timeSet, Address, baseUrl: "/api/codex/v1", CheckContainerCrashed, Container.Name); + } + + private void CheckContainerCrashed(HttpClient client) { if (hasContainerCrashed) throw new Exception("Container has crashed."); } diff --git a/DistTestCore/DistTest.cs b/DistTestCore/DistTest.cs index 943b098e..a6379ecc 100644 --- a/DistTestCore/DistTest.cs +++ b/DistTestCore/DistTest.cs @@ -30,14 +30,8 @@ namespace DistTestCore var startTime = DateTime.UtcNow; fixtureLog = new FixtureLog(logConfig, startTime); statusLog = new StatusLog(logConfig, startTime); - - PeerConnectionTestHelpers = new PeerConnectionTestHelpers(this); - PeerDownloadTestHelpers = new PeerDownloadTestHelpers(this); } - public PeerConnectionTestHelpers PeerConnectionTestHelpers { get; } - public PeerDownloadTestHelpers PeerDownloadTestHelpers { get; } - [OneTimeSetUp] public void GlobalSetup() { @@ -175,6 +169,16 @@ namespace DistTestCore GetTestLog().Debug(msg); } + public PeerConnectionTestHelpers CreatePeerConnectionTestHelpers() + { + return new PeerConnectionTestHelpers(GetTestLog()); + } + + public PeerDownloadTestHelpers CreatePeerDownloadTestHelpers() + { + return new PeerDownloadTestHelpers(GetTestLog(), Get().FileManager); + } + public void Measure(string name, Action action) { Stopwatch.Measure(Get().Log, name, action); diff --git a/DistTestCore/Helpers/FullConnectivityHelper.cs b/DistTestCore/Helpers/FullConnectivityHelper.cs index afa3442b..68dfdbb8 100644 --- a/DistTestCore/Helpers/FullConnectivityHelper.cs +++ b/DistTestCore/Helpers/FullConnectivityHelper.cs @@ -1,4 +1,5 @@ using DistTestCore.Codex; +using Logging; using NUnit.Framework; using Utils; @@ -14,23 +15,23 @@ namespace DistTestCore.Helpers public class FullConnectivityHelper { private static string Nl = Environment.NewLine; - private readonly DistTest test; + private readonly BaseLog log; private readonly IFullConnectivityImplementation implementation; - public FullConnectivityHelper(DistTest test, IFullConnectivityImplementation implementation) + public FullConnectivityHelper(BaseLog log, IFullConnectivityImplementation implementation) { - this.test = test; + this.log = log; this.implementation = implementation; } - public void AssertFullyConnected(IEnumerable nodes) + public void AssertFullyConnected(IEnumerable nodes) { AssertFullyConnected(nodes.ToArray()); } - private void AssertFullyConnected(IOnlineCodexNode[] nodes) + private void AssertFullyConnected(CodexAccess[] nodes) { - test.Log($"Asserting '{implementation.Description()}' for nodes: '{string.Join(",", nodes.Select(n => n.GetName()))}'..."); + Log($"Asserting '{implementation.Description()}' for nodes: '{string.Join(",", nodes.Select(n => n.GetName()))}'..."); var entries = CreateEntries(nodes); var pairs = CreatePairs(entries); @@ -43,19 +44,19 @@ namespace DistTestCore.Helpers { var pairDetails = string.Join(Nl, pairs.SelectMany(p => p.GetResultMessages())); - test.Log($"Connections failed:{Nl}{pairDetails}"); + Log($"Connections failed:{Nl}{pairDetails}"); Assert.Fail(string.Join(Nl, pairs.SelectMany(p => p.GetResultMessages()))); } else { - test.Log($"'{implementation.Description()}' = Success! for nodes: {string.Join(",", nodes.Select(n => n.GetName()))}"); + Log($"'{implementation.Description()}' = Success! for nodes: {string.Join(",", nodes.Select(n => n.GetName()))}"); } } private static void RetryWhilePairs(List pairs, Action action) { - var timeout = DateTime.UtcNow + TimeSpan.FromMinutes(5); + var timeout = DateTime.UtcNow + TimeSpan.FromMinutes(2); while (pairs.Any(p => p.Inconclusive) && timeout > DateTime.UtcNow) { action(); @@ -72,7 +73,7 @@ namespace DistTestCore.Helpers foreach (var pair in selectedPair) { - test.ScopedTestFiles(pair.Check); + pair.Check(); if (pair.Success) { @@ -81,10 +82,10 @@ namespace DistTestCore.Helpers } } - test.Log($"Connections successful:{Nl}{string.Join(Nl, pairDetails)}"); + Log($"Connections successful:{Nl}{string.Join(Nl, pairDetails)}"); } - private Entry[] CreateEntries(IOnlineCodexNode[] nodes) + private Entry[] CreateEntries(CodexAccess[] nodes) { var entries = nodes.Select(n => new Entry(n)).ToArray(); @@ -117,15 +118,20 @@ namespace DistTestCore.Helpers } } + private void Log(string msg) + { + log.Log(msg); + } + public class Entry { - public Entry(IOnlineCodexNode node) + public Entry(CodexAccess node) { Node = node; Response = node.GetDebugInfo(); } - public IOnlineCodexNode Node { get; } + public CodexAccess Node { get; } public CodexDebugResponse Response { get; } public override string ToString() diff --git a/DistTestCore/Helpers/PeerConnectionTestHelpers.cs b/DistTestCore/Helpers/PeerConnectionTestHelpers.cs index fa07b067..65bb4ce0 100644 --- a/DistTestCore/Helpers/PeerConnectionTestHelpers.cs +++ b/DistTestCore/Helpers/PeerConnectionTestHelpers.cs @@ -1,4 +1,5 @@ using DistTestCore.Codex; +using Logging; using static DistTestCore.Helpers.FullConnectivityHelper; namespace DistTestCore.Helpers @@ -7,12 +8,17 @@ namespace DistTestCore.Helpers { private readonly FullConnectivityHelper helper; - public PeerConnectionTestHelpers(DistTest test) + public PeerConnectionTestHelpers(BaseLog log) { - helper = new FullConnectivityHelper(test, this); + helper = new FullConnectivityHelper(log, this); } public void AssertFullyConnected(IEnumerable nodes) + { + AssertFullyConnected(nodes.Select(n => ((OnlineCodexNode)n).CodexAccess)); + } + + public void AssertFullyConnected(IEnumerable nodes) { helper.AssertFullyConnected(nodes); } @@ -57,9 +63,8 @@ namespace DistTestCore.Helpers var peer = allEntries.SingleOrDefault(e => e.Response.table.localNode.peerId == node.peerId); if (peer == null) return $"peerId: {node.peerId} is not known."; - var n = (OnlineCodexNode)peer.Node; - var ip = n.CodexAccess.Container.Pod.PodInfo.Ip; - var discPort = n.CodexAccess.Container.Recipe.GetPortByTag(CodexContainerRecipe.DiscoveryPortTag); + var ip = peer.Node.Container.Pod.PodInfo.Ip; + var discPort = peer.Node.Container.Recipe.GetPortByTag(CodexContainerRecipe.DiscoveryPortTag); return $"{ip}:{discPort.Number}"; } } diff --git a/DistTestCore/Helpers/PeerDownloadTestHelpers.cs b/DistTestCore/Helpers/PeerDownloadTestHelpers.cs index ffb7d52d..cd12e2b9 100644 --- a/DistTestCore/Helpers/PeerDownloadTestHelpers.cs +++ b/DistTestCore/Helpers/PeerDownloadTestHelpers.cs @@ -1,21 +1,30 @@ -using static DistTestCore.Helpers.FullConnectivityHelper; +using DistTestCore.Codex; +using Logging; +using static DistTestCore.Helpers.FullConnectivityHelper; namespace DistTestCore.Helpers { public class PeerDownloadTestHelpers : IFullConnectivityImplementation { private readonly FullConnectivityHelper helper; - private readonly DistTest test; + private readonly BaseLog log; + private readonly FileManager fileManager; private ByteSize testFileSize; - public PeerDownloadTestHelpers(DistTest test) + public PeerDownloadTestHelpers(BaseLog log, FileManager fileManager) { - helper = new FullConnectivityHelper(test, this); + helper = new FullConnectivityHelper(log, this); testFileSize = 1.MB(); - this.test = test; + this.log = log; + this.fileManager = fileManager; } public void AssertFullDownloadInterconnectivity(IEnumerable nodes, ByteSize testFileSize) + { + AssertFullDownloadInterconnectivity(nodes.Select(n => ((OnlineCodexNode)n).CodexAccess), testFileSize); + } + + public void AssertFullDownloadInterconnectivity(IEnumerable nodes, ByteSize testFileSize) { this.testFileSize = testFileSize; helper.AssertFullyConnected(nodes); @@ -33,13 +42,15 @@ namespace DistTestCore.Helpers public PeerConnectionState Check(Entry from, Entry to) { + fileManager.PushFileSet(); var expectedFile = GenerateTestFile(from.Node, to.Node); - var contentId = from.Node.UploadFile(expectedFile); + using var uploadStream = File.OpenRead(expectedFile.Filename); + var contentId = Stopwatch.Measure(log, "Upload", () => from.Node.UploadFile(uploadStream)); try { - var downloadedFile = to.Node.DownloadContent(contentId, expectedFile.Label + "_downloaded"); + var downloadedFile = Stopwatch.Measure(log, "Download", () => DownloadFile(to.Node, contentId, expectedFile.Label + "_downloaded")); expectedFile.AssertIsEqual(downloadedFile); return PeerConnectionState.Connection; } @@ -49,16 +60,29 @@ namespace DistTestCore.Helpers // We consider that as no-connection for the purpose of this test. return PeerConnectionState.NoConnection; } + finally + { + fileManager.PopFileSet(); + } // Should an exception occur during upload, then this try is inconclusive and we try again next loop. } - private TestFile GenerateTestFile(IOnlineCodexNode uploader, IOnlineCodexNode downloader) + private TestFile DownloadFile(CodexAccess node, string contentId, string label) + { + var downloadedFile = fileManager.CreateEmptyTestFile(label); + using var downloadStream = File.OpenWrite(downloadedFile.Filename); + using var stream = node.DownloadFile(contentId); + stream.CopyTo(downloadStream); + return downloadedFile; + } + + private TestFile GenerateTestFile(CodexAccess uploader, CodexAccess downloader) { var up = uploader.GetName().Replace("<", "").Replace(">", ""); var down = downloader.GetName().Replace("<", "").Replace(">", ""); var label = $"~from:{up}-to:{down}~"; - return test.GenerateTestFile(testFileSize, label); + return fileManager.GenerateTestFile(testFileSize, label); } } } diff --git a/DistTestCore/Metrics/dashboard.json b/DistTestCore/Metrics/dashboard.json index 3af1c529..313abf01 100644 --- a/DistTestCore/Metrics/dashboard.json +++ b/DistTestCore/Metrics/dashboard.json @@ -512,6 +512,203 @@ "title": "Bytes used", "type": "timeseries" }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 2 + }, + "id": 24, + "panels": [], + "title": "Network Status", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 3 + }, + "id": 25, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "editorMode": "builder", + "expr": "libp2p_peers", + "instant": false, + "range": true, + "refId": "A" + } + ], + "title": "Peers", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 3 + }, + "id": 26, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "editorMode": "builder", + "expr": "routing_table_nodes", + "instant": false, + "range": true, + "refId": "A" + } + ], + "title": "Routing table nodes", + "type": "timeseries" + }, { "collapsed": false, "gridPos": { diff --git a/KubernetesWorkflow/CrashWatcher.cs b/KubernetesWorkflow/CrashWatcher.cs index 86f8ae1f..7f8bc71b 100644 --- a/KubernetesWorkflow/CrashWatcher.cs +++ b/KubernetesWorkflow/CrashWatcher.cs @@ -43,6 +43,12 @@ namespace KubernetesWorkflow if (workerException != null) throw new Exception("Exception occurred in CrashWatcher worker thread.", workerException); } + public bool HasContainerCrashed() + { + using var client = new Kubernetes(config); + return HasContainerBeenRestarted(client, container.Pod.PodInfo.Name); + } + private void Worker() { try @@ -57,7 +63,7 @@ namespace KubernetesWorkflow private void MonitorContainer(CancellationToken token) { - var client = new Kubernetes(config); + using var client = new Kubernetes(config); while (!token.IsCancellationRequested) { token.WaitHandle.WaitOne(TimeSpan.FromSeconds(1)); @@ -65,9 +71,7 @@ namespace KubernetesWorkflow var pod = container.Pod; var recipe = container.Recipe; var podName = pod.PodInfo.Name; - var podInfo = client.ReadNamespacedPod(podName, k8sNamespace); - - if (podInfo.Status.ContainerStatuses.Any(c => c.RestartCount > 0)) + if (HasContainerBeenRestarted(client, podName)) { DownloadCrashedContainerLogs(client, podName, recipe); return; @@ -75,6 +79,12 @@ namespace KubernetesWorkflow } } + private bool HasContainerBeenRestarted(Kubernetes client, string podName) + { + var podInfo = client.ReadNamespacedPod(podName, k8sNamespace); + return podInfo.Status.ContainerStatuses.Any(c => c.RestartCount > 0); + } + private void DownloadCrashedContainerLogs(Kubernetes client, string podName, ContainerRecipe recipe) { log.Log("Pod crash detected for " + container.Name); diff --git a/LongTests/DownloadConnectivityTests/LongFullyConnectedDownloadTests.cs b/LongTests/DownloadConnectivityTests/LongFullyConnectedDownloadTests.cs index 5bb73238..d0650f67 100644 --- a/LongTests/DownloadConnectivityTests/LongFullyConnectedDownloadTests.cs +++ b/LongTests/DownloadConnectivityTests/LongFullyConnectedDownloadTests.cs @@ -1,5 +1,4 @@ -using DistTestCore.Helpers; -using DistTestCore; +using DistTestCore; using NUnit.Framework; namespace TestsLong.DownloadConnectivityTests @@ -16,7 +15,7 @@ namespace TestsLong.DownloadConnectivityTests { for (var i = 0; i < numberOfNodes; i++) SetupCodexNode(); - PeerDownloadTestHelpers.AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes(), sizeMBs.MB()); + CreatePeerDownloadTestHelpers().AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes(), sizeMBs.MB()); } } } diff --git a/Tests/BasicTests/TwoClientTests.cs b/Tests/BasicTests/TwoClientTests.cs index d03633b2..12bab131 100644 --- a/Tests/BasicTests/TwoClientTests.cs +++ b/Tests/BasicTests/TwoClientTests.cs @@ -8,7 +8,7 @@ namespace Tests.BasicTests public class TwoClientTests : DistTest { [Test] - public void TwoClientsOnePodTest() + public void TwoClientTest() { var group = SetupCodexNodes(2); @@ -18,15 +18,6 @@ namespace Tests.BasicTests PerformTwoClientTest(primary, secondary); } - [Test] - public void TwoClientsTwoPodsTest() - { - var primary = SetupCodexNode(); - var secondary = SetupCodexNode(); - - PerformTwoClientTest(primary, secondary); - } - [Test] public void TwoClientsTwoLocationsTest() { diff --git a/Tests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs b/Tests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs index a1f2c5f4..7cf546b5 100644 --- a/Tests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs +++ b/Tests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs @@ -6,15 +6,36 @@ namespace Tests.DownloadConnectivityTests [TestFixture] public class FullyConnectedDownloadTests : AutoBootstrapDistTest { + [Test] + public void MetricsDoesNotInterfereWithPeerDownload() + { + SetupCodexNodes(2, s => s.EnableMetrics()); + + AssertAllNodesConnected(); + } + + [Test] + public void MarketplaceDoesNotInterfereWithPeerDownload() + { + SetupCodexNodes(2, s => s.EnableMetrics().EnableMarketplace(1000.TestTokens())); + + AssertAllNodesConnected(); + } + [Test] [Combinatorial] public void FullyConnectedDownloadTest( [Values(1, 3, 5)] int numberOfNodes, [Values(1, 10)] int sizeMBs) { - for (var i = 0; i < numberOfNodes; i++) SetupCodexNode(); + SetupCodexNodes(numberOfNodes); - PeerDownloadTestHelpers.AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes(), sizeMBs.MB()); + AssertAllNodesConnected(sizeMBs); + } + + private void AssertAllNodesConnected(int sizeMBs = 10) + { + CreatePeerDownloadTestHelpers().AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes(), sizeMBs.MB()); } } } diff --git a/Tests/PeerDiscoveryTests/LayeredDiscoveryTests.cs b/Tests/PeerDiscoveryTests/LayeredDiscoveryTests.cs index aeb414a6..8cd32a2d 100644 --- a/Tests/PeerDiscoveryTests/LayeredDiscoveryTests.cs +++ b/Tests/PeerDiscoveryTests/LayeredDiscoveryTests.cs @@ -1,5 +1,4 @@ using DistTestCore; -using DistTestCore.Helpers; using NUnit.Framework; namespace Tests.PeerDiscoveryTests @@ -47,7 +46,7 @@ namespace Tests.PeerDiscoveryTests private void AssertAllNodesConnected() { - PeerConnectionTestHelpers.AssertFullyConnected(GetAllOnlineCodexNodes()); + CreatePeerConnectionTestHelpers().AssertFullyConnected(GetAllOnlineCodexNodes()); } } } diff --git a/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs b/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs index 002fe79d..fcb44fdc 100644 --- a/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs +++ b/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs @@ -1,5 +1,4 @@ using DistTestCore; -using DistTestCore.Helpers; using NUnit.Framework; namespace Tests.PeerDiscoveryTests @@ -17,23 +16,36 @@ namespace Tests.PeerDiscoveryTests Assert.That(result.IsPeerFound, Is.False); } + [Test] + public void MetricsDoesNotInterfereWithPeerDiscovery() + { + SetupCodexNodes(2, s => s.EnableMetrics()); + + AssertAllNodesConnected(); + } + + [Test] + public void MarketplaceDoesNotInterfereWithPeerDiscovery() + { + SetupCodexNodes(2, s => s.EnableMarketplace(1000.TestTokens())); + + AssertAllNodesConnected(); + } + [TestCase(2)] [TestCase(3)] [TestCase(10)] [TestCase(20)] - public void VariableNodesInPods(int number) + public void VariableNodes(int number) { - for (var i = 0; i < number; i++) - { - SetupCodexNode(); - } + SetupCodexNodes(number); AssertAllNodesConnected(); } private void AssertAllNodesConnected() { - PeerConnectionTestHelpers.AssertFullyConnected(GetAllOnlineCodexNodes()); + CreatePeerConnectionTestHelpers().AssertFullyConnected(GetAllOnlineCodexNodes()); } } }