From cdd51f6dc71216892ca182e3253e8c3b7d5f5d44 Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 19 Apr 2023 14:57:00 +0200 Subject: [PATCH] Sets up some durability tests. --- DistTestCore/Codex/CodexStartupConfig.cs | 3 +- DistTestCore/CodexNodeGroup.cs | 2 +- DistTestCore/CodexSetup.cs | 35 +++++++++---- DistTestCore/CodexStarter.cs | 2 +- DistTestCore/OnlineCodexNode.cs | 26 ++++++---- DistTestCore/TestLifecycle.cs | 2 +- KubernetesWorkflow/RunningContainers.cs | 2 +- Tests/DurabilityTests/DurabilityTests.cs | 65 ++++++++++++++++++++++++ 8 files changed, 112 insertions(+), 25 deletions(-) create mode 100644 Tests/DurabilityTests/DurabilityTests.cs diff --git a/DistTestCore/Codex/CodexStartupConfig.cs b/DistTestCore/Codex/CodexStartupConfig.cs index 915d4b9..2ebf407 100644 --- a/DistTestCore/Codex/CodexStartupConfig.cs +++ b/DistTestCore/Codex/CodexStartupConfig.cs @@ -10,7 +10,6 @@ namespace DistTestCore.Codex public ByteSize? StorageQuota { get; set; } public bool MetricsEnabled { get; set; } public MarketplaceInitialConfig? MarketplaceConfig { get; set; } - - //public IOnlineCodexNode? BootstrapNode { get; private set; } + public IOnlineCodexNode? BootstrapNode { get; set; } } } diff --git a/DistTestCore/CodexNodeGroup.cs b/DistTestCore/CodexNodeGroup.cs index 4ce577c..ffcea2e 100644 --- a/DistTestCore/CodexNodeGroup.cs +++ b/DistTestCore/CodexNodeGroup.cs @@ -59,7 +59,7 @@ namespace DistTestCore public string Describe() { - return $""; + return $"group:[{Containers.Describe()}]"; } private OnlineCodexNode CreateOnlineCodexNode(RunningContainer c, ICodexNodeFactory factory) diff --git a/DistTestCore/CodexSetup.cs b/DistTestCore/CodexSetup.cs index 39020f6..8f7e253 100644 --- a/DistTestCore/CodexSetup.cs +++ b/DistTestCore/CodexSetup.cs @@ -8,7 +8,7 @@ namespace DistTestCore { ICodexSetup At(Location location); ICodexSetup WithLogLevel(CodexLogLevel level); - //ICodexStartupConfig WithBootstrapNode(IOnlineCodexNode node); + ICodexSetup WithBootstrapNode(IOnlineCodexNode node); ICodexSetup WithStorageQuota(ByteSize storageQuota); ICodexSetup EnableMetrics(); ICodexSetup EnableMarketplace(TestToken initialBalance); @@ -30,7 +30,24 @@ namespace DistTestCore public ICodexNodeGroup BringOnline() { - return starter.BringOnline(this); + var group = starter.BringOnline(this); + ConnectToBootstrapNode(group); + return group; + } + + private void ConnectToBootstrapNode(ICodexNodeGroup group) + { + if (BootstrapNode == null) return; + + // TODO: + // node.ConnectToPeer uses the '/api/codex/vi/connect/' endpoint to make the connection. + // This should be replaced by injecting the bootstrap node's SPR into the env-vars of the new node containers. (Easy!) + // However, NAT isn't figure out yet. So connecting with SPR doesn't (always?) work. + // So for now, ConnectToPeer + foreach (var node in group) + { + node.ConnectToPeer(BootstrapNode); + } } public ICodexSetup At(Location location) @@ -39,11 +56,11 @@ namespace DistTestCore return this; } - //public ICodexSetupConfig WithBootstrapNode(IOnlineCodexNode node) - //{ - // BootstrapNode = node; - // return this; - //} + public ICodexSetup WithBootstrapNode(IOnlineCodexNode node) + { + BootstrapNode = node; + return this; + } public ICodexSetup WithLogLevel(CodexLogLevel level) { @@ -77,13 +94,13 @@ namespace DistTestCore public string Describe() { var args = string.Join(',', DescribeArgs()); - return $"({NumberOfNodes} CodexNodes with [{args}])"; + return $"({NumberOfNodes} CodexNodes with args:[{args}])"; } private IEnumerable DescribeArgs() { if (LogLevel != null) yield return $"LogLevel={LogLevel}"; - //if (BootstrapNode != null) yield return "BootstrapNode=set-not-shown-here"; + if (BootstrapNode != null) yield return $"BootstrapNode={BootstrapNode.GetName()}"; if (StorageQuota != null) yield return $"StorageQuote={StorageQuota}"; } } diff --git a/DistTestCore/CodexStarter.cs b/DistTestCore/CodexStarter.cs index 84da9b9..52f8769 100644 --- a/DistTestCore/CodexStarter.cs +++ b/DistTestCore/CodexStarter.cs @@ -29,7 +29,7 @@ namespace DistTestCore var codexNodeFactory = new CodexNodeFactory(lifecycle, metricAccessFactory, gethStartResult.MarketplaceAccessFactory); var group = CreateCodexGroup(codexSetup, containers, codexNodeFactory); - LogEnd($"Started {codexSetup.NumberOfNodes} nodes at '{group.Containers.RunningPod.Ip}'. They are: [{string.Join(",", group.Select(n => n.GetName()))}]"); + LogEnd($"Started {codexSetup.NumberOfNodes} nodes at '{group.Containers.RunningPod.Ip}'. They are: {group.Describe()}"); LogSeparator(); return group; } diff --git a/DistTestCore/OnlineCodexNode.cs b/DistTestCore/OnlineCodexNode.cs index a0d7542..43e16ec 100644 --- a/DistTestCore/OnlineCodexNode.cs +++ b/DistTestCore/OnlineCodexNode.cs @@ -23,6 +23,7 @@ namespace DistTestCore private const string SuccessfullyConnectedMessage = "Successfully connected to peer"; private const string UploadFailedMessage = "Unable to store block"; private readonly TestLifecycle lifecycle; + private CodexDebugResponse? debugInfo; public OnlineCodexNode(TestLifecycle lifecycle, CodexAccess codexAccess, CodexNodeGroup group, IMetricsAccess metricsAccess, IMarketplaceAccess marketplaceAccess) { @@ -45,9 +46,11 @@ namespace DistTestCore public CodexDebugResponse GetDebugInfo() { - var response = CodexAccess.GetDebugInfo(); - Log($"Got DebugInfo with id: '{response.id}'."); - return response; + if (debugInfo != null) return debugInfo; + + debugInfo = CodexAccess.GetDebugInfo(); + Log($"Got DebugInfo with id: '{debugInfo.id}'."); + return debugInfo; } public ContentId UploadFile(TestFile file) @@ -89,11 +92,6 @@ namespace DistTestCore return lifecycle.DownloadLog(this); } - public string Describe() - { - return $"({GetName()} in {Group.Describe()})"; - } - private string GetPeerMultiAddress(OnlineCodexNode peer, CodexDebugResponse peerInfo) { var multiAddress = peerInfo.addrs.First(); @@ -112,8 +110,16 @@ namespace DistTestCore private void DownloadToFile(string contentId, TestFile file) { using var fileStream = File.OpenWrite(file.Filename); - using var downloadStream = CodexAccess.DownloadFile(contentId); - downloadStream.CopyTo(fileStream); + try + { + using var downloadStream = CodexAccess.DownloadFile(contentId); + downloadStream.CopyTo(fileStream); + } + catch + { + Log($"Failed to download file '{contentId}'."); + throw; + } } private void Log(string msg) diff --git a/DistTestCore/TestLifecycle.cs b/DistTestCore/TestLifecycle.cs index 3158f4e..2b243e1 100644 --- a/DistTestCore/TestLifecycle.cs +++ b/DistTestCore/TestLifecycle.cs @@ -34,7 +34,7 @@ namespace DistTestCore public ICodexNodeLog DownloadLog(OnlineCodexNode node) { var subFile = Log.CreateSubfile(); - var description = node.Describe(); + var description = node.GetName(); var handler = new LogDownloadHandler(node, description, subFile); Log.Log($"Downloading logs for {description} to file '{subFile.FullFilename}'"); diff --git a/KubernetesWorkflow/RunningContainers.cs b/KubernetesWorkflow/RunningContainers.cs index 27a9991..32335ff 100644 --- a/KubernetesWorkflow/RunningContainers.cs +++ b/KubernetesWorkflow/RunningContainers.cs @@ -15,7 +15,7 @@ public string Describe() { - return $"[{RunningPod.Ip}]"; + return string.Join(",", Containers.Select(c => c.GetName())); } } diff --git a/Tests/DurabilityTests/DurabilityTests.cs b/Tests/DurabilityTests/DurabilityTests.cs new file mode 100644 index 0000000..6ffd78b --- /dev/null +++ b/Tests/DurabilityTests/DurabilityTests.cs @@ -0,0 +1,65 @@ +using DistTestCore; +using NUnit.Framework; +using Utils; + +namespace Tests.DurabilityTests +{ + [TestFixture] + public class DurabilityTests : DistTest + { + [Test] + public void BootstrapNodeDisappearsTest() + { + var bootstrapNode = SetupCodexNodes(1).BringOnline(); + var group = SetupCodexNodes(2).WithBootstrapNode(bootstrapNode[0]).BringOnline(); + var primary = group[0]; + var secondary = group[1]; + + // There is 1 minute of time for the nodes to connect to each other. + // (Should be easy, they're in the same pod.) + Time.Sleep(TimeSpan.FromMinutes(1)); + bootstrapNode.BringOffline(); + + var file = GenerateTestFile(10.MB()); + var contentId = primary.UploadFile(file); + var downloadedFile = secondary.DownloadContent(contentId); + + file.AssertIsEqual(downloadedFile); + } + + [Test] + public void DataRetentionTest() + { + var bootstrapNode = SetupCodexNodes(1).BringOnline()[0]; + + var startGroup = SetupCodexNodes(2).WithBootstrapNode(bootstrapNode).BringOnline(); + var finishGroup = SetupCodexNodes(10).WithBootstrapNode(bootstrapNode).BringOnline(); + + var file = GenerateTestFile(10.MB()); + + // Both nodes in the start group have the file. + var content = startGroup[0].UploadFile(file); + DownloadAndAssert(content, file, startGroup[1]); + + // Three nodes of the finish group have the file. + DownloadAndAssert(content, file, finishGroup[0]); + DownloadAndAssert(content, file, finishGroup[1]); + DownloadAndAssert(content, file, finishGroup[2]); + + // The start group goes away. + startGroup.BringOffline(); + + // All nodes in the finish group can access the file. + foreach (var node in finishGroup) + { + DownloadAndAssert(content, file, node); + } + } + + private void DownloadAndAssert(ContentId content, TestFile file, IOnlineCodexNode onlineCodexNode) + { + var downloaded = onlineCodexNode.DownloadContent(content); + file.AssertIsEqual(downloaded); + } + } +}