Sets up some durability tests.

This commit is contained in:
benbierens 2023-04-19 14:57:00 +02:00
parent baf605464c
commit cdd51f6dc7
No known key found for this signature in database
GPG Key ID: FE44815D96D0A1AA
8 changed files with 112 additions and 25 deletions

View File

@ -10,7 +10,6 @@ namespace DistTestCore.Codex
public ByteSize? StorageQuota { get; set; } public ByteSize? StorageQuota { get; set; }
public bool MetricsEnabled { get; set; } public bool MetricsEnabled { get; set; }
public MarketplaceInitialConfig? MarketplaceConfig { get; set; } public MarketplaceInitialConfig? MarketplaceConfig { get; set; }
public IOnlineCodexNode? BootstrapNode { get; set; }
//public IOnlineCodexNode? BootstrapNode { get; private set; }
} }
} }

View File

@ -59,7 +59,7 @@ namespace DistTestCore
public string Describe() public string Describe()
{ {
return $"<CodexNodeGroup@{Containers.Describe()}-{Setup.Describe()}>"; return $"group:[{Containers.Describe()}]";
} }
private OnlineCodexNode CreateOnlineCodexNode(RunningContainer c, ICodexNodeFactory factory) private OnlineCodexNode CreateOnlineCodexNode(RunningContainer c, ICodexNodeFactory factory)

View File

@ -8,7 +8,7 @@ namespace DistTestCore
{ {
ICodexSetup At(Location location); ICodexSetup At(Location location);
ICodexSetup WithLogLevel(CodexLogLevel level); ICodexSetup WithLogLevel(CodexLogLevel level);
//ICodexStartupConfig WithBootstrapNode(IOnlineCodexNode node); ICodexSetup WithBootstrapNode(IOnlineCodexNode node);
ICodexSetup WithStorageQuota(ByteSize storageQuota); ICodexSetup WithStorageQuota(ByteSize storageQuota);
ICodexSetup EnableMetrics(); ICodexSetup EnableMetrics();
ICodexSetup EnableMarketplace(TestToken initialBalance); ICodexSetup EnableMarketplace(TestToken initialBalance);
@ -30,7 +30,24 @@ namespace DistTestCore
public ICodexNodeGroup BringOnline() public ICodexNodeGroup BringOnline()
{ {
return starter.BringOnline(this); var group = starter.BringOnline(this);
ConnectToBootstrapNode(group);
return group;
}
private void ConnectToBootstrapNode(ICodexNodeGroup group)
{
if (BootstrapNode == null) return;
// TODO:
// node.ConnectToPeer uses the '/api/codex/vi/connect/' endpoint to make the connection.
// This should be replaced by injecting the bootstrap node's SPR into the env-vars of the new node containers. (Easy!)
// However, NAT isn't figure out yet. So connecting with SPR doesn't (always?) work.
// So for now, ConnectToPeer
foreach (var node in group)
{
node.ConnectToPeer(BootstrapNode);
}
} }
public ICodexSetup At(Location location) public ICodexSetup At(Location location)
@ -39,11 +56,11 @@ namespace DistTestCore
return this; return this;
} }
//public ICodexSetupConfig WithBootstrapNode(IOnlineCodexNode node) public ICodexSetup WithBootstrapNode(IOnlineCodexNode node)
//{ {
// BootstrapNode = node; BootstrapNode = node;
// return this; return this;
//} }
public ICodexSetup WithLogLevel(CodexLogLevel level) public ICodexSetup WithLogLevel(CodexLogLevel level)
{ {
@ -77,13 +94,13 @@ namespace DistTestCore
public string Describe() public string Describe()
{ {
var args = string.Join(',', DescribeArgs()); var args = string.Join(',', DescribeArgs());
return $"({NumberOfNodes} CodexNodes with [{args}])"; return $"({NumberOfNodes} CodexNodes with args:[{args}])";
} }
private IEnumerable<string> DescribeArgs() private IEnumerable<string> DescribeArgs()
{ {
if (LogLevel != null) yield return $"LogLevel={LogLevel}"; if (LogLevel != null) yield return $"LogLevel={LogLevel}";
//if (BootstrapNode != null) yield return "BootstrapNode=set-not-shown-here"; if (BootstrapNode != null) yield return $"BootstrapNode={BootstrapNode.GetName()}";
if (StorageQuota != null) yield return $"StorageQuote={StorageQuota}"; if (StorageQuota != null) yield return $"StorageQuote={StorageQuota}";
} }
} }

View File

@ -29,7 +29,7 @@ namespace DistTestCore
var codexNodeFactory = new CodexNodeFactory(lifecycle, metricAccessFactory, gethStartResult.MarketplaceAccessFactory); var codexNodeFactory = new CodexNodeFactory(lifecycle, metricAccessFactory, gethStartResult.MarketplaceAccessFactory);
var group = CreateCodexGroup(codexSetup, containers, codexNodeFactory); var group = CreateCodexGroup(codexSetup, containers, codexNodeFactory);
LogEnd($"Started {codexSetup.NumberOfNodes} nodes at '{group.Containers.RunningPod.Ip}'. They are: [{string.Join(",", group.Select(n => n.GetName()))}]"); LogEnd($"Started {codexSetup.NumberOfNodes} nodes at '{group.Containers.RunningPod.Ip}'. They are: {group.Describe()}");
LogSeparator(); LogSeparator();
return group; return group;
} }

View File

@ -23,6 +23,7 @@ namespace DistTestCore
private const string SuccessfullyConnectedMessage = "Successfully connected to peer"; private const string SuccessfullyConnectedMessage = "Successfully connected to peer";
private const string UploadFailedMessage = "Unable to store block"; private const string UploadFailedMessage = "Unable to store block";
private readonly TestLifecycle lifecycle; private readonly TestLifecycle lifecycle;
private CodexDebugResponse? debugInfo;
public OnlineCodexNode(TestLifecycle lifecycle, CodexAccess codexAccess, CodexNodeGroup group, IMetricsAccess metricsAccess, IMarketplaceAccess marketplaceAccess) public OnlineCodexNode(TestLifecycle lifecycle, CodexAccess codexAccess, CodexNodeGroup group, IMetricsAccess metricsAccess, IMarketplaceAccess marketplaceAccess)
{ {
@ -45,9 +46,11 @@ namespace DistTestCore
public CodexDebugResponse GetDebugInfo() public CodexDebugResponse GetDebugInfo()
{ {
var response = CodexAccess.GetDebugInfo(); if (debugInfo != null) return debugInfo;
Log($"Got DebugInfo with id: '{response.id}'.");
return response; debugInfo = CodexAccess.GetDebugInfo();
Log($"Got DebugInfo with id: '{debugInfo.id}'.");
return debugInfo;
} }
public ContentId UploadFile(TestFile file) public ContentId UploadFile(TestFile file)
@ -89,11 +92,6 @@ namespace DistTestCore
return lifecycle.DownloadLog(this); return lifecycle.DownloadLog(this);
} }
public string Describe()
{
return $"({GetName()} in {Group.Describe()})";
}
private string GetPeerMultiAddress(OnlineCodexNode peer, CodexDebugResponse peerInfo) private string GetPeerMultiAddress(OnlineCodexNode peer, CodexDebugResponse peerInfo)
{ {
var multiAddress = peerInfo.addrs.First(); var multiAddress = peerInfo.addrs.First();
@ -112,8 +110,16 @@ namespace DistTestCore
private void DownloadToFile(string contentId, TestFile file) private void DownloadToFile(string contentId, TestFile file)
{ {
using var fileStream = File.OpenWrite(file.Filename); using var fileStream = File.OpenWrite(file.Filename);
using var downloadStream = CodexAccess.DownloadFile(contentId); try
downloadStream.CopyTo(fileStream); {
using var downloadStream = CodexAccess.DownloadFile(contentId);
downloadStream.CopyTo(fileStream);
}
catch
{
Log($"Failed to download file '{contentId}'.");
throw;
}
} }
private void Log(string msg) private void Log(string msg)

View File

@ -34,7 +34,7 @@ namespace DistTestCore
public ICodexNodeLog DownloadLog(OnlineCodexNode node) public ICodexNodeLog DownloadLog(OnlineCodexNode node)
{ {
var subFile = Log.CreateSubfile(); var subFile = Log.CreateSubfile();
var description = node.Describe(); var description = node.GetName();
var handler = new LogDownloadHandler(node, description, subFile); var handler = new LogDownloadHandler(node, description, subFile);
Log.Log($"Downloading logs for {description} to file '{subFile.FullFilename}'"); Log.Log($"Downloading logs for {description} to file '{subFile.FullFilename}'");

View File

@ -15,7 +15,7 @@
public string Describe() public string Describe()
{ {
return $"[{RunningPod.Ip}]"; return string.Join(",", Containers.Select(c => c.GetName()));
} }
} }

View File

@ -0,0 +1,65 @@
using DistTestCore;
using NUnit.Framework;
using Utils;
namespace Tests.DurabilityTests
{
[TestFixture]
public class DurabilityTests : DistTest
{
[Test]
public void BootstrapNodeDisappearsTest()
{
var bootstrapNode = SetupCodexNodes(1).BringOnline();
var group = SetupCodexNodes(2).WithBootstrapNode(bootstrapNode[0]).BringOnline();
var primary = group[0];
var secondary = group[1];
// There is 1 minute of time for the nodes to connect to each other.
// (Should be easy, they're in the same pod.)
Time.Sleep(TimeSpan.FromMinutes(1));
bootstrapNode.BringOffline();
var file = GenerateTestFile(10.MB());
var contentId = primary.UploadFile(file);
var downloadedFile = secondary.DownloadContent(contentId);
file.AssertIsEqual(downloadedFile);
}
[Test]
public void DataRetentionTest()
{
var bootstrapNode = SetupCodexNodes(1).BringOnline()[0];
var startGroup = SetupCodexNodes(2).WithBootstrapNode(bootstrapNode).BringOnline();
var finishGroup = SetupCodexNodes(10).WithBootstrapNode(bootstrapNode).BringOnline();
var file = GenerateTestFile(10.MB());
// Both nodes in the start group have the file.
var content = startGroup[0].UploadFile(file);
DownloadAndAssert(content, file, startGroup[1]);
// Three nodes of the finish group have the file.
DownloadAndAssert(content, file, finishGroup[0]);
DownloadAndAssert(content, file, finishGroup[1]);
DownloadAndAssert(content, file, finishGroup[2]);
// The start group goes away.
startGroup.BringOffline();
// All nodes in the finish group can access the file.
foreach (var node in finishGroup)
{
DownloadAndAssert(content, file, node);
}
}
private void DownloadAndAssert(ContentId content, TestFile file, IOnlineCodexNode onlineCodexNode)
{
var downloaded = onlineCodexNode.DownloadContent(content);
file.AssertIsEqual(downloaded);
}
}
}