status-im
/
cs-codex-dist-tests
mirror of
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Sets up some durability tests.

This commit is contained in:
benbierens 2023-04-19 14:57:00 +02:00
parent baf605464c
commit cdd51f6dc7
No known key found for this signature in database
GPG Key ID: FE44815D96D0A1AA
8 changed files with 112 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
DistTestCore/Codex/CodexStartupConfig.cs
View File

@ -10,7 +10,6 @@ namespace DistTestCore.Codex
public ByteSize? StorageQuota { get; set; }
public bool MetricsEnabled { get; set; }
public MarketplaceInitialConfig? MarketplaceConfig { get; set; }
//public IOnlineCodexNode? BootstrapNode { get; private set; }
public IOnlineCodexNode? BootstrapNode { get; set; }
}
}

2
DistTestCore/CodexNodeGroup.cs
View File

@ -59,7 +59,7 @@ namespace DistTestCore
public string Describe()
{
return $"<CodexNodeGroup@{Containers.Describe()}-{Setup.Describe()}>";
return $"group:[{Containers.Describe()}]";
}
private OnlineCodexNode CreateOnlineCodexNode(RunningContainer c, ICodexNodeFactory factory)

35
DistTestCore/CodexSetup.cs
View File

@ -8,7 +8,7 @@ namespace DistTestCore
{
ICodexSetup At(Location location);
ICodexSetup WithLogLevel(CodexLogLevel level);
//ICodexStartupConfig WithBootstrapNode(IOnlineCodexNode node);
ICodexSetup WithBootstrapNode(IOnlineCodexNode node);
ICodexSetup WithStorageQuota(ByteSize storageQuota);
ICodexSetup EnableMetrics();
ICodexSetup EnableMarketplace(TestToken initialBalance);
@ -30,7 +30,24 @@ namespace DistTestCore
public ICodexNodeGroup BringOnline()
{
return starter.BringOnline(this);
var group = starter.BringOnline(this);
ConnectToBootstrapNode(group);
return group;
}
private void ConnectToBootstrapNode(ICodexNodeGroup group)
{
if (BootstrapNode == null) return;
// TODO:
// node.ConnectToPeer uses the '/api/codex/vi/connect/' endpoint to make the connection.
// This should be replaced by injecting the bootstrap node's SPR into the env-vars of the new node containers. (Easy!)
// However, NAT isn't figure out yet. So connecting with SPR doesn't (always?) work.
// So for now, ConnectToPeer
foreach (var node in group)
{
node.ConnectToPeer(BootstrapNode);
}
}
public ICodexSetup At(Location location)
@ -39,11 +56,11 @@ namespace DistTestCore
return this;
}
//public ICodexSetupConfig WithBootstrapNode(IOnlineCodexNode node)
//{
// BootstrapNode = node;
// return this;
//}
public ICodexSetup WithBootstrapNode(IOnlineCodexNode node)
{
BootstrapNode = node;
return this;
}
public ICodexSetup WithLogLevel(CodexLogLevel level)
{
@ -77,13 +94,13 @@ namespace DistTestCore
public string Describe()
{
var args = string.Join(',', DescribeArgs());
return $"({NumberOfNodes} CodexNodes with [{args}])";
return $"({NumberOfNodes} CodexNodes with args:[{args}])";
}
private IEnumerable<string> DescribeArgs()
{
if (LogLevel != null) yield return $"LogLevel={LogLevel}";
//if (BootstrapNode != null) yield return "BootstrapNode=set-not-shown-here";
if (BootstrapNode != null) yield return $"BootstrapNode={BootstrapNode.GetName()}";
if (StorageQuota != null) yield return $"StorageQuote={StorageQuota}";
}
}

2
DistTestCore/CodexStarter.cs
View File

@ -29,7 +29,7 @@ namespace DistTestCore
var codexNodeFactory = new CodexNodeFactory(lifecycle, metricAccessFactory, gethStartResult.MarketplaceAccessFactory);
var group = CreateCodexGroup(codexSetup, containers, codexNodeFactory);
LogEnd($"Started {codexSetup.NumberOfNodes} nodes at '{group.Containers.RunningPod.Ip}'. They are: [{string.Join(",", group.Select(n => n.GetName()))}]");
LogEnd($"Started {codexSetup.NumberOfNodes} nodes at '{group.Containers.RunningPod.Ip}'. They are: {group.Describe()}");
LogSeparator();
return group;
}

26
DistTestCore/OnlineCodexNode.cs
View File

@ -23,6 +23,7 @@ namespace DistTestCore
private const string SuccessfullyConnectedMessage = "Successfully connected to peer";
private const string UploadFailedMessage = "Unable to store block";
private readonly TestLifecycle lifecycle;
private CodexDebugResponse? debugInfo;
public OnlineCodexNode(TestLifecycle lifecycle, CodexAccess codexAccess, CodexNodeGroup group, IMetricsAccess metricsAccess, IMarketplaceAccess marketplaceAccess)
{
@ -45,9 +46,11 @@ namespace DistTestCore
public CodexDebugResponse GetDebugInfo()
{
var response = CodexAccess.GetDebugInfo();
Log($"Got DebugInfo with id: '{response.id}'.");
return response;
if (debugInfo != null) return debugInfo;
debugInfo = CodexAccess.GetDebugInfo();
Log($"Got DebugInfo with id: '{debugInfo.id}'.");
return debugInfo;
}
public ContentId UploadFile(TestFile file)
@ -89,11 +92,6 @@ namespace DistTestCore
return lifecycle.DownloadLog(this);
}
public string Describe()
{
return $"({GetName()} in {Group.Describe()})";
}
private string GetPeerMultiAddress(OnlineCodexNode peer, CodexDebugResponse peerInfo)
{
var multiAddress = peerInfo.addrs.First();
@ -112,8 +110,16 @@ namespace DistTestCore
private void DownloadToFile(string contentId, TestFile file)
{
using var fileStream = File.OpenWrite(file.Filename);
using var downloadStream = CodexAccess.DownloadFile(contentId);
downloadStream.CopyTo(fileStream);
try
{
using var downloadStream = CodexAccess.DownloadFile(contentId);
downloadStream.CopyTo(fileStream);
}
catch
{
Log($"Failed to download file '{contentId}'.");
throw;
}
}
private void Log(string msg)

2
DistTestCore/TestLifecycle.cs
View File

@ -34,7 +34,7 @@ namespace DistTestCore
public ICodexNodeLog DownloadLog(OnlineCodexNode node)
{
var subFile = Log.CreateSubfile();
var description = node.Describe();
var description = node.GetName();
var handler = new LogDownloadHandler(node, description, subFile);
Log.Log($"Downloading logs for {description} to file '{subFile.FullFilename}'");

2
KubernetesWorkflow/RunningContainers.cs
View File

@ -15,7 +15,7 @@
public string Describe()
{
return $"[{RunningPod.Ip}]";
return string.Join(",", Containers.Select(c => c.GetName()));
}
}

65
Tests/DurabilityTests/DurabilityTests.cs Normal file
View File

@ -0,0 +1,65 @@
using DistTestCore;
using NUnit.Framework;
using Utils;
namespace Tests.DurabilityTests
{
[TestFixture]
public class DurabilityTests : DistTest
{
[Test]
public void BootstrapNodeDisappearsTest()
{
var bootstrapNode = SetupCodexNodes(1).BringOnline();
var group = SetupCodexNodes(2).WithBootstrapNode(bootstrapNode[0]).BringOnline();
var primary = group[0];
var secondary = group[1];
// There is 1 minute of time for the nodes to connect to each other.
// (Should be easy, they're in the same pod.)
Time.Sleep(TimeSpan.FromMinutes(1));
bootstrapNode.BringOffline();
var file = GenerateTestFile(10.MB());
var contentId = primary.UploadFile(file);
var downloadedFile = secondary.DownloadContent(contentId);
file.AssertIsEqual(downloadedFile);
}
[Test]
public void DataRetentionTest()
{
var bootstrapNode = SetupCodexNodes(1).BringOnline()[0];
var startGroup = SetupCodexNodes(2).WithBootstrapNode(bootstrapNode).BringOnline();
var finishGroup = SetupCodexNodes(10).WithBootstrapNode(bootstrapNode).BringOnline();
var file = GenerateTestFile(10.MB());
// Both nodes in the start group have the file.
var content = startGroup[0].UploadFile(file);
DownloadAndAssert(content, file, startGroup[1]);
// Three nodes of the finish group have the file.
DownloadAndAssert(content, file, finishGroup[0]);
DownloadAndAssert(content, file, finishGroup[1]);
DownloadAndAssert(content, file, finishGroup[2]);
// The start group goes away.
startGroup.BringOffline();
// All nodes in the finish group can access the file.
foreach (var node in finishGroup)
{
DownloadAndAssert(content, file, node);
}
}
private void DownloadAndAssert(ContentId content, TestFile file, IOnlineCodexNode onlineCodexNode)
{
var downloaded = onlineCodexNode.DownloadContent(content);
file.AssertIsEqual(downloaded);
}
}
}