codex-storage/cs-codex-dist-tests
1
0
Fork 0
mirror of https://github.com/codex-storage/cs-codex-dist-tests.git synced 2025-02-10 11:06:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

Replaces delete with disk info log.

Browse Source
This commit is contained in:
Ben 2024-06-12 15:28:08 +02:00
parent 4fc9835f43
commit 390b9de54a
No known key found for this signature in database
GPG Key ID: 541B9D8C9F1426A1
8 changed files with 54 additions and 111 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
ProjectPlugins/CodexPlugin/CodexAccess.cs
View File

@ -64,27 +64,17 @@ namespace CodexPlugin
public string UploadFile(FileStream fileStream, Action<Failure> onFailure) public string UploadFile(FileStream fileStream, Action<Failure> onFailure)
{ {
LogDiskSpace("Before upload"); return OnCodex(
var response = OnCodex(
api => api.UploadAsync(fileStream), api => api.UploadAsync(fileStream),
CreateRetryConfig(nameof(UploadFile), onFailure)); CreateRetryConfig(nameof(UploadFile), onFailure));
LogDiskSpace("After upload");
return response;
} }
public Stream DownloadFile(string contentId, Action<Failure> onFailure) public Stream DownloadFile(string contentId, Action<Failure> onFailure)
{ {
LogDiskSpace("Before download");
var fileResponse = OnCodex( var fileResponse = OnCodex(
api => api.DownloadNetworkAsync(contentId), api => api.DownloadNetworkAsync(contentId),
CreateRetryConfig(nameof(DownloadFile), onFailure)); CreateRetryConfig(nameof(DownloadFile), onFailure));
LogDiskSpace("After download");
if (fileResponse.StatusCode != 200) throw new Exception("Download failed with StatusCode: " + fileResponse.StatusCode); if (fileResponse.StatusCode != 200) throw new Exception("Download failed with StatusCode: " + fileResponse.StatusCode);
return fileResponse.Stream; return fileResponse.Stream;
} }
@ -138,13 +128,17 @@ namespace CodexPlugin
return workflow.GetPodInfo(Container); return workflow.GetPodInfo(Container);
} }
public void DeleteRepoFolder() public void LogDiskSpace(string msg)
{ {
var containerNumber = Container.Containers.First().Recipe.Number; try
var dataDir = $"datadir{containerNumber}"; {
var workflow = tools.CreateWorkflow(); var diskInfo = tools.CreateWorkflow().ExecuteCommand(Container.Containers.Single(), "df", "--sync");
workflow.ExecuteCommand(Container.Containers.First(), "rm", "-Rfv", $"/codex/{dataDir}/repo"); Log($"{msg} - Disk info: {diskInfo}");
Log("Deleted repo folder."); }
catch (Exception e)
{
Log("Failed to get disk info: " + e);
}
} }
private T OnCodex<T>(Func<CodexApi, Task<T>> action) private T OnCodex<T>(Func<CodexApi, Task<T>> action)
@ -250,19 +244,6 @@ namespace CodexPlugin
} }
} }
private void LogDiskSpace(string msg)
{
try
{
var diskInfo = tools.CreateWorkflow().ExecuteCommand(Container.Containers.Single(), "df", "-h");
Log($"{GetName()} - {msg} - Disk info: {diskInfo}");
}
catch (Exception e)
{
Log("Failed to get disk info: " + e);
}
}
private void Throw(Failure failure) private void Throw(Failure failure)
{ {
throw failure.Exception; throw failure.Exception;
@ -270,7 +251,7 @@ namespace CodexPlugin
private void Log(string msg) private void Log(string msg)
{ {
log.Log(msg); log.Log($"{GetName()} {msg}");
} }
private void DownloadLog() private void DownloadLog()

15
ProjectPlugins/CodexPlugin/CodexNode.cs
View File

@ -26,7 +26,6 @@ namespace CodexPlugin
CrashWatcher CrashWatcher { get; } CrashWatcher CrashWatcher { get; }
PodInfo GetPodInfo(); PodInfo GetPodInfo();
ITransferSpeeds TransferSpeeds { get; } ITransferSpeeds TransferSpeeds { get; }
void DeleteRepoFolder();
void Stop(bool waitTillStopped); void Stop(bool waitTillStopped);
} }
@ -100,6 +99,8 @@ namespace CodexPlugin
public ContentId UploadFile(TrackedFile file, Action<Failure> onFailure) public ContentId UploadFile(TrackedFile file, Action<Failure> onFailure)
{ {
CodexAccess.LogDiskSpace("Before upload");
using var fileStream = File.OpenRead(file.Filename); using var fileStream = File.OpenRead(file.Filename);
var logMessage = $"Uploading file {file.Describe()}..."; var logMessage = $"Uploading file {file.Describe()}...";
@ -116,6 +117,8 @@ namespace CodexPlugin
if (response.StartsWith(UploadFailedMessage)) FrameworkAssert.Fail("Node failed to store block."); if (response.StartsWith(UploadFailedMessage)) FrameworkAssert.Fail("Node failed to store block.");
Log($"Uploaded file. Received contentId: '{response}'."); Log($"Uploaded file. Received contentId: '{response}'.");
CodexAccess.LogDiskSpace("After upload");
return new ContentId(response); return new ContentId(response);
} }
@ -161,13 +164,9 @@ namespace CodexPlugin
return CodexAccess.GetPodInfo(); return CodexAccess.GetPodInfo();
} }
public void DeleteRepoFolder()
{
CodexAccess.DeleteRepoFolder();
}
public void Stop(bool waitTillStopped) public void Stop(bool waitTillStopped)
{ {
Log("Stopping...");
CrashWatcher.Stop(); CrashWatcher.Stop();
Group.Stop(this, waitTillStopped); Group.Stop(this, waitTillStopped);
} }
@ -203,6 +202,8 @@ namespace CodexPlugin
private void DownloadToFile(string contentId, TrackedFile file, Action<Failure> onFailure) private void DownloadToFile(string contentId, TrackedFile file, Action<Failure> onFailure)
{ {
CodexAccess.LogDiskSpace("Before download");
using var fileStream = File.OpenWrite(file.Filename); using var fileStream = File.OpenWrite(file.Filename);
try try
{ {
@ -214,6 +215,8 @@ namespace CodexPlugin
Log($"Failed to download file '{contentId}'."); Log($"Failed to download file '{contentId}'.");
throw; throw;
} }
CodexAccess.LogDiskSpace("After download");
} }
private void Log(string msg) private void Log(string msg)

4
Tests/CodexLongTests/DownloadConnectivityTests/LongFullyConnectedDownloadTests.cs
View File

@ -15,9 +15,9 @@ namespace CodexLongTests.DownloadConnectivityTests
[Values(10, 15, 20)] int numberOfNodes, [Values(10, 15, 20)] int numberOfNodes,
[Values(10, 100)] int sizeMBs) [Values(10, 100)] int sizeMBs)
{ {
for (var i = 0; i < numberOfNodes; i++) StartCodex(); var nodes = StartCodex(numberOfNodes);
CreatePeerDownloadTestHelpers().AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes(), sizeMBs.MB()); CreatePeerDownloadTestHelpers().AssertFullDownloadInterconnectivity(nodes, sizeMBs.MB());
} }
} }
} }

48
Tests/CodexTests/CodexDistTest.cs
View File

@ -14,8 +14,6 @@ namespace CodexTests
{ {
public class CodexDistTest : DistTest public class CodexDistTest : DistTest
{ {
private readonly Dictionary<TestLifecycle, List<ICodexNode>> onlineCodexNodes = new Dictionary<TestLifecycle, List<ICodexNode>>();
public CodexDistTest() public CodexDistTest()
{ {
ProjectPlugin.Load<CodexPlugin.CodexPlugin>(); ProjectPlugin.Load<CodexPlugin.CodexPlugin>();
@ -31,17 +29,6 @@ namespace CodexTests
localBuilder.Build(); localBuilder.Build();
} }
protected override void LifecycleStart(TestLifecycle lifecycle)
{
onlineCodexNodes.Add(lifecycle, new List<ICodexNode>());
}
protected override void LifecycleStop(TestLifecycle lifecycle)
{
DeleteBlockRepo(onlineCodexNodes[lifecycle]);
onlineCodexNodes.Remove(lifecycle);
}
public ICodexNode StartCodex() public ICodexNode StartCodex()
{ {
return StartCodex(s => { }); return StartCodex(s => { });
@ -64,7 +51,7 @@ namespace CodexTests
setup(s); setup(s);
OnCodexSetup(s); OnCodexSetup(s);
}); });
onlineCodexNodes[Get()].AddRange(group);
return group; return group;
} }
@ -78,11 +65,6 @@ namespace CodexTests
return new PeerDownloadTestHelpers(GetTestLog(), Get().GetFileManager()); return new PeerDownloadTestHelpers(GetTestLog(), Get().GetFileManager());
} }
public IEnumerable<ICodexNode> GetAllOnlineCodexNodes()
{
return onlineCodexNodes[Get()];
}
public void AssertBalance(ICodexContracts contracts, ICodexNode codexNode, Constraint constraint, string msg = "") public void AssertBalance(ICodexContracts contracts, ICodexNode codexNode, Constraint constraint, string msg = "")
{ {
AssertHelpers.RetryAssert(constraint, () => contracts.GetTestTokenBalance(codexNode), nameof(AssertBalance) + msg); AssertHelpers.RetryAssert(constraint, () => contracts.GetTestTokenBalance(codexNode), nameof(AssertBalance) + msg);
@ -126,33 +108,5 @@ namespace CodexTests
protected virtual void OnCodexSetup(ICodexSetup setup) protected virtual void OnCodexSetup(ICodexSetup setup)
{ {
} }
protected override void CollectStatusLogData(TestLifecycle lifecycle, Dictionary<string, string> data)
{
var nodes = onlineCodexNodes[lifecycle];
var upload = nodes.Select(n => n.TransferSpeeds.GetUploadSpeed()).ToList()!.OptionalAverage();
var download = nodes.Select(n => n.TransferSpeeds.GetDownloadSpeed()).ToList()!.OptionalAverage();
if (upload != null) data.Add("avgupload", upload.ToString());
if (download != null) data.Add("avgdownload", download.ToString());
}
private void DeleteBlockRepo(List<ICodexNode> codexNodes)
{
foreach (var node in codexNodes)
{
try
{
if (node.CrashWatcher.HasContainerCrashed())
{
Log("Crash detected!");
}
node.DeleteRepoFolder();
}
catch (Exception ex)
{
Log($"Failed to delete repo folder for node {node.GetName()} : {ex}");
}
}
}
} }
} }

17
Tests/CodexTests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs
View File

@ -1,4 +1,5 @@
using CodexContractsPlugin; using CodexContractsPlugin;
using CodexPlugin;
using GethPlugin; using GethPlugin;
using NUnit.Framework; using NUnit.Framework;
using Utils; using Utils;
@ -11,9 +12,9 @@ namespace CodexTests.DownloadConnectivityTests
[Test] [Test]
public void MetricsDoesNotInterfereWithPeerDownload() public void MetricsDoesNotInterfereWithPeerDownload()
{ {
StartCodex(2, s => s.EnableMetrics()); var nodes = StartCodex(2, s => s.EnableMetrics());
AssertAllNodesConnected(); AssertAllNodesConnected(nodes);
} }
[Test] [Test]
@ -21,10 +22,10 @@ namespace CodexTests.DownloadConnectivityTests
{ {
var geth = Ci.StartGethNode(s => s.IsMiner()); var geth = Ci.StartGethNode(s => s.IsMiner());
var contracts = Ci.StartCodexContracts(geth); var contracts = Ci.StartCodexContracts(geth);
StartCodex(2, s => s.EnableMarketplace(geth, contracts, m => m var nodes = StartCodex(2, s => s.EnableMarketplace(geth, contracts, m => m
.WithInitial(10.Eth(), 1000.TstWei()))); .WithInitial(10.Eth(), 1000.TstWei())));
AssertAllNodesConnected(); AssertAllNodesConnected(nodes);
} }
[Test] [Test]
@ -33,14 +34,14 @@ namespace CodexTests.DownloadConnectivityTests
[Values(2, 5)] int numberOfNodes, [Values(2, 5)] int numberOfNodes,
[Values(1, 10)] int sizeMBs) [Values(1, 10)] int sizeMBs)
{ {
StartCodex(numberOfNodes); var nodes = StartCodex(numberOfNodes);
AssertAllNodesConnected(sizeMBs); AssertAllNodesConnected(nodes, sizeMBs);
} }
private void AssertAllNodesConnected(int sizeMBs = 10) private void AssertAllNodesConnected(IEnumerable<ICodexNode> nodes, int sizeMBs = 10)
{ {
CreatePeerDownloadTestHelpers().AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes(), sizeMBs.MB()); CreatePeerDownloadTestHelpers().AssertFullDownloadInterconnectivity(nodes, sizeMBs.MB());
} }
} }
} }

17
Tests/CodexTests/PeerDiscoveryTests/LayeredDiscoveryTests.cs
View File

@ -1,4 +1,5 @@
using NUnit.Framework; using CodexPlugin;
using NUnit.Framework;
namespace CodexTests.PeerDiscoveryTests namespace CodexTests.PeerDiscoveryTests
{ {
@ -13,7 +14,7 @@ namespace CodexTests.PeerDiscoveryTests
var l1Node = StartCodex(s => s.WithBootstrapNode(root)); var l1Node = StartCodex(s => s.WithBootstrapNode(root));
var l2Target = StartCodex(s => s.WithBootstrapNode(l1Node)); var l2Target = StartCodex(s => s.WithBootstrapNode(l1Node));
AssertAllNodesConnected(); AssertAllNodesConnected(root, l1Source, l1Node, l2Target);
} }
[Test] [Test]
@ -25,7 +26,7 @@ namespace CodexTests.PeerDiscoveryTests
var l2Node = StartCodex(s => s.WithBootstrapNode(l1Node)); var l2Node = StartCodex(s => s.WithBootstrapNode(l1Node));
var l3Target = StartCodex(s => s.WithBootstrapNode(l2Node)); var l3Target = StartCodex(s => s.WithBootstrapNode(l2Node));
AssertAllNodesConnected(); AssertAllNodesConnected(root, l1Source, l1Node, l2Node, l3Target);
} }
[TestCase(3)] [TestCase(3)]
@ -33,18 +34,22 @@ namespace CodexTests.PeerDiscoveryTests
[TestCase(10)] [TestCase(10)]
public void NodeChainTest(int chainLength) public void NodeChainTest(int chainLength)
{ {
var nodes = new List<ICodexNode>();
var node = StartCodex(); var node = StartCodex();
nodes.Add(node);
for (var i = 1; i < chainLength; i++) for (var i = 1; i < chainLength; i++)
{ {
node = StartCodex(s => s.WithBootstrapNode(node)); node = StartCodex(s => s.WithBootstrapNode(node));
nodes.Add(node);
} }
AssertAllNodesConnected(); AssertAllNodesConnected(nodes.ToArray());
} }
private void AssertAllNodesConnected() private void AssertAllNodesConnected(params ICodexNode[] nodes)
{ {
CreatePeerConnectionTestHelpers().AssertFullyConnected(GetAllOnlineCodexNodes()); CreatePeerConnectionTestHelpers().AssertFullyConnected(nodes);
} }
} }
} }

19
Tests/CodexTests/PeerDiscoveryTests/PeerDiscoveryTests.cs
View File

@ -21,9 +21,9 @@ namespace CodexTests.PeerDiscoveryTests
[Test] [Test]
public void MetricsDoesNotInterfereWithPeerDiscovery() public void MetricsDoesNotInterfereWithPeerDiscovery()
{ {
StartCodex(2, s => s.EnableMetrics()); var nodes = StartCodex(2, s => s.EnableMetrics());
AssertAllNodesConnected(); AssertAllNodesConnected(nodes);
} }
[Test] [Test]
@ -31,10 +31,10 @@ namespace CodexTests.PeerDiscoveryTests
{ {
var geth = Ci.StartGethNode(s => s.IsMiner()); var geth = Ci.StartGethNode(s => s.IsMiner());
var contracts = Ci.StartCodexContracts(geth); var contracts = Ci.StartCodexContracts(geth);
StartCodex(2, s => s.EnableMarketplace(geth, contracts, m => m var nodes = StartCodex(2, s => s.EnableMarketplace(geth, contracts, m => m
.WithInitial(10.Eth(), 1000.TstWei()))); .WithInitial(10.Eth(), 1000.TstWei())));
AssertAllNodesConnected(); AssertAllNodesConnected(nodes);
} }
[TestCase(2)] [TestCase(2)]
@ -42,16 +42,15 @@ namespace CodexTests.PeerDiscoveryTests
[TestCase(10)] [TestCase(10)]
public void VariableNodes(int number) public void VariableNodes(int number)
{ {
StartCodex(number); var nodes = StartCodex(number);
AssertAllNodesConnected(); AssertAllNodesConnected(nodes);
} }
private void AssertAllNodesConnected() private void AssertAllNodesConnected(IEnumerable<ICodexNode> nodes)
{ {
var allNodes = GetAllOnlineCodexNodes(); CreatePeerConnectionTestHelpers().AssertFullyConnected(nodes);
CreatePeerConnectionTestHelpers().AssertFullyConnected(allNodes); CheckRoutingTable(nodes);
CheckRoutingTable(allNodes);
} }
private void CheckRoutingTable(IEnumerable<ICodexNode> allNodes) private void CheckRoutingTable(IEnumerable<ICodexNode> allNodes)

2
Tests/DistTestCore/DistTest.cs
View File

@ -215,7 +215,7 @@ namespace DistTestCore
IncludeLogsOnTestFailure(lifecycle); IncludeLogsOnTestFailure(lifecycle);
LifecycleStop(lifecycle); LifecycleStop(lifecycle);
lifecycle.DeleteAllResources(); lifecycle.DeleteAllResources();
lifecycle = null!; lifecycles.Remove(GetCurrentTestName());
}); });
} }