codex-storage
/
cs-codex-dist-tests
mirror of https://github.com/codex-storage/cs-codex-dist-tests.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Replaces delete with disk info log.

This commit is contained in:
Ben 2024-06-12 15:28:08 +02:00
parent 4fc9835f43
commit 390b9de54a
No known key found for this signature in database
GPG Key ID: 541B9D8C9F1426A1
8 changed files with 54 additions and 111 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
ProjectPlugins/CodexPlugin/CodexAccess.cs
View File

@ -64,27 +64,17 @@ namespace CodexPlugin
public string UploadFile(FileStream fileStream, Action<Failure> onFailure)
{
LogDiskSpace("Before upload");
var response = OnCodex(
return OnCodex(
api => api.UploadAsync(fileStream),
CreateRetryConfig(nameof(UploadFile), onFailure));
LogDiskSpace("After upload");
return response;
}
public Stream DownloadFile(string contentId, Action<Failure> onFailure)
{
LogDiskSpace("Before download");
var fileResponse = OnCodex(
api => api.DownloadNetworkAsync(contentId),
CreateRetryConfig(nameof(DownloadFile), onFailure));
LogDiskSpace("After download");
if (fileResponse.StatusCode != 200) throw new Exception("Download failed with StatusCode: " + fileResponse.StatusCode);
return fileResponse.Stream;
}
@ -138,13 +128,17 @@ namespace CodexPlugin
return workflow.GetPodInfo(Container);
}
public void DeleteRepoFolder()
public void LogDiskSpace(string msg)
{
var containerNumber = Container.Containers.First().Recipe.Number;
var dataDir = $"datadir{containerNumber}";
var workflow = tools.CreateWorkflow();
workflow.ExecuteCommand(Container.Containers.First(), "rm", "-Rfv", $"/codex/{dataDir}/repo");
Log("Deleted repo folder.");
try
{
var diskInfo = tools.CreateWorkflow().ExecuteCommand(Container.Containers.Single(), "df", "--sync");
Log($"{msg} - Disk info: {diskInfo}");
}
catch (Exception e)
{
Log("Failed to get disk info: " + e);
}
}
private T OnCodex<T>(Func<CodexApi, Task<T>> action)
@ -250,19 +244,6 @@ namespace CodexPlugin
}
}
private void LogDiskSpace(string msg)
{
try
{
var diskInfo = tools.CreateWorkflow().ExecuteCommand(Container.Containers.Single(), "df", "-h");
Log($"{GetName()} - {msg} - Disk info: {diskInfo}");
}
catch (Exception e)
{
Log("Failed to get disk info: " + e);
}
}
private void Throw(Failure failure)
{
throw failure.Exception;
@ -270,7 +251,7 @@ namespace CodexPlugin
private void Log(string msg)
{
log.Log(msg);
log.Log($"{GetName()} {msg}");
}
private void DownloadLog()

15
ProjectPlugins/CodexPlugin/CodexNode.cs
View File

@ -26,7 +26,6 @@ namespace CodexPlugin
CrashWatcher CrashWatcher { get; }
PodInfo GetPodInfo();
ITransferSpeeds TransferSpeeds { get; }
void DeleteRepoFolder();
void Stop(bool waitTillStopped);
}
@ -100,6 +99,8 @@ namespace CodexPlugin
public ContentId UploadFile(TrackedFile file, Action<Failure> onFailure)
{
CodexAccess.LogDiskSpace("Before upload");
using var fileStream = File.OpenRead(file.Filename);
var logMessage = $"Uploading file {file.Describe()}...";
@ -116,6 +117,8 @@ namespace CodexPlugin
if (response.StartsWith(UploadFailedMessage)) FrameworkAssert.Fail("Node failed to store block.");
Log($"Uploaded file. Received contentId: '{response}'.");
CodexAccess.LogDiskSpace("After upload");
return new ContentId(response);
}
@ -161,13 +164,9 @@ namespace CodexPlugin
return CodexAccess.GetPodInfo();
}
public void DeleteRepoFolder()
{
CodexAccess.DeleteRepoFolder();
}
public void Stop(bool waitTillStopped)
{
Log("Stopping...");
CrashWatcher.Stop();
Group.Stop(this, waitTillStopped);
}
@ -203,6 +202,8 @@ namespace CodexPlugin
private void DownloadToFile(string contentId, TrackedFile file, Action<Failure> onFailure)
{
CodexAccess.LogDiskSpace("Before download");
using var fileStream = File.OpenWrite(file.Filename);
try
{
@ -214,6 +215,8 @@ namespace CodexPlugin
Log($"Failed to download file '{contentId}'.");
throw;
}
CodexAccess.LogDiskSpace("After download");
}
private void Log(string msg)

4
Tests/CodexLongTests/DownloadConnectivityTests/LongFullyConnectedDownloadTests.cs
View File

@ -15,9 +15,9 @@ namespace CodexLongTests.DownloadConnectivityTests
[Values(10, 15, 20)] int numberOfNodes,
[Values(10, 100)] int sizeMBs)
{
for (var i = 0; i < numberOfNodes; i++) StartCodex();
var nodes = StartCodex(numberOfNodes);
CreatePeerDownloadTestHelpers().AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes(), sizeMBs.MB());
CreatePeerDownloadTestHelpers().AssertFullDownloadInterconnectivity(nodes, sizeMBs.MB());
}
}
}

48
Tests/CodexTests/CodexDistTest.cs
View File

@ -14,8 +14,6 @@ namespace CodexTests
{
public class CodexDistTest : DistTest
{
private readonly Dictionary<TestLifecycle, List<ICodexNode>> onlineCodexNodes = new Dictionary<TestLifecycle, List<ICodexNode>>();
public CodexDistTest()
{
ProjectPlugin.Load<CodexPlugin.CodexPlugin>();
@ -31,17 +29,6 @@ namespace CodexTests
localBuilder.Build();
}
protected override void LifecycleStart(TestLifecycle lifecycle)
{
onlineCodexNodes.Add(lifecycle, new List<ICodexNode>());
}
protected override void LifecycleStop(TestLifecycle lifecycle)
{
DeleteBlockRepo(onlineCodexNodes[lifecycle]);
onlineCodexNodes.Remove(lifecycle);
}
public ICodexNode StartCodex()
{
return StartCodex(s => { });
@ -64,7 +51,7 @@ namespace CodexTests
setup(s);
OnCodexSetup(s);
});
onlineCodexNodes[Get()].AddRange(group);
return group;
}
@ -78,11 +65,6 @@ namespace CodexTests
return new PeerDownloadTestHelpers(GetTestLog(), Get().GetFileManager());
}
public IEnumerable<ICodexNode> GetAllOnlineCodexNodes()
{
return onlineCodexNodes[Get()];
}
public void AssertBalance(ICodexContracts contracts, ICodexNode codexNode, Constraint constraint, string msg = "")
{
AssertHelpers.RetryAssert(constraint, () => contracts.GetTestTokenBalance(codexNode), nameof(AssertBalance) + msg);
@ -126,33 +108,5 @@ namespace CodexTests
protected virtual void OnCodexSetup(ICodexSetup setup)
{
}
protected override void CollectStatusLogData(TestLifecycle lifecycle, Dictionary<string, string> data)
{
var nodes = onlineCodexNodes[lifecycle];
var upload = nodes.Select(n => n.TransferSpeeds.GetUploadSpeed()).ToList()!.OptionalAverage();
var download = nodes.Select(n => n.TransferSpeeds.GetDownloadSpeed()).ToList()!.OptionalAverage();
if (upload != null) data.Add("avgupload", upload.ToString());
if (download != null) data.Add("avgdownload", download.ToString());
}
private void DeleteBlockRepo(List<ICodexNode> codexNodes)
{
foreach (var node in codexNodes)
{
try
{
if (node.CrashWatcher.HasContainerCrashed())
{
Log("Crash detected!");
}
node.DeleteRepoFolder();
}
catch (Exception ex)
{
Log($"Failed to delete repo folder for node {node.GetName()} : {ex}");
}
}
}
}
}

17
Tests/CodexTests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs
View File

@ -1,4 +1,5 @@
using CodexContractsPlugin;
using CodexPlugin;
using GethPlugin;
using NUnit.Framework;
using Utils;
@ -11,9 +12,9 @@ namespace CodexTests.DownloadConnectivityTests
[Test]
public void MetricsDoesNotInterfereWithPeerDownload()
{
StartCodex(2, s => s.EnableMetrics());
var nodes = StartCodex(2, s => s.EnableMetrics());
AssertAllNodesConnected();
AssertAllNodesConnected(nodes);
}
[Test]
@ -21,10 +22,10 @@ namespace CodexTests.DownloadConnectivityTests
{
var geth = Ci.StartGethNode(s => s.IsMiner());
var contracts = Ci.StartCodexContracts(geth);
StartCodex(2, s => s.EnableMarketplace(geth, contracts, m => m
var nodes = StartCodex(2, s => s.EnableMarketplace(geth, contracts, m => m
.WithInitial(10.Eth(), 1000.TstWei())));
AssertAllNodesConnected();
AssertAllNodesConnected(nodes);
}
[Test]
@ -33,14 +34,14 @@ namespace CodexTests.DownloadConnectivityTests
[Values(2, 5)] int numberOfNodes,
[Values(1, 10)] int sizeMBs)
{
StartCodex(numberOfNodes);
var nodes = StartCodex(numberOfNodes);
AssertAllNodesConnected(sizeMBs);
AssertAllNodesConnected(nodes, sizeMBs);
}
private void AssertAllNodesConnected(int sizeMBs = 10)
private void AssertAllNodesConnected(IEnumerable<ICodexNode> nodes, int sizeMBs = 10)
{
CreatePeerDownloadTestHelpers().AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes(), sizeMBs.MB());
CreatePeerDownloadTestHelpers().AssertFullDownloadInterconnectivity(nodes, sizeMBs.MB());
}
}
}

17
Tests/CodexTests/PeerDiscoveryTests/LayeredDiscoveryTests.cs
View File

@ -1,4 +1,5 @@
using NUnit.Framework;
using CodexPlugin;
using NUnit.Framework;
namespace CodexTests.PeerDiscoveryTests
{
@ -13,7 +14,7 @@ namespace CodexTests.PeerDiscoveryTests
var l1Node = StartCodex(s => s.WithBootstrapNode(root));
var l2Target = StartCodex(s => s.WithBootstrapNode(l1Node));
AssertAllNodesConnected();
AssertAllNodesConnected(root, l1Source, l1Node, l2Target);
}
[Test]
@ -25,7 +26,7 @@ namespace CodexTests.PeerDiscoveryTests
var l2Node = StartCodex(s => s.WithBootstrapNode(l1Node));
var l3Target = StartCodex(s => s.WithBootstrapNode(l2Node));
AssertAllNodesConnected();
AssertAllNodesConnected(root, l1Source, l1Node, l2Node, l3Target);
}
[TestCase(3)]
@ -33,18 +34,22 @@ namespace CodexTests.PeerDiscoveryTests
[TestCase(10)]
public void NodeChainTest(int chainLength)
{
var nodes = new List<ICodexNode>();
var node = StartCodex();
nodes.Add(node);
for (var i = 1; i < chainLength; i++)
{
node = StartCodex(s => s.WithBootstrapNode(node));
nodes.Add(node);
}
AssertAllNodesConnected();
AssertAllNodesConnected(nodes.ToArray());
}
private void AssertAllNodesConnected()
private void AssertAllNodesConnected(params ICodexNode[] nodes)
{
CreatePeerConnectionTestHelpers().AssertFullyConnected(GetAllOnlineCodexNodes());
CreatePeerConnectionTestHelpers().AssertFullyConnected(nodes);
}
}
}

19
Tests/CodexTests/PeerDiscoveryTests/PeerDiscoveryTests.cs
View File

@ -21,9 +21,9 @@ namespace CodexTests.PeerDiscoveryTests
[Test]
public void MetricsDoesNotInterfereWithPeerDiscovery()
{
StartCodex(2, s => s.EnableMetrics());
var nodes = StartCodex(2, s => s.EnableMetrics());
AssertAllNodesConnected();
AssertAllNodesConnected(nodes);
}
[Test]
@ -31,10 +31,10 @@ namespace CodexTests.PeerDiscoveryTests
{
var geth = Ci.StartGethNode(s => s.IsMiner());
var contracts = Ci.StartCodexContracts(geth);
StartCodex(2, s => s.EnableMarketplace(geth, contracts, m => m
var nodes = StartCodex(2, s => s.EnableMarketplace(geth, contracts, m => m
.WithInitial(10.Eth(), 1000.TstWei())));
AssertAllNodesConnected();
AssertAllNodesConnected(nodes);
}
[TestCase(2)]
@ -42,16 +42,15 @@ namespace CodexTests.PeerDiscoveryTests
[TestCase(10)]
public void VariableNodes(int number)
{
StartCodex(number);
var nodes = StartCodex(number);
AssertAllNodesConnected();
AssertAllNodesConnected(nodes);
}
private void AssertAllNodesConnected()
private void AssertAllNodesConnected(IEnumerable<ICodexNode> nodes)
{
var allNodes = GetAllOnlineCodexNodes();
CreatePeerConnectionTestHelpers().AssertFullyConnected(allNodes);
CheckRoutingTable(allNodes);
CreatePeerConnectionTestHelpers().AssertFullyConnected(nodes);
CheckRoutingTable(nodes);
}
private void CheckRoutingTable(IEnumerable<ICodexNode> allNodes)

2
Tests/DistTestCore/DistTest.cs
View File

@ -215,7 +215,7 @@ namespace DistTestCore
IncludeLogsOnTestFailure(lifecycle);
LifecycleStop(lifecycle);
lifecycle.DeleteAllResources();
lifecycle = null!;
lifecycles.Remove(GetCurrentTestName());
});
}