diff --git a/ProjectPlugins/CodexPlugin/CodexAccess.cs b/ProjectPlugins/CodexPlugin/CodexAccess.cs index 34459f4..a99a2ec 100644 --- a/ProjectPlugins/CodexPlugin/CodexAccess.cs +++ b/ProjectPlugins/CodexPlugin/CodexAccess.cs @@ -64,27 +64,17 @@ namespace CodexPlugin public string UploadFile(FileStream fileStream, Action onFailure) { - LogDiskSpace("Before upload"); - - var response = OnCodex( + return OnCodex( api => api.UploadAsync(fileStream), CreateRetryConfig(nameof(UploadFile), onFailure)); - - LogDiskSpace("After upload"); - - return response; } public Stream DownloadFile(string contentId, Action onFailure) { - LogDiskSpace("Before download"); - var fileResponse = OnCodex( api => api.DownloadNetworkAsync(contentId), CreateRetryConfig(nameof(DownloadFile), onFailure)); - LogDiskSpace("After download"); - if (fileResponse.StatusCode != 200) throw new Exception("Download failed with StatusCode: " + fileResponse.StatusCode); return fileResponse.Stream; } @@ -138,13 +128,17 @@ namespace CodexPlugin return workflow.GetPodInfo(Container); } - public void DeleteRepoFolder() + public void LogDiskSpace(string msg) { - var containerNumber = Container.Containers.First().Recipe.Number; - var dataDir = $"datadir{containerNumber}"; - var workflow = tools.CreateWorkflow(); - workflow.ExecuteCommand(Container.Containers.First(), "rm", "-Rfv", $"/codex/{dataDir}/repo"); - Log("Deleted repo folder."); + try + { + var diskInfo = tools.CreateWorkflow().ExecuteCommand(Container.Containers.Single(), "df", "--sync"); + Log($"{msg} - Disk info: {diskInfo}"); + } + catch (Exception e) + { + Log("Failed to get disk info: " + e); + } } private T OnCodex(Func> action) @@ -250,19 +244,6 @@ namespace CodexPlugin } } - private void LogDiskSpace(string msg) - { - try - { - var diskInfo = tools.CreateWorkflow().ExecuteCommand(Container.Containers.Single(), "df", "-h"); - Log($"{GetName()} - {msg} - Disk info: {diskInfo}"); - } - catch (Exception e) - { - Log("Failed to get disk info: " + e); - } - } - private void Throw(Failure failure) { throw failure.Exception; @@ -270,7 +251,7 @@ namespace CodexPlugin private void Log(string msg) { - log.Log(msg); + log.Log($"{GetName()} {msg}"); } private void DownloadLog() diff --git a/ProjectPlugins/CodexPlugin/CodexNode.cs b/ProjectPlugins/CodexPlugin/CodexNode.cs index 4965dbc..f0e064b 100644 --- a/ProjectPlugins/CodexPlugin/CodexNode.cs +++ b/ProjectPlugins/CodexPlugin/CodexNode.cs @@ -26,7 +26,6 @@ namespace CodexPlugin CrashWatcher CrashWatcher { get; } PodInfo GetPodInfo(); ITransferSpeeds TransferSpeeds { get; } - void DeleteRepoFolder(); void Stop(bool waitTillStopped); } @@ -100,6 +99,8 @@ namespace CodexPlugin public ContentId UploadFile(TrackedFile file, Action onFailure) { + CodexAccess.LogDiskSpace("Before upload"); + using var fileStream = File.OpenRead(file.Filename); var logMessage = $"Uploading file {file.Describe()}..."; @@ -116,6 +117,8 @@ namespace CodexPlugin if (response.StartsWith(UploadFailedMessage)) FrameworkAssert.Fail("Node failed to store block."); Log($"Uploaded file. Received contentId: '{response}'."); + CodexAccess.LogDiskSpace("After upload"); + return new ContentId(response); } @@ -161,13 +164,9 @@ namespace CodexPlugin return CodexAccess.GetPodInfo(); } - public void DeleteRepoFolder() - { - CodexAccess.DeleteRepoFolder(); - } - public void Stop(bool waitTillStopped) { + Log("Stopping..."); CrashWatcher.Stop(); Group.Stop(this, waitTillStopped); } @@ -203,6 +202,8 @@ namespace CodexPlugin private void DownloadToFile(string contentId, TrackedFile file, Action onFailure) { + CodexAccess.LogDiskSpace("Before download"); + using var fileStream = File.OpenWrite(file.Filename); try { @@ -214,6 +215,8 @@ namespace CodexPlugin Log($"Failed to download file '{contentId}'."); throw; } + + CodexAccess.LogDiskSpace("After download"); } private void Log(string msg) diff --git a/Tests/CodexLongTests/DownloadConnectivityTests/LongFullyConnectedDownloadTests.cs b/Tests/CodexLongTests/DownloadConnectivityTests/LongFullyConnectedDownloadTests.cs index 15fc9f3..eaa999e 100644 --- a/Tests/CodexLongTests/DownloadConnectivityTests/LongFullyConnectedDownloadTests.cs +++ b/Tests/CodexLongTests/DownloadConnectivityTests/LongFullyConnectedDownloadTests.cs @@ -15,9 +15,9 @@ namespace CodexLongTests.DownloadConnectivityTests [Values(10, 15, 20)] int numberOfNodes, [Values(10, 100)] int sizeMBs) { - for (var i = 0; i < numberOfNodes; i++) StartCodex(); + var nodes = StartCodex(numberOfNodes); - CreatePeerDownloadTestHelpers().AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes(), sizeMBs.MB()); + CreatePeerDownloadTestHelpers().AssertFullDownloadInterconnectivity(nodes, sizeMBs.MB()); } } } diff --git a/Tests/CodexTests/CodexDistTest.cs b/Tests/CodexTests/CodexDistTest.cs index 437dbf5..e62627f 100644 --- a/Tests/CodexTests/CodexDistTest.cs +++ b/Tests/CodexTests/CodexDistTest.cs @@ -14,8 +14,6 @@ namespace CodexTests { public class CodexDistTest : DistTest { - private readonly Dictionary> onlineCodexNodes = new Dictionary>(); - public CodexDistTest() { ProjectPlugin.Load(); @@ -31,17 +29,6 @@ namespace CodexTests localBuilder.Build(); } - protected override void LifecycleStart(TestLifecycle lifecycle) - { - onlineCodexNodes.Add(lifecycle, new List()); - } - - protected override void LifecycleStop(TestLifecycle lifecycle) - { - DeleteBlockRepo(onlineCodexNodes[lifecycle]); - onlineCodexNodes.Remove(lifecycle); - } - public ICodexNode StartCodex() { return StartCodex(s => { }); @@ -64,7 +51,7 @@ namespace CodexTests setup(s); OnCodexSetup(s); }); - onlineCodexNodes[Get()].AddRange(group); + return group; } @@ -78,11 +65,6 @@ namespace CodexTests return new PeerDownloadTestHelpers(GetTestLog(), Get().GetFileManager()); } - public IEnumerable GetAllOnlineCodexNodes() - { - return onlineCodexNodes[Get()]; - } - public void AssertBalance(ICodexContracts contracts, ICodexNode codexNode, Constraint constraint, string msg = "") { AssertHelpers.RetryAssert(constraint, () => contracts.GetTestTokenBalance(codexNode), nameof(AssertBalance) + msg); @@ -126,33 +108,5 @@ namespace CodexTests protected virtual void OnCodexSetup(ICodexSetup setup) { } - - protected override void CollectStatusLogData(TestLifecycle lifecycle, Dictionary data) - { - var nodes = onlineCodexNodes[lifecycle]; - var upload = nodes.Select(n => n.TransferSpeeds.GetUploadSpeed()).ToList()!.OptionalAverage(); - var download = nodes.Select(n => n.TransferSpeeds.GetDownloadSpeed()).ToList()!.OptionalAverage(); - if (upload != null) data.Add("avgupload", upload.ToString()); - if (download != null) data.Add("avgdownload", download.ToString()); - } - - private void DeleteBlockRepo(List codexNodes) - { - foreach (var node in codexNodes) - { - try - { - if (node.CrashWatcher.HasContainerCrashed()) - { - Log("Crash detected!"); - } - node.DeleteRepoFolder(); - } - catch (Exception ex) - { - Log($"Failed to delete repo folder for node {node.GetName()} : {ex}"); - } - } - } } } diff --git a/Tests/CodexTests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs b/Tests/CodexTests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs index e40d9fb..e6a42c5 100644 --- a/Tests/CodexTests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs +++ b/Tests/CodexTests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs @@ -1,4 +1,5 @@ using CodexContractsPlugin; +using CodexPlugin; using GethPlugin; using NUnit.Framework; using Utils; @@ -11,9 +12,9 @@ namespace CodexTests.DownloadConnectivityTests [Test] public void MetricsDoesNotInterfereWithPeerDownload() { - StartCodex(2, s => s.EnableMetrics()); + var nodes = StartCodex(2, s => s.EnableMetrics()); - AssertAllNodesConnected(); + AssertAllNodesConnected(nodes); } [Test] @@ -21,10 +22,10 @@ namespace CodexTests.DownloadConnectivityTests { var geth = Ci.StartGethNode(s => s.IsMiner()); var contracts = Ci.StartCodexContracts(geth); - StartCodex(2, s => s.EnableMarketplace(geth, contracts, m => m + var nodes = StartCodex(2, s => s.EnableMarketplace(geth, contracts, m => m .WithInitial(10.Eth(), 1000.TstWei()))); - AssertAllNodesConnected(); + AssertAllNodesConnected(nodes); } [Test] @@ -33,14 +34,14 @@ namespace CodexTests.DownloadConnectivityTests [Values(2, 5)] int numberOfNodes, [Values(1, 10)] int sizeMBs) { - StartCodex(numberOfNodes); + var nodes = StartCodex(numberOfNodes); - AssertAllNodesConnected(sizeMBs); + AssertAllNodesConnected(nodes, sizeMBs); } - private void AssertAllNodesConnected(int sizeMBs = 10) + private void AssertAllNodesConnected(IEnumerable nodes, int sizeMBs = 10) { - CreatePeerDownloadTestHelpers().AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes(), sizeMBs.MB()); + CreatePeerDownloadTestHelpers().AssertFullDownloadInterconnectivity(nodes, sizeMBs.MB()); } } } diff --git a/Tests/CodexTests/PeerDiscoveryTests/LayeredDiscoveryTests.cs b/Tests/CodexTests/PeerDiscoveryTests/LayeredDiscoveryTests.cs index 9c884fe..33225d6 100644 --- a/Tests/CodexTests/PeerDiscoveryTests/LayeredDiscoveryTests.cs +++ b/Tests/CodexTests/PeerDiscoveryTests/LayeredDiscoveryTests.cs @@ -1,4 +1,5 @@ -using NUnit.Framework; +using CodexPlugin; +using NUnit.Framework; namespace CodexTests.PeerDiscoveryTests { @@ -13,7 +14,7 @@ namespace CodexTests.PeerDiscoveryTests var l1Node = StartCodex(s => s.WithBootstrapNode(root)); var l2Target = StartCodex(s => s.WithBootstrapNode(l1Node)); - AssertAllNodesConnected(); + AssertAllNodesConnected(root, l1Source, l1Node, l2Target); } [Test] @@ -25,7 +26,7 @@ namespace CodexTests.PeerDiscoveryTests var l2Node = StartCodex(s => s.WithBootstrapNode(l1Node)); var l3Target = StartCodex(s => s.WithBootstrapNode(l2Node)); - AssertAllNodesConnected(); + AssertAllNodesConnected(root, l1Source, l1Node, l2Node, l3Target); } [TestCase(3)] @@ -33,18 +34,22 @@ namespace CodexTests.PeerDiscoveryTests [TestCase(10)] public void NodeChainTest(int chainLength) { + var nodes = new List(); var node = StartCodex(); + nodes.Add(node); + for (var i = 1; i < chainLength; i++) { node = StartCodex(s => s.WithBootstrapNode(node)); + nodes.Add(node); } - AssertAllNodesConnected(); + AssertAllNodesConnected(nodes.ToArray()); } - private void AssertAllNodesConnected() + private void AssertAllNodesConnected(params ICodexNode[] nodes) { - CreatePeerConnectionTestHelpers().AssertFullyConnected(GetAllOnlineCodexNodes()); + CreatePeerConnectionTestHelpers().AssertFullyConnected(nodes); } } } diff --git a/Tests/CodexTests/PeerDiscoveryTests/PeerDiscoveryTests.cs b/Tests/CodexTests/PeerDiscoveryTests/PeerDiscoveryTests.cs index 34da8d4..52619dd 100644 --- a/Tests/CodexTests/PeerDiscoveryTests/PeerDiscoveryTests.cs +++ b/Tests/CodexTests/PeerDiscoveryTests/PeerDiscoveryTests.cs @@ -21,9 +21,9 @@ namespace CodexTests.PeerDiscoveryTests [Test] public void MetricsDoesNotInterfereWithPeerDiscovery() { - StartCodex(2, s => s.EnableMetrics()); + var nodes = StartCodex(2, s => s.EnableMetrics()); - AssertAllNodesConnected(); + AssertAllNodesConnected(nodes); } [Test] @@ -31,10 +31,10 @@ namespace CodexTests.PeerDiscoveryTests { var geth = Ci.StartGethNode(s => s.IsMiner()); var contracts = Ci.StartCodexContracts(geth); - StartCodex(2, s => s.EnableMarketplace(geth, contracts, m => m + var nodes = StartCodex(2, s => s.EnableMarketplace(geth, contracts, m => m .WithInitial(10.Eth(), 1000.TstWei()))); - AssertAllNodesConnected(); + AssertAllNodesConnected(nodes); } [TestCase(2)] @@ -42,16 +42,15 @@ namespace CodexTests.PeerDiscoveryTests [TestCase(10)] public void VariableNodes(int number) { - StartCodex(number); + var nodes = StartCodex(number); - AssertAllNodesConnected(); + AssertAllNodesConnected(nodes); } - private void AssertAllNodesConnected() + private void AssertAllNodesConnected(IEnumerable nodes) { - var allNodes = GetAllOnlineCodexNodes(); - CreatePeerConnectionTestHelpers().AssertFullyConnected(allNodes); - CheckRoutingTable(allNodes); + CreatePeerConnectionTestHelpers().AssertFullyConnected(nodes); + CheckRoutingTable(nodes); } private void CheckRoutingTable(IEnumerable allNodes) diff --git a/Tests/DistTestCore/DistTest.cs b/Tests/DistTestCore/DistTest.cs index a2edfec..7b1d09e 100644 --- a/Tests/DistTestCore/DistTest.cs +++ b/Tests/DistTestCore/DistTest.cs @@ -215,7 +215,7 @@ namespace DistTestCore IncludeLogsOnTestFailure(lifecycle); LifecycleStop(lifecycle); lifecycle.DeleteAllResources(); - lifecycle = null!; + lifecycles.Remove(GetCurrentTestName()); }); }