diff --git a/Framework/Utils/Time.cs b/Framework/Utils/Time.cs index f0f76043..f1a109d9 100644 --- a/Framework/Utils/Time.cs +++ b/Framework/Utils/Time.cs @@ -1,4 +1,6 @@ -namespace Utils +using System.Diagnostics; + +namespace Utils { public static class Time { @@ -105,16 +107,18 @@ { var start = DateTime.UtcNow; var tries = 1; - var exceptions = new List(); + var tryInfo = new List<(Exception, TimeSpan)>(); while (true) { var duration = DateTime.UtcNow - start; if (duration > maxTimeout) { - throw new TimeoutException($"Retry '{description}' timed out after {tries} tries over {FormatDuration(duration)}.", new AggregateException(exceptions)); + var info = FormatTryInfos(tryInfo); + throw new TimeoutException($"Retry '{description}' timed out after {tries} tries over {FormatDuration(duration)}.{Environment.NewLine}{info}"); } + var sw = Stopwatch.StartNew(); try { action(); @@ -122,7 +126,7 @@ } catch (Exception ex) { - exceptions.Add(ex); + tryInfo.Add((ex, sw.Elapsed)); tries++; } @@ -130,6 +134,16 @@ } } + private static string FormatTryInfos(List<(Exception, TimeSpan)> tryInfo) + { + return string.Join(Environment.NewLine, tryInfo.Select(FormatTryInfo).ToArray()); + } + + private static string FormatTryInfo((Exception, TimeSpan) info, int index) + { + return $"Attempt {index} took {FormatDuration(info.Item2)} and failed with exception {info.Item1}."; + } + public static T Retry(Func action, TimeSpan maxTimeout, TimeSpan retryTime, string description) { var start = DateTime.UtcNow; diff --git a/Tests/CodexLongTests/BasicTests/DownloadTests.cs b/Tests/CodexLongTests/BasicTests/DownloadTests.cs index fbfc20b0..0a95a643 100644 --- a/Tests/CodexLongTests/BasicTests/DownloadTests.cs +++ b/Tests/CodexLongTests/BasicTests/DownloadTests.cs @@ -17,8 +17,8 @@ namespace CodexLongTests.BasicTests [Values(1, 3, 5)] int numberOfFiles, [Values(10, 50, 100)] int filesizeMb) { - var host = AddCodex(); - var client = AddCodex(); + var host = StartCodex(); + var client = StartCodex(); var testfiles = new List(); var contentIds = new List(); diff --git a/Tests/CodexLongTests/BasicTests/LargeFileTests.cs b/Tests/CodexLongTests/BasicTests/LargeFileTests.cs index 1a6e21cc..0470e35a 100644 --- a/Tests/CodexLongTests/BasicTests/LargeFileTests.cs +++ b/Tests/CodexLongTests/BasicTests/LargeFileTests.cs @@ -48,7 +48,7 @@ namespace CodexLongTests.BasicTests var expectedFile = GenerateTestFile(sizeMB); - var node = AddCodex(s => s.WithStorageQuota((size + 10).MB())); + var node = StartCodex(s => s.WithStorageQuota((size + 10).MB())); var uploadStart = DateTime.UtcNow; var cid = node.UploadFile(expectedFile); diff --git a/Tests/CodexLongTests/BasicTests/TestInfraTests.cs b/Tests/CodexLongTests/BasicTests/TestInfraTests.cs index 51d41b70..7dcf02b0 100644 --- a/Tests/CodexLongTests/BasicTests/TestInfraTests.cs +++ b/Tests/CodexLongTests/BasicTests/TestInfraTests.cs @@ -11,7 +11,7 @@ namespace CodexLongTests.BasicTests [Ignore("Not supported atm")] public void TestInfraShouldHave1000AddressSpacesPerPod() { - var group = AddCodex(1000, s => s.EnableMetrics()); + var group = StartCodex(1000, s => s.EnableMetrics()); var nodeIds = group.Select(n => n.GetDebugInfo().Id).ToArray(); @@ -26,7 +26,7 @@ namespace CodexLongTests.BasicTests { for (var i = 0; i < 20; i++) { - var n = AddCodex(); + var n = StartCodex(); Assert.That(!string.IsNullOrEmpty(n.GetDebugInfo().Id)); } diff --git a/Tests/CodexLongTests/BasicTests/UploadTests.cs b/Tests/CodexLongTests/BasicTests/UploadTests.cs index 31718aa0..361c3a73 100644 --- a/Tests/CodexLongTests/BasicTests/UploadTests.cs +++ b/Tests/CodexLongTests/BasicTests/UploadTests.cs @@ -17,8 +17,8 @@ namespace CodexLongTests.BasicTests [Values(1, 3, 5)] int numberOfFiles, [Values(10, 50, 100)] int filesizeMb) { - var host = AddCodex(); - var client = AddCodex(); + var host = StartCodex(); + var client = StartCodex(); var testfiles = new List(); var contentIds = new List(); diff --git a/Tests/CodexLongTests/DownloadConnectivityTests/LongFullyConnectedDownloadTests.cs b/Tests/CodexLongTests/DownloadConnectivityTests/LongFullyConnectedDownloadTests.cs index 8f57e38a..15fc9f34 100644 --- a/Tests/CodexLongTests/DownloadConnectivityTests/LongFullyConnectedDownloadTests.cs +++ b/Tests/CodexLongTests/DownloadConnectivityTests/LongFullyConnectedDownloadTests.cs @@ -15,7 +15,7 @@ namespace CodexLongTests.DownloadConnectivityTests [Values(10, 15, 20)] int numberOfNodes, [Values(10, 100)] int sizeMBs) { - for (var i = 0; i < numberOfNodes; i++) AddCodex(); + for (var i = 0; i < numberOfNodes; i++) StartCodex(); CreatePeerDownloadTestHelpers().AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes(), sizeMBs.MB()); } diff --git a/Tests/CodexLongTests/ScalabilityTests/MultiPeerDownloadTests.cs b/Tests/CodexLongTests/ScalabilityTests/MultiPeerDownloadTests.cs index f4c0a5f0..b1640b6d 100644 --- a/Tests/CodexLongTests/ScalabilityTests/MultiPeerDownloadTests.cs +++ b/Tests/CodexLongTests/ScalabilityTests/MultiPeerDownloadTests.cs @@ -16,7 +16,7 @@ namespace CodexTests.ScalabilityTests [Values(100, 1000)] int fileSize ) { - var hosts = AddCodex(numberOfHosts, s => s.WithLogLevel(CodexPlugin.CodexLogLevel.Trace)); + var hosts = StartCodex(numberOfHosts, s => s.WithLogLevel(CodexPlugin.CodexLogLevel.Trace)); var file = GenerateTestFile(fileSize.MB()); var cid = hosts[0].UploadFile(file); var tailOfManifestCid = cid.Id.Substring(cid.Id.Length - 6); @@ -38,7 +38,7 @@ namespace CodexTests.ScalabilityTests foreach (var h in hosts) h.DownloadContent(cid); - var client = AddCodex(s => s.WithLogLevel(CodexPlugin.CodexLogLevel.Trace)); + var client = StartCodex(s => s.WithLogLevel(CodexPlugin.CodexLogLevel.Trace)); var resultFile = client.DownloadContent(cid); resultFile!.AssertIsEqual(file); diff --git a/Tests/CodexLongTests/ScalabilityTests/ScalabilityTests.cs b/Tests/CodexLongTests/ScalabilityTests/ScalabilityTests.cs index 9a224f7d..b87be971 100644 --- a/Tests/CodexLongTests/ScalabilityTests/ScalabilityTests.cs +++ b/Tests/CodexLongTests/ScalabilityTests/ScalabilityTests.cs @@ -24,8 +24,8 @@ public class ScalabilityTests : CodexDistTest { var logLevel = CodexLogLevel.Info; - var bootstrap = AddCodex(s => s.WithLogLevel(logLevel)); - var nodes = AddCodex(numberOfNodes - 1, s => s + var bootstrap = StartCodex(s => s.WithLogLevel(logLevel)); + var nodes = StartCodex(numberOfNodes - 1, s => s .WithBootstrapNode(bootstrap) .WithLogLevel(logLevel) .WithStorageQuota((fileSizeInMb + 50).MB()) @@ -64,8 +64,8 @@ public class ScalabilityTests : CodexDistTest { var logLevel = CodexLogLevel.Info; - var bootstrap = AddCodex(s => s.WithLogLevel(logLevel)); - var nodes = AddCodex(numberOfNodes - 1, s => s + var bootstrap = StartCodex(s => s.WithLogLevel(logLevel)); + var nodes = StartCodex(numberOfNodes - 1, s => s .WithBootstrapNode(bootstrap) .WithLogLevel(logLevel) .WithStorageQuota((fileSizeInMb + 50).MB()) diff --git a/Tests/CodexTests/AutoBootstrapDistTest.cs b/Tests/CodexTests/AutoBootstrapDistTest.cs index 5c70496e..fa103208 100644 --- a/Tests/CodexTests/AutoBootstrapDistTest.cs +++ b/Tests/CodexTests/AutoBootstrapDistTest.cs @@ -8,7 +8,7 @@ namespace CodexTests [SetUp] public void SetUpBootstrapNode() { - BootstrapNode = AddCodex(s => s.WithName("BOOTSTRAP")); + BootstrapNode = StartCodex(s => s.WithName("BOOTSTRAP")); } [TearDown] diff --git a/Tests/CodexTests/BasicTests/ExampleTests.cs b/Tests/CodexTests/BasicTests/ExampleTests.cs index e7453b5e..03562981 100644 --- a/Tests/CodexTests/BasicTests/ExampleTests.cs +++ b/Tests/CodexTests/BasicTests/ExampleTests.cs @@ -13,7 +13,7 @@ namespace CodexTests.BasicTests [Test] public void CodexLogExample() { - var primary = AddCodex(s => s.WithLogLevel(CodexLogLevel.Trace, new CodexLogCustomTopics(CodexLogLevel.Warn, CodexLogLevel.Warn))); + var primary = StartCodex(s => s.WithLogLevel(CodexLogLevel.Trace, new CodexLogCustomTopics(CodexLogLevel.Warn, CodexLogLevel.Warn))); var cid = primary.UploadFile(GenerateTestFile(5.MB())); @@ -28,8 +28,8 @@ namespace CodexTests.BasicTests [Test] public void TwoMetricsExample() { - var group = AddCodex(2, s => s.EnableMetrics()); - var group2 = AddCodex(2, s => s.EnableMetrics()); + var group = StartCodex(2, s => s.EnableMetrics()); + var group2 = StartCodex(2, s => s.EnableMetrics()); var primary = group[0]; var secondary = group[1]; diff --git a/Tests/CodexTests/BasicTests/MarketplaceTests.cs b/Tests/CodexTests/BasicTests/MarketplaceTests.cs index f81d7bd9..6904e56e 100644 --- a/Tests/CodexTests/BasicTests/MarketplaceTests.cs +++ b/Tests/CodexTests/BasicTests/MarketplaceTests.cs @@ -21,7 +21,7 @@ namespace CodexTests.BasicTests var contracts = Ci.StartCodexContracts(geth); var numberOfHosts = 5; - var hosts = AddCodex(numberOfHosts, s => s + var hosts = StartCodex(numberOfHosts, s => s .WithName("Host") .WithLogLevel(CodexLogLevel.Trace, new CodexLogCustomTopics(CodexLogLevel.Error, CodexLogLevel.Error, CodexLogLevel.Warn) { @@ -49,7 +49,7 @@ namespace CodexTests.BasicTests var testFile = GenerateTestFile(fileSize); - var client = AddCodex(s => s + var client = StartCodex(s => s .WithName("Client") .EnableMarketplace(geth, contracts, m => m .WithInitial(10.Eth(), clientInitialBalance))); diff --git a/Tests/CodexTests/BasicTests/OneClientTests.cs b/Tests/CodexTests/BasicTests/OneClientTests.cs index 9d4f5a58..11e81256 100644 --- a/Tests/CodexTests/BasicTests/OneClientTests.cs +++ b/Tests/CodexTests/BasicTests/OneClientTests.cs @@ -10,7 +10,7 @@ namespace CodexTests.BasicTests [Test] public void OneClientTest() { - var primary = AddCodex(); + var primary = StartCodex(); PerformOneClientTest(primary); } @@ -18,11 +18,11 @@ namespace CodexTests.BasicTests [Test] public void RestartTest() { - var primary = AddCodex(); + var primary = StartCodex(); primary.Stop(waitTillStopped: true); - primary = AddCodex(); + primary = StartCodex(); PerformOneClientTest(primary); } diff --git a/Tests/CodexTests/BasicTests/ThreeClientTest.cs b/Tests/CodexTests/BasicTests/ThreeClientTest.cs index bd9cdfb4..2285aa95 100644 --- a/Tests/CodexTests/BasicTests/ThreeClientTest.cs +++ b/Tests/CodexTests/BasicTests/ThreeClientTest.cs @@ -9,8 +9,8 @@ namespace CodexTests.BasicTests [Test] public void ThreeClient() { - var primary = AddCodex(); - var secondary = AddCodex(); + var primary = StartCodex(); + var secondary = StartCodex(); var testFile = GenerateTestFile(10.MB()); diff --git a/Tests/CodexTests/BasicTests/TwoClientTests.cs b/Tests/CodexTests/BasicTests/TwoClientTests.cs index 546fbda3..ab15ff25 100644 --- a/Tests/CodexTests/BasicTests/TwoClientTests.cs +++ b/Tests/CodexTests/BasicTests/TwoClientTests.cs @@ -10,8 +10,8 @@ namespace CodexTests.BasicTests [Test] public void TwoClientTest() { - var uploader = AddCodex(s => s.WithName("Uploader")); - var downloader = AddCodex(s => s.WithName("Downloader").WithBootstrapNode(uploader)); + var uploader = StartCodex(s => s.WithName("Uploader")); + var downloader = StartCodex(s => s.WithName("Downloader").WithBootstrapNode(uploader)); PerformTwoClientTest(uploader, downloader); } diff --git a/Tests/CodexTests/CodexDistTest.cs b/Tests/CodexTests/CodexDistTest.cs index 0578c27a..7058b8e1 100644 --- a/Tests/CodexTests/CodexDistTest.cs +++ b/Tests/CodexTests/CodexDistTest.cs @@ -39,22 +39,22 @@ namespace CodexTests onlineCodexNodes.Remove(lifecycle); } - public ICodexNode AddCodex() + public ICodexNode StartCodex() { - return AddCodex(s => { }); + return StartCodex(s => { }); } - public ICodexNode AddCodex(Action setup) + public ICodexNode StartCodex(Action setup) { - return AddCodex(1, setup)[0]; + return StartCodex(1, setup)[0]; } - public ICodexNodeGroup AddCodex(int numberOfNodes) + public ICodexNodeGroup StartCodex(int numberOfNodes) { - return AddCodex(numberOfNodes, s => { }); + return StartCodex(numberOfNodes, s => { }); } - public ICodexNodeGroup AddCodex(int numberOfNodes, Action setup) + public ICodexNodeGroup StartCodex(int numberOfNodes, Action setup) { var group = Ci.StartCodexNodes(numberOfNodes, s => { diff --git a/Tests/CodexTests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs b/Tests/CodexTests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs index 7f11171f..5b4f1a93 100644 --- a/Tests/CodexTests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs +++ b/Tests/CodexTests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs @@ -11,7 +11,7 @@ namespace CodexTests.DownloadConnectivityTests [Test] public void MetricsDoesNotInterfereWithPeerDownload() { - AddCodex(2, s => s.EnableMetrics()); + StartCodex(2, s => s.EnableMetrics()); AssertAllNodesConnected(); } @@ -21,7 +21,7 @@ namespace CodexTests.DownloadConnectivityTests { var geth = Ci.StartGethNode(s => s.IsMiner()); var contracts = Ci.StartCodexContracts(geth); - AddCodex(2, s => s.EnableMarketplace(geth, contracts, m => m + StartCodex(2, s => s.EnableMarketplace(geth, contracts, m => m .WithInitial(10.Eth(), 1000.TestTokens()))); AssertAllNodesConnected(); @@ -33,7 +33,7 @@ namespace CodexTests.DownloadConnectivityTests [Values(2, 5)] int numberOfNodes, [Values(1, 10)] int sizeMBs) { - AddCodex(numberOfNodes); + StartCodex(numberOfNodes); AssertAllNodesConnected(sizeMBs); } diff --git a/Tests/CodexTests/PeerDiscoveryTests/LayeredDiscoveryTests.cs b/Tests/CodexTests/PeerDiscoveryTests/LayeredDiscoveryTests.cs index 00c0cda9..9c884fe1 100644 --- a/Tests/CodexTests/PeerDiscoveryTests/LayeredDiscoveryTests.cs +++ b/Tests/CodexTests/PeerDiscoveryTests/LayeredDiscoveryTests.cs @@ -8,10 +8,10 @@ namespace CodexTests.PeerDiscoveryTests [Test] public void TwoLayersTest() { - var root = AddCodex(); - var l1Source = AddCodex(s => s.WithBootstrapNode(root)); - var l1Node = AddCodex(s => s.WithBootstrapNode(root)); - var l2Target = AddCodex(s => s.WithBootstrapNode(l1Node)); + var root = StartCodex(); + var l1Source = StartCodex(s => s.WithBootstrapNode(root)); + var l1Node = StartCodex(s => s.WithBootstrapNode(root)); + var l2Target = StartCodex(s => s.WithBootstrapNode(l1Node)); AssertAllNodesConnected(); } @@ -19,11 +19,11 @@ namespace CodexTests.PeerDiscoveryTests [Test] public void ThreeLayersTest() { - var root = AddCodex(); - var l1Source = AddCodex(s => s.WithBootstrapNode(root)); - var l1Node = AddCodex(s => s.WithBootstrapNode(root)); - var l2Node = AddCodex(s => s.WithBootstrapNode(l1Node)); - var l3Target = AddCodex(s => s.WithBootstrapNode(l2Node)); + var root = StartCodex(); + var l1Source = StartCodex(s => s.WithBootstrapNode(root)); + var l1Node = StartCodex(s => s.WithBootstrapNode(root)); + var l2Node = StartCodex(s => s.WithBootstrapNode(l1Node)); + var l3Target = StartCodex(s => s.WithBootstrapNode(l2Node)); AssertAllNodesConnected(); } @@ -33,10 +33,10 @@ namespace CodexTests.PeerDiscoveryTests [TestCase(10)] public void NodeChainTest(int chainLength) { - var node = AddCodex(); + var node = StartCodex(); for (var i = 1; i < chainLength; i++) { - node = AddCodex(s => s.WithBootstrapNode(node)); + node = StartCodex(s => s.WithBootstrapNode(node)); } AssertAllNodesConnected(); diff --git a/Tests/CodexTests/PeerDiscoveryTests/PeerDiscoveryTests.cs b/Tests/CodexTests/PeerDiscoveryTests/PeerDiscoveryTests.cs index 1e0faa4b..82c53fb0 100644 --- a/Tests/CodexTests/PeerDiscoveryTests/PeerDiscoveryTests.cs +++ b/Tests/CodexTests/PeerDiscoveryTests/PeerDiscoveryTests.cs @@ -12,7 +12,7 @@ namespace CodexTests.PeerDiscoveryTests public void CanReportUnknownPeerId() { var unknownId = "16Uiu2HAkv2CHWpff3dj5iuVNERAp8AGKGNgpGjPexJZHSqUstfsK"; - var node = AddCodex(); + var node = StartCodex(); var result = node.GetDebugPeer(unknownId); Assert.That(result.IsPeerFound, Is.False); @@ -21,7 +21,7 @@ namespace CodexTests.PeerDiscoveryTests [Test] public void MetricsDoesNotInterfereWithPeerDiscovery() { - AddCodex(2, s => s.EnableMetrics()); + StartCodex(2, s => s.EnableMetrics()); AssertAllNodesConnected(); } @@ -31,7 +31,7 @@ namespace CodexTests.PeerDiscoveryTests { var geth = Ci.StartGethNode(s => s.IsMiner()); var contracts = Ci.StartCodexContracts(geth); - AddCodex(2, s => s.EnableMarketplace(geth, contracts, m => m + StartCodex(2, s => s.EnableMarketplace(geth, contracts, m => m .WithInitial(10.Eth(), 1000.TestTokens()))); AssertAllNodesConnected(); @@ -42,7 +42,7 @@ namespace CodexTests.PeerDiscoveryTests [TestCase(10)] public void VariableNodes(int number) { - AddCodex(number); + StartCodex(number); AssertAllNodesConnected(); } diff --git a/Tests/CodexTests/UtilityTests/DiscordBotTests.cs b/Tests/CodexTests/UtilityTests/DiscordBotTests.cs index ab197751..d6425256 100644 --- a/Tests/CodexTests/UtilityTests/DiscordBotTests.cs +++ b/Tests/CodexTests/UtilityTests/DiscordBotTests.cs @@ -58,7 +58,7 @@ namespace CodexTests.UtilityTests for (var i = 0; i < numberOfHosts; i++) { - var seller = AddCodex(s => s + var seller = StartCodex(s => s .WithName("Seller") .WithLogLevel(CodexLogLevel.Trace, new CodexLogCustomTopics(CodexLogLevel.Error, CodexLogLevel.Error, CodexLogLevel.Warn) { @@ -82,7 +82,7 @@ namespace CodexTests.UtilityTests var testFile = GenerateTestFile(fileSize); - var buyer = AddCodex(s => s + var buyer = StartCodex(s => s .WithName("Buyer") .EnableMarketplace(geth, contracts, m => m .WithAccount(myAccount) diff --git a/Tests/CodexTests/UtilityTests/LogHelperTests.cs b/Tests/CodexTests/UtilityTests/LogHelperTests.cs index f8561f85..acc73d34 100644 --- a/Tests/CodexTests/UtilityTests/LogHelperTests.cs +++ b/Tests/CodexTests/UtilityTests/LogHelperTests.cs @@ -11,8 +11,8 @@ namespace CodexTests.UtilityTests [Ignore("Used to find the most common log messages.")] public void FindMostCommonLogMessages() { - var uploader = AddCodex(s => s.WithName("uploader").WithLogLevel(CodexLogLevel.Trace)); - var downloader = AddCodex(s => s.WithName("downloader").WithLogLevel(CodexLogLevel.Trace)); + var uploader = StartCodex(s => s.WithName("uploader").WithLogLevel(CodexLogLevel.Trace)); + var downloader = StartCodex(s => s.WithName("downloader").WithLogLevel(CodexLogLevel.Trace)); var cid = uploader.UploadFile(GenerateTestFile(100.MB()));