Rename AddCodex. Better retry logging

Framework/Utils/Time.cs

@@ -1,4 +1,6 @@
-namespace Utils
+using System.Diagnostics;
+
+namespace Utils
 {
     public static class Time
     {
@@ -105,16 +107,18 @@
         {
             var start = DateTime.UtcNow;
             var tries = 1;
-            var exceptions = new List<Exception>();
+            var tryInfo = new List<(Exception, TimeSpan)>();
 
             while (true)
             {
                 var duration = DateTime.UtcNow - start;
                 if (duration > maxTimeout)
                 {
-                    throw new TimeoutException($"Retry '{description}' timed out after {tries} tries over {FormatDuration(duration)}.", new AggregateException(exceptions));
+                    var info = FormatTryInfos(tryInfo);
+                    throw new TimeoutException($"Retry '{description}' timed out after {tries} tries over {FormatDuration(duration)}.{Environment.NewLine}{info}");
                 }
 
+                var sw = Stopwatch.StartNew();
                 try
                 {
                     action();
@@ -122,7 +126,7 @@
                 }
                 catch (Exception ex)
                 {
-                    exceptions.Add(ex);
+                    tryInfo.Add((ex, sw.Elapsed));
                     tries++;
                 }
 
@@ -130,6 +134,16 @@
             }
         }
 
+        private static string FormatTryInfos(List<(Exception, TimeSpan)> tryInfo)
+        {
+            return string.Join(Environment.NewLine, tryInfo.Select(FormatTryInfo).ToArray());
+        }
+
+        private static string FormatTryInfo((Exception, TimeSpan) info, int index)
+        {
+            return $"Attempt {index} took {FormatDuration(info.Item2)} and failed with exception {info.Item1}.";
+        }
+
         public static T Retry<T>(Func<T> action, TimeSpan maxTimeout, TimeSpan retryTime, string description)
         {
             var start = DateTime.UtcNow;

Tests/CodexLongTests/BasicTests/DownloadTests.cs

@@ -17,8 +17,8 @@ namespace CodexLongTests.BasicTests
             [Values(1, 3, 5)] int numberOfFiles,
             [Values(10, 50, 100)] int filesizeMb)
         {
-            var host = AddCodex();
+            var host = StartCodex();
-            var client = AddCodex();
+            var client = StartCodex();
 
             var testfiles = new List<TrackedFile>();
             var contentIds = new List<ContentId>();

Tests/CodexLongTests/BasicTests/LargeFileTests.cs

@@ -48,7 +48,7 @@ namespace CodexLongTests.BasicTests
 
             var expectedFile = GenerateTestFile(sizeMB);
 
-            var node = AddCodex(s => s.WithStorageQuota((size + 10).MB()));
+            var node = StartCodex(s => s.WithStorageQuota((size + 10).MB()));
 
             var uploadStart = DateTime.UtcNow;
             var cid = node.UploadFile(expectedFile);

Tests/CodexLongTests/BasicTests/TestInfraTests.cs

@@ -11,7 +11,7 @@ namespace CodexLongTests.BasicTests
         [Ignore("Not supported atm")]
         public void TestInfraShouldHave1000AddressSpacesPerPod()
         {
-            var group = AddCodex(1000, s => s.EnableMetrics());
+            var group = StartCodex(1000, s => s.EnableMetrics());
 
             var nodeIds = group.Select(n => n.GetDebugInfo().Id).ToArray();
 
@@ -26,7 +26,7 @@ namespace CodexLongTests.BasicTests
         {
             for (var i = 0; i < 20; i++)
             {
-                var n = AddCodex();
+                var n = StartCodex();
 
                 Assert.That(!string.IsNullOrEmpty(n.GetDebugInfo().Id));
             }

Tests/CodexLongTests/BasicTests/UploadTests.cs

@@ -17,8 +17,8 @@ namespace CodexLongTests.BasicTests
             [Values(1, 3, 5)] int numberOfFiles,
             [Values(10, 50, 100)] int filesizeMb)
         {
-            var host = AddCodex();
+            var host = StartCodex();
-            var client = AddCodex();
+            var client = StartCodex();
 
             var testfiles = new List<TrackedFile>();
             var contentIds = new List<ContentId>();

Tests/CodexLongTests/DownloadConnectivityTests/LongFullyConnectedDownloadTests.cs

@@ -15,7 +15,7 @@ namespace CodexLongTests.DownloadConnectivityTests
             [Values(10, 15, 20)] int numberOfNodes,
             [Values(10, 100)] int sizeMBs)
         {
-            for (var i = 0; i < numberOfNodes; i++) AddCodex();
+            for (var i = 0; i < numberOfNodes; i++) StartCodex();
 
             CreatePeerDownloadTestHelpers().AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes(), sizeMBs.MB());
         }

Tests/CodexLongTests/ScalabilityTests/MultiPeerDownloadTests.cs

@@ -16,7 +16,7 @@ namespace CodexTests.ScalabilityTests
             [Values(100, 1000)] int fileSize
         )
         {
-            var hosts = AddCodex(numberOfHosts, s => s.WithLogLevel(CodexPlugin.CodexLogLevel.Trace));
+            var hosts = StartCodex(numberOfHosts, s => s.WithLogLevel(CodexPlugin.CodexLogLevel.Trace));
             var file = GenerateTestFile(fileSize.MB());
             var cid = hosts[0].UploadFile(file);
             var tailOfManifestCid = cid.Id.Substring(cid.Id.Length - 6);
@@ -38,7 +38,7 @@ namespace CodexTests.ScalabilityTests
 
             foreach (var h in hosts) h.DownloadContent(cid);
 
-            var client = AddCodex(s => s.WithLogLevel(CodexPlugin.CodexLogLevel.Trace));
+            var client = StartCodex(s => s.WithLogLevel(CodexPlugin.CodexLogLevel.Trace));
             var resultFile = client.DownloadContent(cid);
             resultFile!.AssertIsEqual(file);
 

Tests/CodexLongTests/ScalabilityTests/ScalabilityTests.cs

@@ -24,8 +24,8 @@ public class ScalabilityTests : CodexDistTest
     {
         var logLevel = CodexLogLevel.Info;
 
-        var bootstrap = AddCodex(s => s.WithLogLevel(logLevel));
+        var bootstrap = StartCodex(s => s.WithLogLevel(logLevel));
-        var nodes = AddCodex(numberOfNodes - 1, s => s
+        var nodes = StartCodex(numberOfNodes - 1, s => s
             .WithBootstrapNode(bootstrap)
             .WithLogLevel(logLevel)
             .WithStorageQuota((fileSizeInMb + 50).MB())
@@ -64,8 +64,8 @@ public class ScalabilityTests : CodexDistTest
     {
         var logLevel = CodexLogLevel.Info;
 
-        var bootstrap = AddCodex(s => s.WithLogLevel(logLevel));
+        var bootstrap = StartCodex(s => s.WithLogLevel(logLevel));
-        var nodes = AddCodex(numberOfNodes - 1, s => s
+        var nodes = StartCodex(numberOfNodes - 1, s => s
             .WithBootstrapNode(bootstrap)
             .WithLogLevel(logLevel)
             .WithStorageQuota((fileSizeInMb + 50).MB())

Tests/CodexTests/AutoBootstrapDistTest.cs

@@ -8,7 +8,7 @@ namespace CodexTests
         [SetUp]
         public void SetUpBootstrapNode()
         {
-            BootstrapNode = AddCodex(s => s.WithName("BOOTSTRAP"));
+            BootstrapNode = StartCodex(s => s.WithName("BOOTSTRAP"));
         }
 
         [TearDown]

Tests/CodexTests/BasicTests/ExampleTests.cs

@@ -13,7 +13,7 @@ namespace CodexTests.BasicTests
         [Test]
         public void CodexLogExample()
         {
-            var primary = AddCodex(s => s.WithLogLevel(CodexLogLevel.Trace, new CodexLogCustomTopics(CodexLogLevel.Warn, CodexLogLevel.Warn)));
+            var primary = StartCodex(s => s.WithLogLevel(CodexLogLevel.Trace, new CodexLogCustomTopics(CodexLogLevel.Warn, CodexLogLevel.Warn)));
 
             var cid = primary.UploadFile(GenerateTestFile(5.MB()));
 
@@ -28,8 +28,8 @@ namespace CodexTests.BasicTests
         [Test]
         public void TwoMetricsExample()
         {
-            var group = AddCodex(2, s => s.EnableMetrics());
+            var group = StartCodex(2, s => s.EnableMetrics());
-            var group2 = AddCodex(2, s => s.EnableMetrics());
+            var group2 = StartCodex(2, s => s.EnableMetrics());
 
             var primary = group[0];
             var secondary = group[1];

Tests/CodexTests/BasicTests/MarketplaceTests.cs

@@ -21,7 +21,7 @@ namespace CodexTests.BasicTests
             var contracts = Ci.StartCodexContracts(geth);
             
             var numberOfHosts = 5;
-            var hosts = AddCodex(numberOfHosts, s => s
+            var hosts = StartCodex(numberOfHosts, s => s
                 .WithName("Host")
                 .WithLogLevel(CodexLogLevel.Trace, new CodexLogCustomTopics(CodexLogLevel.Error, CodexLogLevel.Error, CodexLogLevel.Warn)
                 {
@@ -49,7 +49,7 @@ namespace CodexTests.BasicTests
 
             var testFile = GenerateTestFile(fileSize);
 
-            var client = AddCodex(s => s
+            var client = StartCodex(s => s
                 .WithName("Client")
                 .EnableMarketplace(geth, contracts, m => m
                     .WithInitial(10.Eth(), clientInitialBalance)));

Tests/CodexTests/BasicTests/OneClientTests.cs

@@ -10,7 +10,7 @@ namespace CodexTests.BasicTests
         [Test]
         public void OneClientTest()
         {
-            var primary = AddCodex();
+            var primary = StartCodex();
 
             PerformOneClientTest(primary);
         }
@@ -18,11 +18,11 @@ namespace CodexTests.BasicTests
         [Test]
         public void RestartTest()
         {
-            var primary = AddCodex();
+            var primary = StartCodex();
 
             primary.Stop(waitTillStopped: true);
 
-            primary = AddCodex();
+            primary = StartCodex();
 
             PerformOneClientTest(primary);
         }

Tests/CodexTests/BasicTests/ThreeClientTest.cs

@@ -9,8 +9,8 @@ namespace CodexTests.BasicTests
         [Test]
         public void ThreeClient()
         {
-            var primary = AddCodex();
+            var primary = StartCodex();
-            var secondary = AddCodex();
+            var secondary = StartCodex();
 
             var testFile = GenerateTestFile(10.MB());
 

Tests/CodexTests/BasicTests/TwoClientTests.cs

@@ -10,8 +10,8 @@ namespace CodexTests.BasicTests
         [Test]
         public void TwoClientTest()
         {
-            var uploader = AddCodex(s => s.WithName("Uploader"));
+            var uploader = StartCodex(s => s.WithName("Uploader"));
-            var downloader = AddCodex(s => s.WithName("Downloader").WithBootstrapNode(uploader));
+            var downloader = StartCodex(s => s.WithName("Downloader").WithBootstrapNode(uploader));
 
             PerformTwoClientTest(uploader, downloader);
         }

Tests/CodexTests/CodexDistTest.cs

@@ -39,22 +39,22 @@ namespace CodexTests
             onlineCodexNodes.Remove(lifecycle);
         }
 
-        public ICodexNode AddCodex()
+        public ICodexNode StartCodex()
         {
-            return AddCodex(s => { });
+            return StartCodex(s => { });
         }
 
-        public ICodexNode AddCodex(Action<ICodexSetup> setup)
+        public ICodexNode StartCodex(Action<ICodexSetup> setup)
         {
-            return AddCodex(1, setup)[0];
+            return StartCodex(1, setup)[0];
         }
 
-        public ICodexNodeGroup AddCodex(int numberOfNodes)
+        public ICodexNodeGroup StartCodex(int numberOfNodes)
         {
-            return AddCodex(numberOfNodes, s => { });
+            return StartCodex(numberOfNodes, s => { });
         }
 
-        public ICodexNodeGroup AddCodex(int numberOfNodes, Action<ICodexSetup> setup)
+        public ICodexNodeGroup StartCodex(int numberOfNodes, Action<ICodexSetup> setup)
         {
             var group = Ci.StartCodexNodes(numberOfNodes, s =>
             {

Tests/CodexTests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs

@@ -11,7 +11,7 @@ namespace CodexTests.DownloadConnectivityTests
         [Test]
         public void MetricsDoesNotInterfereWithPeerDownload()
         {
-            AddCodex(2, s => s.EnableMetrics());
+            StartCodex(2, s => s.EnableMetrics());
 
             AssertAllNodesConnected();
         }
@@ -21,7 +21,7 @@ namespace CodexTests.DownloadConnectivityTests
         {
             var geth = Ci.StartGethNode(s => s.IsMiner());
             var contracts = Ci.StartCodexContracts(geth);
-            AddCodex(2, s => s.EnableMarketplace(geth, contracts, m => m
+            StartCodex(2, s => s.EnableMarketplace(geth, contracts, m => m
                 .WithInitial(10.Eth(), 1000.TestTokens())));
 
             AssertAllNodesConnected();
@@ -33,7 +33,7 @@ namespace CodexTests.DownloadConnectivityTests
             [Values(2, 5)] int numberOfNodes,
             [Values(1, 10)] int sizeMBs)
         {
-            AddCodex(numberOfNodes);
+            StartCodex(numberOfNodes);
 
             AssertAllNodesConnected(sizeMBs);
         }

Tests/CodexTests/PeerDiscoveryTests/LayeredDiscoveryTests.cs

@@ -8,10 +8,10 @@ namespace CodexTests.PeerDiscoveryTests
         [Test]
         public void TwoLayersTest()
         {
-            var root = AddCodex();
+            var root = StartCodex();
-            var l1Source = AddCodex(s => s.WithBootstrapNode(root));
+            var l1Source = StartCodex(s => s.WithBootstrapNode(root));
-            var l1Node = AddCodex(s => s.WithBootstrapNode(root));
+            var l1Node = StartCodex(s => s.WithBootstrapNode(root));
-            var l2Target = AddCodex(s => s.WithBootstrapNode(l1Node));
+            var l2Target = StartCodex(s => s.WithBootstrapNode(l1Node));
 
             AssertAllNodesConnected();
         }
@@ -19,11 +19,11 @@ namespace CodexTests.PeerDiscoveryTests
         [Test]
         public void ThreeLayersTest()
         {
-            var root = AddCodex();
+            var root = StartCodex();
-            var l1Source = AddCodex(s => s.WithBootstrapNode(root));
+            var l1Source = StartCodex(s => s.WithBootstrapNode(root));
-            var l1Node = AddCodex(s => s.WithBootstrapNode(root));
+            var l1Node = StartCodex(s => s.WithBootstrapNode(root));
-            var l2Node = AddCodex(s => s.WithBootstrapNode(l1Node));
+            var l2Node = StartCodex(s => s.WithBootstrapNode(l1Node));
-            var l3Target = AddCodex(s => s.WithBootstrapNode(l2Node));
+            var l3Target = StartCodex(s => s.WithBootstrapNode(l2Node));
 
             AssertAllNodesConnected();
         }
@@ -33,10 +33,10 @@ namespace CodexTests.PeerDiscoveryTests
         [TestCase(10)]
         public void NodeChainTest(int chainLength)
         {
-            var node = AddCodex();
+            var node = StartCodex();
             for (var i = 1; i < chainLength; i++)
             {
-                node = AddCodex(s => s.WithBootstrapNode(node));
+                node = StartCodex(s => s.WithBootstrapNode(node));
             }
 
             AssertAllNodesConnected();

Tests/CodexTests/PeerDiscoveryTests/PeerDiscoveryTests.cs

@@ -12,7 +12,7 @@ namespace CodexTests.PeerDiscoveryTests
         public void CanReportUnknownPeerId()
         {
             var unknownId = "16Uiu2HAkv2CHWpff3dj5iuVNERAp8AGKGNgpGjPexJZHSqUstfsK";
-            var node = AddCodex();
+            var node = StartCodex();
 
             var result = node.GetDebugPeer(unknownId);
             Assert.That(result.IsPeerFound, Is.False);
@@ -21,7 +21,7 @@ namespace CodexTests.PeerDiscoveryTests
         [Test]
         public void MetricsDoesNotInterfereWithPeerDiscovery()
         {
-            AddCodex(2, s => s.EnableMetrics());
+            StartCodex(2, s => s.EnableMetrics());
 
             AssertAllNodesConnected();
         }
@@ -31,7 +31,7 @@ namespace CodexTests.PeerDiscoveryTests
         {
             var geth = Ci.StartGethNode(s => s.IsMiner());
             var contracts = Ci.StartCodexContracts(geth);
-            AddCodex(2, s => s.EnableMarketplace(geth, contracts, m => m
+            StartCodex(2, s => s.EnableMarketplace(geth, contracts, m => m
                 .WithInitial(10.Eth(), 1000.TestTokens())));
 
             AssertAllNodesConnected();
@@ -42,7 +42,7 @@ namespace CodexTests.PeerDiscoveryTests
         [TestCase(10)]
         public void VariableNodes(int number)
         {
-            AddCodex(number);
+            StartCodex(number);
 
             AssertAllNodesConnected();
         }

Tests/CodexTests/UtilityTests/DiscordBotTests.cs

@@ -58,7 +58,7 @@ namespace CodexTests.UtilityTests
 
             for (var i = 0; i < numberOfHosts; i++)
             {
-                var seller = AddCodex(s => s
+                var seller = StartCodex(s => s
                     .WithName("Seller")
                     .WithLogLevel(CodexLogLevel.Trace, new CodexLogCustomTopics(CodexLogLevel.Error, CodexLogLevel.Error, CodexLogLevel.Warn)
                     {
@@ -82,7 +82,7 @@ namespace CodexTests.UtilityTests
 
             var testFile = GenerateTestFile(fileSize);
 
-            var buyer = AddCodex(s => s
+            var buyer = StartCodex(s => s
                 .WithName("Buyer")
                 .EnableMarketplace(geth, contracts, m => m
                     .WithAccount(myAccount)

Tests/CodexTests/UtilityTests/LogHelperTests.cs

@@ -11,8 +11,8 @@ namespace CodexTests.UtilityTests
         [Ignore("Used to find the most common log messages.")]
         public void FindMostCommonLogMessages()
         {
-            var uploader = AddCodex(s => s.WithName("uploader").WithLogLevel(CodexLogLevel.Trace));
+            var uploader = StartCodex(s => s.WithName("uploader").WithLogLevel(CodexLogLevel.Trace));
-            var downloader = AddCodex(s => s.WithName("downloader").WithLogLevel(CodexLogLevel.Trace));
+            var downloader = StartCodex(s => s.WithName("downloader").WithLogLevel(CodexLogLevel.Trace));
 
             var cid = uploader.UploadFile(GenerateTestFile(100.MB()));