status-im
/
cs-codex-dist-tests
mirror of
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Rename AddCodex. Better retry logging

This commit is contained in:
benbierens 2024-05-09 09:32:48 +02:00
parent c01f9dbb21
commit c6a757d6fb
No known key found for this signature in database
GPG Key ID: 877D2C2E09A22F3A
20 changed files with 74 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
Framework/Utils/Time.cs
View File

@ -1,4 +1,6 @@
namespace Utils
using System.Diagnostics;
namespace Utils
{
public static class Time
{
@ -105,16 +107,18 @@
{
var start = DateTime.UtcNow;
var tries = 1;
var exceptions = new List<Exception>();
var tryInfo = new List<(Exception, TimeSpan)>();
while (true)
{
var duration = DateTime.UtcNow - start;
if (duration > maxTimeout)
{
throw new TimeoutException($"Retry '{description}' timed out after {tries} tries over {FormatDuration(duration)}.", new AggregateException(exceptions));
var info = FormatTryInfos(tryInfo);
throw new TimeoutException($"Retry '{description}' timed out after {tries} tries over {FormatDuration(duration)}.{Environment.NewLine}{info}");
}
var sw = Stopwatch.StartNew();
try
{
action();
@ -122,7 +126,7 @@
}
catch (Exception ex)
{
exceptions.Add(ex);
tryInfo.Add((ex, sw.Elapsed));
tries++;
}
@ -130,6 +134,16 @@
}
}
private static string FormatTryInfos(List<(Exception, TimeSpan)> tryInfo)
{
return string.Join(Environment.NewLine, tryInfo.Select(FormatTryInfo).ToArray());
}
private static string FormatTryInfo((Exception, TimeSpan) info, int index)
{
return $"Attempt {index} took {FormatDuration(info.Item2)} and failed with exception {info.Item1}.";
}
public static T Retry<T>(Func<T> action, TimeSpan maxTimeout, TimeSpan retryTime, string description)
{
var start = DateTime.UtcNow;

4
Tests/CodexLongTests/BasicTests/DownloadTests.cs
View File

@ -17,8 +17,8 @@ namespace CodexLongTests.BasicTests
[Values(1, 3, 5)] int numberOfFiles,
[Values(10, 50, 100)] int filesizeMb)
{
var host = AddCodex();
var client = AddCodex();
var host = StartCodex();
var client = StartCodex();
var testfiles = new List<TrackedFile>();
var contentIds = new List<ContentId>();

2
Tests/CodexLongTests/BasicTests/LargeFileTests.cs
View File

@ -48,7 +48,7 @@ namespace CodexLongTests.BasicTests
var expectedFile = GenerateTestFile(sizeMB);
var node = AddCodex(s => s.WithStorageQuota((size + 10).MB()));
var node = StartCodex(s => s.WithStorageQuota((size + 10).MB()));
var uploadStart = DateTime.UtcNow;
var cid = node.UploadFile(expectedFile);

4
Tests/CodexLongTests/BasicTests/TestInfraTests.cs
View File

@ -11,7 +11,7 @@ namespace CodexLongTests.BasicTests
[Ignore("Not supported atm")]
public void TestInfraShouldHave1000AddressSpacesPerPod()
{
var group = AddCodex(1000, s => s.EnableMetrics());
var group = StartCodex(1000, s => s.EnableMetrics());
var nodeIds = group.Select(n => n.GetDebugInfo().Id).ToArray();
@ -26,7 +26,7 @@ namespace CodexLongTests.BasicTests
{
for (var i = 0; i < 20; i++)
{
var n = AddCodex();
var n = StartCodex();
Assert.That(!string.IsNullOrEmpty(n.GetDebugInfo().Id));
}

4
Tests/CodexLongTests/BasicTests/UploadTests.cs
View File

@ -17,8 +17,8 @@ namespace CodexLongTests.BasicTests
[Values(1, 3, 5)] int numberOfFiles,
[Values(10, 50, 100)] int filesizeMb)
{
var host = AddCodex();
var client = AddCodex();
var host = StartCodex();
var client = StartCodex();
var testfiles = new List<TrackedFile>();
var contentIds = new List<ContentId>();

2
Tests/CodexLongTests/DownloadConnectivityTests/LongFullyConnectedDownloadTests.cs
View File

@ -15,7 +15,7 @@ namespace CodexLongTests.DownloadConnectivityTests
[Values(10, 15, 20)] int numberOfNodes,
[Values(10, 100)] int sizeMBs)
{
for (var i = 0; i < numberOfNodes; i++) AddCodex();
for (var i = 0; i < numberOfNodes; i++) StartCodex();
CreatePeerDownloadTestHelpers().AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes(), sizeMBs.MB());
}

4
Tests/CodexLongTests/ScalabilityTests/MultiPeerDownloadTests.cs
View File

@ -16,7 +16,7 @@ namespace CodexTests.ScalabilityTests
[Values(100, 1000)] int fileSize
)
{
var hosts = AddCodex(numberOfHosts, s => s.WithLogLevel(CodexPlugin.CodexLogLevel.Trace));
var hosts = StartCodex(numberOfHosts, s => s.WithLogLevel(CodexPlugin.CodexLogLevel.Trace));
var file = GenerateTestFile(fileSize.MB());
var cid = hosts[0].UploadFile(file);
var tailOfManifestCid = cid.Id.Substring(cid.Id.Length - 6);
@ -38,7 +38,7 @@ namespace CodexTests.ScalabilityTests
foreach (var h in hosts) h.DownloadContent(cid);
var client = AddCodex(s => s.WithLogLevel(CodexPlugin.CodexLogLevel.Trace));
var client = StartCodex(s => s.WithLogLevel(CodexPlugin.CodexLogLevel.Trace));
var resultFile = client.DownloadContent(cid);
resultFile!.AssertIsEqual(file);

8
Tests/CodexLongTests/ScalabilityTests/ScalabilityTests.cs
View File

@ -24,8 +24,8 @@ public class ScalabilityTests : CodexDistTest
{
var logLevel = CodexLogLevel.Info;
var bootstrap = AddCodex(s => s.WithLogLevel(logLevel));
var nodes = AddCodex(numberOfNodes - 1, s => s
var bootstrap = StartCodex(s => s.WithLogLevel(logLevel));
var nodes = StartCodex(numberOfNodes - 1, s => s
.WithBootstrapNode(bootstrap)
.WithLogLevel(logLevel)
.WithStorageQuota((fileSizeInMb + 50).MB())
@ -64,8 +64,8 @@ public class ScalabilityTests : CodexDistTest
{
var logLevel = CodexLogLevel.Info;
var bootstrap = AddCodex(s => s.WithLogLevel(logLevel));
var nodes = AddCodex(numberOfNodes - 1, s => s
var bootstrap = StartCodex(s => s.WithLogLevel(logLevel));
var nodes = StartCodex(numberOfNodes - 1, s => s
.WithBootstrapNode(bootstrap)
.WithLogLevel(logLevel)
.WithStorageQuota((fileSizeInMb + 50).MB())

2
Tests/CodexTests/AutoBootstrapDistTest.cs
View File

@ -8,7 +8,7 @@ namespace CodexTests
[SetUp]
public void SetUpBootstrapNode()
{
BootstrapNode = AddCodex(s => s.WithName("BOOTSTRAP"));
BootstrapNode = StartCodex(s => s.WithName("BOOTSTRAP"));
}
[TearDown]

6
Tests/CodexTests/BasicTests/ExampleTests.cs
View File

@ -13,7 +13,7 @@ namespace CodexTests.BasicTests
[Test]
public void CodexLogExample()
{
var primary = AddCodex(s => s.WithLogLevel(CodexLogLevel.Trace, new CodexLogCustomTopics(CodexLogLevel.Warn, CodexLogLevel.Warn)));
var primary = StartCodex(s => s.WithLogLevel(CodexLogLevel.Trace, new CodexLogCustomTopics(CodexLogLevel.Warn, CodexLogLevel.Warn)));
var cid = primary.UploadFile(GenerateTestFile(5.MB()));
@ -28,8 +28,8 @@ namespace CodexTests.BasicTests
[Test]
public void TwoMetricsExample()
{
var group = AddCodex(2, s => s.EnableMetrics());
var group2 = AddCodex(2, s => s.EnableMetrics());
var group = StartCodex(2, s => s.EnableMetrics());
var group2 = StartCodex(2, s => s.EnableMetrics());
var primary = group[0];
var secondary = group[1];

4
Tests/CodexTests/BasicTests/MarketplaceTests.cs
View File

@ -21,7 +21,7 @@ namespace CodexTests.BasicTests
var contracts = Ci.StartCodexContracts(geth);
var numberOfHosts = 5;
var hosts = AddCodex(numberOfHosts, s => s
var hosts = StartCodex(numberOfHosts, s => s
.WithName("Host")
.WithLogLevel(CodexLogLevel.Trace, new CodexLogCustomTopics(CodexLogLevel.Error, CodexLogLevel.Error, CodexLogLevel.Warn)
{
@ -49,7 +49,7 @@ namespace CodexTests.BasicTests
var testFile = GenerateTestFile(fileSize);
var client = AddCodex(s => s
var client = StartCodex(s => s
.WithName("Client")
.EnableMarketplace(geth, contracts, m => m
.WithInitial(10.Eth(), clientInitialBalance)));

6
Tests/CodexTests/BasicTests/OneClientTests.cs
View File

@ -10,7 +10,7 @@ namespace CodexTests.BasicTests
[Test]
public void OneClientTest()
{
var primary = AddCodex();
var primary = StartCodex();
PerformOneClientTest(primary);
}
@ -18,11 +18,11 @@ namespace CodexTests.BasicTests
[Test]
public void RestartTest()
{
var primary = AddCodex();
var primary = StartCodex();
primary.Stop(waitTillStopped: true);
primary = AddCodex();
primary = StartCodex();
PerformOneClientTest(primary);
}

4
Tests/CodexTests/BasicTests/ThreeClientTest.cs
View File

@ -9,8 +9,8 @@ namespace CodexTests.BasicTests
[Test]
public void ThreeClient()
{
var primary = AddCodex();
var secondary = AddCodex();
var primary = StartCodex();
var secondary = StartCodex();
var testFile = GenerateTestFile(10.MB());

4
Tests/CodexTests/BasicTests/TwoClientTests.cs
View File

@ -10,8 +10,8 @@ namespace CodexTests.BasicTests
[Test]
public void TwoClientTest()
{
var uploader = AddCodex(s => s.WithName("Uploader"));
var downloader = AddCodex(s => s.WithName("Downloader").WithBootstrapNode(uploader));
var uploader = StartCodex(s => s.WithName("Uploader"));
var downloader = StartCodex(s => s.WithName("Downloader").WithBootstrapNode(uploader));
PerformTwoClientTest(uploader, downloader);
}

14
Tests/CodexTests/CodexDistTest.cs
View File

@ -39,22 +39,22 @@ namespace CodexTests
onlineCodexNodes.Remove(lifecycle);
}
public ICodexNode AddCodex()
public ICodexNode StartCodex()
{
return AddCodex(s => { });
return StartCodex(s => { });
}
public ICodexNode AddCodex(Action<ICodexSetup> setup)
public ICodexNode StartCodex(Action<ICodexSetup> setup)
{
return AddCodex(1, setup)[0];
return StartCodex(1, setup)[0];
}
public ICodexNodeGroup AddCodex(int numberOfNodes)
public ICodexNodeGroup StartCodex(int numberOfNodes)
{
return AddCodex(numberOfNodes, s => { });
return StartCodex(numberOfNodes, s => { });
}
public ICodexNodeGroup AddCodex(int numberOfNodes, Action<ICodexSetup> setup)
public ICodexNodeGroup StartCodex(int numberOfNodes, Action<ICodexSetup> setup)
{
var group = Ci.StartCodexNodes(numberOfNodes, s =>
{

6
Tests/CodexTests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs
View File

@ -11,7 +11,7 @@ namespace CodexTests.DownloadConnectivityTests
[Test]
public void MetricsDoesNotInterfereWithPeerDownload()
{
AddCodex(2, s => s.EnableMetrics());
StartCodex(2, s => s.EnableMetrics());
AssertAllNodesConnected();
}
@ -21,7 +21,7 @@ namespace CodexTests.DownloadConnectivityTests
{
var geth = Ci.StartGethNode(s => s.IsMiner());
var contracts = Ci.StartCodexContracts(geth);
AddCodex(2, s => s.EnableMarketplace(geth, contracts, m => m
StartCodex(2, s => s.EnableMarketplace(geth, contracts, m => m
.WithInitial(10.Eth(), 1000.TestTokens())));
AssertAllNodesConnected();
@ -33,7 +33,7 @@ namespace CodexTests.DownloadConnectivityTests
[Values(2, 5)] int numberOfNodes,
[Values(1, 10)] int sizeMBs)
{
AddCodex(numberOfNodes);
StartCodex(numberOfNodes);
AssertAllNodesConnected(sizeMBs);
}

22
Tests/CodexTests/PeerDiscoveryTests/LayeredDiscoveryTests.cs
View File

@ -8,10 +8,10 @@ namespace CodexTests.PeerDiscoveryTests
[Test]
public void TwoLayersTest()
{
var root = AddCodex();
var l1Source = AddCodex(s => s.WithBootstrapNode(root));
var l1Node = AddCodex(s => s.WithBootstrapNode(root));
var l2Target = AddCodex(s => s.WithBootstrapNode(l1Node));
var root = StartCodex();
var l1Source = StartCodex(s => s.WithBootstrapNode(root));
var l1Node = StartCodex(s => s.WithBootstrapNode(root));
var l2Target = StartCodex(s => s.WithBootstrapNode(l1Node));
AssertAllNodesConnected();
}
@ -19,11 +19,11 @@ namespace CodexTests.PeerDiscoveryTests
[Test]
public void ThreeLayersTest()
{
var root = AddCodex();
var l1Source = AddCodex(s => s.WithBootstrapNode(root));
var l1Node = AddCodex(s => s.WithBootstrapNode(root));
var l2Node = AddCodex(s => s.WithBootstrapNode(l1Node));
var l3Target = AddCodex(s => s.WithBootstrapNode(l2Node));
var root = StartCodex();
var l1Source = StartCodex(s => s.WithBootstrapNode(root));
var l1Node = StartCodex(s => s.WithBootstrapNode(root));
var l2Node = StartCodex(s => s.WithBootstrapNode(l1Node));
var l3Target = StartCodex(s => s.WithBootstrapNode(l2Node));
AssertAllNodesConnected();
}
@ -33,10 +33,10 @@ namespace CodexTests.PeerDiscoveryTests
[TestCase(10)]
public void NodeChainTest(int chainLength)
{
var node = AddCodex();
var node = StartCodex();
for (var i = 1; i < chainLength; i++)
{
node = AddCodex(s => s.WithBootstrapNode(node));
node = StartCodex(s => s.WithBootstrapNode(node));
}
AssertAllNodesConnected();

8
Tests/CodexTests/PeerDiscoveryTests/PeerDiscoveryTests.cs
View File

@ -12,7 +12,7 @@ namespace CodexTests.PeerDiscoveryTests
public void CanReportUnknownPeerId()
{
var unknownId = "16Uiu2HAkv2CHWpff3dj5iuVNERAp8AGKGNgpGjPexJZHSqUstfsK";
var node = AddCodex();
var node = StartCodex();
var result = node.GetDebugPeer(unknownId);
Assert.That(result.IsPeerFound, Is.False);
@ -21,7 +21,7 @@ namespace CodexTests.PeerDiscoveryTests
[Test]
public void MetricsDoesNotInterfereWithPeerDiscovery()
{
AddCodex(2, s => s.EnableMetrics());
StartCodex(2, s => s.EnableMetrics());
AssertAllNodesConnected();
}
@ -31,7 +31,7 @@ namespace CodexTests.PeerDiscoveryTests
{
var geth = Ci.StartGethNode(s => s.IsMiner());
var contracts = Ci.StartCodexContracts(geth);
AddCodex(2, s => s.EnableMarketplace(geth, contracts, m => m
StartCodex(2, s => s.EnableMarketplace(geth, contracts, m => m
.WithInitial(10.Eth(), 1000.TestTokens())));
AssertAllNodesConnected();
@ -42,7 +42,7 @@ namespace CodexTests.PeerDiscoveryTests
[TestCase(10)]
public void VariableNodes(int number)
{
AddCodex(number);
StartCodex(number);
AssertAllNodesConnected();
}

4
Tests/CodexTests/UtilityTests/DiscordBotTests.cs
View File

@ -58,7 +58,7 @@ namespace CodexTests.UtilityTests
for (var i = 0; i < numberOfHosts; i++)
{
var seller = AddCodex(s => s
var seller = StartCodex(s => s
.WithName("Seller")
.WithLogLevel(CodexLogLevel.Trace, new CodexLogCustomTopics(CodexLogLevel.Error, CodexLogLevel.Error, CodexLogLevel.Warn)
{
@ -82,7 +82,7 @@ namespace CodexTests.UtilityTests
var testFile = GenerateTestFile(fileSize);
var buyer = AddCodex(s => s
var buyer = StartCodex(s => s
.WithName("Buyer")
.EnableMarketplace(geth, contracts, m => m
.WithAccount(myAccount)

4
Tests/CodexTests/UtilityTests/LogHelperTests.cs
View File

@ -11,8 +11,8 @@ namespace CodexTests.UtilityTests
[Ignore("Used to find the most common log messages.")]
public void FindMostCommonLogMessages()
{
var uploader = AddCodex(s => s.WithName("uploader").WithLogLevel(CodexLogLevel.Trace));
var downloader = AddCodex(s => s.WithName("downloader").WithLogLevel(CodexLogLevel.Trace));
var uploader = StartCodex(s => s.WithName("uploader").WithLogLevel(CodexLogLevel.Trace));
var downloader = StartCodex(s => s.WithName("downloader").WithLogLevel(CodexLogLevel.Trace));
var cid = uploader.UploadFile(GenerateTestFile(100.MB()));