status-im
/
cs-codex-dist-tests
mirror of
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Cleanup of peer connection helpers. Adds peer download helpers.

This commit is contained in:
benbierens 2023-05-29 09:13:38 +02:00
parent ab7a334987
commit 1016f568b8
No known key found for this signature in database
GPG Key ID: FE44815D96D0A1AA
6 changed files with 131 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
DistTestCore/DistTest.cs
View File

@ -1,4 +1,5 @@
using DistTestCore.Codex;
using DistTestCore.Helpers;
using DistTestCore.Logs;
using DistTestCore.Marketplace;
using DistTestCore.Metrics;
@ -25,8 +26,14 @@ namespace DistTestCore
testAssemblies = assemblies.Where(a => a.FullName!.ToLowerInvariant().Contains("test")).ToArray();
fixtureLog = new FixtureLog(configuration.GetLogConfig());
PeerConnectionTestHelpers = new PeerConnectionTestHelpers(this);
PeerDownloadTestHelpers = new PeerDownloadTestHelpers(this);
}
public PeerConnectionTestHelpers PeerConnectionTestHelpers { get; }
public PeerDownloadTestHelpers PeerDownloadTestHelpers { get; }
[OneTimeSetUp]
public void GlobalSetup()
{
@ -85,6 +92,17 @@ namespace DistTestCore
return Get().FileManager.GenerateTestFile(size);
}
/// <summary>
/// Any test files generated in 'action' will be deleted after it returns.
/// This helps prevent large tests from filling up discs.
/// </summary>
public void ScopedTestFiles(Action action)
{
Get().FileManager.PushFileSet();
action();
Get().FileManager.PopFileSet();
}
public IOnlineCodexNode SetupCodexBootstrapNode()
{
return SetupCodexBootstrapNode(s => { });
@ -133,18 +151,18 @@ namespace DistTestCore
return Get().CodexStarter.RunningGroups.SelectMany(g => g.Nodes);
}
protected BaseLog GetTestLog()
public BaseLog GetTestLog()
{
return Get().Log;
}
protected void Log(string msg)
public void Log(string msg)
{
TestContext.Progress.WriteLine(msg);
GetTestLog().Log(msg);
}
protected void Debug(string msg)
public void Debug(string msg)
{
TestContext.Progress.WriteLine(msg);
GetTestLog().Debug(msg);

25
DistTestCore/FileManager.cs
View File

@ -9,6 +9,8 @@ namespace DistTestCore
TestFile CreateEmptyTestFile();
TestFile GenerateTestFile(ByteSize size);
void DeleteAllTestFiles();
void PushFileSet();
void PopFileSet();
}
public class FileManager : IFileManager
@ -18,6 +20,7 @@ namespace DistTestCore
private readonly Random random = new Random();
private readonly TestLog log;
private readonly string folder;
private readonly List<List<TestFile>> fileSetStack = new List<List<TestFile>>();
public FileManager(TestLog log, Configuration configuration)
{
@ -31,6 +34,7 @@ namespace DistTestCore
{
var result = new TestFile(Path.Combine(folder, Guid.NewGuid().ToString() + "_test.bin"));
File.Create(result.Filename).Close();
if (fileSetStack.Any()) fileSetStack.Last().Add(result);
return result;
}
@ -47,6 +51,27 @@ namespace DistTestCore
DeleteDirectory();
}
public void PushFileSet()
{
fileSetStack.Add(new List<TestFile>());
}
public void PopFileSet()
{
if (!fileSetStack.Any()) return;
var pop = fileSetStack.Last();
fileSetStack.Remove(pop);
foreach (var file in pop)
{
try
{
File.Delete(file.Filename);
}
catch { }
}
}
private void GenerateFileBytes(TestFile result, ByteSize size)
{
long bytesLeft = size.SizeInBytes;

55
Tests/PeerDiscoveryTests/PeerTestHelpers.cs → DistTestCore/Helpers/PeerConnectionTestHelpers.cs
View File

@ -1,30 +1,34 @@
using DistTestCore.Codex;
using DistTestCore;
using NUnit.Framework;
using Logging;
using Utils;
namespace Tests.PeerDiscoveryTests
namespace DistTestCore.Helpers
{
public static class PeerTestHelpers
public class PeerConnectionTestHelpers
{
private static readonly Random random = new Random();
private readonly Random random = new Random();
private readonly DistTest test;
public static void AssertFullyConnected(IEnumerable<IOnlineCodexNode> nodes, BaseLog? log = null)
public PeerConnectionTestHelpers(DistTest test)
{
AssertFullyConnected(log, nodes.ToArray());
this.test = test;
}
public static void AssertFullyConnected(BaseLog? log = null, params IOnlineCodexNode[] nodes)
public void AssertFullyConnected(IEnumerable<IOnlineCodexNode> nodes)
{
AssertFullyConnected(nodes.ToArray());
}
public void AssertFullyConnected(params IOnlineCodexNode[] nodes)
{
var entries = CreateEntries(nodes);
var pairs = CreatePairs(entries);
RetryWhilePairs(pairs, () =>
{
CheckAndRemoveSuccessful(pairs, log);
CheckAndRemoveSuccessful(pairs);
});
if (pairs.Any())
{
Assert.Fail(string.Join(Environment.NewLine, pairs.Select(p => p.GetMessage())));
@ -34,7 +38,7 @@ namespace Tests.PeerDiscoveryTests
private static void RetryWhilePairs(List<Pair> pairs, Action action)
{
var timeout = DateTime.UtcNow + TimeSpan.FromMinutes(5);
while (pairs.Any() && (timeout > DateTime.UtcNow))
while (pairs.Any() && timeout > DateTime.UtcNow)
{
action();
@ -42,15 +46,21 @@ namespace Tests.PeerDiscoveryTests
}
}
private static void CheckAndRemoveSuccessful(List<Pair> pairs, BaseLog? log)
private void CheckAndRemoveSuccessful(List<Pair> pairs)
{
var checkTasks = pairs.Select(p => Task.Run(p.Check)).ToArray();
var checkTasks = pairs.Select(p => Task.Run(() =>
{
ApplyRandomDelay();
p.Check();
})).ToArray();
Task.WaitAll(checkTasks);
foreach (var pair in pairs.ToArray())
{
if (pair.Success)
{
test.Debug(pair.GetMessage());
pairs.Remove(pair);
}
}
@ -60,7 +70,7 @@ namespace Tests.PeerDiscoveryTests
{
var entries = nodes.Select(n => new Entry(n)).ToArray();
var incorrectDiscoveryEndpoints = entries.SelectMany(e => e.GetInCorrectDiscoveryEndpoints(entries)).ToArray();
if (incorrectDiscoveryEndpoints.Any())
{
Assert.Fail("Some nodes contain peer records with incorrect discovery ip/port information: " +
@ -86,6 +96,12 @@ namespace Tests.PeerDiscoveryTests
}
}
private void ApplyRandomDelay()
{
// Calling all the nodes all at the same time is not exactly nice.
Time.Sleep(TimeSpan.FromMicroseconds(random.Next(10, 100)));
}
public class Entry
{
public Entry(IOnlineCodexNode node)
@ -94,7 +110,7 @@ namespace Tests.PeerDiscoveryTests
Response = node.GetDebugInfo();
}
public IOnlineCodexNode Node { get ; }
public IOnlineCodexNode Node { get; }
public CodexDebugResponse Response { get; }
public IEnumerable<string> GetInCorrectDiscoveryEndpoints(Entry[] allEntries)
@ -137,11 +153,10 @@ namespace Tests.PeerDiscoveryTests
public Entry B { get; }
public bool AKnowsB { get; private set; }
public bool BKnowsA { get; private set; }
public bool Success { get { return AKnowsB && BKnowsA; } }
public bool Success { get { return AKnowsB && BKnowsA; } }
public void Check()
{
ApplyRandomDelay();
aToBTime = Measure(() => AKnowsB = Knows(A, B));
bToATime = Measure(() => BKnowsA = Knows(B, A));
}
@ -179,12 +194,6 @@ namespace Tests.PeerDiscoveryTests
return $" ({aName}->{bName}: {aToBTime.TotalMinutes} seconds, {bName}->{aName}: {bToATime.TotalSeconds} seconds)";
}
private static void ApplyRandomDelay()
{
// Calling all the nodes all at the same time is not exactly nice.
Time.Sleep(TimeSpan.FromMicroseconds(random.Next(10, 100)));
}
private static TimeSpan Measure(Action action)
{
var start = DateTime.UtcNow;

49
DistTestCore/Helpers/PeerDownloadTestHelpers.cs Normal file
View File

@ -0,0 +1,49 @@
namespace DistTestCore.Helpers
{
public class PeerDownloadTestHelpers
{
private readonly DistTest test;
public PeerDownloadTestHelpers(DistTest test)
{
this.test = test;
}
public void AssertFullDownloadInterconnectivity(IEnumerable<IOnlineCodexNode> nodes)
{
AssertFullDownloadInterconnectivity(nodes, 1.MB());
}
public void AssertFullDownloadInterconnectivity(IEnumerable<IOnlineCodexNode> nodes, ByteSize testFileSize)
{
foreach (var node in nodes)
{
var uploader = node;
var downloaders = nodes.Where(n => n != uploader).ToArray();
test.ScopedTestFiles(() =>
{
PerformTest(uploader, downloaders);
});
}
}
private void PerformTest(IOnlineCodexNode uploader, IOnlineCodexNode[] downloaders)
{
// 1 test file per downloader.
var files = downloaders.Select(d => test.GenerateTestFile(1.MB())).ToArray();
// Upload all the test files to the uploader.
var contentIds = files.Select(uploader.UploadFile).ToArray();
// Each downloader should retrieve its own test file.
for (var i = 0; i < downloaders.Length; i++)
{
var expectedFile = files[i];
var downloadedFile = downloaders[i].DownloadContent(contentIds[i]);
expectedFile.AssertIsEqual(downloadedFile);
}
}
}
}

3
Tests/PeerDiscoveryTests/LayeredDiscoveryTests.cs
View File

@ -1,4 +1,5 @@
using DistTestCore;
using DistTestCore.Helpers;
using NUnit.Framework;
using Utils;
@ -54,7 +55,7 @@ namespace Tests.PeerDiscoveryTests
private void AssertAllNodesConnected()
{
PeerTestHelpers.AssertFullyConnected(GetAllOnlineCodexNodes(), GetTestLog());
PeerConnectionTestHelpers.AssertFullyConnected(GetAllOnlineCodexNodes());
}
}
}

3
Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs
View File

@ -1,5 +1,6 @@
using DistTestCore;
using DistTestCore.Codex;
using DistTestCore.Helpers;
using NUnit.Framework;
using Utils;
@ -55,7 +56,7 @@ namespace Tests.PeerDiscoveryTests
private void AssertAllNodesConnected()
{
PeerTestHelpers.AssertFullyConnected(GetAllOnlineCodexNodes(), GetTestLog());
PeerConnectionTestHelpers.AssertFullyConnected(GetAllOnlineCodexNodes());
}
}
}