codex-storage
/
cs-codex-dist-tests
mirror of https://github.com/codex-storage/cs-codex-dist-tests.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Retain for a while in staged variable nodes peer discovery tests

This commit is contained in:
benbierens 2023-05-18 10:42:04 +02:00
parent 830d74a5ae
commit 025b03e678
No known key found for this signature in database
GPG Key ID: FE44815D96D0A1AA
5 changed files with 92 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
DistTestCore/AutoBootstrapDistTest.cs
View File

@ -21,7 +21,10 @@ namespace DistTestCore
[SetUp]
public void SetUpBootstrapNode()
{
BootstrapNode = BringOnline(new CodexSetup(1))[0];
BootstrapNode = BringOnline(new CodexSetup(1)
{
LogLevel = Codex.CodexLogLevel.Trace
})[0];
}
protected IOnlineCodexNode BootstrapNode { get; private set; } = null!;

9
DistTestCore/Codex/CodexAccess.cs
View File

@ -24,7 +24,12 @@ namespace DistTestCore.Codex
public CodexDebugPeerResponse GetDebugPeer(string peerId)
{
return Http(TimeSpan.FromSeconds(2)).HttpGetJson<CodexDebugPeerResponse>($"debug/peer/{peerId}");
return GetDebugPeer(peerId, TimeSpan.FromSeconds(2));
}
public CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout)
{
return Http(timeout).HttpGetJson<CodexDebugPeerResponse>($"debug/peer/{peerId}");
}
public string UploadFile(FileStream fileStream)
@ -62,6 +67,7 @@ namespace DistTestCore.Codex
var nodePeerId = debugInfo.id;
var nodeName = Container.Name;
log.AddStringReplace(nodePeerId, $"___{nodeName}___");
log.AddStringReplace(debugInfo.table.localNode.nodeId, $"__{nodeName}__");
}
catch (Exception e)
{
@ -99,6 +105,7 @@ namespace DistTestCore.Codex
public class CodexDebugTableNodeResponse
{
public string nodeId { get; set; } = string.Empty;
public string peerId { get; set; } = string.Empty;
public string record { get; set; } = string.Empty;
public bool seen { get; set; }
}

9
DistTestCore/OnlineCodexNode.cs
View File

@ -11,6 +11,7 @@ namespace DistTestCore
string GetName();
CodexDebugResponse GetDebugInfo();
CodexDebugPeerResponse GetDebugPeer(string peerId);
CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout);
ContentId UploadFile(TestFile file);
TestFile? DownloadContent(ContentId contentId);
void ConnectToPeer(IOnlineCodexNode node);
@ -48,7 +49,8 @@ namespace DistTestCore
public CodexDebugResponse GetDebugInfo()
{
var debugInfo = CodexAccess.GetDebugInfo();
Log($"Got DebugInfo with id: '{debugInfo.id}'.");
var known = string.Join(",", debugInfo.table.nodes.Select(n => n.peerId));
Log($"Got DebugInfo with id: '{debugInfo.id}'. This node knows: {known}");
return debugInfo;
}
@ -57,6 +59,11 @@ namespace DistTestCore
return CodexAccess.GetDebugPeer(peerId);
}
public CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout)
{
return CodexAccess.GetDebugPeer(peerId, timeout);
}
public ContentId UploadFile(TestFile file)
{
Log($"Uploading file of size {file.GetFileSize()}...");

14
Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs
View File

@ -1,5 +1,7 @@
using DistTestCore;
using DistTestCore.Codex;
using NUnit.Framework;
using Utils;
namespace Tests.PeerDiscoveryTests
{
@ -32,14 +34,24 @@ namespace Tests.PeerDiscoveryTests
[TestCase(3, 3)]
[TestCase(3, 5)]
[TestCase(3, 10)]
[TestCase(5, 10)]
[TestCase(3, 20)]
[TestCase(5, 20)]
public void StagedVariableNodes(int numberOfNodes, int numberOfStages)
{
for (var i = 0; i < numberOfStages; i++)
{
SetupCodexNodes(numberOfNodes);
SetupCodexNodes(numberOfNodes, s => s.WithLogLevel(CodexLogLevel.Trace));
AssertAllNodesConnected();
}
// Retain for a while
for (int i = 0; i < 10; i++)
{
Time.Sleep(TimeSpan.FromSeconds(30));
AssertAllNodesConnected();
}
}
private void AssertAllNodesConnected()

73
Tests/PeerDiscoveryTests/PeerTestHelpers.cs
View File

@ -8,6 +8,8 @@ namespace Tests.PeerDiscoveryTests
{
public static class PeerTestHelpers
{
private static readonly Random random = new Random();
public static void AssertFullyConnected(IEnumerable<IOnlineCodexNode> nodes, BaseLog? log = null)
{
AssertFullyConnected(log, nodes.ToArray());
@ -31,7 +33,7 @@ namespace Tests.PeerDiscoveryTests
private static void RetryWhilePairs(List<Pair> pairs, Action action)
{
var timeout = DateTime.UtcNow + TimeSpan.FromMinutes(2);
var timeout = DateTime.UtcNow + TimeSpan.FromMinutes(5);
while (pairs.Any() && (timeout > DateTime.UtcNow))
{
action();
@ -52,6 +54,10 @@ namespace Tests.PeerDiscoveryTests
pairs.Remove(pair);
if (log != null) log.Log(pair.GetMessage());
}
else
{
pair.IncreaseTimeout();
}
}
}
@ -90,6 +96,10 @@ namespace Tests.PeerDiscoveryTests
public class Pair
{
private TimeSpan timeout = TimeSpan.FromSeconds(60);
private TimeSpan aToBTime = TimeSpan.FromSeconds(0);
private TimeSpan bToATime = TimeSpan.FromSeconds(0);
public Pair(Entry a, Entry b)
{
A = a;
@ -104,11 +114,22 @@ namespace Tests.PeerDiscoveryTests
public void Check()
{
AKnowsB = Knows(A, B);
BKnowsA = Knows(B, A);
ApplyRandomDelay();
aToBTime = Measure(() => AKnowsB = Knows(A, B));
bToATime = Measure(() => BKnowsA = Knows(B, A));
}
public string GetMessage()
{
return GetResultMessage() + GetTimePostfix();
}
public void IncreaseTimeout()
{
//timeout *= 2;
}
private string GetResultMessage()
{
var aName = A.Response.id;
var bName = B.Response.id;
@ -128,23 +149,47 @@ namespace Tests.PeerDiscoveryTests
return $"{aName} and {bName} don't know each other.";
}
private static bool Knows(Entry a, Entry b)
private string GetTimePostfix()
{
var peerId = b.Response.id;
var aName = A.Response.id;
var bName = B.Response.id;
try
return $" ({aName}->{bName}: {aToBTime.TotalMinutes} seconds, {bName}->{aName}: {bToATime.TotalSeconds} seconds)";
}
private static void ApplyRandomDelay()
{
// Calling all the nodes all at the same time is not exactly nice.
Time.Sleep(TimeSpan.FromMicroseconds(random.Next(10, 100)));
}
private static TimeSpan Measure(Action action)
{
var start = DateTime.UtcNow;
action();
return DateTime.UtcNow - start;
}
private bool Knows(Entry a, Entry b)
{
lock (a)
{
var response = a.Node.GetDebugPeer(peerId);
if (!string.IsNullOrEmpty(response.peerId) && response.addresses.Any())
var peerId = b.Response.id;
try
{
var response = a.Node.GetDebugPeer(peerId, timeout);
if (!string.IsNullOrEmpty(response.peerId) && response.addresses.Any())
{
return true;
}
}
catch
{
return true;
}
}
catch
{
}
return false;
return false;
}
}
}
}