Retain for a while in staged variable nodes peer discovery tests
This commit is contained in:
parent
830d74a5ae
commit
025b03e678
|
@ -21,7 +21,10 @@ namespace DistTestCore
|
||||||
[SetUp]
|
[SetUp]
|
||||||
public void SetUpBootstrapNode()
|
public void SetUpBootstrapNode()
|
||||||
{
|
{
|
||||||
BootstrapNode = BringOnline(new CodexSetup(1))[0];
|
BootstrapNode = BringOnline(new CodexSetup(1)
|
||||||
|
{
|
||||||
|
LogLevel = Codex.CodexLogLevel.Trace
|
||||||
|
})[0];
|
||||||
}
|
}
|
||||||
|
|
||||||
protected IOnlineCodexNode BootstrapNode { get; private set; } = null!;
|
protected IOnlineCodexNode BootstrapNode { get; private set; } = null!;
|
||||||
|
|
|
@ -24,7 +24,12 @@ namespace DistTestCore.Codex
|
||||||
|
|
||||||
public CodexDebugPeerResponse GetDebugPeer(string peerId)
|
public CodexDebugPeerResponse GetDebugPeer(string peerId)
|
||||||
{
|
{
|
||||||
return Http(TimeSpan.FromSeconds(2)).HttpGetJson<CodexDebugPeerResponse>($"debug/peer/{peerId}");
|
return GetDebugPeer(peerId, TimeSpan.FromSeconds(2));
|
||||||
|
}
|
||||||
|
|
||||||
|
public CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout)
|
||||||
|
{
|
||||||
|
return Http(timeout).HttpGetJson<CodexDebugPeerResponse>($"debug/peer/{peerId}");
|
||||||
}
|
}
|
||||||
|
|
||||||
public string UploadFile(FileStream fileStream)
|
public string UploadFile(FileStream fileStream)
|
||||||
|
@ -62,6 +67,7 @@ namespace DistTestCore.Codex
|
||||||
var nodePeerId = debugInfo.id;
|
var nodePeerId = debugInfo.id;
|
||||||
var nodeName = Container.Name;
|
var nodeName = Container.Name;
|
||||||
log.AddStringReplace(nodePeerId, $"___{nodeName}___");
|
log.AddStringReplace(nodePeerId, $"___{nodeName}___");
|
||||||
|
log.AddStringReplace(debugInfo.table.localNode.nodeId, $"__{nodeName}__");
|
||||||
}
|
}
|
||||||
catch (Exception e)
|
catch (Exception e)
|
||||||
{
|
{
|
||||||
|
@ -99,6 +105,7 @@ namespace DistTestCore.Codex
|
||||||
public class CodexDebugTableNodeResponse
|
public class CodexDebugTableNodeResponse
|
||||||
{
|
{
|
||||||
public string nodeId { get; set; } = string.Empty;
|
public string nodeId { get; set; } = string.Empty;
|
||||||
|
public string peerId { get; set; } = string.Empty;
|
||||||
public string record { get; set; } = string.Empty;
|
public string record { get; set; } = string.Empty;
|
||||||
public bool seen { get; set; }
|
public bool seen { get; set; }
|
||||||
}
|
}
|
||||||
|
|
|
@ -11,6 +11,7 @@ namespace DistTestCore
|
||||||
string GetName();
|
string GetName();
|
||||||
CodexDebugResponse GetDebugInfo();
|
CodexDebugResponse GetDebugInfo();
|
||||||
CodexDebugPeerResponse GetDebugPeer(string peerId);
|
CodexDebugPeerResponse GetDebugPeer(string peerId);
|
||||||
|
CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout);
|
||||||
ContentId UploadFile(TestFile file);
|
ContentId UploadFile(TestFile file);
|
||||||
TestFile? DownloadContent(ContentId contentId);
|
TestFile? DownloadContent(ContentId contentId);
|
||||||
void ConnectToPeer(IOnlineCodexNode node);
|
void ConnectToPeer(IOnlineCodexNode node);
|
||||||
|
@ -48,7 +49,8 @@ namespace DistTestCore
|
||||||
public CodexDebugResponse GetDebugInfo()
|
public CodexDebugResponse GetDebugInfo()
|
||||||
{
|
{
|
||||||
var debugInfo = CodexAccess.GetDebugInfo();
|
var debugInfo = CodexAccess.GetDebugInfo();
|
||||||
Log($"Got DebugInfo with id: '{debugInfo.id}'.");
|
var known = string.Join(",", debugInfo.table.nodes.Select(n => n.peerId));
|
||||||
|
Log($"Got DebugInfo with id: '{debugInfo.id}'. This node knows: {known}");
|
||||||
return debugInfo;
|
return debugInfo;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -57,6 +59,11 @@ namespace DistTestCore
|
||||||
return CodexAccess.GetDebugPeer(peerId);
|
return CodexAccess.GetDebugPeer(peerId);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout)
|
||||||
|
{
|
||||||
|
return CodexAccess.GetDebugPeer(peerId, timeout);
|
||||||
|
}
|
||||||
|
|
||||||
public ContentId UploadFile(TestFile file)
|
public ContentId UploadFile(TestFile file)
|
||||||
{
|
{
|
||||||
Log($"Uploading file of size {file.GetFileSize()}...");
|
Log($"Uploading file of size {file.GetFileSize()}...");
|
||||||
|
|
|
@ -1,5 +1,7 @@
|
||||||
using DistTestCore;
|
using DistTestCore;
|
||||||
|
using DistTestCore.Codex;
|
||||||
using NUnit.Framework;
|
using NUnit.Framework;
|
||||||
|
using Utils;
|
||||||
|
|
||||||
namespace Tests.PeerDiscoveryTests
|
namespace Tests.PeerDiscoveryTests
|
||||||
{
|
{
|
||||||
|
@ -32,14 +34,24 @@ namespace Tests.PeerDiscoveryTests
|
||||||
[TestCase(3, 3)]
|
[TestCase(3, 3)]
|
||||||
[TestCase(3, 5)]
|
[TestCase(3, 5)]
|
||||||
[TestCase(3, 10)]
|
[TestCase(3, 10)]
|
||||||
|
[TestCase(5, 10)]
|
||||||
|
[TestCase(3, 20)]
|
||||||
|
[TestCase(5, 20)]
|
||||||
public void StagedVariableNodes(int numberOfNodes, int numberOfStages)
|
public void StagedVariableNodes(int numberOfNodes, int numberOfStages)
|
||||||
{
|
{
|
||||||
for (var i = 0; i < numberOfStages; i++)
|
for (var i = 0; i < numberOfStages; i++)
|
||||||
{
|
{
|
||||||
SetupCodexNodes(numberOfNodes);
|
SetupCodexNodes(numberOfNodes, s => s.WithLogLevel(CodexLogLevel.Trace));
|
||||||
|
|
||||||
AssertAllNodesConnected();
|
AssertAllNodesConnected();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Retain for a while
|
||||||
|
for (int i = 0; i < 10; i++)
|
||||||
|
{
|
||||||
|
Time.Sleep(TimeSpan.FromSeconds(30));
|
||||||
|
AssertAllNodesConnected();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void AssertAllNodesConnected()
|
private void AssertAllNodesConnected()
|
||||||
|
|
|
@ -8,6 +8,8 @@ namespace Tests.PeerDiscoveryTests
|
||||||
{
|
{
|
||||||
public static class PeerTestHelpers
|
public static class PeerTestHelpers
|
||||||
{
|
{
|
||||||
|
private static readonly Random random = new Random();
|
||||||
|
|
||||||
public static void AssertFullyConnected(IEnumerable<IOnlineCodexNode> nodes, BaseLog? log = null)
|
public static void AssertFullyConnected(IEnumerable<IOnlineCodexNode> nodes, BaseLog? log = null)
|
||||||
{
|
{
|
||||||
AssertFullyConnected(log, nodes.ToArray());
|
AssertFullyConnected(log, nodes.ToArray());
|
||||||
|
@ -31,7 +33,7 @@ namespace Tests.PeerDiscoveryTests
|
||||||
|
|
||||||
private static void RetryWhilePairs(List<Pair> pairs, Action action)
|
private static void RetryWhilePairs(List<Pair> pairs, Action action)
|
||||||
{
|
{
|
||||||
var timeout = DateTime.UtcNow + TimeSpan.FromMinutes(2);
|
var timeout = DateTime.UtcNow + TimeSpan.FromMinutes(5);
|
||||||
while (pairs.Any() && (timeout > DateTime.UtcNow))
|
while (pairs.Any() && (timeout > DateTime.UtcNow))
|
||||||
{
|
{
|
||||||
action();
|
action();
|
||||||
|
@ -52,6 +54,10 @@ namespace Tests.PeerDiscoveryTests
|
||||||
pairs.Remove(pair);
|
pairs.Remove(pair);
|
||||||
if (log != null) log.Log(pair.GetMessage());
|
if (log != null) log.Log(pair.GetMessage());
|
||||||
}
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
pair.IncreaseTimeout();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -90,6 +96,10 @@ namespace Tests.PeerDiscoveryTests
|
||||||
|
|
||||||
public class Pair
|
public class Pair
|
||||||
{
|
{
|
||||||
|
private TimeSpan timeout = TimeSpan.FromSeconds(60);
|
||||||
|
private TimeSpan aToBTime = TimeSpan.FromSeconds(0);
|
||||||
|
private TimeSpan bToATime = TimeSpan.FromSeconds(0);
|
||||||
|
|
||||||
public Pair(Entry a, Entry b)
|
public Pair(Entry a, Entry b)
|
||||||
{
|
{
|
||||||
A = a;
|
A = a;
|
||||||
|
@ -104,11 +114,22 @@ namespace Tests.PeerDiscoveryTests
|
||||||
|
|
||||||
public void Check()
|
public void Check()
|
||||||
{
|
{
|
||||||
AKnowsB = Knows(A, B);
|
ApplyRandomDelay();
|
||||||
BKnowsA = Knows(B, A);
|
aToBTime = Measure(() => AKnowsB = Knows(A, B));
|
||||||
|
bToATime = Measure(() => BKnowsA = Knows(B, A));
|
||||||
}
|
}
|
||||||
|
|
||||||
public string GetMessage()
|
public string GetMessage()
|
||||||
|
{
|
||||||
|
return GetResultMessage() + GetTimePostfix();
|
||||||
|
}
|
||||||
|
|
||||||
|
public void IncreaseTimeout()
|
||||||
|
{
|
||||||
|
//timeout *= 2;
|
||||||
|
}
|
||||||
|
|
||||||
|
private string GetResultMessage()
|
||||||
{
|
{
|
||||||
var aName = A.Response.id;
|
var aName = A.Response.id;
|
||||||
var bName = B.Response.id;
|
var bName = B.Response.id;
|
||||||
|
@ -128,13 +149,36 @@ namespace Tests.PeerDiscoveryTests
|
||||||
return $"{aName} and {bName} don't know each other.";
|
return $"{aName} and {bName} don't know each other.";
|
||||||
}
|
}
|
||||||
|
|
||||||
private static bool Knows(Entry a, Entry b)
|
private string GetTimePostfix()
|
||||||
|
{
|
||||||
|
var aName = A.Response.id;
|
||||||
|
var bName = B.Response.id;
|
||||||
|
|
||||||
|
return $" ({aName}->{bName}: {aToBTime.TotalMinutes} seconds, {bName}->{aName}: {bToATime.TotalSeconds} seconds)";
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void ApplyRandomDelay()
|
||||||
|
{
|
||||||
|
// Calling all the nodes all at the same time is not exactly nice.
|
||||||
|
Time.Sleep(TimeSpan.FromMicroseconds(random.Next(10, 100)));
|
||||||
|
}
|
||||||
|
|
||||||
|
private static TimeSpan Measure(Action action)
|
||||||
|
{
|
||||||
|
var start = DateTime.UtcNow;
|
||||||
|
action();
|
||||||
|
return DateTime.UtcNow - start;
|
||||||
|
}
|
||||||
|
|
||||||
|
private bool Knows(Entry a, Entry b)
|
||||||
|
{
|
||||||
|
lock (a)
|
||||||
{
|
{
|
||||||
var peerId = b.Response.id;
|
var peerId = b.Response.id;
|
||||||
|
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
var response = a.Node.GetDebugPeer(peerId);
|
var response = a.Node.GetDebugPeer(peerId, timeout);
|
||||||
if (!string.IsNullOrEmpty(response.peerId) && response.addresses.Any())
|
if (!string.IsNullOrEmpty(response.peerId) && response.addresses.Any())
|
||||||
{
|
{
|
||||||
return true;
|
return true;
|
||||||
|
@ -148,4 +192,5 @@ namespace Tests.PeerDiscoveryTests
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue