Merge branch 'feature/logs-update-for-kibana'
This commit is contained in:
benbierens
commit
754e74f6d8
|
|
@ -57,9 +57,11 @@ namespace ContinuousTests
|
|||
private static void PrintHelp()
|
||||
{
|
||||
var nl = Environment.NewLine;
|
||||
Console.WriteLine("CodexNetDownloader lets you download all container logs given a codex-deployment.json file." + nl);
|
||||
Console.WriteLine("ContinuousTests will run a set of tests against a codex deployment given a codex-deployment.json file." + nl +
|
||||
"The tests will run in an endless loop unless otherwise specified, using the test-specific timing values." + nl);
|
||||
|
||||
Console.WriteLine("CodexNetDownloader assumes you are running this tool from *inside* the Kubernetes cluster. " +
|
||||
|
||||
Console.WriteLine("ContinuousTests assumes you are running this tool from *inside* the Kubernetes cluster. " +
|
||||
"If you are not running this from a container inside the cluster, add the argument '--external'." + nl);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ namespace ContinuousTests
|
|||
startupChecker.Check();
|
||||
|
||||
var taskFactory = new TaskFactory();
|
||||
var overviewLog = new FixtureLog(new LogConfig(config.LogPath, false), "Overview");
|
||||
var overviewLog = new FixtureLog(new LogConfig(config.LogPath, false), DateTime.UtcNow, "Overview");
|
||||
overviewLog.Log("Continuous tests starting...");
|
||||
var allTests = testFactory.CreateTests();
|
||||
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ namespace ContinuousTests
|
|||
this.handle = handle;
|
||||
this.cancelToken = cancelToken;
|
||||
testName = handle.Test.GetType().Name;
|
||||
fixtureLog = new FixtureLog(new LogConfig(config.LogPath, true), testName);
|
||||
fixtureLog = new FixtureLog(new LogConfig(config.LogPath, true), DateTime.UtcNow, testName);
|
||||
|
||||
nodes = CreateRandomNodes(handle.Test.RequiredNumberOfNodes);
|
||||
dataFolder = config.DataPath + "-" + Guid.NewGuid();
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ namespace ContinuousTests
|
|||
|
||||
public void Check()
|
||||
{
|
||||
var log = new FixtureLog(new LogConfig(config.LogPath, false), "StartupChecks");
|
||||
var log = new FixtureLog(new LogConfig(config.LogPath, false), DateTime.UtcNow, "StartupChecks");
|
||||
log.Log("Starting continuous test run...");
|
||||
log.Log("Checking configuration...");
|
||||
PreflightCheck(config);
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ namespace DistTestCore
|
|||
private readonly Configuration configuration = new Configuration();
|
||||
private readonly Assembly[] testAssemblies;
|
||||
private readonly FixtureLog fixtureLog;
|
||||
private readonly StatusLog statusLog;
|
||||
private readonly object lifecycleLock = new object();
|
||||
private readonly Dictionary<string, TestLifecycle> lifecycles = new Dictionary<string, TestLifecycle>();
|
||||
|
||||
|
|
@ -24,7 +25,10 @@ namespace DistTestCore
|
|||
var assemblies = AppDomain.CurrentDomain.GetAssemblies();
|
||||
testAssemblies = assemblies.Where(a => a.FullName!.ToLowerInvariant().Contains("test")).ToArray();
|
||||
|
||||
fixtureLog = new FixtureLog(configuration.GetLogConfig());
|
||||
var logConfig = configuration.GetLogConfig();
|
||||
var startTime = DateTime.UtcNow;
|
||||
fixtureLog = new FixtureLog(logConfig, startTime);
|
||||
statusLog = new StatusLog(logConfig, startTime, CodexContainerRecipe.DockerImage);
|
||||
|
||||
PeerConnectionTestHelpers = new PeerConnectionTestHelpers(this);
|
||||
PeerDownloadTestHelpers = new PeerDownloadTestHelpers(this);
|
||||
|
|
@ -183,6 +187,7 @@ namespace DistTestCore
|
|||
private void CreateNewTestLifecycle()
|
||||
{
|
||||
var testName = GetCurrentTestName();
|
||||
fixtureLog.WriteLogTag();
|
||||
Stopwatch.Measure(fixtureLog, $"Setup for {testName}", () =>
|
||||
{
|
||||
lock (lifecycleLock)
|
||||
|
|
@ -195,7 +200,10 @@ namespace DistTestCore
|
|||
private void DisposeTestLifecycle()
|
||||
{
|
||||
var lifecycle = Get();
|
||||
fixtureLog.Log($"{GetCurrentTestName()} = {GetTestResult()} ({lifecycle.GetTestDuration()})");
|
||||
var testResult = GetTestResult();
|
||||
var testDuration = lifecycle.GetTestDuration();
|
||||
fixtureLog.Log($"{GetCurrentTestName()} = {testResult} ({testDuration})");
|
||||
statusLog.ConcludeTest(testResult, testDuration);
|
||||
Stopwatch.Measure(fixtureLog, $"Teardown for {GetCurrentTestName()}", () =>
|
||||
{
|
||||
lifecycle.Log.EndTest();
|
||||
|
|
|
|||
|
|
@ -0,0 +1,211 @@
|
|||
using DistTestCore.Codex;
|
||||
using NUnit.Framework;
|
||||
using Utils;
|
||||
|
||||
namespace DistTestCore.Helpers
|
||||
{
|
||||
public interface IFullConnectivityImplementation
|
||||
{
|
||||
string Description();
|
||||
string ValidateEntry(FullConnectivityHelper.Entry entry, FullConnectivityHelper.Entry[] allEntries);
|
||||
FullConnectivityHelper.PeerConnectionState Check(FullConnectivityHelper.Entry from, FullConnectivityHelper.Entry to);
|
||||
}
|
||||
|
||||
public class FullConnectivityHelper
|
||||
{
|
||||
private static string Nl = Environment.NewLine;
|
||||
private readonly DistTest test;
|
||||
private readonly IFullConnectivityImplementation implementation;
|
||||
|
||||
public FullConnectivityHelper(DistTest test, IFullConnectivityImplementation implementation)
|
||||
{
|
||||
this.test = test;
|
||||
this.implementation = implementation;
|
||||
}
|
||||
|
||||
public void AssertFullyConnected(IEnumerable<IOnlineCodexNode> nodes)
|
||||
{
|
||||
AssertFullyConnected(nodes.ToArray());
|
||||
}
|
||||
|
||||
private void AssertFullyConnected(IOnlineCodexNode[] nodes)
|
||||
{
|
||||
test.Log($"Asserting '{implementation.Description()}' for nodes: '{string.Join(",", nodes.Select(n => n.GetName()))}'...");
|
||||
var entries = CreateEntries(nodes);
|
||||
var pairs = CreatePairs(entries);
|
||||
|
||||
RetryWhilePairs(pairs, () =>
|
||||
{
|
||||
CheckAndRemoveSuccessful(pairs);
|
||||
});
|
||||
|
||||
if (pairs.Any())
|
||||
{
|
||||
var pairDetails = string.Join(Nl, pairs.SelectMany(p => p.GetResultMessages()));
|
||||
|
||||
test.Log($"Connections failed:{Nl}{pairDetails}");
|
||||
|
||||
Assert.Fail(string.Join(Nl, pairs.SelectMany(p => p.GetResultMessages())));
|
||||
}
|
||||
else
|
||||
{
|
||||
test.Log($"'{implementation.Description()}' = Success! for nodes: {string.Join(",", nodes.Select(n => n.GetName()))}");
|
||||
}
|
||||
}
|
||||
|
||||
private static void RetryWhilePairs(List<Pair> pairs, Action action)
|
||||
{
|
||||
var timeout = DateTime.UtcNow + TimeSpan.FromMinutes(5);
|
||||
while (pairs.Any(p => p.Inconclusive) && timeout > DateTime.UtcNow)
|
||||
{
|
||||
action();
|
||||
|
||||
Time.Sleep(TimeSpan.FromSeconds(2));
|
||||
}
|
||||
}
|
||||
|
||||
private void CheckAndRemoveSuccessful(List<Pair> pairs)
|
||||
{
|
||||
// For large sets, don't try and do all of them at once.
|
||||
var selectedPair = pairs.Take(20).ToArray();
|
||||
var pairDetails = new List<string>();
|
||||
|
||||
foreach (var pair in selectedPair)
|
||||
{
|
||||
test.ScopedTestFiles(pair.Check);
|
||||
|
||||
if (pair.Success)
|
||||
{
|
||||
pairDetails.AddRange(pair.GetResultMessages());
|
||||
pairs.Remove(pair);
|
||||
}
|
||||
}
|
||||
|
||||
test.Log($"Connections successful:{Nl}{string.Join(Nl, pairDetails)}");
|
||||
}
|
||||
|
||||
private Entry[] CreateEntries(IOnlineCodexNode[] nodes)
|
||||
{
|
||||
var entries = nodes.Select(n => new Entry(n)).ToArray();
|
||||
|
||||
var errors = entries
|
||||
.Select(e => implementation.ValidateEntry(e, entries))
|
||||
.Where(s => !string.IsNullOrEmpty(s))
|
||||
.ToArray();
|
||||
|
||||
if (errors.Any())
|
||||
{
|
||||
Assert.Fail("Some node entries failed to validate: " + string.Join(Nl, errors));
|
||||
}
|
||||
|
||||
return entries;
|
||||
}
|
||||
|
||||
private List<Pair> CreatePairs(Entry[] entries)
|
||||
{
|
||||
return CreatePairsIterator(entries).ToList();
|
||||
}
|
||||
|
||||
private IEnumerable<Pair> CreatePairsIterator(Entry[] entries)
|
||||
{
|
||||
for (var x = 0; x < entries.Length; x++)
|
||||
{
|
||||
for (var y = x + 1; y < entries.Length; y++)
|
||||
{
|
||||
yield return new Pair(implementation, entries[x], entries[y]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public class Entry
|
||||
{
|
||||
public Entry(IOnlineCodexNode node)
|
||||
{
|
||||
Node = node;
|
||||
Response = node.GetDebugInfo();
|
||||
}
|
||||
|
||||
public IOnlineCodexNode Node { get; }
|
||||
public CodexDebugResponse Response { get; }
|
||||
|
||||
public override string ToString()
|
||||
{
|
||||
if (Response == null || string.IsNullOrEmpty(Response.id)) return "UNKNOWN";
|
||||
return Response.id;
|
||||
}
|
||||
}
|
||||
|
||||
public enum PeerConnectionState
|
||||
{
|
||||
Unknown,
|
||||
Connection,
|
||||
NoConnection,
|
||||
}
|
||||
|
||||
public class Pair
|
||||
{
|
||||
private TimeSpan aToBTime = TimeSpan.FromSeconds(0);
|
||||
private TimeSpan bToATime = TimeSpan.FromSeconds(0);
|
||||
private readonly IFullConnectivityImplementation implementation;
|
||||
|
||||
public Pair(IFullConnectivityImplementation implementation, Entry a, Entry b)
|
||||
{
|
||||
this.implementation = implementation;
|
||||
A = a;
|
||||
B = b;
|
||||
}
|
||||
|
||||
public Entry A { get; }
|
||||
public Entry B { get; }
|
||||
public PeerConnectionState AKnowsB { get; private set; }
|
||||
public PeerConnectionState BKnowsA { get; private set; }
|
||||
public bool Success { get { return AKnowsB == PeerConnectionState.Connection && BKnowsA == PeerConnectionState.Connection; } }
|
||||
public bool Inconclusive { get { return AKnowsB == PeerConnectionState.Unknown || BKnowsA == PeerConnectionState.Unknown; } }
|
||||
|
||||
public void Check()
|
||||
{
|
||||
aToBTime = Measure(() => AKnowsB = Check(A, B));
|
||||
bToATime = Measure(() => BKnowsA = Check(B, A));
|
||||
}
|
||||
|
||||
public override string ToString()
|
||||
{
|
||||
return $"[{string.Join(",", GetResultMessages())}]";
|
||||
}
|
||||
|
||||
public string[] GetResultMessages()
|
||||
{
|
||||
var aName = A.ToString();
|
||||
var bName = B.ToString();
|
||||
|
||||
return new[]
|
||||
{
|
||||
$"[{aName} --> {bName}] = {AKnowsB} ({aToBTime.TotalSeconds} seconds)",
|
||||
$"[{aName} <-- {bName}] = {BKnowsA} ({bToATime.TotalSeconds} seconds)"
|
||||
};
|
||||
}
|
||||
|
||||
private static TimeSpan Measure(Action action)
|
||||
{
|
||||
var start = DateTime.UtcNow;
|
||||
action();
|
||||
return DateTime.UtcNow - start;
|
||||
}
|
||||
|
||||
private PeerConnectionState Check(Entry from, Entry to)
|
||||
{
|
||||
Thread.Sleep(10);
|
||||
|
||||
try
|
||||
{
|
||||
return implementation.Check(from, to);
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Didn't get a conclusive answer. Try again later.
|
||||
return PeerConnectionState.Unknown;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,149 +1,55 @@
|
|||
using DistTestCore.Codex;
|
||||
using NUnit.Framework;
|
||||
using Utils;
|
||||
using static DistTestCore.Helpers.FullConnectivityHelper;
|
||||
|
||||
namespace DistTestCore.Helpers
|
||||
{
|
||||
public class PeerConnectionTestHelpers
|
||||
public class PeerConnectionTestHelpers : IFullConnectivityImplementation
|
||||
{
|
||||
private readonly Random random = new Random();
|
||||
private readonly DistTest test;
|
||||
private readonly FullConnectivityHelper helper;
|
||||
|
||||
public PeerConnectionTestHelpers(DistTest test)
|
||||
{
|
||||
this.test = test;
|
||||
helper = new FullConnectivityHelper(test, this);
|
||||
}
|
||||
|
||||
public void AssertFullyConnected(IEnumerable<IOnlineCodexNode> nodes)
|
||||
{
|
||||
var n = nodes.ToArray();
|
||||
|
||||
AssertFullyConnected(n);
|
||||
|
||||
for (int i = 0; i < 5; i++)
|
||||
{
|
||||
Time.Sleep(TimeSpan.FromSeconds(30));
|
||||
AssertFullyConnected(n);
|
||||
}
|
||||
helper.AssertFullyConnected(nodes);
|
||||
}
|
||||
|
||||
private void AssertFullyConnected(IOnlineCodexNode[] nodes)
|
||||
public string Description()
|
||||
{
|
||||
test.Log($"Asserting peers are fully-connected for nodes: '{string.Join(",", nodes.Select(n => n.GetName()))}'...");
|
||||
var entries = CreateEntries(nodes);
|
||||
var pairs = CreatePairs(entries);
|
||||
|
||||
RetryWhilePairs(pairs, () =>
|
||||
{
|
||||
CheckAndRemoveSuccessful(pairs);
|
||||
});
|
||||
|
||||
if (pairs.Any())
|
||||
{
|
||||
test.Log($"Unsuccessful! Peers are not fully-connected: {string.Join(",", nodes.Select(n => n.GetName()))}");
|
||||
Assert.Fail(string.Join(Environment.NewLine, pairs.Select(p => p.GetMessage())));
|
||||
test.Log(string.Join(Environment.NewLine, pairs.Select(p => p.GetMessage())));
|
||||
}
|
||||
else
|
||||
{
|
||||
test.Log($"Success! Peers are fully-connected: {string.Join(",", nodes.Select(n => n.GetName()))}");
|
||||
}
|
||||
return "Peer Discovery";
|
||||
}
|
||||
|
||||
private static void RetryWhilePairs(List<Pair> pairs, Action action)
|
||||
public string ValidateEntry(Entry entry, Entry[] allEntries)
|
||||
{
|
||||
var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(30);
|
||||
while (pairs.Any() && timeout > DateTime.UtcNow)
|
||||
{
|
||||
action();
|
||||
|
||||
if (pairs.Any()) Time.Sleep(TimeSpan.FromSeconds(2));
|
||||
}
|
||||
}
|
||||
|
||||
private void CheckAndRemoveSuccessful(List<Pair> pairs)
|
||||
{
|
||||
var checkTasks = pairs.Select(p => Task.Run(() =>
|
||||
{
|
||||
ApplyRandomDelay();
|
||||
p.Check();
|
||||
})).ToArray();
|
||||
|
||||
Task.WaitAll(checkTasks);
|
||||
|
||||
foreach (var pair in pairs.ToArray())
|
||||
{
|
||||
if (pair.Success)
|
||||
{
|
||||
test.Log(pair.GetMessage());
|
||||
pairs.Remove(pair);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static Entry[] CreateEntries(IOnlineCodexNode[] nodes)
|
||||
{
|
||||
var entries = nodes.Select(n => new Entry(n)).ToArray();
|
||||
var incorrectDiscoveryEndpoints = entries.SelectMany(e => e.GetInCorrectDiscoveryEndpoints(entries)).ToArray();
|
||||
|
||||
if (incorrectDiscoveryEndpoints.Any())
|
||||
{
|
||||
Assert.Fail("Some nodes contain peer records with incorrect discovery ip/port information: " +
|
||||
string.Join(Environment.NewLine, incorrectDiscoveryEndpoints));
|
||||
}
|
||||
|
||||
return entries;
|
||||
}
|
||||
|
||||
private static List<Pair> CreatePairs(Entry[] entries)
|
||||
{
|
||||
return CreatePairsIterator(entries).ToList();
|
||||
}
|
||||
|
||||
private static IEnumerable<Pair> CreatePairsIterator(Entry[] entries)
|
||||
{
|
||||
for (var x = 0; x < entries.Length; x++)
|
||||
{
|
||||
for (var y = x + 1; y < entries.Length; y++)
|
||||
{
|
||||
yield return new Pair(entries[x], entries[y]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void ApplyRandomDelay()
|
||||
{
|
||||
// Calling all the nodes all at the same time is not exactly nice.
|
||||
Time.Sleep(TimeSpan.FromMicroseconds(random.Next(10, 1000)));
|
||||
}
|
||||
|
||||
public class Entry
|
||||
{
|
||||
public Entry(IOnlineCodexNode node)
|
||||
{
|
||||
Node = node;
|
||||
Response = node.GetDebugInfo();
|
||||
}
|
||||
|
||||
public IOnlineCodexNode Node { get; }
|
||||
public CodexDebugResponse Response { get; }
|
||||
|
||||
public IEnumerable<string> GetInCorrectDiscoveryEndpoints(Entry[] allEntries)
|
||||
{
|
||||
foreach (var peer in Response.table.nodes)
|
||||
var result = string.Empty;
|
||||
foreach (var peer in entry.Response.table.nodes)
|
||||
{
|
||||
var expected = GetExpectedDiscoveryEndpoint(allEntries, peer);
|
||||
if (expected != peer.address)
|
||||
{
|
||||
yield return $"Node:{Node.GetName()} has incorrect peer table entry. Was: '{peer.address}', expected: '{expected}'";
|
||||
result += $"Node:{entry.Node.GetName()} has incorrect peer table entry. Was: '{peer.address}', expected: '{expected}'. ";
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
public override string ToString()
|
||||
public PeerConnectionState Check(Entry from, Entry to)
|
||||
{
|
||||
if (Response == null || string.IsNullOrEmpty(Response.id)) return "UNKNOWN";
|
||||
return Response.id;
|
||||
var peerId = to.Response.id;
|
||||
|
||||
var response = from.Node.GetDebugPeer(peerId);
|
||||
if (!response.IsPeerFound)
|
||||
{
|
||||
return PeerConnectionState.NoConnection;
|
||||
}
|
||||
if (!string.IsNullOrEmpty(response.peerId) && response.addresses.Any())
|
||||
{
|
||||
return PeerConnectionState.Connection;
|
||||
}
|
||||
return PeerConnectionState.Unknown;
|
||||
}
|
||||
|
||||
private static string GetExpectedDiscoveryEndpoint(Entry[] allEntries, CodexDebugTableNodeResponse node)
|
||||
|
|
@ -157,101 +63,4 @@ namespace DistTestCore.Helpers
|
|||
return $"{ip}:{discPort.Number}";
|
||||
}
|
||||
}
|
||||
|
||||
public enum PeerConnectionState
|
||||
{
|
||||
Unknown,
|
||||
Connection,
|
||||
NoConnection,
|
||||
}
|
||||
|
||||
public class Pair
|
||||
{
|
||||
private TimeSpan aToBTime = TimeSpan.FromSeconds(0);
|
||||
private TimeSpan bToATime = TimeSpan.FromSeconds(0);
|
||||
|
||||
public Pair(Entry a, Entry b)
|
||||
{
|
||||
A = a;
|
||||
B = b;
|
||||
}
|
||||
|
||||
public Entry A { get; }
|
||||
public Entry B { get; }
|
||||
public PeerConnectionState AKnowsB { get; private set; }
|
||||
public PeerConnectionState BKnowsA { get; private set; }
|
||||
public bool Success { get { return AKnowsB == PeerConnectionState.Connection && BKnowsA == PeerConnectionState.Connection; } }
|
||||
|
||||
public void Check()
|
||||
{
|
||||
aToBTime = Measure(() => AKnowsB = Knows(A, B));
|
||||
bToATime = Measure(() => BKnowsA = Knows(B, A));
|
||||
}
|
||||
|
||||
public string GetMessage()
|
||||
{
|
||||
return GetResultMessage() + GetTimePostfix();
|
||||
}
|
||||
|
||||
public override string ToString()
|
||||
{
|
||||
return $"[{GetMessage()}]";
|
||||
}
|
||||
|
||||
private string GetResultMessage()
|
||||
{
|
||||
var aName = A.ToString();
|
||||
var bName = B.ToString();
|
||||
|
||||
if (Success)
|
||||
{
|
||||
return $"{aName} and {bName} know each other.";
|
||||
}
|
||||
|
||||
return $"[{aName}-->{bName}] = {AKnowsB} AND [{aName}<--{bName}] = {BKnowsA}";
|
||||
}
|
||||
|
||||
private string GetTimePostfix()
|
||||
{
|
||||
var aName = A.ToString();
|
||||
var bName = B.ToString();
|
||||
|
||||
return $" ({aName}->{bName}: {aToBTime.TotalMinutes} seconds, {bName}->{aName}: {bToATime.TotalSeconds} seconds)";
|
||||
}
|
||||
|
||||
private static TimeSpan Measure(Action action)
|
||||
{
|
||||
var start = DateTime.UtcNow;
|
||||
action();
|
||||
return DateTime.UtcNow - start;
|
||||
}
|
||||
|
||||
private PeerConnectionState Knows(Entry a, Entry b)
|
||||
{
|
||||
lock (a)
|
||||
{
|
||||
var peerId = b.Response.id;
|
||||
|
||||
try
|
||||
{
|
||||
var response = a.Node.GetDebugPeer(peerId);
|
||||
if (!response.IsPeerFound)
|
||||
{
|
||||
return PeerConnectionState.NoConnection;
|
||||
}
|
||||
if (!string.IsNullOrEmpty(response.peerId) && response.addresses.Any())
|
||||
{
|
||||
return PeerConnectionState.Connection;
|
||||
}
|
||||
}
|
||||
catch
|
||||
{
|
||||
}
|
||||
|
||||
// Didn't get a conclusive answer. Try again later.
|
||||
return PeerConnectionState.Unknown;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,74 +1,63 @@
|
|||
using DistTestCore.Codex;
|
||||
using NUnit.Framework;
|
||||
using static DistTestCore.Helpers.FullConnectivityHelper;
|
||||
|
||||
namespace DistTestCore.Helpers
|
||||
{
|
||||
public class PeerDownloadTestHelpers
|
||||
public class PeerDownloadTestHelpers : IFullConnectivityImplementation
|
||||
{
|
||||
private readonly FullConnectivityHelper helper;
|
||||
private readonly DistTest test;
|
||||
private ByteSize testFileSize;
|
||||
|
||||
public PeerDownloadTestHelpers(DistTest test)
|
||||
{
|
||||
helper = new FullConnectivityHelper(test, this);
|
||||
testFileSize = 1.MB();
|
||||
this.test = test;
|
||||
}
|
||||
|
||||
public void AssertFullDownloadInterconnectivity(IEnumerable<IOnlineCodexNode> nodes, ByteSize testFileSize)
|
||||
{
|
||||
test.Log($"Asserting full download interconnectivity for nodes: '{string.Join(",", nodes.Select(n => n.GetName()))}'...");
|
||||
var start = DateTime.UtcNow;
|
||||
|
||||
foreach (var node in nodes)
|
||||
{
|
||||
var uploader = node;
|
||||
var downloaders = nodes.Where(n => n != uploader).ToArray();
|
||||
|
||||
test.ScopedTestFiles(() =>
|
||||
{
|
||||
PerformTest(uploader, downloaders, testFileSize);
|
||||
});
|
||||
this.testFileSize = testFileSize;
|
||||
helper.AssertFullyConnected(nodes);
|
||||
}
|
||||
|
||||
test.Log($"Success! Full download interconnectivity for nodes: {string.Join(",", nodes.Select(n => n.GetName()))}");
|
||||
var timeTaken = DateTime.UtcNow - start;
|
||||
|
||||
AssertTimePerMB(timeTaken, nodes.Count(), testFileSize);
|
||||
public string Description()
|
||||
{
|
||||
return "Download Connectivity";
|
||||
}
|
||||
|
||||
private void AssertTimePerMB(TimeSpan timeTaken, int numberOfNodes, ByteSize size)
|
||||
public string ValidateEntry(Entry entry, Entry[] allEntries)
|
||||
{
|
||||
var numberOfDownloads = numberOfNodes * (numberOfNodes - 1);
|
||||
var timePerDownload = timeTaken / numberOfDownloads;
|
||||
float sizeInMB = size.ToMB();
|
||||
var timePerMB = timePerDownload / sizeInMB;
|
||||
|
||||
test.Log($"Performed {numberOfDownloads} downloads of {size} in {timeTaken.TotalSeconds} seconds, for an average of {timePerMB.TotalSeconds} seconds per MB.");
|
||||
|
||||
Assert.That(timePerMB, Is.LessThan(CodexContainerRecipe.MaxDownloadTimePerMegabyte), "MaxDownloadTimePerMegabyte performance threshold breached.");
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
private void PerformTest(IOnlineCodexNode uploader, IOnlineCodexNode[] downloaders, ByteSize testFileSize)
|
||||
public PeerConnectionState Check(Entry from, Entry to)
|
||||
{
|
||||
// Generate 1 test file per downloader.
|
||||
var files = downloaders.Select(d => GenerateTestFile(uploader, d, testFileSize)).ToArray();
|
||||
var expectedFile = GenerateTestFile(from.Node, to.Node);
|
||||
|
||||
// Upload all the test files to the uploader.
|
||||
var contentIds = files.Select(uploader.UploadFile).ToArray();
|
||||
var contentId = from.Node.UploadFile(expectedFile);
|
||||
|
||||
// Each downloader should retrieve its own test file.
|
||||
for (var i = 0; i < downloaders.Length; i++)
|
||||
try
|
||||
{
|
||||
var expectedFile = files[i];
|
||||
var downloadedFile = downloaders[i].DownloadContent(contentIds[i], $"{expectedFile.Label}DOWNLOADED");
|
||||
|
||||
var downloadedFile = to.Node.DownloadContent(contentId, expectedFile.Label + "_downloaded");
|
||||
expectedFile.AssertIsEqual(downloadedFile);
|
||||
return PeerConnectionState.Connection;
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Should an exception occur during the download or file-content assertion,
|
||||
// We consider that as no-connection for the purpose of this test.
|
||||
return PeerConnectionState.NoConnection;
|
||||
}
|
||||
|
||||
private TestFile GenerateTestFile(IOnlineCodexNode uploader, IOnlineCodexNode downloader, ByteSize testFileSize)
|
||||
// Should an exception occur during upload, then this try is inconclusive and we try again next loop.
|
||||
}
|
||||
|
||||
private TestFile GenerateTestFile(IOnlineCodexNode uploader, IOnlineCodexNode downloader)
|
||||
{
|
||||
var up = uploader.GetName().Replace("<", "").Replace(">", "");
|
||||
var down = downloader.GetName().Replace("<", "").Replace(">", "");
|
||||
var label = $"FROM{up}TO{down}";
|
||||
var label = $"~from:{up}-to:{down}~";
|
||||
return test.GenerateTestFile(testFileSize, label);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -25,6 +25,8 @@ namespace DistTestCore
|
|||
PrometheusStarter = new PrometheusStarter(this, workflowCreator);
|
||||
GethStarter = new GethStarter(this, workflowCreator);
|
||||
testStart = DateTime.UtcNow;
|
||||
|
||||
Log.WriteLogTag();
|
||||
}
|
||||
|
||||
public BaseLog Log { get; }
|
||||
|
|
|
|||
|
|
@ -73,6 +73,14 @@ namespace Logging
|
|||
return new LogFile($"{GetFullName()}_{GetSubfileNumber()}", ext);
|
||||
}
|
||||
|
||||
public void WriteLogTag()
|
||||
{
|
||||
var runId = NameUtils.GetRunId();
|
||||
var category = NameUtils.GetCategoryName();
|
||||
var name = NameUtils.GetTestMethodName();
|
||||
LogFile.WriteRaw($"{runId} {category} {name}");
|
||||
}
|
||||
|
||||
private string ApplyReplacements(string str)
|
||||
{
|
||||
foreach (var replacement in replacements)
|
||||
|
|
|
|||
|
|
@ -1,20 +1,14 @@
|
|||
using NUnit.Framework;
|
||||
|
||||
namespace Logging
|
||||
namespace Logging
|
||||
{
|
||||
public class FixtureLog : BaseLog
|
||||
{
|
||||
private readonly DateTime start;
|
||||
private readonly string fullName;
|
||||
private readonly LogConfig config;
|
||||
|
||||
public FixtureLog(LogConfig config, string name = "")
|
||||
public FixtureLog(LogConfig config, DateTime start, string name = "")
|
||||
: base(config.DebugEnabled)
|
||||
{
|
||||
start = DateTime.UtcNow;
|
||||
var folder = DetermineFolder(config);
|
||||
var fixtureName = GetFixtureName(name);
|
||||
fullName = Path.Combine(folder, fixtureName);
|
||||
fullName = NameUtils.GetFixtureFullName(config, start, name);
|
||||
this.config = config;
|
||||
}
|
||||
|
||||
|
|
@ -32,28 +26,5 @@ namespace Logging
|
|||
{
|
||||
return fullName;
|
||||
}
|
||||
|
||||
private string DetermineFolder(LogConfig config)
|
||||
{
|
||||
return Path.Join(
|
||||
config.LogRoot,
|
||||
$"{start.Year}-{Pad(start.Month)}",
|
||||
Pad(start.Day));
|
||||
}
|
||||
|
||||
private string GetFixtureName(string name)
|
||||
{
|
||||
var test = TestContext.CurrentContext.Test;
|
||||
var className = test.ClassName!.Substring(test.ClassName.LastIndexOf('.') + 1);
|
||||
if (!string.IsNullOrEmpty(name)) className = name;
|
||||
|
||||
return $"{Pad(start.Hour)}-{Pad(start.Minute)}-{Pad(start.Second)}Z_{className.Replace('.', '-')}";
|
||||
}
|
||||
|
||||
private static string Pad(int n)
|
||||
{
|
||||
return n.ToString().PadLeft(2, '0');
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -49,7 +49,7 @@
|
|||
|
||||
private static string GetTimestamp()
|
||||
{
|
||||
return $"[{DateTime.UtcNow.ToString("u")}]";
|
||||
return $"[{DateTime.UtcNow.ToString("o")}]";
|
||||
}
|
||||
|
||||
private void EnsurePathExists(string filename)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,83 @@
|
|||
using NUnit.Framework;
|
||||
|
||||
namespace Logging
|
||||
{
|
||||
public static class NameUtils
|
||||
{
|
||||
public static string GetTestMethodName(string name = "")
|
||||
{
|
||||
if (!string.IsNullOrEmpty(name)) return name;
|
||||
var test = TestContext.CurrentContext.Test;
|
||||
var args = FormatArguments(test);
|
||||
return ReplaceInvalidCharacters($"{test.MethodName}{args}");
|
||||
}
|
||||
|
||||
public static string GetFixtureFullName(LogConfig config, DateTime start, string name)
|
||||
{
|
||||
var folder = DetermineFolder(config, start);
|
||||
var fixtureName = GetFixtureName(name, start);
|
||||
return Path.Combine(folder, fixtureName);
|
||||
}
|
||||
|
||||
public static string GetRawFixtureName()
|
||||
{
|
||||
var test = TestContext.CurrentContext.Test;
|
||||
var className = test.ClassName!.Substring(test.ClassName.LastIndexOf('.') + 1);
|
||||
return className.Replace('.', '-');
|
||||
}
|
||||
|
||||
public static string GetCategoryName()
|
||||
{
|
||||
var test = TestContext.CurrentContext.Test;
|
||||
return test.ClassName!.Substring(0, test.ClassName.LastIndexOf('.'));
|
||||
}
|
||||
|
||||
public static string GetTestId()
|
||||
{
|
||||
return GetEnvVar("TESTID");
|
||||
}
|
||||
|
||||
public static string GetRunId()
|
||||
{
|
||||
return GetEnvVar("RUNID");
|
||||
}
|
||||
|
||||
private static string GetEnvVar(string name)
|
||||
{
|
||||
var v = Environment.GetEnvironmentVariable(name);
|
||||
if (string.IsNullOrEmpty(v)) return $"EnvVar'{name}'NotSet";
|
||||
return v;
|
||||
}
|
||||
|
||||
private static string FormatArguments(TestContext.TestAdapter test)
|
||||
{
|
||||
if (test.Arguments == null || !test.Arguments.Any()) return "";
|
||||
return $"[{string.Join(',', test.Arguments)}]";
|
||||
}
|
||||
|
||||
private static string ReplaceInvalidCharacters(string name)
|
||||
{
|
||||
return name.Replace(":", "_");
|
||||
}
|
||||
|
||||
private static string DetermineFolder(LogConfig config, DateTime start)
|
||||
{
|
||||
return Path.Join(
|
||||
config.LogRoot,
|
||||
$"{start.Year}-{Pad(start.Month)}",
|
||||
Pad(start.Day));
|
||||
}
|
||||
|
||||
private static string GetFixtureName(string name, DateTime start)
|
||||
{
|
||||
var className = GetRawFixtureName();
|
||||
if (!string.IsNullOrEmpty(name)) className = name;
|
||||
return $"{Pad(start.Hour)}-{Pad(start.Minute)}-{Pad(start.Second)}Z_{className.Replace('.', '-')}";
|
||||
}
|
||||
|
||||
private static string Pad(int n)
|
||||
{
|
||||
return n.ToString().PadLeft(2, '0');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,63 @@
|
|||
using Newtonsoft.Json;
|
||||
|
||||
namespace Logging
|
||||
{
|
||||
public class StatusLog
|
||||
{
|
||||
private readonly object fileLock = new object();
|
||||
private readonly string fullName;
|
||||
private readonly string fixtureName;
|
||||
private readonly string codexId;
|
||||
|
||||
public StatusLog(LogConfig config, DateTime start, string codexId, string name = "")
|
||||
{
|
||||
fullName = NameUtils.GetFixtureFullName(config, start, name) + "_STATUS.log";
|
||||
fixtureName = NameUtils.GetRawFixtureName();
|
||||
this.codexId = codexId;
|
||||
}
|
||||
|
||||
public void ConcludeTest(string resultStatus, string testDuration)
|
||||
{
|
||||
Write(new StatusLogJson
|
||||
{
|
||||
@timestamp = DateTime.UtcNow.ToString("o"),
|
||||
runid = NameUtils.GetRunId(),
|
||||
status = resultStatus,
|
||||
testid = NameUtils.GetTestId(),
|
||||
codexid = codexId,
|
||||
category = NameUtils.GetCategoryName(),
|
||||
fixturename = fixtureName,
|
||||
testname = NameUtils.GetTestMethodName(),
|
||||
testduration = testDuration
|
||||
});
|
||||
}
|
||||
|
||||
private void Write(StatusLogJson json)
|
||||
{
|
||||
try
|
||||
{
|
||||
lock (fileLock)
|
||||
{
|
||||
File.AppendAllLines(fullName, new[] { JsonConvert.SerializeObject(json) });
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Console.WriteLine("Unable to write to status log: " + ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public class StatusLogJson
|
||||
{
|
||||
public string @timestamp { get; set; } = string.Empty;
|
||||
public string runid { get; set; } = string.Empty;
|
||||
public string status { get; set; } = string.Empty;
|
||||
public string testid { get; set; } = string.Empty;
|
||||
public string codexid { get; set; } = string.Empty;
|
||||
public string category { get; set; } = string.Empty;
|
||||
public string fixturename { get; set; } = string.Empty;
|
||||
public string testname { get; set; } = string.Empty;
|
||||
public string testduration { get; set;} = string.Empty;
|
||||
}
|
||||
}
|
||||
|
|
@ -10,7 +10,7 @@ namespace Logging
|
|||
public TestLog(string folder, bool debug, string name = "")
|
||||
: base(debug)
|
||||
{
|
||||
methodName = GetMethodName(name);
|
||||
methodName = NameUtils.GetTestMethodName(name);
|
||||
fullName = Path.Combine(folder, methodName);
|
||||
|
||||
Log($"*** Begin: {methodName}");
|
||||
|
|
@ -37,24 +37,5 @@ namespace Logging
|
|||
{
|
||||
return fullName;
|
||||
}
|
||||
|
||||
private string GetMethodName(string name)
|
||||
{
|
||||
if (!string.IsNullOrEmpty(name)) return name;
|
||||
var test = TestContext.CurrentContext.Test;
|
||||
var args = FormatArguments(test);
|
||||
return ReplaceInvalidCharacters($"{test.MethodName}{args}");
|
||||
}
|
||||
|
||||
private static string FormatArguments(TestContext.TestAdapter test)
|
||||
{
|
||||
if (test.Arguments == null || !test.Arguments.Any()) return "";
|
||||
return $"[{string.Join(',', test.Arguments)}]";
|
||||
}
|
||||
|
||||
private static string ReplaceInvalidCharacters(string name)
|
||||
{
|
||||
return name.Replace(":", "_");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue