codex-storage
/
cs-codex-dist-tests
mirror of https://github.com/codex-storage/cs-codex-dist-tests.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into feature/log-continuous-downloader

This commit is contained in:
benbierens 2023-07-21 09:25:41 +02:00
parent 8b3589a489 754e74f6d8
commit 71f45ca5ea
No known key found for this signature in database
GPG Key ID: FE44815D96D0A1AA
17 changed files with 461 additions and 329 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
.github/workflows/dist-tests.yaml vendored
View File

@ -15,19 +15,19 @@ on:
workflow_dispatch:
inputs:
branch:
description: Branch
description: Branch (master)
required: false
type: string
source:
description: Repository with tests
description: Repository with tests (current)
required: false
type: string
nameprefix:
description: Runner job/pod name prefix
description: Runner prefix (cs-codex-dist-tests)
required: false
type: string
namespace:
description: Kubernetes namespace for runner
description: Runner namespace (cs-codex-dist-tests)
required: false
type: string
@ -56,6 +56,8 @@ jobs:
[[ -n "${{ inputs.source }}" ]] && echo "SOURCE=${{ inputs.source }}" >>"$GITHUB_ENV" || echo "SOURCE=${{ env.SOURCE }}" >>"$GITHUB_ENV"
[[ -n "${{ inputs.nameprefix }}" ]] && echo "NAMEPREFIX=${{ inputs.nameprefix }}" >>"$GITHUB_ENV" || echo "NAMEPREFIX=${{ env.NAMEPREFIX }}" >>"$GITHUB_ENV"
[[ -n "${{ inputs.namespace }}" ]] && echo "NAMESPACE=${{ inputs.namespace }}" >>"$GITHUB_ENV" || echo "NAMESPACE=${{ env.NAMESPACE }}" >>"$GITHUB_ENV"
echo "RUNID=$(date +%Y%m%d-%H%M%S)" >> $GITHUB_ENV
echo "TESTID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
- name: Kubectl - Install ${{ env.KUBE_VERSION }}
uses: azure/setup-kubectl@v3
@ -69,5 +71,4 @@ jobs:
- name: Kubectl - Create Job
run: |
export RUNID=$(date +%Y%m%d-%H%M%S)
envsubst < ${{ env.JOB_MANIFEST }} | kubectl apply -f -

6
ContinuousTests/Configuration.cs
View File

@ -60,9 +60,11 @@ namespace ContinuousTests
private static void PrintHelp()
{
var nl = Environment.NewLine;
Console.WriteLine("CodexNetDownloader lets you download all container logs given a codex-deployment.json file." + nl);
Console.WriteLine("ContinuousTests will run a set of tests against a codex deployment given a codex-deployment.json file." + nl +
"The tests will run in an endless loop unless otherwise specified, using the test-specific timing values." + nl);
Console.WriteLine("CodexNetDownloader assumes you are running this tool from *inside* the Kubernetes cluster. " +
Console.WriteLine("ContinuousTests assumes you are running this tool from *inside* the Kubernetes cluster. " +
"If you are not running this from a container inside the cluster, add the argument '--external'." + nl);
}
}

2
ContinuousTests/ContinuousTestRunner.cs
View File

@ -24,7 +24,7 @@ namespace ContinuousTests
startupChecker.Check();
var taskFactory = new TaskFactory();
var overviewLog = new FixtureLog(new LogConfig(config.LogPath, false), "Overview");
var overviewLog = new FixtureLog(new LogConfig(config.LogPath, false), DateTime.UtcNow, "Overview");
overviewLog.Log("Continuous tests starting...");
var allTests = testFactory.CreateTests();

2
ContinuousTests/SingleTestRun.cs
View File

@ -32,7 +32,7 @@ namespace ContinuousTests
this.handle = handle;
this.cancelToken = cancelToken;
testName = handle.Test.GetType().Name;
fixtureLog = new FixtureLog(new LogConfig(config.LogPath, true), testName);
fixtureLog = new FixtureLog(new LogConfig(config.LogPath, true), DateTime.UtcNow, testName);
nodes = CreateRandomNodes(handle.Test.RequiredNumberOfNodes);
dataFolder = config.DataPath + "-" + Guid.NewGuid();

2
ContinuousTests/StartupChecker.cs
View File

@ -19,7 +19,7 @@ namespace ContinuousTests
public void Check()
{
var log = new FixtureLog(new LogConfig(config.LogPath, false), "StartupChecks");
var log = new FixtureLog(new LogConfig(config.LogPath, false), DateTime.UtcNow, "StartupChecks");
log.Log("Starting continuous test run...");
log.Log("Checking configuration...");
PreflightCheck(config);

12
DistTestCore/DistTest.cs
View File

@ -16,6 +16,7 @@ namespace DistTestCore
private readonly Configuration configuration = new Configuration();
private readonly Assembly[] testAssemblies;
private readonly FixtureLog fixtureLog;
private readonly StatusLog statusLog;
private readonly object lifecycleLock = new object();
private readonly Dictionary<string, TestLifecycle> lifecycles = new Dictionary<string, TestLifecycle>();
@ -24,7 +25,10 @@ namespace DistTestCore
var assemblies = AppDomain.CurrentDomain.GetAssemblies();
testAssemblies = assemblies.Where(a => a.FullName!.ToLowerInvariant().Contains("test")).ToArray();
fixtureLog = new FixtureLog(configuration.GetLogConfig());
var logConfig = configuration.GetLogConfig();
var startTime = DateTime.UtcNow;
fixtureLog = new FixtureLog(logConfig, startTime);
statusLog = new StatusLog(logConfig, startTime, CodexContainerRecipe.DockerImage);
PeerConnectionTestHelpers = new PeerConnectionTestHelpers(this);
PeerDownloadTestHelpers = new PeerDownloadTestHelpers(this);
@ -183,6 +187,7 @@ namespace DistTestCore
private void CreateNewTestLifecycle()
{
var testName = GetCurrentTestName();
fixtureLog.WriteLogTag();
Stopwatch.Measure(fixtureLog, $"Setup for {testName}", () =>
{
lock (lifecycleLock)
@ -195,7 +200,10 @@ namespace DistTestCore
private void DisposeTestLifecycle()
{
var lifecycle = Get();
fixtureLog.Log($"{GetCurrentTestName()} = {GetTestResult()} ({lifecycle.GetTestDuration()})");
var testResult = GetTestResult();
var testDuration = lifecycle.GetTestDuration();
fixtureLog.Log($"{GetCurrentTestName()} = {testResult} ({testDuration})");
statusLog.ConcludeTest(testResult, testDuration);
Stopwatch.Measure(fixtureLog, $"Teardown for {GetCurrentTestName()}", () =>
{
lifecycle.Log.EndTest();

211
DistTestCore/Helpers/FullConnectivityHelper.cs Normal file
View File

@ -0,0 +1,211 @@
using DistTestCore.Codex;
using NUnit.Framework;
using Utils;
namespace DistTestCore.Helpers
{
public interface IFullConnectivityImplementation
{
string Description();
string ValidateEntry(FullConnectivityHelper.Entry entry, FullConnectivityHelper.Entry[] allEntries);
FullConnectivityHelper.PeerConnectionState Check(FullConnectivityHelper.Entry from, FullConnectivityHelper.Entry to);
}
public class FullConnectivityHelper
{
private static string Nl = Environment.NewLine;
private readonly DistTest test;
private readonly IFullConnectivityImplementation implementation;
public FullConnectivityHelper(DistTest test, IFullConnectivityImplementation implementation)
{
this.test = test;
this.implementation = implementation;
}
public void AssertFullyConnected(IEnumerable<IOnlineCodexNode> nodes)
{
AssertFullyConnected(nodes.ToArray());
}
private void AssertFullyConnected(IOnlineCodexNode[] nodes)
{
test.Log($"Asserting '{implementation.Description()}' for nodes: '{string.Join(",", nodes.Select(n => n.GetName()))}'...");
var entries = CreateEntries(nodes);
var pairs = CreatePairs(entries);
RetryWhilePairs(pairs, () =>
{
CheckAndRemoveSuccessful(pairs);
});
if (pairs.Any())
{
var pairDetails = string.Join(Nl, pairs.SelectMany(p => p.GetResultMessages()));
test.Log($"Connections failed:{Nl}{pairDetails}");
Assert.Fail(string.Join(Nl, pairs.SelectMany(p => p.GetResultMessages())));
}
else
{
test.Log($"'{implementation.Description()}' = Success! for nodes: {string.Join(",", nodes.Select(n => n.GetName()))}");
}
}
private static void RetryWhilePairs(List<Pair> pairs, Action action)
{
var timeout = DateTime.UtcNow + TimeSpan.FromMinutes(5);
while (pairs.Any(p => p.Inconclusive) && timeout > DateTime.UtcNow)
{
action();
Time.Sleep(TimeSpan.FromSeconds(2));
}
}
private void CheckAndRemoveSuccessful(List<Pair> pairs)
{
// For large sets, don't try and do all of them at once.
var selectedPair = pairs.Take(20).ToArray();
var pairDetails = new List<string>();
foreach (var pair in selectedPair)
{
test.ScopedTestFiles(pair.Check);
if (pair.Success)
{
pairDetails.AddRange(pair.GetResultMessages());
pairs.Remove(pair);
}
}
test.Log($"Connections successful:{Nl}{string.Join(Nl, pairDetails)}");
}
private Entry[] CreateEntries(IOnlineCodexNode[] nodes)
{
var entries = nodes.Select(n => new Entry(n)).ToArray();
var errors = entries
.Select(e => implementation.ValidateEntry(e, entries))
.Where(s => !string.IsNullOrEmpty(s))
.ToArray();
if (errors.Any())
{
Assert.Fail("Some node entries failed to validate: " + string.Join(Nl, errors));
}
return entries;
}
private List<Pair> CreatePairs(Entry[] entries)
{
return CreatePairsIterator(entries).ToList();
}
private IEnumerable<Pair> CreatePairsIterator(Entry[] entries)
{
for (var x = 0; x < entries.Length; x++)
{
for (var y = x + 1; y < entries.Length; y++)
{
yield return new Pair(implementation, entries[x], entries[y]);
}
}
}
public class Entry
{
public Entry(IOnlineCodexNode node)
{
Node = node;
Response = node.GetDebugInfo();
}
public IOnlineCodexNode Node { get; }
public CodexDebugResponse Response { get; }
public override string ToString()
{
if (Response == null || string.IsNullOrEmpty(Response.id)) return "UNKNOWN";
return Response.id;
}
}
public enum PeerConnectionState
{
Unknown,
Connection,
NoConnection,
}
public class Pair
{
private TimeSpan aToBTime = TimeSpan.FromSeconds(0);
private TimeSpan bToATime = TimeSpan.FromSeconds(0);
private readonly IFullConnectivityImplementation implementation;
public Pair(IFullConnectivityImplementation implementation, Entry a, Entry b)
{
this.implementation = implementation;
A = a;
B = b;
}
public Entry A { get; }
public Entry B { get; }
public PeerConnectionState AKnowsB { get; private set; }
public PeerConnectionState BKnowsA { get; private set; }
public bool Success { get { return AKnowsB == PeerConnectionState.Connection && BKnowsA == PeerConnectionState.Connection; } }
public bool Inconclusive { get { return AKnowsB == PeerConnectionState.Unknown || BKnowsA == PeerConnectionState.Unknown; } }
public void Check()
{
aToBTime = Measure(() => AKnowsB = Check(A, B));
bToATime = Measure(() => BKnowsA = Check(B, A));
}
public override string ToString()
{
return $"[{string.Join(",", GetResultMessages())}]";
}
public string[] GetResultMessages()
{
var aName = A.ToString();
var bName = B.ToString();
return new[]
{
$"[{aName} --> {bName}] = {AKnowsB} ({aToBTime.TotalSeconds} seconds)",
$"[{aName} <-- {bName}] = {BKnowsA} ({bToATime.TotalSeconds} seconds)"
};
}
private static TimeSpan Measure(Action action)
{
var start = DateTime.UtcNow;
action();
return DateTime.UtcNow - start;
}
private PeerConnectionState Check(Entry from, Entry to)
{
Thread.Sleep(10);
try
{
return implementation.Check(from, to);
}
catch
{
// Didn't get a conclusive answer. Try again later.
return PeerConnectionState.Unknown;
}
}
}
}
}

253
DistTestCore/Helpers/PeerConnectionTestHelpers.cs
View File

@ -1,257 +1,66 @@
using DistTestCore.Codex;
using NUnit.Framework;
using Utils;
using static DistTestCore.Helpers.FullConnectivityHelper;
namespace DistTestCore.Helpers
{
public class PeerConnectionTestHelpers
public class PeerConnectionTestHelpers : IFullConnectivityImplementation
{
private readonly Random random = new Random();
private readonly DistTest test;
private readonly FullConnectivityHelper helper;
public PeerConnectionTestHelpers(DistTest test)
{
this.test = test;
helper = new FullConnectivityHelper(test, this);
}
public void AssertFullyConnected(IEnumerable<IOnlineCodexNode> nodes)
{
var n = nodes.ToArray();
AssertFullyConnected(n);
for (int i = 0; i < 5; i++)
{
Time.Sleep(TimeSpan.FromSeconds(30));
AssertFullyConnected(n);
}
helper.AssertFullyConnected(nodes);
}
private void AssertFullyConnected(IOnlineCodexNode[] nodes)
public string Description()
{
test.Log($"Asserting peers are fully-connected for nodes: '{string.Join(",", nodes.Select(n => n.GetName()))}'...");
var entries = CreateEntries(nodes);
var pairs = CreatePairs(entries);
RetryWhilePairs(pairs, () =>
{
CheckAndRemoveSuccessful(pairs);
});
if (pairs.Any())
{
test.Log($"Unsuccessful! Peers are not fully-connected: {string.Join(",", nodes.Select(n => n.GetName()))}");
Assert.Fail(string.Join(Environment.NewLine, pairs.Select(p => p.GetMessage())));
test.Log(string.Join(Environment.NewLine, pairs.Select(p => p.GetMessage())));
}
else
{
test.Log($"Success! Peers are fully-connected: {string.Join(",", nodes.Select(n => n.GetName()))}");
}
return "Peer Discovery";
}
private static void RetryWhilePairs(List<Pair> pairs, Action action)
public string ValidateEntry(Entry entry, Entry[] allEntries)
{
var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(30);
while (pairs.Any() && timeout > DateTime.UtcNow)
var result = string.Empty;
foreach (var peer in entry.Response.table.nodes)
{
action();
if (pairs.Any()) Time.Sleep(TimeSpan.FromSeconds(2));
}
}
private void CheckAndRemoveSuccessful(List<Pair> pairs)
{
var checkTasks = pairs.Select(p => Task.Run(() =>
{
ApplyRandomDelay();
p.Check();
})).ToArray();
Task.WaitAll(checkTasks);
foreach (var pair in pairs.ToArray())
{
if (pair.Success)
var expected = GetExpectedDiscoveryEndpoint(allEntries, peer);
if (expected != peer.address)
{
test.Log(pair.GetMessage());
pairs.Remove(pair);
result += $"Node:{entry.Node.GetName()} has incorrect peer table entry. Was: '{peer.address}', expected: '{expected}'. ";
}
}
return result;
}
private static Entry[] CreateEntries(IOnlineCodexNode[] nodes)
public PeerConnectionState Check(Entry from, Entry to)
{
var entries = nodes.Select(n => new Entry(n)).ToArray();
var incorrectDiscoveryEndpoints = entries.SelectMany(e => e.GetInCorrectDiscoveryEndpoints(entries)).ToArray();
var peerId = to.Response.id;
if (incorrectDiscoveryEndpoints.Any())
var response = from.Node.GetDebugPeer(peerId);
if (!response.IsPeerFound)
{
Assert.Fail("Some nodes contain peer records with incorrect discovery ip/port information: " +
string.Join(Environment.NewLine, incorrectDiscoveryEndpoints));
return PeerConnectionState.NoConnection;
}
return entries;
if (!string.IsNullOrEmpty(response.peerId) && response.addresses.Any())
{
return PeerConnectionState.Connection;
}
return PeerConnectionState.Unknown;
}
private static List<Pair> CreatePairs(Entry[] entries)
private static string GetExpectedDiscoveryEndpoint(Entry[] allEntries, CodexDebugTableNodeResponse node)
{
return CreatePairsIterator(entries).ToList();
}
var peer = allEntries.SingleOrDefault(e => e.Response.table.localNode.peerId == node.peerId);
if (peer == null) return $"peerId: {node.peerId} is not known.";
private static IEnumerable<Pair> CreatePairsIterator(Entry[] entries)
{
for (var x = 0; x < entries.Length; x++)
{
for (var y = x + 1; y < entries.Length; y++)
{
yield return new Pair(entries[x], entries[y]);
}
}
}
private void ApplyRandomDelay()
{
// Calling all the nodes all at the same time is not exactly nice.
Time.Sleep(TimeSpan.FromMicroseconds(random.Next(10, 1000)));
}
public class Entry
{
public Entry(IOnlineCodexNode node)
{
Node = node;
Response = node.GetDebugInfo();
}
public IOnlineCodexNode Node { get; }
public CodexDebugResponse Response { get; }
public IEnumerable<string> GetInCorrectDiscoveryEndpoints(Entry[] allEntries)
{
foreach (var peer in Response.table.nodes)
{
var expected = GetExpectedDiscoveryEndpoint(allEntries, peer);
if (expected != peer.address)
{
yield return $"Node:{Node.GetName()} has incorrect peer table entry. Was: '{peer.address}', expected: '{expected}'";
}
}
}
public override string ToString()
{
if (Response == null || string.IsNullOrEmpty(Response.id)) return "UNKNOWN";
return Response.id;
}
private static string GetExpectedDiscoveryEndpoint(Entry[] allEntries, CodexDebugTableNodeResponse node)
{
var peer = allEntries.SingleOrDefault(e => e.Response.table.localNode.peerId == node.peerId);
if (peer == null) return $"peerId: {node.peerId} is not known.";
var n = (OnlineCodexNode)peer.Node;
var ip = n.CodexAccess.Container.Pod.PodInfo.Ip;
var discPort = n.CodexAccess.Container.Recipe.GetPortByTag(CodexContainerRecipe.DiscoveryPortTag);
return $"{ip}:{discPort.Number}";
}
}
public enum PeerConnectionState
{
Unknown,
Connection,
NoConnection,
}
public class Pair
{
private TimeSpan aToBTime = TimeSpan.FromSeconds(0);
private TimeSpan bToATime = TimeSpan.FromSeconds(0);
public Pair(Entry a, Entry b)
{
A = a;
B = b;
}
public Entry A { get; }
public Entry B { get; }
public PeerConnectionState AKnowsB { get; private set; }
public PeerConnectionState BKnowsA { get; private set; }
public bool Success { get { return AKnowsB == PeerConnectionState.Connection && BKnowsA == PeerConnectionState.Connection; } }
public void Check()
{
aToBTime = Measure(() => AKnowsB = Knows(A, B));
bToATime = Measure(() => BKnowsA = Knows(B, A));
}
public string GetMessage()
{
return GetResultMessage() + GetTimePostfix();
}
public override string ToString()
{
return $"[{GetMessage()}]";
}
private string GetResultMessage()
{
var aName = A.ToString();
var bName = B.ToString();
if (Success)
{
return $"{aName} and {bName} know each other.";
}
return $"[{aName}-->{bName}] = {AKnowsB} AND [{aName}<--{bName}] = {BKnowsA}";
}
private string GetTimePostfix()
{
var aName = A.ToString();
var bName = B.ToString();
return $" ({aName}->{bName}: {aToBTime.TotalMinutes} seconds, {bName}->{aName}: {bToATime.TotalSeconds} seconds)";
}
private static TimeSpan Measure(Action action)
{
var start = DateTime.UtcNow;
action();
return DateTime.UtcNow - start;
}
private PeerConnectionState Knows(Entry a, Entry b)
{
lock (a)
{
var peerId = b.Response.id;
try
{
var response = a.Node.GetDebugPeer(peerId);
if (!response.IsPeerFound)
{
return PeerConnectionState.NoConnection;
}
if (!string.IsNullOrEmpty(response.peerId) && response.addresses.Any())
{
return PeerConnectionState.Connection;
}
}
catch
{
}
// Didn't get a conclusive answer. Try again later.
return PeerConnectionState.Unknown;
}
}
var n = (OnlineCodexNode)peer.Node;
var ip = n.CodexAccess.Container.Pod.PodInfo.Ip;
var discPort = n.CodexAccess.Container.Recipe.GetPortByTag(CodexContainerRecipe.DiscoveryPortTag);
return $"{ip}:{discPort.Number}";
}
}
}

73
DistTestCore/Helpers/PeerDownloadTestHelpers.cs
View File

@ -1,74 +1,63 @@
using DistTestCore.Codex;
using NUnit.Framework;
using static DistTestCore.Helpers.FullConnectivityHelper;
namespace DistTestCore.Helpers
{
public class PeerDownloadTestHelpers
public class PeerDownloadTestHelpers : IFullConnectivityImplementation
{
private readonly FullConnectivityHelper helper;
private readonly DistTest test;
private ByteSize testFileSize;
public PeerDownloadTestHelpers(DistTest test)
{
helper = new FullConnectivityHelper(test, this);
testFileSize = 1.MB();
this.test = test;
}
public void AssertFullDownloadInterconnectivity(IEnumerable<IOnlineCodexNode> nodes, ByteSize testFileSize)
{
test.Log($"Asserting full download interconnectivity for nodes: '{string.Join(",", nodes.Select(n => n.GetName()))}'...");
var start = DateTime.UtcNow;
foreach (var node in nodes)
{
var uploader = node;
var downloaders = nodes.Where(n => n != uploader).ToArray();
test.ScopedTestFiles(() =>
{
PerformTest(uploader, downloaders, testFileSize);
});
}
test.Log($"Success! Full download interconnectivity for nodes: {string.Join(",", nodes.Select(n => n.GetName()))}");
var timeTaken = DateTime.UtcNow - start;
AssertTimePerMB(timeTaken, nodes.Count(), testFileSize);
this.testFileSize = testFileSize;
helper.AssertFullyConnected(nodes);
}
private void AssertTimePerMB(TimeSpan timeTaken, int numberOfNodes, ByteSize size)
public string Description()
{
var numberOfDownloads = numberOfNodes * (numberOfNodes - 1);
var timePerDownload = timeTaken / numberOfDownloads;
float sizeInMB = size.ToMB();
var timePerMB = timePerDownload / sizeInMB;
test.Log($"Performed {numberOfDownloads} downloads of {size} in {timeTaken.TotalSeconds} seconds, for an average of {timePerMB.TotalSeconds} seconds per MB.");
Assert.That(timePerMB, Is.LessThan(CodexContainerRecipe.MaxDownloadTimePerMegabyte), "MaxDownloadTimePerMegabyte performance threshold breached.");
return "Download Connectivity";
}
private void PerformTest(IOnlineCodexNode uploader, IOnlineCodexNode[] downloaders, ByteSize testFileSize)
public string ValidateEntry(Entry entry, Entry[] allEntries)
{
// Generate 1 test file per downloader.
var files = downloaders.Select(d => GenerateTestFile(uploader, d, testFileSize)).ToArray();
return string.Empty;
}
// Upload all the test files to the uploader.
var contentIds = files.Select(uploader.UploadFile).ToArray();
public PeerConnectionState Check(Entry from, Entry to)
{
var expectedFile = GenerateTestFile(from.Node, to.Node);
// Each downloader should retrieve its own test file.
for (var i = 0; i < downloaders.Length; i++)
var contentId = from.Node.UploadFile(expectedFile);
try
{
var expectedFile = files[i];
var downloadedFile = downloaders[i].DownloadContent(contentIds[i], $"{expectedFile.Label}DOWNLOADED");
var downloadedFile = to.Node.DownloadContent(contentId, expectedFile.Label + "_downloaded");
expectedFile.AssertIsEqual(downloadedFile);
return PeerConnectionState.Connection;
}
catch
{
// Should an exception occur during the download or file-content assertion,
// We consider that as no-connection for the purpose of this test.
return PeerConnectionState.NoConnection;
}
// Should an exception occur during upload, then this try is inconclusive and we try again next loop.
}
private TestFile GenerateTestFile(IOnlineCodexNode uploader, IOnlineCodexNode downloader, ByteSize testFileSize)
private TestFile GenerateTestFile(IOnlineCodexNode uploader, IOnlineCodexNode downloader)
{
var up = uploader.GetName().Replace("<", "").Replace(">", "");
var down = downloader.GetName().Replace("<", "").Replace(">", "");
var label = $"FROM{up}TO{down}";
var label = $"~from:{up}-to:{down}~";
return test.GenerateTestFile(testFileSize, label);
}
}

2
DistTestCore/TestLifecycle.cs
View File

@ -25,6 +25,8 @@ namespace DistTestCore
PrometheusStarter = new PrometheusStarter(this, workflowCreator);
GethStarter = new GethStarter(this, workflowCreator);
testStart = DateTime.UtcNow;
Log.WriteLogTag();
}
public BaseLog Log { get; }

8
Logging/BaseLog.cs
View File

@ -73,6 +73,14 @@ namespace Logging
return new LogFile($"{GetFullName()}_{GetSubfileNumber()}", ext);
}
public void WriteLogTag()
{
var runId = NameUtils.GetRunId();
var category = NameUtils.GetCategoryName();
var name = NameUtils.GetTestMethodName();
LogFile.WriteRaw($"{runId} {category} {name}");
}
private string ApplyReplacements(string str)
{
foreach (var replacement in replacements)

35
Logging/FixtureLog.cs
View File

@ -1,20 +1,14 @@
using NUnit.Framework;
namespace Logging
namespace Logging
{
public class FixtureLog : BaseLog
{
private readonly DateTime start;
private readonly string fullName;
private readonly LogConfig config;
public FixtureLog(LogConfig config, string name = "")
public FixtureLog(LogConfig config, DateTime start, string name = "")
: base(config.DebugEnabled)
{
start = DateTime.UtcNow;
var folder = DetermineFolder(config);
var fixtureName = GetFixtureName(name);
fullName = Path.Combine(folder, fixtureName);
fullName = NameUtils.GetFixtureFullName(config, start, name);
this.config = config;
}
@ -32,28 +26,5 @@ namespace Logging
{
return fullName;
}
private string DetermineFolder(LogConfig config)
{
return Path.Join(
config.LogRoot,
$"{start.Year}-{Pad(start.Month)}",
Pad(start.Day));
}
private string GetFixtureName(string name)
{
var test = TestContext.CurrentContext.Test;
var className = test.ClassName!.Substring(test.ClassName.LastIndexOf('.') + 1);
if (!string.IsNullOrEmpty(name)) className = name;
return $"{Pad(start.Hour)}-{Pad(start.Minute)}-{Pad(start.Second)}Z_{className.Replace('.', '-')}";
}
private static string Pad(int n)
{
return n.ToString().PadLeft(2, '0');
}
}
}

2
Logging/LogFile.cs
View File

@ -49,7 +49,7 @@
private static string GetTimestamp()
{
return $"[{DateTime.UtcNow.ToString("u")}]";
return $"[{DateTime.UtcNow.ToString("o")}]";
}
private void EnsurePathExists(string filename)

83
Logging/NameUtils.cs Normal file
View File

@ -0,0 +1,83 @@
using NUnit.Framework;
namespace Logging
{
public static class NameUtils
{
public static string GetTestMethodName(string name = "")
{
if (!string.IsNullOrEmpty(name)) return name;
var test = TestContext.CurrentContext.Test;
var args = FormatArguments(test);
return ReplaceInvalidCharacters($"{test.MethodName}{args}");
}
public static string GetFixtureFullName(LogConfig config, DateTime start, string name)
{
var folder = DetermineFolder(config, start);
var fixtureName = GetFixtureName(name, start);
return Path.Combine(folder, fixtureName);
}
public static string GetRawFixtureName()
{
var test = TestContext.CurrentContext.Test;
var className = test.ClassName!.Substring(test.ClassName.LastIndexOf('.') + 1);
return className.Replace('.', '-');
}
public static string GetCategoryName()
{
var test = TestContext.CurrentContext.Test;
return test.ClassName!.Substring(0, test.ClassName.LastIndexOf('.'));
}
public static string GetTestId()
{
return GetEnvVar("TESTID");
}
public static string GetRunId()
{
return GetEnvVar("RUNID");
}
private static string GetEnvVar(string name)
{
var v = Environment.GetEnvironmentVariable(name);
if (string.IsNullOrEmpty(v)) return $"EnvVar'{name}'NotSet";
return v;
}
private static string FormatArguments(TestContext.TestAdapter test)
{
if (test.Arguments == null || !test.Arguments.Any()) return "";
return $"[{string.Join(',', test.Arguments)}]";
}
private static string ReplaceInvalidCharacters(string name)
{
return name.Replace(":", "_");
}
private static string DetermineFolder(LogConfig config, DateTime start)
{
return Path.Join(
config.LogRoot,
$"{start.Year}-{Pad(start.Month)}",
Pad(start.Day));
}
private static string GetFixtureName(string name, DateTime start)
{
var className = GetRawFixtureName();
if (!string.IsNullOrEmpty(name)) className = name;
return $"{Pad(start.Hour)}-{Pad(start.Minute)}-{Pad(start.Second)}Z_{className.Replace('.', '-')}";
}
private static string Pad(int n)
{
return n.ToString().PadLeft(2, '0');
}
}
}

63
Logging/StatusLog.cs Normal file
View File

@ -0,0 +1,63 @@
using Newtonsoft.Json;
namespace Logging
{
public class StatusLog
{
private readonly object fileLock = new object();
private readonly string fullName;
private readonly string fixtureName;
private readonly string codexId;
public StatusLog(LogConfig config, DateTime start, string codexId, string name = "")
{
fullName = NameUtils.GetFixtureFullName(config, start, name) + "_STATUS.log";
fixtureName = NameUtils.GetRawFixtureName();
this.codexId = codexId;
}
public void ConcludeTest(string resultStatus, string testDuration)
{
Write(new StatusLogJson
{
@timestamp = DateTime.UtcNow.ToString("o"),
runid = NameUtils.GetRunId(),
status = resultStatus,
testid = NameUtils.GetTestId(),
codexid = codexId,
category = NameUtils.GetCategoryName(),
fixturename = fixtureName,
testname = NameUtils.GetTestMethodName(),
testduration = testDuration
});
}
private void Write(StatusLogJson json)
{
try
{
lock (fileLock)
{
File.AppendAllLines(fullName, new[] { JsonConvert.SerializeObject(json) });
}
}
catch (Exception ex)
{
Console.WriteLine("Unable to write to status log: " + ex);
}
}
}
public class StatusLogJson
{
public string @timestamp { get; set; } = string.Empty;
public string runid { get; set; } = string.Empty;
public string status { get; set; } = string.Empty;
public string testid { get; set; } = string.Empty;
public string codexid { get; set; } = string.Empty;
public string category { get; set; } = string.Empty;
public string fixturename { get; set; } = string.Empty;
public string testname { get; set; } = string.Empty;
public string testduration { get; set;} = string.Empty;
}
}

21
Logging/TestLog.cs
View File

@ -10,7 +10,7 @@ namespace Logging
public TestLog(string folder, bool debug, string name = "")
: base(debug)
{
methodName = GetMethodName(name);
methodName = NameUtils.GetTestMethodName(name);
fullName = Path.Combine(folder, methodName);
Log($"*** Begin: {methodName}");
@ -37,24 +37,5 @@ namespace Logging
{
return fullName;
}
private string GetMethodName(string name)
{
if (!string.IsNullOrEmpty(name)) return name;
var test = TestContext.CurrentContext.Test;
var args = FormatArguments(test);
return ReplaceInvalidCharacters($"{test.MethodName}{args}");
}
private static string FormatArguments(TestContext.TestAdapter test)
{
if (test.Arguments == null || !test.Arguments.Any()) return "";
return $"[{string.Join(',', test.Arguments)}]";
}
private static string ReplaceInvalidCharacters(string name)
{
return name.Replace(":", "_");
}
}
}

4
docker/job.yaml
View File

@ -28,6 +28,10 @@ spec:
value: ${BRANCH}
- name: SOURCE
value: ${SOURCE}
- name: RUNID
value: ${RUNID}
- name: TESTID
value: ${TESTID}
volumeMounts:
- name: kubeconfig
mountPath: /opt/kubeconfig.yaml