codex-storage
/
cs-codex-dist-tests
mirror of https://github.com/codex-storage/cs-codex-dist-tests.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into feature/docker-image-testruns 

# Conflicts:
#	DistTestCore/Codex/CodexContainerRecipe.cs
This commit is contained in:
benbierens 2023-07-21 09:34:37 +02:00
parent 78434048b7 dc4749aca6
commit dbbd05ea97
No known key found for this signature in database
GPG Key ID: FE44815D96D0A1AA
28 changed files with 632 additions and 374 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
.github/workflows/dist-tests.yaml vendored
View File

@ -15,19 +15,19 @@ on:
workflow_dispatch:
inputs:
branch:
description: Branch
description: Branch (master)
required: false
type: string
source:
description: Repository with tests
description: Repository with tests (current)
required: false
type: string
nameprefix:
description: Runner job/pod name prefix
description: Runner prefix (cs-codex-dist-tests)
required: false
type: string
namespace:
description: Kubernetes namespace for runner
description: Runner namespace (cs-codex-dist-tests)
required: false
type: string
@ -56,6 +56,8 @@ jobs:
[[ -n "${{ inputs.source }}" ]] && echo "SOURCE=${{ inputs.source }}" >>"$GITHUB_ENV" || echo "SOURCE=${{ env.SOURCE }}" >>"$GITHUB_ENV"
[[ -n "${{ inputs.nameprefix }}" ]] && echo "NAMEPREFIX=${{ inputs.nameprefix }}" >>"$GITHUB_ENV" || echo "NAMEPREFIX=${{ env.NAMEPREFIX }}" >>"$GITHUB_ENV"
[[ -n "${{ inputs.namespace }}" ]] && echo "NAMESPACE=${{ inputs.namespace }}" >>"$GITHUB_ENV" || echo "NAMESPACE=${{ env.NAMESPACE }}" >>"$GITHUB_ENV"
echo "RUNID=$(date +%Y%m%d-%H%M%S)" >> $GITHUB_ENV
echo "TESTID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
- name: Kubectl - Install ${{ env.KUBE_VERSION }}
uses: azure/setup-kubectl@v3
@ -69,5 +71,4 @@ jobs:
- name: Kubectl - Create Job
run: |
export RUNID=$(date +%Y%m%d-%H%M%S)
envsubst < ${{ env.JOB_MANIFEST }} | kubectl apply -f -

9
CodexNetDeployer/Program.cs
View File

@ -1,6 +1,9 @@
using ArgsUniform;
using CodexNetDeployer;
using DistTestCore;
using DistTestCore.Codex;
using DistTestCore.Marketplace;
using DistTestCore.Metrics;
using Newtonsoft.Json;
using Configuration = CodexNetDeployer.Configuration;
@ -29,6 +32,12 @@ public class Program
return;
}
Console.WriteLine("Using images:" + nl +
$"\tCodex image: '{CodexContainerRecipe.DockerImage}'" + nl +
$"\tCodex Contracts image: '{CodexContractsContainerRecipe.DockerImage}'" + nl +
$"\tPrometheus image: '{PrometheusContainerRecipe.DockerImage}'" + nl +
$"\tGeth image: '{GethContainerRecipe.DockerImage}'" + nl);
if (!args.Any(a => a == "-y"))
{
Console.WriteLine("Does the above config look good? [y/n]");

5
CodexNetDeployer/deploy-continuous-testnet.sh
View File

@ -3,10 +3,11 @@ dotnet run \
--kube-namespace=codex-continuous-tests \
--nodes=5 \
--validators=3 \
--log-level=Trace \
--storage-quota=2048 \
--storage-sell=1024 \
--min-price=1024 \
--max-collateral=1024 \
--max-duration=3600000 \
--block-ttl=120
--block-ttl=120 \
-y

9
ContinuousTests/Configuration.cs
View File

@ -25,6 +25,9 @@ namespace ContinuousTests
[Uniform("stop", "s", "STOPONFAIL", false, "If true, runner will stop on first test failure and download all cluster container logs. False by default.")]
public bool StopOnFailure { get; set; } = false;
[Uniform("dl-logs", "dl", "DLLOGS", false, "If true, runner will periodically download and save/append container logs to the log path.")]
public bool DownloadContainerLogs { get; set; } = false;
public CodexDeployment CodexDeployment { get; set; } = null!;
public TestRunnerLocation RunnerLocation { get; set; } = TestRunnerLocation.InternalToCluster;
@ -57,9 +60,11 @@ namespace ContinuousTests
private static void PrintHelp()
{
var nl = Environment.NewLine;
Console.WriteLine("CodexNetDownloader lets you download all container logs given a codex-deployment.json file." + nl);
Console.WriteLine("ContinuousTests will run a set of tests against a codex deployment given a codex-deployment.json file." + nl +
"The tests will run in an endless loop unless otherwise specified, using the test-specific timing values." + nl);
Console.WriteLine("CodexNetDownloader assumes you are running this tool from *inside* the Kubernetes cluster. " +
Console.WriteLine("ContinuousTests assumes you are running this tool from *inside* the Kubernetes cluster. " +
"If you are not running this from a container inside the cluster, add the argument '--external'." + nl);
}
}

99
ContinuousTests/ContinuousLogDownloader.cs Normal file
View File

@ -0,0 +1,99 @@
using DistTestCore;
using DistTestCore.Codex;
using KubernetesWorkflow;
namespace ContinuousTests
{
public class ContinuousLogDownloader
{
private readonly TestLifecycle lifecycle;
private readonly CodexDeployment deployment;
private readonly string outputPath;
private readonly CancellationToken cancelToken;
public ContinuousLogDownloader(TestLifecycle lifecycle, CodexDeployment deployment, string outputPath, CancellationToken cancelToken)
{
this.lifecycle = lifecycle;
this.deployment = deployment;
this.outputPath = outputPath;
this.cancelToken = cancelToken;
}
public void Run()
{
while (!cancelToken.IsCancellationRequested)
{
UpdateLogs();
cancelToken.WaitHandle.WaitOne(TimeSpan.FromSeconds(15));
}
// After testing has stopped, we wait a little bit and fetch the logs one more time.
// If our latest fetch was not recent, interesting test-related log activity might
// not have been captured yet.
Thread.Sleep(TimeSpan.FromSeconds(10));
UpdateLogs();
}
private void UpdateLogs()
{
foreach (var container in deployment.CodexContainers)
{
UpdateLog(container);
}
}
private void UpdateLog(RunningContainer container)
{
var filepath = Path.Combine(outputPath, GetLogName(container));
if (!File.Exists(filepath))
{
File.WriteAllLines(filepath, new[] { container.Name });
}
var appender = new LogAppender(filepath);
lifecycle.CodexStarter.DownloadLog(container, appender);
}
private static string GetLogName(RunningContainer container)
{
return container.Name
.Replace("<", "")
.Replace(">", "")
+ ".log";
}
}
public class LogAppender : ILogHandler
{
private readonly string filename;
public LogAppender(string filename)
{
this.filename = filename;
}
public void Log(Stream log)
{
using var reader = new StreamReader(log);
var lines = File.ReadAllLines(filename);
var lastLine = lines.Last();
var recording = lines.Length < 3;
var line = reader.ReadLine();
while (line != null)
{
if (recording)
{
File.AppendAllLines(filename, new[] { line });
}
else
{
recording = line == lastLine;
}
line = reader.ReadLine();
}
}
}
}

17
ContinuousTests/ContinuousTestRunner.cs
View File

@ -24,12 +24,14 @@ namespace ContinuousTests
startupChecker.Check();
var taskFactory = new TaskFactory();
var overviewLog = new FixtureLog(new LogConfig(config.LogPath, false), "Overview");
var overviewLog = new FixtureLog(new LogConfig(config.LogPath, false), DateTime.UtcNow, "Overview");
overviewLog.Log("Continuous tests starting...");
var allTests = testFactory.CreateTests();
ClearAllCustomNamespaces(allTests, overviewLog);
StartLogDownloader(taskFactory);
var testLoops = allTests.Select(t => new TestLoop(taskFactory, config, overviewLog, t.GetType(), t.RunTestEvery, cancelToken)).ToArray();
foreach (var testLoop in testLoops)
@ -61,5 +63,18 @@ namespace ContinuousTests
var (workflowCreator, _) = k8SFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, config.DataPath, test.CustomK8sNamespace, new DefaultTimeSet(), log, config.RunnerLocation);
workflowCreator.CreateWorkflow().DeleteTestResources();
}
private void StartLogDownloader(TaskFactory taskFactory)
{
if (!config.DownloadContainerLogs) return;
var path = Path.Combine(config.LogPath, "containers");
if (!Directory.Exists(path)) Directory.CreateDirectory(path);
var (_, lifecycle) = k8SFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, config.DataPath, config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation);
var downloader = new ContinuousLogDownloader(lifecycle, config.CodexDeployment, path, cancelToken);
taskFactory.Run(downloader.Run);
}
}
}

2
ContinuousTests/SingleTestRun.cs
View File

@ -32,7 +32,7 @@ namespace ContinuousTests
this.handle = handle;
this.cancelToken = cancelToken;
testName = handle.Test.GetType().Name;
fixtureLog = new FixtureLog(new LogConfig(config.LogPath, true), testName);
fixtureLog = new FixtureLog(new LogConfig(config.LogPath, true), DateTime.UtcNow, testName);
nodes = CreateRandomNodes(handle.Test.RequiredNumberOfNodes);
dataFolder = config.DataPath + "-" + Guid.NewGuid();

2
ContinuousTests/StartupChecker.cs
View File

@ -19,7 +19,7 @@ namespace ContinuousTests
public void Check()
{
var log = new FixtureLog(new LogConfig(config.LogPath, false), "StartupChecks");
var log = new FixtureLog(new LogConfig(config.LogPath, false), DateTime.UtcNow, "StartupChecks");
log.Log("Starting continuous test run...");
log.Log("Checking configuration...");
PreflightCheck(config);

4
ContinuousTests/run-continuous-tests.sh → ContinuousTests/run.sh
View File

@ -1,4 +1,6 @@
dotnet run \
--kube-config=/opt/kubeconfig.yaml \
--codex-deployment=codex-deployment.json \
--stop=1
--keep=1 \
--stop=1 \
--dl-logs=1

16
DistTestCore/Codex/CodexAccess.cs
View File

@ -22,17 +22,12 @@ namespace DistTestCore.Codex
public CodexDebugResponse GetDebugInfo()
{
return Http(TimeSpan.FromSeconds(60)).HttpGetJson<CodexDebugResponse>("debug/info");
return Http().HttpGetJson<CodexDebugResponse>("debug/info");
}
public CodexDebugPeerResponse GetDebugPeer(string peerId)
{
return GetDebugPeer(peerId, TimeSpan.FromSeconds(10));
}
public CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout)
{
var http = Http(timeout);
var http = Http();
var str = http.HttpGetString($"debug/peer/{peerId}");
if (str.ToLowerInvariant() == "unable to find peer!")
@ -50,7 +45,8 @@ namespace DistTestCore.Codex
public int GetDebugFutures()
{
return Http().HttpGetJson<CodexDebugFutures>("debug/futures").futures;
// Some Codex images support debug/futures to count the number of open futures.
return 0; // Http().HttpGetJson<CodexDebugFutures>("debug/futures").futures;
}
public CodexDebugThresholdBreaches GetDebugThresholdBreaches()
@ -88,9 +84,9 @@ namespace DistTestCore.Codex
return Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}");
}
private Http Http(TimeSpan? timeoutOverride = null)
private Http Http()
{
return new Http(log, timeSet, Address, baseUrl: "/api/codex/v1", Container.Name, timeoutOverride);
return new Http(log, timeSet, Address, baseUrl: "/api/codex/v1", Container.Name);
}
}
}

4
DistTestCore/Codex/CodexContainerRecipe.cs
View File

@ -5,6 +5,8 @@ namespace DistTestCore.Codex
{
public class CodexContainerRecipe : ContainerRecipeFactory
{
private const string DefaultDockerImage = "codexstorage/nim-codex:sha-14c5270";
public const string MetricsPortTag = "metrics_port";
public const string DiscoveryPortTag = "discovery-port";
@ -87,7 +89,7 @@ namespace DistTestCore.Codex
{
var image = Environment.GetEnvironmentVariable("CODEXDOCKERIMAGE");
if (!string.IsNullOrEmpty(image)) return image;
return "codexstorage/nim-codex:sha-0265cad";
return DefaultDockerImage;
}
}
}

12
DistTestCore/DistTest.cs
View File

@ -16,6 +16,7 @@ namespace DistTestCore
private readonly Configuration configuration = new Configuration();
private readonly Assembly[] testAssemblies;
private readonly FixtureLog fixtureLog;
private readonly StatusLog statusLog;
private readonly object lifecycleLock = new object();
private readonly Dictionary<string, TestLifecycle> lifecycles = new Dictionary<string, TestLifecycle>();
@ -24,7 +25,10 @@ namespace DistTestCore
var assemblies = AppDomain.CurrentDomain.GetAssemblies();
testAssemblies = assemblies.Where(a => a.FullName!.ToLowerInvariant().Contains("test")).ToArray();
fixtureLog = new FixtureLog(configuration.GetLogConfig());
var logConfig = configuration.GetLogConfig();
var startTime = DateTime.UtcNow;
fixtureLog = new FixtureLog(logConfig, startTime);
statusLog = new StatusLog(logConfig, startTime, CodexContainerRecipe.DefaultDockerImage);
PeerConnectionTestHelpers = new PeerConnectionTestHelpers(this);
PeerDownloadTestHelpers = new PeerDownloadTestHelpers(this);
@ -186,6 +190,7 @@ namespace DistTestCore
private void CreateNewTestLifecycle()
{
var testName = GetCurrentTestName();
fixtureLog.WriteLogTag();
Stopwatch.Measure(fixtureLog, $"Setup for {testName}", () =>
{
lock (lifecycleLock)
@ -198,7 +203,10 @@ namespace DistTestCore
private void DisposeTestLifecycle()
{
var lifecycle = Get();
fixtureLog.Log($"{GetCurrentTestName()} = {GetTestResult()} ({lifecycle.GetTestDuration()})");
var testResult = GetTestResult();
var testDuration = lifecycle.GetTestDuration();
fixtureLog.Log($"{GetCurrentTestName()} = {testResult} ({testDuration})");
statusLog.ConcludeTest(testResult, testDuration);
Stopwatch.Measure(fixtureLog, $"Teardown for {GetCurrentTestName()}", () =>
{
lifecycle.Log.EndTest();

211
DistTestCore/Helpers/FullConnectivityHelper.cs Normal file
View File

@ -0,0 +1,211 @@
using DistTestCore.Codex;
using NUnit.Framework;
using Utils;
namespace DistTestCore.Helpers
{
public interface IFullConnectivityImplementation
{
string Description();
string ValidateEntry(FullConnectivityHelper.Entry entry, FullConnectivityHelper.Entry[] allEntries);
FullConnectivityHelper.PeerConnectionState Check(FullConnectivityHelper.Entry from, FullConnectivityHelper.Entry to);
}
public class FullConnectivityHelper
{
private static string Nl = Environment.NewLine;
private readonly DistTest test;
private readonly IFullConnectivityImplementation implementation;
public FullConnectivityHelper(DistTest test, IFullConnectivityImplementation implementation)
{
this.test = test;
this.implementation = implementation;
}
public void AssertFullyConnected(IEnumerable<IOnlineCodexNode> nodes)
{
AssertFullyConnected(nodes.ToArray());
}
private void AssertFullyConnected(IOnlineCodexNode[] nodes)
{
test.Log($"Asserting '{implementation.Description()}' for nodes: '{string.Join(",", nodes.Select(n => n.GetName()))}'...");
var entries = CreateEntries(nodes);
var pairs = CreatePairs(entries);
RetryWhilePairs(pairs, () =>
{
CheckAndRemoveSuccessful(pairs);
});
if (pairs.Any())
{
var pairDetails = string.Join(Nl, pairs.SelectMany(p => p.GetResultMessages()));
test.Log($"Connections failed:{Nl}{pairDetails}");
Assert.Fail(string.Join(Nl, pairs.SelectMany(p => p.GetResultMessages())));
}
else
{
test.Log($"'{implementation.Description()}' = Success! for nodes: {string.Join(",", nodes.Select(n => n.GetName()))}");
}
}
private static void RetryWhilePairs(List<Pair> pairs, Action action)
{
var timeout = DateTime.UtcNow + TimeSpan.FromMinutes(5);
while (pairs.Any(p => p.Inconclusive) && timeout > DateTime.UtcNow)
{
action();
Time.Sleep(TimeSpan.FromSeconds(2));
}
}
private void CheckAndRemoveSuccessful(List<Pair> pairs)
{
// For large sets, don't try and do all of them at once.
var selectedPair = pairs.Take(20).ToArray();
var pairDetails = new List<string>();
foreach (var pair in selectedPair)
{
test.ScopedTestFiles(pair.Check);
if (pair.Success)
{
pairDetails.AddRange(pair.GetResultMessages());
pairs.Remove(pair);
}
}
test.Log($"Connections successful:{Nl}{string.Join(Nl, pairDetails)}");
}
private Entry[] CreateEntries(IOnlineCodexNode[] nodes)
{
var entries = nodes.Select(n => new Entry(n)).ToArray();
var errors = entries
.Select(e => implementation.ValidateEntry(e, entries))
.Where(s => !string.IsNullOrEmpty(s))
.ToArray();
if (errors.Any())
{
Assert.Fail("Some node entries failed to validate: " + string.Join(Nl, errors));
}
return entries;
}
private List<Pair> CreatePairs(Entry[] entries)
{
return CreatePairsIterator(entries).ToList();
}
private IEnumerable<Pair> CreatePairsIterator(Entry[] entries)
{
for (var x = 0; x < entries.Length; x++)
{
for (var y = x + 1; y < entries.Length; y++)
{
yield return new Pair(implementation, entries[x], entries[y]);
}
}
}
public class Entry
{
public Entry(IOnlineCodexNode node)
{
Node = node;
Response = node.GetDebugInfo();
}
public IOnlineCodexNode Node { get; }
public CodexDebugResponse Response { get; }
public override string ToString()
{
if (Response == null || string.IsNullOrEmpty(Response.id)) return "UNKNOWN";
return Response.id;
}
}
public enum PeerConnectionState
{
Unknown,
Connection,
NoConnection,
}
public class Pair
{
private TimeSpan aToBTime = TimeSpan.FromSeconds(0);
private TimeSpan bToATime = TimeSpan.FromSeconds(0);
private readonly IFullConnectivityImplementation implementation;
public Pair(IFullConnectivityImplementation implementation, Entry a, Entry b)
{
this.implementation = implementation;
A = a;
B = b;
}
public Entry A { get; }
public Entry B { get; }
public PeerConnectionState AKnowsB { get; private set; }
public PeerConnectionState BKnowsA { get; private set; }
public bool Success { get { return AKnowsB == PeerConnectionState.Connection && BKnowsA == PeerConnectionState.Connection; } }
public bool Inconclusive { get { return AKnowsB == PeerConnectionState.Unknown || BKnowsA == PeerConnectionState.Unknown; } }
public void Check()
{
aToBTime = Measure(() => AKnowsB = Check(A, B));
bToATime = Measure(() => BKnowsA = Check(B, A));
}
public override string ToString()
{
return $"[{string.Join(",", GetResultMessages())}]";
}
public string[] GetResultMessages()
{
var aName = A.ToString();
var bName = B.ToString();
return new[]
{
$"[{aName} --> {bName}] = {AKnowsB} ({aToBTime.TotalSeconds} seconds)",
$"[{aName} <-- {bName}] = {BKnowsA} ({bToATime.TotalSeconds} seconds)"
};
}
private static TimeSpan Measure(Action action)
{
var start = DateTime.UtcNow;
action();
return DateTime.UtcNow - start;
}
private PeerConnectionState Check(Entry from, Entry to)
{
Thread.Sleep(10);
try
{
return implementation.Check(from, to);
}
catch
{
// Didn't get a conclusive answer. Try again later.
return PeerConnectionState.Unknown;
}
}
}
}
}

242
DistTestCore/Helpers/PeerConnectionTestHelpers.cs
View File

@ -1,149 +1,55 @@
using DistTestCore.Codex;
using NUnit.Framework;
using Utils;
using static DistTestCore.Helpers.FullConnectivityHelper;
namespace DistTestCore.Helpers
{
public class PeerConnectionTestHelpers
public class PeerConnectionTestHelpers : IFullConnectivityImplementation
{
private readonly Random random = new Random();
private readonly DistTest test;
private readonly FullConnectivityHelper helper;
public PeerConnectionTestHelpers(DistTest test)
{
this.test = test;
helper = new FullConnectivityHelper(test, this);
}
public void AssertFullyConnected(IEnumerable<IOnlineCodexNode> nodes)
{
var n = nodes.ToArray();
AssertFullyConnected(n);
for (int i = 0; i < 5; i++)
{
Time.Sleep(TimeSpan.FromSeconds(30));
AssertFullyConnected(n);
}
helper.AssertFullyConnected(nodes);
}
private void AssertFullyConnected(IOnlineCodexNode[] nodes)
public string Description()
{
test.Log($"Asserting peers are fully-connected for nodes: '{string.Join(",", nodes.Select(n => n.GetName()))}'...");
var entries = CreateEntries(nodes);
var pairs = CreatePairs(entries);
RetryWhilePairs(pairs, () =>
{
CheckAndRemoveSuccessful(pairs);
});
if (pairs.Any())
{
test.Log($"Unsuccessful! Peers are not fully-connected: {string.Join(",", nodes.Select(n => n.GetName()))}");
Assert.Fail(string.Join(Environment.NewLine, pairs.Select(p => p.GetMessage())));
test.Log(string.Join(Environment.NewLine, pairs.Select(p => p.GetMessage())));
}
else
{
test.Log($"Success! Peers are fully-connected: {string.Join(",", nodes.Select(n => n.GetName()))}");
}
return "Peer Discovery";
}
private static void RetryWhilePairs(List<Pair> pairs, Action action)
public string ValidateEntry(Entry entry, Entry[] allEntries)
{
var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(30);
while (pairs.Any() && timeout > DateTime.UtcNow)
{
action();
if (pairs.Any()) Time.Sleep(TimeSpan.FromSeconds(2));
}
}
private void CheckAndRemoveSuccessful(List<Pair> pairs)
{
var checkTasks = pairs.Select(p => Task.Run(() =>
{
ApplyRandomDelay();
p.Check();
})).ToArray();
Task.WaitAll(checkTasks);
foreach (var pair in pairs.ToArray())
{
if (pair.Success)
{
test.Log(pair.GetMessage());
pairs.Remove(pair);
}
}
}
private static Entry[] CreateEntries(IOnlineCodexNode[] nodes)
{
var entries = nodes.Select(n => new Entry(n)).ToArray();
var incorrectDiscoveryEndpoints = entries.SelectMany(e => e.GetInCorrectDiscoveryEndpoints(entries)).ToArray();
if (incorrectDiscoveryEndpoints.Any())
{
Assert.Fail("Some nodes contain peer records with incorrect discovery ip/port information: " +
string.Join(Environment.NewLine, incorrectDiscoveryEndpoints));
}
return entries;
}
private static List<Pair> CreatePairs(Entry[] entries)
{
return CreatePairsIterator(entries).ToList();
}
private static IEnumerable<Pair> CreatePairsIterator(Entry[] entries)
{
for (var x = 0; x < entries.Length; x++)
{
for (var y = x + 1; y < entries.Length; y++)
{
yield return new Pair(entries[x], entries[y]);
}
}
}
private void ApplyRandomDelay()
{
// Calling all the nodes all at the same time is not exactly nice.
Time.Sleep(TimeSpan.FromMicroseconds(random.Next(10, 1000)));
}
public class Entry
{
public Entry(IOnlineCodexNode node)
{
Node = node;
Response = node.GetDebugInfo();
}
public IOnlineCodexNode Node { get; }
public CodexDebugResponse Response { get; }
public IEnumerable<string> GetInCorrectDiscoveryEndpoints(Entry[] allEntries)
{
foreach (var peer in Response.table.nodes)
var result = string.Empty;
foreach (var peer in entry.Response.table.nodes)
{
var expected = GetExpectedDiscoveryEndpoint(allEntries, peer);
if (expected != peer.address)
{
yield return $"Node:{Node.GetName()} has incorrect peer table entry. Was: '{peer.address}', expected: '{expected}'";
result += $"Node:{entry.Node.GetName()} has incorrect peer table entry. Was: '{peer.address}', expected: '{expected}'. ";
}
}
return result;
}
public override string ToString()
public PeerConnectionState Check(Entry from, Entry to)
{
if (Response == null || string.IsNullOrEmpty(Response.id)) return "UNKNOWN";
return Response.id;
var peerId = to.Response.id;
var response = from.Node.GetDebugPeer(peerId);
if (!response.IsPeerFound)
{
return PeerConnectionState.NoConnection;
}
if (!string.IsNullOrEmpty(response.peerId) && response.addresses.Any())
{
return PeerConnectionState.Connection;
}
return PeerConnectionState.Unknown;
}
private static string GetExpectedDiscoveryEndpoint(Entry[] allEntries, CodexDebugTableNodeResponse node)
@ -157,102 +63,4 @@ namespace DistTestCore.Helpers
return $"{ip}:{discPort.Number}";
}
}
public enum PeerConnectionState
{
Unknown,
Connection,
NoConnection,
}
public class Pair
{
private readonly TimeSpan timeout = TimeSpan.FromSeconds(60);
private TimeSpan aToBTime = TimeSpan.FromSeconds(0);
private TimeSpan bToATime = TimeSpan.FromSeconds(0);
public Pair(Entry a, Entry b)
{
A = a;
B = b;
}
public Entry A { get; }
public Entry B { get; }
public PeerConnectionState AKnowsB { get; private set; }
public PeerConnectionState BKnowsA { get; private set; }
public bool Success { get { return AKnowsB == PeerConnectionState.Connection && BKnowsA == PeerConnectionState.Connection; } }
public void Check()
{
aToBTime = Measure(() => AKnowsB = Knows(A, B));
bToATime = Measure(() => BKnowsA = Knows(B, A));
}
public string GetMessage()
{
return GetResultMessage() + GetTimePostfix();
}
public override string ToString()
{
return $"[{GetMessage()}]";
}
private string GetResultMessage()
{
var aName = A.ToString();
var bName = B.ToString();
if (Success)
{
return $"{aName} and {bName} know each other.";
}
return $"[{aName}-->{bName}] = {AKnowsB} AND [{aName}<--{bName}] = {BKnowsA}";
}
private string GetTimePostfix()
{
var aName = A.ToString();
var bName = B.ToString();
return $" ({aName}->{bName}: {aToBTime.TotalMinutes} seconds, {bName}->{aName}: {bToATime.TotalSeconds} seconds)";
}
private static TimeSpan Measure(Action action)
{
var start = DateTime.UtcNow;
action();
return DateTime.UtcNow - start;
}
private PeerConnectionState Knows(Entry a, Entry b)
{
lock (a)
{
var peerId = b.Response.id;
try
{
var response = a.Node.GetDebugPeer(peerId, timeout);
if (!response.IsPeerFound)
{
return PeerConnectionState.NoConnection;
}
if (!string.IsNullOrEmpty(response.peerId) && response.addresses.Any())
{
return PeerConnectionState.Connection;
}
}
catch
{
}
// Didn't get a conclusive answer. Try again later.
return PeerConnectionState.Unknown;
}
}
}
}
}

69
DistTestCore/Helpers/PeerDownloadTestHelpers.cs
View File

@ -1,74 +1,63 @@
using DistTestCore.Codex;
using NUnit.Framework;
using static DistTestCore.Helpers.FullConnectivityHelper;
namespace DistTestCore.Helpers
{
public class PeerDownloadTestHelpers
public class PeerDownloadTestHelpers : IFullConnectivityImplementation
{
private readonly FullConnectivityHelper helper;
private readonly DistTest test;
private ByteSize testFileSize;
public PeerDownloadTestHelpers(DistTest test)
{
helper = new FullConnectivityHelper(test, this);
testFileSize = 1.MB();
this.test = test;
}
public void AssertFullDownloadInterconnectivity(IEnumerable<IOnlineCodexNode> nodes, ByteSize testFileSize)
{
test.Log($"Asserting full download interconnectivity for nodes: '{string.Join(",", nodes.Select(n => n.GetName()))}'...");
var start = DateTime.UtcNow;
foreach (var node in nodes)
{
var uploader = node;
var downloaders = nodes.Where(n => n != uploader).ToArray();
test.ScopedTestFiles(() =>
{
PerformTest(uploader, downloaders, testFileSize);
});
this.testFileSize = testFileSize;
helper.AssertFullyConnected(nodes);
}
test.Log($"Success! Full download interconnectivity for nodes: {string.Join(",", nodes.Select(n => n.GetName()))}");
var timeTaken = DateTime.UtcNow - start;
AssertTimePerMB(timeTaken, nodes.Count(), testFileSize);
public string Description()
{
return "Download Connectivity";
}
private void AssertTimePerMB(TimeSpan timeTaken, int numberOfNodes, ByteSize size)
public string ValidateEntry(Entry entry, Entry[] allEntries)
{
var numberOfDownloads = numberOfNodes * (numberOfNodes - 1);
var timePerDownload = timeTaken / numberOfDownloads;
float sizeInMB = size.ToMB();
var timePerMB = timePerDownload / sizeInMB;
test.Log($"Performed {numberOfDownloads} downloads of {size} in {timeTaken.TotalSeconds} seconds, for an average of {timePerMB.TotalSeconds} seconds per MB.");
Assert.That(timePerMB, Is.LessThan(CodexContainerRecipe.MaxDownloadTimePerMegabyte), "MaxDownloadTimePerMegabyte performance threshold breached.");
return string.Empty;
}
private void PerformTest(IOnlineCodexNode uploader, IOnlineCodexNode[] downloaders, ByteSize testFileSize)
public PeerConnectionState Check(Entry from, Entry to)
{
// Generate 1 test file per downloader.
var files = downloaders.Select(d => GenerateTestFile(uploader, d, testFileSize)).ToArray();
var expectedFile = GenerateTestFile(from.Node, to.Node);
// Upload all the test files to the uploader.
var contentIds = files.Select(uploader.UploadFile).ToArray();
var contentId = from.Node.UploadFile(expectedFile);
// Each downloader should retrieve its own test file.
for (var i = 0; i < downloaders.Length; i++)
try
{
var expectedFile = files[i];
var downloadedFile = downloaders[i].DownloadContent(contentIds[i], $"{expectedFile.Label}DOWNLOADED");
var downloadedFile = to.Node.DownloadContent(contentId, expectedFile.Label + "_downloaded");
expectedFile.AssertIsEqual(downloadedFile);
return PeerConnectionState.Connection;
}
catch
{
// Should an exception occur during the download or file-content assertion,
// We consider that as no-connection for the purpose of this test.
return PeerConnectionState.NoConnection;
}
private TestFile GenerateTestFile(IOnlineCodexNode uploader, IOnlineCodexNode downloader, ByteSize testFileSize)
// Should an exception occur during upload, then this try is inconclusive and we try again next loop.
}
private TestFile GenerateTestFile(IOnlineCodexNode uploader, IOnlineCodexNode downloader)
{
var up = uploader.GetName().Replace("<", "").Replace(">", "");
var down = downloader.GetName().Replace("<", "").Replace(">", "");
var label = $"FROM{up}TO{down}";
var label = $"~from:{up}-to:{down}~";
return test.GenerateTestFile(testFileSize, label);
}
}

19
DistTestCore/Http.cs
View File

@ -13,16 +13,14 @@ namespace DistTestCore
private readonly Address address;
private readonly string baseUrl;
private readonly string? logAlias;
private readonly TimeSpan? timeoutOverride;
public Http(BaseLog log, ITimeSet timeSet, Address address, string baseUrl, string? logAlias = null, TimeSpan? timeoutOverride = null)
public Http(BaseLog log, ITimeSet timeSet, Address address, string baseUrl, string? logAlias = null)
{
this.log = log;
this.timeSet = timeSet;
this.address = address;
this.baseUrl = baseUrl;
this.logAlias = logAlias;
this.timeoutOverride = timeoutOverride;
if (!this.baseUrl.StartsWith("/")) this.baseUrl = "/" + this.baseUrl;
if (!this.baseUrl.EndsWith("/")) this.baseUrl += "/";
}
@ -127,24 +125,13 @@ namespace DistTestCore
private T Retry<T>(Func<T> operation, string description)
{
return Time.Retry(operation, GetTimeout(), timeSet.HttpCallRetryDelay(), description);
return Time.Retry(operation, timeSet.HttpCallRetryTime(), timeSet.HttpCallRetryDelay(), description);
}
private HttpClient GetClient()
{
return GetClient(GetTimeout());
}
private TimeSpan GetTimeout()
{
if (timeoutOverride.HasValue) return timeoutOverride.Value;
return timeSet.HttpCallTimeout();
}
private HttpClient GetClient(TimeSpan timeout)
{
var client = new HttpClient();
client.Timeout = timeout;
client.Timeout = timeSet.HttpCallTimeout();
return client;
}
}

9
DistTestCore/OnlineCodexNode.cs
View File

@ -12,7 +12,6 @@ namespace DistTestCore
string GetName();
CodexDebugResponse GetDebugInfo();
CodexDebugPeerResponse GetDebugPeer(string peerId);
CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout);
ContentId UploadFile(TestFile file);
TestFile? DownloadContent(ContentId contentId, string fileLabel = "");
void ConnectToPeer(IOnlineCodexNode node);
@ -60,11 +59,6 @@ namespace DistTestCore
return CodexAccess.GetDebugPeer(peerId);
}
public CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout)
{
return CodexAccess.GetDebugPeer(peerId, timeout);
}
public ContentId UploadFile(TestFile file)
{
using var fileStream = File.OpenRead(file.Filename);
@ -78,9 +72,6 @@ namespace DistTestCore
if (string.IsNullOrEmpty(response)) Assert.Fail("Received empty response.");
if (response.StartsWith(UploadFailedMessage)) Assert.Fail("Node failed to store block.");
var logReplacement = $"(CID:{file.Describe()})";
Log($"ContentId '{response}' is {logReplacement}");
lifecycle.Log.AddStringReplace(response, logReplacement);
Log($"Uploaded file. Received contentId: '{response}'.");
return new ContentId(response);
}

2
DistTestCore/TestLifecycle.cs
View File

@ -25,6 +25,8 @@ namespace DistTestCore
PrometheusStarter = new PrometheusStarter(this, workflowCreator);
GethStarter = new GethStarter(this, workflowCreator);
testStart = DateTime.UtcNow;
Log.WriteLogTag();
}
public BaseLog Log { get; }

6
DistTestCore/Timing.cs
View File

@ -10,7 +10,7 @@ namespace DistTestCore
public interface ITimeSet
{
TimeSpan HttpCallTimeout();
TimeSpan HttpCallRetryTimeout();
TimeSpan HttpCallRetryTime();
TimeSpan HttpCallRetryDelay();
TimeSpan WaitForK8sServiceDelay();
TimeSpan K8sOperationTimeout();
@ -24,7 +24,7 @@ namespace DistTestCore
return TimeSpan.FromSeconds(10);
}
public TimeSpan HttpCallRetryTimeout()
public TimeSpan HttpCallRetryTime()
{
return TimeSpan.FromMinutes(1);
}
@ -57,7 +57,7 @@ namespace DistTestCore
return TimeSpan.FromHours(2);
}
public TimeSpan HttpCallRetryTimeout()
public TimeSpan HttpCallRetryTime()
{
return TimeSpan.FromHours(5);
}

8
Logging/BaseLog.cs
View File

@ -73,6 +73,14 @@ namespace Logging
return new LogFile($"{GetFullName()}_{GetSubfileNumber()}", ext);
}
public void WriteLogTag()
{
var runId = NameUtils.GetRunId();
var category = NameUtils.GetCategoryName();
var name = NameUtils.GetTestMethodName();
LogFile.WriteRaw($"{runId} {category} {name}");
}
private string ApplyReplacements(string str)
{
foreach (var replacement in replacements)

35
Logging/FixtureLog.cs
View File

@ -1,20 +1,14 @@
using NUnit.Framework;
namespace Logging
namespace Logging
{
public class FixtureLog : BaseLog
{
private readonly DateTime start;
private readonly string fullName;
private readonly LogConfig config;
public FixtureLog(LogConfig config, string name = "")
public FixtureLog(LogConfig config, DateTime start, string name = "")
: base(config.DebugEnabled)
{
start = DateTime.UtcNow;
var folder = DetermineFolder(config);
var fixtureName = GetFixtureName(name);
fullName = Path.Combine(folder, fixtureName);
fullName = NameUtils.GetFixtureFullName(config, start, name);
this.config = config;
}
@ -32,28 +26,5 @@ namespace Logging
{
return fullName;
}
private string DetermineFolder(LogConfig config)
{
return Path.Join(
config.LogRoot,
$"{start.Year}-{Pad(start.Month)}",
Pad(start.Day));
}
private string GetFixtureName(string name)
{
var test = TestContext.CurrentContext.Test;
var className = test.ClassName!.Substring(test.ClassName.LastIndexOf('.') + 1);
if (!string.IsNullOrEmpty(name)) className = name;
return $"{Pad(start.Hour)}-{Pad(start.Minute)}-{Pad(start.Second)}Z_{className.Replace('.', '-')}";
}
private static string Pad(int n)
{
return n.ToString().PadLeft(2, '0');
}
}
}

2
Logging/LogFile.cs
View File

@ -49,7 +49,7 @@
private static string GetTimestamp()
{
return $"[{DateTime.UtcNow.ToString("u")}]";
return $"[{DateTime.UtcNow.ToString("o")}]";
}
private void EnsurePathExists(string filename)

83
Logging/NameUtils.cs Normal file
View File

@ -0,0 +1,83 @@
using NUnit.Framework;
namespace Logging
{
public static class NameUtils
{
public static string GetTestMethodName(string name = "")
{
if (!string.IsNullOrEmpty(name)) return name;
var test = TestContext.CurrentContext.Test;
var args = FormatArguments(test);
return ReplaceInvalidCharacters($"{test.MethodName}{args}");
}
public static string GetFixtureFullName(LogConfig config, DateTime start, string name)
{
var folder = DetermineFolder(config, start);
var fixtureName = GetFixtureName(name, start);
return Path.Combine(folder, fixtureName);
}
public static string GetRawFixtureName()
{
var test = TestContext.CurrentContext.Test;
var className = test.ClassName!.Substring(test.ClassName.LastIndexOf('.') + 1);
return className.Replace('.', '-');
}
public static string GetCategoryName()
{
var test = TestContext.CurrentContext.Test;
return test.ClassName!.Substring(0, test.ClassName.LastIndexOf('.'));
}
public static string GetTestId()
{
return GetEnvVar("TESTID");
}
public static string GetRunId()
{
return GetEnvVar("RUNID");
}
private static string GetEnvVar(string name)
{
var v = Environment.GetEnvironmentVariable(name);
if (string.IsNullOrEmpty(v)) return $"EnvVar'{name}'NotSet";
return v;
}
private static string FormatArguments(TestContext.TestAdapter test)
{
if (test.Arguments == null || !test.Arguments.Any()) return "";
return $"[{string.Join(',', test.Arguments)}]";
}
private static string ReplaceInvalidCharacters(string name)
{
return name.Replace(":", "_");
}
private static string DetermineFolder(LogConfig config, DateTime start)
{
return Path.Join(
config.LogRoot,
$"{start.Year}-{Pad(start.Month)}",
Pad(start.Day));
}
private static string GetFixtureName(string name, DateTime start)
{
var className = GetRawFixtureName();
if (!string.IsNullOrEmpty(name)) className = name;
return $"{Pad(start.Hour)}-{Pad(start.Minute)}-{Pad(start.Second)}Z_{className.Replace('.', '-')}";
}
private static string Pad(int n)
{
return n.ToString().PadLeft(2, '0');
}
}
}

63
Logging/StatusLog.cs Normal file
View File

@ -0,0 +1,63 @@
using Newtonsoft.Json;
namespace Logging
{
public class StatusLog
{
private readonly object fileLock = new object();
private readonly string fullName;
private readonly string fixtureName;
private readonly string codexId;
public StatusLog(LogConfig config, DateTime start, string codexId, string name = "")
{
fullName = NameUtils.GetFixtureFullName(config, start, name) + "_STATUS.log";
fixtureName = NameUtils.GetRawFixtureName();
this.codexId = codexId;
}
public void ConcludeTest(string resultStatus, string testDuration)
{
Write(new StatusLogJson
{
@timestamp = DateTime.UtcNow.ToString("o"),
runid = NameUtils.GetRunId(),
status = resultStatus,
testid = NameUtils.GetTestId(),
codexid = codexId,
category = NameUtils.GetCategoryName(),
fixturename = fixtureName,
testname = NameUtils.GetTestMethodName(),
testduration = testDuration
});
}
private void Write(StatusLogJson json)
{
try
{
lock (fileLock)
{
File.AppendAllLines(fullName, new[] { JsonConvert.SerializeObject(json) });
}
}
catch (Exception ex)
{
Console.WriteLine("Unable to write to status log: " + ex);
}
}
}
public class StatusLogJson
{
public string @timestamp { get; set; } = string.Empty;
public string runid { get; set; } = string.Empty;
public string status { get; set; } = string.Empty;
public string testid { get; set; } = string.Empty;
public string codexid { get; set; } = string.Empty;
public string category { get; set; } = string.Empty;
public string fixturename { get; set; } = string.Empty;
public string testname { get; set; } = string.Empty;
public string testduration { get; set;} = string.Empty;
}
}

21
Logging/TestLog.cs
View File

@ -10,7 +10,7 @@ namespace Logging
public TestLog(string folder, bool debug, string name = "")
: base(debug)
{
methodName = GetMethodName(name);
methodName = NameUtils.GetTestMethodName(name);
fullName = Path.Combine(folder, methodName);
Log($"*** Begin: {methodName}");
@ -37,24 +37,5 @@ namespace Logging
{
return fullName;
}
private string GetMethodName(string name)
{
if (!string.IsNullOrEmpty(name)) return name;
var test = TestContext.CurrentContext.Test;
var args = FormatArguments(test);
return ReplaceInvalidCharacters($"{test.MethodName}{args}");
}
private static string FormatArguments(TestContext.TestAdapter test)
{
if (test.Arguments == null || !test.Arguments.Any()) return "";
return $"[{string.Join(',', test.Arguments)}]";
}
private static string ReplaceInvalidCharacters(string name)
{
return name.Replace(":", "_");
}
}
}

22
LongTests/DownloadConnectivityTests/LongFullyConnectedDownloadTests.cs Normal file
View File

@ -0,0 +1,22 @@
using DistTestCore.Helpers;
using DistTestCore;
using NUnit.Framework;
namespace TestsLong.DownloadConnectivityTests
{
[TestFixture]
public class LongFullyConnectedDownloadTests : AutoBootstrapDistTest
{
[Test]
[UseLongTimeouts]
[Combinatorial]
public void FullyConnectedDownloadTest(
[Values(10, 15, 20)] int numberOfNodes,
[Values(10, 100)] int sizeMBs)
{
for (var i = 0; i < numberOfNodes; i++) SetupCodexNode();
PeerDownloadTestHelpers.AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes(), sizeMBs.MB());
}
}
}

4
Tests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs
View File

@ -9,8 +9,8 @@ namespace Tests.DownloadConnectivityTests
[Test]
[Combinatorial]
public void FullyConnectedDownloadTest(
[Values(3, 10, 20)] int numberOfNodes,
[Values(1, 10, 100)] int sizeMBs)
[Values(1, 3, 5)] int numberOfNodes,
[Values(1, 10)] int sizeMBs)
{
for (var i = 0; i < numberOfNodes; i++) SetupCodexNode();

4
docker/job.yaml
View File

@ -28,6 +28,10 @@ spec:
value: ${BRANCH}
- name: SOURCE
value: ${SOURCE}
- name: RUNID
value: ${RUNID}
- name: TESTID
value: ${TESTID}
volumeMounts:
- name: kubeconfig
mountPath: /opt/kubeconfig.yaml