Merge branch 'master' into feature/docker-image-testruns
# Conflicts: # DistTestCore/Codex/CodexContainerRecipe.cs
This commit is contained in:
commit
dbbd05ea97
|
@ -15,19 +15,19 @@ on:
|
||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
inputs:
|
inputs:
|
||||||
branch:
|
branch:
|
||||||
description: Branch
|
description: Branch (master)
|
||||||
required: false
|
required: false
|
||||||
type: string
|
type: string
|
||||||
source:
|
source:
|
||||||
description: Repository with tests
|
description: Repository with tests (current)
|
||||||
required: false
|
required: false
|
||||||
type: string
|
type: string
|
||||||
nameprefix:
|
nameprefix:
|
||||||
description: Runner job/pod name prefix
|
description: Runner prefix (cs-codex-dist-tests)
|
||||||
required: false
|
required: false
|
||||||
type: string
|
type: string
|
||||||
namespace:
|
namespace:
|
||||||
description: Kubernetes namespace for runner
|
description: Runner namespace (cs-codex-dist-tests)
|
||||||
required: false
|
required: false
|
||||||
type: string
|
type: string
|
||||||
|
|
||||||
|
@ -56,6 +56,8 @@ jobs:
|
||||||
[[ -n "${{ inputs.source }}" ]] && echo "SOURCE=${{ inputs.source }}" >>"$GITHUB_ENV" || echo "SOURCE=${{ env.SOURCE }}" >>"$GITHUB_ENV"
|
[[ -n "${{ inputs.source }}" ]] && echo "SOURCE=${{ inputs.source }}" >>"$GITHUB_ENV" || echo "SOURCE=${{ env.SOURCE }}" >>"$GITHUB_ENV"
|
||||||
[[ -n "${{ inputs.nameprefix }}" ]] && echo "NAMEPREFIX=${{ inputs.nameprefix }}" >>"$GITHUB_ENV" || echo "NAMEPREFIX=${{ env.NAMEPREFIX }}" >>"$GITHUB_ENV"
|
[[ -n "${{ inputs.nameprefix }}" ]] && echo "NAMEPREFIX=${{ inputs.nameprefix }}" >>"$GITHUB_ENV" || echo "NAMEPREFIX=${{ env.NAMEPREFIX }}" >>"$GITHUB_ENV"
|
||||||
[[ -n "${{ inputs.namespace }}" ]] && echo "NAMESPACE=${{ inputs.namespace }}" >>"$GITHUB_ENV" || echo "NAMESPACE=${{ env.NAMESPACE }}" >>"$GITHUB_ENV"
|
[[ -n "${{ inputs.namespace }}" ]] && echo "NAMESPACE=${{ inputs.namespace }}" >>"$GITHUB_ENV" || echo "NAMESPACE=${{ env.NAMESPACE }}" >>"$GITHUB_ENV"
|
||||||
|
echo "RUNID=$(date +%Y%m%d-%H%M%S)" >> $GITHUB_ENV
|
||||||
|
echo "TESTID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
|
||||||
|
|
||||||
- name: Kubectl - Install ${{ env.KUBE_VERSION }}
|
- name: Kubectl - Install ${{ env.KUBE_VERSION }}
|
||||||
uses: azure/setup-kubectl@v3
|
uses: azure/setup-kubectl@v3
|
||||||
|
@ -69,5 +71,4 @@ jobs:
|
||||||
|
|
||||||
- name: Kubectl - Create Job
|
- name: Kubectl - Create Job
|
||||||
run: |
|
run: |
|
||||||
export RUNID=$(date +%Y%m%d-%H%M%S)
|
|
||||||
envsubst < ${{ env.JOB_MANIFEST }} | kubectl apply -f -
|
envsubst < ${{ env.JOB_MANIFEST }} | kubectl apply -f -
|
||||||
|
|
|
@ -1,6 +1,9 @@
|
||||||
using ArgsUniform;
|
using ArgsUniform;
|
||||||
using CodexNetDeployer;
|
using CodexNetDeployer;
|
||||||
using DistTestCore;
|
using DistTestCore;
|
||||||
|
using DistTestCore.Codex;
|
||||||
|
using DistTestCore.Marketplace;
|
||||||
|
using DistTestCore.Metrics;
|
||||||
using Newtonsoft.Json;
|
using Newtonsoft.Json;
|
||||||
using Configuration = CodexNetDeployer.Configuration;
|
using Configuration = CodexNetDeployer.Configuration;
|
||||||
|
|
||||||
|
@ -29,6 +32,12 @@ public class Program
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Console.WriteLine("Using images:" + nl +
|
||||||
|
$"\tCodex image: '{CodexContainerRecipe.DockerImage}'" + nl +
|
||||||
|
$"\tCodex Contracts image: '{CodexContractsContainerRecipe.DockerImage}'" + nl +
|
||||||
|
$"\tPrometheus image: '{PrometheusContainerRecipe.DockerImage}'" + nl +
|
||||||
|
$"\tGeth image: '{GethContainerRecipe.DockerImage}'" + nl);
|
||||||
|
|
||||||
if (!args.Any(a => a == "-y"))
|
if (!args.Any(a => a == "-y"))
|
||||||
{
|
{
|
||||||
Console.WriteLine("Does the above config look good? [y/n]");
|
Console.WriteLine("Does the above config look good? [y/n]");
|
||||||
|
|
|
@ -3,10 +3,11 @@ dotnet run \
|
||||||
--kube-namespace=codex-continuous-tests \
|
--kube-namespace=codex-continuous-tests \
|
||||||
--nodes=5 \
|
--nodes=5 \
|
||||||
--validators=3 \
|
--validators=3 \
|
||||||
|
--log-level=Trace \
|
||||||
--storage-quota=2048 \
|
--storage-quota=2048 \
|
||||||
--storage-sell=1024 \
|
--storage-sell=1024 \
|
||||||
--min-price=1024 \
|
--min-price=1024 \
|
||||||
--max-collateral=1024 \
|
--max-collateral=1024 \
|
||||||
--max-duration=3600000 \
|
--max-duration=3600000 \
|
||||||
--block-ttl=120
|
--block-ttl=120 \
|
||||||
|
-y
|
||||||
|
|
|
@ -25,6 +25,9 @@ namespace ContinuousTests
|
||||||
[Uniform("stop", "s", "STOPONFAIL", false, "If true, runner will stop on first test failure and download all cluster container logs. False by default.")]
|
[Uniform("stop", "s", "STOPONFAIL", false, "If true, runner will stop on first test failure and download all cluster container logs. False by default.")]
|
||||||
public bool StopOnFailure { get; set; } = false;
|
public bool StopOnFailure { get; set; } = false;
|
||||||
|
|
||||||
|
[Uniform("dl-logs", "dl", "DLLOGS", false, "If true, runner will periodically download and save/append container logs to the log path.")]
|
||||||
|
public bool DownloadContainerLogs { get; set; } = false;
|
||||||
|
|
||||||
public CodexDeployment CodexDeployment { get; set; } = null!;
|
public CodexDeployment CodexDeployment { get; set; } = null!;
|
||||||
|
|
||||||
public TestRunnerLocation RunnerLocation { get; set; } = TestRunnerLocation.InternalToCluster;
|
public TestRunnerLocation RunnerLocation { get; set; } = TestRunnerLocation.InternalToCluster;
|
||||||
|
@ -57,9 +60,11 @@ namespace ContinuousTests
|
||||||
private static void PrintHelp()
|
private static void PrintHelp()
|
||||||
{
|
{
|
||||||
var nl = Environment.NewLine;
|
var nl = Environment.NewLine;
|
||||||
Console.WriteLine("CodexNetDownloader lets you download all container logs given a codex-deployment.json file." + nl);
|
Console.WriteLine("ContinuousTests will run a set of tests against a codex deployment given a codex-deployment.json file." + nl +
|
||||||
|
"The tests will run in an endless loop unless otherwise specified, using the test-specific timing values." + nl);
|
||||||
|
|
||||||
Console.WriteLine("CodexNetDownloader assumes you are running this tool from *inside* the Kubernetes cluster. " +
|
|
||||||
|
Console.WriteLine("ContinuousTests assumes you are running this tool from *inside* the Kubernetes cluster. " +
|
||||||
"If you are not running this from a container inside the cluster, add the argument '--external'." + nl);
|
"If you are not running this from a container inside the cluster, add the argument '--external'." + nl);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,99 @@
|
||||||
|
using DistTestCore;
|
||||||
|
using DistTestCore.Codex;
|
||||||
|
using KubernetesWorkflow;
|
||||||
|
|
||||||
|
namespace ContinuousTests
|
||||||
|
{
|
||||||
|
public class ContinuousLogDownloader
|
||||||
|
{
|
||||||
|
private readonly TestLifecycle lifecycle;
|
||||||
|
private readonly CodexDeployment deployment;
|
||||||
|
private readonly string outputPath;
|
||||||
|
private readonly CancellationToken cancelToken;
|
||||||
|
|
||||||
|
public ContinuousLogDownloader(TestLifecycle lifecycle, CodexDeployment deployment, string outputPath, CancellationToken cancelToken)
|
||||||
|
{
|
||||||
|
this.lifecycle = lifecycle;
|
||||||
|
this.deployment = deployment;
|
||||||
|
this.outputPath = outputPath;
|
||||||
|
this.cancelToken = cancelToken;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void Run()
|
||||||
|
{
|
||||||
|
while (!cancelToken.IsCancellationRequested)
|
||||||
|
{
|
||||||
|
UpdateLogs();
|
||||||
|
|
||||||
|
cancelToken.WaitHandle.WaitOne(TimeSpan.FromSeconds(15));
|
||||||
|
}
|
||||||
|
|
||||||
|
// After testing has stopped, we wait a little bit and fetch the logs one more time.
|
||||||
|
// If our latest fetch was not recent, interesting test-related log activity might
|
||||||
|
// not have been captured yet.
|
||||||
|
Thread.Sleep(TimeSpan.FromSeconds(10));
|
||||||
|
UpdateLogs();
|
||||||
|
}
|
||||||
|
|
||||||
|
private void UpdateLogs()
|
||||||
|
{
|
||||||
|
foreach (var container in deployment.CodexContainers)
|
||||||
|
{
|
||||||
|
UpdateLog(container);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void UpdateLog(RunningContainer container)
|
||||||
|
{
|
||||||
|
var filepath = Path.Combine(outputPath, GetLogName(container));
|
||||||
|
if (!File.Exists(filepath))
|
||||||
|
{
|
||||||
|
File.WriteAllLines(filepath, new[] { container.Name });
|
||||||
|
}
|
||||||
|
|
||||||
|
var appender = new LogAppender(filepath);
|
||||||
|
|
||||||
|
lifecycle.CodexStarter.DownloadLog(container, appender);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static string GetLogName(RunningContainer container)
|
||||||
|
{
|
||||||
|
return container.Name
|
||||||
|
.Replace("<", "")
|
||||||
|
.Replace(">", "")
|
||||||
|
+ ".log";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public class LogAppender : ILogHandler
|
||||||
|
{
|
||||||
|
private readonly string filename;
|
||||||
|
|
||||||
|
public LogAppender(string filename)
|
||||||
|
{
|
||||||
|
this.filename = filename;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void Log(Stream log)
|
||||||
|
{
|
||||||
|
using var reader = new StreamReader(log);
|
||||||
|
var lines = File.ReadAllLines(filename);
|
||||||
|
var lastLine = lines.Last();
|
||||||
|
var recording = lines.Length < 3;
|
||||||
|
var line = reader.ReadLine();
|
||||||
|
while (line != null)
|
||||||
|
{
|
||||||
|
if (recording)
|
||||||
|
{
|
||||||
|
File.AppendAllLines(filename, new[] { line });
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
recording = line == lastLine;
|
||||||
|
}
|
||||||
|
|
||||||
|
line = reader.ReadLine();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -24,12 +24,14 @@ namespace ContinuousTests
|
||||||
startupChecker.Check();
|
startupChecker.Check();
|
||||||
|
|
||||||
var taskFactory = new TaskFactory();
|
var taskFactory = new TaskFactory();
|
||||||
var overviewLog = new FixtureLog(new LogConfig(config.LogPath, false), "Overview");
|
var overviewLog = new FixtureLog(new LogConfig(config.LogPath, false), DateTime.UtcNow, "Overview");
|
||||||
overviewLog.Log("Continuous tests starting...");
|
overviewLog.Log("Continuous tests starting...");
|
||||||
var allTests = testFactory.CreateTests();
|
var allTests = testFactory.CreateTests();
|
||||||
|
|
||||||
ClearAllCustomNamespaces(allTests, overviewLog);
|
ClearAllCustomNamespaces(allTests, overviewLog);
|
||||||
|
|
||||||
|
StartLogDownloader(taskFactory);
|
||||||
|
|
||||||
var testLoops = allTests.Select(t => new TestLoop(taskFactory, config, overviewLog, t.GetType(), t.RunTestEvery, cancelToken)).ToArray();
|
var testLoops = allTests.Select(t => new TestLoop(taskFactory, config, overviewLog, t.GetType(), t.RunTestEvery, cancelToken)).ToArray();
|
||||||
|
|
||||||
foreach (var testLoop in testLoops)
|
foreach (var testLoop in testLoops)
|
||||||
|
@ -61,5 +63,18 @@ namespace ContinuousTests
|
||||||
var (workflowCreator, _) = k8SFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, config.DataPath, test.CustomK8sNamespace, new DefaultTimeSet(), log, config.RunnerLocation);
|
var (workflowCreator, _) = k8SFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, config.DataPath, test.CustomK8sNamespace, new DefaultTimeSet(), log, config.RunnerLocation);
|
||||||
workflowCreator.CreateWorkflow().DeleteTestResources();
|
workflowCreator.CreateWorkflow().DeleteTestResources();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void StartLogDownloader(TaskFactory taskFactory)
|
||||||
|
{
|
||||||
|
if (!config.DownloadContainerLogs) return;
|
||||||
|
|
||||||
|
var path = Path.Combine(config.LogPath, "containers");
|
||||||
|
if (!Directory.Exists(path)) Directory.CreateDirectory(path);
|
||||||
|
|
||||||
|
var (_, lifecycle) = k8SFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, config.DataPath, config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation);
|
||||||
|
var downloader = new ContinuousLogDownloader(lifecycle, config.CodexDeployment, path, cancelToken);
|
||||||
|
|
||||||
|
taskFactory.Run(downloader.Run);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -32,7 +32,7 @@ namespace ContinuousTests
|
||||||
this.handle = handle;
|
this.handle = handle;
|
||||||
this.cancelToken = cancelToken;
|
this.cancelToken = cancelToken;
|
||||||
testName = handle.Test.GetType().Name;
|
testName = handle.Test.GetType().Name;
|
||||||
fixtureLog = new FixtureLog(new LogConfig(config.LogPath, true), testName);
|
fixtureLog = new FixtureLog(new LogConfig(config.LogPath, true), DateTime.UtcNow, testName);
|
||||||
|
|
||||||
nodes = CreateRandomNodes(handle.Test.RequiredNumberOfNodes);
|
nodes = CreateRandomNodes(handle.Test.RequiredNumberOfNodes);
|
||||||
dataFolder = config.DataPath + "-" + Guid.NewGuid();
|
dataFolder = config.DataPath + "-" + Guid.NewGuid();
|
||||||
|
|
|
@ -19,7 +19,7 @@ namespace ContinuousTests
|
||||||
|
|
||||||
public void Check()
|
public void Check()
|
||||||
{
|
{
|
||||||
var log = new FixtureLog(new LogConfig(config.LogPath, false), "StartupChecks");
|
var log = new FixtureLog(new LogConfig(config.LogPath, false), DateTime.UtcNow, "StartupChecks");
|
||||||
log.Log("Starting continuous test run...");
|
log.Log("Starting continuous test run...");
|
||||||
log.Log("Checking configuration...");
|
log.Log("Checking configuration...");
|
||||||
PreflightCheck(config);
|
PreflightCheck(config);
|
||||||
|
|
|
@ -1,4 +1,6 @@
|
||||||
dotnet run \
|
dotnet run \
|
||||||
--kube-config=/opt/kubeconfig.yaml \
|
--kube-config=/opt/kubeconfig.yaml \
|
||||||
--codex-deployment=codex-deployment.json \
|
--codex-deployment=codex-deployment.json \
|
||||||
--stop=1
|
--keep=1 \
|
||||||
|
--stop=1 \
|
||||||
|
--dl-logs=1
|
|
@ -22,17 +22,12 @@ namespace DistTestCore.Codex
|
||||||
|
|
||||||
public CodexDebugResponse GetDebugInfo()
|
public CodexDebugResponse GetDebugInfo()
|
||||||
{
|
{
|
||||||
return Http(TimeSpan.FromSeconds(60)).HttpGetJson<CodexDebugResponse>("debug/info");
|
return Http().HttpGetJson<CodexDebugResponse>("debug/info");
|
||||||
}
|
}
|
||||||
|
|
||||||
public CodexDebugPeerResponse GetDebugPeer(string peerId)
|
public CodexDebugPeerResponse GetDebugPeer(string peerId)
|
||||||
{
|
{
|
||||||
return GetDebugPeer(peerId, TimeSpan.FromSeconds(10));
|
var http = Http();
|
||||||
}
|
|
||||||
|
|
||||||
public CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout)
|
|
||||||
{
|
|
||||||
var http = Http(timeout);
|
|
||||||
var str = http.HttpGetString($"debug/peer/{peerId}");
|
var str = http.HttpGetString($"debug/peer/{peerId}");
|
||||||
|
|
||||||
if (str.ToLowerInvariant() == "unable to find peer!")
|
if (str.ToLowerInvariant() == "unable to find peer!")
|
||||||
|
@ -50,7 +45,8 @@ namespace DistTestCore.Codex
|
||||||
|
|
||||||
public int GetDebugFutures()
|
public int GetDebugFutures()
|
||||||
{
|
{
|
||||||
return Http().HttpGetJson<CodexDebugFutures>("debug/futures").futures;
|
// Some Codex images support debug/futures to count the number of open futures.
|
||||||
|
return 0; // Http().HttpGetJson<CodexDebugFutures>("debug/futures").futures;
|
||||||
}
|
}
|
||||||
|
|
||||||
public CodexDebugThresholdBreaches GetDebugThresholdBreaches()
|
public CodexDebugThresholdBreaches GetDebugThresholdBreaches()
|
||||||
|
@ -88,9 +84,9 @@ namespace DistTestCore.Codex
|
||||||
return Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}");
|
return Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}");
|
||||||
}
|
}
|
||||||
|
|
||||||
private Http Http(TimeSpan? timeoutOverride = null)
|
private Http Http()
|
||||||
{
|
{
|
||||||
return new Http(log, timeSet, Address, baseUrl: "/api/codex/v1", Container.Name, timeoutOverride);
|
return new Http(log, timeSet, Address, baseUrl: "/api/codex/v1", Container.Name);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -5,6 +5,8 @@ namespace DistTestCore.Codex
|
||||||
{
|
{
|
||||||
public class CodexContainerRecipe : ContainerRecipeFactory
|
public class CodexContainerRecipe : ContainerRecipeFactory
|
||||||
{
|
{
|
||||||
|
private const string DefaultDockerImage = "codexstorage/nim-codex:sha-14c5270";
|
||||||
|
|
||||||
public const string MetricsPortTag = "metrics_port";
|
public const string MetricsPortTag = "metrics_port";
|
||||||
public const string DiscoveryPortTag = "discovery-port";
|
public const string DiscoveryPortTag = "discovery-port";
|
||||||
|
|
||||||
|
@ -87,7 +89,7 @@ namespace DistTestCore.Codex
|
||||||
{
|
{
|
||||||
var image = Environment.GetEnvironmentVariable("CODEXDOCKERIMAGE");
|
var image = Environment.GetEnvironmentVariable("CODEXDOCKERIMAGE");
|
||||||
if (!string.IsNullOrEmpty(image)) return image;
|
if (!string.IsNullOrEmpty(image)) return image;
|
||||||
return "codexstorage/nim-codex:sha-0265cad";
|
return DefaultDockerImage;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -16,6 +16,7 @@ namespace DistTestCore
|
||||||
private readonly Configuration configuration = new Configuration();
|
private readonly Configuration configuration = new Configuration();
|
||||||
private readonly Assembly[] testAssemblies;
|
private readonly Assembly[] testAssemblies;
|
||||||
private readonly FixtureLog fixtureLog;
|
private readonly FixtureLog fixtureLog;
|
||||||
|
private readonly StatusLog statusLog;
|
||||||
private readonly object lifecycleLock = new object();
|
private readonly object lifecycleLock = new object();
|
||||||
private readonly Dictionary<string, TestLifecycle> lifecycles = new Dictionary<string, TestLifecycle>();
|
private readonly Dictionary<string, TestLifecycle> lifecycles = new Dictionary<string, TestLifecycle>();
|
||||||
|
|
||||||
|
@ -24,7 +25,10 @@ namespace DistTestCore
|
||||||
var assemblies = AppDomain.CurrentDomain.GetAssemblies();
|
var assemblies = AppDomain.CurrentDomain.GetAssemblies();
|
||||||
testAssemblies = assemblies.Where(a => a.FullName!.ToLowerInvariant().Contains("test")).ToArray();
|
testAssemblies = assemblies.Where(a => a.FullName!.ToLowerInvariant().Contains("test")).ToArray();
|
||||||
|
|
||||||
fixtureLog = new FixtureLog(configuration.GetLogConfig());
|
var logConfig = configuration.GetLogConfig();
|
||||||
|
var startTime = DateTime.UtcNow;
|
||||||
|
fixtureLog = new FixtureLog(logConfig, startTime);
|
||||||
|
statusLog = new StatusLog(logConfig, startTime, CodexContainerRecipe.DefaultDockerImage);
|
||||||
|
|
||||||
PeerConnectionTestHelpers = new PeerConnectionTestHelpers(this);
|
PeerConnectionTestHelpers = new PeerConnectionTestHelpers(this);
|
||||||
PeerDownloadTestHelpers = new PeerDownloadTestHelpers(this);
|
PeerDownloadTestHelpers = new PeerDownloadTestHelpers(this);
|
||||||
|
@ -186,6 +190,7 @@ namespace DistTestCore
|
||||||
private void CreateNewTestLifecycle()
|
private void CreateNewTestLifecycle()
|
||||||
{
|
{
|
||||||
var testName = GetCurrentTestName();
|
var testName = GetCurrentTestName();
|
||||||
|
fixtureLog.WriteLogTag();
|
||||||
Stopwatch.Measure(fixtureLog, $"Setup for {testName}", () =>
|
Stopwatch.Measure(fixtureLog, $"Setup for {testName}", () =>
|
||||||
{
|
{
|
||||||
lock (lifecycleLock)
|
lock (lifecycleLock)
|
||||||
|
@ -198,7 +203,10 @@ namespace DistTestCore
|
||||||
private void DisposeTestLifecycle()
|
private void DisposeTestLifecycle()
|
||||||
{
|
{
|
||||||
var lifecycle = Get();
|
var lifecycle = Get();
|
||||||
fixtureLog.Log($"{GetCurrentTestName()} = {GetTestResult()} ({lifecycle.GetTestDuration()})");
|
var testResult = GetTestResult();
|
||||||
|
var testDuration = lifecycle.GetTestDuration();
|
||||||
|
fixtureLog.Log($"{GetCurrentTestName()} = {testResult} ({testDuration})");
|
||||||
|
statusLog.ConcludeTest(testResult, testDuration);
|
||||||
Stopwatch.Measure(fixtureLog, $"Teardown for {GetCurrentTestName()}", () =>
|
Stopwatch.Measure(fixtureLog, $"Teardown for {GetCurrentTestName()}", () =>
|
||||||
{
|
{
|
||||||
lifecycle.Log.EndTest();
|
lifecycle.Log.EndTest();
|
||||||
|
|
|
@ -0,0 +1,211 @@
|
||||||
|
using DistTestCore.Codex;
|
||||||
|
using NUnit.Framework;
|
||||||
|
using Utils;
|
||||||
|
|
||||||
|
namespace DistTestCore.Helpers
|
||||||
|
{
|
||||||
|
public interface IFullConnectivityImplementation
|
||||||
|
{
|
||||||
|
string Description();
|
||||||
|
string ValidateEntry(FullConnectivityHelper.Entry entry, FullConnectivityHelper.Entry[] allEntries);
|
||||||
|
FullConnectivityHelper.PeerConnectionState Check(FullConnectivityHelper.Entry from, FullConnectivityHelper.Entry to);
|
||||||
|
}
|
||||||
|
|
||||||
|
public class FullConnectivityHelper
|
||||||
|
{
|
||||||
|
private static string Nl = Environment.NewLine;
|
||||||
|
private readonly DistTest test;
|
||||||
|
private readonly IFullConnectivityImplementation implementation;
|
||||||
|
|
||||||
|
public FullConnectivityHelper(DistTest test, IFullConnectivityImplementation implementation)
|
||||||
|
{
|
||||||
|
this.test = test;
|
||||||
|
this.implementation = implementation;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void AssertFullyConnected(IEnumerable<IOnlineCodexNode> nodes)
|
||||||
|
{
|
||||||
|
AssertFullyConnected(nodes.ToArray());
|
||||||
|
}
|
||||||
|
|
||||||
|
private void AssertFullyConnected(IOnlineCodexNode[] nodes)
|
||||||
|
{
|
||||||
|
test.Log($"Asserting '{implementation.Description()}' for nodes: '{string.Join(",", nodes.Select(n => n.GetName()))}'...");
|
||||||
|
var entries = CreateEntries(nodes);
|
||||||
|
var pairs = CreatePairs(entries);
|
||||||
|
|
||||||
|
RetryWhilePairs(pairs, () =>
|
||||||
|
{
|
||||||
|
CheckAndRemoveSuccessful(pairs);
|
||||||
|
});
|
||||||
|
|
||||||
|
if (pairs.Any())
|
||||||
|
{
|
||||||
|
var pairDetails = string.Join(Nl, pairs.SelectMany(p => p.GetResultMessages()));
|
||||||
|
|
||||||
|
test.Log($"Connections failed:{Nl}{pairDetails}");
|
||||||
|
|
||||||
|
Assert.Fail(string.Join(Nl, pairs.SelectMany(p => p.GetResultMessages())));
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
test.Log($"'{implementation.Description()}' = Success! for nodes: {string.Join(",", nodes.Select(n => n.GetName()))}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void RetryWhilePairs(List<Pair> pairs, Action action)
|
||||||
|
{
|
||||||
|
var timeout = DateTime.UtcNow + TimeSpan.FromMinutes(5);
|
||||||
|
while (pairs.Any(p => p.Inconclusive) && timeout > DateTime.UtcNow)
|
||||||
|
{
|
||||||
|
action();
|
||||||
|
|
||||||
|
Time.Sleep(TimeSpan.FromSeconds(2));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void CheckAndRemoveSuccessful(List<Pair> pairs)
|
||||||
|
{
|
||||||
|
// For large sets, don't try and do all of them at once.
|
||||||
|
var selectedPair = pairs.Take(20).ToArray();
|
||||||
|
var pairDetails = new List<string>();
|
||||||
|
|
||||||
|
foreach (var pair in selectedPair)
|
||||||
|
{
|
||||||
|
test.ScopedTestFiles(pair.Check);
|
||||||
|
|
||||||
|
if (pair.Success)
|
||||||
|
{
|
||||||
|
pairDetails.AddRange(pair.GetResultMessages());
|
||||||
|
pairs.Remove(pair);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
test.Log($"Connections successful:{Nl}{string.Join(Nl, pairDetails)}");
|
||||||
|
}
|
||||||
|
|
||||||
|
private Entry[] CreateEntries(IOnlineCodexNode[] nodes)
|
||||||
|
{
|
||||||
|
var entries = nodes.Select(n => new Entry(n)).ToArray();
|
||||||
|
|
||||||
|
var errors = entries
|
||||||
|
.Select(e => implementation.ValidateEntry(e, entries))
|
||||||
|
.Where(s => !string.IsNullOrEmpty(s))
|
||||||
|
.ToArray();
|
||||||
|
|
||||||
|
if (errors.Any())
|
||||||
|
{
|
||||||
|
Assert.Fail("Some node entries failed to validate: " + string.Join(Nl, errors));
|
||||||
|
}
|
||||||
|
|
||||||
|
return entries;
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<Pair> CreatePairs(Entry[] entries)
|
||||||
|
{
|
||||||
|
return CreatePairsIterator(entries).ToList();
|
||||||
|
}
|
||||||
|
|
||||||
|
private IEnumerable<Pair> CreatePairsIterator(Entry[] entries)
|
||||||
|
{
|
||||||
|
for (var x = 0; x < entries.Length; x++)
|
||||||
|
{
|
||||||
|
for (var y = x + 1; y < entries.Length; y++)
|
||||||
|
{
|
||||||
|
yield return new Pair(implementation, entries[x], entries[y]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public class Entry
|
||||||
|
{
|
||||||
|
public Entry(IOnlineCodexNode node)
|
||||||
|
{
|
||||||
|
Node = node;
|
||||||
|
Response = node.GetDebugInfo();
|
||||||
|
}
|
||||||
|
|
||||||
|
public IOnlineCodexNode Node { get; }
|
||||||
|
public CodexDebugResponse Response { get; }
|
||||||
|
|
||||||
|
public override string ToString()
|
||||||
|
{
|
||||||
|
if (Response == null || string.IsNullOrEmpty(Response.id)) return "UNKNOWN";
|
||||||
|
return Response.id;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public enum PeerConnectionState
|
||||||
|
{
|
||||||
|
Unknown,
|
||||||
|
Connection,
|
||||||
|
NoConnection,
|
||||||
|
}
|
||||||
|
|
||||||
|
public class Pair
|
||||||
|
{
|
||||||
|
private TimeSpan aToBTime = TimeSpan.FromSeconds(0);
|
||||||
|
private TimeSpan bToATime = TimeSpan.FromSeconds(0);
|
||||||
|
private readonly IFullConnectivityImplementation implementation;
|
||||||
|
|
||||||
|
public Pair(IFullConnectivityImplementation implementation, Entry a, Entry b)
|
||||||
|
{
|
||||||
|
this.implementation = implementation;
|
||||||
|
A = a;
|
||||||
|
B = b;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Entry A { get; }
|
||||||
|
public Entry B { get; }
|
||||||
|
public PeerConnectionState AKnowsB { get; private set; }
|
||||||
|
public PeerConnectionState BKnowsA { get; private set; }
|
||||||
|
public bool Success { get { return AKnowsB == PeerConnectionState.Connection && BKnowsA == PeerConnectionState.Connection; } }
|
||||||
|
public bool Inconclusive { get { return AKnowsB == PeerConnectionState.Unknown || BKnowsA == PeerConnectionState.Unknown; } }
|
||||||
|
|
||||||
|
public void Check()
|
||||||
|
{
|
||||||
|
aToBTime = Measure(() => AKnowsB = Check(A, B));
|
||||||
|
bToATime = Measure(() => BKnowsA = Check(B, A));
|
||||||
|
}
|
||||||
|
|
||||||
|
public override string ToString()
|
||||||
|
{
|
||||||
|
return $"[{string.Join(",", GetResultMessages())}]";
|
||||||
|
}
|
||||||
|
|
||||||
|
public string[] GetResultMessages()
|
||||||
|
{
|
||||||
|
var aName = A.ToString();
|
||||||
|
var bName = B.ToString();
|
||||||
|
|
||||||
|
return new[]
|
||||||
|
{
|
||||||
|
$"[{aName} --> {bName}] = {AKnowsB} ({aToBTime.TotalSeconds} seconds)",
|
||||||
|
$"[{aName} <-- {bName}] = {BKnowsA} ({bToATime.TotalSeconds} seconds)"
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private static TimeSpan Measure(Action action)
|
||||||
|
{
|
||||||
|
var start = DateTime.UtcNow;
|
||||||
|
action();
|
||||||
|
return DateTime.UtcNow - start;
|
||||||
|
}
|
||||||
|
|
||||||
|
private PeerConnectionState Check(Entry from, Entry to)
|
||||||
|
{
|
||||||
|
Thread.Sleep(10);
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
return implementation.Check(from, to);
|
||||||
|
}
|
||||||
|
catch
|
||||||
|
{
|
||||||
|
// Didn't get a conclusive answer. Try again later.
|
||||||
|
return PeerConnectionState.Unknown;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -1,258 +1,66 @@
|
||||||
using DistTestCore.Codex;
|
using DistTestCore.Codex;
|
||||||
using NUnit.Framework;
|
using static DistTestCore.Helpers.FullConnectivityHelper;
|
||||||
using Utils;
|
|
||||||
|
|
||||||
namespace DistTestCore.Helpers
|
namespace DistTestCore.Helpers
|
||||||
{
|
{
|
||||||
public class PeerConnectionTestHelpers
|
public class PeerConnectionTestHelpers : IFullConnectivityImplementation
|
||||||
{
|
{
|
||||||
private readonly Random random = new Random();
|
private readonly FullConnectivityHelper helper;
|
||||||
private readonly DistTest test;
|
|
||||||
|
|
||||||
public PeerConnectionTestHelpers(DistTest test)
|
public PeerConnectionTestHelpers(DistTest test)
|
||||||
{
|
{
|
||||||
this.test = test;
|
helper = new FullConnectivityHelper(test, this);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void AssertFullyConnected(IEnumerable<IOnlineCodexNode> nodes)
|
public void AssertFullyConnected(IEnumerable<IOnlineCodexNode> nodes)
|
||||||
{
|
{
|
||||||
var n = nodes.ToArray();
|
helper.AssertFullyConnected(nodes);
|
||||||
|
|
||||||
AssertFullyConnected(n);
|
|
||||||
|
|
||||||
for (int i = 0; i < 5; i++)
|
|
||||||
{
|
|
||||||
Time.Sleep(TimeSpan.FromSeconds(30));
|
|
||||||
AssertFullyConnected(n);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private void AssertFullyConnected(IOnlineCodexNode[] nodes)
|
public string Description()
|
||||||
{
|
{
|
||||||
test.Log($"Asserting peers are fully-connected for nodes: '{string.Join(",", nodes.Select(n => n.GetName()))}'...");
|
return "Peer Discovery";
|
||||||
var entries = CreateEntries(nodes);
|
|
||||||
var pairs = CreatePairs(entries);
|
|
||||||
|
|
||||||
RetryWhilePairs(pairs, () =>
|
|
||||||
{
|
|
||||||
CheckAndRemoveSuccessful(pairs);
|
|
||||||
});
|
|
||||||
|
|
||||||
if (pairs.Any())
|
|
||||||
{
|
|
||||||
test.Log($"Unsuccessful! Peers are not fully-connected: {string.Join(",", nodes.Select(n => n.GetName()))}");
|
|
||||||
Assert.Fail(string.Join(Environment.NewLine, pairs.Select(p => p.GetMessage())));
|
|
||||||
test.Log(string.Join(Environment.NewLine, pairs.Select(p => p.GetMessage())));
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
test.Log($"Success! Peers are fully-connected: {string.Join(",", nodes.Select(n => n.GetName()))}");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private static void RetryWhilePairs(List<Pair> pairs, Action action)
|
public string ValidateEntry(Entry entry, Entry[] allEntries)
|
||||||
{
|
{
|
||||||
var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(30);
|
var result = string.Empty;
|
||||||
while (pairs.Any() && timeout > DateTime.UtcNow)
|
foreach (var peer in entry.Response.table.nodes)
|
||||||
{
|
{
|
||||||
action();
|
var expected = GetExpectedDiscoveryEndpoint(allEntries, peer);
|
||||||
|
if (expected != peer.address)
|
||||||
if (pairs.Any()) Time.Sleep(TimeSpan.FromSeconds(2));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private void CheckAndRemoveSuccessful(List<Pair> pairs)
|
|
||||||
{
|
|
||||||
var checkTasks = pairs.Select(p => Task.Run(() =>
|
|
||||||
{
|
|
||||||
ApplyRandomDelay();
|
|
||||||
p.Check();
|
|
||||||
})).ToArray();
|
|
||||||
|
|
||||||
Task.WaitAll(checkTasks);
|
|
||||||
|
|
||||||
foreach (var pair in pairs.ToArray())
|
|
||||||
{
|
|
||||||
if (pair.Success)
|
|
||||||
{
|
{
|
||||||
test.Log(pair.GetMessage());
|
result += $"Node:{entry.Node.GetName()} has incorrect peer table entry. Was: '{peer.address}', expected: '{expected}'. ";
|
||||||
pairs.Remove(pair);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
private static Entry[] CreateEntries(IOnlineCodexNode[] nodes)
|
public PeerConnectionState Check(Entry from, Entry to)
|
||||||
{
|
{
|
||||||
var entries = nodes.Select(n => new Entry(n)).ToArray();
|
var peerId = to.Response.id;
|
||||||
var incorrectDiscoveryEndpoints = entries.SelectMany(e => e.GetInCorrectDiscoveryEndpoints(entries)).ToArray();
|
|
||||||
|
|
||||||
if (incorrectDiscoveryEndpoints.Any())
|
var response = from.Node.GetDebugPeer(peerId);
|
||||||
|
if (!response.IsPeerFound)
|
||||||
{
|
{
|
||||||
Assert.Fail("Some nodes contain peer records with incorrect discovery ip/port information: " +
|
return PeerConnectionState.NoConnection;
|
||||||
string.Join(Environment.NewLine, incorrectDiscoveryEndpoints));
|
|
||||||
}
|
}
|
||||||
|
if (!string.IsNullOrEmpty(response.peerId) && response.addresses.Any())
|
||||||
return entries;
|
{
|
||||||
|
return PeerConnectionState.Connection;
|
||||||
|
}
|
||||||
|
return PeerConnectionState.Unknown;
|
||||||
}
|
}
|
||||||
|
|
||||||
private static List<Pair> CreatePairs(Entry[] entries)
|
private static string GetExpectedDiscoveryEndpoint(Entry[] allEntries, CodexDebugTableNodeResponse node)
|
||||||
{
|
{
|
||||||
return CreatePairsIterator(entries).ToList();
|
var peer = allEntries.SingleOrDefault(e => e.Response.table.localNode.peerId == node.peerId);
|
||||||
}
|
if (peer == null) return $"peerId: {node.peerId} is not known.";
|
||||||
|
|
||||||
private static IEnumerable<Pair> CreatePairsIterator(Entry[] entries)
|
var n = (OnlineCodexNode)peer.Node;
|
||||||
{
|
var ip = n.CodexAccess.Container.Pod.PodInfo.Ip;
|
||||||
for (var x = 0; x < entries.Length; x++)
|
var discPort = n.CodexAccess.Container.Recipe.GetPortByTag(CodexContainerRecipe.DiscoveryPortTag);
|
||||||
{
|
return $"{ip}:{discPort.Number}";
|
||||||
for (var y = x + 1; y < entries.Length; y++)
|
|
||||||
{
|
|
||||||
yield return new Pair(entries[x], entries[y]);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private void ApplyRandomDelay()
|
|
||||||
{
|
|
||||||
// Calling all the nodes all at the same time is not exactly nice.
|
|
||||||
Time.Sleep(TimeSpan.FromMicroseconds(random.Next(10, 1000)));
|
|
||||||
}
|
|
||||||
|
|
||||||
public class Entry
|
|
||||||
{
|
|
||||||
public Entry(IOnlineCodexNode node)
|
|
||||||
{
|
|
||||||
Node = node;
|
|
||||||
Response = node.GetDebugInfo();
|
|
||||||
}
|
|
||||||
|
|
||||||
public IOnlineCodexNode Node { get; }
|
|
||||||
public CodexDebugResponse Response { get; }
|
|
||||||
|
|
||||||
public IEnumerable<string> GetInCorrectDiscoveryEndpoints(Entry[] allEntries)
|
|
||||||
{
|
|
||||||
foreach (var peer in Response.table.nodes)
|
|
||||||
{
|
|
||||||
var expected = GetExpectedDiscoveryEndpoint(allEntries, peer);
|
|
||||||
if (expected != peer.address)
|
|
||||||
{
|
|
||||||
yield return $"Node:{Node.GetName()} has incorrect peer table entry. Was: '{peer.address}', expected: '{expected}'";
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public override string ToString()
|
|
||||||
{
|
|
||||||
if (Response == null || string.IsNullOrEmpty(Response.id)) return "UNKNOWN";
|
|
||||||
return Response.id;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static string GetExpectedDiscoveryEndpoint(Entry[] allEntries, CodexDebugTableNodeResponse node)
|
|
||||||
{
|
|
||||||
var peer = allEntries.SingleOrDefault(e => e.Response.table.localNode.peerId == node.peerId);
|
|
||||||
if (peer == null) return $"peerId: {node.peerId} is not known.";
|
|
||||||
|
|
||||||
var n = (OnlineCodexNode)peer.Node;
|
|
||||||
var ip = n.CodexAccess.Container.Pod.PodInfo.Ip;
|
|
||||||
var discPort = n.CodexAccess.Container.Recipe.GetPortByTag(CodexContainerRecipe.DiscoveryPortTag);
|
|
||||||
return $"{ip}:{discPort.Number}";
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public enum PeerConnectionState
|
|
||||||
{
|
|
||||||
Unknown,
|
|
||||||
Connection,
|
|
||||||
NoConnection,
|
|
||||||
}
|
|
||||||
|
|
||||||
public class Pair
|
|
||||||
{
|
|
||||||
private readonly TimeSpan timeout = TimeSpan.FromSeconds(60);
|
|
||||||
private TimeSpan aToBTime = TimeSpan.FromSeconds(0);
|
|
||||||
private TimeSpan bToATime = TimeSpan.FromSeconds(0);
|
|
||||||
|
|
||||||
public Pair(Entry a, Entry b)
|
|
||||||
{
|
|
||||||
A = a;
|
|
||||||
B = b;
|
|
||||||
}
|
|
||||||
|
|
||||||
public Entry A { get; }
|
|
||||||
public Entry B { get; }
|
|
||||||
public PeerConnectionState AKnowsB { get; private set; }
|
|
||||||
public PeerConnectionState BKnowsA { get; private set; }
|
|
||||||
public bool Success { get { return AKnowsB == PeerConnectionState.Connection && BKnowsA == PeerConnectionState.Connection; } }
|
|
||||||
|
|
||||||
public void Check()
|
|
||||||
{
|
|
||||||
aToBTime = Measure(() => AKnowsB = Knows(A, B));
|
|
||||||
bToATime = Measure(() => BKnowsA = Knows(B, A));
|
|
||||||
}
|
|
||||||
|
|
||||||
public string GetMessage()
|
|
||||||
{
|
|
||||||
return GetResultMessage() + GetTimePostfix();
|
|
||||||
}
|
|
||||||
|
|
||||||
public override string ToString()
|
|
||||||
{
|
|
||||||
return $"[{GetMessage()}]";
|
|
||||||
}
|
|
||||||
|
|
||||||
private string GetResultMessage()
|
|
||||||
{
|
|
||||||
var aName = A.ToString();
|
|
||||||
var bName = B.ToString();
|
|
||||||
|
|
||||||
if (Success)
|
|
||||||
{
|
|
||||||
return $"{aName} and {bName} know each other.";
|
|
||||||
}
|
|
||||||
|
|
||||||
return $"[{aName}-->{bName}] = {AKnowsB} AND [{aName}<--{bName}] = {BKnowsA}";
|
|
||||||
}
|
|
||||||
|
|
||||||
private string GetTimePostfix()
|
|
||||||
{
|
|
||||||
var aName = A.ToString();
|
|
||||||
var bName = B.ToString();
|
|
||||||
|
|
||||||
return $" ({aName}->{bName}: {aToBTime.TotalMinutes} seconds, {bName}->{aName}: {bToATime.TotalSeconds} seconds)";
|
|
||||||
}
|
|
||||||
|
|
||||||
private static TimeSpan Measure(Action action)
|
|
||||||
{
|
|
||||||
var start = DateTime.UtcNow;
|
|
||||||
action();
|
|
||||||
return DateTime.UtcNow - start;
|
|
||||||
}
|
|
||||||
|
|
||||||
private PeerConnectionState Knows(Entry a, Entry b)
|
|
||||||
{
|
|
||||||
lock (a)
|
|
||||||
{
|
|
||||||
var peerId = b.Response.id;
|
|
||||||
|
|
||||||
try
|
|
||||||
{
|
|
||||||
var response = a.Node.GetDebugPeer(peerId, timeout);
|
|
||||||
if (!response.IsPeerFound)
|
|
||||||
{
|
|
||||||
return PeerConnectionState.NoConnection;
|
|
||||||
}
|
|
||||||
if (!string.IsNullOrEmpty(response.peerId) && response.addresses.Any())
|
|
||||||
{
|
|
||||||
return PeerConnectionState.Connection;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
catch
|
|
||||||
{
|
|
||||||
}
|
|
||||||
|
|
||||||
// Didn't get a conclusive answer. Try again later.
|
|
||||||
return PeerConnectionState.Unknown;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,74 +1,63 @@
|
||||||
using DistTestCore.Codex;
|
using static DistTestCore.Helpers.FullConnectivityHelper;
|
||||||
using NUnit.Framework;
|
|
||||||
|
|
||||||
namespace DistTestCore.Helpers
|
namespace DistTestCore.Helpers
|
||||||
{
|
{
|
||||||
public class PeerDownloadTestHelpers
|
public class PeerDownloadTestHelpers : IFullConnectivityImplementation
|
||||||
{
|
{
|
||||||
|
private readonly FullConnectivityHelper helper;
|
||||||
private readonly DistTest test;
|
private readonly DistTest test;
|
||||||
|
private ByteSize testFileSize;
|
||||||
|
|
||||||
public PeerDownloadTestHelpers(DistTest test)
|
public PeerDownloadTestHelpers(DistTest test)
|
||||||
{
|
{
|
||||||
|
helper = new FullConnectivityHelper(test, this);
|
||||||
|
testFileSize = 1.MB();
|
||||||
this.test = test;
|
this.test = test;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void AssertFullDownloadInterconnectivity(IEnumerable<IOnlineCodexNode> nodes, ByteSize testFileSize)
|
public void AssertFullDownloadInterconnectivity(IEnumerable<IOnlineCodexNode> nodes, ByteSize testFileSize)
|
||||||
{
|
{
|
||||||
test.Log($"Asserting full download interconnectivity for nodes: '{string.Join(",", nodes.Select(n => n.GetName()))}'...");
|
this.testFileSize = testFileSize;
|
||||||
var start = DateTime.UtcNow;
|
helper.AssertFullyConnected(nodes);
|
||||||
|
|
||||||
foreach (var node in nodes)
|
|
||||||
{
|
|
||||||
var uploader = node;
|
|
||||||
var downloaders = nodes.Where(n => n != uploader).ToArray();
|
|
||||||
|
|
||||||
test.ScopedTestFiles(() =>
|
|
||||||
{
|
|
||||||
PerformTest(uploader, downloaders, testFileSize);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
test.Log($"Success! Full download interconnectivity for nodes: {string.Join(",", nodes.Select(n => n.GetName()))}");
|
|
||||||
var timeTaken = DateTime.UtcNow - start;
|
|
||||||
|
|
||||||
AssertTimePerMB(timeTaken, nodes.Count(), testFileSize);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private void AssertTimePerMB(TimeSpan timeTaken, int numberOfNodes, ByteSize size)
|
public string Description()
|
||||||
{
|
{
|
||||||
var numberOfDownloads = numberOfNodes * (numberOfNodes - 1);
|
return "Download Connectivity";
|
||||||
var timePerDownload = timeTaken / numberOfDownloads;
|
|
||||||
float sizeInMB = size.ToMB();
|
|
||||||
var timePerMB = timePerDownload / sizeInMB;
|
|
||||||
|
|
||||||
test.Log($"Performed {numberOfDownloads} downloads of {size} in {timeTaken.TotalSeconds} seconds, for an average of {timePerMB.TotalSeconds} seconds per MB.");
|
|
||||||
|
|
||||||
Assert.That(timePerMB, Is.LessThan(CodexContainerRecipe.MaxDownloadTimePerMegabyte), "MaxDownloadTimePerMegabyte performance threshold breached.");
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private void PerformTest(IOnlineCodexNode uploader, IOnlineCodexNode[] downloaders, ByteSize testFileSize)
|
public string ValidateEntry(Entry entry, Entry[] allEntries)
|
||||||
{
|
{
|
||||||
// Generate 1 test file per downloader.
|
return string.Empty;
|
||||||
var files = downloaders.Select(d => GenerateTestFile(uploader, d, testFileSize)).ToArray();
|
}
|
||||||
|
|
||||||
// Upload all the test files to the uploader.
|
public PeerConnectionState Check(Entry from, Entry to)
|
||||||
var contentIds = files.Select(uploader.UploadFile).ToArray();
|
{
|
||||||
|
var expectedFile = GenerateTestFile(from.Node, to.Node);
|
||||||
|
|
||||||
// Each downloader should retrieve its own test file.
|
var contentId = from.Node.UploadFile(expectedFile);
|
||||||
for (var i = 0; i < downloaders.Length; i++)
|
|
||||||
|
try
|
||||||
{
|
{
|
||||||
var expectedFile = files[i];
|
var downloadedFile = to.Node.DownloadContent(contentId, expectedFile.Label + "_downloaded");
|
||||||
var downloadedFile = downloaders[i].DownloadContent(contentIds[i], $"{expectedFile.Label}DOWNLOADED");
|
|
||||||
|
|
||||||
expectedFile.AssertIsEqual(downloadedFile);
|
expectedFile.AssertIsEqual(downloadedFile);
|
||||||
|
return PeerConnectionState.Connection;
|
||||||
}
|
}
|
||||||
|
catch
|
||||||
|
{
|
||||||
|
// Should an exception occur during the download or file-content assertion,
|
||||||
|
// We consider that as no-connection for the purpose of this test.
|
||||||
|
return PeerConnectionState.NoConnection;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Should an exception occur during upload, then this try is inconclusive and we try again next loop.
|
||||||
}
|
}
|
||||||
|
|
||||||
private TestFile GenerateTestFile(IOnlineCodexNode uploader, IOnlineCodexNode downloader, ByteSize testFileSize)
|
private TestFile GenerateTestFile(IOnlineCodexNode uploader, IOnlineCodexNode downloader)
|
||||||
{
|
{
|
||||||
var up = uploader.GetName().Replace("<", "").Replace(">", "");
|
var up = uploader.GetName().Replace("<", "").Replace(">", "");
|
||||||
var down = downloader.GetName().Replace("<", "").Replace(">", "");
|
var down = downloader.GetName().Replace("<", "").Replace(">", "");
|
||||||
var label = $"FROM{up}TO{down}";
|
var label = $"~from:{up}-to:{down}~";
|
||||||
return test.GenerateTestFile(testFileSize, label);
|
return test.GenerateTestFile(testFileSize, label);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -13,16 +13,14 @@ namespace DistTestCore
|
||||||
private readonly Address address;
|
private readonly Address address;
|
||||||
private readonly string baseUrl;
|
private readonly string baseUrl;
|
||||||
private readonly string? logAlias;
|
private readonly string? logAlias;
|
||||||
private readonly TimeSpan? timeoutOverride;
|
|
||||||
|
|
||||||
public Http(BaseLog log, ITimeSet timeSet, Address address, string baseUrl, string? logAlias = null, TimeSpan? timeoutOverride = null)
|
public Http(BaseLog log, ITimeSet timeSet, Address address, string baseUrl, string? logAlias = null)
|
||||||
{
|
{
|
||||||
this.log = log;
|
this.log = log;
|
||||||
this.timeSet = timeSet;
|
this.timeSet = timeSet;
|
||||||
this.address = address;
|
this.address = address;
|
||||||
this.baseUrl = baseUrl;
|
this.baseUrl = baseUrl;
|
||||||
this.logAlias = logAlias;
|
this.logAlias = logAlias;
|
||||||
this.timeoutOverride = timeoutOverride;
|
|
||||||
if (!this.baseUrl.StartsWith("/")) this.baseUrl = "/" + this.baseUrl;
|
if (!this.baseUrl.StartsWith("/")) this.baseUrl = "/" + this.baseUrl;
|
||||||
if (!this.baseUrl.EndsWith("/")) this.baseUrl += "/";
|
if (!this.baseUrl.EndsWith("/")) this.baseUrl += "/";
|
||||||
}
|
}
|
||||||
|
@ -127,24 +125,13 @@ namespace DistTestCore
|
||||||
|
|
||||||
private T Retry<T>(Func<T> operation, string description)
|
private T Retry<T>(Func<T> operation, string description)
|
||||||
{
|
{
|
||||||
return Time.Retry(operation, GetTimeout(), timeSet.HttpCallRetryDelay(), description);
|
return Time.Retry(operation, timeSet.HttpCallRetryTime(), timeSet.HttpCallRetryDelay(), description);
|
||||||
}
|
}
|
||||||
|
|
||||||
private HttpClient GetClient()
|
private HttpClient GetClient()
|
||||||
{
|
|
||||||
return GetClient(GetTimeout());
|
|
||||||
}
|
|
||||||
|
|
||||||
private TimeSpan GetTimeout()
|
|
||||||
{
|
|
||||||
if (timeoutOverride.HasValue) return timeoutOverride.Value;
|
|
||||||
return timeSet.HttpCallTimeout();
|
|
||||||
}
|
|
||||||
|
|
||||||
private HttpClient GetClient(TimeSpan timeout)
|
|
||||||
{
|
{
|
||||||
var client = new HttpClient();
|
var client = new HttpClient();
|
||||||
client.Timeout = timeout;
|
client.Timeout = timeSet.HttpCallTimeout();
|
||||||
return client;
|
return client;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -12,7 +12,6 @@ namespace DistTestCore
|
||||||
string GetName();
|
string GetName();
|
||||||
CodexDebugResponse GetDebugInfo();
|
CodexDebugResponse GetDebugInfo();
|
||||||
CodexDebugPeerResponse GetDebugPeer(string peerId);
|
CodexDebugPeerResponse GetDebugPeer(string peerId);
|
||||||
CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout);
|
|
||||||
ContentId UploadFile(TestFile file);
|
ContentId UploadFile(TestFile file);
|
||||||
TestFile? DownloadContent(ContentId contentId, string fileLabel = "");
|
TestFile? DownloadContent(ContentId contentId, string fileLabel = "");
|
||||||
void ConnectToPeer(IOnlineCodexNode node);
|
void ConnectToPeer(IOnlineCodexNode node);
|
||||||
|
@ -60,11 +59,6 @@ namespace DistTestCore
|
||||||
return CodexAccess.GetDebugPeer(peerId);
|
return CodexAccess.GetDebugPeer(peerId);
|
||||||
}
|
}
|
||||||
|
|
||||||
public CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout)
|
|
||||||
{
|
|
||||||
return CodexAccess.GetDebugPeer(peerId, timeout);
|
|
||||||
}
|
|
||||||
|
|
||||||
public ContentId UploadFile(TestFile file)
|
public ContentId UploadFile(TestFile file)
|
||||||
{
|
{
|
||||||
using var fileStream = File.OpenRead(file.Filename);
|
using var fileStream = File.OpenRead(file.Filename);
|
||||||
|
@ -78,9 +72,6 @@ namespace DistTestCore
|
||||||
if (string.IsNullOrEmpty(response)) Assert.Fail("Received empty response.");
|
if (string.IsNullOrEmpty(response)) Assert.Fail("Received empty response.");
|
||||||
if (response.StartsWith(UploadFailedMessage)) Assert.Fail("Node failed to store block.");
|
if (response.StartsWith(UploadFailedMessage)) Assert.Fail("Node failed to store block.");
|
||||||
|
|
||||||
var logReplacement = $"(CID:{file.Describe()})";
|
|
||||||
Log($"ContentId '{response}' is {logReplacement}");
|
|
||||||
lifecycle.Log.AddStringReplace(response, logReplacement);
|
|
||||||
Log($"Uploaded file. Received contentId: '{response}'.");
|
Log($"Uploaded file. Received contentId: '{response}'.");
|
||||||
return new ContentId(response);
|
return new ContentId(response);
|
||||||
}
|
}
|
||||||
|
|
|
@ -25,6 +25,8 @@ namespace DistTestCore
|
||||||
PrometheusStarter = new PrometheusStarter(this, workflowCreator);
|
PrometheusStarter = new PrometheusStarter(this, workflowCreator);
|
||||||
GethStarter = new GethStarter(this, workflowCreator);
|
GethStarter = new GethStarter(this, workflowCreator);
|
||||||
testStart = DateTime.UtcNow;
|
testStart = DateTime.UtcNow;
|
||||||
|
|
||||||
|
Log.WriteLogTag();
|
||||||
}
|
}
|
||||||
|
|
||||||
public BaseLog Log { get; }
|
public BaseLog Log { get; }
|
||||||
|
|
|
@ -10,7 +10,7 @@ namespace DistTestCore
|
||||||
public interface ITimeSet
|
public interface ITimeSet
|
||||||
{
|
{
|
||||||
TimeSpan HttpCallTimeout();
|
TimeSpan HttpCallTimeout();
|
||||||
TimeSpan HttpCallRetryTimeout();
|
TimeSpan HttpCallRetryTime();
|
||||||
TimeSpan HttpCallRetryDelay();
|
TimeSpan HttpCallRetryDelay();
|
||||||
TimeSpan WaitForK8sServiceDelay();
|
TimeSpan WaitForK8sServiceDelay();
|
||||||
TimeSpan K8sOperationTimeout();
|
TimeSpan K8sOperationTimeout();
|
||||||
|
@ -24,7 +24,7 @@ namespace DistTestCore
|
||||||
return TimeSpan.FromSeconds(10);
|
return TimeSpan.FromSeconds(10);
|
||||||
}
|
}
|
||||||
|
|
||||||
public TimeSpan HttpCallRetryTimeout()
|
public TimeSpan HttpCallRetryTime()
|
||||||
{
|
{
|
||||||
return TimeSpan.FromMinutes(1);
|
return TimeSpan.FromMinutes(1);
|
||||||
}
|
}
|
||||||
|
@ -57,7 +57,7 @@ namespace DistTestCore
|
||||||
return TimeSpan.FromHours(2);
|
return TimeSpan.FromHours(2);
|
||||||
}
|
}
|
||||||
|
|
||||||
public TimeSpan HttpCallRetryTimeout()
|
public TimeSpan HttpCallRetryTime()
|
||||||
{
|
{
|
||||||
return TimeSpan.FromHours(5);
|
return TimeSpan.FromHours(5);
|
||||||
}
|
}
|
||||||
|
|
|
@ -73,6 +73,14 @@ namespace Logging
|
||||||
return new LogFile($"{GetFullName()}_{GetSubfileNumber()}", ext);
|
return new LogFile($"{GetFullName()}_{GetSubfileNumber()}", ext);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public void WriteLogTag()
|
||||||
|
{
|
||||||
|
var runId = NameUtils.GetRunId();
|
||||||
|
var category = NameUtils.GetCategoryName();
|
||||||
|
var name = NameUtils.GetTestMethodName();
|
||||||
|
LogFile.WriteRaw($"{runId} {category} {name}");
|
||||||
|
}
|
||||||
|
|
||||||
private string ApplyReplacements(string str)
|
private string ApplyReplacements(string str)
|
||||||
{
|
{
|
||||||
foreach (var replacement in replacements)
|
foreach (var replacement in replacements)
|
||||||
|
|
|
@ -1,20 +1,14 @@
|
||||||
using NUnit.Framework;
|
namespace Logging
|
||||||
|
|
||||||
namespace Logging
|
|
||||||
{
|
{
|
||||||
public class FixtureLog : BaseLog
|
public class FixtureLog : BaseLog
|
||||||
{
|
{
|
||||||
private readonly DateTime start;
|
|
||||||
private readonly string fullName;
|
private readonly string fullName;
|
||||||
private readonly LogConfig config;
|
private readonly LogConfig config;
|
||||||
|
|
||||||
public FixtureLog(LogConfig config, string name = "")
|
public FixtureLog(LogConfig config, DateTime start, string name = "")
|
||||||
: base(config.DebugEnabled)
|
: base(config.DebugEnabled)
|
||||||
{
|
{
|
||||||
start = DateTime.UtcNow;
|
fullName = NameUtils.GetFixtureFullName(config, start, name);
|
||||||
var folder = DetermineFolder(config);
|
|
||||||
var fixtureName = GetFixtureName(name);
|
|
||||||
fullName = Path.Combine(folder, fixtureName);
|
|
||||||
this.config = config;
|
this.config = config;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -32,28 +26,5 @@ namespace Logging
|
||||||
{
|
{
|
||||||
return fullName;
|
return fullName;
|
||||||
}
|
}
|
||||||
|
|
||||||
private string DetermineFolder(LogConfig config)
|
|
||||||
{
|
|
||||||
return Path.Join(
|
|
||||||
config.LogRoot,
|
|
||||||
$"{start.Year}-{Pad(start.Month)}",
|
|
||||||
Pad(start.Day));
|
|
||||||
}
|
|
||||||
|
|
||||||
private string GetFixtureName(string name)
|
|
||||||
{
|
|
||||||
var test = TestContext.CurrentContext.Test;
|
|
||||||
var className = test.ClassName!.Substring(test.ClassName.LastIndexOf('.') + 1);
|
|
||||||
if (!string.IsNullOrEmpty(name)) className = name;
|
|
||||||
|
|
||||||
return $"{Pad(start.Hour)}-{Pad(start.Minute)}-{Pad(start.Second)}Z_{className.Replace('.', '-')}";
|
|
||||||
}
|
|
||||||
|
|
||||||
private static string Pad(int n)
|
|
||||||
{
|
|
||||||
return n.ToString().PadLeft(2, '0');
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -49,7 +49,7 @@
|
||||||
|
|
||||||
private static string GetTimestamp()
|
private static string GetTimestamp()
|
||||||
{
|
{
|
||||||
return $"[{DateTime.UtcNow.ToString("u")}]";
|
return $"[{DateTime.UtcNow.ToString("o")}]";
|
||||||
}
|
}
|
||||||
|
|
||||||
private void EnsurePathExists(string filename)
|
private void EnsurePathExists(string filename)
|
||||||
|
|
|
@ -0,0 +1,83 @@
|
||||||
|
using NUnit.Framework;
|
||||||
|
|
||||||
|
namespace Logging
|
||||||
|
{
|
||||||
|
public static class NameUtils
|
||||||
|
{
|
||||||
|
public static string GetTestMethodName(string name = "")
|
||||||
|
{
|
||||||
|
if (!string.IsNullOrEmpty(name)) return name;
|
||||||
|
var test = TestContext.CurrentContext.Test;
|
||||||
|
var args = FormatArguments(test);
|
||||||
|
return ReplaceInvalidCharacters($"{test.MethodName}{args}");
|
||||||
|
}
|
||||||
|
|
||||||
|
public static string GetFixtureFullName(LogConfig config, DateTime start, string name)
|
||||||
|
{
|
||||||
|
var folder = DetermineFolder(config, start);
|
||||||
|
var fixtureName = GetFixtureName(name, start);
|
||||||
|
return Path.Combine(folder, fixtureName);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static string GetRawFixtureName()
|
||||||
|
{
|
||||||
|
var test = TestContext.CurrentContext.Test;
|
||||||
|
var className = test.ClassName!.Substring(test.ClassName.LastIndexOf('.') + 1);
|
||||||
|
return className.Replace('.', '-');
|
||||||
|
}
|
||||||
|
|
||||||
|
public static string GetCategoryName()
|
||||||
|
{
|
||||||
|
var test = TestContext.CurrentContext.Test;
|
||||||
|
return test.ClassName!.Substring(0, test.ClassName.LastIndexOf('.'));
|
||||||
|
}
|
||||||
|
|
||||||
|
public static string GetTestId()
|
||||||
|
{
|
||||||
|
return GetEnvVar("TESTID");
|
||||||
|
}
|
||||||
|
|
||||||
|
public static string GetRunId()
|
||||||
|
{
|
||||||
|
return GetEnvVar("RUNID");
|
||||||
|
}
|
||||||
|
|
||||||
|
private static string GetEnvVar(string name)
|
||||||
|
{
|
||||||
|
var v = Environment.GetEnvironmentVariable(name);
|
||||||
|
if (string.IsNullOrEmpty(v)) return $"EnvVar'{name}'NotSet";
|
||||||
|
return v;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static string FormatArguments(TestContext.TestAdapter test)
|
||||||
|
{
|
||||||
|
if (test.Arguments == null || !test.Arguments.Any()) return "";
|
||||||
|
return $"[{string.Join(',', test.Arguments)}]";
|
||||||
|
}
|
||||||
|
|
||||||
|
private static string ReplaceInvalidCharacters(string name)
|
||||||
|
{
|
||||||
|
return name.Replace(":", "_");
|
||||||
|
}
|
||||||
|
|
||||||
|
private static string DetermineFolder(LogConfig config, DateTime start)
|
||||||
|
{
|
||||||
|
return Path.Join(
|
||||||
|
config.LogRoot,
|
||||||
|
$"{start.Year}-{Pad(start.Month)}",
|
||||||
|
Pad(start.Day));
|
||||||
|
}
|
||||||
|
|
||||||
|
private static string GetFixtureName(string name, DateTime start)
|
||||||
|
{
|
||||||
|
var className = GetRawFixtureName();
|
||||||
|
if (!string.IsNullOrEmpty(name)) className = name;
|
||||||
|
return $"{Pad(start.Hour)}-{Pad(start.Minute)}-{Pad(start.Second)}Z_{className.Replace('.', '-')}";
|
||||||
|
}
|
||||||
|
|
||||||
|
private static string Pad(int n)
|
||||||
|
{
|
||||||
|
return n.ToString().PadLeft(2, '0');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,63 @@
|
||||||
|
using Newtonsoft.Json;
|
||||||
|
|
||||||
|
namespace Logging
|
||||||
|
{
|
||||||
|
public class StatusLog
|
||||||
|
{
|
||||||
|
private readonly object fileLock = new object();
|
||||||
|
private readonly string fullName;
|
||||||
|
private readonly string fixtureName;
|
||||||
|
private readonly string codexId;
|
||||||
|
|
||||||
|
public StatusLog(LogConfig config, DateTime start, string codexId, string name = "")
|
||||||
|
{
|
||||||
|
fullName = NameUtils.GetFixtureFullName(config, start, name) + "_STATUS.log";
|
||||||
|
fixtureName = NameUtils.GetRawFixtureName();
|
||||||
|
this.codexId = codexId;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void ConcludeTest(string resultStatus, string testDuration)
|
||||||
|
{
|
||||||
|
Write(new StatusLogJson
|
||||||
|
{
|
||||||
|
@timestamp = DateTime.UtcNow.ToString("o"),
|
||||||
|
runid = NameUtils.GetRunId(),
|
||||||
|
status = resultStatus,
|
||||||
|
testid = NameUtils.GetTestId(),
|
||||||
|
codexid = codexId,
|
||||||
|
category = NameUtils.GetCategoryName(),
|
||||||
|
fixturename = fixtureName,
|
||||||
|
testname = NameUtils.GetTestMethodName(),
|
||||||
|
testduration = testDuration
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private void Write(StatusLogJson json)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
lock (fileLock)
|
||||||
|
{
|
||||||
|
File.AppendAllLines(fullName, new[] { JsonConvert.SerializeObject(json) });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
Console.WriteLine("Unable to write to status log: " + ex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public class StatusLogJson
|
||||||
|
{
|
||||||
|
public string @timestamp { get; set; } = string.Empty;
|
||||||
|
public string runid { get; set; } = string.Empty;
|
||||||
|
public string status { get; set; } = string.Empty;
|
||||||
|
public string testid { get; set; } = string.Empty;
|
||||||
|
public string codexid { get; set; } = string.Empty;
|
||||||
|
public string category { get; set; } = string.Empty;
|
||||||
|
public string fixturename { get; set; } = string.Empty;
|
||||||
|
public string testname { get; set; } = string.Empty;
|
||||||
|
public string testduration { get; set;} = string.Empty;
|
||||||
|
}
|
||||||
|
}
|
|
@ -10,7 +10,7 @@ namespace Logging
|
||||||
public TestLog(string folder, bool debug, string name = "")
|
public TestLog(string folder, bool debug, string name = "")
|
||||||
: base(debug)
|
: base(debug)
|
||||||
{
|
{
|
||||||
methodName = GetMethodName(name);
|
methodName = NameUtils.GetTestMethodName(name);
|
||||||
fullName = Path.Combine(folder, methodName);
|
fullName = Path.Combine(folder, methodName);
|
||||||
|
|
||||||
Log($"*** Begin: {methodName}");
|
Log($"*** Begin: {methodName}");
|
||||||
|
@ -37,24 +37,5 @@ namespace Logging
|
||||||
{
|
{
|
||||||
return fullName;
|
return fullName;
|
||||||
}
|
}
|
||||||
|
|
||||||
private string GetMethodName(string name)
|
|
||||||
{
|
|
||||||
if (!string.IsNullOrEmpty(name)) return name;
|
|
||||||
var test = TestContext.CurrentContext.Test;
|
|
||||||
var args = FormatArguments(test);
|
|
||||||
return ReplaceInvalidCharacters($"{test.MethodName}{args}");
|
|
||||||
}
|
|
||||||
|
|
||||||
private static string FormatArguments(TestContext.TestAdapter test)
|
|
||||||
{
|
|
||||||
if (test.Arguments == null || !test.Arguments.Any()) return "";
|
|
||||||
return $"[{string.Join(',', test.Arguments)}]";
|
|
||||||
}
|
|
||||||
|
|
||||||
private static string ReplaceInvalidCharacters(string name)
|
|
||||||
{
|
|
||||||
return name.Replace(":", "_");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,22 @@
|
||||||
|
using DistTestCore.Helpers;
|
||||||
|
using DistTestCore;
|
||||||
|
using NUnit.Framework;
|
||||||
|
|
||||||
|
namespace TestsLong.DownloadConnectivityTests
|
||||||
|
{
|
||||||
|
[TestFixture]
|
||||||
|
public class LongFullyConnectedDownloadTests : AutoBootstrapDistTest
|
||||||
|
{
|
||||||
|
[Test]
|
||||||
|
[UseLongTimeouts]
|
||||||
|
[Combinatorial]
|
||||||
|
public void FullyConnectedDownloadTest(
|
||||||
|
[Values(10, 15, 20)] int numberOfNodes,
|
||||||
|
[Values(10, 100)] int sizeMBs)
|
||||||
|
{
|
||||||
|
for (var i = 0; i < numberOfNodes; i++) SetupCodexNode();
|
||||||
|
|
||||||
|
PeerDownloadTestHelpers.AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes(), sizeMBs.MB());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -9,8 +9,8 @@ namespace Tests.DownloadConnectivityTests
|
||||||
[Test]
|
[Test]
|
||||||
[Combinatorial]
|
[Combinatorial]
|
||||||
public void FullyConnectedDownloadTest(
|
public void FullyConnectedDownloadTest(
|
||||||
[Values(3, 10, 20)] int numberOfNodes,
|
[Values(1, 3, 5)] int numberOfNodes,
|
||||||
[Values(1, 10, 100)] int sizeMBs)
|
[Values(1, 10)] int sizeMBs)
|
||||||
{
|
{
|
||||||
for (var i = 0; i < numberOfNodes; i++) SetupCodexNode();
|
for (var i = 0; i < numberOfNodes; i++) SetupCodexNode();
|
||||||
|
|
||||||
|
|
|
@ -28,6 +28,10 @@ spec:
|
||||||
value: ${BRANCH}
|
value: ${BRANCH}
|
||||||
- name: SOURCE
|
- name: SOURCE
|
||||||
value: ${SOURCE}
|
value: ${SOURCE}
|
||||||
|
- name: RUNID
|
||||||
|
value: ${RUNID}
|
||||||
|
- name: TESTID
|
||||||
|
value: ${TESTID}
|
||||||
volumeMounts:
|
volumeMounts:
|
||||||
- name: kubeconfig
|
- name: kubeconfig
|
||||||
mountPath: /opt/kubeconfig.yaml
|
mountPath: /opt/kubeconfig.yaml
|
||||||
|
|
Loading…
Reference in New Issue