2
0
mirror of synced 2025-02-23 05:28:17 +00:00

Restores continuous test runner

This commit is contained in:
benbierens 2023-09-20 13:33:58 +02:00
parent 48da92c737
commit 8cde69a483
No known key found for this signature in database
GPG Key ID: FE44815D96D0A1AA
20 changed files with 135 additions and 326 deletions

View File

@ -7,6 +7,7 @@ namespace Logging
void Log(string message); void Log(string message);
void Debug(string message = "", int skipFrames = 0); void Debug(string message = "", int skipFrames = 0);
void Error(string message); void Error(string message);
void AddStringReplace(string from, string to);
LogFile CreateSubfile(string ext = "log"); LogFile CreateSubfile(string ext = "log");
} }

View File

@ -30,5 +30,10 @@
{ {
backingLog.Log(prefix + message); backingLog.Log(prefix + message);
} }
public void AddStringReplace(string from, string to)
{
backingLog.AddStringReplace(from, to);
}
} }
} }

View File

@ -51,6 +51,8 @@ namespace CodexPlugin
public string UploadFile(FileStream fileStream) public string UploadFile(FileStream fileStream)
{ {
// private const string UploadFailedMessage = "Unable to store block";
return Http().HttpPostStream("upload", fileStream); return Http().HttpPostStream("upload", fileStream);
} }

View File

@ -10,6 +10,13 @@ namespace CodexPlugin
return Plugin(ci).DeployCodexNodes(number, setup); return Plugin(ci).DeployCodexNodes(number, setup);
} }
public static ICodexNodeGroup WrapCodexContainers(this CoreInterface ci, RunningContainer[] containers)
{
// ew, clean this up.
var rcs = new RunningContainers(null!, containers.First().Pod, containers);
return WrapCodexContainers(ci, new[] { rcs });
}
public static ICodexNodeGroup WrapCodexContainers(this CoreInterface ci, RunningContainers[] containers) public static ICodexNodeGroup WrapCodexContainers(this CoreInterface ci, RunningContainers[] containers)
{ {
return Plugin(ci).WrapCodexContainers(ci, containers); return Plugin(ci).WrapCodexContainers(ci, containers);

View File

@ -1,20 +0,0 @@
using DistTestCore;
using DistTestCore.Codex;
using KubernetesWorkflow;
using Logging;
namespace ContinuousTests
{
public class CodexAccessFactory
{
public CodexAccess[] Create(Configuration config, RunningContainer[] containers, BaseLog log, ITimeSet timeSet)
{
return containers.Select(container =>
{
var address = container.ClusterExternalAddress;
if (config.RunnerLocation == RunnerLocation.InternalToCluster) address = container.ClusterInternalAddress;
return new CodexAccess(log, container, timeSet, address);
}).ToArray();
}
}
}

View File

@ -12,10 +12,10 @@
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
<ProjectReference Include="..\ArgsUniform\ArgsUniform.csproj" /> <ProjectReference Include="..\..\Framework\ArgsUniform\ArgsUniform.csproj" />
<ProjectReference Include="..\..\ProjectPlugins\CodexPlugin\CodexPlugin.csproj" />
<ProjectReference Include="..\CodexTests\CodexTests.csproj" />
<ProjectReference Include="..\DistTestCore\DistTestCore.csproj" /> <ProjectReference Include="..\DistTestCore\DistTestCore.csproj" />
<ProjectReference Include="..\KubernetesWorkflow\KubernetesWorkflow.csproj" />
<ProjectReference Include="..\Logging\Logging.csproj" />
</ItemGroup> </ItemGroup>
</Project> </Project>

View File

@ -1,6 +1,5 @@
using ArgsUniform; using ArgsUniform;
using DistTestCore; using CodexPlugin;
using DistTestCore.Codex;
using Newtonsoft.Json; using Newtonsoft.Json;
namespace ContinuousTests namespace ContinuousTests
@ -29,8 +28,6 @@ namespace ContinuousTests
public bool DownloadContainerLogs { get; set; } = false; public bool DownloadContainerLogs { get; set; } = false;
public CodexDeployment CodexDeployment { get; set; } = null!; public CodexDeployment CodexDeployment { get; set; } = null!;
public RunnerLocation RunnerLocation { get; set; }
} }
public class ConfigLoader public class ConfigLoader
@ -40,10 +37,7 @@ namespace ContinuousTests
var uniformArgs = new ArgsUniform<Configuration>(PrintHelp, args); var uniformArgs = new ArgsUniform<Configuration>(PrintHelp, args);
var result = uniformArgs.Parse(true); var result = uniformArgs.Parse(true);
result.CodexDeployment = ParseCodexDeploymentJson(result.CodexDeploymentJson); result.CodexDeployment = ParseCodexDeploymentJson(result.CodexDeploymentJson);
result.RunnerLocation = RunnerLocationUtils.DetermineRunnerLocation(result.CodexDeployment.CodexContainers.First());
return result; return result;
} }

View File

@ -1,8 +1,7 @@
using DistTestCore; using CodexPlugin;
using DistTestCore.Codex; using Core;
using DistTestCore.Logs; using DistTestCore;
using FileUtils; using FileUtils;
using KubernetesWorkflow;
using Logging; using Logging;
namespace ContinuousTests namespace ContinuousTests
@ -22,9 +21,7 @@ namespace ContinuousTests
protected const int DayOne = HourOne * 24; protected const int DayOne = HourOne * 24;
protected const int DayThree = DayOne * 3; protected const int DayThree = DayOne * 3;
private const string UploadFailedMessage = "Unable to store block"; public void Initialize(ICodexNode[] nodes, ILog log, IFileManager fileManager, Configuration configuration, CancellationToken cancelToken)
public void Initialize(CodexAccess[] nodes, BaseLog log, FileManager fileManager, Configuration configuration, CancellationToken cancelToken)
{ {
Nodes = nodes; Nodes = nodes;
Log = log; Log = log;
@ -34,7 +31,7 @@ namespace ContinuousTests
if (nodes != null) if (nodes != null)
{ {
NodeRunner = new NodeRunner(Nodes, configuration, TimeSet, Log, CustomK8sNamespace, EthereumAccountIndex); NodeRunner = new NodeRunner(Nodes, configuration, Log, CustomK8sNamespace);
} }
else else
{ {
@ -42,8 +39,8 @@ namespace ContinuousTests
} }
} }
public CodexAccess[] Nodes { get; private set; } = null!; public ICodexNode[] Nodes { get; private set; } = null!;
public BaseLog Log { get; private set; } = null!; public ILog Log { get; private set; } = null!;
public IFileManager FileManager { get; private set; } = null!; public IFileManager FileManager { get; private set; } = null!;
public Configuration Configuration { get; private set; } = null!; public Configuration Configuration { get; private set; } = null!;
public virtual ITimeSet TimeSet { get { return new DefaultTimeSet(); } } public virtual ITimeSet TimeSet { get { return new DefaultTimeSet(); } }
@ -53,7 +50,6 @@ namespace ContinuousTests
public abstract int RequiredNumberOfNodes { get; } public abstract int RequiredNumberOfNodes { get; }
public abstract TimeSpan RunTestEvery { get; } public abstract TimeSpan RunTestEvery { get; }
public abstract TestFailMode TestFailMode { get; } public abstract TestFailMode TestFailMode { get; }
public virtual int EthereumAccountIndex { get { return -1; } }
public virtual string CustomK8sNamespace { get { return string.Empty; } } public virtual string CustomK8sNamespace { get { return string.Empty; } }
public string Name public string Name
@ -64,52 +60,6 @@ namespace ContinuousTests
} }
} }
public ContentId? UploadFile(CodexAccess node, TestFile file)
{
using var fileStream = File.OpenRead(file.Filename);
var logMessage = $"Uploading file {file.Describe()}...";
var response = Stopwatch.Measure(Log, logMessage, () =>
{
return node.UploadFile(fileStream);
});
if (string.IsNullOrEmpty(response)) return null;
if (response.StartsWith(UploadFailedMessage)) return null;
Log.Log($"Uploaded file. Received contentId: '{response}'.");
return new ContentId(response);
}
public TestFile DownloadFile(CodexAccess node, ContentId contentId, string fileLabel = "")
{
var logMessage = $"Downloading for contentId: '{contentId.Id}'...";
var file = FileManager.CreateEmptyTestFile(fileLabel);
Stopwatch.Measure(Log, logMessage, () => DownloadToFile(node, contentId.Id, file));
Log.Log($"Downloaded file {file.Describe()} to '{file.Filename}'.");
return file;
}
public IDownloadedLog DownloadContainerLog(RunningContainer container, int? tailLines = null)
{
var nodeRunner = new NodeRunner(Nodes, Configuration, TimeSet, Log, Configuration.CodexDeployment.Metadata.KubeNamespace, EthereumAccountIndex);
return nodeRunner.DownloadLog(container, tailLines);
}
private void DownloadToFile(CodexAccess node, string contentId, TestFile file)
{
using var fileStream = File.OpenWrite(file.Filename);
try
{
using var downloadStream = node.DownloadFile(contentId);
downloadStream.CopyTo(fileStream);
}
catch
{
Log.Log($"Failed to download file '{contentId}'.");
throw;
}
}
} }
public enum TestFailMode public enum TestFailMode

View File

@ -1,36 +1,39 @@
using DistTestCore; using DistTestCore.Logs;
using Logging; using Logging;
namespace ContinuousTests namespace ContinuousTests
{ {
public class ContinuousTestRunner public class ContinuousTestRunner
{ {
private readonly K8sFactory k8SFactory = new K8sFactory(); private readonly EntryPointFactory entryPointFactory = new EntryPointFactory();
private readonly ConfigLoader configLoader = new ConfigLoader(); private readonly ConfigLoader configLoader = new ConfigLoader();
private readonly TestFactory testFactory = new TestFactory(); private readonly TestFactory testFactory = new TestFactory();
private readonly Configuration config; private readonly Configuration config;
private readonly StartupChecker startupChecker;
private readonly CancellationToken cancelToken; private readonly CancellationToken cancelToken;
public ContinuousTestRunner(string[] args, CancellationToken cancelToken) public ContinuousTestRunner(string[] args, CancellationToken cancelToken)
{ {
config = configLoader.Load(args); config = configLoader.Load(args);
startupChecker = new StartupChecker(config, cancelToken);
this.cancelToken = cancelToken; this.cancelToken = cancelToken;
} }
public void Run() public void Run()
{ {
var overviewLog = new FixtureLog(new LogConfig(config.LogPath, false), DateTime.UtcNow, "Overview");
var entryPoint = entryPointFactory.CreateEntryPoint(config.KubeConfigFile, config.DataPath, config.CodexDeployment.Metadata.KubeNamespace, overviewLog);
entryPoint.Announce();
var startupChecker = new StartupChecker(entryPoint, config, cancelToken);
startupChecker.Check(); startupChecker.Check();
var taskFactory = new TaskFactory(); var taskFactory = new TaskFactory();
var overviewLog = new FixtureLog(new LogConfig(config.LogPath, false), DateTime.UtcNow, "Overview");
overviewLog.Log("Continuous tests starting..."); overviewLog.Log("Continuous tests starting...");
var allTests = testFactory.CreateTests(); var allTests = testFactory.CreateTests();
ClearAllCustomNamespaces(allTests, overviewLog); ClearAllCustomNamespaces(allTests, overviewLog);
var testLoops = allTests.Select(t => new TestLoop(taskFactory, config, overviewLog, t.GetType(), t.RunTestEvery, startupChecker, cancelToken)).ToArray(); var testLoops = allTests.Select(t => new TestLoop(entryPoint, taskFactory, config, overviewLog, t.GetType(), t.RunTestEvery, startupChecker, cancelToken)).ToArray();
foreach (var testLoop in testLoops) foreach (var testLoop in testLoops)
{ {
@ -58,8 +61,9 @@ namespace ContinuousTests
if (string.IsNullOrEmpty(test.CustomK8sNamespace)) return; if (string.IsNullOrEmpty(test.CustomK8sNamespace)) return;
log.Log($"Clearing namespace '{test.CustomK8sNamespace}'..."); log.Log($"Clearing namespace '{test.CustomK8sNamespace}'...");
var lifecycle = k8SFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, config.DataPath, test.CustomK8sNamespace, new DefaultTimeSet(), log);
lifecycle.WorkflowCreator.CreateWorkflow().DeleteNamespacesStartingWith(); var entryPoint = entryPointFactory.CreateEntryPoint(config.KubeConfigFile, config.DataPath, test.CustomK8sNamespace, log);
entryPoint.Tools.CreateWorkflow().DeleteNamespacesStartingWith(test.CustomK8sNamespace);
} }
} }
} }

View File

@ -0,0 +1,29 @@
using Logging;
using Core;
namespace ContinuousTests
{
public class EntryPointFactory
{
public EntryPoint CreateEntryPoint(string kubeConfigFile, string dataFilePath, string customNamespace, ILog log)
{
var kubeConfig = GetKubeConfig(kubeConfigFile);
var lifecycleConfig = new KubernetesWorkflow.Configuration
(
kubeConfigFile: kubeConfig,
operationTimeout: TimeSpan.FromSeconds(30),
retryDelay: TimeSpan.FromSeconds(10),
kubernetesNamespace: customNamespace
);
return new EntryPoint(log, lifecycleConfig, dataFilePath);
//DefaultContainerRecipe.TestsType = "continuous-tests";
}
private static string? GetKubeConfig(string kubeConfigFile)
{
if (string.IsNullOrEmpty(kubeConfigFile) || kubeConfigFile.ToLowerInvariant() == "null") return null;
return kubeConfigFile;
}
}
}

View File

@ -1,34 +0,0 @@
using DistTestCore.Codex;
using DistTestCore;
using Logging;
namespace ContinuousTests
{
public class K8sFactory
{
public TestLifecycle CreateTestLifecycle(string kubeConfigFile, string logPath, string dataFilePath, string customNamespace, ITimeSet timeSet, BaseLog log)
{
var kubeConfig = GetKubeConfig(kubeConfigFile);
var lifecycleConfig = new DistTestCore.Configuration
(
kubeConfigFile: kubeConfig,
logPath: logPath,
logDebug: false,
dataFilesPath: dataFilePath,
codexLogLevel: CodexLogLevel.Debug,
k8sNamespacePrefix: customNamespace
);
var lifecycle = new TestLifecycle(log, lifecycleConfig, timeSet, string.Empty);
DefaultContainerRecipe.TestsType = "continuous-tests";
DefaultContainerRecipe.ApplicationIds = lifecycle.GetApplicationIds();
return lifecycle;
}
private static string? GetKubeConfig(string kubeConfigFile)
{
if (string.IsNullOrEmpty(kubeConfigFile) || kubeConfigFile.ToLowerInvariant() == "null") return null;
return kubeConfigFile;
}
}
}

View File

@ -1,113 +1,73 @@
using DistTestCore.Codex; using KubernetesWorkflow;
using DistTestCore.Marketplace;
using DistTestCore;
using KubernetesWorkflow;
using NUnit.Framework; using NUnit.Framework;
using Logging; using Logging;
using Utils; using Utils;
using DistTestCore.Logs; using Core;
using CodexPlugin;
namespace ContinuousTests namespace ContinuousTests
{ {
public class NodeRunner public class NodeRunner
{ {
private readonly K8sFactory k8SFactory = new K8sFactory(); private readonly EntryPointFactory entryPointFactory = new EntryPointFactory();
private readonly CodexAccess[] nodes; private readonly ICodexNode[] nodes;
private readonly Configuration config; private readonly Configuration config;
private readonly ITimeSet timeSet; private readonly ILog log;
private readonly BaseLog log;
private readonly string customNamespace; private readonly string customNamespace;
private readonly int ethereumAccountIndex;
public NodeRunner(CodexAccess[] nodes, Configuration config, ITimeSet timeSet, BaseLog log, string customNamespace, int ethereumAccountIndex) public NodeRunner(ICodexNode[] nodes, Configuration config, ILog log, string customNamespace)
{ {
this.nodes = nodes; this.nodes = nodes;
this.config = config; this.config = config;
this.timeSet = timeSet;
this.log = log; this.log = log;
this.customNamespace = customNamespace; this.customNamespace = customNamespace;
this.ethereumAccountIndex = ethereumAccountIndex;
}
public void RunNode(Action<CodexAccess, MarketplaceAccess, TestLifecycle> operation)
{
RunNode(nodes.ToList().PickOneRandom(), operation, 0.TestTokens());
}
public void RunNode(CodexAccess bootstrapNode, Action<CodexAccess, MarketplaceAccess, TestLifecycle> operation)
{
RunNode(bootstrapNode, operation, 0.TestTokens());
} }
public IDownloadedLog DownloadLog(RunningContainer container, int? tailLines = null) public IDownloadedLog DownloadLog(RunningContainer container, int? tailLines = null)
{ {
var subFile = log.CreateSubfile(); var entryPoint = CreateEntryPoint();
var description = container.Name; return entryPoint.CreateInterface().DownloadLog(container, tailLines);
var handler = new LogDownloadHandler(container, description, subFile);
log.Log($"Downloading logs for {description} to file '{subFile.FullFilename}'");
var lifecycle = CreateTestLifecycle();
var flow = lifecycle.WorkflowCreator.CreateWorkflow();
flow.DownloadContainerLog(container, handler, tailLines);
return new DownloadedLog(subFile, description);
} }
public void RunNode(CodexAccess bootstrapNode, Action<CodexAccess, MarketplaceAccess, TestLifecycle> operation, TestToken mintTestTokens) public void RunNode(Action<ICodexSetup> setup, Action<ICodexNode> operation)
{ {
var lifecycle = CreateTestLifecycle(); RunNode(nodes.ToList().PickOneRandom(), setup, operation);
var flow = lifecycle.WorkflowCreator.CreateWorkflow(); }
public void RunNode(ICodexNode bootstrapNode, Action<ICodexSetup> setup, Action<ICodexNode> operation)
{
var entryPoint = CreateEntryPoint();
try try
{ {
var debugInfo = bootstrapNode.GetDebugInfo(); var debugInfo = bootstrapNode.GetDebugInfo();
Assert.That(!string.IsNullOrEmpty(debugInfo.spr)); Assert.That(!string.IsNullOrEmpty(debugInfo.spr));
var startupConfig = new StartupConfig(); var node = entryPoint.CreateInterface().StartCodexNode(s =>
startupConfig.NameOverride = "TransientNode";
var codexStartConfig = new CodexStartupConfig(CodexLogLevel.Trace);
codexStartConfig.MarketplaceConfig = new MarketplaceInitialConfig(0.Eth(), 0.TestTokens(), false);
codexStartConfig.MarketplaceConfig.AccountIndexOverride = ethereumAccountIndex;
codexStartConfig.BootstrapSpr = debugInfo.spr;
startupConfig.Add(codexStartConfig);
startupConfig.Add(config.CodexDeployment.GethStartResult);
var rc = flow.Start(1, Location.Unspecified, new CodexContainerRecipe(), startupConfig);
var account = config.CodexDeployment.GethStartResult.CompanionNode.Accounts[ethereumAccountIndex];
var marketplaceNetwork = config.CodexDeployment.GethStartResult.MarketplaceNetwork;
if (mintTestTokens.Amount > 0)
{ {
var tokenAddress = marketplaceNetwork.Marketplace.TokenAddress; setup(s);
var interaction = marketplaceNetwork.Bootstrap.StartInteraction(lifecycle); s.WithBootstrapNode(bootstrapNode);
interaction.MintTestTokens(new[] { account.Account }, mintTestTokens.Amount, tokenAddress); });
}
var container = rc.Containers[0];
var address = lifecycle.Configuration.GetAddress(container);
var codexAccess = new CodexAccess(log, container, lifecycle.TimeSet, address);
var marketAccess = new MarketplaceAccess(lifecycle, marketplaceNetwork, account, codexAccess);
try try
{ {
operation(codexAccess, marketAccess, lifecycle); operation(node);
} }
catch catch
{ {
lifecycle.DownloadLog(container); DownloadLog(node.Container);
throw; throw;
} }
} }
finally finally
{ {
flow.DeleteNamespacesStartingWith(); entryPoint.Tools.CreateWorkflow().DeleteNamespace();
} }
} }
private TestLifecycle CreateTestLifecycle() private EntryPoint CreateEntryPoint()
{ {
return k8SFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, config.DataPath, customNamespace, timeSet, log); return entryPointFactory.CreateEntryPoint(config.KubeConfigFile, config.DataPath, customNamespace, log);
} }
} }
} }

View File

@ -1,33 +1,34 @@
using DistTestCore.Codex; using Logging;
using DistTestCore;
using Logging;
using Utils; using Utils;
using KubernetesWorkflow; using KubernetesWorkflow;
using NUnit.Framework.Internal; using NUnit.Framework.Internal;
using System.Reflection; using System.Reflection;
using static Program; using static Program;
using FileUtils; using FileUtils;
using CodexPlugin;
using DistTestCore.Logs;
using Core;
namespace ContinuousTests namespace ContinuousTests
{ {
public class SingleTestRun public class SingleTestRun
{ {
private readonly CodexAccessFactory codexNodeFactory = new CodexAccessFactory();
private readonly List<Exception> exceptions = new List<Exception>(); private readonly List<Exception> exceptions = new List<Exception>();
private readonly EntryPoint entryPoint;
private readonly TaskFactory taskFactory; private readonly TaskFactory taskFactory;
private readonly Configuration config; private readonly Configuration config;
private readonly BaseLog overviewLog; private readonly BaseLog overviewLog;
private readonly TestHandle handle; private readonly TestHandle handle;
private readonly CancellationToken cancelToken; private readonly CancellationToken cancelToken;
private readonly CodexAccess[] nodes; private readonly ICodexNode[] nodes;
private readonly FileManager fileManager;
private readonly FixtureLog fixtureLog; private readonly FixtureLog fixtureLog;
private readonly string testName; private readonly string testName;
private readonly string dataFolder; private readonly string dataFolder;
private static int failureCount = 0; private static int failureCount = 0;
public SingleTestRun(TaskFactory taskFactory, Configuration config, BaseLog overviewLog, TestHandle handle, StartupChecker startupChecker, CancellationToken cancelToken) public SingleTestRun(EntryPoint entryPoint, TaskFactory taskFactory, Configuration config, BaseLog overviewLog, TestHandle handle, StartupChecker startupChecker, CancellationToken cancelToken)
{ {
this.entryPoint = entryPoint;
this.taskFactory = taskFactory; this.taskFactory = taskFactory;
this.config = config; this.config = config;
this.overviewLog = overviewLog; this.overviewLog = overviewLog;
@ -39,7 +40,6 @@ namespace ContinuousTests
nodes = CreateRandomNodes(); nodes = CreateRandomNodes();
dataFolder = config.DataPath + "-" + Guid.NewGuid(); dataFolder = config.DataPath + "-" + Guid.NewGuid();
fileManager = new FileManager(fixtureLog, CreateFileManagerConfiguration().GetFileManagerFolder());
} }
public void Run(EventWaitHandle runFinishedHandle) public void Run(EventWaitHandle runFinishedHandle)
@ -49,7 +49,7 @@ namespace ContinuousTests
try try
{ {
RunTest(); RunTest();
fileManager.DeleteAllTestFiles(); entryPoint.Tools.GetFileManager().DeleteAllFiles();
Directory.Delete(dataFolder, true); Directory.Delete(dataFolder, true);
runFinishedHandle.Set(); runFinishedHandle.Set();
} }
@ -142,14 +142,14 @@ namespace ContinuousTests
private void DownloadClusterLogs() private void DownloadClusterLogs()
{ {
var k8sFactory = new K8sFactory(); var entryPointFactory = new EntryPointFactory();
var log = new NullLog(); var log = new NullLog();
log.FullFilename = Path.Combine(config.LogPath, "NODE"); log.FullFilename = Path.Combine(config.LogPath, "NODE");
var lifecycle = k8sFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), log); var entryPoint = entryPointFactory.CreateEntryPoint(config.KubeConfigFile, config.DataPath, config.CodexDeployment.Metadata.KubeNamespace, log);
foreach (var container in config.CodexDeployment.CodexContainers) foreach (var container in config.CodexDeployment.CodexContainers)
{ {
lifecycle.DownloadLog(container); entryPoint.CreateInterface().DownloadLog(container);
} }
} }
@ -197,7 +197,7 @@ namespace ContinuousTests
private void InitializeTest(string name) private void InitializeTest(string name)
{ {
Log($" > Running TestMoment '{name}'"); Log($" > Running TestMoment '{name}'");
handle.Test.Initialize(nodes, fixtureLog, fileManager, config, cancelToken); handle.Test.Initialize(nodes, fixtureLog, entryPoint.Tools.GetFileManager(), config, cancelToken);
} }
private void DecommissionTest() private void DecommissionTest()
@ -223,11 +223,11 @@ namespace ContinuousTests
return $"({string.Join(",", nodes.Select(n => n.Container.Name))})"; return $"({string.Join(",", nodes.Select(n => n.Container.Name))})";
} }
private CodexAccess[] CreateRandomNodes() private ICodexNode[] CreateRandomNodes()
{ {
var containers = SelectRandomContainers(); var containers = SelectRandomContainers();
fixtureLog.Log("Selected nodes: " + string.Join(",", containers.Select(c => c.Name))); fixtureLog.Log("Selected nodes: " + string.Join(",", containers.Select(c => c.Name)));
return codexNodeFactory.Create(config, containers, fixtureLog, handle.Test.TimeSet); return entryPoint.CreateInterface().WrapCodexContainers(containers).ToArray();
} }
private RunningContainer[] SelectRandomContainers() private RunningContainer[] SelectRandomContainers()
@ -243,11 +243,5 @@ namespace ContinuousTests
} }
return result; return result;
} }
private DistTestCore.Configuration CreateFileManagerConfiguration()
{
return new DistTestCore.Configuration(null, string.Empty, false, dataFolder,
CodexLogLevel.Error, string.Empty);
}
} }
} }

View File

@ -1,5 +1,6 @@
using DistTestCore.Codex; using CodexPlugin;
using DistTestCore; using Core;
using DistTestCore.Logs;
using Logging; using Logging;
namespace ContinuousTests namespace ContinuousTests
@ -7,12 +8,13 @@ namespace ContinuousTests
public class StartupChecker public class StartupChecker
{ {
private readonly TestFactory testFactory = new TestFactory(); private readonly TestFactory testFactory = new TestFactory();
private readonly CodexAccessFactory codexNodeFactory = new CodexAccessFactory(); private readonly EntryPoint entryPoint;
private readonly Configuration config; private readonly Configuration config;
private readonly CancellationToken cancelToken; private readonly CancellationToken cancelToken;
public StartupChecker(Configuration config, CancellationToken cancelToken) public StartupChecker(EntryPoint entryPoint, Configuration config, CancellationToken cancelToken)
{ {
this.entryPoint = entryPoint;
this.config = config; this.config = config;
this.cancelToken = cancelToken; this.cancelToken = cancelToken;
LogReplacements = new List<BaseLogStringReplacement>(); LogReplacements = new List<BaseLogStringReplacement>();
@ -61,13 +63,13 @@ namespace ContinuousTests
private void CheckCodexNodes(BaseLog log, Configuration config) private void CheckCodexNodes(BaseLog log, Configuration config)
{ {
var nodes = codexNodeFactory.Create(config, config.CodexDeployment.CodexContainers, log, new DefaultTimeSet()); var nodes = entryPoint.CreateInterface().WrapCodexContainers(config.CodexDeployment.CodexContainers);
var pass = true; var pass = true;
foreach (var n in nodes) foreach (var n in nodes)
{ {
cancelToken.ThrowIfCancellationRequested(); cancelToken.ThrowIfCancellationRequested();
log.Log($"Checking {n.Container.Name} @ '{n.Address.Host}:{n.Address.Port}'..."); log.Log($"Checking {n.Container.Name} @ '{n.Container.Address.Host}:{n.Container.Address.Port}'...");
if (EnsureOnline(log, n)) if (EnsureOnline(log, n))
{ {
@ -75,7 +77,7 @@ namespace ContinuousTests
} }
else else
{ {
log.Error($"No response from '{n.Address.Host}'."); log.Error($"No response from '{n.Container.Address.Host}'.");
pass = false; pass = false;
} }
} }
@ -85,7 +87,7 @@ namespace ContinuousTests
} }
} }
private bool EnsureOnline(BaseLog log, CodexAccess n) private bool EnsureOnline(BaseLog log, ICodexNode n)
{ {
try try
{ {
@ -107,30 +109,9 @@ namespace ContinuousTests
var errors = new List<string>(); var errors = new List<string>();
CheckRequiredNumberOfNodes(tests, errors); CheckRequiredNumberOfNodes(tests, errors);
CheckCustomNamespaceClashes(tests, errors); CheckCustomNamespaceClashes(tests, errors);
CheckEthereumIndexClashes(tests, errors);
return errors; return errors;
} }
private void CheckEthereumIndexClashes(ContinuousTest[] tests, List<string> errors)
{
var offLimits = config.CodexDeployment.CodexContainers.Length;
foreach (var test in tests)
{
if (test.EthereumAccountIndex != -1)
{
if (test.EthereumAccountIndex <= offLimits)
{
errors.Add($"Test '{test.Name}' has selected 'EthereumAccountIndex' = {test.EthereumAccountIndex}. All accounts up to and including {offLimits} are being used by the targetted Codex net. Select a different 'EthereumAccountIndex'.");
}
}
}
DuplicatesCheck(tests, errors,
considerCondition: t => t.EthereumAccountIndex != -1,
getValue: t => t.EthereumAccountIndex,
propertyName: nameof(ContinuousTest.EthereumAccountIndex));
}
private void CheckCustomNamespaceClashes(ContinuousTest[] tests, List<string> errors) private void CheckCustomNamespaceClashes(ContinuousTest[] tests, List<string> errors)
{ {
DuplicatesCheck(tests, errors, DuplicatesCheck(tests, errors,

View File

@ -1,9 +1,11 @@
using Logging; using Core;
using Logging;
namespace ContinuousTests namespace ContinuousTests
{ {
public class TestLoop public class TestLoop
{ {
private readonly EntryPoint entryPoint;
private readonly TaskFactory taskFactory; private readonly TaskFactory taskFactory;
private readonly Configuration config; private readonly Configuration config;
private readonly BaseLog overviewLog; private readonly BaseLog overviewLog;
@ -13,8 +15,9 @@ namespace ContinuousTests
private readonly CancellationToken cancelToken; private readonly CancellationToken cancelToken;
private readonly EventWaitHandle runFinishedHandle = new EventWaitHandle(true, EventResetMode.ManualReset); private readonly EventWaitHandle runFinishedHandle = new EventWaitHandle(true, EventResetMode.ManualReset);
public TestLoop(TaskFactory taskFactory, Configuration config, BaseLog overviewLog, Type testType, TimeSpan runsEvery, StartupChecker startupChecker, CancellationToken cancelToken) public TestLoop(Core.EntryPoint entryPoint, TaskFactory taskFactory, Configuration config, BaseLog overviewLog, Type testType, TimeSpan runsEvery, StartupChecker startupChecker, CancellationToken cancelToken)
{ {
this.entryPoint = entryPoint;
this.taskFactory = taskFactory; this.taskFactory = taskFactory;
this.config = config; this.config = config;
this.overviewLog = overviewLog; this.overviewLog = overviewLog;
@ -60,7 +63,7 @@ namespace ContinuousTests
{ {
var test = (ContinuousTest)Activator.CreateInstance(testType)!; var test = (ContinuousTest)Activator.CreateInstance(testType)!;
var handle = new TestHandle(test); var handle = new TestHandle(test);
var run = new SingleTestRun(taskFactory, config, overviewLog, handle, startupChecker, cancelToken); var run = new SingleTestRun(entryPoint, taskFactory, config, overviewLog, handle, startupChecker, cancelToken);
runFinishedHandle.Reset(); runFinishedHandle.Reset();
run.Run(runFinishedHandle); run.Run(runFinishedHandle);

View File

@ -1,4 +1,4 @@
using DistTestCore; using CodexPlugin;
using FileUtils; using FileUtils;
using NUnit.Framework; using NUnit.Framework;
using Utils; using Utils;
@ -12,19 +12,19 @@ namespace ContinuousTests.Tests
public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure; public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure;
private ContentId? cid; private ContentId? cid;
private TestFile file = null!; private TrackedFile file = null!;
[TestMoment(t: Zero)] [TestMoment(t: Zero)]
public void UploadTestFile() public void UploadTestFile()
{ {
var filesize = 80.MB(); var filesize = 80.MB();
file = FileManager.GenerateTestFile(filesize); file = FileManager.GenerateFile(filesize);
cid = UploadFile(Nodes[0], file); cid = Nodes[0].UploadFile(file);
Assert.That(cid, Is.Not.Null); Assert.That(cid, Is.Not.Null);
var dl = DownloadFile(Nodes[0], cid!); var dl = Nodes[0].DownloadContent(cid);
file.AssertIsEqual(dl); file.AssertIsEqual(dl);
} }
} }

View File

@ -1,4 +1,4 @@
using DistTestCore.Codex; using CodexPlugin;
using DistTestCore.Helpers; using DistTestCore.Helpers;
using NUnit.Framework; using NUnit.Framework;
@ -37,7 +37,7 @@ namespace ContinuousTests.Tests
} }
} }
private string AreAllPresent(CodexAccess n, string[] allIds) private string AreAllPresent(ICodexNode n, string[] allIds)
{ {
var info = n.GetDebugInfo(); var info = n.GetDebugInfo();
var known = info.table.nodes.Select(n => n.nodeId).ToArray(); var known = info.table.nodes.Select(n => n.nodeId).ToArray();

View File

@ -1,60 +0,0 @@
using DistTestCore;
using DistTestCore.Codex;
using NUnit.Framework;
namespace ContinuousTests.Tests
{
public class ThresholdChecks : ContinuousTest
{
public override int RequiredNumberOfNodes => 1;
public override TimeSpan RunTestEvery => TimeSpan.FromSeconds(30);
public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure;
private static readonly List<string> previousBreaches = new List<string>();
[TestMoment(t: 0)]
public void CheckAllThresholds()
{
var allNodes = CreateAccessToAllNodes();
foreach (var n in allNodes) CheckThresholds(n);
}
private void CheckThresholds(CodexAccess n)
{
var breaches = n.GetDebugThresholdBreaches();
if (breaches.breaches.Any())
{
var newBreaches = new List<string>();
foreach (var b in breaches.breaches)
{
if (!previousBreaches.Contains(b))
{
newBreaches.Add(b);
previousBreaches.Add(b);
}
}
if (newBreaches.Any())
{
Assert.Fail(string.Join(",", newBreaches.Select(b => FormatBreach(n, b))));
Program.Cancellation.Cts.Cancel();
}
}
}
private string FormatBreach(CodexAccess n, string breach)
{
return $"{n.Container.Name} = '{breach}'";
}
private CodexAccess[] CreateAccessToAllNodes()
{
// Normally, a continuous test accesses only a subset of the nodes in the deployment.
// This time, we want to check all of them.
var factory = new CodexAccessFactory();
var allContainers = Configuration.CodexDeployment.CodexContainers;
return factory.Create(Configuration, allContainers, Log, new DefaultTimeSet());
}
}
}

View File

@ -1,4 +1,4 @@
using DistTestCore; using CodexPlugin;
using FileUtils; using FileUtils;
using NUnit.Framework; using NUnit.Framework;
using Utils; using Utils;
@ -12,21 +12,21 @@ namespace ContinuousTests.Tests
public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure; public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure;
private ContentId? cid; private ContentId? cid;
private TestFile file = null!; private TrackedFile file = null!;
[TestMoment(t: Zero)] [TestMoment(t: Zero)]
public void UploadTestFile() public void UploadTestFile()
{ {
file = FileManager.GenerateTestFile(80.MB()); file = FileManager.GenerateFile(80.MB());
cid = UploadFile(Nodes[0], file); cid = Nodes[0].UploadFile(file);
Assert.That(cid, Is.Not.Null); Assert.That(cid, Is.Not.Null);
} }
[TestMoment(t: 10)] [TestMoment(t: 10)]
public void DownloadTestFile() public void DownloadTestFile()
{ {
var dl = DownloadFile(Nodes[1], cid!); var dl = Nodes[1].DownloadContent(cid!);
file.AssertIsEqual(dl); file.AssertIsEqual(dl);
} }

View File

@ -43,8 +43,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DistTestCore", "Tests\DistT
EndProject EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CodexNetDeployer", "Tools\CodexNetDeployer\CodexNetDeployer.csproj", "{3417D508-E2F4-4974-8988-BB124046D9E2}" Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CodexNetDeployer", "Tools\CodexNetDeployer\CodexNetDeployer.csproj", "{3417D508-E2F4-4974-8988-BB124046D9E2}"
EndProject EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CodexNetDownloader", "Tools\CodexNetDownloader\CodexNetDownloader.csproj", "{8BB4E60B-2381-436C-BDA9-72D2A31F8DFA}"
EndProject
Global Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU Debug|Any CPU = Debug|Any CPU
@ -115,10 +113,6 @@ Global
{3417D508-E2F4-4974-8988-BB124046D9E2}.Debug|Any CPU.Build.0 = Debug|Any CPU {3417D508-E2F4-4974-8988-BB124046D9E2}.Debug|Any CPU.Build.0 = Debug|Any CPU
{3417D508-E2F4-4974-8988-BB124046D9E2}.Release|Any CPU.ActiveCfg = Release|Any CPU {3417D508-E2F4-4974-8988-BB124046D9E2}.Release|Any CPU.ActiveCfg = Release|Any CPU
{3417D508-E2F4-4974-8988-BB124046D9E2}.Release|Any CPU.Build.0 = Release|Any CPU {3417D508-E2F4-4974-8988-BB124046D9E2}.Release|Any CPU.Build.0 = Release|Any CPU
{8BB4E60B-2381-436C-BDA9-72D2A31F8DFA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{8BB4E60B-2381-436C-BDA9-72D2A31F8DFA}.Debug|Any CPU.Build.0 = Debug|Any CPU
{8BB4E60B-2381-436C-BDA9-72D2A31F8DFA}.Release|Any CPU.ActiveCfg = Release|Any CPU
{8BB4E60B-2381-436C-BDA9-72D2A31F8DFA}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection EndGlobalSection
GlobalSection(SolutionProperties) = preSolution GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE HideSolutionNode = FALSE
@ -140,7 +134,6 @@ Global
{562EC700-6984-4C9A-83BF-3BF4E3EB1A64} = {88C2A621-8A98-4D07-8625-7900FC8EF89E} {562EC700-6984-4C9A-83BF-3BF4E3EB1A64} = {88C2A621-8A98-4D07-8625-7900FC8EF89E}
{E849B7BA-FDCC-4CFF-998F-845ED2F1BF40} = {88C2A621-8A98-4D07-8625-7900FC8EF89E} {E849B7BA-FDCC-4CFF-998F-845ED2F1BF40} = {88C2A621-8A98-4D07-8625-7900FC8EF89E}
{3417D508-E2F4-4974-8988-BB124046D9E2} = {7591C5B3-D86E-4AE4-8ED2-B272D17FE7E3} {3417D508-E2F4-4974-8988-BB124046D9E2} = {7591C5B3-D86E-4AE4-8ED2-B272D17FE7E3}
{8BB4E60B-2381-436C-BDA9-72D2A31F8DFA} = {7591C5B3-D86E-4AE4-8ED2-B272D17FE7E3}
EndGlobalSection EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {237BF0AA-9EC4-4659-AD9A-65DEB974250C} SolutionGuid = {237BF0AA-9EC4-4659-AD9A-65DEB974250C}