Merge branch 'plugin-refactor'
This commit is contained in:
benbierens
commit
e1a529175a
|
|
@ -41,7 +41,7 @@ env:
|
|||
SOURCE: ${{ format('{0}/{1}', github.server_url, github.repository) }}
|
||||
NAMEPREFIX: codex-dist-tests
|
||||
NAMESPACE: default
|
||||
COMMAND: dotnet test Tests
|
||||
COMMAND: dotnet test Tests/CodexTests
|
||||
JOB_MANIFEST: docker/job.yaml
|
||||
KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
|
||||
KUBE_VERSION: v1.26.1
|
||||
|
|
|
|||
|
|
@ -0,0 +1,86 @@
|
|||
# Distributed System Tests for Nim-Codex
|
||||
|
||||
## Contributing plugins
|
||||
The testing framework was created for testing Codex. However, it's been designed such that other distributed/containerized projects can 'easily' be added. In order to add your project to the framework you must:
|
||||
1. Create a library assembly in the project plugins folder.
|
||||
1. It must contain a type that implements the `IProjectPlugin` interface from the `Core` assembly.
|
||||
1. If your plugin wants to expose any specific methods or objects to the code using the framework (the tests and tools), it must implement extensions for the `CoreInterface` type.
|
||||
|
||||
## Constructors & Tools
|
||||
Your implementation of `IProjectPlugin` must have a public constructor with a single argument of type `IPluginTools`, for example:
|
||||
```C#
|
||||
public class MyPlugin : IProjectPlugin
|
||||
{
|
||||
public MyPlugin(IPluginTools tools)
|
||||
{
|
||||
...
|
||||
}
|
||||
|
||||
...
|
||||
}
|
||||
```
|
||||
|
||||
`IPluginTools` provides your plugin access to all framework functionality, such as logging, tracked file management, container lifecycle management, and a means to create HTTP clients for containers. (Without having to figure out addresses manually.)
|
||||
|
||||
## Plugin Interfaces
|
||||
The `IProjectPlugin` interface requires the implementation of two methods.
|
||||
1. `Announce` - It is considered polite to use the logging functionality provided by the `IPluginTools` to announce that your plugin has been loaded. You may also want to log some manner of version information at this time if applicable.
|
||||
1. `Decommission` - Should your plugin have any active system resources, free them in this method.
|
||||
|
||||
There are a few optional interfaces your plugin may choose to implement. The framework will automatically use these interfaces.
|
||||
1. `IHasLogPrefix` - Implementing this interface allows you to provide a string with will be prepended to all log statements made by your plugin.
|
||||
1. `IHasMetadata` - This allows you to provide metadata in the form of key/value pairs. This metadata can be accessed by code that uses your plugin.
|
||||
|
||||
## Core Interface
|
||||
Any functionality your plugin wants to expose to code which uses the framework will have to be added on to the `CoreInterface` type. You can accomplish this by using C# extension methods. The framework provides a `GetPlugin` method to access your plugin instance from the `CoreInterface` type:
|
||||
```C#
|
||||
public static class CoreInterfaceExtensions
|
||||
{
|
||||
public static MyPluginReturnType DoSomethingCool(this CoreInterface ci, string someArgument)
|
||||
{
|
||||
return Plugin(ci).SomethingCool(someArgument);
|
||||
}
|
||||
|
||||
private static MyPlugin Plugin(CoreInterface ci)
|
||||
{
|
||||
return ci.GetPlugin<MyPlugin>();
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
While technically you can build whatever you like on top of the `CoreInterface` and your own plugin types, I recommend that you follow the approach explained below.
|
||||
|
||||
## Deploying, Wrapping, and Starting
|
||||
When building a plugin, it is important to make as few assumptions as possible about how it will be used by whoever is going to use the framework. For this reason, I recommend you expose three kinds of methods using your `CoreInterface` extensions:
|
||||
1. Deploy - This kind of method should deploy your project, creating and configuring containers as needed and returning containers as a result. If your project requires additional information, you can create a new class type to contain both it and the containers created.
|
||||
1. Wrap - This kind of method should, when given the previously mentioned container information, create some kind of convenient accessor or interactor object. This object should abstract away for example details of a REST API of your project, allowing users of your plugin to write their code using a set of methods and types that nicely model your project's domain.
|
||||
1. Start - This kind of method does both, simply calling a Deploy method first, then a Wrap method, and returns the result.
|
||||
|
||||
Here's an example:
|
||||
```C#
|
||||
public static class CoreInterfaceExtensions
|
||||
{
|
||||
public static RunningContainers DeployMyProject(this CoreInterface ci, string someArgument)
|
||||
{
|
||||
// `RunningContainers` is a framework type. It contains all necessary information about a deployed container. It is serializable.
|
||||
// Should you need to return any additional information, create a new type that contains it as well as the container information. Make sure it is serializable.
|
||||
return Plugin(ci).DeployMyProjectContainer(someArgument); // <-- This method should use the `PluginTools.CreateWorkflow()` tool to deploy a container with a configuration that matches someArguments.
|
||||
}
|
||||
|
||||
public static IMyProjectNode WrapMyProjectContainer(this CoreInterface ci, RunningContainers container)
|
||||
{
|
||||
return Plugin(ci).WrapMyContainerProject(container); // <-- This method probably will use the 'PluginTools.CreateHttp()` tool to create an HTTP client for the container, then wrap it in an object that
|
||||
// represents the API of your project.
|
||||
}
|
||||
|
||||
public static IMyProjectNode StartMyProject(this CoreInterface ci, string someArgument)
|
||||
{
|
||||
// Start is now nothing more than a convenience method, combining the previous two.
|
||||
var rc = ci.DeployMyProject(someArgument);
|
||||
return WrapMyProjectContainer(ci, rc);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
The primary reason to decouple deploying and wrapping functionalities is that some use cases require these steps to be performed by separate applications, and different moments in time. For this reason, whatever is returned by the deploy methods should be serializable. After deserialization at some later time, it should then be valid input for the wrap method. The Codex continuous tests system is a clear example of this use case: The `CodexNetDeployer` tool uses deploy methods to create Codex nodes. Then it writes the returned objects to a JSON file. Some time later, the `CodexContinousTests` application uses this JSON file to reconstruct the objects created by the deploy methods. It then uses the wrap methods to create accessors and interactors, which are used for testing.
|
||||
|
||||
|
|
@ -1,24 +1,40 @@
|
|||
# Distributed System Tests for Nim-Codex
|
||||
|
||||
## Contributing tests
|
||||
Do you want to write some tests for Codex using this distributed test setup? Great! Here's what you do.
|
||||
Do you want to write some tests using this distributed test setup? Great! Here's what you do.
|
||||
|
||||
1. Create a branch. Name it something descriptive, but start it with `tests/` please. [Example: `tests/data-redundancy`.]
|
||||
1. Checkout your branch, and decide if your tests will be 'short' tests (minutes to hours), or 'long' tests (hours to days), or both! Create a folder for your tests in the matching folders (`/Tests`, `/LongTests`) and don't worry: You can always move your tests later if you like. [Example, short: `/Tests/DataRedundancy/`, long: `/LongTests/DataRedundancy/`]
|
||||
1. Create one or more code files in your new folder, and write some tests! Here are some tips to help you get started. You can always take a look at the example tests found in [`/Tests/BasicTests/ExampleTests.cs`](/Tests/BasicTests/ExampleTests.cs)
|
||||
1. Set up a standard NUnit test fixture.
|
||||
1. Inherrit from `DistTest` or `AutoBootstrapDistTest`.
|
||||
1. When using `DistTest`:
|
||||
1. You must start your own Codex bootstrap node. You can use `SetupCodexBootstrapNode(...)` for this.
|
||||
1. When you start other Codex nodes with `SetupCodexNodes(...)` you can pass the bootstrap node by adding the `.WithBootstrapNode(...)` option.
|
||||
1. When using `AutoBootstrapDistTest`:
|
||||
1. The test-infra creates the bootstrap node for you, and automatically passes it to each Codex node you create in your tests. Handy for keeping your tests clean and to-the-point.
|
||||
1. When using the auto-bootstrap, you have no control over the bootstrap node from your tests. You can't (for example) shut it down during the course of the test. If you need this level of control for your scenario, use the `DistTest` instead.
|
||||
1. You can generate files of random test data by calling `GenerateTestFile(...)`.
|
||||
1. If your test needs a long time to run, add the `[UseLongTimeouts]` function attribute. This will greatly increase maximum time-out values for operations like for example uploading and downloading files.
|
||||
1. You can enable access to the Codex node metrics by adding the option `.EnableMetrics()`. Enabling metrics will make the test-infra download and save all Codex metrics in case of a test failure. (The metrics are stored as CSV, in the same location as the test log file.)
|
||||
1. You can enable access to the blockchain marketplace by adding the option `.EnableMarketplace(...)`.
|
||||
1. Enabling metrics and/or enabling the marketplace takes extra resources from the test-infra and increases the time needed during Codex node setup. Please don't enable these features unless your tests need them.
|
||||
1. Tip: Codex nodes can be named. Use the option `WithName(...)` and make reading your test logs a little nicer!
|
||||
1. Tip: Commit often.
|
||||
1. Once you're happy with your tests, please create a pull-request and ask (another) Codex core contributor to review your changes.
|
||||
1. Checkout your branch.
|
||||
1. Create a new assembly in the `/Tests` folder. This can be an NUnit test assembly or simply a console app.
|
||||
1. Add Project references to `Core`, as well as any project plugin you'll be using.
|
||||
1. Write tests! Use existing tests for inspiration.
|
||||
|
||||
## Tips for writing tests for Codex
|
||||
### Transient tests
|
||||
1. Add new code files to `Tests/CodexTests`
|
||||
1. Inherrit from `CodexDistTest` or `AutoBootstrapDistTest`.
|
||||
1. When using `CodexDistTest`:
|
||||
1. You must start your own Codex bootstrap node. You can use `AddCodex(...)` for this.
|
||||
1. When you start other Codex nodes with `AddCodex(...)` you can pass the bootstrap node by adding the `.WithBootstrapNode(...)` option.
|
||||
1. When using `AutoBootstrapDistTest`:
|
||||
1. The test-infra creates the bootstrap node for you, and automatically passes it to each Codex node you create in your tests. Handy for keeping your tests clean and to-the-point.
|
||||
1. When using the auto-bootstrap, you have no control over the bootstrap node from your tests. You can't (for example) shut it down during the course of the test. If you need this level of control for your scenario, use the `CodexDistTest` instead.
|
||||
1. If your test needs a long time to run, add the `[UseLongTimeouts]` function attribute. This will greatly increase maximum time-out values for operations like for example uploading and downloading files.
|
||||
### Continuous tests
|
||||
1. Add new code files to `Tests/CodexContinousTests/Tests`
|
||||
1. Inherrit from `ContinuousTest`
|
||||
1. Define one or more methods and decorate them with the `[TestMoment(...)]` attribute.
|
||||
1. The TestMoment takes a number of seconds as argument. Each moment will be executed by the continuous test runner applying the given seconds as delay. (Non-cumulative. So two moments at T:10 will be executed one after another without delay, in this case the order of execution should not be depended upon.)
|
||||
1. Continuous tests automatically receive access to the Codex nodes that the tests are being run against.
|
||||
1. Additionally, Continuous tests can start their own transient Codex nodes and bootstrap them against the persistent nodes.
|
||||
|
||||
### Tips for either type of test
|
||||
1. You can generate files of random test data by calling `GenerateTestFile(...)`.
|
||||
1. You can enable access to the Codex node metrics by adding the option `.EnableMetrics()`. Enabling metrics will make the test-infra download and save all Codex metrics in case of a test failure. (The metrics are stored as CSV, in the same location as the test log file.)
|
||||
1. You can enable access to the blockchain marketplace by adding the option `.EnableMarketplace(...)`.
|
||||
1. Enabling metrics and/or enabling the marketplace takes extra resources from the test-infra and increases the time needed during Codex node setup. Please don't enable these features unless your tests need them.
|
||||
1. Tip: Codex nodes can be named. Use the option `WithName(...)` and make reading your test logs a little nicer!
|
||||
1. Tip: Commit often.
|
||||
|
||||
## Don't forget
|
||||
1. Once you're happy with your tests, please create a pull-request and ask a Codex core contributor to review your changes.
|
||||
|
|
|
|||
|
|
@ -1,127 +0,0 @@
|
|||
using DistTestCore;
|
||||
using DistTestCore.Codex;
|
||||
using DistTestCore.Marketplace;
|
||||
using KubernetesWorkflow;
|
||||
using Utils;
|
||||
|
||||
namespace CodexNetDeployer
|
||||
{
|
||||
public class CodexNodeStarter
|
||||
{
|
||||
private readonly Configuration config;
|
||||
private readonly TestLifecycle lifecycle;
|
||||
private readonly GethStartResult gethResult;
|
||||
private string bootstrapSpr = "";
|
||||
private int validatorsLeft;
|
||||
|
||||
public CodexNodeStarter(Configuration config, TestLifecycle lifecycle, GethStartResult gethResult, int numberOfValidators)
|
||||
{
|
||||
this.config = config;
|
||||
this.lifecycle = lifecycle;
|
||||
this.gethResult = gethResult;
|
||||
validatorsLeft = numberOfValidators;
|
||||
}
|
||||
|
||||
public CodexNodeStartResult? Start(int i)
|
||||
{
|
||||
Console.Write($" - {i} = ");
|
||||
var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
|
||||
var workflowStartup = new StartupConfig();
|
||||
workflowStartup.Add(gethResult);
|
||||
workflowStartup.Add(CreateCodexStartupConfig(bootstrapSpr, i, validatorsLeft));
|
||||
workflowStartup.NameOverride = GetCodexContainerName(i);
|
||||
|
||||
var containers = workflow.Start(1, Location.Unspecified, new CodexContainerRecipe(), workflowStartup);
|
||||
|
||||
var container = containers.Containers.First();
|
||||
var codexAccess = new CodexAccess(lifecycle.Log, container, lifecycle.TimeSet, lifecycle.Configuration.GetAddress(container));
|
||||
var account = gethResult.MarketplaceNetwork.Bootstrap.AllAccounts.Accounts[i];
|
||||
var tokenAddress = gethResult.MarketplaceNetwork.Marketplace.TokenAddress;
|
||||
var marketAccess = new MarketplaceAccess(lifecycle, gethResult.MarketplaceNetwork, account, codexAccess);
|
||||
|
||||
try
|
||||
{
|
||||
var debugInfo = codexAccess.GetDebugInfo();
|
||||
if (!string.IsNullOrWhiteSpace(debugInfo.spr))
|
||||
{
|
||||
Console.Write("Online\t");
|
||||
|
||||
var interaction = gethResult.MarketplaceNetwork.Bootstrap.StartInteraction(lifecycle);
|
||||
interaction.MintTestTokens(new[] { account.Account }, config.InitialTestTokens, tokenAddress);
|
||||
Console.Write("Tokens minted\t");
|
||||
|
||||
var response = marketAccess.MakeStorageAvailable(
|
||||
totalSpace: config.StorageSell!.Value.MB(),
|
||||
minPriceForTotalSpace: config.MinPrice.TestTokens(),
|
||||
maxCollateral: config.MaxCollateral.TestTokens(),
|
||||
maxDuration: TimeSpan.FromSeconds(config.MaxDuration));
|
||||
|
||||
if (!string.IsNullOrEmpty(response))
|
||||
{
|
||||
Console.Write("Storage available\tOK" + Environment.NewLine);
|
||||
|
||||
if (string.IsNullOrEmpty(bootstrapSpr)) bootstrapSpr = debugInfo.spr;
|
||||
validatorsLeft--;
|
||||
return new CodexNodeStartResult(workflow, container, codexAccess);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Console.WriteLine("Exception:" + ex.ToString());
|
||||
}
|
||||
|
||||
Console.Write("Unknown failure. Downloading container log." + Environment.NewLine);
|
||||
lifecycle.DownloadLog(container);
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private string GetCodexContainerName(int i)
|
||||
{
|
||||
if (i == 0) return "BOOTSTRAP";
|
||||
return "CODEX" + i;
|
||||
}
|
||||
|
||||
private CodexStartupConfig CreateCodexStartupConfig(string bootstrapSpr, int i, int validatorsLeft)
|
||||
{
|
||||
var codexStart = new CodexStartupConfig(config.CodexLogLevel);
|
||||
|
||||
if (!string.IsNullOrEmpty(bootstrapSpr)) codexStart.BootstrapSpr = bootstrapSpr;
|
||||
codexStart.StorageQuota = config.StorageQuota!.Value.MB();
|
||||
var marketplaceConfig = new MarketplaceInitialConfig(100000.Eth(), 0.TestTokens(), validatorsLeft > 0);
|
||||
marketplaceConfig.AccountIndexOverride = i;
|
||||
codexStart.MarketplaceConfig = marketplaceConfig;
|
||||
codexStart.MetricsMode = config.Metrics;
|
||||
|
||||
if (config.BlockTTL != Configuration.SecondsIn1Day)
|
||||
{
|
||||
codexStart.BlockTTL = config.BlockTTL;
|
||||
}
|
||||
if (config.BlockMI != Configuration.TenMinutes)
|
||||
{
|
||||
codexStart.BlockMaintenanceInterval = TimeSpan.FromSeconds(config.BlockMI);
|
||||
}
|
||||
if (config.BlockMN != 1000)
|
||||
{
|
||||
codexStart.BlockMaintenanceNumber = config.BlockMN;
|
||||
}
|
||||
|
||||
return codexStart;
|
||||
}
|
||||
}
|
||||
|
||||
public class CodexNodeStartResult
|
||||
{
|
||||
public CodexNodeStartResult(StartupWorkflow workflow, RunningContainer container, CodexAccess access)
|
||||
{
|
||||
Workflow = workflow;
|
||||
Container = container;
|
||||
Access = access;
|
||||
}
|
||||
|
||||
public StartupWorkflow Workflow { get; }
|
||||
public RunningContainer Container { get; }
|
||||
public CodexAccess Access { get; }
|
||||
}
|
||||
}
|
||||
|
|
@ -1,156 +0,0 @@
|
|||
using DistTestCore;
|
||||
using DistTestCore.Codex;
|
||||
using KubernetesWorkflow;
|
||||
using Logging;
|
||||
using Utils;
|
||||
|
||||
namespace CodexNetDeployer
|
||||
{
|
||||
public class Deployer
|
||||
{
|
||||
private readonly Configuration config;
|
||||
private readonly DefaultTimeSet timeset;
|
||||
private readonly PeerConnectivityChecker peerConnectivityChecker;
|
||||
|
||||
public Deployer(Configuration config)
|
||||
{
|
||||
this.config = config;
|
||||
timeset = new DefaultTimeSet();
|
||||
peerConnectivityChecker = new PeerConnectivityChecker();
|
||||
}
|
||||
|
||||
public CodexDeployment Deploy()
|
||||
{
|
||||
Log("Initializing...");
|
||||
var lifecycle = CreateTestLifecycle();
|
||||
|
||||
Log("Preparing configuration...");
|
||||
// We trick the Geth companion node into unlocking all of its accounts, by saying we want to start 999 codex nodes.
|
||||
var setup = new CodexSetup(999, config.CodexLogLevel);
|
||||
setup.WithStorageQuota(config.StorageQuota!.Value.MB()).EnableMarketplace(0.TestTokens());
|
||||
setup.MetricsMode = config.Metrics;
|
||||
|
||||
Log("Creating Geth instance and deploying contracts...");
|
||||
var gethStarter = new GethStarter(lifecycle);
|
||||
var gethResults = gethStarter.BringOnlineMarketplaceFor(setup);
|
||||
|
||||
Log("Geth started. Codex contracts deployed.");
|
||||
Log("Warning: It can take up to 45 minutes for the Geth node to finish unlocking all if its 1000 preconfigured accounts.");
|
||||
|
||||
// It takes a second for the geth node to unlock a single account. Let's assume 3.
|
||||
// We can't start the codex nodes until their accounts are definitely unlocked. So
|
||||
// We wait:
|
||||
Thread.Sleep(TimeSpan.FromSeconds(3.0 * config.NumberOfCodexNodes!.Value));
|
||||
|
||||
Log("Starting Codex nodes...");
|
||||
|
||||
// Each node must have its own IP, so it needs it own pod. Start them 1 at a time.
|
||||
var codexStarter = new CodexNodeStarter(config, lifecycle, gethResults, config.NumberOfValidators!.Value);
|
||||
var startResults = new List<CodexNodeStartResult>();
|
||||
for (var i = 0; i < config.NumberOfCodexNodes; i++)
|
||||
{
|
||||
var result = codexStarter.Start(i);
|
||||
if (result != null) startResults.Add(result);
|
||||
}
|
||||
|
||||
var (prometheusContainer, grafanaStartInfo) = StartMetricsService(lifecycle, setup, startResults.Select(r => r.Container));
|
||||
|
||||
CheckPeerConnectivity(startResults);
|
||||
CheckContainerRestarts(startResults);
|
||||
|
||||
return new CodexDeployment(gethResults, startResults.Select(r => r.Container).ToArray(), prometheusContainer, grafanaStartInfo, CreateMetadata());
|
||||
}
|
||||
|
||||
private TestLifecycle CreateTestLifecycle()
|
||||
{
|
||||
var kubeConfig = GetKubeConfig(config.KubeConfigFile);
|
||||
|
||||
var lifecycleConfig = new DistTestCore.Configuration
|
||||
(
|
||||
kubeConfigFile: kubeConfig,
|
||||
logPath: "null",
|
||||
logDebug: false,
|
||||
dataFilesPath: "notUsed",
|
||||
codexLogLevel: config.CodexLogLevel,
|
||||
k8sNamespacePrefix: config.KubeNamespace
|
||||
);
|
||||
|
||||
var lifecycle = new TestLifecycle(new NullLog(), lifecycleConfig, timeset, string.Empty);
|
||||
DefaultContainerRecipe.TestsType = config.TestsTypePodLabel;
|
||||
DefaultContainerRecipe.ApplicationIds = lifecycle.GetApplicationIds();
|
||||
return lifecycle;
|
||||
}
|
||||
|
||||
private (RunningContainer?, GrafanaStartInfo?) StartMetricsService(TestLifecycle lifecycle, CodexSetup setup, IEnumerable<RunningContainer> codexContainers)
|
||||
{
|
||||
if (setup.MetricsMode == DistTestCore.Metrics.MetricsMode.None) return (null, null);
|
||||
|
||||
Log("Starting metrics service...");
|
||||
var runningContainers = new[] { new RunningContainers(null!, null!, codexContainers.ToArray()) };
|
||||
var prometheusContainer = lifecycle.PrometheusStarter.CollectMetricsFor(runningContainers).Containers.Single();
|
||||
|
||||
if (setup.MetricsMode == DistTestCore.Metrics.MetricsMode.Record) return (prometheusContainer, null);
|
||||
|
||||
Log("Starting dashboard service...");
|
||||
var grafanaStartInfo = lifecycle.GrafanaStarter.StartDashboard(prometheusContainer, setup);
|
||||
return (prometheusContainer, grafanaStartInfo);
|
||||
}
|
||||
|
||||
private string? GetKubeConfig(string kubeConfigFile)
|
||||
{
|
||||
if (string.IsNullOrEmpty(kubeConfigFile) || kubeConfigFile.ToLowerInvariant() == "null") return null;
|
||||
return kubeConfigFile;
|
||||
}
|
||||
|
||||
private void CheckPeerConnectivity(List<CodexNodeStartResult> codexContainers)
|
||||
{
|
||||
if (!config.CheckPeerConnection) return;
|
||||
|
||||
Log("Starting peer-connectivity check for deployed nodes...");
|
||||
peerConnectivityChecker.CheckConnectivity(codexContainers);
|
||||
Log("Check passed.");
|
||||
}
|
||||
|
||||
private void CheckContainerRestarts(List<CodexNodeStartResult> startResults)
|
||||
{
|
||||
var crashes = new List<RunningContainer>();
|
||||
foreach (var startResult in startResults)
|
||||
{
|
||||
var watcher = startResult.Workflow.CreateCrashWatcher(startResult.Container);
|
||||
if (watcher.HasContainerCrashed()) crashes.Add(startResult.Container);
|
||||
}
|
||||
|
||||
if (!crashes.Any())
|
||||
{
|
||||
Log("Container restart check passed.");
|
||||
}
|
||||
else
|
||||
{
|
||||
Log($"Deployment failed. The following containers have crashed: {string.Join(",", crashes.Select(c => c.Name))}");
|
||||
throw new Exception("Deployment failed: One or more containers crashed.");
|
||||
}
|
||||
}
|
||||
|
||||
private DeploymentMetadata CreateMetadata()
|
||||
{
|
||||
return new DeploymentMetadata(
|
||||
kubeNamespace: config.KubeNamespace,
|
||||
numberOfCodexNodes: config.NumberOfCodexNodes!.Value,
|
||||
numberOfValidators: config.NumberOfValidators!.Value,
|
||||
storageQuotaMB: config.StorageQuota!.Value,
|
||||
codexLogLevel: config.CodexLogLevel,
|
||||
initialTestTokens: config.InitialTestTokens,
|
||||
minPrice: config.MinPrice,
|
||||
maxCollateral: config.MaxCollateral,
|
||||
maxDuration: config.MaxDuration,
|
||||
blockTTL: config.BlockTTL,
|
||||
blockMI: config.BlockMI,
|
||||
blockMN: config.BlockMN);
|
||||
}
|
||||
|
||||
private void Log(string msg)
|
||||
{
|
||||
Console.WriteLine(msg);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,34 +0,0 @@
|
|||
using DistTestCore.Helpers;
|
||||
using Logging;
|
||||
|
||||
namespace CodexNetDeployer
|
||||
{
|
||||
public class PeerConnectivityChecker
|
||||
{
|
||||
public void CheckConnectivity(List<CodexNodeStartResult> startResults)
|
||||
{
|
||||
var log = new ConsoleLog();
|
||||
var checker = new PeerConnectionTestHelpers(log);
|
||||
var access = startResults.Select(r => r.Access);
|
||||
|
||||
checker.AssertFullyConnected(access);
|
||||
}
|
||||
}
|
||||
|
||||
public class ConsoleLog : BaseLog
|
||||
{
|
||||
public ConsoleLog() : base(false)
|
||||
{
|
||||
}
|
||||
|
||||
protected override string GetFullName()
|
||||
{
|
||||
return "CONSOLE";
|
||||
}
|
||||
|
||||
public override void Log(string message)
|
||||
{
|
||||
Console.WriteLine(message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,16 +0,0 @@
|
|||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<OutputType>Exe</OutputType>
|
||||
<TargetFramework>net7.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\ArgsUniform\ArgsUniform.csproj" />
|
||||
<ProjectReference Include="..\ContinuousTests\ContinuousTests.csproj" />
|
||||
<ProjectReference Include="..\DistTestCore\DistTestCore.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
|
|
@ -1,19 +0,0 @@
|
|||
using ArgsUniform;
|
||||
using DistTestCore.Codex;
|
||||
|
||||
namespace CodexNetDownloader
|
||||
{
|
||||
public class Configuration
|
||||
{
|
||||
[Uniform("output-path", "o", "OUTPUT", true, "Path where files will be written.")]
|
||||
public string OutputPath { get; set; } = "output";
|
||||
|
||||
[Uniform("codex-deployment", "c", "CODEXDEPLOYMENT", true, "Path to codex-deployment JSON file.")]
|
||||
public string CodexDeploymentJson { get; set; } = string.Empty;
|
||||
|
||||
[Uniform("kube-config", "kc", "KUBECONFIG", true, "Path to Kubeconfig file. Use 'null' (default) to use local cluster.")]
|
||||
public string KubeConfigFile { get; set; } = "null";
|
||||
|
||||
public CodexDeployment CodexDeployment { get; set; } = null!;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,48 +0,0 @@
|
|||
using ArgsUniform;
|
||||
using ContinuousTests;
|
||||
using DistTestCore;
|
||||
using DistTestCore.Codex;
|
||||
using Logging;
|
||||
using Newtonsoft.Json;
|
||||
|
||||
public class Program
|
||||
{
|
||||
public static void Main(string[] args)
|
||||
{
|
||||
var nl = Environment.NewLine;
|
||||
Console.WriteLine("CodexNetDownloader" + nl);
|
||||
|
||||
var uniformArgs = new ArgsUniform<CodexNetDownloader.Configuration>(PrintHelp, args);
|
||||
var config = uniformArgs.Parse(true);
|
||||
|
||||
config.CodexDeployment = ParseCodexDeploymentJson(config.CodexDeploymentJson);
|
||||
|
||||
if (!Directory.Exists(config.OutputPath)) Directory.CreateDirectory(config.OutputPath);
|
||||
|
||||
var k8sFactory = new K8sFactory();
|
||||
var lifecycle = k8sFactory.CreateTestLifecycle(config.KubeConfigFile, config.OutputPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog());
|
||||
|
||||
foreach (var container in config.CodexDeployment.CodexContainers)
|
||||
{
|
||||
lifecycle.DownloadLog(container);
|
||||
}
|
||||
|
||||
Console.WriteLine("Done!");
|
||||
}
|
||||
|
||||
private static CodexDeployment ParseCodexDeploymentJson(string filename)
|
||||
{
|
||||
var d = JsonConvert.DeserializeObject<CodexDeployment>(File.ReadAllText(filename))!;
|
||||
if (d == null) throw new Exception("Unable to parse " + filename);
|
||||
return d;
|
||||
}
|
||||
|
||||
private static void PrintHelp()
|
||||
{
|
||||
var nl = Environment.NewLine;
|
||||
Console.WriteLine("CodexNetDownloader lets you download all container logs given a codex-deployment.json file." + nl);
|
||||
|
||||
Console.WriteLine("CodexNetDownloader assumes you are running this tool from *inside* the Kubernetes cluster. " +
|
||||
"If you are not running this from a container inside the cluster, add the argument '--external'." + nl);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,20 +0,0 @@
|
|||
using DistTestCore;
|
||||
using DistTestCore.Codex;
|
||||
using KubernetesWorkflow;
|
||||
using Logging;
|
||||
|
||||
namespace ContinuousTests
|
||||
{
|
||||
public class CodexAccessFactory
|
||||
{
|
||||
public CodexAccess[] Create(Configuration config, RunningContainer[] containers, BaseLog log, ITimeSet timeSet)
|
||||
{
|
||||
return containers.Select(container =>
|
||||
{
|
||||
var address = container.ClusterExternalAddress;
|
||||
if (config.RunnerLocation == RunnerLocation.InternalToCluster) address = container.ClusterInternalAddress;
|
||||
return new CodexAccess(log, container, timeSet, address);
|
||||
}).ToArray();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,119 +0,0 @@
|
|||
using DistTestCore;
|
||||
using DistTestCore.Codex;
|
||||
using DistTestCore.Logs;
|
||||
using KubernetesWorkflow;
|
||||
using Logging;
|
||||
|
||||
namespace ContinuousTests
|
||||
{
|
||||
public abstract class ContinuousTestLongTimeouts : ContinuousTest
|
||||
{
|
||||
public override ITimeSet TimeSet => new LongTimeSet();
|
||||
}
|
||||
|
||||
public abstract class ContinuousTest
|
||||
{
|
||||
protected const int Zero = 0;
|
||||
protected const int MinuteOne = 60;
|
||||
protected const int MinuteFive = MinuteOne * 5;
|
||||
protected const int HourOne = MinuteOne * 60;
|
||||
protected const int HourThree = HourOne * 3;
|
||||
protected const int DayOne = HourOne * 24;
|
||||
protected const int DayThree = DayOne * 3;
|
||||
|
||||
private const string UploadFailedMessage = "Unable to store block";
|
||||
|
||||
public void Initialize(CodexAccess[] nodes, BaseLog log, FileManager fileManager, Configuration configuration, CancellationToken cancelToken)
|
||||
{
|
||||
Nodes = nodes;
|
||||
Log = log;
|
||||
FileManager = fileManager;
|
||||
Configuration = configuration;
|
||||
CancelToken = cancelToken;
|
||||
|
||||
if (nodes != null)
|
||||
{
|
||||
NodeRunner = new NodeRunner(Nodes, configuration, TimeSet, Log, CustomK8sNamespace, EthereumAccountIndex);
|
||||
}
|
||||
else
|
||||
{
|
||||
NodeRunner = null!;
|
||||
}
|
||||
}
|
||||
|
||||
public CodexAccess[] Nodes { get; private set; } = null!;
|
||||
public BaseLog Log { get; private set; } = null!;
|
||||
public IFileManager FileManager { get; private set; } = null!;
|
||||
public Configuration Configuration { get; private set; } = null!;
|
||||
public virtual ITimeSet TimeSet { get { return new DefaultTimeSet(); } }
|
||||
public CancellationToken CancelToken { get; private set; } = new CancellationToken();
|
||||
public NodeRunner NodeRunner { get; private set; } = null!;
|
||||
|
||||
public abstract int RequiredNumberOfNodes { get; }
|
||||
public abstract TimeSpan RunTestEvery { get; }
|
||||
public abstract TestFailMode TestFailMode { get; }
|
||||
public virtual int EthereumAccountIndex { get { return -1; } }
|
||||
public virtual string CustomK8sNamespace { get { return string.Empty; } }
|
||||
|
||||
public string Name
|
||||
{
|
||||
get
|
||||
{
|
||||
return GetType().Name;
|
||||
}
|
||||
}
|
||||
|
||||
public ContentId? UploadFile(CodexAccess node, TestFile file)
|
||||
{
|
||||
using var fileStream = File.OpenRead(file.Filename);
|
||||
|
||||
var logMessage = $"Uploading file {file.Describe()}...";
|
||||
var response = Stopwatch.Measure(Log, logMessage, () =>
|
||||
{
|
||||
return node.UploadFile(fileStream);
|
||||
});
|
||||
|
||||
if (string.IsNullOrEmpty(response)) return null;
|
||||
if (response.StartsWith(UploadFailedMessage)) return null;
|
||||
|
||||
Log.Log($"Uploaded file. Received contentId: '{response}'.");
|
||||
return new ContentId(response);
|
||||
}
|
||||
|
||||
public TestFile DownloadFile(CodexAccess node, ContentId contentId, string fileLabel = "")
|
||||
{
|
||||
var logMessage = $"Downloading for contentId: '{contentId.Id}'...";
|
||||
var file = FileManager.CreateEmptyTestFile(fileLabel);
|
||||
Stopwatch.Measure(Log, logMessage, () => DownloadToFile(node, contentId.Id, file));
|
||||
Log.Log($"Downloaded file {file.Describe()} to '{file.Filename}'.");
|
||||
return file;
|
||||
}
|
||||
|
||||
public IDownloadedLog DownloadContainerLog(RunningContainer container, int? tailLines = null)
|
||||
{
|
||||
var nodeRunner = new NodeRunner(Nodes, Configuration, TimeSet, Log, Configuration.CodexDeployment.Metadata.KubeNamespace, EthereumAccountIndex);
|
||||
return nodeRunner.DownloadLog(container, tailLines);
|
||||
}
|
||||
|
||||
private void DownloadToFile(CodexAccess node, string contentId, TestFile file)
|
||||
{
|
||||
using var fileStream = File.OpenWrite(file.Filename);
|
||||
try
|
||||
{
|
||||
using var downloadStream = node.DownloadFile(contentId);
|
||||
downloadStream.CopyTo(fileStream);
|
||||
}
|
||||
catch
|
||||
{
|
||||
Log.Log($"Failed to download file '{contentId}'.");
|
||||
throw;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public enum TestFailMode
|
||||
{
|
||||
StopAfterFirstFailure,
|
||||
AlwaysRunAllMoments
|
||||
}
|
||||
}
|
||||
|
|
@ -1,34 +0,0 @@
|
|||
using DistTestCore.Codex;
|
||||
using DistTestCore;
|
||||
using Logging;
|
||||
|
||||
namespace ContinuousTests
|
||||
{
|
||||
public class K8sFactory
|
||||
{
|
||||
public TestLifecycle CreateTestLifecycle(string kubeConfigFile, string logPath, string dataFilePath, string customNamespace, ITimeSet timeSet, BaseLog log)
|
||||
{
|
||||
var kubeConfig = GetKubeConfig(kubeConfigFile);
|
||||
var lifecycleConfig = new DistTestCore.Configuration
|
||||
(
|
||||
kubeConfigFile: kubeConfig,
|
||||
logPath: logPath,
|
||||
logDebug: false,
|
||||
dataFilesPath: dataFilePath,
|
||||
codexLogLevel: CodexLogLevel.Debug,
|
||||
k8sNamespacePrefix: customNamespace
|
||||
);
|
||||
|
||||
var lifecycle = new TestLifecycle(log, lifecycleConfig, timeSet, string.Empty);
|
||||
DefaultContainerRecipe.TestsType = "continuous-tests";
|
||||
DefaultContainerRecipe.ApplicationIds = lifecycle.GetApplicationIds();
|
||||
return lifecycle;
|
||||
}
|
||||
|
||||
private static string? GetKubeConfig(string kubeConfigFile)
|
||||
{
|
||||
if (string.IsNullOrEmpty(kubeConfigFile) || kubeConfigFile.ToLowerInvariant() == "null") return null;
|
||||
return kubeConfigFile;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,113 +0,0 @@
|
|||
using DistTestCore.Codex;
|
||||
using DistTestCore.Marketplace;
|
||||
using DistTestCore;
|
||||
using KubernetesWorkflow;
|
||||
using NUnit.Framework;
|
||||
using Logging;
|
||||
using Utils;
|
||||
using DistTestCore.Logs;
|
||||
|
||||
namespace ContinuousTests
|
||||
{
|
||||
public class NodeRunner
|
||||
{
|
||||
private readonly K8sFactory k8SFactory = new K8sFactory();
|
||||
private readonly CodexAccess[] nodes;
|
||||
private readonly Configuration config;
|
||||
private readonly ITimeSet timeSet;
|
||||
private readonly BaseLog log;
|
||||
private readonly string customNamespace;
|
||||
private readonly int ethereumAccountIndex;
|
||||
|
||||
public NodeRunner(CodexAccess[] nodes, Configuration config, ITimeSet timeSet, BaseLog log, string customNamespace, int ethereumAccountIndex)
|
||||
{
|
||||
this.nodes = nodes;
|
||||
this.config = config;
|
||||
this.timeSet = timeSet;
|
||||
this.log = log;
|
||||
this.customNamespace = customNamespace;
|
||||
this.ethereumAccountIndex = ethereumAccountIndex;
|
||||
}
|
||||
|
||||
public void RunNode(Action<CodexAccess, MarketplaceAccess, TestLifecycle> operation)
|
||||
{
|
||||
RunNode(nodes.ToList().PickOneRandom(), operation, 0.TestTokens());
|
||||
}
|
||||
|
||||
public void RunNode(CodexAccess bootstrapNode, Action<CodexAccess, MarketplaceAccess, TestLifecycle> operation)
|
||||
{
|
||||
RunNode(bootstrapNode, operation, 0.TestTokens());
|
||||
}
|
||||
|
||||
public IDownloadedLog DownloadLog(RunningContainer container, int? tailLines = null)
|
||||
{
|
||||
var subFile = log.CreateSubfile();
|
||||
var description = container.Name;
|
||||
var handler = new LogDownloadHandler(container, description, subFile);
|
||||
|
||||
log.Log($"Downloading logs for {description} to file '{subFile.FullFilename}'");
|
||||
|
||||
var lifecycle = CreateTestLifecycle();
|
||||
var flow = lifecycle.WorkflowCreator.CreateWorkflow();
|
||||
flow.DownloadContainerLog(container, handler, tailLines);
|
||||
|
||||
return new DownloadedLog(subFile, description);
|
||||
}
|
||||
|
||||
public void RunNode(CodexAccess bootstrapNode, Action<CodexAccess, MarketplaceAccess, TestLifecycle> operation, TestToken mintTestTokens)
|
||||
{
|
||||
var lifecycle = CreateTestLifecycle();
|
||||
var flow = lifecycle.WorkflowCreator.CreateWorkflow();
|
||||
|
||||
try
|
||||
{
|
||||
var debugInfo = bootstrapNode.GetDebugInfo();
|
||||
Assert.That(!string.IsNullOrEmpty(debugInfo.spr));
|
||||
|
||||
var startupConfig = new StartupConfig();
|
||||
startupConfig.NameOverride = "TransientNode";
|
||||
var codexStartConfig = new CodexStartupConfig(CodexLogLevel.Trace);
|
||||
codexStartConfig.MarketplaceConfig = new MarketplaceInitialConfig(0.Eth(), 0.TestTokens(), false);
|
||||
codexStartConfig.MarketplaceConfig.AccountIndexOverride = ethereumAccountIndex;
|
||||
codexStartConfig.BootstrapSpr = debugInfo.spr;
|
||||
startupConfig.Add(codexStartConfig);
|
||||
startupConfig.Add(config.CodexDeployment.GethStartResult);
|
||||
var rc = flow.Start(1, Location.Unspecified, new CodexContainerRecipe(), startupConfig);
|
||||
|
||||
var account = config.CodexDeployment.GethStartResult.CompanionNode.Accounts[ethereumAccountIndex];
|
||||
|
||||
var marketplaceNetwork = config.CodexDeployment.GethStartResult.MarketplaceNetwork;
|
||||
if (mintTestTokens.Amount > 0)
|
||||
{
|
||||
var tokenAddress = marketplaceNetwork.Marketplace.TokenAddress;
|
||||
var interaction = marketplaceNetwork.Bootstrap.StartInteraction(lifecycle);
|
||||
interaction.MintTestTokens(new[] { account.Account }, mintTestTokens.Amount, tokenAddress);
|
||||
}
|
||||
|
||||
var container = rc.Containers[0];
|
||||
var address = lifecycle.Configuration.GetAddress(container);
|
||||
var codexAccess = new CodexAccess(log, container, lifecycle.TimeSet, address);
|
||||
var marketAccess = new MarketplaceAccess(lifecycle, marketplaceNetwork, account, codexAccess);
|
||||
|
||||
try
|
||||
{
|
||||
operation(codexAccess, marketAccess, lifecycle);
|
||||
}
|
||||
catch
|
||||
{
|
||||
lifecycle.DownloadLog(container);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
flow.DeleteTestResources();
|
||||
}
|
||||
}
|
||||
|
||||
private TestLifecycle CreateTestLifecycle()
|
||||
{
|
||||
return k8SFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, config.DataPath, customNamespace, timeSet, log);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,60 +0,0 @@
|
|||
using DistTestCore;
|
||||
using DistTestCore.Codex;
|
||||
using NUnit.Framework;
|
||||
|
||||
namespace ContinuousTests.Tests
|
||||
{
|
||||
public class ThresholdChecks : ContinuousTest
|
||||
{
|
||||
public override int RequiredNumberOfNodes => 1;
|
||||
public override TimeSpan RunTestEvery => TimeSpan.FromSeconds(30);
|
||||
public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure;
|
||||
|
||||
private static readonly List<string> previousBreaches = new List<string>();
|
||||
|
||||
[TestMoment(t: 0)]
|
||||
public void CheckAllThresholds()
|
||||
{
|
||||
var allNodes = CreateAccessToAllNodes();
|
||||
foreach (var n in allNodes) CheckThresholds(n);
|
||||
}
|
||||
|
||||
private void CheckThresholds(CodexAccess n)
|
||||
{
|
||||
var breaches = n.GetDebugThresholdBreaches();
|
||||
if (breaches.breaches.Any())
|
||||
{
|
||||
var newBreaches = new List<string>();
|
||||
foreach (var b in breaches.breaches)
|
||||
{
|
||||
if (!previousBreaches.Contains(b))
|
||||
{
|
||||
newBreaches.Add(b);
|
||||
previousBreaches.Add(b);
|
||||
}
|
||||
}
|
||||
|
||||
if (newBreaches.Any())
|
||||
{
|
||||
Assert.Fail(string.Join(",", newBreaches.Select(b => FormatBreach(n, b))));
|
||||
|
||||
Program.Cancellation.Cts.Cancel();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private string FormatBreach(CodexAccess n, string breach)
|
||||
{
|
||||
return $"{n.Container.Name} = '{breach}'";
|
||||
}
|
||||
|
||||
private CodexAccess[] CreateAccessToAllNodes()
|
||||
{
|
||||
// Normally, a continuous test accesses only a subset of the nodes in the deployment.
|
||||
// This time, we want to check all of them.
|
||||
var factory = new CodexAccessFactory();
|
||||
var allContainers = Configuration.CodexDeployment.CodexContainers;
|
||||
return factory.Create(Configuration, allContainers, Log, new DefaultTimeSet());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,30 +0,0 @@
|
|||
using NUnit.Framework;
|
||||
|
||||
namespace DistTestCore
|
||||
{
|
||||
public class AutoBootstrapDistTest : DistTest
|
||||
{
|
||||
public override IOnlineCodexNode SetupCodexBootstrapNode(Action<ICodexSetup> setup)
|
||||
{
|
||||
throw new Exception("AutoBootstrapDistTest creates and attaches a single bootstrap node for you. " +
|
||||
"If you want to control the bootstrap node from your test, please use DistTest instead.");
|
||||
}
|
||||
|
||||
public override ICodexNodeGroup SetupCodexNodes(int numberOfNodes, Action<ICodexSetup> setup)
|
||||
{
|
||||
var codexSetup = CreateCodexSetup(numberOfNodes);
|
||||
setup(codexSetup);
|
||||
codexSetup.WithBootstrapNode(BootstrapNode);
|
||||
return BringOnline(codexSetup);
|
||||
}
|
||||
|
||||
[SetUp]
|
||||
public void SetUpBootstrapNode()
|
||||
{
|
||||
var setup = CreateCodexSetup(1).WithName("BOOTSTRAP");
|
||||
BootstrapNode = BringOnline(setup)[0];
|
||||
}
|
||||
|
||||
protected IOnlineCodexNode BootstrapNode { get; private set; } = null!;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,42 +0,0 @@
|
|||
using Logging;
|
||||
|
||||
namespace DistTestCore
|
||||
{
|
||||
public class BaseStarter
|
||||
{
|
||||
protected readonly TestLifecycle lifecycle;
|
||||
private Stopwatch? stopwatch;
|
||||
|
||||
public BaseStarter(TestLifecycle lifecycle)
|
||||
{
|
||||
this.lifecycle = lifecycle;
|
||||
}
|
||||
|
||||
protected void LogStart(string msg)
|
||||
{
|
||||
Log(msg);
|
||||
stopwatch = Stopwatch.Begin(lifecycle.Log, GetClassName());
|
||||
}
|
||||
|
||||
protected void LogEnd(string msg)
|
||||
{
|
||||
stopwatch!.End(msg);
|
||||
stopwatch = null;
|
||||
}
|
||||
|
||||
protected void Log(string msg)
|
||||
{
|
||||
lifecycle.Log.Log($"{GetClassName()} {msg}");
|
||||
}
|
||||
|
||||
protected void Debug(string msg)
|
||||
{
|
||||
lifecycle.Log.Debug($"{GetClassName()} {msg}", 1);
|
||||
}
|
||||
|
||||
private string GetClassName()
|
||||
{
|
||||
return $"({GetType().Name})";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,32 +0,0 @@
|
|||
using DistTestCore.Codex;
|
||||
using DistTestCore.Marketplace;
|
||||
using DistTestCore.Metrics;
|
||||
|
||||
namespace DistTestCore
|
||||
{
|
||||
public interface ICodexNodeFactory
|
||||
{
|
||||
OnlineCodexNode CreateOnlineCodexNode(CodexAccess access, CodexNodeGroup group);
|
||||
}
|
||||
|
||||
public class CodexNodeFactory : ICodexNodeFactory
|
||||
{
|
||||
private readonly TestLifecycle lifecycle;
|
||||
private readonly IMetricsAccessFactory metricsAccessFactory;
|
||||
private readonly IMarketplaceAccessFactory marketplaceAccessFactory;
|
||||
|
||||
public CodexNodeFactory(TestLifecycle lifecycle, IMetricsAccessFactory metricsAccessFactory, IMarketplaceAccessFactory marketplaceAccessFactory)
|
||||
{
|
||||
this.lifecycle = lifecycle;
|
||||
this.metricsAccessFactory = metricsAccessFactory;
|
||||
this.marketplaceAccessFactory = marketplaceAccessFactory;
|
||||
}
|
||||
|
||||
public OnlineCodexNode CreateOnlineCodexNode(CodexAccess access, CodexNodeGroup group)
|
||||
{
|
||||
var metricsAccess = metricsAccessFactory.CreateMetricsAccess(access.Container);
|
||||
var marketplaceAccess = marketplaceAccessFactory.CreateMarketplaceAccess(access);
|
||||
return new OnlineCodexNode(lifecycle, access, group, metricsAccess, marketplaceAccess);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,159 +0,0 @@
|
|||
using DistTestCore.Codex;
|
||||
using DistTestCore.Marketplace;
|
||||
using DistTestCore.Metrics;
|
||||
using KubernetesWorkflow;
|
||||
using Logging;
|
||||
|
||||
namespace DistTestCore
|
||||
{
|
||||
public class CodexStarter : BaseStarter
|
||||
{
|
||||
public CodexStarter(TestLifecycle lifecycle)
|
||||
: base(lifecycle)
|
||||
{
|
||||
}
|
||||
|
||||
public List<CodexNodeGroup> RunningGroups { get; } = new List<CodexNodeGroup>();
|
||||
|
||||
public ICodexNodeGroup BringOnline(CodexSetup codexSetup)
|
||||
{
|
||||
LogSeparator();
|
||||
LogStart($"Starting {codexSetup.Describe()}...");
|
||||
var gethStartResult = lifecycle.GethStarter.BringOnlineMarketplaceFor(codexSetup);
|
||||
gethStartResult = lifecycle.GethStarter.BringOnlineValidatorFor(codexSetup, gethStartResult);
|
||||
|
||||
var startupConfig = CreateStartupConfig(gethStartResult, codexSetup);
|
||||
|
||||
var containers = StartCodexContainers(startupConfig, codexSetup.NumberOfNodes, codexSetup.Location);
|
||||
|
||||
var metricAccessFactory = CollectMetrics(codexSetup, containers);
|
||||
|
||||
var codexNodeFactory = new CodexNodeFactory(lifecycle, metricAccessFactory, gethStartResult.MarketplaceAccessFactory);
|
||||
|
||||
var group = CreateCodexGroup(codexSetup, containers, codexNodeFactory);
|
||||
lifecycle.SetCodexVersion(group.Version);
|
||||
|
||||
var nl = Environment.NewLine;
|
||||
var podInfos = string.Join(nl, containers.Containers().Select(c => $"Container: '{c.Name}' runs at '{c.Pod.PodInfo.K8SNodeName}'={c.Pod.PodInfo.Ip}"));
|
||||
LogEnd($"Started {codexSetup.NumberOfNodes} nodes " +
|
||||
$"of image '{containers.Containers().First().Recipe.Image}' " +
|
||||
$"and version '{group.Version}'{nl}" +
|
||||
podInfos);
|
||||
LogSeparator();
|
||||
|
||||
return group;
|
||||
}
|
||||
|
||||
public void BringOffline(CodexNodeGroup group)
|
||||
{
|
||||
LogStart($"Stopping {group.Describe()}...");
|
||||
var workflow = CreateWorkflow();
|
||||
foreach (var c in group.Containers)
|
||||
{
|
||||
StopCrashWatcher(c);
|
||||
workflow.Stop(c);
|
||||
}
|
||||
RunningGroups.Remove(group);
|
||||
LogEnd("Stopped.");
|
||||
}
|
||||
|
||||
public void DeleteAllResources()
|
||||
{
|
||||
var workflow = CreateWorkflow();
|
||||
workflow.DeleteTestResources();
|
||||
|
||||
RunningGroups.Clear();
|
||||
}
|
||||
|
||||
public void DownloadLog(RunningContainer container, ILogHandler logHandler, int? tailLines)
|
||||
{
|
||||
var workflow = CreateWorkflow();
|
||||
workflow.DownloadContainerLog(container, logHandler, tailLines);
|
||||
}
|
||||
|
||||
private IMetricsAccessFactory CollectMetrics(CodexSetup codexSetup, RunningContainers[] containers)
|
||||
{
|
||||
if (codexSetup.MetricsMode == MetricsMode.None) return new MetricsUnavailableAccessFactory();
|
||||
|
||||
var runningContainers = lifecycle.PrometheusStarter.CollectMetricsFor(containers);
|
||||
|
||||
if (codexSetup.MetricsMode == MetricsMode.Dashboard)
|
||||
{
|
||||
lifecycle.GrafanaStarter.StartDashboard(runningContainers.Containers.First(), codexSetup);
|
||||
}
|
||||
|
||||
return new CodexNodeMetricsAccessFactory(lifecycle, runningContainers);
|
||||
}
|
||||
|
||||
private StartupConfig CreateStartupConfig(GethStartResult gethStartResult, CodexSetup codexSetup)
|
||||
{
|
||||
var startupConfig = new StartupConfig();
|
||||
startupConfig.NameOverride = codexSetup.NameOverride;
|
||||
startupConfig.Add(codexSetup);
|
||||
startupConfig.Add(gethStartResult);
|
||||
return startupConfig;
|
||||
}
|
||||
|
||||
private RunningContainers[] StartCodexContainers(StartupConfig startupConfig, int numberOfNodes, Location location)
|
||||
{
|
||||
var result = new List<RunningContainers>();
|
||||
var recipe = new CodexContainerRecipe();
|
||||
for (var i = 0; i < numberOfNodes; i++)
|
||||
{
|
||||
var workflow = CreateWorkflow();
|
||||
var rc = workflow.Start(1, location, recipe, startupConfig);
|
||||
CreateCrashWatcher(workflow, rc);
|
||||
result.Add(rc);
|
||||
}
|
||||
return result.ToArray();
|
||||
}
|
||||
|
||||
private CodexNodeGroup CreateCodexGroup(CodexSetup codexSetup, RunningContainers[] runningContainers, CodexNodeFactory codexNodeFactory)
|
||||
{
|
||||
var group = new CodexNodeGroup(lifecycle, codexSetup, runningContainers, codexNodeFactory);
|
||||
RunningGroups.Add(group);
|
||||
|
||||
try
|
||||
{
|
||||
Stopwatch.Measure(lifecycle.Log, "EnsureOnline", group.EnsureOnline, debug: true);
|
||||
}
|
||||
catch
|
||||
{
|
||||
CodexNodesNotOnline(runningContainers);
|
||||
throw;
|
||||
}
|
||||
|
||||
return group;
|
||||
}
|
||||
|
||||
private void CodexNodesNotOnline(RunningContainers[] runningContainers)
|
||||
{
|
||||
Log("Codex nodes failed to start");
|
||||
foreach (var container in runningContainers.Containers()) lifecycle.DownloadLog(container);
|
||||
}
|
||||
|
||||
private StartupWorkflow CreateWorkflow()
|
||||
{
|
||||
return lifecycle.WorkflowCreator.CreateWorkflow();
|
||||
}
|
||||
|
||||
private void LogSeparator()
|
||||
{
|
||||
Log("----------------------------------------------------------------------------");
|
||||
}
|
||||
|
||||
private void CreateCrashWatcher(StartupWorkflow workflow, RunningContainers rc)
|
||||
{
|
||||
var c = rc.Containers.Single();
|
||||
c.CrashWatcher = workflow.CreateCrashWatcher(c);
|
||||
}
|
||||
|
||||
private void StopCrashWatcher(RunningContainers containers)
|
||||
{
|
||||
foreach (var c in containers.Containers)
|
||||
{
|
||||
c.CrashWatcher?.Stop();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,144 +0,0 @@
|
|||
using DistTestCore.Codex;
|
||||
using KubernetesWorkflow;
|
||||
using System.Net.NetworkInformation;
|
||||
using Utils;
|
||||
|
||||
namespace DistTestCore
|
||||
{
|
||||
public class Configuration
|
||||
{
|
||||
private readonly string? kubeConfigFile;
|
||||
private readonly string logPath;
|
||||
private readonly bool logDebug;
|
||||
private readonly string dataFilesPath;
|
||||
private readonly CodexLogLevel codexLogLevel;
|
||||
private readonly string k8sNamespacePrefix;
|
||||
private static RunnerLocation? runnerLocation = null;
|
||||
|
||||
public Configuration()
|
||||
{
|
||||
kubeConfigFile = GetNullableEnvVarOrDefault("KUBECONFIG", null);
|
||||
logPath = GetEnvVarOrDefault("LOGPATH", "CodexTestLogs");
|
||||
logDebug = GetEnvVarOrDefault("LOGDEBUG", "false").ToLowerInvariant() == "true";
|
||||
dataFilesPath = GetEnvVarOrDefault("DATAFILEPATH", "TestDataFiles");
|
||||
codexLogLevel = ParseEnum.Parse<CodexLogLevel>(GetEnvVarOrDefault("LOGLEVEL", nameof(CodexLogLevel.Trace)));
|
||||
k8sNamespacePrefix = "ct-";
|
||||
}
|
||||
|
||||
public Configuration(string? kubeConfigFile, string logPath, bool logDebug, string dataFilesPath, CodexLogLevel codexLogLevel, string k8sNamespacePrefix)
|
||||
{
|
||||
this.kubeConfigFile = kubeConfigFile;
|
||||
this.logPath = logPath;
|
||||
this.logDebug = logDebug;
|
||||
this.dataFilesPath = dataFilesPath;
|
||||
this.codexLogLevel = codexLogLevel;
|
||||
this.k8sNamespacePrefix = k8sNamespacePrefix;
|
||||
}
|
||||
|
||||
public KubernetesWorkflow.Configuration GetK8sConfiguration(ITimeSet timeSet)
|
||||
{
|
||||
return new KubernetesWorkflow.Configuration(
|
||||
k8sNamespacePrefix: k8sNamespacePrefix,
|
||||
kubeConfigFile: kubeConfigFile,
|
||||
operationTimeout: timeSet.K8sOperationTimeout(),
|
||||
retryDelay: timeSet.WaitForK8sServiceDelay()
|
||||
);
|
||||
}
|
||||
|
||||
public Logging.LogConfig GetLogConfig()
|
||||
{
|
||||
return new Logging.LogConfig(logPath, debugEnabled: logDebug);
|
||||
}
|
||||
|
||||
public string GetFileManagerFolder()
|
||||
{
|
||||
return dataFilesPath;
|
||||
}
|
||||
|
||||
public CodexLogLevel GetCodexLogLevel()
|
||||
{
|
||||
return codexLogLevel;
|
||||
}
|
||||
|
||||
public Address GetAddress(RunningContainer container)
|
||||
{
|
||||
if (runnerLocation == null)
|
||||
{
|
||||
runnerLocation = RunnerLocationUtils.DetermineRunnerLocation(container);
|
||||
}
|
||||
|
||||
if (runnerLocation == RunnerLocation.InternalToCluster)
|
||||
{
|
||||
return container.ClusterInternalAddress;
|
||||
}
|
||||
return container.ClusterExternalAddress;
|
||||
}
|
||||
|
||||
private static string GetEnvVarOrDefault(string varName, string defaultValue)
|
||||
{
|
||||
var v = Environment.GetEnvironmentVariable(varName);
|
||||
if (v == null) return defaultValue;
|
||||
return v;
|
||||
}
|
||||
|
||||
private static string? GetNullableEnvVarOrDefault(string varName, string? defaultValue)
|
||||
{
|
||||
var v = Environment.GetEnvironmentVariable(varName);
|
||||
if (v == null) return defaultValue;
|
||||
return v;
|
||||
}
|
||||
}
|
||||
|
||||
public enum RunnerLocation
|
||||
{
|
||||
ExternalToCluster,
|
||||
InternalToCluster,
|
||||
}
|
||||
|
||||
public static class RunnerLocationUtils
|
||||
{
|
||||
private static bool alreadyDidThat = false;
|
||||
|
||||
public static RunnerLocation DetermineRunnerLocation(RunningContainer container)
|
||||
{
|
||||
// We want to be sure we don't ping more often than strictly necessary.
|
||||
// If we have already determined the location during this application
|
||||
// lifetime, don't do it again.
|
||||
if (alreadyDidThat) throw new Exception("We already did that.");
|
||||
alreadyDidThat = true;
|
||||
|
||||
if (PingHost(container.Pod.PodInfo.Ip))
|
||||
{
|
||||
return RunnerLocation.InternalToCluster;
|
||||
}
|
||||
if (PingHost(Format(container.ClusterExternalAddress)))
|
||||
{
|
||||
return RunnerLocation.ExternalToCluster;
|
||||
}
|
||||
|
||||
throw new Exception("Unable to determine runner location.");
|
||||
}
|
||||
|
||||
private static string Format(Address host)
|
||||
{
|
||||
return host.Host
|
||||
.Replace("http://", "")
|
||||
.Replace("https://", "");
|
||||
}
|
||||
|
||||
private static bool PingHost(string host)
|
||||
{
|
||||
try
|
||||
{
|
||||
using var pinger = new Ping();
|
||||
PingReply reply = pinger.Send(host);
|
||||
return reply.Status == IPStatus.Success;
|
||||
}
|
||||
catch (PingException)
|
||||
{
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,40 +0,0 @@
|
|||
using KubernetesWorkflow;
|
||||
using Logging;
|
||||
|
||||
namespace DistTestCore
|
||||
{
|
||||
public abstract class DefaultContainerRecipe : ContainerRecipeFactory
|
||||
{
|
||||
public static string TestsType { get; set; } = "NotSet";
|
||||
public static ApplicationIds? ApplicationIds { get; set; } = null;
|
||||
|
||||
protected abstract void InitializeRecipe(StartupConfig config);
|
||||
|
||||
protected override void Initialize(StartupConfig config)
|
||||
{
|
||||
Add("tests-type", TestsType);
|
||||
Add("runid", NameUtils.GetRunId());
|
||||
Add("testid", NameUtils.GetTestId());
|
||||
Add("category", NameUtils.GetCategoryName());
|
||||
Add("fixturename", NameUtils.GetRawFixtureName());
|
||||
Add("testname", NameUtils.GetTestMethodName());
|
||||
|
||||
if (ApplicationIds != null)
|
||||
{
|
||||
Add("codexid", ApplicationIds.CodexId);
|
||||
Add("gethid", ApplicationIds.GethId);
|
||||
Add("prometheusid", ApplicationIds.PrometheusId);
|
||||
Add("codexcontractsid", ApplicationIds.CodexContractsId);
|
||||
Add("grafanaid", ApplicationIds.GrafanaId);
|
||||
}
|
||||
Add("app", AppName);
|
||||
|
||||
InitializeRecipe(config);
|
||||
}
|
||||
|
||||
private void Add(string name, string value)
|
||||
{
|
||||
AddPodLabel(name, value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,218 +0,0 @@
|
|||
using Logging;
|
||||
using NUnit.Framework;
|
||||
using Utils;
|
||||
|
||||
namespace DistTestCore
|
||||
{
|
||||
public interface IFileManager
|
||||
{
|
||||
TestFile CreateEmptyTestFile(string label = "");
|
||||
TestFile GenerateTestFile(ByteSize size, string label = "");
|
||||
void DeleteAllTestFiles();
|
||||
void PushFileSet();
|
||||
void PopFileSet();
|
||||
}
|
||||
|
||||
public class FileManager : IFileManager
|
||||
{
|
||||
public const int ChunkSize = 1024 * 1024 * 100;
|
||||
private static NumberSource folderNumberSource = new NumberSource(0);
|
||||
private readonly Random random = new Random();
|
||||
private readonly BaseLog log;
|
||||
private readonly string folder;
|
||||
private readonly List<List<TestFile>> fileSetStack = new List<List<TestFile>>();
|
||||
|
||||
public FileManager(BaseLog log, Configuration configuration)
|
||||
{
|
||||
folder = Path.Combine(configuration.GetFileManagerFolder(), folderNumberSource.GetNextNumber().ToString("D5"));
|
||||
|
||||
EnsureDirectory();
|
||||
this.log = log;
|
||||
}
|
||||
|
||||
public TestFile CreateEmptyTestFile(string label = "")
|
||||
{
|
||||
var path = Path.Combine(folder, Guid.NewGuid().ToString() + "_test.bin");
|
||||
var result = new TestFile(log, path, label);
|
||||
File.Create(result.Filename).Close();
|
||||
if (fileSetStack.Any()) fileSetStack.Last().Add(result);
|
||||
return result;
|
||||
}
|
||||
|
||||
public TestFile GenerateTestFile(ByteSize size, string label)
|
||||
{
|
||||
var sw = Stopwatch.Begin(log);
|
||||
var result = GenerateFile(size, label);
|
||||
sw.End($"Generated file '{result.Describe()}'.");
|
||||
return result;
|
||||
}
|
||||
|
||||
public void DeleteAllTestFiles()
|
||||
{
|
||||
DeleteDirectory();
|
||||
}
|
||||
|
||||
public void PushFileSet()
|
||||
{
|
||||
fileSetStack.Add(new List<TestFile>());
|
||||
}
|
||||
|
||||
public void PopFileSet()
|
||||
{
|
||||
if (!fileSetStack.Any()) return;
|
||||
var pop = fileSetStack.Last();
|
||||
fileSetStack.Remove(pop);
|
||||
|
||||
foreach (var file in pop)
|
||||
{
|
||||
try
|
||||
{
|
||||
File.Delete(file.Filename);
|
||||
}
|
||||
catch { }
|
||||
}
|
||||
}
|
||||
|
||||
private TestFile GenerateFile(ByteSize size, string label)
|
||||
{
|
||||
var result = CreateEmptyTestFile(label);
|
||||
CheckSpaceAvailable(result, size);
|
||||
|
||||
GenerateFileBytes(result, size);
|
||||
return result;
|
||||
}
|
||||
|
||||
private void CheckSpaceAvailable(TestFile testFile, ByteSize size)
|
||||
{
|
||||
var file = new FileInfo(testFile.Filename);
|
||||
var drive = new DriveInfo(file.Directory!.Root.FullName);
|
||||
|
||||
var spaceAvailable = drive.TotalFreeSpace;
|
||||
|
||||
if (spaceAvailable < size.SizeInBytes)
|
||||
{
|
||||
var msg = $"Inconclusive: Not enough disk space to perform test. " +
|
||||
$"{Formatter.FormatByteSize(size.SizeInBytes)} required. " +
|
||||
$"{Formatter.FormatByteSize(spaceAvailable)} available.";
|
||||
|
||||
log.Log(msg);
|
||||
Assert.Inconclusive(msg);
|
||||
}
|
||||
}
|
||||
|
||||
private void GenerateFileBytes(TestFile result, ByteSize size)
|
||||
{
|
||||
long bytesLeft = size.SizeInBytes;
|
||||
int chunkSize = ChunkSize;
|
||||
while (bytesLeft > 0)
|
||||
{
|
||||
try
|
||||
{
|
||||
var length = Math.Min(bytesLeft, chunkSize);
|
||||
AppendRandomBytesToFile(result, length);
|
||||
bytesLeft -= length;
|
||||
}
|
||||
catch
|
||||
{
|
||||
chunkSize = chunkSize / 2;
|
||||
if (chunkSize < 1024) throw;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void AppendRandomBytesToFile(TestFile result, long length)
|
||||
{
|
||||
var bytes = new byte[length];
|
||||
random.NextBytes(bytes);
|
||||
using var stream = new FileStream(result.Filename, FileMode.Append);
|
||||
stream.Write(bytes, 0, bytes.Length);
|
||||
}
|
||||
|
||||
private void EnsureDirectory()
|
||||
{
|
||||
if (!Directory.Exists(folder)) Directory.CreateDirectory(folder);
|
||||
}
|
||||
|
||||
private void DeleteDirectory()
|
||||
{
|
||||
Directory.Delete(folder, true);
|
||||
}
|
||||
}
|
||||
|
||||
public class TestFile
|
||||
{
|
||||
private readonly BaseLog log;
|
||||
|
||||
public TestFile(BaseLog log, string filename, string label)
|
||||
{
|
||||
this.log = log;
|
||||
Filename = filename;
|
||||
Label = label;
|
||||
}
|
||||
|
||||
public string Filename { get; }
|
||||
public string Label { get; }
|
||||
|
||||
public void AssertIsEqual(TestFile? actual)
|
||||
{
|
||||
var sw = Stopwatch.Begin(log);
|
||||
try
|
||||
{
|
||||
AssertEqual(actual);
|
||||
}
|
||||
finally
|
||||
{
|
||||
sw.End($"{nameof(TestFile)}.{nameof(AssertIsEqual)}");
|
||||
}
|
||||
}
|
||||
|
||||
public string Describe()
|
||||
{
|
||||
var sizePostfix = $" ({Formatter.FormatByteSize(GetFileSize())})";
|
||||
if (!string.IsNullOrEmpty(Label)) return Label + sizePostfix;
|
||||
return $"'{Filename}'{sizePostfix}";
|
||||
}
|
||||
|
||||
private void AssertEqual(TestFile? actual)
|
||||
{
|
||||
if (actual == null) Assert.Fail("TestFile is null.");
|
||||
if (actual == this || actual!.Filename == Filename) Assert.Fail("TestFile is compared to itself.");
|
||||
|
||||
Assert.That(actual.GetFileSize(), Is.EqualTo(GetFileSize()), "Files are not of equal length.");
|
||||
|
||||
using var streamExpected = new FileStream(Filename, FileMode.Open, FileAccess.Read);
|
||||
using var streamActual = new FileStream(actual.Filename, FileMode.Open, FileAccess.Read);
|
||||
|
||||
var bytesExpected = new byte[FileManager.ChunkSize];
|
||||
var bytesActual = new byte[FileManager.ChunkSize];
|
||||
|
||||
var readExpected = 0;
|
||||
var readActual = 0;
|
||||
|
||||
while (true)
|
||||
{
|
||||
readExpected = streamExpected.Read(bytesExpected, 0, FileManager.ChunkSize);
|
||||
readActual = streamActual.Read(bytesActual, 0, FileManager.ChunkSize);
|
||||
|
||||
if (readExpected == 0 && readActual == 0)
|
||||
{
|
||||
log.Log($"OK: '{Describe()}' is equal to '{actual.Describe()}'.");
|
||||
return;
|
||||
}
|
||||
|
||||
Assert.That(readActual, Is.EqualTo(readExpected), "Unable to read buffers of equal length.");
|
||||
|
||||
for (var i = 0; i < readActual; i++)
|
||||
{
|
||||
if (bytesExpected[i] != bytesActual[i]) Assert.Fail("File contents not equal.");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private long GetFileSize()
|
||||
{
|
||||
var info = new FileInfo(Filename);
|
||||
return info.Length;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,99 +0,0 @@
|
|||
using DistTestCore.Marketplace;
|
||||
|
||||
namespace DistTestCore
|
||||
{
|
||||
public class GethStarter : BaseStarter
|
||||
{
|
||||
private readonly MarketplaceNetworkCache marketplaceNetworkCache;
|
||||
private readonly GethCompanionNodeStarter companionNodeStarter;
|
||||
|
||||
public GethStarter(TestLifecycle lifecycle)
|
||||
: base(lifecycle)
|
||||
{
|
||||
marketplaceNetworkCache = new MarketplaceNetworkCache(
|
||||
new GethBootstrapNodeStarter(lifecycle),
|
||||
new CodexContractsStarter(lifecycle));
|
||||
companionNodeStarter = new GethCompanionNodeStarter(lifecycle);
|
||||
}
|
||||
|
||||
public GethStartResult BringOnlineMarketplaceFor(CodexSetup codexSetup)
|
||||
{
|
||||
if (codexSetup.MarketplaceConfig == null) return CreateMarketplaceUnavailableResult();
|
||||
|
||||
var marketplaceNetwork = marketplaceNetworkCache.Get();
|
||||
var companionNode = StartCompanionNode(codexSetup, marketplaceNetwork);
|
||||
|
||||
LogStart("Setting up initial balance...");
|
||||
TransferInitialBalance(marketplaceNetwork, codexSetup.MarketplaceConfig, companionNode);
|
||||
LogEnd($"Initial balance of {codexSetup.MarketplaceConfig.InitialTestTokens} set for {codexSetup.NumberOfNodes} nodes.");
|
||||
|
||||
return CreateGethStartResult(marketplaceNetwork, companionNode);
|
||||
}
|
||||
|
||||
public GethStartResult BringOnlineValidatorFor(CodexSetup codexSetup, GethStartResult previousResult)
|
||||
{
|
||||
// allow marketplace and validator to be enabled on the same Codex node
|
||||
if (previousResult.CompanionNode != null || (codexSetup.EnableValidator ?? false) == false) return previousResult;
|
||||
|
||||
var marketplaceNetwork = marketplaceNetworkCache.Get();
|
||||
var companionNode = StartCompanionNode(codexSetup, marketplaceNetwork);
|
||||
|
||||
return CreateGethStartResult(marketplaceNetwork, companionNode);
|
||||
}
|
||||
|
||||
private void TransferInitialBalance(MarketplaceNetwork marketplaceNetwork, MarketplaceInitialConfig marketplaceConfig, GethCompanionNodeInfo companionNode)
|
||||
{
|
||||
if (marketplaceConfig.InitialTestTokens.Amount == 0) return;
|
||||
|
||||
var interaction = marketplaceNetwork.StartInteraction(lifecycle);
|
||||
var tokenAddress = marketplaceNetwork.Marketplace.TokenAddress;
|
||||
|
||||
var accounts = companionNode.Accounts.Select(a => a.Account).ToArray();
|
||||
interaction.MintTestTokens(accounts, marketplaceConfig.InitialTestTokens.Amount, tokenAddress);
|
||||
}
|
||||
|
||||
private GethStartResult CreateGethStartResult(MarketplaceNetwork marketplaceNetwork, GethCompanionNodeInfo companionNode)
|
||||
{
|
||||
return new GethStartResult(CreateMarketplaceAccessFactory(marketplaceNetwork), marketplaceNetwork, companionNode);
|
||||
}
|
||||
|
||||
private GethStartResult CreateMarketplaceUnavailableResult()
|
||||
{
|
||||
return new GethStartResult(new MarketplaceUnavailableAccessFactory(), null!, null!);
|
||||
}
|
||||
|
||||
private IMarketplaceAccessFactory CreateMarketplaceAccessFactory(MarketplaceNetwork marketplaceNetwork)
|
||||
{
|
||||
return new GethMarketplaceAccessFactory(lifecycle, marketplaceNetwork);
|
||||
}
|
||||
|
||||
private GethCompanionNodeInfo StartCompanionNode(CodexSetup codexSetup, MarketplaceNetwork marketplaceNetwork)
|
||||
{
|
||||
return companionNodeStarter.StartCompanionNodeFor(codexSetup, marketplaceNetwork);
|
||||
}
|
||||
}
|
||||
|
||||
public class MarketplaceNetworkCache
|
||||
{
|
||||
private readonly GethBootstrapNodeStarter bootstrapNodeStarter;
|
||||
private readonly CodexContractsStarter codexContractsStarter;
|
||||
private MarketplaceNetwork? network;
|
||||
|
||||
public MarketplaceNetworkCache(GethBootstrapNodeStarter bootstrapNodeStarter, CodexContractsStarter codexContractsStarter)
|
||||
{
|
||||
this.bootstrapNodeStarter = bootstrapNodeStarter;
|
||||
this.codexContractsStarter = codexContractsStarter;
|
||||
}
|
||||
|
||||
public MarketplaceNetwork Get()
|
||||
{
|
||||
if (network == null)
|
||||
{
|
||||
var bootstrapInfo = bootstrapNodeStarter.StartGethBootstrapNode();
|
||||
var marketplaceInfo = codexContractsStarter.Start(bootstrapInfo);
|
||||
network = new MarketplaceNetwork(bootstrapInfo, marketplaceInfo );
|
||||
}
|
||||
return network;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,193 +0,0 @@
|
|||
using DistTestCore.Metrics;
|
||||
using IdentityModel.Client;
|
||||
using KubernetesWorkflow;
|
||||
using Newtonsoft.Json;
|
||||
using System.Reflection;
|
||||
using Utils;
|
||||
|
||||
namespace DistTestCore
|
||||
{
|
||||
public class GrafanaStarter : BaseStarter
|
||||
{
|
||||
private const string StorageQuotaThresholdReplaceToken = "\"<CODEX_STORAGEQUOTA>\"";
|
||||
private const string BytesUsedGraphAxisSoftMaxReplaceToken = "\"<CODEX_BYTESUSED_SOFTMAX>\"";
|
||||
|
||||
public GrafanaStarter(TestLifecycle lifecycle)
|
||||
: base(lifecycle)
|
||||
{
|
||||
}
|
||||
|
||||
public GrafanaStartInfo StartDashboard(RunningContainer prometheusContainer, CodexSetup codexSetup)
|
||||
{
|
||||
LogStart($"Starting dashboard server");
|
||||
|
||||
var grafanaContainer = StartGrafanaContainer();
|
||||
var grafanaAddress = lifecycle.Configuration.GetAddress(grafanaContainer);
|
||||
|
||||
var http = new Http(lifecycle.Log, new DefaultTimeSet(), grafanaAddress, "api/", AddBasicAuth);
|
||||
|
||||
Log("Connecting datasource...");
|
||||
AddDataSource(http, prometheusContainer);
|
||||
|
||||
Log("Uploading dashboard configurations...");
|
||||
var jsons = ReadEachDashboardJsonFile(codexSetup);
|
||||
var dashboardUrls = jsons.Select(j => UploadDashboard(http, grafanaContainer, j)).ToArray();
|
||||
|
||||
LogEnd("Dashboard server started.");
|
||||
|
||||
return new GrafanaStartInfo(dashboardUrls, grafanaContainer);
|
||||
}
|
||||
|
||||
private RunningContainer StartGrafanaContainer()
|
||||
{
|
||||
var startupConfig = new StartupConfig();
|
||||
|
||||
var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
|
||||
var grafanaContainers = workflow.Start(1, Location.Unspecified, new GrafanaContainerRecipe(), startupConfig);
|
||||
if (grafanaContainers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 dashboard container to be created.");
|
||||
|
||||
return grafanaContainers.Containers.First();
|
||||
}
|
||||
|
||||
private void AddBasicAuth(HttpClient client)
|
||||
{
|
||||
client.SetBasicAuthentication(
|
||||
GrafanaContainerRecipe.DefaultAdminUser,
|
||||
GrafanaContainerRecipe.DefaultAdminPassword);
|
||||
}
|
||||
|
||||
private static void AddDataSource(Http http, RunningContainer prometheusContainer)
|
||||
{
|
||||
var prometheusAddress = prometheusContainer.ClusterExternalAddress;
|
||||
var prometheusUrl = prometheusAddress.Host + ":" + prometheusAddress.Port;
|
||||
var response = http.HttpPostJson<GrafanaDataSourceRequest, GrafanaDataSourceResponse>("datasources", new GrafanaDataSourceRequest
|
||||
{
|
||||
uid = "c89eaad3-9184-429f-ac94-8ba0b1824dbb",
|
||||
name = "CodexPrometheus",
|
||||
type = "prometheus",
|
||||
url = prometheusUrl,
|
||||
access = "proxy",
|
||||
basicAuth = false,
|
||||
jsonData = new GrafanaDataSourceJsonData
|
||||
{
|
||||
httpMethod = "POST"
|
||||
}
|
||||
});
|
||||
|
||||
if (response.message != "Datasource added")
|
||||
{
|
||||
throw new Exception("Test infra failure: Failed to add datasource to dashboard: " + response.message);
|
||||
}
|
||||
}
|
||||
|
||||
public static string UploadDashboard(Http http, RunningContainer grafanaContainer, string dashboardJson)
|
||||
{
|
||||
var request = GetDashboardCreateRequest(dashboardJson);
|
||||
var response = http.HttpPostString("dashboards/db", request);
|
||||
var jsonResponse = JsonConvert.DeserializeObject<GrafanaPostDashboardResponse>(response);
|
||||
if (jsonResponse == null || string.IsNullOrEmpty(jsonResponse.url)) throw new Exception("Failed to upload dashboard.");
|
||||
|
||||
var grafanaAddress = grafanaContainer.ClusterExternalAddress;
|
||||
return grafanaAddress.Host + ":" + grafanaAddress.Port + jsonResponse.url;
|
||||
}
|
||||
|
||||
private static string[] ReadEachDashboardJsonFile(CodexSetup codexSetup)
|
||||
{
|
||||
var assembly = Assembly.GetExecutingAssembly();
|
||||
var resourceNames = new[]
|
||||
{
|
||||
"DistTestCore.Metrics.dashboard.json"
|
||||
};
|
||||
|
||||
return resourceNames.Select(r => GetManifestResource(assembly, r, codexSetup)).ToArray();
|
||||
}
|
||||
|
||||
private static string GetManifestResource(Assembly assembly, string resourceName, CodexSetup codexSetup)
|
||||
{
|
||||
using var stream = assembly.GetManifestResourceStream(resourceName);
|
||||
if (stream == null) throw new Exception("Unable to find resource " + resourceName);
|
||||
using var reader = new StreamReader(stream);
|
||||
return ApplyReplacements(reader.ReadToEnd(), codexSetup);
|
||||
}
|
||||
|
||||
private static string ApplyReplacements(string input, CodexSetup codexSetup)
|
||||
{
|
||||
var quotaString = GetQuotaString(codexSetup);
|
||||
var softMaxString = GetSoftMaxString(codexSetup);
|
||||
|
||||
return input
|
||||
.Replace(StorageQuotaThresholdReplaceToken, quotaString)
|
||||
.Replace(BytesUsedGraphAxisSoftMaxReplaceToken, softMaxString);
|
||||
}
|
||||
|
||||
private static string GetQuotaString(CodexSetup codexSetup)
|
||||
{
|
||||
return GetCodexStorageQuotaInBytes(codexSetup).ToString();
|
||||
}
|
||||
|
||||
private static string GetSoftMaxString(CodexSetup codexSetup)
|
||||
{
|
||||
var quota = GetCodexStorageQuotaInBytes(codexSetup);
|
||||
var softMax = Convert.ToInt64(quota * 1.1); // + 10%, for nice viewing.
|
||||
return softMax.ToString();
|
||||
}
|
||||
|
||||
private static long GetCodexStorageQuotaInBytes(CodexSetup codexSetup)
|
||||
{
|
||||
if (codexSetup.StorageQuota != null) return codexSetup.StorageQuota.SizeInBytes;
|
||||
|
||||
// Codex default: 8GB
|
||||
return 8.GB().SizeInBytes;
|
||||
}
|
||||
|
||||
private static string GetDashboardCreateRequest(string dashboardJson)
|
||||
{
|
||||
return $"{{\"dashboard\": {dashboardJson} ,\"message\": \"Default Codex Dashboard\",\"overwrite\": false}}";
|
||||
}
|
||||
}
|
||||
|
||||
public class GrafanaStartInfo
|
||||
{
|
||||
public GrafanaStartInfo(string[] dashboardUrls, RunningContainer container)
|
||||
{
|
||||
DashboardUrls = dashboardUrls;
|
||||
Container = container;
|
||||
}
|
||||
|
||||
public string[] DashboardUrls { get; }
|
||||
public RunningContainer Container { get; }
|
||||
}
|
||||
|
||||
public class GrafanaDataSourceRequest
|
||||
{
|
||||
public string uid { get; set; } = string.Empty;
|
||||
public string name { get; set; } = string.Empty;
|
||||
public string type { get; set; } = string.Empty;
|
||||
public string url { get; set; } = string.Empty;
|
||||
public string access { get; set; } = string.Empty;
|
||||
public bool basicAuth { get; set; }
|
||||
public GrafanaDataSourceJsonData jsonData { get; set; } = new();
|
||||
}
|
||||
|
||||
public class GrafanaDataSourceResponse
|
||||
{
|
||||
public int id { get; set; }
|
||||
public string message { get; set; } = string.Empty;
|
||||
public string name { get; set; } = string.Empty;
|
||||
}
|
||||
|
||||
public class GrafanaDataSourceJsonData
|
||||
{
|
||||
public string httpMethod { get; set; } = string.Empty;
|
||||
}
|
||||
|
||||
public class GrafanaPostDashboardResponse
|
||||
{
|
||||
public int id { get; set; }
|
||||
public string slug { get; set; } = string.Empty;
|
||||
public string status { get; set; } = string.Empty;
|
||||
public string uid { get; set; } = string.Empty;
|
||||
public string url { get; set; } = string.Empty;
|
||||
public int version { get; set; }
|
||||
}
|
||||
}
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
using NUnit.Framework;
|
||||
|
||||
namespace DistTestCore.Logs
|
||||
{
|
||||
[AttributeUsage(AttributeTargets.Method, AllowMultiple = false)]
|
||||
public class DontDownloadLogsAndMetricsOnFailureAttribute : PropertyAttribute
|
||||
{
|
||||
public const string DontDownloadKey = "DontDownloadLogsAndMetrics";
|
||||
|
||||
public DontDownloadLogsAndMetricsOnFailureAttribute()
|
||||
: base(DontDownloadKey)
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,16 +0,0 @@
|
|||
using KubernetesWorkflow;
|
||||
|
||||
namespace DistTestCore.Marketplace
|
||||
{
|
||||
public class CodexContractsContainerConfig
|
||||
{
|
||||
public CodexContractsContainerConfig(string bootstrapNodeIp, Port jsonRpcPort)
|
||||
{
|
||||
BootstrapNodeIp = bootstrapNodeIp;
|
||||
JsonRpcPort = jsonRpcPort;
|
||||
}
|
||||
|
||||
public string BootstrapNodeIp { get; }
|
||||
public Port JsonRpcPort { get; }
|
||||
}
|
||||
}
|
||||
|
|
@ -1,42 +0,0 @@
|
|||
using KubernetesWorkflow;
|
||||
using NethereumWorkflow;
|
||||
|
||||
namespace DistTestCore.Marketplace
|
||||
{
|
||||
public class GethBootstrapNodeInfo
|
||||
{
|
||||
public GethBootstrapNodeInfo(RunningContainers runningContainers, AllGethAccounts allAccounts, string pubKey, Port discoveryPort)
|
||||
{
|
||||
RunningContainers = runningContainers;
|
||||
AllAccounts = allAccounts;
|
||||
Account = allAccounts.Accounts[0];
|
||||
PubKey = pubKey;
|
||||
DiscoveryPort = discoveryPort;
|
||||
}
|
||||
|
||||
public RunningContainers RunningContainers { get; }
|
||||
public AllGethAccounts AllAccounts { get; }
|
||||
public GethAccount Account { get; }
|
||||
public string PubKey { get; }
|
||||
public Port DiscoveryPort { get; }
|
||||
|
||||
public NethereumInteraction StartInteraction(TestLifecycle lifecycle)
|
||||
{
|
||||
var address = lifecycle.Configuration.GetAddress(RunningContainers.Containers[0]);
|
||||
var account = Account;
|
||||
|
||||
var creator = new NethereumInteractionCreator(lifecycle.Log, address.Host, address.Port, account.PrivateKey);
|
||||
return creator.CreateWorkflow();
|
||||
}
|
||||
}
|
||||
|
||||
public class AllGethAccounts
|
||||
{
|
||||
public GethAccount[] Accounts { get; }
|
||||
|
||||
public AllGethAccounts(GethAccount[] accounts)
|
||||
{
|
||||
Accounts = accounts;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,40 +0,0 @@
|
|||
using KubernetesWorkflow;
|
||||
|
||||
namespace DistTestCore.Marketplace
|
||||
{
|
||||
public class GethBootstrapNodeStarter : BaseStarter
|
||||
{
|
||||
public GethBootstrapNodeStarter(TestLifecycle lifecycle)
|
||||
: base(lifecycle)
|
||||
{
|
||||
}
|
||||
|
||||
public GethBootstrapNodeInfo StartGethBootstrapNode()
|
||||
{
|
||||
LogStart("Starting Geth bootstrap node...");
|
||||
var startupConfig = CreateBootstrapStartupConfig();
|
||||
|
||||
var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
|
||||
var containers = workflow.Start(1, Location.Unspecified, new GethContainerRecipe(), startupConfig);
|
||||
if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 Geth bootstrap node to be created. Test infra failure.");
|
||||
var bootstrapContainer = containers.Containers[0];
|
||||
|
||||
var extractor = new ContainerInfoExtractor(lifecycle.Log, workflow, bootstrapContainer);
|
||||
var accounts = extractor.ExtractAccounts();
|
||||
var pubKey = extractor.ExtractPubKey();
|
||||
var discoveryPort = bootstrapContainer.Recipe.GetPortByTag(GethContainerRecipe.DiscoveryPortTag);
|
||||
var result = new GethBootstrapNodeInfo(containers, accounts, pubKey, discoveryPort);
|
||||
|
||||
LogEnd($"Geth bootstrap node started with account '{result.Account.Account}'");
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private StartupConfig CreateBootstrapStartupConfig()
|
||||
{
|
||||
var config = new StartupConfig();
|
||||
config.Add(new GethStartupConfig(true, null!, 0, 0));
|
||||
return config;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,38 +0,0 @@
|
|||
using KubernetesWorkflow;
|
||||
using NethereumWorkflow;
|
||||
|
||||
namespace DistTestCore.Marketplace
|
||||
{
|
||||
public class GethCompanionNodeInfo
|
||||
{
|
||||
public GethCompanionNodeInfo(RunningContainer runningContainer, GethAccount[] accounts)
|
||||
{
|
||||
RunningContainer = runningContainer;
|
||||
Accounts = accounts;
|
||||
}
|
||||
|
||||
public RunningContainer RunningContainer { get; }
|
||||
public GethAccount[] Accounts { get; }
|
||||
|
||||
public NethereumInteraction StartInteraction(TestLifecycle lifecycle, GethAccount account)
|
||||
{
|
||||
var address = lifecycle.Configuration.GetAddress(RunningContainer);
|
||||
var privateKey = account.PrivateKey;
|
||||
|
||||
var creator = new NethereumInteractionCreator(lifecycle.Log, address.Host, address.Port, privateKey);
|
||||
return creator.CreateWorkflow();
|
||||
}
|
||||
}
|
||||
|
||||
public class GethAccount
|
||||
{
|
||||
public GethAccount(string account, string privateKey)
|
||||
{
|
||||
Account = account;
|
||||
PrivateKey = privateKey;
|
||||
}
|
||||
|
||||
public string Account { get; }
|
||||
public string PrivateKey { get; }
|
||||
}
|
||||
}
|
||||
|
|
@ -1,77 +0,0 @@
|
|||
using KubernetesWorkflow;
|
||||
using Utils;
|
||||
|
||||
namespace DistTestCore.Marketplace
|
||||
{
|
||||
public class GethCompanionNodeStarter : BaseStarter
|
||||
{
|
||||
private int companionAccountIndex = 0;
|
||||
|
||||
public GethCompanionNodeStarter(TestLifecycle lifecycle)
|
||||
: base(lifecycle)
|
||||
{
|
||||
}
|
||||
|
||||
public GethCompanionNodeInfo StartCompanionNodeFor(CodexSetup codexSetup, MarketplaceNetwork marketplace)
|
||||
{
|
||||
LogStart($"Initializing companion for {codexSetup.NumberOfNodes} Codex nodes.");
|
||||
|
||||
var config = CreateCompanionNodeStartupConfig(marketplace.Bootstrap, codexSetup.NumberOfNodes);
|
||||
|
||||
var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
|
||||
var containers = workflow.Start(1, Location.Unspecified, new GethContainerRecipe(), CreateStartupConfig(config));
|
||||
if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected one Geth companion node to be created. Test infra failure.");
|
||||
var container = containers.Containers[0];
|
||||
|
||||
var node = CreateCompanionInfo(container, marketplace, config);
|
||||
EnsureCompanionNodeIsSynced(node, marketplace);
|
||||
|
||||
LogEnd($"Initialized one companion node for {codexSetup.NumberOfNodes} Codex nodes. Their accounts: [{string.Join(",", node.Accounts.Select(a => a.Account))}]");
|
||||
return node;
|
||||
}
|
||||
|
||||
private GethCompanionNodeInfo CreateCompanionInfo(RunningContainer container, MarketplaceNetwork marketplace, GethStartupConfig config)
|
||||
{
|
||||
var accounts = ExtractAccounts(marketplace, config);
|
||||
return new GethCompanionNodeInfo(container, accounts);
|
||||
}
|
||||
|
||||
private static GethAccount[] ExtractAccounts(MarketplaceNetwork marketplace, GethStartupConfig config)
|
||||
{
|
||||
return marketplace.Bootstrap.AllAccounts.Accounts
|
||||
.Skip(1 + config.CompanionAccountStartIndex)
|
||||
.Take(config.NumberOfCompanionAccounts)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private void EnsureCompanionNodeIsSynced(GethCompanionNodeInfo node, MarketplaceNetwork marketplace)
|
||||
{
|
||||
try
|
||||
{
|
||||
Time.WaitUntil(() =>
|
||||
{
|
||||
var interaction = node.StartInteraction(lifecycle, node.Accounts.First());
|
||||
return interaction.IsSynced(marketplace.Marketplace.Address, marketplace.Marketplace.Abi);
|
||||
}, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(3));
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
throw new Exception("Geth companion node did not sync within timeout. Test infra failure.", e);
|
||||
}
|
||||
}
|
||||
|
||||
private GethStartupConfig CreateCompanionNodeStartupConfig(GethBootstrapNodeInfo bootstrapNode, int numberOfAccounts)
|
||||
{
|
||||
var config = new GethStartupConfig(false, bootstrapNode, companionAccountIndex, numberOfAccounts);
|
||||
companionAccountIndex += numberOfAccounts;
|
||||
return config;
|
||||
}
|
||||
|
||||
private StartupConfig CreateStartupConfig(GethStartupConfig gethConfig)
|
||||
{
|
||||
var config = new StartupConfig();
|
||||
config.Add(gethConfig);
|
||||
return config;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,73 +0,0 @@
|
|||
using KubernetesWorkflow;
|
||||
|
||||
namespace DistTestCore.Marketplace
|
||||
{
|
||||
public class GethContainerRecipe : DefaultContainerRecipe
|
||||
{
|
||||
private const string defaultArgs = "--ipcdisable --syncmode full";
|
||||
|
||||
public const string HttpPortTag = "http_port";
|
||||
public const string DiscoveryPortTag = "disc_port";
|
||||
public const string AccountsFilename = "accounts.csv";
|
||||
|
||||
public override string AppName => "geth";
|
||||
public override string Image => "codexstorage/dist-tests-geth:latest";
|
||||
|
||||
protected override void InitializeRecipe(StartupConfig startupConfig)
|
||||
{
|
||||
var config = startupConfig.Get<GethStartupConfig>();
|
||||
|
||||
var args = CreateArgs(config);
|
||||
|
||||
AddEnvVar("GETH_ARGS", args);
|
||||
}
|
||||
|
||||
private string CreateArgs(GethStartupConfig config)
|
||||
{
|
||||
var discovery = AddInternalPort(tag: DiscoveryPortTag);
|
||||
|
||||
if (config.IsBootstrapNode)
|
||||
{
|
||||
return CreateBootstapArgs(discovery);
|
||||
}
|
||||
|
||||
return CreateCompanionArgs(discovery, config);
|
||||
}
|
||||
|
||||
private string CreateBootstapArgs(Port discovery)
|
||||
{
|
||||
AddEnvVar("ENABLE_MINER", "1");
|
||||
UnlockAccounts(0, 1);
|
||||
var exposedPort = AddExposedPort(tag: HttpPortTag);
|
||||
return $"--http.port {exposedPort.Number} --port {discovery.Number} --discovery.port {discovery.Number} {defaultArgs}";
|
||||
}
|
||||
|
||||
private string CreateCompanionArgs(Port discovery, GethStartupConfig config)
|
||||
{
|
||||
UnlockAccounts(
|
||||
config.CompanionAccountStartIndex + 1,
|
||||
config.NumberOfCompanionAccounts);
|
||||
|
||||
var port = AddInternalPort();
|
||||
var authRpc = AddInternalPort();
|
||||
var httpPort = AddExposedPort(tag: HttpPortTag);
|
||||
|
||||
var bootPubKey = config.BootstrapNode.PubKey;
|
||||
var bootIp = config.BootstrapNode.RunningContainers.Containers[0].Pod.PodInfo.Ip;
|
||||
var bootPort = config.BootstrapNode.DiscoveryPort.Number;
|
||||
var bootstrapArg = $"--bootnodes enode://{bootPubKey}@{bootIp}:{bootPort} --nat=extip:{bootIp}";
|
||||
|
||||
return $"--port {port.Number} --discovery.port {discovery.Number} --authrpc.port {authRpc.Number} --http.addr 0.0.0.0 --http.port {httpPort.Number} --ws --ws.addr 0.0.0.0 --ws.port {httpPort.Number} {bootstrapArg} {defaultArgs}";
|
||||
}
|
||||
|
||||
private void UnlockAccounts(int startIndex, int numberOfAccounts)
|
||||
{
|
||||
if (startIndex < 0) throw new ArgumentException();
|
||||
if (numberOfAccounts < 1) throw new ArgumentException();
|
||||
if (startIndex + numberOfAccounts > 1000) throw new ArgumentException("Out of accounts!");
|
||||
|
||||
AddEnvVar("UNLOCK_START_INDEX", startIndex.ToString());
|
||||
AddEnvVar("UNLOCK_NUMBER", numberOfAccounts.ToString());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,19 +0,0 @@
|
|||
using Newtonsoft.Json;
|
||||
|
||||
namespace DistTestCore.Marketplace
|
||||
{
|
||||
public class GethStartResult
|
||||
{
|
||||
public GethStartResult(IMarketplaceAccessFactory marketplaceAccessFactory, MarketplaceNetwork marketplaceNetwork, GethCompanionNodeInfo companionNode)
|
||||
{
|
||||
MarketplaceAccessFactory = marketplaceAccessFactory;
|
||||
MarketplaceNetwork = marketplaceNetwork;
|
||||
CompanionNode = companionNode;
|
||||
}
|
||||
|
||||
[JsonIgnore]
|
||||
public IMarketplaceAccessFactory MarketplaceAccessFactory { get; }
|
||||
public MarketplaceNetwork MarketplaceNetwork { get; }
|
||||
public GethCompanionNodeInfo CompanionNode { get; }
|
||||
}
|
||||
}
|
||||
|
|
@ -1,18 +0,0 @@
|
|||
namespace DistTestCore.Marketplace
|
||||
{
|
||||
public class GethStartupConfig
|
||||
{
|
||||
public GethStartupConfig(bool isBootstrapNode, GethBootstrapNodeInfo bootstrapNode, int companionAccountStartIndex, int numberOfCompanionAccounts)
|
||||
{
|
||||
IsBootstrapNode = isBootstrapNode;
|
||||
BootstrapNode = bootstrapNode;
|
||||
CompanionAccountStartIndex = companionAccountStartIndex;
|
||||
NumberOfCompanionAccounts = numberOfCompanionAccounts;
|
||||
}
|
||||
|
||||
public bool IsBootstrapNode { get; }
|
||||
public GethBootstrapNodeInfo BootstrapNode { get; }
|
||||
public int CompanionAccountStartIndex { get; }
|
||||
public int NumberOfCompanionAccounts { get; }
|
||||
}
|
||||
}
|
||||
|
|
@ -1,41 +0,0 @@
|
|||
using DistTestCore.Codex;
|
||||
|
||||
namespace DistTestCore.Marketplace
|
||||
{
|
||||
public interface IMarketplaceAccessFactory
|
||||
{
|
||||
IMarketplaceAccess CreateMarketplaceAccess(CodexAccess access);
|
||||
}
|
||||
|
||||
public class MarketplaceUnavailableAccessFactory : IMarketplaceAccessFactory
|
||||
{
|
||||
public IMarketplaceAccess CreateMarketplaceAccess(CodexAccess access)
|
||||
{
|
||||
return new MarketplaceUnavailable();
|
||||
}
|
||||
}
|
||||
|
||||
public class GethMarketplaceAccessFactory : IMarketplaceAccessFactory
|
||||
{
|
||||
private readonly TestLifecycle lifecycle;
|
||||
private readonly MarketplaceNetwork marketplaceNetwork;
|
||||
|
||||
public GethMarketplaceAccessFactory(TestLifecycle lifecycle, MarketplaceNetwork marketplaceNetwork)
|
||||
{
|
||||
this.lifecycle = lifecycle;
|
||||
this.marketplaceNetwork = marketplaceNetwork;
|
||||
}
|
||||
|
||||
public IMarketplaceAccess CreateMarketplaceAccess(CodexAccess access)
|
||||
{
|
||||
var companionNode = GetGethCompanionNode(access);
|
||||
return new MarketplaceAccess(lifecycle, marketplaceNetwork, companionNode, access);
|
||||
}
|
||||
|
||||
private GethAccount GetGethCompanionNode(CodexAccess access)
|
||||
{
|
||||
var account = access.Container.Recipe.Additionals.Single(a => a is GethAccount);
|
||||
return (GethAccount)account;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,17 +0,0 @@
|
|||
namespace DistTestCore.Marketplace
|
||||
{
|
||||
public class MarketplaceInitialConfig
|
||||
{
|
||||
public MarketplaceInitialConfig(Ether initialEth, TestToken initialTestTokens, bool isValidator)
|
||||
{
|
||||
InitialEth = initialEth;
|
||||
InitialTestTokens = initialTestTokens;
|
||||
IsValidator = isValidator;
|
||||
}
|
||||
|
||||
public Ether InitialEth { get; }
|
||||
public TestToken InitialTestTokens { get; }
|
||||
public bool IsValidator { get; }
|
||||
public int? AccountIndexOverride { get; set; }
|
||||
}
|
||||
}
|
||||
|
|
@ -1,21 +0,0 @@
|
|||
using NethereumWorkflow;
|
||||
|
||||
namespace DistTestCore.Marketplace
|
||||
{
|
||||
public class MarketplaceNetwork
|
||||
{
|
||||
public MarketplaceNetwork(GethBootstrapNodeInfo bootstrap, MarketplaceInfo marketplace)
|
||||
{
|
||||
Bootstrap = bootstrap;
|
||||
Marketplace = marketplace;
|
||||
}
|
||||
|
||||
public GethBootstrapNodeInfo Bootstrap { get; }
|
||||
public MarketplaceInfo Marketplace { get; }
|
||||
|
||||
public NethereumInteraction StartInteraction(TestLifecycle lifecycle)
|
||||
{
|
||||
return Bootstrap.StartInteraction(lifecycle);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,25 +0,0 @@
|
|||
using KubernetesWorkflow;
|
||||
|
||||
namespace DistTestCore.Metrics
|
||||
{
|
||||
public class GrafanaContainerRecipe : DefaultContainerRecipe
|
||||
{
|
||||
public override string AppName => "grafana";
|
||||
public override string Image => "grafana/grafana-oss:10.0.3";
|
||||
|
||||
public const string DefaultAdminUser = "adminium";
|
||||
public const string DefaultAdminPassword = "passwordium";
|
||||
|
||||
protected override void InitializeRecipe(StartupConfig startupConfig)
|
||||
{
|
||||
AddExposedPort(3000);
|
||||
|
||||
AddEnvVar("GF_AUTH_ANONYMOUS_ENABLED", "true");
|
||||
AddEnvVar("GF_AUTH_ANONYMOUS_ORG_NAME", "Main Org.");
|
||||
AddEnvVar("GF_AUTH_ANONYMOUS_ORG_ROLE", "Editor");
|
||||
|
||||
AddEnvVar("GF_SECURITY_ADMIN_USER", DefaultAdminUser);
|
||||
AddEnvVar("GF_SECURITY_ADMIN_PASSWORD", DefaultAdminPassword);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,81 +0,0 @@
|
|||
using DistTestCore.Helpers;
|
||||
using KubernetesWorkflow;
|
||||
using Logging;
|
||||
using NUnit.Framework;
|
||||
using NUnit.Framework.Constraints;
|
||||
using Utils;
|
||||
|
||||
namespace DistTestCore.Metrics
|
||||
{
|
||||
public interface IMetricsAccess
|
||||
{
|
||||
void AssertThat(string metricName, IResolveConstraint constraint, string message = "");
|
||||
}
|
||||
|
||||
public class MetricsAccess : IMetricsAccess
|
||||
{
|
||||
private readonly BaseLog log;
|
||||
private readonly ITimeSet timeSet;
|
||||
private readonly MetricsQuery query;
|
||||
private readonly RunningContainer node;
|
||||
|
||||
public MetricsAccess(BaseLog log, ITimeSet timeSet, MetricsQuery query, RunningContainer node)
|
||||
{
|
||||
this.log = log;
|
||||
this.timeSet = timeSet;
|
||||
this.query = query;
|
||||
this.node = node;
|
||||
}
|
||||
|
||||
public void AssertThat(string metricName, IResolveConstraint constraint, string message = "")
|
||||
{
|
||||
AssertHelpers.RetryAssert(constraint, () =>
|
||||
{
|
||||
var metricSet = GetMetricWithTimeout(metricName);
|
||||
var metricValue = metricSet.Values[0].Value;
|
||||
|
||||
log.Log($"{node.Name} metric '{metricName}' = {metricValue}");
|
||||
return metricValue;
|
||||
}, message);
|
||||
}
|
||||
|
||||
public Metrics? GetAllMetrics()
|
||||
{
|
||||
return query.GetAllMetricsForNode(node);
|
||||
}
|
||||
|
||||
private MetricsSet GetMetricWithTimeout(string metricName)
|
||||
{
|
||||
var start = DateTime.UtcNow;
|
||||
|
||||
while (true)
|
||||
{
|
||||
var mostRecent = GetMostRecent(metricName);
|
||||
if (mostRecent != null) return mostRecent;
|
||||
if (DateTime.UtcNow - start > timeSet.WaitForMetricTimeout())
|
||||
{
|
||||
Assert.Fail($"Timeout: Unable to get metric '{metricName}'.");
|
||||
throw new TimeoutException();
|
||||
}
|
||||
|
||||
Time.Sleep(TimeSpan.FromSeconds(2));
|
||||
}
|
||||
}
|
||||
|
||||
private MetricsSet? GetMostRecent(string metricName)
|
||||
{
|
||||
var result = query.GetMostRecent(metricName, node);
|
||||
if (result == null) return null;
|
||||
return result.Sets.LastOrDefault();
|
||||
}
|
||||
}
|
||||
|
||||
public class MetricsUnavailable : IMetricsAccess
|
||||
{
|
||||
public void AssertThat(string metricName, IResolveConstraint constraint, string message = "")
|
||||
{
|
||||
Assert.Fail("Incorrect test setup: Metrics were not enabled for this group of Codex nodes. Add 'EnableMetrics()' after 'SetupCodexNodes()' to enable it.");
|
||||
throw new InvalidOperationException();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,35 +0,0 @@
|
|||
using KubernetesWorkflow;
|
||||
|
||||
namespace DistTestCore.Metrics
|
||||
{
|
||||
public interface IMetricsAccessFactory
|
||||
{
|
||||
IMetricsAccess CreateMetricsAccess(RunningContainer codexContainer);
|
||||
}
|
||||
|
||||
public class MetricsUnavailableAccessFactory : IMetricsAccessFactory
|
||||
{
|
||||
public IMetricsAccess CreateMetricsAccess(RunningContainer codexContainer)
|
||||
{
|
||||
return new MetricsUnavailable();
|
||||
}
|
||||
}
|
||||
|
||||
public class CodexNodeMetricsAccessFactory : IMetricsAccessFactory
|
||||
{
|
||||
private readonly TestLifecycle lifecycle;
|
||||
private readonly RunningContainers prometheusContainer;
|
||||
|
||||
public CodexNodeMetricsAccessFactory(TestLifecycle lifecycle, RunningContainers prometheusContainer)
|
||||
{
|
||||
this.lifecycle = lifecycle;
|
||||
this.prometheusContainer = prometheusContainer;
|
||||
}
|
||||
|
||||
public IMetricsAccess CreateMetricsAccess(RunningContainer codexContainer)
|
||||
{
|
||||
var query = new MetricsQuery(lifecycle, prometheusContainer);
|
||||
return new MetricsAccess(lifecycle.Log, lifecycle.TimeSet, query, codexContainer);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,9 +0,0 @@
|
|||
namespace DistTestCore.Metrics
|
||||
{
|
||||
public enum MetricsMode
|
||||
{
|
||||
None,
|
||||
Record,
|
||||
Dashboard
|
||||
}
|
||||
}
|
||||
|
|
@ -1,52 +0,0 @@
|
|||
using DistTestCore.Codex;
|
||||
using DistTestCore.Metrics;
|
||||
using KubernetesWorkflow;
|
||||
using System.Text;
|
||||
|
||||
namespace DistTestCore
|
||||
{
|
||||
public class PrometheusStarter : BaseStarter
|
||||
{
|
||||
public PrometheusStarter(TestLifecycle lifecycle)
|
||||
: base(lifecycle)
|
||||
{
|
||||
}
|
||||
|
||||
public RunningContainers CollectMetricsFor(RunningContainers[] containers)
|
||||
{
|
||||
LogStart($"Starting metrics server for {containers.Describe()}");
|
||||
var startupConfig = new StartupConfig();
|
||||
startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(containers.Containers())));
|
||||
|
||||
var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
|
||||
var runningContainers = workflow.Start(1, Location.Unspecified, new PrometheusContainerRecipe(), startupConfig);
|
||||
if (runningContainers.Containers.Length != 1) throw new InvalidOperationException("Expected only 1 Prometheus container to be created.");
|
||||
|
||||
return runningContainers;
|
||||
}
|
||||
|
||||
private string GeneratePrometheusConfig(RunningContainer[] nodes)
|
||||
{
|
||||
var config = "";
|
||||
config += "global:\n";
|
||||
config += " scrape_interval: 10s\n";
|
||||
config += " scrape_timeout: 10s\n";
|
||||
config += "\n";
|
||||
config += "scrape_configs:\n";
|
||||
config += " - job_name: services\n";
|
||||
config += " metrics_path: /metrics\n";
|
||||
config += " static_configs:\n";
|
||||
config += " - targets:\n";
|
||||
|
||||
foreach (var node in nodes)
|
||||
{
|
||||
var ip = node.Pod.PodInfo.Ip;
|
||||
var port = node.Recipe.GetPortByTag(CodexContainerRecipe.MetricsPortTag).Number;
|
||||
config += $" - '{ip}:{port}'\n";
|
||||
}
|
||||
|
||||
var bytes = Encoding.ASCII.GetBytes(config);
|
||||
return Convert.ToBase64String(bytes);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,93 +0,0 @@
|
|||
using DistTestCore.Codex;
|
||||
using DistTestCore.Logs;
|
||||
using DistTestCore.Marketplace;
|
||||
using DistTestCore.Metrics;
|
||||
using KubernetesWorkflow;
|
||||
using Logging;
|
||||
using Utils;
|
||||
|
||||
namespace DistTestCore
|
||||
{
|
||||
public class TestLifecycle
|
||||
{
|
||||
private readonly DateTime testStart;
|
||||
|
||||
public TestLifecycle(BaseLog log, Configuration configuration, ITimeSet timeSet, string testNamespace)
|
||||
{
|
||||
Log = log;
|
||||
Configuration = configuration;
|
||||
TimeSet = timeSet;
|
||||
|
||||
WorkflowCreator = new WorkflowCreator(log, configuration.GetK8sConfiguration(timeSet), testNamespace);
|
||||
|
||||
FileManager = new FileManager(Log, configuration);
|
||||
CodexStarter = new CodexStarter(this);
|
||||
PrometheusStarter = new PrometheusStarter(this);
|
||||
GrafanaStarter = new GrafanaStarter(this);
|
||||
GethStarter = new GethStarter(this);
|
||||
testStart = DateTime.UtcNow;
|
||||
CodexVersion = null;
|
||||
|
||||
Log.WriteLogTag();
|
||||
}
|
||||
|
||||
public BaseLog Log { get; }
|
||||
public Configuration Configuration { get; }
|
||||
public ITimeSet TimeSet { get; }
|
||||
public WorkflowCreator WorkflowCreator { get; }
|
||||
public FileManager FileManager { get; }
|
||||
public CodexStarter CodexStarter { get; }
|
||||
public PrometheusStarter PrometheusStarter { get; }
|
||||
public GrafanaStarter GrafanaStarter { get; }
|
||||
public GethStarter GethStarter { get; }
|
||||
public CodexDebugVersionResponse? CodexVersion { get; private set; }
|
||||
|
||||
public void DeleteAllResources()
|
||||
{
|
||||
CodexStarter.DeleteAllResources();
|
||||
FileManager.DeleteAllTestFiles();
|
||||
}
|
||||
|
||||
public IDownloadedLog DownloadLog(RunningContainer container, int? tailLines = null)
|
||||
{
|
||||
var subFile = Log.CreateSubfile();
|
||||
var description = container.Name;
|
||||
var handler = new LogDownloadHandler(container, description, subFile);
|
||||
|
||||
Log.Log($"Downloading logs for {description} to file '{subFile.FullFilename}'");
|
||||
CodexStarter.DownloadLog(container, handler, tailLines);
|
||||
|
||||
return new DownloadedLog(subFile, description);
|
||||
}
|
||||
|
||||
public string GetTestDuration()
|
||||
{
|
||||
var testDuration = DateTime.UtcNow - testStart;
|
||||
return Time.FormatDuration(testDuration);
|
||||
}
|
||||
|
||||
public void SetCodexVersion(CodexDebugVersionResponse version)
|
||||
{
|
||||
if (CodexVersion == null) CodexVersion = version;
|
||||
}
|
||||
|
||||
public ApplicationIds GetApplicationIds()
|
||||
{
|
||||
return new ApplicationIds(
|
||||
codexId: GetCodexId(),
|
||||
gethId: new GethContainerRecipe().Image,
|
||||
prometheusId: new PrometheusContainerRecipe().Image,
|
||||
codexContractsId: new CodexContractsContainerRecipe().Image,
|
||||
grafanaId: new GrafanaContainerRecipe().Image
|
||||
);
|
||||
}
|
||||
|
||||
private string GetCodexId()
|
||||
{
|
||||
var v = CodexVersion;
|
||||
if (v == null) return new CodexContainerRecipe().Image;
|
||||
if (v.version != "untagged build") return v.version;
|
||||
return v.revision;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,85 +0,0 @@
|
|||
using NUnit.Framework;
|
||||
|
||||
namespace DistTestCore
|
||||
{
|
||||
[AttributeUsage(AttributeTargets.Method, AllowMultiple = false)]
|
||||
public class UseLongTimeoutsAttribute : PropertyAttribute
|
||||
{
|
||||
}
|
||||
|
||||
public interface ITimeSet
|
||||
{
|
||||
TimeSpan HttpCallTimeout();
|
||||
TimeSpan HttpCallRetryTime();
|
||||
TimeSpan HttpCallRetryDelay();
|
||||
TimeSpan WaitForK8sServiceDelay();
|
||||
TimeSpan K8sOperationTimeout();
|
||||
TimeSpan WaitForMetricTimeout();
|
||||
}
|
||||
|
||||
public class DefaultTimeSet : ITimeSet
|
||||
{
|
||||
public TimeSpan HttpCallTimeout()
|
||||
{
|
||||
return TimeSpan.FromMinutes(5);
|
||||
}
|
||||
|
||||
public TimeSpan HttpCallRetryTime()
|
||||
{
|
||||
return TimeSpan.FromMinutes(1);
|
||||
}
|
||||
|
||||
public TimeSpan HttpCallRetryDelay()
|
||||
{
|
||||
return TimeSpan.FromSeconds(1);
|
||||
}
|
||||
|
||||
public TimeSpan WaitForK8sServiceDelay()
|
||||
{
|
||||
return TimeSpan.FromSeconds(10);
|
||||
}
|
||||
|
||||
public TimeSpan K8sOperationTimeout()
|
||||
{
|
||||
return TimeSpan.FromMinutes(30);
|
||||
}
|
||||
|
||||
public TimeSpan WaitForMetricTimeout()
|
||||
{
|
||||
return TimeSpan.FromSeconds(30);
|
||||
}
|
||||
}
|
||||
|
||||
public class LongTimeSet : ITimeSet
|
||||
{
|
||||
public TimeSpan HttpCallTimeout()
|
||||
{
|
||||
return TimeSpan.FromHours(2);
|
||||
}
|
||||
|
||||
public TimeSpan HttpCallRetryTime()
|
||||
{
|
||||
return TimeSpan.FromHours(5);
|
||||
}
|
||||
|
||||
public TimeSpan HttpCallRetryDelay()
|
||||
{
|
||||
return TimeSpan.FromSeconds(2);
|
||||
}
|
||||
|
||||
public TimeSpan WaitForK8sServiceDelay()
|
||||
{
|
||||
return TimeSpan.FromSeconds(10);
|
||||
}
|
||||
|
||||
public TimeSpan K8sOperationTimeout()
|
||||
{
|
||||
return TimeSpan.FromMinutes(15);
|
||||
}
|
||||
|
||||
public TimeSpan WaitForMetricTimeout()
|
||||
{
|
||||
return TimeSpan.FromMinutes(5);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,15 +1,14 @@
|
|||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<OutputType>Exe</OutputType>
|
||||
<TargetFramework>net7.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\ArgsUniform\ArgsUniform.csproj" />
|
||||
<ProjectReference Include="..\DistTestCore\DistTestCore.csproj" />
|
||||
<ProjectReference Include="..\FileUtils\FileUtils.csproj" />
|
||||
<ProjectReference Include="..\KubernetesWorkflow\KubernetesWorkflow.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
|
|
@ -0,0 +1,50 @@
|
|||
using KubernetesWorkflow;
|
||||
|
||||
namespace Core
|
||||
{
|
||||
public sealed class CoreInterface
|
||||
{
|
||||
private readonly EntryPoint entryPoint;
|
||||
|
||||
internal CoreInterface(EntryPoint entryPoint)
|
||||
{
|
||||
this.entryPoint = entryPoint;
|
||||
}
|
||||
|
||||
public T GetPlugin<T>() where T : IProjectPlugin
|
||||
{
|
||||
return entryPoint.GetPlugin<T>();
|
||||
}
|
||||
|
||||
public IDownloadedLog DownloadLog(IHasContainer containerSource, int? tailLines = null)
|
||||
{
|
||||
return DownloadLog(containerSource.Container, tailLines);
|
||||
}
|
||||
|
||||
public IDownloadedLog DownloadLog(RunningContainer container, int? tailLines = null)
|
||||
{
|
||||
var workflow = entryPoint.Tools.CreateWorkflow();
|
||||
var file = entryPoint.Tools.GetLog().CreateSubfile();
|
||||
entryPoint.Tools.GetLog().Log($"Downloading container log for '{container.Name}' to file '{file.FullFilename}'...");
|
||||
var logHandler = new LogDownloadHandler(container.Name, file);
|
||||
workflow.DownloadContainerLog(container, logHandler, tailLines);
|
||||
return logHandler.DownloadLog();
|
||||
}
|
||||
|
||||
public string ExecuteContainerCommand(IHasContainer containerSource, string command, params string[] args)
|
||||
{
|
||||
return ExecuteContainerCommand(containerSource.Container, command, args);
|
||||
}
|
||||
|
||||
public string ExecuteContainerCommand(RunningContainer container, string command, params string[] args)
|
||||
{
|
||||
var workflow = entryPoint.Tools.CreateWorkflow();
|
||||
return workflow.ExecuteCommand(container, command, args);
|
||||
}
|
||||
}
|
||||
|
||||
public interface IHasContainer
|
||||
{
|
||||
RunningContainer Container { get; }
|
||||
}
|
||||
}
|
||||
|
|
@ -1,27 +1,24 @@
|
|||
using Logging;
|
||||
using NUnit.Framework;
|
||||
|
||||
namespace DistTestCore.Logs
|
||||
namespace Core
|
||||
{
|
||||
public interface IDownloadedLog
|
||||
{
|
||||
void AssertLogContains(string expectedString);
|
||||
bool DoesLogContain(string expectedString);
|
||||
string[] FindLinesThatContain(params string[] tags);
|
||||
void DeleteFile();
|
||||
}
|
||||
|
||||
public class DownloadedLog : IDownloadedLog
|
||||
internal class DownloadedLog : IDownloadedLog
|
||||
{
|
||||
private readonly LogFile logFile;
|
||||
private readonly string owner;
|
||||
|
||||
public DownloadedLog(LogFile logFile, string owner)
|
||||
internal DownloadedLog(LogFile logFile)
|
||||
{
|
||||
this.logFile = logFile;
|
||||
this.owner = owner;
|
||||
}
|
||||
|
||||
public void AssertLogContains(string expectedString)
|
||||
public bool DoesLogContain(string expectedString)
|
||||
{
|
||||
using var file = File.OpenRead(logFile.FullFilename);
|
||||
using var streamReader = new StreamReader(file);
|
||||
|
|
@ -29,11 +26,12 @@ namespace DistTestCore.Logs
|
|||
var line = streamReader.ReadLine();
|
||||
while (line != null)
|
||||
{
|
||||
if (line.Contains(expectedString)) return;
|
||||
if (line.Contains(expectedString)) return true;
|
||||
line = streamReader.ReadLine();
|
||||
}
|
||||
|
||||
Assert.Fail($"{owner} Unable to find string '{expectedString}' in CodexNode log file {logFile.FullFilename}");
|
||||
//Assert.Fail($"{owner} Unable to find string '{expectedString}' in CodexNode log file {logFile.FullFilename}");
|
||||
return false;
|
||||
}
|
||||
|
||||
public string[] FindLinesThatContain(params string[] tags)
|
||||
|
|
@ -0,0 +1,52 @@
|
|||
using KubernetesWorkflow;
|
||||
using Logging;
|
||||
|
||||
namespace Core
|
||||
{
|
||||
public class EntryPoint
|
||||
{
|
||||
private readonly IToolsFactory toolsFactory;
|
||||
private readonly PluginManager manager = new PluginManager();
|
||||
|
||||
public EntryPoint(ILog log, Configuration configuration, string fileManagerRootFolder, ITimeSet timeSet)
|
||||
{
|
||||
toolsFactory = new ToolsFactory(log, configuration, fileManagerRootFolder, timeSet);
|
||||
|
||||
Tools = toolsFactory.CreateTools();
|
||||
manager.InstantiatePlugins(PluginFinder.GetPluginTypes(), toolsFactory);
|
||||
}
|
||||
|
||||
public EntryPoint(ILog log, Configuration configuration, string fileManagerRootFolder)
|
||||
: this(log, configuration, fileManagerRootFolder, new DefaultTimeSet())
|
||||
{
|
||||
}
|
||||
|
||||
public IPluginTools Tools { get; }
|
||||
|
||||
public void Announce()
|
||||
{
|
||||
manager.AnnouncePlugins();
|
||||
}
|
||||
|
||||
public Dictionary<string, string> GetPluginMetadata()
|
||||
{
|
||||
return manager.GatherPluginMetadata().Get();
|
||||
}
|
||||
|
||||
public CoreInterface CreateInterface()
|
||||
{
|
||||
return new CoreInterface(this);
|
||||
}
|
||||
|
||||
public void Decommission(bool deleteKubernetesResources, bool deleteTrackedFiles)
|
||||
{
|
||||
manager.DecommissionPlugins(deleteKubernetesResources, deleteTrackedFiles);
|
||||
Tools.Decommission(deleteKubernetesResources, deleteTrackedFiles);
|
||||
}
|
||||
|
||||
internal T GetPlugin<T>() where T : IProjectPlugin
|
||||
{
|
||||
return manager.GetPlugin<T>();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -5,23 +5,35 @@ using System.Net.Http.Headers;
|
|||
using System.Net.Http.Json;
|
||||
using Utils;
|
||||
|
||||
namespace DistTestCore
|
||||
namespace Core
|
||||
{
|
||||
public class Http
|
||||
public interface IHttp
|
||||
{
|
||||
private readonly BaseLog log;
|
||||
string HttpGetString(string route);
|
||||
T HttpGetJson<T>(string route);
|
||||
TResponse HttpPostJson<TRequest, TResponse>(string route, TRequest body);
|
||||
string HttpPostJson<TRequest>(string route, TRequest body);
|
||||
string HttpPostString(string route, string body);
|
||||
string HttpPostStream(string route, Stream stream);
|
||||
Stream HttpGetStream(string route);
|
||||
T TryJsonDeserialize<T>(string json);
|
||||
}
|
||||
|
||||
internal class Http : IHttp
|
||||
{
|
||||
private readonly ILog log;
|
||||
private readonly ITimeSet timeSet;
|
||||
private readonly Address address;
|
||||
private readonly string baseUrl;
|
||||
private readonly Action<HttpClient> onClientCreated;
|
||||
private readonly string? logAlias;
|
||||
|
||||
public Http(BaseLog log, ITimeSet timeSet, Address address, string baseUrl, string? logAlias = null)
|
||||
internal Http(ILog log, ITimeSet timeSet, Address address, string baseUrl, string? logAlias = null)
|
||||
: this(log, timeSet, address, baseUrl, DoNothing, logAlias)
|
||||
{
|
||||
}
|
||||
|
||||
public Http(BaseLog log, ITimeSet timeSet, Address address, string baseUrl, Action<HttpClient> onClientCreated, string? logAlias = null)
|
||||
internal Http(ILog log, ITimeSet timeSet, Address address, string baseUrl, Action<HttpClient> onClientCreated, string? logAlias = null)
|
||||
{
|
||||
this.log = log;
|
||||
this.timeSet = timeSet;
|
||||
|
|
@ -1,25 +1,23 @@
|
|||
using KubernetesWorkflow;
|
||||
using Logging;
|
||||
|
||||
namespace DistTestCore.Logs
|
||||
namespace Core
|
||||
{
|
||||
public class LogDownloadHandler : LogHandler, ILogHandler
|
||||
internal class LogDownloadHandler : LogHandler, ILogHandler
|
||||
{
|
||||
private readonly RunningContainer container;
|
||||
private readonly LogFile log;
|
||||
|
||||
public LogDownloadHandler(RunningContainer container, string description, LogFile log)
|
||||
internal LogDownloadHandler(string description, LogFile log)
|
||||
{
|
||||
this.container = container;
|
||||
this.log = log;
|
||||
|
||||
log.Write($"{description} -->> {log.FullFilename}");
|
||||
log.WriteRaw(description);
|
||||
}
|
||||
|
||||
public DownloadedLog DownloadLog()
|
||||
internal IDownloadedLog DownloadLog()
|
||||
{
|
||||
return new DownloadedLog(log, container.Name);
|
||||
return new DownloadedLog(log);
|
||||
}
|
||||
|
||||
protected override void ProcessLine(string line)
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
using System.Reflection;
|
||||
|
||||
namespace Core
|
||||
{
|
||||
internal static class PluginFinder
|
||||
{
|
||||
private static Type[]? pluginTypes = null;
|
||||
|
||||
internal static Type[] GetPluginTypes()
|
||||
{
|
||||
if (pluginTypes != null) return pluginTypes;
|
||||
|
||||
// Reflection can be costly. Do this only once.
|
||||
FindAndLoadPluginAssemblies();
|
||||
|
||||
var assemblies = AppDomain.CurrentDomain.GetAssemblies();
|
||||
pluginTypes = assemblies.SelectMany(a => a.GetTypes().Where(t =>
|
||||
typeof(IProjectPlugin).IsAssignableFrom(t) &&
|
||||
!t.IsAbstract)
|
||||
).ToArray();
|
||||
|
||||
return pluginTypes;
|
||||
}
|
||||
|
||||
private static void FindAndLoadPluginAssemblies()
|
||||
{
|
||||
var files = Directory.GetFiles(".");
|
||||
foreach (var file in files)
|
||||
{
|
||||
var f = file.ToLowerInvariant();
|
||||
if (f.Contains("plugin") && f.EndsWith("dll"))
|
||||
{
|
||||
var name = Path.GetFileNameWithoutExtension(file);
|
||||
try
|
||||
{
|
||||
Assembly.Load(name);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
throw new Exception($"Failed to load plugin from file '{name}'.", ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,78 @@
|
|||
namespace Core
|
||||
{
|
||||
internal class PluginManager
|
||||
{
|
||||
private readonly List<PluginToolsPair> pairs = new List<PluginToolsPair>();
|
||||
|
||||
internal void InstantiatePlugins(Type[] pluginTypes, IToolsFactory provider)
|
||||
{
|
||||
pairs.Clear();
|
||||
foreach (var pluginType in pluginTypes)
|
||||
{
|
||||
var tools = provider.CreateTools();
|
||||
var plugin = InstantiatePlugins(pluginType, tools);
|
||||
|
||||
ApplyLogPrefix(plugin, tools);
|
||||
}
|
||||
}
|
||||
|
||||
internal void AnnouncePlugins()
|
||||
{
|
||||
foreach (var pair in pairs) pair.Plugin.Announce();
|
||||
}
|
||||
|
||||
internal PluginMetadata GatherPluginMetadata()
|
||||
{
|
||||
var metadata = new PluginMetadata();
|
||||
foreach (var pair in pairs)
|
||||
{
|
||||
if (pair.Plugin is IHasMetadata m)
|
||||
{
|
||||
m.AddMetadata(metadata);
|
||||
}
|
||||
}
|
||||
return metadata;
|
||||
}
|
||||
|
||||
internal void DecommissionPlugins(bool deleteKubernetesResources, bool deleteTrackedFiles)
|
||||
{
|
||||
foreach (var pair in pairs)
|
||||
{
|
||||
pair.Plugin.Decommission();
|
||||
pair.Tools.Decommission(deleteKubernetesResources, deleteTrackedFiles);
|
||||
}
|
||||
}
|
||||
|
||||
internal T GetPlugin<T>() where T : IProjectPlugin
|
||||
{
|
||||
return (T)pairs.Single(p => p.Plugin.GetType() == typeof(T)).Plugin;
|
||||
}
|
||||
|
||||
private IProjectPlugin InstantiatePlugins(Type pluginType, PluginTools tools)
|
||||
{
|
||||
var plugin = (IProjectPlugin)Activator.CreateInstance(pluginType, args: tools)!;
|
||||
pairs.Add(new PluginToolsPair(plugin, tools));
|
||||
return plugin;
|
||||
}
|
||||
|
||||
private void ApplyLogPrefix(IProjectPlugin plugin, PluginTools tools)
|
||||
{
|
||||
if (plugin is IHasLogPrefix hasLogPrefix)
|
||||
{
|
||||
tools.ApplyLogPrefix(hasLogPrefix.LogPrefix);
|
||||
}
|
||||
}
|
||||
|
||||
private class PluginToolsPair
|
||||
{
|
||||
public PluginToolsPair(IProjectPlugin plugin, IPluginTools tools)
|
||||
{
|
||||
Plugin = plugin;
|
||||
Tools = tools;
|
||||
}
|
||||
|
||||
public IProjectPlugin Plugin { get; }
|
||||
public IPluginTools Tools { get; }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
namespace Core
|
||||
{
|
||||
internal interface IPluginMetadata
|
||||
{
|
||||
Dictionary<string, string> Get();
|
||||
}
|
||||
|
||||
public interface IAddMetadata
|
||||
{
|
||||
void Add(string key, string value);
|
||||
}
|
||||
|
||||
internal class PluginMetadata : IPluginMetadata, IAddMetadata
|
||||
{
|
||||
private readonly Dictionary<string, string> metadata = new Dictionary<string, string>();
|
||||
|
||||
public void Add(string key, string value)
|
||||
{
|
||||
metadata.Add(key, value);
|
||||
}
|
||||
|
||||
public Dictionary<string, string> Get()
|
||||
{
|
||||
return new Dictionary<string, string>(metadata);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,85 @@
|
|||
using FileUtils;
|
||||
using KubernetesWorkflow;
|
||||
using Logging;
|
||||
using Utils;
|
||||
|
||||
namespace Core
|
||||
{
|
||||
public interface IPluginTools : IWorkflowTool, ILogTool, IHttpFactoryTool, IFileTool
|
||||
{
|
||||
void Decommission(bool deleteKubernetesResources, bool deleteTrackedFiles);
|
||||
}
|
||||
|
||||
public interface IWorkflowTool
|
||||
{
|
||||
IStartupWorkflow CreateWorkflow(string? namespaceOverride = null);
|
||||
}
|
||||
|
||||
public interface ILogTool
|
||||
{
|
||||
ILog GetLog();
|
||||
}
|
||||
|
||||
public interface IHttpFactoryTool
|
||||
{
|
||||
IHttp CreateHttp(Address address, string baseUrl, Action<HttpClient> onClientCreated, string? logAlias = null);
|
||||
IHttp CreateHttp(Address address, string baseUrl, string? logAlias = null);
|
||||
}
|
||||
|
||||
public interface IFileTool
|
||||
{
|
||||
IFileManager GetFileManager();
|
||||
}
|
||||
|
||||
internal class PluginTools : IPluginTools
|
||||
{
|
||||
private readonly ITimeSet timeSet;
|
||||
private readonly WorkflowCreator workflowCreator;
|
||||
private readonly IFileManager fileManager;
|
||||
private ILog log;
|
||||
|
||||
internal PluginTools(ILog log, WorkflowCreator workflowCreator, string fileManagerRootFolder, ITimeSet timeSet)
|
||||
{
|
||||
this.log = log;
|
||||
this.workflowCreator = workflowCreator;
|
||||
this.timeSet = timeSet;
|
||||
fileManager = new FileManager(log, fileManagerRootFolder);
|
||||
}
|
||||
|
||||
public void ApplyLogPrefix(string prefix)
|
||||
{
|
||||
log = new LogPrefixer(log, prefix);
|
||||
}
|
||||
|
||||
public IHttp CreateHttp(Address address, string baseUrl, Action<HttpClient> onClientCreated, string? logAlias = null)
|
||||
{
|
||||
return new Http(log, timeSet, address, baseUrl, onClientCreated, logAlias);
|
||||
}
|
||||
|
||||
public IHttp CreateHttp(Address address, string baseUrl, string? logAlias = null)
|
||||
{
|
||||
return new Http(log, timeSet, address, baseUrl, logAlias);
|
||||
}
|
||||
|
||||
public IStartupWorkflow CreateWorkflow(string? namespaceOverride = null)
|
||||
{
|
||||
return workflowCreator.CreateWorkflow(namespaceOverride);
|
||||
}
|
||||
|
||||
public void Decommission(bool deleteKubernetesResources, bool deleteTrackedFiles)
|
||||
{
|
||||
if (deleteKubernetesResources) CreateWorkflow().DeleteNamespace();
|
||||
if (deleteTrackedFiles) fileManager.DeleteAllFiles();
|
||||
}
|
||||
|
||||
public IFileManager GetFileManager()
|
||||
{
|
||||
return fileManager;
|
||||
}
|
||||
|
||||
public ILog GetLog()
|
||||
{
|
||||
return log;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
using Utils;
|
||||
|
||||
namespace Core
|
||||
{
|
||||
public interface IProjectPlugin
|
||||
{
|
||||
void Announce();
|
||||
void Decommission();
|
||||
}
|
||||
|
||||
public interface IHasLogPrefix
|
||||
{
|
||||
string LogPrefix { get; }
|
||||
}
|
||||
|
||||
public interface IHasMetadata
|
||||
{
|
||||
void AddMetadata(IAddMetadata metadata);
|
||||
}
|
||||
|
||||
public static class ProjectPlugin
|
||||
{
|
||||
/// <summary>
|
||||
/// On some platforms and in some cases, not all required plugin assemblies are automatically loaded into the app domain.
|
||||
/// In this case, the runtime needs a slight push to load it before the EntryPoint class is instantiated.
|
||||
/// Used ProjectPlugin.Load<>() before you create an EntryPoint to ensure all plugins you want to use are loaded.
|
||||
/// </summary>
|
||||
public static void Load<T>() where T : IProjectPlugin
|
||||
{
|
||||
var type = typeof(T);
|
||||
FrameworkAssert.That(type != null, $"Unable to load plugin.");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
using Newtonsoft.Json;
|
||||
|
||||
namespace Core
|
||||
{
|
||||
public static class SerializeGate
|
||||
{
|
||||
/// <summary>
|
||||
/// SerializeGate was added to help ensure deployment objects are serializable
|
||||
/// and remain viable after deserialization.
|
||||
/// Tools can be built on top of the core interface that rely on deployment objects being serializable.
|
||||
/// Insert the serialization gate after deployment but before wrapping to ensure any future changes
|
||||
/// don't break this requirement.
|
||||
/// </summary>
|
||||
public static T Gate<T>(T anything)
|
||||
{
|
||||
var json = JsonConvert.SerializeObject(anything);
|
||||
return JsonConvert.DeserializeObject<T>(json)!;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
namespace Core
|
||||
{
|
||||
public interface ITimeSet
|
||||
{
|
||||
TimeSpan HttpCallTimeout();
|
||||
TimeSpan HttpCallRetryTime();
|
||||
TimeSpan HttpCallRetryDelay();
|
||||
TimeSpan WaitForK8sServiceDelay();
|
||||
TimeSpan K8sOperationTimeout();
|
||||
}
|
||||
|
||||
public class DefaultTimeSet : ITimeSet
|
||||
{
|
||||
public TimeSpan HttpCallTimeout()
|
||||
{
|
||||
return TimeSpan.FromMinutes(5);
|
||||
}
|
||||
|
||||
public TimeSpan HttpCallRetryTime()
|
||||
{
|
||||
return TimeSpan.FromMinutes(1);
|
||||
}
|
||||
|
||||
public TimeSpan HttpCallRetryDelay()
|
||||
{
|
||||
return TimeSpan.FromSeconds(1);
|
||||
}
|
||||
|
||||
public TimeSpan WaitForK8sServiceDelay()
|
||||
{
|
||||
return TimeSpan.FromSeconds(10);
|
||||
}
|
||||
|
||||
public TimeSpan K8sOperationTimeout()
|
||||
{
|
||||
return TimeSpan.FromMinutes(30);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
using KubernetesWorkflow;
|
||||
using Logging;
|
||||
|
||||
namespace Core
|
||||
{
|
||||
internal interface IToolsFactory
|
||||
{
|
||||
PluginTools CreateTools();
|
||||
}
|
||||
|
||||
internal class ToolsFactory : IToolsFactory
|
||||
{
|
||||
private readonly ILog log;
|
||||
private readonly WorkflowCreator workflowCreator;
|
||||
private readonly string fileManagerRootFolder;
|
||||
private readonly ITimeSet timeSet;
|
||||
|
||||
public ToolsFactory(ILog log, Configuration configuration, string fileManagerRootFolder, ITimeSet timeSet)
|
||||
{
|
||||
this.log = log;
|
||||
workflowCreator = new WorkflowCreator(log, configuration);
|
||||
this.fileManagerRootFolder = fileManagerRootFolder;
|
||||
this.timeSet = timeSet;
|
||||
}
|
||||
|
||||
public PluginTools CreateTools()
|
||||
{
|
||||
return new PluginTools(log, workflowCreator, fileManagerRootFolder, timeSet);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,157 @@
|
|||
using Logging;
|
||||
using Utils;
|
||||
|
||||
namespace FileUtils
|
||||
{
|
||||
public interface IFileManager
|
||||
{
|
||||
TrackedFile CreateEmptyFile(string label = "");
|
||||
TrackedFile GenerateFile(ByteSize size, string label = "");
|
||||
void DeleteAllFiles();
|
||||
void ScopedFiles(Action action);
|
||||
T ScopedFiles<T>(Func<T> action);
|
||||
}
|
||||
|
||||
public class FileManager : IFileManager
|
||||
{
|
||||
public const int ChunkSize = 1024 * 1024 * 100;
|
||||
private static NumberSource folderNumberSource = new NumberSource(0);
|
||||
private readonly Random random = new Random();
|
||||
private readonly ILog log;
|
||||
private readonly string rootFolder;
|
||||
private readonly string folder;
|
||||
private readonly List<List<TrackedFile>> fileSetStack = new List<List<TrackedFile>>();
|
||||
|
||||
public FileManager(ILog log, string rootFolder)
|
||||
{
|
||||
folder = Path.Combine(rootFolder, folderNumberSource.GetNextNumber().ToString("D5"));
|
||||
|
||||
this.log = log;
|
||||
this.rootFolder = rootFolder;
|
||||
}
|
||||
|
||||
public TrackedFile CreateEmptyFile(string label = "")
|
||||
{
|
||||
var path = Path.Combine(folder, Guid.NewGuid().ToString() + ".bin");
|
||||
EnsureDirectory();
|
||||
|
||||
var result = new TrackedFile(log, path, label);
|
||||
File.Create(result.Filename).Close();
|
||||
if (fileSetStack.Any()) fileSetStack.Last().Add(result);
|
||||
return result;
|
||||
}
|
||||
|
||||
public TrackedFile GenerateFile(ByteSize size, string label)
|
||||
{
|
||||
var sw = Stopwatch.Begin(log);
|
||||
var result = GenerateRandomFile(size, label);
|
||||
sw.End($"Generated file '{result.Describe()}'.");
|
||||
return result;
|
||||
}
|
||||
|
||||
public void DeleteAllFiles()
|
||||
{
|
||||
DeleteDirectory();
|
||||
}
|
||||
|
||||
public void ScopedFiles(Action action)
|
||||
{
|
||||
PushFileSet();
|
||||
action();
|
||||
PopFileSet();
|
||||
}
|
||||
|
||||
public T ScopedFiles<T>(Func<T> action)
|
||||
{
|
||||
PushFileSet();
|
||||
var result = action();
|
||||
PopFileSet();
|
||||
return result;
|
||||
}
|
||||
|
||||
private void PushFileSet()
|
||||
{
|
||||
fileSetStack.Add(new List<TrackedFile>());
|
||||
}
|
||||
|
||||
private void PopFileSet()
|
||||
{
|
||||
if (!fileSetStack.Any()) return;
|
||||
var pop = fileSetStack.Last();
|
||||
fileSetStack.Remove(pop);
|
||||
|
||||
foreach (var file in pop)
|
||||
{
|
||||
File.Delete(file.Filename);
|
||||
}
|
||||
|
||||
// If the folder is now empty, delete it too.
|
||||
if (!Directory.GetFiles(folder).Any()) DeleteDirectory();
|
||||
}
|
||||
|
||||
private TrackedFile GenerateRandomFile(ByteSize size, string label)
|
||||
{
|
||||
var result = CreateEmptyFile(label);
|
||||
CheckSpaceAvailable(result, size);
|
||||
|
||||
GenerateFileBytes(result, size);
|
||||
return result;
|
||||
}
|
||||
|
||||
private void CheckSpaceAvailable(TrackedFile testFile, ByteSize size)
|
||||
{
|
||||
var file = new FileInfo(testFile.Filename);
|
||||
var drive = new DriveInfo(file.Directory!.Root.FullName);
|
||||
|
||||
var spaceAvailable = drive.TotalFreeSpace;
|
||||
|
||||
if (spaceAvailable < size.SizeInBytes)
|
||||
{
|
||||
var msg = $"Not enough disk space. " +
|
||||
$"{Formatter.FormatByteSize(size.SizeInBytes)} required. " +
|
||||
$"{Formatter.FormatByteSize(spaceAvailable)} available.";
|
||||
|
||||
log.Log(msg);
|
||||
throw new Exception(msg);
|
||||
}
|
||||
}
|
||||
|
||||
private void GenerateFileBytes(TrackedFile result, ByteSize size)
|
||||
{
|
||||
long bytesLeft = size.SizeInBytes;
|
||||
int chunkSize = ChunkSize;
|
||||
while (bytesLeft > 0)
|
||||
{
|
||||
try
|
||||
{
|
||||
var length = Math.Min(bytesLeft, chunkSize);
|
||||
AppendRandomBytesToFile(result, length);
|
||||
bytesLeft -= length;
|
||||
}
|
||||
catch
|
||||
{
|
||||
chunkSize = chunkSize / 2;
|
||||
if (chunkSize < 1024) throw;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void AppendRandomBytesToFile(TrackedFile result, long length)
|
||||
{
|
||||
var bytes = new byte[length];
|
||||
random.NextBytes(bytes);
|
||||
using var stream = new FileStream(result.Filename, FileMode.Append);
|
||||
stream.Write(bytes, 0, bytes.Length);
|
||||
}
|
||||
|
||||
private void EnsureDirectory()
|
||||
{
|
||||
Directory.CreateDirectory(folder);
|
||||
}
|
||||
|
||||
private void DeleteDirectory()
|
||||
{
|
||||
if (Directory.Exists(folder)) Directory.Delete(folder, true);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net7.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\Logging\Logging.csproj" />
|
||||
<ProjectReference Include="..\Utils\Utils.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
|
|
@ -0,0 +1,82 @@
|
|||
using Logging;
|
||||
using Utils;
|
||||
|
||||
namespace FileUtils
|
||||
{
|
||||
public class TrackedFile
|
||||
{
|
||||
private readonly ILog log;
|
||||
|
||||
public TrackedFile(ILog log, string filename, string label)
|
||||
{
|
||||
this.log = log;
|
||||
Filename = filename;
|
||||
Label = label;
|
||||
}
|
||||
|
||||
public string Filename { get; }
|
||||
public string Label { get; }
|
||||
|
||||
public void AssertIsEqual(TrackedFile? actual)
|
||||
{
|
||||
var sw = Stopwatch.Begin(log);
|
||||
try
|
||||
{
|
||||
AssertEqual(actual);
|
||||
}
|
||||
finally
|
||||
{
|
||||
sw.End($"{nameof(TrackedFile)}.{nameof(AssertIsEqual)}");
|
||||
}
|
||||
}
|
||||
|
||||
public string Describe()
|
||||
{
|
||||
var sizePostfix = $" ({Formatter.FormatByteSize(GetFileSize())})";
|
||||
if (!string.IsNullOrEmpty(Label)) return Label + sizePostfix;
|
||||
return $"'{Filename}'{sizePostfix}";
|
||||
}
|
||||
|
||||
private void AssertEqual(TrackedFile? actual)
|
||||
{
|
||||
if (actual == null) FrameworkAssert.Fail("TestFile is null.");
|
||||
if (actual == this || actual!.Filename == Filename) FrameworkAssert.Fail("TestFile is compared to itself.");
|
||||
|
||||
FrameworkAssert.That(actual.GetFileSize() == GetFileSize(), "Files are not of equal length.");
|
||||
|
||||
using var streamExpected = new FileStream(Filename, FileMode.Open, FileAccess.Read);
|
||||
using var streamActual = new FileStream(actual.Filename, FileMode.Open, FileAccess.Read);
|
||||
|
||||
var bytesExpected = new byte[FileManager.ChunkSize];
|
||||
var bytesActual = new byte[FileManager.ChunkSize];
|
||||
|
||||
var readExpected = 0;
|
||||
var readActual = 0;
|
||||
|
||||
while (true)
|
||||
{
|
||||
readExpected = streamExpected.Read(bytesExpected, 0, FileManager.ChunkSize);
|
||||
readActual = streamActual.Read(bytesActual, 0, FileManager.ChunkSize);
|
||||
|
||||
if (readExpected == 0 && readActual == 0)
|
||||
{
|
||||
log.Log($"OK: '{Describe()}' is equal to '{actual.Describe()}'.");
|
||||
return;
|
||||
}
|
||||
|
||||
FrameworkAssert.That(readActual == readExpected, "Unable to read buffers of equal length.");
|
||||
|
||||
for (var i = 0; i < readActual; i++)
|
||||
{
|
||||
if (bytesExpected[i] != bytesActual[i]) FrameworkAssert.Fail("File contents not equal.");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private long GetFileSize()
|
||||
{
|
||||
var info = new FileInfo(Filename);
|
||||
return info.Length;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
using Newtonsoft.Json;
|
||||
|
||||
namespace KubernetesWorkflow
|
||||
{
|
||||
public class Configuration
|
||||
{
|
||||
public Configuration(string? kubeConfigFile, TimeSpan operationTimeout, TimeSpan retryDelay, string kubernetesNamespace)
|
||||
{
|
||||
KubeConfigFile = kubeConfigFile;
|
||||
OperationTimeout = operationTimeout;
|
||||
RetryDelay = retryDelay;
|
||||
KubernetesNamespace = kubernetesNamespace;
|
||||
}
|
||||
|
||||
public string? KubeConfigFile { get; }
|
||||
public TimeSpan OperationTimeout { get; }
|
||||
public TimeSpan RetryDelay { get; }
|
||||
public string KubernetesNamespace { get; }
|
||||
public bool AllowNamespaceOverride { get; set; } = true;
|
||||
public bool AddAppPodLabel { get; set; } = true;
|
||||
|
||||
[JsonIgnore]
|
||||
public IK8sHooks Hooks { get; set; } = new DoNothingK8sHooks();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,53 @@
|
|||
using Newtonsoft.Json.Linq;
|
||||
|
||||
namespace KubernetesWorkflow
|
||||
{
|
||||
public class ContainerAdditionals
|
||||
{
|
||||
public ContainerAdditionals(Additional[] additionals)
|
||||
{
|
||||
Additionals = additionals;
|
||||
}
|
||||
|
||||
public static ContainerAdditionals CreateFromUserData(IEnumerable<object> userData)
|
||||
{
|
||||
return new ContainerAdditionals(userData.Select(ConvertToAdditional).ToArray());
|
||||
}
|
||||
|
||||
public Additional[] Additionals { get; }
|
||||
|
||||
public T? Get<T>()
|
||||
{
|
||||
var typeName = GetTypeName(typeof(T));
|
||||
var userData = Additionals.SingleOrDefault(a => a.Type == typeName);
|
||||
if (userData == null) return default(T);
|
||||
var jobject = (JObject)userData.UserData;
|
||||
return jobject.ToObject<T>();
|
||||
}
|
||||
|
||||
private static Additional ConvertToAdditional(object userData)
|
||||
{
|
||||
var typeName = GetTypeName(userData.GetType());
|
||||
return new Additional(typeName, userData);
|
||||
}
|
||||
|
||||
private static string GetTypeName(Type type)
|
||||
{
|
||||
var typeName = type.FullName;
|
||||
if (string.IsNullOrEmpty(typeName)) throw new Exception("Object type fullname is null or empty: " + type);
|
||||
return typeName;
|
||||
}
|
||||
}
|
||||
|
||||
public class Additional
|
||||
{
|
||||
public Additional(string type, object userData)
|
||||
{
|
||||
Type = type;
|
||||
UserData = userData;
|
||||
}
|
||||
|
||||
public string Type { get; }
|
||||
public object UserData { get; }
|
||||
}
|
||||
}
|
||||
|
|
@ -2,9 +2,10 @@
|
|||
{
|
||||
public class ContainerRecipe
|
||||
{
|
||||
public ContainerRecipe(int number, string image, ContainerResources resources, Port[] exposedPorts, Port[] internalPorts, EnvVar[] envVars, PodLabels podLabels, PodAnnotations podAnnotations, VolumeMount[] volumes, object[] additionals)
|
||||
public ContainerRecipe(int number, string? nameOverride, string image, ContainerResources resources, Port[] exposedPorts, Port[] internalPorts, EnvVar[] envVars, PodLabels podLabels, PodAnnotations podAnnotations, VolumeMount[] volumes, ContainerAdditionals additionals)
|
||||
{
|
||||
Number = number;
|
||||
NameOverride = nameOverride;
|
||||
Image = image;
|
||||
Resources = resources;
|
||||
ExposedPorts = exposedPorts;
|
||||
|
|
@ -14,10 +15,20 @@
|
|||
PodAnnotations = podAnnotations;
|
||||
Volumes = volumes;
|
||||
Additionals = additionals;
|
||||
|
||||
if (NameOverride != null)
|
||||
{
|
||||
Name = $"{K8sNameUtils.Format(NameOverride)}-{Number}";
|
||||
}
|
||||
else
|
||||
{
|
||||
Name = $"ctnr{Number}";
|
||||
}
|
||||
}
|
||||
|
||||
public string Name { get { return $"ctnr{Number}"; } }
|
||||
public string Name { get; }
|
||||
public int Number { get; }
|
||||
public string? NameOverride { get; }
|
||||
public ContainerResources Resources { get; }
|
||||
public string Image { get; }
|
||||
public Port[] ExposedPorts { get; }
|
||||
|
|
@ -26,11 +37,11 @@
|
|||
public PodLabels PodLabels { get; }
|
||||
public PodAnnotations PodAnnotations { get; }
|
||||
public VolumeMount[] Volumes { get; }
|
||||
public object[] Additionals { get; }
|
||||
public ContainerAdditionals Additionals { get; }
|
||||
|
||||
public Port GetPortByTag(string tag)
|
||||
public Port? GetPortByTag(string tag)
|
||||
{
|
||||
return ExposedPorts.Concat(InternalPorts).Single(p => p.Tag == tag);
|
||||
return ExposedPorts.Concat(InternalPorts).SingleOrDefault(p => p.Tag == tag);
|
||||
}
|
||||
|
||||
public override string ToString()
|
||||
|
|
@ -12,6 +12,7 @@ namespace KubernetesWorkflow
|
|||
private readonly List<VolumeMount> volumeMounts = new List<VolumeMount>();
|
||||
private readonly List<object> additionals = new List<object>();
|
||||
private RecipeComponentFactory factory = null!;
|
||||
private ContainerResources resources = new ContainerResources();
|
||||
|
||||
public ContainerRecipe CreateRecipe(int index, int containerNumber, RecipeComponentFactory factory, StartupConfig config)
|
||||
{
|
||||
|
|
@ -21,14 +22,14 @@ namespace KubernetesWorkflow
|
|||
|
||||
Initialize(config);
|
||||
|
||||
var recipe = new ContainerRecipe(containerNumber, Image, Resources,
|
||||
var recipe = new ContainerRecipe(containerNumber, config.NameOverride, Image, resources,
|
||||
exposedPorts.ToArray(),
|
||||
internalPorts.ToArray(),
|
||||
envVars.ToArray(),
|
||||
podLabels.Clone(),
|
||||
podAnnotations.Clone(),
|
||||
volumeMounts.ToArray(),
|
||||
additionals.ToArray());
|
||||
ContainerAdditionals.CreateFromUserData(additionals));
|
||||
|
||||
exposedPorts.Clear();
|
||||
internalPorts.Clear();
|
||||
|
|
@ -38,29 +39,25 @@ namespace KubernetesWorkflow
|
|||
volumeMounts.Clear();
|
||||
additionals.Clear();
|
||||
this.factory = null!;
|
||||
resources = new ContainerResources();
|
||||
|
||||
return recipe;
|
||||
}
|
||||
|
||||
public abstract string AppName { get; }
|
||||
public abstract string Image { get; }
|
||||
public ContainerResources Resources { get; } = new ContainerResources();
|
||||
protected int ContainerNumber { get; private set; } = 0;
|
||||
protected int Index { get; private set; } = 0;
|
||||
protected abstract void Initialize(StartupConfig config);
|
||||
|
||||
protected Port AddExposedPort(string tag = "")
|
||||
{
|
||||
var p = factory.CreatePort(tag);
|
||||
exposedPorts.Add(p);
|
||||
return p;
|
||||
return AddExposedPort(factory.CreatePort(tag));
|
||||
}
|
||||
|
||||
protected Port AddExposedPort(int number, string tag = "")
|
||||
{
|
||||
var p = factory.CreatePort(number, tag);
|
||||
exposedPorts.Add(p);
|
||||
return p;
|
||||
return AddExposedPort(factory.CreatePort(number, tag));
|
||||
}
|
||||
|
||||
protected Port AddInternalPort(string tag = "")
|
||||
|
|
@ -112,5 +109,36 @@ namespace KubernetesWorkflow
|
|||
{
|
||||
additionals.Add(userData);
|
||||
}
|
||||
|
||||
protected void SetResourcesRequest(int milliCPUs, ByteSize memory)
|
||||
{
|
||||
SetResourcesRequest(new ContainerResourceSet(milliCPUs, memory));
|
||||
}
|
||||
|
||||
protected void SetResourceLimits(int milliCPUs, ByteSize memory)
|
||||
{
|
||||
SetResourceLimits(new ContainerResourceSet(milliCPUs, memory));
|
||||
}
|
||||
|
||||
protected void SetResourcesRequest(ContainerResourceSet requests)
|
||||
{
|
||||
resources.Requests = requests;
|
||||
}
|
||||
|
||||
protected void SetResourceLimits(ContainerResourceSet limits)
|
||||
{
|
||||
resources.Limits = limits;
|
||||
}
|
||||
|
||||
private Port AddExposedPort(Port port)
|
||||
{
|
||||
if (exposedPorts.Any())
|
||||
{
|
||||
throw new NotImplementedException("Current implementation only support 1 exposed port per container recipe. " +
|
||||
$"Methods for determining container addresses in {nameof(StartupWorkflow)} currently rely on this constraint.");
|
||||
}
|
||||
exposedPorts.Add(port);
|
||||
return port;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -5,7 +5,7 @@ namespace KubernetesWorkflow
|
|||
{
|
||||
public class CrashWatcher
|
||||
{
|
||||
private readonly BaseLog log;
|
||||
private readonly ILog log;
|
||||
private readonly KubernetesClientConfiguration config;
|
||||
private readonly string k8sNamespace;
|
||||
private readonly RunningContainer container;
|
||||
|
|
@ -14,7 +14,7 @@ namespace KubernetesWorkflow
|
|||
private Task? worker;
|
||||
private Exception? workerException;
|
||||
|
||||
public CrashWatcher(BaseLog log, KubernetesClientConfiguration config, string k8sNamespace, RunningContainer container)
|
||||
public CrashWatcher(ILog log, KubernetesClientConfiguration config, string k8sNamespace, RunningContainer container)
|
||||
{
|
||||
this.log = log;
|
||||
this.config = config;
|
||||
|
|
@ -7,13 +7,13 @@ namespace KubernetesWorkflow
|
|||
{
|
||||
public class K8sController
|
||||
{
|
||||
private readonly BaseLog log;
|
||||
private readonly ILog log;
|
||||
private readonly K8sCluster cluster;
|
||||
private readonly KnownK8sPods knownPods;
|
||||
private readonly WorkflowNumberSource workflowNumberSource;
|
||||
private readonly K8sClient client;
|
||||
|
||||
public K8sController(BaseLog log, K8sCluster cluster, KnownK8sPods knownPods, WorkflowNumberSource workflowNumberSource, string testNamespace)
|
||||
public K8sController(ILog log, K8sCluster cluster, KnownK8sPods knownPods, WorkflowNumberSource workflowNumberSource, string k8sNamespace)
|
||||
{
|
||||
this.log = log;
|
||||
this.cluster = cluster;
|
||||
|
|
@ -21,7 +21,7 @@ namespace KubernetesWorkflow
|
|||
this.workflowNumberSource = workflowNumberSource;
|
||||
client = new K8sClient(cluster.GetK8sClientConfig());
|
||||
|
||||
K8sTestNamespace = cluster.Configuration.K8sNamespacePrefix + testNamespace;
|
||||
K8sNamespace = k8sNamespace;
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
|
|
@ -54,7 +54,7 @@ namespace KubernetesWorkflow
|
|||
public void DownloadPodLog(RunningPod pod, ContainerRecipe recipe, ILogHandler logHandler, int? tailLines)
|
||||
{
|
||||
log.Debug();
|
||||
using var stream = client.Run(c => c.ReadNamespacedPodLog(pod.PodInfo.Name, K8sTestNamespace, recipe.Name, tailLines: tailLines));
|
||||
using var stream = client.Run(c => c.ReadNamespacedPodLog(pod.PodInfo.Name, K8sNamespace, recipe.Name, tailLines: tailLines));
|
||||
logHandler.Log(stream);
|
||||
}
|
||||
|
||||
|
|
@ -63,7 +63,7 @@ namespace KubernetesWorkflow
|
|||
var cmdAndArgs = $"{containerName}: {command} ({string.Join(",", args)})";
|
||||
log.Debug(cmdAndArgs);
|
||||
|
||||
var runner = new CommandRunner(client, K8sTestNamespace, pod, containerName, command, args);
|
||||
var runner = new CommandRunner(client, K8sNamespace, pod, containerName, command, args);
|
||||
runner.Run();
|
||||
var result = runner.GetStdOut();
|
||||
|
||||
|
|
@ -71,12 +71,12 @@ namespace KubernetesWorkflow
|
|||
return result;
|
||||
}
|
||||
|
||||
public void DeleteAllResources()
|
||||
public void DeleteAllNamespacesStartingWith(string prefix)
|
||||
{
|
||||
log.Debug();
|
||||
|
||||
var all = client.Run(c => c.ListNamespace().Items);
|
||||
var namespaces = all.Select(n => n.Name()).Where(n => n.StartsWith(cluster.Configuration.K8sNamespacePrefix));
|
||||
var namespaces = all.Select(n => n.Name()).Where(n => n.StartsWith(prefix));
|
||||
|
||||
foreach (var ns in namespaces)
|
||||
{
|
||||
|
|
@ -88,12 +88,12 @@ namespace KubernetesWorkflow
|
|||
}
|
||||
}
|
||||
|
||||
public void DeleteTestNamespace()
|
||||
public void DeleteNamespace()
|
||||
{
|
||||
log.Debug();
|
||||
if (IsTestNamespaceOnline())
|
||||
{
|
||||
client.Run(c => c.DeleteNamespace(K8sTestNamespace, null, null, gracePeriodSeconds: 0));
|
||||
client.Run(c => c.DeleteNamespace(K8sNamespace, null, null, gracePeriodSeconds: 0));
|
||||
}
|
||||
WaitUntilNamespaceDeleted();
|
||||
}
|
||||
|
|
@ -145,7 +145,7 @@ namespace KubernetesWorkflow
|
|||
|
||||
#region Namespace management
|
||||
|
||||
private string K8sTestNamespace { get; }
|
||||
private string K8sNamespace { get; }
|
||||
|
||||
private void EnsureTestNamespace()
|
||||
{
|
||||
|
|
@ -156,8 +156,8 @@ namespace KubernetesWorkflow
|
|||
ApiVersion = "v1",
|
||||
Metadata = new V1ObjectMeta
|
||||
{
|
||||
Name = K8sTestNamespace,
|
||||
Labels = new Dictionary<string, string> { { "name", K8sTestNamespace } }
|
||||
Name = K8sNamespace,
|
||||
Labels = new Dictionary<string, string> { { "name", K8sNamespace } }
|
||||
}
|
||||
};
|
||||
client.Run(c => c.CreateNamespace(namespaceSpec));
|
||||
|
|
@ -168,7 +168,7 @@ namespace KubernetesWorkflow
|
|||
|
||||
private bool IsTestNamespaceOnline()
|
||||
{
|
||||
return IsNamespaceOnline(K8sTestNamespace);
|
||||
return IsNamespaceOnline(K8sNamespace);
|
||||
}
|
||||
|
||||
private bool IsNamespaceOnline(string name)
|
||||
|
|
@ -185,7 +185,7 @@ namespace KubernetesWorkflow
|
|||
Metadata = new V1ObjectMeta
|
||||
{
|
||||
Name = "isolate-policy",
|
||||
NamespaceProperty = K8sTestNamespace
|
||||
NamespaceProperty = K8sNamespace
|
||||
},
|
||||
Spec = new V1NetworkPolicySpec
|
||||
{
|
||||
|
|
@ -314,7 +314,7 @@ namespace KubernetesWorkflow
|
|||
}
|
||||
};
|
||||
|
||||
c.CreateNamespacedNetworkPolicy(body, K8sTestNamespace);
|
||||
c.CreateNamespacedNetworkPolicy(body, K8sNamespace);
|
||||
});
|
||||
}
|
||||
|
||||
|
|
@ -352,7 +352,7 @@ namespace KubernetesWorkflow
|
|||
}
|
||||
};
|
||||
|
||||
client.Run(c => c.CreateNamespacedDeployment(deploymentSpec, K8sTestNamespace));
|
||||
client.Run(c => c.CreateNamespacedDeployment(deploymentSpec, K8sNamespace));
|
||||
WaitUntilDeploymentOnline(deploymentSpec.Metadata.Name);
|
||||
|
||||
return deploymentSpec.Metadata.Name;
|
||||
|
|
@ -360,7 +360,7 @@ namespace KubernetesWorkflow
|
|||
|
||||
private void DeleteDeployment(string deploymentName)
|
||||
{
|
||||
client.Run(c => c.DeleteNamespacedDeployment(deploymentName, K8sTestNamespace));
|
||||
client.Run(c => c.DeleteNamespacedDeployment(deploymentName, K8sNamespace));
|
||||
WaitUntilDeploymentOffline(deploymentName);
|
||||
}
|
||||
|
||||
|
|
@ -399,8 +399,8 @@ namespace KubernetesWorkflow
|
|||
{
|
||||
return new V1ObjectMeta
|
||||
{
|
||||
Name = "deploy-" + workflowNumberSource.WorkflowNumber,
|
||||
NamespaceProperty = K8sTestNamespace,
|
||||
Name = string.Join('-',containerRecipes.Select(r => r.Name)),
|
||||
NamespaceProperty = K8sNamespace,
|
||||
Labels = GetSelector(containerRecipes),
|
||||
Annotations = GetAnnotations(containerRecipes)
|
||||
};
|
||||
|
|
@ -495,7 +495,7 @@ namespace KubernetesWorkflow
|
|||
}
|
||||
}
|
||||
}
|
||||
}, K8sTestNamespace));
|
||||
}, K8sNamespace));
|
||||
|
||||
return new V1Volume
|
||||
{
|
||||
|
|
@ -571,7 +571,7 @@ namespace KubernetesWorkflow
|
|||
}
|
||||
};
|
||||
|
||||
client.Run(c => c.CreateNamespacedService(serviceSpec, K8sTestNamespace));
|
||||
client.Run(c => c.CreateNamespacedService(serviceSpec, K8sNamespace));
|
||||
|
||||
ReadBackServiceAndMapPorts(serviceSpec, containerRecipes, result);
|
||||
|
||||
|
|
@ -581,7 +581,7 @@ namespace KubernetesWorkflow
|
|||
private void ReadBackServiceAndMapPorts(V1Service serviceSpec, ContainerRecipe[] containerRecipes, List<ContainerRecipePortMapEntry> result)
|
||||
{
|
||||
// For each container-recipe, we need to figure out which service-ports it was assigned by K8s.
|
||||
var readback = client.Run(c => c.ReadNamespacedService(serviceSpec.Metadata.Name, K8sTestNamespace));
|
||||
var readback = client.Run(c => c.ReadNamespacedService(serviceSpec.Metadata.Name, K8sNamespace));
|
||||
foreach (var r in containerRecipes)
|
||||
{
|
||||
if (r.ExposedPorts.Any())
|
||||
|
|
@ -610,7 +610,7 @@ namespace KubernetesWorkflow
|
|||
|
||||
private void DeleteService(string serviceName)
|
||||
{
|
||||
client.Run(c => c.DeleteNamespacedService(serviceName, K8sTestNamespace));
|
||||
client.Run(c => c.DeleteNamespacedService(serviceName, K8sNamespace));
|
||||
}
|
||||
|
||||
private V1ObjectMeta CreateServiceMetadata()
|
||||
|
|
@ -618,7 +618,7 @@ namespace KubernetesWorkflow
|
|||
return new V1ObjectMeta
|
||||
{
|
||||
Name = "service-" + workflowNumberSource.WorkflowNumber,
|
||||
NamespaceProperty = K8sTestNamespace
|
||||
NamespaceProperty = K8sNamespace
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -672,7 +672,7 @@ namespace KubernetesWorkflow
|
|||
{
|
||||
WaitUntil(() =>
|
||||
{
|
||||
var deployment = client.Run(c => c.ReadNamespacedDeployment(deploymentName, K8sTestNamespace));
|
||||
var deployment = client.Run(c => c.ReadNamespacedDeployment(deploymentName, K8sNamespace));
|
||||
return deployment?.Status.AvailableReplicas != null && deployment.Status.AvailableReplicas > 0;
|
||||
});
|
||||
}
|
||||
|
|
@ -681,7 +681,7 @@ namespace KubernetesWorkflow
|
|||
{
|
||||
WaitUntil(() =>
|
||||
{
|
||||
var deployments = client.Run(c => c.ListNamespacedDeployment(K8sTestNamespace));
|
||||
var deployments = client.Run(c => c.ListNamespacedDeployment(K8sNamespace));
|
||||
var deployment = deployments.Items.SingleOrDefault(d => d.Metadata.Name == deploymentName);
|
||||
return deployment == null || deployment.Status.AvailableReplicas == 0;
|
||||
});
|
||||
|
|
@ -691,7 +691,7 @@ namespace KubernetesWorkflow
|
|||
{
|
||||
WaitUntil(() =>
|
||||
{
|
||||
var pods = client.Run(c => c.ListNamespacedPod(K8sTestNamespace)).Items;
|
||||
var pods = client.Run(c => c.ListNamespacedPod(K8sNamespace)).Items;
|
||||
var pod = pods.SingleOrDefault(p => p.Metadata.Name == podName);
|
||||
return pod == null;
|
||||
});
|
||||
|
|
@ -714,12 +714,12 @@ namespace KubernetesWorkflow
|
|||
|
||||
public CrashWatcher CreateCrashWatcher(RunningContainer container)
|
||||
{
|
||||
return new CrashWatcher(log, cluster.GetK8sClientConfig(), K8sTestNamespace, container);
|
||||
return new CrashWatcher(log, cluster.GetK8sClientConfig(), K8sNamespace, container);
|
||||
}
|
||||
|
||||
private PodInfo FetchNewPod()
|
||||
{
|
||||
var pods = client.Run(c => c.ListNamespacedPod(K8sTestNamespace)).Items;
|
||||
var pods = client.Run(c => c.ListNamespacedPod(K8sNamespace)).Items;
|
||||
|
||||
var newPods = pods.Where(p => !knownPods.Contains(p.Name())).ToArray();
|
||||
if (newPods.Length != 1) throw new InvalidOperationException("Expected only 1 pod to be created. Test infra failure.");
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
namespace KubernetesWorkflow
|
||||
{
|
||||
public interface IK8sHooks
|
||||
{
|
||||
void OnContainersStarted(RunningContainers runningContainers);
|
||||
void OnContainersStopped(RunningContainers runningContainers);
|
||||
void OnContainerRecipeCreated(ContainerRecipe recipe);
|
||||
}
|
||||
|
||||
public class DoNothingK8sHooks : IK8sHooks
|
||||
{
|
||||
public void OnContainersStarted(RunningContainers runningContainers)
|
||||
{
|
||||
}
|
||||
|
||||
public void OnContainersStopped(RunningContainers runningContainers)
|
||||
{
|
||||
}
|
||||
|
||||
public void OnContainerRecipeCreated(ContainerRecipe recipe)
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
namespace KubernetesWorkflow
|
||||
{
|
||||
public static class K8sNameUtils
|
||||
{
|
||||
public static string Format(string s)
|
||||
{
|
||||
var result = s.ToLowerInvariant()
|
||||
.Replace(" ", "-")
|
||||
.Replace(":", "-")
|
||||
.Replace("/", "-")
|
||||
.Replace("\\", "-")
|
||||
.Replace("[", "-")
|
||||
.Replace("]", "-")
|
||||
.Replace(",", "-");
|
||||
|
||||
return result.Trim('-');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -9,6 +9,7 @@
|
|||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="KubernetesClient" Version="10.1.4" />
|
||||
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
|
|
@ -6,7 +6,7 @@
|
|||
|
||||
public void Add(string key, string value)
|
||||
{
|
||||
labels.Add(key, Format(value));
|
||||
labels.Add(key, K8sNameUtils.Format(value));
|
||||
}
|
||||
|
||||
public PodLabels Clone()
|
||||
|
|
@ -21,19 +21,6 @@
|
|||
labels.Clear();
|
||||
}
|
||||
|
||||
private static string Format(string s)
|
||||
{
|
||||
var result = s.ToLowerInvariant()
|
||||
.Replace(":", "-")
|
||||
.Replace("/", "-")
|
||||
.Replace("\\", "-")
|
||||
.Replace("[", "-")
|
||||
.Replace("]", "-")
|
||||
.Replace(",", "-");
|
||||
|
||||
return result.Trim('-');
|
||||
}
|
||||
|
||||
internal Dictionary<string, string> GetLabels()
|
||||
{
|
||||
return labels;
|
||||
|
|
@ -0,0 +1,55 @@
|
|||
using System.Net.NetworkInformation;
|
||||
using Utils;
|
||||
|
||||
namespace KubernetesWorkflow
|
||||
{
|
||||
internal enum RunnerLocation
|
||||
{
|
||||
ExternalToCluster,
|
||||
InternalToCluster,
|
||||
}
|
||||
|
||||
internal static class RunnerLocationUtils
|
||||
{
|
||||
private static RunnerLocation? knownLocation = null;
|
||||
|
||||
internal static RunnerLocation DetermineRunnerLocation(RunningContainer container)
|
||||
{
|
||||
if (knownLocation != null) return knownLocation.Value;
|
||||
|
||||
if (PingHost(container.Pod.PodInfo.Ip))
|
||||
{
|
||||
knownLocation = RunnerLocation.InternalToCluster;
|
||||
}
|
||||
if (PingHost(Format(container.ClusterExternalAddress)))
|
||||
{
|
||||
knownLocation = RunnerLocation.ExternalToCluster;
|
||||
}
|
||||
|
||||
if (knownLocation == null) throw new Exception("Unable to determine location relative to kubernetes cluster.");
|
||||
return knownLocation.Value;
|
||||
}
|
||||
|
||||
private static string Format(Address host)
|
||||
{
|
||||
return host.Host
|
||||
.Replace("http://", "")
|
||||
.Replace("https://", "");
|
||||
}
|
||||
|
||||
private static bool PingHost(string host)
|
||||
{
|
||||
try
|
||||
{
|
||||
using var pinger = new Ping();
|
||||
PingReply reply = pinger.Send(host);
|
||||
return reply.Status == IPStatus.Success;
|
||||
}
|
||||
catch (PingException)
|
||||
{
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -42,7 +42,17 @@ namespace KubernetesWorkflow
|
|||
public Address ClusterInternalAddress { get; }
|
||||
|
||||
[JsonIgnore]
|
||||
public CrashWatcher? CrashWatcher { get; set; }
|
||||
public Address Address
|
||||
{
|
||||
get
|
||||
{
|
||||
if (RunnerLocationUtils.DetermineRunnerLocation(this) == RunnerLocation.InternalToCluster)
|
||||
{
|
||||
return ClusterInternalAddress;
|
||||
}
|
||||
return ClusterExternalAddress;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static class RunningContainersExtensions
|
||||
|
|
@ -3,22 +3,33 @@ using Utils;
|
|||
|
||||
namespace KubernetesWorkflow
|
||||
{
|
||||
public class StartupWorkflow
|
||||
public interface IStartupWorkflow
|
||||
{
|
||||
private readonly BaseLog log;
|
||||
RunningContainers Start(int numberOfContainers, Location location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig);
|
||||
CrashWatcher CreateCrashWatcher(RunningContainer container);
|
||||
void Stop(RunningContainers runningContainers);
|
||||
void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null);
|
||||
string ExecuteCommand(RunningContainer container, string command, params string[] args);
|
||||
void DeleteNamespace();
|
||||
void DeleteNamespacesStartingWith(string namespacePrefix);
|
||||
}
|
||||
|
||||
public class StartupWorkflow : IStartupWorkflow
|
||||
{
|
||||
private readonly ILog log;
|
||||
private readonly WorkflowNumberSource numberSource;
|
||||
private readonly K8sCluster cluster;
|
||||
private readonly KnownK8sPods knownK8SPods;
|
||||
private readonly string testNamespace;
|
||||
private readonly string k8sNamespace;
|
||||
private readonly RecipeComponentFactory componentFactory = new RecipeComponentFactory();
|
||||
|
||||
internal StartupWorkflow(BaseLog log, WorkflowNumberSource numberSource, K8sCluster cluster, KnownK8sPods knownK8SPods, string testNamespace)
|
||||
internal StartupWorkflow(ILog log, WorkflowNumberSource numberSource, K8sCluster cluster, KnownK8sPods knownK8SPods, string k8sNamespace)
|
||||
{
|
||||
this.log = log;
|
||||
this.numberSource = numberSource;
|
||||
this.cluster = cluster;
|
||||
this.knownK8SPods = knownK8SPods;
|
||||
this.testNamespace = testNamespace;
|
||||
this.k8sNamespace = k8sNamespace;
|
||||
}
|
||||
|
||||
public RunningContainers Start(int numberOfContainers, Location location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
|
||||
|
|
@ -26,16 +37,18 @@ namespace KubernetesWorkflow
|
|||
return K8s(controller =>
|
||||
{
|
||||
var recipes = CreateRecipes(numberOfContainers, recipeFactory, startupConfig);
|
||||
|
||||
var runningPod = controller.BringOnline(recipes, location);
|
||||
var containers = CreateContainers(runningPod, recipes, startupConfig);
|
||||
|
||||
return new RunningContainers(startupConfig, runningPod, CreateContainers(runningPod, recipes, startupConfig));
|
||||
var rc = new RunningContainers(startupConfig, runningPod, containers);
|
||||
cluster.Configuration.Hooks.OnContainersStarted(rc);
|
||||
return rc;
|
||||
});
|
||||
}
|
||||
|
||||
public CrashWatcher CreateCrashWatcher(RunningContainer container)
|
||||
{
|
||||
return K8s(controller => controller.CreateCrashWatcher(container));
|
||||
return K8s(c => c.CreateCrashWatcher(container));
|
||||
}
|
||||
|
||||
public void Stop(RunningContainers runningContainers)
|
||||
|
|
@ -43,10 +56,11 @@ namespace KubernetesWorkflow
|
|||
K8s(controller =>
|
||||
{
|
||||
controller.Stop(runningContainers.RunningPod);
|
||||
cluster.Configuration.Hooks.OnContainersStopped(runningContainers);
|
||||
});
|
||||
}
|
||||
|
||||
public void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines)
|
||||
public void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null)
|
||||
{
|
||||
K8s(controller =>
|
||||
{
|
||||
|
|
@ -62,19 +76,19 @@ namespace KubernetesWorkflow
|
|||
});
|
||||
}
|
||||
|
||||
public void DeleteAllResources()
|
||||
public void DeleteNamespace()
|
||||
{
|
||||
K8s(controller =>
|
||||
{
|
||||
controller.DeleteAllResources();
|
||||
controller.DeleteNamespace();
|
||||
});
|
||||
}
|
||||
|
||||
public void DeleteTestResources()
|
||||
public void DeleteNamespacesStartingWith(string namespacePrefix)
|
||||
{
|
||||
K8s(controller =>
|
||||
{
|
||||
controller.DeleteTestNamespace();
|
||||
controller.DeleteAllNamespacesStartingWith(namespacePrefix);
|
||||
});
|
||||
}
|
||||
|
||||
|
|
@ -118,11 +132,10 @@ namespace KubernetesWorkflow
|
|||
private Address GetContainerInternalAddress(ContainerRecipe recipe)
|
||||
{
|
||||
var serviceName = "service-" + numberSource.WorkflowNumber;
|
||||
var namespaceName = cluster.Configuration.K8sNamespacePrefix + testNamespace;
|
||||
var port = GetInternalPort(recipe);
|
||||
|
||||
return new Address(
|
||||
$"http://{serviceName}.{namespaceName}.svc.cluster.local",
|
||||
$"http://{serviceName}.{k8sNamespace}.svc.cluster.local",
|
||||
port);
|
||||
}
|
||||
|
||||
|
|
@ -144,7 +157,10 @@ namespace KubernetesWorkflow
|
|||
var result = new List<ContainerRecipe>();
|
||||
for (var i = 0; i < numberOfContainers; i++)
|
||||
{
|
||||
result.Add(recipeFactory.CreateRecipe(i, numberSource.GetContainerNumber(), componentFactory, startupConfig));
|
||||
var recipe = recipeFactory.CreateRecipe(i, numberSource.GetContainerNumber(), componentFactory, startupConfig);
|
||||
if (cluster.Configuration.AddAppPodLabel) recipe.PodLabels.Add("app", recipeFactory.AppName);
|
||||
cluster.Configuration.Hooks.OnContainerRecipeCreated(recipe);
|
||||
result.Add(recipe);
|
||||
}
|
||||
|
||||
return result.ToArray();
|
||||
|
|
@ -152,14 +168,14 @@ namespace KubernetesWorkflow
|
|||
|
||||
private void K8s(Action<K8sController> action)
|
||||
{
|
||||
var controller = new K8sController(log, cluster, knownK8SPods, numberSource, testNamespace);
|
||||
var controller = new K8sController(log, cluster, knownK8SPods, numberSource, k8sNamespace);
|
||||
action(controller);
|
||||
controller.Dispose();
|
||||
}
|
||||
|
||||
private T K8s<T>(Func<K8sController, T> action)
|
||||
{
|
||||
var controller = new K8sController(log, cluster, knownK8SPods, numberSource, testNamespace);
|
||||
var controller = new K8sController(log, cluster, knownK8SPods, numberSource, k8sNamespace);
|
||||
var result = action(controller);
|
||||
controller.Dispose();
|
||||
return result;
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
using Logging;
|
||||
using Utils;
|
||||
|
||||
namespace KubernetesWorkflow
|
||||
{
|
||||
public class WorkflowCreator
|
||||
{
|
||||
private readonly NumberSource numberSource = new NumberSource(0);
|
||||
private readonly NumberSource containerNumberSource = new NumberSource(0);
|
||||
private readonly KnownK8sPods knownPods = new KnownK8sPods();
|
||||
private readonly K8sCluster cluster;
|
||||
private readonly ILog log;
|
||||
private readonly Configuration configuration;
|
||||
private readonly string k8sNamespace;
|
||||
|
||||
public WorkflowCreator(ILog log, Configuration configuration)
|
||||
{
|
||||
this.log = log;
|
||||
this.configuration = configuration;
|
||||
cluster = new K8sCluster(configuration);
|
||||
k8sNamespace = configuration.KubernetesNamespace.ToLowerInvariant();
|
||||
}
|
||||
|
||||
public IStartupWorkflow CreateWorkflow(string? namespaceOverride = null)
|
||||
{
|
||||
var workflowNumberSource = new WorkflowNumberSource(numberSource.GetNextNumber(),
|
||||
containerNumberSource);
|
||||
|
||||
return new StartupWorkflow(log, workflowNumberSource, cluster, knownPods, GetNamespace(namespaceOverride));
|
||||
}
|
||||
|
||||
private string GetNamespace(string? namespaceOverride)
|
||||
{
|
||||
if (namespaceOverride != null)
|
||||
{
|
||||
if (!configuration.AllowNamespaceOverride) throw new Exception("Namespace override is not allowed.");
|
||||
return namespaceOverride;
|
||||
}
|
||||
return k8sNamespace;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -2,12 +2,20 @@
|
|||
|
||||
namespace Logging
|
||||
{
|
||||
public abstract class BaseLog
|
||||
public interface ILog
|
||||
{
|
||||
void Log(string message);
|
||||
void Debug(string message = "", int skipFrames = 0);
|
||||
void Error(string message);
|
||||
void AddStringReplace(string from, string to);
|
||||
LogFile CreateSubfile(string ext = "log");
|
||||
}
|
||||
|
||||
public abstract class BaseLog : ILog
|
||||
{
|
||||
private readonly NumberSource subfileNumberSource = new NumberSource(0);
|
||||
private readonly bool debug;
|
||||
private readonly List<BaseLogStringReplacement> replacements = new List<BaseLogStringReplacement>();
|
||||
private bool hasFailed;
|
||||
private LogFile? logFile;
|
||||
|
||||
protected BaseLog(bool debug)
|
||||
|
|
@ -26,10 +34,6 @@ namespace Logging
|
|||
}
|
||||
}
|
||||
|
||||
public virtual void EndTest()
|
||||
{
|
||||
}
|
||||
|
||||
public virtual void Log(string message)
|
||||
{
|
||||
LogFile.Write(ApplyReplacements(message));
|
||||
|
|
@ -50,13 +54,6 @@ namespace Logging
|
|||
Log($"[ERROR] {message}");
|
||||
}
|
||||
|
||||
public virtual void MarkAsFailed()
|
||||
{
|
||||
if (hasFailed) return;
|
||||
hasFailed = true;
|
||||
LogFile.ConcatToFilename("_FAILED");
|
||||
}
|
||||
|
||||
public virtual void AddStringReplace(string from, string to)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(from)) return;
|
||||
|
|
@ -74,14 +71,6 @@ namespace Logging
|
|||
return new LogFile($"{GetFullName()}_{GetSubfileNumber()}", ext);
|
||||
}
|
||||
|
||||
public void WriteLogTag()
|
||||
{
|
||||
var runId = NameUtils.GetRunId();
|
||||
var category = NameUtils.GetCategoryName();
|
||||
var name = NameUtils.GetTestMethodName();
|
||||
LogFile.WriteRaw($"{runId} {category} {name}");
|
||||
}
|
||||
|
||||
private string ApplyReplacements(string str)
|
||||
{
|
||||
foreach (var replacement in replacements)
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
namespace Logging
|
||||
{
|
||||
public class ConsoleLog : BaseLog
|
||||
{
|
||||
public ConsoleLog() : base(false)
|
||||
{
|
||||
}
|
||||
|
||||
protected override string GetFullName()
|
||||
{
|
||||
return "CONSOLE";
|
||||
}
|
||||
|
||||
public override void Log(string message)
|
||||
{
|
||||
Console.WriteLine(message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
namespace Logging
|
||||
{
|
||||
public class LogPrefixer : ILog
|
||||
{
|
||||
private readonly ILog backingLog;
|
||||
private readonly string prefix;
|
||||
|
||||
public LogPrefixer(ILog backingLog, string prefix)
|
||||
{
|
||||
this.backingLog = backingLog;
|
||||
this.prefix = prefix;
|
||||
}
|
||||
|
||||
public LogFile CreateSubfile(string ext = "log")
|
||||
{
|
||||
return backingLog.CreateSubfile(ext);
|
||||
}
|
||||
|
||||
public void Debug(string message = "", int skipFrames = 0)
|
||||
{
|
||||
backingLog.Debug(prefix + message, skipFrames);
|
||||
}
|
||||
|
||||
public void Error(string message)
|
||||
{
|
||||
backingLog.Error(prefix + message);
|
||||
}
|
||||
|
||||
public void Log(string message)
|
||||
{
|
||||
backingLog.Log(prefix + message);
|
||||
}
|
||||
|
||||
public void AddStringReplace(string from, string to)
|
||||
{
|
||||
backingLog.AddStringReplace(from, to);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
namespace Logging
|
||||
{
|
||||
public class LogSplitter : ILog
|
||||
{
|
||||
private readonly ILog[] targetLogs;
|
||||
|
||||
public LogSplitter(params ILog[] targetLogs)
|
||||
{
|
||||
this.targetLogs = targetLogs;
|
||||
}
|
||||
|
||||
public void AddStringReplace(string from, string to)
|
||||
{
|
||||
OnAll(l => l.AddStringReplace(from, to));
|
||||
}
|
||||
|
||||
public LogFile CreateSubfile(string ext = "log")
|
||||
{
|
||||
return targetLogs.First().CreateSubfile(ext);
|
||||
}
|
||||
|
||||
public void Debug(string message = "", int skipFrames = 0)
|
||||
{
|
||||
OnAll(l => l.Debug(message, skipFrames + 2));
|
||||
}
|
||||
|
||||
public void Error(string message)
|
||||
{
|
||||
OnAll(l => l.Error(message));
|
||||
}
|
||||
|
||||
public void Log(string message)
|
||||
{
|
||||
OnAll(l => l.Log(message));
|
||||
}
|
||||
|
||||
private void OnAll(Action<ILog> action)
|
||||
{
|
||||
foreach (var t in targetLogs) action(t);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -7,12 +7,6 @@
|
|||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="nunit" Version="3.13.3" />
|
||||
<PackageReference Include="NUnit3TestAdapter" Version="4.4.2" />
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.5.0" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\Utils\Utils.csproj" />
|
||||
</ItemGroup>
|
||||
|
|
@ -1,8 +1,8 @@
|
|||
namespace Logging
|
||||
{
|
||||
public class NullLog : TestLog
|
||||
public class NullLog : BaseLog
|
||||
{
|
||||
public NullLog() : base("NULL", false, "NULL")
|
||||
public NullLog() : base(false)
|
||||
{
|
||||
}
|
||||
|
||||
|
|
@ -26,10 +26,6 @@
|
|||
Console.WriteLine("Error: " + message);
|
||||
}
|
||||
|
||||
public override void MarkAsFailed()
|
||||
{
|
||||
}
|
||||
|
||||
public override void AddStringReplace(string from, string to)
|
||||
{
|
||||
}
|
||||
|
|
@ -5,25 +5,25 @@ namespace Logging
|
|||
public class Stopwatch
|
||||
{
|
||||
private readonly DateTime start = DateTime.UtcNow;
|
||||
private readonly BaseLog log;
|
||||
private readonly ILog log;
|
||||
private readonly string name;
|
||||
private readonly bool debug;
|
||||
|
||||
private Stopwatch(BaseLog log, string name, bool debug)
|
||||
private Stopwatch(ILog log, string name, bool debug)
|
||||
{
|
||||
this.log = log;
|
||||
this.name = name;
|
||||
this.debug = debug;
|
||||
}
|
||||
|
||||
public static void Measure(BaseLog log, string name, Action action, bool debug = false)
|
||||
public static void Measure(ILog log, string name, Action action, bool debug = false)
|
||||
{
|
||||
var sw = Begin(log, name, debug);
|
||||
action();
|
||||
sw.End();
|
||||
}
|
||||
|
||||
public static T Measure<T>(BaseLog log, string name, Func<T> action, bool debug = false)
|
||||
public static T Measure<T>(ILog log, string name, Func<T> action, bool debug = false)
|
||||
{
|
||||
var sw = Begin(log, name, debug);
|
||||
var result = action();
|
||||
|
|
@ -31,22 +31,22 @@ namespace Logging
|
|||
return result;
|
||||
}
|
||||
|
||||
public static Stopwatch Begin(BaseLog log)
|
||||
public static Stopwatch Begin(ILog log)
|
||||
{
|
||||
return Begin(log, "");
|
||||
}
|
||||
|
||||
public static Stopwatch Begin(BaseLog log, string name)
|
||||
public static Stopwatch Begin(ILog log, string name)
|
||||
{
|
||||
return Begin(log, name, false);
|
||||
}
|
||||
|
||||
public static Stopwatch Begin(BaseLog log, bool debug)
|
||||
public static Stopwatch Begin(ILog log, bool debug)
|
||||
{
|
||||
return Begin(log, "", debug);
|
||||
}
|
||||
|
||||
public static Stopwatch Begin(BaseLog log, string name, bool debug)
|
||||
public static Stopwatch Begin(ILog log, string name, bool debug)
|
||||
{
|
||||
return new Stopwatch(log, name, debug);
|
||||
}
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
using Nethereum.Hex.HexTypes;
|
||||
using System.Numerics;
|
||||
|
||||
namespace NethereumWorkflow
|
||||
{
|
||||
public static class ConversionExtensions
|
||||
{
|
||||
public static HexBigInteger ToHexBig(this decimal amount)
|
||||
{
|
||||
var bigint = ToBig(amount);
|
||||
var str = bigint.ToString("X");
|
||||
return new HexBigInteger(str);
|
||||
}
|
||||
|
||||
public static BigInteger ToBig(this decimal amount)
|
||||
{
|
||||
return new BigInteger(amount);
|
||||
}
|
||||
|
||||
public static decimal ToDecimal(this HexBigInteger hexBigInteger)
|
||||
{
|
||||
return ToDecimal(hexBigInteger.Value);
|
||||
}
|
||||
|
||||
public static decimal ToDecimal(this BigInteger bigInteger)
|
||||
{
|
||||
return (decimal)bigInteger;
|
||||
}
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue