status-im
/
cs-codex-dist-tests
mirror of
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #68 from codex-storage/feature/deploy-local-codex 

Feature/deploy local codex
This commit is contained in:
Ben Bierens 2023-09-29 10:45:15 +02:00 committed by GitHub
parent 453a1a35a2 ce995b3df7
commit b96751cd2c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 277 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
Framework/KubernetesWorkflow/K8sNameUtils.cs
View File

@ -13,7 +13,10 @@
.Replace("]", "-")
.Replace(",", "-");
return result.Trim('-');
result = result.Trim('-');
if (result.Length > 62) result = result.Substring(0, 62);
return result;
}
}
}

8
ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs
View File

@ -16,12 +16,9 @@ namespace CodexPlugin
public static readonly TimeSpan MaxDownloadTimePerMegabyte = TimeSpan.FromSeconds(2.0);
public override string AppName => "codex";
public override string Image { get; }
public override string Image => GetDockerImage();
public CodexContainerRecipe()
{
Image = GetDockerImage();
}
public static string DockerImageOverride { get; set; } = string.Empty;
protected override void Initialize(StartupConfig startupConfig)
{
@ -122,6 +119,7 @@ namespace CodexPlugin
{
var image = Environment.GetEnvironmentVariable("CODEXDOCKERIMAGE");
if (!string.IsNullOrEmpty(image)) return image;
if (!string.IsNullOrEmpty(DockerImageOverride)) return DockerImageOverride;
return DefaultDockerImage;
}
}

129
ProjectPlugins/CodexPlugin/LocalCodexBuilder.cs Normal file
View File

@ -0,0 +1,129 @@
using CodexPlugin;
using Logging;
using System.Diagnostics;
namespace CodexNetDeployer
{
public class LocalCodexBuilder
{
private readonly ILog log;
private readonly string? repoPath;
private readonly string? dockerUsername;
public LocalCodexBuilder(ILog log, string? repoPath, string? dockerUsername)
{
this.log = new LogPrefixer(log, "(LocalCodexBuilder) ");
this.repoPath = repoPath;
this.dockerUsername = dockerUsername;
}
public LocalCodexBuilder(ILog log, string? repoPath)
: this(log, repoPath, Environment.GetEnvironmentVariable("DOCKERUSERNAME"))
{
}
public LocalCodexBuilder(ILog log)
: this(log, Environment.GetEnvironmentVariable("CODEXREPOPATH"))
{
}
public void Intialize()
{
if (!IsEnabled()) return;
if (string.IsNullOrEmpty(dockerUsername)) throw new Exception("Docker username required. (Pass to constructor or set 'DOCKERUSERNAME' environment variable.)");
if (string.IsNullOrEmpty(repoPath)) throw new Exception("Codex repo path required. (Pass to constructor or set 'CODEXREPOPATH' environment variable.)");
if (!Directory.Exists(repoPath)) throw new Exception($"Path '{repoPath}' does not exist.");
var files = Directory.GetFiles(repoPath);
if (!files.Any(f => f.ToLowerInvariant().EndsWith("codex.nim"))) throw new Exception($"Path '{repoPath}' does not appear to be the Codex repo root.");
Log($"Codex docker image will be built in path '{repoPath}'.");
Log("Please note this can take several minutes. If you're not trying to use a Codex image with local code changes,");
Log("Consider using the default test image or consider setting the 'CODEXDOCKERIMAGE' environment variable to use an already built image.");
CodexContainerRecipe.DockerImageOverride = $"Using docker image locally built in path '{repoPath}'.";
}
public void Build()
{
if (!IsEnabled()) return;
Log("Docker login...");
DockerLogin();
Log($"Logged in. Building Codex image in path '{repoPath}'...");
var customImage = GenerateImageName();
Docker($"build", "-t", customImage, "-f", "./codex.Dockerfile",
"--build-arg=\"MAKE_PARALLEL=4\"",
"--build-arg=\"NIMFLAGS=-d:disableMarchNative -d:codex_enable_api_debug_peers=true -d:codex_enable_api_debug_fetch=true -d:codex_enable_simulated_proof_failures\"",
"--build-arg=\"NAT_IP_AUTO=true\"",
"..");
Log($"Image '{customImage}' built successfully. Pushing...");
Docker("push", customImage);
CodexContainerRecipe.DockerImageOverride = customImage;
Log("Image pushed. Good to go!");
}
private void DockerLogin()
{
var dockerPassword = Environment.GetEnvironmentVariable("DOCKERPASSWORD");
try
{
if (string.IsNullOrEmpty(dockerUsername) || string.IsNullOrEmpty(dockerPassword))
{
Log("Environment variable 'DOCKERPASSWORD' not provided.");
Log("Trying system default...");
Docker("login");
}
else
{
Docker("login", "-u", dockerUsername, "-p", dockerPassword);
}
}
catch
{
Log("Docker login failed.");
Log("Please check the docker username and password provided by the constructor arguments and/or");
Log("set by 'DOCKERUSERNAME' and 'DOCKERPASSWORD' environment variables.");
Log("Note: You can use a docker access token as DOCKERPASSWORD.");
throw;
}
}
private string GenerateImageName()
{
var tag = Environment.GetEnvironmentVariable("DOCKERTAG");
if (string.IsNullOrEmpty(tag)) return $"{dockerUsername!}/nim-codex-autoimage:{Guid.NewGuid().ToString().ToLowerInvariant()}";
return $"{dockerUsername}/nim-codex-autoimage:{tag}";
}
private void Docker(params string[] args)
{
var dockerPath = Path.Combine(repoPath!, "docker");
var startInfo = new ProcessStartInfo()
{
FileName = "docker",
Arguments = string.Join(" ", args),
WorkingDirectory = dockerPath,
};
var process = Process.Start(startInfo);
if (process == null) throw new Exception("Failed to start docker process.");
if (!process.WaitForExit(TimeSpan.FromMinutes(10))) throw new Exception("Docker processed timed out after 10 minutes.");
if (process.ExitCode != 0) throw new Exception("Docker process exited with error.");
}
private bool IsEnabled()
{
return !string.IsNullOrEmpty(repoPath);
}
private void Log(string msg)
{
log.Log(msg);
}
}
}

8
README.md
View File

@ -10,15 +10,19 @@ Nethereum: v4.14.0
## Tests/CodexTests and Tests/CodexLongTests
These are test assemblies that use NUnit3 to perform tests against transient Codex nodes.
Read more [HERE](/Tests/CodexTests/README.md)
## Tests/ContinousTests
A console application that runs tests in an endless loop against a persistent deployment of Codex nodes.
Read more [HERE](/Tests/CodexContinuousTests/README.md)
## Tools/CodexNetDeployer
A console application that can deploy Codex nodes.
Read more [HERE](/Tools/CodexNetDeployer/README.MD)
## Test logs
Because tests potentially take a long time to run, logging is in place to help you investigate failures afterwards. Should a test fail, all Codex terminal output (as well as metrics if they have been enabled) will be downloaded and stored along with a detailed, step-by-step log of the test. If something's gone wrong and you're here to discover the details, head for the logs.
## Framework architecture
The framework is designed to be extended by project-specific plugins. These plugins contribute functionality and abstractions to the framework. Users of the framework use these to perform tasks such as testing and deploying.
![Architecture](/docs/FrameworkArchitecture.png)
## How to contribute a plugin
If you want to add support for your project to the testing framework, follow the steps [HERE](/CONTRIBUTINGPLUGINS.MD)

8
Tests/CodexContinuousTests/Configuration.cs
View File

@ -27,9 +27,6 @@ namespace ContinuousTests
[Uniform("target-duration", "td", "TARGETDURATION", false, "If greater than zero, runner will run for this many seconds before stopping.")]
public int TargetDurationSeconds { get; set; } = 0;
[Uniform("dl-logs", "dl", "DLLOGS", false, "If true, runner will periodically download and save/append container logs to the log path.")]
public bool DownloadContainerLogs { get; set; } = false;
public CodexDeployment CodexDeployment { get; set; } = null!;
}
@ -55,10 +52,7 @@ namespace ContinuousTests
{
var nl = Environment.NewLine;
Console.WriteLine("ContinuousTests will run a set of tests against a codex deployment given a codex-deployment.json file." + nl +
"The tests will run in an endless loop unless otherwise specified, using the test-specific timing values." + nl);
Console.WriteLine("ContinuousTests assumes you are running this tool from *inside* the Kubernetes cluster. " +
"If you are not running this from a container inside the cluster, add the argument '--external'." + nl);
"The tests will run in an endless loop unless otherwise specified." + nl);
}
}
}

3
Tests/CodexContinuousTests/NodeRunner.cs
View File

@ -37,6 +37,9 @@ namespace ContinuousTests
public void RunNode(ICodexNode bootstrapNode, Action<ICodexSetup> setup, Action<ICodexNode> operation)
{
var entryPoint = CreateEntryPoint();
// We have to be sure that the transient node we start is using the same image as whatever's already in the deployed network.
// Therefore, we use the image of the bootstrap node.
CodexContainerRecipe.DockerImageOverride = bootstrapNode.Container.Recipe.Image;
try
{

20
Tests/CodexContinuousTests/README.md Normal file
View File

@ -0,0 +1,20 @@
# Codex Continuous Tests
This CLI tool runs tests in an endless loop, using a network of Codex nodes in a kubernetes cluster. Run `dotnet run -- --help` to view all CLI options.
## Choosing tests
By default, all tests in the `CodexContinuousTests/Tests` folder will be used. If you want to limit your test run to a subset of tests, please delete or disable the other test code files. TODO: We'd like a CLI option for selecting tests. Similar to `dotnet test --filter`, maybe?
## Where do I get a `codex-deployment.json`
See [THIS](../../Tools/CodexNetDeployer/README.MD)
## Output
The test runner will produce a folder with all the test logs. They are sorted by timestamp and reflect the names of the tests. When a test fails, the log file for that specific test will be postfixed with `_FAILED`.
### Pass and fail conditions
While individual tests can pass or fail for a number of times and/or over a length of time as configurable with the CLI argument, the test run entirely is not considered passed or failed until either of the following conditions are met:
1. Failed: The number of test failures has reached the specifid number, or the test runner was manually cancelled.
1. Passed: The failed condition was not reached within the time specified by the target-duration option.
## Transient nodes
The continuous tests runner is designed to use a network of Codex nodes deployed in a kubernetes cluster. The runner will not influence or manage the lifecycle of the nodes in this deployment. However, some test cases require transient nodes.
A transient node is a node started and managed by the test runner on behalf of a specific test. The runner facilitates the tests to start and stop transient nodes, as well as bootstrap those nodes against (permanent) nodes from the deployed network. The test runner makes sure that the transient nodes use the same docker image as the permanent nodes, to avoid version conflicts. However, the image used for transient nodes can be overwritten by setting the `CODEXDOCKERIMAGE` environment variable. The use of a local Codex repository for building override images is not supported for transient nodes.

1
Tests/CodexContinuousTests/run.sh
View File

@ -3,5 +3,4 @@ dotnet run \
--codex-deployment=codex-deployment.json \
--keep=1 \
--stop=10 \
--dl-logs=1 \
--target-duration=172800 # 48 hours

9
Tests/CodexTests/AutoBootstrapDistTest.cs
View File

@ -5,13 +5,16 @@ namespace Tests
{
public class AutoBootstrapDistTest : CodexDistTest
{
private readonly List<ICodexNode> onlineCodexNodes = new List<ICodexNode>();
[SetUp]
public void SetUpBootstrapNode()
{
BootstrapNode = AddCodex(s => s.WithName("BOOTSTRAP"));
onlineCodexNodes.Add(BootstrapNode);
}
[TearDown]
public void TearDownBootstrapNode()
{
BootstrapNode = null;
}
protected override void OnCodexSetup(ICodexSetup setup)

16
Tests/CodexTests/CodexDistTest.cs
View File

@ -1,9 +1,12 @@
using CodexContractsPlugin;
using CodexNetDeployer;
using CodexPlugin;
using Core;
using DistTestCore;
using DistTestCore.Helpers;
using DistTestCore.Logs;
using GethPlugin;
using NUnit.Framework;
using NUnit.Framework.Constraints;
namespace Tests
@ -20,6 +23,19 @@ namespace Tests
ProjectPlugin.Load<MetricsPlugin.MetricsPlugin>();
}
[TearDown]
public void TearDownCodexFixture()
{
onlineCodexNodes.Clear();
}
protected override void Initialize(FixtureLog fixtureLog)
{
var localBuilder = new LocalCodexBuilder(fixtureLog);
localBuilder.Intialize();
localBuilder.Build();
}
public ICodexNode AddCodex()
{
return AddCodex(s => { });

8
Tests/CodexTests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs
View File

@ -1,4 +1,6 @@
using NUnit.Framework;
using CodexContractsPlugin;
using GethPlugin;
using NUnit.Framework;
using Utils;
namespace Tests.DownloadConnectivityTests
@ -17,7 +19,9 @@ namespace Tests.DownloadConnectivityTests
[Test]
public void MarketplaceDoesNotInterfereWithPeerDownload()
{
//AddCodex(2, s => s.EnableMetrics().EnableMarketplace(1000.TestTokens()));
var geth = Ci.StartGethNode(s => s.IsMiner());
var contracts = Ci.StartCodexContracts(geth);
AddCodex(2, s => s.EnableMarketplace(geth, contracts, 10.Eth(), 1000.TestTokens()));
AssertAllNodesConnected();
}

8
Tests/CodexTests/PeerDiscoveryTests/PeerDiscoveryTests.cs
View File

@ -1,4 +1,6 @@
using NUnit.Framework;
using CodexContractsPlugin;
using GethPlugin;
using NUnit.Framework;
namespace Tests.PeerDiscoveryTests
{
@ -26,7 +28,9 @@ namespace Tests.PeerDiscoveryTests
[Test]
public void MarketplaceDoesNotInterfereWithPeerDiscovery()
{
//AddCodex(2, s => s.EnableMarketplace(1000.TestTokens()));
var geth = Ci.StartGethNode(s => s.IsMiner());
var contracts = Ci.StartCodexContracts(geth);
AddCodex(2, s => s.EnableMarketplace(geth, contracts, 10.Eth(), 1000.TestTokens()));
AssertAllNodesConnected();
}

27
Tests/CodexTests/README.md Normal file
View File

@ -0,0 +1,27 @@
# Codex Tests
This is an NUnit test assembly that can be used with the standard dotnet test runner. For all its CLI options, run `dotnet test --help`.
## Example tests
Running all the tests in the assembly can take a while. In order to check basic viability of your setup as well as the Codex image you're using, consider running only the example tests using the filter option: `dotnet test --filter=Example`.
## Output
The test runner will produce a folder named `CodexTestLogs` with all the test logs. They are sorted by timestamp and reflect the names of the test fixtures and individual tests. When a test fails, the log file for that specific test will be postfixed with `_FAILED`. The same applies to the fixture log file. The `STATUS` files contain the test results in JSON, for easy machine reading.
## Overrides
The following environment variables allow you to override specific aspects of the behaviour of the tests.
| Variable | Description |
|------------------|-------------------------------------------------------------------------------------------------------------|
| RUNID | A pod-label 'runid' is added to each pod created during the tests. Use this to set the value of that label. |
| TESTID | Similar to RUNID, except the label is 'testid'. |
| CODEXDOCKERIMAGE | If set, this will be used instead of the default Codex docker image. |
## Using a local Codex repository
If you have a clone of the Codex git repository, and you want to run the tests using your local modifications, the following environment variable options are for you. Please note that any changes made in Codex's 'vendor' directory will be discarded during the build process.
| Variable | Description |
|----------------|--------------------------------------------------------------------------------------------------------------------------|
| CODEXREPOPATH | Path to the Codex repository. |
| DOCKERUSERNAME | Username of your Dockerhub account. |
| DOCKERPASSWORD | Password OR access-token for your Dockerhub account. You can omit this variable to use your system-default account. |
| DOCKERTAG | Optional. Tag used for docker image that will be built and pushed to the Dockerhub account. Random ID used when not set. |

6
Tests/DistTestCore/DistTest.cs
View File

@ -32,6 +32,8 @@ namespace DistTestCore
statusLog = new StatusLog(logConfig, startTime);
globalEntryPoint = new EntryPoint(fixtureLog, configuration.GetK8sConfiguration(new DefaultTimeSet(), TestNamespacePrefix), configuration.GetFileManagerFolder());
Initialize(fixtureLog);
}
[OneTimeSetUp]
@ -140,6 +142,10 @@ namespace DistTestCore
Stopwatch.Measure(Get().Log, name, action);
}
protected virtual void Initialize(FixtureLog fixtureLog)
{
}
protected TestLifecycle Get()
{
lock (lifecycleLock)

15
Tools/CodexNetDeployer/Configuration.cs
View File

@ -14,6 +14,11 @@ namespace CodexNetDeployer
[Uniform("kube-namespace", "kn", "KUBENAMESPACE", true, "Kubernetes namespace to be used for deployment.")]
public string KubeNamespace { get; set; } = string.Empty;
[Uniform("codex-local-repo", "cr", "CODEXLOCALREPOPATH", false, "If set, instead of using the default Codex docker image, the local repository will be used to build an image. " +
"This requires the 'DOCKERUSERNAME' and 'DOCKERPASSWORD' environment variables to be set. (You can omit the password to use your system default, or use a docker access token as DOCKERPASSWORD.) You can set " +
"'DOCKERTAG' to define the image tag. If not set, one will be generated.")]
public string CodexLocalRepoPath { get; set; } = string.Empty;
[Uniform("nodes", "n", "NODES", true, "Number of Codex nodes to be created.")]
public int? NumberOfCodexNodes { get; set; }
@ -59,13 +64,16 @@ namespace CodexNetDeployer
[Uniform("check-connect", "cc", "CHECKCONNECT", false, "If true, deployer check ensure peer-connectivity between all deployed nodes after deployment. Default is false.")]
public bool CheckPeerConnection { get; set; } = false;
public List<string> Validate()
{
var errors = new List<string>();
StringIsSet(nameof(KubeNamespace), KubeNamespace, errors);
StringIsSet(nameof(KubeConfigFile), KubeConfigFile, errors);
StringIsSet(nameof(TestsTypePodLabel), TestsTypePodLabel, errors);
ForEachProperty(
onString: (n, v) => StringIsSet(n, v, errors),
onInt: (n, v) => IntIsOverZero(n, v, errors));
if (NumberOfValidators > NumberOfCodexNodes)
@ -80,12 +88,11 @@ namespace CodexNetDeployer
return errors;
}
private void ForEachProperty(Action<string, string> onString, Action<string, int?> onInt)
private void ForEachProperty(Action<string, int?> onInt)
{
var properties = GetType().GetProperties();
foreach (var p in properties)
{
if (p.PropertyType == typeof(string)) onString(p.Name, (string)p.GetValue(this)!);
if (p.PropertyType == typeof(int?)) onInt(p.Name, (int?)p.GetValue(this)!);
if (p.PropertyType == typeof(int)) onInt(p.Name, (int)p.GetValue(this)!);
}

6
Tools/CodexNetDeployer/Deployer.cs
View File

@ -13,11 +13,13 @@ namespace CodexNetDeployer
private readonly Configuration config;
private readonly PeerConnectivityChecker peerConnectivityChecker;
private readonly EntryPoint entryPoint;
private readonly LocalCodexBuilder localCodexBuilder;
public Deployer(Configuration config)
{
this.config = config;
peerConnectivityChecker = new PeerConnectivityChecker();
localCodexBuilder = new LocalCodexBuilder(new ConsoleLog(), config.CodexLocalRepoPath);
ProjectPlugin.Load<CodexPlugin.CodexPlugin>();
ProjectPlugin.Load<CodexContractsPlugin.CodexContractsPlugin>();
@ -30,6 +32,8 @@ namespace CodexNetDeployer
{
var ep = CreateEntryPoint(new ConsoleLog());
localCodexBuilder.Intialize();
Log("Using plugins:" + Environment.NewLine);
var metadata = ep.GetPluginMetadata();
var longestKey = metadata.Keys.Max(k => k.Length);
@ -44,6 +48,8 @@ namespace CodexNetDeployer
public CodexDeployment Deploy()
{
localCodexBuilder.Build();
Log("Initializing...");
var ci = entryPoint.CreateInterface();

2
Tools/CodexNetDeployer/Program.cs
View File

@ -45,7 +45,7 @@ public class Program
private static void PrintHelp()
{
var nl = Environment.NewLine;
Console.WriteLine("CodexNetDeployer allows you to easily deploy multiple Codex nodes in a Kubernetes cluster. " +
Console.WriteLine("CodexNetDeployer allows you to deploy multiple Codex nodes in a Kubernetes cluster. " +
"The deployer will set up the required supporting services, deploy the Codex on-chain contracts, start and bootstrap the Codex instances. " +
"All Kubernetes objects will be created in the namespace provided, allowing you to easily find, modify, and delete them afterwards." + nl);
}

26
Tools/CodexNetDeployer/README.MD Normal file
View File

@ -0,0 +1,26 @@
# CodexNetDeployer
This CLI tool allows you to create a network of Codex nodes in a kubernetes cluster. There are many arguments that allow you to configure every aspect of the deployment. Each argument can also be controlled using an environment variable. I won't copy-paste the list here because then it'll have to be kept up-to-date. Just run:
`dotnet run -- --help`
## Output
After the deployment has successfully finished, a `codex-deployment.json` file will be created. This file contains all the information necessary to interface with the deployed network. It can be used by other tools, for example the CodexContinuousTests runner CLI tool.
## Overrides
The arguments allow you to configure quite a bit, but not everything. Here are some environment variables the CodexNetDeployer will respond to. None of these are required.
| Variable | Description |
|------------------|--------------------------------------------------------------------------------------------------------------|
| RUNID | A pod-label 'runid' is added to each pod created during deployment. Use this to set the value of that label. |
| TESTID | Similar to RUNID, except the label is 'testid'. |
| CODEXDOCKERIMAGE | If set, this will be used instead of the default Codex docker image. |
## Using a local Codex repository
If you have a clone of the Codex git repository, and you want to deploy a network using your local modifications, the following environment variable options are for you. Please note that any changes made in Codex's 'vendor' directory will be discarded during the build process.
| Variable | Description |
|----------------|--------------------------------------------------------------------------------------------------------------------------|
| CODEXREPOPATH | Path to the Codex repository. |
| DOCKERUSERNAME | Username of your Dockerhub account. |
| DOCKERPASSWORD | Password OR access-token for your Dockerhub account. You can omit this variable to use your system-default account. |
| DOCKERTAG | Optional. Tag used for docker image that will be built and pushed to the Dockerhub account. Random ID used when not set. |