From 0f12bcd08643437cb4980a85988f8e3714e13c11 Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 28 Sep 2023 11:31:09 +0200 Subject: [PATCH 1/9] Sets up local builder in codex plugin. --- .../CodexPlugin/CodexContainerRecipe.cs | 8 +- .../CodexPlugin/LocalCodexBuilder.cs | 116 ++++++++++++++++++ Tools/CodexNetDeployer/Configuration.cs | 45 ++++++- Tools/CodexNetDeployer/Deployer.cs | 6 + 4 files changed, 165 insertions(+), 10 deletions(-) create mode 100644 ProjectPlugins/CodexPlugin/LocalCodexBuilder.cs diff --git a/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs b/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs index bee2d4a..ebef993 100644 --- a/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs +++ b/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs @@ -16,12 +16,9 @@ namespace CodexPlugin public static readonly TimeSpan MaxDownloadTimePerMegabyte = TimeSpan.FromSeconds(2.0); public override string AppName => "codex"; - public override string Image { get; } + public override string Image => GetDockerImage(); - public CodexContainerRecipe() - { - Image = GetDockerImage(); - } + public static string DockerImageOverride { get; set; } = string.Empty; protected override void Initialize(StartupConfig startupConfig) { @@ -122,6 +119,7 @@ namespace CodexPlugin { var image = Environment.GetEnvironmentVariable("CODEXDOCKERIMAGE"); if (!string.IsNullOrEmpty(image)) return image; + if (!string.IsNullOrEmpty(DockerImageOverride)) return DockerImageOverride; return DefaultDockerImage; } } diff --git a/ProjectPlugins/CodexPlugin/LocalCodexBuilder.cs b/ProjectPlugins/CodexPlugin/LocalCodexBuilder.cs new file mode 100644 index 0000000..5e4b2d0 --- /dev/null +++ b/ProjectPlugins/CodexPlugin/LocalCodexBuilder.cs @@ -0,0 +1,116 @@ +using CodexPlugin; +using Logging; +using System.Diagnostics; + +namespace CodexNetDeployer +{ + public class LocalCodexBuilder + { + private readonly ILog log; + private readonly string? repoPath; + private readonly string? dockerUsername; + + public LocalCodexBuilder(ILog log, string? repoPath, string? dockerUsername) + { + this.log = log; + this.repoPath = repoPath; + this.dockerUsername = dockerUsername; + } + + public LocalCodexBuilder(ILog log, string? repoPath) + : this(log, repoPath, Environment.GetEnvironmentVariable("DOCKERUSERNAME")) + { + } + + public LocalCodexBuilder(ILog log) + : this(log, Environment.GetEnvironmentVariable("CODEXREPOPATH")) + { + } + + public void Intialize() + { + if (!IsEnabled()) return; + + if (string.IsNullOrEmpty(dockerUsername)) throw new Exception("Docker username required. (Pass to constructor or set 'DOCKERUSERNAME' environment variable.)"); + if (string.IsNullOrEmpty(repoPath)) throw new Exception("Codex repo path required. (Pass to constructor or set 'CODEXREPOPATH' environment variable.)"); + if (!Directory.Exists(repoPath)) throw new Exception($"Path '{repoPath}' does not exist."); + var files = Directory.GetFiles(repoPath); + if (!files.Any(f => f.ToLowerInvariant().EndsWith("codex.nim"))) throw new Exception($"Path '{repoPath}' does not appear to be the Codex repo root."); + + Log($"Codex docker image will be built in path '{repoPath}'."); + Log("Please note this can take several minutes. If you're not trying to use a Codex image with local code changes,"); + Log("Consider using the default test image or consider setting the 'CODEXDOCKERIMAGE' environment variable to use an already built image."); + CodexContainerRecipe.DockerImageOverride = $"Using docker image locally built in path '{repoPath}'."; + } + + public void Build() + { + if (!IsEnabled()) return; + Log("Docker login..."); + DockerLogin(); + + Log($"Logged in. Building Codex image in path '{repoPath}'..."); + + var customImage = GenerateImageName(); + Docker($"build", "-t", customImage, "-f", "./codex.Dockerfile", + "--build-arg=\"MAKE_PARALLEL=4\"", + "--build-arg=\"NIMFLAGS=-d:disableMarchNative -d:codex_enable_api_debug_peers=true -d:codex_enable_api_debug_fetch=true -d:codex_enable_simulated_proof_failures\"", + "--build-arg=\"NAT_IP_AUTO=true\"", + ".."); + + Log($"Image '{customImage}' built successfully. Pushing..."); + + Docker("push", customImage); + + CodexContainerRecipe.DockerImageOverride = customImage; + Log("Image pushed. Good to go!"); + } + + private void DockerLogin() + { + var dockerPassword = Environment.GetEnvironmentVariable("DOCKERPASSWORD"); + + if (string.IsNullOrEmpty(dockerUsername) || string.IsNullOrEmpty(dockerPassword)) + { + Log("Environment variable 'DOCKERPASSWORD' not provided."); + Log("Trying system default..."); + Docker("login"); + } + else + { + Docker("login", "-u", dockerUsername, "-p", dockerPassword); + } + } + + private string GenerateImageName() + { + return $"{dockerUsername!}/nim-codex-autoimage:{Guid.NewGuid().ToString().ToLowerInvariant()}"; + } + + private void Docker(params string[] args) + { + var dockerPath = Path.Combine(repoPath!, "docker"); + + var startInfo = new ProcessStartInfo() + { + FileName = "docker", + Arguments = string.Join(" ", args), + WorkingDirectory = dockerPath, + }; + var process = Process.Start(startInfo); + if (process == null) throw new Exception("Failed to start docker process."); + if (!process.WaitForExit(TimeSpan.FromMinutes(10))) throw new Exception("Docker processed timed out after 10 minutes."); + if (process.ExitCode != 0) throw new Exception("Docker process exited with error."); + } + + private bool IsEnabled() + { + return !string.IsNullOrEmpty(repoPath); + } + + private void Log(string msg) + { + log.Log(msg); + } + } +} diff --git a/Tools/CodexNetDeployer/Configuration.cs b/Tools/CodexNetDeployer/Configuration.cs index 3ff1020..fa94c15 100644 --- a/Tools/CodexNetDeployer/Configuration.cs +++ b/Tools/CodexNetDeployer/Configuration.cs @@ -1,5 +1,6 @@ using ArgsUniform; using CodexPlugin; +using static Org.BouncyCastle.Math.EC.ECCurve; namespace CodexNetDeployer { @@ -14,6 +15,10 @@ namespace CodexNetDeployer [Uniform("kube-namespace", "kn", "KUBENAMESPACE", true, "Kubernetes namespace to be used for deployment.")] public string KubeNamespace { get; set; } = string.Empty; + [Uniform("codex-local-repo", "cr", "CODEXLOCALREPOPATH", false, "If set, instead of using the default Codex docker image, the local repository will be used to build an image. " + + "This requires the 'DOCKERUSERNAME' and 'DOCKERPASSWORD' environment variables to be set.")] + public string CodexLocalRepoPath { get; set; } = string.Empty; + [Uniform("nodes", "n", "NODES", true, "Number of Codex nodes to be created.")] public int? NumberOfCodexNodes { get; set; } @@ -59,13 +64,44 @@ namespace CodexNetDeployer [Uniform("check-connect", "cc", "CHECKCONNECT", false, "If true, deployer check ensure peer-connectivity between all deployed nodes after deployment. Default is false.")] public bool CheckPeerConnection { get; set; } = false; - - public List Validate() + + public Configuration() + { + // dotnet run \ + //--kube - config =/ opt / kubeconfig.yaml \ + //--kube -namespace=codex-continuous-tests \ + //--nodes=5 \ + //--validators=3 \ + //--log-level=Trace \ + //--storage-quota=2048 \ + //--storage-sell=1024 \ + //--min-price=1024 \ + //--max-collateral=1024 \ + //--max-duration=3600000 \ + //--block-ttl=180 \ + //--block-mi=120 \ + //--block-mn=10000 \ + //--metrics=1 \ + //--check-connect=1 + + KubeNamespace = "autodockertest"; + NumberOfCodexNodes = 3; + NumberOfValidators = 1; + StorageQuota = 2048; + StorageSell = 1024; + + CodexLocalRepoPath = "D:/Projects/nim-codex"; + } + + public List Validate() { var errors = new List(); + StringIsSet(nameof(KubeNamespace), KubeNamespace, errors); + StringIsSet(nameof(KubeConfigFile), KubeConfigFile, errors); + StringIsSet(nameof(TestsTypePodLabel), TestsTypePodLabel, errors); + ForEachProperty( - onString: (n, v) => StringIsSet(n, v, errors), onInt: (n, v) => IntIsOverZero(n, v, errors)); if (NumberOfValidators > NumberOfCodexNodes) @@ -80,12 +116,11 @@ namespace CodexNetDeployer return errors; } - private void ForEachProperty(Action onString, Action onInt) + private void ForEachProperty(Action onInt) { var properties = GetType().GetProperties(); foreach (var p in properties) { - if (p.PropertyType == typeof(string)) onString(p.Name, (string)p.GetValue(this)!); if (p.PropertyType == typeof(int?)) onInt(p.Name, (int?)p.GetValue(this)!); if (p.PropertyType == typeof(int)) onInt(p.Name, (int)p.GetValue(this)!); } diff --git a/Tools/CodexNetDeployer/Deployer.cs b/Tools/CodexNetDeployer/Deployer.cs index 5c1365f..d95f6bf 100644 --- a/Tools/CodexNetDeployer/Deployer.cs +++ b/Tools/CodexNetDeployer/Deployer.cs @@ -13,11 +13,13 @@ namespace CodexNetDeployer private readonly Configuration config; private readonly PeerConnectivityChecker peerConnectivityChecker; private readonly EntryPoint entryPoint; + private readonly LocalCodexBuilder localCodexBuilder; public Deployer(Configuration config) { this.config = config; peerConnectivityChecker = new PeerConnectivityChecker(); + localCodexBuilder = new LocalCodexBuilder(new ConsoleLog(), config.CodexLocalRepoPath); ProjectPlugin.Load(); ProjectPlugin.Load(); @@ -30,6 +32,8 @@ namespace CodexNetDeployer { var ep = CreateEntryPoint(new ConsoleLog()); + localCodexBuilder.Intialize(); + Log("Using plugins:" + Environment.NewLine); var metadata = ep.GetPluginMetadata(); var longestKey = metadata.Keys.Max(k => k.Length); @@ -44,6 +48,8 @@ namespace CodexNetDeployer public CodexDeployment Deploy() { + localCodexBuilder.Build(); + Log("Initializing..."); var ci = entryPoint.CreateInterface(); From b0f72cf453bf45f5de5a83d1e73561bdffe3f6bc Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 28 Sep 2023 12:22:35 +0200 Subject: [PATCH 2/9] Automated local docker build is working. --- Framework/KubernetesWorkflow/K8sNameUtils.cs | 5 ++- .../CodexPlugin/LocalCodexBuilder.cs | 27 +++++++++++---- Tests/CodexTests/CodexDistTest.cs | 9 +++++ Tests/DistTestCore/DistTest.cs | 6 ++++ Tools/CodexNetDeployer/Configuration.cs | 34 ++----------------- 5 files changed, 42 insertions(+), 39 deletions(-) diff --git a/Framework/KubernetesWorkflow/K8sNameUtils.cs b/Framework/KubernetesWorkflow/K8sNameUtils.cs index c888870..65c2228 100644 --- a/Framework/KubernetesWorkflow/K8sNameUtils.cs +++ b/Framework/KubernetesWorkflow/K8sNameUtils.cs @@ -13,7 +13,10 @@ .Replace("]", "-") .Replace(",", "-"); - return result.Trim('-'); + result = result.Trim('-'); + if (result.Length > 62) result = result.Substring(0, 62); + + return result; } } } diff --git a/ProjectPlugins/CodexPlugin/LocalCodexBuilder.cs b/ProjectPlugins/CodexPlugin/LocalCodexBuilder.cs index 5e4b2d0..bd86442 100644 --- a/ProjectPlugins/CodexPlugin/LocalCodexBuilder.cs +++ b/ProjectPlugins/CodexPlugin/LocalCodexBuilder.cs @@ -70,21 +70,34 @@ namespace CodexNetDeployer { var dockerPassword = Environment.GetEnvironmentVariable("DOCKERPASSWORD"); - if (string.IsNullOrEmpty(dockerUsername) || string.IsNullOrEmpty(dockerPassword)) + try { - Log("Environment variable 'DOCKERPASSWORD' not provided."); - Log("Trying system default..."); - Docker("login"); + if (string.IsNullOrEmpty(dockerUsername) || string.IsNullOrEmpty(dockerPassword)) + { + Log("Environment variable 'DOCKERPASSWORD' not provided."); + Log("Trying system default..."); + Docker("login"); + } + else + { + Docker("login", "-u", dockerUsername, "-p", dockerPassword); + } } - else + catch { - Docker("login", "-u", dockerUsername, "-p", dockerPassword); + Log("Docker login failed."); + Log("Please check the docker username and password provided by the constructor arguments and/or"); + Log("set by 'DOCKERUSERNAME' and 'DOCKERPASSWORD' environment variables."); + Log("Note: You can use a docker access token as DOCKERPASSWORD."); + throw; } } private string GenerateImageName() { - return $"{dockerUsername!}/nim-codex-autoimage:{Guid.NewGuid().ToString().ToLowerInvariant()}"; + var tag = Environment.GetEnvironmentVariable("DOCKERTAG"); + if (string.IsNullOrEmpty(tag)) return $"{dockerUsername!}/nim-codex-autoimage:{Guid.NewGuid().ToString().ToLowerInvariant()}"; + return $"{dockerUsername}/nim-codex-autoimage:{tag}"; } private void Docker(params string[] args) diff --git a/Tests/CodexTests/CodexDistTest.cs b/Tests/CodexTests/CodexDistTest.cs index 32c6ec2..f2e863d 100644 --- a/Tests/CodexTests/CodexDistTest.cs +++ b/Tests/CodexTests/CodexDistTest.cs @@ -1,8 +1,10 @@ using CodexContractsPlugin; +using CodexNetDeployer; using CodexPlugin; using Core; using DistTestCore; using DistTestCore.Helpers; +using DistTestCore.Logs; using GethPlugin; using NUnit.Framework.Constraints; @@ -20,6 +22,13 @@ namespace Tests ProjectPlugin.Load(); } + protected override void Initialize(FixtureLog fixtureLog) + { + var localBuilder = new LocalCodexBuilder(fixtureLog); + localBuilder.Intialize(); + localBuilder.Build(); + } + public ICodexNode AddCodex() { return AddCodex(s => { }); diff --git a/Tests/DistTestCore/DistTest.cs b/Tests/DistTestCore/DistTest.cs index 8e9932a..95e69a9 100644 --- a/Tests/DistTestCore/DistTest.cs +++ b/Tests/DistTestCore/DistTest.cs @@ -32,6 +32,8 @@ namespace DistTestCore statusLog = new StatusLog(logConfig, startTime); globalEntryPoint = new EntryPoint(fixtureLog, configuration.GetK8sConfiguration(new DefaultTimeSet(), TestNamespacePrefix), configuration.GetFileManagerFolder()); + + Initialize(fixtureLog); } [OneTimeSetUp] @@ -140,6 +142,10 @@ namespace DistTestCore Stopwatch.Measure(Get().Log, name, action); } + protected virtual void Initialize(FixtureLog fixtureLog) + { + } + protected TestLifecycle Get() { lock (lifecycleLock) diff --git a/Tools/CodexNetDeployer/Configuration.cs b/Tools/CodexNetDeployer/Configuration.cs index fa94c15..3ec20bf 100644 --- a/Tools/CodexNetDeployer/Configuration.cs +++ b/Tools/CodexNetDeployer/Configuration.cs @@ -1,6 +1,5 @@ using ArgsUniform; using CodexPlugin; -using static Org.BouncyCastle.Math.EC.ECCurve; namespace CodexNetDeployer { @@ -16,7 +15,8 @@ namespace CodexNetDeployer public string KubeNamespace { get; set; } = string.Empty; [Uniform("codex-local-repo", "cr", "CODEXLOCALREPOPATH", false, "If set, instead of using the default Codex docker image, the local repository will be used to build an image. " + - "This requires the 'DOCKERUSERNAME' and 'DOCKERPASSWORD' environment variables to be set.")] + "This requires the 'DOCKERUSERNAME' and 'DOCKERPASSWORD' environment variables to be set. (You can omit the password to use your system default, or use a docker access token as DOCKERPASSWORD.) You can set " + + "'DOCKERTAG' to define the image tag. If not set, one will be generated.")] public string CodexLocalRepoPath { get; set; } = string.Empty; [Uniform("nodes", "n", "NODES", true, "Number of Codex nodes to be created.")] @@ -65,35 +65,7 @@ namespace CodexNetDeployer [Uniform("check-connect", "cc", "CHECKCONNECT", false, "If true, deployer check ensure peer-connectivity between all deployed nodes after deployment. Default is false.")] public bool CheckPeerConnection { get; set; } = false; - public Configuration() - { - // dotnet run \ - //--kube - config =/ opt / kubeconfig.yaml \ - //--kube -namespace=codex-continuous-tests \ - //--nodes=5 \ - //--validators=3 \ - //--log-level=Trace \ - //--storage-quota=2048 \ - //--storage-sell=1024 \ - //--min-price=1024 \ - //--max-collateral=1024 \ - //--max-duration=3600000 \ - //--block-ttl=180 \ - //--block-mi=120 \ - //--block-mn=10000 \ - //--metrics=1 \ - //--check-connect=1 - - KubeNamespace = "autodockertest"; - NumberOfCodexNodes = 3; - NumberOfValidators = 1; - StorageQuota = 2048; - StorageSell = 1024; - - CodexLocalRepoPath = "D:/Projects/nim-codex"; - } - - public List Validate() + public List Validate() { var errors = new List(); From 260294fb351cccd6baadcea999453a547d651a44 Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 28 Sep 2023 14:00:13 +0200 Subject: [PATCH 3/9] Fixes marketplace interference check tests. --- .../FullyConnectedDownloadTests.cs | 8 ++++++-- Tests/CodexTests/PeerDiscoveryTests/PeerDiscoveryTests.cs | 8 ++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/Tests/CodexTests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs b/Tests/CodexTests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs index ed66240..567c611 100644 --- a/Tests/CodexTests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs +++ b/Tests/CodexTests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs @@ -1,4 +1,6 @@ -using NUnit.Framework; +using CodexContractsPlugin; +using GethPlugin; +using NUnit.Framework; using Utils; namespace Tests.DownloadConnectivityTests @@ -17,7 +19,9 @@ namespace Tests.DownloadConnectivityTests [Test] public void MarketplaceDoesNotInterfereWithPeerDownload() { - //AddCodex(2, s => s.EnableMetrics().EnableMarketplace(1000.TestTokens())); + var geth = Ci.StartGethNode(s => s.IsMiner()); + var contracts = Ci.StartCodexContracts(geth); + AddCodex(2, s => s.EnableMarketplace(geth, contracts, 10.Eth(), 1000.TestTokens())); AssertAllNodesConnected(); } diff --git a/Tests/CodexTests/PeerDiscoveryTests/PeerDiscoveryTests.cs b/Tests/CodexTests/PeerDiscoveryTests/PeerDiscoveryTests.cs index 4141925..9327e7b 100644 --- a/Tests/CodexTests/PeerDiscoveryTests/PeerDiscoveryTests.cs +++ b/Tests/CodexTests/PeerDiscoveryTests/PeerDiscoveryTests.cs @@ -1,4 +1,6 @@ -using NUnit.Framework; +using CodexContractsPlugin; +using GethPlugin; +using NUnit.Framework; namespace Tests.PeerDiscoveryTests { @@ -26,7 +28,9 @@ namespace Tests.PeerDiscoveryTests [Test] public void MarketplaceDoesNotInterfereWithPeerDiscovery() { - //AddCodex(2, s => s.EnableMarketplace(1000.TestTokens())); + var geth = Ci.StartGethNode(s => s.IsMiner()); + var contracts = Ci.StartCodexContracts(geth); + AddCodex(2, s => s.EnableMarketplace(geth, contracts, 10.Eth(), 1000.TestTokens())); AssertAllNodesConnected(); } From 993f2ff813b8333a64399a5903ae3319aa88f41d Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 28 Sep 2023 14:23:48 +0200 Subject: [PATCH 4/9] Fixes issue where autobootstrap node doesn't clear old bootstrap node handles. --- Tests/CodexTests/AutoBootstrapDistTest.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Tests/CodexTests/AutoBootstrapDistTest.cs b/Tests/CodexTests/AutoBootstrapDistTest.cs index 046ff0a..86c7c61 100644 --- a/Tests/CodexTests/AutoBootstrapDistTest.cs +++ b/Tests/CodexTests/AutoBootstrapDistTest.cs @@ -14,6 +14,12 @@ namespace Tests onlineCodexNodes.Add(BootstrapNode); } + [TearDown] + public void TearDownBootstrapNode() + { + BootstrapNode = null; + } + protected override void OnCodexSetup(ICodexSetup setup) { if (BootstrapNode != null) setup.WithBootstrapNode(BootstrapNode); From 3e1b361b7b15935d99bff3bf9b5f969e90e7385a Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 28 Sep 2023 15:04:21 +0200 Subject: [PATCH 5/9] Fixes cleanup of online codexnodes in Codex test fixture --- Tests/CodexTests/AutoBootstrapDistTest.cs | 3 --- Tests/CodexTests/CodexDistTest.cs | 7 +++++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/Tests/CodexTests/AutoBootstrapDistTest.cs b/Tests/CodexTests/AutoBootstrapDistTest.cs index 86c7c61..869453a 100644 --- a/Tests/CodexTests/AutoBootstrapDistTest.cs +++ b/Tests/CodexTests/AutoBootstrapDistTest.cs @@ -5,13 +5,10 @@ namespace Tests { public class AutoBootstrapDistTest : CodexDistTest { - private readonly List onlineCodexNodes = new List(); - [SetUp] public void SetUpBootstrapNode() { BootstrapNode = AddCodex(s => s.WithName("BOOTSTRAP")); - onlineCodexNodes.Add(BootstrapNode); } [TearDown] diff --git a/Tests/CodexTests/CodexDistTest.cs b/Tests/CodexTests/CodexDistTest.cs index f2e863d..d521ad4 100644 --- a/Tests/CodexTests/CodexDistTest.cs +++ b/Tests/CodexTests/CodexDistTest.cs @@ -6,6 +6,7 @@ using DistTestCore; using DistTestCore.Helpers; using DistTestCore.Logs; using GethPlugin; +using NUnit.Framework; using NUnit.Framework.Constraints; namespace Tests @@ -22,6 +23,12 @@ namespace Tests ProjectPlugin.Load(); } + [TearDown] + public void TearDownCodexFixture() + { + onlineCodexNodes.Clear(); + } + protected override void Initialize(FixtureLog fixtureLog) { var localBuilder = new LocalCodexBuilder(fixtureLog); From 8d2ce8ae7a8963f34ccadb62e7d553af3d6c8012 Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 28 Sep 2023 15:29:29 +0200 Subject: [PATCH 6/9] Adds Readme to codex net deployer --- Tools/CodexNetDeployer/README.MD | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 Tools/CodexNetDeployer/README.MD diff --git a/Tools/CodexNetDeployer/README.MD b/Tools/CodexNetDeployer/README.MD new file mode 100644 index 0000000..c60dd71 --- /dev/null +++ b/Tools/CodexNetDeployer/README.MD @@ -0,0 +1,25 @@ +# CodexNetDeployer + +This CLI tool allows you to create a network of Codex nodes in a kubernetes cluster. There are many arguments that allow you to configure every aspect of the deployment. Each argument can also be controlled using an environment variable. I won't copy-paste the list here because then it'll have to be kept up-to-date. Just run: +`dotnet run -- --help` + +## Output +After the deployment has successfully finished, a `codex-deployment.json` file will be created. This file contains all the information necessary to interface with the deployed network. It can be used by other tools, for example the CodexContinuousTests runner CLI tool. + +## Overrides +The arguments allow you to configure quite a bit, but not everything. Here are some environment variables the CodexNetDeployer will respond to. None of these are required. + +| Variable | Description | +|------------------|--------------------------------------------------------------------------------------------------------------| +| RUNID | A pod-label 'runid' is added to each pod created during deployment. Use this to set the value of that label. | +| TESTID | Similar to RUNID, except the label is 'testid'. | +| CODEXDOCKERIMAGE | If set, this will be used instead of the default Codex docker image. | + +## Using a local Codex repository +If you have a clone of the Codex git repository, and you want to deploy a network using your local modifications, the following environment variable options are for you. Please note that any changes made in Codex's 'vendor' directory will be discarded during the build process. + +| Variable | Description | +|----------------|---------------------------------------------------------------------------------------------------------------------| +| CODEXREPOPATH | Path to the Codex repository. | +| DOCKERUSERNAME | Username of your Dockerhub account. | +| DOCKERPASSWORD | Password OR access-token for your Dockerhub account. You can omit this variable to use your system-default account. | From dcdcd110a1d73d89621700708a5819193019031d Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 29 Sep 2023 10:19:59 +0200 Subject: [PATCH 7/9] Updates READMEs --- README.md | 8 +++++-- Tests/CodexContinuousTests/Configuration.cs | 8 +------ Tests/CodexContinuousTests/NodeRunner.cs | 3 +++ Tests/CodexContinuousTests/README.md | 20 ++++++++++++++++ Tests/CodexContinuousTests/run.sh | 1 - Tests/CodexTests/README.md | 26 +++++++++++++++++++++ Tools/CodexNetDeployer/Program.cs | 2 +- 7 files changed, 57 insertions(+), 11 deletions(-) create mode 100644 Tests/CodexContinuousTests/README.md create mode 100644 Tests/CodexTests/README.md diff --git a/README.md b/README.md index ee26ed4..7b22bd2 100644 --- a/README.md +++ b/README.md @@ -10,15 +10,19 @@ Nethereum: v4.14.0 ## Tests/CodexTests and Tests/CodexLongTests These are test assemblies that use NUnit3 to perform tests against transient Codex nodes. +Read more [HERE](/Tests/CodexTests/README.md) ## Tests/ContinousTests A console application that runs tests in an endless loop against a persistent deployment of Codex nodes. +Read more [HERE](/Tests/CodexContinuousTests/README.md) ## Tools/CodexNetDeployer A console application that can deploy Codex nodes. +Read more [HERE](/Tools/CodexNetDeployer/README.MD) -## Test logs -Because tests potentially take a long time to run, logging is in place to help you investigate failures afterwards. Should a test fail, all Codex terminal output (as well as metrics if they have been enabled) will be downloaded and stored along with a detailed, step-by-step log of the test. If something's gone wrong and you're here to discover the details, head for the logs. +## Framework architecture +The framework is designed to be extended by project-specific plugins. These plugins contribute functionality and abstractions to the framework. Users of the framework use these to perform tasks such as testing and deploying. +![Architecture](/docs/FrameworkArchitecture.png) ## How to contribute a plugin If you want to add support for your project to the testing framework, follow the steps [HERE](/CONTRIBUTINGPLUGINS.MD) diff --git a/Tests/CodexContinuousTests/Configuration.cs b/Tests/CodexContinuousTests/Configuration.cs index e84a253..f8480b8 100644 --- a/Tests/CodexContinuousTests/Configuration.cs +++ b/Tests/CodexContinuousTests/Configuration.cs @@ -27,9 +27,6 @@ namespace ContinuousTests [Uniform("target-duration", "td", "TARGETDURATION", false, "If greater than zero, runner will run for this many seconds before stopping.")] public int TargetDurationSeconds { get; set; } = 0; - [Uniform("dl-logs", "dl", "DLLOGS", false, "If true, runner will periodically download and save/append container logs to the log path.")] - public bool DownloadContainerLogs { get; set; } = false; - public CodexDeployment CodexDeployment { get; set; } = null!; } @@ -55,10 +52,7 @@ namespace ContinuousTests { var nl = Environment.NewLine; Console.WriteLine("ContinuousTests will run a set of tests against a codex deployment given a codex-deployment.json file." + nl + - "The tests will run in an endless loop unless otherwise specified, using the test-specific timing values." + nl); - - Console.WriteLine("ContinuousTests assumes you are running this tool from *inside* the Kubernetes cluster. " + - "If you are not running this from a container inside the cluster, add the argument '--external'." + nl); + "The tests will run in an endless loop unless otherwise specified." + nl); } } } diff --git a/Tests/CodexContinuousTests/NodeRunner.cs b/Tests/CodexContinuousTests/NodeRunner.cs index 85f01b2..b9e78e2 100644 --- a/Tests/CodexContinuousTests/NodeRunner.cs +++ b/Tests/CodexContinuousTests/NodeRunner.cs @@ -37,6 +37,9 @@ namespace ContinuousTests public void RunNode(ICodexNode bootstrapNode, Action setup, Action operation) { var entryPoint = CreateEntryPoint(); + // We have to be sure that the transient node we start is using the same image as whatever's already in the deployed network. + // Therefore, we use the image of the bootstrap node. + CodexContainerRecipe.DockerImageOverride = bootstrapNode.Container.Recipe.Image; try { diff --git a/Tests/CodexContinuousTests/README.md b/Tests/CodexContinuousTests/README.md new file mode 100644 index 0000000..ca8eae5 --- /dev/null +++ b/Tests/CodexContinuousTests/README.md @@ -0,0 +1,20 @@ +# Codex Continuous Tests +This CLI tool runs tests in an endless loop, using a network of Codex nodes in a kubernetes cluster. Run `dotnet run -- --help` to view all CLI options. + +## Choosing tests +By default, all tests in the `CodexContinuousTests/Tests` folder will be used. If you want to limit your test run to a subset of tests, please delete or disable the other test code files. TODO: We'd like a CLI option for selecting tests. Similar to `dotnet test --filter`, maybe? + +## Where do I get a `codex-deployment.json` +See [THIS](../../Tools/CodexNetDeployer/README.MD) + +## Output +The test runner will produce a folder with all the test logs. They are sorted by timestamp and reflect the names of the tests. When a test fails, the log file for that specific test will be postfixed with `_FAILED`. + +### Pass and fail conditions +While individual tests can pass or fail for a number of times and/or over a length of time as configurable with the CLI argument, the test run entirely is not considered passed or failed until either of the following conditions are met: +1. Failed: The number of test failures has reached the specifid number, or the test runner was manually cancelled. +1. Passed: The failed condition was not reached within the time specified by the target-duration option. + +## Transient nodes +The continuous tests runner is designed to use a network of Codex nodes deployed in a kubernetes cluster. The runner will not influence or manage the lifecycle of the nodes in this deployment. However, some test cases require transient nodes. +A transient node is a node started and managed by the test runner on behalf of a specific test. The runner facilitates the tests to start and stop transient nodes, as well as bootstrap those nodes against (permanent) nodes from the deployed network. The test runner makes sure that the transient nodes use the same docker image as the permanent nodes, to avoid version conflicts. However, the image used for transient nodes can be overwritten by setting the `CODEXDOCKERIMAGE` environment variable. The use of a local Codex repository for building override images is not supported for transient nodes. diff --git a/Tests/CodexContinuousTests/run.sh b/Tests/CodexContinuousTests/run.sh index 72822a8..59536db 100644 --- a/Tests/CodexContinuousTests/run.sh +++ b/Tests/CodexContinuousTests/run.sh @@ -3,5 +3,4 @@ dotnet run \ --codex-deployment=codex-deployment.json \ --keep=1 \ --stop=10 \ - --dl-logs=1 \ --target-duration=172800 # 48 hours diff --git a/Tests/CodexTests/README.md b/Tests/CodexTests/README.md new file mode 100644 index 0000000..7c39829 --- /dev/null +++ b/Tests/CodexTests/README.md @@ -0,0 +1,26 @@ +# Codex Tests +This is an NUnit test assembly that can be used with the standard dotnet test runner. For all its CLI options, run `dotnet test --help`. + +## Example tests +Running all the tests in the assembly can take a while. In order to check basic viability of your setup as well as the Codex image you're using, consider running only the example tests using the filter option: `dotnet test --filter=Example`. + +## Output +The test runner will produce a folder named `CodexTestLogs` with all the test logs. They are sorted by timestamp and reflect the names of the test fixtures and individual tests. When a test fails, the log file for that specific test will be postfixed with `_FAILED`. The same applies to the fixture log file. The `STATUS` files contain the test results in JSON, for easy machine reading. + +## Overrides +The following environment variables allow you to override specific aspects of the behaviour of the tests. + +| Variable | Description | +|------------------|-------------------------------------------------------------------------------------------------------------| +| RUNID | A pod-label 'runid' is added to each pod created during the tests. Use this to set the value of that label. | +| TESTID | Similar to RUNID, except the label is 'testid'. | +| CODEXDOCKERIMAGE | If set, this will be used instead of the default Codex docker image. | + +## Using a local Codex repository +If you have a clone of the Codex git repository, and you want to run the tests using your local modifications, the following environment variable options are for you. Please note that any changes made in Codex's 'vendor' directory will be discarded during the build process. + +| Variable | Description | +|----------------|---------------------------------------------------------------------------------------------------------------------| +| CODEXREPOPATH | Path to the Codex repository. | +| DOCKERUSERNAME | Username of your Dockerhub account. | +| DOCKERPASSWORD | Password OR access-token for your Dockerhub account. You can omit this variable to use your system-default account. | diff --git a/Tools/CodexNetDeployer/Program.cs b/Tools/CodexNetDeployer/Program.cs index 73fef7f..1d10364 100644 --- a/Tools/CodexNetDeployer/Program.cs +++ b/Tools/CodexNetDeployer/Program.cs @@ -45,7 +45,7 @@ public class Program private static void PrintHelp() { var nl = Environment.NewLine; - Console.WriteLine("CodexNetDeployer allows you to easily deploy multiple Codex nodes in a Kubernetes cluster. " + + Console.WriteLine("CodexNetDeployer allows you to deploy multiple Codex nodes in a Kubernetes cluster. " + "The deployer will set up the required supporting services, deploy the Codex on-chain contracts, start and bootstrap the Codex instances. " + "All Kubernetes objects will be created in the namespace provided, allowing you to easily find, modify, and delete them afterwards." + nl); } From ac7fa4905d571fc013b3e1ae149cec3dac2b3c7e Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 29 Sep 2023 10:40:48 +0200 Subject: [PATCH 8/9] Adds dockertag description --- Tests/CodexTests/README.md | 11 ++++++----- Tools/CodexNetDeployer/README.MD | 11 ++++++----- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/Tests/CodexTests/README.md b/Tests/CodexTests/README.md index 7c39829..d691d97 100644 --- a/Tests/CodexTests/README.md +++ b/Tests/CodexTests/README.md @@ -19,8 +19,9 @@ The following environment variables allow you to override specific aspects of th ## Using a local Codex repository If you have a clone of the Codex git repository, and you want to run the tests using your local modifications, the following environment variable options are for you. Please note that any changes made in Codex's 'vendor' directory will be discarded during the build process. -| Variable | Description | -|----------------|---------------------------------------------------------------------------------------------------------------------| -| CODEXREPOPATH | Path to the Codex repository. | -| DOCKERUSERNAME | Username of your Dockerhub account. | -| DOCKERPASSWORD | Password OR access-token for your Dockerhub account. You can omit this variable to use your system-default account. | +| Variable | Description | +|----------------|--------------------------------------------------------------------------------------------------------------------------| +| CODEXREPOPATH | Path to the Codex repository. | +| DOCKERUSERNAME | Username of your Dockerhub account. | +| DOCKERPASSWORD | Password OR access-token for your Dockerhub account. You can omit this variable to use your system-default account. | +| DOCKERTAG | Optional. Tag used for docker image that will be built and pushed to the Dockerhub account. Random ID used when not set. | diff --git a/Tools/CodexNetDeployer/README.MD b/Tools/CodexNetDeployer/README.MD index c60dd71..d85486a 100644 --- a/Tools/CodexNetDeployer/README.MD +++ b/Tools/CodexNetDeployer/README.MD @@ -18,8 +18,9 @@ The arguments allow you to configure quite a bit, but not everything. Here are s ## Using a local Codex repository If you have a clone of the Codex git repository, and you want to deploy a network using your local modifications, the following environment variable options are for you. Please note that any changes made in Codex's 'vendor' directory will be discarded during the build process. -| Variable | Description | -|----------------|---------------------------------------------------------------------------------------------------------------------| -| CODEXREPOPATH | Path to the Codex repository. | -| DOCKERUSERNAME | Username of your Dockerhub account. | -| DOCKERPASSWORD | Password OR access-token for your Dockerhub account. You can omit this variable to use your system-default account. | +| Variable | Description | +|----------------|--------------------------------------------------------------------------------------------------------------------------| +| CODEXREPOPATH | Path to the Codex repository. | +| DOCKERUSERNAME | Username of your Dockerhub account. | +| DOCKERPASSWORD | Password OR access-token for your Dockerhub account. You can omit this variable to use your system-default account. | +| DOCKERTAG | Optional. Tag used for docker image that will be built and pushed to the Dockerhub account. Random ID used when not set. | From ce995b3df772ecf69cee2126ac94c78f6d15f224 Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 29 Sep 2023 10:43:21 +0200 Subject: [PATCH 9/9] Adds log prefixer to local builder --- ProjectPlugins/CodexPlugin/LocalCodexBuilder.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ProjectPlugins/CodexPlugin/LocalCodexBuilder.cs b/ProjectPlugins/CodexPlugin/LocalCodexBuilder.cs index bd86442..9f1b76c 100644 --- a/ProjectPlugins/CodexPlugin/LocalCodexBuilder.cs +++ b/ProjectPlugins/CodexPlugin/LocalCodexBuilder.cs @@ -12,7 +12,7 @@ namespace CodexNetDeployer public LocalCodexBuilder(ILog log, string? repoPath, string? dockerUsername) { - this.log = log; + this.log = new LogPrefixer(log, "(LocalCodexBuilder) "); this.repoPath = repoPath; this.dockerUsername = dockerUsername; }