Merge pull request #68 from codex-storage/feature/deploy-local-codex

Feature/deploy local codex

Framework/KubernetesWorkflow/K8sNameUtils.cs

@@ -13,7 +13,10 @@
                 .Replace("]", "-")
                 .Replace(",", "-");
 
-            return result.Trim('-');
+            result = result.Trim('-');
+            if (result.Length > 62) result = result.Substring(0, 62);
+
+            return result;
         }
     }
 }

ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs

@@ -16,12 +16,9 @@ namespace CodexPlugin
         public static readonly TimeSpan MaxDownloadTimePerMegabyte = TimeSpan.FromSeconds(2.0);
 
         public override string AppName => "codex";
-        public override string Image { get; }
+        public override string Image => GetDockerImage();
 
-        public CodexContainerRecipe()
+        public static string DockerImageOverride { get; set; } = string.Empty;
-        {
-            Image = GetDockerImage();
-        }
 
         protected override void Initialize(StartupConfig startupConfig)
         {
@@ -122,6 +119,7 @@ namespace CodexPlugin
         {
             var image = Environment.GetEnvironmentVariable("CODEXDOCKERIMAGE");
             if (!string.IsNullOrEmpty(image)) return image;
+            if (!string.IsNullOrEmpty(DockerImageOverride)) return DockerImageOverride;
             return DefaultDockerImage;
         }
     }

ProjectPlugins/CodexPlugin/LocalCodexBuilder.cs

@@ -0,0 +1,129 @@
+using CodexPlugin;
+using Logging;
+using System.Diagnostics;
+
+namespace CodexNetDeployer
+{
+    public class LocalCodexBuilder
+    {
+        private readonly ILog log;
+        private readonly string? repoPath;
+        private readonly string? dockerUsername;
+
+        public LocalCodexBuilder(ILog log, string? repoPath, string? dockerUsername)
+        {
+            this.log = new LogPrefixer(log, "(LocalCodexBuilder) ");
+            this.repoPath = repoPath;
+            this.dockerUsername = dockerUsername;
+        }
+
+        public LocalCodexBuilder(ILog log, string? repoPath)
+            : this(log, repoPath, Environment.GetEnvironmentVariable("DOCKERUSERNAME"))
+        {
+        }
+
+        public LocalCodexBuilder(ILog log)
+            : this(log, Environment.GetEnvironmentVariable("CODEXREPOPATH"))
+        {
+        }
+
+        public void Intialize()
+        {
+            if (!IsEnabled()) return;
+
+            if (string.IsNullOrEmpty(dockerUsername)) throw new Exception("Docker username required. (Pass to constructor or set 'DOCKERUSERNAME' environment variable.)");
+            if (string.IsNullOrEmpty(repoPath)) throw new Exception("Codex repo path required. (Pass to constructor or set 'CODEXREPOPATH' environment variable.)");
+            if (!Directory.Exists(repoPath)) throw new Exception($"Path '{repoPath}' does not exist.");
+            var files = Directory.GetFiles(repoPath);
+            if (!files.Any(f => f.ToLowerInvariant().EndsWith("codex.nim"))) throw new Exception($"Path '{repoPath}' does not appear to be the Codex repo root.");
+
+            Log($"Codex docker image will be built in path '{repoPath}'.");
+            Log("Please note this can take several minutes. If you're not trying to use a Codex image with local code changes,");
+            Log("Consider using the default test image or consider setting the 'CODEXDOCKERIMAGE' environment variable to use an already built image.");
+            CodexContainerRecipe.DockerImageOverride = $"Using docker image locally built in path '{repoPath}'.";
+        }
+
+        public void Build()
+        {
+            if (!IsEnabled()) return;
+            Log("Docker login...");
+            DockerLogin();
+
+            Log($"Logged in. Building Codex image in path '{repoPath}'...");
+
+            var customImage = GenerateImageName();
+            Docker($"build", "-t", customImage, "-f", "./codex.Dockerfile",
+                "--build-arg=\"MAKE_PARALLEL=4\"",
+                "--build-arg=\"NIMFLAGS=-d:disableMarchNative -d:codex_enable_api_debug_peers=true -d:codex_enable_api_debug_fetch=true -d:codex_enable_simulated_proof_failures\"",
+                "--build-arg=\"NAT_IP_AUTO=true\"",
+                "..");
+
+            Log($"Image '{customImage}' built successfully. Pushing...");
+
+            Docker("push", customImage);
+
+            CodexContainerRecipe.DockerImageOverride = customImage;
+            Log("Image pushed. Good to go!");
+        }
+
+        private void DockerLogin()
+        {
+            var dockerPassword = Environment.GetEnvironmentVariable("DOCKERPASSWORD");
+
+            try
+            {
+                if (string.IsNullOrEmpty(dockerUsername) || string.IsNullOrEmpty(dockerPassword))
+                {
+                    Log("Environment variable 'DOCKERPASSWORD' not provided.");
+                    Log("Trying system default...");
+                    Docker("login");
+                }
+                else
+                {
+                    Docker("login", "-u", dockerUsername, "-p", dockerPassword);
+                }
+            }
+            catch
+            {
+                Log("Docker login failed.");
+                Log("Please check the docker username and password provided by the constructor arguments and/or");
+                Log("set by 'DOCKERUSERNAME' and 'DOCKERPASSWORD' environment variables.");
+                Log("Note: You can use a docker access token as DOCKERPASSWORD.");
+                throw;
+            }
+        }
+
+        private string GenerateImageName()
+        {
+            var tag = Environment.GetEnvironmentVariable("DOCKERTAG");
+            if (string.IsNullOrEmpty(tag)) return $"{dockerUsername!}/nim-codex-autoimage:{Guid.NewGuid().ToString().ToLowerInvariant()}";
+            return $"{dockerUsername}/nim-codex-autoimage:{tag}";
+        }
+
+        private void Docker(params string[] args)
+        {
+            var dockerPath = Path.Combine(repoPath!, "docker");
+
+            var startInfo = new ProcessStartInfo()
+            {
+                FileName = "docker",
+                Arguments = string.Join(" ", args),
+                WorkingDirectory = dockerPath,
+            };
+            var process = Process.Start(startInfo);
+            if (process == null) throw new Exception("Failed to start docker process.");
+            if (!process.WaitForExit(TimeSpan.FromMinutes(10))) throw new Exception("Docker processed timed out after 10 minutes.");
+            if (process.ExitCode != 0) throw new Exception("Docker process exited with error.");
+        }
+
+        private bool IsEnabled()
+        {
+            return !string.IsNullOrEmpty(repoPath);
+        }
+
+        private void Log(string msg)
+        {
+            log.Log(msg);
+        }
+    }
+}

README.md

@@ -10,15 +10,19 @@ Nethereum: v4.14.0
 
 ## Tests/CodexTests and Tests/CodexLongTests
 These are test assemblies that use NUnit3 to perform tests against transient Codex nodes.
+Read more [HERE](/Tests/CodexTests/README.md)
 
 ## Tests/ContinousTests
 A console application that runs tests in an endless loop against a persistent deployment of Codex nodes.
+Read more [HERE](/Tests/CodexContinuousTests/README.md)
 
 ## Tools/CodexNetDeployer
 A console application that can deploy Codex nodes.
+Read more [HERE](/Tools/CodexNetDeployer/README.MD)
 
-## Test logs
+## Framework architecture
-Because tests potentially take a long time to run, logging is in place to help you investigate failures afterwards. Should a test fail, all Codex terminal output (as well as metrics if they have been enabled) will be downloaded and stored along with a detailed, step-by-step log of the test. If something's gone wrong and you're here to discover the details, head for the logs.
+The framework is designed to be extended by project-specific plugins. These plugins contribute functionality and abstractions to the framework. Users of the framework use these to perform tasks such as testing and deploying.
+![Architecture](/docs/FrameworkArchitecture.png)
 
 ## How to contribute a plugin
 If you want to add support for your project to the testing framework, follow the steps [HERE](/CONTRIBUTINGPLUGINS.MD)

Tests/CodexContinuousTests/Configuration.cs

@@ -27,9 +27,6 @@ namespace ContinuousTests
         [Uniform("target-duration", "td", "TARGETDURATION", false, "If greater than zero, runner will run for this many seconds before stopping.")]
         public int TargetDurationSeconds { get; set; } = 0;
 
-        [Uniform("dl-logs", "dl", "DLLOGS", false, "If true, runner will periodically download and save/append container logs to the log path.")]
-        public bool DownloadContainerLogs { get; set; } = false;
-
         public CodexDeployment CodexDeployment { get; set; } = null!;
     }
 
@@ -55,10 +52,7 @@ namespace ContinuousTests
         {
             var nl = Environment.NewLine;
             Console.WriteLine("ContinuousTests will run a set of tests against a codex deployment given a codex-deployment.json file." + nl +
-                "The tests will run in an endless loop unless otherwise specified, using the test-specific timing values." + nl);
+                "The tests will run in an endless loop unless otherwise specified." + nl);
-
-            Console.WriteLine("ContinuousTests assumes you are running this tool from *inside* the Kubernetes cluster. " +
-                "If you are not running this from a container inside the cluster, add the argument '--external'." + nl);
         }
     }
 }

Tests/CodexContinuousTests/NodeRunner.cs

@@ -37,6 +37,9 @@ namespace ContinuousTests
         public void RunNode(ICodexNode bootstrapNode, Action<ICodexSetup> setup, Action<ICodexNode> operation)
         {
             var entryPoint = CreateEntryPoint();
+            // We have to be sure that the transient node we start is using the same image as whatever's already in the deployed network.
+            // Therefore, we use the image of the bootstrap node.
+            CodexContainerRecipe.DockerImageOverride = bootstrapNode.Container.Recipe.Image;
 
             try
             {

Tests/CodexContinuousTests/README.md

@@ -0,0 +1,20 @@
+# Codex Continuous Tests
+This CLI tool runs tests in an endless loop, using a network of Codex nodes in a kubernetes cluster. Run `dotnet run -- --help` to view all CLI options.
+
+## Choosing tests
+By default, all tests in the `CodexContinuousTests/Tests` folder will be used. If you want to limit your test run to a subset of tests, please delete or disable the other test code files. TODO: We'd like a CLI option for selecting tests. Similar to `dotnet test --filter`, maybe?
+
+## Where do I get a `codex-deployment.json`
+See [THIS](../../Tools/CodexNetDeployer/README.MD)
+
+## Output
+The test runner will produce a folder with all the test logs. They are sorted by timestamp and reflect the names of the tests. When a test fails, the log file for that specific test will be postfixed with `_FAILED`.
+
+### Pass and fail conditions
+While individual tests can pass or fail for a number of times and/or over a length of time as configurable with the CLI argument, the test run entirely is not considered passed or failed until either of the following conditions are met:
+1. Failed: The number of test failures has reached the specifid number, or the test runner was manually cancelled.
+1. Passed: The failed condition was not reached within the time specified by the target-duration option.
+
+## Transient nodes
+The continuous tests runner is designed to use a network of Codex nodes deployed in a kubernetes cluster. The runner will not influence or manage the lifecycle of the nodes in this deployment. However, some test cases require transient nodes.
+A transient node is a node started and managed by the test runner on behalf of a specific test. The runner facilitates the tests to start and stop transient nodes, as well as bootstrap those nodes against (permanent) nodes from the deployed network. The test runner makes sure that the transient nodes use the same docker image as the permanent nodes, to avoid version conflicts. However, the image used for transient nodes can be overwritten by setting the `CODEXDOCKERIMAGE` environment variable. The use of a local Codex repository for building override images is not supported for transient nodes.

Tests/CodexContinuousTests/run.sh

@@ -3,5 +3,4 @@ dotnet run \
     --codex-deployment=codex-deployment.json \
     --keep=1 \
     --stop=10 \
-    --dl-logs=1 \
     --target-duration=172800 # 48 hours

Tests/CodexTests/AutoBootstrapDistTest.cs

@@ -5,13 +5,16 @@ namespace Tests
 {
     public class AutoBootstrapDistTest : CodexDistTest
     {
-        private readonly List<ICodexNode> onlineCodexNodes = new List<ICodexNode>();
-
         [SetUp]
         public void SetUpBootstrapNode()
         {
             BootstrapNode = AddCodex(s => s.WithName("BOOTSTRAP"));
-            onlineCodexNodes.Add(BootstrapNode);
+        }
+
+        [TearDown]
+        public void TearDownBootstrapNode()
+        {
+            BootstrapNode = null;
         }
 
         protected override void OnCodexSetup(ICodexSetup setup)

Tests/CodexTests/CodexDistTest.cs

@@ -1,9 +1,12 @@
 using CodexContractsPlugin;
+using CodexNetDeployer;
 using CodexPlugin;
 using Core;
 using DistTestCore;
 using DistTestCore.Helpers;
+using DistTestCore.Logs;
 using GethPlugin;
+using NUnit.Framework;
 using NUnit.Framework.Constraints;
 
 namespace Tests
@@ -20,6 +23,19 @@ namespace Tests
             ProjectPlugin.Load<MetricsPlugin.MetricsPlugin>();
         }
 
+        [TearDown]
+        public void TearDownCodexFixture()
+        {
+            onlineCodexNodes.Clear();
+        }
+
+        protected override void Initialize(FixtureLog fixtureLog)
+        {
+            var localBuilder = new LocalCodexBuilder(fixtureLog);
+            localBuilder.Intialize();
+            localBuilder.Build();
+        }
+
         public ICodexNode AddCodex()
         {
             return AddCodex(s => { });

Tests/CodexTests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs

@@ -1,4 +1,6 @@
-using NUnit.Framework;
+using CodexContractsPlugin;
+using GethPlugin;
+using NUnit.Framework;
 using Utils;
 
 namespace Tests.DownloadConnectivityTests
@@ -17,7 +19,9 @@ namespace Tests.DownloadConnectivityTests
         [Test]
         public void MarketplaceDoesNotInterfereWithPeerDownload()
         {
-            //AddCodex(2, s => s.EnableMetrics().EnableMarketplace(1000.TestTokens()));
+            var geth = Ci.StartGethNode(s => s.IsMiner());
+            var contracts = Ci.StartCodexContracts(geth);
+            AddCodex(2, s => s.EnableMarketplace(geth, contracts, 10.Eth(), 1000.TestTokens()));
 
             AssertAllNodesConnected();
         }

Tests/CodexTests/PeerDiscoveryTests/PeerDiscoveryTests.cs

@@ -1,4 +1,6 @@
-using NUnit.Framework;
+using CodexContractsPlugin;
+using GethPlugin;
+using NUnit.Framework;
 
 namespace Tests.PeerDiscoveryTests
 {
@@ -26,7 +28,9 @@ namespace Tests.PeerDiscoveryTests
         [Test]
         public void MarketplaceDoesNotInterfereWithPeerDiscovery()
         {
-            //AddCodex(2, s => s.EnableMarketplace(1000.TestTokens()));
+            var geth = Ci.StartGethNode(s => s.IsMiner());
+            var contracts = Ci.StartCodexContracts(geth);
+            AddCodex(2, s => s.EnableMarketplace(geth, contracts, 10.Eth(), 1000.TestTokens()));
 
             AssertAllNodesConnected();
         }

Tests/CodexTests/README.md

@@ -0,0 +1,27 @@
+# Codex Tests
+This is an NUnit test assembly that can be used with the standard dotnet test runner. For all its CLI options, run `dotnet test --help`.
+
+## Example tests
+Running all the tests in the assembly can take a while. In order to check basic viability of your setup as well as the Codex image you're using, consider running only the example tests using the filter option: `dotnet test --filter=Example`.
+
+## Output
+The test runner will produce a folder named `CodexTestLogs` with all the test logs. They are sorted by timestamp and reflect the names of the test fixtures and individual tests. When a test fails, the log file for that specific test will be postfixed with `_FAILED`. The same applies to the fixture log file. The `STATUS` files contain the test results in JSON, for easy machine reading.
+
+## Overrides
+The following environment variables allow you to override specific aspects of the behaviour of the tests.
+
+| Variable         | Description                                                                                                 |
+|------------------|-------------------------------------------------------------------------------------------------------------|
+| RUNID            | A pod-label 'runid' is added to each pod created during the tests. Use this to set the value of that label. |
+| TESTID           | Similar to RUNID, except the label is 'testid'.                                                             |
+| CODEXDOCKERIMAGE | If set, this will be used instead of the default Codex docker image.                                        |
+
+## Using a local Codex repository
+If you have a clone of the Codex git repository, and you want to run the tests using your local modifications, the following environment variable options are for you. Please note that any changes made in Codex's 'vendor' directory will be discarded during the build process.
+
+| Variable       | Description                                                                                                              |
+|----------------|--------------------------------------------------------------------------------------------------------------------------|
+| CODEXREPOPATH  | Path to the Codex repository.                                                                                            |
+| DOCKERUSERNAME | Username of your Dockerhub account.                                                                                      |
+| DOCKERPASSWORD | Password OR access-token for your Dockerhub account. You can omit this variable to use your system-default account.      |
+| DOCKERTAG      | Optional. Tag used for docker image that will be built and pushed to the Dockerhub account. Random ID used when not set. |

Tests/DistTestCore/DistTest.cs

@@ -32,6 +32,8 @@ namespace DistTestCore
             statusLog = new StatusLog(logConfig, startTime);
 
             globalEntryPoint = new EntryPoint(fixtureLog, configuration.GetK8sConfiguration(new DefaultTimeSet(), TestNamespacePrefix), configuration.GetFileManagerFolder());
+
+            Initialize(fixtureLog);
         }
 
         [OneTimeSetUp]
@@ -140,6 +142,10 @@ namespace DistTestCore
             Stopwatch.Measure(Get().Log, name, action);
         }
 
+        protected virtual void Initialize(FixtureLog fixtureLog)
+        {
+        }
+
         protected TestLifecycle Get()
         {
             lock (lifecycleLock)

Tools/CodexNetDeployer/Configuration.cs

@@ -14,6 +14,11 @@ namespace CodexNetDeployer
         [Uniform("kube-namespace", "kn", "KUBENAMESPACE", true, "Kubernetes namespace to be used for deployment.")]
         public string KubeNamespace { get; set; } = string.Empty;
 
+        [Uniform("codex-local-repo", "cr", "CODEXLOCALREPOPATH", false, "If set, instead of using the default Codex docker image, the local repository will be used to build an image. " +
+            "This requires the 'DOCKERUSERNAME' and 'DOCKERPASSWORD' environment variables to be set. (You can omit the password to use your system default, or use a docker access token as DOCKERPASSWORD.) You can set " +
+            "'DOCKERTAG' to define the image tag. If not set, one will be generated.")]
+        public string CodexLocalRepoPath { get; set; } = string.Empty;
+
         [Uniform("nodes", "n", "NODES", true, "Number of Codex nodes to be created.")]
         public int? NumberOfCodexNodes { get; set; }
 
@@ -59,13 +64,16 @@ namespace CodexNetDeployer
 
         [Uniform("check-connect", "cc", "CHECKCONNECT", false, "If true, deployer check ensure peer-connectivity between all deployed nodes after deployment. Default is false.")]
         public bool CheckPeerConnection { get; set; } = false;
-       
+
         public List<string> Validate()
         {
             var errors = new List<string>();
 
+            StringIsSet(nameof(KubeNamespace), KubeNamespace, errors);
+            StringIsSet(nameof(KubeConfigFile), KubeConfigFile, errors);
+            StringIsSet(nameof(TestsTypePodLabel), TestsTypePodLabel, errors);
+
             ForEachProperty(
-                onString: (n, v) => StringIsSet(n, v, errors),
                 onInt: (n, v) => IntIsOverZero(n, v, errors));
 
             if (NumberOfValidators > NumberOfCodexNodes)
@@ -80,12 +88,11 @@ namespace CodexNetDeployer
             return errors;
         }
 
-        private void ForEachProperty(Action<string, string> onString, Action<string, int?> onInt)
+        private void ForEachProperty(Action<string, int?> onInt)
         {
             var properties = GetType().GetProperties();
             foreach (var p in properties)
             {
-                if (p.PropertyType == typeof(string)) onString(p.Name, (string)p.GetValue(this)!);
                 if (p.PropertyType == typeof(int?)) onInt(p.Name, (int?)p.GetValue(this)!);
                 if (p.PropertyType == typeof(int)) onInt(p.Name, (int)p.GetValue(this)!);
             }

Tools/CodexNetDeployer/Deployer.cs

@@ -13,11 +13,13 @@ namespace CodexNetDeployer
         private readonly Configuration config;
         private readonly PeerConnectivityChecker peerConnectivityChecker;
         private readonly EntryPoint entryPoint;
+        private readonly LocalCodexBuilder localCodexBuilder;
 
         public Deployer(Configuration config)
         {
             this.config = config;
             peerConnectivityChecker = new PeerConnectivityChecker();
+            localCodexBuilder = new LocalCodexBuilder(new ConsoleLog(), config.CodexLocalRepoPath);
 
             ProjectPlugin.Load<CodexPlugin.CodexPlugin>();
             ProjectPlugin.Load<CodexContractsPlugin.CodexContractsPlugin>();
@@ -30,6 +32,8 @@ namespace CodexNetDeployer
         {
             var ep = CreateEntryPoint(new ConsoleLog());
 
+            localCodexBuilder.Intialize();
+
             Log("Using plugins:" + Environment.NewLine);
             var metadata = ep.GetPluginMetadata();
             var longestKey = metadata.Keys.Max(k => k.Length);
@@ -44,6 +48,8 @@ namespace CodexNetDeployer
 
         public CodexDeployment Deploy()
         {
+            localCodexBuilder.Build();
+
             Log("Initializing...");
             var ci = entryPoint.CreateInterface();
 

Tools/CodexNetDeployer/Program.cs

@@ -45,7 +45,7 @@ public class Program
     private static void PrintHelp()
     {
         var nl = Environment.NewLine;
-        Console.WriteLine("CodexNetDeployer allows you to easily deploy multiple Codex nodes in a Kubernetes cluster. " +
+        Console.WriteLine("CodexNetDeployer allows you to deploy multiple Codex nodes in a Kubernetes cluster. " +
             "The deployer will set up the required supporting services, deploy the Codex on-chain contracts, start and bootstrap the Codex instances. " +
             "All Kubernetes objects will be created in the namespace provided, allowing you to easily find, modify, and delete them afterwards." + nl);
     }

Tools/CodexNetDeployer/README.MD

@@ -0,0 +1,26 @@
+# CodexNetDeployer
+
+This CLI tool allows you to create a network of Codex nodes in a kubernetes cluster. There are many arguments that allow you to configure every aspect of the deployment.  Each argument can also be controlled using an environment variable. I won't copy-paste the list here because then it'll have to be kept up-to-date. Just run:
+`dotnet run -- --help`
+
+## Output
+After the deployment has successfully finished, a `codex-deployment.json` file will be created. This file contains all the information necessary to interface with the deployed network. It can be used by other tools, for example the CodexContinuousTests runner CLI tool.
+
+## Overrides
+The arguments allow you to configure quite a bit, but not everything. Here are some environment variables the CodexNetDeployer will respond to. None of these are required.
+
+| Variable         | Description                                                                                                  |
+|------------------|--------------------------------------------------------------------------------------------------------------|
+| RUNID            | A pod-label 'runid' is added to each pod created during deployment. Use this to set the value of that label. |
+| TESTID           | Similar to RUNID, except the label is 'testid'.                                                              |
+| CODEXDOCKERIMAGE | If set, this will be used instead of the default Codex docker image.                                         |
+
+## Using a local Codex repository
+If you have a clone of the Codex git repository, and you want to deploy a network using your local modifications, the following environment variable options are for you. Please note that any changes made in Codex's 'vendor' directory will be discarded during the build process.
+
+| Variable       | Description                                                                                                              |
+|----------------|--------------------------------------------------------------------------------------------------------------------------|
+| CODEXREPOPATH  | Path to the Codex repository.                                                                                            |
+| DOCKERUSERNAME | Username of your Dockerhub account.                                                                                      |
+| DOCKERPASSWORD | Password OR access-token for your Dockerhub account. You can omit this variable to use your system-default account.      |
+| DOCKERTAG      | Optional. Tag used for docker image that will be built and pushed to the Dockerhub account. Random ID used when not set. |