From 40046d9bcb46b063ea3eae2fda2d5f480adb6b71 Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 22 Jun 2023 08:29:33 +0200 Subject: [PATCH 1/9] New console app --- CodexNetDeployer/CodexNetDeployer.csproj | 10 ++++++++++ CodexNetDeployer/Program.cs | 7 +++++++ cs-codex-dist-testing.sln | 8 +++++++- 3 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 CodexNetDeployer/CodexNetDeployer.csproj create mode 100644 CodexNetDeployer/Program.cs diff --git a/CodexNetDeployer/CodexNetDeployer.csproj b/CodexNetDeployer/CodexNetDeployer.csproj new file mode 100644 index 00000000..f02677bf --- /dev/null +++ b/CodexNetDeployer/CodexNetDeployer.csproj @@ -0,0 +1,10 @@ + + + + Exe + net7.0 + enable + enable + + + diff --git a/CodexNetDeployer/Program.cs b/CodexNetDeployer/Program.cs new file mode 100644 index 00000000..a08476fe --- /dev/null +++ b/CodexNetDeployer/Program.cs @@ -0,0 +1,7 @@ +public class Program +{ + public static void Main(string[] args) + { + Console.WriteLine("Hello, World!"); + } +} diff --git a/cs-codex-dist-testing.sln b/cs-codex-dist-testing.sln index 7946ff45..cec7e4cd 100644 --- a/cs-codex-dist-testing.sln +++ b/cs-codex-dist-testing.sln @@ -17,7 +17,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Logging", "Logging\Logging. EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "NethereumWorkflow", "Nethereum\NethereumWorkflow.csproj", "{D6C3555E-D52D-4993-A87B-71AB650398FD}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ContinuousTests", "ContinuousTests\ContinuousTests.csproj", "{025B7074-0A09-4FCC-9BB9-03AE2A961EA1}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ContinuousTests", "ContinuousTests\ContinuousTests.csproj", "{025B7074-0A09-4FCC-9BB9-03AE2A961EA1}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CodexNetDeployer", "CodexNetDeployer\CodexNetDeployer.csproj", "{871CAF12-14BE-4509-BC6E-20FDF0B1083A}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -57,6 +59,10 @@ Global {025B7074-0A09-4FCC-9BB9-03AE2A961EA1}.Debug|Any CPU.Build.0 = Debug|Any CPU {025B7074-0A09-4FCC-9BB9-03AE2A961EA1}.Release|Any CPU.ActiveCfg = Release|Any CPU {025B7074-0A09-4FCC-9BB9-03AE2A961EA1}.Release|Any CPU.Build.0 = Release|Any CPU + {871CAF12-14BE-4509-BC6E-20FDF0B1083A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {871CAF12-14BE-4509-BC6E-20FDF0B1083A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {871CAF12-14BE-4509-BC6E-20FDF0B1083A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {871CAF12-14BE-4509-BC6E-20FDF0B1083A}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE From e5a7f04c4e175c252a466e2a5180102fe15db170 Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 22 Jun 2023 09:51:25 +0200 Subject: [PATCH 2/9] Arg and env-var parsing --- CodexNetDeployer/ArgOrVar.cs | 89 ++++++++++++++++++++++++ CodexNetDeployer/CodexNetDeployer.csproj | 4 ++ CodexNetDeployer/Configuration.cs | 84 ++++++++++++++++++++++ CodexNetDeployer/Program.cs | 43 +++++++++++- 4 files changed, 218 insertions(+), 2 deletions(-) create mode 100644 CodexNetDeployer/ArgOrVar.cs create mode 100644 CodexNetDeployer/Configuration.cs diff --git a/CodexNetDeployer/ArgOrVar.cs b/CodexNetDeployer/ArgOrVar.cs new file mode 100644 index 00000000..df701719 --- /dev/null +++ b/CodexNetDeployer/ArgOrVar.cs @@ -0,0 +1,89 @@ +namespace CodexNetDeployer +{ + public class ArgOrVar + { + public static readonly ArgVar CodexImage = new ArgVar("codex-image", "CODEXIMAGE", "Docker image of Codex."); + public static readonly ArgVar GethImage = new ArgVar("geth-image", "GETHIMAGE", "Docker image of Geth."); + public static readonly ArgVar ContractsImage = new ArgVar("contracts-image", "CONTRACTSIMAGE", "Docker image of Codex Contracts deployer."); + public static readonly ArgVar KubeConfigFile = new ArgVar("kube-config", "KUBECONFIG", "Path to Kubeconfig file."); + public static readonly ArgVar KubeNamespace = new ArgVar("kube-namespace", "KUBENAMESPACE", "Kubernetes namespace to be used for deployment."); + public static readonly ArgVar NumberOfCodexNodes = new ArgVar("nodes", "NODES", "Number of Codex nodes to be created."); + public static readonly ArgVar StorageQuota = new ArgVar("storage-quota", "STORAGEQUOTA", "Storage quota in megabytes used by each Codex node."); + + private readonly string[] args; + + public ArgOrVar(string[] args) + { + this.args = args; + } + + public string Get(ArgVar key, string defaultValue = "") + { + var argKey = $"--{key.Arg}="; + var arg = args.FirstOrDefault(a => a.StartsWith(argKey)); + if (arg != null) + { + return arg.Substring(argKey.Length); + } + + var env = Environment.GetEnvironmentVariable(key.Var); + if (env != null) + { + return env; + } + + return defaultValue; + } + + public int? GetInt(ArgVar key) + { + var str = Get(key); + if (string.IsNullOrEmpty(str)) return null; + if (int.TryParse(str, out int result)) + { + return result; + } + return null; + } + + public void PrintHelp() + { + var nl = Environment.NewLine; + Console.WriteLine("CodexNetDeployer allows you to easily deploy multiple Codex nodes in a Kubernetes cluster. " + + "The deployer will set up the required supporting services, deploy the Codex on-chain contracts, start and bootstrap the Codex instances. " + + "All Kubernetes objects will be created in the namespace provided, allowing you to easily find, modify, and delete them afterwards." + nl); + + + Console.Write("\t[ CLI argument ] or [ Environment variable ]"); + Console.CursorLeft = 70; + Console.Write("(Description)" + nl); + var fields = GetType().GetFields();// System.Reflection.BindingFlags.Public & System.Reflection.BindingFlags.Static); + foreach (var field in fields) + { + var value = (ArgVar)field.GetValue(null)!; + value.PrintHelp(); + } + } + } + + public class ArgVar + { + public ArgVar(string arg, string var, string description) + { + Arg = arg; + Var = var; + Description = description; + } + + public string Arg { get; } + public string Var { get; } + public string Description { get; } + + public void PrintHelp() + { + Console.Write($"\t[ --{Arg}=... ] or [ {Var}=... ]"); + Console.CursorLeft = 70; + Console.Write(Description + Environment.NewLine); + } + } +} diff --git a/CodexNetDeployer/CodexNetDeployer.csproj b/CodexNetDeployer/CodexNetDeployer.csproj index f02677bf..80f6f4c2 100644 --- a/CodexNetDeployer/CodexNetDeployer.csproj +++ b/CodexNetDeployer/CodexNetDeployer.csproj @@ -7,4 +7,8 @@ enable + + + + diff --git a/CodexNetDeployer/Configuration.cs b/CodexNetDeployer/Configuration.cs new file mode 100644 index 00000000..3abb7a4b --- /dev/null +++ b/CodexNetDeployer/Configuration.cs @@ -0,0 +1,84 @@ +namespace CodexNetDeployer +{ + public class Configuration + { + public Configuration( + string codexImage, + string gethImage, + string contractsImage, + string kubeConfigFile, + string kubeNamespace, + int? numberOfCodexNodes, + int? storageQuota) + { + CodexImage = codexImage; + GethImage = gethImage; + ContractsImage = contractsImage; + KubeConfigFile = kubeConfigFile; + KubeNamespace = kubeNamespace; + NumberOfCodexNodes = numberOfCodexNodes; + StorageQuota = storageQuota; + } + + public string CodexImage { get; } + public string GethImage { get; } + public string ContractsImage { get; } + public string KubeConfigFile { get; } + public string KubeNamespace { get; } + public int? NumberOfCodexNodes { get; } + public int? StorageQuota { get; } + + public void PrintConfig() + { + ForEachProperty(onString: Print, onInt: Print); + } + + public List Validate() + { + var errors = new List(); + + ForEachProperty( + onString: (n, v) => StringIsSet(n, v, errors), + onInt: (n, v) => IntIsOverZero(n, v, errors)); + + return errors; + } + + private void ForEachProperty(Action onString, Action onInt) + { + var properties = GetType().GetProperties(); + foreach (var p in properties) + { + if (p.PropertyType == typeof(string)) onString(p.Name, (string)p.GetValue(this)!); + if (p.PropertyType == typeof(int?)) onInt(p.Name, (int?)p.GetValue(this)!); + } + } + + private static void IntIsOverZero(string variable, int? value, List errors) + { + if (value == null || value.Value < 1) + { + errors.Add($"{variable} is must be set and must be greater than 0."); + } + } + + private static void StringIsSet(string variable, string value, List errors) + { + if (string.IsNullOrWhiteSpace(value)) + { + errors.Add($"{variable} is must be set."); + } + } + + private static void Print(string variable, string value) + { + Console.WriteLine($"\t{variable}: '{value}'"); + } + + private static void Print(string variable, int? value) + { + if (value != null) Print(variable, value.ToString()!); + else Print(variable, ""); + } + } +} diff --git a/CodexNetDeployer/Program.cs b/CodexNetDeployer/Program.cs index a08476fe..64a445f8 100644 --- a/CodexNetDeployer/Program.cs +++ b/CodexNetDeployer/Program.cs @@ -1,7 +1,46 @@ -public class Program +using CodexNetDeployer; +using DistTestCore.Codex; +using DistTestCore.Marketplace; + +public class Program { public static void Main(string[] args) { - Console.WriteLine("Hello, World!"); + var nl = Environment.NewLine; + Console.WriteLine("CodexNetDeployer" + nl + nl); + + var argOrVar = new ArgOrVar(args); + + if (args.Any(a => a == "-h" || a == "--help" || a == "-?")) + { + argOrVar.PrintHelp(); + return; + } + + var config = new Configuration( + codexImage: argOrVar.Get(ArgOrVar.CodexImage, CodexContainerRecipe.DockerImage), + gethImage: argOrVar.Get(ArgOrVar.GethImage, GethContainerRecipe.DockerImage), + contractsImage: argOrVar.Get(ArgOrVar.ContractsImage, CodexContractsContainerRecipe.DockerImage), + kubeConfigFile: argOrVar.Get(ArgOrVar.KubeConfigFile), + kubeNamespace: argOrVar.Get(ArgOrVar.KubeNamespace), + numberOfCodexNodes: argOrVar.GetInt(ArgOrVar.NumberOfCodexNodes), + storageQuota: argOrVar.GetInt(ArgOrVar.StorageQuota) + ); + + Console.WriteLine("Using:"); + config.PrintConfig(); + Console.WriteLine(nl); + + var errors = config.Validate(); + if (errors.Any()) + { + Console.WriteLine($"Configuration errors: ({errors.Count})"); + foreach ( var error in errors ) Console.WriteLine("\t" + error); + Console.WriteLine(nl); + argOrVar.PrintHelp(); + return; + } + + } } From ee5a46694040bb546b8bca9ac89271bf2f797fcf Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 22 Jun 2023 10:17:12 +0200 Subject: [PATCH 3/9] Wiring up the starter class --- CodexNetDeployer/ArgOrVar.cs | 5 +++- CodexNetDeployer/Configuration.cs | 12 +++++++-- CodexNetDeployer/Deployer.cs | 41 +++++++++++++++++++++++++++++++ CodexNetDeployer/NullLog.cs | 41 +++++++++++++++++++++++++++++++ CodexNetDeployer/Program.cs | 15 ++++++++++- DistTestCore/Configuration.cs | 19 ++++++++------ Logging/BaseLog.cs | 12 ++++----- Utils/ParseEnum.cs | 10 ++++++++ 8 files changed, 138 insertions(+), 17 deletions(-) create mode 100644 CodexNetDeployer/Deployer.cs create mode 100644 CodexNetDeployer/NullLog.cs create mode 100644 Utils/ParseEnum.cs diff --git a/CodexNetDeployer/ArgOrVar.cs b/CodexNetDeployer/ArgOrVar.cs index df701719..90c34ae1 100644 --- a/CodexNetDeployer/ArgOrVar.cs +++ b/CodexNetDeployer/ArgOrVar.cs @@ -9,6 +9,7 @@ public static readonly ArgVar KubeNamespace = new ArgVar("kube-namespace", "KUBENAMESPACE", "Kubernetes namespace to be used for deployment."); public static readonly ArgVar NumberOfCodexNodes = new ArgVar("nodes", "NODES", "Number of Codex nodes to be created."); public static readonly ArgVar StorageQuota = new ArgVar("storage-quota", "STORAGEQUOTA", "Storage quota in megabytes used by each Codex node."); + public static readonly ArgVar LogLevel = new ArgVar("log-level", "LOGLEVEL", "Log level used by each Codex node. [Trace, Debug*, Info, Warn, Error]"); private readonly string[] args; @@ -53,11 +54,13 @@ "The deployer will set up the required supporting services, deploy the Codex on-chain contracts, start and bootstrap the Codex instances. " + "All Kubernetes objects will be created in the namespace provided, allowing you to easily find, modify, and delete them afterwards." + nl); + Console.WriteLine("CodexNetDeployer assumes you are running this tool from *outside* the Kubernetes cluster you want to deploy to. " + + "If you are running this from a container inside the cluster, add the argument '--internal'." + nl); Console.Write("\t[ CLI argument ] or [ Environment variable ]"); Console.CursorLeft = 70; Console.Write("(Description)" + nl); - var fields = GetType().GetFields();// System.Reflection.BindingFlags.Public & System.Reflection.BindingFlags.Static); + var fields = GetType().GetFields(); foreach (var field in fields) { var value = (ArgVar)field.GetValue(null)!; diff --git a/CodexNetDeployer/Configuration.cs b/CodexNetDeployer/Configuration.cs index 3abb7a4b..0a3504c7 100644 --- a/CodexNetDeployer/Configuration.cs +++ b/CodexNetDeployer/Configuration.cs @@ -1,4 +1,6 @@ -namespace CodexNetDeployer +using DistTestCore; + +namespace CodexNetDeployer { public class Configuration { @@ -9,7 +11,9 @@ string kubeConfigFile, string kubeNamespace, int? numberOfCodexNodes, - int? storageQuota) + int? storageQuota, + string codexLogLevel, + TestRunnerLocation runnerLocation) { CodexImage = codexImage; GethImage = gethImage; @@ -18,6 +22,8 @@ KubeNamespace = kubeNamespace; NumberOfCodexNodes = numberOfCodexNodes; StorageQuota = storageQuota; + CodexLogLevel = codexLogLevel; + RunnerLocation = runnerLocation; } public string CodexImage { get; } @@ -27,6 +33,8 @@ public string KubeNamespace { get; } public int? NumberOfCodexNodes { get; } public int? StorageQuota { get; } + public string CodexLogLevel { get; } + public TestRunnerLocation RunnerLocation { get; } public void PrintConfig() { diff --git a/CodexNetDeployer/Deployer.cs b/CodexNetDeployer/Deployer.cs new file mode 100644 index 00000000..0683e5c4 --- /dev/null +++ b/CodexNetDeployer/Deployer.cs @@ -0,0 +1,41 @@ +using DistTestCore; +using DistTestCore.Codex; +using Utils; + +namespace CodexNetDeployer +{ + public class Deployer + { + private readonly Configuration config; + + public Deployer(Configuration config) + { + this.config = config; + } + + public void Deploy() + { + var log = new NullLog(); + var lifecycleConfig = new DistTestCore.Configuration + ( + kubeConfigFile: config.KubeConfigFile, + logPath: "null", + logDebug: false, + dataFilesPath: "notUsed", + codexLogLevel: ParseEnum.Parse(config.CodexLogLevel), + runnerLocation: config.RunnerLocation + ); + + var timeset = new DefaultTimeSet(); + var kubeConfig = new KubernetesWorkflow.Configuration( + k8sNamespacePrefix: config.KubeNamespace, + kubeConfigFile: config.KubeConfigFile, + operationTimeout: timeset.K8sOperationTimeout(), + retryDelay: timeset.WaitForK8sServiceDelay()); + + var lifecycle = new TestLifecycle(log, lifecycleConfig, timeset); + var workflowCreator = new KubernetesWorkflow.WorkflowCreator(log, kubeConfig); + var starter = new CodexStarter(lifecycle, workflowCreator); + } + } +} diff --git a/CodexNetDeployer/NullLog.cs b/CodexNetDeployer/NullLog.cs new file mode 100644 index 00000000..c64190b9 --- /dev/null +++ b/CodexNetDeployer/NullLog.cs @@ -0,0 +1,41 @@ +using Logging; + +namespace CodexNetDeployer +{ + public class NullLog : TestLog + { + public NullLog() : base("NULL", false, "NULL") + { + } + + protected override LogFile CreateLogFile() + { + return null!; + } + + public override void Log(string message) + { + } + + public override void Debug(string message = "", int skipFrames = 0) + { + } + + public override void Error(string message) + { + Console.WriteLine("Error: " + message); + } + + public override void MarkAsFailed() + { + } + + public override void AddStringReplace(string from, string to) + { + } + + public override void Delete() + { + } + } +} diff --git a/CodexNetDeployer/Program.cs b/CodexNetDeployer/Program.cs index 64a445f8..582d2a1a 100644 --- a/CodexNetDeployer/Program.cs +++ b/CodexNetDeployer/Program.cs @@ -1,6 +1,8 @@ using CodexNetDeployer; +using DistTestCore; using DistTestCore.Codex; using DistTestCore.Marketplace; +using Configuration = CodexNetDeployer.Configuration; public class Program { @@ -17,6 +19,12 @@ public class Program return; } + var location = TestRunnerLocation.ExternalToCluster; + if (args.Any(a => a == "--internal")) + { + location = TestRunnerLocation.InternalToCluster; + } + var config = new Configuration( codexImage: argOrVar.Get(ArgOrVar.CodexImage, CodexContainerRecipe.DockerImage), gethImage: argOrVar.Get(ArgOrVar.GethImage, GethContainerRecipe.DockerImage), @@ -24,7 +32,9 @@ public class Program kubeConfigFile: argOrVar.Get(ArgOrVar.KubeConfigFile), kubeNamespace: argOrVar.Get(ArgOrVar.KubeNamespace), numberOfCodexNodes: argOrVar.GetInt(ArgOrVar.NumberOfCodexNodes), - storageQuota: argOrVar.GetInt(ArgOrVar.StorageQuota) + storageQuota: argOrVar.GetInt(ArgOrVar.StorageQuota), + codexLogLevel: argOrVar.Get(ArgOrVar.LogLevel), + runnerLocation: location ); Console.WriteLine("Using:"); @@ -41,6 +51,9 @@ public class Program return; } + var deployer = new Deployer(config); + deployer.Deploy(); + Console.WriteLine("Done!"); } } diff --git a/DistTestCore/Configuration.cs b/DistTestCore/Configuration.cs index af76127e..03514cd8 100644 --- a/DistTestCore/Configuration.cs +++ b/DistTestCore/Configuration.cs @@ -19,8 +19,18 @@ namespace DistTestCore logPath = GetEnvVarOrDefault("LOGPATH", "CodexTestLogs"); logDebug = GetEnvVarOrDefault("LOGDEBUG", "false").ToLowerInvariant() == "true"; dataFilesPath = GetEnvVarOrDefault("DATAFILEPATH", "TestDataFiles"); - codexLogLevel = ParseEnum(GetEnvVarOrDefault("LOGLEVEL", nameof(CodexLogLevel.Trace))); - runnerLocation = ParseEnum(GetEnvVarOrDefault("RUNNERLOCATION", nameof(TestRunnerLocation.ExternalToCluster))); + codexLogLevel = ParseEnum.Parse(GetEnvVarOrDefault("LOGLEVEL", nameof(CodexLogLevel.Trace))); + runnerLocation = ParseEnum.Parse(GetEnvVarOrDefault("RUNNERLOCATION", nameof(TestRunnerLocation.ExternalToCluster))); + } + + public Configuration(string? kubeConfigFile, string logPath, bool logDebug, string dataFilesPath, CodexLogLevel codexLogLevel, TestRunnerLocation runnerLocation) + { + this.kubeConfigFile = kubeConfigFile; + this.logPath = logPath; + this.logDebug = logDebug; + this.dataFilesPath = dataFilesPath; + this.codexLogLevel = codexLogLevel; + this.runnerLocation = runnerLocation; } public KubernetesWorkflow.Configuration GetK8sConfiguration(ITimeSet timeSet) @@ -75,11 +85,6 @@ namespace DistTestCore if (v == null) return defaultValue; return v; } - - private static T ParseEnum(string value) - { - return (T)Enum.Parse(typeof(T), value, true); - } } public enum TestRunnerLocation diff --git a/Logging/BaseLog.cs b/Logging/BaseLog.cs index 2c69f9c8..c3122569 100644 --- a/Logging/BaseLog.cs +++ b/Logging/BaseLog.cs @@ -25,12 +25,12 @@ namespace Logging } } - public void Log(string message) + public virtual void Log(string message) { LogFile.Write(ApplyReplacements(message)); } - public void Debug(string message = "", int skipFrames = 0) + public virtual void Debug(string message = "", int skipFrames = 0) { if (debug) { @@ -40,25 +40,25 @@ namespace Logging } } - public void Error(string message) + public virtual void Error(string message) { Log($"[ERROR] {message}"); } - public void MarkAsFailed() + public virtual void MarkAsFailed() { if (hasFailed) return; hasFailed = true; LogFile.ConcatToFilename("_FAILED"); } - public void AddStringReplace(string from, string to) + public virtual void AddStringReplace(string from, string to) { if (string.IsNullOrWhiteSpace(from)) return; replacements.Add(new BaseLogStringReplacement(from, to)); } - public void Delete() + public virtual void Delete() { File.Delete(LogFile.FullFilename); } diff --git a/Utils/ParseEnum.cs b/Utils/ParseEnum.cs new file mode 100644 index 00000000..3b3d0cb4 --- /dev/null +++ b/Utils/ParseEnum.cs @@ -0,0 +1,10 @@ +namespace Utils +{ + public static class ParseEnum + { + public static T Parse(string value) + { + return (T)Enum.Parse(typeof(T), value, true); + } + } +} From 7fdca9ffdb17e350d5362913131a5c041fb7afdf Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 22 Jun 2023 10:33:21 +0200 Subject: [PATCH 4/9] all wired up --- CodexNetDeployer/Configuration.cs | 5 +++-- CodexNetDeployer/Deployer.cs | 30 ++++++++++++++++++++++++++---- CodexNetDeployer/Program.cs | 3 ++- 3 files changed, 31 insertions(+), 7 deletions(-) diff --git a/CodexNetDeployer/Configuration.cs b/CodexNetDeployer/Configuration.cs index 0a3504c7..3e5346ca 100644 --- a/CodexNetDeployer/Configuration.cs +++ b/CodexNetDeployer/Configuration.cs @@ -1,4 +1,5 @@ using DistTestCore; +using DistTestCore.Codex; namespace CodexNetDeployer { @@ -12,7 +13,7 @@ namespace CodexNetDeployer string kubeNamespace, int? numberOfCodexNodes, int? storageQuota, - string codexLogLevel, + CodexLogLevel codexLogLevel, TestRunnerLocation runnerLocation) { CodexImage = codexImage; @@ -33,7 +34,7 @@ namespace CodexNetDeployer public string KubeNamespace { get; } public int? NumberOfCodexNodes { get; } public int? StorageQuota { get; } - public string CodexLogLevel { get; } + public CodexLogLevel CodexLogLevel { get; } public TestRunnerLocation RunnerLocation { get; } public void PrintConfig() diff --git a/CodexNetDeployer/Deployer.cs b/CodexNetDeployer/Deployer.cs index 0683e5c4..03f428ad 100644 --- a/CodexNetDeployer/Deployer.cs +++ b/CodexNetDeployer/Deployer.cs @@ -1,6 +1,4 @@ using DistTestCore; -using DistTestCore.Codex; -using Utils; namespace CodexNetDeployer { @@ -14,6 +12,25 @@ namespace CodexNetDeployer } public void Deploy() + { + Log("Initializing..."); + var starter = CreateStarter(); + + Log("Preparing configuration..."); + var setup = new CodexSetup(config.NumberOfCodexNodes!.Value, config.CodexLogLevel); + + Log("Creating resources..."); + var group = (CodexNodeGroup) starter.BringOnline(setup); + + var containers = group.Containers; + foreach (var container in containers.Containers) + { + var pod = container.Pod.PodInfo; + Log($"Container '{container.Name}' online. Pod: '{pod.Name}@{pod.Ip}' on '{pod.K8SNodeName}'."); + } + } + + private CodexStarter CreateStarter() { var log = new NullLog(); var lifecycleConfig = new DistTestCore.Configuration @@ -22,7 +39,7 @@ namespace CodexNetDeployer logPath: "null", logDebug: false, dataFilesPath: "notUsed", - codexLogLevel: ParseEnum.Parse(config.CodexLogLevel), + codexLogLevel: config.CodexLogLevel, runnerLocation: config.RunnerLocation ); @@ -35,7 +52,12 @@ namespace CodexNetDeployer var lifecycle = new TestLifecycle(log, lifecycleConfig, timeset); var workflowCreator = new KubernetesWorkflow.WorkflowCreator(log, kubeConfig); - var starter = new CodexStarter(lifecycle, workflowCreator); + return new CodexStarter(lifecycle, workflowCreator); + } + + private void Log(string msg) + { + Console.WriteLine(msg); } } } diff --git a/CodexNetDeployer/Program.cs b/CodexNetDeployer/Program.cs index 582d2a1a..6f61fc7d 100644 --- a/CodexNetDeployer/Program.cs +++ b/CodexNetDeployer/Program.cs @@ -2,6 +2,7 @@ using DistTestCore; using DistTestCore.Codex; using DistTestCore.Marketplace; +using Utils; using Configuration = CodexNetDeployer.Configuration; public class Program @@ -33,7 +34,7 @@ public class Program kubeNamespace: argOrVar.Get(ArgOrVar.KubeNamespace), numberOfCodexNodes: argOrVar.GetInt(ArgOrVar.NumberOfCodexNodes), storageQuota: argOrVar.GetInt(ArgOrVar.StorageQuota), - codexLogLevel: argOrVar.Get(ArgOrVar.LogLevel), + codexLogLevel: ParseEnum.Parse(argOrVar.Get(ArgOrVar.LogLevel, nameof(CodexLogLevel.Debug))), runnerLocation: location ); From cd69565c2e62d28f6f45d7cb4a8b329e62a5093f Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 22 Jun 2023 14:37:37 +0200 Subject: [PATCH 5/9] Successful local deploy --- CodexNetDeployer/Deployer.cs | 75 +++++++++++++++---- CodexNetDeployer/NullLog.cs | 2 + CodexNetDeployer/Program.cs | 8 ++ DistTestCore/Codex/CodexContainerRecipe.cs | 8 +- DistTestCore/Configuration.cs | 2 +- DistTestCore/GethStarter.cs | 2 + .../Marketplace/MarketplaceInitialConfig.cs | 1 + DistTestCore/TestLifecycle.cs | 7 +- 8 files changed, 85 insertions(+), 20 deletions(-) diff --git a/CodexNetDeployer/Deployer.cs b/CodexNetDeployer/Deployer.cs index 03f428ad..40849b50 100644 --- a/CodexNetDeployer/Deployer.cs +++ b/CodexNetDeployer/Deployer.cs @@ -1,41 +1,84 @@ using DistTestCore; +using DistTestCore.Codex; +using DistTestCore.Marketplace; +using KubernetesWorkflow; +using System.ComponentModel; namespace CodexNetDeployer { public class Deployer { private readonly Configuration config; + private readonly NullLog log; + private readonly DefaultTimeSet timeset; public Deployer(Configuration config) { this.config = config; + log = new NullLog(); + timeset = new DefaultTimeSet(); } public void Deploy() { Log("Initializing..."); - var starter = CreateStarter(); + var (workflowCreator, lifecycle) = CreateFacilities(); Log("Preparing configuration..."); - var setup = new CodexSetup(config.NumberOfCodexNodes!.Value, config.CodexLogLevel); + // We trick the Geth companion node into unlocking all of its accounts, by saying we want to start 999 codex nodes. + var setup = new CodexSetup(999, config.CodexLogLevel); + setup.WithStorageQuota(config.StorageQuota!.Value.MB()).EnableMarketplace(0.TestTokens()); - Log("Creating resources..."); - var group = (CodexNodeGroup) starter.BringOnline(setup); + Log("Creating Geth instance and deploying contracts..."); + var gethStarter = new GethStarter(lifecycle, workflowCreator); + var gethResults = gethStarter.BringOnlineMarketplaceFor(setup); - var containers = group.Containers; - foreach (var container in containers.Containers) + Log("Geth started. Codex contracts deployed."); + + Log("Starting Codex nodes..."); + + // Each node must have its own IP, so it needs it own pod. Start them 1 at a time. + var bootstrapSpr = ""; // The first one will be used to bootstrap the others. + for (var i = 0; i < config.NumberOfCodexNodes; i++) { - var pod = container.Pod.PodInfo; - Log($"Container '{container.Name}' online. Pod: '{pod.Name}@{pod.Ip}' on '{pod.K8SNodeName}'."); + Console.Write($" - {i} = "); + var workflow = workflowCreator.CreateWorkflow(); + var workflowStartup = new StartupConfig(); + var codexStart = new CodexStartupConfig(config.CodexLogLevel); + workflowStartup.Add(codexStart); + if (!string.IsNullOrEmpty(bootstrapSpr)) codexStart.BootstrapSpr = bootstrapSpr; + codexStart.StorageQuota = config.StorageQuota.Value.MB(); + var marketplaceConfig = new MarketplaceInitialConfig(100000.Eth(), 0.TestTokens()); + marketplaceConfig.AccountIndexOverride = i; + codexStart.MarketplaceConfig = marketplaceConfig; + workflowStartup.Add(gethResults); + + var containers = workflow.Start(1, Location.Unspecified, new CodexContainerRecipe(), workflowStartup); + + var container = containers.Containers.First(); + var address = lifecycle.Configuration.GetAddress(container); + var codexNode = new CodexNode(log, timeset, address); + var debugInfo = codexNode.GetDebugInfo(); + + if (!string.IsNullOrWhiteSpace(debugInfo.spr)) + { + var pod = container.Pod.PodInfo; + Console.Write($"Online ({pod.Name} at {pod.Ip} on '{pod.K8SNodeName}'" + Environment.NewLine); + + if (string.IsNullOrEmpty(bootstrapSpr)) bootstrapSpr = debugInfo.spr; + } + else + { + Console.Write("Unknown failure." + Environment.NewLine); + } } } - private CodexStarter CreateStarter() + private (WorkflowCreator, TestLifecycle) CreateFacilities() { - var log = new NullLog(); var lifecycleConfig = new DistTestCore.Configuration ( - kubeConfigFile: config.KubeConfigFile, + kubeConfigFile: null, //config.KubeConfigFile, logPath: "null", logDebug: false, dataFilesPath: "notUsed", @@ -43,16 +86,16 @@ namespace CodexNetDeployer runnerLocation: config.RunnerLocation ); - var timeset = new DefaultTimeSet(); var kubeConfig = new KubernetesWorkflow.Configuration( k8sNamespacePrefix: config.KubeNamespace, - kubeConfigFile: config.KubeConfigFile, + kubeConfigFile: null, // config.KubeConfigFile, operationTimeout: timeset.K8sOperationTimeout(), retryDelay: timeset.WaitForK8sServiceDelay()); - var lifecycle = new TestLifecycle(log, lifecycleConfig, timeset); - var workflowCreator = new KubernetesWorkflow.WorkflowCreator(log, kubeConfig); - return new CodexStarter(lifecycle, workflowCreator); + var workflowCreator = new WorkflowCreator(log, kubeConfig); + var lifecycle = new TestLifecycle(log, lifecycleConfig, timeset, workflowCreator); + + return (workflowCreator, lifecycle); } private void Log(string msg) diff --git a/CodexNetDeployer/NullLog.cs b/CodexNetDeployer/NullLog.cs index c64190b9..8417d392 100644 --- a/CodexNetDeployer/NullLog.cs +++ b/CodexNetDeployer/NullLog.cs @@ -15,10 +15,12 @@ namespace CodexNetDeployer public override void Log(string message) { + //Console.WriteLine(message); } public override void Debug(string message = "", int skipFrames = 0) { + //Console.WriteLine(message); } public override void Error(string message) diff --git a/CodexNetDeployer/Program.cs b/CodexNetDeployer/Program.cs index 6f61fc7d..ce59a6dc 100644 --- a/CodexNetDeployer/Program.cs +++ b/CodexNetDeployer/Program.cs @@ -9,6 +9,14 @@ public class Program { public static void Main(string[] args) { + args = new[] + { + @"--kube-config=C:\Users\Ben\.kube\codex-tests-ams3-dev-kubeconfig.yaml", + "--kube-namespace=testing-deployer", + "--nodes=3", + "--storage-quota=1024" + }; + var nl = Environment.NewLine; Console.WriteLine("CodexNetDeployer" + nl + nl); diff --git a/DistTestCore/Codex/CodexContainerRecipe.cs b/DistTestCore/Codex/CodexContainerRecipe.cs index f3279274..4cf4be1c 100644 --- a/DistTestCore/Codex/CodexContainerRecipe.cs +++ b/DistTestCore/Codex/CodexContainerRecipe.cs @@ -50,7 +50,7 @@ namespace DistTestCore.Codex { var gethConfig = startupConfig.Get(); var companionNode = gethConfig.CompanionNode; - var companionNodeAccount = companionNode.Accounts[Index]; + var companionNodeAccount = companionNode.Accounts[GetAccountIndex(config.MarketplaceConfig)]; Additional(companionNodeAccount); var ip = companionNode.RunningContainer.Pod.PodInfo.Ip; @@ -62,5 +62,11 @@ namespace DistTestCore.Codex AddEnvVar("PERSISTENCE", "1"); } } + + private int GetAccountIndex(MarketplaceInitialConfig marketplaceConfig) + { + if (marketplaceConfig.AccountIndexOverride != null) return marketplaceConfig.AccountIndexOverride.Value; + return Index; + } } } diff --git a/DistTestCore/Configuration.cs b/DistTestCore/Configuration.cs index 03514cd8..fceb4e09 100644 --- a/DistTestCore/Configuration.cs +++ b/DistTestCore/Configuration.cs @@ -17,7 +17,7 @@ namespace DistTestCore { kubeConfigFile = GetNullableEnvVarOrDefault("KUBECONFIG", null); logPath = GetEnvVarOrDefault("LOGPATH", "CodexTestLogs"); - logDebug = GetEnvVarOrDefault("LOGDEBUG", "false").ToLowerInvariant() == "true"; + logDebug = GetEnvVarOrDefault("LOGDEBUG", "true").ToLowerInvariant() == "true"; dataFilesPath = GetEnvVarOrDefault("DATAFILEPATH", "TestDataFiles"); codexLogLevel = ParseEnum.Parse(GetEnvVarOrDefault("LOGLEVEL", nameof(CodexLogLevel.Trace))); runnerLocation = ParseEnum.Parse(GetEnvVarOrDefault("RUNNERLOCATION", nameof(TestRunnerLocation.ExternalToCluster))); diff --git a/DistTestCore/GethStarter.cs b/DistTestCore/GethStarter.cs index 92af53f0..3b9a9b13 100644 --- a/DistTestCore/GethStarter.cs +++ b/DistTestCore/GethStarter.cs @@ -33,6 +33,8 @@ namespace DistTestCore private void TransferInitialBalance(MarketplaceNetwork marketplaceNetwork, MarketplaceInitialConfig marketplaceConfig, GethCompanionNodeInfo companionNode) { + if (marketplaceConfig.InitialTestTokens.Amount == 0) return; + var interaction = marketplaceNetwork.StartInteraction(lifecycle); var tokenAddress = marketplaceNetwork.Marketplace.TokenAddress; diff --git a/DistTestCore/Marketplace/MarketplaceInitialConfig.cs b/DistTestCore/Marketplace/MarketplaceInitialConfig.cs index 1b66199c..b802f333 100644 --- a/DistTestCore/Marketplace/MarketplaceInitialConfig.cs +++ b/DistTestCore/Marketplace/MarketplaceInitialConfig.cs @@ -10,5 +10,6 @@ public Ether InitialEth { get; } public TestToken InitialTestTokens { get; } + public int? AccountIndexOverride { get; set; } } } diff --git a/DistTestCore/TestLifecycle.cs b/DistTestCore/TestLifecycle.cs index ffd34b18..667b96cf 100644 --- a/DistTestCore/TestLifecycle.cs +++ b/DistTestCore/TestLifecycle.cs @@ -7,15 +7,18 @@ namespace DistTestCore { public class TestLifecycle { - private readonly WorkflowCreator workflowCreator; private DateTime testStart = DateTime.MinValue; public TestLifecycle(TestLog log, Configuration configuration, ITimeSet timeSet) + : this(log, configuration, timeSet, new WorkflowCreator(log, configuration.GetK8sConfiguration(timeSet))) + { + } + + public TestLifecycle(TestLog log, Configuration configuration, ITimeSet timeSet, WorkflowCreator workflowCreator) { Log = log; Configuration = configuration; TimeSet = timeSet; - workflowCreator = new WorkflowCreator(log, configuration.GetK8sConfiguration(timeSet)); FileManager = new FileManager(Log, configuration); CodexStarter = new CodexStarter(this, workflowCreator); From f242ad5a16645bc8a746acb32ec634f4af6d46b2 Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 22 Jun 2023 15:58:18 +0200 Subject: [PATCH 6/9] Adds option to declare validators. --- CodexNetDeployer/ArgOrVar.cs | 1 + CodexNetDeployer/Configuration.cs | 8 ++++++++ CodexNetDeployer/Deployer.cs | 5 +++-- CodexNetDeployer/Program.cs | 4 +++- DistTestCore/Codex/CodexContainerRecipe.cs | 5 +++++ DistTestCore/CodexSetup.cs | 8 +++++++- DistTestCore/Marketplace/MarketplaceInitialConfig.cs | 4 +++- 7 files changed, 30 insertions(+), 5 deletions(-) diff --git a/CodexNetDeployer/ArgOrVar.cs b/CodexNetDeployer/ArgOrVar.cs index 90c34ae1..f4e35cd3 100644 --- a/CodexNetDeployer/ArgOrVar.cs +++ b/CodexNetDeployer/ArgOrVar.cs @@ -8,6 +8,7 @@ public static readonly ArgVar KubeConfigFile = new ArgVar("kube-config", "KUBECONFIG", "Path to Kubeconfig file."); public static readonly ArgVar KubeNamespace = new ArgVar("kube-namespace", "KUBENAMESPACE", "Kubernetes namespace to be used for deployment."); public static readonly ArgVar NumberOfCodexNodes = new ArgVar("nodes", "NODES", "Number of Codex nodes to be created."); + public static readonly ArgVar NumberOfValidatorNodes = new ArgVar("validators", "VALIDATORS", "Number of Codex nodes that will be validating."); public static readonly ArgVar StorageQuota = new ArgVar("storage-quota", "STORAGEQUOTA", "Storage quota in megabytes used by each Codex node."); public static readonly ArgVar LogLevel = new ArgVar("log-level", "LOGLEVEL", "Log level used by each Codex node. [Trace, Debug*, Info, Warn, Error]"); diff --git a/CodexNetDeployer/Configuration.cs b/CodexNetDeployer/Configuration.cs index 3e5346ca..23830474 100644 --- a/CodexNetDeployer/Configuration.cs +++ b/CodexNetDeployer/Configuration.cs @@ -12,6 +12,7 @@ namespace CodexNetDeployer string kubeConfigFile, string kubeNamespace, int? numberOfCodexNodes, + int? numberOfValidators, int? storageQuota, CodexLogLevel codexLogLevel, TestRunnerLocation runnerLocation) @@ -22,6 +23,7 @@ namespace CodexNetDeployer KubeConfigFile = kubeConfigFile; KubeNamespace = kubeNamespace; NumberOfCodexNodes = numberOfCodexNodes; + NumberOfValidators = numberOfValidators; StorageQuota = storageQuota; CodexLogLevel = codexLogLevel; RunnerLocation = runnerLocation; @@ -33,6 +35,7 @@ namespace CodexNetDeployer public string KubeConfigFile { get; } public string KubeNamespace { get; } public int? NumberOfCodexNodes { get; } + public int? NumberOfValidators { get; } public int? StorageQuota { get; } public CodexLogLevel CodexLogLevel { get; } public TestRunnerLocation RunnerLocation { get; } @@ -50,6 +53,11 @@ namespace CodexNetDeployer onString: (n, v) => StringIsSet(n, v, errors), onInt: (n, v) => IntIsOverZero(n, v, errors)); + if (NumberOfValidators > NumberOfCodexNodes) + { + errors.Add($"{nameof(NumberOfValidators)} ({NumberOfValidators}) may not be greater than {nameof(NumberOfCodexNodes)} ({NumberOfCodexNodes})."); + } + return errors; } diff --git a/CodexNetDeployer/Deployer.cs b/CodexNetDeployer/Deployer.cs index 40849b50..a4298b94 100644 --- a/CodexNetDeployer/Deployer.cs +++ b/CodexNetDeployer/Deployer.cs @@ -39,16 +39,16 @@ namespace CodexNetDeployer // Each node must have its own IP, so it needs it own pod. Start them 1 at a time. var bootstrapSpr = ""; // The first one will be used to bootstrap the others. + int validatorsLeft = config.NumberOfValidators!.Value; for (var i = 0; i < config.NumberOfCodexNodes; i++) { Console.Write($" - {i} = "); var workflow = workflowCreator.CreateWorkflow(); var workflowStartup = new StartupConfig(); var codexStart = new CodexStartupConfig(config.CodexLogLevel); - workflowStartup.Add(codexStart); if (!string.IsNullOrEmpty(bootstrapSpr)) codexStart.BootstrapSpr = bootstrapSpr; codexStart.StorageQuota = config.StorageQuota.Value.MB(); - var marketplaceConfig = new MarketplaceInitialConfig(100000.Eth(), 0.TestTokens()); + var marketplaceConfig = new MarketplaceInitialConfig(100000.Eth(), 0.TestTokens(), validatorsLeft > 0); marketplaceConfig.AccountIndexOverride = i; codexStart.MarketplaceConfig = marketplaceConfig; workflowStartup.Add(gethResults); @@ -66,6 +66,7 @@ namespace CodexNetDeployer Console.Write($"Online ({pod.Name} at {pod.Ip} on '{pod.K8SNodeName}'" + Environment.NewLine); if (string.IsNullOrEmpty(bootstrapSpr)) bootstrapSpr = debugInfo.spr; + validatorsLeft--; } else { diff --git a/CodexNetDeployer/Program.cs b/CodexNetDeployer/Program.cs index ce59a6dc..eb3efc30 100644 --- a/CodexNetDeployer/Program.cs +++ b/CodexNetDeployer/Program.cs @@ -13,7 +13,8 @@ public class Program { @"--kube-config=C:\Users\Ben\.kube\codex-tests-ams3-dev-kubeconfig.yaml", "--kube-namespace=testing-deployer", - "--nodes=3", + "--nodes=5", + "--validators=3", "--storage-quota=1024" }; @@ -41,6 +42,7 @@ public class Program kubeConfigFile: argOrVar.Get(ArgOrVar.KubeConfigFile), kubeNamespace: argOrVar.Get(ArgOrVar.KubeNamespace), numberOfCodexNodes: argOrVar.GetInt(ArgOrVar.NumberOfCodexNodes), + numberOfValidators: argOrVar.GetInt(ArgOrVar.NumberOfValidatorNodes), storageQuota: argOrVar.GetInt(ArgOrVar.StorageQuota), codexLogLevel: ParseEnum.Parse(argOrVar.Get(ArgOrVar.LogLevel, nameof(CodexLogLevel.Debug))), runnerLocation: location diff --git a/DistTestCore/Codex/CodexContainerRecipe.cs b/DistTestCore/Codex/CodexContainerRecipe.cs index 4cf4be1c..b7c3b946 100644 --- a/DistTestCore/Codex/CodexContainerRecipe.cs +++ b/DistTestCore/Codex/CodexContainerRecipe.cs @@ -60,6 +60,11 @@ namespace DistTestCore.Codex AddEnvVar("ETH_ACCOUNT", companionNodeAccount.Account); AddEnvVar("ETH_MARKETPLACE_ADDRESS", gethConfig.MarketplaceNetwork.Marketplace.Address); AddEnvVar("PERSISTENCE", "1"); + + if (config.MarketplaceConfig.IsValidator) + { + AddEnvVar("VALIDATOR", "1"); + } } } diff --git a/DistTestCore/CodexSetup.cs b/DistTestCore/CodexSetup.cs index 83c5b9b9..8c1a73cc 100644 --- a/DistTestCore/CodexSetup.cs +++ b/DistTestCore/CodexSetup.cs @@ -13,6 +13,7 @@ namespace DistTestCore ICodexSetup EnableMetrics(); ICodexSetup EnableMarketplace(TestToken initialBalance); ICodexSetup EnableMarketplace(TestToken initialBalance, Ether initialEther); + ICodexSetup EnableMarketplace(TestToken initialBalance, Ether initialEther, bool isValidator); } public class CodexSetup : CodexStartupConfig, ICodexSetup @@ -62,7 +63,12 @@ namespace DistTestCore public ICodexSetup EnableMarketplace(TestToken initialBalance, Ether initialEther) { - MarketplaceConfig = new MarketplaceInitialConfig(initialEther, initialBalance); + return EnableMarketplace(initialBalance, initialEther, false); + } + + public ICodexSetup EnableMarketplace(TestToken initialBalance, Ether initialEther, bool isValidator) + { + MarketplaceConfig = new MarketplaceInitialConfig(initialEther, initialBalance, isValidator); return this; } diff --git a/DistTestCore/Marketplace/MarketplaceInitialConfig.cs b/DistTestCore/Marketplace/MarketplaceInitialConfig.cs index b802f333..c51d79f8 100644 --- a/DistTestCore/Marketplace/MarketplaceInitialConfig.cs +++ b/DistTestCore/Marketplace/MarketplaceInitialConfig.cs @@ -2,14 +2,16 @@ { public class MarketplaceInitialConfig { - public MarketplaceInitialConfig(Ether initialEth, TestToken initialTestTokens) + public MarketplaceInitialConfig(Ether initialEth, TestToken initialTestTokens, bool isValidator) { InitialEth = initialEth; InitialTestTokens = initialTestTokens; + IsValidator = isValidator; } public Ether InitialEth { get; } public TestToken InitialTestTokens { get; } + public bool IsValidator { get; } public int? AccountIndexOverride { get; set; } } } From 4f0a278df1d838a0b9c26cd24c5f4e9ba9def47c Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 23 Jun 2023 08:18:48 +0200 Subject: [PATCH 7/9] Flips internal/external default. --- CodexNetDeployer/ArgOrVar.cs | 4 ++-- CodexNetDeployer/Deployer.cs | 6 ++++-- CodexNetDeployer/Program.cs | 9 +++++---- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/CodexNetDeployer/ArgOrVar.cs b/CodexNetDeployer/ArgOrVar.cs index f4e35cd3..5844b68a 100644 --- a/CodexNetDeployer/ArgOrVar.cs +++ b/CodexNetDeployer/ArgOrVar.cs @@ -55,8 +55,8 @@ "The deployer will set up the required supporting services, deploy the Codex on-chain contracts, start and bootstrap the Codex instances. " + "All Kubernetes objects will be created in the namespace provided, allowing you to easily find, modify, and delete them afterwards." + nl); - Console.WriteLine("CodexNetDeployer assumes you are running this tool from *outside* the Kubernetes cluster you want to deploy to. " + - "If you are running this from a container inside the cluster, add the argument '--internal'." + nl); + Console.WriteLine("CodexNetDeployer assumes you are running this tool from *inside* the Kubernetes cluster you want to deploy to. " + + "If you are not running this from a container inside the cluster, add the argument '--external'." + nl); Console.Write("\t[ CLI argument ] or [ Environment variable ]"); Console.CursorLeft = 70; diff --git a/CodexNetDeployer/Deployer.cs b/CodexNetDeployer/Deployer.cs index a4298b94..137ce882 100644 --- a/CodexNetDeployer/Deployer.cs +++ b/CodexNetDeployer/Deployer.cs @@ -46,12 +46,14 @@ namespace CodexNetDeployer var workflow = workflowCreator.CreateWorkflow(); var workflowStartup = new StartupConfig(); var codexStart = new CodexStartupConfig(config.CodexLogLevel); + workflowStartup.Add(gethResults); + workflowStartup.Add(codexStart); + if (!string.IsNullOrEmpty(bootstrapSpr)) codexStart.BootstrapSpr = bootstrapSpr; codexStart.StorageQuota = config.StorageQuota.Value.MB(); var marketplaceConfig = new MarketplaceInitialConfig(100000.Eth(), 0.TestTokens(), validatorsLeft > 0); marketplaceConfig.AccountIndexOverride = i; codexStart.MarketplaceConfig = marketplaceConfig; - workflowStartup.Add(gethResults); var containers = workflow.Start(1, Location.Unspecified, new CodexContainerRecipe(), workflowStartup); @@ -63,7 +65,7 @@ namespace CodexNetDeployer if (!string.IsNullOrWhiteSpace(debugInfo.spr)) { var pod = container.Pod.PodInfo; - Console.Write($"Online ({pod.Name} at {pod.Ip} on '{pod.K8SNodeName}'" + Environment.NewLine); + Console.Write($"Online ({pod.Name} at {pod.Ip} on '{pod.K8SNodeName}')" + Environment.NewLine); if (string.IsNullOrEmpty(bootstrapSpr)) bootstrapSpr = debugInfo.spr; validatorsLeft--; diff --git a/CodexNetDeployer/Program.cs b/CodexNetDeployer/Program.cs index eb3efc30..805b6548 100644 --- a/CodexNetDeployer/Program.cs +++ b/CodexNetDeployer/Program.cs @@ -15,7 +15,8 @@ public class Program "--kube-namespace=testing-deployer", "--nodes=5", "--validators=3", - "--storage-quota=1024" + "--storage-quota=1024", + "--external" }; var nl = Environment.NewLine; @@ -29,10 +30,10 @@ public class Program return; } - var location = TestRunnerLocation.ExternalToCluster; - if (args.Any(a => a == "--internal")) + var location = TestRunnerLocation.InternalToCluster; + if (args.Any(a => a == "--external")) { - location = TestRunnerLocation.InternalToCluster; + location = TestRunnerLocation.ExternalToCluster; } var config = new Configuration( From aa9b667940291bd8d1966eea2f149c1fa1b2567b Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 23 Jun 2023 08:44:27 +0200 Subject: [PATCH 8/9] Writes codex-deployment.json which can be used by the test running. --- CodexNetDeployer/CodexNodeStarter.cs | 75 +++++++++++++++++++++++++++ CodexNetDeployer/Deployer.cs | 45 +++------------- CodexNetDeployer/Program.cs | 7 ++- DistTestCore/Codex/CodexDeployment.cs | 17 ++++++ 4 files changed, 106 insertions(+), 38 deletions(-) create mode 100644 CodexNetDeployer/CodexNodeStarter.cs create mode 100644 DistTestCore/Codex/CodexDeployment.cs diff --git a/CodexNetDeployer/CodexNodeStarter.cs b/CodexNetDeployer/CodexNodeStarter.cs new file mode 100644 index 00000000..ee071dba --- /dev/null +++ b/CodexNetDeployer/CodexNodeStarter.cs @@ -0,0 +1,75 @@ +using DistTestCore; +using DistTestCore.Codex; +using DistTestCore.Marketplace; +using KubernetesWorkflow; +using Logging; + +namespace CodexNetDeployer +{ + public class CodexNodeStarter + { + private readonly Configuration config; + private readonly WorkflowCreator workflowCreator; + private readonly TestLifecycle lifecycle; + private readonly BaseLog log; + private readonly ITimeSet timeSet; + private readonly GethStartResult gethResult; + private string bootstrapSpr = ""; + private int validatorsLeft; + + public CodexNodeStarter(Configuration config, WorkflowCreator workflowCreator, TestLifecycle lifecycle, BaseLog log, ITimeSet timeSet, GethStartResult gethResult, int numberOfValidators) + { + this.config = config; + this.workflowCreator = workflowCreator; + this.lifecycle = lifecycle; + this.log = log; + this.timeSet = timeSet; + this.gethResult = gethResult; + this.validatorsLeft = numberOfValidators; + } + + public RunningContainer? Start(int i) + { + Console.Write($" - {i} = "); + var workflow = workflowCreator.CreateWorkflow(); + var workflowStartup = new StartupConfig(); + workflowStartup.Add(gethResult); + workflowStartup.Add(CreateCodexStartupConfig(bootstrapSpr, i, validatorsLeft)); + + var containers = workflow.Start(1, Location.Unspecified, new CodexContainerRecipe(), workflowStartup); + + var container = containers.Containers.First(); + var address = lifecycle.Configuration.GetAddress(container); + var codexNode = new CodexNode(log, timeSet, address); + var debugInfo = codexNode.GetDebugInfo(); + + if (!string.IsNullOrWhiteSpace(debugInfo.spr)) + { + var pod = container.Pod.PodInfo; + Console.Write($"Online ({pod.Name} at {pod.Ip} on '{pod.K8SNodeName}')" + Environment.NewLine); + + if (string.IsNullOrEmpty(bootstrapSpr)) bootstrapSpr = debugInfo.spr; + validatorsLeft--; + return container; + } + else + { + Console.Write("Unknown failure." + Environment.NewLine); + return null; + } + } + + private CodexStartupConfig CreateCodexStartupConfig(string bootstrapSpr, int i, int validatorsLeft) + { + var codexStart = new CodexStartupConfig(config.CodexLogLevel); + + if (!string.IsNullOrEmpty(bootstrapSpr)) codexStart.BootstrapSpr = bootstrapSpr; + codexStart.StorageQuota = config.StorageQuota!.Value.MB(); + var marketplaceConfig = new MarketplaceInitialConfig(100000.Eth(), 0.TestTokens(), validatorsLeft > 0); + marketplaceConfig.AccountIndexOverride = i; + codexStart.MarketplaceConfig = marketplaceConfig; + + return codexStart; + } + } +} diff --git a/CodexNetDeployer/Deployer.cs b/CodexNetDeployer/Deployer.cs index 137ce882..b6a1df2b 100644 --- a/CodexNetDeployer/Deployer.cs +++ b/CodexNetDeployer/Deployer.cs @@ -1,8 +1,6 @@ using DistTestCore; using DistTestCore.Codex; -using DistTestCore.Marketplace; using KubernetesWorkflow; -using System.ComponentModel; namespace CodexNetDeployer { @@ -19,7 +17,7 @@ namespace CodexNetDeployer timeset = new DefaultTimeSet(); } - public void Deploy() + public CodexDeployment Deploy() { Log("Initializing..."); var (workflowCreator, lifecycle) = CreateFacilities(); @@ -34,47 +32,20 @@ namespace CodexNetDeployer var gethResults = gethStarter.BringOnlineMarketplaceFor(setup); Log("Geth started. Codex contracts deployed."); + Log("Warning: It can take up to 45 minutes for the Geth node to finish unlocking all if its 1000 preconfigured accounts."); Log("Starting Codex nodes..."); // Each node must have its own IP, so it needs it own pod. Start them 1 at a time. - var bootstrapSpr = ""; // The first one will be used to bootstrap the others. - int validatorsLeft = config.NumberOfValidators!.Value; + var codexStarter = new CodexNodeStarter(config, workflowCreator, lifecycle, log, timeset, gethResults, config.NumberOfValidators!.Value); + var codexContainers = new List(); for (var i = 0; i < config.NumberOfCodexNodes; i++) { - Console.Write($" - {i} = "); - var workflow = workflowCreator.CreateWorkflow(); - var workflowStartup = new StartupConfig(); - var codexStart = new CodexStartupConfig(config.CodexLogLevel); - workflowStartup.Add(gethResults); - workflowStartup.Add(codexStart); - - if (!string.IsNullOrEmpty(bootstrapSpr)) codexStart.BootstrapSpr = bootstrapSpr; - codexStart.StorageQuota = config.StorageQuota.Value.MB(); - var marketplaceConfig = new MarketplaceInitialConfig(100000.Eth(), 0.TestTokens(), validatorsLeft > 0); - marketplaceConfig.AccountIndexOverride = i; - codexStart.MarketplaceConfig = marketplaceConfig; - - var containers = workflow.Start(1, Location.Unspecified, new CodexContainerRecipe(), workflowStartup); - - var container = containers.Containers.First(); - var address = lifecycle.Configuration.GetAddress(container); - var codexNode = new CodexNode(log, timeset, address); - var debugInfo = codexNode.GetDebugInfo(); - - if (!string.IsNullOrWhiteSpace(debugInfo.spr)) - { - var pod = container.Pod.PodInfo; - Console.Write($"Online ({pod.Name} at {pod.Ip} on '{pod.K8SNodeName}')" + Environment.NewLine); - - if (string.IsNullOrEmpty(bootstrapSpr)) bootstrapSpr = debugInfo.spr; - validatorsLeft--; - } - else - { - Console.Write("Unknown failure." + Environment.NewLine); - } + var container = codexStarter.Start(i); + if (container != null) codexContainers.Add(container); } + + return new CodexDeployment(gethResults, codexContainers.ToArray()); } private (WorkflowCreator, TestLifecycle) CreateFacilities() diff --git a/CodexNetDeployer/Program.cs b/CodexNetDeployer/Program.cs index 805b6548..8c29f3de 100644 --- a/CodexNetDeployer/Program.cs +++ b/CodexNetDeployer/Program.cs @@ -2,6 +2,7 @@ using DistTestCore; using DistTestCore.Codex; using DistTestCore.Marketplace; +using Newtonsoft.Json; using Utils; using Configuration = CodexNetDeployer.Configuration; @@ -64,7 +65,11 @@ public class Program } var deployer = new Deployer(config); - deployer.Deploy(); + var deployment = deployer.Deploy(); + + Console.WriteLine("Writing codex-deployment.json..."); + + File.WriteAllText("codex-deployment.json", JsonConvert.SerializeObject(deployment, Formatting.Indented)); Console.WriteLine("Done!"); } diff --git a/DistTestCore/Codex/CodexDeployment.cs b/DistTestCore/Codex/CodexDeployment.cs new file mode 100644 index 00000000..f86ad318 --- /dev/null +++ b/DistTestCore/Codex/CodexDeployment.cs @@ -0,0 +1,17 @@ +using DistTestCore.Marketplace; +using KubernetesWorkflow; + +namespace DistTestCore.Codex +{ + public class CodexDeployment + { + public CodexDeployment(GethStartResult gethStartResult, RunningContainer[] codexContainers) + { + GethStartResult = gethStartResult; + CodexContainers = codexContainers; + } + + public GethStartResult GethStartResult { get; } + public RunningContainer[] CodexContainers { get; } + } +} From acfd1f1f101a2c8d8d845d75bad49f56c0e4ee01 Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 23 Jun 2023 08:45:16 +0200 Subject: [PATCH 9/9] Removes local debug help --- CodexNetDeployer/Deployer.cs | 4 ++-- CodexNetDeployer/Program.cs | 10 ---------- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/CodexNetDeployer/Deployer.cs b/CodexNetDeployer/Deployer.cs index b6a1df2b..a5aa56b1 100644 --- a/CodexNetDeployer/Deployer.cs +++ b/CodexNetDeployer/Deployer.cs @@ -52,7 +52,7 @@ namespace CodexNetDeployer { var lifecycleConfig = new DistTestCore.Configuration ( - kubeConfigFile: null, //config.KubeConfigFile, + kubeConfigFile: config.KubeConfigFile, logPath: "null", logDebug: false, dataFilesPath: "notUsed", @@ -62,7 +62,7 @@ namespace CodexNetDeployer var kubeConfig = new KubernetesWorkflow.Configuration( k8sNamespacePrefix: config.KubeNamespace, - kubeConfigFile: null, // config.KubeConfigFile, + kubeConfigFile: config.KubeConfigFile, operationTimeout: timeset.K8sOperationTimeout(), retryDelay: timeset.WaitForK8sServiceDelay()); diff --git a/CodexNetDeployer/Program.cs b/CodexNetDeployer/Program.cs index 8c29f3de..462667c6 100644 --- a/CodexNetDeployer/Program.cs +++ b/CodexNetDeployer/Program.cs @@ -10,16 +10,6 @@ public class Program { public static void Main(string[] args) { - args = new[] - { - @"--kube-config=C:\Users\Ben\.kube\codex-tests-ams3-dev-kubeconfig.yaml", - "--kube-namespace=testing-deployer", - "--nodes=5", - "--validators=3", - "--storage-quota=1024", - "--external" - }; - var nl = Environment.NewLine; Console.WriteLine("CodexNetDeployer" + nl + nl);