From df8a0eed8ac2a8788cbb583e58edade165355748 Mon Sep 17 00:00:00 2001 From: ThatBen Date: Fri, 18 Apr 2025 15:47:44 +0200 Subject: [PATCH] Uses label of docker image to find compatible contracts images. --- Framework/Core/PluginManager.cs | 13 ++- Framework/Core/ProjectPlugin.cs | 6 + .../CodexContractsContainerRecipe.cs | 10 +- .../CodexContractsPlugin.cs | 17 ++- .../CodexContractsStarter.cs | 6 +- .../CoreInterfaceExtensions.cs | 5 + .../CodexContractsPlugin/VersionRegistry.cs | 103 ++++++++++++++++++ .../CodexDiscordBotPlugin.cs | 4 + .../CodexPlugin/CodexContainerRecipe.cs | 17 +-- .../CodexPlugin/CodexDockerImage.cs | 20 ++++ ProjectPlugins/CodexPlugin/CodexPlugin.cs | 12 +- .../CodexPlugin/ContainerCodexStarter.cs | 5 +- .../CodexPlugin/LocalCodexBuilder.cs | 4 +- .../DeployAndRunPlugin/DeployAndRunPlugin.cs | 4 + ProjectPlugins/GethPlugin/GethPlugin.cs | 4 + ProjectPlugins/MetricsPlugin/MetricsPlugin.cs | 4 + Tests/CodexContinuousTests/NodeRunner.cs | 2 +- 17 files changed, 210 insertions(+), 26 deletions(-) create mode 100644 ProjectPlugins/CodexContractsPlugin/VersionRegistry.cs create mode 100644 ProjectPlugins/CodexPlugin/CodexDockerImage.cs diff --git a/Framework/Core/PluginManager.cs b/Framework/Core/PluginManager.cs index 27b08fe3..c8bac789 100644 --- a/Framework/Core/PluginManager.cs +++ b/Framework/Core/PluginManager.cs @@ -1,6 +1,6 @@ namespace Core { - internal class PluginManager + internal class PluginManager : IPluginAccess { private readonly List pairs = new List(); @@ -14,6 +14,7 @@ ApplyLogPrefix(plugin, tools); } + AwakePlugins(); } internal void AnnouncePlugins() @@ -43,7 +44,7 @@ } } - internal T GetPlugin() where T : IProjectPlugin + public T GetPlugin() where T : IProjectPlugin { return (T)pairs.Single(p => p.Plugin.GetType() == typeof(T)).Plugin; } @@ -55,6 +56,14 @@ return plugin; } + private void AwakePlugins() + { + foreach (var p in pairs) + { + p.Plugin.Awake(this); + } + } + private void ApplyLogPrefix(IProjectPlugin plugin, PluginTools tools) { if (plugin is IHasLogPrefix hasLogPrefix) diff --git a/Framework/Core/ProjectPlugin.cs b/Framework/Core/ProjectPlugin.cs index 72f3c16a..a777ef80 100644 --- a/Framework/Core/ProjectPlugin.cs +++ b/Framework/Core/ProjectPlugin.cs @@ -4,6 +4,7 @@ namespace Core { public interface IProjectPlugin { + void Awake(IPluginAccess access); void Announce(); void Decommission(); } @@ -18,6 +19,11 @@ namespace Core void AddMetadata(IAddMetadata metadata); } + public interface IPluginAccess + { + T GetPlugin() where T : IProjectPlugin; + } + public static class ProjectPlugin { /// diff --git a/ProjectPlugins/CodexContractsPlugin/CodexContractsContainerRecipe.cs b/ProjectPlugins/CodexContractsPlugin/CodexContractsContainerRecipe.cs index 66adf592..e09a0ee7 100644 --- a/ProjectPlugins/CodexContractsPlugin/CodexContractsContainerRecipe.cs +++ b/ProjectPlugins/CodexContractsPlugin/CodexContractsContainerRecipe.cs @@ -6,13 +6,17 @@ namespace CodexContractsPlugin { public class CodexContractsContainerRecipe : ContainerRecipeFactory { - public static string DockerImage { get; } = "codexstorage/codex-contracts-eth:latest-dist-tests"; - public const string MarketplaceAddressFilename = "/hardhat/deployments/codexdisttestnetwork/Marketplace.json"; public const string MarketplaceArtifactFilename = "/hardhat/artifacts/contracts/Marketplace.sol/Marketplace.json"; + private readonly VersionRegistry versionRegistry; public override string AppName => "codex-contracts"; - public override string Image => DockerImage; + public override string Image => versionRegistry.GetContractsDockerImage(); + + public CodexContractsContainerRecipe(VersionRegistry versionRegistry) + { + this.versionRegistry = versionRegistry; + } protected override void Initialize(StartupConfig startupConfig) { diff --git a/ProjectPlugins/CodexContractsPlugin/CodexContractsPlugin.cs b/ProjectPlugins/CodexContractsPlugin/CodexContractsPlugin.cs index 416b17aa..6e02280d 100644 --- a/ProjectPlugins/CodexContractsPlugin/CodexContractsPlugin.cs +++ b/ProjectPlugins/CodexContractsPlugin/CodexContractsPlugin.cs @@ -7,15 +7,23 @@ namespace CodexContractsPlugin { private readonly IPluginTools tools; private readonly CodexContractsStarter starter; + private readonly VersionRegistry versionRegistry; + private readonly CodexContractsContainerRecipe recipe; public CodexContractsPlugin(IPluginTools tools) { this.tools = tools; - starter = new CodexContractsStarter(tools); + versionRegistry = new VersionRegistry(tools.GetLog()); + recipe = new CodexContractsContainerRecipe(versionRegistry); + starter = new CodexContractsStarter(tools, recipe); } public string LogPrefix => "(CodexContracts) "; + public void Awake(IPluginAccess access) + { + } + public void Announce() { tools.GetLog().Log($"Loaded Codex-Marketplace SmartContracts"); @@ -23,7 +31,7 @@ namespace CodexContractsPlugin public void AddMetadata(IAddMetadata metadata) { - metadata.Add("codexcontractsid", CodexContractsContainerRecipe.DockerImage); + metadata.Add("codexcontractsid", recipe.Image); } public void Decommission() @@ -40,5 +48,10 @@ namespace CodexContractsPlugin deployment = SerializeGate.Gate(deployment); return starter.Wrap(gethNode, deployment); } + + public void SetCodexDockerImageProvider(ICodexDockerImageProvider provider) + { + versionRegistry.SetProvider(provider); + } } } diff --git a/ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs b/ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs index be74a3b2..5ecc22a9 100644 --- a/ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs +++ b/ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs @@ -12,10 +12,12 @@ namespace CodexContractsPlugin public class CodexContractsStarter { private readonly IPluginTools tools; + private readonly CodexContractsContainerRecipe recipe; - public CodexContractsStarter(IPluginTools tools) + public CodexContractsStarter(IPluginTools tools, CodexContractsContainerRecipe recipe) { this.tools = tools; + this.recipe = recipe; } public CodexContractsDeployment Deploy(CoreInterface ci, IGethNode gethNode) @@ -26,7 +28,7 @@ namespace CodexContractsPlugin var startupConfig = CreateStartupConfig(gethNode); startupConfig.NameOverride = "codex-contracts"; - var containers = workflow.Start(1, new CodexContractsContainerRecipe(), startupConfig).WaitForOnline(); + var containers = workflow.Start(1, recipe, startupConfig).WaitForOnline(); if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 Codex contracts container to be created. Test infra failure."); var container = containers.Containers[0]; diff --git a/ProjectPlugins/CodexContractsPlugin/CoreInterfaceExtensions.cs b/ProjectPlugins/CodexContractsPlugin/CoreInterfaceExtensions.cs index 7d68a860..ea123bc9 100644 --- a/ProjectPlugins/CodexContractsPlugin/CoreInterfaceExtensions.cs +++ b/ProjectPlugins/CodexContractsPlugin/CoreInterfaceExtensions.cs @@ -21,6 +21,11 @@ namespace CodexContractsPlugin return WrapCodexContractsDeployment(ci, gethNode, deployment); } + public static void SetCodexDockerImageProvider(this CoreInterface ci, ICodexDockerImageProvider provider) + { + Plugin(ci).SetCodexDockerImageProvider(provider); + } + private static CodexContractsPlugin Plugin(CoreInterface ci) { return ci.GetPlugin(); diff --git a/ProjectPlugins/CodexContractsPlugin/VersionRegistry.cs b/ProjectPlugins/CodexContractsPlugin/VersionRegistry.cs new file mode 100644 index 00000000..9ed4e8b4 --- /dev/null +++ b/ProjectPlugins/CodexContractsPlugin/VersionRegistry.cs @@ -0,0 +1,103 @@ +using System.Diagnostics; +using Logging; + +namespace CodexContractsPlugin +{ + public interface ICodexDockerImageProvider + { + string GetCodexDockerImage(); + } + + public class VersionRegistry + { + private ICodexDockerImageProvider provider = new ExceptionProvider(); + private readonly Dictionary cache = new Dictionary(); + private readonly ILog log; + + public VersionRegistry(ILog log) + { + this.log = log; + } + + public void SetProvider(ICodexDockerImageProvider provider) + { + this.provider = provider; + } + + public string GetContractsDockerImage() + { + try + { + var codexImage = provider.GetCodexDockerImage(); + return GetContractsDockerImage(codexImage); + } + catch (Exception exc) + { + throw new Exception("Failed to get contracts docker image.", exc); + } + } + + private string GetContractsDockerImage(string codexImage) + { + if (cache.TryGetValue(codexImage, out string? value)) + { + return value; + } + var result = GetContractsImage(codexImage); + cache.Add(codexImage, result); + return result; + } + + private string GetContractsImage(string codexImage) + { + var inspectResult = InspectCodexImage(codexImage); + var image = ParseCodexContractsImageName(inspectResult); + log.Log($"From codex docker image '{codexImage}', determined codex-contracts docker image: '{image}'"); + return image; + } + + private string InspectCodexImage(string img) + { + Execute("docker", $"pull {img}"); + return Execute("docker", $"inspect {img}"); + } + + private string ParseCodexContractsImageName(string inspectResult) + { + // It is a nice json structure. But we only need this one line. + // "storage.codex.nim-codex.blockchain-image": "codexstorage/codex-contracts-eth:sha-0bf1385-dist-tests" + var lines = inspectResult.Split('\n', StringSplitOptions.RemoveEmptyEntries); + var line = lines.Single(l => l.Contains("storage.codex.nim-codex.blockchain-image")); + var tokens = line.Split(' ', StringSplitOptions.RemoveEmptyEntries); + return tokens.Last().Replace("\"", "").Trim(); + } + + private string Execute(string cmd, string args) + { + var startInfo = new ProcessStartInfo( + fileName: cmd, + arguments: args + ); + startInfo.RedirectStandardOutput = true; + startInfo.RedirectStandardError = true; + + var process = Process.Start(startInfo); + if (process == null || process.HasExited) + { + throw new Exception("Failed to start: " + cmd + args); + } + + process.WaitForExit(); + return process.StandardOutput.ReadToEnd(); + } + } + + internal class ExceptionProvider : ICodexDockerImageProvider + { + public string GetCodexDockerImage() + { + throw new InvalidOperationException("CodexContractsPlugin has not yet received a CodexDockerImageProvider " + + "and so cannot select a compatible contracts docker image."); + } + } +} diff --git a/ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.cs b/ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.cs index 556ef152..e1354806 100644 --- a/ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.cs +++ b/ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.cs @@ -17,6 +17,10 @@ namespace CodexDiscordBotPlugin public string LogPrefix => "(DiscordBot) "; + public void Awake(IPluginAccess access) + { + } + public void Announce() { tools.GetLog().Log($"Codex DiscordBot (BiblioTech) loaded."); diff --git a/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs b/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs index 9dfcf2cf..6a0b76e7 100644 --- a/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs +++ b/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs @@ -7,7 +7,6 @@ namespace CodexPlugin { public class CodexContainerRecipe : ContainerRecipeFactory { - private const string DefaultDockerImage = "codexstorage/nim-codex:latest-dist-tests"; public const string ApiPortTag = "codex_api_port"; public const string ListenPortTag = "codex_listen_port"; public const string MetricsPortTag = "codex_metrics_port"; @@ -16,11 +15,15 @@ namespace CodexPlugin // Used by tests for time-constraint assertions. public static readonly TimeSpan MaxUploadTimePerMegabyte = TimeSpan.FromSeconds(2.0); public static readonly TimeSpan MaxDownloadTimePerMegabyte = TimeSpan.FromSeconds(2.0); + private readonly CodexDockerImage codexDockerImage; public override string AppName => "codex"; - public override string Image => GetDockerImage(); + public override string Image => codexDockerImage.GetCodexDockerImage(); - public static string DockerImageOverride { get; set; } = string.Empty; + public CodexContainerRecipe(CodexDockerImage codexDockerImage) + { + this.codexDockerImage = codexDockerImage; + } protected override void Initialize(StartupConfig startupConfig) { @@ -163,13 +166,5 @@ namespace CodexPlugin // Default Codex quota: 8 Gb, using +20% to be safe. return 8.GB().Multiply(1.2); } - - private string GetDockerImage() - { - var image = Environment.GetEnvironmentVariable("CODEXDOCKERIMAGE"); - if (!string.IsNullOrEmpty(image)) return image; - if (!string.IsNullOrEmpty(DockerImageOverride)) return DockerImageOverride; - return DefaultDockerImage; - } } } diff --git a/ProjectPlugins/CodexPlugin/CodexDockerImage.cs b/ProjectPlugins/CodexPlugin/CodexDockerImage.cs new file mode 100644 index 00000000..f86470db --- /dev/null +++ b/ProjectPlugins/CodexPlugin/CodexDockerImage.cs @@ -0,0 +1,20 @@ +using CodexContractsPlugin; + +namespace CodexPlugin +{ + public class CodexDockerImage : ICodexDockerImageProvider + { + private const string DefaultDockerImage = "codexstorage/nim-codex:sha-c9a5ef8-dist-tests"; + //"codexstorage/nim-codex:latest-dist-tests"; + + public static string Override { get; set; } = string.Empty; + + public string GetCodexDockerImage() + { + var image = Environment.GetEnvironmentVariable("CODEXDOCKERIMAGE"); + if (!string.IsNullOrEmpty(image)) return image; + if (!string.IsNullOrEmpty(Override)) return Override; + return DefaultDockerImage; + } + } +} diff --git a/ProjectPlugins/CodexPlugin/CodexPlugin.cs b/ProjectPlugins/CodexPlugin/CodexPlugin.cs index 02fbeaa6..277b6101 100644 --- a/ProjectPlugins/CodexPlugin/CodexPlugin.cs +++ b/ProjectPlugins/CodexPlugin/CodexPlugin.cs @@ -13,12 +13,15 @@ namespace CodexPlugin private readonly CodexLogLevel defaultLogLevel = CodexLogLevel.Trace; private readonly CodexHooksFactory hooksFactory = new CodexHooksFactory(); private readonly ProcessControlMap processControlMap = new ProcessControlMap(); + private readonly CodexDockerImage codexDockerImage = new CodexDockerImage(); + private readonly CodexContainerRecipe recipe; private readonly CodexWrapper codexWrapper; public CodexPlugin(IPluginTools tools) { this.tools = tools; + recipe = new CodexContainerRecipe(codexDockerImage); codexStarter = CreateCodexStarter(); codexWrapper = new CodexWrapper(tools, processControlMap, hooksFactory); } @@ -28,7 +31,7 @@ namespace CodexPlugin if (UseContainers) { Log("Using Containerized Codex instances"); - return new ContainerCodexStarter(tools, processControlMap); + return new ContainerCodexStarter(tools, recipe, processControlMap); } Log("Using Binary Codex instances"); @@ -37,8 +40,15 @@ namespace CodexPlugin public string LogPrefix => "(Codex) "; + public void Awake(IPluginAccess access) + { + access.GetPlugin().SetCodexDockerImageProvider(codexDockerImage); + } + public void Announce() { + // give codex docker image to contracts plugin. + Log($"Loaded with Codex ID: '{codexWrapper.GetCodexId()}' - Revision: {codexWrapper.GetCodexRevision()}"); } diff --git a/ProjectPlugins/CodexPlugin/ContainerCodexStarter.cs b/ProjectPlugins/CodexPlugin/ContainerCodexStarter.cs index 0627bc1f..4103fb8c 100644 --- a/ProjectPlugins/CodexPlugin/ContainerCodexStarter.cs +++ b/ProjectPlugins/CodexPlugin/ContainerCodexStarter.cs @@ -10,12 +10,13 @@ namespace CodexPlugin { private readonly IPluginTools pluginTools; private readonly ProcessControlMap processControlMap; - private readonly CodexContainerRecipe recipe = new CodexContainerRecipe(); + private readonly CodexContainerRecipe recipe; private readonly ApiChecker apiChecker; - public ContainerCodexStarter(IPluginTools pluginTools, ProcessControlMap processControlMap) + public ContainerCodexStarter(IPluginTools pluginTools, CodexContainerRecipe recipe, ProcessControlMap processControlMap) { this.pluginTools = pluginTools; + this.recipe = recipe; this.processControlMap = processControlMap; apiChecker = new ApiChecker(pluginTools); } diff --git a/ProjectPlugins/CodexPlugin/LocalCodexBuilder.cs b/ProjectPlugins/CodexPlugin/LocalCodexBuilder.cs index 9f1b76c9..90bc1cb1 100644 --- a/ProjectPlugins/CodexPlugin/LocalCodexBuilder.cs +++ b/ProjectPlugins/CodexPlugin/LocalCodexBuilder.cs @@ -40,7 +40,7 @@ namespace CodexNetDeployer Log($"Codex docker image will be built in path '{repoPath}'."); Log("Please note this can take several minutes. If you're not trying to use a Codex image with local code changes,"); Log("Consider using the default test image or consider setting the 'CODEXDOCKERIMAGE' environment variable to use an already built image."); - CodexContainerRecipe.DockerImageOverride = $"Using docker image locally built in path '{repoPath}'."; + CodexDockerImage.Override = $"Using docker image locally built in path '{repoPath}'."; } public void Build() @@ -62,7 +62,7 @@ namespace CodexNetDeployer Docker("push", customImage); - CodexContainerRecipe.DockerImageOverride = customImage; + CodexDockerImage.Override = customImage; Log("Image pushed. Good to go!"); } diff --git a/ProjectPlugins/DeployAndRunPlugin/DeployAndRunPlugin.cs b/ProjectPlugins/DeployAndRunPlugin/DeployAndRunPlugin.cs index c290c0d5..e1c60c24 100644 --- a/ProjectPlugins/DeployAndRunPlugin/DeployAndRunPlugin.cs +++ b/ProjectPlugins/DeployAndRunPlugin/DeployAndRunPlugin.cs @@ -13,6 +13,10 @@ namespace DeployAndRunPlugin this.tools = tools; } + public void Awake(IPluginAccess access) + { + } + public void Announce() { tools.GetLog().Log("Deploy-and-Run plugin loaded."); diff --git a/ProjectPlugins/GethPlugin/GethPlugin.cs b/ProjectPlugins/GethPlugin/GethPlugin.cs index 07249eca..76f92314 100644 --- a/ProjectPlugins/GethPlugin/GethPlugin.cs +++ b/ProjectPlugins/GethPlugin/GethPlugin.cs @@ -16,6 +16,10 @@ namespace GethPlugin public string LogPrefix => "(Geth) "; + public void Awake(IPluginAccess access) + { + } + public void Announce() { tools.GetLog().Log($"Loaded Geth plugin."); diff --git a/ProjectPlugins/MetricsPlugin/MetricsPlugin.cs b/ProjectPlugins/MetricsPlugin/MetricsPlugin.cs index a796ba06..75e27eca 100644 --- a/ProjectPlugins/MetricsPlugin/MetricsPlugin.cs +++ b/ProjectPlugins/MetricsPlugin/MetricsPlugin.cs @@ -18,6 +18,10 @@ namespace MetricsPlugin public string LogPrefix => "(Metrics) "; + public void Awake(IPluginAccess access) + { + } + public void Announce() { tools.GetLog().Log($"Prometheus plugin loaded with '{starter.GetPrometheusId()}'."); diff --git a/Tests/CodexContinuousTests/NodeRunner.cs b/Tests/CodexContinuousTests/NodeRunner.cs index f2b3c64f..ae18f7e9 100644 --- a/Tests/CodexContinuousTests/NodeRunner.cs +++ b/Tests/CodexContinuousTests/NodeRunner.cs @@ -33,7 +33,7 @@ namespace ContinuousTests var entryPoint = CreateEntryPoint(); // We have to be sure that the transient node we start is using the same image as whatever's already in the deployed network. // Therefore, we use the image of the bootstrap node. - CodexContainerRecipe.DockerImageOverride = bootstrapNode.GetImageName(); + CodexDockerImage.Override = bootstrapNode.GetImageName(); try {