logos-storage/logos-storage-nim-cs-dist-tests
1
0
Fork 0
mirror of https://github.com/logos-storage/logos-storage-nim-cs-dist-tests.git synced 2026-01-02 13:33:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'feature/automatic-contracts-image-version-detection'

Browse Source
This commit is contained in:
ThatBen 2025-04-22 19:49:12 +02:00
commit ac7aae972c
No known key found for this signature in database
GPG Key ID: 62C543548433D43E
19 changed files with 62 additions and 182 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
ProjectPlugins/CodexClient/CodexTypes.cs
View File

@ -17,6 +17,7 @@ namespace CodexClient
{
public string Version { get; set; } = string.Empty;
public string Revision { get; set; } = string.Empty;
public string Contracts { get; set; } = string.Empty;
public bool IsValid()
{

3
ProjectPlugins/CodexClient/Mapper.cs
View File

@ -168,7 +168,8 @@ namespace CodexClient
return new DebugInfoVersion
{
Version = obj.Version,
Revision = obj.Revision
Revision = obj.Revision,
Contracts = obj.Contracts
};
}

3
ProjectPlugins/CodexClient/openapi.yaml
View File

@ -124,6 +124,9 @@ components:
revision:
type: string
example: 0c647d8
contracts:
type: string
example: 0b537c7
PeersTable:
type: object

16
ProjectPlugins/CodexContractsPlugin/CodexContractsContainerRecipe.cs
View File

@ -1,4 +1,5 @@
using GethPlugin;
using CodexClient;
using GethPlugin;
using KubernetesWorkflow;
using KubernetesWorkflow.Recipe;
@ -8,14 +9,14 @@ namespace CodexContractsPlugin
{
public const string MarketplaceAddressFilename = "/hardhat/deployments/codexdisttestnetwork/Marketplace.json";
public const string MarketplaceArtifactFilename = "/hardhat/artifacts/contracts/Marketplace.sol/Marketplace.json";
private readonly VersionRegistry versionRegistry;
private readonly DebugInfoVersion versionInfo;
public override string AppName => "codex-contracts";
public override string Image => versionRegistry.GetContractsDockerImage();
public override string Image => GetContractsDockerImage();
public CodexContractsContainerRecipe(VersionRegistry versionRegistry)
public CodexContractsContainerRecipe(DebugInfoVersion versionInfo)
{
this.versionRegistry = versionRegistry;
this.versionInfo = versionInfo;
}
protected override void Initialize(StartupConfig startupConfig)
@ -30,5 +31,10 @@ namespace CodexContractsPlugin
AddEnvVar("HARDHAT_NETWORK", "codexdisttestnetwork");
AddEnvVar("KEEP_ALIVE", "1");
}
private string GetContractsDockerImage()
{
return $"codexstorage/codex-contracts-eth:sha-{versionInfo.Contracts}-dist-tests";
}
}
}

17
ProjectPlugins/CodexContractsPlugin/CodexContractsPlugin.cs
View File

@ -7,15 +7,11 @@ namespace CodexContractsPlugin
{
private readonly IPluginTools tools;
private readonly CodexContractsStarter starter;
private readonly VersionRegistry versionRegistry;
private readonly CodexContractsContainerRecipe recipe;
public CodexContractsPlugin(IPluginTools tools)
{
this.tools = tools;
versionRegistry = new VersionRegistry(tools.GetLog());
recipe = new CodexContractsContainerRecipe(versionRegistry);
starter = new CodexContractsStarter(tools, recipe);
starter = new CodexContractsStarter(tools);
}
public string LogPrefix => "(CodexContracts) ";
@ -31,16 +27,16 @@ namespace CodexContractsPlugin
public void AddMetadata(IAddMetadata metadata)
{
metadata.Add("codexcontractsid", recipe.Image);
metadata.Add("codexcontractsid", "dynamic");
}
public void Decommission()
{
}
public CodexContractsDeployment DeployContracts(CoreInterface ci, IGethNode gethNode)
public CodexContractsDeployment DeployContracts(CoreInterface ci, IGethNode gethNode, CodexClient.DebugInfoVersion versionInfo)
{
return starter.Deploy(ci, gethNode);
return starter.Deploy(ci, gethNode, versionInfo);
}
public ICodexContracts WrapDeploy(IGethNode gethNode, CodexContractsDeployment deployment)
@ -48,10 +44,5 @@ namespace CodexContractsPlugin
deployment = SerializeGate.Gate(deployment);
return starter.Wrap(gethNode, deployment);
}
public void SetCodexDockerImageProvider(ICodexDockerImageProvider provider)
{
versionRegistry.SetProvider(provider);
}
}
}

1
ProjectPlugins/CodexContractsPlugin/CodexContractsPlugin.csproj
View File

@ -13,6 +13,7 @@
<ItemGroup>
<ProjectReference Include="..\..\Framework\Core\Core.csproj" />
<ProjectReference Include="..\CodexClient\CodexClient.csproj" />
<ProjectReference Include="..\GethPlugin\GethPlugin.csproj" />
</ItemGroup>

12
ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs
View File

@ -1,4 +1,5 @@
using CodexContractsPlugin.Marketplace;
using CodexClient;
using CodexContractsPlugin.Marketplace;
using Core;
using GethPlugin;
using KubernetesWorkflow;
@ -12,15 +13,13 @@ namespace CodexContractsPlugin
public class CodexContractsStarter
{
private readonly IPluginTools tools;
private readonly CodexContractsContainerRecipe recipe;
public CodexContractsStarter(IPluginTools tools, CodexContractsContainerRecipe recipe)
public CodexContractsStarter(IPluginTools tools)
{
this.tools = tools;
this.recipe = recipe;
}
public CodexContractsDeployment Deploy(CoreInterface ci, IGethNode gethNode)
public CodexContractsDeployment Deploy(CoreInterface ci, IGethNode gethNode, DebugInfoVersion versionInfo)
{
Log("Starting Codex SmartContracts container...");
@ -28,6 +27,9 @@ namespace CodexContractsPlugin
var startupConfig = CreateStartupConfig(gethNode);
startupConfig.NameOverride = "codex-contracts";
var recipe = new CodexContractsContainerRecipe(versionInfo);
Log($"Using image: {recipe.Image}");
var containers = workflow.Start(1, recipe, startupConfig).WaitForOnline();
if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 Codex contracts container to be created. Test infra failure.");
var container = containers.Containers[0];

16
ProjectPlugins/CodexContractsPlugin/CoreInterfaceExtensions.cs
View File

@ -1,13 +1,14 @@
using Core;
using CodexClient;
using Core;
using GethPlugin;
namespace CodexContractsPlugin
{
public static class CoreInterfaceExtensions
{
public static CodexContractsDeployment DeployCodexContracts(this CoreInterface ci, IGethNode gethNode)
public static CodexContractsDeployment DeployCodexContracts(this CoreInterface ci, IGethNode gethNode, DebugInfoVersion versionInfo)
{
return Plugin(ci).DeployContracts(ci, gethNode);
return Plugin(ci).DeployContracts(ci, gethNode, versionInfo);
}
public static ICodexContracts WrapCodexContractsDeployment(this CoreInterface ci, IGethNode gethNode, CodexContractsDeployment deployment)
@ -15,17 +16,12 @@ namespace CodexContractsPlugin
return Plugin(ci).WrapDeploy(gethNode, deployment);
}
public static ICodexContracts StartCodexContracts(this CoreInterface ci, IGethNode gethNode)
public static ICodexContracts StartCodexContracts(this CoreInterface ci, IGethNode gethNode, DebugInfoVersion versionInfo)
{
var deployment = DeployCodexContracts(ci, gethNode);
var deployment = DeployCodexContracts(ci, gethNode, versionInfo);
return WrapCodexContractsDeployment(ci, gethNode, deployment);
}
public static void SetCodexDockerImageProvider(this CoreInterface ci, ICodexDockerImageProvider provider)
{
Plugin(ci).SetCodexDockerImageProvider(provider);
}
private static CodexContractsPlugin Plugin(CoreInterface ci)
{
return ci.GetPlugin<CodexContractsPlugin>();

125
ProjectPlugins/CodexContractsPlugin/VersionRegistry.cs
View File

@ -1,125 +0,0 @@
using System.Diagnostics;
using Logging;
namespace CodexContractsPlugin
{
public interface ICodexDockerImageProvider
{
string GetCodexDockerImage();
}
public class VersionRegistry
{
private ICodexDockerImageProvider provider = new ExceptionProvider();
private static readonly Dictionary<string, string> cache = new Dictionary<string, string>();
private static readonly object cacheLock = new object();
private readonly ILog log;
public VersionRegistry(ILog log)
{
this.log = log;
}
public void SetProvider(ICodexDockerImageProvider provider)
{
this.provider = provider;
}
public string GetContractsDockerImage()
{
try
{
var codexImage = provider.GetCodexDockerImage();
return GetContractsDockerImage(codexImage);
}
catch (Exception exc)
{
throw new Exception("Failed to get contracts docker image.", exc);
}
}
private string GetContractsDockerImage(string codexImage)
{
lock (cacheLock)
{
if (cache.TryGetValue(codexImage, out string? value))
{
return value;
}
var result = GetContractsImage(codexImage);
cache.Add(codexImage, result);
return result;
}
}
private string GetContractsImage(string codexImage)
{
var inspectResult = InspectCodexImage(codexImage);
var image = ParseCodexContractsImageName(inspectResult);
log.Log($"From codex docker image '{codexImage}', determined codex-contracts docker image: '{image}'");
return image;
}
private string InspectCodexImage(string img)
{
Execute("docker", $"pull {img}");
return Execute("docker", $"inspect {img}");
}
private string ParseCodexContractsImageName(string inspectResult)
{
// It is a nice json structure. But we only need this one line.
// "storage.codex.nim-codex.blockchain-image": "codexstorage/codex-contracts-eth:sha-0bf1385-dist-tests"
var lines = inspectResult.Split('\n', StringSplitOptions.RemoveEmptyEntries);
var line = lines.Single(l => l.Contains("storage.codex.nim-codex.blockchain-image"));
var tokens = line.Split(' ', StringSplitOptions.RemoveEmptyEntries);
return tokens.Last().Replace("\"", "").Trim();
}
private string Execute(string cmd, string args)
{
var startInfo = new ProcessStartInfo(
fileName: cmd,
arguments: args
);
startInfo.RedirectStandardOutput = true;
startInfo.RedirectStandardError = true;
var process = Process.Start(startInfo);
if (process == null)
{
throw new Exception("Failed to start: " + cmd + args);
}
KillAfterTimeout(process);
process.WaitForExit();
return process.StandardOutput.ReadToEnd();
}
private void KillAfterTimeout(Process process)
{
// There's a known issue that some docker commands on some platforms
// will fail to stop on their own. This has been known since 2019 and it's not fixed.
// So we will issue a kill to the process ourselves if it exceeds a timeout.
Task.Run(() =>
{
Thread.Sleep(TimeSpan.FromSeconds(30.0));
if (process != null && !process.HasExited)
{
process.Kill();
}
});
}
}
internal class ExceptionProvider : ICodexDockerImageProvider
{
public string GetCodexDockerImage()
{
throw new InvalidOperationException("CodexContractsPlugin has not yet received a CodexDockerImageProvider " +
"and so cannot select a compatible contracts docker image.");
}
}
}

2
ProjectPlugins/CodexPlugin/ApiChecker.cs
View File

@ -10,7 +10,7 @@ namespace CodexPlugin
public class ApiChecker
{
// <INSERT-OPENAPI-YAML-HASH>
private const string OpenApiYamlHash = "1A-F7-DF-C3-E1-C6-98-FF-32-20-21-9B-26-40-B0-51-08-35-C2-E7-DB-41-49-93-60-A9-CE-47-B5-AD-3D-A3";
private const string OpenApiYamlHash = "06-B9-41-E8-C8-6C-DE-01-86-83-F3-9A-E4-AC-E7-30-D9-E6-64-60-E0-21-81-9E-4E-C5-93-77-2C-71-79-14";
private const string OpenApiFilePath = "/codex/openapi.yaml";
private const string DisableEnvironmentVariable = "CODEXPLUGIN_DISABLE_APICHECK";

6
ProjectPlugins/CodexPlugin/CodexDockerImage.cs
View File

@ -1,8 +1,6 @@
using CodexContractsPlugin;
namespace CodexPlugin
namespace CodexPlugin
{
public class CodexDockerImage : ICodexDockerImageProvider
public class CodexDockerImage
{
private const string DefaultDockerImage = "codexstorage/nim-codex:latest-dist-tests";

1
ProjectPlugins/CodexPlugin/CodexPlugin.cs
View File

@ -42,7 +42,6 @@ namespace CodexPlugin
public void Awake(IPluginAccess access)
{
access.GetPlugin<CodexContractsPlugin.CodexContractsPlugin>().SetCodexDockerImageProvider(codexDockerImage);
}
public void Announce()

3
Tests/CodexReleaseTests/DataTests/DataExpiryTest.cs
View File

@ -50,8 +50,9 @@ namespace CodexReleaseTests.DataTests
var blockTtl = TimeSpan.FromMinutes(1.0);
var interval = TimeSpan.FromSeconds(10.0);
var bootstrapNode = StartCodex();
var geth = StartGethNode(s => s.IsMiner());
var contracts = Ci.StartCodexContracts(geth);
var contracts = Ci.StartCodexContracts(geth, bootstrapNode.Version);
var node = StartCodex(s => s
.EnableMarketplace(geth, contracts, m => m.WithInitial(100.Eth(), 100.Tst()))
.WithBlockTTL(blockTtl)

5
Tests/CodexReleaseTests/MarketTests/MarketplaceAutoBootstrapDistTest.cs
View File

@ -6,7 +6,6 @@ using CodexTests;
using DistTestCore;
using GethPlugin;
using Nethereum.Hex.HexConvertors.Extensions;
using NUnit.Framework;
using Utils;
namespace CodexReleaseTests.MarketTests
@ -21,14 +20,14 @@ namespace CodexReleaseTests.MarketTests
{
base.LifecycleStart(lifecycle);
var geth = StartGethNode(s => s.IsMiner());
var contracts = Ci.StartCodexContracts(geth);
var contracts = Ci.StartCodexContracts(geth, BootstrapNode.Version);
handles.Add(lifecycle, new MarketplaceHandle(geth, contracts));
}
protected override void LifecycleStop(TestLifecycle lifecycle, DistTestResult result)
{
base.LifecycleStop(lifecycle, result);
handles.Remove(lifecycle);
base.LifecycleStop(lifecycle, result);
}
protected IGethNode GetGeth()

21
Tests/ExperimentalTests/AutoBootstrapDistTest.cs
View File

@ -1,37 +1,40 @@
using CodexClient;
using CodexPlugin;
using DistTestCore;
using NUnit.Framework;
namespace CodexTests
{
public class AutoBootstrapDistTest : CodexDistTest
{
private readonly Dictionary<TestLifecycle, ICodexNode> bootstrapNodes = new Dictionary<TestLifecycle, ICodexNode>();
private bool isBooting = false;
[SetUp]
public void SetUpBootstrapNode()
protected override void LifecycleStart(TestLifecycle tl)
{
var tl = Get();
base.LifecycleStart(tl);
if (!bootstrapNodes.ContainsKey(tl))
{
isBooting = true;
bootstrapNodes.Add(tl, StartCodex(s => s.WithName("BOOTSTRAP_" + tl.TestNamespace)));
isBooting = false;
}
}
[TearDown]
public void TearDownBootstrapNode()
protected override void LifecycleStop(TestLifecycle lifecycle, DistTestResult result)
{
bootstrapNodes.Remove(Get());
bootstrapNodes.Remove(lifecycle);
base.LifecycleStop(lifecycle, result);
}
protected override void OnCodexSetup(ICodexSetup setup)
{
if (isBooting) return;
var node = BootstrapNode;
if (node != null) setup.WithBootstrapNode(node);
}
protected ICodexNode? BootstrapNode
protected ICodexNode BootstrapNode
{
get
{
@ -40,7 +43,7 @@ namespace CodexTests
{
return node;
}
return null;
throw new InvalidOperationException("Bootstrap node not yet started.");
}
}
}

2
Tests/ExperimentalTests/BasicTests/MarketplaceTests.cs
View File

@ -31,7 +31,7 @@ namespace ExperimentalTests.BasicTests
);
var geth = StartGethNode(s => s.IsMiner().WithName("disttest-geth"));
var contracts = Ci.StartCodexContracts(geth);
var contracts = Ci.StartCodexContracts(geth, BootstrapNode.Version);
var numberOfHosts = 5;
var hosts = StartCodex(numberOfHosts, s => s

2
Tests/ExperimentalTests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs
View File

@ -21,7 +21,7 @@ namespace ExperimentalTests.DownloadConnectivityTests
public void MarketplaceDoesNotInterfereWithPeerDownload()
{
var geth = StartGethNode(s => s.IsMiner());
var contracts = Ci.StartCodexContracts(geth);
var contracts = Ci.StartCodexContracts(geth, BootstrapNode.Version);
var nodes = StartCodex(2, s => s.EnableMarketplace(geth, contracts, m => m
.WithInitial(10.Eth(), 1000.TstWei())));

2
Tests/ExperimentalTests/PeerDiscoveryTests/PeerDiscoveryTests.cs
View File

@ -31,7 +31,7 @@ namespace ExperimentalTests.PeerDiscoveryTests
public void MarketplaceDoesNotInterfereWithPeerDiscovery()
{
var geth = StartGethNode(s => s.IsMiner());
var contracts = Ci.StartCodexContracts(geth);
var contracts = Ci.StartCodexContracts(geth, BootstrapNode.Version);
var nodes = StartCodex(2, s => s.EnableMarketplace(geth, contracts, m => m
.WithInitial(10.Eth(), 1000.TstWei())));

6
Tools/CodexNetDeployer/Deployer.cs
View File

@ -64,8 +64,12 @@ namespace CodexNetDeployer
var gethDeployment = DeployGeth(ci);
var gethNode = ci.WrapGethDeployment(gethDeployment, new BlockCache());
var bootNode = ci.StartCodexNode();
var versionInfo = bootNode.GetDebugInfo().Version;
bootNode.Stop(waitTillStopped: true);
Log("Geth started. Deploying Codex contracts...");
var contractsDeployment = ci.DeployCodexContracts(gethNode);
var contractsDeployment = ci.DeployCodexContracts(gethNode, versionInfo);
var contracts = ci.WrapCodexContractsDeployment(gethNode, contractsDeployment);
Log("Codex contracts deployed.");