From 9db35be2ec4b6a74ee360dcf07c1ab9227e86b5c Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 24 Apr 2023 14:09:23 +0200 Subject: [PATCH] Allows for nethereum interactions with companion nodes. Verifies that marketplace contract is available before proceeding with codex-node setup. --- DistTestCore/Codex/CodexContainerRecipe.cs | 5 +++ DistTestCore/Codex/CodexStartupConfig.cs | 2 +- DistTestCore/CodexSetup.cs | 23 ++-------- DistTestCore/GethStarter.cs | 2 +- .../CodexContractsContainerRecipe.cs | 1 + .../Marketplace/CodexContractsStarter.cs | 7 +++- .../Marketplace/ContainerInfoExtractor.cs | 42 ++++++++++++++----- .../Marketplace/GethBootstrapNodeStarter.cs | 2 +- .../Marketplace/GethCompanionNodeInfo.cs | 17 +++++++- .../Marketplace/GethCompanionNodeStarter.cs | 18 ++++++-- .../Marketplace/GethContainerRecipe.cs | 4 +- DistTestCore/OnlineCodexNode.cs | 5 +-- Nethereum/NethereumInteraction.cs | 24 ++++++++++- Tests/BasicTests/ExampleTests.cs | 5 ++- Tests/DurabilityTests/DurabilityTests.cs | 9 ++-- 15 files changed, 114 insertions(+), 52 deletions(-) diff --git a/DistTestCore/Codex/CodexContainerRecipe.cs b/DistTestCore/Codex/CodexContainerRecipe.cs index 53f5078..10995ff 100644 --- a/DistTestCore/Codex/CodexContainerRecipe.cs +++ b/DistTestCore/Codex/CodexContainerRecipe.cs @@ -21,6 +21,11 @@ namespace DistTestCore.Codex var listenPort = AddInternalPort(); AddEnvVar("LISTEN_ADDRS", $"/ip4/0.0.0.0/tcp/{listenPort.Number}"); + if (!string.IsNullOrEmpty(config.BootstrapSpr)) + { + AddEnvVar("BOOTSTRAP_SPR", config.BootstrapSpr); + } + if (config.LogLevel != null) { AddEnvVar("LOG_LEVEL", config.LogLevel.ToString()!.ToUpperInvariant()); diff --git a/DistTestCore/Codex/CodexStartupConfig.cs b/DistTestCore/Codex/CodexStartupConfig.cs index 2ebf407..ae27bdf 100644 --- a/DistTestCore/Codex/CodexStartupConfig.cs +++ b/DistTestCore/Codex/CodexStartupConfig.cs @@ -10,6 +10,6 @@ namespace DistTestCore.Codex public ByteSize? StorageQuota { get; set; } public bool MetricsEnabled { get; set; } public MarketplaceInitialConfig? MarketplaceConfig { get; set; } - public IOnlineCodexNode? BootstrapNode { get; set; } + public string? BootstrapSpr { get; set; } } } diff --git a/DistTestCore/CodexSetup.cs b/DistTestCore/CodexSetup.cs index 8f7e253..618cd06 100644 --- a/DistTestCore/CodexSetup.cs +++ b/DistTestCore/CodexSetup.cs @@ -30,24 +30,7 @@ namespace DistTestCore public ICodexNodeGroup BringOnline() { - var group = starter.BringOnline(this); - ConnectToBootstrapNode(group); - return group; - } - - private void ConnectToBootstrapNode(ICodexNodeGroup group) - { - if (BootstrapNode == null) return; - - // TODO: - // node.ConnectToPeer uses the '/api/codex/vi/connect/' endpoint to make the connection. - // This should be replaced by injecting the bootstrap node's SPR into the env-vars of the new node containers. (Easy!) - // However, NAT isn't figure out yet. So connecting with SPR doesn't (always?) work. - // So for now, ConnectToPeer - foreach (var node in group) - { - node.ConnectToPeer(BootstrapNode); - } + return starter.BringOnline(this); } public ICodexSetup At(Location location) @@ -58,7 +41,7 @@ namespace DistTestCore public ICodexSetup WithBootstrapNode(IOnlineCodexNode node) { - BootstrapNode = node; + BootstrapSpr = node.GetDebugInfo().spr; return this; } @@ -100,7 +83,7 @@ namespace DistTestCore private IEnumerable DescribeArgs() { if (LogLevel != null) yield return $"LogLevel={LogLevel}"; - if (BootstrapNode != null) yield return $"BootstrapNode={BootstrapNode.GetName()}"; + if (BootstrapSpr != null) yield return $"BootstrapNode={BootstrapSpr}"; if (StorageQuota != null) yield return $"StorageQuote={StorageQuota}"; } } diff --git a/DistTestCore/GethStarter.cs b/DistTestCore/GethStarter.cs index b3c782b..d933eaa 100644 --- a/DistTestCore/GethStarter.cs +++ b/DistTestCore/GethStarter.cs @@ -62,7 +62,7 @@ namespace DistTestCore private GethCompanionNodeInfo[] StartCompanionNodes(CodexSetup codexSetup, MarketplaceNetwork marketplaceNetwork) { - return companionNodeStarter.StartCompanionNodesFor(codexSetup, marketplaceNetwork.Bootstrap); + return companionNodeStarter.StartCompanionNodesFor(codexSetup, marketplaceNetwork); } } diff --git a/DistTestCore/Marketplace/CodexContractsContainerRecipe.cs b/DistTestCore/Marketplace/CodexContractsContainerRecipe.cs index d2a93a7..42bbeca 100644 --- a/DistTestCore/Marketplace/CodexContractsContainerRecipe.cs +++ b/DistTestCore/Marketplace/CodexContractsContainerRecipe.cs @@ -6,6 +6,7 @@ namespace DistTestCore.Marketplace { public const string DockerImage = "thatbenbierens/codex-contracts-deployment"; public const string MarketplaceAddressFilename = "/usr/app/deployments/codexdisttestnetwork/Marketplace.json"; + public const string MarketplaceArtifactFilename = "/usr/app/artifacts/contracts/Marketplace.sol/Marketplace.json"; protected override string Image => DockerImage; diff --git a/DistTestCore/Marketplace/CodexContractsStarter.cs b/DistTestCore/Marketplace/CodexContractsStarter.cs index 841e830..cb2c414 100644 --- a/DistTestCore/Marketplace/CodexContractsStarter.cs +++ b/DistTestCore/Marketplace/CodexContractsStarter.cs @@ -32,13 +32,14 @@ namespace DistTestCore.Marketplace var extractor = new ContainerInfoExtractor(workflow, container); var marketplaceAddress = extractor.ExtractMarketplaceAddress(); + var abi = extractor.ExtractMarketplaceAbi(); var interaction = bootstrapNode.StartInteraction(lifecycle.Log); var tokenAddress = interaction.GetTokenAddress(marketplaceAddress); LogEnd("Contracts deployed."); - return new MarketplaceInfo(marketplaceAddress, tokenAddress); + return new MarketplaceInfo(marketplaceAddress, abi, tokenAddress); } private void WaitUntil(Func predicate) @@ -57,13 +58,15 @@ namespace DistTestCore.Marketplace public class MarketplaceInfo { - public MarketplaceInfo(string address, string tokenAddress) + public MarketplaceInfo(string address, string abi, string tokenAddress) { Address = address; + Abi = abi; TokenAddress = tokenAddress; } public string Address { get; } + public string Abi { get; } public string TokenAddress { get; } } diff --git a/DistTestCore/Marketplace/ContainerInfoExtractor.cs b/DistTestCore/Marketplace/ContainerInfoExtractor.cs index c2e4501..a2f4a96 100644 --- a/DistTestCore/Marketplace/ContainerInfoExtractor.cs +++ b/DistTestCore/Marketplace/ContainerInfoExtractor.cs @@ -1,5 +1,9 @@ -using KubernetesWorkflow; +using IdentityModel.OidcClient; +using KubernetesWorkflow; using Newtonsoft.Json; +using Newtonsoft.Json.Converters; +using Newtonsoft.Json.Linq; +using Utils; namespace DistTestCore.Marketplace { @@ -30,9 +34,9 @@ namespace DistTestCore.Marketplace return pubKey; } - public string ExtractBootstrapPrivateKey() + public string ExtractPrivateKey() { - var privKey = Retry(FetchBootstrapPrivateKey); + var privKey = Retry(FetchPrivateKey); if (string.IsNullOrEmpty(privKey)) throw new InvalidOperationException("Unable to fetch private key from geth node. Test infra failure."); return privKey; @@ -46,14 +50,23 @@ namespace DistTestCore.Marketplace return marketplaceAddress; } + public string ExtractMarketplaceAbi() + { + var marketplaceAbi = Retry(FetchMarketplaceAbi); + if (string.IsNullOrEmpty(marketplaceAbi)) throw new InvalidOperationException("Unable to fetch marketplace artifacts from codex-contracts node. Test infra failure."); + + return marketplaceAbi; + } + private string Retry(Func fetch) { - var result = Catch(fetch); - if (string.IsNullOrEmpty(result)) + var result = string.Empty; + Time.WaitUntil(() => { - Thread.Sleep(TimeSpan.FromSeconds(5)); - result = fetch(); - } + result = Catch(fetch); + return !string.IsNullOrEmpty(result); + }, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(3)); + return result; } @@ -74,9 +87,9 @@ namespace DistTestCore.Marketplace return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.AccountFilename); } - private string FetchBootstrapPrivateKey() + private string FetchPrivateKey() { - return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.BootstrapPrivateKeyFilename); + return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.PrivateKeyFilename); } private string FetchMarketplaceAddress() @@ -86,6 +99,15 @@ namespace DistTestCore.Marketplace return marketplace!.address; } + private string FetchMarketplaceAbi() + { + var json = workflow.ExecuteCommand(container, "cat", CodexContractsContainerRecipe.MarketplaceArtifactFilename); + + var artifact = JObject.Parse(json); + var abi = artifact["abi"]; + return abi!.ToString(Formatting.None); + } + private string FetchPubKey() { var enodeFinder = new PubKeyFinder(); diff --git a/DistTestCore/Marketplace/GethBootstrapNodeStarter.cs b/DistTestCore/Marketplace/GethBootstrapNodeStarter.cs index e0efc61..1a68ee9 100644 --- a/DistTestCore/Marketplace/GethBootstrapNodeStarter.cs +++ b/DistTestCore/Marketplace/GethBootstrapNodeStarter.cs @@ -22,7 +22,7 @@ namespace DistTestCore.Marketplace var extractor = new ContainerInfoExtractor(workflow, bootstrapContainer); var account = extractor.ExtractAccount(); var pubKey = extractor.ExtractPubKey(); - var privateKey = extractor.ExtractBootstrapPrivateKey(); + var privateKey = extractor.ExtractPrivateKey(); var discoveryPort = bootstrapContainer.Recipe.GetPortByTag(GethContainerRecipe.DiscoveryPortTag); LogEnd($"Geth bootstrap node started with account '{account}'"); diff --git a/DistTestCore/Marketplace/GethCompanionNodeInfo.cs b/DistTestCore/Marketplace/GethCompanionNodeInfo.cs index 9b7bd23..3657059 100644 --- a/DistTestCore/Marketplace/GethCompanionNodeInfo.cs +++ b/DistTestCore/Marketplace/GethCompanionNodeInfo.cs @@ -1,16 +1,31 @@ using KubernetesWorkflow; +using Logging; +using NethereumWorkflow; namespace DistTestCore.Marketplace { public class GethCompanionNodeInfo { - public GethCompanionNodeInfo(RunningContainer runningContainer, string account) + public GethCompanionNodeInfo(RunningContainer runningContainer, string account, string privateKey) { RunningContainer = runningContainer; Account = account; + PrivateKey = privateKey; } public RunningContainer RunningContainer { get; } public string Account { get; } + public string PrivateKey { get; } + + public NethereumInteraction StartInteraction(TestLog log) + { + var ip = RunningContainer.Pod.Cluster.IP; + var port = RunningContainer.ServicePorts[0].Number; + var account = Account; + var privateKey = PrivateKey; + + var creator = new NethereumInteractionCreator(log, ip, port, account, privateKey); + return creator.CreateWorkflow(); + } } } diff --git a/DistTestCore/Marketplace/GethCompanionNodeStarter.cs b/DistTestCore/Marketplace/GethCompanionNodeStarter.cs index 047af28..b087c8a 100644 --- a/DistTestCore/Marketplace/GethCompanionNodeStarter.cs +++ b/DistTestCore/Marketplace/GethCompanionNodeStarter.cs @@ -9,11 +9,11 @@ namespace DistTestCore.Marketplace { } - public GethCompanionNodeInfo[] StartCompanionNodesFor(CodexSetup codexSetup, GethBootstrapNodeInfo bootstrapNode) + public GethCompanionNodeInfo[] StartCompanionNodesFor(CodexSetup codexSetup, MarketplaceNetwork marketplace) { LogStart($"Initializing companions for {codexSetup.NumberOfNodes} Codex nodes."); - var startupConfig = CreateCompanionNodeStartupConfig(bootstrapNode); + var startupConfig = CreateCompanionNodeStartupConfig(marketplace.Bootstrap); var workflow = workflowCreator.CreateWorkflow(); var containers = workflow.Start(codexSetup.NumberOfNodes, Location.Unspecified, new GethContainerRecipe(), startupConfig); @@ -21,6 +21,11 @@ namespace DistTestCore.Marketplace var result = containers.Containers.Select(c => CreateCompanionInfo(workflow, c)).ToArray(); + foreach (var node in result) + { + EnsureCompanionNodeIsSynced(node, marketplace); + } + LogEnd($"Initialized {codexSetup.NumberOfNodes} companion nodes. Their accounts: [{string.Join(",", result.Select(c => c.Account))}]"); return result; @@ -30,7 +35,14 @@ namespace DistTestCore.Marketplace { var extractor = new ContainerInfoExtractor(workflow, container); var account = extractor.ExtractAccount(); - return new GethCompanionNodeInfo(container, account); + var privKey = extractor.ExtractPrivateKey(); + return new GethCompanionNodeInfo(container, account, privKey); + } + + private void EnsureCompanionNodeIsSynced(GethCompanionNodeInfo node, MarketplaceNetwork marketplace) + { + var interaction = node.StartInteraction(lifecycle.Log); + interaction.EnsureSynced(marketplace.Marketplace.Address, marketplace.Marketplace.Abi); } private StartupConfig CreateCompanionNodeStartupConfig(GethBootstrapNodeInfo bootstrapNode) diff --git a/DistTestCore/Marketplace/GethContainerRecipe.cs b/DistTestCore/Marketplace/GethContainerRecipe.cs index 85309cc..8f2a826 100644 --- a/DistTestCore/Marketplace/GethContainerRecipe.cs +++ b/DistTestCore/Marketplace/GethContainerRecipe.cs @@ -8,7 +8,7 @@ namespace DistTestCore.Marketplace public const string HttpPortTag = "http_port"; public const string DiscoveryPortTag = "disc_port"; public const string AccountFilename = "account_string.txt"; - public const string BootstrapPrivateKeyFilename = "bootstrap_private.key"; + public const string PrivateKeyFilename = "private.key"; protected override string Image => DockerImage; @@ -44,7 +44,7 @@ namespace DistTestCore.Marketplace { var port = AddInternalPort(); var authRpc = AddInternalPort(); - var httpPort = AddInternalPort(tag: HttpPortTag); + var httpPort = AddExposedPort(tag: HttpPortTag); var bootPubKey = config.BootstrapNode.PubKey; var bootIp = config.BootstrapNode.RunningContainers.Containers[0].Pod.Ip; diff --git a/DistTestCore/OnlineCodexNode.cs b/DistTestCore/OnlineCodexNode.cs index 43e16ec..149bc0e 100644 --- a/DistTestCore/OnlineCodexNode.cs +++ b/DistTestCore/OnlineCodexNode.cs @@ -23,7 +23,6 @@ namespace DistTestCore private const string SuccessfullyConnectedMessage = "Successfully connected to peer"; private const string UploadFailedMessage = "Unable to store block"; private readonly TestLifecycle lifecycle; - private CodexDebugResponse? debugInfo; public OnlineCodexNode(TestLifecycle lifecycle, CodexAccess codexAccess, CodexNodeGroup group, IMetricsAccess metricsAccess, IMarketplaceAccess marketplaceAccess) { @@ -46,9 +45,7 @@ namespace DistTestCore public CodexDebugResponse GetDebugInfo() { - if (debugInfo != null) return debugInfo; - - debugInfo = CodexAccess.GetDebugInfo(); + var debugInfo = CodexAccess.GetDebugInfo(); Log($"Got DebugInfo with id: '{debugInfo.id}'."); return debugInfo; } diff --git a/Nethereum/NethereumInteraction.cs b/Nethereum/NethereumInteraction.cs index 735bd6f..c7384a8 100644 --- a/Nethereum/NethereumInteraction.cs +++ b/Nethereum/NethereumInteraction.cs @@ -72,6 +72,28 @@ namespace NethereumWorkflow Task.WaitAll(tasks); } + public void EnsureSynced(string marketplaceAddress, string marketplaceAbi) + { + Time.WaitUntil(() => + { + return !Time.Wait(web3.Eth.Syncing.SendRequestAsync()).IsSyncing; + }, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(1)); + + + Time.WaitUntil(() => + { + try + { + var contract = web3.Eth.GetContract(marketplaceAbi, marketplaceAddress); + return contract != null; + } + catch + { + return false; + } + }, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(1)); + } + private HexBigInteger ToHexBig(decimal amount) { var bigint = ToBig(amount); @@ -106,7 +128,7 @@ namespace NethereumWorkflow } [Function("balanceOf", "uint256")] - public class GetTokenBalanceFunction :FunctionMessage + public class GetTokenBalanceFunction : FunctionMessage { [Parameter("address", "owner", 1)] public string Owner { get; set; } diff --git a/Tests/BasicTests/ExampleTests.cs b/Tests/BasicTests/ExampleTests.cs index 31cc56a..e1beb7d 100644 --- a/Tests/BasicTests/ExampleTests.cs +++ b/Tests/BasicTests/ExampleTests.cs @@ -51,6 +51,7 @@ namespace Tests.BasicTests public void MarketplaceExample() { var primary = SetupCodexNodes(1) + .WithLogLevel(CodexLogLevel.Trace) .WithStorageQuota(11.GB()) .EnableMarketplace(initialBalance: 234.TestTokens()) .BringOnline()[0]; @@ -58,11 +59,11 @@ namespace Tests.BasicTests primary.Marketplace.AssertThatBalance(Is.EqualTo(234.TestTokens())); var secondary = SetupCodexNodes(1) + .WithLogLevel(CodexLogLevel.Trace) + .WithBootstrapNode(primary) .EnableMarketplace(initialBalance: 1000.TestTokens()) .BringOnline()[0]; - primary.ConnectToPeer(secondary); - primary.Marketplace.MakeStorageAvailable( size: 10.GB(), minPricePerBytePerSecond: 1.TestTokens(), diff --git a/Tests/DurabilityTests/DurabilityTests.cs b/Tests/DurabilityTests/DurabilityTests.cs index 6ffd78b..53d6ab3 100644 --- a/Tests/DurabilityTests/DurabilityTests.cs +++ b/Tests/DurabilityTests/DurabilityTests.cs @@ -1,4 +1,5 @@ using DistTestCore; +using DistTestCore.Codex; using NUnit.Framework; using Utils; @@ -17,7 +18,7 @@ namespace Tests.DurabilityTests // There is 1 minute of time for the nodes to connect to each other. // (Should be easy, they're in the same pod.) - Time.Sleep(TimeSpan.FromMinutes(1)); + Time.Sleep(TimeSpan.FromMinutes(6)); bootstrapNode.BringOffline(); var file = GenerateTestFile(10.MB()); @@ -30,10 +31,10 @@ namespace Tests.DurabilityTests [Test] public void DataRetentionTest() { - var bootstrapNode = SetupCodexNodes(1).BringOnline()[0]; + var bootstrapNode = SetupCodexNodes(1).WithLogLevel(CodexLogLevel.Trace).BringOnline()[0]; - var startGroup = SetupCodexNodes(2).WithBootstrapNode(bootstrapNode).BringOnline(); - var finishGroup = SetupCodexNodes(10).WithBootstrapNode(bootstrapNode).BringOnline(); + var startGroup = SetupCodexNodes(2).WithLogLevel(CodexLogLevel.Trace).WithBootstrapNode(bootstrapNode).BringOnline(); + var finishGroup = SetupCodexNodes(10).WithLogLevel(CodexLogLevel.Trace).WithBootstrapNode(bootstrapNode).BringOnline(); var file = GenerateTestFile(10.MB());