Allows for nethereum interactions with companion nodes. Verifies that marketplace contract is available before proceeding with codex-node setup.

DistTestCore/Codex/CodexContainerRecipe.cs

@@ -21,6 +21,11 @@ namespace DistTestCore.Codex
             var listenPort = AddInternalPort();
             AddEnvVar("LISTEN_ADDRS", $"/ip4/0.0.0.0/tcp/{listenPort.Number}");
 
+            if (!string.IsNullOrEmpty(config.BootstrapSpr))
+            {
+                AddEnvVar("BOOTSTRAP_SPR", config.BootstrapSpr);
+            }
+
             if (config.LogLevel != null)
             {
                 AddEnvVar("LOG_LEVEL", config.LogLevel.ToString()!.ToUpperInvariant());

DistTestCore/Codex/CodexStartupConfig.cs

@@ -10,6 +10,6 @@ namespace DistTestCore.Codex
         public ByteSize? StorageQuota { get; set; }
         public bool MetricsEnabled { get; set; }
         public MarketplaceInitialConfig? MarketplaceConfig { get; set; }
-        public IOnlineCodexNode? BootstrapNode { get; set; }
+        public string? BootstrapSpr { get; set; }
     }
 }

DistTestCore/CodexSetup.cs

@@ -30,24 +30,7 @@ namespace DistTestCore
 
         public ICodexNodeGroup BringOnline()
         {
-            var group = starter.BringOnline(this);
+            return starter.BringOnline(this);
-            ConnectToBootstrapNode(group);
-            return group;
-        }
-
-        private void ConnectToBootstrapNode(ICodexNodeGroup group)
-        {
-            if (BootstrapNode == null) return;
-
-            // TODO:
-            // node.ConnectToPeer uses the '/api/codex/vi/connect/' endpoint to make the connection.
-            // This should be replaced by injecting the bootstrap node's SPR into the env-vars of the new node containers. (Easy!)
-            // However, NAT isn't figure out yet. So connecting with SPR doesn't (always?) work.
-            // So for now, ConnectToPeer
-            foreach (var node in group)
-            {
-                node.ConnectToPeer(BootstrapNode);
-            }
         }
 
         public ICodexSetup At(Location location)
@@ -58,7 +41,7 @@ namespace DistTestCore
 
         public ICodexSetup WithBootstrapNode(IOnlineCodexNode node)
         {
-            BootstrapNode = node;
+            BootstrapSpr = node.GetDebugInfo().spr;
             return this;
         }
 
@@ -100,7 +83,7 @@ namespace DistTestCore
         private IEnumerable<string> DescribeArgs()
         {
             if (LogLevel != null) yield return $"LogLevel={LogLevel}";
-            if (BootstrapNode != null) yield return $"BootstrapNode={BootstrapNode.GetName()}";
+            if (BootstrapSpr != null) yield return $"BootstrapNode={BootstrapSpr}";
             if (StorageQuota != null) yield return $"StorageQuote={StorageQuota}";
         }
     }

DistTestCore/GethStarter.cs

@@ -62,7 +62,7 @@ namespace DistTestCore
 
         private GethCompanionNodeInfo[] StartCompanionNodes(CodexSetup codexSetup, MarketplaceNetwork marketplaceNetwork)
         {
-            return companionNodeStarter.StartCompanionNodesFor(codexSetup, marketplaceNetwork.Bootstrap);
+            return companionNodeStarter.StartCompanionNodesFor(codexSetup, marketplaceNetwork);
         }
     }
 

DistTestCore/Marketplace/CodexContractsContainerRecipe.cs

@@ -6,6 +6,7 @@ namespace DistTestCore.Marketplace
     {
         public const string DockerImage = "thatbenbierens/codex-contracts-deployment";
         public const string MarketplaceAddressFilename = "/usr/app/deployments/codexdisttestnetwork/Marketplace.json";
+        public const string MarketplaceArtifactFilename = "/usr/app/artifacts/contracts/Marketplace.sol/Marketplace.json";
 
         protected override string Image => DockerImage;
 

DistTestCore/Marketplace/CodexContractsStarter.cs

@@ -32,13 +32,14 @@ namespace DistTestCore.Marketplace
 
             var extractor = new ContainerInfoExtractor(workflow, container);
             var marketplaceAddress = extractor.ExtractMarketplaceAddress();
+            var abi = extractor.ExtractMarketplaceAbi();
 
             var interaction = bootstrapNode.StartInteraction(lifecycle.Log);
             var tokenAddress = interaction.GetTokenAddress(marketplaceAddress);
 
             LogEnd("Contracts deployed.");
 
-            return new MarketplaceInfo(marketplaceAddress, tokenAddress);
+            return new MarketplaceInfo(marketplaceAddress, abi, tokenAddress);
         }
 
         private void WaitUntil(Func<bool> predicate)
@@ -57,13 +58,15 @@ namespace DistTestCore.Marketplace
 
     public class MarketplaceInfo
     {
-        public MarketplaceInfo(string address, string tokenAddress)
+        public MarketplaceInfo(string address, string abi, string tokenAddress)
         {
             Address = address;
+            Abi = abi;
             TokenAddress = tokenAddress;
         }
 
         public string Address { get; }
+        public string Abi { get; }
         public string TokenAddress { get; }
     }
 

DistTestCore/Marketplace/ContainerInfoExtractor.cs

@@ -1,5 +1,9 @@
-using KubernetesWorkflow;
+using IdentityModel.OidcClient;
+using KubernetesWorkflow;
 using Newtonsoft.Json;
+using Newtonsoft.Json.Converters;
+using Newtonsoft.Json.Linq;
+using Utils;
 
 namespace DistTestCore.Marketplace
 {
@@ -30,9 +34,9 @@ namespace DistTestCore.Marketplace
             return pubKey;
         }
 
-        public string ExtractBootstrapPrivateKey()
+        public string ExtractPrivateKey()
         {
-            var privKey = Retry(FetchBootstrapPrivateKey);
+            var privKey = Retry(FetchPrivateKey);
             if (string.IsNullOrEmpty(privKey)) throw new InvalidOperationException("Unable to fetch private key from geth node. Test infra failure.");
 
             return privKey;
@@ -46,14 +50,23 @@ namespace DistTestCore.Marketplace
             return marketplaceAddress;
         }
 
+        public string ExtractMarketplaceAbi()
+        {
+            var marketplaceAbi = Retry(FetchMarketplaceAbi);
+            if (string.IsNullOrEmpty(marketplaceAbi)) throw new InvalidOperationException("Unable to fetch marketplace artifacts from codex-contracts node. Test infra failure.");
+
+            return marketplaceAbi;
+        }
+
         private string Retry(Func<string> fetch)
         {
-            var result = Catch(fetch);
+            var result = string.Empty;
-            if (string.IsNullOrEmpty(result))
+            Time.WaitUntil(() =>
             {
-                Thread.Sleep(TimeSpan.FromSeconds(5));
+                result = Catch(fetch);
-                result = fetch();
+                return !string.IsNullOrEmpty(result);
-            }
+            }, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(3));
+
             return result;
         }
 
@@ -74,9 +87,9 @@ namespace DistTestCore.Marketplace
             return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.AccountFilename);
         }
 
-        private string FetchBootstrapPrivateKey()
+        private string FetchPrivateKey()
         {
-            return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.BootstrapPrivateKeyFilename);
+            return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.PrivateKeyFilename);
         }
 
         private string FetchMarketplaceAddress()
@@ -86,6 +99,15 @@ namespace DistTestCore.Marketplace
             return marketplace!.address;
         }
 
+        private string FetchMarketplaceAbi()
+        {
+            var json = workflow.ExecuteCommand(container, "cat", CodexContractsContainerRecipe.MarketplaceArtifactFilename);
+
+            var artifact = JObject.Parse(json);
+            var abi = artifact["abi"];
+            return abi!.ToString(Formatting.None);
+        }
+
         private string FetchPubKey()
         {
             var enodeFinder = new PubKeyFinder();

DistTestCore/Marketplace/GethBootstrapNodeStarter.cs

@@ -22,7 +22,7 @@ namespace DistTestCore.Marketplace
             var extractor = new ContainerInfoExtractor(workflow, bootstrapContainer);
             var account = extractor.ExtractAccount();
             var pubKey = extractor.ExtractPubKey();
-            var privateKey = extractor.ExtractBootstrapPrivateKey();
+            var privateKey = extractor.ExtractPrivateKey();
             var discoveryPort = bootstrapContainer.Recipe.GetPortByTag(GethContainerRecipe.DiscoveryPortTag);
 
             LogEnd($"Geth bootstrap node started with account '{account}'");

DistTestCore/Marketplace/GethCompanionNodeInfo.cs

@@ -1,16 +1,31 @@
 using KubernetesWorkflow;
+using Logging;
+using NethereumWorkflow;
 
 namespace DistTestCore.Marketplace
 {
     public class GethCompanionNodeInfo
     {
-        public GethCompanionNodeInfo(RunningContainer runningContainer, string account)
+        public GethCompanionNodeInfo(RunningContainer runningContainer, string account, string privateKey)
         {
             RunningContainer = runningContainer;
             Account = account;
+            PrivateKey = privateKey;
         }
 
         public RunningContainer RunningContainer { get; }
         public string Account { get; }
+        public string PrivateKey { get; }
+
+        public NethereumInteraction StartInteraction(TestLog log)
+        {
+            var ip = RunningContainer.Pod.Cluster.IP;
+            var port = RunningContainer.ServicePorts[0].Number;
+            var account = Account;
+            var privateKey = PrivateKey;
+
+            var creator = new NethereumInteractionCreator(log, ip, port, account, privateKey);
+            return creator.CreateWorkflow();
+        }
     }
 }

DistTestCore/Marketplace/GethCompanionNodeStarter.cs

@@ -9,11 +9,11 @@ namespace DistTestCore.Marketplace
         {
         }
 
-        public GethCompanionNodeInfo[] StartCompanionNodesFor(CodexSetup codexSetup, GethBootstrapNodeInfo bootstrapNode)
+        public GethCompanionNodeInfo[] StartCompanionNodesFor(CodexSetup codexSetup, MarketplaceNetwork marketplace)
         {
             LogStart($"Initializing companions for {codexSetup.NumberOfNodes} Codex nodes.");
 
-            var startupConfig = CreateCompanionNodeStartupConfig(bootstrapNode);
+            var startupConfig = CreateCompanionNodeStartupConfig(marketplace.Bootstrap);
 
             var workflow = workflowCreator.CreateWorkflow();
             var containers = workflow.Start(codexSetup.NumberOfNodes, Location.Unspecified, new GethContainerRecipe(), startupConfig);
@@ -21,6 +21,11 @@ namespace DistTestCore.Marketplace
 
             var result = containers.Containers.Select(c => CreateCompanionInfo(workflow, c)).ToArray();
 
+            foreach (var node in result)
+            {
+                EnsureCompanionNodeIsSynced(node, marketplace);
+            }
+
             LogEnd($"Initialized {codexSetup.NumberOfNodes} companion nodes. Their accounts: [{string.Join(",", result.Select(c => c.Account))}]");
 
             return result;
@@ -30,7 +35,14 @@ namespace DistTestCore.Marketplace
         {
             var extractor = new ContainerInfoExtractor(workflow, container);
             var account = extractor.ExtractAccount();
-            return new GethCompanionNodeInfo(container, account);
+            var privKey = extractor.ExtractPrivateKey();
+            return new GethCompanionNodeInfo(container, account, privKey);
+        }
+
+        private void EnsureCompanionNodeIsSynced(GethCompanionNodeInfo node, MarketplaceNetwork marketplace)
+        {
+            var interaction = node.StartInteraction(lifecycle.Log);
+            interaction.EnsureSynced(marketplace.Marketplace.Address, marketplace.Marketplace.Abi);
         }
 
         private StartupConfig CreateCompanionNodeStartupConfig(GethBootstrapNodeInfo bootstrapNode)

DistTestCore/Marketplace/GethContainerRecipe.cs

@@ -8,7 +8,7 @@ namespace DistTestCore.Marketplace
         public const string HttpPortTag = "http_port";
         public const string DiscoveryPortTag = "disc_port";
         public const string AccountFilename = "account_string.txt";
-        public const string BootstrapPrivateKeyFilename = "bootstrap_private.key";
+        public const string PrivateKeyFilename = "private.key";
 
         protected override string Image => DockerImage;
 
@@ -44,7 +44,7 @@ namespace DistTestCore.Marketplace
         {
             var port = AddInternalPort();
             var authRpc = AddInternalPort();
-            var httpPort = AddInternalPort(tag: HttpPortTag);
+            var httpPort = AddExposedPort(tag: HttpPortTag);
 
             var bootPubKey = config.BootstrapNode.PubKey;
             var bootIp = config.BootstrapNode.RunningContainers.Containers[0].Pod.Ip;

DistTestCore/OnlineCodexNode.cs

@@ -23,7 +23,6 @@ namespace DistTestCore
         private const string SuccessfullyConnectedMessage = "Successfully connected to peer";
         private const string UploadFailedMessage = "Unable to store block";
         private readonly TestLifecycle lifecycle;
-        private CodexDebugResponse? debugInfo;
 
         public OnlineCodexNode(TestLifecycle lifecycle, CodexAccess codexAccess, CodexNodeGroup group, IMetricsAccess metricsAccess, IMarketplaceAccess marketplaceAccess)
         {
@@ -46,9 +45,7 @@ namespace DistTestCore
 
         public CodexDebugResponse GetDebugInfo()
         {
-            if (debugInfo != null) return debugInfo;
+            var debugInfo = CodexAccess.GetDebugInfo();
-
-            debugInfo = CodexAccess.GetDebugInfo();
             Log($"Got DebugInfo with id: '{debugInfo.id}'.");
             return debugInfo;
         }

Nethereum/NethereumInteraction.cs

@@ -72,6 +72,28 @@ namespace NethereumWorkflow
             Task.WaitAll(tasks);
         }
 
+        public void EnsureSynced(string marketplaceAddress, string marketplaceAbi)
+        {
+            Time.WaitUntil(() =>
+            {
+                return !Time.Wait(web3.Eth.Syncing.SendRequestAsync()).IsSyncing;
+            }, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(1));
+
+
+            Time.WaitUntil(() =>
+            {
+                try
+                {
+                    var contract = web3.Eth.GetContract(marketplaceAbi, marketplaceAddress);
+                    return contract != null;
+                }
+                catch
+                {
+                    return false;
+                }
+            }, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(1));
+        }
+
         private HexBigInteger ToHexBig(decimal amount)
         {
             var bigint = ToBig(amount);
@@ -106,7 +128,7 @@ namespace NethereumWorkflow
     }
 
     [Function("balanceOf", "uint256")]
-    public class GetTokenBalanceFunction :FunctionMessage
+    public class GetTokenBalanceFunction : FunctionMessage
     {
         [Parameter("address", "owner", 1)]
         public string Owner { get; set; }

Tests/BasicTests/ExampleTests.cs

@@ -51,6 +51,7 @@ namespace Tests.BasicTests
         public void MarketplaceExample()
         {
             var primary = SetupCodexNodes(1)
+                            .WithLogLevel(CodexLogLevel.Trace)
                             .WithStorageQuota(11.GB())
                             .EnableMarketplace(initialBalance: 234.TestTokens())
                             .BringOnline()[0];
@@ -58,11 +59,11 @@ namespace Tests.BasicTests
             primary.Marketplace.AssertThatBalance(Is.EqualTo(234.TestTokens()));
 
             var secondary = SetupCodexNodes(1)
+                            .WithLogLevel(CodexLogLevel.Trace)
+                            .WithBootstrapNode(primary)
                             .EnableMarketplace(initialBalance: 1000.TestTokens())
                             .BringOnline()[0];
 
-            primary.ConnectToPeer(secondary);
-
             primary.Marketplace.MakeStorageAvailable(
                 size: 10.GB(),
                 minPricePerBytePerSecond: 1.TestTokens(),

Tests/DurabilityTests/DurabilityTests.cs

@@ -1,4 +1,5 @@
 using DistTestCore;
+using DistTestCore.Codex;
 using NUnit.Framework;
 using Utils;
 
@@ -17,7 +18,7 @@ namespace Tests.DurabilityTests
 
             // There is 1 minute of time for the nodes to connect to each other.
             // (Should be easy, they're in the same pod.)
-            Time.Sleep(TimeSpan.FromMinutes(1));
+            Time.Sleep(TimeSpan.FromMinutes(6));
             bootstrapNode.BringOffline();
 
             var file = GenerateTestFile(10.MB());
@@ -30,10 +31,10 @@ namespace Tests.DurabilityTests
         [Test]
         public void DataRetentionTest()
         {
-            var bootstrapNode = SetupCodexNodes(1).BringOnline()[0];
+            var bootstrapNode = SetupCodexNodes(1).WithLogLevel(CodexLogLevel.Trace).BringOnline()[0];
 
-            var startGroup = SetupCodexNodes(2).WithBootstrapNode(bootstrapNode).BringOnline();
+            var startGroup = SetupCodexNodes(2).WithLogLevel(CodexLogLevel.Trace).WithBootstrapNode(bootstrapNode).BringOnline();
-            var finishGroup = SetupCodexNodes(10).WithBootstrapNode(bootstrapNode).BringOnline();
+            var finishGroup = SetupCodexNodes(10).WithLogLevel(CodexLogLevel.Trace).WithBootstrapNode(bootstrapNode).BringOnline();
 
             var file = GenerateTestFile(10.MB());