Supports multiple codex nodes from a single geth companion node.

DistTestCore/Codex/CodexContainerRecipe.cs

@@ -5,7 +5,8 @@ namespace DistTestCore.Codex
 {
     public class CodexContainerRecipe : ContainerRecipeFactory
     {
-        public const string DockerImage = "thatbenbierens/nim-codex:sha-bf5512b";
+        //public const string DockerImage = "thatbenbierens/nim-codex:sha-bf5512b";
+        public const string DockerImage = "thatbenbierens/codexlocal:latest";
         public const string MetricsPortTag = "metrics_port";
 
         protected override string Image => DockerImage;
@@ -43,14 +44,15 @@ namespace DistTestCore.Codex
             if (config.MarketplaceConfig != null)
             {
                 var gethConfig = startupConfig.Get<GethStartResult>();
-                var companionNode = gethConfig.CompanionNodes[Index];
+                var companionNode = gethConfig.CompanionNode;
-                Additional(companionNode);
+                var companionNodeAccount = companionNode.Accounts[Index];
+                Additional(companionNodeAccount);
 
                 var ip = companionNode.RunningContainer.Pod.Ip;
                 var port = companionNode.RunningContainer.Recipe.GetPortByTag(GethContainerRecipe.HttpPortTag).Number;
 
                 AddEnvVar("ETH_PROVIDER", $"ws://{ip}:{port}");
-                AddEnvVar("ETH_ACCOUNT", companionNode.Account);
+                AddEnvVar("ETH_ACCOUNT", companionNodeAccount.Account);
                 AddEnvVar("ETH_MARKETPLACE_ADDRESS", gethConfig.MarketplaceNetwork.Marketplace.Address);
             }
         }

DistTestCore/GethStarter.cs

@@ -22,37 +22,37 @@ namespace DistTestCore
             if (codexSetup.MarketplaceConfig == null) return CreateMarketplaceUnavailableResult();
 
             var marketplaceNetwork = marketplaceNetworkCache.Get();
-            var companionNodes = StartCompanionNodes(codexSetup, marketplaceNetwork);
+            var companionNode = StartCompanionNode(codexSetup, marketplaceNetwork);
 
             LogStart("Setting up initial balance...");
-            TransferInitialBalance(marketplaceNetwork, codexSetup.MarketplaceConfig, companionNodes);
+            TransferInitialBalance(marketplaceNetwork, codexSetup.MarketplaceConfig, companionNode);
             LogEnd($"Initial balance of {codexSetup.MarketplaceConfig.InitialTestTokens} set for {codexSetup.NumberOfNodes} nodes.");
 
-            return CreateGethStartResult(marketplaceNetwork, companionNodes);
+            return CreateGethStartResult(marketplaceNetwork, companionNode);
         }
 
-        private void TransferInitialBalance(MarketplaceNetwork marketplaceNetwork, MarketplaceInitialConfig marketplaceConfig, GethCompanionNodeInfo[] companionNodes)
+        private void TransferInitialBalance(MarketplaceNetwork marketplaceNetwork, MarketplaceInitialConfig marketplaceConfig, GethCompanionNodeInfo companionNode)
         {
             var interaction = marketplaceNetwork.StartInteraction(lifecycle.Log);
             var tokenAddress = marketplaceNetwork.Marketplace.TokenAddress;
 
-            foreach (var node in companionNodes)
+            foreach (var account in companionNode.Accounts)
             {
-                interaction.TransferWeiTo(node.Account, marketplaceConfig.InitialEth.Wei);
+                interaction.TransferWeiTo(account.Account, marketplaceConfig.InitialEth.Wei);
-                interaction.MintTestTokens(node.Account, marketplaceConfig.InitialTestTokens.Amount, tokenAddress);
+                interaction.MintTestTokens(account.Account, marketplaceConfig.InitialTestTokens.Amount, tokenAddress);
             }
 
             interaction.WaitForAllTransactions();
         }
 
-        private GethStartResult CreateGethStartResult(MarketplaceNetwork marketplaceNetwork, GethCompanionNodeInfo[] companionNodes)
+        private GethStartResult CreateGethStartResult(MarketplaceNetwork marketplaceNetwork, GethCompanionNodeInfo companionNode)
         {
-            return new GethStartResult(CreateMarketplaceAccessFactory(marketplaceNetwork), marketplaceNetwork, companionNodes);
+            return new GethStartResult(CreateMarketplaceAccessFactory(marketplaceNetwork), marketplaceNetwork, companionNode);
         }
 
         private GethStartResult CreateMarketplaceUnavailableResult()
         {
-            return new GethStartResult(new MarketplaceUnavailableAccessFactory(), null!, Array.Empty<GethCompanionNodeInfo>());
+            return new GethStartResult(new MarketplaceUnavailableAccessFactory(), null!, null!);
         }
 
         private IMarketplaceAccessFactory CreateMarketplaceAccessFactory(MarketplaceNetwork marketplaceNetwork)
@@ -60,9 +60,9 @@ namespace DistTestCore
             return new GethMarketplaceAccessFactory(lifecycle.Log, marketplaceNetwork);
         }
 
-        private GethCompanionNodeInfo[] StartCompanionNodes(CodexSetup codexSetup, MarketplaceNetwork marketplaceNetwork)
+        private GethCompanionNodeInfo StartCompanionNode(CodexSetup codexSetup, MarketplaceNetwork marketplaceNetwork)
         {
-            return companionNodeStarter.StartCompanionNodesFor(codexSetup, marketplaceNetwork);
+            return companionNodeStarter.StartCompanionNodeFor(codexSetup, marketplaceNetwork);
         }
     }
 

DistTestCore/Marketplace/ContainerInfoExtractor.cs

@@ -19,10 +19,10 @@ namespace DistTestCore.Marketplace
             this.container = container;
         }
 
-        public string ExtractAccount()
+        public string ExtractAccount(int? orderNumber)
         {
             log.Debug();
-            var account = Retry(FetchAccount);
+            var account = Retry(() => FetchAccount(orderNumber));
             if (string.IsNullOrEmpty(account)) throw new InvalidOperationException("Unable to fetch account for geth node. Test infra failure.");
 
             return account;
@@ -37,10 +37,10 @@ namespace DistTestCore.Marketplace
             return pubKey;
         }
 
-        public string ExtractPrivateKey()
+        public string ExtractPrivateKey(int? orderNumber)
         {
             log.Debug();
-            var privKey = Retry(FetchPrivateKey);
+            var privKey = Retry(() => FetchPrivateKey(orderNumber));
             if (string.IsNullOrEmpty(privKey)) throw new InvalidOperationException("Unable to fetch private key from geth node. Test infra failure.");
 
             return privKey;
@@ -88,14 +88,14 @@ namespace DistTestCore.Marketplace
             }
         }
 
-        private string FetchAccount()
+        private string FetchAccount(int? orderNumber)
         {
-            return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.AccountFilename);
+            return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.GetAccountFilename(orderNumber));
         }
 
-        private string FetchPrivateKey()
+        private string FetchPrivateKey(int? orderNumber)
         {
-            return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.PrivateKeyFilename);
+            return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.GetPrivateKeyFilename(orderNumber));
         }
 
         private string FetchMarketplaceAddress()

DistTestCore/Marketplace/GethBootstrapNodeStarter.cs

@@ -20,9 +20,9 @@ namespace DistTestCore.Marketplace
             var bootstrapContainer = containers.Containers[0];
 
             var extractor = new ContainerInfoExtractor(lifecycle.Log, workflow, bootstrapContainer);
-            var account = extractor.ExtractAccount();
+            var account = extractor.ExtractAccount(null);
             var pubKey = extractor.ExtractPubKey();
-            var privateKey = extractor.ExtractPrivateKey();
+            var privateKey = extractor.ExtractPrivateKey(null);
             var discoveryPort = bootstrapContainer.Recipe.GetPortByTag(GethContainerRecipe.DiscoveryPortTag);
 
             LogEnd($"Geth bootstrap node started with account '{account}'");
@@ -33,7 +33,7 @@ namespace DistTestCore.Marketplace
         private StartupConfig CreateBootstrapStartupConfig()
         {
             var config = new StartupConfig();
-            config.Add(new GethStartupConfig(true, null!));
+            config.Add(new GethStartupConfig(true, null!, 0));
             return config;
         }
     }

DistTestCore/Marketplace/GethCompanionNodeInfo.cs

@@ -6,26 +6,36 @@ namespace DistTestCore.Marketplace
 {
     public class GethCompanionNodeInfo
     {
-        public GethCompanionNodeInfo(RunningContainer runningContainer, string account, string privateKey)
+        public GethCompanionNodeInfo(RunningContainer runningContainer, GethCompanionAccount[] accounts)
         {
             RunningContainer = runningContainer;
+            Accounts = accounts;
+        }
+
+        public RunningContainer RunningContainer { get; }
+        public GethCompanionAccount[] Accounts { get; }
+        
+        public NethereumInteraction StartInteraction(BaseLog log, GethCompanionAccount account)
+        {
+            var ip = RunningContainer.Pod.Cluster.IP;
+            var port = RunningContainer.ServicePorts[0].Number;
+            var accountStr = account.Account;
+            var privateKey = account.PrivateKey;
+
+            var creator = new NethereumInteractionCreator(log, ip, port, accountStr, privateKey);
+            return creator.CreateWorkflow();
+        }
+    }
+
+    public class GethCompanionAccount
+    {
+        public GethCompanionAccount(string account, string privateKey)
+        {
             Account = account;
             PrivateKey = privateKey;
         }
 
-        public RunningContainer RunningContainer { get; }
         public string Account { get; }
         public string PrivateKey { get; }
-
-        public NethereumInteraction StartInteraction(BaseLog log)
-        {
-            var ip = RunningContainer.Pod.Cluster.IP;
-            var port = RunningContainer.ServicePorts[0].Number;
-            var account = Account;
-            var privateKey = PrivateKey;
-
-            var creator = new NethereumInteractionCreator(log, ip, port, account, privateKey);
-            return creator.CreateWorkflow();
-        }
     }
 }

DistTestCore/Marketplace/GethCompanionNodeStarter.cs

@@ -1,4 +1,5 @@
 using KubernetesWorkflow;
+using Utils;
 
 namespace DistTestCore.Marketplace
 {
@@ -9,41 +10,56 @@ namespace DistTestCore.Marketplace
         {
         }
 
-        public GethCompanionNodeInfo[] StartCompanionNodesFor(CodexSetup codexSetup, MarketplaceNetwork marketplace)
+        public GethCompanionNodeInfo StartCompanionNodeFor(CodexSetup codexSetup, MarketplaceNetwork marketplace)
         {
-            LogStart($"Initializing companions for {codexSetup.NumberOfNodes} Codex nodes.");
+            LogStart($"Initializing companion for {codexSetup.NumberOfNodes} Codex nodes.");
 
-            var startupConfig = CreateCompanionNodeStartupConfig(marketplace.Bootstrap);
+            var startupConfig = CreateCompanionNodeStartupConfig(marketplace.Bootstrap, codexSetup.NumberOfNodes);
 
             var workflow = workflowCreator.CreateWorkflow();
-            var containers = workflow.Start(codexSetup.NumberOfNodes, Location.Unspecified, new GethContainerRecipe(), startupConfig);
+            var containers = workflow.Start(1, Location.Unspecified, new GethContainerRecipe(), startupConfig);
-            if (containers.Containers.Length != codexSetup.NumberOfNodes) throw new InvalidOperationException("Expected a Geth companion node to be created for each Codex node. Test infra failure.");
+            WaitForAccountCreation(codexSetup.NumberOfNodes);
+            if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected one Geth companion node to be created. Test infra failure.");
+            var container = containers.Containers[0];
 
-            var result = containers.Containers.Select(c => CreateCompanionInfo(workflow, c)).ToArray();
+            var node = CreateCompanionInfo(workflow, container, codexSetup.NumberOfNodes);
+            EnsureCompanionNodeIsSynced(node, marketplace);
 
-            foreach (var node in result)
+            LogEnd($"Initialized one companion node for {codexSetup.NumberOfNodes} Codex nodes. Their accounts: [{string.Join(",", node.Accounts.Select(a => a.Account))}]");
-            {
+            return node;
-                EnsureCompanionNodeIsSynced(node, marketplace);
-            }
-
-            LogEnd($"Initialized {codexSetup.NumberOfNodes} companion nodes. Their accounts: [{string.Join(",", result.Select(c => c.Account))}]");
-
-            return result;
         }
 
-        private GethCompanionNodeInfo CreateCompanionInfo(StartupWorkflow workflow, RunningContainer container)
+        private void WaitForAccountCreation(int numberOfNodes)
+        {
+            // We wait proportional to the number of account the node has to create. It takes a few seconds for each one to generate the keys and create the files
+            // we will be trying to read in 'ExtractAccount', later on in the start-up process.
+            Time.Sleep(TimeSpan.FromSeconds(4.5 * numberOfNodes));
+        }
+
+        private GethCompanionNodeInfo CreateCompanionInfo(StartupWorkflow workflow, RunningContainer container, int numberOfAccounts)
         {
             var extractor = new ContainerInfoExtractor(lifecycle.Log, workflow, container);
-            var account = extractor.ExtractAccount();
+            var accounts = ExtractAccounts(extractor, numberOfAccounts).ToArray();
-            var privKey = extractor.ExtractPrivateKey();
+            return new GethCompanionNodeInfo(container, accounts);
-            return new GethCompanionNodeInfo(container, account, privKey);
+        }
+
+        private IEnumerable<GethCompanionAccount> ExtractAccounts(ContainerInfoExtractor extractor, int numberOfAccounts)
+        {
+            for (int i = 0; i < numberOfAccounts; i++) yield return ExtractAccount(extractor, i + 1);
+        }
+
+        private GethCompanionAccount ExtractAccount(ContainerInfoExtractor extractor, int orderNumber)
+        {
+            var account = extractor.ExtractAccount(orderNumber);
+            var privKey = extractor.ExtractPrivateKey(orderNumber);
+            return new GethCompanionAccount(account, privKey);
         }
 
         private void EnsureCompanionNodeIsSynced(GethCompanionNodeInfo node, MarketplaceNetwork marketplace)
         {
             try
             {
-                var interaction = node.StartInteraction(lifecycle.Log);
+                var interaction = node.StartInteraction(lifecycle.Log, node.Accounts.First());
                 interaction.EnsureSynced(marketplace.Marketplace.Address, marketplace.Marketplace.Abi);
             }
             catch (Exception e)
@@ -52,10 +68,10 @@ namespace DistTestCore.Marketplace
             }
         }
 
-        private StartupConfig CreateCompanionNodeStartupConfig(GethBootstrapNodeInfo bootstrapNode)
+        private StartupConfig CreateCompanionNodeStartupConfig(GethBootstrapNodeInfo bootstrapNode, int numberOfAccounts)
         {
             var config = new StartupConfig();
-            config.Add(new GethStartupConfig(false, bootstrapNode));
+            config.Add(new GethStartupConfig(false, bootstrapNode, numberOfAccounts));
             return config;
         }
     }

DistTestCore/Marketplace/GethContainerRecipe.cs

@@ -7,10 +7,20 @@ namespace DistTestCore.Marketplace
         public const string DockerImage = "thatbenbierens/geth-confenv:latest";
         public const string HttpPortTag = "http_port";
         public const string DiscoveryPortTag = "disc_port";
-        public const string AccountFilename = "account_string.txt";
-        public const string PrivateKeyFilename = "private.key";
         private const string defaultArgs = "--ipcdisable --syncmode full";
 
+        public static string GetAccountFilename(int? orderNumber)
+        {
+            if (orderNumber == null) return "account_string.txt";
+            return $"account_string_{orderNumber.Value}.txt";
+        }
+
+        public static string GetPrivateKeyFilename(int? orderNumber)
+        {
+            if (orderNumber == null) return "private.key";
+            return $"private_{orderNumber.Value}.key";
+        }
+
         protected override string Image => DockerImage;
 
         protected override void Initialize(StartupConfig startupConfig)
@@ -43,6 +53,8 @@ namespace DistTestCore.Marketplace
 
         private string CreateCompanionArgs(Port discovery, GethStartupConfig config)
         {
+            AddEnvVar("NUMBER_OF_ACCOUNTS", config.NumberOfCompanionAccounts.ToString());
+
             var port = AddInternalPort();
             var authRpc = AddInternalPort();
             var httpPort = AddExposedPort(tag: HttpPortTag);

DistTestCore/Marketplace/GethStartResult.cs

@@ -2,15 +2,15 @@
 {
     public class GethStartResult
     {
-        public GethStartResult(IMarketplaceAccessFactory marketplaceAccessFactory, MarketplaceNetwork marketplaceNetwork, GethCompanionNodeInfo[] companionNodes)
+        public GethStartResult(IMarketplaceAccessFactory marketplaceAccessFactory, MarketplaceNetwork marketplaceNetwork, GethCompanionNodeInfo companionNode)
         {
             MarketplaceAccessFactory = marketplaceAccessFactory;
             MarketplaceNetwork = marketplaceNetwork;
-            CompanionNodes = companionNodes;
+            CompanionNode = companionNode;
         }
 
         public IMarketplaceAccessFactory MarketplaceAccessFactory { get; }
         public MarketplaceNetwork MarketplaceNetwork { get; }
-        public GethCompanionNodeInfo[] CompanionNodes { get; }
+        public GethCompanionNodeInfo CompanionNode { get; }
     }
 }

DistTestCore/Marketplace/GethStartupConfig.cs

@@ -2,13 +2,15 @@
 {
     public class GethStartupConfig
     {
-        public GethStartupConfig(bool isBootstrapNode, GethBootstrapNodeInfo bootstrapNode)
+        public GethStartupConfig(bool isBootstrapNode, GethBootstrapNodeInfo bootstrapNode, int numberOfCompanionAccounts)
         {
             IsBootstrapNode = isBootstrapNode;
             BootstrapNode = bootstrapNode;
+            NumberOfCompanionAccounts = numberOfCompanionAccounts;
         }
 
         public bool IsBootstrapNode { get; }
         public GethBootstrapNodeInfo BootstrapNode { get; }
+        public int NumberOfCompanionAccounts { get; }
     }
 }

DistTestCore/Marketplace/MarketplaceAccess.cs

@@ -19,14 +19,14 @@ namespace DistTestCore.Marketplace
     {
         private readonly TestLog log;
         private readonly MarketplaceNetwork marketplaceNetwork;
-        private readonly GethCompanionNodeInfo companionNode;
+        private readonly GethCompanionAccount account;
         private readonly CodexAccess codexAccess;
 
-        public MarketplaceAccess(TestLog log, MarketplaceNetwork marketplaceNetwork, GethCompanionNodeInfo companionNode, CodexAccess codexAccess)
+        public MarketplaceAccess(TestLog log, MarketplaceNetwork marketplaceNetwork, GethCompanionAccount account, CodexAccess codexAccess)
         {
             this.log = log;
             this.marketplaceNetwork = marketplaceNetwork;
-            this.companionNode = companionNode;
+            this.account = account;
             this.codexAccess = codexAccess;
         }
 
@@ -104,11 +104,10 @@ namespace DistTestCore.Marketplace
         public TestToken GetBalance()
         {
             var interaction = marketplaceNetwork.StartInteraction(log);
-            var account = companionNode.Account;
+            var amount = interaction.GetBalance(marketplaceNetwork.Marketplace.TokenAddress, account.Account);
-            var amount = interaction.GetBalance(marketplaceNetwork.Marketplace.TokenAddress, account);
             var balance = new TestToken(amount);
 
-            Log($"Balance of {account} is {balance}.");
+            Log($"Balance of {account.Account} is {balance}.");
 
             return balance;
         }

DistTestCore/Marketplace/MarketplaceAccessFactory.cs

@@ -33,10 +33,10 @@ namespace DistTestCore.Marketplace
             return new MarketplaceAccess(log, marketplaceNetwork, companionNode, access);
         }
 
-        private GethCompanionNodeInfo GetGethCompanionNode(CodexAccess access)
+        private GethCompanionAccount GetGethCompanionNode(CodexAccess access)
         {
-            var node = access.Container.Recipe.Additionals.Single(a => a is GethCompanionNodeInfo);
+            var account = access.Container.Recipe.Additionals.Single(a => a is GethCompanionAccount);
-            return (GethCompanionNodeInfo)node;
+            return (GethCompanionAccount)account;
         }
     }
 }

Nethereum/NethereumInteraction.cs

@@ -92,7 +92,7 @@ namespace NethereumWorkflow
                 var numberOfBlocks = ToDecimal(number);
                 return !sync.IsSyncing && numberOfBlocks > 256;
 
-            }, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(1));
+            }, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(3));
         }
 
         private void WaitForContract(string marketplaceAddress, string marketplaceAbi)
@@ -109,7 +109,7 @@ namespace NethereumWorkflow
                 {
                     return false;
                 }
-            }, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(1));
+            }, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(3));
         }
 
         private HexBigInteger ToHexBig(decimal amount)

Tests/BasicTests/ExampleTests.cs

@@ -43,28 +43,30 @@ namespace Tests.BasicTests
         [Test]
         public void MarketplaceExample()
         {
-            var primary = SetupCodexNode(s => s
+            var sellerInitialBalance = 234.TestTokens();
+            var buyerInitialBalance = 1000.TestTokens();
+
+            var seller = SetupCodexNode(s => s
                             .WithLogLevel(CodexLogLevel.Trace)
                             .WithStorageQuota(11.GB())
-                            .EnableMarketplace(initialBalance: 234.TestTokens()));
+                            .EnableMarketplace(sellerInitialBalance));
 
-            primary.Marketplace.AssertThatBalance(Is.EqualTo(234.TestTokens()));
+            seller.Marketplace.AssertThatBalance(Is.EqualTo(sellerInitialBalance));
-
+            seller.Marketplace.MakeStorageAvailable(
-            var secondary = SetupCodexNode(s => s
-                            .WithLogLevel(CodexLogLevel.Trace)
-                            .WithBootstrapNode(primary)
-                            .EnableMarketplace(initialBalance: 1000.TestTokens()));
-
-            primary.Marketplace.MakeStorageAvailable(
                 size: 10.GB(),
                 minPricePerBytePerSecond: 1.TestTokens(),
                 maxCollateral: 20.TestTokens(),
                 maxDuration: TimeSpan.FromMinutes(3));
 
             var testFile = GenerateTestFile(10.MB());
-            var contentId = secondary.UploadFile(testFile);
 
-            secondary.Marketplace.RequestStorage(contentId,
+            var buyer = SetupCodexNode(s => s
+                .WithLogLevel(CodexLogLevel.Trace)
+                .WithBootstrapNode(seller)
+                .EnableMarketplace(buyerInitialBalance));
+            
+            var contentId = buyer.UploadFile(testFile);
+            buyer.Marketplace.RequestStorage(contentId,
                 pricePerBytePerSecond: 2.TestTokens(),
                 requiredCollateral: 10.TestTokens(),
                 minRequiredNumberOfNodes: 1,
@@ -73,12 +75,12 @@ namespace Tests.BasicTests
 
             Time.Sleep(TimeSpan.FromMinutes(1));
 
-            primary.Marketplace.AssertThatBalance(Is.LessThan(234.TestTokens()), "Collateral was not placed.");
+            seller.Marketplace.AssertThatBalance(Is.LessThan(sellerInitialBalance), "Collateral was not placed.");
 
             Time.Sleep(TimeSpan.FromMinutes(2));
 
-            primary.Marketplace.AssertThatBalance(Is.GreaterThan(234.TestTokens()), "Storer was not paid for storage.");
+            seller.Marketplace.AssertThatBalance(Is.GreaterThan(sellerInitialBalance), "Seller was not paid for storage.");
-            secondary.Marketplace.AssertThatBalance(Is.LessThan(1000.TestTokens()), "Contractor was not charged for storage.");
+            buyer.Marketplace.AssertThatBalance(Is.LessThan(buyerInitialBalance), "Buyer was not charged for storage.");
         }
     }
 }