status-im
/
cs-codex-dist-tests
mirror of
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Supports multiple codex nodes from a single geth companion node.

This commit is contained in:
benbierens 2023-04-26 11:12:33 +02:00
parent ff68b8f0f1
commit ef546a435b
No known key found for this signature in database
GPG Key ID: FE44815D96D0A1AA
13 changed files with 136 additions and 93 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
DistTestCore/Codex/CodexContainerRecipe.cs
View File

@ -5,7 +5,8 @@ namespace DistTestCore.Codex
{
public class CodexContainerRecipe : ContainerRecipeFactory
{
public const string DockerImage = "thatbenbierens/nim-codex:sha-bf5512b";
//public const string DockerImage = "thatbenbierens/nim-codex:sha-bf5512b";
public const string DockerImage = "thatbenbierens/codexlocal:latest";
public const string MetricsPortTag = "metrics_port";
protected override string Image => DockerImage;
@ -43,14 +44,15 @@ namespace DistTestCore.Codex
if (config.MarketplaceConfig != null)
{
var gethConfig = startupConfig.Get<GethStartResult>();
var companionNode = gethConfig.CompanionNodes[Index];
Additional(companionNode);
var companionNode = gethConfig.CompanionNode;
var companionNodeAccount = companionNode.Accounts[Index];
Additional(companionNodeAccount);
var ip = companionNode.RunningContainer.Pod.Ip;
var port = companionNode.RunningContainer.Recipe.GetPortByTag(GethContainerRecipe.HttpPortTag).Number;
AddEnvVar("ETH_PROVIDER", $"ws://{ip}:{port}");
AddEnvVar("ETH_ACCOUNT", companionNode.Account);
AddEnvVar("ETH_ACCOUNT", companionNodeAccount.Account);
AddEnvVar("ETH_MARKETPLACE_ADDRESS", gethConfig.MarketplaceNetwork.Marketplace.Address);
}
}

24
DistTestCore/GethStarter.cs
View File

@ -22,37 +22,37 @@ namespace DistTestCore
if (codexSetup.MarketplaceConfig == null) return CreateMarketplaceUnavailableResult();
var marketplaceNetwork = marketplaceNetworkCache.Get();
var companionNodes = StartCompanionNodes(codexSetup, marketplaceNetwork);
var companionNode = StartCompanionNode(codexSetup, marketplaceNetwork);
LogStart("Setting up initial balance...");
TransferInitialBalance(marketplaceNetwork, codexSetup.MarketplaceConfig, companionNodes);
TransferInitialBalance(marketplaceNetwork, codexSetup.MarketplaceConfig, companionNode);
LogEnd($"Initial balance of {codexSetup.MarketplaceConfig.InitialTestTokens} set for {codexSetup.NumberOfNodes} nodes.");
return CreateGethStartResult(marketplaceNetwork, companionNodes);
return CreateGethStartResult(marketplaceNetwork, companionNode);
}
private void TransferInitialBalance(MarketplaceNetwork marketplaceNetwork, MarketplaceInitialConfig marketplaceConfig, GethCompanionNodeInfo[] companionNodes)
private void TransferInitialBalance(MarketplaceNetwork marketplaceNetwork, MarketplaceInitialConfig marketplaceConfig, GethCompanionNodeInfo companionNode)
{
var interaction = marketplaceNetwork.StartInteraction(lifecycle.Log);
var tokenAddress = marketplaceNetwork.Marketplace.TokenAddress;
foreach (var node in companionNodes)
foreach (var account in companionNode.Accounts)
{
interaction.TransferWeiTo(node.Account, marketplaceConfig.InitialEth.Wei);
interaction.MintTestTokens(node.Account, marketplaceConfig.InitialTestTokens.Amount, tokenAddress);
interaction.TransferWeiTo(account.Account, marketplaceConfig.InitialEth.Wei);
interaction.MintTestTokens(account.Account, marketplaceConfig.InitialTestTokens.Amount, tokenAddress);
}
interaction.WaitForAllTransactions();
}
private GethStartResult CreateGethStartResult(MarketplaceNetwork marketplaceNetwork, GethCompanionNodeInfo[] companionNodes)
private GethStartResult CreateGethStartResult(MarketplaceNetwork marketplaceNetwork, GethCompanionNodeInfo companionNode)
{
return new GethStartResult(CreateMarketplaceAccessFactory(marketplaceNetwork), marketplaceNetwork, companionNodes);
return new GethStartResult(CreateMarketplaceAccessFactory(marketplaceNetwork), marketplaceNetwork, companionNode);
}
private GethStartResult CreateMarketplaceUnavailableResult()
{
return new GethStartResult(new MarketplaceUnavailableAccessFactory(), null!, Array.Empty<GethCompanionNodeInfo>());
return new GethStartResult(new MarketplaceUnavailableAccessFactory(), null!, null!);
}
private IMarketplaceAccessFactory CreateMarketplaceAccessFactory(MarketplaceNetwork marketplaceNetwork)
@ -60,9 +60,9 @@ namespace DistTestCore
return new GethMarketplaceAccessFactory(lifecycle.Log, marketplaceNetwork);
}
private GethCompanionNodeInfo[] StartCompanionNodes(CodexSetup codexSetup, MarketplaceNetwork marketplaceNetwork)
private GethCompanionNodeInfo StartCompanionNode(CodexSetup codexSetup, MarketplaceNetwork marketplaceNetwork)
{
return companionNodeStarter.StartCompanionNodesFor(codexSetup, marketplaceNetwork);
return companionNodeStarter.StartCompanionNodeFor(codexSetup, marketplaceNetwork);
}
}

16
DistTestCore/Marketplace/ContainerInfoExtractor.cs
View File

@ -19,10 +19,10 @@ namespace DistTestCore.Marketplace
this.container = container;
}
public string ExtractAccount()
public string ExtractAccount(int? orderNumber)
{
log.Debug();
var account = Retry(FetchAccount);
var account = Retry(() => FetchAccount(orderNumber));
if (string.IsNullOrEmpty(account)) throw new InvalidOperationException("Unable to fetch account for geth node. Test infra failure.");
return account;
@ -37,10 +37,10 @@ namespace DistTestCore.Marketplace
return pubKey;
}
public string ExtractPrivateKey()
public string ExtractPrivateKey(int? orderNumber)
{
log.Debug();
var privKey = Retry(FetchPrivateKey);
var privKey = Retry(() => FetchPrivateKey(orderNumber));
if (string.IsNullOrEmpty(privKey)) throw new InvalidOperationException("Unable to fetch private key from geth node. Test infra failure.");
return privKey;
@ -88,14 +88,14 @@ namespace DistTestCore.Marketplace
}
}
private string FetchAccount()
private string FetchAccount(int? orderNumber)
{
return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.AccountFilename);
return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.GetAccountFilename(orderNumber));
}
private string FetchPrivateKey()
private string FetchPrivateKey(int? orderNumber)
{
return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.PrivateKeyFilename);
return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.GetPrivateKeyFilename(orderNumber));
}
private string FetchMarketplaceAddress()

6
DistTestCore/Marketplace/GethBootstrapNodeStarter.cs
View File

@ -20,9 +20,9 @@ namespace DistTestCore.Marketplace
var bootstrapContainer = containers.Containers[0];
var extractor = new ContainerInfoExtractor(lifecycle.Log, workflow, bootstrapContainer);
var account = extractor.ExtractAccount();
var account = extractor.ExtractAccount(null);
var pubKey = extractor.ExtractPubKey();
var privateKey = extractor.ExtractPrivateKey();
var privateKey = extractor.ExtractPrivateKey(null);
var discoveryPort = bootstrapContainer.Recipe.GetPortByTag(GethContainerRecipe.DiscoveryPortTag);
LogEnd($"Geth bootstrap node started with account '{account}'");
@ -33,7 +33,7 @@ namespace DistTestCore.Marketplace
private StartupConfig CreateBootstrapStartupConfig()
{
var config = new StartupConfig();
config.Add(new GethStartupConfig(true, null!));
config.Add(new GethStartupConfig(true, null!, 0));
return config;
}
}

36
DistTestCore/Marketplace/GethCompanionNodeInfo.cs
View File

@ -6,26 +6,36 @@ namespace DistTestCore.Marketplace
{
public class GethCompanionNodeInfo
{
public GethCompanionNodeInfo(RunningContainer runningContainer, string account, string privateKey)
public GethCompanionNodeInfo(RunningContainer runningContainer, GethCompanionAccount[] accounts)
{
RunningContainer = runningContainer;
Accounts = accounts;
}
public RunningContainer RunningContainer { get; }
public GethCompanionAccount[] Accounts { get; }
public NethereumInteraction StartInteraction(BaseLog log, GethCompanionAccount account)
{
var ip = RunningContainer.Pod.Cluster.IP;
var port = RunningContainer.ServicePorts[0].Number;
var accountStr = account.Account;
var privateKey = account.PrivateKey;
var creator = new NethereumInteractionCreator(log, ip, port, accountStr, privateKey);
return creator.CreateWorkflow();
}
}
public class GethCompanionAccount
{
public GethCompanionAccount(string account, string privateKey)
{
Account = account;
PrivateKey = privateKey;
}
public RunningContainer RunningContainer { get; }
public string Account { get; }
public string PrivateKey { get; }
public NethereumInteraction StartInteraction(BaseLog log)
{
var ip = RunningContainer.Pod.Cluster.IP;
var port = RunningContainer.ServicePorts[0].Number;
var account = Account;
var privateKey = PrivateKey;
var creator = new NethereumInteractionCreator(log, ip, port, account, privateKey);
return creator.CreateWorkflow();
}
}
}

58
DistTestCore/Marketplace/GethCompanionNodeStarter.cs
View File

@ -1,4 +1,5 @@
using KubernetesWorkflow;
using Utils;
namespace DistTestCore.Marketplace
{
@ -9,41 +10,56 @@ namespace DistTestCore.Marketplace
{
}
public GethCompanionNodeInfo[] StartCompanionNodesFor(CodexSetup codexSetup, MarketplaceNetwork marketplace)
public GethCompanionNodeInfo StartCompanionNodeFor(CodexSetup codexSetup, MarketplaceNetwork marketplace)
{
LogStart($"Initializing companions for {codexSetup.NumberOfNodes} Codex nodes.");
LogStart($"Initializing companion for {codexSetup.NumberOfNodes} Codex nodes.");
var startupConfig = CreateCompanionNodeStartupConfig(marketplace.Bootstrap);
var startupConfig = CreateCompanionNodeStartupConfig(marketplace.Bootstrap, codexSetup.NumberOfNodes);
var workflow = workflowCreator.CreateWorkflow();
var containers = workflow.Start(codexSetup.NumberOfNodes, Location.Unspecified, new GethContainerRecipe(), startupConfig);
if (containers.Containers.Length != codexSetup.NumberOfNodes) throw new InvalidOperationException("Expected a Geth companion node to be created for each Codex node. Test infra failure.");
var containers = workflow.Start(1, Location.Unspecified, new GethContainerRecipe(), startupConfig);
WaitForAccountCreation(codexSetup.NumberOfNodes);
if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected one Geth companion node to be created. Test infra failure.");
var container = containers.Containers[0];
var result = containers.Containers.Select(c => CreateCompanionInfo(workflow, c)).ToArray();
var node = CreateCompanionInfo(workflow, container, codexSetup.NumberOfNodes);
EnsureCompanionNodeIsSynced(node, marketplace);
foreach (var node in result)
{
EnsureCompanionNodeIsSynced(node, marketplace);
}
LogEnd($"Initialized {codexSetup.NumberOfNodes} companion nodes. Their accounts: [{string.Join(",", result.Select(c => c.Account))}]");
return result;
LogEnd($"Initialized one companion node for {codexSetup.NumberOfNodes} Codex nodes. Their accounts: [{string.Join(",", node.Accounts.Select(a => a.Account))}]");
return node;
}
private GethCompanionNodeInfo CreateCompanionInfo(StartupWorkflow workflow, RunningContainer container)
private void WaitForAccountCreation(int numberOfNodes)
{
// We wait proportional to the number of account the node has to create. It takes a few seconds for each one to generate the keys and create the files
// we will be trying to read in 'ExtractAccount', later on in the start-up process.
Time.Sleep(TimeSpan.FromSeconds(4.5 * numberOfNodes));
}
private GethCompanionNodeInfo CreateCompanionInfo(StartupWorkflow workflow, RunningContainer container, int numberOfAccounts)
{
var extractor = new ContainerInfoExtractor(lifecycle.Log, workflow, container);
var account = extractor.ExtractAccount();
var privKey = extractor.ExtractPrivateKey();
return new GethCompanionNodeInfo(container, account, privKey);
var accounts = ExtractAccounts(extractor, numberOfAccounts).ToArray();
return new GethCompanionNodeInfo(container, accounts);
}
private IEnumerable<GethCompanionAccount> ExtractAccounts(ContainerInfoExtractor extractor, int numberOfAccounts)
{
for (int i = 0; i < numberOfAccounts; i++) yield return ExtractAccount(extractor, i + 1);
}
private GethCompanionAccount ExtractAccount(ContainerInfoExtractor extractor, int orderNumber)
{
var account = extractor.ExtractAccount(orderNumber);
var privKey = extractor.ExtractPrivateKey(orderNumber);
return new GethCompanionAccount(account, privKey);
}
private void EnsureCompanionNodeIsSynced(GethCompanionNodeInfo node, MarketplaceNetwork marketplace)
{
try
{
var interaction = node.StartInteraction(lifecycle.Log);
var interaction = node.StartInteraction(lifecycle.Log, node.Accounts.First());
interaction.EnsureSynced(marketplace.Marketplace.Address, marketplace.Marketplace.Abi);
}
catch (Exception e)
@ -52,10 +68,10 @@ namespace DistTestCore.Marketplace
}
}
private StartupConfig CreateCompanionNodeStartupConfig(GethBootstrapNodeInfo bootstrapNode)
private StartupConfig CreateCompanionNodeStartupConfig(GethBootstrapNodeInfo bootstrapNode, int numberOfAccounts)
{
var config = new StartupConfig();
config.Add(new GethStartupConfig(false, bootstrapNode));
config.Add(new GethStartupConfig(false, bootstrapNode, numberOfAccounts));
return config;
}
}

16
DistTestCore/Marketplace/GethContainerRecipe.cs
View File

@ -7,10 +7,20 @@ namespace DistTestCore.Marketplace
public const string DockerImage = "thatbenbierens/geth-confenv:latest";
public const string HttpPortTag = "http_port";
public const string DiscoveryPortTag = "disc_port";
public const string AccountFilename = "account_string.txt";
public const string PrivateKeyFilename = "private.key";
private const string defaultArgs = "--ipcdisable --syncmode full";
public static string GetAccountFilename(int? orderNumber)
{
if (orderNumber == null) return "account_string.txt";
return $"account_string_{orderNumber.Value}.txt";
}
public static string GetPrivateKeyFilename(int? orderNumber)
{
if (orderNumber == null) return "private.key";
return $"private_{orderNumber.Value}.key";
}
protected override string Image => DockerImage;
protected override void Initialize(StartupConfig startupConfig)
@ -43,6 +53,8 @@ namespace DistTestCore.Marketplace
private string CreateCompanionArgs(Port discovery, GethStartupConfig config)
{
AddEnvVar("NUMBER_OF_ACCOUNTS", config.NumberOfCompanionAccounts.ToString());
var port = AddInternalPort();
var authRpc = AddInternalPort();
var httpPort = AddExposedPort(tag: HttpPortTag);

6
DistTestCore/Marketplace/GethStartResult.cs
View File

@ -2,15 +2,15 @@
{
public class GethStartResult
{
public GethStartResult(IMarketplaceAccessFactory marketplaceAccessFactory, MarketplaceNetwork marketplaceNetwork, GethCompanionNodeInfo[] companionNodes)
public GethStartResult(IMarketplaceAccessFactory marketplaceAccessFactory, MarketplaceNetwork marketplaceNetwork, GethCompanionNodeInfo companionNode)
{
MarketplaceAccessFactory = marketplaceAccessFactory;
MarketplaceNetwork = marketplaceNetwork;
CompanionNodes = companionNodes;
CompanionNode = companionNode;
}
public IMarketplaceAccessFactory MarketplaceAccessFactory { get; }
public MarketplaceNetwork MarketplaceNetwork { get; }
public GethCompanionNodeInfo[] CompanionNodes { get; }
public GethCompanionNodeInfo CompanionNode { get; }
}
}

4
DistTestCore/Marketplace/GethStartupConfig.cs
View File

@ -2,13 +2,15 @@
{
public class GethStartupConfig
{
public GethStartupConfig(bool isBootstrapNode, GethBootstrapNodeInfo bootstrapNode)
public GethStartupConfig(bool isBootstrapNode, GethBootstrapNodeInfo bootstrapNode, int numberOfCompanionAccounts)
{
IsBootstrapNode = isBootstrapNode;
BootstrapNode = bootstrapNode;
NumberOfCompanionAccounts = numberOfCompanionAccounts;
}
public bool IsBootstrapNode { get; }
public GethBootstrapNodeInfo BootstrapNode { get; }
public int NumberOfCompanionAccounts { get; }
}
}

11
DistTestCore/Marketplace/MarketplaceAccess.cs
View File

@ -19,14 +19,14 @@ namespace DistTestCore.Marketplace
{
private readonly TestLog log;
private readonly MarketplaceNetwork marketplaceNetwork;
private readonly GethCompanionNodeInfo companionNode;
private readonly GethCompanionAccount account;
private readonly CodexAccess codexAccess;
public MarketplaceAccess(TestLog log, MarketplaceNetwork marketplaceNetwork, GethCompanionNodeInfo companionNode, CodexAccess codexAccess)
public MarketplaceAccess(TestLog log, MarketplaceNetwork marketplaceNetwork, GethCompanionAccount account, CodexAccess codexAccess)
{
this.log = log;
this.marketplaceNetwork = marketplaceNetwork;
this.companionNode = companionNode;
this.account = account;
this.codexAccess = codexAccess;
}
@ -104,11 +104,10 @@ namespace DistTestCore.Marketplace
public TestToken GetBalance()
{
var interaction = marketplaceNetwork.StartInteraction(log);
var account = companionNode.Account;
var amount = interaction.GetBalance(marketplaceNetwork.Marketplace.TokenAddress, account);
var amount = interaction.GetBalance(marketplaceNetwork.Marketplace.TokenAddress, account.Account);
var balance = new TestToken(amount);
Log($"Balance of {account} is {balance}.");
Log($"Balance of {account.Account} is {balance}.");
return balance;
}

6
DistTestCore/Marketplace/MarketplaceAccessFactory.cs
View File

@ -33,10 +33,10 @@ namespace DistTestCore.Marketplace
return new MarketplaceAccess(log, marketplaceNetwork, companionNode, access);
}
private GethCompanionNodeInfo GetGethCompanionNode(CodexAccess access)
private GethCompanionAccount GetGethCompanionNode(CodexAccess access)
{
var node = access.Container.Recipe.Additionals.Single(a => a is GethCompanionNodeInfo);
return (GethCompanionNodeInfo)node;
var account = access.Container.Recipe.Additionals.Single(a => a is GethCompanionAccount);
return (GethCompanionAccount)account;
}
}
}

4
Nethereum/NethereumInteraction.cs
View File

@ -92,7 +92,7 @@ namespace NethereumWorkflow
var numberOfBlocks = ToDecimal(number);
return !sync.IsSyncing && numberOfBlocks > 256;
}, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(1));
}, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(3));
}
private void WaitForContract(string marketplaceAddress, string marketplaceAbi)
@ -109,7 +109,7 @@ namespace NethereumWorkflow
{
return false;
}
}, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(1));
}, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(3));
}
private HexBigInteger ToHexBig(decimal amount)

32
Tests/BasicTests/ExampleTests.cs
View File

@ -43,28 +43,30 @@ namespace Tests.BasicTests
[Test]
public void MarketplaceExample()
{
var primary = SetupCodexNode(s => s
var sellerInitialBalance = 234.TestTokens();
var buyerInitialBalance = 1000.TestTokens();
var seller = SetupCodexNode(s => s
.WithLogLevel(CodexLogLevel.Trace)
.WithStorageQuota(11.GB())
.EnableMarketplace(initialBalance: 234.TestTokens()));
.EnableMarketplace(sellerInitialBalance));
primary.Marketplace.AssertThatBalance(Is.EqualTo(234.TestTokens()));
var secondary = SetupCodexNode(s => s
.WithLogLevel(CodexLogLevel.Trace)
.WithBootstrapNode(primary)
.EnableMarketplace(initialBalance: 1000.TestTokens()));
primary.Marketplace.MakeStorageAvailable(
seller.Marketplace.AssertThatBalance(Is.EqualTo(sellerInitialBalance));
seller.Marketplace.MakeStorageAvailable(
size: 10.GB(),
minPricePerBytePerSecond: 1.TestTokens(),
maxCollateral: 20.TestTokens(),
maxDuration: TimeSpan.FromMinutes(3));
var testFile = GenerateTestFile(10.MB());
var contentId = secondary.UploadFile(testFile);
secondary.Marketplace.RequestStorage(contentId,
var buyer = SetupCodexNode(s => s
.WithLogLevel(CodexLogLevel.Trace)
.WithBootstrapNode(seller)
.EnableMarketplace(buyerInitialBalance));
var contentId = buyer.UploadFile(testFile);
buyer.Marketplace.RequestStorage(contentId,
pricePerBytePerSecond: 2.TestTokens(),
requiredCollateral: 10.TestTokens(),
minRequiredNumberOfNodes: 1,
@ -73,12 +75,12 @@ namespace Tests.BasicTests
Time.Sleep(TimeSpan.FromMinutes(1));
primary.Marketplace.AssertThatBalance(Is.LessThan(234.TestTokens()), "Collateral was not placed.");
seller.Marketplace.AssertThatBalance(Is.LessThan(sellerInitialBalance), "Collateral was not placed.");
Time.Sleep(TimeSpan.FromMinutes(2));
primary.Marketplace.AssertThatBalance(Is.GreaterThan(234.TestTokens()), "Storer was not paid for storage.");
secondary.Marketplace.AssertThatBalance(Is.LessThan(1000.TestTokens()), "Contractor was not charged for storage.");
seller.Marketplace.AssertThatBalance(Is.GreaterThan(sellerInitialBalance), "Seller was not paid for storage.");
buyer.Marketplace.AssertThatBalance(Is.LessThan(buyerInitialBalance), "Buyer was not charged for storage.");
}
}
}