diff --git a/DistTestCore/AutoBootstrapDistTest.cs b/DistTestCore/AutoBootstrapDistTest.cs new file mode 100644 index 00000000..352e2ecf --- /dev/null +++ b/DistTestCore/AutoBootstrapDistTest.cs @@ -0,0 +1,30 @@ +namespace DistTestCore +{ + public class AutoBootstrapDistTest : DistTest + { + private IOnlineCodexNode? bootstrapNode; + + public override IOnlineCodexNode SetupCodexBootstrapNode(Action setup) + { + throw new Exception("AutoBootstrapDistTest creates and attaches a single boostrap node for you. " + + "If you want to control the bootstrap node from your test, please use DistTest instead."); + } + + public override ICodexNodeGroup SetupCodexNodes(int numberOfNodes, Action setup) + { + var codexSetup = new CodexSetup(numberOfNodes); + setup(codexSetup); + codexSetup.WithBootstrapNode(EnsureBootstapNode()); + return BringOnline(codexSetup); + } + + private IOnlineCodexNode EnsureBootstapNode() + { + if (bootstrapNode == null) + { + bootstrapNode = base.SetupCodexBootstrapNode(s => { }); + } + return bootstrapNode; + } + } +} diff --git a/DistTestCore/BaseStarter.cs b/DistTestCore/BaseStarter.cs index 7d259e79..eb344497 100644 --- a/DistTestCore/BaseStarter.cs +++ b/DistTestCore/BaseStarter.cs @@ -1,4 +1,5 @@ using KubernetesWorkflow; +using Logging; namespace DistTestCore { diff --git a/DistTestCore/Codex/CodexAccess.cs b/DistTestCore/Codex/CodexAccess.cs index ae84a91d..006b66e5 100644 --- a/DistTestCore/Codex/CodexAccess.cs +++ b/DistTestCore/Codex/CodexAccess.cs @@ -1,11 +1,15 @@ using KubernetesWorkflow; +using Logging; namespace DistTestCore.Codex { public class CodexAccess { - public CodexAccess(RunningContainer runningContainer) + private readonly BaseLog log; + + public CodexAccess(BaseLog log, RunningContainer runningContainer) { + this.log = log; Container = runningContainer; } @@ -31,16 +35,16 @@ namespace DistTestCore.Codex return Http().HttpPostJson("sales/availability", request); } - public CodexSalesRequestStorageResponse RequestStorage(CodexSalesRequestStorageRequest request, string contentId) + public string RequestStorage(CodexSalesRequestStorageRequest request, string contentId) { - return Http().HttpPostJson($"storage/request/{contentId}", request); + return Http().HttpPostJson($"storage/request/{contentId}", request); } private Http Http() { var ip = Container.Pod.Cluster.IP; var port = Container.ServicePorts[0].Number; - return new Http(ip, port, baseUrl: "/api/codex/v1"); + return new Http(log, ip, port, baseUrl: "/api/codex/v1"); } public string ConnectToPeer(string peerId, string peerMultiAddress) @@ -55,9 +59,31 @@ namespace DistTestCore.Codex public string[] addrs { get; set; } = new string[0]; public string repo { get; set; } = string.Empty; public string spr { get; set; } = string.Empty; + public EnginePeerResponse[] enginePeers { get; set; } = Array.Empty(); + public SwitchPeerResponse[] switchPeers { get; set; } = Array.Empty(); public CodexDebugVersionResponse codex { get; set; } = new(); } + public class EnginePeerResponse + { + public string peerId { get; set; } = string.Empty; + public EnginePeerContextResponse context { get; set; } = new(); + } + + public class EnginePeerContextResponse + { + public int blocks { get; set; } = 0; + public int peerWants { get; set; } = 0; + public int exchanged { get; set; } = 0; + public string lastExchange { get; set; } = string.Empty; + } + + public class SwitchPeerResponse + { + public string peerId { get; set; } = string.Empty; + public string key { get; set; } = string.Empty; + } + public class CodexDebugVersionResponse { public string version { get; set; } = string.Empty; @@ -91,9 +117,4 @@ namespace DistTestCore.Codex public uint? nodes { get; set; } public uint? tolerance { get; set;} } - - public class CodexSalesRequestStorageResponse - { - public string purchaseId { get; set; } = string.Empty; - } } diff --git a/DistTestCore/Codex/CodexContainerRecipe.cs b/DistTestCore/Codex/CodexContainerRecipe.cs index 01f49339..c3095b1d 100644 --- a/DistTestCore/Codex/CodexContainerRecipe.cs +++ b/DistTestCore/Codex/CodexContainerRecipe.cs @@ -5,7 +5,8 @@ namespace DistTestCore.Codex { public class CodexContainerRecipe : ContainerRecipeFactory { - public const string DockerImage = "thatbenbierens/nim-codex:sha-bf5512b"; + //public const string DockerImage = "thatbenbierens/nim-codex:sha-9716635"; + public const string DockerImage = "thatbenbierens/codexlocal:latest"; public const string MetricsPortTag = "metrics_port"; protected override string Image => DockerImage; @@ -21,6 +22,11 @@ namespace DistTestCore.Codex var listenPort = AddInternalPort(); AddEnvVar("LISTEN_ADDRS", $"/ip4/0.0.0.0/tcp/{listenPort.Number}"); + if (!string.IsNullOrEmpty(config.BootstrapSpr)) + { + AddEnvVar("BOOTSTRAP_SPR", config.BootstrapSpr); + } + if (config.LogLevel != null) { AddEnvVar("LOG_LEVEL", config.LogLevel.ToString()!.ToUpperInvariant()); @@ -38,14 +44,15 @@ namespace DistTestCore.Codex if (config.MarketplaceConfig != null) { var gethConfig = startupConfig.Get(); - var companionNode = gethConfig.CompanionNodes[Index]; - Additional(companionNode); + var companionNode = gethConfig.CompanionNode; + var companionNodeAccount = companionNode.Accounts[Index]; + Additional(companionNodeAccount); var ip = companionNode.RunningContainer.Pod.Ip; - var port = companionNode.RunningContainer.Recipe.GetPortByTag(GethContainerRecipe.WsPortTag).Number; + var port = companionNode.RunningContainer.Recipe.GetPortByTag(GethContainerRecipe.HttpPortTag).Number; AddEnvVar("ETH_PROVIDER", $"ws://{ip}:{port}"); - AddEnvVar("ETH_ACCOUNT", companionNode.Account); + AddEnvVar("ETH_ACCOUNT", companionNodeAccount.Account); AddEnvVar("ETH_MARKETPLACE_ADDRESS", gethConfig.MarketplaceNetwork.Marketplace.Address); } } diff --git a/DistTestCore/Codex/CodexStartupConfig.cs b/DistTestCore/Codex/CodexStartupConfig.cs index 2ebf4077..9b3cb8a5 100644 --- a/DistTestCore/Codex/CodexStartupConfig.cs +++ b/DistTestCore/Codex/CodexStartupConfig.cs @@ -5,11 +5,12 @@ namespace DistTestCore.Codex { public class CodexStartupConfig { + public string? NameOverride { get; set; } public Location Location { get; set; } public CodexLogLevel? LogLevel { get; set; } public ByteSize? StorageQuota { get; set; } public bool MetricsEnabled { get; set; } public MarketplaceInitialConfig? MarketplaceConfig { get; set; } - public IOnlineCodexNode? BootstrapNode { get; set; } + public string? BootstrapSpr { get; set; } } } diff --git a/DistTestCore/CodexNodeGroup.cs b/DistTestCore/CodexNodeGroup.cs index ffcea2e4..b0930d39 100644 --- a/DistTestCore/CodexNodeGroup.cs +++ b/DistTestCore/CodexNodeGroup.cs @@ -64,7 +64,7 @@ namespace DistTestCore private OnlineCodexNode CreateOnlineCodexNode(RunningContainer c, ICodexNodeFactory factory) { - var access = new CodexAccess(c); + var access = new CodexAccess(lifecycle.Log, c); EnsureOnline(access); return factory.CreateOnlineCodexNode(access, this); } @@ -75,6 +75,10 @@ namespace DistTestCore { var debugInfo = access.GetDebugInfo(); if (debugInfo == null || string.IsNullOrEmpty(debugInfo.id)) throw new InvalidOperationException("Unable to get debug-info from codex node at startup."); + + var nodePeerId = debugInfo.id; + var nodeName = access.Container.Name; + lifecycle.Log.AddStringReplace(nodePeerId, $"___{nodeName}___"); } catch (Exception e) { diff --git a/DistTestCore/CodexSetup.cs b/DistTestCore/CodexSetup.cs index 8f7e2539..5b5f3c0c 100644 --- a/DistTestCore/CodexSetup.cs +++ b/DistTestCore/CodexSetup.cs @@ -6,6 +6,7 @@ namespace DistTestCore { public interface ICodexSetup { + ICodexSetup WithName(string name); ICodexSetup At(Location location); ICodexSetup WithLogLevel(CodexLogLevel level); ICodexSetup WithBootstrapNode(IOnlineCodexNode node); @@ -13,41 +14,21 @@ namespace DistTestCore ICodexSetup EnableMetrics(); ICodexSetup EnableMarketplace(TestToken initialBalance); ICodexSetup EnableMarketplace(TestToken initialBalance, Ether initialEther); - ICodexNodeGroup BringOnline(); } public class CodexSetup : CodexStartupConfig, ICodexSetup { - private readonly CodexStarter starter; - public int NumberOfNodes { get; } - public CodexSetup(CodexStarter starter, int numberOfNodes) + public CodexSetup(int numberOfNodes) { - this.starter = starter; NumberOfNodes = numberOfNodes; } - public ICodexNodeGroup BringOnline() + public ICodexSetup WithName(string name) { - var group = starter.BringOnline(this); - ConnectToBootstrapNode(group); - return group; - } - - private void ConnectToBootstrapNode(ICodexNodeGroup group) - { - if (BootstrapNode == null) return; - - // TODO: - // node.ConnectToPeer uses the '/api/codex/vi/connect/' endpoint to make the connection. - // This should be replaced by injecting the bootstrap node's SPR into the env-vars of the new node containers. (Easy!) - // However, NAT isn't figure out yet. So connecting with SPR doesn't (always?) work. - // So for now, ConnectToPeer - foreach (var node in group) - { - node.ConnectToPeer(BootstrapNode); - } + NameOverride = name; + return this; } public ICodexSetup At(Location location) @@ -58,7 +39,7 @@ namespace DistTestCore public ICodexSetup WithBootstrapNode(IOnlineCodexNode node) { - BootstrapNode = node; + BootstrapSpr = node.GetDebugInfo().spr; return this; } @@ -100,7 +81,7 @@ namespace DistTestCore private IEnumerable DescribeArgs() { if (LogLevel != null) yield return $"LogLevel={LogLevel}"; - if (BootstrapNode != null) yield return $"BootstrapNode={BootstrapNode.GetName()}"; + if (BootstrapSpr != null) yield return $"BootstrapNode={BootstrapSpr}"; if (StorageQuota != null) yield return $"StorageQuote={StorageQuota}"; } } diff --git a/DistTestCore/CodexStarter.cs b/DistTestCore/CodexStarter.cs index 52f87699..900967a9 100644 --- a/DistTestCore/CodexStarter.cs +++ b/DistTestCore/CodexStarter.cs @@ -1,4 +1,5 @@ using DistTestCore.Codex; +using DistTestCore.Marketplace; using KubernetesWorkflow; namespace DistTestCore @@ -18,10 +19,7 @@ namespace DistTestCore LogStart($"Starting {codexSetup.Describe()}..."); var gethStartResult = lifecycle.GethStarter.BringOnlineMarketplaceFor(codexSetup); - var startupConfig = new StartupConfig(); - startupConfig.Add(codexSetup); - startupConfig.Add(gethStartResult); - + var startupConfig = CreateStartupConfig(gethStartResult, codexSetup); var containers = StartCodexContainers(startupConfig, codexSetup.NumberOfNodes, codexSetup.Location); var metricAccessFactory = lifecycle.PrometheusStarter.CollectMetricsFor(codexSetup, containers); @@ -56,7 +54,16 @@ namespace DistTestCore var workflow = CreateWorkflow(); workflow.DownloadContainerLog(container, logHandler); } - + + private StartupConfig CreateStartupConfig(GethStartResult gethStartResult, CodexSetup codexSetup) + { + var startupConfig = new StartupConfig(); + startupConfig.NameOverride = codexSetup.NameOverride; + startupConfig.Add(codexSetup); + startupConfig.Add(gethStartResult); + return startupConfig; + } + private RunningContainers StartCodexContainers(StartupConfig startupConfig, int numberOfNodes, Location location) { var workflow = CreateWorkflow(); diff --git a/DistTestCore/Configuration.cs b/DistTestCore/Configuration.cs index b0d1affb..5de9afcd 100644 --- a/DistTestCore/Configuration.cs +++ b/DistTestCore/Configuration.cs @@ -21,7 +21,7 @@ namespace DistTestCore public Logging.LogConfig GetLogConfig() { - return new Logging.LogConfig("CodexTestLogs"); + return new Logging.LogConfig("CodexTestLogs", debugEnabled: false); } public string GetFileManagerFolder() diff --git a/DistTestCore/DistTest.cs b/DistTestCore/DistTest.cs index ca3f70bd..a85f689d 100644 --- a/DistTestCore/DistTest.cs +++ b/DistTestCore/DistTest.cs @@ -5,6 +5,7 @@ using DistTestCore.Metrics; using KubernetesWorkflow; using Logging; using NUnit.Framework; +using System.Reflection; using Utils; namespace DistTestCore @@ -13,22 +14,30 @@ namespace DistTestCore public abstract class DistTest { private readonly Configuration configuration = new Configuration(); + private readonly Assembly[] testAssemblies; private FixtureLog fixtureLog = null!; private TestLifecycle lifecycle = null!; private DateTime testStart = DateTime.MinValue; + public DistTest() + { + var assemblies = AppDomain.CurrentDomain.GetAssemblies(); + testAssemblies = assemblies.Where(a => a.FullName!.ToLowerInvariant().Contains("test")).ToArray(); + } + [OneTimeSetUp] public void GlobalSetup() { // Previous test run may have been interrupted. // Begin by cleaning everything up. + Timing.UseLongTimeouts = false; fixtureLog = new FixtureLog(configuration.GetLogConfig()); try { Stopwatch.Measure(fixtureLog, "Global setup", () => { - var wc = new WorkflowCreator(configuration.GetK8sConfiguration()); + var wc = new WorkflowCreator(fixtureLog, configuration.GetK8sConfiguration()); wc.CreateWorkflow().DeleteAllResources(); }); } @@ -48,6 +57,8 @@ namespace DistTestCore [SetUp] public void SetUpDistTest() { + Timing.UseLongTimeouts = ShouldUseLongTimeouts(); + if (GlobalTestFailure.HasFailed) { Assert.Inconclusive("Skip test: Previous test failed during clean up."); @@ -77,9 +88,67 @@ namespace DistTestCore return lifecycle.FileManager.GenerateTestFile(size); } - public ICodexSetup SetupCodexNodes(int numberOfNodes) + public IOnlineCodexNode SetupCodexBootstrapNode() { - return new CodexSetup(lifecycle.CodexStarter, numberOfNodes); + return SetupCodexBootstrapNode(s => { }); + } + + public virtual IOnlineCodexNode SetupCodexBootstrapNode(Action setup) + { + return SetupCodexNode(s => + { + setup(s); + s.WithName("Bootstrap"); + }); + } + + public IOnlineCodexNode SetupCodexNode() + { + return SetupCodexNode(s => { }); + } + + public IOnlineCodexNode SetupCodexNode(Action setup) + { + return SetupCodexNodes(1, setup)[0]; + } + + public ICodexNodeGroup SetupCodexNodes(int numberOfNodes) + { + return SetupCodexNodes(numberOfNodes, s => { }); + } + + public virtual ICodexNodeGroup SetupCodexNodes(int numberOfNodes, Action setup) + { + var codexSetup = new CodexSetup(numberOfNodes); + + setup(codexSetup); + + return BringOnline(codexSetup); + } + + public ICodexNodeGroup BringOnline(ICodexSetup codexSetup) + { + return lifecycle.CodexStarter.BringOnline((CodexSetup)codexSetup); + } + + protected BaseLog Log + { + get { return lifecycle.Log; } + } + + private bool ShouldUseLongTimeouts() + { + // Don't be fooled! TestContext.CurrentTest.Test allows you easy access to the attributes of the current test. + // But this doesn't work for tests making use of [TestCase]. So instead, we use reflection here to figure out + // if the attribute is present. + var currentTest = TestContext.CurrentContext.Test; + var className = currentTest.ClassName; + var methodName = currentTest.MethodName; + + var testClasses = testAssemblies.SelectMany(a => a.GetTypes()).Where(c => c.FullName == className).ToArray(); + var testMethods = testClasses.SelectMany(c => c.GetMethods()).Where(m => m.Name == methodName).ToArray(); + + return testMethods.Any(m => m.GetCustomAttribute() != null); } private void CreateNewTestLifecycle() diff --git a/DistTestCore/GethStarter.cs b/DistTestCore/GethStarter.cs index b3c782b0..914b680c 100644 --- a/DistTestCore/GethStarter.cs +++ b/DistTestCore/GethStarter.cs @@ -22,37 +22,37 @@ namespace DistTestCore if (codexSetup.MarketplaceConfig == null) return CreateMarketplaceUnavailableResult(); var marketplaceNetwork = marketplaceNetworkCache.Get(); - var companionNodes = StartCompanionNodes(codexSetup, marketplaceNetwork); + var companionNode = StartCompanionNode(codexSetup, marketplaceNetwork); LogStart("Setting up initial balance..."); - TransferInitialBalance(marketplaceNetwork, codexSetup.MarketplaceConfig, companionNodes); + TransferInitialBalance(marketplaceNetwork, codexSetup.MarketplaceConfig, companionNode); LogEnd($"Initial balance of {codexSetup.MarketplaceConfig.InitialTestTokens} set for {codexSetup.NumberOfNodes} nodes."); - return CreateGethStartResult(marketplaceNetwork, companionNodes); + return CreateGethStartResult(marketplaceNetwork, companionNode); } - private void TransferInitialBalance(MarketplaceNetwork marketplaceNetwork, MarketplaceInitialConfig marketplaceConfig, GethCompanionNodeInfo[] companionNodes) + private void TransferInitialBalance(MarketplaceNetwork marketplaceNetwork, MarketplaceInitialConfig marketplaceConfig, GethCompanionNodeInfo companionNode) { var interaction = marketplaceNetwork.StartInteraction(lifecycle.Log); var tokenAddress = marketplaceNetwork.Marketplace.TokenAddress; - foreach (var node in companionNodes) + foreach (var account in companionNode.Accounts) { - interaction.TransferWeiTo(node.Account, marketplaceConfig.InitialEth.Wei); - interaction.MintTestTokens(node.Account, marketplaceConfig.InitialTestTokens.Amount, tokenAddress); + interaction.TransferWeiTo(account.Account, marketplaceConfig.InitialEth.Wei); + interaction.MintTestTokens(account.Account, marketplaceConfig.InitialTestTokens.Amount, tokenAddress); } interaction.WaitForAllTransactions(); } - private GethStartResult CreateGethStartResult(MarketplaceNetwork marketplaceNetwork, GethCompanionNodeInfo[] companionNodes) + private GethStartResult CreateGethStartResult(MarketplaceNetwork marketplaceNetwork, GethCompanionNodeInfo companionNode) { - return new GethStartResult(CreateMarketplaceAccessFactory(marketplaceNetwork), marketplaceNetwork, companionNodes); + return new GethStartResult(CreateMarketplaceAccessFactory(marketplaceNetwork), marketplaceNetwork, companionNode); } private GethStartResult CreateMarketplaceUnavailableResult() { - return new GethStartResult(new MarketplaceUnavailableAccessFactory(), null!, Array.Empty()); + return new GethStartResult(new MarketplaceUnavailableAccessFactory(), null!, null!); } private IMarketplaceAccessFactory CreateMarketplaceAccessFactory(MarketplaceNetwork marketplaceNetwork) @@ -60,9 +60,9 @@ namespace DistTestCore return new GethMarketplaceAccessFactory(lifecycle.Log, marketplaceNetwork); } - private GethCompanionNodeInfo[] StartCompanionNodes(CodexSetup codexSetup, MarketplaceNetwork marketplaceNetwork) + private GethCompanionNodeInfo StartCompanionNode(CodexSetup codexSetup, MarketplaceNetwork marketplaceNetwork) { - return companionNodeStarter.StartCompanionNodesFor(codexSetup, marketplaceNetwork.Bootstrap); + return companionNodeStarter.StartCompanionNodeFor(codexSetup, marketplaceNetwork); } } diff --git a/DistTestCore/Http.cs b/DistTestCore/Http.cs index 62ccbfbf..2969596e 100644 --- a/DistTestCore/Http.cs +++ b/DistTestCore/Http.cs @@ -1,4 +1,5 @@ -using Newtonsoft.Json; +using Logging; +using Newtonsoft.Json; using NUnit.Framework; using System.Net.Http.Headers; using System.Net.Http.Json; @@ -8,12 +9,14 @@ namespace DistTestCore { public class Http { + private readonly BaseLog log; private readonly string ip; private readonly int port; private readonly string baseUrl; - public Http(string ip, int port, string baseUrl) + public Http(BaseLog log, string ip, int port, string baseUrl) { + this.log = log; this.ip = ip; this.port = port; this.baseUrl = baseUrl; @@ -28,8 +31,11 @@ namespace DistTestCore { using var client = GetClient(); var url = GetUrl() + route; + Log(url, ""); var result = Time.Wait(client.GetAsync(url)); - return Time.Wait(result.Content.ReadAsStringAsync()); + var str = Time.Wait(result.Content.ReadAsStringAsync()); + Log(url, str); + return str; ; }); } @@ -41,16 +47,23 @@ namespace DistTestCore public TResponse HttpPostJson(string route, TRequest body) { - var json = Retry(() => + var json = HttpPostJson(route, body); + return TryJsonDeserialize(json); + } + + public string HttpPostJson(string route, TRequest body) + { + return Retry(() => { using var client = GetClient(); var url = GetUrl() + route; using var content = JsonContent.Create(body); + Log(url, JsonConvert.SerializeObject(body)); var result = Time.Wait(client.PostAsync(url, content)); - return Time.Wait(result.Content.ReadAsStringAsync()); + var str= Time.Wait(result.Content.ReadAsStringAsync()); + Log(url, str); + return str; }); - - return TryJsonDeserialize(json); } public string HttpPostStream(string route, Stream stream) @@ -59,12 +72,13 @@ namespace DistTestCore { using var client = GetClient(); var url = GetUrl() + route; - + Log(url, "~ STREAM ~"); var content = new StreamContent(stream); content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream"); var response = Time.Wait(client.PostAsync(url, content)); - - return Time.Wait(response.Content.ReadAsStringAsync()); + var str =Time.Wait(response.Content.ReadAsStringAsync()); + Log(url, str); + return str; }); } @@ -74,7 +88,7 @@ namespace DistTestCore { var client = GetClient(); var url = GetUrl() + route; - + Log(url, "~ STREAM ~"); return Time.Wait(client.GetStreamAsync(url)); }); } @@ -84,6 +98,11 @@ namespace DistTestCore return $"http://{ip}:{port}{baseUrl}"; } + private void Log(string url, string message) + { + log.Debug($"({url}) = '{message}'", 3); + } + private static T Retry(Func operation) { var retryCounter = 0; diff --git a/DistTestCore/Marketplace/CodexContractsContainerRecipe.cs b/DistTestCore/Marketplace/CodexContractsContainerRecipe.cs index d2a93a79..42bbeca0 100644 --- a/DistTestCore/Marketplace/CodexContractsContainerRecipe.cs +++ b/DistTestCore/Marketplace/CodexContractsContainerRecipe.cs @@ -6,6 +6,7 @@ namespace DistTestCore.Marketplace { public const string DockerImage = "thatbenbierens/codex-contracts-deployment"; public const string MarketplaceAddressFilename = "/usr/app/deployments/codexdisttestnetwork/Marketplace.json"; + public const string MarketplaceArtifactFilename = "/usr/app/artifacts/contracts/Marketplace.sol/Marketplace.json"; protected override string Image => DockerImage; diff --git a/DistTestCore/Marketplace/CodexContractsStarter.cs b/DistTestCore/Marketplace/CodexContractsStarter.cs index 841e830a..412c25ad 100644 --- a/DistTestCore/Marketplace/CodexContractsStarter.cs +++ b/DistTestCore/Marketplace/CodexContractsStarter.cs @@ -30,15 +30,16 @@ namespace DistTestCore.Marketplace return logHandler.Found; }); - var extractor = new ContainerInfoExtractor(workflow, container); + var extractor = new ContainerInfoExtractor(lifecycle.Log, workflow, container); var marketplaceAddress = extractor.ExtractMarketplaceAddress(); + var abi = extractor.ExtractMarketplaceAbi(); var interaction = bootstrapNode.StartInteraction(lifecycle.Log); var tokenAddress = interaction.GetTokenAddress(marketplaceAddress); LogEnd("Contracts deployed."); - return new MarketplaceInfo(marketplaceAddress, tokenAddress); + return new MarketplaceInfo(marketplaceAddress, abi, tokenAddress); } private void WaitUntil(Func predicate) @@ -57,13 +58,15 @@ namespace DistTestCore.Marketplace public class MarketplaceInfo { - public MarketplaceInfo(string address, string tokenAddress) + public MarketplaceInfo(string address, string abi, string tokenAddress) { Address = address; + Abi = abi; TokenAddress = tokenAddress; } public string Address { get; } + public string Abi { get; } public string TokenAddress { get; } } diff --git a/DistTestCore/Marketplace/ContainerInfoExtractor.cs b/DistTestCore/Marketplace/ContainerInfoExtractor.cs index 09d8aef9..281a6130 100644 --- a/DistTestCore/Marketplace/ContainerInfoExtractor.cs +++ b/DistTestCore/Marketplace/ContainerInfoExtractor.cs @@ -1,23 +1,28 @@ using KubernetesWorkflow; +using Logging; using Newtonsoft.Json; -using System.Text; +using Newtonsoft.Json.Linq; +using Utils; namespace DistTestCore.Marketplace { public class ContainerInfoExtractor { + private readonly BaseLog log; private readonly StartupWorkflow workflow; private readonly RunningContainer container; - public ContainerInfoExtractor(StartupWorkflow workflow, RunningContainer container) + public ContainerInfoExtractor(BaseLog log, StartupWorkflow workflow, RunningContainer container) { + this.log = log; this.workflow = workflow; this.container = container; } - public string ExtractAccount() + public string ExtractAccount(int? orderNumber) { - var account = Retry(FetchAccount); + log.Debug(); + var account = Retry(() => FetchAccount(orderNumber)); if (string.IsNullOrEmpty(account)) throw new InvalidOperationException("Unable to fetch account for geth node. Test infra failure."); return account; @@ -25,15 +30,17 @@ namespace DistTestCore.Marketplace public string ExtractPubKey() { + log.Debug(); var pubKey = Retry(FetchPubKey); if (string.IsNullOrEmpty(pubKey)) throw new InvalidOperationException("Unable to fetch enode from geth node. Test infra failure."); return pubKey; } - public string ExtractBootstrapPrivateKey() + public string ExtractPrivateKey(int? orderNumber) { - var privKey = Retry(FetchBootstrapPrivateKey); + log.Debug(); + var privKey = Retry(() => FetchPrivateKey(orderNumber)); if (string.IsNullOrEmpty(privKey)) throw new InvalidOperationException("Unable to fetch private key from geth node. Test infra failure."); return privKey; @@ -41,20 +48,31 @@ namespace DistTestCore.Marketplace public string ExtractMarketplaceAddress() { + log.Debug(); var marketplaceAddress = Retry(FetchMarketplaceAddress); if (string.IsNullOrEmpty(marketplaceAddress)) throw new InvalidOperationException("Unable to fetch marketplace account from codex-contracts node. Test infra failure."); return marketplaceAddress; } + public string ExtractMarketplaceAbi() + { + log.Debug(); + var marketplaceAbi = Retry(FetchMarketplaceAbi); + if (string.IsNullOrEmpty(marketplaceAbi)) throw new InvalidOperationException("Unable to fetch marketplace artifacts from codex-contracts node. Test infra failure."); + + return marketplaceAbi; + } + private string Retry(Func fetch) { - var result = Catch(fetch); - if (string.IsNullOrEmpty(result)) + var result = string.Empty; + Time.WaitUntil(() => { - Thread.Sleep(TimeSpan.FromSeconds(5)); - result = fetch(); - } + result = Catch(fetch); + return !string.IsNullOrEmpty(result); + }, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(3)); + return result; } @@ -70,14 +88,14 @@ namespace DistTestCore.Marketplace } } - private string FetchAccount() + private string FetchAccount(int? orderNumber) { - return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.AccountFilename); + return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.GetAccountFilename(orderNumber)); } - private string FetchBootstrapPrivateKey() + private string FetchPrivateKey(int? orderNumber) { - return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.BootstrapPrivateKeyFilename); + return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.GetPrivateKeyFilename(orderNumber)); } private string FetchMarketplaceAddress() @@ -87,6 +105,15 @@ namespace DistTestCore.Marketplace return marketplace!.address; } + private string FetchMarketplaceAbi() + { + var json = workflow.ExecuteCommand(container, "cat", CodexContractsContainerRecipe.MarketplaceArtifactFilename); + + var artifact = JObject.Parse(json); + var abi = artifact["abi"]; + return abi!.ToString(Formatting.None); + } + private string FetchPubKey() { var enodeFinder = new PubKeyFinder(); @@ -97,7 +124,8 @@ namespace DistTestCore.Marketplace public class PubKeyFinder : LogHandler, ILogHandler { - private const string openTag = "self=\"enode://"; + private const string openTag = "self=enode://"; + private const string openTagQuote = "self=\"enode://"; private string pubKey = string.Empty; public string GetPubKey() @@ -109,13 +137,17 @@ namespace DistTestCore.Marketplace { if (line.Contains(openTag)) { - ExtractPubKey(line); + ExtractPubKey(openTag, line); + } + else if (line.Contains(openTagQuote)) + { + ExtractPubKey(openTagQuote, line); } } - private void ExtractPubKey(string line) + private void ExtractPubKey(string tag, string line) { - var openIndex = line.IndexOf(openTag) + openTag.Length; + var openIndex = line.IndexOf(tag) + tag.Length; var closeIndex = line.IndexOf("@"); pubKey = line.Substring( diff --git a/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs b/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs index 0ea4c690..0c19fbf7 100644 --- a/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs +++ b/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs @@ -21,7 +21,7 @@ namespace DistTestCore.Marketplace public string PrivateKey { get; } public Port DiscoveryPort { get; } - public NethereumInteraction StartInteraction(TestLog log) + public NethereumInteraction StartInteraction(BaseLog log) { var ip = RunningContainers.RunningPod.Cluster.IP; var port = RunningContainers.Containers[0].ServicePorts[0].Number; diff --git a/DistTestCore/Marketplace/GethBootstrapNodeStarter.cs b/DistTestCore/Marketplace/GethBootstrapNodeStarter.cs index e0efc61b..300893b2 100644 --- a/DistTestCore/Marketplace/GethBootstrapNodeStarter.cs +++ b/DistTestCore/Marketplace/GethBootstrapNodeStarter.cs @@ -19,10 +19,10 @@ namespace DistTestCore.Marketplace if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 Geth bootstrap node to be created. Test infra failure."); var bootstrapContainer = containers.Containers[0]; - var extractor = new ContainerInfoExtractor(workflow, bootstrapContainer); - var account = extractor.ExtractAccount(); + var extractor = new ContainerInfoExtractor(lifecycle.Log, workflow, bootstrapContainer); + var account = extractor.ExtractAccount(null); var pubKey = extractor.ExtractPubKey(); - var privateKey = extractor.ExtractBootstrapPrivateKey(); + var privateKey = extractor.ExtractPrivateKey(null); var discoveryPort = bootstrapContainer.Recipe.GetPortByTag(GethContainerRecipe.DiscoveryPortTag); LogEnd($"Geth bootstrap node started with account '{account}'"); @@ -33,7 +33,7 @@ namespace DistTestCore.Marketplace private StartupConfig CreateBootstrapStartupConfig() { var config = new StartupConfig(); - config.Add(new GethStartupConfig(true, null!)); + config.Add(new GethStartupConfig(true, null!, 0)); return config; } } diff --git a/DistTestCore/Marketplace/GethCompanionNodeInfo.cs b/DistTestCore/Marketplace/GethCompanionNodeInfo.cs index 9b7bd23d..64e8ad90 100644 --- a/DistTestCore/Marketplace/GethCompanionNodeInfo.cs +++ b/DistTestCore/Marketplace/GethCompanionNodeInfo.cs @@ -1,16 +1,41 @@ using KubernetesWorkflow; +using Logging; +using NethereumWorkflow; namespace DistTestCore.Marketplace { public class GethCompanionNodeInfo { - public GethCompanionNodeInfo(RunningContainer runningContainer, string account) + public GethCompanionNodeInfo(RunningContainer runningContainer, GethCompanionAccount[] accounts) { RunningContainer = runningContainer; - Account = account; + Accounts = accounts; } public RunningContainer RunningContainer { get; } + public GethCompanionAccount[] Accounts { get; } + + public NethereumInteraction StartInteraction(BaseLog log, GethCompanionAccount account) + { + var ip = RunningContainer.Pod.Cluster.IP; + var port = RunningContainer.ServicePorts[0].Number; + var accountStr = account.Account; + var privateKey = account.PrivateKey; + + var creator = new NethereumInteractionCreator(log, ip, port, accountStr, privateKey); + return creator.CreateWorkflow(); + } + } + + public class GethCompanionAccount + { + public GethCompanionAccount(string account, string privateKey) + { + Account = account; + PrivateKey = privateKey; + } + public string Account { get; } + public string PrivateKey { get; } } } diff --git a/DistTestCore/Marketplace/GethCompanionNodeStarter.cs b/DistTestCore/Marketplace/GethCompanionNodeStarter.cs index 047af289..80c1bcdc 100644 --- a/DistTestCore/Marketplace/GethCompanionNodeStarter.cs +++ b/DistTestCore/Marketplace/GethCompanionNodeStarter.cs @@ -1,4 +1,5 @@ using KubernetesWorkflow; +using Utils; namespace DistTestCore.Marketplace { @@ -9,34 +10,68 @@ namespace DistTestCore.Marketplace { } - public GethCompanionNodeInfo[] StartCompanionNodesFor(CodexSetup codexSetup, GethBootstrapNodeInfo bootstrapNode) + public GethCompanionNodeInfo StartCompanionNodeFor(CodexSetup codexSetup, MarketplaceNetwork marketplace) { - LogStart($"Initializing companions for {codexSetup.NumberOfNodes} Codex nodes."); + LogStart($"Initializing companion for {codexSetup.NumberOfNodes} Codex nodes."); - var startupConfig = CreateCompanionNodeStartupConfig(bootstrapNode); + var startupConfig = CreateCompanionNodeStartupConfig(marketplace.Bootstrap, codexSetup.NumberOfNodes); var workflow = workflowCreator.CreateWorkflow(); - var containers = workflow.Start(codexSetup.NumberOfNodes, Location.Unspecified, new GethContainerRecipe(), startupConfig); - if (containers.Containers.Length != codexSetup.NumberOfNodes) throw new InvalidOperationException("Expected a Geth companion node to be created for each Codex node. Test infra failure."); + var containers = workflow.Start(1, Location.Unspecified, new GethContainerRecipe(), startupConfig); + WaitForAccountCreation(codexSetup.NumberOfNodes); + if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected one Geth companion node to be created. Test infra failure."); + var container = containers.Containers[0]; - var result = containers.Containers.Select(c => CreateCompanionInfo(workflow, c)).ToArray(); + var node = CreateCompanionInfo(workflow, container, codexSetup.NumberOfNodes); + EnsureCompanionNodeIsSynced(node, marketplace); - LogEnd($"Initialized {codexSetup.NumberOfNodes} companion nodes. Their accounts: [{string.Join(",", result.Select(c => c.Account))}]"); - - return result; + LogEnd($"Initialized one companion node for {codexSetup.NumberOfNodes} Codex nodes. Their accounts: [{string.Join(",", node.Accounts.Select(a => a.Account))}]"); + return node; } - private GethCompanionNodeInfo CreateCompanionInfo(StartupWorkflow workflow, RunningContainer container) + private void WaitForAccountCreation(int numberOfNodes) { - var extractor = new ContainerInfoExtractor(workflow, container); - var account = extractor.ExtractAccount(); - return new GethCompanionNodeInfo(container, account); + // We wait proportional to the number of account the node has to create. It takes a few seconds for each one to generate the keys and create the files + // we will be trying to read in 'ExtractAccount', later on in the start-up process. + Time.Sleep(TimeSpan.FromSeconds(4.5 * numberOfNodes)); } - private StartupConfig CreateCompanionNodeStartupConfig(GethBootstrapNodeInfo bootstrapNode) + private GethCompanionNodeInfo CreateCompanionInfo(StartupWorkflow workflow, RunningContainer container, int numberOfAccounts) + { + var extractor = new ContainerInfoExtractor(lifecycle.Log, workflow, container); + var accounts = ExtractAccounts(extractor, numberOfAccounts).ToArray(); + return new GethCompanionNodeInfo(container, accounts); + } + + private IEnumerable ExtractAccounts(ContainerInfoExtractor extractor, int numberOfAccounts) + { + for (int i = 0; i < numberOfAccounts; i++) yield return ExtractAccount(extractor, i + 1); + } + + private GethCompanionAccount ExtractAccount(ContainerInfoExtractor extractor, int orderNumber) + { + var account = extractor.ExtractAccount(orderNumber); + var privKey = extractor.ExtractPrivateKey(orderNumber); + return new GethCompanionAccount(account, privKey); + } + + private void EnsureCompanionNodeIsSynced(GethCompanionNodeInfo node, MarketplaceNetwork marketplace) + { + try + { + var interaction = node.StartInteraction(lifecycle.Log, node.Accounts.First()); + interaction.EnsureSynced(marketplace.Marketplace.Address, marketplace.Marketplace.Abi); + } + catch (Exception e) + { + throw new Exception("Geth companion node did not sync within timeout. Test infra failure.", e); + } + } + + private StartupConfig CreateCompanionNodeStartupConfig(GethBootstrapNodeInfo bootstrapNode, int numberOfAccounts) { var config = new StartupConfig(); - config.Add(new GethStartupConfig(false, bootstrapNode)); + config.Add(new GethStartupConfig(false, bootstrapNode, numberOfAccounts)); return config; } } diff --git a/DistTestCore/Marketplace/GethContainerRecipe.cs b/DistTestCore/Marketplace/GethContainerRecipe.cs index 5067cf7e..f2693acd 100644 --- a/DistTestCore/Marketplace/GethContainerRecipe.cs +++ b/DistTestCore/Marketplace/GethContainerRecipe.cs @@ -6,10 +6,20 @@ namespace DistTestCore.Marketplace { public const string DockerImage = "thatbenbierens/geth-confenv:latest"; public const string HttpPortTag = "http_port"; - public const string WsPortTag = "ws_port"; public const string DiscoveryPortTag = "disc_port"; - public const string AccountFilename = "account_string.txt"; - public const string BootstrapPrivateKeyFilename = "bootstrap_private.key"; + private const string defaultArgs = "--ipcdisable --syncmode full"; + + public static string GetAccountFilename(int? orderNumber) + { + if (orderNumber == null) return "account_string.txt"; + return $"account_string_{orderNumber.Value}.txt"; + } + + public static string GetPrivateKeyFilename(int? orderNumber) + { + if (orderNumber == null) return "private.key"; + return $"private_{orderNumber.Value}.key"; + } protected override string Image => DockerImage; @@ -28,22 +38,33 @@ namespace DistTestCore.Marketplace if (config.IsBootstrapNode) { - AddEnvVar("IS_BOOTSTRAP", "1"); - var exposedPort = AddExposedPort(tag: HttpPortTag); - return $"--http.port {exposedPort.Number} --discovery.port {discovery.Number} --nodiscover"; + return CreateBootstapArgs(discovery); } + return CreateCompanionArgs(discovery, config); + } + + private string CreateBootstapArgs(Port discovery) + { + AddEnvVar("IS_BOOTSTRAP", "1"); + var exposedPort = AddExposedPort(tag: HttpPortTag); + return $"--http.port {exposedPort.Number} --port {discovery.Number} --discovery.port {discovery.Number} {defaultArgs}"; + } + + private string CreateCompanionArgs(Port discovery, GethStartupConfig config) + { + AddEnvVar("NUMBER_OF_ACCOUNTS", config.NumberOfCompanionAccounts.ToString()); + var port = AddInternalPort(); var authRpc = AddInternalPort(); - var httpPort = AddInternalPort(tag: HttpPortTag); - var wsPort = AddInternalPort(tag: WsPortTag); + var httpPort = AddExposedPort(tag: HttpPortTag); var bootPubKey = config.BootstrapNode.PubKey; var bootIp = config.BootstrapNode.RunningContainers.Containers[0].Pod.Ip; var bootPort = config.BootstrapNode.DiscoveryPort.Number; - var bootstrapArg = $"--bootnodes enode://{bootPubKey}@{bootIp}:{bootPort}"; + var bootstrapArg = $"--bootnodes enode://{bootPubKey}@{bootIp}:{bootPort} --nat=extip:{bootIp}"; - return $"--port {port.Number} --discovery.port {discovery.Number} --authrpc.port {authRpc.Number} --http.port {httpPort.Number} --ws --ws.addr 0.0.0.0 --ws.port {wsPort.Number} --nodiscover {bootstrapArg}"; + return $"--port {port.Number} --discovery.port {discovery.Number} --authrpc.port {authRpc.Number} --http.addr 0.0.0.0 --http.port {httpPort.Number} --ws --ws.addr 0.0.0.0 --ws.port {httpPort.Number} {bootstrapArg} {defaultArgs}"; } } } diff --git a/DistTestCore/Marketplace/GethStartResult.cs b/DistTestCore/Marketplace/GethStartResult.cs index 2f0d24d2..412d2887 100644 --- a/DistTestCore/Marketplace/GethStartResult.cs +++ b/DistTestCore/Marketplace/GethStartResult.cs @@ -2,15 +2,15 @@ { public class GethStartResult { - public GethStartResult(IMarketplaceAccessFactory marketplaceAccessFactory, MarketplaceNetwork marketplaceNetwork, GethCompanionNodeInfo[] companionNodes) + public GethStartResult(IMarketplaceAccessFactory marketplaceAccessFactory, MarketplaceNetwork marketplaceNetwork, GethCompanionNodeInfo companionNode) { MarketplaceAccessFactory = marketplaceAccessFactory; MarketplaceNetwork = marketplaceNetwork; - CompanionNodes = companionNodes; + CompanionNode = companionNode; } public IMarketplaceAccessFactory MarketplaceAccessFactory { get; } public MarketplaceNetwork MarketplaceNetwork { get; } - public GethCompanionNodeInfo[] CompanionNodes { get; } + public GethCompanionNodeInfo CompanionNode { get; } } } diff --git a/DistTestCore/Marketplace/GethStartupConfig.cs b/DistTestCore/Marketplace/GethStartupConfig.cs index a8026f1a..bc9671fc 100644 --- a/DistTestCore/Marketplace/GethStartupConfig.cs +++ b/DistTestCore/Marketplace/GethStartupConfig.cs @@ -2,13 +2,15 @@ { public class GethStartupConfig { - public GethStartupConfig(bool isBootstrapNode, GethBootstrapNodeInfo bootstrapNode) + public GethStartupConfig(bool isBootstrapNode, GethBootstrapNodeInfo bootstrapNode, int numberOfCompanionAccounts) { IsBootstrapNode = isBootstrapNode; BootstrapNode = bootstrapNode; + NumberOfCompanionAccounts = numberOfCompanionAccounts; } public bool IsBootstrapNode { get; } public GethBootstrapNodeInfo BootstrapNode { get; } + public int NumberOfCompanionAccounts { get; } } } diff --git a/DistTestCore/Marketplace/MarketplaceAccess.cs b/DistTestCore/Marketplace/MarketplaceAccess.cs index 881ce931..b66679c0 100644 --- a/DistTestCore/Marketplace/MarketplaceAccess.cs +++ b/DistTestCore/Marketplace/MarketplaceAccess.cs @@ -19,14 +19,14 @@ namespace DistTestCore.Marketplace { private readonly TestLog log; private readonly MarketplaceNetwork marketplaceNetwork; - private readonly GethCompanionNodeInfo companionNode; + private readonly GethCompanionAccount account; private readonly CodexAccess codexAccess; - public MarketplaceAccess(TestLog log, MarketplaceNetwork marketplaceNetwork, GethCompanionNodeInfo companionNode, CodexAccess codexAccess) + public MarketplaceAccess(TestLog log, MarketplaceNetwork marketplaceNetwork, GethCompanionAccount account, CodexAccess codexAccess) { this.log = log; this.marketplaceNetwork = marketplaceNetwork; - this.companionNode = companionNode; + this.account = account; this.codexAccess = codexAccess; } @@ -52,9 +52,14 @@ namespace DistTestCore.Marketplace var response = codexAccess.RequestStorage(request, contentId.Id); - Log($"Storage requested successfully. PurchaseId: {response.purchaseId}"); + if (response == "Purchasing not available") + { + throw new InvalidOperationException(response); + } - return response.purchaseId; + Log($"Storage requested successfully. PurchaseId: {response}"); + + return response; } public string MakeStorageAvailable(ByteSize size, TestToken minPricePerBytePerSecond, TestToken maxCollateral, TimeSpan maxDuration) @@ -99,18 +104,17 @@ namespace DistTestCore.Marketplace public TestToken GetBalance() { var interaction = marketplaceNetwork.StartInteraction(log); - var account = companionNode.Account; - var amount = interaction.GetBalance(marketplaceNetwork.Marketplace.TokenAddress, account); + var amount = interaction.GetBalance(marketplaceNetwork.Marketplace.TokenAddress, account.Account); var balance = new TestToken(amount); - Log($"Balance of {account} is {balance}."); + Log($"Balance of {account.Account} is {balance}."); return balance; } private void Log(string msg) { - log.Log($"{codexAccess.Container.GetName()} {msg}"); + log.Log($"{codexAccess.Container.Name} {msg}"); } } diff --git a/DistTestCore/Marketplace/MarketplaceAccessFactory.cs b/DistTestCore/Marketplace/MarketplaceAccessFactory.cs index b0389e1d..ea4786c7 100644 --- a/DistTestCore/Marketplace/MarketplaceAccessFactory.cs +++ b/DistTestCore/Marketplace/MarketplaceAccessFactory.cs @@ -33,10 +33,10 @@ namespace DistTestCore.Marketplace return new MarketplaceAccess(log, marketplaceNetwork, companionNode, access); } - private GethCompanionNodeInfo GetGethCompanionNode(CodexAccess access) + private GethCompanionAccount GetGethCompanionNode(CodexAccess access) { - var node = access.Container.Recipe.Additionals.Single(a => a is GethCompanionNodeInfo); - return (GethCompanionNodeInfo)node; + var account = access.Container.Recipe.Additionals.Single(a => a is GethCompanionAccount); + return (GethCompanionAccount)account; } } } diff --git a/DistTestCore/Marketplace/MarketplaceNetwork.cs b/DistTestCore/Marketplace/MarketplaceNetwork.cs index 5f43fa91..83784449 100644 --- a/DistTestCore/Marketplace/MarketplaceNetwork.cs +++ b/DistTestCore/Marketplace/MarketplaceNetwork.cs @@ -14,7 +14,7 @@ namespace DistTestCore.Marketplace public GethBootstrapNodeInfo Bootstrap { get; } public MarketplaceInfo Marketplace { get; } - public NethereumInteraction StartInteraction(TestLog log) + public NethereumInteraction StartInteraction(BaseLog log) { return Bootstrap.StartInteraction(log); } diff --git a/DistTestCore/Metrics/MetricsAccess.cs b/DistTestCore/Metrics/MetricsAccess.cs index fffaea81..f0d4ff93 100644 --- a/DistTestCore/Metrics/MetricsAccess.cs +++ b/DistTestCore/Metrics/MetricsAccess.cs @@ -29,7 +29,7 @@ namespace DistTestCore.Metrics var metricSet = GetMetricWithTimeout(metricName); var metricValue = metricSet.Values[0].Value; - log.Log($"{node.GetName()} metric '{metricName}' = {metricValue}"); + log.Log($"{node.Name} metric '{metricName}' = {metricValue}"); Assert.That(metricValue, constraint, message); } diff --git a/DistTestCore/Metrics/MetricsAccessFactory.cs b/DistTestCore/Metrics/MetricsAccessFactory.cs index dad95b8e..6f93886b 100644 --- a/DistTestCore/Metrics/MetricsAccessFactory.cs +++ b/DistTestCore/Metrics/MetricsAccessFactory.cs @@ -28,7 +28,7 @@ namespace DistTestCore.Metrics public IMetricsAccess CreateMetricsAccess(RunningContainer codexContainer) { - var query = new MetricsQuery(prometheusContainer); + var query = new MetricsQuery(lifecycle.Log, prometheusContainer); return new MetricsAccess(lifecycle.Log, query, codexContainer); } } diff --git a/DistTestCore/Metrics/MetricsQuery.cs b/DistTestCore/Metrics/MetricsQuery.cs index c06fc8de..cc8bd671 100644 --- a/DistTestCore/Metrics/MetricsQuery.cs +++ b/DistTestCore/Metrics/MetricsQuery.cs @@ -1,5 +1,6 @@ using DistTestCore.Codex; using KubernetesWorkflow; +using Logging; using System.Globalization; namespace DistTestCore.Metrics @@ -8,11 +9,12 @@ namespace DistTestCore.Metrics { private readonly Http http; - public MetricsQuery(RunningContainers runningContainers) + public MetricsQuery(BaseLog log, RunningContainers runningContainers) { RunningContainers = runningContainers; http = new Http( + log, runningContainers.RunningPod.Cluster.IP, runningContainers.Containers[0].ServicePorts[0].Number, "api/v1"); diff --git a/DistTestCore/OnlineCodexNode.cs b/DistTestCore/OnlineCodexNode.cs index 43e16ec2..91cf11a6 100644 --- a/DistTestCore/OnlineCodexNode.cs +++ b/DistTestCore/OnlineCodexNode.cs @@ -16,6 +16,7 @@ namespace DistTestCore ICodexNodeLog DownloadLog(); IMetricsAccess Metrics { get; } IMarketplaceAccess Marketplace { get; } + ICodexSetup BringOffline(); } public class OnlineCodexNode : IOnlineCodexNode @@ -23,7 +24,6 @@ namespace DistTestCore private const string SuccessfullyConnectedMessage = "Successfully connected to peer"; private const string UploadFailedMessage = "Unable to store block"; private readonly TestLifecycle lifecycle; - private CodexDebugResponse? debugInfo; public OnlineCodexNode(TestLifecycle lifecycle, CodexAccess codexAccess, CodexNodeGroup group, IMetricsAccess metricsAccess, IMarketplaceAccess marketplaceAccess) { @@ -41,14 +41,12 @@ namespace DistTestCore public string GetName() { - return CodexAccess.Container.GetName(); + return CodexAccess.Container.Name; } public CodexDebugResponse GetDebugInfo() { - if (debugInfo != null) return debugInfo; - - debugInfo = CodexAccess.GetDebugInfo(); + var debugInfo = CodexAccess.GetDebugInfo(); Log($"Got DebugInfo with id: '{debugInfo.id}'."); return debugInfo; } @@ -92,6 +90,11 @@ namespace DistTestCore return lifecycle.DownloadLog(this); } + public ICodexSetup BringOffline() + { + return Group.BringOffline(); + } + private string GetPeerMultiAddress(OnlineCodexNode peer, CodexDebugResponse peerInfo) { var multiAddress = peerInfo.addrs.First(); diff --git a/DistTestCore/Stopwatch.cs b/DistTestCore/Stopwatch.cs deleted file mode 100644 index 878912f0..00000000 --- a/DistTestCore/Stopwatch.cs +++ /dev/null @@ -1,36 +0,0 @@ -using Logging; -using Utils; - -namespace DistTestCore -{ - public class Stopwatch - { - private readonly DateTime start = DateTime.UtcNow; - private readonly BaseLog log; - private readonly string name; - - public Stopwatch(BaseLog log, string name) - { - this.log = log; - this.name = name; - } - - public static void Measure(BaseLog log, string name, Action action) - { - var sw = Begin(log, name); - action(); - sw.End(); - } - - public static Stopwatch Begin(BaseLog log, string name) - { - return new Stopwatch(log, name); - } - - public void End(string msg = "") - { - var duration = DateTime.UtcNow - start; - log.Log($"{name} {msg} ({Time.FormatDuration(duration)})"); - } - } -} diff --git a/DistTestCore/TestLifecycle.cs b/DistTestCore/TestLifecycle.cs index 2b243e19..4dddf34b 100644 --- a/DistTestCore/TestLifecycle.cs +++ b/DistTestCore/TestLifecycle.cs @@ -11,7 +11,7 @@ namespace DistTestCore public TestLifecycle(TestLog log, Configuration configuration) { Log = log; - workflowCreator = new WorkflowCreator(configuration.GetK8sConfiguration()); + workflowCreator = new WorkflowCreator(log, configuration.GetK8sConfiguration()); FileManager = new FileManager(Log, configuration); CodexStarter = new CodexStarter(this, workflowCreator); diff --git a/DistTestCore/Timing.cs b/DistTestCore/Timing.cs index 67653ece..3cfc1bc3 100644 --- a/DistTestCore/Timing.cs +++ b/DistTestCore/Timing.cs @@ -6,15 +6,12 @@ namespace DistTestCore [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] public class UseLongTimeoutsAttribute : PropertyAttribute { - public UseLongTimeoutsAttribute() - : base(Timing.UseLongTimeoutsKey) - { - } } public static class Timing { - public const string UseLongTimeoutsKey = "UseLongTimeouts"; + public static bool UseLongTimeouts { get; set; } + public static TimeSpan HttpCallTimeout() { @@ -48,8 +45,7 @@ namespace DistTestCore private static ITimeSet GetTimes() { - var testProperties = TestContext.CurrentContext.Test.Properties; - if (testProperties.ContainsKey(UseLongTimeoutsKey)) return new LongTimeSet(); + if (UseLongTimeouts) return new LongTimeSet(); return new DefaultTimeSet(); } } diff --git a/DistTestCore/Tokens.cs b/DistTestCore/Tokens.cs index 07d16929..5593ffcf 100644 --- a/DistTestCore/Tokens.cs +++ b/DistTestCore/Tokens.cs @@ -1,6 +1,6 @@ namespace DistTestCore { - public class Ether + public class Ether : IComparable { public Ether(decimal wei) { @@ -9,6 +9,11 @@ public decimal Wei { get; } + public int CompareTo(Ether? other) + { + return Wei.CompareTo(other!.Wei); + } + public override bool Equals(object? obj) { return obj is Ether ether && Wei == ether.Wei; @@ -25,7 +30,7 @@ } } - public class TestToken + public class TestToken : IComparable { public TestToken(decimal amount) { @@ -34,6 +39,11 @@ public decimal Amount { get; } + public int CompareTo(TestToken? other) + { + return Amount.CompareTo(other!.Amount); + } + public override bool Equals(object? obj) { return obj is TestToken token && Amount == token.Amount; diff --git a/KubernetesWorkflow/K8sController.cs b/KubernetesWorkflow/K8sController.cs index 611c8fec..21e82661 100644 --- a/KubernetesWorkflow/K8sController.cs +++ b/KubernetesWorkflow/K8sController.cs @@ -1,18 +1,21 @@ using k8s; using k8s.Models; +using Logging; using Utils; namespace KubernetesWorkflow { public class K8sController { + private readonly BaseLog log; private readonly K8sCluster cluster; private readonly KnownK8sPods knownPods; private readonly WorkflowNumberSource workflowNumberSource; private readonly Kubernetes client; - public K8sController(K8sCluster cluster, KnownK8sPods knownPods, WorkflowNumberSource workflowNumberSource) + public K8sController(BaseLog log, K8sCluster cluster, KnownK8sPods knownPods, WorkflowNumberSource workflowNumberSource) { + this.log = log; this.cluster = cluster; this.knownPods = knownPods; this.workflowNumberSource = workflowNumberSource; @@ -27,6 +30,7 @@ namespace KubernetesWorkflow public RunningPod BringOnline(ContainerRecipe[] containerRecipes, Location location) { + log.Debug(); EnsureTestNamespace(); var deploymentName = CreateDeployment(containerRecipes, location); @@ -38,6 +42,7 @@ namespace KubernetesWorkflow public void Stop(RunningPod pod) { + log.Debug(); if (!string.IsNullOrEmpty(pod.ServiceName)) DeleteService(pod.ServiceName); DeleteDeployment(pod.DeploymentName); WaitUntilDeploymentOffline(pod.DeploymentName); @@ -46,12 +51,14 @@ namespace KubernetesWorkflow public void DownloadPodLog(RunningPod pod, ContainerRecipe recipe, ILogHandler logHandler) { + log.Debug(); using var stream = client.ReadNamespacedPodLog(pod.Name, K8sNamespace, recipe.Name); logHandler.Log(stream); } public string ExecuteCommand(RunningPod pod, string containerName, string command, params string[] args) { + log.Debug($"{containerName}: {command} ({string.Join(",", args)})"); var runner = new CommandRunner(client, K8sNamespace, pod, containerName, command, args); runner.Run(); return runner.GetStdOut(); @@ -59,6 +66,7 @@ namespace KubernetesWorkflow public void DeleteAllResources() { + log.Debug(); DeleteNamespace(); WaitUntilNamespaceDeleted(); @@ -346,7 +354,15 @@ namespace KubernetesWorkflow private void WaitUntil(Func predicate) { - Time.WaitUntil(predicate, cluster.K8sOperationTimeout(), cluster.WaitForK8sServiceDelay()); + var sw = Stopwatch.Begin(log, true); + try + { + Time.WaitUntil(predicate, cluster.K8sOperationTimeout(), cluster.WaitForK8sServiceDelay()); + } + finally + { + sw.End("", 1); + } } #endregion diff --git a/KubernetesWorkflow/KubernetesWorkflow.csproj b/KubernetesWorkflow/KubernetesWorkflow.csproj index 655a8fd9..dbfbf965 100644 --- a/KubernetesWorkflow/KubernetesWorkflow.csproj +++ b/KubernetesWorkflow/KubernetesWorkflow.csproj @@ -12,6 +12,7 @@ + diff --git a/KubernetesWorkflow/RunningContainers.cs b/KubernetesWorkflow/RunningContainers.cs index 32335ff9..616056c2 100644 --- a/KubernetesWorkflow/RunningContainers.cs +++ b/KubernetesWorkflow/RunningContainers.cs @@ -15,26 +15,35 @@ public string Describe() { - return string.Join(",", Containers.Select(c => c.GetName())); + return string.Join(",", Containers.Select(c => c.Name)); } } public class RunningContainer { - public RunningContainer(RunningPod pod, ContainerRecipe recipe, Port[] servicePorts) + public RunningContainer(RunningPod pod, ContainerRecipe recipe, Port[] servicePorts, StartupConfig startupConfig) { Pod = pod; Recipe = recipe; ServicePorts = servicePorts; + Name = GetContainerName(recipe, startupConfig); } - public string GetName() - { - return $"<{Recipe.Name}>"; - } - + public string Name { get; } public RunningPod Pod { get; } public ContainerRecipe Recipe { get; } public Port[] ServicePorts { get; } + + private string GetContainerName(ContainerRecipe recipe, StartupConfig startupConfig) + { + if (!string.IsNullOrEmpty(startupConfig.NameOverride)) + { + return $"<{startupConfig.NameOverride}{recipe.Number}>"; + } + else + { + return $"<{recipe.Name}>"; + } + } } } diff --git a/KubernetesWorkflow/StartupConfig.cs b/KubernetesWorkflow/StartupConfig.cs index 406cea96..76a96b67 100644 --- a/KubernetesWorkflow/StartupConfig.cs +++ b/KubernetesWorkflow/StartupConfig.cs @@ -4,6 +4,8 @@ { private readonly List configs = new List(); + public string? NameOverride { get; set; } + public void Add(object config) { configs.Add(config); diff --git a/KubernetesWorkflow/StartupWorkflow.cs b/KubernetesWorkflow/StartupWorkflow.cs index d37699f7..8aaba75e 100644 --- a/KubernetesWorkflow/StartupWorkflow.cs +++ b/KubernetesWorkflow/StartupWorkflow.cs @@ -1,16 +1,18 @@ -using System.IO; +using Logging; namespace KubernetesWorkflow { public class StartupWorkflow { + private readonly BaseLog log; private readonly WorkflowNumberSource numberSource; private readonly K8sCluster cluster; private readonly KnownK8sPods knownK8SPods; private readonly RecipeComponentFactory componentFactory = new RecipeComponentFactory(); - internal StartupWorkflow(WorkflowNumberSource numberSource, K8sCluster cluster, KnownK8sPods knownK8SPods) + internal StartupWorkflow(BaseLog log, WorkflowNumberSource numberSource, K8sCluster cluster, KnownK8sPods knownK8SPods) { + this.log = log; this.numberSource = numberSource; this.cluster = cluster; this.knownK8SPods = knownK8SPods; @@ -24,7 +26,7 @@ namespace KubernetesWorkflow var runningPod = controller.BringOnline(recipes, location); - return new RunningContainers(startupConfig, runningPod, CreateContainers(runningPod, recipes)); + return new RunningContainers(startupConfig, runningPod, CreateContainers(runningPod, recipes, startupConfig)); }); } @@ -60,13 +62,15 @@ namespace KubernetesWorkflow }); } - private static RunningContainer[] CreateContainers(RunningPod runningPod, ContainerRecipe[] recipes) + private RunningContainer[] CreateContainers(RunningPod runningPod, ContainerRecipe[] recipes, StartupConfig startupConfig) { - return recipes.Select(r => new RunningContainer(runningPod, r, runningPod.GetServicePortsForContainerRecipe(r))).ToArray(); + log.Debug(); + return recipes.Select(r => new RunningContainer(runningPod, r, runningPod.GetServicePortsForContainerRecipe(r), startupConfig)).ToArray(); } private ContainerRecipe[] CreateRecipes(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig) { + log.Debug(); var result = new List(); for (var i = 0; i < numberOfContainers; i++) { @@ -78,14 +82,14 @@ namespace KubernetesWorkflow private void K8s(Action action) { - var controller = new K8sController(cluster, knownK8SPods, numberSource); + var controller = new K8sController(log, cluster, knownK8SPods, numberSource); action(controller); controller.Dispose(); } private T K8s(Func action) { - var controller = new K8sController(cluster, knownK8SPods, numberSource); + var controller = new K8sController(log, cluster, knownK8SPods, numberSource); var result = action(controller); controller.Dispose(); return result; diff --git a/KubernetesWorkflow/WorkflowCreator.cs b/KubernetesWorkflow/WorkflowCreator.cs index 51c1c29a..1f54beae 100644 --- a/KubernetesWorkflow/WorkflowCreator.cs +++ b/KubernetesWorkflow/WorkflowCreator.cs @@ -1,4 +1,5 @@ -using Utils; +using Logging; +using Utils; namespace KubernetesWorkflow { @@ -9,10 +10,12 @@ namespace KubernetesWorkflow private readonly NumberSource containerNumberSource = new NumberSource(0); private readonly KnownK8sPods knownPods = new KnownK8sPods(); private readonly K8sCluster cluster; + private readonly BaseLog log; - public WorkflowCreator(Configuration configuration) + public WorkflowCreator(BaseLog log, Configuration configuration) { cluster = new K8sCluster(configuration); + this.log = log; } public StartupWorkflow CreateWorkflow() @@ -21,7 +24,7 @@ namespace KubernetesWorkflow servicePortNumberSource, containerNumberSource); - return new StartupWorkflow(workflowNumberSource, cluster, knownPods); + return new StartupWorkflow(log, workflowNumberSource, cluster, knownPods); } } } diff --git a/Logging/BaseLog.cs b/Logging/BaseLog.cs index e19ec128..fb4769b0 100644 --- a/Logging/BaseLog.cs +++ b/Logging/BaseLog.cs @@ -1,10 +1,19 @@ -namespace Logging +using Utils; + +namespace Logging { public abstract class BaseLog { + private readonly bool debug; + private readonly List replacements = new List(); private bool hasFailed; private LogFile? logFile; - + + protected BaseLog(bool debug) + { + this.debug = debug; + } + protected abstract LogFile CreateLogFile(); protected LogFile LogFile @@ -18,7 +27,17 @@ public void Log(string message) { - LogFile.Write(message); + LogFile.Write(ApplyReplacements(message)); + } + + public void Debug(string message = "", int skipFrames = 0) + { + if (debug) + { + var callerName = DebugStack.GetCallerName(skipFrames); + // We don't use Log because in the debug output we should not have any replacements. + LogFile.Write($"(debug)({callerName}) {message}"); + } } public void Error(string message) @@ -32,5 +51,38 @@ hasFailed = true; LogFile.ConcatToFilename("_FAILED"); } + + public void AddStringReplace(string from, string to) + { + replacements.Add(new BaseLogStringReplacement(from, to)); + } + + private string ApplyReplacements(string str) + { + foreach (var replacement in replacements) + { + str = replacement.Apply(str); + } + return str; + } + } + + public class BaseLogStringReplacement + { + private readonly string from; + private readonly string to; + + public BaseLogStringReplacement(string from, string to) + { + this.from = from; + this.to = to; + + if (string.IsNullOrEmpty(from) || string.IsNullOrEmpty(to) || from == to) throw new ArgumentException(); + } + + public string Apply(string msg) + { + return msg.Replace(from, to); + } } } diff --git a/Logging/FixtureLog.cs b/Logging/FixtureLog.cs index 7bb80abd..1502ec9a 100644 --- a/Logging/FixtureLog.cs +++ b/Logging/FixtureLog.cs @@ -6,18 +6,21 @@ namespace Logging { private readonly DateTime start; private readonly string fullName; + private readonly LogConfig config; public FixtureLog(LogConfig config) + : base(config.DebugEnabled) { start = DateTime.UtcNow; var folder = DetermineFolder(config); var fixtureName = GetFixtureName(); fullName = Path.Combine(folder, fixtureName); + this.config = config; } public TestLog CreateTestLog() { - return new TestLog(fullName); + return new TestLog(fullName, config.DebugEnabled); } protected override LogFile CreateLogFile() diff --git a/Logging/LogConfig.cs b/Logging/LogConfig.cs index b7bc937c..eb91fdb5 100644 --- a/Logging/LogConfig.cs +++ b/Logging/LogConfig.cs @@ -2,11 +2,13 @@ { public class LogConfig { - public LogConfig(string logRoot) + public LogConfig(string logRoot, bool debugEnabled) { LogRoot = logRoot; + DebugEnabled = debugEnabled; } public string LogRoot { get; } + public bool DebugEnabled { get; } } } diff --git a/Logging/Stopwatch.cs b/Logging/Stopwatch.cs new file mode 100644 index 00000000..9f8a346d --- /dev/null +++ b/Logging/Stopwatch.cs @@ -0,0 +1,61 @@ +using Utils; + +namespace Logging +{ + public class Stopwatch + { + private readonly DateTime start = DateTime.UtcNow; + private readonly BaseLog log; + private readonly string name; + private readonly bool debug; + + private Stopwatch(BaseLog log, string name, bool debug) + { + this.log = log; + this.name = name; + this.debug = debug; + } + + public static void Measure(BaseLog log, string name, Action action, bool debug = false) + { + var sw = Begin(log, name, debug); + action(); + sw.End(); + } + + public static Stopwatch Begin(BaseLog log) + { + return Begin(log, ""); + } + + public static Stopwatch Begin(BaseLog log, string name) + { + return Begin(log, name, false); + } + + public static Stopwatch Begin(BaseLog log, bool debug) + { + return Begin(log, "", debug); + } + + public static Stopwatch Begin(BaseLog log, string name, bool debug) + { + return new Stopwatch(log, name, debug); + } + + public void End(string msg = "", int skipFrames = 0) + { + var duration = DateTime.UtcNow - start; + var entry = $"{name} {msg} ({Time.FormatDuration(duration)})"; + + if (debug) + { + log.Debug(entry, 1 + skipFrames); + } + else + { + log.Log(entry); + } + } + } +} diff --git a/Logging/TestLog.cs b/Logging/TestLog.cs index 066d84a8..39f70a0b 100644 --- a/Logging/TestLog.cs +++ b/Logging/TestLog.cs @@ -9,7 +9,8 @@ namespace Logging private readonly string methodName; private readonly string fullName; - public TestLog(string folder) + public TestLog(string folder, bool debug) + : base(debug) { methodName = GetMethodName(); fullName = Path.Combine(folder, methodName); diff --git a/LongTests/BasicTests/LargeFileTests.cs b/LongTests/BasicTests/LargeFileTests.cs index 3db5d596..63b33f51 100644 --- a/LongTests/BasicTests/LargeFileTests.cs +++ b/LongTests/BasicTests/LargeFileTests.cs @@ -10,10 +10,9 @@ namespace TestsLong.BasicTests [Test, UseLongTimeouts] public void OneClientLargeFileTest() { - var primary = SetupCodexNodes(1) + var primary = SetupCodexNode(s => s .WithLogLevel(CodexLogLevel.Warn) - .WithStorageQuota(20.GB()) - .BringOnline()[0]; + .WithStorageQuota(20.GB())); var testFile = GenerateTestFile(10.GB()); diff --git a/LongTests/BasicTests/TestInfraTests.cs b/LongTests/BasicTests/TestInfraTests.cs index 7acd78e9..0b5e6401 100644 --- a/LongTests/BasicTests/TestInfraTests.cs +++ b/LongTests/BasicTests/TestInfraTests.cs @@ -9,9 +9,7 @@ namespace TestsLong.BasicTests [Test, UseLongTimeouts] public void TestInfraShouldHave1000AddressSpacesPerPod() { - var group = SetupCodexNodes(1000) - .EnableMetrics() // Increases use of port address space per node. - .BringOnline(); + var group = SetupCodexNodes(1000, s => s.EnableMetrics()); // Increases use of port address space per node. var nodeIds = group.Select(n => n.GetDebugInfo().id).ToArray(); @@ -24,7 +22,7 @@ namespace TestsLong.BasicTests { for (var i = 0; i < 20; i++) { - var n = SetupCodexNodes(1).BringOnline()[0]; + var n = SetupCodexNode(); Assert.That(!string.IsNullOrEmpty(n.GetDebugInfo().id)); } @@ -33,10 +31,9 @@ namespace TestsLong.BasicTests [Test, UseLongTimeouts] public void DownloadConsistencyTest() { - var primary = SetupCodexNodes(1) + var primary = SetupCodexNode(s => s .WithLogLevel(CodexLogLevel.Trace) - .WithStorageQuota(2.MB()) - .BringOnline()[0]; + .WithStorageQuota(2.MB())); var testFile = GenerateTestFile(1.MB()); diff --git a/Nethereum/NethereumInteraction.cs b/Nethereum/NethereumInteraction.cs index 735bd6f1..3d74e54a 100644 --- a/Nethereum/NethereumInteraction.cs +++ b/Nethereum/NethereumInteraction.cs @@ -11,11 +11,11 @@ namespace NethereumWorkflow public class NethereumInteraction { private readonly List openTasks = new List(); - private readonly TestLog log; + private readonly BaseLog log; private readonly Web3 web3; private readonly string rootAccount; - internal NethereumInteraction(TestLog log, Web3 web3, string rootAccount) + internal NethereumInteraction(BaseLog log, Web3 web3, string rootAccount) { this.log = log; this.web3 = web3; @@ -24,6 +24,7 @@ namespace NethereumWorkflow public string GetTokenAddress(string marketplaceAddress) { + log.Debug(marketplaceAddress); var function = new GetTokenFunction(); var handler = web3.Eth.GetContractQueryHandler(); @@ -32,6 +33,7 @@ namespace NethereumWorkflow public void TransferWeiTo(string account, decimal amount) { + log.Debug($"{amount} --> {account}"); if (amount < 1 || string.IsNullOrEmpty(account)) throw new ArgumentException("Invalid arguments for AddToBalance"); var value = ToHexBig(amount); @@ -41,6 +43,7 @@ namespace NethereumWorkflow public void MintTestTokens(string account, decimal amount, string tokenAddress) { + log.Debug($"({tokenAddress}) {amount} --> {account}"); if (amount < 1 || string.IsNullOrEmpty(account)) throw new ArgumentException("Invalid arguments for MintTestTokens"); var function = new MintTokensFunction @@ -55,6 +58,7 @@ namespace NethereumWorkflow public decimal GetBalance(string tokenAddress, string account) { + log.Debug($"({tokenAddress}) {account}"); var function = new GetTokenBalanceFunction { Owner = account @@ -72,6 +76,42 @@ namespace NethereumWorkflow Task.WaitAll(tasks); } + public void EnsureSynced(string marketplaceAddress, string marketplaceAbi) + { + WaitUntilSynced(); + WaitForContract(marketplaceAddress, marketplaceAbi); + } + + private void WaitUntilSynced() + { + log.Debug(); + Time.WaitUntil(() => + { + var sync = Time.Wait(web3.Eth.Syncing.SendRequestAsync()); + var number = Time.Wait(web3.Eth.Blocks.GetBlockNumber.SendRequestAsync()); + var numberOfBlocks = ToDecimal(number); + return !sync.IsSyncing && numberOfBlocks > 256; + + }, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(3)); + } + + private void WaitForContract(string marketplaceAddress, string marketplaceAbi) + { + log.Debug(); + Time.WaitUntil(() => + { + try + { + var contract = web3.Eth.GetContract(marketplaceAbi, marketplaceAddress); + return contract != null; + } + catch + { + return false; + } + }, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(3)); + } + private HexBigInteger ToHexBig(decimal amount) { var bigint = ToBig(amount); @@ -84,6 +124,11 @@ namespace NethereumWorkflow return new BigInteger(amount); } + private decimal ToDecimal(HexBigInteger hexBigInteger) + { + return ToDecimal(hexBigInteger.Value); + } + private decimal ToDecimal(BigInteger bigInteger) { return (decimal)bigInteger; @@ -106,7 +151,7 @@ namespace NethereumWorkflow } [Function("balanceOf", "uint256")] - public class GetTokenBalanceFunction :FunctionMessage + public class GetTokenBalanceFunction : FunctionMessage { [Parameter("address", "owner", 1)] public string Owner { get; set; } diff --git a/Nethereum/NethereumInteractionCreator.cs b/Nethereum/NethereumInteractionCreator.cs index 3695fa15..ff7e9fd0 100644 --- a/Nethereum/NethereumInteractionCreator.cs +++ b/Nethereum/NethereumInteractionCreator.cs @@ -5,13 +5,13 @@ namespace NethereumWorkflow { public class NethereumInteractionCreator { - private readonly TestLog log; + private readonly BaseLog log; private readonly string ip; private readonly int port; private readonly string rootAccount; private readonly string privateKey; - public NethereumInteractionCreator(TestLog log, string ip, int port, string rootAccount, string privateKey) + public NethereumInteractionCreator(BaseLog log, string ip, int port, string rootAccount, string privateKey) { this.log = log; this.ip = ip; diff --git a/Tests/BasicTests/DownloadTests.cs b/Tests/BasicTests/DownloadTests.cs index bba4da1f..0f61daf7 100644 --- a/Tests/BasicTests/DownloadTests.cs +++ b/Tests/BasicTests/DownloadTests.cs @@ -1,38 +1,26 @@ using DistTestCore; -using KubernetesWorkflow; using NUnit.Framework; + namespace Tests.ParallelTests { [TestFixture] public class DownloadTests : DistTest { - [Test] - public void ThreeNodeDownloads() + [TestCase(3, 500)] + [TestCase(5, 100)] + [TestCase(10, 256)] + [UseLongTimeouts] + public void ParallelDownload(int numberOfNodes, int filesizeMb) { - ParallelDownload(3, 5000.MB()); - } - [Test] - public void FiveNodeDownloads() - { - ParallelDownload(5, 1000.MB()); - } - [Test] - public void TenNodeDownloads() - { - ParallelDownload(10, 256.MB()); - } - - void ParallelDownload(int numberOfNodes, ByteSize filesize) - { - var group = SetupCodexNodes(numberOfNodes).BringOnline(); - var host = SetupCodexNodes(1).BringOnline()[0]; + var group = SetupCodexNodes(numberOfNodes); + var host = SetupCodexNode(); foreach (var node in group) { host.ConnectToPeer(node); } - var testFile = GenerateTestFile(filesize); + var testFile = GenerateTestFile(filesizeMb.MB()); var contentId = host.UploadFile(testFile); var list = new List>(); diff --git a/Tests/BasicTests/ExampleTests.cs b/Tests/BasicTests/ExampleTests.cs index 31cc56a5..afafe9a0 100644 --- a/Tests/BasicTests/ExampleTests.cs +++ b/Tests/BasicTests/ExampleTests.cs @@ -11,9 +11,7 @@ namespace Tests.BasicTests [Test] public void CodexLogExample() { - var primary = SetupCodexNodes(1) - .WithLogLevel(CodexLogLevel.Trace) - .BringOnline()[0]; + var primary = SetupCodexNode(s => s.WithLogLevel(CodexLogLevel.Trace)); primary.UploadFile(GenerateTestFile(5.MB())); @@ -25,13 +23,8 @@ namespace Tests.BasicTests [Test] public void TwoMetricsExample() { - var group = SetupCodexNodes(2) - .EnableMetrics() - .BringOnline(); - - var group2 = SetupCodexNodes(2) - .EnableMetrics() - .BringOnline(); + var group = SetupCodexNodes(2, s => s.EnableMetrics()); + var group2 = SetupCodexNodes(2, s => s.EnableMetrics()); var primary = group[0]; var secondary = group[1]; @@ -50,29 +43,30 @@ namespace Tests.BasicTests [Test] public void MarketplaceExample() { - var primary = SetupCodexNodes(1) + var sellerInitialBalance = 234.TestTokens(); + var buyerInitialBalance = 1000.TestTokens(); + + var seller = SetupCodexNode(s => s + .WithLogLevel(CodexLogLevel.Trace) .WithStorageQuota(11.GB()) - .EnableMarketplace(initialBalance: 234.TestTokens()) - .BringOnline()[0]; + .EnableMarketplace(sellerInitialBalance)); - primary.Marketplace.AssertThatBalance(Is.EqualTo(234.TestTokens())); - - var secondary = SetupCodexNodes(1) - .EnableMarketplace(initialBalance: 1000.TestTokens()) - .BringOnline()[0]; - - primary.ConnectToPeer(secondary); - - primary.Marketplace.MakeStorageAvailable( + seller.Marketplace.AssertThatBalance(Is.EqualTo(sellerInitialBalance)); + seller.Marketplace.MakeStorageAvailable( size: 10.GB(), minPricePerBytePerSecond: 1.TestTokens(), maxCollateral: 20.TestTokens(), maxDuration: TimeSpan.FromMinutes(3)); var testFile = GenerateTestFile(10.MB()); - var contentId = secondary.UploadFile(testFile); - secondary.Marketplace.RequestStorage(contentId, + var buyer = SetupCodexNode(s => s + .WithLogLevel(CodexLogLevel.Trace) + .WithBootstrapNode(seller) + .EnableMarketplace(buyerInitialBalance)); + + var contentId = buyer.UploadFile(testFile); + buyer.Marketplace.RequestStorage(contentId, pricePerBytePerSecond: 2.TestTokens(), requiredCollateral: 10.TestTokens(), minRequiredNumberOfNodes: 1, @@ -81,12 +75,12 @@ namespace Tests.BasicTests Time.Sleep(TimeSpan.FromMinutes(1)); - primary.Marketplace.AssertThatBalance(Is.LessThan(234.TestTokens()), "Collateral was not placed."); + seller.Marketplace.AssertThatBalance(Is.LessThan(sellerInitialBalance), "Collateral was not placed."); Time.Sleep(TimeSpan.FromMinutes(2)); - primary.Marketplace.AssertThatBalance(Is.GreaterThan(234.TestTokens()), "Storer was not paid for storage."); - secondary.Marketplace.AssertThatBalance(Is.LessThan(1000.TestTokens()), "Contractor was not charged for storage."); + seller.Marketplace.AssertThatBalance(Is.GreaterThan(sellerInitialBalance), "Seller was not paid for storage."); + buyer.Marketplace.AssertThatBalance(Is.LessThan(buyerInitialBalance), "Buyer was not charged for storage."); } } } diff --git a/Tests/BasicTests/OneClientTests.cs b/Tests/BasicTests/OneClientTests.cs index a2c0c57d..b22e53bf 100644 --- a/Tests/BasicTests/OneClientTests.cs +++ b/Tests/BasicTests/OneClientTests.cs @@ -9,7 +9,7 @@ namespace Tests.BasicTests [Test] public void OneClientTest() { - var primary = SetupCodexNodes(1).BringOnline()[0]; + var primary = SetupCodexNode(); PerformOneClientTest(primary); } @@ -17,11 +17,11 @@ namespace Tests.BasicTests [Test] public void RestartTest() { - var group = SetupCodexNodes(1).BringOnline(); + var primary = SetupCodexNode(); - var setup = group.BringOffline(); + var setup = primary.BringOffline(); - var primary = setup.BringOnline()[0]; + primary = BringOnline(setup)[0]; PerformOneClientTest(primary); } diff --git a/Tests/BasicTests/PeerTests.cs b/Tests/BasicTests/PeerTests.cs new file mode 100644 index 00000000..f7fa700c --- /dev/null +++ b/Tests/BasicTests/PeerTests.cs @@ -0,0 +1,85 @@ +using DistTestCore; +using DistTestCore.Codex; +using NUnit.Framework; + +namespace Tests.BasicTests +{ + [TestFixture] + public class PeerTests : DistTest + { + [Test] + public void TwoNodes() + { + var primary = SetupCodexBootstrapNode(); + var secondary = SetupCodexNode(s => s.WithBootstrapNode(primary)); + + primary.ConnectToPeer(secondary); // TODO REMOVE THIS: This is required for the switchPeers to show up. + + // This is required for the enginePeers to show up. + //var file = GenerateTestFile(10.MB()); + //var contentId = primary.UploadFile(file); + //var file2 = secondary.DownloadContent(contentId); + //file.AssertIsEqual(file2); + + AssertKnowEachother(primary, secondary); + } + + [TestCase(2)] + [TestCase(3)] + [TestCase(10)] + public void VariableNodes(int number) + { + var bootstrap = SetupCodexBootstrapNode(); + var nodes = SetupCodexNodes(number, s => s.WithBootstrapNode(bootstrap)); + + var file = GenerateTestFile(10.MB()); + var contentId = nodes.First().UploadFile(file); + var file2 = nodes.Last().DownloadContent(contentId); + file.AssertIsEqual(file2); + + // + foreach (var node in nodes) bootstrap.ConnectToPeer(node); + for (var x = 0; x < number; x++) + { + for (var y = x + 1; y < number; y++) + { + nodes[x].ConnectToPeer(nodes[y]); + } + } + // + + foreach (var node in nodes) AssertKnowEachother(node, bootstrap); + + for (var x = 0; x < number; x++) + { + for (var y = x + 1; y < number; y++) + { + AssertKnowEachother(nodes[x], nodes[y]); + } + } + } + + private void AssertKnowEachother(IOnlineCodexNode a, IOnlineCodexNode b) + { + AssertKnowEachother(a.GetDebugInfo(), b.GetDebugInfo()); + } + + private void AssertKnowEachother(CodexDebugResponse a, CodexDebugResponse b) + { + AssertKnows(a, b); + AssertKnows(b, a); + } + + private void AssertKnows(CodexDebugResponse a, CodexDebugResponse b) + { + //var enginePeers = string.Join(",", a.enginePeers.Select(p => p.peerId)); + var switchPeers = string.Join(",", a.switchPeers.Select(p => p.peerId)); + + //Log.Debug($"Looking for {b.id} in engine-peers [{enginePeers}]"); + Log.Debug($"{a.id} is looking for {b.id} in switch-peers [{switchPeers}]"); + + //Assert.That(a.enginePeers.Any(p => p.peerId == b.id), $"{a.id} was looking for '{b.id}' in engine-peers [{enginePeers}] but it was not found."); + Assert.That(a.switchPeers.Any(p => p.peerId == b.id), $"{a.id} was looking for '{b.id}' in switch-peers [{switchPeers}] but it was not found."); + } + } +} diff --git a/Tests/BasicTests/TwoClientTests.cs b/Tests/BasicTests/TwoClientTests.cs index cde3b048..90af450d 100644 --- a/Tests/BasicTests/TwoClientTests.cs +++ b/Tests/BasicTests/TwoClientTests.cs @@ -10,7 +10,7 @@ namespace Tests.BasicTests [Test] public void TwoClientsOnePodTest() { - var group = SetupCodexNodes(2).BringOnline(); + var group = SetupCodexNodes(2); var primary = group[0]; var secondary = group[1]; @@ -21,9 +21,8 @@ namespace Tests.BasicTests [Test] public void TwoClientsTwoPodsTest() { - var primary = SetupCodexNodes(1).BringOnline()[0]; - - var secondary = SetupCodexNodes(1).BringOnline()[0]; + var primary = SetupCodexNode(); + var secondary = SetupCodexNode(); PerformTwoClientTest(primary, secondary); } @@ -32,13 +31,8 @@ namespace Tests.BasicTests [Ignore("Requires Location map to be configured for k8s cluster.")] public void TwoClientsTwoLocationsTest() { - var primary = SetupCodexNodes(1) - .At(Location.BensLaptop) - .BringOnline()[0]; - - var secondary = SetupCodexNodes(1) - .At(Location.BensOldGamingMachine) - .BringOnline()[0]; + var primary = SetupCodexNode(s => s.At(Location.BensLaptop)); + var secondary = SetupCodexNode(s => s.At(Location.BensOldGamingMachine)); PerformTwoClientTest(primary, secondary); } diff --git a/Tests/BasicTests/UploadTests.cs b/Tests/BasicTests/UploadTests.cs index a6035240..8cbbe7a1 100644 --- a/Tests/BasicTests/UploadTests.cs +++ b/Tests/BasicTests/UploadTests.cs @@ -1,30 +1,19 @@ using DistTestCore; -using KubernetesWorkflow; using NUnit.Framework; + namespace Tests.ParallelTests { [TestFixture] public class UploadTests : DistTest { - [Test] - public void ThreeNodeUploads() + [TestCase(3, 50)] + [TestCase(5, 75)] + [TestCase(10, 25)] + [UseLongTimeouts] + public void ParallelUpload(int numberOfNodes, int filesizeMb) { - ParallelUpload(3, 50.MB()); - } - [Test] - public void FiveNodeUploads() - { - ParallelUpload(5, 750.MB()); - } - [Test] - public void TenNodeUploads() - { - ParallelUpload(10, 25.MB()); - } - void ParallelUpload(int numberOfNodes, ByteSize filesize) - { - var group = SetupCodexNodes(numberOfNodes).BringOnline(); - var host = SetupCodexNodes(1).BringOnline()[0]; + var group = SetupCodexNodes(numberOfNodes); + var host = SetupCodexNode(); foreach (var node in group) { @@ -36,7 +25,7 @@ namespace Tests.ParallelTests for (int i = 0; i < group.Count(); i++) { - testfiles.Add(GenerateTestFile(filesize)); + testfiles.Add(GenerateTestFile(filesizeMb.MB())); var n = i; contentIds.Add(Task.Run(() => { return host.UploadFile(testfiles[n]); })); } diff --git a/Tests/DurabilityTests/DurabilityTests.cs b/Tests/DurabilityTests/DurabilityTests.cs index 6ffd78bb..584d1c66 100644 --- a/Tests/DurabilityTests/DurabilityTests.cs +++ b/Tests/DurabilityTests/DurabilityTests.cs @@ -1,4 +1,5 @@ using DistTestCore; +using DistTestCore.Codex; using NUnit.Framework; using Utils; @@ -10,14 +11,14 @@ namespace Tests.DurabilityTests [Test] public void BootstrapNodeDisappearsTest() { - var bootstrapNode = SetupCodexNodes(1).BringOnline(); - var group = SetupCodexNodes(2).WithBootstrapNode(bootstrapNode[0]).BringOnline(); + var bootstrapNode = SetupCodexNode(); + var group = SetupCodexNodes(2, s => s.WithBootstrapNode(bootstrapNode)); var primary = group[0]; var secondary = group[1]; // There is 1 minute of time for the nodes to connect to each other. // (Should be easy, they're in the same pod.) - Time.Sleep(TimeSpan.FromMinutes(1)); + Time.Sleep(TimeSpan.FromMinutes(6)); bootstrapNode.BringOffline(); var file = GenerateTestFile(10.MB()); @@ -30,10 +31,10 @@ namespace Tests.DurabilityTests [Test] public void DataRetentionTest() { - var bootstrapNode = SetupCodexNodes(1).BringOnline()[0]; + var bootstrapNode = SetupCodexNode(s => s.WithLogLevel(CodexLogLevel.Trace)); - var startGroup = SetupCodexNodes(2).WithBootstrapNode(bootstrapNode).BringOnline(); - var finishGroup = SetupCodexNodes(10).WithBootstrapNode(bootstrapNode).BringOnline(); + var startGroup = SetupCodexNodes(2, s => s.WithLogLevel(CodexLogLevel.Trace).WithBootstrapNode(bootstrapNode)); + var finishGroup = SetupCodexNodes(10, s => s.WithLogLevel(CodexLogLevel.Trace).WithBootstrapNode(bootstrapNode)); var file = GenerateTestFile(10.MB()); diff --git a/Utils/DebugStack.cs b/Utils/DebugStack.cs new file mode 100644 index 00000000..d10a3786 --- /dev/null +++ b/Utils/DebugStack.cs @@ -0,0 +1,12 @@ +using System.Diagnostics; + +namespace Utils +{ + public class DebugStack + { + public static string GetCallerName(int skipFrames = 0) + { + return new StackFrame(2 + skipFrames, true).GetMethod()!.Name; + } + } +}