From b92c1b970d1bdb328962da3261e61f60827cf1af Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 1 Jun 2023 09:35:18 +0200 Subject: [PATCH 01/13] Sets up support for internal and external container addresses --- DistTestCore/Codex/CodexAccess.cs | 20 ++++++------- DistTestCore/Codex/CodexContainerRecipe.cs | 4 +-- DistTestCore/CodexNodeGroup.cs | 2 +- DistTestCore/Configuration.cs | 20 +++++++++++++ DistTestCore/GethStarter.cs | 4 +-- DistTestCore/Http.cs | 13 ++++----- .../Marketplace/CodexContractsStarter.cs | 2 +- .../Marketplace/GethBootstrapNodeInfo.cs | 8 ++---- DistTestCore/Marketplace/MarketplaceAccess.cs | 10 +++---- .../Marketplace/MarketplaceAccessFactory.cs | 8 +++--- .../Marketplace/MarketplaceNetwork.cs | 7 ++--- DistTestCore/Metrics/MetricsAccessFactory.cs | 2 +- DistTestCore/Metrics/MetricsQuery.cs | 12 ++++---- DistTestCore/TestLifecycle.cs | 2 ++ KubernetesWorkflow/RunningContainers.cs | 18 +++++++++++- KubernetesWorkflow/StartupWorkflow.cs | 28 ++++++++++++++++++- Nethereum/NethereumInteraction.cs | 4 +-- Nethereum/NethereumInteractionCreator.cs | 2 +- Tests/BasicTests/ExampleTests.cs | 1 - 19 files changed, 111 insertions(+), 56 deletions(-) diff --git a/DistTestCore/Codex/CodexAccess.cs b/DistTestCore/Codex/CodexAccess.cs index 587f7f09..4d6f1dab 100644 --- a/DistTestCore/Codex/CodexAccess.cs +++ b/DistTestCore/Codex/CodexAccess.cs @@ -1,17 +1,14 @@ using KubernetesWorkflow; -using Logging; namespace DistTestCore.Codex { public class CodexAccess { - private readonly BaseLog log; - private readonly ITimeSet timeSet; + private readonly TestLifecycle lifecycle; - public CodexAccess(BaseLog log, ITimeSet timeSet, RunningContainer runningContainer) + public CodexAccess(TestLifecycle lifecycle, RunningContainer runningContainer) { - this.log = log; - this.timeSet = timeSet; + this.lifecycle = lifecycle; Container = runningContainer; } @@ -79,21 +76,20 @@ namespace DistTestCore.Codex var nodePeerId = debugInfo.id; var nodeName = Container.Name; - log.AddStringReplace(nodePeerId, nodeName); - log.AddStringReplace(debugInfo.table.localNode.nodeId, nodeName); + lifecycle.Log.AddStringReplace(nodePeerId, nodeName); + lifecycle.Log.AddStringReplace(debugInfo.table.localNode.nodeId, nodeName); } catch (Exception e) { - log.Error($"Failed to start codex node: {e}. Test infra failure."); + lifecycle.Log.Error($"Failed to start codex node: {e}. Test infra failure."); throw new InvalidOperationException($"Failed to start codex node. Test infra failure.", e); } } private Http Http(TimeSpan? timeoutOverride = null) { - var ip = Container.Pod.Cluster.HostAddress; - var port = Container.ServicePorts[0].Number; - return new Http(log, timeSet, ip, port, baseUrl: "/api/codex/v1", timeoutOverride); + var address = lifecycle.Configuration.GetAddress(Container); + return new Http(lifecycle.Log, lifecycle.TimeSet, address, baseUrl: "/api/codex/v1", timeoutOverride); } } diff --git a/DistTestCore/Codex/CodexContainerRecipe.cs b/DistTestCore/Codex/CodexContainerRecipe.cs index dfaf3a1a..a723baa9 100644 --- a/DistTestCore/Codex/CodexContainerRecipe.cs +++ b/DistTestCore/Codex/CodexContainerRecipe.cs @@ -1,5 +1,4 @@ -using System.Runtime.InteropServices; -using DistTestCore.Marketplace; +using DistTestCore.Marketplace; using KubernetesWorkflow; namespace DistTestCore.Codex @@ -56,6 +55,7 @@ namespace DistTestCore.Codex AddEnvVar("ETH_PROVIDER", $"ws://{ip}:{port}"); AddEnvVar("ETH_ACCOUNT", companionNodeAccount.Account); AddEnvVar("ETH_MARKETPLACE_ADDRESS", gethConfig.MarketplaceNetwork.Marketplace.Address); + AddEnvVar("PERSISTENCE", "1"); } } } diff --git a/DistTestCore/CodexNodeGroup.cs b/DistTestCore/CodexNodeGroup.cs index 2005410a..77d46e64 100644 --- a/DistTestCore/CodexNodeGroup.cs +++ b/DistTestCore/CodexNodeGroup.cs @@ -69,7 +69,7 @@ namespace DistTestCore private OnlineCodexNode CreateOnlineCodexNode(RunningContainer c, ICodexNodeFactory factory) { - var access = new CodexAccess(lifecycle.Log, lifecycle.TimeSet, c); + var access = new CodexAccess(lifecycle, c); return factory.CreateOnlineCodexNode(access, this); } } diff --git a/DistTestCore/Configuration.cs b/DistTestCore/Configuration.cs index ac7d470e..b9f21de3 100644 --- a/DistTestCore/Configuration.cs +++ b/DistTestCore/Configuration.cs @@ -34,5 +34,25 @@ namespace DistTestCore { return CodexLogLevel.Trace; } + + public TestRunnerLocation GetTestRunnerLocation() + { + return TestRunnerLocation.ExternalToCluster; + } + + public RunningContainerAddress GetAddress(RunningContainer container) + { + if (GetTestRunnerLocation() == TestRunnerLocation.InternalToCluster) + { + return container.ClusterInternalAddress; + } + return container.ClusterExternalAddress; + } + } + + public enum TestRunnerLocation + { + ExternalToCluster, + InternalToCluster, } } diff --git a/DistTestCore/GethStarter.cs b/DistTestCore/GethStarter.cs index 3ac8ce02..92af53f0 100644 --- a/DistTestCore/GethStarter.cs +++ b/DistTestCore/GethStarter.cs @@ -33,7 +33,7 @@ namespace DistTestCore private void TransferInitialBalance(MarketplaceNetwork marketplaceNetwork, MarketplaceInitialConfig marketplaceConfig, GethCompanionNodeInfo companionNode) { - var interaction = marketplaceNetwork.StartInteraction(lifecycle.Log); + var interaction = marketplaceNetwork.StartInteraction(lifecycle); var tokenAddress = marketplaceNetwork.Marketplace.TokenAddress; var accounts = companionNode.Accounts.Select(a => a.Account).ToArray(); @@ -52,7 +52,7 @@ namespace DistTestCore private IMarketplaceAccessFactory CreateMarketplaceAccessFactory(MarketplaceNetwork marketplaceNetwork) { - return new GethMarketplaceAccessFactory(lifecycle.Log, marketplaceNetwork); + return new GethMarketplaceAccessFactory(lifecycle, marketplaceNetwork); } private GethCompanionNodeInfo StartCompanionNode(CodexSetup codexSetup, MarketplaceNetwork marketplaceNetwork) diff --git a/DistTestCore/Http.cs b/DistTestCore/Http.cs index b7c7481c..30bf48ca 100644 --- a/DistTestCore/Http.cs +++ b/DistTestCore/Http.cs @@ -1,4 +1,5 @@ -using Logging; +using KubernetesWorkflow; +using Logging; using Newtonsoft.Json; using System.Net.Http.Headers; using System.Net.Http.Json; @@ -10,17 +11,15 @@ namespace DistTestCore { private readonly BaseLog log; private readonly ITimeSet timeSet; - private readonly string host; - private readonly int port; + private readonly RunningContainerAddress address; private readonly string baseUrl; private readonly TimeSpan? timeoutOverride; - public Http(BaseLog log, ITimeSet timeSet, string host, int port, string baseUrl, TimeSpan? timeoutOverride = null) + public Http(BaseLog log, ITimeSet timeSet, RunningContainerAddress address, string baseUrl, TimeSpan? timeoutOverride = null) { this.log = log; this.timeSet = timeSet; - this.host = host; - this.port = port; + this.address = address; this.baseUrl = baseUrl; this.timeoutOverride = timeoutOverride; if (!this.baseUrl.StartsWith("/")) this.baseUrl = "/" + this.baseUrl; @@ -110,7 +109,7 @@ namespace DistTestCore private string GetUrl() { - return $"{host}:{port}{baseUrl}"; + return $"{address.Host}:{address.Port}{baseUrl}"; } private void Log(string url, string message) diff --git a/DistTestCore/Marketplace/CodexContractsStarter.cs b/DistTestCore/Marketplace/CodexContractsStarter.cs index 412c25ad..2571ef17 100644 --- a/DistTestCore/Marketplace/CodexContractsStarter.cs +++ b/DistTestCore/Marketplace/CodexContractsStarter.cs @@ -34,7 +34,7 @@ namespace DistTestCore.Marketplace var marketplaceAddress = extractor.ExtractMarketplaceAddress(); var abi = extractor.ExtractMarketplaceAbi(); - var interaction = bootstrapNode.StartInteraction(lifecycle.Log); + var interaction = bootstrapNode.StartInteraction(lifecycle); var tokenAddress = interaction.GetTokenAddress(marketplaceAddress); LogEnd("Contracts deployed."); diff --git a/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs b/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs index ab38cb18..6825ba58 100644 --- a/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs +++ b/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs @@ -1,5 +1,4 @@ using KubernetesWorkflow; -using Logging; using NethereumWorkflow; namespace DistTestCore.Marketplace @@ -21,13 +20,12 @@ namespace DistTestCore.Marketplace public string PubKey { get; } public Port DiscoveryPort { get; } - public NethereumInteraction StartInteraction(BaseLog log) + public NethereumInteraction StartInteraction(TestLifecycle lifecycle) { - var ip = RunningContainers.RunningPod.Cluster.HostAddress; - var port = RunningContainers.Containers[0].ServicePorts[0].Number; + var address = lifecycle.Configuration.GetAddress(RunningContainers.Containers[0]); var account = Account; - var creator = new NethereumInteractionCreator(log, ip, port, account.PrivateKey); + var creator = new NethereumInteractionCreator(lifecycle.Log, address.Host, address.Port, account.PrivateKey); return creator.CreateWorkflow(); } } diff --git a/DistTestCore/Marketplace/MarketplaceAccess.cs b/DistTestCore/Marketplace/MarketplaceAccess.cs index 1cc19e7f..33f68e66 100644 --- a/DistTestCore/Marketplace/MarketplaceAccess.cs +++ b/DistTestCore/Marketplace/MarketplaceAccess.cs @@ -17,14 +17,14 @@ namespace DistTestCore.Marketplace public class MarketplaceAccess : IMarketplaceAccess { - private readonly TestLog log; + private readonly TestLifecycle lifecycle; private readonly MarketplaceNetwork marketplaceNetwork; private readonly GethAccount account; private readonly CodexAccess codexAccess; - public MarketplaceAccess(TestLog log, MarketplaceNetwork marketplaceNetwork, GethAccount account, CodexAccess codexAccess) + public MarketplaceAccess(TestLifecycle lifecycle, MarketplaceNetwork marketplaceNetwork, GethAccount account, CodexAccess codexAccess) { - this.log = log; + this.lifecycle = lifecycle; this.marketplaceNetwork = marketplaceNetwork; this.account = account; this.codexAccess = codexAccess; @@ -103,7 +103,7 @@ namespace DistTestCore.Marketplace public TestToken GetBalance() { - var interaction = marketplaceNetwork.StartInteraction(log); + var interaction = marketplaceNetwork.StartInteraction(lifecycle); var amount = interaction.GetBalance(marketplaceNetwork.Marketplace.TokenAddress, account.Account); var balance = new TestToken(amount); @@ -114,7 +114,7 @@ namespace DistTestCore.Marketplace private void Log(string msg) { - log.Log($"{codexAccess.Container.Name} {msg}"); + lifecycle.Log.Log($"{codexAccess.Container.Name} {msg}"); } } diff --git a/DistTestCore/Marketplace/MarketplaceAccessFactory.cs b/DistTestCore/Marketplace/MarketplaceAccessFactory.cs index cd37d818..5ad52eb5 100644 --- a/DistTestCore/Marketplace/MarketplaceAccessFactory.cs +++ b/DistTestCore/Marketplace/MarketplaceAccessFactory.cs @@ -18,19 +18,19 @@ namespace DistTestCore.Marketplace public class GethMarketplaceAccessFactory : IMarketplaceAccessFactory { - private readonly TestLog log; + private readonly TestLifecycle lifecycle; private readonly MarketplaceNetwork marketplaceNetwork; - public GethMarketplaceAccessFactory(TestLog log, MarketplaceNetwork marketplaceNetwork) + public GethMarketplaceAccessFactory(TestLifecycle lifecycle, MarketplaceNetwork marketplaceNetwork) { - this.log = log; + this.lifecycle = lifecycle; this.marketplaceNetwork = marketplaceNetwork; } public IMarketplaceAccess CreateMarketplaceAccess(CodexAccess access) { var companionNode = GetGethCompanionNode(access); - return new MarketplaceAccess(log, marketplaceNetwork, companionNode, access); + return new MarketplaceAccess(lifecycle, marketplaceNetwork, companionNode, access); } private GethAccount GetGethCompanionNode(CodexAccess access) diff --git a/DistTestCore/Marketplace/MarketplaceNetwork.cs b/DistTestCore/Marketplace/MarketplaceNetwork.cs index 83784449..bba80a29 100644 --- a/DistTestCore/Marketplace/MarketplaceNetwork.cs +++ b/DistTestCore/Marketplace/MarketplaceNetwork.cs @@ -1,5 +1,4 @@ -using Logging; -using NethereumWorkflow; +using NethereumWorkflow; namespace DistTestCore.Marketplace { @@ -14,9 +13,9 @@ namespace DistTestCore.Marketplace public GethBootstrapNodeInfo Bootstrap { get; } public MarketplaceInfo Marketplace { get; } - public NethereumInteraction StartInteraction(BaseLog log) + public NethereumInteraction StartInteraction(TestLifecycle lifecycle) { - return Bootstrap.StartInteraction(log); + return Bootstrap.StartInteraction(lifecycle); } } } diff --git a/DistTestCore/Metrics/MetricsAccessFactory.cs b/DistTestCore/Metrics/MetricsAccessFactory.cs index 24103ab5..18dae041 100644 --- a/DistTestCore/Metrics/MetricsAccessFactory.cs +++ b/DistTestCore/Metrics/MetricsAccessFactory.cs @@ -28,7 +28,7 @@ namespace DistTestCore.Metrics public IMetricsAccess CreateMetricsAccess(RunningContainer codexContainer) { - var query = new MetricsQuery(lifecycle.Log, lifecycle.TimeSet, prometheusContainer); + var query = new MetricsQuery(lifecycle, prometheusContainer); return new MetricsAccess(lifecycle.Log, lifecycle.TimeSet, query, codexContainer); } } diff --git a/DistTestCore/Metrics/MetricsQuery.cs b/DistTestCore/Metrics/MetricsQuery.cs index f1626904..fc19867c 100644 --- a/DistTestCore/Metrics/MetricsQuery.cs +++ b/DistTestCore/Metrics/MetricsQuery.cs @@ -1,6 +1,5 @@ using DistTestCore.Codex; using KubernetesWorkflow; -using Logging; using System.Globalization; namespace DistTestCore.Metrics @@ -9,15 +8,16 @@ namespace DistTestCore.Metrics { private readonly Http http; - public MetricsQuery(BaseLog log, ITimeSet timeSet, RunningContainers runningContainers) + public MetricsQuery(TestLifecycle lifecycle, RunningContainers runningContainers) { RunningContainers = runningContainers; + var address = lifecycle.Configuration.GetAddress(runningContainers.Containers[0]); + http = new Http( - log, - timeSet, - runningContainers.RunningPod.Cluster.HostAddress, - runningContainers.Containers[0].ServicePorts[0].Number, + lifecycle.Log, + lifecycle.TimeSet, + address, "api/v1"); } diff --git a/DistTestCore/TestLifecycle.cs b/DistTestCore/TestLifecycle.cs index 15057458..ffd34b18 100644 --- a/DistTestCore/TestLifecycle.cs +++ b/DistTestCore/TestLifecycle.cs @@ -13,6 +13,7 @@ namespace DistTestCore public TestLifecycle(TestLog log, Configuration configuration, ITimeSet timeSet) { Log = log; + Configuration = configuration; TimeSet = timeSet; workflowCreator = new WorkflowCreator(log, configuration.GetK8sConfiguration(timeSet)); @@ -24,6 +25,7 @@ namespace DistTestCore } public TestLog Log { get; } + public Configuration Configuration { get; } public ITimeSet TimeSet { get; } public FileManager FileManager { get; } public CodexStarter CodexStarter { get; } diff --git a/KubernetesWorkflow/RunningContainers.cs b/KubernetesWorkflow/RunningContainers.cs index 616056c2..0b7b3fdb 100644 --- a/KubernetesWorkflow/RunningContainers.cs +++ b/KubernetesWorkflow/RunningContainers.cs @@ -21,18 +21,22 @@ public class RunningContainer { - public RunningContainer(RunningPod pod, ContainerRecipe recipe, Port[] servicePorts, StartupConfig startupConfig) + public RunningContainer(RunningPod pod, ContainerRecipe recipe, Port[] servicePorts, StartupConfig startupConfig, RunningContainerAddress clusterExternalAddress, RunningContainerAddress clusterInternalAddress) { Pod = pod; Recipe = recipe; ServicePorts = servicePorts; Name = GetContainerName(recipe, startupConfig); + ClusterExternalAddress = clusterExternalAddress; + ClusterInternalAddress = clusterInternalAddress; } public string Name { get; } public RunningPod Pod { get; } public ContainerRecipe Recipe { get; } public Port[] ServicePorts { get; } + public RunningContainerAddress ClusterExternalAddress { get; } + public RunningContainerAddress ClusterInternalAddress { get; } private string GetContainerName(ContainerRecipe recipe, StartupConfig startupConfig) { @@ -46,4 +50,16 @@ } } } + + public class RunningContainerAddress + { + public RunningContainerAddress(string host, int port) + { + Host = host; + Port = port; + } + + public string Host { get; } + public int Port { get; } + } } diff --git a/KubernetesWorkflow/StartupWorkflow.cs b/KubernetesWorkflow/StartupWorkflow.cs index 20cea0f1..f5546c58 100644 --- a/KubernetesWorkflow/StartupWorkflow.cs +++ b/KubernetesWorkflow/StartupWorkflow.cs @@ -80,10 +80,36 @@ namespace KubernetesWorkflow var servicePorts = runningPod.GetServicePortsForContainerRecipe(r); log.Debug($"{r} -> service ports: {string.Join(",", servicePorts.Select(p => p.Number))}"); - return new RunningContainer(runningPod, r, servicePorts, startupConfig); + return new RunningContainer(runningPod, r, servicePorts, startupConfig, + GetContainerExternalAddress(runningPod, servicePorts), + GetContainerInternalAddress(servicePorts)); + }).ToArray(); } + private RunningContainerAddress GetContainerExternalAddress(RunningPod pod, Port[] servicePorts) + { + return new RunningContainerAddress( + pod.Cluster.HostAddress, + GetServicePort(servicePorts)); + } + + private RunningContainerAddress GetContainerInternalAddress(Port[] servicePorts) + { + var serviceName = "service-" + numberSource.WorkflowNumber; + var namespaceName = cluster.Configuration.K8sNamespacePrefix + testNamespace; + + return new RunningContainerAddress( + $"http://{serviceName}.{namespaceName}.svc.cluster.local", + GetServicePort(servicePorts)); + } + + private static int GetServicePort(Port[] servicePorts) + { + if (servicePorts.Any()) return servicePorts.First().Number; + return 0; + } + private ContainerRecipe[] CreateRecipes(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig) { log.Debug(); diff --git a/Nethereum/NethereumInteraction.cs b/Nethereum/NethereumInteraction.cs index 46d6da01..624f20c7 100644 --- a/Nethereum/NethereumInteraction.cs +++ b/Nethereum/NethereumInteraction.cs @@ -131,7 +131,7 @@ namespace NethereumWorkflow public class MintTokensFunction : FunctionMessage { [Parameter("address", "holder", 1)] - public string Holder { get; set; } + public string Holder { get; set; } = string.Empty; [Parameter("uint256", "amount", 2)] public BigInteger Amount { get; set; } @@ -141,6 +141,6 @@ namespace NethereumWorkflow public class GetTokenBalanceFunction : FunctionMessage { [Parameter("address", "owner", 1)] - public string Owner { get; set; } + public string Owner { get; set; } = string.Empty; } } diff --git a/Nethereum/NethereumInteractionCreator.cs b/Nethereum/NethereumInteractionCreator.cs index 3ac0431e..ab5449c3 100644 --- a/Nethereum/NethereumInteractionCreator.cs +++ b/Nethereum/NethereumInteractionCreator.cs @@ -26,7 +26,7 @@ namespace NethereumWorkflow private Web3 CreateWeb3() { var account = new Nethereum.Web3.Accounts.Account(privateKey); - return new Web3(account, $"http://{ip}:{port}"); + return new Web3(account, $"{ip}:{port}"); } } } diff --git a/Tests/BasicTests/ExampleTests.cs b/Tests/BasicTests/ExampleTests.cs index 3a4884cc..976ccc30 100644 --- a/Tests/BasicTests/ExampleTests.cs +++ b/Tests/BasicTests/ExampleTests.cs @@ -1,5 +1,4 @@ using DistTestCore; -using DistTestCore.Codex; using NUnit.Framework; using Utils; From 61d2185bb63829f546125198c627378f774a492b Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 1 Jun 2023 10:14:32 +0200 Subject: [PATCH 02/13] Attempt internal connection using pod IP and exposed port. --- KubernetesWorkflow/StartupWorkflow.cs | 15 +++++++++------ Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs | 1 - 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/KubernetesWorkflow/StartupWorkflow.cs b/KubernetesWorkflow/StartupWorkflow.cs index f5546c58..be86a7f4 100644 --- a/KubernetesWorkflow/StartupWorkflow.cs +++ b/KubernetesWorkflow/StartupWorkflow.cs @@ -82,7 +82,7 @@ namespace KubernetesWorkflow return new RunningContainer(runningPod, r, servicePorts, startupConfig, GetContainerExternalAddress(runningPod, servicePorts), - GetContainerInternalAddress(servicePorts)); + GetContainerInternalAddress(runningPod, r, servicePorts)); }).ToArray(); } @@ -94,14 +94,17 @@ namespace KubernetesWorkflow GetServicePort(servicePorts)); } - private RunningContainerAddress GetContainerInternalAddress(Port[] servicePorts) + private RunningContainerAddress GetContainerInternalAddress(RunningPod pod, ContainerRecipe recipe, Port[] servicePorts) { - var serviceName = "service-" + numberSource.WorkflowNumber; - var namespaceName = cluster.Configuration.K8sNamespacePrefix + testNamespace; + var ip = pod.Ip; + var port = recipe.ExposedPorts.First().Number; + //var serviceName = "service-" + numberSource.WorkflowNumber; + //var namespaceName = cluster.Configuration.K8sNamespacePrefix + testNamespace; return new RunningContainerAddress( - $"http://{serviceName}.{namespaceName}.svc.cluster.local", - GetServicePort(servicePorts)); + //$"http://{serviceName}.{namespaceName}.svc.cluster.local", + $"http://{ip}", + port); } private static int GetServicePort(Port[] servicePorts) diff --git a/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs b/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs index 41b0a66c..a77cc3aa 100644 --- a/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs +++ b/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs @@ -1,5 +1,4 @@ using DistTestCore; -using DistTestCore.Codex; using DistTestCore.Helpers; using NUnit.Framework; using Utils; From 97478467875f37c2515de7257ff4b131b9f74d93 Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 1 Jun 2023 12:27:54 +0200 Subject: [PATCH 03/13] Increase retry timeout --- .../BasicTests/DownloadTests.cs | 4 +- .../BasicTests/UploadTests.cs | 2 +- Tests/DurabilityTests/DurabilityTests.cs | 66 ------------------- Utils/Time.cs | 8 +-- 4 files changed, 7 insertions(+), 73 deletions(-) rename {Tests => LongTests}/BasicTests/DownloadTests.cs (95%) rename {Tests => LongTests}/BasicTests/UploadTests.cs (97%) delete mode 100644 Tests/DurabilityTests/DurabilityTests.cs diff --git a/Tests/BasicTests/DownloadTests.cs b/LongTests/BasicTests/DownloadTests.cs similarity index 95% rename from Tests/BasicTests/DownloadTests.cs rename to LongTests/BasicTests/DownloadTests.cs index 36a635f9..50344620 100644 --- a/Tests/BasicTests/DownloadTests.cs +++ b/LongTests/BasicTests/DownloadTests.cs @@ -1,7 +1,7 @@ using DistTestCore; using NUnit.Framework; -namespace Tests.ParallelTests +namespace TestsLong.BasicTests { [TestFixture] public class DownloadTests : DistTest @@ -23,7 +23,7 @@ namespace Tests.ParallelTests var testFile = GenerateTestFile(filesizeMb.MB()); var contentId = host.UploadFile(testFile); var list = new List>(); - + foreach (var node in group) { list.Add(Task.Run(() => { return node.DownloadContent(contentId); })); diff --git a/Tests/BasicTests/UploadTests.cs b/LongTests/BasicTests/UploadTests.cs similarity index 97% rename from Tests/BasicTests/UploadTests.cs rename to LongTests/BasicTests/UploadTests.cs index ea3b41c4..ab005feb 100644 --- a/Tests/BasicTests/UploadTests.cs +++ b/LongTests/BasicTests/UploadTests.cs @@ -1,7 +1,7 @@ using DistTestCore; using NUnit.Framework; -namespace Tests.ParallelTests +namespace TestsLong.BasicTests { [TestFixture] public class UploadTests : DistTest diff --git a/Tests/DurabilityTests/DurabilityTests.cs b/Tests/DurabilityTests/DurabilityTests.cs deleted file mode 100644 index 12674005..00000000 --- a/Tests/DurabilityTests/DurabilityTests.cs +++ /dev/null @@ -1,66 +0,0 @@ -using DistTestCore; -using DistTestCore.Codex; -using NUnit.Framework; -using Utils; - -namespace Tests.DurabilityTests -{ - [TestFixture] - public class DurabilityTests : DistTest - { - [Test] - public void BootstrapNodeDisappearsTest() - { - var bootstrapNode = SetupCodexBootstrapNode(); - var group = SetupCodexNodes(2, s => s.WithBootstrapNode(bootstrapNode)); - var primary = group[0]; - var secondary = group[1]; - - // There is 1 minute of time f or the nodes to connect to each other. - // (Should be easy, they're in the same pod.) - Time.Sleep(TimeSpan.FromMinutes(6)); - bootstrapNode.BringOffline(); - - var file = GenerateTestFile(10.MB()); - var contentId = primary.UploadFile(file); - var downloadedFile = secondary.DownloadContent(contentId); - - file.AssertIsEqual(downloadedFile); - } - - [Test] - public void DataRetentionTest() - { - var bootstrapNode = SetupCodexBootstrapNode(); - - var startGroup = SetupCodexNodes(2, s => s.WithBootstrapNode(bootstrapNode)); - var finishGroup = SetupCodexNodes(10, s => s.WithBootstrapNode(bootstrapNode)); - - var file = GenerateTestFile(10.MB()); - - // Both nodes in the start group have the file. - var content = startGroup[0].UploadFile(file); - DownloadAndAssert(content, file, startGroup[1]); - - // Three nodes of the finish group have the file. - DownloadAndAssert(content, file, finishGroup[0]); - DownloadAndAssert(content, file, finishGroup[1]); - DownloadAndAssert(content, file, finishGroup[2]); - - // The start group goes away. - startGroup.BringOffline(); - - // All nodes in the finish group can access the file. - foreach (var node in finishGroup) - { - DownloadAndAssert(content, file, node); - } - } - - private void DownloadAndAssert(ContentId content, TestFile file, IOnlineCodexNode onlineCodexNode) - { - var downloaded = onlineCodexNode.DownloadContent(content); - file.AssertIsEqual(downloaded); - } - } -} diff --git a/Utils/Time.cs b/Utils/Time.cs index 6ae26400..5d6979b2 100644 --- a/Utils/Time.cs +++ b/Utils/Time.cs @@ -41,22 +41,22 @@ public static void Retry(Action action, string description) { - Retry(action, TimeSpan.FromMinutes(1), description); + Retry(action, TimeSpan.FromMinutes(2), description); } public static T Retry(Func action, string description) { - return Retry(action, TimeSpan.FromMinutes(1), description); + return Retry(action, TimeSpan.FromMinutes(2), description); } public static void Retry(Action action, TimeSpan timeout, string description) { - Retry(action, timeout, TimeSpan.FromSeconds(1), description); + Retry(action, timeout, TimeSpan.FromSeconds(2), description); } public static T Retry(Func action, TimeSpan timeout, string description) { - return Retry(action, timeout, TimeSpan.FromSeconds(1), description); + return Retry(action, timeout, TimeSpan.FromSeconds(2), description); } public static void Retry(Action action, TimeSpan timeout, TimeSpan retryTime, string description) From 4d654e106c0f995a2a51a4580bece47743698efa Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 1 Jun 2023 12:31:14 +0200 Subject: [PATCH 04/13] Remaps running container internal address to service endpoint --- KubernetesWorkflow/StartupWorkflow.cs | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/KubernetesWorkflow/StartupWorkflow.cs b/KubernetesWorkflow/StartupWorkflow.cs index be86a7f4..96706a9d 100644 --- a/KubernetesWorkflow/StartupWorkflow.cs +++ b/KubernetesWorkflow/StartupWorkflow.cs @@ -82,7 +82,7 @@ namespace KubernetesWorkflow return new RunningContainer(runningPod, r, servicePorts, startupConfig, GetContainerExternalAddress(runningPod, servicePorts), - GetContainerInternalAddress(runningPod, r, servicePorts)); + GetContainerInternalAddress(r)); }).ToArray(); } @@ -94,16 +94,14 @@ namespace KubernetesWorkflow GetServicePort(servicePorts)); } - private RunningContainerAddress GetContainerInternalAddress(RunningPod pod, ContainerRecipe recipe, Port[] servicePorts) + private RunningContainerAddress GetContainerInternalAddress(ContainerRecipe recipe) { - var ip = pod.Ip; + var serviceName = "service-" + numberSource.WorkflowNumber; + var namespaceName = cluster.Configuration.K8sNamespacePrefix + testNamespace; var port = recipe.ExposedPorts.First().Number; - //var serviceName = "service-" + numberSource.WorkflowNumber; - //var namespaceName = cluster.Configuration.K8sNamespacePrefix + testNamespace; return new RunningContainerAddress( - //$"http://{serviceName}.{namespaceName}.svc.cluster.local", - $"http://{ip}", + $"http://{serviceName}.{namespaceName}.svc.cluster.local", port); } From 9eae7cf25e09a856e9cb06fdc9fbe81d5d94e877 Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 1 Jun 2023 12:44:48 +0200 Subject: [PATCH 05/13] Fixes issue where containers with no exposed ports cannot start. --- KubernetesWorkflow/StartupWorkflow.cs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/KubernetesWorkflow/StartupWorkflow.cs b/KubernetesWorkflow/StartupWorkflow.cs index 96706a9d..59fe617b 100644 --- a/KubernetesWorkflow/StartupWorkflow.cs +++ b/KubernetesWorkflow/StartupWorkflow.cs @@ -98,7 +98,7 @@ namespace KubernetesWorkflow { var serviceName = "service-" + numberSource.WorkflowNumber; var namespaceName = cluster.Configuration.K8sNamespacePrefix + testNamespace; - var port = recipe.ExposedPorts.First().Number; + var port = GetInternalPort(recipe); return new RunningContainerAddress( $"http://{serviceName}.{namespaceName}.svc.cluster.local", @@ -111,6 +111,12 @@ namespace KubernetesWorkflow return 0; } + private static int GetInternalPort(ContainerRecipe recipe) + { + if (recipe.ExposedPorts.Any()) return recipe.ExposedPorts.First().Number; + return 0; + } + private ContainerRecipe[] CreateRecipes(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig) { log.Debug(); From b16eb4e88abb455fa697c31d0afa9a16f13d75bc Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 1 Jun 2023 14:03:16 +0200 Subject: [PATCH 06/13] Fixes issue where fetching of enode fails --- DistTestCore/Marketplace/ContainerInfoExtractor.cs | 1 + Utils/Time.cs | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/DistTestCore/Marketplace/ContainerInfoExtractor.cs b/DistTestCore/Marketplace/ContainerInfoExtractor.cs index 1aadc44d..ba1fd9fe 100644 --- a/DistTestCore/Marketplace/ContainerInfoExtractor.cs +++ b/DistTestCore/Marketplace/ContainerInfoExtractor.cs @@ -107,6 +107,7 @@ namespace DistTestCore.Marketplace public string GetPubKey() { + if (string.IsNullOrEmpty(pubKey)) throw new Exception("Not found yet exception."); return pubKey; } diff --git a/Utils/Time.cs b/Utils/Time.cs index 5d6979b2..6ae26400 100644 --- a/Utils/Time.cs +++ b/Utils/Time.cs @@ -41,22 +41,22 @@ public static void Retry(Action action, string description) { - Retry(action, TimeSpan.FromMinutes(2), description); + Retry(action, TimeSpan.FromMinutes(1), description); } public static T Retry(Func action, string description) { - return Retry(action, TimeSpan.FromMinutes(2), description); + return Retry(action, TimeSpan.FromMinutes(1), description); } public static void Retry(Action action, TimeSpan timeout, string description) { - Retry(action, timeout, TimeSpan.FromSeconds(2), description); + Retry(action, timeout, TimeSpan.FromSeconds(1), description); } public static T Retry(Func action, TimeSpan timeout, string description) { - return Retry(action, timeout, TimeSpan.FromSeconds(2), description); + return Retry(action, timeout, TimeSpan.FromSeconds(1), description); } public static void Retry(Action action, TimeSpan timeout, TimeSpan retryTime, string description) From b590e24de8d8ba3044c0d54c55e24f6a4a3ba60b Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 1 Jun 2023 14:18:09 +0200 Subject: [PATCH 07/13] Debugging marketplace deploy in cluster --- DistTestCore/Marketplace/CodexContractsStarter.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/DistTestCore/Marketplace/CodexContractsStarter.cs b/DistTestCore/Marketplace/CodexContractsStarter.cs index 2571ef17..8bf1f66f 100644 --- a/DistTestCore/Marketplace/CodexContractsStarter.cs +++ b/DistTestCore/Marketplace/CodexContractsStarter.cs @@ -14,7 +14,7 @@ namespace DistTestCore.Marketplace public MarketplaceInfo Start(GethBootstrapNodeInfo bootstrapNode) { - LogStart("Deploying Codex contracts..."); + LogStart("Deploying Codex Marketplace..."); var workflow = workflowCreator.CreateWorkflow(); var startupConfig = CreateStartupConfig(bootstrapNode.RunningContainers.Containers[0]); @@ -29,6 +29,7 @@ namespace DistTestCore.Marketplace workflow.DownloadContainerLog(container, logHandler); return logHandler.Found; }); + Log("Contracts deployed. Extracting addresses..."); var extractor = new ContainerInfoExtractor(lifecycle.Log, workflow, container); var marketplaceAddress = extractor.ExtractMarketplaceAddress(); @@ -37,14 +38,14 @@ namespace DistTestCore.Marketplace var interaction = bootstrapNode.StartInteraction(lifecycle); var tokenAddress = interaction.GetTokenAddress(marketplaceAddress); - LogEnd("Contracts deployed."); + LogEnd("Extract completed. Marketplace deployed."); return new MarketplaceInfo(marketplaceAddress, abi, tokenAddress); } private void WaitUntil(Func predicate) { - Time.WaitUntil(predicate, TimeSpan.FromMinutes(2), TimeSpan.FromSeconds(1)); + Time.WaitUntil(predicate, TimeSpan.FromMinutes(3), TimeSpan.FromSeconds(2)); } private StartupConfig CreateStartupConfig(RunningContainer bootstrapContainer) From 99c9b254873961cd4c30d584934c1fa5cb569742 Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 1 Jun 2023 15:04:04 +0200 Subject: [PATCH 08/13] Adds better check for successful deployment of codex contracts --- .../Marketplace/CodexContractsStarter.cs | 18 +++++++++--------- KubernetesWorkflow/K8sController.cs | 1 - 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/DistTestCore/Marketplace/CodexContractsStarter.cs b/DistTestCore/Marketplace/CodexContractsStarter.cs index 8bf1f66f..0f656350 100644 --- a/DistTestCore/Marketplace/CodexContractsStarter.cs +++ b/DistTestCore/Marketplace/CodexContractsStarter.cs @@ -5,7 +5,6 @@ namespace DistTestCore.Marketplace { public class CodexContractsStarter : BaseStarter { - private const string readyString = "Done! Sleeping indefinitely..."; public CodexContractsStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator) : base(lifecycle, workflowCreator) @@ -25,7 +24,7 @@ namespace DistTestCore.Marketplace WaitUntil(() => { - var logHandler = new ContractsReadyLogHandler(readyString); + var logHandler = new ContractsReadyLogHandler(); workflow.DownloadContainerLog(container, logHandler); return logHandler.Found; }); @@ -73,18 +72,19 @@ namespace DistTestCore.Marketplace public class ContractsReadyLogHandler : LogHandler { - private readonly string targetString; - - public ContractsReadyLogHandler(string targetString) - { - this.targetString = targetString; - } + // Log should contain 'Compiled 15 Solidity files successfully' at some point. + private const string RequiredCompiledString = "Solidity files successfully"; + // When script is done, it prints the ready-string. + private const string ReadyString = "Done! Sleeping indefinitely..."; + public bool SeenCompileString { get; private set; } public bool Found { get; private set; } protected override void ProcessLine(string line) { - if (line.Contains(targetString)) Found = true; + if (line.Contains(RequiredCompiledString)) SeenCompileString = true; + + if (SeenCompileString && line.Contains(ReadyString)) Found = true; } } } diff --git a/KubernetesWorkflow/K8sController.cs b/KubernetesWorkflow/K8sController.cs index 90d0b32c..539b2b26 100644 --- a/KubernetesWorkflow/K8sController.cs +++ b/KubernetesWorkflow/K8sController.cs @@ -22,7 +22,6 @@ namespace KubernetesWorkflow client = new K8sClient(cluster.GetK8sClientConfig()); K8sTestNamespace = cluster.Configuration.K8sNamespacePrefix + testNamespace; - log.Debug($"Test namespace: '{K8sTestNamespace}'"); } public void Dispose() From 37f7f5293ff852d1d5ae8793b3cefb9904748055 Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 1 Jun 2023 15:20:21 +0200 Subject: [PATCH 09/13] Adds debug logging for containerlog-searching operations --- DistTestCore/BaseStarter.cs | 5 +++++ DistTestCore/Marketplace/CodexContractsStarter.cs | 10 +++++++++- DistTestCore/Marketplace/ContainerInfoExtractor.cs | 10 +++++++++- 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/DistTestCore/BaseStarter.cs b/DistTestCore/BaseStarter.cs index eb344497..83a6b8d9 100644 --- a/DistTestCore/BaseStarter.cs +++ b/DistTestCore/BaseStarter.cs @@ -32,6 +32,11 @@ namespace DistTestCore lifecycle.Log.Log($"{GetClassName()} {msg}"); } + protected void Debug(string msg) + { + lifecycle.Log.Debug($"{GetClassName()} {msg}", 1); + } + private string GetClassName() { return $"({GetType().Name})"; diff --git a/DistTestCore/Marketplace/CodexContractsStarter.cs b/DistTestCore/Marketplace/CodexContractsStarter.cs index 0f656350..68d64fe4 100644 --- a/DistTestCore/Marketplace/CodexContractsStarter.cs +++ b/DistTestCore/Marketplace/CodexContractsStarter.cs @@ -24,7 +24,7 @@ namespace DistTestCore.Marketplace WaitUntil(() => { - var logHandler = new ContractsReadyLogHandler(); + var logHandler = new ContractsReadyLogHandler(Debug); workflow.DownloadContainerLog(container, logHandler); return logHandler.Found; }); @@ -76,12 +76,20 @@ namespace DistTestCore.Marketplace private const string RequiredCompiledString = "Solidity files successfully"; // When script is done, it prints the ready-string. private const string ReadyString = "Done! Sleeping indefinitely..."; + private readonly Action debug; + + public ContractsReadyLogHandler(Action debug) + { + this.debug = debug; + debug($"Looking for '{RequiredCompiledString}' and '{ReadyString}' in container logs..."); + } public bool SeenCompileString { get; private set; } public bool Found { get; private set; } protected override void ProcessLine(string line) { + debug(line); if (line.Contains(RequiredCompiledString)) SeenCompileString = true; if (SeenCompileString && line.Contains(ReadyString)) Found = true; diff --git a/DistTestCore/Marketplace/ContainerInfoExtractor.cs b/DistTestCore/Marketplace/ContainerInfoExtractor.cs index ba1fd9fe..7fd5638a 100644 --- a/DistTestCore/Marketplace/ContainerInfoExtractor.cs +++ b/DistTestCore/Marketplace/ContainerInfoExtractor.cs @@ -79,7 +79,7 @@ namespace DistTestCore.Marketplace private string FetchPubKey() { - var enodeFinder = new PubKeyFinder(); + var enodeFinder = new PubKeyFinder(s => log.Debug(s)); workflow.DownloadContainerLog(container, enodeFinder); return enodeFinder.GetPubKey(); } @@ -103,8 +103,15 @@ namespace DistTestCore.Marketplace { private const string openTag = "self=enode://"; private const string openTagQuote = "self=\"enode://"; + private readonly Action debug; private string pubKey = string.Empty; + public PubKeyFinder(Action debug) + { + this.debug = debug; + debug($"Looking for '{openTag}' in container logs..."); + } + public string GetPubKey() { if (string.IsNullOrEmpty(pubKey)) throw new Exception("Not found yet exception."); @@ -113,6 +120,7 @@ namespace DistTestCore.Marketplace protected override void ProcessLine(string line) { + debug(line); if (line.Contains(openTag)) { ExtractPubKey(openTag, line); From 56c716a4ea89265e7696e083c69304c7362718bf Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 1 Jun 2023 15:33:50 +0200 Subject: [PATCH 10/13] force update to new contracts image --- DistTestCore/Marketplace/CodexContractsContainerRecipe.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DistTestCore/Marketplace/CodexContractsContainerRecipe.cs b/DistTestCore/Marketplace/CodexContractsContainerRecipe.cs index 1df6d2e2..1ff0c06d 100644 --- a/DistTestCore/Marketplace/CodexContractsContainerRecipe.cs +++ b/DistTestCore/Marketplace/CodexContractsContainerRecipe.cs @@ -7,7 +7,7 @@ namespace DistTestCore.Marketplace #if Arm64 public const string DockerImage = "emizzle/codex-contracts-deployment:latest"; #else - public const string DockerImage = "thatbenbierens/codex-contracts-deployment:nomint"; + public const string DockerImage = "thatbenbierens/codex-contracts-deployment:nomint2"; #endif public const string MarketplaceAddressFilename = "/usr/app/deployments/codexdisttestnetwork/Marketplace.json"; public const string MarketplaceArtifactFilename = "/usr/app/artifacts/contracts/Marketplace.sol/Marketplace.json"; From 44e237e60e76914acec50f87e4298b4971ad1b74 Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 1 Jun 2023 15:56:26 +0200 Subject: [PATCH 11/13] Fixes incorrect call to node interaction from gethCompantionNodeStarter --- DistTestCore/Marketplace/GethCompanionNodeInfo.cs | 8 +++----- DistTestCore/Marketplace/GethCompanionNodeStarter.cs | 2 +- KubernetesWorkflow/K8sController.cs | 9 +++++++-- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/DistTestCore/Marketplace/GethCompanionNodeInfo.cs b/DistTestCore/Marketplace/GethCompanionNodeInfo.cs index 6c4b8e8b..3230d2a5 100644 --- a/DistTestCore/Marketplace/GethCompanionNodeInfo.cs +++ b/DistTestCore/Marketplace/GethCompanionNodeInfo.cs @@ -1,5 +1,4 @@ using KubernetesWorkflow; -using Logging; using NethereumWorkflow; namespace DistTestCore.Marketplace @@ -15,13 +14,12 @@ namespace DistTestCore.Marketplace public RunningContainer RunningContainer { get; } public GethAccount[] Accounts { get; } - public NethereumInteraction StartInteraction(BaseLog log, GethAccount account) + public NethereumInteraction StartInteraction(TestLifecycle lifecycle, GethAccount account) { - var ip = RunningContainer.Pod.Cluster.HostAddress; - var port = RunningContainer.ServicePorts[0].Number; + var address = lifecycle.Configuration.GetAddress(RunningContainer); var privateKey = account.PrivateKey; - var creator = new NethereumInteractionCreator(log, ip, port, privateKey); + var creator = new NethereumInteractionCreator(lifecycle.Log, address.Host, address.Port, privateKey); return creator.CreateWorkflow(); } } diff --git a/DistTestCore/Marketplace/GethCompanionNodeStarter.cs b/DistTestCore/Marketplace/GethCompanionNodeStarter.cs index a71c6615..2008e9c2 100644 --- a/DistTestCore/Marketplace/GethCompanionNodeStarter.cs +++ b/DistTestCore/Marketplace/GethCompanionNodeStarter.cs @@ -50,7 +50,7 @@ namespace DistTestCore.Marketplace { Time.WaitUntil(() => { - var interaction = node.StartInteraction(lifecycle.Log, node.Accounts.First()); + var interaction = node.StartInteraction(lifecycle, node.Accounts.First()); return interaction.IsSynced(marketplace.Marketplace.Address, marketplace.Marketplace.Abi); }, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(3)); } diff --git a/KubernetesWorkflow/K8sController.cs b/KubernetesWorkflow/K8sController.cs index 539b2b26..f094134f 100644 --- a/KubernetesWorkflow/K8sController.cs +++ b/KubernetesWorkflow/K8sController.cs @@ -59,10 +59,15 @@ namespace KubernetesWorkflow public string ExecuteCommand(RunningPod pod, string containerName, string command, params string[] args) { - log.Debug($"{containerName}: {command} ({string.Join(",", args)})"); + var cmdAndArgs = $"{containerName}: {command} ({string.Join(",", args)})"; + log.Debug(cmdAndArgs); + var runner = new CommandRunner(client, K8sTestNamespace, pod, containerName, command, args); runner.Run(); - return runner.GetStdOut(); + var result = runner.GetStdOut(); + + log.Debug($"{cmdAndArgs} = '{result}'"); + return result; } public void DeleteAllResources() From 9328f04f4ad1b5f90e156f93d91e7f61a3ce2e76 Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 1 Jun 2023 16:09:38 +0200 Subject: [PATCH 12/13] Adds retry for assertions of metrics and account balances. --- DistTestCore/Marketplace/MarketplaceAccess.cs | 6 ++++-- DistTestCore/Metrics/MetricsAccess.cs | 11 +++++++---- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/DistTestCore/Marketplace/MarketplaceAccess.cs b/DistTestCore/Marketplace/MarketplaceAccess.cs index 33f68e66..ad9bc7a8 100644 --- a/DistTestCore/Marketplace/MarketplaceAccess.cs +++ b/DistTestCore/Marketplace/MarketplaceAccess.cs @@ -1,5 +1,4 @@ using DistTestCore.Codex; -using Logging; using NUnit.Framework; using NUnit.Framework.Constraints; using System.Numerics; @@ -98,7 +97,10 @@ namespace DistTestCore.Marketplace public void AssertThatBalance(IResolveConstraint constraint, string message = "") { - Assert.That(GetBalance(), constraint, message); + Time.Retry(() => + { + Assert.That(GetBalance(), constraint, message); + }, nameof(AssertThatBalance)); } public TestToken GetBalance() diff --git a/DistTestCore/Metrics/MetricsAccess.cs b/DistTestCore/Metrics/MetricsAccess.cs index e52c1750..2032839d 100644 --- a/DistTestCore/Metrics/MetricsAccess.cs +++ b/DistTestCore/Metrics/MetricsAccess.cs @@ -28,12 +28,15 @@ namespace DistTestCore.Metrics public void AssertThat(string metricName, IResolveConstraint constraint, string message = "") { - var metricSet = GetMetricWithTimeout(metricName); - var metricValue = metricSet.Values[0].Value; + Time.Retry(() => + { + var metricSet = GetMetricWithTimeout(metricName); + var metricValue = metricSet.Values[0].Value; - log.Log($"{node.Name} metric '{metricName}' = {metricValue}"); + log.Log($"{node.Name} metric '{metricName}' = {metricValue}"); - Assert.That(metricValue, constraint, message); + Assert.That(metricValue, constraint, message); + }, nameof(AssertThat)); } public Metrics? GetAllMetrics() From 09e550df79c6ab4dc3376691e326f787896d82c5 Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 1 Jun 2023 16:28:34 +0200 Subject: [PATCH 13/13] Adds AssertHelper for better less log-spamming retry-assert. --- DistTestCore/Helpers/AssertHelpers.cs | 22 +++++++++++++++++++ DistTestCore/Marketplace/MarketplaceAccess.cs | 6 ++--- DistTestCore/Metrics/MetricsAccess.cs | 10 ++++----- Utils/Time.cs | 5 +++++ 4 files changed, 34 insertions(+), 9 deletions(-) create mode 100644 DistTestCore/Helpers/AssertHelpers.cs diff --git a/DistTestCore/Helpers/AssertHelpers.cs b/DistTestCore/Helpers/AssertHelpers.cs new file mode 100644 index 00000000..fdbffab6 --- /dev/null +++ b/DistTestCore/Helpers/AssertHelpers.cs @@ -0,0 +1,22 @@ +using NUnit.Framework.Constraints; +using NUnit.Framework; +using Utils; + +namespace DistTestCore.Helpers +{ + public static class AssertHelpers + { + public static void RetryAssert(IResolveConstraint constraint, Func actual, string message) + { + try + { + var c = constraint.Resolve(); + Time.WaitUntil(() => c.ApplyTo(actual()).IsSuccess); + } + catch (TimeoutException) + { + Assert.That(actual(), constraint, message); + } + } + } +} diff --git a/DistTestCore/Marketplace/MarketplaceAccess.cs b/DistTestCore/Marketplace/MarketplaceAccess.cs index ad9bc7a8..7bac8b23 100644 --- a/DistTestCore/Marketplace/MarketplaceAccess.cs +++ b/DistTestCore/Marketplace/MarketplaceAccess.cs @@ -1,4 +1,5 @@ using DistTestCore.Codex; +using DistTestCore.Helpers; using NUnit.Framework; using NUnit.Framework.Constraints; using System.Numerics; @@ -97,10 +98,7 @@ namespace DistTestCore.Marketplace public void AssertThatBalance(IResolveConstraint constraint, string message = "") { - Time.Retry(() => - { - Assert.That(GetBalance(), constraint, message); - }, nameof(AssertThatBalance)); + AssertHelpers.RetryAssert(constraint, GetBalance, message); } public TestToken GetBalance() diff --git a/DistTestCore/Metrics/MetricsAccess.cs b/DistTestCore/Metrics/MetricsAccess.cs index 2032839d..5ea09404 100644 --- a/DistTestCore/Metrics/MetricsAccess.cs +++ b/DistTestCore/Metrics/MetricsAccess.cs @@ -1,4 +1,5 @@ -using KubernetesWorkflow; +using DistTestCore.Helpers; +using KubernetesWorkflow; using Logging; using NUnit.Framework; using NUnit.Framework.Constraints; @@ -28,15 +29,14 @@ namespace DistTestCore.Metrics public void AssertThat(string metricName, IResolveConstraint constraint, string message = "") { - Time.Retry(() => + AssertHelpers.RetryAssert(constraint, () => { var metricSet = GetMetricWithTimeout(metricName); var metricValue = metricSet.Values[0].Value; log.Log($"{node.Name} metric '{metricName}' = {metricValue}"); - - Assert.That(metricValue, constraint, message); - }, nameof(AssertThat)); + return metricValue; + }, message); } public Metrics? GetAllMetrics() diff --git a/Utils/Time.cs b/Utils/Time.cs index 6ae26400..102831ec 100644 --- a/Utils/Time.cs +++ b/Utils/Time.cs @@ -22,6 +22,11 @@ result += $"{d.Seconds} secs"; return result; } + + public static void WaitUntil(Func predicate) + { + WaitUntil(predicate, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(1)); + } public static void WaitUntil(Func predicate, TimeSpan timeout, TimeSpan retryTime) {