From 14d663143ee08a2d4b824d20673ccb58070b4f4f Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 31 Mar 2023 08:39:24 +0200 Subject: [PATCH 01/46] Examples of how to execute commands in a container. Example of setup with Nethereum. --- CodexDistTestCore/CodexDistTestCore.csproj | 1 + CodexDistTestCore/DistTest.cs | 2 +- CodexDistTestCore/K8sManager.cs | 6 + CodexDistTestCore/K8sOperations.cs | 23 +++ CodexDistTestCore/TryContract.cs | 100 +++++++++++ Tests/BasicTests/SimpleTests.cs | 197 ++++++++++++--------- 6 files changed, 240 insertions(+), 89 deletions(-) create mode 100644 CodexDistTestCore/TryContract.cs diff --git a/CodexDistTestCore/CodexDistTestCore.csproj b/CodexDistTestCore/CodexDistTestCore.csproj index a42e9be..142eefb 100644 --- a/CodexDistTestCore/CodexDistTestCore.csproj +++ b/CodexDistTestCore/CodexDistTestCore.csproj @@ -9,6 +9,7 @@ + diff --git a/CodexDistTestCore/DistTest.cs b/CodexDistTestCore/DistTest.cs index c1d5c70..a8f1f3d 100644 --- a/CodexDistTestCore/DistTest.cs +++ b/CodexDistTestCore/DistTest.cs @@ -8,7 +8,7 @@ namespace CodexDistTestCore { private TestLog log = null!; private FileManager fileManager = null!; - private K8sManager k8sManager = null!; + public K8sManager k8sManager = null!; private MetricsAggregator metricsAggregator = null!; [OneTimeSetUp] diff --git a/CodexDistTestCore/K8sManager.cs b/CodexDistTestCore/K8sManager.cs index f9dace8..dd37089 100644 --- a/CodexDistTestCore/K8sManager.cs +++ b/CodexDistTestCore/K8sManager.cs @@ -43,6 +43,12 @@ return online.Origin; } + public void ExampleOfCMD(IOnlineCodexNode node) + { + var n = (OnlineCodexNode)node; + K8s(k => k.ExampleOfCommandExecution(n)); + } + public void DeleteAllResources() { K8s(k => k.DeleteAllResources()); diff --git a/CodexDistTestCore/K8sOperations.cs b/CodexDistTestCore/K8sOperations.cs index 7826c7d..6bad053 100644 --- a/CodexDistTestCore/K8sOperations.cs +++ b/CodexDistTestCore/K8sOperations.cs @@ -25,6 +25,29 @@ namespace CodexDistTestCore client.Dispose(); } + private Task Callback(Stream stdIn, Stream stdOut, Stream stdErr) + { + using var streamReader = new StreamReader(stdOut); + var lines = new List(); + var line = streamReader.ReadLine(); + while (line != null) + { + lines.Add(line); + line = streamReader.ReadLine(); + } + + Assert.That(lines.Any(l => l.Contains("FOO76543"))); + + + return Task.CompletedTask; + } + + public void ExampleOfCommandExecution(OnlineCodexNode node) + { + Utils.Wait(client.NamespacedPodExecAsync( + node.Group.PodInfo!.Name, K8sNamespace, node.Container.Name, new[] { "echo", "FOO76543" }, false, Callback, new CancellationToken())); + } + public void BringOnline(CodexNodeGroup online, OfflineCodexNodes offline) { EnsureTestNamespace(); diff --git a/CodexDistTestCore/TryContract.cs b/CodexDistTestCore/TryContract.cs new file mode 100644 index 0000000..766054b --- /dev/null +++ b/CodexDistTestCore/TryContract.cs @@ -0,0 +1,100 @@ +using Nethereum.Web3; +using Nethereum.ABI.FunctionEncoding.Attributes; +using Nethereum.Contracts.CQS; +using Nethereum.Util; +using Nethereum.Web3.Accounts; +using Nethereum.Hex.HexConvertors.Extensions; +using Nethereum.Contracts; +using Nethereum.Contracts.Extensions; +using System.Numerics; +using NUnit.Framework; + +// https://docs.nethereum.com/en/latest/nethereum-smartcontrats-gettingstarted/ + +namespace CodexDistTestCore +{ + public class TryContract + { + [Test] + public void DoThing() + { + var url = "http://testchain.nethereum.com:8545"; + var privateKey = "0x7580e7fb49df1c861f0050fae31c2224c6aba908e116b8da44ee8cd927b990b0"; + var account = new Account(privateKey); + var web3 = new Web3(account, url); + + // Deploy contract: + var deploymentMessage = new StandardTokenDeployment + { + TotalSupply = 100000 + }; + var deploymentHandler = web3.Eth.GetContractDeploymentHandler(); + var transactionReceipt = Utils.Wait(deploymentHandler.SendRequestAndWaitForReceiptAsync(deploymentMessage)); + var contractAddress = transactionReceipt.ContractAddress; + + // Get balance: + var balanceOfFunctionMessage = new BalanceOfFunction() + { + Owner = account.Address, + }; + + var balanceHandler = web3.Eth.GetContractQueryHandler(); + var balance = Utils.Wait(balanceHandler.QueryAsync(contractAddress, balanceOfFunctionMessage)); + long asInt = ((long)balance); + + // Transfer: + var receiverAddress = "0xde0B295669a9FD93d5F28D9Ec85E40f4cb697BAe"; + var transferHandler = web3.Eth.GetContractTransactionHandler(); + var transfer = new TransferFunction() + { + To = receiverAddress, + TokenAmount = 100 + }; + var transferReceipt = Utils.Wait(transferHandler.SendRequestAndWaitForReceiptAsync(contractAddress, transfer)); + + // Signing: + var signedTransaction = Utils.Wait(transferHandler.SignTransactionAsync(contractAddress, transfer)); + } + } + + public class StandardTokenDeployment : ContractDeploymentMessage + { + + public static string BYTECODE = "0x60606040526040516020806106f5833981016040528080519060200190919050505b80600160005060003373ffffffffffffffffffffffffffffffffffffffff16815260200190815260200160002060005081905550806000600050819055505b506106868061006f6000396000f360606040523615610074576000357c010000000000000000000000000000000000000000000000000000000090048063095ea7b31461008157806318160ddd146100b657806323b872dd146100d957806370a0823114610117578063a9059cbb14610143578063dd62ed3e1461017857610074565b61007f5b610002565b565b005b6100a060048080359060200190919080359060200190919050506101ad565b6040518082815260200191505060405180910390f35b6100c36004805050610674565b6040518082815260200191505060405180910390f35b6101016004808035906020019091908035906020019091908035906020019091905050610281565b6040518082815260200191505060405180910390f35b61012d600480803590602001909190505061048d565b6040518082815260200191505060405180910390f35b61016260048080359060200190919080359060200190919050506104cb565b6040518082815260200191505060405180910390f35b610197600480803590602001909190803590602001909190505061060b565b6040518082815260200191505060405180910390f35b600081600260005060003373ffffffffffffffffffffffffffffffffffffffff16815260200190815260200160002060005060008573ffffffffffffffffffffffffffffffffffffffff168152602001908152602001600020600050819055508273ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff167f8c5be1e5ebec7d5bd14f71427d1e84f3dd0314c0f7b2291e5b200ac8c7c3b925846040518082815260200191505060405180910390a36001905061027b565b92915050565b600081600160005060008673ffffffffffffffffffffffffffffffffffffffff168152602001908152602001600020600050541015801561031b575081600260005060008673ffffffffffffffffffffffffffffffffffffffff16815260200190815260200160002060005060003373ffffffffffffffffffffffffffffffffffffffff1681526020019081526020016000206000505410155b80156103275750600082115b1561047c5781600160005060008573ffffffffffffffffffffffffffffffffffffffff1681526020019081526020016000206000828282505401925050819055508273ffffffffffffffffffffffffffffffffffffffff168473ffffffffffffffffffffffffffffffffffffffff167fddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef846040518082815260200191505060405180910390a381600160005060008673ffffffffffffffffffffffffffffffffffffffff16815260200190815260200160002060008282825054039250508190555081600260005060008673ffffffffffffffffffffffffffffffffffffffff16815260200190815260200160002060005060003373ffffffffffffffffffffffffffffffffffffffff1681526020019081526020016000206000828282505403925050819055506001905061048656610485565b60009050610486565b5b9392505050565b6000600160005060008373ffffffffffffffffffffffffffffffffffffffff1681526020019081526020016000206000505490506104c6565b919050565b600081600160005060003373ffffffffffffffffffffffffffffffffffffffff168152602001908152602001600020600050541015801561050c5750600082115b156105fb5781600160005060003373ffffffffffffffffffffffffffffffffffffffff16815260200190815260200160002060008282825054039250508190555081600160005060008573ffffffffffffffffffffffffffffffffffffffff1681526020019081526020016000206000828282505401925050819055508273ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff167fddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef846040518082815260200191505060405180910390a36001905061060556610604565b60009050610605565b5b92915050565b6000600260005060008473ffffffffffffffffffffffffffffffffffffffff16815260200190815260200160002060005060008373ffffffffffffffffffffffffffffffffffffffff16815260200190815260200160002060005054905061066e565b92915050565b60006000600050549050610683565b9056"; + + public StandardTokenDeployment() : base(BYTECODE) { } + + [Parameter("uint256", "totalSupply")] + public BigInteger TotalSupply { get; set; } + } + + [Function("balanceOf", "uint256")] + public class BalanceOfFunction : FunctionMessage + { + [Parameter("address", "_owner", 1)] + public string Owner { get; set; } + } + + [Function("transfer", "bool")] + public class TransferFunction : FunctionMessage + { + [Parameter("address", "_to", 1)] + public string To { get; set; } + + [Parameter("uint256", "_value", 2)] + public BigInteger TokenAmount { get; set; } + } + + [Event("Transfer")] + public class TransferEventDTO : IEventDTO + { + [Parameter("address", "_from", 1, true)] + public string From { get; set; } + + [Parameter("address", "_to", 2, true)] + public string To { get; set; } + + [Parameter("uint256", "_value", 3, false)] + public BigInteger Value { get; set; } + } +} diff --git a/Tests/BasicTests/SimpleTests.cs b/Tests/BasicTests/SimpleTests.cs index 9a8c5c8..ab73e71 100644 --- a/Tests/BasicTests/SimpleTests.cs +++ b/Tests/BasicTests/SimpleTests.cs @@ -7,105 +7,126 @@ namespace Tests.BasicTests [TestFixture] public class SimpleTests : DistTest { - [Test] - public void GetDebugInfo() - { - var dockerImage = new CodexDockerImage(); + //[Test] + //public void GetDebugInfo() + //{ + // var dockerImage = new CodexDockerImage(); - var node = SetupCodexNodes(1).BringOnline()[0]; + // var node = SetupCodexNodes(1).BringOnline()[0]; - var debugInfo = node.GetDebugInfo(); + // var debugInfo = node.GetDebugInfo(); - Assert.That(debugInfo.spr, Is.Not.Empty); - Assert.That(debugInfo.codex.revision, Is.EqualTo(dockerImage.GetExpectedImageRevision())); - } + // Assert.That(debugInfo.spr, Is.Not.Empty); + // Assert.That(debugInfo.codex.revision, Is.EqualTo(dockerImage.GetExpectedImageRevision())); + //} - [Test, DontDownloadLogsAndMetricsOnFailure] - public void CanAccessLogs() - { - var node = SetupCodexNodes(1).BringOnline()[0]; + //[Test, DontDownloadLogsAndMetricsOnFailure] + //public void CanAccessLogs() + //{ + // var node = SetupCodexNodes(1).BringOnline()[0]; - var log = node.DownloadLog(); + // var log = node.DownloadLog(); - log.AssertLogContains("Started codex node"); - } + // log.AssertLogContains("Started codex node"); + //} + + //[Test] + //public void TwoMetricsExample() + //{ + // var group = SetupCodexNodes(2) + // .EnableMetrics() + // .BringOnline(); + + // var metrics = GatherMetrics(group); + + // var group2 = SetupCodexNodes(2) + // .EnableMetrics() + // .BringOnline(); + + // var metrics2 = GatherMetrics(group2); + + // var primary = group[0]; + // var secondary = group[1]; + // var primary2 = group2[0]; + // var secondary2 = group2[1]; + + // primary.ConnectToPeer(secondary); + // primary2.ConnectToPeer(secondary2); + + // Thread.Sleep(TimeSpan.FromMinutes(5)); + + // metrics.AssertThat(primary, "libp2p_peers", Is.EqualTo(1)); + // metrics2.AssertThat(primary2, "libp2p_peers", Is.EqualTo(1)); + //} + + //[Test] + //public void OneClientTest() + //{ + // var primary = SetupCodexNodes(1).BringOnline()[0]; + + // var testFile = GenerateTestFile(1.MB()); + + // var contentId = primary.UploadFile(testFile); + + // var downloadedFile = primary.DownloadContent(contentId); + + // testFile.AssertIsEqual(downloadedFile); + //} + + //[Test] + //public void TwoClientsOnePodTest() + //{ + // var group = SetupCodexNodes(2).BringOnline(); + + // var primary = group[0]; + // var secondary = group[1]; + + // PerformTwoClientTest(primary, secondary); + //} + + //[Test] + //public void TwoClientsTwoPodsTest() + //{ + // var primary = SetupCodexNodes(1).BringOnline()[0]; + + // var secondary = SetupCodexNodes(1).BringOnline()[0]; + + // PerformTwoClientTest(primary, secondary); + //} + + //[Test] + //public void TwoClientsTwoLocationsTest() + //{ + // var primary = SetupCodexNodes(1) + // .At(Location.BensLaptop) + // .BringOnline()[0]; + + // var secondary = SetupCodexNodes(1) + // .At(Location.BensOldGamingMachine) + // .BringOnline()[0]; + + // PerformTwoClientTest(primary, secondary); + //} + + //[Test] + //public void RequestStorageTest() + //{ + // var primary = SetupCodexNodes(1) + // .Enable(startupCoins:100) + // .BringOnline()[0]; + + // GiveCoins(primary, 0); + + // AssertBalance(primary, 100); + + //} [Test] - public void TwoMetricsExample() - { - var group = SetupCodexNodes(2) - .EnableMetrics() - .BringOnline(); - - var metrics = GatherMetrics(group); - - var group2 = SetupCodexNodes(2) - .EnableMetrics() - .BringOnline(); - - var metrics2 = GatherMetrics(group2); - - var primary = group[0]; - var secondary = group[1]; - var primary2 = group2[0]; - var secondary2 = group2[1]; - - primary.ConnectToPeer(secondary); - primary2.ConnectToPeer(secondary2); - - Thread.Sleep(TimeSpan.FromMinutes(5)); - - metrics.AssertThat(primary, "libp2p_peers", Is.EqualTo(1)); - metrics2.AssertThat(primary2, "libp2p_peers", Is.EqualTo(1)); - } - - [Test] - public void OneClientTest() + public void DoCommand() { var primary = SetupCodexNodes(1).BringOnline()[0]; - var testFile = GenerateTestFile(1.MB()); - - var contentId = primary.UploadFile(testFile); - - var downloadedFile = primary.DownloadContent(contentId); - - testFile.AssertIsEqual(downloadedFile); - } - - [Test] - public void TwoClientsOnePodTest() - { - var group = SetupCodexNodes(2).BringOnline(); - - var primary = group[0]; - var secondary = group[1]; - - PerformTwoClientTest(primary, secondary); - } - - [Test] - public void TwoClientsTwoPodsTest() - { - var primary = SetupCodexNodes(1).BringOnline()[0]; - - var secondary = SetupCodexNodes(1).BringOnline()[0]; - - PerformTwoClientTest(primary, secondary); - } - - [Test] - public void TwoClientsTwoLocationsTest() - { - var primary = SetupCodexNodes(1) - .At(Location.BensLaptop) - .BringOnline()[0]; - - var secondary = SetupCodexNodes(1) - .At(Location.BensOldGamingMachine) - .BringOnline()[0]; - - PerformTwoClientTest(primary, secondary); + k8sManager.ExampleOfCMD(primary); } private void PerformTwoClientTest(IOnlineCodexNode primary, IOnlineCodexNode secondary) From adbcfb997492a2dfe7e83aa73b249842baf5a112 Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 10 Apr 2023 09:05:27 +0200 Subject: [PATCH 02/46] Setting up test interface for marketplace --- CodexDistTestCore/K8sManager.cs | 4 +- CodexDistTestCore/K8sOperations.cs | 1 + .../Marketplace/MarketplaceAccess.cs | 66 +++++++++++++++++++ .../Marketplace/MarketplaceInitialConfig.cs | 12 ++++ .../{ => Metrics}/K8sPrometheusSpecs.cs | 4 +- .../{ => Metrics}/MetricsAccess.cs | 2 +- .../{ => Metrics}/MetricsAggregator.cs | 4 +- .../{ => Metrics}/MetricsDownloader.cs | 2 +- .../{ => Metrics}/MetricsQuery.cs | 4 +- CodexDistTestCore/OfflineCodexNodes.cs | 14 +++- CodexDistTestCore/OnlineCodexNode.cs | 4 ++ CodexDistTestCore/TryContract.cs | 1 + Tests/BasicTests/SimpleTests.cs | 26 ++++++++ 13 files changed, 133 insertions(+), 11 deletions(-) create mode 100644 CodexDistTestCore/Marketplace/MarketplaceAccess.cs create mode 100644 CodexDistTestCore/Marketplace/MarketplaceInitialConfig.cs rename CodexDistTestCore/{ => Metrics}/K8sPrometheusSpecs.cs (98%) rename CodexDistTestCore/{ => Metrics}/MetricsAccess.cs (98%) rename CodexDistTestCore/{ => Metrics}/MetricsAggregator.cs (97%) rename CodexDistTestCore/{ => Metrics}/MetricsDownloader.cs (98%) rename CodexDistTestCore/{ => Metrics}/MetricsQuery.cs (99%) diff --git a/CodexDistTestCore/K8sManager.cs b/CodexDistTestCore/K8sManager.cs index 46bf07d..5ee9109 100644 --- a/CodexDistTestCore/K8sManager.cs +++ b/CodexDistTestCore/K8sManager.cs @@ -1,4 +1,6 @@ -namespace CodexDistTestCore +using CodexDistTestCore.Metrics; + +namespace CodexDistTestCore { public interface IK8sManager { diff --git a/CodexDistTestCore/K8sOperations.cs b/CodexDistTestCore/K8sOperations.cs index 6bad053..2354e9d 100644 --- a/CodexDistTestCore/K8sOperations.cs +++ b/CodexDistTestCore/K8sOperations.cs @@ -1,4 +1,5 @@ using CodexDistTestCore.Config; +using CodexDistTestCore.Metrics; using k8s; using k8s.KubeConfigModels; using k8s.Models; diff --git a/CodexDistTestCore/Marketplace/MarketplaceAccess.cs b/CodexDistTestCore/Marketplace/MarketplaceAccess.cs new file mode 100644 index 0000000..25bb63f --- /dev/null +++ b/CodexDistTestCore/Marketplace/MarketplaceAccess.cs @@ -0,0 +1,66 @@ +using NUnit.Framework; +using NUnit.Framework.Constraints; + +namespace CodexDistTestCore.Marketplace +{ + public interface IMarketplaceAccess + { + void AdvertiseStorage(ByteSize size, float pricePerMBPerSecond, float collateral); + void AdvertiseContract(ContentId contentId, float maxPricePerMBPerSecond, float minRequiredCollateral, float minRequiredNumberOfDuplicates); + void AssertThatBalance(IResolveConstraint constraint, string message = ""); + float GetBalance(); + } + + public class MarketplaceAccess : IMarketplaceAccess + { + public void AdvertiseContract(ContentId contentId, float maxPricePerMBPerSecond, float minRequiredCollateral, float minRequiredNumberOfDuplicates) + { + throw new NotImplementedException(); + } + + public void AdvertiseStorage(ByteSize size, float pricePerMBPerSecond, float collateral) + { + throw new NotImplementedException(); + } + + public void AssertThatBalance(IResolveConstraint constraint, string message = "") + { + throw new NotImplementedException(); + } + + public float GetBalance() + { + throw new NotImplementedException(); + } + } + + public class MarketplaceUnavailable : IMarketplaceAccess + { + public void AdvertiseContract(ContentId contentId, float maxPricePerMBPerSecond, float minRequiredCollateral, float minRequiredNumberOfDuplicates) + { + Unavailable(); + } + + public void AdvertiseStorage(ByteSize size, float pricePerMBPerSecond, float collateral) + { + Unavailable(); + } + + public void AssertThatBalance(IResolveConstraint constraint, string message = "") + { + Unavailable(); + } + + public float GetBalance() + { + Unavailable(); + return 0.0f; + } + + private void Unavailable() + { + Assert.Fail("Incorrect test setup: Marketplace was not enabled for this group of Codex nodes. Add 'EnableMarketplace(...)' after 'SetupCodexNodes()' to enable it."); + throw new InvalidOperationException(); + } + } +} diff --git a/CodexDistTestCore/Marketplace/MarketplaceInitialConfig.cs b/CodexDistTestCore/Marketplace/MarketplaceInitialConfig.cs new file mode 100644 index 0000000..f7761f5 --- /dev/null +++ b/CodexDistTestCore/Marketplace/MarketplaceInitialConfig.cs @@ -0,0 +1,12 @@ +namespace CodexDistTestCore.Marketplace +{ + public class MarketplaceInitialConfig + { + public MarketplaceInitialConfig(int initialBalance) + { + InitialBalance = initialBalance; + } + + public int InitialBalance { get; } + } +} diff --git a/CodexDistTestCore/K8sPrometheusSpecs.cs b/CodexDistTestCore/Metrics/K8sPrometheusSpecs.cs similarity index 98% rename from CodexDistTestCore/K8sPrometheusSpecs.cs rename to CodexDistTestCore/Metrics/K8sPrometheusSpecs.cs index dcda941..0fc0865 100644 --- a/CodexDistTestCore/K8sPrometheusSpecs.cs +++ b/CodexDistTestCore/Metrics/K8sPrometheusSpecs.cs @@ -1,7 +1,7 @@ using CodexDistTestCore.Config; using k8s.Models; -namespace CodexDistTestCore +namespace CodexDistTestCore.Metrics { public class K8sPrometheusSpecs { @@ -31,7 +31,7 @@ namespace CodexDistTestCore var deploymentSpec = new V1Deployment { ApiVersion = "apps/v1", - Metadata = new V1ObjectMeta + Metadata = new V1ObjectMeta { Name = GetDeploymentName(), NamespaceProperty = K8sCluster.K8sNamespace diff --git a/CodexDistTestCore/MetricsAccess.cs b/CodexDistTestCore/Metrics/MetricsAccess.cs similarity index 98% rename from CodexDistTestCore/MetricsAccess.cs rename to CodexDistTestCore/Metrics/MetricsAccess.cs index feb3fe7..2f6456e 100644 --- a/CodexDistTestCore/MetricsAccess.cs +++ b/CodexDistTestCore/Metrics/MetricsAccess.cs @@ -1,7 +1,7 @@ using NUnit.Framework; using NUnit.Framework.Constraints; -namespace CodexDistTestCore +namespace CodexDistTestCore.Metrics { public interface IMetricsAccess { diff --git a/CodexDistTestCore/MetricsAggregator.cs b/CodexDistTestCore/Metrics/MetricsAggregator.cs similarity index 97% rename from CodexDistTestCore/MetricsAggregator.cs rename to CodexDistTestCore/Metrics/MetricsAggregator.cs index 59e884d..adf0c3f 100644 --- a/CodexDistTestCore/MetricsAggregator.cs +++ b/CodexDistTestCore/Metrics/MetricsAggregator.cs @@ -1,7 +1,7 @@ using NUnit.Framework; using System.Text; -namespace CodexDistTestCore +namespace CodexDistTestCore.Metrics { public class MetricsAggregator { @@ -27,7 +27,7 @@ namespace CodexDistTestCore log.Log("Metrics service started."); - foreach(var node in nodes) + foreach (var node in nodes) { node.Metrics = new MetricsAccess(query, node); } diff --git a/CodexDistTestCore/MetricsDownloader.cs b/CodexDistTestCore/Metrics/MetricsDownloader.cs similarity index 98% rename from CodexDistTestCore/MetricsDownloader.cs rename to CodexDistTestCore/Metrics/MetricsDownloader.cs index 22ef526..18fd10b 100644 --- a/CodexDistTestCore/MetricsDownloader.cs +++ b/CodexDistTestCore/Metrics/MetricsDownloader.cs @@ -1,6 +1,6 @@ using System.Globalization; -namespace CodexDistTestCore +namespace CodexDistTestCore.Metrics { public class MetricsDownloader { diff --git a/CodexDistTestCore/MetricsQuery.cs b/CodexDistTestCore/Metrics/MetricsQuery.cs similarity index 99% rename from CodexDistTestCore/MetricsQuery.cs rename to CodexDistTestCore/Metrics/MetricsQuery.cs index d8ba684..c028a6c 100644 --- a/CodexDistTestCore/MetricsQuery.cs +++ b/CodexDistTestCore/Metrics/MetricsQuery.cs @@ -1,7 +1,7 @@ using CodexDistTestCore.Config; using System.Globalization; -namespace CodexDistTestCore +namespace CodexDistTestCore.Metrics { public class MetricsQuery { @@ -107,7 +107,7 @@ namespace CodexDistTestCore { Timestamp = ToTimestamp(value[0]), Value = ToValue(value[1]) - }; + }; } private string GetInstanceNameForNode(OnlineCodexNode node) diff --git a/CodexDistTestCore/OfflineCodexNodes.cs b/CodexDistTestCore/OfflineCodexNodes.cs index f5527af..d82ed53 100644 --- a/CodexDistTestCore/OfflineCodexNodes.cs +++ b/CodexDistTestCore/OfflineCodexNodes.cs @@ -1,4 +1,6 @@ -namespace CodexDistTestCore +using CodexDistTestCore.Marketplace; + +namespace CodexDistTestCore { public interface IOfflineCodexNodes { @@ -7,6 +9,7 @@ IOfflineCodexNodes WithBootstrapNode(IOnlineCodexNode node); IOfflineCodexNodes WithStorageQuota(ByteSize storageQuota); IOfflineCodexNodes EnableMetrics(); + IOfflineCodexNodes EnableMarketplace(int initialBalance); ICodexNodeGroup BringOnline(); } @@ -36,6 +39,7 @@ public IOnlineCodexNode? BootstrapNode { get; private set; } public ByteSize? StorageQuota { get; private set; } public bool MetricsEnabled { get; private set; } + public MarketplaceInitialConfig? MarketplaceConfig { get; private set; } public OfflineCodexNodes(IK8sManager k8SManager, int numberOfNodes) { @@ -80,6 +84,12 @@ return this; } + public IOfflineCodexNodes EnableMarketplace(int initialBalance) + { + MarketplaceConfig = new MarketplaceInitialConfig(initialBalance); + return this; + } + public string Describe() { var args = string.Join(',', DescribeArgs()); @@ -89,7 +99,7 @@ private IEnumerable DescribeArgs() { if (LogLevel != null) yield return ($"LogLevel={LogLevel}"); - if (BootstrapNode != null) yield return ("BootstrapNode=set"); + if (BootstrapNode != null) yield return ("BootstrapNode=set-not-shown-here"); if (StorageQuota != null) yield return ($"StorageQuote={StorageQuota.SizeInBytes}"); } } diff --git a/CodexDistTestCore/OnlineCodexNode.cs b/CodexDistTestCore/OnlineCodexNode.cs index a9cf511..64f9c6d 100644 --- a/CodexDistTestCore/OnlineCodexNode.cs +++ b/CodexDistTestCore/OnlineCodexNode.cs @@ -1,4 +1,6 @@ using CodexDistTestCore.Config; +using CodexDistTestCore.Marketplace; +using CodexDistTestCore.Metrics; using NUnit.Framework; namespace CodexDistTestCore @@ -11,6 +13,7 @@ namespace CodexDistTestCore void ConnectToPeer(IOnlineCodexNode node); ICodexNodeLog DownloadLog(); IMetricsAccess Metrics { get; } + IMarketplaceAccess Marketplace { get; } } public class OnlineCodexNode : IOnlineCodexNode @@ -32,6 +35,7 @@ namespace CodexDistTestCore public CodexNodeContainer Container { get; } public CodexNodeGroup Group { get; internal set; } = null!; public IMetricsAccess Metrics { get; set; } = new MetricsUnavailable(); + public IMarketplaceAccess Marketplace { set; get; } = new MarketplaceUnavailable(); public string GetName() { diff --git a/CodexDistTestCore/TryContract.cs b/CodexDistTestCore/TryContract.cs index 766054b..efc9d5a 100644 --- a/CodexDistTestCore/TryContract.cs +++ b/CodexDistTestCore/TryContract.cs @@ -16,6 +16,7 @@ namespace CodexDistTestCore public class TryContract { [Test] + [Ignore("aaa")] public void DoThing() { var url = "http://testchain.nethereum.com:8545"; diff --git a/Tests/BasicTests/SimpleTests.cs b/Tests/BasicTests/SimpleTests.cs index 1fd90d4..3785ac3 100644 --- a/Tests/BasicTests/SimpleTests.cs +++ b/Tests/BasicTests/SimpleTests.cs @@ -39,6 +39,32 @@ namespace Tests.BasicTests primary2.Metrics.AssertThat("libp2p_peers", Is.EqualTo(1)); } + [Test] + public void MarketplaceExample() + { + var primary = SetupCodexNodes(1) + .WithStorageQuota(10.GB()) + .EnableMarketplace(initialBalance: 20) + .BringOnline()[0]; + + var secondary = SetupCodexNodes(1) + .EnableMarketplace(initialBalance: 1000) + .BringOnline()[0]; + + primary.ConnectToPeer(secondary); + primary.Marketplace.AdvertiseStorage(10.GB(), pricePerMBPerSecond: 0.01f, collateral: 20); + + var testFile = GenerateTestFile(10.MB()); + var contentId = secondary.UploadFile(testFile); + secondary.Marketplace.AdvertiseContract(contentId, maxPricePerMBPerSecond: 0.02f, minRequiredCollateral: 10, minRequiredNumberOfDuplicates: 1); + + primary.Marketplace.AssertThatBalance(Is.LessThan(20), "Collateral was not placed."); + var primaryBalance = primary.Marketplace.GetBalance(); + + secondary.Marketplace.AssertThatBalance(Is.LessThan(1000), "Contractor was not charged for storage."); + primary.Marketplace.AssertThatBalance(Is.GreaterThan(primaryBalance), "Storer was not paid for storage."); + } + [Test] public void OneClientTest() { From 124a4e373803a241c8454e5e6e279bb0a1b7afba Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 10 Apr 2023 10:09:41 +0200 Subject: [PATCH 03/46] setting up bootstrap geth node --- CodexDistTestCore/K8sManager.cs | 44 ++++-- CodexDistTestCore/K8sOperations.cs | 84 ++++++++---- CodexDistTestCore/Marketplace/K8sGethSpecs.cs | 128 ++++++++++++++++++ .../Marketplace/MarketplaceController.cs | 38 ++++++ 4 files changed, 262 insertions(+), 32 deletions(-) create mode 100644 CodexDistTestCore/Marketplace/K8sGethSpecs.cs create mode 100644 CodexDistTestCore/Marketplace/MarketplaceController.cs diff --git a/CodexDistTestCore/K8sManager.cs b/CodexDistTestCore/K8sManager.cs index 5ee9109..d10f8ea 100644 --- a/CodexDistTestCore/K8sManager.cs +++ b/CodexDistTestCore/K8sManager.cs @@ -1,4 +1,5 @@ -using CodexDistTestCore.Metrics; +using CodexDistTestCore.Marketplace; +using CodexDistTestCore.Metrics; namespace CodexDistTestCore { @@ -17,18 +18,25 @@ namespace CodexDistTestCore private readonly TestLog log; private readonly IFileManager fileManager; private readonly MetricsAggregator metricsAggregator; + private readonly MarketplaceController marketplaceController; public K8sManager(TestLog log, IFileManager fileManager) { this.log = log; this.fileManager = fileManager; metricsAggregator = new MetricsAggregator(log, this); + marketplaceController = new MarketplaceController(log, this); } public ICodexNodeGroup BringOnline(OfflineCodexNodes offline) { var online = CreateOnlineCodexNodes(offline); + if (offline.MarketplaceConfig != null) + { + BringOnlineMarketplace(); + } + K8s(k => k.BringOnline(online, offline)); log.Log($"{online.Describe()} online."); @@ -52,10 +60,9 @@ namespace CodexDistTestCore return online.Origin; } - public void ExampleOfCMD(IOnlineCodexNode node) + public string ExecuteCommand(PodInfo pod, string containerName, string command, params string[] arguments) { - var n = (OnlineCodexNode)node; - K8s(k => k.ExampleOfCommandExecution(n)); + return K8s(k => k.ExecuteCommand(pod, containerName, command, arguments)); } public void DeleteAllResources() @@ -77,9 +84,12 @@ namespace CodexDistTestCore { var spec = new K8sPrometheusSpecs(codexGroupNumberSource.GetNextServicePort(), prometheusNumber, config); - PrometheusInfo? info = null; - K8s(k => info = k.BringOnlinePrometheus(spec)); - return info!; + return K8s(k => k.BringOnlinePrometheus(spec)); + } + + public PodInfo BringOnlineGethBootstrapNode() + { + return K8s(k => k.BringOnlineGethBootstrapNode()); } public void DownloadAllMetrics() @@ -89,9 +99,12 @@ namespace CodexDistTestCore private void BringOnlineMetrics(CodexNodeGroup group) { - var onlineNodes = group.Nodes.Cast().ToArray(); + metricsAggregator.BeginCollectingMetricsFor(DowncastNodes(group)); + } - metricsAggregator.BeginCollectingMetricsFor(onlineNodes); + private void BringOnlineMarketplace() + { + marketplaceController.BringOnlineMarketplace(); } private CodexNodeGroup CreateOnlineCodexNodes(OfflineCodexNodes offline) @@ -124,5 +137,18 @@ namespace CodexDistTestCore action(k8s); k8s.Close(); } + + private T K8s(Func action) + { + var k8s = new K8sOperations(knownPods); + var result = action(k8s); + k8s.Close(); + return result; + } + + private static OnlineCodexNode[] DowncastNodes(CodexNodeGroup group) + { + return group.Nodes.Cast().ToArray(); + } } } diff --git a/CodexDistTestCore/K8sOperations.cs b/CodexDistTestCore/K8sOperations.cs index 2354e9d..576495c 100644 --- a/CodexDistTestCore/K8sOperations.cs +++ b/CodexDistTestCore/K8sOperations.cs @@ -3,6 +3,7 @@ using CodexDistTestCore.Metrics; using k8s; using k8s.KubeConfigModels; using k8s.Models; +using Nethereum.Merkle.Patricia; using NUnit.Framework; namespace CodexDistTestCore @@ -26,29 +27,6 @@ namespace CodexDistTestCore client.Dispose(); } - private Task Callback(Stream stdIn, Stream stdOut, Stream stdErr) - { - using var streamReader = new StreamReader(stdOut); - var lines = new List(); - var line = streamReader.ReadLine(); - while (line != null) - { - lines.Add(line); - line = streamReader.ReadLine(); - } - - Assert.That(lines.Any(l => l.Contains("FOO76543"))); - - - return Task.CompletedTask; - } - - public void ExampleOfCommandExecution(OnlineCodexNode node) - { - Utils.Wait(client.NamespacedPodExecAsync( - node.Group.PodInfo!.Name, K8sNamespace, node.Container.Name, new[] { "echo", "FOO76543" }, false, Callback, new CancellationToken())); - } - public void BringOnline(CodexNodeGroup online, OfflineCodexNodes offline) { EnsureTestNamespace(); @@ -82,6 +60,13 @@ namespace CodexDistTestCore logHandler.Log(stream); } + public string ExecuteCommand(PodInfo pod, string containerName, string command, params string[] arguments) + { + var runner = new CommandRunner(client, pod, containerName, command, arguments); + runner.Run(); + return runner.GetStdOut(); + } + public PrometheusInfo BringOnlinePrometheus(K8sPrometheusSpecs spec) { EnsureTestNamespace(); @@ -93,6 +78,14 @@ namespace CodexDistTestCore return new PrometheusInfo(spec.ServicePort, FetchNewPod()); } + public PodInfo BringOnlineGethBootstrapNode() + { + EnsureTestNamespace(); + + return FetchNewPod(); + + } + private void FetchPodInfo(CodexNodeGroup online) { online.PodInfo = FetchNewPod(); @@ -344,5 +337,50 @@ namespace CodexDistTestCore { return client.ListNamespace().Items.Any(n => n.Metadata.Name == K8sNamespace); } + + private class CommandRunner + { + private readonly Kubernetes client; + private readonly PodInfo pod; + private readonly string containerName; + private readonly string command; + private readonly string[] arguments; + private readonly List lines = new List(); + + public CommandRunner(Kubernetes client, PodInfo pod, string containerName, string command, string[] arguments) + { + this.client = client; + this.pod = pod; + this.containerName = containerName; + this.command = command; + this.arguments = arguments; + } + + public void Run() + { + var input = new[] { command }.Concat(arguments).ToArray(); + + Utils.Wait(client.NamespacedPodExecAsync( + pod.Name, K8sCluster.K8sNamespace, containerName, input, false, Callback, new CancellationToken())); + } + + public string GetStdOut() + { + return string.Join(Environment.NewLine, lines); + } + + private Task Callback(Stream stdIn, Stream stdOut, Stream stdErr) + { + using var streamReader = new StreamReader(stdOut); + var line = streamReader.ReadLine(); + while (line != null) + { + lines.Add(line); + line = streamReader.ReadLine(); + } + + return Task.CompletedTask; + } + } } } diff --git a/CodexDistTestCore/Marketplace/K8sGethSpecs.cs b/CodexDistTestCore/Marketplace/K8sGethSpecs.cs new file mode 100644 index 0000000..c2fd79e --- /dev/null +++ b/CodexDistTestCore/Marketplace/K8sGethSpecs.cs @@ -0,0 +1,128 @@ +using CodexDistTestCore.Config; +using k8s.Models; + +namespace CodexDistTestCore.Marketplace +{ + public class K8sGethBoostrapSpecs + { + public const string ContainerName = "dtest-gethb"; + private const string dockerImage = "thatbenbierens/prometheus-envconf:latest"; // todo - bake modified geth image and post it. + private const string portName = "gethb"; + private const string genesisJsonBase64 = "ewogICAgImNvbmZpZyI6IHsKICAgICAgImNoYWluSWQiOiAxMjM0NSwKICAgICAgImhvbWVzdGVhZEJsb2NrIjogMCwKICAgICAgImVpcDE1MEJsb2NrIjogMCwKICAgICAgImVpcDE1NUJsb2NrIjogMCwKICAgICAgImVpcDE1OEJsb2NrIjogMCwKICAgICAgImJ5emFudGl1bUJsb2NrIjogMCwKICAgICAgImNvbnN0YW50aW5vcGxlQmxvY2siOiAwLAogICAgICAicGV0ZXJzYnVyZ0Jsb2NrIjogMCwKICAgICAgImlzdGFuYnVsQmxvY2siOiAwLAogICAgICAibXVpckdsYWNpZXJCbG9jayI6IDAsCiAgICAgICJiZXJsaW5CbG9jayI6IDAsCiAgICAgICJsb25kb25CbG9jayI6IDAsCiAgICAgICJhcnJvd0dsYWNpZXJCbG9jayI6IDAsCiAgICAgICJncmF5R2xhY2llckJsb2NrIjogMCwKICAgICAgImNsaXF1ZSI6IHsKICAgICAgICAicGVyaW9kIjogNSwKICAgICAgICAiZXBvY2giOiAzMDAwMAogICAgICB9CiAgICB9LAogICAgImRpZmZpY3VsdHkiOiAiMSIsCiAgICAiZ2FzTGltaXQiOiAiODAwMDAwMDAwIiwKICAgICJleHRyYWRhdGEiOiAiMHgwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwN2RmOWE4NzVhMTc0YjNiYzU2NWU2NDI0YTAwNTBlYmMxYjJkMWQ4MjAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAiLAogICAgImFsbG9jIjogewogICAgICAiQUNDT1VOVF9IRVJFIjogeyAiYmFsYW5jZSI6ICI1MDAwMDAiIH0KICAgIH0KICB9"; + + public K8sGethBoostrapSpecs(int servicePort) + { + ServicePort = servicePort; + } + + public int ServicePort { get; } + + public string GetDeploymentName() + { + return "test-gethb"; + } + + public V1Deployment CreateGethBootstrapDeployment() + { + var deploymentSpec = new V1Deployment + { + ApiVersion = "apps/v1", + Metadata = new V1ObjectMeta + { + Name = GetDeploymentName(), + NamespaceProperty = K8sCluster.K8sNamespace + }, + Spec = new V1DeploymentSpec + { + Replicas = 1, + Selector = new V1LabelSelector + { + MatchLabels = CreateSelector() + }, + Template = new V1PodTemplateSpec + { + Metadata = new V1ObjectMeta + { + Labels = CreateSelector() + }, + Spec = new V1PodSpec + { + Containers = new List + { + new V1Container + { + Name = ContainerName, + Image = dockerImage, + Ports = new List + { + new V1ContainerPort + { + ContainerPort = 9090, + Name = portName + } + }, + Env = new List + { + new V1EnvVar + { + Name = "GETH_ARGS", + Value = "--qwerty" + }, + new V1EnvVar + { + Name = "GENSIS_JSON", + Value = genesisJsonBase64 + }, + new V1EnvVar + { + Name = "IS_BOOTSTRAP", + Value = "1" + } + } + } + } + } + } + } + }; + + return deploymentSpec; + } + + public V1Service CreateGethBootstrapService() + { + var serviceSpec = new V1Service + { + ApiVersion = "v1", + Metadata = new V1ObjectMeta + { + Name = "codex-gethb-service", + NamespaceProperty = K8sCluster.K8sNamespace + }, + Spec = new V1ServiceSpec + { + Type = "NodePort", + Selector = CreateSelector(), + Ports = new List + { + new V1ServicePort + { + Name = "gethb-service", + Protocol = "TCP", + Port = 9090, + TargetPort = portName, + NodePort = ServicePort + } + } + } + }; + + return serviceSpec; + } + + private Dictionary CreateSelector() + { + return new Dictionary { { "test-gethb", "dtest-gethb" } }; + } + } +} diff --git a/CodexDistTestCore/Marketplace/MarketplaceController.cs b/CodexDistTestCore/Marketplace/MarketplaceController.cs new file mode 100644 index 0000000..d0f5947 --- /dev/null +++ b/CodexDistTestCore/Marketplace/MarketplaceController.cs @@ -0,0 +1,38 @@ +namespace CodexDistTestCore.Marketplace +{ + public class MarketplaceController + { + private readonly TestLog log; + private readonly K8sManager k8sManager; + private PodInfo? gethBootstrapNode; + private string bootstrapAccount = string.Empty; + private string bootstrapGenesisJson = string.Empty; + + public MarketplaceController(TestLog log, K8sManager k8sManager) + { + this.log = log; + this.k8sManager = k8sManager; + } + + public void BringOnlineMarketplace() + { + if (gethBootstrapNode != null) return; + + log.Log("Starting Geth bootstrap node..."); + gethBootstrapNode = k8sManager.BringOnlineGethBootstrapNode(); + ExtractAccountAndGenesisJson(); + log.Log("Geth boothstrap node started."); + } + + private void ExtractAccountAndGenesisJson() + { + bootstrapAccount = ExecuteCommand("cat", "account_string.txt"); + bootstrapGenesisJson = ExecuteCommand("cat", "genesis.json"); + } + + private string ExecuteCommand(string command, params string[] arguments) + { + return k8sManager.ExecuteCommand(gethBootstrapNode!, K8sGethBoostrapSpecs.ContainerName, command, arguments); + } + } +} From 2f694bac8d081aa587383e3a3e4621118e4e9f80 Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 10 Apr 2023 10:21:26 +0200 Subject: [PATCH 04/46] pushes wip docker image for geth-confenv --- CodexDistTestCore/Marketplace/K8sGethSpecs.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CodexDistTestCore/Marketplace/K8sGethSpecs.cs b/CodexDistTestCore/Marketplace/K8sGethSpecs.cs index c2fd79e..3485ddc 100644 --- a/CodexDistTestCore/Marketplace/K8sGethSpecs.cs +++ b/CodexDistTestCore/Marketplace/K8sGethSpecs.cs @@ -6,7 +6,7 @@ namespace CodexDistTestCore.Marketplace public class K8sGethBoostrapSpecs { public const string ContainerName = "dtest-gethb"; - private const string dockerImage = "thatbenbierens/prometheus-envconf:latest"; // todo - bake modified geth image and post it. + private const string dockerImage = "thatbenbierens/geth-confenv:latest"; private const string portName = "gethb"; private const string genesisJsonBase64 = "ewogICAgImNvbmZpZyI6IHsKICAgICAgImNoYWluSWQiOiAxMjM0NSwKICAgICAgImhvbWVzdGVhZEJsb2NrIjogMCwKICAgICAgImVpcDE1MEJsb2NrIjogMCwKICAgICAgImVpcDE1NUJsb2NrIjogMCwKICAgICAgImVpcDE1OEJsb2NrIjogMCwKICAgICAgImJ5emFudGl1bUJsb2NrIjogMCwKICAgICAgImNvbnN0YW50aW5vcGxlQmxvY2siOiAwLAogICAgICAicGV0ZXJzYnVyZ0Jsb2NrIjogMCwKICAgICAgImlzdGFuYnVsQmxvY2siOiAwLAogICAgICAibXVpckdsYWNpZXJCbG9jayI6IDAsCiAgICAgICJiZXJsaW5CbG9jayI6IDAsCiAgICAgICJsb25kb25CbG9jayI6IDAsCiAgICAgICJhcnJvd0dsYWNpZXJCbG9jayI6IDAsCiAgICAgICJncmF5R2xhY2llckJsb2NrIjogMCwKICAgICAgImNsaXF1ZSI6IHsKICAgICAgICAicGVyaW9kIjogNSwKICAgICAgICAiZXBvY2giOiAzMDAwMAogICAgICB9CiAgICB9LAogICAgImRpZmZpY3VsdHkiOiAiMSIsCiAgICAiZ2FzTGltaXQiOiAiODAwMDAwMDAwIiwKICAgICJleHRyYWRhdGEiOiAiMHgwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwN2RmOWE4NzVhMTc0YjNiYzU2NWU2NDI0YTAwNTBlYmMxYjJkMWQ4MjAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAiLAogICAgImFsbG9jIjogewogICAgICAiQUNDT1VOVF9IRVJFIjogeyAiYmFsYW5jZSI6ICI1MDAwMDAiIH0KICAgIH0KICB9"; From e800197cdd6af66dc7efd0246043a147649860dc Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 10 Apr 2023 14:00:12 +0200 Subject: [PATCH 05/46] Sets up bootstrap geth node --- CodexDistTestCore/K8sManager.cs | 4 ++- CodexDistTestCore/K8sOperations.cs | 29 +++++++++++++++++-- CodexDistTestCore/Marketplace/K8sGethSpecs.cs | 12 ++++---- .../Marketplace/MarketplaceController.cs | 7 ++++- 4 files changed, 41 insertions(+), 11 deletions(-) diff --git a/CodexDistTestCore/K8sManager.cs b/CodexDistTestCore/K8sManager.cs index d10f8ea..635a8fe 100644 --- a/CodexDistTestCore/K8sManager.cs +++ b/CodexDistTestCore/K8sManager.cs @@ -89,7 +89,9 @@ namespace CodexDistTestCore public PodInfo BringOnlineGethBootstrapNode() { - return K8s(k => k.BringOnlineGethBootstrapNode()); + var spec = new K8sGethBoostrapSpecs(codexGroupNumberSource.GetNextServicePort()); + + return K8s(k => k.BringOnlineGethBootstrapNode(spec)); } public void DownloadAllMetrics() diff --git a/CodexDistTestCore/K8sOperations.cs b/CodexDistTestCore/K8sOperations.cs index 576495c..86ab636 100644 --- a/CodexDistTestCore/K8sOperations.cs +++ b/CodexDistTestCore/K8sOperations.cs @@ -1,4 +1,5 @@ using CodexDistTestCore.Config; +using CodexDistTestCore.Marketplace; using CodexDistTestCore.Metrics; using k8s; using k8s.KubeConfigModels; @@ -78,12 +79,15 @@ namespace CodexDistTestCore return new PrometheusInfo(spec.ServicePort, FetchNewPod()); } - public PodInfo BringOnlineGethBootstrapNode() + public PodInfo BringOnlineGethBootstrapNode(K8sGethBoostrapSpecs spec) { EnsureTestNamespace(); - return FetchNewPod(); + CreateGethBootstrapDeployment(spec); + CreateGethBootstrapService(spec); + WaitUntilGethBootstrapOnline(spec); + return FetchNewPod(); } private void FetchPodInfo(CodexNodeGroup online) @@ -140,7 +144,16 @@ namespace CodexDistTestCore private void WaitUntilPrometheusOnline(K8sPrometheusSpecs spec) { - var deploymentName = spec.GetDeploymentName(); + WaitUntilDeploymentOnline(spec.GetDeploymentName()); + } + + private void WaitUntilGethBootstrapOnline(K8sGethBoostrapSpecs spec) + { + WaitUntilDeploymentOnline(spec.GetDeploymentName()); + } + + private void WaitUntilDeploymentOnline(string deploymentName) + { WaitUntil(() => { var deployment = client.ReadNamespacedDeployment(deploymentName, K8sNamespace); @@ -216,6 +229,11 @@ namespace CodexDistTestCore client.CreateNamespacedService(spec.CreatePrometheusService(), K8sNamespace); } + private void CreateGethBootstrapService(K8sGethBoostrapSpecs spec) + { + client.CreateNamespacedService(spec.CreateGethBootstrapService(), K8sNamespace); + } + #endregion #region Deployment management @@ -298,6 +316,11 @@ namespace CodexDistTestCore client.CreateNamespacedDeployment(spec.CreatePrometheusDeployment(), K8sNamespace); } + private void CreateGethBootstrapDeployment(K8sGethBoostrapSpecs spec) + { + client.CreateNamespacedDeployment(spec.CreateGethBootstrapDeployment(), K8sNamespace); + } + #endregion #region Namespace management diff --git a/CodexDistTestCore/Marketplace/K8sGethSpecs.cs b/CodexDistTestCore/Marketplace/K8sGethSpecs.cs index 3485ddc..0682b49 100644 --- a/CodexDistTestCore/Marketplace/K8sGethSpecs.cs +++ b/CodexDistTestCore/Marketplace/K8sGethSpecs.cs @@ -63,14 +63,14 @@ namespace CodexDistTestCore.Marketplace }, Env = new List { + //new V1EnvVar + //{ + // Name = "GETH_ARGS", + // Value = "--qwerty" + //}, new V1EnvVar { - Name = "GETH_ARGS", - Value = "--qwerty" - }, - new V1EnvVar - { - Name = "GENSIS_JSON", + Name = "GENESIS_JSON", Value = genesisJsonBase64 }, new V1EnvVar diff --git a/CodexDistTestCore/Marketplace/MarketplaceController.cs b/CodexDistTestCore/Marketplace/MarketplaceController.cs index d0f5947..38b5ad1 100644 --- a/CodexDistTestCore/Marketplace/MarketplaceController.cs +++ b/CodexDistTestCore/Marketplace/MarketplaceController.cs @@ -1,4 +1,6 @@ -namespace CodexDistTestCore.Marketplace +using NUnit.Framework; + +namespace CodexDistTestCore.Marketplace { public class MarketplaceController { @@ -28,6 +30,9 @@ { bootstrapAccount = ExecuteCommand("cat", "account_string.txt"); bootstrapGenesisJson = ExecuteCommand("cat", "genesis.json"); + + Assert.That(bootstrapAccount, Is.Not.Empty, "Unable to fetch account for bootstrap geth node. Test infra failure."); + Assert.That(bootstrapGenesisJson, Is.Not.Empty, "Unable to fetch genesis-json for bootstrap geth node. Test infra failure."); } private string ExecuteCommand(string command, params string[] arguments) From ab8318e1022675c85af0d804613a64d72f8475ba Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 10 Apr 2023 14:48:16 +0200 Subject: [PATCH 06/46] Successful spin-up of geth companion node --- CodexDistTestCore/CodexNodeContainer.cs | 25 +++++++-- CodexDistTestCore/CodexNodeGroup.cs | 2 + CodexDistTestCore/K8sManager.cs | 8 +-- CodexDistTestCore/K8sOperations.cs | 17 ++++-- .../Marketplace/GethCompanionNodeContainer.cs | 53 +++++++++++++++++++ CodexDistTestCore/Marketplace/K8sGethSpecs.cs | 8 ++- .../Marketplace/MarketplaceController.cs | 26 +++++++-- Tests/BasicTests/SimpleTests.cs | 34 +++++------- 8 files changed, 136 insertions(+), 37 deletions(-) create mode 100644 CodexDistTestCore/Marketplace/GethCompanionNodeContainer.cs diff --git a/CodexDistTestCore/CodexNodeContainer.cs b/CodexDistTestCore/CodexNodeContainer.cs index 4a1f239..d9a87be 100644 --- a/CodexDistTestCore/CodexNodeContainer.cs +++ b/CodexDistTestCore/CodexNodeContainer.cs @@ -1,8 +1,10 @@ -namespace CodexDistTestCore +using CodexDistTestCore.Marketplace; + +namespace CodexDistTestCore { public class CodexNodeContainer { - public CodexNodeContainer(string name, int servicePort, string servicePortName, int apiPort, string containerPortName, int discoveryPort, int listenPort, string dataDir, int metricsPort) + public CodexNodeContainer(string name, int servicePort, string servicePortName, int apiPort, string containerPortName, int discoveryPort, int listenPort, string dataDir, int metricsPort, GethCompanionNodeContainer? gethCompanionNodeContainer) { Name = name; ServicePort = servicePort; @@ -13,6 +15,7 @@ ListenPort = listenPort; DataDir = dataDir; MetricsPort = metricsPort; + GethCompanionNodeContainer = gethCompanionNodeContainer; } public string Name { get; } @@ -24,6 +27,8 @@ public int ListenPort { get; } public string DataDir { get; } public int MetricsPort { get; } + + public GethCompanionNodeContainer? GethCompanionNodeContainer { get; } } public class CodexGroupNumberSource @@ -71,7 +76,8 @@ discoveryPort: codexPortSource.GetNextNumber(), listenPort: codexPortSource.GetNextNumber(), dataDir: $"datadir{n}", - metricsPort: GetMetricsPort(offline) + metricsPort: GetMetricsPort(offline), + CreateGethNodeContainer(offline, n) ); } @@ -80,5 +86,18 @@ if (offline.MetricsEnabled) return codexPortSource.GetNextNumber(); return 0; } + + private GethCompanionNodeContainer? CreateGethNodeContainer(OfflineCodexNodes offline, int n) + { + if (offline.MarketplaceConfig == null) return null; + + return new GethCompanionNodeContainer( + name: $"geth-node{n}", + servicePort: groupContainerFactory.GetNextServicePort(), + servicePortName: groupContainerFactory.GetNextServicePortName(), + apiPort: codexPortSource.GetNextNumber(), + containerPortName: $"geth-{n}" + ); + } } } diff --git a/CodexDistTestCore/CodexNodeGroup.cs b/CodexDistTestCore/CodexNodeGroup.cs index 975eccf..a3a82d3 100644 --- a/CodexDistTestCore/CodexNodeGroup.cs +++ b/CodexDistTestCore/CodexNodeGroup.cs @@ -1,4 +1,5 @@ using CodexDistTestCore.Config; +using CodexDistTestCore.Marketplace; using k8s.Models; using System.Collections; @@ -45,6 +46,7 @@ namespace CodexDistTestCore public V1Deployment? Deployment { get; set; } public V1Service? Service { get; set; } public PodInfo? PodInfo { get; set; } + public GethInfo? GethInfo { get; set; } public CodexNodeContainer[] GetContainers() { diff --git a/CodexDistTestCore/K8sManager.cs b/CodexDistTestCore/K8sManager.cs index 635a8fe..5e8716b 100644 --- a/CodexDistTestCore/K8sManager.cs +++ b/CodexDistTestCore/K8sManager.cs @@ -34,7 +34,7 @@ namespace CodexDistTestCore if (offline.MarketplaceConfig != null) { - BringOnlineMarketplace(); + online.GethInfo = BringOnlineMarketplace(); } K8s(k => k.BringOnline(online, offline)); @@ -87,7 +87,7 @@ namespace CodexDistTestCore return K8s(k => k.BringOnlinePrometheus(spec)); } - public PodInfo BringOnlineGethBootstrapNode() + public GethInfo BringOnlineGethBootstrapNode() { var spec = new K8sGethBoostrapSpecs(codexGroupNumberSource.GetNextServicePort()); @@ -104,9 +104,9 @@ namespace CodexDistTestCore metricsAggregator.BeginCollectingMetricsFor(DowncastNodes(group)); } - private void BringOnlineMarketplace() + private GethInfo BringOnlineMarketplace() { - marketplaceController.BringOnlineMarketplace(); + return marketplaceController.BringOnlineMarketplace(); } private CodexNodeGroup CreateOnlineCodexNodes(OfflineCodexNodes offline) diff --git a/CodexDistTestCore/K8sOperations.cs b/CodexDistTestCore/K8sOperations.cs index 86ab636..22e7998 100644 --- a/CodexDistTestCore/K8sOperations.cs +++ b/CodexDistTestCore/K8sOperations.cs @@ -2,10 +2,9 @@ using CodexDistTestCore.Marketplace; using CodexDistTestCore.Metrics; using k8s; -using k8s.KubeConfigModels; using k8s.Models; -using Nethereum.Merkle.Patricia; using NUnit.Framework; +using System.Numerics; namespace CodexDistTestCore { @@ -79,7 +78,7 @@ namespace CodexDistTestCore return new PrometheusInfo(spec.ServicePort, FetchNewPod()); } - public PodInfo BringOnlineGethBootstrapNode(K8sGethBoostrapSpecs spec) + public GethInfo BringOnlineGethBootstrapNode(K8sGethBoostrapSpecs spec) { EnsureTestNamespace(); @@ -87,7 +86,7 @@ namespace CodexDistTestCore CreateGethBootstrapService(spec); WaitUntilGethBootstrapOnline(spec); - return FetchNewPod(); + return new GethInfo(spec, FetchNewPod()); } private void FetchPodInfo(CodexNodeGroup online) @@ -213,6 +212,11 @@ namespace CodexDistTestCore TargetPort = container.ContainerPortName, NodePort = container.ServicePort }); + + if (container.GethCompanionNodeContainer != null) + { + result.Add(container.GethCompanionNodeContainer.CreateServicePort()); + } } return result; } @@ -299,6 +303,11 @@ namespace CodexDistTestCore }, Env = dockerImage.CreateEnvironmentVariables(offline, container) }); + + if (container.GethCompanionNodeContainer != null) + { + result.Add(container.GethCompanionNodeContainer.CreateDeploymentContainer(online.GethInfo!)); + } } return result; diff --git a/CodexDistTestCore/Marketplace/GethCompanionNodeContainer.cs b/CodexDistTestCore/Marketplace/GethCompanionNodeContainer.cs new file mode 100644 index 0000000..eeb84ca --- /dev/null +++ b/CodexDistTestCore/Marketplace/GethCompanionNodeContainer.cs @@ -0,0 +1,53 @@ +using k8s.Models; + +namespace CodexDistTestCore.Marketplace +{ + public class GethCompanionNodeContainer + { + public GethCompanionNodeContainer(string name, int servicePort, string servicePortName, int apiPort, string containerPortName) + { + Name = name; + ServicePort = servicePort; + ServicePortName = servicePortName; + ApiPort = apiPort; + ContainerPortName = containerPortName; + } + + public string Name { get; } + public int ServicePort { get; } + public string ServicePortName { get; } + public int ApiPort { get; } + public string ContainerPortName { get; } + + public V1Container CreateDeploymentContainer(GethInfo gethInfo) + { + return new V1Container + { + Name = Name, + Image = GethDockerImage.Image, + Ports = new List + { + new V1ContainerPort + { + ContainerPort = ApiPort, + Name = ContainerPortName + } + }, + // todo: use env vars to connect this node to the bootstrap node provided by gethInfo.podInfo & gethInfo.servicePort & gethInfo.genesisJsonBase64 + //Env = dockerImage.CreateEnvironmentVariables(offline, container) + }; + } + + public V1ServicePort CreateServicePort() + { + return new V1ServicePort + { + Name = ServicePortName, + Protocol = "TCP", + Port = ApiPort, + TargetPort = ContainerPortName, + NodePort = ServicePort + }; + } + } +} diff --git a/CodexDistTestCore/Marketplace/K8sGethSpecs.cs b/CodexDistTestCore/Marketplace/K8sGethSpecs.cs index 0682b49..dd0204d 100644 --- a/CodexDistTestCore/Marketplace/K8sGethSpecs.cs +++ b/CodexDistTestCore/Marketplace/K8sGethSpecs.cs @@ -3,10 +3,14 @@ using k8s.Models; namespace CodexDistTestCore.Marketplace { + public static class GethDockerImage + { + public const string Image = "thatbenbierens/geth-confenv:latest"; + } + public class K8sGethBoostrapSpecs { public const string ContainerName = "dtest-gethb"; - private const string dockerImage = "thatbenbierens/geth-confenv:latest"; private const string portName = "gethb"; private const string genesisJsonBase64 = "ewogICAgImNvbmZpZyI6IHsKICAgICAgImNoYWluSWQiOiAxMjM0NSwKICAgICAgImhvbWVzdGVhZEJsb2NrIjogMCwKICAgICAgImVpcDE1MEJsb2NrIjogMCwKICAgICAgImVpcDE1NUJsb2NrIjogMCwKICAgICAgImVpcDE1OEJsb2NrIjogMCwKICAgICAgImJ5emFudGl1bUJsb2NrIjogMCwKICAgICAgImNvbnN0YW50aW5vcGxlQmxvY2siOiAwLAogICAgICAicGV0ZXJzYnVyZ0Jsb2NrIjogMCwKICAgICAgImlzdGFuYnVsQmxvY2siOiAwLAogICAgICAibXVpckdsYWNpZXJCbG9jayI6IDAsCiAgICAgICJiZXJsaW5CbG9jayI6IDAsCiAgICAgICJsb25kb25CbG9jayI6IDAsCiAgICAgICJhcnJvd0dsYWNpZXJCbG9jayI6IDAsCiAgICAgICJncmF5R2xhY2llckJsb2NrIjogMCwKICAgICAgImNsaXF1ZSI6IHsKICAgICAgICAicGVyaW9kIjogNSwKICAgICAgICAiZXBvY2giOiAzMDAwMAogICAgICB9CiAgICB9LAogICAgImRpZmZpY3VsdHkiOiAiMSIsCiAgICAiZ2FzTGltaXQiOiAiODAwMDAwMDAwIiwKICAgICJleHRyYWRhdGEiOiAiMHgwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwN2RmOWE4NzVhMTc0YjNiYzU2NWU2NDI0YTAwNTBlYmMxYjJkMWQ4MjAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAiLAogICAgImFsbG9jIjogewogICAgICAiQUNDT1VOVF9IRVJFIjogeyAiYmFsYW5jZSI6ICI1MDAwMDAiIH0KICAgIH0KICB9"; @@ -52,7 +56,7 @@ namespace CodexDistTestCore.Marketplace new V1Container { Name = ContainerName, - Image = dockerImage, + Image = GethDockerImage.Image, Ports = new List { new V1ContainerPort diff --git a/CodexDistTestCore/Marketplace/MarketplaceController.cs b/CodexDistTestCore/Marketplace/MarketplaceController.cs index 38b5ad1..fd68ade 100644 --- a/CodexDistTestCore/Marketplace/MarketplaceController.cs +++ b/CodexDistTestCore/Marketplace/MarketplaceController.cs @@ -1,4 +1,5 @@ using NUnit.Framework; +using System.Text; namespace CodexDistTestCore.Marketplace { @@ -6,7 +7,7 @@ namespace CodexDistTestCore.Marketplace { private readonly TestLog log; private readonly K8sManager k8sManager; - private PodInfo? gethBootstrapNode; + private GethInfo? gethBootstrapNode; private string bootstrapAccount = string.Empty; private string bootstrapGenesisJson = string.Empty; @@ -16,14 +17,16 @@ namespace CodexDistTestCore.Marketplace this.k8sManager = k8sManager; } - public void BringOnlineMarketplace() + public GethInfo BringOnlineMarketplace() { - if (gethBootstrapNode != null) return; + if (gethBootstrapNode != null) return gethBootstrapNode; log.Log("Starting Geth bootstrap node..."); gethBootstrapNode = k8sManager.BringOnlineGethBootstrapNode(); ExtractAccountAndGenesisJson(); log.Log("Geth boothstrap node started."); + + return gethBootstrapNode; } private void ExtractAccountAndGenesisJson() @@ -33,11 +36,26 @@ namespace CodexDistTestCore.Marketplace Assert.That(bootstrapAccount, Is.Not.Empty, "Unable to fetch account for bootstrap geth node. Test infra failure."); Assert.That(bootstrapGenesisJson, Is.Not.Empty, "Unable to fetch genesis-json for bootstrap geth node. Test infra failure."); + + gethBootstrapNode!.GenesisJsonBase64 = Convert.ToBase64String(Encoding.ASCII.GetBytes(bootstrapGenesisJson)); } private string ExecuteCommand(string command, params string[] arguments) { - return k8sManager.ExecuteCommand(gethBootstrapNode!, K8sGethBoostrapSpecs.ContainerName, command, arguments); + return k8sManager.ExecuteCommand(gethBootstrapNode!.Pod, K8sGethBoostrapSpecs.ContainerName, command, arguments); } } + + public class GethInfo + { + public GethInfo(K8sGethBoostrapSpecs spec, PodInfo pod) + { + Spec = spec; + Pod = pod; + } + + public K8sGethBoostrapSpecs Spec { get; } + public PodInfo Pod { get; } + public string GenesisJsonBase64 { get; set; } = string.Empty; + } } diff --git a/Tests/BasicTests/SimpleTests.cs b/Tests/BasicTests/SimpleTests.cs index 3785ac3..0812b78 100644 --- a/Tests/BasicTests/SimpleTests.cs +++ b/Tests/BasicTests/SimpleTests.cs @@ -6,14 +6,6 @@ namespace Tests.BasicTests [TestFixture] public class SimpleTests : DistTest { - [Test] - public void DoCommand() - { - var primary = SetupCodexNodes(1).BringOnline()[0]; - - k8sManager.ExampleOfCMD(primary); - } - [Test] public void TwoMetricsExample() { @@ -47,22 +39,24 @@ namespace Tests.BasicTests .EnableMarketplace(initialBalance: 20) .BringOnline()[0]; - var secondary = SetupCodexNodes(1) - .EnableMarketplace(initialBalance: 1000) - .BringOnline()[0]; + //var secondary = SetupCodexNodes(1) + // .EnableMarketplace(initialBalance: 1000) + // .BringOnline()[0]; - primary.ConnectToPeer(secondary); - primary.Marketplace.AdvertiseStorage(10.GB(), pricePerMBPerSecond: 0.01f, collateral: 20); + //primary.ConnectToPeer(secondary); + //primary.Marketplace.AdvertiseStorage(10.GB(), pricePerMBPerSecond: 0.01f, collateral: 20); - var testFile = GenerateTestFile(10.MB()); - var contentId = secondary.UploadFile(testFile); - secondary.Marketplace.AdvertiseContract(contentId, maxPricePerMBPerSecond: 0.02f, minRequiredCollateral: 10, minRequiredNumberOfDuplicates: 1); + //var testFile = GenerateTestFile(10.MB()); + //var contentId = secondary.UploadFile(testFile); + //secondary.Marketplace.AdvertiseContract(contentId, maxPricePerMBPerSecond: 0.02f, minRequiredCollateral: 10, minRequiredNumberOfDuplicates: 1); - primary.Marketplace.AssertThatBalance(Is.LessThan(20), "Collateral was not placed."); - var primaryBalance = primary.Marketplace.GetBalance(); + //primary.Marketplace.AssertThatBalance(Is.LessThan(20), "Collateral was not placed."); + //var primaryBalance = primary.Marketplace.GetBalance(); - secondary.Marketplace.AssertThatBalance(Is.LessThan(1000), "Contractor was not charged for storage."); - primary.Marketplace.AssertThatBalance(Is.GreaterThan(primaryBalance), "Storer was not paid for storage."); + //secondary.Marketplace.AssertThatBalance(Is.LessThan(1000), "Contractor was not charged for storage."); + //primary.Marketplace.AssertThatBalance(Is.GreaterThan(primaryBalance), "Storer was not paid for storage."); + + var aa = 0; } [Test] From 230f9f3bd06213698cdd36eec77efdc947a15e37 Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 10 Apr 2023 15:21:45 +0200 Subject: [PATCH 07/46] Fixes port clashes for multiple geth nodes per pod --- CodexDistTestCore/CodexNodeContainer.cs | 1 + .../Marketplace/GethCompanionNodeContainer.cs | 18 ++++++++++++++++-- .../Marketplace/MarketplaceController.cs | 14 ++++++++++++-- Tests/BasicTests/SimpleTests.cs | 2 +- 4 files changed, 30 insertions(+), 5 deletions(-) diff --git a/CodexDistTestCore/CodexNodeContainer.cs b/CodexDistTestCore/CodexNodeContainer.cs index d9a87be..915a2b4 100644 --- a/CodexDistTestCore/CodexNodeContainer.cs +++ b/CodexDistTestCore/CodexNodeContainer.cs @@ -96,6 +96,7 @@ namespace CodexDistTestCore servicePort: groupContainerFactory.GetNextServicePort(), servicePortName: groupContainerFactory.GetNextServicePortName(), apiPort: codexPortSource.GetNextNumber(), + rpcPort: codexPortSource.GetNextNumber(), containerPortName: $"geth-{n}" ); } diff --git a/CodexDistTestCore/Marketplace/GethCompanionNodeContainer.cs b/CodexDistTestCore/Marketplace/GethCompanionNodeContainer.cs index eeb84ca..a4b452b 100644 --- a/CodexDistTestCore/Marketplace/GethCompanionNodeContainer.cs +++ b/CodexDistTestCore/Marketplace/GethCompanionNodeContainer.cs @@ -4,12 +4,13 @@ namespace CodexDistTestCore.Marketplace { public class GethCompanionNodeContainer { - public GethCompanionNodeContainer(string name, int servicePort, string servicePortName, int apiPort, string containerPortName) + public GethCompanionNodeContainer(string name, int servicePort, string servicePortName, int apiPort, int rpcPort, string containerPortName) { Name = name; ServicePort = servicePort; ServicePortName = servicePortName; ApiPort = apiPort; + RpcPort = rpcPort; ContainerPortName = containerPortName; } @@ -17,6 +18,7 @@ namespace CodexDistTestCore.Marketplace public int ServicePort { get; } public string ServicePortName { get; } public int ApiPort { get; } + public int RpcPort { get; } public string ContainerPortName { get; } public V1Container CreateDeploymentContainer(GethInfo gethInfo) @@ -34,7 +36,19 @@ namespace CodexDistTestCore.Marketplace } }, // todo: use env vars to connect this node to the bootstrap node provided by gethInfo.podInfo & gethInfo.servicePort & gethInfo.genesisJsonBase64 - //Env = dockerImage.CreateEnvironmentVariables(offline, container) + Env = new List + { + new V1EnvVar + { + Name = "GETH_ARGS", + Value = $"--port {ApiPort} --discovery.port {ApiPort} --authrpc.port {RpcPort}" + }, + new V1EnvVar + { + Name = "GENESIS_JSON", + Value = gethInfo.GenesisJsonBase64 + } + } }; } diff --git a/CodexDistTestCore/Marketplace/MarketplaceController.cs b/CodexDistTestCore/Marketplace/MarketplaceController.cs index fd68ade..e832335 100644 --- a/CodexDistTestCore/Marketplace/MarketplaceController.cs +++ b/CodexDistTestCore/Marketplace/MarketplaceController.cs @@ -31,8 +31,12 @@ namespace CodexDistTestCore.Marketplace private void ExtractAccountAndGenesisJson() { - bootstrapAccount = ExecuteCommand("cat", "account_string.txt"); - bootstrapGenesisJson = ExecuteCommand("cat", "genesis.json"); + FetchAccountAndGenesisJson(); + if (string.IsNullOrEmpty(bootstrapAccount) || string.IsNullOrEmpty(bootstrapGenesisJson)) + { + Thread.Sleep(TimeSpan.FromSeconds(15)); + FetchAccountAndGenesisJson(); + } Assert.That(bootstrapAccount, Is.Not.Empty, "Unable to fetch account for bootstrap geth node. Test infra failure."); Assert.That(bootstrapGenesisJson, Is.Not.Empty, "Unable to fetch genesis-json for bootstrap geth node. Test infra failure."); @@ -40,6 +44,12 @@ namespace CodexDistTestCore.Marketplace gethBootstrapNode!.GenesisJsonBase64 = Convert.ToBase64String(Encoding.ASCII.GetBytes(bootstrapGenesisJson)); } + private void FetchAccountAndGenesisJson() + { + bootstrapAccount = ExecuteCommand("cat", "account_string.txt"); + bootstrapGenesisJson = ExecuteCommand("cat", "genesis.json"); + } + private string ExecuteCommand(string command, params string[] arguments) { return k8sManager.ExecuteCommand(gethBootstrapNode!.Pod, K8sGethBoostrapSpecs.ContainerName, command, arguments); diff --git a/Tests/BasicTests/SimpleTests.cs b/Tests/BasicTests/SimpleTests.cs index 0812b78..7763876 100644 --- a/Tests/BasicTests/SimpleTests.cs +++ b/Tests/BasicTests/SimpleTests.cs @@ -34,7 +34,7 @@ namespace Tests.BasicTests [Test] public void MarketplaceExample() { - var primary = SetupCodexNodes(1) + var primary = SetupCodexNodes(4) .WithStorageQuota(10.GB()) .EnableMarketplace(initialBalance: 20) .BringOnline()[0]; From 388aacf71c54b115524b87e78358b39695b497b4 Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 10 Apr 2023 15:54:13 +0200 Subject: [PATCH 08/46] Fetch account string for each geth companion node --- CodexDistTestCore/K8sManager.cs | 33 +++++++++++++++---- CodexDistTestCore/Marketplace/K8sGethSpecs.cs | 2 ++ .../Marketplace/MarketplaceAccess.cs | 27 +++++++++++++++ .../Marketplace/MarketplaceController.cs | 10 +++--- 4 files changed, 61 insertions(+), 11 deletions(-) diff --git a/CodexDistTestCore/K8sManager.cs b/CodexDistTestCore/K8sManager.cs index 5e8716b..c9e27ff 100644 --- a/CodexDistTestCore/K8sManager.cs +++ b/CodexDistTestCore/K8sManager.cs @@ -30,23 +30,27 @@ namespace CodexDistTestCore public ICodexNodeGroup BringOnline(OfflineCodexNodes offline) { - var online = CreateOnlineCodexNodes(offline); + var group = CreateOnlineCodexNodes(offline); if (offline.MarketplaceConfig != null) { - online.GethInfo = BringOnlineMarketplace(); + group.GethInfo = BringOnlineMarketplace(); } - K8s(k => k.BringOnline(online, offline)); - - log.Log($"{online.Describe()} online."); + K8s(k => k.BringOnline(group, offline)); if (offline.MetricsEnabled) { - BringOnlineMetrics(online); + BringOnlineMetrics(group); + } + if (offline.MarketplaceConfig != null) + { + ConnectMarketplace(group); } - return online; + log.Log($"{group.Describe()} online."); + + return group; } public IOfflineCodexNodes BringOffline(ICodexNodeGroup node) @@ -109,6 +113,21 @@ namespace CodexDistTestCore return marketplaceController.BringOnlineMarketplace(); } + private void ConnectMarketplace(CodexNodeGroup group) + { + foreach (var node in DowncastNodes(group)) + { + ConnectMarketplace(group, node); + } + } + + private void ConnectMarketplace(CodexNodeGroup group, OnlineCodexNode node) + { + var access = new MarketplaceAccess(this, log); + access.Initialize(group.PodInfo!, node.Container.GethCompanionNodeContainer!); + node.Marketplace = access; + } + private CodexNodeGroup CreateOnlineCodexNodes(OfflineCodexNodes offline) { var containers = CreateContainers(offline); diff --git a/CodexDistTestCore/Marketplace/K8sGethSpecs.cs b/CodexDistTestCore/Marketplace/K8sGethSpecs.cs index dd0204d..751fb5e 100644 --- a/CodexDistTestCore/Marketplace/K8sGethSpecs.cs +++ b/CodexDistTestCore/Marketplace/K8sGethSpecs.cs @@ -6,6 +6,8 @@ namespace CodexDistTestCore.Marketplace public static class GethDockerImage { public const string Image = "thatbenbierens/geth-confenv:latest"; + public const string AccountFilename = "account_string.txt"; + public const string GenesisFilename = "genesis.json"; } public class K8sGethBoostrapSpecs diff --git a/CodexDistTestCore/Marketplace/MarketplaceAccess.cs b/CodexDistTestCore/Marketplace/MarketplaceAccess.cs index 25bb63f..a49eab4 100644 --- a/CodexDistTestCore/Marketplace/MarketplaceAccess.cs +++ b/CodexDistTestCore/Marketplace/MarketplaceAccess.cs @@ -13,6 +13,33 @@ namespace CodexDistTestCore.Marketplace public class MarketplaceAccess : IMarketplaceAccess { + private readonly K8sManager k8sManager; + private readonly TestLog log; + private string account = string.Empty; + + public MarketplaceAccess(K8sManager k8sManager, TestLog log) + { + this.k8sManager = k8sManager; + this.log = log; + } + + public void Initialize(PodInfo pod, GethCompanionNodeContainer gethCompanionNodeContainer) + { + FetchAccount(pod, gethCompanionNodeContainer); + if (string.IsNullOrEmpty(account)) + { + Thread.Sleep(TimeSpan.FromSeconds(15)); + FetchAccount(pod, gethCompanionNodeContainer); + } + Assert.That(account, Is.Not.Empty, "Unable to fetch account for geth companion node. Test infra failure."); + log.Log($"Initialized Geth companion node with account '{account}'"); + } + + private void FetchAccount(PodInfo pod, GethCompanionNodeContainer gethCompanionNodeContainer) + { + account = k8sManager.ExecuteCommand(pod, gethCompanionNodeContainer.Name, "cat", GethDockerImage.AccountFilename); + } + public void AdvertiseContract(ContentId contentId, float maxPricePerMBPerSecond, float minRequiredCollateral, float minRequiredNumberOfDuplicates) { throw new NotImplementedException(); diff --git a/CodexDistTestCore/Marketplace/MarketplaceController.cs b/CodexDistTestCore/Marketplace/MarketplaceController.cs index e832335..d4dbbce 100644 --- a/CodexDistTestCore/Marketplace/MarketplaceController.cs +++ b/CodexDistTestCore/Marketplace/MarketplaceController.cs @@ -38,16 +38,18 @@ namespace CodexDistTestCore.Marketplace FetchAccountAndGenesisJson(); } - Assert.That(bootstrapAccount, Is.Not.Empty, "Unable to fetch account for bootstrap geth node. Test infra failure."); - Assert.That(bootstrapGenesisJson, Is.Not.Empty, "Unable to fetch genesis-json for bootstrap geth node. Test infra failure."); + Assert.That(bootstrapAccount, Is.Not.Empty, "Unable to fetch account for geth bootstrap node. Test infra failure."); + Assert.That(bootstrapGenesisJson, Is.Not.Empty, "Unable to fetch genesis-json for geth bootstrap node. Test infra failure."); gethBootstrapNode!.GenesisJsonBase64 = Convert.ToBase64String(Encoding.ASCII.GetBytes(bootstrapGenesisJson)); + + log.Log($"Initialized geth bootstrap node with account '{bootstrapAccount}'"); } private void FetchAccountAndGenesisJson() { - bootstrapAccount = ExecuteCommand("cat", "account_string.txt"); - bootstrapGenesisJson = ExecuteCommand("cat", "genesis.json"); + bootstrapAccount = ExecuteCommand("cat", GethDockerImage.AccountFilename); + bootstrapGenesisJson = ExecuteCommand("cat", GethDockerImage.GenesisFilename); } private string ExecuteCommand(string command, params string[] arguments) From 8a6ababba61c9ba390413e96cf192bda2e165444 Mon Sep 17 00:00:00 2001 From: benbierens Date: Tue, 11 Apr 2023 10:12:24 +0200 Subject: [PATCH 09/46] begin splitting geth companion node to its own pod. --- CodexDistTestCore/Config/CodexDockerImage.cs | 15 +++++- CodexDistTestCore/K8sManager.cs | 4 +- .../Marketplace/MarketplaceAccess.cs | 47 +++++++++++++------ .../Marketplace/MarketplaceController.cs | 8 ++++ 4 files changed, 56 insertions(+), 18 deletions(-) diff --git a/CodexDistTestCore/Config/CodexDockerImage.cs b/CodexDistTestCore/Config/CodexDockerImage.cs index 333ec0d..5855f71 100644 --- a/CodexDistTestCore/Config/CodexDockerImage.cs +++ b/CodexDistTestCore/Config/CodexDockerImage.cs @@ -14,10 +14,10 @@ namespace CodexDistTestCore.Config return "b20483"; } - public List CreateEnvironmentVariables(OfflineCodexNodes node, CodexNodeContainer environment) + public List CreateEnvironmentVariables(OfflineCodexNodes node, CodexNodeContainer container) { var formatter = new EnvFormatter(); - formatter.Create(node, environment); + formatter.Create(node, container); return formatter.Result; } @@ -50,6 +50,17 @@ namespace CodexDistTestCore.Config AddVar("METRICS_ADDR", "0.0.0.0"); AddVar("METRICS_PORT", container.MetricsPort.ToString()); } + if (container.GethCompanionNodeContainer != null) + { + // well, darn: To get the account here, the geth companion needs to have been started and account string fetched. + // but the codex node and geth companion node are in the same pod. so they start at the same time. + // so we cannot start the codex node with the correct account string at the same time as we start the geth node that + // is supposed to generate that string. + // begin rework: Separate pod for geth companion node. +//ETH_PROVIDER +//ETH_ACCOUNT +//ETH_DEPLOYMENT + } } private void AddVar(string key, string value) diff --git a/CodexDistTestCore/K8sManager.cs b/CodexDistTestCore/K8sManager.cs index c9e27ff..3e1eeac 100644 --- a/CodexDistTestCore/K8sManager.cs +++ b/CodexDistTestCore/K8sManager.cs @@ -123,8 +123,8 @@ namespace CodexDistTestCore private void ConnectMarketplace(CodexNodeGroup group, OnlineCodexNode node) { - var access = new MarketplaceAccess(this, log); - access.Initialize(group.PodInfo!, node.Container.GethCompanionNodeContainer!); + var access = new MarketplaceAccess(this, marketplaceController, log, group, node.Container.GethCompanionNodeContainer!); + access.Initialize(); node.Marketplace = access; } diff --git a/CodexDistTestCore/Marketplace/MarketplaceAccess.cs b/CodexDistTestCore/Marketplace/MarketplaceAccess.cs index a49eab4..9dcd1c9 100644 --- a/CodexDistTestCore/Marketplace/MarketplaceAccess.cs +++ b/CodexDistTestCore/Marketplace/MarketplaceAccess.cs @@ -14,30 +14,33 @@ namespace CodexDistTestCore.Marketplace public class MarketplaceAccess : IMarketplaceAccess { private readonly K8sManager k8sManager; + private readonly MarketplaceController marketplaceController; private readonly TestLog log; + private readonly CodexNodeGroup group; + private readonly GethCompanionNodeContainer gethCompanionNodeContainer; private string account = string.Empty; - public MarketplaceAccess(K8sManager k8sManager, TestLog log) + public MarketplaceAccess( + K8sManager k8sManager, + MarketplaceController marketplaceController, + TestLog log, + CodexNodeGroup group, + GethCompanionNodeContainer gethCompanionNodeContainer) { this.k8sManager = k8sManager; + this.marketplaceController = marketplaceController; this.log = log; + this.group = group; + this.gethCompanionNodeContainer = gethCompanionNodeContainer; } - public void Initialize(PodInfo pod, GethCompanionNodeContainer gethCompanionNodeContainer) + public void Initialize() { - FetchAccount(pod, gethCompanionNodeContainer); - if (string.IsNullOrEmpty(account)) - { - Thread.Sleep(TimeSpan.FromSeconds(15)); - FetchAccount(pod, gethCompanionNodeContainer); - } - Assert.That(account, Is.Not.Empty, "Unable to fetch account for geth companion node. Test infra failure."); - log.Log($"Initialized Geth companion node with account '{account}'"); - } + EnsureAccount(); - private void FetchAccount(PodInfo pod, GethCompanionNodeContainer gethCompanionNodeContainer) - { - account = k8sManager.ExecuteCommand(pod, gethCompanionNodeContainer.Name, "cat", GethDockerImage.AccountFilename); + marketplaceController.AddToBalance(account, group.Origin.MarketplaceConfig!.InitialBalance); + + log.Log($"Initialized Geth companion node with account '{account}' and initial balance {group.Origin.MarketplaceConfig!.InitialBalance}"); } public void AdvertiseContract(ContentId contentId, float maxPricePerMBPerSecond, float minRequiredCollateral, float minRequiredNumberOfDuplicates) @@ -59,6 +62,22 @@ namespace CodexDistTestCore.Marketplace { throw new NotImplementedException(); } + + private void EnsureAccount() + { + FetchAccount(); + if (string.IsNullOrEmpty(account)) + { + Thread.Sleep(TimeSpan.FromSeconds(15)); + FetchAccount(); + } + Assert.That(account, Is.Not.Empty, "Unable to fetch account for geth companion node. Test infra failure."); + } + + private void FetchAccount() + { + account = k8sManager.ExecuteCommand(group.PodInfo!, gethCompanionNodeContainer.Name, "cat", GethDockerImage.AccountFilename); + } } public class MarketplaceUnavailable : IMarketplaceAccess diff --git a/CodexDistTestCore/Marketplace/MarketplaceController.cs b/CodexDistTestCore/Marketplace/MarketplaceController.cs index d4dbbce..9b2c960 100644 --- a/CodexDistTestCore/Marketplace/MarketplaceController.cs +++ b/CodexDistTestCore/Marketplace/MarketplaceController.cs @@ -29,6 +29,14 @@ namespace CodexDistTestCore.Marketplace return gethBootstrapNode; } + public void AddToBalance(string account, int amount) + { + if (amount < 1 || string.IsNullOrEmpty(account)) Assert.Fail("Invalid arguments for AddToBalance"); + + // call the bootstrap node and convince it to give 'account' 'amount' tokens somehow. + throw new NotImplementedException(); + } + private void ExtractAccountAndGenesisJson() { FetchAccountAndGenesisJson(); From c977e37ab8533100d9d393cbbb95f7d6c712b348 Mon Sep 17 00:00:00 2001 From: benbierens Date: Tue, 11 Apr 2023 11:00:39 +0200 Subject: [PATCH 10/46] wip --- CodexDistTestCore/CodexNodeContainer.cs | 35 +++---------- CodexDistTestCore/K8sManager.cs | 7 +-- .../Marketplace/GethCompanionNodeContainer.cs | 50 +------------------ CodexDistTestCore/Marketplace/K8sGethSpecs.cs | 32 ++++++++++++ .../Marketplace/MarketplaceAccess.cs | 8 +-- .../Marketplace/MarketplaceController.cs | 30 ++++++++--- Tests/BasicTests/SimpleTests.cs | 7 +-- 7 files changed, 75 insertions(+), 94 deletions(-) diff --git a/CodexDistTestCore/CodexNodeContainer.cs b/CodexDistTestCore/CodexNodeContainer.cs index 915a2b4..ccf9b87 100644 --- a/CodexDistTestCore/CodexNodeContainer.cs +++ b/CodexDistTestCore/CodexNodeContainer.cs @@ -1,10 +1,8 @@ -using CodexDistTestCore.Marketplace; - -namespace CodexDistTestCore +namespace CodexDistTestCore { public class CodexNodeContainer { - public CodexNodeContainer(string name, int servicePort, string servicePortName, int apiPort, string containerPortName, int discoveryPort, int listenPort, string dataDir, int metricsPort, GethCompanionNodeContainer? gethCompanionNodeContainer) + public CodexNodeContainer(string name, int servicePort, string servicePortName, int apiPort, string containerPortName, int discoveryPort, int listenPort, string dataDir, int metricsPort) { Name = name; ServicePort = servicePort; @@ -15,7 +13,6 @@ namespace CodexDistTestCore ListenPort = listenPort; DataDir = dataDir; MetricsPort = metricsPort; - GethCompanionNodeContainer = gethCompanionNodeContainer; } public string Name { get; } @@ -27,8 +24,6 @@ namespace CodexDistTestCore public int ListenPort { get; } public string DataDir { get; } public int MetricsPort { get; } - - public GethCompanionNodeContainer? GethCompanionNodeContainer { get; } } public class CodexGroupNumberSource @@ -57,11 +52,11 @@ namespace CodexDistTestCore { private readonly NumberSource containerNameSource = new NumberSource(1); private readonly NumberSource codexPortSource = new NumberSource(8080); - private readonly CodexGroupNumberSource groupContainerFactory; + private readonly CodexGroupNumberSource numberSource; - public CodexNodeContainerFactory(CodexGroupNumberSource groupContainerFactory) + public CodexNodeContainerFactory(CodexGroupNumberSource numberSource) { - this.groupContainerFactory = groupContainerFactory; + this.numberSource = numberSource; } public CodexNodeContainer CreateNext(OfflineCodexNodes offline) @@ -69,15 +64,14 @@ namespace CodexDistTestCore var n = containerNameSource.GetNextNumber(); return new CodexNodeContainer( name: $"codex-node{n}", - servicePort: groupContainerFactory.GetNextServicePort(), - servicePortName: groupContainerFactory.GetNextServicePortName(), + servicePort: numberSource.GetNextServicePort(), + servicePortName: numberSource.GetNextServicePortName(), apiPort: codexPortSource.GetNextNumber(), containerPortName: $"api-{n}", discoveryPort: codexPortSource.GetNextNumber(), listenPort: codexPortSource.GetNextNumber(), dataDir: $"datadir{n}", - metricsPort: GetMetricsPort(offline), - CreateGethNodeContainer(offline, n) + metricsPort: GetMetricsPort(offline) ); } @@ -87,18 +81,5 @@ namespace CodexDistTestCore return 0; } - private GethCompanionNodeContainer? CreateGethNodeContainer(OfflineCodexNodes offline, int n) - { - if (offline.MarketplaceConfig == null) return null; - - return new GethCompanionNodeContainer( - name: $"geth-node{n}", - servicePort: groupContainerFactory.GetNextServicePort(), - servicePortName: groupContainerFactory.GetNextServicePortName(), - apiPort: codexPortSource.GetNextNumber(), - rpcPort: codexPortSource.GetNextNumber(), - containerPortName: $"geth-{n}" - ); - } } } diff --git a/CodexDistTestCore/K8sManager.cs b/CodexDistTestCore/K8sManager.cs index 3e1eeac..5346c40 100644 --- a/CodexDistTestCore/K8sManager.cs +++ b/CodexDistTestCore/K8sManager.cs @@ -34,7 +34,7 @@ namespace CodexDistTestCore if (offline.MarketplaceConfig != null) { - group.GethInfo = BringOnlineMarketplace(); + group.GethInfo = marketplaceController.BringOnlineMarketplace(offline); } K8s(k => k.BringOnline(group, offline)); @@ -108,11 +108,6 @@ namespace CodexDistTestCore metricsAggregator.BeginCollectingMetricsFor(DowncastNodes(group)); } - private GethInfo BringOnlineMarketplace() - { - return marketplaceController.BringOnlineMarketplace(); - } - private void ConnectMarketplace(CodexNodeGroup group) { foreach (var node in DowncastNodes(group)) diff --git a/CodexDistTestCore/Marketplace/GethCompanionNodeContainer.cs b/CodexDistTestCore/Marketplace/GethCompanionNodeContainer.cs index a4b452b..d1ded2e 100644 --- a/CodexDistTestCore/Marketplace/GethCompanionNodeContainer.cs +++ b/CodexDistTestCore/Marketplace/GethCompanionNodeContainer.cs @@ -4,64 +4,18 @@ namespace CodexDistTestCore.Marketplace { public class GethCompanionNodeContainer { - public GethCompanionNodeContainer(string name, int servicePort, string servicePortName, int apiPort, int rpcPort, string containerPortName) + public GethCompanionNodeContainer(string name, int apiPort, int rpcPort, string containerPortName) { Name = name; - ServicePort = servicePort; - ServicePortName = servicePortName; ApiPort = apiPort; RpcPort = rpcPort; ContainerPortName = containerPortName; } public string Name { get; } - public int ServicePort { get; } - public string ServicePortName { get; } public int ApiPort { get; } public int RpcPort { get; } public string ContainerPortName { get; } - - public V1Container CreateDeploymentContainer(GethInfo gethInfo) - { - return new V1Container - { - Name = Name, - Image = GethDockerImage.Image, - Ports = new List - { - new V1ContainerPort - { - ContainerPort = ApiPort, - Name = ContainerPortName - } - }, - // todo: use env vars to connect this node to the bootstrap node provided by gethInfo.podInfo & gethInfo.servicePort & gethInfo.genesisJsonBase64 - Env = new List - { - new V1EnvVar - { - Name = "GETH_ARGS", - Value = $"--port {ApiPort} --discovery.port {ApiPort} --authrpc.port {RpcPort}" - }, - new V1EnvVar - { - Name = "GENESIS_JSON", - Value = gethInfo.GenesisJsonBase64 - } - } - }; - } - - public V1ServicePort CreateServicePort() - { - return new V1ServicePort - { - Name = ServicePortName, - Protocol = "TCP", - Port = ApiPort, - TargetPort = ContainerPortName, - NodePort = ServicePort - }; - } + } } diff --git a/CodexDistTestCore/Marketplace/K8sGethSpecs.cs b/CodexDistTestCore/Marketplace/K8sGethSpecs.cs index 751fb5e..5247ce6 100644 --- a/CodexDistTestCore/Marketplace/K8sGethSpecs.cs +++ b/CodexDistTestCore/Marketplace/K8sGethSpecs.cs @@ -1,5 +1,6 @@ using CodexDistTestCore.Config; using k8s.Models; +using System.Xml.Linq; namespace CodexDistTestCore.Marketplace { @@ -126,6 +127,37 @@ namespace CodexDistTestCore.Marketplace return serviceSpec; } + public V1Deployment CreateGethCompanionDeployment(GethInfo gethInfo) + { + return new V1Container + { + Name = Name, + Image = GethDockerImage.Image, + Ports = new List + { + new V1ContainerPort + { + ContainerPort = ApiPort, + Name = ContainerPortName + } + }, + // todo: use env vars to connect this node to the bootstrap node provided by gethInfo.podInfo & gethInfo.servicePort & gethInfo.genesisJsonBase64 + Env = new List + { + new V1EnvVar + { + Name = "GETH_ARGS", + Value = $"--port {ApiPort} --discovery.port {ApiPort} --authrpc.port {RpcPort}" + }, + new V1EnvVar + { + Name = "GENESIS_JSON", + Value = gethInfo.GenesisJsonBase64 + } + } + }; + } + private Dictionary CreateSelector() { return new Dictionary { { "test-gethb", "dtest-gethb" } }; diff --git a/CodexDistTestCore/Marketplace/MarketplaceAccess.cs b/CodexDistTestCore/Marketplace/MarketplaceAccess.cs index 9dcd1c9..248e4ac 100644 --- a/CodexDistTestCore/Marketplace/MarketplaceAccess.cs +++ b/CodexDistTestCore/Marketplace/MarketplaceAccess.cs @@ -5,8 +5,8 @@ namespace CodexDistTestCore.Marketplace { public interface IMarketplaceAccess { - void AdvertiseStorage(ByteSize size, float pricePerMBPerSecond, float collateral); - void AdvertiseContract(ContentId contentId, float maxPricePerMBPerSecond, float minRequiredCollateral, float minRequiredNumberOfDuplicates); + void MakeStorageAvailable(ByteSize size, int minPricePerBytePerSecond, float maxCollateral); + void RequestStorage(ContentId contentId, int pricePerBytePerSecond, float requiredCollateral, float minRequiredNumberOfNodes); void AssertThatBalance(IResolveConstraint constraint, string message = ""); float GetBalance(); } @@ -48,7 +48,7 @@ namespace CodexDistTestCore.Marketplace throw new NotImplementedException(); } - public void AdvertiseStorage(ByteSize size, float pricePerMBPerSecond, float collateral) + public void MakeStorageAvailable(ByteSize size, float pricePerMBPerSecond, float collateral) { throw new NotImplementedException(); } @@ -87,7 +87,7 @@ namespace CodexDistTestCore.Marketplace Unavailable(); } - public void AdvertiseStorage(ByteSize size, float pricePerMBPerSecond, float collateral) + public void MakeStorageAvailable(ByteSize size, float pricePerMBPerSecond, float collateral) { Unavailable(); } diff --git a/CodexDistTestCore/Marketplace/MarketplaceController.cs b/CodexDistTestCore/Marketplace/MarketplaceController.cs index 9b2c960..deaf747 100644 --- a/CodexDistTestCore/Marketplace/MarketplaceController.cs +++ b/CodexDistTestCore/Marketplace/MarketplaceController.cs @@ -17,18 +17,34 @@ namespace CodexDistTestCore.Marketplace this.k8sManager = k8sManager; } - public GethInfo BringOnlineMarketplace() + public GethInfo BringOnlineMarketplace(OfflineCodexNodes offline) { if (gethBootstrapNode != null) return gethBootstrapNode; log.Log("Starting Geth bootstrap node..."); gethBootstrapNode = k8sManager.BringOnlineGethBootstrapNode(); ExtractAccountAndGenesisJson(); - log.Log("Geth boothstrap node started."); + log.Log($"Geth boothstrap node started. Initializing companions for {offline.NumberOfNodes} Codex nodes."); + + + return gethBootstrapNode; } + + private GethCompanionNodeContainer? CreateGethNodeContainer(OfflineCodexNodes offline, int n) + { + return new GethCompanionNodeContainer( + name: $"geth-node{n}", + servicePort: numberSource.GetNextServicePort(), + servicePortName: numberSource.GetNextServicePortName(), + apiPort: codexPortSource.GetNextNumber(), + rpcPort: codexPortSource.GetNextNumber(), + containerPortName: $"geth-{n}" + ); + } + public void AddToBalance(string account, int amount) { if (amount < 1 || string.IsNullOrEmpty(account)) Assert.Fail("Invalid arguments for AddToBalance"); @@ -62,20 +78,22 @@ namespace CodexDistTestCore.Marketplace private string ExecuteCommand(string command, params string[] arguments) { - return k8sManager.ExecuteCommand(gethBootstrapNode!.Pod, K8sGethBoostrapSpecs.ContainerName, command, arguments); + return k8sManager.ExecuteCommand(gethBootstrapNode!.BootstrapPod, K8sGethBoostrapSpecs.ContainerName, command, arguments); } } public class GethInfo { - public GethInfo(K8sGethBoostrapSpecs spec, PodInfo pod) + public GethInfo(K8sGethBoostrapSpecs spec, PodInfo bootstrapPod, PodInfo companionPod) { Spec = spec; - Pod = pod; + BootstrapPod = bootstrapPod; + CompanionPod = companionPod; } public K8sGethBoostrapSpecs Spec { get; } - public PodInfo Pod { get; } + public PodInfo BootstrapPod { get; } + public PodInfo CompanionPod { get; } public string GenesisJsonBase64 { get; set; } = string.Empty; } } diff --git a/Tests/BasicTests/SimpleTests.cs b/Tests/BasicTests/SimpleTests.cs index 7763876..233b730 100644 --- a/Tests/BasicTests/SimpleTests.cs +++ b/Tests/BasicTests/SimpleTests.cs @@ -34,7 +34,7 @@ namespace Tests.BasicTests [Test] public void MarketplaceExample() { - var primary = SetupCodexNodes(4) + var primary = SetupCodexNodes(1) .WithStorageQuota(10.GB()) .EnableMarketplace(initialBalance: 20) .BringOnline()[0]; @@ -44,11 +44,12 @@ namespace Tests.BasicTests // .BringOnline()[0]; //primary.ConnectToPeer(secondary); - //primary.Marketplace.AdvertiseStorage(10.GB(), pricePerMBPerSecond: 0.01f, collateral: 20); + //primary.Marketplace.MakeStorageAvailable(10.GB(), minPricePerBytePerSecond: 1, maxCollateral: 20); //var testFile = GenerateTestFile(10.MB()); //var contentId = secondary.UploadFile(testFile); - //secondary.Marketplace.AdvertiseContract(contentId, maxPricePerMBPerSecond: 0.02f, minRequiredCollateral: 10, minRequiredNumberOfDuplicates: 1); + //secondary.Marketplace.RequestStorage(contentId, pricePerBytePerSecond: 2, + // requiredCollateral: 10, minRequiredNumberOfNodes: 1); //primary.Marketplace.AssertThatBalance(Is.LessThan(20), "Collateral was not placed."); //var primaryBalance = primary.Marketplace.GetBalance(); From d33eb5300347285bd85b9951813956be2876841c Mon Sep 17 00:00:00 2001 From: benbierens Date: Tue, 11 Apr 2023 12:06:33 +0200 Subject: [PATCH 11/46] splits geth nodes into their own pod --- CodexDistTestCore/CodexNodeGroup.cs | 2 +- CodexDistTestCore/K8sManager.cs | 22 +++-- CodexDistTestCore/K8sOperations.cs | 37 +++++--- .../Marketplace/GethCompanionNodeContainer.cs | 20 +++- CodexDistTestCore/Marketplace/K8sGethSpecs.cs | 69 +++++++++++--- .../Marketplace/MarketplaceAccess.cs | 27 +++--- .../Marketplace/MarketplaceController.cs | 92 +++++++++++-------- 7 files changed, 180 insertions(+), 89 deletions(-) diff --git a/CodexDistTestCore/CodexNodeGroup.cs b/CodexDistTestCore/CodexNodeGroup.cs index a3a82d3..1ad5781 100644 --- a/CodexDistTestCore/CodexNodeGroup.cs +++ b/CodexDistTestCore/CodexNodeGroup.cs @@ -46,7 +46,7 @@ namespace CodexDistTestCore public V1Deployment? Deployment { get; set; } public V1Service? Service { get; set; } public PodInfo? PodInfo { get; set; } - public GethInfo? GethInfo { get; set; } + public GethCompanionGroup? GethCompanionGroup { get; set; } public CodexNodeContainer[] GetContainers() { diff --git a/CodexDistTestCore/K8sManager.cs b/CodexDistTestCore/K8sManager.cs index 5346c40..9686bcc 100644 --- a/CodexDistTestCore/K8sManager.cs +++ b/CodexDistTestCore/K8sManager.cs @@ -34,7 +34,7 @@ namespace CodexDistTestCore if (offline.MarketplaceConfig != null) { - group.GethInfo = marketplaceController.BringOnlineMarketplace(offline); + group.GethCompanionGroup = marketplaceController.BringOnlineMarketplace(offline); } K8s(k => k.BringOnline(group, offline)); @@ -91,13 +91,21 @@ namespace CodexDistTestCore return K8s(k => k.BringOnlinePrometheus(spec)); } - public GethInfo BringOnlineGethBootstrapNode() + public K8sGethBoostrapSpecs CreateGethBootstrapNodeSpec() { - var spec = new K8sGethBoostrapSpecs(codexGroupNumberSource.GetNextServicePort()); + return new K8sGethBoostrapSpecs(codexGroupNumberSource.GetNextServicePort()); + } + public PodInfo BringOnlineGethBootstrapNode(K8sGethBoostrapSpecs spec) + { return K8s(k => k.BringOnlineGethBootstrapNode(spec)); } + public PodInfo BringOnlineGethCompanionGroup(GethBootstrapInfo info, GethCompanionGroup group) + { + return K8s(k => k.BringOnlineGethCompanionGroup(info, group)); + } + public void DownloadAllMetrics() { metricsAggregator.DownloadAllMetrics(); @@ -110,15 +118,15 @@ namespace CodexDistTestCore private void ConnectMarketplace(CodexNodeGroup group) { - foreach (var node in DowncastNodes(group)) + for (var i = 0; i < group.Nodes.Length; i++) { - ConnectMarketplace(group, node); + ConnectMarketplace(group, group.Nodes[i], group.GethCompanionGroup!.Containers[i]); } } - private void ConnectMarketplace(CodexNodeGroup group, OnlineCodexNode node) + private void ConnectMarketplace(CodexNodeGroup group, OnlineCodexNode node, GethCompanionNodeContainer container) { - var access = new MarketplaceAccess(this, marketplaceController, log, group, node.Container.GethCompanionNodeContainer!); + var access = new MarketplaceAccess(this, marketplaceController, log, group, container); access.Initialize(); node.Marketplace = access; } diff --git a/CodexDistTestCore/K8sOperations.cs b/CodexDistTestCore/K8sOperations.cs index 22e7998..c0c7234 100644 --- a/CodexDistTestCore/K8sOperations.cs +++ b/CodexDistTestCore/K8sOperations.cs @@ -4,7 +4,6 @@ using CodexDistTestCore.Metrics; using k8s; using k8s.Models; using NUnit.Framework; -using System.Numerics; namespace CodexDistTestCore { @@ -78,7 +77,7 @@ namespace CodexDistTestCore return new PrometheusInfo(spec.ServicePort, FetchNewPod()); } - public GethInfo BringOnlineGethBootstrapNode(K8sGethBoostrapSpecs spec) + public PodInfo BringOnlineGethBootstrapNode(K8sGethBoostrapSpecs spec) { EnsureTestNamespace(); @@ -86,7 +85,17 @@ namespace CodexDistTestCore CreateGethBootstrapService(spec); WaitUntilGethBootstrapOnline(spec); - return new GethInfo(spec, FetchNewPod()); + return FetchNewPod(); + } + + public PodInfo BringOnlineGethCompanionGroup(GethBootstrapInfo info, GethCompanionGroup group) + { + EnsureTestNamespace(); + + CreateGethCompanionDeployment(info, group); + WaitUntilGethCompanionGroupOnline(info.Spec, group); + + return FetchNewPod(); } private void FetchPodInfo(CodexNodeGroup online) @@ -148,7 +157,12 @@ namespace CodexDistTestCore private void WaitUntilGethBootstrapOnline(K8sGethBoostrapSpecs spec) { - WaitUntilDeploymentOnline(spec.GetDeploymentName()); + WaitUntilDeploymentOnline(spec.GetBootstrapDeploymentName()); + } + + private void WaitUntilGethCompanionGroupOnline(K8sGethBoostrapSpecs spec, GethCompanionGroup group) + { + WaitUntilDeploymentOnline(spec.GetCompanionDeploymentName(group)); } private void WaitUntilDeploymentOnline(string deploymentName) @@ -212,11 +226,6 @@ namespace CodexDistTestCore TargetPort = container.ContainerPortName, NodePort = container.ServicePort }); - - if (container.GethCompanionNodeContainer != null) - { - result.Add(container.GethCompanionNodeContainer.CreateServicePort()); - } } return result; } @@ -303,11 +312,6 @@ namespace CodexDistTestCore }, Env = dockerImage.CreateEnvironmentVariables(offline, container) }); - - if (container.GethCompanionNodeContainer != null) - { - result.Add(container.GethCompanionNodeContainer.CreateDeploymentContainer(online.GethInfo!)); - } } return result; @@ -330,6 +334,11 @@ namespace CodexDistTestCore client.CreateNamespacedDeployment(spec.CreateGethBootstrapDeployment(), K8sNamespace); } + private void CreateGethCompanionDeployment(GethBootstrapInfo info, GethCompanionGroup group) + { + client.CreateNamespacedDeployment(info.Spec.CreateGethCompanionDeployment(group, info), K8sNamespace); + } + #endregion #region Namespace management diff --git a/CodexDistTestCore/Marketplace/GethCompanionNodeContainer.cs b/CodexDistTestCore/Marketplace/GethCompanionNodeContainer.cs index d1ded2e..de5e59e 100644 --- a/CodexDistTestCore/Marketplace/GethCompanionNodeContainer.cs +++ b/CodexDistTestCore/Marketplace/GethCompanionNodeContainer.cs @@ -1,7 +1,18 @@ -using k8s.Models; - -namespace CodexDistTestCore.Marketplace +namespace CodexDistTestCore.Marketplace { + public class GethCompanionGroup + { + public GethCompanionGroup(int number, GethCompanionNodeContainer[] containers) + { + Number = number; + Containers = containers; + } + + public int Number { get; } + public GethCompanionNodeContainer[] Containers { get; } + public PodInfo? Pod { get; set; } + } + public class GethCompanionNodeContainer { public GethCompanionNodeContainer(string name, int apiPort, int rpcPort, string containerPortName) @@ -16,6 +27,7 @@ namespace CodexDistTestCore.Marketplace public int ApiPort { get; } public int RpcPort { get; } public string ContainerPortName { get; } - + + public string Account { get; set; } = string.Empty; } } diff --git a/CodexDistTestCore/Marketplace/K8sGethSpecs.cs b/CodexDistTestCore/Marketplace/K8sGethSpecs.cs index 5247ce6..6318c4d 100644 --- a/CodexDistTestCore/Marketplace/K8sGethSpecs.cs +++ b/CodexDistTestCore/Marketplace/K8sGethSpecs.cs @@ -1,6 +1,5 @@ using CodexDistTestCore.Config; using k8s.Models; -using System.Xml.Linq; namespace CodexDistTestCore.Marketplace { @@ -24,11 +23,16 @@ namespace CodexDistTestCore.Marketplace public int ServicePort { get; } - public string GetDeploymentName() + public string GetBootstrapDeploymentName() { return "test-gethb"; } + public string GetCompanionDeploymentName(GethCompanionGroup group) + { + return "test-geth" + group.Number; + } + public V1Deployment CreateGethBootstrapDeployment() { var deploymentSpec = new V1Deployment @@ -36,7 +40,7 @@ namespace CodexDistTestCore.Marketplace ApiVersion = "apps/v1", Metadata = new V1ObjectMeta { - Name = GetDeploymentName(), + Name = GetBootstrapDeploymentName(), NamespaceProperty = K8sCluster.K8sNamespace }, Spec = new V1DeploymentSpec @@ -44,13 +48,13 @@ namespace CodexDistTestCore.Marketplace Replicas = 1, Selector = new V1LabelSelector { - MatchLabels = CreateSelector() + MatchLabels = CreateBootstrapSelector() }, Template = new V1PodTemplateSpec { Metadata = new V1ObjectMeta { - Labels = CreateSelector() + Labels = CreateBootstrapSelector() }, Spec = new V1PodSpec { @@ -109,7 +113,7 @@ namespace CodexDistTestCore.Marketplace Spec = new V1ServiceSpec { Type = "NodePort", - Selector = CreateSelector(), + Selector = CreateBootstrapSelector(), Ports = new List { new V1ServicePort @@ -127,18 +131,52 @@ namespace CodexDistTestCore.Marketplace return serviceSpec; } - public V1Deployment CreateGethCompanionDeployment(GethInfo gethInfo) + public V1Deployment CreateGethCompanionDeployment(GethCompanionGroup group, GethBootstrapInfo info) + { + var deploymentSpec = new V1Deployment + { + ApiVersion = "apps/v1", + Metadata = new V1ObjectMeta + { + Name = GetCompanionDeploymentName(group), + NamespaceProperty = K8sCluster.K8sNamespace + }, + Spec = new V1DeploymentSpec + { + Replicas = 1, + Selector = new V1LabelSelector + { + MatchLabels = CreateCompanionSelector() + }, + Template = new V1PodTemplateSpec + { + Metadata = new V1ObjectMeta + { + Labels = CreateCompanionSelector() + }, + Spec = new V1PodSpec + { + Containers = group.Containers.Select(c => CreateContainer(c, info)).ToList() + } + } + } + }; + + return deploymentSpec; + } + + private static V1Container CreateContainer(GethCompanionNodeContainer container, GethBootstrapInfo info) { return new V1Container { - Name = Name, + Name = container.Name, Image = GethDockerImage.Image, Ports = new List { new V1ContainerPort { - ContainerPort = ApiPort, - Name = ContainerPortName + ContainerPort = container.ApiPort, + Name = container.ContainerPortName } }, // todo: use env vars to connect this node to the bootstrap node provided by gethInfo.podInfo & gethInfo.servicePort & gethInfo.genesisJsonBase64 @@ -147,20 +185,25 @@ namespace CodexDistTestCore.Marketplace new V1EnvVar { Name = "GETH_ARGS", - Value = $"--port {ApiPort} --discovery.port {ApiPort} --authrpc.port {RpcPort}" + Value = $"--port {container.ApiPort} --discovery.port {container.ApiPort} --authrpc.port {container.RpcPort}" }, new V1EnvVar { Name = "GENESIS_JSON", - Value = gethInfo.GenesisJsonBase64 + Value = info.GenesisJsonBase64 } } }; } - private Dictionary CreateSelector() + private Dictionary CreateBootstrapSelector() { return new Dictionary { { "test-gethb", "dtest-gethb" } }; } + + private Dictionary CreateCompanionSelector() + { + return new Dictionary { { "test-gethc", "dtest-gethc" } }; + } } } diff --git a/CodexDistTestCore/Marketplace/MarketplaceAccess.cs b/CodexDistTestCore/Marketplace/MarketplaceAccess.cs index 248e4ac..7782634 100644 --- a/CodexDistTestCore/Marketplace/MarketplaceAccess.cs +++ b/CodexDistTestCore/Marketplace/MarketplaceAccess.cs @@ -17,38 +17,37 @@ namespace CodexDistTestCore.Marketplace private readonly MarketplaceController marketplaceController; private readonly TestLog log; private readonly CodexNodeGroup group; - private readonly GethCompanionNodeContainer gethCompanionNodeContainer; - private string account = string.Empty; + private readonly GethCompanionNodeContainer container; public MarketplaceAccess( K8sManager k8sManager, MarketplaceController marketplaceController, TestLog log, - CodexNodeGroup group, - GethCompanionNodeContainer gethCompanionNodeContainer) + CodexNodeGroup group, + GethCompanionNodeContainer container) { this.k8sManager = k8sManager; this.marketplaceController = marketplaceController; this.log = log; this.group = group; - this.gethCompanionNodeContainer = gethCompanionNodeContainer; + this.container = container; } public void Initialize() { EnsureAccount(); - marketplaceController.AddToBalance(account, group.Origin.MarketplaceConfig!.InitialBalance); + marketplaceController.AddToBalance(container.Account, group.Origin.MarketplaceConfig!.InitialBalance); - log.Log($"Initialized Geth companion node with account '{account}' and initial balance {group.Origin.MarketplaceConfig!.InitialBalance}"); + log.Log($"Initialized Geth companion node with account '{container.Account}' and initial balance {group.Origin.MarketplaceConfig!.InitialBalance}"); } - public void AdvertiseContract(ContentId contentId, float maxPricePerMBPerSecond, float minRequiredCollateral, float minRequiredNumberOfDuplicates) + public void RequestStorage(ContentId contentId, int pricePerBytePerSecond, float requiredCollateral, float minRequiredNumberOfNodes) { throw new NotImplementedException(); } - public void MakeStorageAvailable(ByteSize size, float pricePerMBPerSecond, float collateral) + public void MakeStorageAvailable(ByteSize size, int minPricePerBytePerSecond, float maxCollateral) { throw new NotImplementedException(); } @@ -66,28 +65,28 @@ namespace CodexDistTestCore.Marketplace private void EnsureAccount() { FetchAccount(); - if (string.IsNullOrEmpty(account)) + if (string.IsNullOrEmpty(container.Account)) { Thread.Sleep(TimeSpan.FromSeconds(15)); FetchAccount(); } - Assert.That(account, Is.Not.Empty, "Unable to fetch account for geth companion node. Test infra failure."); + Assert.That(container.Account, Is.Not.Empty, "Unable to fetch account for geth companion node. Test infra failure."); } private void FetchAccount() { - account = k8sManager.ExecuteCommand(group.PodInfo!, gethCompanionNodeContainer.Name, "cat", GethDockerImage.AccountFilename); + container.Account = k8sManager.ExecuteCommand(group.PodInfo!, container.Name, "cat", GethDockerImage.AccountFilename); } } public class MarketplaceUnavailable : IMarketplaceAccess { - public void AdvertiseContract(ContentId contentId, float maxPricePerMBPerSecond, float minRequiredCollateral, float minRequiredNumberOfDuplicates) + public void RequestStorage(ContentId contentId, int pricePerBytePerSecond, float requiredCollateral, float minRequiredNumberOfNodes) { Unavailable(); } - public void MakeStorageAvailable(ByteSize size, float pricePerMBPerSecond, float collateral) + public void MakeStorageAvailable(ByteSize size, int minPricePerBytePerSecond, float maxCollateral) { Unavailable(); } diff --git a/CodexDistTestCore/Marketplace/MarketplaceController.cs b/CodexDistTestCore/Marketplace/MarketplaceController.cs index deaf747..9edea24 100644 --- a/CodexDistTestCore/Marketplace/MarketplaceController.cs +++ b/CodexDistTestCore/Marketplace/MarketplaceController.cs @@ -7,9 +7,9 @@ namespace CodexDistTestCore.Marketplace { private readonly TestLog log; private readonly K8sManager k8sManager; - private GethInfo? gethBootstrapNode; - private string bootstrapAccount = string.Empty; - private string bootstrapGenesisJson = string.Empty; + private readonly NumberSource companionGroupNumberSource = new NumberSource(0); + private List companionGroups = new List(); + private GethBootstrapInfo? bootstrapInfo; public MarketplaceController(TestLog log, K8sManager k8sManager) { @@ -17,30 +17,47 @@ namespace CodexDistTestCore.Marketplace this.k8sManager = k8sManager; } - public GethInfo BringOnlineMarketplace(OfflineCodexNodes offline) + public GethCompanionGroup BringOnlineMarketplace(OfflineCodexNodes offline) { - if (gethBootstrapNode != null) return gethBootstrapNode; + if (bootstrapInfo == null) + { + BringOnlineBootstrapNode(); + } - log.Log("Starting Geth bootstrap node..."); - gethBootstrapNode = k8sManager.BringOnlineGethBootstrapNode(); - ExtractAccountAndGenesisJson(); - log.Log($"Geth boothstrap node started. Initializing companions for {offline.NumberOfNodes} Codex nodes."); + log.Log($"Initializing companions for {offline.NumberOfNodes} Codex nodes."); + var group = new GethCompanionGroup(companionGroupNumberSource.GetNextNumber(), CreateCompanionContainers(offline)); + group.Pod = k8sManager.BringOnlineGethCompanionGroup(bootstrapInfo!, group); + companionGroups.Add(group); - - - return gethBootstrapNode; + log.Log("Initialized companion nodes."); + return group; } + private void BringOnlineBootstrapNode() + { + log.Log("Starting Geth bootstrap node..."); + var spec = k8sManager.CreateGethBootstrapNodeSpec(); + var pod = k8sManager.BringOnlineGethBootstrapNode(spec); + var (account, genesisJson) = ExtractAccountAndGenesisJson(); + bootstrapInfo = new GethBootstrapInfo(spec, pod, account, genesisJson); + log.Log($"Geth boothstrap node started."); + } - private GethCompanionNodeContainer? CreateGethNodeContainer(OfflineCodexNodes offline, int n) + private GethCompanionNodeContainer[] CreateCompanionContainers(OfflineCodexNodes offline) + { + var numberSource = new NumberSource(8080); + var result = new List(); + for (var i = 0; i < offline.NumberOfNodes; i++) result.Add(CreateGethNodeContainer(numberSource, i)); + return result.ToArray(); + } + + private GethCompanionNodeContainer CreateGethNodeContainer(NumberSource numberSource, int n) { return new GethCompanionNodeContainer( name: $"geth-node{n}", - servicePort: numberSource.GetNextServicePort(), - servicePortName: numberSource.GetNextServicePortName(), - apiPort: codexPortSource.GetNextNumber(), - rpcPort: codexPortSource.GetNextNumber(), + apiPort: numberSource.GetNextNumber(), + rpcPort: numberSource.GetNextNumber(), containerPortName: $"geth-{n}" ); } @@ -53,47 +70,50 @@ namespace CodexDistTestCore.Marketplace throw new NotImplementedException(); } - private void ExtractAccountAndGenesisJson() + private (string, string) ExtractAccountAndGenesisJson() { - FetchAccountAndGenesisJson(); - if (string.IsNullOrEmpty(bootstrapAccount) || string.IsNullOrEmpty(bootstrapGenesisJson)) + var (account, genesisJson) = FetchAccountAndGenesisJson(); + if (string.IsNullOrEmpty(account) || string.IsNullOrEmpty(genesisJson)) { Thread.Sleep(TimeSpan.FromSeconds(15)); - FetchAccountAndGenesisJson(); + (account, genesisJson) = FetchAccountAndGenesisJson(); } - Assert.That(bootstrapAccount, Is.Not.Empty, "Unable to fetch account for geth bootstrap node. Test infra failure."); - Assert.That(bootstrapGenesisJson, Is.Not.Empty, "Unable to fetch genesis-json for geth bootstrap node. Test infra failure."); + Assert.That(account, Is.Not.Empty, "Unable to fetch account for geth bootstrap node. Test infra failure."); + Assert.That(genesisJson, Is.Not.Empty, "Unable to fetch genesis-json for geth bootstrap node. Test infra failure."); - gethBootstrapNode!.GenesisJsonBase64 = Convert.ToBase64String(Encoding.ASCII.GetBytes(bootstrapGenesisJson)); + var encoded = Convert.ToBase64String(Encoding.ASCII.GetBytes(genesisJson)); - log.Log($"Initialized geth bootstrap node with account '{bootstrapAccount}'"); + log.Log($"Initialized geth bootstrap node with account '{account}'"); + return (account, encoded); } - private void FetchAccountAndGenesisJson() + private (string, string) FetchAccountAndGenesisJson() { - bootstrapAccount = ExecuteCommand("cat", GethDockerImage.AccountFilename); - bootstrapGenesisJson = ExecuteCommand("cat", GethDockerImage.GenesisFilename); + var bootstrapAccount = ExecuteCommand("cat", GethDockerImage.AccountFilename); + var bootstrapGenesisJson = ExecuteCommand("cat", GethDockerImage.GenesisFilename); + return (bootstrapAccount, bootstrapGenesisJson); } private string ExecuteCommand(string command, params string[] arguments) { - return k8sManager.ExecuteCommand(gethBootstrapNode!.BootstrapPod, K8sGethBoostrapSpecs.ContainerName, command, arguments); + return k8sManager.ExecuteCommand(bootstrapInfo!.Pod, K8sGethBoostrapSpecs.ContainerName, command, arguments); } } - public class GethInfo + public class GethBootstrapInfo { - public GethInfo(K8sGethBoostrapSpecs spec, PodInfo bootstrapPod, PodInfo companionPod) + public GethBootstrapInfo(K8sGethBoostrapSpecs spec, PodInfo pod, string account, string genesisJsonBase64) { Spec = spec; - BootstrapPod = bootstrapPod; - CompanionPod = companionPod; + Pod = pod; + Account = account; + GenesisJsonBase64 = genesisJsonBase64; } public K8sGethBoostrapSpecs Spec { get; } - public PodInfo BootstrapPod { get; } - public PodInfo CompanionPod { get; } - public string GenesisJsonBase64 { get; set; } = string.Empty; + public PodInfo Pod { get; } + public string Account { get; } + public string GenesisJsonBase64 { get; } } } From 1512bd271605ecb8786aa30b84d997acbe322c43 Mon Sep 17 00:00:00 2001 From: benbierens Date: Tue, 11 Apr 2023 12:17:21 +0200 Subject: [PATCH 12/46] This needs cleaning up --- CodexDistTestCore/CodexNodeContainer.cs | 6 +++++- CodexDistTestCore/Config/CodexDockerImage.cs | 14 +++++--------- CodexDistTestCore/K8sManager.cs | 7 +++---- CodexDistTestCore/K8sOperations.cs | 12 ++++++------ 4 files changed, 19 insertions(+), 20 deletions(-) diff --git a/CodexDistTestCore/CodexNodeContainer.cs b/CodexDistTestCore/CodexNodeContainer.cs index ccf9b87..71994ab 100644 --- a/CodexDistTestCore/CodexNodeContainer.cs +++ b/CodexDistTestCore/CodexNodeContainer.cs @@ -1,4 +1,6 @@ -namespace CodexDistTestCore +using CodexDistTestCore.Marketplace; + +namespace CodexDistTestCore { public class CodexNodeContainer { @@ -24,6 +26,8 @@ public int ListenPort { get; } public string DataDir { get; } public int MetricsPort { get; } + + public GethCompanionNodeContainer? GethCompanionNodeContainer { get; set; } // :C } public class CodexGroupNumberSource diff --git a/CodexDistTestCore/Config/CodexDockerImage.cs b/CodexDistTestCore/Config/CodexDockerImage.cs index 5855f71..a252a26 100644 --- a/CodexDistTestCore/Config/CodexDockerImage.cs +++ b/CodexDistTestCore/Config/CodexDockerImage.cs @@ -50,16 +50,12 @@ namespace CodexDistTestCore.Config AddVar("METRICS_ADDR", "0.0.0.0"); AddVar("METRICS_PORT", container.MetricsPort.ToString()); } - if (container.GethCompanionNodeContainer != null) + if (node.MarketplaceConfig != null) { - // well, darn: To get the account here, the geth companion needs to have been started and account string fetched. - // but the codex node and geth companion node are in the same pod. so they start at the same time. - // so we cannot start the codex node with the correct account string at the same time as we start the geth node that - // is supposed to generate that string. - // begin rework: Separate pod for geth companion node. -//ETH_PROVIDER -//ETH_ACCOUNT -//ETH_DEPLOYMENT + //ETH_PROVIDER + //ETH_ACCOUNT + //ETH_DEPLOYMENT + AddVar("ETH_ACCOUNT", container.GethCompanionNodeContainer!.Account); } } diff --git a/CodexDistTestCore/K8sManager.cs b/CodexDistTestCore/K8sManager.cs index 9686bcc..06bcafb 100644 --- a/CodexDistTestCore/K8sManager.cs +++ b/CodexDistTestCore/K8sManager.cs @@ -35,6 +35,7 @@ namespace CodexDistTestCore if (offline.MarketplaceConfig != null) { group.GethCompanionGroup = marketplaceController.BringOnlineMarketplace(offline); + ConnectMarketplace(group); } K8s(k => k.BringOnline(group, offline)); @@ -43,10 +44,6 @@ namespace CodexDistTestCore { BringOnlineMetrics(group); } - if (offline.MarketplaceConfig != null) - { - ConnectMarketplace(group); - } log.Log($"{group.Describe()} online."); @@ -126,6 +123,8 @@ namespace CodexDistTestCore private void ConnectMarketplace(CodexNodeGroup group, OnlineCodexNode node, GethCompanionNodeContainer container) { + node.Container.GethCompanionNodeContainer = container; // :c + var access = new MarketplaceAccess(this, marketplaceController, log, group, container); access.Initialize(); node.Marketplace = access; diff --git a/CodexDistTestCore/K8sOperations.cs b/CodexDistTestCore/K8sOperations.cs index c0c7234..580829e 100644 --- a/CodexDistTestCore/K8sOperations.cs +++ b/CodexDistTestCore/K8sOperations.cs @@ -292,10 +292,10 @@ namespace CodexDistTestCore }; } - private List CreateDeploymentContainers(CodexNodeGroup online, OfflineCodexNodes offline) + private List CreateDeploymentContainers(CodexNodeGroup group, OfflineCodexNodes offline) { var result = new List(); - var containers = online.GetContainers(); + var containers = group.GetContainers(); foreach (var container in containers) { result.Add(new V1Container @@ -317,11 +317,11 @@ namespace CodexDistTestCore return result; } - private void DeleteDeployment(CodexNodeGroup online) + private void DeleteDeployment(CodexNodeGroup group) { - if (online.Deployment == null) return; - client.DeleteNamespacedDeployment(online.Deployment.Name(), K8sNamespace); - online.Deployment = null; + if (group.Deployment == null) return; + client.DeleteNamespacedDeployment(group.Deployment.Name(), K8sNamespace); + group.Deployment = null; } private void CreatePrometheusDeployment(K8sPrometheusSpecs spec) From de16ba3e362a04249774ef34fa4e4fb3d7e7c0dd Mon Sep 17 00:00:00 2001 From: benbierens Date: Tue, 11 Apr 2023 16:13:39 +0200 Subject: [PATCH 13/46] Configures geth bootstrap node to mine blocks --- CodexDistTestCore/Marketplace/K8sGethSpecs.cs | 10 ++-- .../Marketplace/MarketplaceAccess.cs | 2 +- .../Marketplace/MarketplaceController.cs | 47 ++++++++++++++----- 3 files changed, 41 insertions(+), 18 deletions(-) diff --git a/CodexDistTestCore/Marketplace/K8sGethSpecs.cs b/CodexDistTestCore/Marketplace/K8sGethSpecs.cs index 6318c4d..47e8fb7 100644 --- a/CodexDistTestCore/Marketplace/K8sGethSpecs.cs +++ b/CodexDistTestCore/Marketplace/K8sGethSpecs.cs @@ -14,7 +14,7 @@ namespace CodexDistTestCore.Marketplace { public const string ContainerName = "dtest-gethb"; private const string portName = "gethb"; - private const string genesisJsonBase64 = "ewogICAgImNvbmZpZyI6IHsKICAgICAgImNoYWluSWQiOiAxMjM0NSwKICAgICAgImhvbWVzdGVhZEJsb2NrIjogMCwKICAgICAgImVpcDE1MEJsb2NrIjogMCwKICAgICAgImVpcDE1NUJsb2NrIjogMCwKICAgICAgImVpcDE1OEJsb2NrIjogMCwKICAgICAgImJ5emFudGl1bUJsb2NrIjogMCwKICAgICAgImNvbnN0YW50aW5vcGxlQmxvY2siOiAwLAogICAgICAicGV0ZXJzYnVyZ0Jsb2NrIjogMCwKICAgICAgImlzdGFuYnVsQmxvY2siOiAwLAogICAgICAibXVpckdsYWNpZXJCbG9jayI6IDAsCiAgICAgICJiZXJsaW5CbG9jayI6IDAsCiAgICAgICJsb25kb25CbG9jayI6IDAsCiAgICAgICJhcnJvd0dsYWNpZXJCbG9jayI6IDAsCiAgICAgICJncmF5R2xhY2llckJsb2NrIjogMCwKICAgICAgImNsaXF1ZSI6IHsKICAgICAgICAicGVyaW9kIjogNSwKICAgICAgICAiZXBvY2giOiAzMDAwMAogICAgICB9CiAgICB9LAogICAgImRpZmZpY3VsdHkiOiAiMSIsCiAgICAiZ2FzTGltaXQiOiAiODAwMDAwMDAwIiwKICAgICJleHRyYWRhdGEiOiAiMHgwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwN2RmOWE4NzVhMTc0YjNiYzU2NWU2NDI0YTAwNTBlYmMxYjJkMWQ4MjAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAiLAogICAgImFsbG9jIjogewogICAgICAiQUNDT1VOVF9IRVJFIjogeyAiYmFsYW5jZSI6ICI1MDAwMDAiIH0KICAgIH0KICB9"; + private const string genesisJsonBase64 = "ewogICAgImNvbmZpZyI6IHsKICAgICAgImNoYWluSWQiOiA3ODk5ODgsCiAgICAgICJob21lc3RlYWRCbG9jayI6IDAsCiAgICAgICJlaXAxNTBCbG9jayI6IDAsCiAgICAgICJlaXAxNTVCbG9jayI6IDAsCiAgICAgICJlaXAxNThCbG9jayI6IDAsCiAgICAgICJieXphbnRpdW1CbG9jayI6IDAsCiAgICAgICJjb25zdGFudGlub3BsZUJsb2NrIjogMCwKICAgICAgInBldGVyc2J1cmdCbG9jayI6IDAsCiAgICAgICJpc3RhbmJ1bEJsb2NrIjogMCwKICAgICAgIm11aXJHbGFjaWVyQmxvY2siOiAwLAogICAgICAiYmVybGluQmxvY2siOiAwLAogICAgICAibG9uZG9uQmxvY2siOiAwLAogICAgICAiYXJyb3dHbGFjaWVyQmxvY2siOiAwLAogICAgICAiZ3JheUdsYWNpZXJCbG9jayI6IDAsCiAgICAgICJjbGlxdWUiOiB7CiAgICAgICAgInBlcmlvZCI6IDUsCiAgICAgICAgImVwb2NoIjogMzAwMDAKICAgICAgfQogICAgfSwKICAgICJkaWZmaWN1bHR5IjogIjEiLAogICAgImdhc0xpbWl0IjogIjgwMDAwMDAwMCIsCiAgICAiZXh0cmFkYXRhIjogIjB4MDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMEFDQ09VTlRfSEVSRTAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAiLAogICAgImFsbG9jIjogewogICAgICAiMHhBQ0NPVU5UX0hFUkUiOiB7ICJiYWxhbmNlIjogIjUwMDAwMCIgfQogICAgfQogIH0="; public K8sGethBoostrapSpecs(int servicePort) { @@ -68,7 +68,7 @@ namespace CodexDistTestCore.Marketplace { new V1ContainerPort { - ContainerPort = 9090, + ContainerPort = 8545, Name = portName } }, @@ -77,7 +77,7 @@ namespace CodexDistTestCore.Marketplace //new V1EnvVar //{ // Name = "GETH_ARGS", - // Value = "--qwerty" + // Value = "" //}, new V1EnvVar { @@ -120,7 +120,7 @@ namespace CodexDistTestCore.Marketplace { Name = "gethb-service", Protocol = "TCP", - Port = 9090, + Port = 8545, TargetPort = portName, NodePort = ServicePort } @@ -185,7 +185,7 @@ namespace CodexDistTestCore.Marketplace new V1EnvVar { Name = "GETH_ARGS", - Value = $"--port {container.ApiPort} --discovery.port {container.ApiPort} --authrpc.port {container.RpcPort}" + Value = $"--port {container.ApiPort} --discovery.port {container.ApiPort} --http.port {container.RpcPort}" }, new V1EnvVar { diff --git a/CodexDistTestCore/Marketplace/MarketplaceAccess.cs b/CodexDistTestCore/Marketplace/MarketplaceAccess.cs index 7782634..e1b99b0 100644 --- a/CodexDistTestCore/Marketplace/MarketplaceAccess.cs +++ b/CodexDistTestCore/Marketplace/MarketplaceAccess.cs @@ -75,7 +75,7 @@ namespace CodexDistTestCore.Marketplace private void FetchAccount() { - container.Account = k8sManager.ExecuteCommand(group.PodInfo!, container.Name, "cat", GethDockerImage.AccountFilename); + container.Account = k8sManager.ExecuteCommand(group.GethCompanionGroup!.Pod!, container.Name, "cat", GethDockerImage.AccountFilename); } } diff --git a/CodexDistTestCore/Marketplace/MarketplaceController.cs b/CodexDistTestCore/Marketplace/MarketplaceController.cs index 9edea24..43ed636 100644 --- a/CodexDistTestCore/Marketplace/MarketplaceController.cs +++ b/CodexDistTestCore/Marketplace/MarketplaceController.cs @@ -1,4 +1,8 @@ -using NUnit.Framework; +using CodexDistTestCore.Config; +using Nethereum.Web3; +using Nethereum.Web3.Accounts; +using Nethereum.Web3.Accounts.Managed; +using NUnit.Framework; using System.Text; namespace CodexDistTestCore.Marketplace @@ -39,7 +43,7 @@ namespace CodexDistTestCore.Marketplace log.Log("Starting Geth bootstrap node..."); var spec = k8sManager.CreateGethBootstrapNodeSpec(); var pod = k8sManager.BringOnlineGethBootstrapNode(spec); - var (account, genesisJson) = ExtractAccountAndGenesisJson(); + var (account, genesisJson) = ExtractAccountAndGenesisJson(pod); bootstrapInfo = new GethBootstrapInfo(spec, pod, account, genesisJson); log.Log($"Geth boothstrap node started."); } @@ -62,21 +66,40 @@ namespace CodexDistTestCore.Marketplace ); } - public void AddToBalance(string account, int amount) + + private readonly K8sCluster k8sCluster = new K8sCluster(); + + public void AddToBalance(string account, decimal amount) { if (amount < 1 || string.IsNullOrEmpty(account)) Assert.Fail("Invalid arguments for AddToBalance"); // call the bootstrap node and convince it to give 'account' 'amount' tokens somehow. - throw new NotImplementedException(); + + var ip = k8sCluster.GetIp(); + var port = bootstrapInfo!.Spec.ServicePort; + + var bootstrapaccount = new ManagedAccount(bootstrapInfo.Account, "qwerty!@#$%^"); + var web3 = new Web3(bootstrapaccount, $"http://{ip}:{port}"); + + var blockNumber1 = Utils.Wait(web3.Eth.Blocks.GetBlockNumber.SendRequestAsync()); + Thread.Sleep(TimeSpan.FromSeconds(12)); + var blockNumber2 = Utils.Wait(web3.Eth.Blocks.GetBlockNumber.SendRequestAsync()); + + + + var receipt = Utils.Wait(web3.Eth.GetEtherTransferService().TransferEtherAndWaitForReceiptAsync(account, amount)); + + var a = 0; + } - private (string, string) ExtractAccountAndGenesisJson() + private (string, string) ExtractAccountAndGenesisJson(PodInfo pod) { - var (account, genesisJson) = FetchAccountAndGenesisJson(); + var (account, genesisJson) = FetchAccountAndGenesisJson(pod); if (string.IsNullOrEmpty(account) || string.IsNullOrEmpty(genesisJson)) { Thread.Sleep(TimeSpan.FromSeconds(15)); - (account, genesisJson) = FetchAccountAndGenesisJson(); + (account, genesisJson) = FetchAccountAndGenesisJson(pod); } Assert.That(account, Is.Not.Empty, "Unable to fetch account for geth bootstrap node. Test infra failure."); @@ -88,16 +111,16 @@ namespace CodexDistTestCore.Marketplace return (account, encoded); } - private (string, string) FetchAccountAndGenesisJson() + private (string, string) FetchAccountAndGenesisJson(PodInfo pod) { - var bootstrapAccount = ExecuteCommand("cat", GethDockerImage.AccountFilename); - var bootstrapGenesisJson = ExecuteCommand("cat", GethDockerImage.GenesisFilename); + var bootstrapAccount = ExecuteCommand(pod, "cat", GethDockerImage.AccountFilename); + var bootstrapGenesisJson = ExecuteCommand(pod, "cat", GethDockerImage.GenesisFilename); return (bootstrapAccount, bootstrapGenesisJson); } - private string ExecuteCommand(string command, params string[] arguments) + private string ExecuteCommand(PodInfo pod, string command, params string[] arguments) { - return k8sManager.ExecuteCommand(bootstrapInfo!.Pod, K8sGethBoostrapSpecs.ContainerName, command, arguments); + return k8sManager.ExecuteCommand(pod, K8sGethBoostrapSpecs.ContainerName, command, arguments); } } From f87163acdc0361bf852084a62c69ba30d52cb47e Mon Sep 17 00:00:00 2001 From: benbierens Date: Tue, 11 Apr 2023 16:43:53 +0200 Subject: [PATCH 14/46] very ugly wip --- CodexDistTestCore/Marketplace/K8sGethSpecs.cs | 12 ++++++------ .../Marketplace/MarketplaceController.cs | 17 +++++++++++++---- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/CodexDistTestCore/Marketplace/K8sGethSpecs.cs b/CodexDistTestCore/Marketplace/K8sGethSpecs.cs index 47e8fb7..dcd9740 100644 --- a/CodexDistTestCore/Marketplace/K8sGethSpecs.cs +++ b/CodexDistTestCore/Marketplace/K8sGethSpecs.cs @@ -14,7 +14,7 @@ namespace CodexDistTestCore.Marketplace { public const string ContainerName = "dtest-gethb"; private const string portName = "gethb"; - private const string genesisJsonBase64 = "ewogICAgImNvbmZpZyI6IHsKICAgICAgImNoYWluSWQiOiA3ODk5ODgsCiAgICAgICJob21lc3RlYWRCbG9jayI6IDAsCiAgICAgICJlaXAxNTBCbG9jayI6IDAsCiAgICAgICJlaXAxNTVCbG9jayI6IDAsCiAgICAgICJlaXAxNThCbG9jayI6IDAsCiAgICAgICJieXphbnRpdW1CbG9jayI6IDAsCiAgICAgICJjb25zdGFudGlub3BsZUJsb2NrIjogMCwKICAgICAgInBldGVyc2J1cmdCbG9jayI6IDAsCiAgICAgICJpc3RhbmJ1bEJsb2NrIjogMCwKICAgICAgIm11aXJHbGFjaWVyQmxvY2siOiAwLAogICAgICAiYmVybGluQmxvY2siOiAwLAogICAgICAibG9uZG9uQmxvY2siOiAwLAogICAgICAiYXJyb3dHbGFjaWVyQmxvY2siOiAwLAogICAgICAiZ3JheUdsYWNpZXJCbG9jayI6IDAsCiAgICAgICJjbGlxdWUiOiB7CiAgICAgICAgInBlcmlvZCI6IDUsCiAgICAgICAgImVwb2NoIjogMzAwMDAKICAgICAgfQogICAgfSwKICAgICJkaWZmaWN1bHR5IjogIjEiLAogICAgImdhc0xpbWl0IjogIjgwMDAwMDAwMCIsCiAgICAiZXh0cmFkYXRhIjogIjB4MDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMEFDQ09VTlRfSEVSRTAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAiLAogICAgImFsbG9jIjogewogICAgICAiMHhBQ0NPVU5UX0hFUkUiOiB7ICJiYWxhbmNlIjogIjUwMDAwMCIgfQogICAgfQogIH0="; + private const string genesisJsonBase64 = "ewogICAgImNvbmZpZyI6IHsKICAgICAgImNoYWluSWQiOiA3ODk5ODgsCiAgICAgICJob21lc3RlYWRCbG9jayI6IDAsCiAgICAgICJlaXAxNTBCbG9jayI6IDAsCiAgICAgICJlaXAxNTVCbG9jayI6IDAsCiAgICAgICJlaXAxNThCbG9jayI6IDAsCiAgICAgICJieXphbnRpdW1CbG9jayI6IDAsCiAgICAgICJjb25zdGFudGlub3BsZUJsb2NrIjogMCwKICAgICAgInBldGVyc2J1cmdCbG9jayI6IDAsCiAgICAgICJpc3RhbmJ1bEJsb2NrIjogMCwKICAgICAgIm11aXJHbGFjaWVyQmxvY2siOiAwLAogICAgICAiYmVybGluQmxvY2siOiAwLAogICAgICAibG9uZG9uQmxvY2siOiAwLAogICAgICAiYXJyb3dHbGFjaWVyQmxvY2siOiAwLAogICAgICAiZ3JheUdsYWNpZXJCbG9jayI6IDAsCiAgICAgICJjbGlxdWUiOiB7CiAgICAgICAgInBlcmlvZCI6IDUsCiAgICAgICAgImVwb2NoIjogMzAwMDAKICAgICAgfQogICAgfSwKICAgICJkaWZmaWN1bHR5IjogIjEiLAogICAgImdhc0xpbWl0IjogIjgwMDAwMDAwMCIsCiAgICAiZXh0cmFkYXRhIjogIjB4MDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMEFDQ09VTlRfSEVSRTAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAiLAogICAgImFsbG9jIjogewogICAgICAiMHhBQ0NPVU5UX0hFUkUiOiB7ICJiYWxhbmNlIjogIjUwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAiIH0KICAgIH0KICB9"; public K8sGethBoostrapSpecs(int servicePort) { @@ -74,11 +74,11 @@ namespace CodexDistTestCore.Marketplace }, Env = new List { - //new V1EnvVar - //{ - // Name = "GETH_ARGS", - // Value = "" - //}, + new V1EnvVar + { + Name = "GETH_ARGS", + Value = "" + }, new V1EnvVar { Name = "GENESIS_JSON", diff --git a/CodexDistTestCore/Marketplace/MarketplaceController.cs b/CodexDistTestCore/Marketplace/MarketplaceController.cs index 43ed636..17ce1cf 100644 --- a/CodexDistTestCore/Marketplace/MarketplaceController.cs +++ b/CodexDistTestCore/Marketplace/MarketplaceController.cs @@ -3,6 +3,7 @@ using Nethereum.Web3; using Nethereum.Web3.Accounts; using Nethereum.Web3.Accounts.Managed; using NUnit.Framework; +using System.Numerics; using System.Text; namespace CodexDistTestCore.Marketplace @@ -78,16 +79,24 @@ namespace CodexDistTestCore.Marketplace var ip = k8sCluster.GetIp(); var port = bootstrapInfo!.Spec.ServicePort; - var bootstrapaccount = new ManagedAccount(bootstrapInfo.Account, "qwerty!@#$%^"); - var web3 = new Web3(bootstrapaccount, $"http://{ip}:{port}"); + //var bootstrapaccount = new ManagedAccount(bootstrapInfo.Account, "qwerty!@#$%^"); + var web3 = new Web3($"http://{ip}:{port}"); var blockNumber1 = Utils.Wait(web3.Eth.Blocks.GetBlockNumber.SendRequestAsync()); - Thread.Sleep(TimeSpan.FromSeconds(12)); + Thread.Sleep(TimeSpan.FromSeconds(5)); var blockNumber2 = Utils.Wait(web3.Eth.Blocks.GetBlockNumber.SendRequestAsync()); + var bootstrapBalance = Utils.Wait(web3.Eth.GetBalance.SendRequestAsync(bootstrapInfo.Account)); + var targetBalance = Utils.Wait(web3.Eth.GetBalance.SendRequestAsync(account)); + var bigint = new BigInteger(amount); + var str = bigint.ToString("X"); + var value = new Nethereum.Hex.HexTypes.HexBigInteger(str); + var aaa = Utils.Wait(web3.Eth.TransactionManager.SendTransactionAsync(bootstrapInfo.Account, account, value)); - var receipt = Utils.Wait(web3.Eth.GetEtherTransferService().TransferEtherAndWaitForReceiptAsync(account, amount)); + //var receipt = Utils.Wait(web3.Eth.GetEtherTransferService().TransferEtherAndWaitForReceiptAsync(account, amount)); + + targetBalance = Utils.Wait(web3.Eth.GetBalance.SendRequestAsync(account)); var a = 0; From bdd977d8a9f574914b5200ec87bddf686249920d Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 12 Apr 2023 08:40:23 +0200 Subject: [PATCH 15/46] wrap up with basic ETH balance control --- .../Marketplace/GethCompanionNodeContainer.cs | 4 +- CodexDistTestCore/Marketplace/K8sGethSpecs.cs | 2 +- .../Marketplace/MarketplaceAccess.cs | 10 ++--- .../Marketplace/MarketplaceController.cs | 38 +++++++++++-------- Tests/BasicTests/SimpleTests.cs | 16 ++++++-- 5 files changed, 44 insertions(+), 26 deletions(-) diff --git a/CodexDistTestCore/Marketplace/GethCompanionNodeContainer.cs b/CodexDistTestCore/Marketplace/GethCompanionNodeContainer.cs index de5e59e..74d687c 100644 --- a/CodexDistTestCore/Marketplace/GethCompanionNodeContainer.cs +++ b/CodexDistTestCore/Marketplace/GethCompanionNodeContainer.cs @@ -15,16 +15,18 @@ public class GethCompanionNodeContainer { - public GethCompanionNodeContainer(string name, int apiPort, int rpcPort, string containerPortName) + public GethCompanionNodeContainer(string name, int apiPort, int rpcPort, string containerPortName, int authRpcPort) { Name = name; ApiPort = apiPort; + AuthRpcPort = authRpcPort; RpcPort = rpcPort; ContainerPortName = containerPortName; } public string Name { get; } public int ApiPort { get; } + public int AuthRpcPort { get; } public int RpcPort { get; } public string ContainerPortName { get; } diff --git a/CodexDistTestCore/Marketplace/K8sGethSpecs.cs b/CodexDistTestCore/Marketplace/K8sGethSpecs.cs index dcd9740..f99d520 100644 --- a/CodexDistTestCore/Marketplace/K8sGethSpecs.cs +++ b/CodexDistTestCore/Marketplace/K8sGethSpecs.cs @@ -185,7 +185,7 @@ namespace CodexDistTestCore.Marketplace new V1EnvVar { Name = "GETH_ARGS", - Value = $"--port {container.ApiPort} --discovery.port {container.ApiPort} --http.port {container.RpcPort}" + Value = $"--port {container.ApiPort} --discovery.port {container.ApiPort} --authrpc.port {container.AuthRpcPort} --http.port {container.RpcPort}" }, new V1EnvVar { diff --git a/CodexDistTestCore/Marketplace/MarketplaceAccess.cs b/CodexDistTestCore/Marketplace/MarketplaceAccess.cs index e1b99b0..e9bf39e 100644 --- a/CodexDistTestCore/Marketplace/MarketplaceAccess.cs +++ b/CodexDistTestCore/Marketplace/MarketplaceAccess.cs @@ -8,7 +8,7 @@ namespace CodexDistTestCore.Marketplace void MakeStorageAvailable(ByteSize size, int minPricePerBytePerSecond, float maxCollateral); void RequestStorage(ContentId contentId, int pricePerBytePerSecond, float requiredCollateral, float minRequiredNumberOfNodes); void AssertThatBalance(IResolveConstraint constraint, string message = ""); - float GetBalance(); + decimal GetBalance(); } public class MarketplaceAccess : IMarketplaceAccess @@ -57,9 +57,9 @@ namespace CodexDistTestCore.Marketplace throw new NotImplementedException(); } - public float GetBalance() + public decimal GetBalance() { - throw new NotImplementedException(); + return marketplaceController.GetBalance(container.Account); } private void EnsureAccount() @@ -96,10 +96,10 @@ namespace CodexDistTestCore.Marketplace Unavailable(); } - public float GetBalance() + public decimal GetBalance() { Unavailable(); - return 0.0f; + return 0; } private void Unavailable() diff --git a/CodexDistTestCore/Marketplace/MarketplaceController.cs b/CodexDistTestCore/Marketplace/MarketplaceController.cs index 17ce1cf..21bfc08 100644 --- a/CodexDistTestCore/Marketplace/MarketplaceController.cs +++ b/CodexDistTestCore/Marketplace/MarketplaceController.cs @@ -62,12 +62,12 @@ namespace CodexDistTestCore.Marketplace return new GethCompanionNodeContainer( name: $"geth-node{n}", apiPort: numberSource.GetNextNumber(), + authRpcPort: numberSource.GetNextNumber(), rpcPort: numberSource.GetNextNumber(), containerPortName: $"geth-{n}" ); } - private readonly K8sCluster k8sCluster = new K8sCluster(); public void AddToBalance(string account, decimal amount) @@ -76,30 +76,38 @@ namespace CodexDistTestCore.Marketplace // call the bootstrap node and convince it to give 'account' 'amount' tokens somehow. - var ip = k8sCluster.GetIp(); - var port = bootstrapInfo!.Spec.ServicePort; + var web3 = CreateWeb3(); - //var bootstrapaccount = new ManagedAccount(bootstrapInfo.Account, "qwerty!@#$%^"); - var web3 = new Web3($"http://{ip}:{port}"); + //var blockNumber1 = Utils.Wait(web3.Eth.Blocks.GetBlockNumber.SendRequestAsync()); + //Thread.Sleep(TimeSpan.FromSeconds(5)); + //var blockNumber2 = Utils.Wait(web3.Eth.Blocks.GetBlockNumber.SendRequestAsync()); - var blockNumber1 = Utils.Wait(web3.Eth.Blocks.GetBlockNumber.SendRequestAsync()); - Thread.Sleep(TimeSpan.FromSeconds(5)); - var blockNumber2 = Utils.Wait(web3.Eth.Blocks.GetBlockNumber.SendRequestAsync()); - - var bootstrapBalance = Utils.Wait(web3.Eth.GetBalance.SendRequestAsync(bootstrapInfo.Account)); - var targetBalance = Utils.Wait(web3.Eth.GetBalance.SendRequestAsync(account)); + //var bootstrapBalance = Utils.Wait(web3.Eth.GetBalance.SendRequestAsync(bootstrapInfo.Account)); + var bigint = new BigInteger(amount); var str = bigint.ToString("X"); var value = new Nethereum.Hex.HexTypes.HexBigInteger(str); - var aaa = Utils.Wait(web3.Eth.TransactionManager.SendTransactionAsync(bootstrapInfo.Account, account, value)); + var aaa = Utils.Wait(web3.Eth.TransactionManager.SendTransactionAsync(bootstrapInfo!.Account, account, value)); + var receipt = Utils.Wait(web3.Eth.TransactionManager.TransactionReceiptService.PollForReceiptAsync(aaa)); //var receipt = Utils.Wait(web3.Eth.GetEtherTransferService().TransferEtherAndWaitForReceiptAsync(account, amount)); + //var targetBalance = Utils.Wait(web3.Eth.GetBalance.SendRequestAsync(account)); + } - targetBalance = Utils.Wait(web3.Eth.GetBalance.SendRequestAsync(account)); - - var a = 0; + public decimal GetBalance(string account) + { + var web3 = CreateWeb3(); + var bigInt = Utils.Wait(web3.Eth.GetBalance.SendRequestAsync(account)); + return (decimal)bigInt.Value; + } + private Web3 CreateWeb3() + { + var ip = k8sCluster.GetIp(); + var port = bootstrapInfo!.Spec.ServicePort; + //var bootstrapaccount = new ManagedAccount(bootstrapInfo.Account, "qwerty!@#$%^"); + return new Web3($"http://{ip}:{port}"); } private (string, string) ExtractAccountAndGenesisJson(PodInfo pod) diff --git a/Tests/BasicTests/SimpleTests.cs b/Tests/BasicTests/SimpleTests.cs index 233b730..e15ce98 100644 --- a/Tests/BasicTests/SimpleTests.cs +++ b/Tests/BasicTests/SimpleTests.cs @@ -34,10 +34,20 @@ namespace Tests.BasicTests [Test] public void MarketplaceExample() { - var primary = SetupCodexNodes(1) + var group = SetupCodexNodes(4) .WithStorageQuota(10.GB()) .EnableMarketplace(initialBalance: 20) - .BringOnline()[0]; + .BringOnline(); + + foreach (var node in group) + { + Assert.That(node.Marketplace.GetBalance(), Is.EqualTo(20)); + } + + // WIP: Balance is now only ETH. + // todo: All nodes should have plenty of ETH to pay for transactions. + // todo: Upload our own token, use this exclusively. ETH should be invisibile to the tests. + //var secondary = SetupCodexNodes(1) // .EnableMarketplace(initialBalance: 1000) @@ -56,8 +66,6 @@ namespace Tests.BasicTests //secondary.Marketplace.AssertThatBalance(Is.LessThan(1000), "Contractor was not charged for storage."); //primary.Marketplace.AssertThatBalance(Is.GreaterThan(primaryBalance), "Storer was not paid for storage."); - - var aa = 0; } [Test] From 7b91c83f5bc470a6094d32c8c2d321b07e2a2421 Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 12 Apr 2023 13:53:55 +0200 Subject: [PATCH 16/46] Moving everything around --- CodexDistTestCore/CodexNodeGroup.cs | 10 -- CodexDistTestCore/OfflineCodexNodes.cs | 9 -- CodexDistTestCore/TestLog.cs | 144 ------------------ CodexDistTestCore/Utils.cs | 9 -- DistTestCore/ByteSize.cs | 57 +++++++ DistTestCore/Codex/CodexAccess.cs | 43 ++++++ DistTestCore/Codex/CodexContainerRecipe.cs | 35 +++++ DistTestCore/Codex/CodexLogLevel.cs | 11 ++ DistTestCore/Codex/CodexStartupConfig.cs | 12 ++ DistTestCore/DistTestCore.csproj | 20 +++ DistTestCore/Http.cs | 100 ++++++++++++ DistTestCore/TestLifecycle.cs | 31 ++++ KubernetesWorkflow/ContainerRecipe.cs | 42 +++++ KubernetesWorkflow/ContainerRecipeFactory.cs | 60 ++++++++ KubernetesWorkflow/K8sCluster.cs | 40 +++++ KubernetesWorkflow/K8sController.cs | 25 +++ KubernetesWorkflow/KubernetesWorkflow.csproj | 14 ++ KubernetesWorkflow/Location.cs | 9 ++ KubernetesWorkflow/RecipeComponentFactory.cs | 24 +++ KubernetesWorkflow/RunningContainers.cs | 30 ++++ KubernetesWorkflow/RunningPod.cs | 24 +++ KubernetesWorkflow/StartupConfig.cs | 18 +++ KubernetesWorkflow/StartupWorkflow.cs | 40 +++++ KubernetesWorkflow/WorkflowCreator.cs | 13 ++ .../Config => Logging}/LogConfig.cs | 2 +- Logging/LogFile.cs | 73 +++++++++ Logging/Logging.csproj | 19 +++ Logging/TestLog.cs | 74 +++++++++ LongTests/BasicTests/LargeFileTests.cs | 2 +- LongTests/BasicTests/TestInfraTests.cs | 2 +- .../{LongTests.csproj => TestsLong.csproj} | 0 Utils/NumberSource.cs | 19 +++ Utils/Time.cs | 16 ++ Utils/Utils.csproj | 10 ++ cs-codex-dist-testing.sln | 22 ++- 35 files changed, 882 insertions(+), 177 deletions(-) delete mode 100644 CodexDistTestCore/TestLog.cs create mode 100644 DistTestCore/ByteSize.cs create mode 100644 DistTestCore/Codex/CodexAccess.cs create mode 100644 DistTestCore/Codex/CodexContainerRecipe.cs create mode 100644 DistTestCore/Codex/CodexLogLevel.cs create mode 100644 DistTestCore/Codex/CodexStartupConfig.cs create mode 100644 DistTestCore/DistTestCore.csproj create mode 100644 DistTestCore/Http.cs create mode 100644 DistTestCore/TestLifecycle.cs create mode 100644 KubernetesWorkflow/ContainerRecipe.cs create mode 100644 KubernetesWorkflow/ContainerRecipeFactory.cs create mode 100644 KubernetesWorkflow/K8sCluster.cs create mode 100644 KubernetesWorkflow/K8sController.cs create mode 100644 KubernetesWorkflow/KubernetesWorkflow.csproj create mode 100644 KubernetesWorkflow/Location.cs create mode 100644 KubernetesWorkflow/RecipeComponentFactory.cs create mode 100644 KubernetesWorkflow/RunningContainers.cs create mode 100644 KubernetesWorkflow/RunningPod.cs create mode 100644 KubernetesWorkflow/StartupConfig.cs create mode 100644 KubernetesWorkflow/StartupWorkflow.cs create mode 100644 KubernetesWorkflow/WorkflowCreator.cs rename {CodexDistTestCore/Config => Logging}/LogConfig.cs (72%) create mode 100644 Logging/LogFile.cs create mode 100644 Logging/Logging.csproj create mode 100644 Logging/TestLog.cs rename LongTests/{LongTests.csproj => TestsLong.csproj} (100%) create mode 100644 Utils/NumberSource.cs create mode 100644 Utils/Time.cs create mode 100644 Utils/Utils.csproj diff --git a/CodexDistTestCore/CodexNodeGroup.cs b/CodexDistTestCore/CodexNodeGroup.cs index 1ad5781..12031ac 100644 --- a/CodexDistTestCore/CodexNodeGroup.cs +++ b/CodexDistTestCore/CodexNodeGroup.cs @@ -99,15 +99,5 @@ namespace CodexDistTestCore } } - public class PodInfo - { - public PodInfo(string name, string ip) - { - Name = name; - Ip = ip; - } - public string Name { get; } - public string Ip { get; } - } } diff --git a/CodexDistTestCore/OfflineCodexNodes.cs b/CodexDistTestCore/OfflineCodexNodes.cs index d82ed53..84003a6 100644 --- a/CodexDistTestCore/OfflineCodexNodes.cs +++ b/CodexDistTestCore/OfflineCodexNodes.cs @@ -13,15 +13,6 @@ namespace CodexDistTestCore ICodexNodeGroup BringOnline(); } - public enum CodexLogLevel - { - Trace, - Debug, - Info, - Warn, - Error - } - public enum Location { Unspecified, diff --git a/CodexDistTestCore/TestLog.cs b/CodexDistTestCore/TestLog.cs deleted file mode 100644 index 3947678..0000000 --- a/CodexDistTestCore/TestLog.cs +++ /dev/null @@ -1,144 +0,0 @@ -using CodexDistTestCore.Config; -using NUnit.Framework; - -namespace CodexDistTestCore -{ - public class TestLog - { - private readonly NumberSource subfileNumberSource = new NumberSource(0); - private readonly LogFile file; - private readonly DateTime now; - - public TestLog() - { - now = DateTime.UtcNow; - - var name = GetTestName(); - file = new LogFile(now, name); - - Log($"Begin: {name}"); - } - - public void Log(string message) - { - file.Write(message); - } - - public void Error(string message) - { - Log($"[ERROR] {message}"); - } - - public void EndTest() - { - var result = TestContext.CurrentContext.Result; - - Log($"Finished: {GetTestName()} = {result.Outcome.Status}"); - if (!string.IsNullOrEmpty(result.Message)) - { - Log(result.Message); - Log($"{result.StackTrace}"); - } - - if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed) - { - RenameLogFile(); - } - } - - private void RenameLogFile() - { - file.ConcatToFilename("_FAILED"); - } - - public LogFile CreateSubfile(string ext = "log") - { - return new LogFile(now, $"{GetTestName()}_{subfileNumberSource.GetNextNumber().ToString().PadLeft(6, '0')}", ext); - } - - private static string GetTestName() - { - var test = TestContext.CurrentContext.Test; - var className = test.ClassName!.Substring(test.ClassName.LastIndexOf('.') + 1); - var args = FormatArguments(test); - return $"{className}.{test.MethodName}{args}"; - } - - private static string FormatArguments(TestContext.TestAdapter test) - { - if (test.Arguments == null || !test.Arguments.Any()) return ""; - return $"[{string.Join(',', test.Arguments)}]"; - } - } - - public class LogFile - { - private readonly DateTime now; - private string name; - private readonly string ext; - private readonly string filepath; - - public LogFile(DateTime now, string name, string ext = "log") - { - this.now = now; - this.name = name; - this.ext = ext; - - filepath = Path.Join( - LogConfig.LogRoot, - $"{now.Year}-{Pad(now.Month)}", - Pad(now.Day)); - - Directory.CreateDirectory(filepath); - - GenerateFilename(); - } - - public string FullFilename { get; private set; } = string.Empty; - public string FilenameWithoutPath { get; private set; } = string.Empty; - - public void Write(string message) - { - WriteRaw($"{GetTimestamp()} {message}"); - } - - public void WriteRaw(string message) - { - try - { - File.AppendAllLines(FullFilename, new[] { message }); - } - catch (Exception ex) - { - Console.WriteLine("Writing to log has failed: " + ex); - } - } - - public void ConcatToFilename(string toAdd) - { - var oldFullName = FullFilename; - - name += toAdd; - - GenerateFilename(); - - File.Move(oldFullName, FullFilename); - } - - private static string Pad(int n) - { - return n.ToString().PadLeft(2, '0'); - } - - private static string GetTimestamp() - { - return $"[{DateTime.UtcNow.ToString("u")}]"; - } - - private void GenerateFilename() - { - FilenameWithoutPath = $"{Pad(now.Hour)}-{Pad(now.Minute)}-{Pad(now.Second)}Z_{name.Replace('.', '-')}.{ext}"; - FullFilename = Path.Combine(filepath, FilenameWithoutPath); - } - } -} diff --git a/CodexDistTestCore/Utils.cs b/CodexDistTestCore/Utils.cs index e0149a1..ae3e761 100644 --- a/CodexDistTestCore/Utils.cs +++ b/CodexDistTestCore/Utils.cs @@ -2,15 +2,6 @@ { public static class Utils { - public static void Sleep(TimeSpan span) - { - Thread.Sleep(span); - } - public static T Wait(Task task) - { - task.Wait(); - return task.Result; - } } } diff --git a/DistTestCore/ByteSize.cs b/DistTestCore/ByteSize.cs new file mode 100644 index 0000000..4f13da8 --- /dev/null +++ b/DistTestCore/ByteSize.cs @@ -0,0 +1,57 @@ +namespace DistTestCore +{ + public class ByteSize + { + public ByteSize(long sizeInBytes) + { + SizeInBytes = sizeInBytes; + } + + public long SizeInBytes { get; } + } + + public static class IntExtensions + { + private const long Kilo = 1024; + + public static ByteSize KB(this long i) + { + return new ByteSize(i * Kilo); + } + + public static ByteSize MB(this long i) + { + return (i * Kilo).KB(); + } + + public static ByteSize GB(this long i) + { + return (i * Kilo).MB(); + } + + public static ByteSize TB(this long i) + { + return (i * Kilo).GB(); + } + + public static ByteSize KB(this int i) + { + return Convert.ToInt64(i).KB(); + } + + public static ByteSize MB(this int i) + { + return Convert.ToInt64(i).MB(); + } + + public static ByteSize GB(this int i) + { + return Convert.ToInt64(i).GB(); + } + + public static ByteSize TB(this int i) + { + return Convert.ToInt64(i).TB(); + } + } +} diff --git a/DistTestCore/Codex/CodexAccess.cs b/DistTestCore/Codex/CodexAccess.cs new file mode 100644 index 0000000..a7c826e --- /dev/null +++ b/DistTestCore/Codex/CodexAccess.cs @@ -0,0 +1,43 @@ +using KubernetesWorkflow; + +namespace DistTestCore.Codex +{ + public class CodexAccess + { + private readonly RunningContainer runningContainer; + + public CodexAccess(RunningContainer runningContainer) + { + this.runningContainer = runningContainer; + } + + public CodexDebugResponse GetDebugInfo() + { + var response = Http().HttpGetJson("debug/info"); + //Log($"Got DebugInfo with id: '{response.id}'."); + return response; + } + + private Http Http() + { + var ip = runningContainer.Pod.Cluster.GetIp(); + var port = runningContainer.ServicePorts[0].Number; + return new Http(ip, port, baseUrl: "/api/codex/v1"); + } + } + + public class CodexDebugResponse + { + public string id { get; set; } = string.Empty; + public string[] addrs { get; set; } = new string[0]; + public string repo { get; set; } = string.Empty; + public string spr { get; set; } = string.Empty; + public CodexDebugVersionResponse codex { get; set; } = new(); + } + + public class CodexDebugVersionResponse + { + public string version { get; set; } = string.Empty; + public string revision { get; set; } = string.Empty; + } +} diff --git a/DistTestCore/Codex/CodexContainerRecipe.cs b/DistTestCore/Codex/CodexContainerRecipe.cs new file mode 100644 index 0000000..9c7b87d --- /dev/null +++ b/DistTestCore/Codex/CodexContainerRecipe.cs @@ -0,0 +1,35 @@ +using KubernetesWorkflow; + +namespace DistTestCore.Codex +{ + public class CodexContainerRecipe : ContainerRecipeFactory + { + protected override string Image => "thatbenbierens/nim-codex:sha-b204837"; + + protected override void Initialize(StartupConfig startupConfig) + { + var config = startupConfig.Get(); + + AddExposedPortAndVar("API_PORT"); + AddEnvVar("DATA_DIR", $"datadir{ContainerNumber}"); + AddInternalPortAndVar("DISC_PORT"); + + var listenPort = AddInternalPort(); + AddEnvVar("LISTEN_ADDRS", $"/ip4/0.0.0.0/tcp/{listenPort.Number}"); + + if (config.LogLevel != null) + { + AddEnvVar("LOG_LEVEL", config.LogLevel.ToString()!.ToUpperInvariant()); + } + if (config.StorageQuota != null) + { + AddEnvVar("STORAGE_QUOTA", config.StorageQuota.SizeInBytes.ToString()!); + } + if (config.MetricsEnabled) + { + AddEnvVar("METRICS_ADDR", "0.0.0.0"); + AddInternalPortAndVar("METRICS_PORT"); + } + } + } +} diff --git a/DistTestCore/Codex/CodexLogLevel.cs b/DistTestCore/Codex/CodexLogLevel.cs new file mode 100644 index 0000000..cde0eb7 --- /dev/null +++ b/DistTestCore/Codex/CodexLogLevel.cs @@ -0,0 +1,11 @@ +namespace DistTestCore.Codex +{ + public enum CodexLogLevel + { + Trace, + Debug, + Info, + Warn, + Error + } +} diff --git a/DistTestCore/Codex/CodexStartupConfig.cs b/DistTestCore/Codex/CodexStartupConfig.cs new file mode 100644 index 0000000..fdbffcf --- /dev/null +++ b/DistTestCore/Codex/CodexStartupConfig.cs @@ -0,0 +1,12 @@ +using KubernetesWorkflow; + +namespace DistTestCore.Codex +{ + public class CodexStartupConfig + { + public Location Location { get; set; } + public CodexLogLevel? LogLevel { get; set; } + public ByteSize? StorageQuota { get; set; } + public bool MetricsEnabled { get; set; } + } +} diff --git a/DistTestCore/DistTestCore.csproj b/DistTestCore/DistTestCore.csproj new file mode 100644 index 0000000..ae8e964 --- /dev/null +++ b/DistTestCore/DistTestCore.csproj @@ -0,0 +1,20 @@ + + + + net6.0 + DistTestCore + enable + enable + + + + + + + + + + + + + diff --git a/DistTestCore/Http.cs b/DistTestCore/Http.cs new file mode 100644 index 0000000..365920b --- /dev/null +++ b/DistTestCore/Http.cs @@ -0,0 +1,100 @@ +using Newtonsoft.Json; +using NUnit.Framework; +using System.Net.Http.Headers; + +namespace DistTestCore +{ + public class Http + { + private readonly string ip; + private readonly int port; + private readonly string baseUrl; + + public Http(string ip, int port, string baseUrl) + { + this.ip = ip; + this.port = port; + this.baseUrl = baseUrl; + + if (!this.baseUrl.StartsWith("/")) this.baseUrl = "/" + this.baseUrl; + if (!this.baseUrl.EndsWith("/")) this.baseUrl += "/"; + } + + public string HttpGetString(string route) + { + return Retry(() => + { + using var client = GetClient(); + var url = GetUrl() + route; + var result = Utils.Wait(client.GetAsync(url)); + return Utils.Wait(result.Content.ReadAsStringAsync()); + }); + } + + public T HttpGetJson(string route) + { + return JsonConvert.DeserializeObject(HttpGetString(route))!; + } + + public string HttpPostStream(string route, Stream stream) + { + return Retry(() => + { + using var client = GetClient(); + var url = GetUrl() + route; + + var content = new StreamContent(stream); + content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream"); + var response = Utils.Wait(client.PostAsync(url, content)); + + return Utils.Wait(response.Content.ReadAsStringAsync()); + }); + } + + public Stream HttpGetStream(string route) + { + return Retry(() => + { + var client = GetClient(); + var url = GetUrl() + route; + + return Utils.Wait(client.GetStreamAsync(url)); + }); + } + + private string GetUrl() + { + return $"http://{ip}:{port}{baseUrl}"; + } + + private static T Retry(Func operation) + { + var retryCounter = 0; + + while (true) + { + try + { + return operation(); + } + catch (Exception exception) + { + Timing.HttpCallRetryDelay(); + retryCounter++; + if (retryCounter > Timing.HttpCallRetryCount()) + { + Assert.Fail(exception.Message); + throw; + } + } + } + } + + private static HttpClient GetClient() + { + var client = new HttpClient(); + client.Timeout = Timing.HttpCallTimeout(); + return client; + } + } +} diff --git a/DistTestCore/TestLifecycle.cs b/DistTestCore/TestLifecycle.cs new file mode 100644 index 0000000..e3d2186 --- /dev/null +++ b/DistTestCore/TestLifecycle.cs @@ -0,0 +1,31 @@ +using DistTestCore.Codex; +using KubernetesWorkflow; + +namespace DistTestCore +{ + public class TestLifecycle + { + private readonly WorkflowCreator workflowCreator = new WorkflowCreator(); + + public void SetUpCodexNodes() + { + var config = new CodexStartupConfig() + { + StorageQuota = 10.MB(), + Location = Location.Unspecified, + LogLevel = CodexLogLevel.Error, + MetricsEnabled = false, + }; + + var workflow = workflowCreator.CreateWorkflow(); + var startupConfig = new StartupConfig(); + startupConfig.Add(config); + var containers = workflow.Start(3, new CodexContainerRecipe(), startupConfig); + + foreach (var c in containers.Containers) + { + var access = new CodexAccess(c); + } + } + } +} diff --git a/KubernetesWorkflow/ContainerRecipe.cs b/KubernetesWorkflow/ContainerRecipe.cs new file mode 100644 index 0000000..db8420e --- /dev/null +++ b/KubernetesWorkflow/ContainerRecipe.cs @@ -0,0 +1,42 @@ +namespace KubernetesWorkflow +{ + public class ContainerRecipe + { + public ContainerRecipe(string name, string image, Port[] exposedPorts, Port[] internalPorts, EnvVar[] envVars) + { + Name = name; + Image = image; + ExposedPorts = exposedPorts; + InternalPorts = internalPorts; + EnvVars = envVars; + } + + public string Name { get; } + public string Image { get; } + public Port[] ExposedPorts { get; } + public Port[] InternalPorts { get; } + public EnvVar[] EnvVars { get; } + } + + public class Port + { + public Port(int number) + { + Number = number; + } + + public int Number { get; } + } + + public class EnvVar + { + public EnvVar(string name, string value) + { + Name = name; + Value = value; + } + + public string Name { get; } + public string Value { get; } + } +} diff --git a/KubernetesWorkflow/ContainerRecipeFactory.cs b/KubernetesWorkflow/ContainerRecipeFactory.cs new file mode 100644 index 0000000..d4dfb0a --- /dev/null +++ b/KubernetesWorkflow/ContainerRecipeFactory.cs @@ -0,0 +1,60 @@ +namespace KubernetesWorkflow +{ + public abstract class ContainerRecipeFactory + { + private readonly List exposedPorts = new List(); + private readonly List internalPorts = new List(); + private readonly List envVars = new List(); + private RecipeComponentFactory factory = null!; + + public ContainerRecipe CreateRecipe(int containerNumber, RecipeComponentFactory factory, StartupConfig config) + { + this.factory = factory; + ContainerNumber = containerNumber; + + Initialize(config); + + var name = $"ctnr{containerNumber}"; + + return new ContainerRecipe(name, Image, exposedPorts.ToArray(), internalPorts.ToArray(), envVars.ToArray()); + } + + protected abstract string Image { get; } + protected int ContainerNumber { get; private set; } = 0; + protected abstract void Initialize(StartupConfig config); + + protected Port AddExposedPort() + { + var p = factory.CreatePort(); + exposedPorts.Add(p); + return p; + } + + protected Port AddInternalPort() + { + var p = factory.CreatePort(); + internalPorts.Add(p); + return p; + } + + protected void AddExposedPortAndVar(string name) + { + AddEnvVar(name, AddExposedPort()); + } + + protected void AddInternalPortAndVar(string name) + { + AddEnvVar(name, AddInternalPort()); + } + + protected void AddEnvVar(string name, string value) + { + envVars.Add(factory.CreateEnvVar(name, value)); + } + + protected void AddEnvVar(string name, Port value) + { + envVars.Add(factory.CreateEnvVar(name, value.Number)); + } + } +} diff --git a/KubernetesWorkflow/K8sCluster.cs b/KubernetesWorkflow/K8sCluster.cs new file mode 100644 index 0000000..d325a75 --- /dev/null +++ b/KubernetesWorkflow/K8sCluster.cs @@ -0,0 +1,40 @@ +using k8s; + +namespace KubernetesWorkflow +{ + public class K8sCluster + { + public const string K8sNamespace = "codex-test-namespace"; + private const string KubeConfigFile = "C:\\kube\\config"; + private readonly Dictionary K8sNodeLocationMap = new Dictionary + { + { Location.BensLaptop, "worker01" }, + { Location.BensOldGamingMachine, "worker02" }, + }; + + private KubernetesClientConfiguration? config; + + public KubernetesClientConfiguration GetK8sClientConfig() + { + if (config != null) return config; + //config = KubernetesClientConfiguration.BuildConfigFromConfigFile(KubeConfigFile); + config = KubernetesClientConfiguration.BuildDefaultConfig(); + return config; + } + + public string GetIp() + { + var c = GetK8sClientConfig(); + + var host = c.Host.Replace("https://", ""); + + return host.Substring(0, host.IndexOf(':')); + } + + public string GetNodeLabelForLocation(Location location) + { + if (location == Location.Unspecified) return string.Empty; + return K8sNodeLocationMap[location]; + } + } +} diff --git a/KubernetesWorkflow/K8sController.cs b/KubernetesWorkflow/K8sController.cs new file mode 100644 index 0000000..3f6bef6 --- /dev/null +++ b/KubernetesWorkflow/K8sController.cs @@ -0,0 +1,25 @@ +namespace KubernetesWorkflow +{ + public class K8sController + { + private readonly K8sCluster cluster; + + public K8sController(K8sCluster cluster) + { + this.cluster = cluster; + } + + public RunningPod BringOnline(ContainerRecipe[] containerRecipes) + { + // Ensure namespace + // create deployment + // create service if necessary + // wait until deployment online + // fetch pod info + + // for each container, there is now an array of service ports available. + + return null!; + } + } +} diff --git a/KubernetesWorkflow/KubernetesWorkflow.csproj b/KubernetesWorkflow/KubernetesWorkflow.csproj new file mode 100644 index 0000000..557b45c --- /dev/null +++ b/KubernetesWorkflow/KubernetesWorkflow.csproj @@ -0,0 +1,14 @@ + + + + net6.0 + KubernetesWorkflow + enable + enable + + + + + + + diff --git a/KubernetesWorkflow/Location.cs b/KubernetesWorkflow/Location.cs new file mode 100644 index 0000000..3b01284 --- /dev/null +++ b/KubernetesWorkflow/Location.cs @@ -0,0 +1,9 @@ +namespace KubernetesWorkflow +{ + public enum Location + { + Unspecified, + BensLaptop, + BensOldGamingMachine + } +} diff --git a/KubernetesWorkflow/RecipeComponentFactory.cs b/KubernetesWorkflow/RecipeComponentFactory.cs new file mode 100644 index 0000000..6a40cc9 --- /dev/null +++ b/KubernetesWorkflow/RecipeComponentFactory.cs @@ -0,0 +1,24 @@ +using System.Globalization; + +namespace KubernetesWorkflow +{ + public class RecipeComponentFactory + { + private NumberSource portNumberSource = new NumberSource(8080); + + public Port CreatePort() + { + return new Port(portNumberSource.GetNextNumber()); + } + + public EnvVar CreateEnvVar(string name, int value) + { + return CreateEnvVar(name, value.ToString(CultureInfo.InvariantCulture)); + } + + public EnvVar CreateEnvVar(string name, string value) + { + return new EnvVar(name, value); + } + } +} diff --git a/KubernetesWorkflow/RunningContainers.cs b/KubernetesWorkflow/RunningContainers.cs new file mode 100644 index 0000000..49fc65f --- /dev/null +++ b/KubernetesWorkflow/RunningContainers.cs @@ -0,0 +1,30 @@ +namespace KubernetesWorkflow +{ + public class RunningContainers + { + public RunningContainers(StartupConfig startupConfig, RunningPod runningPod, RunningContainer[] containers) + { + StartupConfig = startupConfig; + RunningPod = runningPod; + Containers = containers; + } + + public StartupConfig StartupConfig { get; } + public RunningPod RunningPod { get; } + public RunningContainer[] Containers { get; } + } + + public class RunningContainer + { + public RunningContainer(RunningPod pod, ContainerRecipe recipe, Port[] servicePorts) + { + Pod = pod; + Recipe = recipe; + ServicePorts = servicePorts; + } + + public RunningPod Pod { get; } + public ContainerRecipe Recipe { get; } + public Port[] ServicePorts { get; } + } +} diff --git a/KubernetesWorkflow/RunningPod.cs b/KubernetesWorkflow/RunningPod.cs new file mode 100644 index 0000000..63378e2 --- /dev/null +++ b/KubernetesWorkflow/RunningPod.cs @@ -0,0 +1,24 @@ +namespace KubernetesWorkflow +{ + public class RunningPod + { + private readonly Dictionary servicePortMap; + + public RunningPod(K8sCluster cluster, string name, string ip, Dictionary servicePortMap) + { + Cluster = cluster; + Name = name; + Ip = ip; + this.servicePortMap = servicePortMap; + } + + public K8sCluster Cluster { get; } + public string Name { get; } + public string Ip { get; } + + public Port[] GetServicePortsForContainerRecipe(ContainerRecipe containerRecipe) + { + return servicePortMap[containerRecipe]; + } + } +} diff --git a/KubernetesWorkflow/StartupConfig.cs b/KubernetesWorkflow/StartupConfig.cs new file mode 100644 index 0000000..7c13347 --- /dev/null +++ b/KubernetesWorkflow/StartupConfig.cs @@ -0,0 +1,18 @@ +namespace KubernetesWorkflow +{ + public class StartupConfig + { + private readonly List configs = new List(); + + public void Add(object config) + { + configs.Add(config); + } + + public T Get() + { + var match = configs.Single(c => c.GetType() == typeof(T)); + return (T)match; + } + } +} diff --git a/KubernetesWorkflow/StartupWorkflow.cs b/KubernetesWorkflow/StartupWorkflow.cs new file mode 100644 index 0000000..eb714f7 --- /dev/null +++ b/KubernetesWorkflow/StartupWorkflow.cs @@ -0,0 +1,40 @@ +namespace KubernetesWorkflow +{ + public class StartupWorkflow + { + private readonly NumberSource containerNumberSource; + private readonly K8sController k8SController; + private readonly RecipeComponentFactory componentFactory = new RecipeComponentFactory(); + + public StartupWorkflow(NumberSource containerNumberSource, K8sController k8SController) + { + this.containerNumberSource = containerNumberSource; + this.k8SController = k8SController; + } + + public RunningContainers Start(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig) + { + var recipes = CreateRecipes(numberOfContainers, recipeFactory, startupConfig); + + var runningPod = k8SController.BringOnline(recipes); + + return new RunningContainers(startupConfig, runningPod, CreateContainers(runningPod, recipes)); + } + + private static RunningContainer[] CreateContainers(RunningPod runningPod, ContainerRecipe[] recipes) + { + return recipes.Select(r => new RunningContainer(runningPod, r, runningPod.GetServicePortsForContainerRecipe(r))).ToArray(); + } + + private ContainerRecipe[] CreateRecipes(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig) + { + var result = new List(); + for (var i = 0; i < numberOfContainers; i++) + { + result.Add(recipeFactory.CreateRecipe(containerNumberSource.GetNextNumber(), componentFactory, startupConfig)); + } + + return result.ToArray(); + } + } +} diff --git a/KubernetesWorkflow/WorkflowCreator.cs b/KubernetesWorkflow/WorkflowCreator.cs new file mode 100644 index 0000000..25f5439 --- /dev/null +++ b/KubernetesWorkflow/WorkflowCreator.cs @@ -0,0 +1,13 @@ +namespace KubernetesWorkflow +{ + public class WorkflowCreator + { + private readonly NumberSource containerNumberSource = new NumberSource(0); + private readonly K8sController controller = new K8sController(new K8sCluster()); + + public StartupWorkflow CreateWorkflow() + { + return new StartupWorkflow(containerNumberSource, controller); + } + } +} diff --git a/CodexDistTestCore/Config/LogConfig.cs b/Logging/LogConfig.cs similarity index 72% rename from CodexDistTestCore/Config/LogConfig.cs rename to Logging/LogConfig.cs index a14470e..c15b1fe 100644 --- a/CodexDistTestCore/Config/LogConfig.cs +++ b/Logging/LogConfig.cs @@ -1,4 +1,4 @@ -namespace CodexDistTestCore.Config +namespace Logging { public class LogConfig { diff --git a/Logging/LogFile.cs b/Logging/LogFile.cs new file mode 100644 index 0000000..a43186c --- /dev/null +++ b/Logging/LogFile.cs @@ -0,0 +1,73 @@ +namespace Logging +{ + public class LogFile + { + private readonly DateTime now; + private string name; + private readonly string ext; + private readonly string filepath; + + public LogFile(DateTime now, string name, string ext = "log") + { + this.now = now; + this.name = name; + this.ext = ext; + + filepath = Path.Join( + LogConfig.LogRoot, + $"{now.Year}-{Pad(now.Month)}", + Pad(now.Day)); + + Directory.CreateDirectory(filepath); + + GenerateFilename(); + } + + public string FullFilename { get; private set; } = string.Empty; + public string FilenameWithoutPath { get; private set; } = string.Empty; + + public void Write(string message) + { + WriteRaw($"{GetTimestamp()} {message}"); + } + + public void WriteRaw(string message) + { + try + { + File.AppendAllLines(FullFilename, new[] { message }); + } + catch (Exception ex) + { + Console.WriteLine("Writing to log has failed: " + ex); + } + } + + public void ConcatToFilename(string toAdd) + { + var oldFullName = FullFilename; + + name += toAdd; + + GenerateFilename(); + + File.Move(oldFullName, FullFilename); + } + + private static string Pad(int n) + { + return n.ToString().PadLeft(2, '0'); + } + + private static string GetTimestamp() + { + return $"[{DateTime.UtcNow.ToString("u")}]"; + } + + private void GenerateFilename() + { + FilenameWithoutPath = $"{Pad(now.Hour)}-{Pad(now.Minute)}-{Pad(now.Second)}Z_{name.Replace('.', '-')}.{ext}"; + FullFilename = Path.Combine(filepath, FilenameWithoutPath); + } + } +} diff --git a/Logging/Logging.csproj b/Logging/Logging.csproj new file mode 100644 index 0000000..fad5ec4 --- /dev/null +++ b/Logging/Logging.csproj @@ -0,0 +1,19 @@ + + + + net6.0 + Logging + enable + enable + + + + + + + + + + + + diff --git a/Logging/TestLog.cs b/Logging/TestLog.cs new file mode 100644 index 0000000..83b6cb1 --- /dev/null +++ b/Logging/TestLog.cs @@ -0,0 +1,74 @@ +using NUnit.Framework; +using Utils; + +namespace Logging +{ + public class TestLog + { + private readonly NumberSource subfileNumberSource = new NumberSource(0); + private readonly LogFile file; + private readonly DateTime now; + + public TestLog() + { + now = DateTime.UtcNow; + + var name = GetTestName(); + file = new LogFile(now, name); + + Log($"Begin: {name}"); + } + + public void Log(string message) + { + file.Write(message); + } + + public void Error(string message) + { + Log($"[ERROR] {message}"); + } + + public void EndTest() + { + var result = TestContext.CurrentContext.Result; + + Log($"Finished: {GetTestName()} = {result.Outcome.Status}"); + if (!string.IsNullOrEmpty(result.Message)) + { + Log(result.Message); + Log($"{result.StackTrace}"); + } + + if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed) + { + RenameLogFile(); + } + } + + private void RenameLogFile() + { + file.ConcatToFilename("_FAILED"); + } + + public LogFile CreateSubfile(string ext = "log") + { + return new LogFile(now, $"{GetTestName()}_{subfileNumberSource.GetNextNumber().ToString().PadLeft(6, '0')}", ext); + } + + private static string GetTestName() + { + var test = TestContext.CurrentContext.Test; + var className = test.ClassName!.Substring(test.ClassName.LastIndexOf('.') + 1); + var args = FormatArguments(test); + return $"{className}.{test.MethodName}{args}"; + } + + private static string FormatArguments(TestContext.TestAdapter test) + { + if (test.Arguments == null || !test.Arguments.Any()) return ""; + return $"[{string.Join(',', test.Arguments)}]"; + } + } + +} diff --git a/LongTests/BasicTests/LargeFileTests.cs b/LongTests/BasicTests/LargeFileTests.cs index b8ab20e..a7f0c9b 100644 --- a/LongTests/BasicTests/LargeFileTests.cs +++ b/LongTests/BasicTests/LargeFileTests.cs @@ -1,7 +1,7 @@ using CodexDistTestCore; using NUnit.Framework; -namespace LongTests.BasicTests +namespace TestsLong.BasicTests { [TestFixture] public class LargeFileTests : DistTest diff --git a/LongTests/BasicTests/TestInfraTests.cs b/LongTests/BasicTests/TestInfraTests.cs index 705b4fc..c39069c 100644 --- a/LongTests/BasicTests/TestInfraTests.cs +++ b/LongTests/BasicTests/TestInfraTests.cs @@ -1,7 +1,7 @@ using CodexDistTestCore; using NUnit.Framework; -namespace LongTests.BasicTests +namespace TestsLong.BasicTests { public class TestInfraTests : DistTest { diff --git a/LongTests/LongTests.csproj b/LongTests/TestsLong.csproj similarity index 100% rename from LongTests/LongTests.csproj rename to LongTests/TestsLong.csproj diff --git a/Utils/NumberSource.cs b/Utils/NumberSource.cs new file mode 100644 index 0000000..2c7266f --- /dev/null +++ b/Utils/NumberSource.cs @@ -0,0 +1,19 @@ +namespace Utils +{ + public class NumberSource + { + private int number; + + public NumberSource(int start) + { + number = start; + } + + public int GetNextNumber() + { + var n = number; + number++; + return n; + } + } +} diff --git a/Utils/Time.cs b/Utils/Time.cs new file mode 100644 index 0000000..98c5979 --- /dev/null +++ b/Utils/Time.cs @@ -0,0 +1,16 @@ +namespace Utils +{ + public static class Time + { + public static void Sleep(TimeSpan span) + { + Thread.Sleep(span); + } + + public static T Wait(Task task) + { + task.Wait(); + return task.Result; + } + } +} diff --git a/Utils/Utils.csproj b/Utils/Utils.csproj new file mode 100644 index 0000000..42c2303 --- /dev/null +++ b/Utils/Utils.csproj @@ -0,0 +1,10 @@ + + + + net6.0 + Utils + enable + enable + + + diff --git a/cs-codex-dist-testing.sln b/cs-codex-dist-testing.sln index 9a7a7ad..43b21e9 100644 --- a/cs-codex-dist-testing.sln +++ b/cs-codex-dist-testing.sln @@ -3,12 +3,18 @@ Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 17 VisualStudioVersion = 17.4.33213.308 MinimumVisualStudioVersion = 10.0.40219.1 -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Tests", "Tests\Tests.csproj", "{57F57B85-A537-4D3A-B7AE-B72C66B74AAB}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tests", "Tests\Tests.csproj", "{57F57B85-A537-4D3A-B7AE-B72C66B74AAB}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LongTests", "LongTests\LongTests.csproj", "{AFCE270E-F844-4A7C-9006-69AE622BB1F4}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TestsLong", "LongTests\TestsLong.csproj", "{AFCE270E-F844-4A7C-9006-69AE622BB1F4}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CodexDistTestCore", "CodexDistTestCore\CodexDistTestCore.csproj", "{19306DE1-CEE5-4F7B-AA5D-FD91926D853D}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DistTestCore", "DistTestCore\DistTestCore.csproj", "{47F31305-6E68-4827-8E39-7B41DAA1CE7A}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "KubernetesWorkflow", "KubernetesWorkflow\KubernetesWorkflow.csproj", "{359123AA-3D9B-4442-80F4-19E32E3EC9EA}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Utils", "Utils\Utils.csproj", "{957DE3B8-9571-450A-8609-B267DCA8727C}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -27,6 +33,18 @@ Global {19306DE1-CEE5-4F7B-AA5D-FD91926D853D}.Debug|Any CPU.Build.0 = Debug|Any CPU {19306DE1-CEE5-4F7B-AA5D-FD91926D853D}.Release|Any CPU.ActiveCfg = Release|Any CPU {19306DE1-CEE5-4F7B-AA5D-FD91926D853D}.Release|Any CPU.Build.0 = Release|Any CPU + {47F31305-6E68-4827-8E39-7B41DAA1CE7A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {47F31305-6E68-4827-8E39-7B41DAA1CE7A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {47F31305-6E68-4827-8E39-7B41DAA1CE7A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {47F31305-6E68-4827-8E39-7B41DAA1CE7A}.Release|Any CPU.Build.0 = Release|Any CPU + {359123AA-3D9B-4442-80F4-19E32E3EC9EA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {359123AA-3D9B-4442-80F4-19E32E3EC9EA}.Debug|Any CPU.Build.0 = Debug|Any CPU + {359123AA-3D9B-4442-80F4-19E32E3EC9EA}.Release|Any CPU.ActiveCfg = Release|Any CPU + {359123AA-3D9B-4442-80F4-19E32E3EC9EA}.Release|Any CPU.Build.0 = Release|Any CPU + {957DE3B8-9571-450A-8609-B267DCA8727C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {957DE3B8-9571-450A-8609-B267DCA8727C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {957DE3B8-9571-450A-8609-B267DCA8727C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {957DE3B8-9571-450A-8609-B267DCA8727C}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE From 2bcf5127374ca29ec4ecc3cba4b23990410f6cd9 Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 12 Apr 2023 15:11:36 +0200 Subject: [PATCH 17/46] Implements KubernetesWorkflow assembly. --- CodexDistTestCore/K8sOperations.cs | 38 -- KubernetesWorkflow/ContainerRecipe.cs | 7 +- KubernetesWorkflow/ContainerRecipeFactory.cs | 4 +- KubernetesWorkflow/K8sCluster.cs | 11 + KubernetesWorkflow/K8sController.cs | 405 ++++++++++++++++++- KubernetesWorkflow/KnownK8sPods.cs | 17 + KubernetesWorkflow/KubernetesWorkflow.csproj | 4 + KubernetesWorkflow/RecipeComponentFactory.cs | 1 + KubernetesWorkflow/StartupWorkflow.cs | 49 ++- KubernetesWorkflow/WorkflowCreator.cs | 14 +- KubernetesWorkflow/WorkflowNumberSource.cs | 28 ++ cs-codex-dist-testing.sln | 6 + 12 files changed, 516 insertions(+), 68 deletions(-) create mode 100644 KubernetesWorkflow/KnownK8sPods.cs create mode 100644 KubernetesWorkflow/WorkflowNumberSource.cs diff --git a/CodexDistTestCore/K8sOperations.cs b/CodexDistTestCore/K8sOperations.cs index 580829e..73f04b0 100644 --- a/CodexDistTestCore/K8sOperations.cs +++ b/CodexDistTestCore/K8sOperations.cs @@ -341,44 +341,6 @@ namespace CodexDistTestCore #endregion - #region Namespace management - - private void EnsureTestNamespace() - { - if (IsTestNamespaceOnline()) return; - - var namespaceSpec = new V1Namespace - { - ApiVersion = "v1", - Metadata = new V1ObjectMeta - { - Name = K8sNamespace, - Labels = new Dictionary { { "name", K8sNamespace } } - } - }; - client.CreateNamespace(namespaceSpec); - } - - private void DeleteNamespace() - { - if (IsTestNamespaceOnline()) - { - client.DeleteNamespace(K8sNamespace, null, null, gracePeriodSeconds: 0); - } - } - - private string K8sNamespace - { - get { return K8sCluster.K8sNamespace; } - } - - #endregion - - private bool IsTestNamespaceOnline() - { - return client.ListNamespace().Items.Any(n => n.Metadata.Name == K8sNamespace); - } - private class CommandRunner { private readonly Kubernetes client; diff --git a/KubernetesWorkflow/ContainerRecipe.cs b/KubernetesWorkflow/ContainerRecipe.cs index db8420e..7e8d90e 100644 --- a/KubernetesWorkflow/ContainerRecipe.cs +++ b/KubernetesWorkflow/ContainerRecipe.cs @@ -2,16 +2,17 @@ { public class ContainerRecipe { - public ContainerRecipe(string name, string image, Port[] exposedPorts, Port[] internalPorts, EnvVar[] envVars) + public ContainerRecipe(int number, string image, Port[] exposedPorts, Port[] internalPorts, EnvVar[] envVars) { - Name = name; + Number = number; Image = image; ExposedPorts = exposedPorts; InternalPorts = internalPorts; EnvVars = envVars; } - public string Name { get; } + public string Name { get { return $"ctnr{Number}"; } } + public int Number { get; } public string Image { get; } public Port[] ExposedPorts { get; } public Port[] InternalPorts { get; } diff --git a/KubernetesWorkflow/ContainerRecipeFactory.cs b/KubernetesWorkflow/ContainerRecipeFactory.cs index d4dfb0a..fc06f33 100644 --- a/KubernetesWorkflow/ContainerRecipeFactory.cs +++ b/KubernetesWorkflow/ContainerRecipeFactory.cs @@ -14,9 +14,7 @@ Initialize(config); - var name = $"ctnr{containerNumber}"; - - return new ContainerRecipe(name, Image, exposedPorts.ToArray(), internalPorts.ToArray(), envVars.ToArray()); + return new ContainerRecipe(containerNumber, Image, exposedPorts.ToArray(), internalPorts.ToArray(), envVars.ToArray()); } protected abstract string Image { get; } diff --git a/KubernetesWorkflow/K8sCluster.cs b/KubernetesWorkflow/K8sCluster.cs index d325a75..7d6e8eb 100644 --- a/KubernetesWorkflow/K8sCluster.cs +++ b/KubernetesWorkflow/K8sCluster.cs @@ -36,5 +36,16 @@ namespace KubernetesWorkflow if (location == Location.Unspecified) return string.Empty; return K8sNodeLocationMap[location]; } + + // make configurable from test env! + public TimeSpan K8sOperationTimeout() + { + return TimeSpan.FromMinutes(5); + } + + public TimeSpan WaitForK8sServiceDelay() + { + return TimeSpan.FromSeconds(5); + } } } diff --git a/KubernetesWorkflow/K8sController.cs b/KubernetesWorkflow/K8sController.cs index 3f6bef6..4049607 100644 --- a/KubernetesWorkflow/K8sController.cs +++ b/KubernetesWorkflow/K8sController.cs @@ -1,25 +1,410 @@ -namespace KubernetesWorkflow +using k8s; +using k8s.Models; + +namespace KubernetesWorkflow { public class K8sController { private readonly K8sCluster cluster; + private readonly KnownK8sPods knownPods; + private readonly WorkflowNumberSource workflowNumberSource; + private readonly Kubernetes client; - public K8sController(K8sCluster cluster) + public K8sController(K8sCluster cluster, KnownK8sPods knownPods, WorkflowNumberSource workflowNumberSource) { this.cluster = cluster; + this.knownPods = knownPods; + this.workflowNumberSource = workflowNumberSource; + + client = new Kubernetes(cluster.GetK8sClientConfig()); } - public RunningPod BringOnline(ContainerRecipe[] containerRecipes) + public void Dispose() { - // Ensure namespace - // create deployment - // create service if necessary - // wait until deployment online - // fetch pod info + client.Dispose(); + } + + public RunningPod BringOnline(ContainerRecipe[] containerRecipes, Location location) + { + EnsureTestNamespace(); - // for each container, there is now an array of service ports available. + CreateDeployment(containerRecipes, location); + var servicePortsMap = CreateService(containerRecipes); + var (podName, podIp) = FetchNewPod(); - return null!; + return new RunningPod(cluster, podName, podIp, servicePortsMap); + } + + public void DeleteAllResources() + { + DeleteNamespace(); + + WaitUntilNamespaceDeleted(); + } + + #region Namespace management + + private void EnsureTestNamespace() + { + if (IsTestNamespaceOnline()) return; + + var namespaceSpec = new V1Namespace + { + ApiVersion = "v1", + Metadata = new V1ObjectMeta + { + Name = K8sNamespace, + Labels = new Dictionary { { "name", K8sNamespace } } + } + }; + client.CreateNamespace(namespaceSpec); + WaitUntilNamespaceCreated(); + } + + private void DeleteNamespace() + { + if (IsTestNamespaceOnline()) + { + client.DeleteNamespace(K8sNamespace, null, null, gracePeriodSeconds: 0); + } + } + + private string K8sNamespace + { + get { return K8sCluster.K8sNamespace; } + } + + private bool IsTestNamespaceOnline() + { + return client.ListNamespace().Items.Any(n => n.Metadata.Name == K8sNamespace); + } + + #endregion + + #region Deployment management + + private void CreateDeployment(ContainerRecipe[] containerRecipes, Location location) + { + var deploymentSpec = new V1Deployment + { + ApiVersion = "apps/v1", + Metadata = CreateDeploymentMetadata(), + Spec = new V1DeploymentSpec + { + Replicas = 1, + Selector = new V1LabelSelector + { + MatchLabels = GetSelector() + }, + Template = new V1PodTemplateSpec + { + Metadata = new V1ObjectMeta + { + Labels = GetSelector() + }, + Spec = new V1PodSpec + { + NodeSelector = CreateNodeSelector(location), + Containers = CreateDeploymentContainers(containerRecipes) + } + } + } + }; + + client.CreateNamespacedDeployment(deploymentSpec, K8sNamespace); + WaitUntilDeploymentCreated(deploymentSpec); + } + + private IDictionary CreateNodeSelector(Location location) + { + if (location == Location.Unspecified) return new Dictionary(); + + return new Dictionary + { + { "codex-test-location", cluster.GetNodeLabelForLocation(location) } + }; + } + + private IDictionary GetSelector() + { + return new Dictionary { { "codex-test-node", "dist-test-" + workflowNumberSource.WorkflowNumber } }; + } + + private V1ObjectMeta CreateDeploymentMetadata() + { + return new V1ObjectMeta + { + Name = "deploy-" + workflowNumberSource.WorkflowNumber, + NamespaceProperty = K8sCluster.K8sNamespace + }; + } + + private List CreateDeploymentContainers(ContainerRecipe[] containerRecipes) + { + return containerRecipes.Select(r => CreateDeploymentContainer(r)).ToList(); + } + + private V1Container CreateDeploymentContainer(ContainerRecipe recipe) + { + return new V1Container + { + Name = recipe.Name, + Image = recipe.Image, + Ports = CreateContainerPorts(recipe), + Env = CreateEnv(recipe) + }; + } + + private List CreateEnv(ContainerRecipe recipe) + { + return recipe.EnvVars.Select(CreateEnvVar).ToList(); + } + + private V1EnvVar CreateEnvVar(EnvVar envVar) + { + return new V1EnvVar + { + Name = envVar.Name, + Value = envVar.Value, + }; + } + + private List CreateContainerPorts(ContainerRecipe recipe) + { + var exposedPorts = recipe.ExposedPorts.Select(p => CreateContainerPort(recipe, p)); + var internalPorts = recipe.InternalPorts.Select(p => CreateContainerPort(recipe, p)); + return exposedPorts.Concat(internalPorts).ToList(); + } + + private V1ContainerPort CreateContainerPort(ContainerRecipe recipe, Port port) + { + return new V1ContainerPort + { + Name = GetNameForPort(recipe, port), + ContainerPort = port.Number + }; + } + + private string GetNameForPort(ContainerRecipe recipe, Port port) + { + return $"P{workflowNumberSource.WorkflowNumber}-{recipe.Number}-{port.Number}"; + } + + + //private void DeleteDeployment(CodexNodeGroup group) + //{ + // if (group.Deployment == null) return; + // client.DeleteNamespacedDeployment(group.Deployment.Name(), K8sNamespace); + // group.Deployment = null; + //} + + //private void CreatePrometheusDeployment(K8sPrometheusSpecs spec) + //{ + // client.CreateNamespacedDeployment(spec.CreatePrometheusDeployment(), K8sNamespace); + //} + + //private void CreateGethBootstrapDeployment(K8sGethBoostrapSpecs spec) + //{ + // client.CreateNamespacedDeployment(spec.CreateGethBootstrapDeployment(), K8sNamespace); + //} + + //private void CreateGethCompanionDeployment(GethBootstrapInfo info, GethCompanionGroup group) + //{ + // client.CreateNamespacedDeployment(info.Spec.CreateGethCompanionDeployment(group, info), K8sNamespace); + //} + + #endregion + + #region Service management + + private Dictionary CreateService(ContainerRecipe[] containerRecipes) + { + var result = new Dictionary(); + + var ports = CreateServicePorts(result, containerRecipes); + + if (!ports.Any()) + { + // None of these container-recipes wish to expose anything via a serice port. + // So, we don't have to create a service. + return result; + } + + var serviceSpec = new V1Service + { + ApiVersion = "v1", + Metadata = CreateServiceMetadata(), + Spec = new V1ServiceSpec + { + Type = "NodePort", + Selector = GetSelector(), + Ports = ports + } + }; + + client.CreateNamespacedService(serviceSpec, K8sNamespace); + return result; + } + + private V1ObjectMeta CreateServiceMetadata() + { + return new V1ObjectMeta + { + Name = "deploy-" + workflowNumberSource.WorkflowNumber, + NamespaceProperty = K8sCluster.K8sNamespace + }; + } + + private List CreateServicePorts(Dictionary servicePorts, ContainerRecipe[] recipes) + { + var result = new List(); + foreach (var recipe in recipes) + { + result.AddRange(CreateServicePorts(servicePorts, recipe)); + } + return result; + } + + private List CreateServicePorts(Dictionary servicePorts, ContainerRecipe recipe) + { + var result = new List(); + var usedPorts = new List(); + foreach (var port in recipe.ExposedPorts) + { + var servicePort = workflowNumberSource.GetServicePort(); + usedPorts.Add(new Port(servicePort)); + + result.Add(new V1ServicePort + { + Name = GetNameForPort(recipe, port), + Protocol = "TCP", + Port = port.Number, + TargetPort = GetNameForPort(recipe, port), + NodePort = servicePort + }); + } + + servicePorts.Add(recipe, usedPorts.ToArray()); + return result; + } + + //private void DeleteService(CodexNodeGroup online) + //{ + // if (online.Service == null) return; + // client.DeleteNamespacedService(online.Service.Name(), K8sNamespace); + // online.Service = null; + //} + + //private void CreatePrometheusService(K8sPrometheusSpecs spec) + //{ + // client.CreateNamespacedService(spec.CreatePrometheusService(), K8sNamespace); + //} + + //private void CreateGethBootstrapService(K8sGethBoostrapSpecs spec) + //{ + // client.CreateNamespacedService(spec.CreateGethBootstrapService(), K8sNamespace); + //} + + #endregion + + #region Waiting + + //private void WaitUntilOnline(CodexNodeGroup online) + //{ + // WaitUntil(() => + // { + // online.Deployment = client.ReadNamespacedDeployment(online.Deployment.Name(), K8sNamespace); + // return online.Deployment?.Status.AvailableReplicas != null && online.Deployment.Status.AvailableReplicas > 0; + // }); + //} + + //private void WaitUntilOffline(string deploymentName) + //{ + // WaitUntil(() => + // { + // var deployment = client.ReadNamespacedDeployment(deploymentName, K8sNamespace); + // return deployment == null || deployment.Status.AvailableReplicas == 0; + // }); + //} + + //private void WaitUntilZeroPods() + //{ + // WaitUntil(() => !client.ListNamespacedPod(K8sNamespace).Items.Any()); + //} + + private void WaitUntilNamespaceCreated() + { + WaitUntil(() => IsTestNamespaceOnline()); + } + + private void WaitUntilNamespaceDeleted() + { + WaitUntil(() => !IsTestNamespaceOnline()); + } + + //private void WaitUntilPrometheusOnline(K8sPrometheusSpecs spec) + //{ + // WaitUntilDeploymentOnline(spec.GetDeploymentName()); + //} + + //private void WaitUntilGethBootstrapOnline(K8sGethBoostrapSpecs spec) + //{ + // WaitUntilDeploymentOnline(spec.GetBootstrapDeploymentName()); + //} + + //private void WaitUntilGethCompanionGroupOnline(K8sGethBoostrapSpecs spec, GethCompanionGroup group) + //{ + // WaitUntilDeploymentOnline(spec.GetCompanionDeploymentName(group)); + //} + + private void WaitUntilDeploymentCreated(V1Deployment deploymentSpec) + { + WaitUntilDeploymentOnline(deploymentSpec.Metadata.Name); + } + + private void WaitUntilDeploymentOnline(string deploymentName) + { + WaitUntil(() => + { + var deployment = client.ReadNamespacedDeployment(deploymentName, K8sNamespace); + return deployment?.Status.AvailableReplicas != null && deployment.Status.AvailableReplicas > 0; + }); + } + + private void WaitUntil(Func predicate) + { + var start = DateTime.UtcNow; + var state = predicate(); + while (!state) + { + if (DateTime.UtcNow - start > cluster.K8sOperationTimeout()) + { + throw new TimeoutException("K8s operation timed out."); + } + + cluster.WaitForK8sServiceDelay(); + state = predicate(); + } + } + + #endregion + + private (string, string) FetchNewPod() + { + var pods = client.ListNamespacedPod(K8sNamespace).Items; + + var newPods = pods.Where(p => !knownPods.Contains(p.Name())).ToArray(); + if (newPods.Length != 1) throw new InvalidOperationException("Expected only 1 pod to be created. Test infra failure."); + + var newPod = newPods.Single(); + var name = newPod.Name(); + var ip = newPod.Status.PodIP; + + if (string.IsNullOrEmpty(name)) throw new InvalidOperationException("Invalid pod name received. Test infra failure."); + if (string.IsNullOrEmpty(ip)) throw new InvalidOperationException("Invalid pod IP received. Test infra failure."); + + knownPods.Add(name); + return (name, ip); } } } diff --git a/KubernetesWorkflow/KnownK8sPods.cs b/KubernetesWorkflow/KnownK8sPods.cs new file mode 100644 index 0000000..6d80eb6 --- /dev/null +++ b/KubernetesWorkflow/KnownK8sPods.cs @@ -0,0 +1,17 @@ +namespace KubernetesWorkflow +{ + public class KnownK8sPods + { + private readonly List knownActivePodNames = new List(); + + public bool Contains(string name) + { + return knownActivePodNames.Contains(name); + } + + public void Add(string name) + { + knownActivePodNames.Add(name); + } + } +} diff --git a/KubernetesWorkflow/KubernetesWorkflow.csproj b/KubernetesWorkflow/KubernetesWorkflow.csproj index 557b45c..655a8fd 100644 --- a/KubernetesWorkflow/KubernetesWorkflow.csproj +++ b/KubernetesWorkflow/KubernetesWorkflow.csproj @@ -11,4 +11,8 @@ + + + + diff --git a/KubernetesWorkflow/RecipeComponentFactory.cs b/KubernetesWorkflow/RecipeComponentFactory.cs index 6a40cc9..f99f345 100644 --- a/KubernetesWorkflow/RecipeComponentFactory.cs +++ b/KubernetesWorkflow/RecipeComponentFactory.cs @@ -1,4 +1,5 @@ using System.Globalization; +using Utils; namespace KubernetesWorkflow { diff --git a/KubernetesWorkflow/StartupWorkflow.cs b/KubernetesWorkflow/StartupWorkflow.cs index eb714f7..8486590 100644 --- a/KubernetesWorkflow/StartupWorkflow.cs +++ b/KubernetesWorkflow/StartupWorkflow.cs @@ -2,23 +2,36 @@ { public class StartupWorkflow { - private readonly NumberSource containerNumberSource; - private readonly K8sController k8SController; + private readonly WorkflowNumberSource numberSource; + private readonly K8sCluster cluster; + private readonly KnownK8sPods knownK8SPods; private readonly RecipeComponentFactory componentFactory = new RecipeComponentFactory(); - public StartupWorkflow(NumberSource containerNumberSource, K8sController k8SController) + internal StartupWorkflow(WorkflowNumberSource numberSource, K8sCluster cluster, KnownK8sPods knownK8SPods) { - this.containerNumberSource = containerNumberSource; - this.k8SController = k8SController; + this.numberSource = numberSource; + this.cluster = cluster; + this.knownK8SPods = knownK8SPods; } - public RunningContainers Start(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig) + public RunningContainers Start(int numberOfContainers, Location location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig) { - var recipes = CreateRecipes(numberOfContainers, recipeFactory, startupConfig); + return K8s(controller => + { + var recipes = CreateRecipes(numberOfContainers, recipeFactory, startupConfig); - var runningPod = k8SController.BringOnline(recipes); + var runningPod = controller.BringOnline(recipes, location); - return new RunningContainers(startupConfig, runningPod, CreateContainers(runningPod, recipes)); + return new RunningContainers(startupConfig, runningPod, CreateContainers(runningPod, recipes)); + }); + } + + public void DeleteAllResources() + { + K8s(controller => + { + controller.DeleteAllResources(); + }); } private static RunningContainer[] CreateContainers(RunningPod runningPod, ContainerRecipe[] recipes) @@ -31,10 +44,26 @@ var result = new List(); for (var i = 0; i < numberOfContainers; i++) { - result.Add(recipeFactory.CreateRecipe(containerNumberSource.GetNextNumber(), componentFactory, startupConfig)); + result.Add(recipeFactory.CreateRecipe(numberSource.GetContainerNumber(), componentFactory, startupConfig)); } return result.ToArray(); } + + private void K8s(Action action) + { + var controller = new K8sController(cluster, knownK8SPods, numberSource); + action(controller); + controller.Dispose(); + } + + private T K8s(Func action) + { + var controller = new K8sController(cluster, knownK8SPods, numberSource); + var result = action(controller); + controller.Dispose(); + return result; + } + } } diff --git a/KubernetesWorkflow/WorkflowCreator.cs b/KubernetesWorkflow/WorkflowCreator.cs index 25f5439..450bdd6 100644 --- a/KubernetesWorkflow/WorkflowCreator.cs +++ b/KubernetesWorkflow/WorkflowCreator.cs @@ -1,13 +1,19 @@ -namespace KubernetesWorkflow +using Utils; + +namespace KubernetesWorkflow { public class WorkflowCreator { - private readonly NumberSource containerNumberSource = new NumberSource(0); - private readonly K8sController controller = new K8sController(new K8sCluster()); + private readonly NumberSource numberSource = new NumberSource(0); + private readonly NumberSource servicePortNumberSource = new NumberSource(30001); + private readonly K8sCluster cluster = new K8sCluster(); + private readonly KnownK8sPods knownPods = new KnownK8sPods(); public StartupWorkflow CreateWorkflow() { - return new StartupWorkflow(containerNumberSource, controller); + var workflowNumberSource = new WorkflowNumberSource(numberSource.GetNextNumber(), servicePortNumberSource); + + return new StartupWorkflow(workflowNumberSource, cluster, knownPods); } } } diff --git a/KubernetesWorkflow/WorkflowNumberSource.cs b/KubernetesWorkflow/WorkflowNumberSource.cs new file mode 100644 index 0000000..018b97b --- /dev/null +++ b/KubernetesWorkflow/WorkflowNumberSource.cs @@ -0,0 +1,28 @@ +using Utils; + +namespace KubernetesWorkflow +{ + public class WorkflowNumberSource + { + private readonly NumberSource containerNumberSource = new NumberSource(0); + private readonly NumberSource servicePortNumberSource; + + public WorkflowNumberSource(int workflowNumber, NumberSource servicePortNumberSource) + { + WorkflowNumber = workflowNumber; + this.servicePortNumberSource = servicePortNumberSource; + } + + public int WorkflowNumber { get; } + + public int GetContainerNumber() + { + return containerNumberSource.GetNextNumber(); + } + + public int GetServicePort() + { + return servicePortNumberSource.GetNextNumber(); + } + } +} diff --git a/cs-codex-dist-testing.sln b/cs-codex-dist-testing.sln index 43b21e9..de9f58f 100644 --- a/cs-codex-dist-testing.sln +++ b/cs-codex-dist-testing.sln @@ -15,6 +15,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "KubernetesWorkflow", "Kuber EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Utils", "Utils\Utils.csproj", "{957DE3B8-9571-450A-8609-B267DCA8727C}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Logging", "Logging\Logging.csproj", "{8481A4A6-4BDD-41B0-A3EB-EF53F7BD40D1}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -45,6 +47,10 @@ Global {957DE3B8-9571-450A-8609-B267DCA8727C}.Debug|Any CPU.Build.0 = Debug|Any CPU {957DE3B8-9571-450A-8609-B267DCA8727C}.Release|Any CPU.ActiveCfg = Release|Any CPU {957DE3B8-9571-450A-8609-B267DCA8727C}.Release|Any CPU.Build.0 = Release|Any CPU + {8481A4A6-4BDD-41B0-A3EB-EF53F7BD40D1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {8481A4A6-4BDD-41B0-A3EB-EF53F7BD40D1}.Debug|Any CPU.Build.0 = Debug|Any CPU + {8481A4A6-4BDD-41B0-A3EB-EF53F7BD40D1}.Release|Any CPU.ActiveCfg = Release|Any CPU + {8481A4A6-4BDD-41B0-A3EB-EF53F7BD40D1}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE From 7c8a278cd99a5982ab4cc6df033d5649c2fd8a87 Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 12 Apr 2023 15:22:09 +0200 Subject: [PATCH 18/46] Cleanup and make kubernetesworkflow configurable --- KubernetesWorkflow/Configuration.cs | 32 ++++++++++ KubernetesWorkflow/K8sCluster.cs | 34 ++++++----- KubernetesWorkflow/K8sController.cs | 84 +-------------------------- KubernetesWorkflow/WorkflowCreator.cs | 7 ++- 4 files changed, 61 insertions(+), 96 deletions(-) create mode 100644 KubernetesWorkflow/Configuration.cs diff --git a/KubernetesWorkflow/Configuration.cs b/KubernetesWorkflow/Configuration.cs new file mode 100644 index 0000000..b5a4779 --- /dev/null +++ b/KubernetesWorkflow/Configuration.cs @@ -0,0 +1,32 @@ +namespace KubernetesWorkflow +{ + public class Configuration + { + public Configuration(string k8sNamespace, string? kubeConfigFile, TimeSpan operationTimeout, TimeSpan retryDelay, ConfigurationLocationEntry[] locationMap) + { + K8sNamespace = k8sNamespace; + KubeConfigFile = kubeConfigFile; + OperationTimeout = operationTimeout; + RetryDelay = retryDelay; + LocationMap = locationMap; + } + + public string K8sNamespace { get; } + public string? KubeConfigFile { get; } + public TimeSpan OperationTimeout { get; } + public TimeSpan RetryDelay { get; } + public ConfigurationLocationEntry[] LocationMap { get; } + } + + public class ConfigurationLocationEntry + { + public ConfigurationLocationEntry(Location location, string workerName) + { + Location = location; + WorkerName = workerName; + } + + public Location Location { get; } + public string WorkerName { get; } + } +} diff --git a/KubernetesWorkflow/K8sCluster.cs b/KubernetesWorkflow/K8sCluster.cs index 7d6e8eb..c823a09 100644 --- a/KubernetesWorkflow/K8sCluster.cs +++ b/KubernetesWorkflow/K8sCluster.cs @@ -4,21 +4,28 @@ namespace KubernetesWorkflow { public class K8sCluster { - public const string K8sNamespace = "codex-test-namespace"; - private const string KubeConfigFile = "C:\\kube\\config"; - private readonly Dictionary K8sNodeLocationMap = new Dictionary - { - { Location.BensLaptop, "worker01" }, - { Location.BensOldGamingMachine, "worker02" }, - }; - private KubernetesClientConfiguration? config; + public K8sCluster(Configuration configuration) + { + Configuration = configuration; + } + + public Configuration Configuration { get; } + public KubernetesClientConfiguration GetK8sClientConfig() { if (config != null) return config; - //config = KubernetesClientConfiguration.BuildConfigFromConfigFile(KubeConfigFile); - config = KubernetesClientConfiguration.BuildDefaultConfig(); + + if (Configuration.KubeConfigFile != null) + { + config = KubernetesClientConfiguration.BuildConfigFromConfigFile(Configuration.KubeConfigFile); + } + else + { + config = KubernetesClientConfiguration.BuildDefaultConfig(); + } + return config; } @@ -34,18 +41,17 @@ namespace KubernetesWorkflow public string GetNodeLabelForLocation(Location location) { if (location == Location.Unspecified) return string.Empty; - return K8sNodeLocationMap[location]; + return Configuration.LocationMap.Single(l => l.Location == location).WorkerName; } - // make configurable from test env! public TimeSpan K8sOperationTimeout() { - return TimeSpan.FromMinutes(5); + return Configuration.OperationTimeout; } public TimeSpan WaitForK8sServiceDelay() { - return TimeSpan.FromSeconds(5); + return Configuration.RetryDelay; } } } diff --git a/KubernetesWorkflow/K8sController.cs b/KubernetesWorkflow/K8sController.cs index 4049607..df68f18 100644 --- a/KubernetesWorkflow/K8sController.cs +++ b/KubernetesWorkflow/K8sController.cs @@ -71,7 +71,7 @@ namespace KubernetesWorkflow private string K8sNamespace { - get { return K8sCluster.K8sNamespace; } + get { return cluster.Configuration.K8sNamespace; } } private bool IsTestNamespaceOnline() @@ -135,7 +135,7 @@ namespace KubernetesWorkflow return new V1ObjectMeta { Name = "deploy-" + workflowNumberSource.WorkflowNumber, - NamespaceProperty = K8sCluster.K8sNamespace + NamespaceProperty = K8sNamespace }; } @@ -190,29 +190,6 @@ namespace KubernetesWorkflow return $"P{workflowNumberSource.WorkflowNumber}-{recipe.Number}-{port.Number}"; } - - //private void DeleteDeployment(CodexNodeGroup group) - //{ - // if (group.Deployment == null) return; - // client.DeleteNamespacedDeployment(group.Deployment.Name(), K8sNamespace); - // group.Deployment = null; - //} - - //private void CreatePrometheusDeployment(K8sPrometheusSpecs spec) - //{ - // client.CreateNamespacedDeployment(spec.CreatePrometheusDeployment(), K8sNamespace); - //} - - //private void CreateGethBootstrapDeployment(K8sGethBoostrapSpecs spec) - //{ - // client.CreateNamespacedDeployment(spec.CreateGethBootstrapDeployment(), K8sNamespace); - //} - - //private void CreateGethCompanionDeployment(GethBootstrapInfo info, GethCompanionGroup group) - //{ - // client.CreateNamespacedDeployment(info.Spec.CreateGethCompanionDeployment(group, info), K8sNamespace); - //} - #endregion #region Service management @@ -251,7 +228,7 @@ namespace KubernetesWorkflow return new V1ObjectMeta { Name = "deploy-" + workflowNumberSource.WorkflowNumber, - NamespaceProperty = K8sCluster.K8sNamespace + NamespaceProperty = K8sNamespace }; } @@ -288,50 +265,10 @@ namespace KubernetesWorkflow return result; } - //private void DeleteService(CodexNodeGroup online) - //{ - // if (online.Service == null) return; - // client.DeleteNamespacedService(online.Service.Name(), K8sNamespace); - // online.Service = null; - //} - - //private void CreatePrometheusService(K8sPrometheusSpecs spec) - //{ - // client.CreateNamespacedService(spec.CreatePrometheusService(), K8sNamespace); - //} - - //private void CreateGethBootstrapService(K8sGethBoostrapSpecs spec) - //{ - // client.CreateNamespacedService(spec.CreateGethBootstrapService(), K8sNamespace); - //} - #endregion #region Waiting - //private void WaitUntilOnline(CodexNodeGroup online) - //{ - // WaitUntil(() => - // { - // online.Deployment = client.ReadNamespacedDeployment(online.Deployment.Name(), K8sNamespace); - // return online.Deployment?.Status.AvailableReplicas != null && online.Deployment.Status.AvailableReplicas > 0; - // }); - //} - - //private void WaitUntilOffline(string deploymentName) - //{ - // WaitUntil(() => - // { - // var deployment = client.ReadNamespacedDeployment(deploymentName, K8sNamespace); - // return deployment == null || deployment.Status.AvailableReplicas == 0; - // }); - //} - - //private void WaitUntilZeroPods() - //{ - // WaitUntil(() => !client.ListNamespacedPod(K8sNamespace).Items.Any()); - //} - private void WaitUntilNamespaceCreated() { WaitUntil(() => IsTestNamespaceOnline()); @@ -342,21 +279,6 @@ namespace KubernetesWorkflow WaitUntil(() => !IsTestNamespaceOnline()); } - //private void WaitUntilPrometheusOnline(K8sPrometheusSpecs spec) - //{ - // WaitUntilDeploymentOnline(spec.GetDeploymentName()); - //} - - //private void WaitUntilGethBootstrapOnline(K8sGethBoostrapSpecs spec) - //{ - // WaitUntilDeploymentOnline(spec.GetBootstrapDeploymentName()); - //} - - //private void WaitUntilGethCompanionGroupOnline(K8sGethBoostrapSpecs spec, GethCompanionGroup group) - //{ - // WaitUntilDeploymentOnline(spec.GetCompanionDeploymentName(group)); - //} - private void WaitUntilDeploymentCreated(V1Deployment deploymentSpec) { WaitUntilDeploymentOnline(deploymentSpec.Metadata.Name); diff --git a/KubernetesWorkflow/WorkflowCreator.cs b/KubernetesWorkflow/WorkflowCreator.cs index 450bdd6..88af0e8 100644 --- a/KubernetesWorkflow/WorkflowCreator.cs +++ b/KubernetesWorkflow/WorkflowCreator.cs @@ -6,8 +6,13 @@ namespace KubernetesWorkflow { private readonly NumberSource numberSource = new NumberSource(0); private readonly NumberSource servicePortNumberSource = new NumberSource(30001); - private readonly K8sCluster cluster = new K8sCluster(); private readonly KnownK8sPods knownPods = new KnownK8sPods(); + private readonly K8sCluster cluster; + + public WorkflowCreator(Configuration configuration) + { + cluster = new K8sCluster(configuration); + } public StartupWorkflow CreateWorkflow() { From 68d089874de8b4ee8ebbaf42f956a98c3c4e72a5 Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 12 Apr 2023 16:06:04 +0200 Subject: [PATCH 19/46] wiring up the dist-test backend --- CodexDistTestCore/Config/K8sCluster.cs | 2 +- DistTestCore/CodexSetupConfig.cs | 97 ++++++++++++++++++ DistTestCore/CodexStarter.cs | 26 +++++ DistTestCore/Configuration.cs | 32 ++++++ DistTestCore/DistTest.cs | 123 +++++++++++++++++++++++ DistTestCore/FileManager.cs | 113 +++++++++++++++++++++ DistTestCore/TestLifecycle.cs | 33 +++---- DistTestCore/Timing.cs | 132 +++++++++++++++++++++++++ Logging/LogConfig.cs | 7 +- Logging/LogFile.cs | 4 +- Logging/TestLog.cs | 9 +- 11 files changed, 550 insertions(+), 28 deletions(-) create mode 100644 DistTestCore/CodexSetupConfig.cs create mode 100644 DistTestCore/CodexStarter.cs create mode 100644 DistTestCore/Configuration.cs create mode 100644 DistTestCore/DistTest.cs create mode 100644 DistTestCore/FileManager.cs create mode 100644 DistTestCore/Timing.cs diff --git a/CodexDistTestCore/Config/K8sCluster.cs b/CodexDistTestCore/Config/K8sCluster.cs index a290193..a17e6fd 100644 --- a/CodexDistTestCore/Config/K8sCluster.cs +++ b/CodexDistTestCore/Config/K8sCluster.cs @@ -4,7 +4,7 @@ namespace CodexDistTestCore.Config { public class K8sCluster { - public const string K8sNamespace = "codex-test-namespace"; + public const string K8sNamespace = ""; private const string KubeConfigFile = "C:\\kube\\config"; private readonly Dictionary K8sNodeLocationMap = new Dictionary { diff --git a/DistTestCore/CodexSetupConfig.cs b/DistTestCore/CodexSetupConfig.cs new file mode 100644 index 0000000..1f1a632 --- /dev/null +++ b/DistTestCore/CodexSetupConfig.cs @@ -0,0 +1,97 @@ +using DistTestCore.Codex; + +namespace DistTestCore +{ + public interface ICodexSetupConfig + { + ICodexSetupConfig At(Location location); + ICodexSetupConfig WithLogLevel(CodexLogLevel level); + //ICodexStartupConfig WithBootstrapNode(IOnlineCodexNode node); + ICodexSetupConfig WithStorageQuota(ByteSize storageQuota); + ICodexSetupConfig EnableMetrics(); + //ICodexSetupConfig EnableMarketplace(int initialBalance); + ICodexNodeGroup BringOnline(); + } + + public enum Location + { + Unspecified, + BensLaptop, + BensOldGamingMachine, + } + + public class CodexSetupConfig : ICodexSetupConfig + { + private readonly CodexStarter starter; + + public int NumberOfNodes { get; } + public Location Location { get; private set; } + public CodexLogLevel? LogLevel { get; private set; } + //public IOnlineCodexNode? BootstrapNode { get; private set; } + public ByteSize? StorageQuota { get; private set; } + public bool MetricsEnabled { get; private set; } + //public MarketplaceInitialConfig? MarketplaceConfig { get; private set; } + + public CodexSetupConfig(CodexStarter starter, int numberOfNodes) + { + this.starter = starter; + NumberOfNodes = numberOfNodes; + Location = Location.Unspecified; + MetricsEnabled = false; + } + + public ICodexNodeGroup BringOnline() + { + return starter.BringOnline(this); + } + + public ICodexSetupConfig At(Location location) + { + Location = location; + return this; + } + + //public ICodexSetupConfig WithBootstrapNode(IOnlineCodexNode node) + //{ + // BootstrapNode = node; + // return this; + //} + + public ICodexSetupConfig WithLogLevel(CodexLogLevel level) + { + LogLevel = level; + return this; + } + + public ICodexSetupConfig WithStorageQuota(ByteSize storageQuota) + { + StorageQuota = storageQuota; + return this; + } + + public ICodexSetupConfig EnableMetrics() + { + MetricsEnabled = true; + return this; + } + + //public ICodexSetupConfig EnableMarketplace(int initialBalance) + //{ + // MarketplaceConfig = new MarketplaceInitialConfig(initialBalance); + // return this; + //} + + public string Describe() + { + var args = string.Join(',', DescribeArgs()); + return $"{NumberOfNodes} CodexNodes with [{args}]"; + } + + private IEnumerable DescribeArgs() + { + if (LogLevel != null) yield return $"LogLevel={LogLevel}"; + //if (BootstrapNode != null) yield return "BootstrapNode=set-not-shown-here"; + if (StorageQuota != null) yield return $"StorageQuote={StorageQuota.SizeInBytes}"; + } + } +} diff --git a/DistTestCore/CodexStarter.cs b/DistTestCore/CodexStarter.cs new file mode 100644 index 0000000..f295d0a --- /dev/null +++ b/DistTestCore/CodexStarter.cs @@ -0,0 +1,26 @@ +using KubernetesWorkflow; +using Logging; + +namespace DistTestCore +{ + public class CodexStarter + { + private readonly WorkflowCreator workflowCreator; + + public CodexStarter(TestLog log, Configuration configuration) + { + workflowCreator = new WorkflowCreator(configuration.GetK8sConfiguration()); + } + + public ICodexNodeGroup BringOnline(CodexSetupConfig codexSetupConfig) + { + + } + + public void DeleteAllResources() + { + var workflow = workflowCreator.CreateWorkflow(); + workflow.DeleteAllResources(); + } + } +} diff --git a/DistTestCore/Configuration.cs b/DistTestCore/Configuration.cs new file mode 100644 index 0000000..f11437c --- /dev/null +++ b/DistTestCore/Configuration.cs @@ -0,0 +1,32 @@ +using KubernetesWorkflow; + +namespace DistTestCore +{ + public class Configuration + { + public KubernetesWorkflow.Configuration GetK8sConfiguration() + { + return new KubernetesWorkflow.Configuration( + k8sNamespace: "codex-test-ns", + kubeConfigFile: null, + operationTimeout: Timing.K8sOperationTimeout(), + retryDelay: Timing.K8sServiceDelay(), + locationMap: new[] + { + new ConfigurationLocationEntry(Location.BensOldGamingMachine, "worker01"), + new ConfigurationLocationEntry(Location.BensLaptop, "worker02"), + } + ); + } + + public Logging.LogConfig GetLogConfig() + { + return new Logging.LogConfig("D:/CodexTestLogs"); + } + + public string GetFileManagerFolder() + { + return "TestDataFiles"; + } + } +} diff --git a/DistTestCore/DistTest.cs b/DistTestCore/DistTest.cs new file mode 100644 index 0000000..995fffa --- /dev/null +++ b/DistTestCore/DistTest.cs @@ -0,0 +1,123 @@ +using NUnit.Framework; + +namespace DistTestCore +{ + [SetUpFixture] + public abstract class DistTest + { + private TestLifecycle lifecycle = null!; + + [OneTimeSetUp] + public void GlobalSetup() + { + // Previous test run may have been interrupted. + // Begin by cleaning everything up. + CreateNewTestLifecycle(); + + try + { + lifecycle.DeleteAllResources(); + } + catch (Exception ex) + { + GlobalTestFailure.HasFailed = true; + Error($"Global setup cleanup failed with: {ex}"); + throw; + } + Log("Global setup cleanup successful"); + } + + [SetUp] + public void SetUpDistTest() + { + if (GlobalTestFailure.HasFailed) + { + Assert.Inconclusive("Skip test: Previous test failed during clean up."); + } + else + { + CreateNewTestLifecycle(); + } + } + + [TearDown] + public void TearDownDistTest() + { + try + { + lifecycle.Log.EndTest(); + IncludeLogsAndMetricsOnTestFailure(); + lifecycle.DeleteAllResources(); + } + catch (Exception ex) + { + Error("Cleanup failed: " + ex.Message); + GlobalTestFailure.HasFailed = true; + } + } + + public TestFile GenerateTestFile(ByteSize size) + { + return lifecycle.FileManager.GenerateTestFile(size); + } + + public ICodexSetupConfig SetupCodexNodes(int numberOfNodes) + { + return new CodexSetupConfig(lifecycle.CodexStarter, numberOfNodes); + } + + private void IncludeLogsAndMetricsOnTestFailure() + { + var result = TestContext.CurrentContext.Result; + if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed) + { + if (IsDownloadingLogsAndMetricsEnabled()) + { + log.Log("Downloading all CodexNode logs and metrics because of test failure..."); + k8sManager.ForEachOnlineGroup(DownloadLogs); + k8sManager.DownloadAllMetrics(); + } + else + { + log.Log("Skipping download of all CodexNode logs and metrics due to [DontDownloadLogsAndMetricsOnFailure] attribute."); + } + } + } + + private void Log(string msg) + { + lifecycle.Log.Log(msg); + } + + private void Error(string msg) + { + lifecycle.Log.Error(msg); + } + + private void CreateNewTestLifecycle() + { + lifecycle = new TestLifecycle(new Configuration()); + } + + private void DownloadLogs(CodexNodeGroup group) + { + foreach (var node in group) + { + var downloader = new PodLogDownloader(log, k8sManager); + var n = (OnlineCodexNode)node; + downloader.DownloadLog(n); + } + } + + private bool IsDownloadingLogsAndMetricsEnabled() + { + var testProperties = TestContext.CurrentContext.Test.Properties; + return !testProperties.ContainsKey(PodLogDownloader.DontDownloadLogsOnFailureKey); + } + } + + public static class GlobalTestFailure + { + public static bool HasFailed { get; set; } = false; + } +} diff --git a/DistTestCore/FileManager.cs b/DistTestCore/FileManager.cs new file mode 100644 index 0000000..10f126b --- /dev/null +++ b/DistTestCore/FileManager.cs @@ -0,0 +1,113 @@ +using Logging; +using NUnit.Framework; + +namespace DistTestCore +{ + public interface IFileManager + { + TestFile CreateEmptyTestFile(); + TestFile GenerateTestFile(ByteSize size); + void DeleteAllTestFiles(); + } + + public class FileManager : IFileManager + { + public const int ChunkSize = 1024 * 1024; + private readonly Random random = new Random(); + private readonly List activeFiles = new List(); + private readonly TestLog log; + private readonly string folder; + + public FileManager(TestLog log, Configuration configuration) + { + folder = configuration.GetFileManagerFolder(); + + if (!Directory.Exists(folder)) Directory.CreateDirectory(folder); + this.log = log; + } + + public TestFile CreateEmptyTestFile() + { + var result = new TestFile(Path.Combine(folder, Guid.NewGuid().ToString() + "_test.bin")); + File.Create(result.Filename).Close(); + activeFiles.Add(result); + return result; + } + + public TestFile GenerateTestFile(ByteSize size) + { + var result = CreateEmptyTestFile(); + GenerateFileBytes(result, size); + log.Log($"Generated {size.SizeInBytes} bytes of content for file '{result.Filename}'."); + return result; + } + + public void DeleteAllTestFiles() + { + foreach (var file in activeFiles) File.Delete(file.Filename); + activeFiles.Clear(); + } + + private void GenerateFileBytes(TestFile result, ByteSize size) + { + long bytesLeft = size.SizeInBytes; + while (bytesLeft > 0) + { + var length = Math.Min(bytesLeft, ChunkSize); + AppendRandomBytesToFile(result, length); + bytesLeft -= length; + } + } + + private void AppendRandomBytesToFile(TestFile result, long length) + { + var bytes = new byte[length]; + random.NextBytes(bytes); + using var stream = new FileStream(result.Filename, FileMode.Append); + stream.Write(bytes, 0, bytes.Length); + } + } + + public class TestFile + { + public TestFile(string filename) + { + Filename = filename; + } + + public string Filename { get; } + + public long GetFileSize() + { + var info = new FileInfo(Filename); + return info.Length; + } + + public void AssertIsEqual(TestFile? actual) + { + if (actual == null) Assert.Fail("TestFile is null."); + if (actual == this || actual!.Filename == Filename) Assert.Fail("TestFile is compared to itself."); + + Assert.That(actual.GetFileSize(), Is.EqualTo(GetFileSize()), "Files are not of equal length."); + + using var streamExpected = new FileStream(Filename, FileMode.Open, FileAccess.Read); + using var streamActual = new FileStream(actual.Filename, FileMode.Open, FileAccess.Read); + + var bytesExpected = new byte[FileManager.ChunkSize]; + var bytesActual = new byte[FileManager.ChunkSize]; + + var readExpected = 0; + var readActual = 0; + + while (true) + { + readExpected = streamExpected.Read(bytesExpected, 0, FileManager.ChunkSize); + readActual = streamActual.Read(bytesActual, 0, FileManager.ChunkSize); + + if (readExpected == 0 && readActual == 0) return; + Assert.That(readActual, Is.EqualTo(readExpected), "Unable to read buffers of equal length."); + CollectionAssert.AreEqual(bytesExpected, bytesActual, "Files are not binary-equal."); + } + } + } +} diff --git a/DistTestCore/TestLifecycle.cs b/DistTestCore/TestLifecycle.cs index e3d2186..6ff92f9 100644 --- a/DistTestCore/TestLifecycle.cs +++ b/DistTestCore/TestLifecycle.cs @@ -1,31 +1,24 @@ -using DistTestCore.Codex; -using KubernetesWorkflow; +using Logging; namespace DistTestCore { public class TestLifecycle { - private readonly WorkflowCreator workflowCreator = new WorkflowCreator(); - - public void SetUpCodexNodes() + public TestLifecycle(Configuration configuration) { - var config = new CodexStartupConfig() - { - StorageQuota = 10.MB(), - Location = Location.Unspecified, - LogLevel = CodexLogLevel.Error, - MetricsEnabled = false, - }; + Log = new TestLog(configuration.GetLogConfig()); + FileManager = new FileManager(Log, configuration); + CodexStarter = new CodexStarter(Log, configuration); + } - var workflow = workflowCreator.CreateWorkflow(); - var startupConfig = new StartupConfig(); - startupConfig.Add(config); - var containers = workflow.Start(3, new CodexContainerRecipe(), startupConfig); + public TestLog Log { get; } + public FileManager FileManager { get; } + public CodexStarter CodexStarter { get; } - foreach (var c in containers.Containers) - { - var access = new CodexAccess(c); - } + public void DeleteAllResources() + { + CodexStarter.DeleteAllResources(); + FileManager.DeleteAllTestFiles(); } } } diff --git a/DistTestCore/Timing.cs b/DistTestCore/Timing.cs new file mode 100644 index 0000000..67653ec --- /dev/null +++ b/DistTestCore/Timing.cs @@ -0,0 +1,132 @@ +using NUnit.Framework; +using Utils; + +namespace DistTestCore +{ + [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] + public class UseLongTimeoutsAttribute : PropertyAttribute + { + public UseLongTimeoutsAttribute() + : base(Timing.UseLongTimeoutsKey) + { + } + } + + public static class Timing + { + public const string UseLongTimeoutsKey = "UseLongTimeouts"; + + public static TimeSpan HttpCallTimeout() + { + return GetTimes().HttpCallTimeout(); + } + + public static int HttpCallRetryCount() + { + return GetTimes().HttpCallRetryCount(); + } + + public static void HttpCallRetryDelay() + { + Time.Sleep(GetTimes().HttpCallRetryDelay()); + } + + public static TimeSpan K8sServiceDelay() + { + return GetTimes().WaitForK8sServiceDelay(); + } + + public static TimeSpan K8sOperationTimeout() + { + return GetTimes().K8sOperationTimeout(); + } + + public static TimeSpan WaitForMetricTimeout() + { + return GetTimes().WaitForMetricTimeout(); + } + + private static ITimeSet GetTimes() + { + var testProperties = TestContext.CurrentContext.Test.Properties; + if (testProperties.ContainsKey(UseLongTimeoutsKey)) return new LongTimeSet(); + return new DefaultTimeSet(); + } + } + + public interface ITimeSet + { + TimeSpan HttpCallTimeout(); + int HttpCallRetryCount(); + TimeSpan HttpCallRetryDelay(); + TimeSpan WaitForK8sServiceDelay(); + TimeSpan K8sOperationTimeout(); + TimeSpan WaitForMetricTimeout(); + } + + public class DefaultTimeSet : ITimeSet + { + public TimeSpan HttpCallTimeout() + { + return TimeSpan.FromSeconds(10); + } + + public int HttpCallRetryCount() + { + return 5; + } + + public TimeSpan HttpCallRetryDelay() + { + return TimeSpan.FromSeconds(3); + } + + public TimeSpan WaitForK8sServiceDelay() + { + return TimeSpan.FromSeconds(1); + } + + public TimeSpan K8sOperationTimeout() + { + return TimeSpan.FromMinutes(5); + } + + public TimeSpan WaitForMetricTimeout() + { + return TimeSpan.FromSeconds(30); + } + } + + public class LongTimeSet : ITimeSet + { + public TimeSpan HttpCallTimeout() + { + return TimeSpan.FromHours(2); + } + + public int HttpCallRetryCount() + { + return 2; + } + + public TimeSpan HttpCallRetryDelay() + { + return TimeSpan.FromMinutes(5); + } + + public TimeSpan WaitForK8sServiceDelay() + { + return TimeSpan.FromSeconds(10); + } + + public TimeSpan K8sOperationTimeout() + { + return TimeSpan.FromMinutes(15); + } + + public TimeSpan WaitForMetricTimeout() + { + return TimeSpan.FromMinutes(5); + } + } +} diff --git a/Logging/LogConfig.cs b/Logging/LogConfig.cs index c15b1fe..b7bc937 100644 --- a/Logging/LogConfig.cs +++ b/Logging/LogConfig.cs @@ -2,6 +2,11 @@ { public class LogConfig { - public const string LogRoot = "D:/CodexTestLogs"; + public LogConfig(string logRoot) + { + LogRoot = logRoot; + } + + public string LogRoot { get; } } } diff --git a/Logging/LogFile.cs b/Logging/LogFile.cs index a43186c..3a0063b 100644 --- a/Logging/LogFile.cs +++ b/Logging/LogFile.cs @@ -7,14 +7,14 @@ private readonly string ext; private readonly string filepath; - public LogFile(DateTime now, string name, string ext = "log") + public LogFile(LogConfig config, DateTime now, string name, string ext = "log") { this.now = now; this.name = name; this.ext = ext; filepath = Path.Join( - LogConfig.LogRoot, + config.LogRoot, $"{now.Year}-{Pad(now.Month)}", Pad(now.Day)); diff --git a/Logging/TestLog.cs b/Logging/TestLog.cs index 83b6cb1..7eb3979 100644 --- a/Logging/TestLog.cs +++ b/Logging/TestLog.cs @@ -8,13 +8,15 @@ namespace Logging private readonly NumberSource subfileNumberSource = new NumberSource(0); private readonly LogFile file; private readonly DateTime now; + private readonly LogConfig config; - public TestLog() + public TestLog(LogConfig config) { + this.config = config; now = DateTime.UtcNow; var name = GetTestName(); - file = new LogFile(now, name); + file = new LogFile(config, now, name); Log($"Begin: {name}"); } @@ -53,7 +55,7 @@ namespace Logging public LogFile CreateSubfile(string ext = "log") { - return new LogFile(now, $"{GetTestName()}_{subfileNumberSource.GetNextNumber().ToString().PadLeft(6, '0')}", ext); + return new LogFile(config, now, $"{GetTestName()}_{subfileNumberSource.GetNextNumber().ToString().PadLeft(6, '0')}", ext); } private static string GetTestName() @@ -70,5 +72,4 @@ namespace Logging return $"[{string.Join(',', test.Arguments)}]"; } } - } From bb81d7f0377d80d05d1904fd0f924282dc5adeb1 Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 12 Apr 2023 16:12:04 +0200 Subject: [PATCH 20/46] rewiring codex node starter --- DistTestCore/Codex/CodexStartupConfig.cs | 7 ++++--- DistTestCore/CodexSetupConfig.cs | 20 +++----------------- DistTestCore/CodexStarter.cs | 9 ++++++++- DistTestCore/DistTestCore.csproj | 1 + KubernetesWorkflow/StartupConfig.cs | 2 +- 5 files changed, 17 insertions(+), 22 deletions(-) diff --git a/DistTestCore/Codex/CodexStartupConfig.cs b/DistTestCore/Codex/CodexStartupConfig.cs index fdbffcf..a978b5c 100644 --- a/DistTestCore/Codex/CodexStartupConfig.cs +++ b/DistTestCore/Codex/CodexStartupConfig.cs @@ -1,6 +1,4 @@ -using KubernetesWorkflow; - -namespace DistTestCore.Codex +namespace DistTestCore.Codex { public class CodexStartupConfig { @@ -8,5 +6,8 @@ namespace DistTestCore.Codex public CodexLogLevel? LogLevel { get; set; } public ByteSize? StorageQuota { get; set; } public bool MetricsEnabled { get; set; } + + //public IOnlineCodexNode? BootstrapNode { get; private set; } + //public MarketplaceInitialConfig? MarketplaceConfig { get; private set; } } } diff --git a/DistTestCore/CodexSetupConfig.cs b/DistTestCore/CodexSetupConfig.cs index 1f1a632..c7dfef8 100644 --- a/DistTestCore/CodexSetupConfig.cs +++ b/DistTestCore/CodexSetupConfig.cs @@ -1,4 +1,5 @@ using DistTestCore.Codex; +using KubernetesWorkflow; namespace DistTestCore { @@ -12,32 +13,17 @@ namespace DistTestCore //ICodexSetupConfig EnableMarketplace(int initialBalance); ICodexNodeGroup BringOnline(); } - - public enum Location - { - Unspecified, - BensLaptop, - BensOldGamingMachine, - } - - public class CodexSetupConfig : ICodexSetupConfig + + public class CodexSetupConfig : CodexStartupConfig, ICodexSetupConfig { private readonly CodexStarter starter; public int NumberOfNodes { get; } - public Location Location { get; private set; } - public CodexLogLevel? LogLevel { get; private set; } - //public IOnlineCodexNode? BootstrapNode { get; private set; } - public ByteSize? StorageQuota { get; private set; } - public bool MetricsEnabled { get; private set; } - //public MarketplaceInitialConfig? MarketplaceConfig { get; private set; } public CodexSetupConfig(CodexStarter starter, int numberOfNodes) { this.starter = starter; NumberOfNodes = numberOfNodes; - Location = Location.Unspecified; - MetricsEnabled = false; } public ICodexNodeGroup BringOnline() diff --git a/DistTestCore/CodexStarter.cs b/DistTestCore/CodexStarter.cs index f295d0a..0222925 100644 --- a/DistTestCore/CodexStarter.cs +++ b/DistTestCore/CodexStarter.cs @@ -1,4 +1,5 @@ -using KubernetesWorkflow; +using DistTestCore.Codex; +using KubernetesWorkflow; using Logging; namespace DistTestCore @@ -14,7 +15,13 @@ namespace DistTestCore public ICodexNodeGroup BringOnline(CodexSetupConfig codexSetupConfig) { + var workflow = workflowCreator.CreateWorkflow(); + var startupConfig = new StartupConfig(); + startupConfig.Add(codexSetupConfig); + var runningContainers = workflow.Start(codexSetupConfig.NumberOfNodes, codexSetupConfig.Location, new CodexContainerRecipe(), startupConfig); + + // create access objects. Easy, right? } public void DeleteAllResources() diff --git a/DistTestCore/DistTestCore.csproj b/DistTestCore/DistTestCore.csproj index ae8e964..c57463d 100644 --- a/DistTestCore/DistTestCore.csproj +++ b/DistTestCore/DistTestCore.csproj @@ -16,5 +16,6 @@ + diff --git a/KubernetesWorkflow/StartupConfig.cs b/KubernetesWorkflow/StartupConfig.cs index 7c13347..406cea9 100644 --- a/KubernetesWorkflow/StartupConfig.cs +++ b/KubernetesWorkflow/StartupConfig.cs @@ -11,7 +11,7 @@ public T Get() { - var match = configs.Single(c => c.GetType() == typeof(T)); + var match = configs.Single(c => typeof(T).IsAssignableFrom(c.GetType())); return (T)match; } } From f5c60f0bcadc5a82d3476a6ab955b7b2e04e4417 Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 13 Apr 2023 09:33:10 +0200 Subject: [PATCH 21/46] OneClient test passed --- DistTestCore/Codex/CodexAccess.cs | 25 ++- DistTestCore/Codex/CodexStartupConfig.cs | 4 +- DistTestCore/CodexNodeGroup.cs | 78 +++++++ .../{CodexSetupConfig.cs => CodexSetup.cs} | 22 +- DistTestCore/CodexStarter.cs | 13 +- DistTestCore/DistTest.cs | 54 ++--- DistTestCore/Http.cs | 11 +- DistTestCore/OnlineCodexNode.cs | 126 +++++++++++ DistTestCore/TestLifecycle.cs | 2 +- KubernetesWorkflow/K8sController.cs | 2 +- Tests/BasicTests/SimpleTests.cs | 196 +++++++++--------- Tests/Tests.csproj | 2 +- 12 files changed, 379 insertions(+), 156 deletions(-) create mode 100644 DistTestCore/CodexNodeGroup.cs rename DistTestCore/{CodexSetupConfig.cs => CodexSetup.cs} (73%) create mode 100644 DistTestCore/OnlineCodexNode.cs diff --git a/DistTestCore/Codex/CodexAccess.cs b/DistTestCore/Codex/CodexAccess.cs index a7c826e..ca6953f 100644 --- a/DistTestCore/Codex/CodexAccess.cs +++ b/DistTestCore/Codex/CodexAccess.cs @@ -4,13 +4,13 @@ namespace DistTestCore.Codex { public class CodexAccess { - private readonly RunningContainer runningContainer; - public CodexAccess(RunningContainer runningContainer) { - this.runningContainer = runningContainer; + Container = runningContainer; } + public RunningContainer Container { get; } + public CodexDebugResponse GetDebugInfo() { var response = Http().HttpGetJson("debug/info"); @@ -18,12 +18,27 @@ namespace DistTestCore.Codex return response; } + public string UploadFile(FileStream fileStream) + { + return Http().HttpPostStream("upload", fileStream); + } + + public Stream DownloadFile(string contentId) + { + return Http().HttpGetStream("download/" + contentId); + } + private Http Http() { - var ip = runningContainer.Pod.Cluster.GetIp(); - var port = runningContainer.ServicePorts[0].Number; + var ip = Container.Pod.Cluster.GetIp(); + var port = Container.ServicePorts[0].Number; return new Http(ip, port, baseUrl: "/api/codex/v1"); } + + public string ConnectToPeer(string peerId, string peerMultiAddress) + { + return Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}"); + } } public class CodexDebugResponse diff --git a/DistTestCore/Codex/CodexStartupConfig.cs b/DistTestCore/Codex/CodexStartupConfig.cs index a978b5c..f83d629 100644 --- a/DistTestCore/Codex/CodexStartupConfig.cs +++ b/DistTestCore/Codex/CodexStartupConfig.cs @@ -1,4 +1,6 @@ -namespace DistTestCore.Codex +using KubernetesWorkflow; + +namespace DistTestCore.Codex { public class CodexStartupConfig { diff --git a/DistTestCore/CodexNodeGroup.cs b/DistTestCore/CodexNodeGroup.cs new file mode 100644 index 0000000..4ca3a6c --- /dev/null +++ b/DistTestCore/CodexNodeGroup.cs @@ -0,0 +1,78 @@ +using DistTestCore.Codex; +using KubernetesWorkflow; +using System.Collections; + +namespace DistTestCore +{ + public interface ICodexNodeGroup : IEnumerable + { + //ICodexSetup BringOffline(); + IOnlineCodexNode this[int index] { get; } + } + + public class CodexNodeGroup : ICodexNodeGroup + { + private readonly TestLifecycle lifecycle; + + public CodexNodeGroup(TestLifecycle lifecycle, CodexSetup setup, RunningContainers containers) + { + this.lifecycle = lifecycle; + Setup = setup; + Containers = containers; + Nodes = containers.Containers.Select(c => CreateOnlineCodexNode(c)).ToArray(); + } + + public IOnlineCodexNode this[int index] + { + get + { + return Nodes[index]; + } + } + + //public ICodexSetup BringOffline() + //{ + // //return k8SManager.BringOffline(this); + //} + + public CodexSetup Setup { get; } + public RunningContainers Containers { get; } + public OnlineCodexNode[] Nodes { get; } + + //public GethCompanionGroup? GethCompanionGroup { get; set; } + + //public CodexNodeContainer[] GetContainers() + //{ + // return Nodes.Select(n => n.Container).ToArray(); + //} + + public IEnumerator GetEnumerator() + { + return Nodes.Cast().GetEnumerator(); + } + + IEnumerator IEnumerable.GetEnumerator() + { + return Nodes.GetEnumerator(); + } + + //public CodexNodeLog DownloadLog(IOnlineCodexNode node) + //{ + // var logDownloader = new PodLogDownloader(log, k8SManager); + // var n = (OnlineCodexNode)node; + // return logDownloader.DownloadLog(n); + //} + + public string Describe() + { + var orderNumber = Containers.RunningPod.Ip; + return $"CodexNodeGroup@{orderNumber}-{Setup.Describe()}"; + } + + private OnlineCodexNode CreateOnlineCodexNode(RunningContainer c) + { + var access = new CodexAccess(c); + return new OnlineCodexNode(lifecycle, access, this); + } + } +} diff --git a/DistTestCore/CodexSetupConfig.cs b/DistTestCore/CodexSetup.cs similarity index 73% rename from DistTestCore/CodexSetupConfig.cs rename to DistTestCore/CodexSetup.cs index c7dfef8..ddc6ef0 100644 --- a/DistTestCore/CodexSetupConfig.cs +++ b/DistTestCore/CodexSetup.cs @@ -3,24 +3,24 @@ using KubernetesWorkflow; namespace DistTestCore { - public interface ICodexSetupConfig + public interface ICodexSetup { - ICodexSetupConfig At(Location location); - ICodexSetupConfig WithLogLevel(CodexLogLevel level); + ICodexSetup At(Location location); + ICodexSetup WithLogLevel(CodexLogLevel level); //ICodexStartupConfig WithBootstrapNode(IOnlineCodexNode node); - ICodexSetupConfig WithStorageQuota(ByteSize storageQuota); - ICodexSetupConfig EnableMetrics(); + ICodexSetup WithStorageQuota(ByteSize storageQuota); + ICodexSetup EnableMetrics(); //ICodexSetupConfig EnableMarketplace(int initialBalance); ICodexNodeGroup BringOnline(); } - public class CodexSetupConfig : CodexStartupConfig, ICodexSetupConfig + public class CodexSetup : CodexStartupConfig, ICodexSetup { private readonly CodexStarter starter; public int NumberOfNodes { get; } - public CodexSetupConfig(CodexStarter starter, int numberOfNodes) + public CodexSetup(CodexStarter starter, int numberOfNodes) { this.starter = starter; NumberOfNodes = numberOfNodes; @@ -31,7 +31,7 @@ namespace DistTestCore return starter.BringOnline(this); } - public ICodexSetupConfig At(Location location) + public ICodexSetup At(Location location) { Location = location; return this; @@ -43,19 +43,19 @@ namespace DistTestCore // return this; //} - public ICodexSetupConfig WithLogLevel(CodexLogLevel level) + public ICodexSetup WithLogLevel(CodexLogLevel level) { LogLevel = level; return this; } - public ICodexSetupConfig WithStorageQuota(ByteSize storageQuota) + public ICodexSetup WithStorageQuota(ByteSize storageQuota) { StorageQuota = storageQuota; return this; } - public ICodexSetupConfig EnableMetrics() + public ICodexSetup EnableMetrics() { MetricsEnabled = true; return this; diff --git a/DistTestCore/CodexStarter.cs b/DistTestCore/CodexStarter.cs index 0222925..86ed8c3 100644 --- a/DistTestCore/CodexStarter.cs +++ b/DistTestCore/CodexStarter.cs @@ -1,27 +1,28 @@ using DistTestCore.Codex; using KubernetesWorkflow; -using Logging; namespace DistTestCore { public class CodexStarter { private readonly WorkflowCreator workflowCreator; + private readonly TestLifecycle lifecycle; - public CodexStarter(TestLog log, Configuration configuration) + public CodexStarter(TestLifecycle lifecycle, Configuration configuration) { workflowCreator = new WorkflowCreator(configuration.GetK8sConfiguration()); + this.lifecycle = lifecycle; } - public ICodexNodeGroup BringOnline(CodexSetupConfig codexSetupConfig) + public ICodexNodeGroup BringOnline(CodexSetup codexSetup) { var workflow = workflowCreator.CreateWorkflow(); var startupConfig = new StartupConfig(); - startupConfig.Add(codexSetupConfig); + startupConfig.Add(codexSetup); - var runningContainers = workflow.Start(codexSetupConfig.NumberOfNodes, codexSetupConfig.Location, new CodexContainerRecipe(), startupConfig); + var runningContainers = workflow.Start(codexSetup.NumberOfNodes, codexSetup.Location, new CodexContainerRecipe(), startupConfig); - // create access objects. Easy, right? + return new CodexNodeGroup(lifecycle, codexSetup, runningContainers); } public void DeleteAllResources() diff --git a/DistTestCore/DistTest.cs b/DistTestCore/DistTest.cs index 995fffa..9a43a7b 100644 --- a/DistTestCore/DistTest.cs +++ b/DistTestCore/DistTest.cs @@ -61,27 +61,27 @@ namespace DistTestCore return lifecycle.FileManager.GenerateTestFile(size); } - public ICodexSetupConfig SetupCodexNodes(int numberOfNodes) + public ICodexSetup SetupCodexNodes(int numberOfNodes) { - return new CodexSetupConfig(lifecycle.CodexStarter, numberOfNodes); + return new CodexSetup(lifecycle.CodexStarter, numberOfNodes); } private void IncludeLogsAndMetricsOnTestFailure() { - var result = TestContext.CurrentContext.Result; - if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed) - { - if (IsDownloadingLogsAndMetricsEnabled()) - { - log.Log("Downloading all CodexNode logs and metrics because of test failure..."); - k8sManager.ForEachOnlineGroup(DownloadLogs); - k8sManager.DownloadAllMetrics(); - } - else - { - log.Log("Skipping download of all CodexNode logs and metrics due to [DontDownloadLogsAndMetricsOnFailure] attribute."); - } - } + //var result = TestContext.CurrentContext.Result; + //if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed) + //{ + // if (IsDownloadingLogsAndMetricsEnabled()) + // { + // log.Log("Downloading all CodexNode logs and metrics because of test failure..."); + // k8sManager.ForEachOnlineGroup(DownloadLogs); + // k8sManager.DownloadAllMetrics(); + // } + // else + // { + // log.Log("Skipping download of all CodexNode logs and metrics due to [DontDownloadLogsAndMetricsOnFailure] attribute."); + // } + //} } private void Log(string msg) @@ -101,19 +101,19 @@ namespace DistTestCore private void DownloadLogs(CodexNodeGroup group) { - foreach (var node in group) - { - var downloader = new PodLogDownloader(log, k8sManager); - var n = (OnlineCodexNode)node; - downloader.DownloadLog(n); - } + //foreach (var node in group) + //{ + // var downloader = new PodLogDownloader(log, k8sManager); + // var n = (OnlineCodexNode)node; + // downloader.DownloadLog(n); + //} } - private bool IsDownloadingLogsAndMetricsEnabled() - { - var testProperties = TestContext.CurrentContext.Test.Properties; - return !testProperties.ContainsKey(PodLogDownloader.DontDownloadLogsOnFailureKey); - } + //private bool IsDownloadingLogsAndMetricsEnabled() + //{ + // var testProperties = TestContext.CurrentContext.Test.Properties; + // return !testProperties.ContainsKey(PodLogDownloader.DontDownloadLogsOnFailureKey); + //} } public static class GlobalTestFailure diff --git a/DistTestCore/Http.cs b/DistTestCore/Http.cs index 365920b..f75768b 100644 --- a/DistTestCore/Http.cs +++ b/DistTestCore/Http.cs @@ -1,6 +1,7 @@ using Newtonsoft.Json; using NUnit.Framework; using System.Net.Http.Headers; +using Utils; namespace DistTestCore { @@ -26,8 +27,8 @@ namespace DistTestCore { using var client = GetClient(); var url = GetUrl() + route; - var result = Utils.Wait(client.GetAsync(url)); - return Utils.Wait(result.Content.ReadAsStringAsync()); + var result = Time.Wait(client.GetAsync(url)); + return Time.Wait(result.Content.ReadAsStringAsync()); }); } @@ -45,9 +46,9 @@ namespace DistTestCore var content = new StreamContent(stream); content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream"); - var response = Utils.Wait(client.PostAsync(url, content)); + var response = Time.Wait(client.PostAsync(url, content)); - return Utils.Wait(response.Content.ReadAsStringAsync()); + return Time.Wait(response.Content.ReadAsStringAsync()); }); } @@ -58,7 +59,7 @@ namespace DistTestCore var client = GetClient(); var url = GetUrl() + route; - return Utils.Wait(client.GetStreamAsync(url)); + return Time.Wait(client.GetStreamAsync(url)); }); } diff --git a/DistTestCore/OnlineCodexNode.cs b/DistTestCore/OnlineCodexNode.cs new file mode 100644 index 0000000..a4082a8 --- /dev/null +++ b/DistTestCore/OnlineCodexNode.cs @@ -0,0 +1,126 @@ +using DistTestCore.Codex; +using NUnit.Framework; + +namespace DistTestCore +{ + public interface IOnlineCodexNode + { + CodexDebugResponse GetDebugInfo(); + ContentId UploadFile(TestFile file); + TestFile? DownloadContent(ContentId contentId); + void ConnectToPeer(IOnlineCodexNode node); + //ICodexNodeLog DownloadLog(); + //IMetricsAccess Metrics { get; } + //IMarketplaceAccess Marketplace { get; } + } + + public class OnlineCodexNode : IOnlineCodexNode + { + private const string SuccessfullyConnectedMessage = "Successfully connected to peer"; + private const string UploadFailedMessage = "Unable to store block"; + private readonly TestLifecycle lifecycle; + + public OnlineCodexNode(TestLifecycle lifecycle, CodexAccess codexAccess, CodexNodeGroup group) + { + this.lifecycle = lifecycle; + CodexAccess = codexAccess; + Group = group; + } + + public CodexAccess CodexAccess { get; } + public CodexNodeGroup Group { get; } + + public string GetName() + { + return $"<{CodexAccess.Container.Recipe.Name}>"; + } + + public CodexDebugResponse GetDebugInfo() + { + var response = CodexAccess.GetDebugInfo(); + Log($"Got DebugInfo with id: '{response.id}'."); + return response; + } + + public ContentId UploadFile(TestFile file) + { + Log($"Uploading file of size {file.GetFileSize()}..."); + using var fileStream = File.OpenRead(file.Filename); + var response = CodexAccess.UploadFile(fileStream); + if (response.StartsWith(UploadFailedMessage)) + { + Assert.Fail("Node failed to store block."); + } + Log($"Uploaded file. Received contentId: '{response}'."); + return new ContentId(response); + } + + public TestFile? DownloadContent(ContentId contentId) + { + Log($"Downloading for contentId: '{contentId.Id}'..."); + var file = lifecycle.FileManager.CreateEmptyTestFile(); + DownloadToFile(contentId.Id, file); + Log($"Downloaded file of size {file.GetFileSize()} to '{file.Filename}'."); + return file; + } + + public void ConnectToPeer(IOnlineCodexNode node) + { + var peer = (OnlineCodexNode)node; + + Log($"Connecting to peer {peer.GetName()}..."); + var peerInfo = node.GetDebugInfo(); + var response = CodexAccess.ConnectToPeer(peerInfo.id, GetPeerMultiAddress(peer, peerInfo)); + + Assert.That(response, Is.EqualTo(SuccessfullyConnectedMessage), "Unable to connect codex nodes."); + Log($"Successfully connected to peer {peer.GetName()}."); + } + + //public ICodexNodeLog DownloadLog() + //{ + // return Group.DownloadLog(this); + //} + + public string Describe() + { + return $"{Group.Describe()} contains {GetName()}"; + } + + private string GetPeerMultiAddress(OnlineCodexNode peer, CodexDebugResponse peerInfo) + { + var multiAddress = peerInfo.addrs.First(); + // Todo: Is there a case where First address in list is not the way? + + if (Group == peer.Group) + { + return multiAddress; + } + + // The peer we want to connect is in a different pod. + // We must replace the default IP with the pod IP in the multiAddress. + return multiAddress.Replace("0.0.0.0", peer.Group.Containers.RunningPod.Ip); + } + + private void DownloadToFile(string contentId, TestFile file) + { + using var fileStream = File.OpenWrite(file.Filename); + using var downloadStream = CodexAccess.DownloadFile(contentId); + downloadStream.CopyTo(fileStream); + } + + private void Log(string msg) + { + lifecycle.Log.Log($"{GetName()}: {msg}"); + } + } + + public class ContentId + { + public ContentId(string id) + { + Id = id; + } + + public string Id { get; } + } +} diff --git a/DistTestCore/TestLifecycle.cs b/DistTestCore/TestLifecycle.cs index 6ff92f9..7f35056 100644 --- a/DistTestCore/TestLifecycle.cs +++ b/DistTestCore/TestLifecycle.cs @@ -8,7 +8,7 @@ namespace DistTestCore { Log = new TestLog(configuration.GetLogConfig()); FileManager = new FileManager(Log, configuration); - CodexStarter = new CodexStarter(Log, configuration); + CodexStarter = new CodexStarter(this, configuration); } public TestLog Log { get; } diff --git a/KubernetesWorkflow/K8sController.cs b/KubernetesWorkflow/K8sController.cs index df68f18..3df4549 100644 --- a/KubernetesWorkflow/K8sController.cs +++ b/KubernetesWorkflow/K8sController.cs @@ -187,7 +187,7 @@ namespace KubernetesWorkflow private string GetNameForPort(ContainerRecipe recipe, Port port) { - return $"P{workflowNumberSource.WorkflowNumber}-{recipe.Number}-{port.Number}"; + return $"p{workflowNumberSource.WorkflowNumber}-{recipe.Number}-{port.Number}"; } #endregion diff --git a/Tests/BasicTests/SimpleTests.cs b/Tests/BasicTests/SimpleTests.cs index e15ce98..f4214e6 100644 --- a/Tests/BasicTests/SimpleTests.cs +++ b/Tests/BasicTests/SimpleTests.cs @@ -1,4 +1,4 @@ -using CodexDistTestCore; +using DistTestCore; using NUnit.Framework; namespace Tests.BasicTests @@ -6,68 +6,6 @@ namespace Tests.BasicTests [TestFixture] public class SimpleTests : DistTest { - [Test] - public void TwoMetricsExample() - { - var group = SetupCodexNodes(2) - .EnableMetrics() - .BringOnline(); - - var group2 = SetupCodexNodes(2) - .EnableMetrics() - .BringOnline(); - - var primary = group[0]; - var secondary = group[1]; - var primary2 = group2[0]; - var secondary2 = group2[1]; - - primary.ConnectToPeer(secondary); - primary2.ConnectToPeer(secondary2); - - Thread.Sleep(TimeSpan.FromMinutes(5)); - - primary.Metrics.AssertThat("libp2p_peers", Is.EqualTo(1)); - primary2.Metrics.AssertThat("libp2p_peers", Is.EqualTo(1)); - } - - [Test] - public void MarketplaceExample() - { - var group = SetupCodexNodes(4) - .WithStorageQuota(10.GB()) - .EnableMarketplace(initialBalance: 20) - .BringOnline(); - - foreach (var node in group) - { - Assert.That(node.Marketplace.GetBalance(), Is.EqualTo(20)); - } - - // WIP: Balance is now only ETH. - // todo: All nodes should have plenty of ETH to pay for transactions. - // todo: Upload our own token, use this exclusively. ETH should be invisibile to the tests. - - - //var secondary = SetupCodexNodes(1) - // .EnableMarketplace(initialBalance: 1000) - // .BringOnline()[0]; - - //primary.ConnectToPeer(secondary); - //primary.Marketplace.MakeStorageAvailable(10.GB(), minPricePerBytePerSecond: 1, maxCollateral: 20); - - //var testFile = GenerateTestFile(10.MB()); - //var contentId = secondary.UploadFile(testFile); - //secondary.Marketplace.RequestStorage(contentId, pricePerBytePerSecond: 2, - // requiredCollateral: 10, minRequiredNumberOfNodes: 1); - - //primary.Marketplace.AssertThatBalance(Is.LessThan(20), "Collateral was not placed."); - //var primaryBalance = primary.Marketplace.GetBalance(); - - //secondary.Marketplace.AssertThatBalance(Is.LessThan(1000), "Contractor was not charged for storage."); - //primary.Marketplace.AssertThatBalance(Is.GreaterThan(primaryBalance), "Storer was not paid for storage."); - } - [Test] public void OneClientTest() { @@ -82,53 +20,115 @@ namespace Tests.BasicTests testFile.AssertIsEqual(downloadedFile); } - [Test] - public void TwoClientsOnePodTest() - { - var group = SetupCodexNodes(2).BringOnline(); + //[Test] + //public void TwoClientsOnePodTest() + //{ + // var group = SetupCodexNodes(2).BringOnline(); - var primary = group[0]; - var secondary = group[1]; + // var primary = group[0]; + // var secondary = group[1]; - PerformTwoClientTest(primary, secondary); - } + // PerformTwoClientTest(primary, secondary); + //} - [Test] - public void TwoClientsTwoPodsTest() - { - var primary = SetupCodexNodes(1).BringOnline()[0]; + //[Test] + //public void TwoClientsTwoPodsTest() + //{ + // var primary = SetupCodexNodes(1).BringOnline()[0]; - var secondary = SetupCodexNodes(1).BringOnline()[0]; + // var secondary = SetupCodexNodes(1).BringOnline()[0]; - PerformTwoClientTest(primary, secondary); - } + // PerformTwoClientTest(primary, secondary); + //} - [Test] - [Ignore("Requires Location map to be configured for k8s cluster.")] - public void TwoClientsTwoLocationsTest() - { - var primary = SetupCodexNodes(1) - .At(Location.BensLaptop) - .BringOnline()[0]; + //[Test] + //[Ignore("Requires Location map to be configured for k8s cluster.")] + //public void TwoClientsTwoLocationsTest() + //{ + // var primary = SetupCodexNodes(1) + // .At(Location.BensLaptop) + // .BringOnline()[0]; - var secondary = SetupCodexNodes(1) - .At(Location.BensOldGamingMachine) - .BringOnline()[0]; + // var secondary = SetupCodexNodes(1) + // .At(Location.BensOldGamingMachine) + // .BringOnline()[0]; - PerformTwoClientTest(primary, secondary); - } + // PerformTwoClientTest(primary, secondary); + //} - private void PerformTwoClientTest(IOnlineCodexNode primary, IOnlineCodexNode secondary) - { - primary.ConnectToPeer(secondary); + //[Test] + //public void TwoMetricsExample() + //{ + // var group = SetupCodexNodes(2) + // .EnableMetrics() + // .BringOnline(); - var testFile = GenerateTestFile(1.MB()); + // var group2 = SetupCodexNodes(2) + // .EnableMetrics() + // .BringOnline(); - var contentId = primary.UploadFile(testFile); + // var primary = group[0]; + // var secondary = group[1]; + // var primary2 = group2[0]; + // var secondary2 = group2[1]; - var downloadedFile = secondary.DownloadContent(contentId); + // primary.ConnectToPeer(secondary); + // primary2.ConnectToPeer(secondary2); - testFile.AssertIsEqual(downloadedFile); - } + // Thread.Sleep(TimeSpan.FromMinutes(5)); + + // primary.Metrics.AssertThat("libp2p_peers", Is.EqualTo(1)); + // primary2.Metrics.AssertThat("libp2p_peers", Is.EqualTo(1)); + //} + + //[Test] + //public void MarketplaceExample() + //{ + // var group = SetupCodexNodes(4) + // .WithStorageQuota(10.GB()) + // .EnableMarketplace(initialBalance: 20) + // .BringOnline(); + + // foreach (var node in group) + // { + // Assert.That(node.Marketplace.GetBalance(), Is.EqualTo(20)); + // } + + // // WIP: Balance is now only ETH. + // // todo: All nodes should have plenty of ETH to pay for transactions. + // // todo: Upload our own token, use this exclusively. ETH should be invisibile to the tests. + + + // //var secondary = SetupCodexNodes(1) + // // .EnableMarketplace(initialBalance: 1000) + // // .BringOnline()[0]; + + // //primary.ConnectToPeer(secondary); + // //primary.Marketplace.MakeStorageAvailable(10.GB(), minPricePerBytePerSecond: 1, maxCollateral: 20); + + // //var testFile = GenerateTestFile(10.MB()); + // //var contentId = secondary.UploadFile(testFile); + // //secondary.Marketplace.RequestStorage(contentId, pricePerBytePerSecond: 2, + // // requiredCollateral: 10, minRequiredNumberOfNodes: 1); + + // //primary.Marketplace.AssertThatBalance(Is.LessThan(20), "Collateral was not placed."); + // //var primaryBalance = primary.Marketplace.GetBalance(); + + // //secondary.Marketplace.AssertThatBalance(Is.LessThan(1000), "Contractor was not charged for storage."); + // //primary.Marketplace.AssertThatBalance(Is.GreaterThan(primaryBalance), "Storer was not paid for storage."); + //} + + //private void PerformTwoClientTest(IOnlineCodexNode primary, IOnlineCodexNode secondary) + //{ + // primary.ConnectToPeer(secondary); + + // var testFile = GenerateTestFile(1.MB()); + + // var contentId = primary.UploadFile(testFile); + + // var downloadedFile = secondary.DownloadContent(contentId); + + // testFile.AssertIsEqual(downloadedFile); + //} } } diff --git a/Tests/Tests.csproj b/Tests/Tests.csproj index fc5152f..136951d 100644 --- a/Tests/Tests.csproj +++ b/Tests/Tests.csproj @@ -13,7 +13,7 @@ - + From 56063bbbf1dfc109b7c0df18e9335fa9b698c28c Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 13 Apr 2023 10:11:33 +0200 Subject: [PATCH 22/46] two-client tests pass --- DistTestCore/Codex/CodexAccess.cs | 2 +- KubernetesWorkflow/ContainerRecipeFactory.cs | 9 ++- KubernetesWorkflow/K8sCluster.cs | 43 ++++++------ KubernetesWorkflow/K8sController.cs | 2 +- Tests/BasicTests/SimpleTests.cs | 71 ++++++++++---------- 5 files changed, 67 insertions(+), 60 deletions(-) diff --git a/DistTestCore/Codex/CodexAccess.cs b/DistTestCore/Codex/CodexAccess.cs index ca6953f..d0c0f72 100644 --- a/DistTestCore/Codex/CodexAccess.cs +++ b/DistTestCore/Codex/CodexAccess.cs @@ -30,7 +30,7 @@ namespace DistTestCore.Codex private Http Http() { - var ip = Container.Pod.Cluster.GetIp(); + var ip = Container.Pod.Cluster.IP; var port = Container.ServicePorts[0].Number; return new Http(ip, port, baseUrl: "/api/codex/v1"); } diff --git a/KubernetesWorkflow/ContainerRecipeFactory.cs b/KubernetesWorkflow/ContainerRecipeFactory.cs index fc06f33..60ea91e 100644 --- a/KubernetesWorkflow/ContainerRecipeFactory.cs +++ b/KubernetesWorkflow/ContainerRecipeFactory.cs @@ -14,7 +14,14 @@ Initialize(config); - return new ContainerRecipe(containerNumber, Image, exposedPorts.ToArray(), internalPorts.ToArray(), envVars.ToArray()); + var recipe = new ContainerRecipe(containerNumber, Image, exposedPorts.ToArray(), internalPorts.ToArray(), envVars.ToArray()); + + exposedPorts.Clear(); + internalPorts.Clear(); + envVars.Clear(); + this.factory = null!; + + return recipe; } protected abstract string Image { get; } diff --git a/KubernetesWorkflow/K8sCluster.cs b/KubernetesWorkflow/K8sCluster.cs index c823a09..4d5a772 100644 --- a/KubernetesWorkflow/K8sCluster.cs +++ b/KubernetesWorkflow/K8sCluster.cs @@ -4,40 +4,21 @@ namespace KubernetesWorkflow { public class K8sCluster { - private KubernetesClientConfiguration? config; - public K8sCluster(Configuration configuration) { Configuration = configuration; } public Configuration Configuration { get; } + public string IP { get; private set; } = string.Empty; public KubernetesClientConfiguration GetK8sClientConfig() { - if (config != null) return config; - - if (Configuration.KubeConfigFile != null) - { - config = KubernetesClientConfiguration.BuildConfigFromConfigFile(Configuration.KubeConfigFile); - } - else - { - config = KubernetesClientConfiguration.BuildDefaultConfig(); - } - + var config = GetConfig(); + UpdateIp(config); return config; } - public string GetIp() - { - var c = GetK8sClientConfig(); - - var host = c.Host.Replace("https://", ""); - - return host.Substring(0, host.IndexOf(':')); - } - public string GetNodeLabelForLocation(Location location) { if (location == Location.Unspecified) return string.Empty; @@ -53,5 +34,23 @@ namespace KubernetesWorkflow { return Configuration.RetryDelay; } + + private KubernetesClientConfiguration GetConfig() + { + if (Configuration.KubeConfigFile != null) + { + return KubernetesClientConfiguration.BuildConfigFromConfigFile(Configuration.KubeConfigFile); + } + else + { + return KubernetesClientConfiguration.BuildDefaultConfig(); + } + } + + private void UpdateIp(KubernetesClientConfiguration config) + { + var host = config.Host.Replace("https://", ""); + IP = host.Substring(0, host.IndexOf(':')); + } } } diff --git a/KubernetesWorkflow/K8sController.cs b/KubernetesWorkflow/K8sController.cs index 3df4549..295397b 100644 --- a/KubernetesWorkflow/K8sController.cs +++ b/KubernetesWorkflow/K8sController.cs @@ -227,7 +227,7 @@ namespace KubernetesWorkflow { return new V1ObjectMeta { - Name = "deploy-" + workflowNumberSource.WorkflowNumber, + Name = "service-" + workflowNumberSource.WorkflowNumber, NamespaceProperty = K8sNamespace }; } diff --git a/Tests/BasicTests/SimpleTests.cs b/Tests/BasicTests/SimpleTests.cs index f4214e6..4c33425 100644 --- a/Tests/BasicTests/SimpleTests.cs +++ b/Tests/BasicTests/SimpleTests.cs @@ -1,4 +1,5 @@ using DistTestCore; +using KubernetesWorkflow; using NUnit.Framework; namespace Tests.BasicTests @@ -20,41 +21,41 @@ namespace Tests.BasicTests testFile.AssertIsEqual(downloadedFile); } - //[Test] - //public void TwoClientsOnePodTest() - //{ - // var group = SetupCodexNodes(2).BringOnline(); + [Test] + public void TwoClientsOnePodTest() + { + var group = SetupCodexNodes(2).BringOnline(); - // var primary = group[0]; - // var secondary = group[1]; + var primary = group[0]; + var secondary = group[1]; - // PerformTwoClientTest(primary, secondary); - //} + PerformTwoClientTest(primary, secondary); + } - //[Test] - //public void TwoClientsTwoPodsTest() - //{ - // var primary = SetupCodexNodes(1).BringOnline()[0]; + [Test] + public void TwoClientsTwoPodsTest() + { + var primary = SetupCodexNodes(1).BringOnline()[0]; - // var secondary = SetupCodexNodes(1).BringOnline()[0]; + var secondary = SetupCodexNodes(1).BringOnline()[0]; - // PerformTwoClientTest(primary, secondary); - //} + PerformTwoClientTest(primary, secondary); + } - //[Test] - //[Ignore("Requires Location map to be configured for k8s cluster.")] - //public void TwoClientsTwoLocationsTest() - //{ - // var primary = SetupCodexNodes(1) - // .At(Location.BensLaptop) - // .BringOnline()[0]; + [Test] + [Ignore("Requires Location map to be configured for k8s cluster.")] + public void TwoClientsTwoLocationsTest() + { + var primary = SetupCodexNodes(1) + .At(Location.BensLaptop) + .BringOnline()[0]; - // var secondary = SetupCodexNodes(1) - // .At(Location.BensOldGamingMachine) - // .BringOnline()[0]; + var secondary = SetupCodexNodes(1) + .At(Location.BensOldGamingMachine) + .BringOnline()[0]; - // PerformTwoClientTest(primary, secondary); - //} + PerformTwoClientTest(primary, secondary); + } //[Test] //public void TwoMetricsExample() @@ -118,17 +119,17 @@ namespace Tests.BasicTests // //primary.Marketplace.AssertThatBalance(Is.GreaterThan(primaryBalance), "Storer was not paid for storage."); //} - //private void PerformTwoClientTest(IOnlineCodexNode primary, IOnlineCodexNode secondary) - //{ - // primary.ConnectToPeer(secondary); + private void PerformTwoClientTest(IOnlineCodexNode primary, IOnlineCodexNode secondary) + { + primary.ConnectToPeer(secondary); - // var testFile = GenerateTestFile(1.MB()); + var testFile = GenerateTestFile(1.MB()); - // var contentId = primary.UploadFile(testFile); + var contentId = primary.UploadFile(testFile); - // var downloadedFile = secondary.DownloadContent(contentId); + var downloadedFile = secondary.DownloadContent(contentId); - // testFile.AssertIsEqual(downloadedFile); - //} + testFile.AssertIsEqual(downloadedFile); + } } } From cbf0fbf5b572d684a17a6bc59c888c15f97bf469 Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 13 Apr 2023 11:07:36 +0200 Subject: [PATCH 23/46] Implements restart test. --- DistTestCore/CodexNodeGroup.cs | 25 ++++++--- DistTestCore/CodexStarter.cs | 15 +++++- KubernetesWorkflow/K8sController.cs | 63 +++++++++++++++++----- KubernetesWorkflow/RunningPod.cs | 6 ++- KubernetesWorkflow/StartupWorkflow.cs | 8 +++ KubernetesWorkflow/WorkflowCreator.cs | 5 +- KubernetesWorkflow/WorkflowNumberSource.cs | 5 +- Tests/BasicTests/SimpleTests.cs | 25 +++++++-- 8 files changed, 121 insertions(+), 31 deletions(-) diff --git a/DistTestCore/CodexNodeGroup.cs b/DistTestCore/CodexNodeGroup.cs index 4ca3a6c..e486162 100644 --- a/DistTestCore/CodexNodeGroup.cs +++ b/DistTestCore/CodexNodeGroup.cs @@ -6,7 +6,7 @@ namespace DistTestCore { public interface ICodexNodeGroup : IEnumerable { - //ICodexSetup BringOffline(); + ICodexSetup BringOffline(); IOnlineCodexNode this[int index] { get; } } @@ -30,14 +30,23 @@ namespace DistTestCore } } - //public ICodexSetup BringOffline() - //{ - // //return k8SManager.BringOffline(this); - //} + public ICodexSetup BringOffline() + { + var result = Setup; + var containers = Containers; - public CodexSetup Setup { get; } - public RunningContainers Containers { get; } - public OnlineCodexNode[] Nodes { get; } + // Clear everything. Prevent accidental use. + Setup = null!; + Containers = null!; + Nodes = Array.Empty(); + + lifecycle.CodexStarter.BringOffline(containers); + return result; + } + + public CodexSetup Setup { get; private set; } + public RunningContainers Containers { get; private set; } + public OnlineCodexNode[] Nodes { get; private set; } //public GethCompanionGroup? GethCompanionGroup { get; set; } diff --git a/DistTestCore/CodexStarter.cs b/DistTestCore/CodexStarter.cs index 86ed8c3..8ad0aa8 100644 --- a/DistTestCore/CodexStarter.cs +++ b/DistTestCore/CodexStarter.cs @@ -16,7 +16,7 @@ namespace DistTestCore public ICodexNodeGroup BringOnline(CodexSetup codexSetup) { - var workflow = workflowCreator.CreateWorkflow(); + var workflow = CreateWorkflow(); var startupConfig = new StartupConfig(); startupConfig.Add(codexSetup); @@ -25,10 +25,21 @@ namespace DistTestCore return new CodexNodeGroup(lifecycle, codexSetup, runningContainers); } + public void BringOffline(RunningContainers runningContainers) + { + var workflow = CreateWorkflow(); + workflow.Stop(runningContainers); + } + public void DeleteAllResources() { - var workflow = workflowCreator.CreateWorkflow(); + var workflow = CreateWorkflow(); workflow.DeleteAllResources(); } + + private StartupWorkflow CreateWorkflow() + { + return workflowCreator.CreateWorkflow(); + } } } diff --git a/KubernetesWorkflow/K8sController.cs b/KubernetesWorkflow/K8sController.cs index 295397b..a3b25fa 100644 --- a/KubernetesWorkflow/K8sController.cs +++ b/KubernetesWorkflow/K8sController.cs @@ -28,11 +28,19 @@ namespace KubernetesWorkflow { EnsureTestNamespace(); - CreateDeployment(containerRecipes, location); - var servicePortsMap = CreateService(containerRecipes); + var deploymentName = CreateDeployment(containerRecipes, location); + var (serviceName, servicePortsMap) = CreateService(containerRecipes); var (podName, podIp) = FetchNewPod(); - return new RunningPod(cluster, podName, podIp, servicePortsMap); + return new RunningPod(cluster, podName, podIp, deploymentName, serviceName, servicePortsMap); + } + + public void Stop(RunningPod pod) + { + if (!string.IsNullOrEmpty(pod.ServiceName)) DeleteService(pod.ServiceName); + DeleteDeployment(pod.DeploymentName); + WaitUntilDeploymentOffline(pod.DeploymentName); + WaitUntilPodOffline(pod.Name); } public void DeleteAllResources() @@ -83,7 +91,7 @@ namespace KubernetesWorkflow #region Deployment management - private void CreateDeployment(ContainerRecipe[] containerRecipes, Location location) + private string CreateDeployment(ContainerRecipe[] containerRecipes, Location location) { var deploymentSpec = new V1Deployment { @@ -112,7 +120,15 @@ namespace KubernetesWorkflow }; client.CreateNamespacedDeployment(deploymentSpec, K8sNamespace); - WaitUntilDeploymentCreated(deploymentSpec); + WaitUntilDeploymentOnline(deploymentSpec.Metadata.Name); + + return deploymentSpec.Metadata.Name; + } + + private void DeleteDeployment(string deploymentName) + { + client.DeleteNamespacedDeployment(deploymentName, K8sNamespace); + WaitUntilDeploymentOffline(deploymentName); } private IDictionary CreateNodeSelector(Location location) @@ -194,7 +210,7 @@ namespace KubernetesWorkflow #region Service management - private Dictionary CreateService(ContainerRecipe[] containerRecipes) + private (string, Dictionary) CreateService(ContainerRecipe[] containerRecipes) { var result = new Dictionary(); @@ -204,7 +220,7 @@ namespace KubernetesWorkflow { // None of these container-recipes wish to expose anything via a serice port. // So, we don't have to create a service. - return result; + return (string.Empty, result); } var serviceSpec = new V1Service @@ -220,7 +236,13 @@ namespace KubernetesWorkflow }; client.CreateNamespacedService(serviceSpec, K8sNamespace); - return result; + + return (serviceSpec.Metadata.Name, result); + } + + private void DeleteService(string serviceName) + { + client.DeleteNamespacedService(serviceName, K8sNamespace); } private V1ObjectMeta CreateServiceMetadata() @@ -279,11 +301,6 @@ namespace KubernetesWorkflow WaitUntil(() => !IsTestNamespaceOnline()); } - private void WaitUntilDeploymentCreated(V1Deployment deploymentSpec) - { - WaitUntilDeploymentOnline(deploymentSpec.Metadata.Name); - } - private void WaitUntilDeploymentOnline(string deploymentName) { WaitUntil(() => @@ -293,6 +310,26 @@ namespace KubernetesWorkflow }); } + private void WaitUntilDeploymentOffline(string deploymentName) + { + WaitUntil(() => + { + var deployments = client.ListNamespacedDeployment(K8sNamespace); + var deployment = deployments.Items.SingleOrDefault(d => d.Metadata.Name == deploymentName); + return deployment == null || deployment.Status.AvailableReplicas == 0; + }); + } + + private void WaitUntilPodOffline(string podName) + { + WaitUntil(() => + { + var pods = client.ListNamespacedPod(K8sNamespace).Items; + var pod = pods.SingleOrDefault(p => p.Metadata.Name == podName); + return pod == null; + }); + } + private void WaitUntil(Func predicate) { var start = DateTime.UtcNow; diff --git a/KubernetesWorkflow/RunningPod.cs b/KubernetesWorkflow/RunningPod.cs index 63378e2..b676903 100644 --- a/KubernetesWorkflow/RunningPod.cs +++ b/KubernetesWorkflow/RunningPod.cs @@ -4,17 +4,21 @@ { private readonly Dictionary servicePortMap; - public RunningPod(K8sCluster cluster, string name, string ip, Dictionary servicePortMap) + public RunningPod(K8sCluster cluster, string name, string ip, string deploymentName, string serviceName, Dictionary servicePortMap) { Cluster = cluster; Name = name; Ip = ip; + DeploymentName = deploymentName; + ServiceName = serviceName; this.servicePortMap = servicePortMap; } public K8sCluster Cluster { get; } public string Name { get; } public string Ip { get; } + internal string DeploymentName { get; } + internal string ServiceName { get; } public Port[] GetServicePortsForContainerRecipe(ContainerRecipe containerRecipe) { diff --git a/KubernetesWorkflow/StartupWorkflow.cs b/KubernetesWorkflow/StartupWorkflow.cs index 8486590..c3faa50 100644 --- a/KubernetesWorkflow/StartupWorkflow.cs +++ b/KubernetesWorkflow/StartupWorkflow.cs @@ -26,6 +26,14 @@ }); } + public void Stop(RunningContainers runningContainers) + { + K8s(controller => + { + controller.Stop(runningContainers.RunningPod); + }); + } + public void DeleteAllResources() { K8s(controller => diff --git a/KubernetesWorkflow/WorkflowCreator.cs b/KubernetesWorkflow/WorkflowCreator.cs index 88af0e8..51c1c29 100644 --- a/KubernetesWorkflow/WorkflowCreator.cs +++ b/KubernetesWorkflow/WorkflowCreator.cs @@ -6,6 +6,7 @@ namespace KubernetesWorkflow { private readonly NumberSource numberSource = new NumberSource(0); private readonly NumberSource servicePortNumberSource = new NumberSource(30001); + private readonly NumberSource containerNumberSource = new NumberSource(0); private readonly KnownK8sPods knownPods = new KnownK8sPods(); private readonly K8sCluster cluster; @@ -16,7 +17,9 @@ namespace KubernetesWorkflow public StartupWorkflow CreateWorkflow() { - var workflowNumberSource = new WorkflowNumberSource(numberSource.GetNextNumber(), servicePortNumberSource); + var workflowNumberSource = new WorkflowNumberSource(numberSource.GetNextNumber(), + servicePortNumberSource, + containerNumberSource); return new StartupWorkflow(workflowNumberSource, cluster, knownPods); } diff --git a/KubernetesWorkflow/WorkflowNumberSource.cs b/KubernetesWorkflow/WorkflowNumberSource.cs index 018b97b..8cbab34 100644 --- a/KubernetesWorkflow/WorkflowNumberSource.cs +++ b/KubernetesWorkflow/WorkflowNumberSource.cs @@ -4,13 +4,14 @@ namespace KubernetesWorkflow { public class WorkflowNumberSource { - private readonly NumberSource containerNumberSource = new NumberSource(0); private readonly NumberSource servicePortNumberSource; + private readonly NumberSource containerNumberSource; - public WorkflowNumberSource(int workflowNumber, NumberSource servicePortNumberSource) + public WorkflowNumberSource(int workflowNumber, NumberSource servicePortNumberSource, NumberSource containerNumberSource) { WorkflowNumber = workflowNumber; this.servicePortNumberSource = servicePortNumberSource; + this.containerNumberSource = containerNumberSource; } public int WorkflowNumber { get; } diff --git a/Tests/BasicTests/SimpleTests.cs b/Tests/BasicTests/SimpleTests.cs index 4c33425..a723471 100644 --- a/Tests/BasicTests/SimpleTests.cs +++ b/Tests/BasicTests/SimpleTests.cs @@ -12,13 +12,19 @@ namespace Tests.BasicTests { var primary = SetupCodexNodes(1).BringOnline()[0]; - var testFile = GenerateTestFile(1.MB()); + PerformOneClientTest(primary); + } - var contentId = primary.UploadFile(testFile); + [Test] + public void RestartTest() + { + var group = SetupCodexNodes(1).BringOnline(); - var downloadedFile = primary.DownloadContent(contentId); + var setup = group.BringOffline(); - testFile.AssertIsEqual(downloadedFile); + var primary = setup.BringOnline()[0]; + + PerformOneClientTest(primary); } [Test] @@ -119,6 +125,17 @@ namespace Tests.BasicTests // //primary.Marketplace.AssertThatBalance(Is.GreaterThan(primaryBalance), "Storer was not paid for storage."); //} + private void PerformOneClientTest(IOnlineCodexNode primary) + { + var testFile = GenerateTestFile(1.MB()); + + var contentId = primary.UploadFile(testFile); + + var downloadedFile = primary.DownloadContent(contentId); + + testFile.AssertIsEqual(downloadedFile); + } + private void PerformTwoClientTest(IOnlineCodexNode primary, IOnlineCodexNode secondary) { primary.ConnectToPeer(secondary); From 7eab4840efc2e4a7ac9e3cc194c46e1bb8b9a52b Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 13 Apr 2023 11:30:19 +0200 Subject: [PATCH 24/46] Log accessing test passes --- DistTestCore/CodexLogs/CodexNodeLog.cs | 35 ++++++++++++++++++++ DistTestCore/CodexLogs/LogDownloadHandler.cs | 35 ++++++++++++++++++++ DistTestCore/CodexStarter.cs | 8 +++++ DistTestCore/OnlineCodexNode.cs | 11 +++--- DistTestCore/TestLifecycle.cs | 14 +++++++- KubernetesWorkflow/K8sController.cs | 6 ++++ KubernetesWorkflow/StartupWorkflow.cs | 12 +++++++ Tests/BasicTests/SimpleTests.cs | 15 +++++++++ 8 files changed, 130 insertions(+), 6 deletions(-) create mode 100644 DistTestCore/CodexLogs/CodexNodeLog.cs create mode 100644 DistTestCore/CodexLogs/LogDownloadHandler.cs diff --git a/DistTestCore/CodexLogs/CodexNodeLog.cs b/DistTestCore/CodexLogs/CodexNodeLog.cs new file mode 100644 index 0000000..5003bf8 --- /dev/null +++ b/DistTestCore/CodexLogs/CodexNodeLog.cs @@ -0,0 +1,35 @@ +using Logging; +using NUnit.Framework; + +namespace DistTestCore.CodexLogs +{ + public interface ICodexNodeLog + { + void AssertLogContains(string expectedString); + } + + public class CodexNodeLog : ICodexNodeLog + { + private readonly LogFile logFile; + + public CodexNodeLog(LogFile logFile) + { + this.logFile = logFile; + } + + public void AssertLogContains(string expectedString) + { + using var file = File.OpenRead(logFile.FullFilename); + using var streamReader = new StreamReader(file); + + var line = streamReader.ReadLine(); + while (line != null) + { + if (line.Contains(expectedString)) return; + line = streamReader.ReadLine(); + } + + Assert.Fail($"Unable to find string '{expectedString}' in CodexNode log file {logFile.FilenameWithoutPath}"); + } + } +} diff --git a/DistTestCore/CodexLogs/LogDownloadHandler.cs b/DistTestCore/CodexLogs/LogDownloadHandler.cs new file mode 100644 index 0000000..3848b28 --- /dev/null +++ b/DistTestCore/CodexLogs/LogDownloadHandler.cs @@ -0,0 +1,35 @@ +using KubernetesWorkflow; +using Logging; + +namespace DistTestCore.CodexLogs +{ + public class LogDownloadHandler : ILogHandler + { + private readonly string description; + private readonly LogFile log; + + public LogDownloadHandler(string description, LogFile log) + { + this.description = description; + this.log = log; + } + + public CodexNodeLog CreateCodexNodeLog() + { + return new CodexNodeLog(log); + } + + public void Log(Stream stream) + { + log.Write($"{description} -->> {log.FilenameWithoutPath}"); + log.WriteRaw(description); + var reader = new StreamReader(stream); + var line = reader.ReadLine(); + while (line != null) + { + log.WriteRaw(line); + line = reader.ReadLine(); + } + } + } +} diff --git a/DistTestCore/CodexStarter.cs b/DistTestCore/CodexStarter.cs index 8ad0aa8..41c685f 100644 --- a/DistTestCore/CodexStarter.cs +++ b/DistTestCore/CodexStarter.cs @@ -1,5 +1,7 @@ using DistTestCore.Codex; +using DistTestCore.CodexLogs; using KubernetesWorkflow; +using Nethereum.Merkle.Patricia; namespace DistTestCore { @@ -37,6 +39,12 @@ namespace DistTestCore workflow.DeleteAllResources(); } + public void DownloadLog(RunningContainer container, ILogHandler logHandler) + { + var workflow = CreateWorkflow(); + workflow.DownloadContainerLog(container, logHandler); + } + private StartupWorkflow CreateWorkflow() { return workflowCreator.CreateWorkflow(); diff --git a/DistTestCore/OnlineCodexNode.cs b/DistTestCore/OnlineCodexNode.cs index a4082a8..9ecaef5 100644 --- a/DistTestCore/OnlineCodexNode.cs +++ b/DistTestCore/OnlineCodexNode.cs @@ -1,4 +1,5 @@ using DistTestCore.Codex; +using DistTestCore.CodexLogs; using NUnit.Framework; namespace DistTestCore @@ -9,7 +10,7 @@ namespace DistTestCore ContentId UploadFile(TestFile file); TestFile? DownloadContent(ContentId contentId); void ConnectToPeer(IOnlineCodexNode node); - //ICodexNodeLog DownloadLog(); + ICodexNodeLog DownloadLog(); //IMetricsAccess Metrics { get; } //IMarketplaceAccess Marketplace { get; } } @@ -76,10 +77,10 @@ namespace DistTestCore Log($"Successfully connected to peer {peer.GetName()}."); } - //public ICodexNodeLog DownloadLog() - //{ - // return Group.DownloadLog(this); - //} + public ICodexNodeLog DownloadLog() + { + return lifecycle.DownloadLog(this); + } public string Describe() { diff --git a/DistTestCore/TestLifecycle.cs b/DistTestCore/TestLifecycle.cs index 7f35056..95f4508 100644 --- a/DistTestCore/TestLifecycle.cs +++ b/DistTestCore/TestLifecycle.cs @@ -1,4 +1,5 @@ -using Logging; +using DistTestCore.CodexLogs; +using Logging; namespace DistTestCore { @@ -20,5 +21,16 @@ namespace DistTestCore CodexStarter.DeleteAllResources(); FileManager.DeleteAllTestFiles(); } + + public ICodexNodeLog DownloadLog(OnlineCodexNode node) + { + var subFile = Log.CreateSubfile(); + var description = node.Describe(); + var handler = new LogDownloadHandler(description, subFile); + + CodexStarter.DownloadLog(node.CodexAccess.Container, handler); + + return new CodexNodeLog(subFile); + } } } diff --git a/KubernetesWorkflow/K8sController.cs b/KubernetesWorkflow/K8sController.cs index a3b25fa..eb492a9 100644 --- a/KubernetesWorkflow/K8sController.cs +++ b/KubernetesWorkflow/K8sController.cs @@ -43,6 +43,12 @@ namespace KubernetesWorkflow WaitUntilPodOffline(pod.Name); } + public void DownloadPodLog(RunningPod pod, ContainerRecipe recipe, ILogHandler logHandler) + { + var stream = client.ReadNamespacedPodLog(pod.Name, K8sNamespace, recipe.Name); + logHandler.Log(stream); + } + public void DeleteAllResources() { DeleteNamespace(); diff --git a/KubernetesWorkflow/StartupWorkflow.cs b/KubernetesWorkflow/StartupWorkflow.cs index c3faa50..9dff328 100644 --- a/KubernetesWorkflow/StartupWorkflow.cs +++ b/KubernetesWorkflow/StartupWorkflow.cs @@ -34,6 +34,14 @@ }); } + public void DownloadContainerLog(RunningContainer container, ILogHandler logHandler) + { + K8s(controller => + { + controller.DownloadPodLog(container.Pod, container.Recipe, logHandler); + }); + } + public void DeleteAllResources() { K8s(controller => @@ -72,6 +80,10 @@ controller.Dispose(); return result; } + } + public interface ILogHandler + { + void Log(Stream log); } } diff --git a/Tests/BasicTests/SimpleTests.cs b/Tests/BasicTests/SimpleTests.cs index a723471..6f6ce14 100644 --- a/Tests/BasicTests/SimpleTests.cs +++ b/Tests/BasicTests/SimpleTests.cs @@ -1,4 +1,5 @@ using DistTestCore; +using DistTestCore.Codex; using KubernetesWorkflow; using NUnit.Framework; @@ -63,6 +64,20 @@ namespace Tests.BasicTests PerformTwoClientTest(primary, secondary); } + [Test] + public void CodexLogExample() + { + var primary = SetupCodexNodes(1) + .WithLogLevel(CodexLogLevel.Trace) + .BringOnline()[0]; + + primary.UploadFile(GenerateTestFile(5.MB())); + + var log = primary.DownloadLog(); + + log.AssertLogContains("Uploaded file"); + } + //[Test] //public void TwoMetricsExample() //{ From 31e034ab6776f285ed70cab65e49193602830e0c Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 13 Apr 2023 11:53:54 +0200 Subject: [PATCH 25/46] Restores automatic log download on test failure --- CodexDistTestCore/PodLogDownloader.cs | 9 ---- .../CodexNodeLog.cs | 2 +- ...ownloadLogsAndMetricsOnFailureAttribute.cs | 15 ++++++ .../LogDownloadHandler.cs | 2 +- DistTestCore/CodexNodeGroup.cs | 7 ++- DistTestCore/CodexStarter.cs | 26 +++++++-- DistTestCore/DistTest.cs | 54 +++++++++---------- DistTestCore/OnlineCodexNode.cs | 2 +- DistTestCore/TestLifecycle.cs | 3 +- 9 files changed, 71 insertions(+), 49 deletions(-) rename DistTestCore/{CodexLogs => CodexLogsAndMetrics}/CodexNodeLog.cs (95%) create mode 100644 DistTestCore/CodexLogsAndMetrics/DontDownloadLogsAndMetricsOnFailureAttribute.cs rename DistTestCore/{CodexLogs => CodexLogsAndMetrics}/LogDownloadHandler.cs (95%) diff --git a/CodexDistTestCore/PodLogDownloader.cs b/CodexDistTestCore/PodLogDownloader.cs index 4df84d2..e09a57c 100644 --- a/CodexDistTestCore/PodLogDownloader.cs +++ b/CodexDistTestCore/PodLogDownloader.cs @@ -7,15 +7,6 @@ namespace CodexDistTestCore void Log(Stream log); } - [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] - public class DontDownloadLogsAndMetricsOnFailureAttribute : PropertyAttribute - { - public DontDownloadLogsAndMetricsOnFailureAttribute() - : base(Timing.UseLongTimeoutsKey) - { - } - } - public class PodLogDownloader { public const string DontDownloadLogsOnFailureKey = "DontDownloadLogsOnFailure"; diff --git a/DistTestCore/CodexLogs/CodexNodeLog.cs b/DistTestCore/CodexLogsAndMetrics/CodexNodeLog.cs similarity index 95% rename from DistTestCore/CodexLogs/CodexNodeLog.cs rename to DistTestCore/CodexLogsAndMetrics/CodexNodeLog.cs index 5003bf8..a4a9cb0 100644 --- a/DistTestCore/CodexLogs/CodexNodeLog.cs +++ b/DistTestCore/CodexLogsAndMetrics/CodexNodeLog.cs @@ -1,7 +1,7 @@ using Logging; using NUnit.Framework; -namespace DistTestCore.CodexLogs +namespace DistTestCore.CodexLogsAndMetrics { public interface ICodexNodeLog { diff --git a/DistTestCore/CodexLogsAndMetrics/DontDownloadLogsAndMetricsOnFailureAttribute.cs b/DistTestCore/CodexLogsAndMetrics/DontDownloadLogsAndMetricsOnFailureAttribute.cs new file mode 100644 index 0000000..0cf1cbe --- /dev/null +++ b/DistTestCore/CodexLogsAndMetrics/DontDownloadLogsAndMetricsOnFailureAttribute.cs @@ -0,0 +1,15 @@ +using NUnit.Framework; + +namespace DistTestCore.CodexLogsAndMetrics +{ + [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] + public class DontDownloadLogsAndMetricsOnFailureAttribute : PropertyAttribute + { + public const string DontDownloadKey = "DontDownloadLogsAndMetrics"; + + public DontDownloadLogsAndMetricsOnFailureAttribute() + : base(DontDownloadKey) + { + } + } +} diff --git a/DistTestCore/CodexLogs/LogDownloadHandler.cs b/DistTestCore/CodexLogsAndMetrics/LogDownloadHandler.cs similarity index 95% rename from DistTestCore/CodexLogs/LogDownloadHandler.cs rename to DistTestCore/CodexLogsAndMetrics/LogDownloadHandler.cs index 3848b28..59c14d8 100644 --- a/DistTestCore/CodexLogs/LogDownloadHandler.cs +++ b/DistTestCore/CodexLogsAndMetrics/LogDownloadHandler.cs @@ -1,7 +1,7 @@ using KubernetesWorkflow; using Logging; -namespace DistTestCore.CodexLogs +namespace DistTestCore.CodexLogsAndMetrics { public class LogDownloadHandler : ILogHandler { diff --git a/DistTestCore/CodexNodeGroup.cs b/DistTestCore/CodexNodeGroup.cs index e486162..2c5c312 100644 --- a/DistTestCore/CodexNodeGroup.cs +++ b/DistTestCore/CodexNodeGroup.cs @@ -32,15 +32,14 @@ namespace DistTestCore public ICodexSetup BringOffline() { - var result = Setup; - var containers = Containers; + lifecycle.CodexStarter.BringOffline(this); + var result = Setup; // Clear everything. Prevent accidental use. Setup = null!; - Containers = null!; Nodes = Array.Empty(); + Containers = null!; - lifecycle.CodexStarter.BringOffline(containers); return result; } diff --git a/DistTestCore/CodexStarter.cs b/DistTestCore/CodexStarter.cs index 41c685f..d7bc0ab 100644 --- a/DistTestCore/CodexStarter.cs +++ b/DistTestCore/CodexStarter.cs @@ -1,7 +1,5 @@ using DistTestCore.Codex; -using DistTestCore.CodexLogs; using KubernetesWorkflow; -using Nethereum.Merkle.Patricia; namespace DistTestCore { @@ -16,27 +14,40 @@ namespace DistTestCore this.lifecycle = lifecycle; } + public List RunningGroups { get; } = new List(); + public ICodexNodeGroup BringOnline(CodexSetup codexSetup) { + Log($"Starting {codexSetup.Describe()}..."); + var workflow = CreateWorkflow(); var startupConfig = new StartupConfig(); startupConfig.Add(codexSetup); var runningContainers = workflow.Start(codexSetup.NumberOfNodes, codexSetup.Location, new CodexContainerRecipe(), startupConfig); - return new CodexNodeGroup(lifecycle, codexSetup, runningContainers); + var group = new CodexNodeGroup(lifecycle, codexSetup, runningContainers); + RunningGroups.Add(group); + + Log($"Started at '{group.Containers.RunningPod.Ip}'"); + return group; } - public void BringOffline(RunningContainers runningContainers) + public void BringOffline(CodexNodeGroup group) { + Log($"Stopping {group.Describe()}..."); var workflow = CreateWorkflow(); - workflow.Stop(runningContainers); + workflow.Stop(group.Containers); + RunningGroups.Remove(group); + Log("Stopped."); } public void DeleteAllResources() { var workflow = CreateWorkflow(); workflow.DeleteAllResources(); + + RunningGroups.Clear(); } public void DownloadLog(RunningContainer container, ILogHandler logHandler) @@ -49,5 +60,10 @@ namespace DistTestCore { return workflowCreator.CreateWorkflow(); } + + private void Log(string msg) + { + lifecycle.Log.Log(msg); + } } } diff --git a/DistTestCore/DistTest.cs b/DistTestCore/DistTest.cs index 9a43a7b..d1830ba 100644 --- a/DistTestCore/DistTest.cs +++ b/DistTestCore/DistTest.cs @@ -1,4 +1,5 @@ -using NUnit.Framework; +using DistTestCore.CodexLogsAndMetrics; +using NUnit.Framework; namespace DistTestCore { @@ -68,20 +69,20 @@ namespace DistTestCore private void IncludeLogsAndMetricsOnTestFailure() { - //var result = TestContext.CurrentContext.Result; - //if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed) - //{ - // if (IsDownloadingLogsAndMetricsEnabled()) - // { - // log.Log("Downloading all CodexNode logs and metrics because of test failure..."); - // k8sManager.ForEachOnlineGroup(DownloadLogs); - // k8sManager.DownloadAllMetrics(); - // } - // else - // { - // log.Log("Skipping download of all CodexNode logs and metrics due to [DontDownloadLogsAndMetricsOnFailure] attribute."); - // } - //} + var result = TestContext.CurrentContext.Result; + if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed) + { + if (IsDownloadingLogsAndMetricsEnabled()) + { + Log("Downloading all CodexNode logs and metrics because of test failure..."); + DownloadAllLogs(); + //k8sManager.DownloadAllMetrics(); + } + else + { + Log("Skipping download of all CodexNode logs and metrics due to [DontDownloadLogsAndMetricsOnFailure] attribute."); + } + } } private void Log(string msg) @@ -99,21 +100,20 @@ namespace DistTestCore lifecycle = new TestLifecycle(new Configuration()); } - private void DownloadLogs(CodexNodeGroup group) + private void DownloadAllLogs() { - //foreach (var node in group) - //{ - // var downloader = new PodLogDownloader(log, k8sManager); - // var n = (OnlineCodexNode)node; - // downloader.DownloadLog(n); - //} + var allNodes = lifecycle.CodexStarter.RunningGroups.SelectMany(g => g.Nodes); + foreach (var node in allNodes) + { + lifecycle.DownloadLog(node); + } } - //private bool IsDownloadingLogsAndMetricsEnabled() - //{ - // var testProperties = TestContext.CurrentContext.Test.Properties; - // return !testProperties.ContainsKey(PodLogDownloader.DontDownloadLogsOnFailureKey); - //} + private bool IsDownloadingLogsAndMetricsEnabled() + { + var testProperties = TestContext.CurrentContext.Test.Properties; + return !testProperties.ContainsKey(DontDownloadLogsAndMetricsOnFailureAttribute.DontDownloadKey); + } } public static class GlobalTestFailure diff --git a/DistTestCore/OnlineCodexNode.cs b/DistTestCore/OnlineCodexNode.cs index 9ecaef5..8f85f91 100644 --- a/DistTestCore/OnlineCodexNode.cs +++ b/DistTestCore/OnlineCodexNode.cs @@ -1,5 +1,5 @@ using DistTestCore.Codex; -using DistTestCore.CodexLogs; +using DistTestCore.CodexLogsAndMetrics; using NUnit.Framework; namespace DistTestCore diff --git a/DistTestCore/TestLifecycle.cs b/DistTestCore/TestLifecycle.cs index 95f4508..1267216 100644 --- a/DistTestCore/TestLifecycle.cs +++ b/DistTestCore/TestLifecycle.cs @@ -1,4 +1,4 @@ -using DistTestCore.CodexLogs; +using DistTestCore.CodexLogsAndMetrics; using Logging; namespace DistTestCore @@ -28,6 +28,7 @@ namespace DistTestCore var description = node.Describe(); var handler = new LogDownloadHandler(description, subFile); + Log.Log($"Downloading logs for {description} to file {subFile.FilenameWithoutPath}"); CodexStarter.DownloadLog(node.CodexAccess.Container, handler); return new CodexNodeLog(subFile); From 33a3f85136d48f777da2e3c82cdc9b0eabf994b6 Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 13 Apr 2023 14:36:17 +0200 Subject: [PATCH 26/46] Metrics example test passes --- DistTestCore/Codex/CodexContainerRecipe.cs | 4 +- DistTestCore/CodexNodeFactory.cs | 28 +++ DistTestCore/CodexNodeGroup.cs | 11 +- DistTestCore/CodexStarter.cs | 38 +++- DistTestCore/Metrics/MetricsAccess.cs | 65 ++++++ DistTestCore/Metrics/MetricsAccessFactory.cs | 33 +++ DistTestCore/Metrics/MetricsDownloader.cs | 98 +++++++++ DistTestCore/Metrics/MetricsQuery.cs | 195 ++++++++++++++++++ .../Metrics/PrometheusContainerRecipe.cs | 17 ++ .../Metrics/PrometheusStartupConfig.cs | 12 ++ DistTestCore/OnlineCodexNode.cs | 7 +- DistTestCore/PrometheusStarter.cs | 65 ++++++ DistTestCore/TestLifecycle.cs | 9 +- KubernetesWorkflow/ContainerRecipe.cs | 9 +- KubernetesWorkflow/ContainerRecipeFactory.cs | 16 +- KubernetesWorkflow/K8sController.cs | 2 +- KubernetesWorkflow/RecipeComponentFactory.cs | 4 +- KubernetesWorkflow/RunningContainers.cs | 5 + Tests/BasicTests/SimpleTests.cs | 38 ++-- 19 files changed, 604 insertions(+), 52 deletions(-) create mode 100644 DistTestCore/CodexNodeFactory.cs create mode 100644 DistTestCore/Metrics/MetricsAccess.cs create mode 100644 DistTestCore/Metrics/MetricsAccessFactory.cs create mode 100644 DistTestCore/Metrics/MetricsDownloader.cs create mode 100644 DistTestCore/Metrics/MetricsQuery.cs create mode 100644 DistTestCore/Metrics/PrometheusContainerRecipe.cs create mode 100644 DistTestCore/Metrics/PrometheusStartupConfig.cs create mode 100644 DistTestCore/PrometheusStarter.cs diff --git a/DistTestCore/Codex/CodexContainerRecipe.cs b/DistTestCore/Codex/CodexContainerRecipe.cs index 9c7b87d..e49dd84 100644 --- a/DistTestCore/Codex/CodexContainerRecipe.cs +++ b/DistTestCore/Codex/CodexContainerRecipe.cs @@ -4,6 +4,8 @@ namespace DistTestCore.Codex { public class CodexContainerRecipe : ContainerRecipeFactory { + public const string MetricsPortTag = "metrics_port"; + protected override string Image => "thatbenbierens/nim-codex:sha-b204837"; protected override void Initialize(StartupConfig startupConfig) @@ -28,7 +30,7 @@ namespace DistTestCore.Codex if (config.MetricsEnabled) { AddEnvVar("METRICS_ADDR", "0.0.0.0"); - AddInternalPortAndVar("METRICS_PORT"); + AddInternalPortAndVar("METRICS_PORT", tag: MetricsPortTag); } } } diff --git a/DistTestCore/CodexNodeFactory.cs b/DistTestCore/CodexNodeFactory.cs new file mode 100644 index 0000000..3dce8aa --- /dev/null +++ b/DistTestCore/CodexNodeFactory.cs @@ -0,0 +1,28 @@ +using DistTestCore.Codex; +using DistTestCore.Metrics; + +namespace DistTestCore +{ + public interface ICodexNodeFactory + { + OnlineCodexNode CreateOnlineCodexNode(CodexAccess access, CodexNodeGroup group); + } + + public class CodexNodeFactory : ICodexNodeFactory + { + private readonly TestLifecycle lifecycle; + private readonly IMetricsAccessFactory metricsAccessFactory; + + public CodexNodeFactory(TestLifecycle lifecycle, IMetricsAccessFactory metricsAccessFactory) + { + this.lifecycle = lifecycle; + this.metricsAccessFactory = metricsAccessFactory; + } + + public OnlineCodexNode CreateOnlineCodexNode(CodexAccess access, CodexNodeGroup group) + { + var metricsAccess = metricsAccessFactory.CreateMetricsAccess(access.Container); + return new OnlineCodexNode(lifecycle, access, group, metricsAccess); + } + } +} diff --git a/DistTestCore/CodexNodeGroup.cs b/DistTestCore/CodexNodeGroup.cs index 2c5c312..da4313f 100644 --- a/DistTestCore/CodexNodeGroup.cs +++ b/DistTestCore/CodexNodeGroup.cs @@ -14,12 +14,12 @@ namespace DistTestCore { private readonly TestLifecycle lifecycle; - public CodexNodeGroup(TestLifecycle lifecycle, CodexSetup setup, RunningContainers containers) + public CodexNodeGroup(TestLifecycle lifecycle, CodexSetup setup, RunningContainers containers, ICodexNodeFactory codexNodeFactory) { this.lifecycle = lifecycle; Setup = setup; Containers = containers; - Nodes = containers.Containers.Select(c => CreateOnlineCodexNode(c)).ToArray(); + Nodes = containers.Containers.Select(c => CreateOnlineCodexNode(c, codexNodeFactory)).ToArray(); } public IOnlineCodexNode this[int index] @@ -73,14 +73,13 @@ namespace DistTestCore public string Describe() { - var orderNumber = Containers.RunningPod.Ip; - return $"CodexNodeGroup@{orderNumber}-{Setup.Describe()}"; + return $"CodexNodeGroup@{Containers.Describe()}-{Setup.Describe()}"; } - private OnlineCodexNode CreateOnlineCodexNode(RunningContainer c) + private OnlineCodexNode CreateOnlineCodexNode(RunningContainer c, ICodexNodeFactory factory) { var access = new CodexAccess(c); - return new OnlineCodexNode(lifecycle, access, this); + return factory.CreateOnlineCodexNode(access, this); } } } diff --git a/DistTestCore/CodexStarter.cs b/DistTestCore/CodexStarter.cs index d7bc0ab..65c9a32 100644 --- a/DistTestCore/CodexStarter.cs +++ b/DistTestCore/CodexStarter.cs @@ -5,31 +5,27 @@ namespace DistTestCore { public class CodexStarter { - private readonly WorkflowCreator workflowCreator; private readonly TestLifecycle lifecycle; + private readonly WorkflowCreator workflowCreator; - public CodexStarter(TestLifecycle lifecycle, Configuration configuration) + public CodexStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator) { - workflowCreator = new WorkflowCreator(configuration.GetK8sConfiguration()); this.lifecycle = lifecycle; + this.workflowCreator = workflowCreator; } public List RunningGroups { get; } = new List(); public ICodexNodeGroup BringOnline(CodexSetup codexSetup) { - Log($"Starting {codexSetup.Describe()}..."); + var containers = StartCodexContainers(codexSetup); - var workflow = CreateWorkflow(); - var startupConfig = new StartupConfig(); - startupConfig.Add(codexSetup); + var metricAccessFactory = lifecycle.PrometheusStarter.CollectMetricsFor(codexSetup, containers); - var runningContainers = workflow.Start(codexSetup.NumberOfNodes, codexSetup.Location, new CodexContainerRecipe(), startupConfig); + var codexNodeFactory = new CodexNodeFactory(lifecycle, metricAccessFactory); - var group = new CodexNodeGroup(lifecycle, codexSetup, runningContainers); - RunningGroups.Add(group); + var group = CreateCodexGroup(codexSetup, containers, codexNodeFactory); - Log($"Started at '{group.Containers.RunningPod.Ip}'"); return group; } @@ -55,6 +51,26 @@ namespace DistTestCore var workflow = CreateWorkflow(); workflow.DownloadContainerLog(container, logHandler); } + + private RunningContainers StartCodexContainers(CodexSetup codexSetup) + { + Log($"Starting {codexSetup.Describe()}..."); + + var workflow = CreateWorkflow(); + var startupConfig = new StartupConfig(); + startupConfig.Add(codexSetup); + + return workflow.Start(codexSetup.NumberOfNodes, codexSetup.Location, new CodexContainerRecipe(), startupConfig); + } + + private CodexNodeGroup CreateCodexGroup(CodexSetup codexSetup, RunningContainers runningContainers, CodexNodeFactory codexNodeFactory) + { + var group = new CodexNodeGroup(lifecycle, codexSetup, runningContainers, codexNodeFactory); + RunningGroups.Add(group); + + Log($"Started at '{group.Containers.RunningPod.Ip}'"); + return group; + } private StartupWorkflow CreateWorkflow() { diff --git a/DistTestCore/Metrics/MetricsAccess.cs b/DistTestCore/Metrics/MetricsAccess.cs new file mode 100644 index 0000000..3287ea7 --- /dev/null +++ b/DistTestCore/Metrics/MetricsAccess.cs @@ -0,0 +1,65 @@ +using KubernetesWorkflow; +using NUnit.Framework; +using NUnit.Framework.Constraints; +using Utils; + +namespace DistTestCore.Metrics +{ + public interface IMetricsAccess + { + void AssertThat(string metricName, IResolveConstraint constraint, string message = ""); + } + + public class MetricsUnavailable : IMetricsAccess + { + public void AssertThat(string metricName, IResolveConstraint constraint, string message = "") + { + Assert.Fail("Incorrect test setup: Metrics were not enabled for this group of Codex nodes. Add 'EnableMetrics()' after 'SetupCodexNodes()' to enable it."); + throw new InvalidOperationException(); + } + } + + public class MetricsAccess : IMetricsAccess + { + private readonly MetricsQuery query; + private readonly RunningContainer node; + + public MetricsAccess(MetricsQuery query, RunningContainer node) + { + this.query = query; + this.node = node; + } + + public void AssertThat(string metricName, IResolveConstraint constraint, string message = "") + { + var metricSet = GetMetricWithTimeout(metricName); + var metricValue = metricSet.Values[0].Value; + Assert.That(metricValue, constraint, message); + } + + private MetricsSet GetMetricWithTimeout(string metricName) + { + var start = DateTime.UtcNow; + + while (true) + { + var mostRecent = GetMostRecent(metricName); + if (mostRecent != null) return mostRecent; + if (DateTime.UtcNow - start > Timing.WaitForMetricTimeout()) + { + Assert.Fail($"Timeout: Unable to get metric '{metricName}'."); + throw new TimeoutException(); + } + + Time.Sleep(TimeSpan.FromSeconds(2)); + } + } + + private MetricsSet? GetMostRecent(string metricName) + { + var result = query.GetMostRecent(metricName, node); + if (result == null) return null; + return result.Sets.LastOrDefault(); + } + } +} diff --git a/DistTestCore/Metrics/MetricsAccessFactory.cs b/DistTestCore/Metrics/MetricsAccessFactory.cs new file mode 100644 index 0000000..fcf5dfb --- /dev/null +++ b/DistTestCore/Metrics/MetricsAccessFactory.cs @@ -0,0 +1,33 @@ +using KubernetesWorkflow; + +namespace DistTestCore.Metrics +{ + public interface IMetricsAccessFactory + { + IMetricsAccess CreateMetricsAccess(RunningContainer codexContainer); + } + + public class MetricsUnavailableAccessFactory : IMetricsAccessFactory + { + public IMetricsAccess CreateMetricsAccess(RunningContainer codexContainer) + { + return new MetricsUnavailable(); + } + } + + public class CodexNodeMetricsAccessFactory : IMetricsAccessFactory + { + private readonly RunningContainers prometheusContainer; + + public CodexNodeMetricsAccessFactory(RunningContainers prometheusContainer) + { + this.prometheusContainer = prometheusContainer; + } + + public IMetricsAccess CreateMetricsAccess(RunningContainer codexContainer) + { + var query = new MetricsQuery(prometheusContainer); + return new MetricsAccess(query, codexContainer); + } + } +} diff --git a/DistTestCore/Metrics/MetricsDownloader.cs b/DistTestCore/Metrics/MetricsDownloader.cs new file mode 100644 index 0000000..3d79752 --- /dev/null +++ b/DistTestCore/Metrics/MetricsDownloader.cs @@ -0,0 +1,98 @@ +using Logging; +using System.Globalization; + +namespace DistTestCore.Metrics +{ + public class MetricsDownloader + { + private readonly TestLog log; + private readonly Dictionary activePrometheuses; + + public MetricsDownloader(TestLog log, Dictionary activePrometheuses) + { + this.log = log; + this.activePrometheuses = activePrometheuses; + } + + public void DownloadAllMetrics() + { + foreach (var pair in activePrometheuses) + { + DownloadAllMetrics(pair.Key, pair.Value); + } + } + + private void DownloadAllMetrics(MetricsQuery query, OnlineCodexNode[] nodes) + { + foreach (var node in nodes) + { + DownloadAllMetricsForNode(query, node); + } + } + + private void DownloadAllMetricsForNode(MetricsQuery query, OnlineCodexNode node) + { + var metrics = query.GetAllMetricsForNode(node.CodexAccess.Container); + if (metrics == null || metrics.Sets.Length == 0 || metrics.Sets.All(s => s.Values.Length == 0)) return; + + var headers = new[] { "timestamp" }.Concat(metrics.Sets.Select(s => s.Name)).ToArray(); + var map = CreateValueMap(metrics); + + WriteToFile(node.GetName(), headers, map); + } + + private void WriteToFile(string nodeName, string[] headers, Dictionary> map) + { + var file = log.CreateSubfile("csv"); + log.Log($"Downloading metrics for {nodeName} to file {file.FilenameWithoutPath}"); + + file.WriteRaw(string.Join(",", headers)); + + foreach (var pair in map) + { + file.WriteRaw(string.Join(",", new[] { FormatTimestamp(pair.Key) }.Concat(pair.Value))); + } + } + + private Dictionary> CreateValueMap(Metrics metrics) + { + var map = CreateForAllTimestamps(metrics); + foreach (var metric in metrics.Sets) + { + AddToMap(map, metric); + } + return map; + + } + + private Dictionary> CreateForAllTimestamps(Metrics metrics) + { + var result = new Dictionary>(); + var timestamps = metrics.Sets.SelectMany(s => s.Values).Select(v => v.Timestamp).Distinct().ToArray(); + foreach (var timestamp in timestamps) result.Add(timestamp, new List()); + return result; + } + + private void AddToMap(Dictionary> map, MetricsSet metric) + { + foreach (var key in map.Keys) + { + map[key].Add(GetValueAtTimestamp(key, metric)); + } + } + + private string GetValueAtTimestamp(DateTime key, MetricsSet metric) + { + var value = metric.Values.SingleOrDefault(v => v.Timestamp == key); + if (value == null) return ""; + return value.Value.ToString(CultureInfo.InvariantCulture); + } + + private string FormatTimestamp(DateTime key) + { + var origin = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc); + var diff = key - origin; + return Math.Floor(diff.TotalSeconds).ToString(CultureInfo.InvariantCulture); + } + } +} diff --git a/DistTestCore/Metrics/MetricsQuery.cs b/DistTestCore/Metrics/MetricsQuery.cs new file mode 100644 index 0000000..c06fc8d --- /dev/null +++ b/DistTestCore/Metrics/MetricsQuery.cs @@ -0,0 +1,195 @@ +using DistTestCore.Codex; +using KubernetesWorkflow; +using System.Globalization; + +namespace DistTestCore.Metrics +{ + public class MetricsQuery + { + private readonly Http http; + + public MetricsQuery(RunningContainers runningContainers) + { + RunningContainers = runningContainers; + + http = new Http( + runningContainers.RunningPod.Cluster.IP, + runningContainers.Containers[0].ServicePorts[0].Number, + "api/v1"); + } + + public RunningContainers RunningContainers { get; } + + public Metrics? GetMostRecent(string metricName, RunningContainer node) + { + var response = GetLastOverTime(metricName, GetInstanceStringForNode(node)); + if (response == null) return null; + + return new Metrics + { + Sets = response.data.result.Select(r => + { + return new MetricsSet + { + Instance = r.metric.instance, + Values = MapSingleValue(r.value) + }; + }).ToArray() + }; + } + + public Metrics? GetMetrics(string metricName) + { + var response = GetAll(metricName); + if (response == null) return null; + return MapResponseToMetrics(response); + } + + public Metrics? GetAllMetricsForNode(RunningContainer node) + { + var response = http.HttpGetJson($"query?query={GetInstanceStringForNode(node)}{GetQueryTimeRange()}"); + if (response.status != "success") return null; + return MapResponseToMetrics(response); + } + + private PrometheusQueryResponse? GetLastOverTime(string metricName, string instanceString) + { + var response = http.HttpGetJson($"query?query=last_over_time({metricName}{instanceString}{GetQueryTimeRange()})"); + if (response.status != "success") return null; + return response; + } + + private PrometheusQueryResponse? GetAll(string metricName) + { + var response = http.HttpGetJson($"query?query={metricName}{GetQueryTimeRange()}"); + if (response.status != "success") return null; + return response; + } + + private Metrics MapResponseToMetrics(PrometheusQueryResponse response) + { + return new Metrics + { + Sets = response.data.result.Select(r => + { + return new MetricsSet + { + Name = r.metric.__name__, + Instance = r.metric.instance, + Values = MapMultipleValues(r.values) + }; + }).ToArray() + }; + } + + private MetricsSetValue[] MapSingleValue(object[] value) + { + if (value != null && value.Length > 0) + { + return new[] + { + MapValue(value) + }; + } + return Array.Empty(); + } + + private MetricsSetValue[] MapMultipleValues(object[][] values) + { + if (values != null && values.Length > 0) + { + return values.Select(v => MapValue(v)).ToArray(); + } + return Array.Empty(); + } + + private MetricsSetValue MapValue(object[] value) + { + if (value.Length != 2) throw new InvalidOperationException("Expected value to be [double, string]."); + + return new MetricsSetValue + { + Timestamp = ToTimestamp(value[0]), + Value = ToValue(value[1]) + }; + } + + private string GetInstanceNameForNode(RunningContainer node) + { + var ip = node.Pod.Ip; + var port = node.Recipe.GetPortByTag(CodexContainerRecipe.MetricsPortTag).Number; + return $"{ip}:{port}"; + } + + private string GetInstanceStringForNode(RunningContainer node) + { + return "{instance=\"" + GetInstanceNameForNode(node) + "\"}"; + } + + private string GetQueryTimeRange() + { + return "[12h]"; + } + + private double ToValue(object v) + { + return Convert.ToDouble(v, CultureInfo.InvariantCulture); + } + + private DateTime ToTimestamp(object v) + { + var unixSeconds = ToValue(v); + return new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc).AddSeconds(unixSeconds); + } + } + + public class Metrics + { + public MetricsSet[] Sets { get; set; } = Array.Empty(); + } + + public class MetricsSet + { + public string Name { get; set; } = string.Empty; + public string Instance { get; set; } = string.Empty; + public MetricsSetValue[] Values { get; set; } = Array.Empty(); + } + + public class MetricsSetValue + { + public DateTime Timestamp { get; set; } + public double Value { get; set; } + } + + public class PrometheusQueryResponse + { + public string status { get; set; } = string.Empty; + public PrometheusQueryResponseData data { get; set; } = new(); + } + + public class PrometheusQueryResponseData + { + public string resultType { get; set; } = string.Empty; + public PrometheusQueryResponseDataResultEntry[] result { get; set; } = Array.Empty(); + } + + public class PrometheusQueryResponseDataResultEntry + { + public ResultEntryMetric metric { get; set; } = new(); + public object[] value { get; set; } = Array.Empty(); + public object[][] values { get; set; } = Array.Empty(); + } + + public class ResultEntryMetric + { + public string __name__ { get; set; } = string.Empty; + public string instance { get; set; } = string.Empty; + public string job { get; set; } = string.Empty; + } + + public class PrometheusAllNamesResponse + { + public string status { get; set; } = string.Empty; + public string[] data { get; set; } = Array.Empty(); + } +} diff --git a/DistTestCore/Metrics/PrometheusContainerRecipe.cs b/DistTestCore/Metrics/PrometheusContainerRecipe.cs new file mode 100644 index 0000000..5152ff7 --- /dev/null +++ b/DistTestCore/Metrics/PrometheusContainerRecipe.cs @@ -0,0 +1,17 @@ +using KubernetesWorkflow; + +namespace DistTestCore.Metrics +{ + public class PrometheusContainerRecipe : ContainerRecipeFactory + { + protected override string Image => "thatbenbierens/prometheus-envconf:latest"; + + protected override void Initialize(StartupConfig startupConfig) + { + var config = startupConfig.Get(); + + AddExposedPortAndVar("PROM_PORT"); + AddEnvVar("PROM_CONFIG", config.PrometheusConfigBase64); + } + } +} diff --git a/DistTestCore/Metrics/PrometheusStartupConfig.cs b/DistTestCore/Metrics/PrometheusStartupConfig.cs new file mode 100644 index 0000000..7bf7fe6 --- /dev/null +++ b/DistTestCore/Metrics/PrometheusStartupConfig.cs @@ -0,0 +1,12 @@ +namespace DistTestCore.Metrics +{ + public class PrometheusStartupConfig + { + public PrometheusStartupConfig(string prometheusConfigBase64) + { + PrometheusConfigBase64 = prometheusConfigBase64; + } + + public string PrometheusConfigBase64 { get; } + } +} diff --git a/DistTestCore/OnlineCodexNode.cs b/DistTestCore/OnlineCodexNode.cs index 8f85f91..0efa51a 100644 --- a/DistTestCore/OnlineCodexNode.cs +++ b/DistTestCore/OnlineCodexNode.cs @@ -1,5 +1,6 @@ using DistTestCore.Codex; using DistTestCore.CodexLogsAndMetrics; +using DistTestCore.Metrics; using NUnit.Framework; namespace DistTestCore @@ -11,7 +12,7 @@ namespace DistTestCore TestFile? DownloadContent(ContentId contentId); void ConnectToPeer(IOnlineCodexNode node); ICodexNodeLog DownloadLog(); - //IMetricsAccess Metrics { get; } + IMetricsAccess Metrics { get; } //IMarketplaceAccess Marketplace { get; } } @@ -21,15 +22,17 @@ namespace DistTestCore private const string UploadFailedMessage = "Unable to store block"; private readonly TestLifecycle lifecycle; - public OnlineCodexNode(TestLifecycle lifecycle, CodexAccess codexAccess, CodexNodeGroup group) + public OnlineCodexNode(TestLifecycle lifecycle, CodexAccess codexAccess, CodexNodeGroup group, IMetricsAccess metricsAccess) { this.lifecycle = lifecycle; CodexAccess = codexAccess; Group = group; + Metrics = metricsAccess; } public CodexAccess CodexAccess { get; } public CodexNodeGroup Group { get; } + public IMetricsAccess Metrics { get; } public string GetName() { diff --git a/DistTestCore/PrometheusStarter.cs b/DistTestCore/PrometheusStarter.cs new file mode 100644 index 0000000..a58b3e4 --- /dev/null +++ b/DistTestCore/PrometheusStarter.cs @@ -0,0 +1,65 @@ +using DistTestCore.Codex; +using DistTestCore.Metrics; +using KubernetesWorkflow; +using System.Text; + +namespace DistTestCore +{ + public class PrometheusStarter + { + private readonly TestLifecycle lifecycle; + private readonly WorkflowCreator workflowCreator; + + public PrometheusStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator) + { + this.lifecycle = lifecycle; + this.workflowCreator = workflowCreator; + } + + public IMetricsAccessFactory CollectMetricsFor(CodexSetup codexSetup, RunningContainers containers) + { + if (!codexSetup.MetricsEnabled) return new MetricsUnavailableAccessFactory(); + + Log($"Starting metrics server for {containers.Describe()}"); + var startupConfig = new StartupConfig(); + startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(containers.Containers))); + + var workflow = workflowCreator.CreateWorkflow(); + var runningContainers = workflow.Start(1, Location.Unspecified, new PrometheusContainerRecipe(), startupConfig); + if (runningContainers.Containers.Length != 1) throw new InvalidOperationException("Expected only 1 Prometheus container to be created."); + + Log("Metrics server started."); + + return new CodexNodeMetricsAccessFactory(runningContainers); + } + + private string GeneratePrometheusConfig(RunningContainer[] nodes) + { + var config = ""; + config += "global:\n"; + config += " scrape_interval: 30s\n"; + config += " scrape_timeout: 10s\n"; + config += "\n"; + config += "scrape_configs:\n"; + config += " - job_name: services\n"; + config += " metrics_path: /metrics\n"; + config += " static_configs:\n"; + config += " - targets:\n"; + + foreach (var node in nodes) + { + var ip = node.Pod.Ip; + var port = node.Recipe.GetPortByTag(CodexContainerRecipe.MetricsPortTag).Number; + config += $" - '{ip}:{port}'\n"; + } + + var bytes = Encoding.ASCII.GetBytes(config); + return Convert.ToBase64String(bytes); + } + + private void Log(string msg) + { + lifecycle.Log.Log(msg); + } + } +} diff --git a/DistTestCore/TestLifecycle.cs b/DistTestCore/TestLifecycle.cs index 1267216..a5127a7 100644 --- a/DistTestCore/TestLifecycle.cs +++ b/DistTestCore/TestLifecycle.cs @@ -1,20 +1,27 @@ using DistTestCore.CodexLogsAndMetrics; +using KubernetesWorkflow; using Logging; namespace DistTestCore { public class TestLifecycle { + private readonly WorkflowCreator workflowCreator; + public TestLifecycle(Configuration configuration) { Log = new TestLog(configuration.GetLogConfig()); + workflowCreator = new WorkflowCreator(configuration.GetK8sConfiguration()); + FileManager = new FileManager(Log, configuration); - CodexStarter = new CodexStarter(this, configuration); + CodexStarter = new CodexStarter(this, workflowCreator); + PrometheusStarter = new PrometheusStarter(this, workflowCreator); } public TestLog Log { get; } public FileManager FileManager { get; } public CodexStarter CodexStarter { get; } + public PrometheusStarter PrometheusStarter { get; } public void DeleteAllResources() { diff --git a/KubernetesWorkflow/ContainerRecipe.cs b/KubernetesWorkflow/ContainerRecipe.cs index 7e8d90e..f676c7f 100644 --- a/KubernetesWorkflow/ContainerRecipe.cs +++ b/KubernetesWorkflow/ContainerRecipe.cs @@ -17,16 +17,23 @@ public Port[] ExposedPorts { get; } public Port[] InternalPorts { get; } public EnvVar[] EnvVars { get; } + + public Port GetPortByTag(string tag) + { + return ExposedPorts.Concat(InternalPorts).Single(p => p.Tag == tag); + } } public class Port { - public Port(int number) + public Port(int number, string tag) { Number = number; + Tag = tag; } public int Number { get; } + public string Tag { get; } } public class EnvVar diff --git a/KubernetesWorkflow/ContainerRecipeFactory.cs b/KubernetesWorkflow/ContainerRecipeFactory.cs index 60ea91e..6c6a3ee 100644 --- a/KubernetesWorkflow/ContainerRecipeFactory.cs +++ b/KubernetesWorkflow/ContainerRecipeFactory.cs @@ -28,28 +28,28 @@ protected int ContainerNumber { get; private set; } = 0; protected abstract void Initialize(StartupConfig config); - protected Port AddExposedPort() + protected Port AddExposedPort(string tag = "") { - var p = factory.CreatePort(); + var p = factory.CreatePort(tag); exposedPorts.Add(p); return p; } - protected Port AddInternalPort() + protected Port AddInternalPort(string tag = "") { - var p = factory.CreatePort(); + var p = factory.CreatePort(tag); internalPorts.Add(p); return p; } - protected void AddExposedPortAndVar(string name) + protected void AddExposedPortAndVar(string name, string tag = "") { - AddEnvVar(name, AddExposedPort()); + AddEnvVar(name, AddExposedPort(tag)); } - protected void AddInternalPortAndVar(string name) + protected void AddInternalPortAndVar(string name, string tag = "") { - AddEnvVar(name, AddInternalPort()); + AddEnvVar(name, AddInternalPort(tag)); } protected void AddEnvVar(string name, string value) diff --git a/KubernetesWorkflow/K8sController.cs b/KubernetesWorkflow/K8sController.cs index eb492a9..394f86e 100644 --- a/KubernetesWorkflow/K8sController.cs +++ b/KubernetesWorkflow/K8sController.cs @@ -277,7 +277,7 @@ namespace KubernetesWorkflow foreach (var port in recipe.ExposedPorts) { var servicePort = workflowNumberSource.GetServicePort(); - usedPorts.Add(new Port(servicePort)); + usedPorts.Add(new Port(servicePort, "")); result.Add(new V1ServicePort { diff --git a/KubernetesWorkflow/RecipeComponentFactory.cs b/KubernetesWorkflow/RecipeComponentFactory.cs index f99f345..cf1f67b 100644 --- a/KubernetesWorkflow/RecipeComponentFactory.cs +++ b/KubernetesWorkflow/RecipeComponentFactory.cs @@ -7,9 +7,9 @@ namespace KubernetesWorkflow { private NumberSource portNumberSource = new NumberSource(8080); - public Port CreatePort() + public Port CreatePort(string tag) { - return new Port(portNumberSource.GetNextNumber()); + return new Port(portNumberSource.GetNextNumber(), tag); } public EnvVar CreateEnvVar(string name, int value) diff --git a/KubernetesWorkflow/RunningContainers.cs b/KubernetesWorkflow/RunningContainers.cs index 49fc65f..783c6f8 100644 --- a/KubernetesWorkflow/RunningContainers.cs +++ b/KubernetesWorkflow/RunningContainers.cs @@ -12,6 +12,11 @@ public StartupConfig StartupConfig { get; } public RunningPod RunningPod { get; } public RunningContainer[] Containers { get; } + + public string Describe() + { + return $"[{RunningPod.Ip}]"; + } } public class RunningContainer diff --git a/Tests/BasicTests/SimpleTests.cs b/Tests/BasicTests/SimpleTests.cs index 6f6ce14..bf0c331 100644 --- a/Tests/BasicTests/SimpleTests.cs +++ b/Tests/BasicTests/SimpleTests.cs @@ -78,30 +78,30 @@ namespace Tests.BasicTests log.AssertLogContains("Uploaded file"); } - //[Test] - //public void TwoMetricsExample() - //{ - // var group = SetupCodexNodes(2) - // .EnableMetrics() - // .BringOnline(); + [Test] + public void TwoMetricsExample() + { + var group = SetupCodexNodes(2) + .EnableMetrics() + .BringOnline(); - // var group2 = SetupCodexNodes(2) - // .EnableMetrics() - // .BringOnline(); + var group2 = SetupCodexNodes(2) + .EnableMetrics() + .BringOnline(); - // var primary = group[0]; - // var secondary = group[1]; - // var primary2 = group2[0]; - // var secondary2 = group2[1]; + var primary = group[0]; + var secondary = group[1]; + var primary2 = group2[0]; + var secondary2 = group2[1]; - // primary.ConnectToPeer(secondary); - // primary2.ConnectToPeer(secondary2); + primary.ConnectToPeer(secondary); + primary2.ConnectToPeer(secondary2); - // Thread.Sleep(TimeSpan.FromMinutes(5)); + Thread.Sleep(TimeSpan.FromMinutes(5)); - // primary.Metrics.AssertThat("libp2p_peers", Is.EqualTo(1)); - // primary2.Metrics.AssertThat("libp2p_peers", Is.EqualTo(1)); - //} + primary.Metrics.AssertThat("libp2p_peers", Is.EqualTo(1)); + primary2.Metrics.AssertThat("libp2p_peers", Is.EqualTo(1)); + } //[Test] //public void MarketplaceExample() From 9a458832784c029b2e31c356a600c45a00a7e5f9 Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 13 Apr 2023 15:02:51 +0200 Subject: [PATCH 27/46] Restores downloading of metrics on test failure --- DistTestCore/DistTest.cs | 26 ++++++++++++++++++++--- DistTestCore/Metrics/MetricsAccess.cs | 5 +++++ DistTestCore/Metrics/MetricsDownloader.cs | 26 ++++------------------- 3 files changed, 32 insertions(+), 25 deletions(-) diff --git a/DistTestCore/DistTest.cs b/DistTestCore/DistTest.cs index d1830ba..0cfea37 100644 --- a/DistTestCore/DistTest.cs +++ b/DistTestCore/DistTest.cs @@ -1,4 +1,5 @@ using DistTestCore.CodexLogsAndMetrics; +using DistTestCore.Metrics; using NUnit.Framework; namespace DistTestCore @@ -76,7 +77,7 @@ namespace DistTestCore { Log("Downloading all CodexNode logs and metrics because of test failure..."); DownloadAllLogs(); - //k8sManager.DownloadAllMetrics(); + DownloadAllMetrics(); } else { @@ -102,10 +103,29 @@ namespace DistTestCore private void DownloadAllLogs() { - var allNodes = lifecycle.CodexStarter.RunningGroups.SelectMany(g => g.Nodes); - foreach (var node in allNodes) + OnEachCodexNode(node => { lifecycle.DownloadLog(node); + }); + } + + private void DownloadAllMetrics() + { + var metricsDownloader = new MetricsDownloader(lifecycle.Log); + + OnEachCodexNode(node => + { + var m = (MetricsAccess)node.Metrics; + metricsDownloader.DownloadAllMetricsForNode(node.GetName(), m); + }); + } + + private void OnEachCodexNode(Action action) + { + var allNodes = lifecycle.CodexStarter.RunningGroups.SelectMany(g => g.Nodes); + foreach (var node in allNodes) + { + action(node); } } diff --git a/DistTestCore/Metrics/MetricsAccess.cs b/DistTestCore/Metrics/MetricsAccess.cs index 3287ea7..e5bd2b3 100644 --- a/DistTestCore/Metrics/MetricsAccess.cs +++ b/DistTestCore/Metrics/MetricsAccess.cs @@ -37,6 +37,11 @@ namespace DistTestCore.Metrics Assert.That(metricValue, constraint, message); } + public Metrics? GetAllMetrics() + { + return query.GetAllMetricsForNode(node); + } + private MetricsSet GetMetricWithTimeout(string metricName) { var start = DateTime.UtcNow; diff --git a/DistTestCore/Metrics/MetricsDownloader.cs b/DistTestCore/Metrics/MetricsDownloader.cs index 3d79752..1ea56f3 100644 --- a/DistTestCore/Metrics/MetricsDownloader.cs +++ b/DistTestCore/Metrics/MetricsDownloader.cs @@ -6,39 +6,21 @@ namespace DistTestCore.Metrics public class MetricsDownloader { private readonly TestLog log; - private readonly Dictionary activePrometheuses; - public MetricsDownloader(TestLog log, Dictionary activePrometheuses) + public MetricsDownloader(TestLog log) { this.log = log; - this.activePrometheuses = activePrometheuses; } - public void DownloadAllMetrics() + public void DownloadAllMetricsForNode(string nodeName, MetricsAccess access) { - foreach (var pair in activePrometheuses) - { - DownloadAllMetrics(pair.Key, pair.Value); - } - } - - private void DownloadAllMetrics(MetricsQuery query, OnlineCodexNode[] nodes) - { - foreach (var node in nodes) - { - DownloadAllMetricsForNode(query, node); - } - } - - private void DownloadAllMetricsForNode(MetricsQuery query, OnlineCodexNode node) - { - var metrics = query.GetAllMetricsForNode(node.CodexAccess.Container); + var metrics = access.GetAllMetrics(); if (metrics == null || metrics.Sets.Length == 0 || metrics.Sets.All(s => s.Values.Length == 0)) return; var headers = new[] { "timestamp" }.Concat(metrics.Sets.Select(s => s.Name)).ToArray(); var map = CreateValueMap(metrics); - WriteToFile(node.GetName(), headers, map); + WriteToFile(nodeName, headers, map); } private void WriteToFile(string nodeName, string[] headers, Dictionary> map) From 07fbda3f9a39002426f53e1533c9328f28ead453 Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 13 Apr 2023 15:04:01 +0200 Subject: [PATCH 28/46] Allows for metrics collection when some codex groups don't have metrics enabled. --- DistTestCore/DistTest.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/DistTestCore/DistTest.cs b/DistTestCore/DistTest.cs index 0cfea37..a088e7c 100644 --- a/DistTestCore/DistTest.cs +++ b/DistTestCore/DistTest.cs @@ -115,8 +115,11 @@ namespace DistTestCore OnEachCodexNode(node => { - var m = (MetricsAccess)node.Metrics; - metricsDownloader.DownloadAllMetricsForNode(node.GetName(), m); + var m = node.Metrics as MetricsAccess; + if (m != null) + { + metricsDownloader.DownloadAllMetricsForNode(node.GetName(), m); + } }); } From 4fd00607dfd42526bee153456376198e4b4b438e Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 14 Apr 2023 09:54:07 +0200 Subject: [PATCH 29/46] Setting up Geth starters --- CodexDistTestCore/K8sOperations.cs | 42 +------------- CodexDistTestCore/Marketplace/K8sGethSpecs.cs | 4 +- DistTestCore/CodexStarter.cs | 2 + DistTestCore/DistTest.cs | 2 +- DistTestCore/GethStarter.cs | 44 ++++++++++++++ .../CodexNodeLog.cs | 2 +- ...ownloadLogsAndMetricsOnFailureAttribute.cs | 2 +- .../LogDownloadHandler.cs | 2 +- .../Marketplace/GethBootstrapNodeInfo.cs | 18 ++++++ .../Marketplace/GethBootstrapNodeStarter.cs | 47 +++++++++++++++ .../Marketplace/GethCompanionNodeInfo.cs | 16 +++++ .../Marketplace/GethCompanionNodeStarter.cs | 50 ++++++++++++++++ .../Marketplace/GethContainerRecipe.cs | 37 ++++++++++++ DistTestCore/Marketplace/GethInfoExtractor.cs | 58 +++++++++++++++++++ DistTestCore/Marketplace/GethStartupConfig.cs | 14 +++++ DistTestCore/OnlineCodexNode.cs | 2 +- DistTestCore/TestLifecycle.cs | 4 +- KubernetesWorkflow/CommandRunner.cs | 52 +++++++++++++++++ KubernetesWorkflow/K8sController.cs | 7 +++ KubernetesWorkflow/StartupWorkflow.cs | 8 +++ 20 files changed, 363 insertions(+), 50 deletions(-) create mode 100644 DistTestCore/GethStarter.cs rename DistTestCore/{CodexLogsAndMetrics => Logs}/CodexNodeLog.cs (95%) rename DistTestCore/{CodexLogsAndMetrics => Logs}/DontDownloadLogsAndMetricsOnFailureAttribute.cs (90%) rename DistTestCore/{CodexLogsAndMetrics => Logs}/LogDownloadHandler.cs (95%) create mode 100644 DistTestCore/Marketplace/GethBootstrapNodeInfo.cs create mode 100644 DistTestCore/Marketplace/GethBootstrapNodeStarter.cs create mode 100644 DistTestCore/Marketplace/GethCompanionNodeInfo.cs create mode 100644 DistTestCore/Marketplace/GethCompanionNodeStarter.cs create mode 100644 DistTestCore/Marketplace/GethContainerRecipe.cs create mode 100644 DistTestCore/Marketplace/GethInfoExtractor.cs create mode 100644 DistTestCore/Marketplace/GethStartupConfig.cs create mode 100644 KubernetesWorkflow/CommandRunner.cs diff --git a/CodexDistTestCore/K8sOperations.cs b/CodexDistTestCore/K8sOperations.cs index 73f04b0..0cc3ea1 100644 --- a/CodexDistTestCore/K8sOperations.cs +++ b/CodexDistTestCore/K8sOperations.cs @@ -343,47 +343,7 @@ namespace CodexDistTestCore private class CommandRunner { - private readonly Kubernetes client; - private readonly PodInfo pod; - private readonly string containerName; - private readonly string command; - private readonly string[] arguments; - private readonly List lines = new List(); - - public CommandRunner(Kubernetes client, PodInfo pod, string containerName, string command, string[] arguments) - { - this.client = client; - this.pod = pod; - this.containerName = containerName; - this.command = command; - this.arguments = arguments; - } - - public void Run() - { - var input = new[] { command }.Concat(arguments).ToArray(); - - Utils.Wait(client.NamespacedPodExecAsync( - pod.Name, K8sCluster.K8sNamespace, containerName, input, false, Callback, new CancellationToken())); - } - - public string GetStdOut() - { - return string.Join(Environment.NewLine, lines); - } - - private Task Callback(Stream stdIn, Stream stdOut, Stream stdErr) - { - using var streamReader = new StreamReader(stdOut); - var line = streamReader.ReadLine(); - while (line != null) - { - lines.Add(line); - line = streamReader.ReadLine(); - } - - return Task.CompletedTask; - } + } } } diff --git a/CodexDistTestCore/Marketplace/K8sGethSpecs.cs b/CodexDistTestCore/Marketplace/K8sGethSpecs.cs index f99d520..bfaf3d4 100644 --- a/CodexDistTestCore/Marketplace/K8sGethSpecs.cs +++ b/CodexDistTestCore/Marketplace/K8sGethSpecs.cs @@ -6,15 +6,13 @@ namespace CodexDistTestCore.Marketplace public static class GethDockerImage { public const string Image = "thatbenbierens/geth-confenv:latest"; - public const string AccountFilename = "account_string.txt"; - public const string GenesisFilename = "genesis.json"; + } public class K8sGethBoostrapSpecs { public const string ContainerName = "dtest-gethb"; private const string portName = "gethb"; - private const string genesisJsonBase64 = "ewogICAgImNvbmZpZyI6IHsKICAgICAgImNoYWluSWQiOiA3ODk5ODgsCiAgICAgICJob21lc3RlYWRCbG9jayI6IDAsCiAgICAgICJlaXAxNTBCbG9jayI6IDAsCiAgICAgICJlaXAxNTVCbG9jayI6IDAsCiAgICAgICJlaXAxNThCbG9jayI6IDAsCiAgICAgICJieXphbnRpdW1CbG9jayI6IDAsCiAgICAgICJjb25zdGFudGlub3BsZUJsb2NrIjogMCwKICAgICAgInBldGVyc2J1cmdCbG9jayI6IDAsCiAgICAgICJpc3RhbmJ1bEJsb2NrIjogMCwKICAgICAgIm11aXJHbGFjaWVyQmxvY2siOiAwLAogICAgICAiYmVybGluQmxvY2siOiAwLAogICAgICAibG9uZG9uQmxvY2siOiAwLAogICAgICAiYXJyb3dHbGFjaWVyQmxvY2siOiAwLAogICAgICAiZ3JheUdsYWNpZXJCbG9jayI6IDAsCiAgICAgICJjbGlxdWUiOiB7CiAgICAgICAgInBlcmlvZCI6IDUsCiAgICAgICAgImVwb2NoIjogMzAwMDAKICAgICAgfQogICAgfSwKICAgICJkaWZmaWN1bHR5IjogIjEiLAogICAgImdhc0xpbWl0IjogIjgwMDAwMDAwMCIsCiAgICAiZXh0cmFkYXRhIjogIjB4MDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMEFDQ09VTlRfSEVSRTAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAiLAogICAgImFsbG9jIjogewogICAgICAiMHhBQ0NPVU5UX0hFUkUiOiB7ICJiYWxhbmNlIjogIjUwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAiIH0KICAgIH0KICB9"; public K8sGethBoostrapSpecs(int servicePort) { diff --git a/DistTestCore/CodexStarter.cs b/DistTestCore/CodexStarter.cs index 65c9a32..b85fba4 100644 --- a/DistTestCore/CodexStarter.cs +++ b/DistTestCore/CodexStarter.cs @@ -18,6 +18,8 @@ namespace DistTestCore public ICodexNodeGroup BringOnline(CodexSetup codexSetup) { + var something = lifecycle.GethStarter.BringOnlineMarketplaceFor(codexSetup); + var containers = StartCodexContainers(codexSetup); var metricAccessFactory = lifecycle.PrometheusStarter.CollectMetricsFor(codexSetup, containers); diff --git a/DistTestCore/DistTest.cs b/DistTestCore/DistTest.cs index a088e7c..e3631f4 100644 --- a/DistTestCore/DistTest.cs +++ b/DistTestCore/DistTest.cs @@ -1,4 +1,4 @@ -using DistTestCore.CodexLogsAndMetrics; +using DistTestCore.Logs; using DistTestCore.Metrics; using NUnit.Framework; diff --git a/DistTestCore/GethStarter.cs b/DistTestCore/GethStarter.cs new file mode 100644 index 0000000..bf26523 --- /dev/null +++ b/DistTestCore/GethStarter.cs @@ -0,0 +1,44 @@ +using DistTestCore.Marketplace; +using KubernetesWorkflow; + +namespace DistTestCore +{ + public class GethStarter + { + private readonly TestLifecycle lifecycle; + private readonly WorkflowCreator workflowCreator; + private readonly GethBootstrapNodeStarter bootstrapNodeStarter; + private GethBootstrapNodeInfo? bootstrapNode; + + public GethStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator) + { + this.lifecycle = lifecycle; + this.workflowCreator = workflowCreator; + + bootstrapNodeStarter = new GethBootstrapNodeStarter(lifecycle, workflowCreator); + } + + public object BringOnlineMarketplaceFor(CodexSetup codexSetup) + { + EnsureBootstrapNode(); + StartCompanionNodes(codexSetup); + return null!; + } + + private void EnsureBootstrapNode() + { + if (bootstrapNode != null) return; + bootstrapNode = bootstrapNodeStarter.StartGethBootstrapNode(); + } + + private void StartCompanionNodes(CodexSetup codexSetup) + { + throw new NotImplementedException(); + } + + private void Log(string msg) + { + lifecycle.Log.Log(msg); + } + } +} diff --git a/DistTestCore/CodexLogsAndMetrics/CodexNodeLog.cs b/DistTestCore/Logs/CodexNodeLog.cs similarity index 95% rename from DistTestCore/CodexLogsAndMetrics/CodexNodeLog.cs rename to DistTestCore/Logs/CodexNodeLog.cs index a4a9cb0..ac92678 100644 --- a/DistTestCore/CodexLogsAndMetrics/CodexNodeLog.cs +++ b/DistTestCore/Logs/CodexNodeLog.cs @@ -1,7 +1,7 @@ using Logging; using NUnit.Framework; -namespace DistTestCore.CodexLogsAndMetrics +namespace DistTestCore.Logs { public interface ICodexNodeLog { diff --git a/DistTestCore/CodexLogsAndMetrics/DontDownloadLogsAndMetricsOnFailureAttribute.cs b/DistTestCore/Logs/DontDownloadLogsAndMetricsOnFailureAttribute.cs similarity index 90% rename from DistTestCore/CodexLogsAndMetrics/DontDownloadLogsAndMetricsOnFailureAttribute.cs rename to DistTestCore/Logs/DontDownloadLogsAndMetricsOnFailureAttribute.cs index 0cf1cbe..b95d875 100644 --- a/DistTestCore/CodexLogsAndMetrics/DontDownloadLogsAndMetricsOnFailureAttribute.cs +++ b/DistTestCore/Logs/DontDownloadLogsAndMetricsOnFailureAttribute.cs @@ -1,6 +1,6 @@ using NUnit.Framework; -namespace DistTestCore.CodexLogsAndMetrics +namespace DistTestCore.Logs { [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] public class DontDownloadLogsAndMetricsOnFailureAttribute : PropertyAttribute diff --git a/DistTestCore/CodexLogsAndMetrics/LogDownloadHandler.cs b/DistTestCore/Logs/LogDownloadHandler.cs similarity index 95% rename from DistTestCore/CodexLogsAndMetrics/LogDownloadHandler.cs rename to DistTestCore/Logs/LogDownloadHandler.cs index 59c14d8..b6136c3 100644 --- a/DistTestCore/CodexLogsAndMetrics/LogDownloadHandler.cs +++ b/DistTestCore/Logs/LogDownloadHandler.cs @@ -1,7 +1,7 @@ using KubernetesWorkflow; using Logging; -namespace DistTestCore.CodexLogsAndMetrics +namespace DistTestCore.Logs { public class LogDownloadHandler : ILogHandler { diff --git a/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs b/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs new file mode 100644 index 0000000..8f897ef --- /dev/null +++ b/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs @@ -0,0 +1,18 @@ +using KubernetesWorkflow; + +namespace DistTestCore.Marketplace +{ + public class GethBootstrapNodeInfo + { + public GethBootstrapNodeInfo(RunningContainers runningContainers, string account, string genesisJsonBase64) + { + RunningContainers = runningContainers; + Account = account; + GenesisJsonBase64 = genesisJsonBase64; + } + + public RunningContainers RunningContainers { get; } + public string Account { get; } + public string GenesisJsonBase64 { get; } + } +} diff --git a/DistTestCore/Marketplace/GethBootstrapNodeStarter.cs b/DistTestCore/Marketplace/GethBootstrapNodeStarter.cs new file mode 100644 index 0000000..7c114c4 --- /dev/null +++ b/DistTestCore/Marketplace/GethBootstrapNodeStarter.cs @@ -0,0 +1,47 @@ +using KubernetesWorkflow; + +namespace DistTestCore.Marketplace +{ + public class GethBootstrapNodeStarter + { + private const string bootstrapGenesisJsonBase64 = "ewogICAgImNvbmZpZyI6IHsKICAgICAgImNoYWluSWQiOiA3ODk5ODgsCiAgICAgICJob21lc3RlYWRCbG9jayI6IDAsCiAgICAgICJlaXAxNTBCbG9jayI6IDAsCiAgICAgICJlaXAxNTVCbG9jayI6IDAsCiAgICAgICJlaXAxNThCbG9jayI6IDAsCiAgICAgICJieXphbnRpdW1CbG9jayI6IDAsCiAgICAgICJjb25zdGFudGlub3BsZUJsb2NrIjogMCwKICAgICAgInBldGVyc2J1cmdCbG9jayI6IDAsCiAgICAgICJpc3RhbmJ1bEJsb2NrIjogMCwKICAgICAgIm11aXJHbGFjaWVyQmxvY2siOiAwLAogICAgICAiYmVybGluQmxvY2siOiAwLAogICAgICAibG9uZG9uQmxvY2siOiAwLAogICAgICAiYXJyb3dHbGFjaWVyQmxvY2siOiAwLAogICAgICAiZ3JheUdsYWNpZXJCbG9jayI6IDAsCiAgICAgICJjbGlxdWUiOiB7CiAgICAgICAgInBlcmlvZCI6IDUsCiAgICAgICAgImVwb2NoIjogMzAwMDAKICAgICAgfQogICAgfSwKICAgICJkaWZmaWN1bHR5IjogIjEiLAogICAgImdhc0xpbWl0IjogIjgwMDAwMDAwMCIsCiAgICAiZXh0cmFkYXRhIjogIjB4MDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMEFDQ09VTlRfSEVSRTAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAiLAogICAgImFsbG9jIjogewogICAgICAiMHhBQ0NPVU5UX0hFUkUiOiB7ICJiYWxhbmNlIjogIjUwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAiIH0KICAgIH0KICB9"; + private readonly TestLifecycle lifecycle; + private readonly WorkflowCreator workflowCreator; + + public GethBootstrapNodeStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator) + { + this.lifecycle = lifecycle; + this.workflowCreator = workflowCreator; + } + + public GethBootstrapNodeInfo StartGethBootstrapNode() + { + Log("Starting Geth bootstrap node..."); + var startupConfig = CreateBootstrapStartupConfig(); + + var workflow = workflowCreator.CreateWorkflow(); + var containers = workflow.Start(1, Location.Unspecified, new GethContainerRecipe(), startupConfig); + if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 Geth bootstrap node to be created. Test infra failure."); + + var extractor = new GethInfoExtractor(workflow, containers.Containers[0]); + var account = extractor.ExtractAccount(); + var genesisJsonBase64 = extractor.ExtractGenesisJsonBase64(); + + Log($"Geth bootstrap node started with account '{account}'"); + + return new GethBootstrapNodeInfo(containers, account, genesisJsonBase64); + } + + private StartupConfig CreateBootstrapStartupConfig() + { + var config = new StartupConfig(); + config.Add(new GethStartupConfig(true, bootstrapGenesisJsonBase64)); + return config; + } + + private void Log(string msg) + { + lifecycle.Log.Log(msg); + } + } +} diff --git a/DistTestCore/Marketplace/GethCompanionNodeInfo.cs b/DistTestCore/Marketplace/GethCompanionNodeInfo.cs new file mode 100644 index 0000000..9b7bd23 --- /dev/null +++ b/DistTestCore/Marketplace/GethCompanionNodeInfo.cs @@ -0,0 +1,16 @@ +using KubernetesWorkflow; + +namespace DistTestCore.Marketplace +{ + public class GethCompanionNodeInfo + { + public GethCompanionNodeInfo(RunningContainer runningContainer, string account) + { + RunningContainer = runningContainer; + Account = account; + } + + public RunningContainer RunningContainer { get; } + public string Account { get; } + } +} diff --git a/DistTestCore/Marketplace/GethCompanionNodeStarter.cs b/DistTestCore/Marketplace/GethCompanionNodeStarter.cs new file mode 100644 index 0000000..b8b178b --- /dev/null +++ b/DistTestCore/Marketplace/GethCompanionNodeStarter.cs @@ -0,0 +1,50 @@ +using KubernetesWorkflow; + +namespace DistTestCore.Marketplace +{ + public class GethCompanionNodeStarter + { + private readonly TestLifecycle lifecycle; + private readonly WorkflowCreator workflowCreator; + + public GethCompanionNodeStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator) + { + this.lifecycle = lifecycle; + this.workflowCreator = workflowCreator; + } + + public GethCompanionNodeInfo[] StartCompanionNodesFor(CodexSetup codexSetup, GethBootstrapNodeInfo bootstrapNode) + { + Log($"Initializing companions for {codexSetup.NumberOfNodes} Codex nodes."); + + var startupConfig = CreateCompanionNodeStartupConfig(bootstrapNode); + + var workflow = workflowCreator.CreateWorkflow(); + var containers = workflow.Start(codexSetup.NumberOfNodes, Location.Unspecified, new GethContainerRecipe(), startupConfig); + if (containers.Containers.Length != codexSetup.NumberOfNodes) throw new InvalidOperationException("Expected a Geth companion node to be created for each Codex node. Test infra failure."); + + Log("Initialized companion nodes."); + + return containers.Containers.Select(c => CreateCompanionInfo(workflow, c)).ToArray(); + } + + private GethCompanionNodeInfo CreateCompanionInfo(StartupWorkflow workflow, RunningContainer container) + { + var extractor = new GethInfoExtractor(workflow, container); + var account = extractor.ExtractAccount(); + return new GethCompanionNodeInfo(container, account); + } + + private StartupConfig CreateCompanionNodeStartupConfig(GethBootstrapNodeInfo bootstrapNode) + { + var config = new StartupConfig(); + config.Add(new GethStartupConfig(false, bootstrapNode.GenesisJsonBase64)); + return config; + } + + private void Log(string msg) + { + lifecycle.Log.Log(msg); + } + } +} diff --git a/DistTestCore/Marketplace/GethContainerRecipe.cs b/DistTestCore/Marketplace/GethContainerRecipe.cs new file mode 100644 index 0000000..3df72f6 --- /dev/null +++ b/DistTestCore/Marketplace/GethContainerRecipe.cs @@ -0,0 +1,37 @@ +using KubernetesWorkflow; + +namespace DistTestCore.Marketplace +{ + public class GethContainerRecipe : ContainerRecipeFactory + { + protected override string Image => "thatbenbierens/geth-confenv:latest"; + public const string AccountFilename = "account_string.txt"; + public const string GenesisFilename = "genesis.json"; + + protected override void Initialize(StartupConfig startupConfig) + { + var config = startupConfig.Get(); + + var args = CreateArgs(config); + + AddEnvVar("GETH_ARGS", args); + AddEnvVar("GENESIS_JSON", config.GenesisJsonBase64); + } + + private string CreateArgs(GethStartupConfig config) + { + if (config.IsBootstrapNode) + { + AddEnvVar("IS_BOOTSTRAP", "1"); + var exposedPort = AddExposedPort(); + return $"--http.port {exposedPort.Number}"; + } + + var port = AddInternalPort(); + var discovery = AddInternalPort(); + var authRpc = AddInternalPort(); + var httpPort = AddInternalPort(); + return $"--port {port.Number} --discovery.port {discovery.Number} --authrpc.port {authRpc.Number} --http.port {httpPort.Number}"; + } + } +} diff --git a/DistTestCore/Marketplace/GethInfoExtractor.cs b/DistTestCore/Marketplace/GethInfoExtractor.cs new file mode 100644 index 0000000..f27e396 --- /dev/null +++ b/DistTestCore/Marketplace/GethInfoExtractor.cs @@ -0,0 +1,58 @@ +using KubernetesWorkflow; +using System.Text; + +namespace DistTestCore.Marketplace +{ + public class GethInfoExtractor + { + private readonly StartupWorkflow workflow; + private readonly RunningContainer container; + + public GethInfoExtractor(StartupWorkflow workflow, RunningContainer container) + { + this.workflow = workflow; + this.container = container; + } + + public string ExtractAccount() + { + var account = Retry(FetchAccount); + + if (string.IsNullOrEmpty(account)) throw new InvalidOperationException("Unable to fetch account for geth node. Test infra failure."); + + return account; + } + + public string ExtractGenesisJsonBase64() + { + var genesisJson = Retry(FetchGenesisJson); + + if (string.IsNullOrEmpty(genesisJson)) throw new InvalidOperationException("Unable to fetch genesis-json for geth node. Test infra failure."); + + var encoded = Convert.ToBase64String(Encoding.ASCII.GetBytes(genesisJson)); + + return encoded; + } + + private string Retry(Func fetch) + { + var result = fetch(); + if (string.IsNullOrEmpty(result)) + { + Thread.Sleep(TimeSpan.FromSeconds(5)); + result = fetch(); + } + return result; + } + + private string FetchGenesisJson() + { + return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.GenesisFilename); + } + + private string FetchAccount() + { + return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.AccountFilename); + } + } +} diff --git a/DistTestCore/Marketplace/GethStartupConfig.cs b/DistTestCore/Marketplace/GethStartupConfig.cs new file mode 100644 index 0000000..60164bb --- /dev/null +++ b/DistTestCore/Marketplace/GethStartupConfig.cs @@ -0,0 +1,14 @@ +namespace DistTestCore.Marketplace +{ + public class GethStartupConfig + { + public GethStartupConfig(bool isBootstrapNode, string genesisJsonBase64) + { + IsBootstrapNode = isBootstrapNode; + GenesisJsonBase64 = genesisJsonBase64; + } + + public bool IsBootstrapNode { get; } + public string GenesisJsonBase64 { get; } + } +} diff --git a/DistTestCore/OnlineCodexNode.cs b/DistTestCore/OnlineCodexNode.cs index 0efa51a..76ab592 100644 --- a/DistTestCore/OnlineCodexNode.cs +++ b/DistTestCore/OnlineCodexNode.cs @@ -1,5 +1,5 @@ using DistTestCore.Codex; -using DistTestCore.CodexLogsAndMetrics; +using DistTestCore.Logs; using DistTestCore.Metrics; using NUnit.Framework; diff --git a/DistTestCore/TestLifecycle.cs b/DistTestCore/TestLifecycle.cs index a5127a7..63d3c7a 100644 --- a/DistTestCore/TestLifecycle.cs +++ b/DistTestCore/TestLifecycle.cs @@ -1,4 +1,4 @@ -using DistTestCore.CodexLogsAndMetrics; +using DistTestCore.Logs; using KubernetesWorkflow; using Logging; @@ -16,12 +16,14 @@ namespace DistTestCore FileManager = new FileManager(Log, configuration); CodexStarter = new CodexStarter(this, workflowCreator); PrometheusStarter = new PrometheusStarter(this, workflowCreator); + GethStarter = new GethStarter(this, workflowCreator); } public TestLog Log { get; } public FileManager FileManager { get; } public CodexStarter CodexStarter { get; } public PrometheusStarter PrometheusStarter { get; } + public GethStarter GethStarter { get; } public void DeleteAllResources() { diff --git a/KubernetesWorkflow/CommandRunner.cs b/KubernetesWorkflow/CommandRunner.cs new file mode 100644 index 0000000..a045500 --- /dev/null +++ b/KubernetesWorkflow/CommandRunner.cs @@ -0,0 +1,52 @@ +using k8s; +using Utils; + +namespace KubernetesWorkflow +{ + public class CommandRunner + { + private readonly Kubernetes client; + private readonly string k8sNamespace; + private readonly RunningPod pod; + private readonly string containerName; + private readonly string command; + private readonly string[] arguments; + private readonly List lines = new List(); + + public CommandRunner(Kubernetes client, string k8sNamespace, RunningPod pod, string containerName, string command, string[] arguments) + { + this.client = client; + this.k8sNamespace = k8sNamespace; + this.pod = pod; + this.containerName = containerName; + this.command = command; + this.arguments = arguments; + } + + public void Run() + { + var input = new[] { command }.Concat(arguments).ToArray(); + + Time.Wait(client.NamespacedPodExecAsync( + pod.Name, k8sNamespace, containerName, input, false, Callback, new CancellationToken())); + } + + public string GetStdOut() + { + return string.Join(Environment.NewLine, lines); + } + + private Task Callback(Stream stdIn, Stream stdOut, Stream stdErr) + { + using var streamReader = new StreamReader(stdOut); + var line = streamReader.ReadLine(); + while (line != null) + { + lines.Add(line); + line = streamReader.ReadLine(); + } + + return Task.CompletedTask; + } + } +} diff --git a/KubernetesWorkflow/K8sController.cs b/KubernetesWorkflow/K8sController.cs index 394f86e..0804e8f 100644 --- a/KubernetesWorkflow/K8sController.cs +++ b/KubernetesWorkflow/K8sController.cs @@ -49,6 +49,13 @@ namespace KubernetesWorkflow logHandler.Log(stream); } + public string ExecuteCommand(RunningPod pod, string containerName, string command, params string[] args) + { + var runner = new CommandRunner(client, K8sNamespace, pod, containerName, command, args); + runner.Run(); + return runner.GetStdOut(); + } + public void DeleteAllResources() { DeleteNamespace(); diff --git a/KubernetesWorkflow/StartupWorkflow.cs b/KubernetesWorkflow/StartupWorkflow.cs index 9dff328..e78a5b2 100644 --- a/KubernetesWorkflow/StartupWorkflow.cs +++ b/KubernetesWorkflow/StartupWorkflow.cs @@ -42,6 +42,14 @@ }); } + public string ExecuteCommand(RunningContainer container, string command, params string[] args) + { + return K8s(controller => + { + return controller.ExecuteCommand(container.Pod, container.Recipe.Name, command, args); + }); + } + public void DeleteAllResources() { K8s(controller => From 3d908bab6c272f63a91291c7066e64df0203a16f Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 14 Apr 2023 10:51:35 +0200 Subject: [PATCH 30/46] Setting up nethereum library --- .../Marketplace/MarketplaceController.cs | 135 ------------------ .../Marketplace/MarketplaceInitialConfig.cs | 8 -- DistTestCore/Codex/CodexContainerRecipe.cs | 17 ++- DistTestCore/Codex/CodexStartupConfig.cs | 5 +- DistTestCore/CodexNodeFactory.cs | 8 +- DistTestCore/CodexSetup.cs | 13 +- DistTestCore/CodexStarter.cs | 23 ++- DistTestCore/DistTestCore.csproj | 2 +- DistTestCore/GethStarter.cs | 39 +++-- .../Marketplace/GethContainerRecipe.cs | 3 +- DistTestCore/Marketplace/GethStartResult.cs | 16 +++ DistTestCore/Marketplace/MarketplaceAccess.cs | 85 +++++++++++ .../Marketplace/MarketplaceAccessFactory.cs | 24 ++++ .../Marketplace/MarketplaceInitialConfig.cs | 12 ++ DistTestCore/OnlineCodexNode.cs | 7 +- KubernetesWorkflow/ContainerRecipeFactory.cs | 4 +- KubernetesWorkflow/StartupWorkflow.cs | 2 +- Nethereum/NethereumWorkflow.cs | 41 ++++++ Nethereum/NethereumWorkflow.csproj | 19 +++ Nethereum/NethereumWorkflowCreator.cs | 29 ++++ cs-codex-dist-testing.sln | 14 +- 21 files changed, 318 insertions(+), 188 deletions(-) create mode 100644 DistTestCore/Marketplace/GethStartResult.cs create mode 100644 DistTestCore/Marketplace/MarketplaceAccess.cs create mode 100644 DistTestCore/Marketplace/MarketplaceAccessFactory.cs create mode 100644 DistTestCore/Marketplace/MarketplaceInitialConfig.cs create mode 100644 Nethereum/NethereumWorkflow.cs create mode 100644 Nethereum/NethereumWorkflow.csproj create mode 100644 Nethereum/NethereumWorkflowCreator.cs diff --git a/CodexDistTestCore/Marketplace/MarketplaceController.cs b/CodexDistTestCore/Marketplace/MarketplaceController.cs index 21bfc08..8bab9e9 100644 --- a/CodexDistTestCore/Marketplace/MarketplaceController.cs +++ b/CodexDistTestCore/Marketplace/MarketplaceController.cs @@ -1,7 +1,4 @@ using CodexDistTestCore.Config; -using Nethereum.Web3; -using Nethereum.Web3.Accounts; -using Nethereum.Web3.Accounts.Managed; using NUnit.Framework; using System.Numerics; using System.Text; @@ -22,138 +19,6 @@ namespace CodexDistTestCore.Marketplace this.k8sManager = k8sManager; } - public GethCompanionGroup BringOnlineMarketplace(OfflineCodexNodes offline) - { - if (bootstrapInfo == null) - { - BringOnlineBootstrapNode(); - } - log.Log($"Initializing companions for {offline.NumberOfNodes} Codex nodes."); - - var group = new GethCompanionGroup(companionGroupNumberSource.GetNextNumber(), CreateCompanionContainers(offline)); - group.Pod = k8sManager.BringOnlineGethCompanionGroup(bootstrapInfo!, group); - companionGroups.Add(group); - - log.Log("Initialized companion nodes."); - return group; - } - - private void BringOnlineBootstrapNode() - { - log.Log("Starting Geth bootstrap node..."); - var spec = k8sManager.CreateGethBootstrapNodeSpec(); - var pod = k8sManager.BringOnlineGethBootstrapNode(spec); - var (account, genesisJson) = ExtractAccountAndGenesisJson(pod); - bootstrapInfo = new GethBootstrapInfo(spec, pod, account, genesisJson); - log.Log($"Geth boothstrap node started."); - } - - private GethCompanionNodeContainer[] CreateCompanionContainers(OfflineCodexNodes offline) - { - var numberSource = new NumberSource(8080); - var result = new List(); - for (var i = 0; i < offline.NumberOfNodes; i++) result.Add(CreateGethNodeContainer(numberSource, i)); - return result.ToArray(); - } - - private GethCompanionNodeContainer CreateGethNodeContainer(NumberSource numberSource, int n) - { - return new GethCompanionNodeContainer( - name: $"geth-node{n}", - apiPort: numberSource.GetNextNumber(), - authRpcPort: numberSource.GetNextNumber(), - rpcPort: numberSource.GetNextNumber(), - containerPortName: $"geth-{n}" - ); - } - - private readonly K8sCluster k8sCluster = new K8sCluster(); - - public void AddToBalance(string account, decimal amount) - { - if (amount < 1 || string.IsNullOrEmpty(account)) Assert.Fail("Invalid arguments for AddToBalance"); - - // call the bootstrap node and convince it to give 'account' 'amount' tokens somehow. - - var web3 = CreateWeb3(); - - //var blockNumber1 = Utils.Wait(web3.Eth.Blocks.GetBlockNumber.SendRequestAsync()); - //Thread.Sleep(TimeSpan.FromSeconds(5)); - //var blockNumber2 = Utils.Wait(web3.Eth.Blocks.GetBlockNumber.SendRequestAsync()); - - //var bootstrapBalance = Utils.Wait(web3.Eth.GetBalance.SendRequestAsync(bootstrapInfo.Account)); - - - var bigint = new BigInteger(amount); - var str = bigint.ToString("X"); - var value = new Nethereum.Hex.HexTypes.HexBigInteger(str); - var aaa = Utils.Wait(web3.Eth.TransactionManager.SendTransactionAsync(bootstrapInfo!.Account, account, value)); - var receipt = Utils.Wait(web3.Eth.TransactionManager.TransactionReceiptService.PollForReceiptAsync(aaa)); - - //var receipt = Utils.Wait(web3.Eth.GetEtherTransferService().TransferEtherAndWaitForReceiptAsync(account, amount)); - //var targetBalance = Utils.Wait(web3.Eth.GetBalance.SendRequestAsync(account)); - } - - public decimal GetBalance(string account) - { - var web3 = CreateWeb3(); - var bigInt = Utils.Wait(web3.Eth.GetBalance.SendRequestAsync(account)); - return (decimal)bigInt.Value; - } - - private Web3 CreateWeb3() - { - var ip = k8sCluster.GetIp(); - var port = bootstrapInfo!.Spec.ServicePort; - //var bootstrapaccount = new ManagedAccount(bootstrapInfo.Account, "qwerty!@#$%^"); - return new Web3($"http://{ip}:{port}"); - } - - private (string, string) ExtractAccountAndGenesisJson(PodInfo pod) - { - var (account, genesisJson) = FetchAccountAndGenesisJson(pod); - if (string.IsNullOrEmpty(account) || string.IsNullOrEmpty(genesisJson)) - { - Thread.Sleep(TimeSpan.FromSeconds(15)); - (account, genesisJson) = FetchAccountAndGenesisJson(pod); - } - - Assert.That(account, Is.Not.Empty, "Unable to fetch account for geth bootstrap node. Test infra failure."); - Assert.That(genesisJson, Is.Not.Empty, "Unable to fetch genesis-json for geth bootstrap node. Test infra failure."); - - var encoded = Convert.ToBase64String(Encoding.ASCII.GetBytes(genesisJson)); - - log.Log($"Initialized geth bootstrap node with account '{account}'"); - return (account, encoded); - } - - private (string, string) FetchAccountAndGenesisJson(PodInfo pod) - { - var bootstrapAccount = ExecuteCommand(pod, "cat", GethDockerImage.AccountFilename); - var bootstrapGenesisJson = ExecuteCommand(pod, "cat", GethDockerImage.GenesisFilename); - return (bootstrapAccount, bootstrapGenesisJson); - } - - private string ExecuteCommand(PodInfo pod, string command, params string[] arguments) - { - return k8sManager.ExecuteCommand(pod, K8sGethBoostrapSpecs.ContainerName, command, arguments); - } - } - - public class GethBootstrapInfo - { - public GethBootstrapInfo(K8sGethBoostrapSpecs spec, PodInfo pod, string account, string genesisJsonBase64) - { - Spec = spec; - Pod = pod; - Account = account; - GenesisJsonBase64 = genesisJsonBase64; - } - - public K8sGethBoostrapSpecs Spec { get; } - public PodInfo Pod { get; } - public string Account { get; } - public string GenesisJsonBase64 { get; } } } diff --git a/CodexDistTestCore/Marketplace/MarketplaceInitialConfig.cs b/CodexDistTestCore/Marketplace/MarketplaceInitialConfig.cs index f7761f5..23ad25f 100644 --- a/CodexDistTestCore/Marketplace/MarketplaceInitialConfig.cs +++ b/CodexDistTestCore/Marketplace/MarketplaceInitialConfig.cs @@ -1,12 +1,4 @@ namespace CodexDistTestCore.Marketplace { - public class MarketplaceInitialConfig - { - public MarketplaceInitialConfig(int initialBalance) - { - InitialBalance = initialBalance; - } - public int InitialBalance { get; } - } } diff --git a/DistTestCore/Codex/CodexContainerRecipe.cs b/DistTestCore/Codex/CodexContainerRecipe.cs index e49dd84..31a9a7f 100644 --- a/DistTestCore/Codex/CodexContainerRecipe.cs +++ b/DistTestCore/Codex/CodexContainerRecipe.cs @@ -1,4 +1,5 @@ -using KubernetesWorkflow; +using DistTestCore.Marketplace; +using KubernetesWorkflow; namespace DistTestCore.Codex { @@ -32,6 +33,20 @@ namespace DistTestCore.Codex AddEnvVar("METRICS_ADDR", "0.0.0.0"); AddInternalPortAndVar("METRICS_PORT", tag: MetricsPortTag); } + + if (config.MarketplaceConfig != null) + { + var gethConfig = startupConfig.Get(); + var companionNode = gethConfig.CompanionNodes[Index]; + + // Bootstrap node access from within the cluster: + //var ip = gethConfig.BootstrapNode.RunningContainers.RunningPod.Ip; + //var port = gethConfig.BootstrapNode.RunningContainers.Containers[0].Recipe.GetPortByTag(GethContainerRecipe.HttpPortTag); + + //AddEnvVar("ETH_PROVIDER", "todo"); + //AddEnvVar("ETH_ACCOUNT", companionNode.Account); + //AddEnvVar("ETH_DEPLOYMENT", "todo"); + } } } } diff --git a/DistTestCore/Codex/CodexStartupConfig.cs b/DistTestCore/Codex/CodexStartupConfig.cs index f83d629..915d4b9 100644 --- a/DistTestCore/Codex/CodexStartupConfig.cs +++ b/DistTestCore/Codex/CodexStartupConfig.cs @@ -1,4 +1,5 @@ -using KubernetesWorkflow; +using DistTestCore.Marketplace; +using KubernetesWorkflow; namespace DistTestCore.Codex { @@ -8,8 +9,8 @@ namespace DistTestCore.Codex public CodexLogLevel? LogLevel { get; set; } public ByteSize? StorageQuota { get; set; } public bool MetricsEnabled { get; set; } + public MarketplaceInitialConfig? MarketplaceConfig { get; set; } //public IOnlineCodexNode? BootstrapNode { get; private set; } - //public MarketplaceInitialConfig? MarketplaceConfig { get; private set; } } } diff --git a/DistTestCore/CodexNodeFactory.cs b/DistTestCore/CodexNodeFactory.cs index 3dce8aa..8320111 100644 --- a/DistTestCore/CodexNodeFactory.cs +++ b/DistTestCore/CodexNodeFactory.cs @@ -1,4 +1,5 @@ using DistTestCore.Codex; +using DistTestCore.Marketplace; using DistTestCore.Metrics; namespace DistTestCore @@ -12,17 +13,20 @@ namespace DistTestCore { private readonly TestLifecycle lifecycle; private readonly IMetricsAccessFactory metricsAccessFactory; + private readonly IMarketplaceAccessFactory marketplaceAccessFactory; - public CodexNodeFactory(TestLifecycle lifecycle, IMetricsAccessFactory metricsAccessFactory) + public CodexNodeFactory(TestLifecycle lifecycle, IMetricsAccessFactory metricsAccessFactory, IMarketplaceAccessFactory marketplaceAccessFactory) { this.lifecycle = lifecycle; this.metricsAccessFactory = metricsAccessFactory; + this.marketplaceAccessFactory = marketplaceAccessFactory; } public OnlineCodexNode CreateOnlineCodexNode(CodexAccess access, CodexNodeGroup group) { var metricsAccess = metricsAccessFactory.CreateMetricsAccess(access.Container); - return new OnlineCodexNode(lifecycle, access, group, metricsAccess); + var marketplaceAccess = marketplaceAccessFactory.CreateMarketplaceAccess(); + return new OnlineCodexNode(lifecycle, access, group, metricsAccess, marketplaceAccess); } } } diff --git a/DistTestCore/CodexSetup.cs b/DistTestCore/CodexSetup.cs index ddc6ef0..5acedcb 100644 --- a/DistTestCore/CodexSetup.cs +++ b/DistTestCore/CodexSetup.cs @@ -1,4 +1,5 @@ using DistTestCore.Codex; +using DistTestCore.Marketplace; using KubernetesWorkflow; namespace DistTestCore @@ -10,7 +11,7 @@ namespace DistTestCore //ICodexStartupConfig WithBootstrapNode(IOnlineCodexNode node); ICodexSetup WithStorageQuota(ByteSize storageQuota); ICodexSetup EnableMetrics(); - //ICodexSetupConfig EnableMarketplace(int initialBalance); + ICodexSetup EnableMarketplace(int initialBalance); ICodexNodeGroup BringOnline(); } @@ -61,11 +62,11 @@ namespace DistTestCore return this; } - //public ICodexSetupConfig EnableMarketplace(int initialBalance) - //{ - // MarketplaceConfig = new MarketplaceInitialConfig(initialBalance); - // return this; - //} + public ICodexSetup EnableMarketplace(int initialBalance) + { + MarketplaceConfig = new MarketplaceInitialConfig(initialBalance); + return this; + } public string Describe() { diff --git a/DistTestCore/CodexStarter.cs b/DistTestCore/CodexStarter.cs index b85fba4..e6d4484 100644 --- a/DistTestCore/CodexStarter.cs +++ b/DistTestCore/CodexStarter.cs @@ -18,16 +18,21 @@ namespace DistTestCore public ICodexNodeGroup BringOnline(CodexSetup codexSetup) { - var something = lifecycle.GethStarter.BringOnlineMarketplaceFor(codexSetup); + Log($"Starting {codexSetup.Describe()}..."); + var gethStartResult = lifecycle.GethStarter.BringOnlineMarketplaceFor(codexSetup); - var containers = StartCodexContainers(codexSetup); + var startupConfig = new StartupConfig(); + startupConfig.Add(codexSetup); + startupConfig.Add(gethStartResult); + + var containers = StartCodexContainers(startupConfig, codexSetup.NumberOfNodes, codexSetup.Location); var metricAccessFactory = lifecycle.PrometheusStarter.CollectMetricsFor(codexSetup, containers); - var codexNodeFactory = new CodexNodeFactory(lifecycle, metricAccessFactory); + var codexNodeFactory = new CodexNodeFactory(lifecycle, metricAccessFactory, gethStartResult.MarketplaceAccessFactory); var group = CreateCodexGroup(codexSetup, containers, codexNodeFactory); - + Log($"Started at '{group.Containers.RunningPod.Ip}'"); return group; } @@ -54,15 +59,10 @@ namespace DistTestCore workflow.DownloadContainerLog(container, logHandler); } - private RunningContainers StartCodexContainers(CodexSetup codexSetup) + private RunningContainers StartCodexContainers(StartupConfig startupConfig, int numberOfNodes, Location location) { - Log($"Starting {codexSetup.Describe()}..."); - var workflow = CreateWorkflow(); - var startupConfig = new StartupConfig(); - startupConfig.Add(codexSetup); - - return workflow.Start(codexSetup.NumberOfNodes, codexSetup.Location, new CodexContainerRecipe(), startupConfig); + return workflow.Start(numberOfNodes, location, new CodexContainerRecipe(), startupConfig); } private CodexNodeGroup CreateCodexGroup(CodexSetup codexSetup, RunningContainers runningContainers, CodexNodeFactory codexNodeFactory) @@ -70,7 +70,6 @@ namespace DistTestCore var group = new CodexNodeGroup(lifecycle, codexSetup, runningContainers, codexNodeFactory); RunningGroups.Add(group); - Log($"Started at '{group.Containers.RunningPod.Ip}'"); return group; } diff --git a/DistTestCore/DistTestCore.csproj b/DistTestCore/DistTestCore.csproj index c57463d..944e748 100644 --- a/DistTestCore/DistTestCore.csproj +++ b/DistTestCore/DistTestCore.csproj @@ -8,7 +8,6 @@ - @@ -17,5 +16,6 @@ + diff --git a/DistTestCore/GethStarter.cs b/DistTestCore/GethStarter.cs index bf26523..c0db927 100644 --- a/DistTestCore/GethStarter.cs +++ b/DistTestCore/GethStarter.cs @@ -6,23 +6,43 @@ namespace DistTestCore public class GethStarter { private readonly TestLifecycle lifecycle; - private readonly WorkflowCreator workflowCreator; private readonly GethBootstrapNodeStarter bootstrapNodeStarter; + private readonly GethCompanionNodeStarter companionNodeStarter; private GethBootstrapNodeInfo? bootstrapNode; public GethStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator) { this.lifecycle = lifecycle; - this.workflowCreator = workflowCreator; bootstrapNodeStarter = new GethBootstrapNodeStarter(lifecycle, workflowCreator); + companionNodeStarter = new GethCompanionNodeStarter(lifecycle, workflowCreator); } - public object BringOnlineMarketplaceFor(CodexSetup codexSetup) + public GethStartResult BringOnlineMarketplaceFor(CodexSetup codexSetup) { + if (codexSetup.MarketplaceConfig == null) return CreateMarketplaceUnavailableResult(); + EnsureBootstrapNode(); - StartCompanionNodes(codexSetup); - return null!; + var companionNodes = StartCompanionNodes(codexSetup); + + TransferInitialBalance(codexSetup.MarketplaceConfig.InitialBalance, bootstrapNode, companionNodes); + + return new GethStartResult(CreateMarketplaceAccessFactory(), bootstrapNode!, companionNodes); + } + + private void TransferInitialBalance(int initialBalance, GethBootstrapNodeInfo? bootstrapNode, GethCompanionNodeInfo[] companionNodes) + { + aaaa + } + + private GethStartResult CreateMarketplaceUnavailableResult() + { + return new GethStartResult(new MarketplaceUnavailableAccessFactory(), null!, Array.Empty()); + } + + private IMarketplaceAccessFactory CreateMarketplaceAccessFactory() + { + throw new NotImplementedException(); } private void EnsureBootstrapNode() @@ -31,14 +51,9 @@ namespace DistTestCore bootstrapNode = bootstrapNodeStarter.StartGethBootstrapNode(); } - private void StartCompanionNodes(CodexSetup codexSetup) + private GethCompanionNodeInfo[] StartCompanionNodes(CodexSetup codexSetup) { - throw new NotImplementedException(); - } - - private void Log(string msg) - { - lifecycle.Log.Log(msg); + return companionNodeStarter.StartCompanionNodesFor(codexSetup, bootstrapNode!); } } } diff --git a/DistTestCore/Marketplace/GethContainerRecipe.cs b/DistTestCore/Marketplace/GethContainerRecipe.cs index 3df72f6..2db8529 100644 --- a/DistTestCore/Marketplace/GethContainerRecipe.cs +++ b/DistTestCore/Marketplace/GethContainerRecipe.cs @@ -5,6 +5,7 @@ namespace DistTestCore.Marketplace public class GethContainerRecipe : ContainerRecipeFactory { protected override string Image => "thatbenbierens/geth-confenv:latest"; + public const string HttpPortTag = "http_port"; public const string AccountFilename = "account_string.txt"; public const string GenesisFilename = "genesis.json"; @@ -30,7 +31,7 @@ namespace DistTestCore.Marketplace var port = AddInternalPort(); var discovery = AddInternalPort(); var authRpc = AddInternalPort(); - var httpPort = AddInternalPort(); + var httpPort = AddInternalPort(tag: HttpPortTag); return $"--port {port.Number} --discovery.port {discovery.Number} --authrpc.port {authRpc.Number} --http.port {httpPort.Number}"; } } diff --git a/DistTestCore/Marketplace/GethStartResult.cs b/DistTestCore/Marketplace/GethStartResult.cs new file mode 100644 index 0000000..f1d9c97 --- /dev/null +++ b/DistTestCore/Marketplace/GethStartResult.cs @@ -0,0 +1,16 @@ +namespace DistTestCore.Marketplace +{ + public class GethStartResult + { + public GethStartResult(IMarketplaceAccessFactory marketplaceAccessFactory, GethBootstrapNodeInfo bootstrapNode, GethCompanionNodeInfo[] companionNodes) + { + MarketplaceAccessFactory = marketplaceAccessFactory; + BootstrapNode = bootstrapNode; + CompanionNodes = companionNodes; + } + + public IMarketplaceAccessFactory MarketplaceAccessFactory { get; } + public GethBootstrapNodeInfo BootstrapNode { get; } + public GethCompanionNodeInfo[] CompanionNodes { get; } + } +} diff --git a/DistTestCore/Marketplace/MarketplaceAccess.cs b/DistTestCore/Marketplace/MarketplaceAccess.cs new file mode 100644 index 0000000..aa55cbf --- /dev/null +++ b/DistTestCore/Marketplace/MarketplaceAccess.cs @@ -0,0 +1,85 @@ +using Logging; +using NUnit.Framework; +using NUnit.Framework.Constraints; + +namespace DistTestCore.Marketplace +{ + public interface IMarketplaceAccess + { + void MakeStorageAvailable(ByteSize size, int minPricePerBytePerSecond, float maxCollateral); + void RequestStorage(ContentId contentId, int pricePerBytePerSecond, float requiredCollateral, float minRequiredNumberOfNodes); + void AssertThatBalance(IResolveConstraint constraint, string message = ""); + decimal GetBalance(); + } + + public class MarketplaceAccess : IMarketplaceAccess + { + private readonly TestLog log; + private readonly CodexNodeGroup group; + + public MarketplaceAccess(TestLog log, CodexNodeGroup group) + { + this.log = log; + this.group = group; + } + + public void Initialize() + { + EnsureAccount(); + + marketplaceController.AddToBalance(container.Account, group.Origin.MarketplaceConfig!.InitialBalance); + + log.Log($"Initialized Geth companion node with account '{container.Account}' and initial balance {group.Origin.MarketplaceConfig!.InitialBalance}"); + } + + public void RequestStorage(ContentId contentId, int pricePerBytePerSecond, float requiredCollateral, float minRequiredNumberOfNodes) + { + throw new NotImplementedException(); + } + + public void MakeStorageAvailable(ByteSize size, int minPricePerBytePerSecond, float maxCollateral) + { + throw new NotImplementedException(); + } + + public void AssertThatBalance(IResolveConstraint constraint, string message = "") + { + throw new NotImplementedException(); + } + + public decimal GetBalance() + { + return marketplaceController.GetBalance(container.Account); + } + } + + public class MarketplaceUnavailable : IMarketplaceAccess + { + public void RequestStorage(ContentId contentId, int pricePerBytePerSecond, float requiredCollateral, float minRequiredNumberOfNodes) + { + Unavailable(); + } + + public void MakeStorageAvailable(ByteSize size, int minPricePerBytePerSecond, float maxCollateral) + { + Unavailable(); + } + + public void AssertThatBalance(IResolveConstraint constraint, string message = "") + { + Unavailable(); + } + + public decimal GetBalance() + { + Unavailable(); + return 0; + } + + private void Unavailable() + { + Assert.Fail("Incorrect test setup: Marketplace was not enabled for this group of Codex nodes. Add 'EnableMarketplace(...)' after 'SetupCodexNodes()' to enable it."); + throw new InvalidOperationException(); + } + } +} diff --git a/DistTestCore/Marketplace/MarketplaceAccessFactory.cs b/DistTestCore/Marketplace/MarketplaceAccessFactory.cs new file mode 100644 index 0000000..e3de573 --- /dev/null +++ b/DistTestCore/Marketplace/MarketplaceAccessFactory.cs @@ -0,0 +1,24 @@ +namespace DistTestCore.Marketplace +{ + public interface IMarketplaceAccessFactory + { + IMarketplaceAccess CreateMarketplaceAccess(); + } + + public class MarketplaceUnavailableAccessFactory : IMarketplaceAccessFactory + { + public IMarketplaceAccess CreateMarketplaceAccess() + { + return new MarketplaceUnavailable(); + } + } + + public class GethMarketplaceAccessFactory : IMarketplaceAccessFactory + { + public IMarketplaceAccess CreateMarketplaceAccess() + { + + return new MarketplaceAccess(query, codexContainer); + } + } +} diff --git a/DistTestCore/Marketplace/MarketplaceInitialConfig.cs b/DistTestCore/Marketplace/MarketplaceInitialConfig.cs new file mode 100644 index 0000000..a815842 --- /dev/null +++ b/DistTestCore/Marketplace/MarketplaceInitialConfig.cs @@ -0,0 +1,12 @@ +namespace DistTestCore.Marketplace +{ + public class MarketplaceInitialConfig + { + public MarketplaceInitialConfig(int initialBalance) + { + InitialBalance = initialBalance; + } + + public int InitialBalance { get; } + } +} diff --git a/DistTestCore/OnlineCodexNode.cs b/DistTestCore/OnlineCodexNode.cs index 76ab592..7075ef3 100644 --- a/DistTestCore/OnlineCodexNode.cs +++ b/DistTestCore/OnlineCodexNode.cs @@ -1,5 +1,6 @@ using DistTestCore.Codex; using DistTestCore.Logs; +using DistTestCore.Marketplace; using DistTestCore.Metrics; using NUnit.Framework; @@ -13,7 +14,7 @@ namespace DistTestCore void ConnectToPeer(IOnlineCodexNode node); ICodexNodeLog DownloadLog(); IMetricsAccess Metrics { get; } - //IMarketplaceAccess Marketplace { get; } + IMarketplaceAccess Marketplace { get; } } public class OnlineCodexNode : IOnlineCodexNode @@ -22,17 +23,19 @@ namespace DistTestCore private const string UploadFailedMessage = "Unable to store block"; private readonly TestLifecycle lifecycle; - public OnlineCodexNode(TestLifecycle lifecycle, CodexAccess codexAccess, CodexNodeGroup group, IMetricsAccess metricsAccess) + public OnlineCodexNode(TestLifecycle lifecycle, CodexAccess codexAccess, CodexNodeGroup group, IMetricsAccess metricsAccess, IMarketplaceAccess marketplaceAccess) { this.lifecycle = lifecycle; CodexAccess = codexAccess; Group = group; Metrics = metricsAccess; + Marketplace = marketplaceAccess; } public CodexAccess CodexAccess { get; } public CodexNodeGroup Group { get; } public IMetricsAccess Metrics { get; } + public IMarketplaceAccess Marketplace { get; } public string GetName() { diff --git a/KubernetesWorkflow/ContainerRecipeFactory.cs b/KubernetesWorkflow/ContainerRecipeFactory.cs index 6c6a3ee..136b209 100644 --- a/KubernetesWorkflow/ContainerRecipeFactory.cs +++ b/KubernetesWorkflow/ContainerRecipeFactory.cs @@ -7,10 +7,11 @@ private readonly List envVars = new List(); private RecipeComponentFactory factory = null!; - public ContainerRecipe CreateRecipe(int containerNumber, RecipeComponentFactory factory, StartupConfig config) + public ContainerRecipe CreateRecipe(int index, int containerNumber, RecipeComponentFactory factory, StartupConfig config) { this.factory = factory; ContainerNumber = containerNumber; + Index = index; Initialize(config); @@ -26,6 +27,7 @@ protected abstract string Image { get; } protected int ContainerNumber { get; private set; } = 0; + protected int Index { get; private set; } = 0; protected abstract void Initialize(StartupConfig config); protected Port AddExposedPort(string tag = "") diff --git a/KubernetesWorkflow/StartupWorkflow.cs b/KubernetesWorkflow/StartupWorkflow.cs index e78a5b2..f575b15 100644 --- a/KubernetesWorkflow/StartupWorkflow.cs +++ b/KubernetesWorkflow/StartupWorkflow.cs @@ -68,7 +68,7 @@ var result = new List(); for (var i = 0; i < numberOfContainers; i++) { - result.Add(recipeFactory.CreateRecipe(numberSource.GetContainerNumber(), componentFactory, startupConfig)); + result.Add(recipeFactory.CreateRecipe(i ,numberSource.GetContainerNumber(), componentFactory, startupConfig)); } return result.ToArray(); diff --git a/Nethereum/NethereumWorkflow.cs b/Nethereum/NethereumWorkflow.cs new file mode 100644 index 0000000..795ef27 --- /dev/null +++ b/Nethereum/NethereumWorkflow.cs @@ -0,0 +1,41 @@ +using Nethereum.Hex.HexTypes; +using Nethereum.Web3; +using System.Numerics; +using Utils; + +namespace NethereumWorkflow +{ + public class NethereumWorkflow + { + private readonly Web3 web3; + private readonly string rootAccount; + + internal NethereumWorkflow(Web3 web3, string rootAccount) + { + this.web3 = web3; + this.rootAccount = rootAccount; + } + + public void AddToBalance(string account, decimal amount) + { + if (amount < 1 || string.IsNullOrEmpty(account)) throw new ArgumentException("Invalid arguments for AddToBalance"); + + var value = ToHexBig(amount); + var transactionId = Time.Wait(web3.Eth.TransactionManager.SendTransactionAsync(rootAccount, account, value)); + Time.Wait(web3.Eth.TransactionManager.TransactionReceiptService.PollForReceiptAsync(transactionId)); + } + + public decimal GetBalance(string account) + { + var bigInt = Time.Wait(web3.Eth.GetBalance.SendRequestAsync(account)); + return (decimal)bigInt.Value; + } + + private HexBigInteger ToHexBig(decimal amount) + { + var bigint = new BigInteger(amount); + var str = bigint.ToString("X"); + return new HexBigInteger(str); + } + } +} diff --git a/Nethereum/NethereumWorkflow.csproj b/Nethereum/NethereumWorkflow.csproj new file mode 100644 index 0000000..99771c5 --- /dev/null +++ b/Nethereum/NethereumWorkflow.csproj @@ -0,0 +1,19 @@ + + + + net6.0 + NethereumWorkflow + enable + enable + + + + + + + + + + + + diff --git a/Nethereum/NethereumWorkflowCreator.cs b/Nethereum/NethereumWorkflowCreator.cs new file mode 100644 index 0000000..33ebf92 --- /dev/null +++ b/Nethereum/NethereumWorkflowCreator.cs @@ -0,0 +1,29 @@ +using Nethereum.Web3; + +namespace NethereumWorkflow +{ + public class NethereumWorkflowCreator + { + private readonly string ip; + private readonly int port; + private readonly string rootAccount; + + public NethereumWorkflowCreator(string ip, int port, string rootAccount) + { + this.ip = ip; + this.port = port; + this.rootAccount = rootAccount; + } + + public NethereumWorkflow CreateWorkflow() + { + return new NethereumWorkflow(CreateWeb3(), rootAccount); + } + + private Web3 CreateWeb3() + { + //var bootstrapaccount = new ManagedAccount(bootstrapInfo.Account, "qwerty!@#$%^"); + return new Web3($"http://{ip}:{port}"); + } + } +} diff --git a/cs-codex-dist-testing.sln b/cs-codex-dist-testing.sln index de9f58f..b47da0c 100644 --- a/cs-codex-dist-testing.sln +++ b/cs-codex-dist-testing.sln @@ -9,13 +9,15 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TestsLong", "LongTests\Test EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CodexDistTestCore", "CodexDistTestCore\CodexDistTestCore.csproj", "{19306DE1-CEE5-4F7B-AA5D-FD91926D853D}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DistTestCore", "DistTestCore\DistTestCore.csproj", "{47F31305-6E68-4827-8E39-7B41DAA1CE7A}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DistTestCore", "DistTestCore\DistTestCore.csproj", "{47F31305-6E68-4827-8E39-7B41DAA1CE7A}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "KubernetesWorkflow", "KubernetesWorkflow\KubernetesWorkflow.csproj", "{359123AA-3D9B-4442-80F4-19E32E3EC9EA}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "KubernetesWorkflow", "KubernetesWorkflow\KubernetesWorkflow.csproj", "{359123AA-3D9B-4442-80F4-19E32E3EC9EA}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Utils", "Utils\Utils.csproj", "{957DE3B8-9571-450A-8609-B267DCA8727C}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Utils", "Utils\Utils.csproj", "{957DE3B8-9571-450A-8609-B267DCA8727C}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Logging", "Logging\Logging.csproj", "{8481A4A6-4BDD-41B0-A3EB-EF53F7BD40D1}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Logging", "Logging\Logging.csproj", "{8481A4A6-4BDD-41B0-A3EB-EF53F7BD40D1}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Nethereum", "Nethereum\Nethereum.csproj", "{D6C3555E-D52D-4993-A87B-71AB650398FD}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -51,6 +53,10 @@ Global {8481A4A6-4BDD-41B0-A3EB-EF53F7BD40D1}.Debug|Any CPU.Build.0 = Debug|Any CPU {8481A4A6-4BDD-41B0-A3EB-EF53F7BD40D1}.Release|Any CPU.ActiveCfg = Release|Any CPU {8481A4A6-4BDD-41B0-A3EB-EF53F7BD40D1}.Release|Any CPU.Build.0 = Release|Any CPU + {D6C3555E-D52D-4993-A87B-71AB650398FD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D6C3555E-D52D-4993-A87B-71AB650398FD}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D6C3555E-D52D-4993-A87B-71AB650398FD}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D6C3555E-D52D-4993-A87B-71AB650398FD}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE From 419ea1854fdc64b583dac8d04c41caf988b41af9 Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 14 Apr 2023 12:37:05 +0200 Subject: [PATCH 31/46] Marketplace test passes --- CodexDistTestCore/CodexDistTestCore.csproj | 2 - DistTestCore/Codex/CodexContainerRecipe.cs | 4 +- DistTestCore/CodexNodeFactory.cs | 2 +- DistTestCore/CodexStarter.cs | 1 - DistTestCore/DistTest.cs | 13 +++- DistTestCore/DistTestCore.csproj | 2 +- DistTestCore/GethStarter.cs | 59 +++++++++++++------ .../Marketplace/GethBootstrapNodeInfo.cs | 11 ++++ .../Marketplace/GethContainerRecipe.cs | 4 +- DistTestCore/Marketplace/MarketplaceAccess.cs | 22 +++---- .../Marketplace/MarketplaceAccessFactory.cs | 30 ++++++++-- .../Metrics/PrometheusContainerRecipe.cs | 4 +- KubernetesWorkflow/ContainerRecipe.cs | 4 +- KubernetesWorkflow/ContainerRecipeFactory.cs | 9 ++- ...eumWorkflow.cs => NethereumInteraction.cs} | 27 +++++++-- ...ator.cs => NethereumInteractionCreator.cs} | 13 ++-- Tests/BasicTests/SimpleTests.cs | 56 +++++++++--------- cs-codex-dist-testing.sln | 2 +- 18 files changed, 175 insertions(+), 90 deletions(-) rename Nethereum/{NethereumWorkflow.cs => NethereumInteraction.cs} (60%) rename Nethereum/{NethereumWorkflowCreator.cs => NethereumInteractionCreator.cs} (57%) diff --git a/CodexDistTestCore/CodexDistTestCore.csproj b/CodexDistTestCore/CodexDistTestCore.csproj index 142eefb..cb1fb2e 100644 --- a/CodexDistTestCore/CodexDistTestCore.csproj +++ b/CodexDistTestCore/CodexDistTestCore.csproj @@ -8,8 +8,6 @@ - - diff --git a/DistTestCore/Codex/CodexContainerRecipe.cs b/DistTestCore/Codex/CodexContainerRecipe.cs index 31a9a7f..50d6441 100644 --- a/DistTestCore/Codex/CodexContainerRecipe.cs +++ b/DistTestCore/Codex/CodexContainerRecipe.cs @@ -5,9 +5,10 @@ namespace DistTestCore.Codex { public class CodexContainerRecipe : ContainerRecipeFactory { + public const string DockerImage = "thatbenbierens/nim-codex:sha-b204837"; public const string MetricsPortTag = "metrics_port"; - protected override string Image => "thatbenbierens/nim-codex:sha-b204837"; + protected override string Image => DockerImage; protected override void Initialize(StartupConfig startupConfig) { @@ -38,6 +39,7 @@ namespace DistTestCore.Codex { var gethConfig = startupConfig.Get(); var companionNode = gethConfig.CompanionNodes[Index]; + Additional(companionNode); // Bootstrap node access from within the cluster: //var ip = gethConfig.BootstrapNode.RunningContainers.RunningPod.Ip; diff --git a/DistTestCore/CodexNodeFactory.cs b/DistTestCore/CodexNodeFactory.cs index 8320111..9b67158 100644 --- a/DistTestCore/CodexNodeFactory.cs +++ b/DistTestCore/CodexNodeFactory.cs @@ -25,7 +25,7 @@ namespace DistTestCore public OnlineCodexNode CreateOnlineCodexNode(CodexAccess access, CodexNodeGroup group) { var metricsAccess = metricsAccessFactory.CreateMetricsAccess(access.Container); - var marketplaceAccess = marketplaceAccessFactory.CreateMarketplaceAccess(); + var marketplaceAccess = marketplaceAccessFactory.CreateMarketplaceAccess(access); return new OnlineCodexNode(lifecycle, access, group, metricsAccess, marketplaceAccess); } } diff --git a/DistTestCore/CodexStarter.cs b/DistTestCore/CodexStarter.cs index e6d4484..e90c0f0 100644 --- a/DistTestCore/CodexStarter.cs +++ b/DistTestCore/CodexStarter.cs @@ -69,7 +69,6 @@ namespace DistTestCore { var group = new CodexNodeGroup(lifecycle, codexSetup, runningContainers, codexNodeFactory); RunningGroups.Add(group); - return group; } diff --git a/DistTestCore/DistTest.cs b/DistTestCore/DistTest.cs index e3631f4..e6c9ac0 100644 --- a/DistTestCore/DistTest.cs +++ b/DistTestCore/DistTest.cs @@ -1,5 +1,8 @@ -using DistTestCore.Logs; +using DistTestCore.Codex; +using DistTestCore.Logs; +using DistTestCore.Marketplace; using DistTestCore.Metrics; +using Logging; using NUnit.Framework; namespace DistTestCore @@ -8,6 +11,7 @@ namespace DistTestCore public abstract class DistTest { private TestLifecycle lifecycle = null!; + private TestLog log = null!; [OneTimeSetUp] public void GlobalSetup() @@ -26,7 +30,10 @@ namespace DistTestCore Error($"Global setup cleanup failed with: {ex}"); throw; } - Log("Global setup cleanup successful"); + log.Log("Global setup cleanup successful"); + log.Log($"Codex image: {CodexContainerRecipe.DockerImage}"); + log.Log($"Prometheus image: {PrometheusContainerRecipe.DockerImage}"); + log.Log($"Geth image: {GethContainerRecipe.DockerImage}"); } [SetUp] @@ -38,6 +45,7 @@ namespace DistTestCore } else { + log.Log($"Run: {TestContext.CurrentContext.Test.Name}"); CreateNewTestLifecycle(); } } @@ -47,6 +55,7 @@ namespace DistTestCore { try { + log.Log($"{TestContext.CurrentContext.Test.Name} = {TestContext.CurrentContext.Result.Outcome.Status}"); lifecycle.Log.EndTest(); IncludeLogsAndMetricsOnTestFailure(); lifecycle.DeleteAllResources(); diff --git a/DistTestCore/DistTestCore.csproj b/DistTestCore/DistTestCore.csproj index 944e748..ccd5f66 100644 --- a/DistTestCore/DistTestCore.csproj +++ b/DistTestCore/DistTestCore.csproj @@ -16,6 +16,6 @@ - + diff --git a/DistTestCore/GethStarter.cs b/DistTestCore/GethStarter.cs index c0db927..3e01a5e 100644 --- a/DistTestCore/GethStarter.cs +++ b/DistTestCore/GethStarter.cs @@ -5,34 +5,41 @@ namespace DistTestCore { public class GethStarter { - private readonly TestLifecycle lifecycle; - private readonly GethBootstrapNodeStarter bootstrapNodeStarter; + private readonly GethBootstrapNodeCache bootstrapNodeCache; private readonly GethCompanionNodeStarter companionNodeStarter; - private GethBootstrapNodeInfo? bootstrapNode; + private readonly TestLifecycle lifecycle; public GethStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator) { - this.lifecycle = lifecycle; - - bootstrapNodeStarter = new GethBootstrapNodeStarter(lifecycle, workflowCreator); + bootstrapNodeCache = new GethBootstrapNodeCache(new GethBootstrapNodeStarter(lifecycle, workflowCreator)); companionNodeStarter = new GethCompanionNodeStarter(lifecycle, workflowCreator); + this.lifecycle = lifecycle; } public GethStartResult BringOnlineMarketplaceFor(CodexSetup codexSetup) { if (codexSetup.MarketplaceConfig == null) return CreateMarketplaceUnavailableResult(); - EnsureBootstrapNode(); - var companionNodes = StartCompanionNodes(codexSetup); + var bootstrapNode = bootstrapNodeCache.Get(); + var companionNodes = StartCompanionNodes(codexSetup, bootstrapNode); - TransferInitialBalance(codexSetup.MarketplaceConfig.InitialBalance, bootstrapNode, companionNodes); + TransferInitialBalance(bootstrapNode, codexSetup.MarketplaceConfig.InitialBalance, companionNodes); - return new GethStartResult(CreateMarketplaceAccessFactory(), bootstrapNode!, companionNodes); + return CreateGethStartResult(bootstrapNode, companionNodes); } - private void TransferInitialBalance(int initialBalance, GethBootstrapNodeInfo? bootstrapNode, GethCompanionNodeInfo[] companionNodes) + private void TransferInitialBalance(GethBootstrapNodeInfo bootstrapNode, int initialBalance, GethCompanionNodeInfo[] companionNodes) { - aaaa + var interaction = bootstrapNode.StartInteraction(lifecycle.Log); + foreach (var node in companionNodes) + { + interaction.TransferTo(node.Account, initialBalance); + } + } + + private GethStartResult CreateGethStartResult(GethBootstrapNodeInfo bootstrapNode, GethCompanionNodeInfo[] companionNodes) + { + return new GethStartResult(CreateMarketplaceAccessFactory(bootstrapNode), bootstrapNode, companionNodes); } private GethStartResult CreateMarketplaceUnavailableResult() @@ -40,20 +47,34 @@ namespace DistTestCore return new GethStartResult(new MarketplaceUnavailableAccessFactory(), null!, Array.Empty()); } - private IMarketplaceAccessFactory CreateMarketplaceAccessFactory() + private IMarketplaceAccessFactory CreateMarketplaceAccessFactory(GethBootstrapNodeInfo bootstrapNode) { - throw new NotImplementedException(); + return new GethMarketplaceAccessFactory(lifecycle.Log, bootstrapNode!); } - private void EnsureBootstrapNode() + private GethCompanionNodeInfo[] StartCompanionNodes(CodexSetup codexSetup, GethBootstrapNodeInfo bootstrapNode) { - if (bootstrapNode != null) return; - bootstrapNode = bootstrapNodeStarter.StartGethBootstrapNode(); + return companionNodeStarter.StartCompanionNodesFor(codexSetup, bootstrapNode); + } + } + + public class GethBootstrapNodeCache + { + private readonly GethBootstrapNodeStarter bootstrapNodeStarter; + private GethBootstrapNodeInfo? bootstrapNode; + + public GethBootstrapNodeCache(GethBootstrapNodeStarter bootstrapNodeStarter) + { + this.bootstrapNodeStarter = bootstrapNodeStarter; } - private GethCompanionNodeInfo[] StartCompanionNodes(CodexSetup codexSetup) + public GethBootstrapNodeInfo Get() { - return companionNodeStarter.StartCompanionNodesFor(codexSetup, bootstrapNode!); + if (bootstrapNode == null) + { + bootstrapNode = bootstrapNodeStarter.StartGethBootstrapNode(); + } + return bootstrapNode; } } } diff --git a/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs b/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs index 8f897ef..39b6715 100644 --- a/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs +++ b/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs @@ -1,4 +1,6 @@ using KubernetesWorkflow; +using Logging; +using NethereumWorkflow; namespace DistTestCore.Marketplace { @@ -14,5 +16,14 @@ namespace DistTestCore.Marketplace public RunningContainers RunningContainers { get; } public string Account { get; } public string GenesisJsonBase64 { get; } + + public NethereumInteraction StartInteraction(TestLog log) + { + var ip = RunningContainers.RunningPod.Cluster.IP; + var port = RunningContainers.Containers[0].ServicePorts[0].Number; + + var creator = new NethereumInteractionCreator(log, ip, port, Account); + return creator.CreateWorkflow(); + } } } diff --git a/DistTestCore/Marketplace/GethContainerRecipe.cs b/DistTestCore/Marketplace/GethContainerRecipe.cs index 2db8529..2dbfbde 100644 --- a/DistTestCore/Marketplace/GethContainerRecipe.cs +++ b/DistTestCore/Marketplace/GethContainerRecipe.cs @@ -4,10 +4,12 @@ namespace DistTestCore.Marketplace { public class GethContainerRecipe : ContainerRecipeFactory { - protected override string Image => "thatbenbierens/geth-confenv:latest"; + public const string DockerImage = "thatbenbierens/geth-confenv:latest"; public const string HttpPortTag = "http_port"; public const string AccountFilename = "account_string.txt"; public const string GenesisFilename = "genesis.json"; + + protected override string Image => DockerImage; protected override void Initialize(StartupConfig startupConfig) { diff --git a/DistTestCore/Marketplace/MarketplaceAccess.cs b/DistTestCore/Marketplace/MarketplaceAccess.cs index aa55cbf..8de63dd 100644 --- a/DistTestCore/Marketplace/MarketplaceAccess.cs +++ b/DistTestCore/Marketplace/MarketplaceAccess.cs @@ -15,21 +15,14 @@ namespace DistTestCore.Marketplace public class MarketplaceAccess : IMarketplaceAccess { private readonly TestLog log; - private readonly CodexNodeGroup group; + private readonly GethBootstrapNodeInfo bootstrapNode; + private readonly GethCompanionNodeInfo companionNode; - public MarketplaceAccess(TestLog log, CodexNodeGroup group) + public MarketplaceAccess(TestLog log, GethBootstrapNodeInfo bootstrapNode, GethCompanionNodeInfo companionNode) { this.log = log; - this.group = group; - } - - public void Initialize() - { - EnsureAccount(); - - marketplaceController.AddToBalance(container.Account, group.Origin.MarketplaceConfig!.InitialBalance); - - log.Log($"Initialized Geth companion node with account '{container.Account}' and initial balance {group.Origin.MarketplaceConfig!.InitialBalance}"); + this.bootstrapNode = bootstrapNode; + this.companionNode = companionNode; } public void RequestStorage(ContentId contentId, int pricePerBytePerSecond, float requiredCollateral, float minRequiredNumberOfNodes) @@ -44,12 +37,13 @@ namespace DistTestCore.Marketplace public void AssertThatBalance(IResolveConstraint constraint, string message = "") { - throw new NotImplementedException(); + Assert.That(GetBalance(), constraint, message); } public decimal GetBalance() { - return marketplaceController.GetBalance(container.Account); + var interaction = bootstrapNode.StartInteraction(log); + return interaction.GetBalance(companionNode.Account); } } diff --git a/DistTestCore/Marketplace/MarketplaceAccessFactory.cs b/DistTestCore/Marketplace/MarketplaceAccessFactory.cs index e3de573..fba1155 100644 --- a/DistTestCore/Marketplace/MarketplaceAccessFactory.cs +++ b/DistTestCore/Marketplace/MarketplaceAccessFactory.cs @@ -1,13 +1,16 @@ -namespace DistTestCore.Marketplace +using DistTestCore.Codex; +using Logging; + +namespace DistTestCore.Marketplace { public interface IMarketplaceAccessFactory { - IMarketplaceAccess CreateMarketplaceAccess(); + IMarketplaceAccess CreateMarketplaceAccess(CodexAccess access); } public class MarketplaceUnavailableAccessFactory : IMarketplaceAccessFactory { - public IMarketplaceAccess CreateMarketplaceAccess() + public IMarketplaceAccess CreateMarketplaceAccess(CodexAccess access) { return new MarketplaceUnavailable(); } @@ -15,10 +18,25 @@ public class GethMarketplaceAccessFactory : IMarketplaceAccessFactory { - public IMarketplaceAccess CreateMarketplaceAccess() + private readonly TestLog log; + private readonly GethBootstrapNodeInfo bootstrapNode; + + public GethMarketplaceAccessFactory(TestLog log, GethBootstrapNodeInfo bootstrapNode) { - - return new MarketplaceAccess(query, codexContainer); + this.log = log; + this.bootstrapNode = bootstrapNode; + } + + public IMarketplaceAccess CreateMarketplaceAccess(CodexAccess access) + { + var companionNode = GetGethCompanionNode(access); + return new MarketplaceAccess(log, bootstrapNode, companionNode); + } + + private GethCompanionNodeInfo GetGethCompanionNode(CodexAccess access) + { + var node = access.Container.Recipe.Additionals.Single(a => a is GethCompanionNodeInfo); + return (GethCompanionNodeInfo)node; } } } diff --git a/DistTestCore/Metrics/PrometheusContainerRecipe.cs b/DistTestCore/Metrics/PrometheusContainerRecipe.cs index 5152ff7..46587cf 100644 --- a/DistTestCore/Metrics/PrometheusContainerRecipe.cs +++ b/DistTestCore/Metrics/PrometheusContainerRecipe.cs @@ -4,7 +4,9 @@ namespace DistTestCore.Metrics { public class PrometheusContainerRecipe : ContainerRecipeFactory { - protected override string Image => "thatbenbierens/prometheus-envconf:latest"; + public const string DockerImage = "thatbenbierens/prometheus-envconf:latest"; + + protected override string Image => DockerImage; protected override void Initialize(StartupConfig startupConfig) { diff --git a/KubernetesWorkflow/ContainerRecipe.cs b/KubernetesWorkflow/ContainerRecipe.cs index f676c7f..9fbbb9f 100644 --- a/KubernetesWorkflow/ContainerRecipe.cs +++ b/KubernetesWorkflow/ContainerRecipe.cs @@ -2,13 +2,14 @@ { public class ContainerRecipe { - public ContainerRecipe(int number, string image, Port[] exposedPorts, Port[] internalPorts, EnvVar[] envVars) + public ContainerRecipe(int number, string image, Port[] exposedPorts, Port[] internalPorts, EnvVar[] envVars, object[] additionals) { Number = number; Image = image; ExposedPorts = exposedPorts; InternalPorts = internalPorts; EnvVars = envVars; + Additionals = additionals; } public string Name { get { return $"ctnr{Number}"; } } @@ -17,6 +18,7 @@ public Port[] ExposedPorts { get; } public Port[] InternalPorts { get; } public EnvVar[] EnvVars { get; } + public object[] Additionals { get; } public Port GetPortByTag(string tag) { diff --git a/KubernetesWorkflow/ContainerRecipeFactory.cs b/KubernetesWorkflow/ContainerRecipeFactory.cs index 136b209..07dc58e 100644 --- a/KubernetesWorkflow/ContainerRecipeFactory.cs +++ b/KubernetesWorkflow/ContainerRecipeFactory.cs @@ -5,6 +5,7 @@ private readonly List exposedPorts = new List(); private readonly List internalPorts = new List(); private readonly List envVars = new List(); + private readonly List additionals = new List(); private RecipeComponentFactory factory = null!; public ContainerRecipe CreateRecipe(int index, int containerNumber, RecipeComponentFactory factory, StartupConfig config) @@ -15,11 +16,12 @@ Initialize(config); - var recipe = new ContainerRecipe(containerNumber, Image, exposedPorts.ToArray(), internalPorts.ToArray(), envVars.ToArray()); + var recipe = new ContainerRecipe(containerNumber, Image, exposedPorts.ToArray(), internalPorts.ToArray(), envVars.ToArray(), additionals.ToArray()); exposedPorts.Clear(); internalPorts.Clear(); envVars.Clear(); + additionals.Clear(); this.factory = null!; return recipe; @@ -63,5 +65,10 @@ { envVars.Add(factory.CreateEnvVar(name, value.Number)); } + + protected void Additional(object userData) + { + additionals.Add(userData); + } } } diff --git a/Nethereum/NethereumWorkflow.cs b/Nethereum/NethereumInteraction.cs similarity index 60% rename from Nethereum/NethereumWorkflow.cs rename to Nethereum/NethereumInteraction.cs index 795ef27..832f32f 100644 --- a/Nethereum/NethereumWorkflow.cs +++ b/Nethereum/NethereumInteraction.cs @@ -1,34 +1,41 @@ -using Nethereum.Hex.HexTypes; +using Logging; +using Nethereum.Hex.HexTypes; using Nethereum.Web3; using System.Numerics; using Utils; namespace NethereumWorkflow { - public class NethereumWorkflow + public class NethereumInteraction { + private readonly TestLog log; private readonly Web3 web3; private readonly string rootAccount; - internal NethereumWorkflow(Web3 web3, string rootAccount) + internal NethereumInteraction(TestLog log, Web3 web3, string rootAccount) { + this.log = log; this.web3 = web3; this.rootAccount = rootAccount; } - public void AddToBalance(string account, decimal amount) + public void TransferTo(string account, decimal amount) { if (amount < 1 || string.IsNullOrEmpty(account)) throw new ArgumentException("Invalid arguments for AddToBalance"); var value = ToHexBig(amount); var transactionId = Time.Wait(web3.Eth.TransactionManager.SendTransactionAsync(rootAccount, account, value)); Time.Wait(web3.Eth.TransactionManager.TransactionReceiptService.PollForReceiptAsync(transactionId)); + + Log($"Transferred {amount} to {account}"); } public decimal GetBalance(string account) { var bigInt = Time.Wait(web3.Eth.GetBalance.SendRequestAsync(account)); - return (decimal)bigInt.Value; + var result = ToDecimal(bigInt); + Log($"Balance of {account} is {result}"); + return result; } private HexBigInteger ToHexBig(decimal amount) @@ -37,5 +44,15 @@ namespace NethereumWorkflow var str = bigint.ToString("X"); return new HexBigInteger(str); } + + private decimal ToDecimal(HexBigInteger hexBigInteger) + { + return (decimal)hexBigInteger.Value; + } + + private void Log(string msg) + { + log.Log(msg); + } } } diff --git a/Nethereum/NethereumWorkflowCreator.cs b/Nethereum/NethereumInteractionCreator.cs similarity index 57% rename from Nethereum/NethereumWorkflowCreator.cs rename to Nethereum/NethereumInteractionCreator.cs index 33ebf92..b0b41f7 100644 --- a/Nethereum/NethereumWorkflowCreator.cs +++ b/Nethereum/NethereumInteractionCreator.cs @@ -1,23 +1,26 @@ -using Nethereum.Web3; +using Logging; +using Nethereum.Web3; namespace NethereumWorkflow { - public class NethereumWorkflowCreator + public class NethereumInteractionCreator { + private readonly TestLog log; private readonly string ip; private readonly int port; private readonly string rootAccount; - public NethereumWorkflowCreator(string ip, int port, string rootAccount) + public NethereumInteractionCreator(TestLog log, string ip, int port, string rootAccount) { + this.log = log; this.ip = ip; this.port = port; this.rootAccount = rootAccount; } - public NethereumWorkflow CreateWorkflow() + public NethereumInteraction CreateWorkflow() { - return new NethereumWorkflow(CreateWeb3(), rootAccount); + return new NethereumInteraction(log, CreateWeb3(), rootAccount); } private Web3 CreateWeb3() diff --git a/Tests/BasicTests/SimpleTests.cs b/Tests/BasicTests/SimpleTests.cs index bf0c331..943b598 100644 --- a/Tests/BasicTests/SimpleTests.cs +++ b/Tests/BasicTests/SimpleTests.cs @@ -103,42 +103,42 @@ namespace Tests.BasicTests primary2.Metrics.AssertThat("libp2p_peers", Is.EqualTo(1)); } - //[Test] - //public void MarketplaceExample() - //{ - // var group = SetupCodexNodes(4) - // .WithStorageQuota(10.GB()) - // .EnableMarketplace(initialBalance: 20) - // .BringOnline(); + [Test] + public void MarketplaceExample() + { + var group = SetupCodexNodes(4) + .WithStorageQuota(10.GB()) + .EnableMarketplace(initialBalance: 20) + .BringOnline(); - // foreach (var node in group) - // { - // Assert.That(node.Marketplace.GetBalance(), Is.EqualTo(20)); - // } + foreach (var node in group) + { + Assert.That(node.Marketplace.GetBalance(), Is.EqualTo(20)); + } - // // WIP: Balance is now only ETH. - // // todo: All nodes should have plenty of ETH to pay for transactions. - // // todo: Upload our own token, use this exclusively. ETH should be invisibile to the tests. + // WIP: Balance is now only ETH. + // todo: All nodes should have plenty of ETH to pay for transactions. + // todo: Upload our own token, use this exclusively. ETH should be invisibile to the tests. - // //var secondary = SetupCodexNodes(1) - // // .EnableMarketplace(initialBalance: 1000) - // // .BringOnline()[0]; + //var secondary = SetupCodexNodes(1) + // .EnableMarketplace(initialBalance: 1000) + // .BringOnline()[0]; - // //primary.ConnectToPeer(secondary); - // //primary.Marketplace.MakeStorageAvailable(10.GB(), minPricePerBytePerSecond: 1, maxCollateral: 20); + //primary.ConnectToPeer(secondary); + //primary.Marketplace.MakeStorageAvailable(10.GB(), minPricePerBytePerSecond: 1, maxCollateral: 20); - // //var testFile = GenerateTestFile(10.MB()); - // //var contentId = secondary.UploadFile(testFile); - // //secondary.Marketplace.RequestStorage(contentId, pricePerBytePerSecond: 2, - // // requiredCollateral: 10, minRequiredNumberOfNodes: 1); + //var testFile = GenerateTestFile(10.MB()); + //var contentId = secondary.UploadFile(testFile); + //secondary.Marketplace.RequestStorage(contentId, pricePerBytePerSecond: 2, + // requiredCollateral: 10, minRequiredNumberOfNodes: 1); - // //primary.Marketplace.AssertThatBalance(Is.LessThan(20), "Collateral was not placed."); - // //var primaryBalance = primary.Marketplace.GetBalance(); + //primary.Marketplace.AssertThatBalance(Is.LessThan(20), "Collateral was not placed."); + //var primaryBalance = primary.Marketplace.GetBalance(); - // //secondary.Marketplace.AssertThatBalance(Is.LessThan(1000), "Contractor was not charged for storage."); - // //primary.Marketplace.AssertThatBalance(Is.GreaterThan(primaryBalance), "Storer was not paid for storage."); - //} + //secondary.Marketplace.AssertThatBalance(Is.LessThan(1000), "Contractor was not charged for storage."); + //primary.Marketplace.AssertThatBalance(Is.GreaterThan(primaryBalance), "Storer was not paid for storage."); + } private void PerformOneClientTest(IOnlineCodexNode primary) { diff --git a/cs-codex-dist-testing.sln b/cs-codex-dist-testing.sln index b47da0c..1f5c04f 100644 --- a/cs-codex-dist-testing.sln +++ b/cs-codex-dist-testing.sln @@ -17,7 +17,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Utils", "Utils\Utils.csproj EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Logging", "Logging\Logging.csproj", "{8481A4A6-4BDD-41B0-A3EB-EF53F7BD40D1}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Nethereum", "Nethereum\Nethereum.csproj", "{D6C3555E-D52D-4993-A87B-71AB650398FD}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NethereumWorkflow", "Nethereum\NethereumWorkflow.csproj", "{D6C3555E-D52D-4993-A87B-71AB650398FD}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution From 60e653b63cd7ef3b8e52fce2a0bb03e9830c7a7a Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 14 Apr 2023 14:53:39 +0200 Subject: [PATCH 32/46] Better logging + time measurement --- DistTestCore/CodexNodeGroup.cs | 2 +- DistTestCore/CodexSetup.cs | 2 +- DistTestCore/DistTest.cs | 65 ++++++++++++--- DistTestCore/Logs/CodexNodeLog.cs | 2 +- DistTestCore/Logs/LogDownloadHandler.cs | 2 +- DistTestCore/Metrics/MetricsDownloader.cs | 2 +- DistTestCore/OnlineCodexNode.cs | 2 +- DistTestCore/Stopwatch.cs | 19 +++++ DistTestCore/TestLifecycle.cs | 6 +- Logging/BaseLog.cs | 36 ++++++++ Logging/FixtureLog.cs | 49 +++++++++++ Logging/LogFile.cs | 42 +++------- Logging/TestLog.cs | 51 +++++------- .../{SimpleTests.cs => ExampleTests.cs} | 83 +------------------ Tests/BasicTests/OneClientTests.cs | 41 +++++++++ Tests/BasicTests/TwoClientTests.cs | 59 +++++++++++++ Utils/Time.cs | 10 +++ 17 files changed, 309 insertions(+), 164 deletions(-) create mode 100644 DistTestCore/Stopwatch.cs create mode 100644 Logging/BaseLog.cs create mode 100644 Logging/FixtureLog.cs rename Tests/BasicTests/{SimpleTests.cs => ExampleTests.cs} (56%) create mode 100644 Tests/BasicTests/OneClientTests.cs create mode 100644 Tests/BasicTests/TwoClientTests.cs diff --git a/DistTestCore/CodexNodeGroup.cs b/DistTestCore/CodexNodeGroup.cs index da4313f..25d46c1 100644 --- a/DistTestCore/CodexNodeGroup.cs +++ b/DistTestCore/CodexNodeGroup.cs @@ -73,7 +73,7 @@ namespace DistTestCore public string Describe() { - return $"CodexNodeGroup@{Containers.Describe()}-{Setup.Describe()}"; + return $""; } private OnlineCodexNode CreateOnlineCodexNode(RunningContainer c, ICodexNodeFactory factory) diff --git a/DistTestCore/CodexSetup.cs b/DistTestCore/CodexSetup.cs index 5acedcb..612c37e 100644 --- a/DistTestCore/CodexSetup.cs +++ b/DistTestCore/CodexSetup.cs @@ -71,7 +71,7 @@ namespace DistTestCore public string Describe() { var args = string.Join(',', DescribeArgs()); - return $"{NumberOfNodes} CodexNodes with [{args}]"; + return $"({NumberOfNodes} CodexNodes with [{args}])"; } private IEnumerable DescribeArgs() diff --git a/DistTestCore/DistTest.cs b/DistTestCore/DistTest.cs index e6c9ac0..75711c8 100644 --- a/DistTestCore/DistTest.cs +++ b/DistTestCore/DistTest.cs @@ -2,27 +2,35 @@ using DistTestCore.Logs; using DistTestCore.Marketplace; using DistTestCore.Metrics; +using KubernetesWorkflow; using Logging; using NUnit.Framework; +using Utils; namespace DistTestCore { [SetUpFixture] public abstract class DistTest { + private readonly Configuration configuration = new Configuration(); + private FixtureLog fixtureLog = null!; private TestLifecycle lifecycle = null!; - private TestLog log = null!; + private DateTime testStart = DateTime.MinValue; [OneTimeSetUp] public void GlobalSetup() { // Previous test run may have been interrupted. // Begin by cleaning everything up. - CreateNewTestLifecycle(); + fixtureLog = new FixtureLog(configuration.GetLogConfig()); try { - lifecycle.DeleteAllResources(); + Stopwatch.Measure(fixtureLog, "Global setup", () => + { + var wc = new WorkflowCreator(configuration.GetK8sConfiguration()); + wc.CreateWorkflow().DeleteAllResources(); + }); } catch (Exception ex) { @@ -30,10 +38,11 @@ namespace DistTestCore Error($"Global setup cleanup failed with: {ex}"); throw; } - log.Log("Global setup cleanup successful"); - log.Log($"Codex image: {CodexContainerRecipe.DockerImage}"); - log.Log($"Prometheus image: {PrometheusContainerRecipe.DockerImage}"); - log.Log($"Geth image: {GethContainerRecipe.DockerImage}"); + + fixtureLog.Log("Global setup cleanup successful"); + fixtureLog.Log($"Codex image: {CodexContainerRecipe.DockerImage}"); + fixtureLog.Log($"Prometheus image: {PrometheusContainerRecipe.DockerImage}"); + fixtureLog.Log($"Geth image: {GethContainerRecipe.DockerImage}"); } [SetUp] @@ -45,7 +54,6 @@ namespace DistTestCore } else { - log.Log($"Run: {TestContext.CurrentContext.Test.Name}"); CreateNewTestLifecycle(); } } @@ -55,10 +63,7 @@ namespace DistTestCore { try { - log.Log($"{TestContext.CurrentContext.Test.Name} = {TestContext.CurrentContext.Result.Outcome.Status}"); - lifecycle.Log.EndTest(); - IncludeLogsAndMetricsOnTestFailure(); - lifecycle.DeleteAllResources(); + DisposeTestLifecycle(); } catch (Exception ex) { @@ -82,6 +87,8 @@ namespace DistTestCore var result = TestContext.CurrentContext.Result; if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed) { + fixtureLog.MarkAsFailed(); + if (IsDownloadingLogsAndMetricsEnabled()) { Log("Downloading all CodexNode logs and metrics because of test failure..."); @@ -107,7 +114,29 @@ namespace DistTestCore private void CreateNewTestLifecycle() { - lifecycle = new TestLifecycle(new Configuration()); + Stopwatch.Measure(fixtureLog, $"Setup for {GetCurrentTestName()}", () => + { + lifecycle = new TestLifecycle(fixtureLog.CreateTestLog(), configuration); + testStart = DateTime.UtcNow; + }); + } + + private void DisposeTestLifecycle() + { + fixtureLog.Log($"{GetCurrentTestName()} = {GetTestResult()} ({GetTestDuration()})"); + Stopwatch.Measure(fixtureLog, $"Teardown for {GetCurrentTestName()}", () => + { + lifecycle.Log.EndTest(); + IncludeLogsAndMetricsOnTestFailure(); + lifecycle.DeleteAllResources(); + lifecycle = null!; + }); + } + + private string GetTestDuration() + { + var testDuration = DateTime.UtcNow - testStart; + return Time.FormatDuration(testDuration); } private void DownloadAllLogs() @@ -141,6 +170,16 @@ namespace DistTestCore } } + private string GetCurrentTestName() + { + return $"[{TestContext.CurrentContext.Test.Name}]"; + } + + private string GetTestResult() + { + return TestContext.CurrentContext.Result.Outcome.Status.ToString(); + } + private bool IsDownloadingLogsAndMetricsEnabled() { var testProperties = TestContext.CurrentContext.Test.Properties; diff --git a/DistTestCore/Logs/CodexNodeLog.cs b/DistTestCore/Logs/CodexNodeLog.cs index ac92678..3b65241 100644 --- a/DistTestCore/Logs/CodexNodeLog.cs +++ b/DistTestCore/Logs/CodexNodeLog.cs @@ -29,7 +29,7 @@ namespace DistTestCore.Logs line = streamReader.ReadLine(); } - Assert.Fail($"Unable to find string '{expectedString}' in CodexNode log file {logFile.FilenameWithoutPath}"); + Assert.Fail($"Unable to find string '{expectedString}' in CodexNode log file {logFile.FullFilename}"); } } } diff --git a/DistTestCore/Logs/LogDownloadHandler.cs b/DistTestCore/Logs/LogDownloadHandler.cs index b6136c3..a01ad9e 100644 --- a/DistTestCore/Logs/LogDownloadHandler.cs +++ b/DistTestCore/Logs/LogDownloadHandler.cs @@ -21,7 +21,7 @@ namespace DistTestCore.Logs public void Log(Stream stream) { - log.Write($"{description} -->> {log.FilenameWithoutPath}"); + log.Write($"{description} -->> {log.FullFilename}"); log.WriteRaw(description); var reader = new StreamReader(stream); var line = reader.ReadLine(); diff --git a/DistTestCore/Metrics/MetricsDownloader.cs b/DistTestCore/Metrics/MetricsDownloader.cs index 1ea56f3..4a458dd 100644 --- a/DistTestCore/Metrics/MetricsDownloader.cs +++ b/DistTestCore/Metrics/MetricsDownloader.cs @@ -26,7 +26,7 @@ namespace DistTestCore.Metrics private void WriteToFile(string nodeName, string[] headers, Dictionary> map) { var file = log.CreateSubfile("csv"); - log.Log($"Downloading metrics for {nodeName} to file {file.FilenameWithoutPath}"); + log.Log($"Downloading metrics for {nodeName} to file {file.FullFilename}"); file.WriteRaw(string.Join(",", headers)); diff --git a/DistTestCore/OnlineCodexNode.cs b/DistTestCore/OnlineCodexNode.cs index 7075ef3..a12de9f 100644 --- a/DistTestCore/OnlineCodexNode.cs +++ b/DistTestCore/OnlineCodexNode.cs @@ -90,7 +90,7 @@ namespace DistTestCore public string Describe() { - return $"{Group.Describe()} contains {GetName()}"; + return $"({Group.Describe()} contains {GetName()})"; } private string GetPeerMultiAddress(OnlineCodexNode peer, CodexDebugResponse peerInfo) diff --git a/DistTestCore/Stopwatch.cs b/DistTestCore/Stopwatch.cs new file mode 100644 index 0000000..5be09c0 --- /dev/null +++ b/DistTestCore/Stopwatch.cs @@ -0,0 +1,19 @@ +using Logging; +using Utils; + +namespace DistTestCore +{ + public class Stopwatch + { + public static void Measure(BaseLog log, string name, Action action) + { + var start = DateTime.UtcNow; + + action(); + + var duration = DateTime.UtcNow - start; + + log.Log($"{name} ({Time.FormatDuration(duration)})"); + } + } +} diff --git a/DistTestCore/TestLifecycle.cs b/DistTestCore/TestLifecycle.cs index 63d3c7a..2257610 100644 --- a/DistTestCore/TestLifecycle.cs +++ b/DistTestCore/TestLifecycle.cs @@ -8,9 +8,9 @@ namespace DistTestCore { private readonly WorkflowCreator workflowCreator; - public TestLifecycle(Configuration configuration) + public TestLifecycle(TestLog log, Configuration configuration) { - Log = new TestLog(configuration.GetLogConfig()); + Log = log; workflowCreator = new WorkflowCreator(configuration.GetK8sConfiguration()); FileManager = new FileManager(Log, configuration); @@ -37,7 +37,7 @@ namespace DistTestCore var description = node.Describe(); var handler = new LogDownloadHandler(description, subFile); - Log.Log($"Downloading logs for {description} to file {subFile.FilenameWithoutPath}"); + Log.Log($"Downloading logs for {description} to file '{subFile.FullFilename}'"); CodexStarter.DownloadLog(node.CodexAccess.Container, handler); return new CodexNodeLog(subFile); diff --git a/Logging/BaseLog.cs b/Logging/BaseLog.cs new file mode 100644 index 0000000..e19ec12 --- /dev/null +++ b/Logging/BaseLog.cs @@ -0,0 +1,36 @@ +namespace Logging +{ + public abstract class BaseLog + { + private bool hasFailed; + private LogFile? logFile; + + protected abstract LogFile CreateLogFile(); + + protected LogFile LogFile + { + get + { + if (logFile == null) logFile = CreateLogFile(); + return logFile; + } + } + + public void Log(string message) + { + LogFile.Write(message); + } + + public void Error(string message) + { + Log($"[ERROR] {message}"); + } + + public void MarkAsFailed() + { + if (hasFailed) return; + hasFailed = true; + LogFile.ConcatToFilename("_FAILED"); + } + } +} diff --git a/Logging/FixtureLog.cs b/Logging/FixtureLog.cs new file mode 100644 index 0000000..e06f4bc --- /dev/null +++ b/Logging/FixtureLog.cs @@ -0,0 +1,49 @@ +using NUnit.Framework; + +namespace Logging +{ + public class FixtureLog : BaseLog + { + private readonly DateTime start; + private readonly string fullName; + + public FixtureLog(LogConfig config) + { + start = DateTime.UtcNow; + var folder = DetermineFolder(config); // "root/2023-04 /14" + var fixtureName = GetFixtureName(); // "11-09-23Z_ExampleTests" + fullName = Path.Combine(folder, fixtureName); + } + + public TestLog CreateTestLog() + { + return new TestLog(fullName); + } + + protected override LogFile CreateLogFile() + { + return new LogFile(fullName, "log"); + } + + private string DetermineFolder(LogConfig config) + { + return Path.Join( + config.LogRoot, + $"{start.Year}-{Pad(start.Month)}", + Pad(start.Day)); + } + + private string GetFixtureName() + { + var test = TestContext.CurrentContext.Test; + var className = test.ClassName!.Substring(test.ClassName.LastIndexOf('.') + 1); + return $"{Pad(start.Hour)}-{Pad(start.Minute)}-{Pad(start.Second)}Z_{className.Replace('.', '-')}"; + } + + private static string Pad(int n) + { + return n.ToString().PadLeft(2, '0'); + } + + } +} diff --git a/Logging/LogFile.cs b/Logging/LogFile.cs index 3a0063b..96ed16b 100644 --- a/Logging/LogFile.cs +++ b/Logging/LogFile.cs @@ -2,29 +2,19 @@ { public class LogFile { - private readonly DateTime now; - private string name; - private readonly string ext; - private readonly string filepath; + private readonly string extension; + private string filename; - public LogFile(LogConfig config, DateTime now, string name, string ext = "log") + public LogFile(string filename, string extension) { - this.now = now; - this.name = name; - this.ext = ext; + this.filename = filename; + this.extension = extension; + FullFilename = filename + "." + extension; - filepath = Path.Join( - config.LogRoot, - $"{now.Year}-{Pad(now.Month)}", - Pad(now.Day)); - - Directory.CreateDirectory(filepath); - - GenerateFilename(); + EnsurePathExists(filename); } - public string FullFilename { get; private set; } = string.Empty; - public string FilenameWithoutPath { get; private set; } = string.Empty; + public string FullFilename { get; private set; } public void Write(string message) { @@ -47,27 +37,21 @@ { var oldFullName = FullFilename; - name += toAdd; - - GenerateFilename(); + filename += toAdd; + FullFilename = filename + "." + extension; File.Move(oldFullName, FullFilename); } - private static string Pad(int n) - { - return n.ToString().PadLeft(2, '0'); - } - private static string GetTimestamp() { return $"[{DateTime.UtcNow.ToString("u")}]"; } - private void GenerateFilename() + private void EnsurePathExists(string filename) { - FilenameWithoutPath = $"{Pad(now.Hour)}-{Pad(now.Minute)}-{Pad(now.Second)}Z_{name.Replace('.', '-')}.{ext}"; - FullFilename = Path.Combine(filepath, FilenameWithoutPath); + var path = new FileInfo(filename).Directory!.FullName; + Directory.CreateDirectory(path); } } } diff --git a/Logging/TestLog.cs b/Logging/TestLog.cs index 7eb3979..8c4dc17 100644 --- a/Logging/TestLog.cs +++ b/Logging/TestLog.cs @@ -3,39 +3,30 @@ using Utils; namespace Logging { - public class TestLog + public class TestLog : BaseLog { private readonly NumberSource subfileNumberSource = new NumberSource(0); - private readonly LogFile file; - private readonly DateTime now; - private readonly LogConfig config; + private readonly string methodName; + private readonly string fullName; - public TestLog(LogConfig config) + public TestLog(string folder) { - this.config = config; - now = DateTime.UtcNow; + methodName = GetMethodName(); + fullName = Path.Combine(folder, methodName); - var name = GetTestName(); - file = new LogFile(config, now, name); - - Log($"Begin: {name}"); + Log($"Begin: {methodName}"); } - public void Log(string message) + public LogFile CreateSubfile(string ext = "log") { - file.Write(message); - } - - public void Error(string message) - { - Log($"[ERROR] {message}"); + return new LogFile($"{fullName}_{GetSubfileNumber()}", ext); } public void EndTest() { var result = TestContext.CurrentContext.Result; - Log($"Finished: {GetTestName()} = {result.Outcome.Status}"); + Log($"Finished: {methodName} = {result.Outcome.Status}"); if (!string.IsNullOrEmpty(result.Message)) { Log(result.Message); @@ -44,26 +35,24 @@ namespace Logging if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed) { - RenameLogFile(); + MarkAsFailed(); } } - - private void RenameLogFile() + protected override LogFile CreateLogFile() { - file.ConcatToFilename("_FAILED"); + return new LogFile(fullName, "log"); } - public LogFile CreateSubfile(string ext = "log") - { - return new LogFile(config, now, $"{GetTestName()}_{subfileNumberSource.GetNextNumber().ToString().PadLeft(6, '0')}", ext); - } - - private static string GetTestName() + private string GetMethodName() { var test = TestContext.CurrentContext.Test; - var className = test.ClassName!.Substring(test.ClassName.LastIndexOf('.') + 1); var args = FormatArguments(test); - return $"{className}.{test.MethodName}{args}"; + return $"{test.MethodName}{args}"; + } + + private string GetSubfileNumber() + { + return subfileNumberSource.GetNextNumber().ToString().PadLeft(6, '0'); } private static string FormatArguments(TestContext.TestAdapter test) diff --git a/Tests/BasicTests/SimpleTests.cs b/Tests/BasicTests/ExampleTests.cs similarity index 56% rename from Tests/BasicTests/SimpleTests.cs rename to Tests/BasicTests/ExampleTests.cs index 943b598..a3e3ae4 100644 --- a/Tests/BasicTests/SimpleTests.cs +++ b/Tests/BasicTests/ExampleTests.cs @@ -1,69 +1,12 @@ using DistTestCore; using DistTestCore.Codex; -using KubernetesWorkflow; using NUnit.Framework; namespace Tests.BasicTests { [TestFixture] - public class SimpleTests : DistTest + public class ExampleTests : DistTest { - [Test] - public void OneClientTest() - { - var primary = SetupCodexNodes(1).BringOnline()[0]; - - PerformOneClientTest(primary); - } - - [Test] - public void RestartTest() - { - var group = SetupCodexNodes(1).BringOnline(); - - var setup = group.BringOffline(); - - var primary = setup.BringOnline()[0]; - - PerformOneClientTest(primary); - } - - [Test] - public void TwoClientsOnePodTest() - { - var group = SetupCodexNodes(2).BringOnline(); - - var primary = group[0]; - var secondary = group[1]; - - PerformTwoClientTest(primary, secondary); - } - - [Test] - public void TwoClientsTwoPodsTest() - { - var primary = SetupCodexNodes(1).BringOnline()[0]; - - var secondary = SetupCodexNodes(1).BringOnline()[0]; - - PerformTwoClientTest(primary, secondary); - } - - [Test] - [Ignore("Requires Location map to be configured for k8s cluster.")] - public void TwoClientsTwoLocationsTest() - { - var primary = SetupCodexNodes(1) - .At(Location.BensLaptop) - .BringOnline()[0]; - - var secondary = SetupCodexNodes(1) - .At(Location.BensOldGamingMachine) - .BringOnline()[0]; - - PerformTwoClientTest(primary, secondary); - } - [Test] public void CodexLogExample() { @@ -139,29 +82,5 @@ namespace Tests.BasicTests //secondary.Marketplace.AssertThatBalance(Is.LessThan(1000), "Contractor was not charged for storage."); //primary.Marketplace.AssertThatBalance(Is.GreaterThan(primaryBalance), "Storer was not paid for storage."); } - - private void PerformOneClientTest(IOnlineCodexNode primary) - { - var testFile = GenerateTestFile(1.MB()); - - var contentId = primary.UploadFile(testFile); - - var downloadedFile = primary.DownloadContent(contentId); - - testFile.AssertIsEqual(downloadedFile); - } - - private void PerformTwoClientTest(IOnlineCodexNode primary, IOnlineCodexNode secondary) - { - primary.ConnectToPeer(secondary); - - var testFile = GenerateTestFile(1.MB()); - - var contentId = primary.UploadFile(testFile); - - var downloadedFile = secondary.DownloadContent(contentId); - - testFile.AssertIsEqual(downloadedFile); - } } } diff --git a/Tests/BasicTests/OneClientTests.cs b/Tests/BasicTests/OneClientTests.cs new file mode 100644 index 0000000..76ff79b --- /dev/null +++ b/Tests/BasicTests/OneClientTests.cs @@ -0,0 +1,41 @@ +using DistTestCore; +using NUnit.Framework; + +namespace Tests.BasicTests +{ + [TestFixture] + public class OneClientTests : DistTest + { + [Test] + public void OneClientTest() + { + var primary = SetupCodexNodes(1).BringOnline()[0]; + + PerformOneClientTest(primary); + } + + [Test] + [Ignore("Unstable.")] + public void RestartTest() + { + var group = SetupCodexNodes(1).BringOnline(); + + var setup = group.BringOffline(); + + var primary = setup.BringOnline()[0]; + + PerformOneClientTest(primary); + } + + private void PerformOneClientTest(IOnlineCodexNode primary) + { + var testFile = GenerateTestFile(1.MB()); + + var contentId = primary.UploadFile(testFile); + + var downloadedFile = primary.DownloadContent(contentId); + + testFile.AssertIsEqual(downloadedFile); + } + } +} diff --git a/Tests/BasicTests/TwoClientTests.cs b/Tests/BasicTests/TwoClientTests.cs new file mode 100644 index 0000000..cde3b04 --- /dev/null +++ b/Tests/BasicTests/TwoClientTests.cs @@ -0,0 +1,59 @@ +using DistTestCore; +using KubernetesWorkflow; +using NUnit.Framework; + +namespace Tests.BasicTests +{ + [TestFixture] + public class TwoClientTests : DistTest + { + [Test] + public void TwoClientsOnePodTest() + { + var group = SetupCodexNodes(2).BringOnline(); + + var primary = group[0]; + var secondary = group[1]; + + PerformTwoClientTest(primary, secondary); + } + + [Test] + public void TwoClientsTwoPodsTest() + { + var primary = SetupCodexNodes(1).BringOnline()[0]; + + var secondary = SetupCodexNodes(1).BringOnline()[0]; + + PerformTwoClientTest(primary, secondary); + } + + [Test] + [Ignore("Requires Location map to be configured for k8s cluster.")] + public void TwoClientsTwoLocationsTest() + { + var primary = SetupCodexNodes(1) + .At(Location.BensLaptop) + .BringOnline()[0]; + + var secondary = SetupCodexNodes(1) + .At(Location.BensOldGamingMachine) + .BringOnline()[0]; + + PerformTwoClientTest(primary, secondary); + } + + private void PerformTwoClientTest(IOnlineCodexNode primary, IOnlineCodexNode secondary) + { + primary.ConnectToPeer(secondary); + + var testFile = GenerateTestFile(1.MB()); + + var contentId = primary.UploadFile(testFile); + + var downloadedFile = secondary.DownloadContent(contentId); + + testFile.AssertIsEqual(downloadedFile); + } + } +} diff --git a/Utils/Time.cs b/Utils/Time.cs index 98c5979..afe4f29 100644 --- a/Utils/Time.cs +++ b/Utils/Time.cs @@ -12,5 +12,15 @@ task.Wait(); return task.Result; } + + public static string FormatDuration(TimeSpan d) + { + var result = ""; + if (d.Days > 0) result += $"{d.Days} days, "; + if (d.Hours > 0) result += $"{d.Hours} hours, "; + if (d.Minutes > 0) result += $"{d.Minutes} mins, "; + result += $"{d.Seconds} secs"; + return result; + } } } From 3f159b8ece92fed6dfa56ef7860e7c463d011f58 Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 17 Apr 2023 07:56:08 +0200 Subject: [PATCH 33/46] Removes old backend --- CodexDistTestCore/ByteSize.cs | 57 --- CodexDistTestCore/CodexAPI.cs | 17 - CodexDistTestCore/CodexDistTestCore.csproj | 16 - CodexDistTestCore/CodexNodeContainer.cs | 89 ----- CodexDistTestCore/CodexNodeGroup.cs | 103 ------ CodexDistTestCore/CodexNodeLog.cs | 34 -- CodexDistTestCore/Config/CodexDockerImage.cs | 72 ---- CodexDistTestCore/Config/FileManagerConfig.cs | 7 - CodexDistTestCore/Config/K8sCluster.cs | 40 -- CodexDistTestCore/DistTest.cs | 120 ------ CodexDistTestCore/FileManager.cs | 110 ------ CodexDistTestCore/Http.cs | 100 ----- CodexDistTestCore/K8sManager.cs | 177 --------- CodexDistTestCore/K8sOperations.cs | 349 ------------------ CodexDistTestCore/KnownK8sPods.cs | 17 - .../Marketplace/GethCompanionNodeContainer.cs | 35 -- CodexDistTestCore/Marketplace/K8sGethSpecs.cs | 207 ----------- .../Marketplace/MarketplaceAccess.cs | 111 ------ .../Marketplace/MarketplaceController.cs | 24 -- .../Marketplace/MarketplaceInitialConfig.cs | 4 - .../Metrics/K8sPrometheusSpecs.cs | 122 ------ CodexDistTestCore/Metrics/MetricsAccess.cs | 63 ---- .../Metrics/MetricsAggregator.cs | 78 ---- .../Metrics/MetricsDownloader.cs | 97 ----- CodexDistTestCore/Metrics/MetricsQuery.cs | 190 ---------- CodexDistTestCore/NumberSource.cs | 19 - CodexDistTestCore/OfflineCodexNodes.cs | 97 ----- CodexDistTestCore/OnlineCodexNode.cs | 141 ------- CodexDistTestCore/PodLogDownloader.cs | 64 ---- CodexDistTestCore/Timing.cs | 131 ------- CodexDistTestCore/TryContract.cs | 101 ----- CodexDistTestCore/Utils.cs | 7 - LongTests/BasicTests/LargeFileTests.cs | 3 +- LongTests/BasicTests/TestInfraTests.cs | 3 +- LongTests/TestsLong.csproj | 2 +- cs-codex-dist-testing.sln | 8 +- 36 files changed, 6 insertions(+), 2809 deletions(-) delete mode 100644 CodexDistTestCore/ByteSize.cs delete mode 100644 CodexDistTestCore/CodexAPI.cs delete mode 100644 CodexDistTestCore/CodexDistTestCore.csproj delete mode 100644 CodexDistTestCore/CodexNodeContainer.cs delete mode 100644 CodexDistTestCore/CodexNodeGroup.cs delete mode 100644 CodexDistTestCore/CodexNodeLog.cs delete mode 100644 CodexDistTestCore/Config/CodexDockerImage.cs delete mode 100644 CodexDistTestCore/Config/FileManagerConfig.cs delete mode 100644 CodexDistTestCore/Config/K8sCluster.cs delete mode 100644 CodexDistTestCore/DistTest.cs delete mode 100644 CodexDistTestCore/FileManager.cs delete mode 100644 CodexDistTestCore/Http.cs delete mode 100644 CodexDistTestCore/K8sManager.cs delete mode 100644 CodexDistTestCore/K8sOperations.cs delete mode 100644 CodexDistTestCore/KnownK8sPods.cs delete mode 100644 CodexDistTestCore/Marketplace/GethCompanionNodeContainer.cs delete mode 100644 CodexDistTestCore/Marketplace/K8sGethSpecs.cs delete mode 100644 CodexDistTestCore/Marketplace/MarketplaceAccess.cs delete mode 100644 CodexDistTestCore/Marketplace/MarketplaceController.cs delete mode 100644 CodexDistTestCore/Marketplace/MarketplaceInitialConfig.cs delete mode 100644 CodexDistTestCore/Metrics/K8sPrometheusSpecs.cs delete mode 100644 CodexDistTestCore/Metrics/MetricsAccess.cs delete mode 100644 CodexDistTestCore/Metrics/MetricsAggregator.cs delete mode 100644 CodexDistTestCore/Metrics/MetricsDownloader.cs delete mode 100644 CodexDistTestCore/Metrics/MetricsQuery.cs delete mode 100644 CodexDistTestCore/NumberSource.cs delete mode 100644 CodexDistTestCore/OfflineCodexNodes.cs delete mode 100644 CodexDistTestCore/OnlineCodexNode.cs delete mode 100644 CodexDistTestCore/PodLogDownloader.cs delete mode 100644 CodexDistTestCore/Timing.cs delete mode 100644 CodexDistTestCore/TryContract.cs delete mode 100644 CodexDistTestCore/Utils.cs diff --git a/CodexDistTestCore/ByteSize.cs b/CodexDistTestCore/ByteSize.cs deleted file mode 100644 index e8f5c92..0000000 --- a/CodexDistTestCore/ByteSize.cs +++ /dev/null @@ -1,57 +0,0 @@ -namespace CodexDistTestCore -{ - public class ByteSize - { - public ByteSize(long sizeInBytes) - { - SizeInBytes = sizeInBytes; - } - - public long SizeInBytes { get; } - } - - public static class IntExtensions - { - private const long Kilo = 1024; - - public static ByteSize KB(this long i) - { - return new ByteSize(i * Kilo); - } - - public static ByteSize MB(this long i) - { - return (i * Kilo).KB(); - } - - public static ByteSize GB(this long i) - { - return (i * Kilo).MB(); - } - - public static ByteSize TB(this long i) - { - return (i * Kilo).GB(); - } - - public static ByteSize KB(this int i) - { - return Convert.ToInt64(i).KB(); - } - - public static ByteSize MB(this int i) - { - return Convert.ToInt64(i).MB(); - } - - public static ByteSize GB(this int i) - { - return Convert.ToInt64(i).GB(); - } - - public static ByteSize TB(this int i) - { - return Convert.ToInt64(i).TB(); - } - } -} diff --git a/CodexDistTestCore/CodexAPI.cs b/CodexDistTestCore/CodexAPI.cs deleted file mode 100644 index 997cd3c..0000000 --- a/CodexDistTestCore/CodexAPI.cs +++ /dev/null @@ -1,17 +0,0 @@ -namespace CodexDistTestCore -{ - public class CodexDebugResponse - { - public string id { get; set; } = string.Empty; - public string[] addrs { get; set; } = new string[0]; - public string repo { get; set; } = string.Empty; - public string spr { get; set; } = string.Empty; - public CodexDebugVersionResponse codex { get; set; } = new(); - } - - public class CodexDebugVersionResponse - { - public string version { get; set; } = string.Empty; - public string revision { get; set; } = string.Empty; - } -} diff --git a/CodexDistTestCore/CodexDistTestCore.csproj b/CodexDistTestCore/CodexDistTestCore.csproj deleted file mode 100644 index cb1fb2e..0000000 --- a/CodexDistTestCore/CodexDistTestCore.csproj +++ /dev/null @@ -1,16 +0,0 @@ - - - - net6.0 - CodexDistTestCore - enable - enable - - - - - - - - - diff --git a/CodexDistTestCore/CodexNodeContainer.cs b/CodexDistTestCore/CodexNodeContainer.cs deleted file mode 100644 index 71994ab..0000000 --- a/CodexDistTestCore/CodexNodeContainer.cs +++ /dev/null @@ -1,89 +0,0 @@ -using CodexDistTestCore.Marketplace; - -namespace CodexDistTestCore -{ - public class CodexNodeContainer - { - public CodexNodeContainer(string name, int servicePort, string servicePortName, int apiPort, string containerPortName, int discoveryPort, int listenPort, string dataDir, int metricsPort) - { - Name = name; - ServicePort = servicePort; - ServicePortName = servicePortName; - ApiPort = apiPort; - ContainerPortName = containerPortName; - DiscoveryPort = discoveryPort; - ListenPort = listenPort; - DataDir = dataDir; - MetricsPort = metricsPort; - } - - public string Name { get; } - public int ServicePort { get; } - public string ServicePortName { get; } - public int ApiPort { get; } - public string ContainerPortName { get; } - public int DiscoveryPort { get; } - public int ListenPort { get; } - public string DataDir { get; } - public int MetricsPort { get; } - - public GethCompanionNodeContainer? GethCompanionNodeContainer { get; set; } // :C - } - - public class CodexGroupNumberSource - { - private readonly NumberSource codexNodeGroupNumberSource = new NumberSource(0); - private readonly NumberSource groupContainerNameSource = new NumberSource(1); - private readonly NumberSource servicePortSource = new NumberSource(30001); - - public int GetNextCodexNodeGroupNumber() - { - return codexNodeGroupNumberSource.GetNextNumber(); - } - - public string GetNextServicePortName() - { - return $"node{groupContainerNameSource.GetNextNumber()}"; - } - - public int GetNextServicePort() - { - return servicePortSource.GetNextNumber(); - } - } - - public class CodexNodeContainerFactory - { - private readonly NumberSource containerNameSource = new NumberSource(1); - private readonly NumberSource codexPortSource = new NumberSource(8080); - private readonly CodexGroupNumberSource numberSource; - - public CodexNodeContainerFactory(CodexGroupNumberSource numberSource) - { - this.numberSource = numberSource; - } - - public CodexNodeContainer CreateNext(OfflineCodexNodes offline) - { - var n = containerNameSource.GetNextNumber(); - return new CodexNodeContainer( - name: $"codex-node{n}", - servicePort: numberSource.GetNextServicePort(), - servicePortName: numberSource.GetNextServicePortName(), - apiPort: codexPortSource.GetNextNumber(), - containerPortName: $"api-{n}", - discoveryPort: codexPortSource.GetNextNumber(), - listenPort: codexPortSource.GetNextNumber(), - dataDir: $"datadir{n}", - metricsPort: GetMetricsPort(offline) - ); - } - - private int GetMetricsPort(OfflineCodexNodes offline) - { - if (offline.MetricsEnabled) return codexPortSource.GetNextNumber(); - return 0; - } - - } -} diff --git a/CodexDistTestCore/CodexNodeGroup.cs b/CodexDistTestCore/CodexNodeGroup.cs deleted file mode 100644 index 12031ac..0000000 --- a/CodexDistTestCore/CodexNodeGroup.cs +++ /dev/null @@ -1,103 +0,0 @@ -using CodexDistTestCore.Config; -using CodexDistTestCore.Marketplace; -using k8s.Models; -using System.Collections; - -namespace CodexDistTestCore -{ - public interface ICodexNodeGroup : IEnumerable - { - IOfflineCodexNodes BringOffline(); - IOnlineCodexNode this[int index] { get; } - } - - public class CodexNodeGroup : ICodexNodeGroup - { - private readonly TestLog log; - private readonly IK8sManager k8SManager; - - public CodexNodeGroup(TestLog log, int orderNumber, OfflineCodexNodes origin, IK8sManager k8SManager, OnlineCodexNode[] nodes) - { - this.log = log; - OrderNumber = orderNumber; - Origin = origin; - this.k8SManager = k8SManager; - Nodes = nodes; - - foreach (var n in nodes) n.Group = this; - } - - public IOnlineCodexNode this[int index] - { - get - { - return Nodes[index]; - } - } - - public IOfflineCodexNodes BringOffline() - { - return k8SManager.BringOffline(this); - } - - public int OrderNumber { get; } - public OfflineCodexNodes Origin { get; } - public OnlineCodexNode[] Nodes { get; } - public V1Deployment? Deployment { get; set; } - public V1Service? Service { get; set; } - public PodInfo? PodInfo { get; set; } - public GethCompanionGroup? GethCompanionGroup { get; set; } - - public CodexNodeContainer[] GetContainers() - { - return Nodes.Select(n => n.Container).ToArray(); - } - - public IEnumerator GetEnumerator() - { - return Nodes.Cast().GetEnumerator(); - } - - IEnumerator IEnumerable.GetEnumerator() - { - return Nodes.GetEnumerator(); - } - - public V1ObjectMeta GetServiceMetadata() - { - return new V1ObjectMeta - { - Name = "codex-test-entrypoint-" + OrderNumber, - NamespaceProperty = K8sCluster.K8sNamespace - }; - } - - public V1ObjectMeta GetDeploymentMetadata() - { - return new V1ObjectMeta - { - Name = "codex-test-node-" + OrderNumber, - NamespaceProperty = K8sCluster.K8sNamespace - }; - } - - public CodexNodeLog DownloadLog(IOnlineCodexNode node) - { - var logDownloader = new PodLogDownloader(log, k8SManager); - var n = (OnlineCodexNode)node; - return logDownloader.DownloadLog(n); - } - - public Dictionary GetSelector() - { - return new Dictionary { { "codex-test-node", "dist-test-" + OrderNumber } }; - } - - public string Describe() - { - return $"CodexNodeGroup#{OrderNumber}-{Origin.Describe()}"; - } - } - - -} diff --git a/CodexDistTestCore/CodexNodeLog.cs b/CodexDistTestCore/CodexNodeLog.cs deleted file mode 100644 index 1a0572a..0000000 --- a/CodexDistTestCore/CodexNodeLog.cs +++ /dev/null @@ -1,34 +0,0 @@ -using NUnit.Framework; - -namespace CodexDistTestCore -{ - public interface ICodexNodeLog - { - void AssertLogContains(string expectedString); - } - - public class CodexNodeLog : ICodexNodeLog - { - private readonly LogFile logFile; - - public CodexNodeLog(LogFile logFile) - { - this.logFile = logFile; - } - - public void AssertLogContains(string expectedString) - { - using var file = File.OpenRead(logFile.FullFilename); - using var streamReader = new StreamReader(file); - - var line = streamReader.ReadLine(); - while (line != null) - { - if (line.Contains(expectedString)) return; - line = streamReader.ReadLine(); - } - - Assert.Fail($"Unable to find string '{expectedString}' in CodexNode log file {logFile.FilenameWithoutPath}"); - } - } -} diff --git a/CodexDistTestCore/Config/CodexDockerImage.cs b/CodexDistTestCore/Config/CodexDockerImage.cs deleted file mode 100644 index a252a26..0000000 --- a/CodexDistTestCore/Config/CodexDockerImage.cs +++ /dev/null @@ -1,72 +0,0 @@ -using k8s.Models; - -namespace CodexDistTestCore.Config -{ - public class CodexDockerImage - { - public string GetImageTag() - { - return "thatbenbierens/nim-codex:sha-b204837"; - } - - public string GetExpectedImageRevision() - { - return "b20483"; - } - - public List CreateEnvironmentVariables(OfflineCodexNodes node, CodexNodeContainer container) - { - var formatter = new EnvFormatter(); - formatter.Create(node, container); - return formatter.Result; - } - - private class EnvFormatter - { - public List Result { get; } = new List(); - - public void Create(OfflineCodexNodes node, CodexNodeContainer container) - { - AddVar("API_PORT", container.ApiPort.ToString()); - AddVar("DATA_DIR", container.DataDir); - AddVar("DISC_PORT", container.DiscoveryPort.ToString()); - AddVar("LISTEN_ADDRS", $"/ip4/0.0.0.0/tcp/{container.ListenPort}"); - - if (node.BootstrapNode != null) - { - var debugInfo = node.BootstrapNode.GetDebugInfo(); - AddVar("BOOTSTRAP_SPR", debugInfo.spr); - } - if (node.LogLevel != null) - { - AddVar("LOG_LEVEL", node.LogLevel.ToString()!.ToUpperInvariant()); - } - if (node.StorageQuota != null) - { - AddVar("STORAGE_QUOTA", node.StorageQuota.SizeInBytes.ToString()!); - } - if (node.MetricsEnabled) - { - AddVar("METRICS_ADDR", "0.0.0.0"); - AddVar("METRICS_PORT", container.MetricsPort.ToString()); - } - if (node.MarketplaceConfig != null) - { - //ETH_PROVIDER - //ETH_ACCOUNT - //ETH_DEPLOYMENT - AddVar("ETH_ACCOUNT", container.GethCompanionNodeContainer!.Account); - } - } - - private void AddVar(string key, string value) - { - Result.Add(new V1EnvVar - { - Name = key, - Value = value - }); - } - } - } -} diff --git a/CodexDistTestCore/Config/FileManagerConfig.cs b/CodexDistTestCore/Config/FileManagerConfig.cs deleted file mode 100644 index f7befc2..0000000 --- a/CodexDistTestCore/Config/FileManagerConfig.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace CodexDistTestCore.Config -{ - public class FileManagerConfig - { - public const string Folder = "TestDataFiles"; - } -} diff --git a/CodexDistTestCore/Config/K8sCluster.cs b/CodexDistTestCore/Config/K8sCluster.cs deleted file mode 100644 index a17e6fd..0000000 --- a/CodexDistTestCore/Config/K8sCluster.cs +++ /dev/null @@ -1,40 +0,0 @@ -using k8s; - -namespace CodexDistTestCore.Config -{ - public class K8sCluster - { - public const string K8sNamespace = ""; - private const string KubeConfigFile = "C:\\kube\\config"; - private readonly Dictionary K8sNodeLocationMap = new Dictionary - { - { Location.BensLaptop, "worker01" }, - { Location.BensOldGamingMachine, "worker02" }, - }; - - private KubernetesClientConfiguration? config; - - public KubernetesClientConfiguration GetK8sClientConfig() - { - if (config != null) return config; - //config = KubernetesClientConfiguration.BuildConfigFromConfigFile(KubeConfigFile); - config = KubernetesClientConfiguration.BuildDefaultConfig(); - return config; - } - - public string GetIp() - { - var c = GetK8sClientConfig(); - - var host = c.Host.Replace("https://", ""); - - return host.Substring(0, host.IndexOf(':')); - } - - public string GetNodeLabelForLocation(Location location) - { - if (location == Location.Unspecified) return string.Empty; - return K8sNodeLocationMap[location]; - } - } -} diff --git a/CodexDistTestCore/DistTest.cs b/CodexDistTestCore/DistTest.cs deleted file mode 100644 index bf95aa7..0000000 --- a/CodexDistTestCore/DistTest.cs +++ /dev/null @@ -1,120 +0,0 @@ -using CodexDistTestCore.Config; -using NUnit.Framework; - -namespace CodexDistTestCore -{ - [SetUpFixture] - public abstract class DistTest - { - private TestLog log = null!; - private FileManager fileManager = null!; - public K8sManager k8sManager = null!; - - [OneTimeSetUp] - public void GlobalSetup() - { - // Previous test run may have been interrupted. - // Begin by cleaning everything up. - log = new TestLog(); - fileManager = new FileManager(log); - k8sManager = new K8sManager(log, fileManager); - - try - { - k8sManager.DeleteAllResources(); - fileManager.DeleteAllTestFiles(); - } - catch (Exception ex) - { - GlobalTestFailure.HasFailed = true; - log.Error($"Global setup cleanup failed with: {ex}"); - throw; - } - log.Log("Global setup cleanup successful"); - } - - [SetUp] - public void SetUpDistTest() - { - if (GlobalTestFailure.HasFailed) - { - Assert.Inconclusive("Skip test: Previous test failed during clean up."); - } - else - { - var dockerImage = new CodexDockerImage(); - log = new TestLog(); - log.Log($"Using docker image '{dockerImage.GetImageTag()}'"); - - fileManager = new FileManager(log); - k8sManager = new K8sManager(log, fileManager); - } - } - - [TearDown] - public void TearDownDistTest() - { - try - { - log.EndTest(); - IncludeLogsAndMetricsOnTestFailure(); - k8sManager.DeleteAllResources(); - fileManager.DeleteAllTestFiles(); - } - catch (Exception ex) - { - log.Error("Cleanup failed: " + ex.Message); - GlobalTestFailure.HasFailed = true; - } - } - - public TestFile GenerateTestFile(ByteSize size) - { - return fileManager.GenerateTestFile(size); - } - - public IOfflineCodexNodes SetupCodexNodes(int numberOfNodes) - { - return new OfflineCodexNodes(k8sManager, numberOfNodes); - } - - private void IncludeLogsAndMetricsOnTestFailure() - { - var result = TestContext.CurrentContext.Result; - if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed) - { - if (IsDownloadingLogsAndMetricsEnabled()) - { - log.Log("Downloading all CodexNode logs and metrics because of test failure..."); - k8sManager.ForEachOnlineGroup(DownloadLogs); - k8sManager.DownloadAllMetrics(); - } - else - { - log.Log("Skipping download of all CodexNode logs and metrics due to [DontDownloadLogsAndMetricsOnFailure] attribute."); - } - } - } - - private void DownloadLogs(CodexNodeGroup group) - { - foreach (var node in group) - { - var downloader = new PodLogDownloader(log, k8sManager); - var n = (OnlineCodexNode)node; - downloader.DownloadLog(n); - } - } - - private bool IsDownloadingLogsAndMetricsEnabled() - { - var testProperties = TestContext.CurrentContext.Test.Properties; - return !testProperties.ContainsKey(PodLogDownloader.DontDownloadLogsOnFailureKey); - } - } - - public static class GlobalTestFailure - { - public static bool HasFailed { get; set; } = false; - } -} diff --git a/CodexDistTestCore/FileManager.cs b/CodexDistTestCore/FileManager.cs deleted file mode 100644 index 6fbd55f..0000000 --- a/CodexDistTestCore/FileManager.cs +++ /dev/null @@ -1,110 +0,0 @@ -using CodexDistTestCore.Config; -using NUnit.Framework; - -namespace CodexDistTestCore -{ - public interface IFileManager - { - TestFile CreateEmptyTestFile(); - TestFile GenerateTestFile(ByteSize size); - void DeleteAllTestFiles(); - } - - public class FileManager : IFileManager - { - public const int ChunkSize = 1024 * 1024; - private readonly Random random = new Random(); - private readonly List activeFiles = new List(); - private readonly TestLog log; - - public FileManager(TestLog log) - { - if (!Directory.Exists(FileManagerConfig.Folder)) Directory.CreateDirectory(FileManagerConfig.Folder); - this.log = log; - } - - public TestFile CreateEmptyTestFile() - { - var result = new TestFile(Path.Combine(FileManagerConfig.Folder, Guid.NewGuid().ToString() + "_test.bin")); - File.Create(result.Filename).Close(); - activeFiles.Add(result); - return result; - } - - public TestFile GenerateTestFile(ByteSize size) - { - var result = CreateEmptyTestFile(); - GenerateFileBytes(result, size); - log.Log($"Generated {size.SizeInBytes} bytes of content for file '{result.Filename}'."); - return result; - } - - public void DeleteAllTestFiles() - { - foreach (var file in activeFiles) File.Delete(file.Filename); - activeFiles.Clear(); - } - - private void GenerateFileBytes(TestFile result, ByteSize size) - { - long bytesLeft = size.SizeInBytes; - while (bytesLeft > 0) - { - var length = Math.Min(bytesLeft, ChunkSize); - AppendRandomBytesToFile(result, length); - bytesLeft -= length; - } - } - - private void AppendRandomBytesToFile(TestFile result, long length) - { - var bytes = new byte[length]; - random.NextBytes(bytes); - using var stream = new FileStream(result.Filename, FileMode.Append); - stream.Write(bytes, 0, bytes.Length); - } - } - - public class TestFile - { - public TestFile(string filename) - { - Filename = filename; - } - - public string Filename { get; } - - public long GetFileSize() - { - var info = new FileInfo(Filename); - return info.Length; - } - - public void AssertIsEqual(TestFile? actual) - { - if (actual == null) Assert.Fail("TestFile is null."); - if (actual == this || actual!.Filename == Filename) Assert.Fail("TestFile is compared to itself."); - - Assert.That(actual.GetFileSize(), Is.EqualTo(GetFileSize()), "Files are not of equal length."); - - using var streamExpected = new FileStream(Filename, FileMode.Open, FileAccess.Read); - using var streamActual = new FileStream(actual.Filename, FileMode.Open, FileAccess.Read); - - var bytesExpected = new byte[FileManager.ChunkSize]; - var bytesActual = new byte[FileManager.ChunkSize]; - - var readExpected = 0; - var readActual = 0; - - while (true) - { - readExpected = streamExpected.Read(bytesExpected, 0, FileManager.ChunkSize); - readActual = streamActual.Read(bytesActual, 0, FileManager.ChunkSize); - - if (readExpected == 0 && readActual == 0) return; - Assert.That(readActual, Is.EqualTo(readExpected), "Unable to read buffers of equal length."); - CollectionAssert.AreEqual(bytesExpected, bytesActual, "Files are not binary-equal."); - } - } - } -} diff --git a/CodexDistTestCore/Http.cs b/CodexDistTestCore/Http.cs deleted file mode 100644 index fd7e31a..0000000 --- a/CodexDistTestCore/Http.cs +++ /dev/null @@ -1,100 +0,0 @@ -using Newtonsoft.Json; -using NUnit.Framework; -using System.Net.Http.Headers; - -namespace CodexDistTestCore -{ - public class Http - { - private readonly string ip; - private readonly int port; - private readonly string baseUrl; - - public Http(string ip, int port, string baseUrl) - { - this.ip = ip; - this.port = port; - this.baseUrl = baseUrl; - - if (!this.baseUrl.StartsWith("/")) this.baseUrl = "/" + this.baseUrl; - if (!this.baseUrl.EndsWith("/")) this.baseUrl += "/"; - } - - public string HttpGetString(string route) - { - return Retry(() => - { - using var client = GetClient(); - var url = GetUrl() + route; - var result = Utils.Wait(client.GetAsync(url)); - return Utils.Wait(result.Content.ReadAsStringAsync()); - }); - } - - public T HttpGetJson(string route) - { - return JsonConvert.DeserializeObject(HttpGetString(route))!; - } - - public string HttpPostStream(string route, Stream stream) - { - return Retry(() => - { - using var client = GetClient(); - var url = GetUrl() + route; - - var content = new StreamContent(stream); - content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream"); - var response = Utils.Wait(client.PostAsync(url, content)); - - return Utils.Wait(response.Content.ReadAsStringAsync()); - }); - } - - public Stream HttpGetStream(string route) - { - return Retry(() => - { - var client = GetClient(); - var url = GetUrl() + route; - - return Utils.Wait(client.GetStreamAsync(url)); - }); - } - - private string GetUrl() - { - return $"http://{ip}:{port}{baseUrl}"; - } - - private static T Retry(Func operation) - { - var retryCounter = 0; - - while (true) - { - try - { - return operation(); - } - catch (Exception exception) - { - Timing.HttpCallRetryDelay(); - retryCounter++; - if (retryCounter > Timing.HttpCallRetryCount()) - { - Assert.Fail(exception.Message); - throw; - } - } - } - } - - private static HttpClient GetClient() - { - var client = new HttpClient(); - client.Timeout = Timing.HttpCallTimeout(); - return client; - } - } -} diff --git a/CodexDistTestCore/K8sManager.cs b/CodexDistTestCore/K8sManager.cs deleted file mode 100644 index 06bcafb..0000000 --- a/CodexDistTestCore/K8sManager.cs +++ /dev/null @@ -1,177 +0,0 @@ -using CodexDistTestCore.Marketplace; -using CodexDistTestCore.Metrics; - -namespace CodexDistTestCore -{ - public interface IK8sManager - { - ICodexNodeGroup BringOnline(OfflineCodexNodes node); - IOfflineCodexNodes BringOffline(ICodexNodeGroup node); - void FetchPodLog(OnlineCodexNode node, IPodLogHandler logHandler); - } - - public class K8sManager : IK8sManager - { - private readonly CodexGroupNumberSource codexGroupNumberSource = new CodexGroupNumberSource(); - private readonly List onlineCodexNodeGroups = new List(); - private readonly KnownK8sPods knownPods = new KnownK8sPods(); - private readonly TestLog log; - private readonly IFileManager fileManager; - private readonly MetricsAggregator metricsAggregator; - private readonly MarketplaceController marketplaceController; - - public K8sManager(TestLog log, IFileManager fileManager) - { - this.log = log; - this.fileManager = fileManager; - metricsAggregator = new MetricsAggregator(log, this); - marketplaceController = new MarketplaceController(log, this); - } - - public ICodexNodeGroup BringOnline(OfflineCodexNodes offline) - { - var group = CreateOnlineCodexNodes(offline); - - if (offline.MarketplaceConfig != null) - { - group.GethCompanionGroup = marketplaceController.BringOnlineMarketplace(offline); - ConnectMarketplace(group); - } - - K8s(k => k.BringOnline(group, offline)); - - if (offline.MetricsEnabled) - { - BringOnlineMetrics(group); - } - - log.Log($"{group.Describe()} online."); - - return group; - } - - public IOfflineCodexNodes BringOffline(ICodexNodeGroup node) - { - var online = GetAndRemoveActiveNodeFor(node); - - K8s(k => k.BringOffline(online)); - - log.Log($"{online.Describe()} offline."); - - return online.Origin; - } - - public string ExecuteCommand(PodInfo pod, string containerName, string command, params string[] arguments) - { - return K8s(k => k.ExecuteCommand(pod, containerName, command, arguments)); - } - - public void DeleteAllResources() - { - K8s(k => k.DeleteAllResources()); - } - - public void ForEachOnlineGroup(Action action) - { - foreach (var group in onlineCodexNodeGroups) action(group); - } - - public void FetchPodLog(OnlineCodexNode node, IPodLogHandler logHandler) - { - K8s(k => k.FetchPodLog(node, logHandler)); - } - - public PrometheusInfo BringOnlinePrometheus(string config, int prometheusNumber) - { - var spec = new K8sPrometheusSpecs(codexGroupNumberSource.GetNextServicePort(), prometheusNumber, config); - - return K8s(k => k.BringOnlinePrometheus(spec)); - } - - public K8sGethBoostrapSpecs CreateGethBootstrapNodeSpec() - { - return new K8sGethBoostrapSpecs(codexGroupNumberSource.GetNextServicePort()); - } - - public PodInfo BringOnlineGethBootstrapNode(K8sGethBoostrapSpecs spec) - { - return K8s(k => k.BringOnlineGethBootstrapNode(spec)); - } - - public PodInfo BringOnlineGethCompanionGroup(GethBootstrapInfo info, GethCompanionGroup group) - { - return K8s(k => k.BringOnlineGethCompanionGroup(info, group)); - } - - public void DownloadAllMetrics() - { - metricsAggregator.DownloadAllMetrics(); - } - - private void BringOnlineMetrics(CodexNodeGroup group) - { - metricsAggregator.BeginCollectingMetricsFor(DowncastNodes(group)); - } - - private void ConnectMarketplace(CodexNodeGroup group) - { - for (var i = 0; i < group.Nodes.Length; i++) - { - ConnectMarketplace(group, group.Nodes[i], group.GethCompanionGroup!.Containers[i]); - } - } - - private void ConnectMarketplace(CodexNodeGroup group, OnlineCodexNode node, GethCompanionNodeContainer container) - { - node.Container.GethCompanionNodeContainer = container; // :c - - var access = new MarketplaceAccess(this, marketplaceController, log, group, container); - access.Initialize(); - node.Marketplace = access; - } - - private CodexNodeGroup CreateOnlineCodexNodes(OfflineCodexNodes offline) - { - var containers = CreateContainers(offline); - var online = containers.Select(c => new OnlineCodexNode(log, fileManager, c)).ToArray(); - var result = new CodexNodeGroup(log, codexGroupNumberSource.GetNextCodexNodeGroupNumber(), offline, this, online); - onlineCodexNodeGroups.Add(result); - return result; - } - - private CodexNodeContainer[] CreateContainers(OfflineCodexNodes offline) - { - var factory = new CodexNodeContainerFactory(codexGroupNumberSource); - var containers = new List(); - for (var i = 0; i < offline.NumberOfNodes; i++) containers.Add(factory.CreateNext(offline)); - return containers.ToArray(); - } - - private CodexNodeGroup GetAndRemoveActiveNodeFor(ICodexNodeGroup node) - { - var n = (CodexNodeGroup)node; - onlineCodexNodeGroups.Remove(n); - return n; - } - - private void K8s(Action action) - { - var k8s = new K8sOperations(knownPods); - action(k8s); - k8s.Close(); - } - - private T K8s(Func action) - { - var k8s = new K8sOperations(knownPods); - var result = action(k8s); - k8s.Close(); - return result; - } - - private static OnlineCodexNode[] DowncastNodes(CodexNodeGroup group) - { - return group.Nodes.Cast().ToArray(); - } - } -} diff --git a/CodexDistTestCore/K8sOperations.cs b/CodexDistTestCore/K8sOperations.cs deleted file mode 100644 index 0cc3ea1..0000000 --- a/CodexDistTestCore/K8sOperations.cs +++ /dev/null @@ -1,349 +0,0 @@ -using CodexDistTestCore.Config; -using CodexDistTestCore.Marketplace; -using CodexDistTestCore.Metrics; -using k8s; -using k8s.Models; -using NUnit.Framework; - -namespace CodexDistTestCore -{ - public class K8sOperations - { - private readonly CodexDockerImage dockerImage = new CodexDockerImage(); - private readonly K8sCluster k8sCluster = new K8sCluster(); - private readonly Kubernetes client; - private readonly KnownK8sPods knownPods; - - public K8sOperations(KnownK8sPods knownPods) - { - this.knownPods = knownPods; - - client = new Kubernetes(k8sCluster.GetK8sClientConfig()); - } - - public void Close() - { - client.Dispose(); - } - - public void BringOnline(CodexNodeGroup online, OfflineCodexNodes offline) - { - EnsureTestNamespace(); - - CreateDeployment(online, offline); - CreateService(online); - - WaitUntilOnline(online); - FetchPodInfo(online); - } - - public void BringOffline(CodexNodeGroup online) - { - var deploymentName = online.Deployment.Name(); - DeleteDeployment(online); - DeleteService(online); - WaitUntilOffline(deploymentName); - } - - public void DeleteAllResources() - { - DeleteNamespace(); - - WaitUntilZeroPods(); - WaitUntilNamespaceDeleted(); - } - - public void FetchPodLog(OnlineCodexNode node, IPodLogHandler logHandler) - { - var stream = client.ReadNamespacedPodLog(node.Group.PodInfo!.Name, K8sNamespace, node.Container.Name); - logHandler.Log(stream); - } - - public string ExecuteCommand(PodInfo pod, string containerName, string command, params string[] arguments) - { - var runner = new CommandRunner(client, pod, containerName, command, arguments); - runner.Run(); - return runner.GetStdOut(); - } - - public PrometheusInfo BringOnlinePrometheus(K8sPrometheusSpecs spec) - { - EnsureTestNamespace(); - - CreatePrometheusDeployment(spec); - CreatePrometheusService(spec); - WaitUntilPrometheusOnline(spec); - - return new PrometheusInfo(spec.ServicePort, FetchNewPod()); - } - - public PodInfo BringOnlineGethBootstrapNode(K8sGethBoostrapSpecs spec) - { - EnsureTestNamespace(); - - CreateGethBootstrapDeployment(spec); - CreateGethBootstrapService(spec); - WaitUntilGethBootstrapOnline(spec); - - return FetchNewPod(); - } - - public PodInfo BringOnlineGethCompanionGroup(GethBootstrapInfo info, GethCompanionGroup group) - { - EnsureTestNamespace(); - - CreateGethCompanionDeployment(info, group); - WaitUntilGethCompanionGroupOnline(info.Spec, group); - - return FetchNewPod(); - } - - private void FetchPodInfo(CodexNodeGroup online) - { - online.PodInfo = FetchNewPod(); - } - - private PodInfo FetchNewPod() - { - var pods = client.ListNamespacedPod(K8sNamespace).Items; - - var newPods = pods.Where(p => !knownPods.Contains(p.Name())).ToArray(); - Assert.That(newPods.Length, Is.EqualTo(1), "Expected only 1 pod to be created. Test infra failure."); - - var newPod = newPods.Single(); - var info = new PodInfo(newPod.Name(), newPod.Status.PodIP); - - Assert.That(!string.IsNullOrEmpty(info.Name), "Invalid pod name received. Test infra failure."); - Assert.That(!string.IsNullOrEmpty(info.Ip), "Invalid pod IP received. Test infra failure."); - - knownPods.Add(newPod.Name()); - return info; - } - - #region Waiting - - private void WaitUntilOnline(CodexNodeGroup online) - { - WaitUntil(() => - { - online.Deployment = client.ReadNamespacedDeployment(online.Deployment.Name(), K8sNamespace); - return online.Deployment?.Status.AvailableReplicas != null && online.Deployment.Status.AvailableReplicas > 0; - }); - } - - private void WaitUntilOffline(string deploymentName) - { - WaitUntil(() => - { - var deployment = client.ReadNamespacedDeployment(deploymentName, K8sNamespace); - return deployment == null || deployment.Status.AvailableReplicas == 0; - }); - } - - private void WaitUntilZeroPods() - { - WaitUntil(() => !client.ListNamespacedPod(K8sNamespace).Items.Any()); - } - - private void WaitUntilNamespaceDeleted() - { - WaitUntil(() => !IsTestNamespaceOnline()); - } - - private void WaitUntilPrometheusOnline(K8sPrometheusSpecs spec) - { - WaitUntilDeploymentOnline(spec.GetDeploymentName()); - } - - private void WaitUntilGethBootstrapOnline(K8sGethBoostrapSpecs spec) - { - WaitUntilDeploymentOnline(spec.GetBootstrapDeploymentName()); - } - - private void WaitUntilGethCompanionGroupOnline(K8sGethBoostrapSpecs spec, GethCompanionGroup group) - { - WaitUntilDeploymentOnline(spec.GetCompanionDeploymentName(group)); - } - - private void WaitUntilDeploymentOnline(string deploymentName) - { - WaitUntil(() => - { - var deployment = client.ReadNamespacedDeployment(deploymentName, K8sNamespace); - return deployment?.Status.AvailableReplicas != null && deployment.Status.AvailableReplicas > 0; - }); - } - - private void WaitUntil(Func predicate) - { - var start = DateTime.UtcNow; - var state = predicate(); - while (!state) - { - if (DateTime.UtcNow - start > Timing.K8sOperationTimeout()) - { - Assert.Fail("K8s operation timed out."); - throw new TimeoutException(); - } - - Timing.WaitForK8sServiceDelay(); - state = predicate(); - } - } - - #endregion - - #region Service management - - private void CreateService(CodexNodeGroup online) - { - var serviceSpec = new V1Service - { - ApiVersion = "v1", - Metadata = online.GetServiceMetadata(), - Spec = new V1ServiceSpec - { - Type = "NodePort", - Selector = online.GetSelector(), - Ports = CreateServicePorts(online) - } - }; - - online.Service = client.CreateNamespacedService(serviceSpec, K8sNamespace); - } - - private List CreateServicePorts(CodexNodeGroup online) - { - var result = new List(); - var containers = online.GetContainers(); - foreach (var container in containers) - { - result.Add(new V1ServicePort - { - Name = container.ServicePortName, - Protocol = "TCP", - Port = container.ApiPort, - TargetPort = container.ContainerPortName, - NodePort = container.ServicePort - }); - } - return result; - } - - private void DeleteService(CodexNodeGroup online) - { - if (online.Service == null) return; - client.DeleteNamespacedService(online.Service.Name(), K8sNamespace); - online.Service = null; - } - - private void CreatePrometheusService(K8sPrometheusSpecs spec) - { - client.CreateNamespacedService(spec.CreatePrometheusService(), K8sNamespace); - } - - private void CreateGethBootstrapService(K8sGethBoostrapSpecs spec) - { - client.CreateNamespacedService(spec.CreateGethBootstrapService(), K8sNamespace); - } - - #endregion - - #region Deployment management - - private void CreateDeployment(CodexNodeGroup online, OfflineCodexNodes offline) - { - var deploymentSpec = new V1Deployment - { - ApiVersion = "apps/v1", - Metadata = online.GetDeploymentMetadata(), - Spec = new V1DeploymentSpec - { - Replicas = 1, - Selector = new V1LabelSelector - { - MatchLabels = online.GetSelector() - }, - Template = new V1PodTemplateSpec - { - Metadata = new V1ObjectMeta - { - Labels = online.GetSelector() - }, - Spec = new V1PodSpec - { - NodeSelector = CreateNodeSelector(offline), - Containers = CreateDeploymentContainers(online, offline) - } - } - } - }; - - online.Deployment = client.CreateNamespacedDeployment(deploymentSpec, K8sNamespace); - } - - private IDictionary CreateNodeSelector(OfflineCodexNodes offline) - { - if (offline.Location == Location.Unspecified) return new Dictionary(); - - return new Dictionary - { - { "codex-test-location", k8sCluster.GetNodeLabelForLocation(offline.Location) } - }; - } - - private List CreateDeploymentContainers(CodexNodeGroup group, OfflineCodexNodes offline) - { - var result = new List(); - var containers = group.GetContainers(); - foreach (var container in containers) - { - result.Add(new V1Container - { - Name = container.Name, - Image = dockerImage.GetImageTag(), - Ports = new List - { - new V1ContainerPort - { - ContainerPort = container.ApiPort, - Name = container.ContainerPortName - } - }, - Env = dockerImage.CreateEnvironmentVariables(offline, container) - }); - } - - return result; - } - - private void DeleteDeployment(CodexNodeGroup group) - { - if (group.Deployment == null) return; - client.DeleteNamespacedDeployment(group.Deployment.Name(), K8sNamespace); - group.Deployment = null; - } - - private void CreatePrometheusDeployment(K8sPrometheusSpecs spec) - { - client.CreateNamespacedDeployment(spec.CreatePrometheusDeployment(), K8sNamespace); - } - - private void CreateGethBootstrapDeployment(K8sGethBoostrapSpecs spec) - { - client.CreateNamespacedDeployment(spec.CreateGethBootstrapDeployment(), K8sNamespace); - } - - private void CreateGethCompanionDeployment(GethBootstrapInfo info, GethCompanionGroup group) - { - client.CreateNamespacedDeployment(info.Spec.CreateGethCompanionDeployment(group, info), K8sNamespace); - } - - #endregion - - private class CommandRunner - { - - } - } -} diff --git a/CodexDistTestCore/KnownK8sPods.cs b/CodexDistTestCore/KnownK8sPods.cs deleted file mode 100644 index 940a147..0000000 --- a/CodexDistTestCore/KnownK8sPods.cs +++ /dev/null @@ -1,17 +0,0 @@ -namespace CodexDistTestCore -{ - public class KnownK8sPods - { - private readonly List knownActivePodNames = new List(); - - public bool Contains(string name) - { - return knownActivePodNames.Contains(name); - } - - public void Add(string name) - { - knownActivePodNames.Add(name); - } - } -} diff --git a/CodexDistTestCore/Marketplace/GethCompanionNodeContainer.cs b/CodexDistTestCore/Marketplace/GethCompanionNodeContainer.cs deleted file mode 100644 index 74d687c..0000000 --- a/CodexDistTestCore/Marketplace/GethCompanionNodeContainer.cs +++ /dev/null @@ -1,35 +0,0 @@ -namespace CodexDistTestCore.Marketplace -{ - public class GethCompanionGroup - { - public GethCompanionGroup(int number, GethCompanionNodeContainer[] containers) - { - Number = number; - Containers = containers; - } - - public int Number { get; } - public GethCompanionNodeContainer[] Containers { get; } - public PodInfo? Pod { get; set; } - } - - public class GethCompanionNodeContainer - { - public GethCompanionNodeContainer(string name, int apiPort, int rpcPort, string containerPortName, int authRpcPort) - { - Name = name; - ApiPort = apiPort; - AuthRpcPort = authRpcPort; - RpcPort = rpcPort; - ContainerPortName = containerPortName; - } - - public string Name { get; } - public int ApiPort { get; } - public int AuthRpcPort { get; } - public int RpcPort { get; } - public string ContainerPortName { get; } - - public string Account { get; set; } = string.Empty; - } -} diff --git a/CodexDistTestCore/Marketplace/K8sGethSpecs.cs b/CodexDistTestCore/Marketplace/K8sGethSpecs.cs deleted file mode 100644 index bfaf3d4..0000000 --- a/CodexDistTestCore/Marketplace/K8sGethSpecs.cs +++ /dev/null @@ -1,207 +0,0 @@ -using CodexDistTestCore.Config; -using k8s.Models; - -namespace CodexDistTestCore.Marketplace -{ - public static class GethDockerImage - { - public const string Image = "thatbenbierens/geth-confenv:latest"; - - } - - public class K8sGethBoostrapSpecs - { - public const string ContainerName = "dtest-gethb"; - private const string portName = "gethb"; - - public K8sGethBoostrapSpecs(int servicePort) - { - ServicePort = servicePort; - } - - public int ServicePort { get; } - - public string GetBootstrapDeploymentName() - { - return "test-gethb"; - } - - public string GetCompanionDeploymentName(GethCompanionGroup group) - { - return "test-geth" + group.Number; - } - - public V1Deployment CreateGethBootstrapDeployment() - { - var deploymentSpec = new V1Deployment - { - ApiVersion = "apps/v1", - Metadata = new V1ObjectMeta - { - Name = GetBootstrapDeploymentName(), - NamespaceProperty = K8sCluster.K8sNamespace - }, - Spec = new V1DeploymentSpec - { - Replicas = 1, - Selector = new V1LabelSelector - { - MatchLabels = CreateBootstrapSelector() - }, - Template = new V1PodTemplateSpec - { - Metadata = new V1ObjectMeta - { - Labels = CreateBootstrapSelector() - }, - Spec = new V1PodSpec - { - Containers = new List - { - new V1Container - { - Name = ContainerName, - Image = GethDockerImage.Image, - Ports = new List - { - new V1ContainerPort - { - ContainerPort = 8545, - Name = portName - } - }, - Env = new List - { - new V1EnvVar - { - Name = "GETH_ARGS", - Value = "" - }, - new V1EnvVar - { - Name = "GENESIS_JSON", - Value = genesisJsonBase64 - }, - new V1EnvVar - { - Name = "IS_BOOTSTRAP", - Value = "1" - } - } - } - } - } - } - } - }; - - return deploymentSpec; - } - - public V1Service CreateGethBootstrapService() - { - var serviceSpec = new V1Service - { - ApiVersion = "v1", - Metadata = new V1ObjectMeta - { - Name = "codex-gethb-service", - NamespaceProperty = K8sCluster.K8sNamespace - }, - Spec = new V1ServiceSpec - { - Type = "NodePort", - Selector = CreateBootstrapSelector(), - Ports = new List - { - new V1ServicePort - { - Name = "gethb-service", - Protocol = "TCP", - Port = 8545, - TargetPort = portName, - NodePort = ServicePort - } - } - } - }; - - return serviceSpec; - } - - public V1Deployment CreateGethCompanionDeployment(GethCompanionGroup group, GethBootstrapInfo info) - { - var deploymentSpec = new V1Deployment - { - ApiVersion = "apps/v1", - Metadata = new V1ObjectMeta - { - Name = GetCompanionDeploymentName(group), - NamespaceProperty = K8sCluster.K8sNamespace - }, - Spec = new V1DeploymentSpec - { - Replicas = 1, - Selector = new V1LabelSelector - { - MatchLabels = CreateCompanionSelector() - }, - Template = new V1PodTemplateSpec - { - Metadata = new V1ObjectMeta - { - Labels = CreateCompanionSelector() - }, - Spec = new V1PodSpec - { - Containers = group.Containers.Select(c => CreateContainer(c, info)).ToList() - } - } - } - }; - - return deploymentSpec; - } - - private static V1Container CreateContainer(GethCompanionNodeContainer container, GethBootstrapInfo info) - { - return new V1Container - { - Name = container.Name, - Image = GethDockerImage.Image, - Ports = new List - { - new V1ContainerPort - { - ContainerPort = container.ApiPort, - Name = container.ContainerPortName - } - }, - // todo: use env vars to connect this node to the bootstrap node provided by gethInfo.podInfo & gethInfo.servicePort & gethInfo.genesisJsonBase64 - Env = new List - { - new V1EnvVar - { - Name = "GETH_ARGS", - Value = $"--port {container.ApiPort} --discovery.port {container.ApiPort} --authrpc.port {container.AuthRpcPort} --http.port {container.RpcPort}" - }, - new V1EnvVar - { - Name = "GENESIS_JSON", - Value = info.GenesisJsonBase64 - } - } - }; - } - - private Dictionary CreateBootstrapSelector() - { - return new Dictionary { { "test-gethb", "dtest-gethb" } }; - } - - private Dictionary CreateCompanionSelector() - { - return new Dictionary { { "test-gethc", "dtest-gethc" } }; - } - } -} diff --git a/CodexDistTestCore/Marketplace/MarketplaceAccess.cs b/CodexDistTestCore/Marketplace/MarketplaceAccess.cs deleted file mode 100644 index e9bf39e..0000000 --- a/CodexDistTestCore/Marketplace/MarketplaceAccess.cs +++ /dev/null @@ -1,111 +0,0 @@ -using NUnit.Framework; -using NUnit.Framework.Constraints; - -namespace CodexDistTestCore.Marketplace -{ - public interface IMarketplaceAccess - { - void MakeStorageAvailable(ByteSize size, int minPricePerBytePerSecond, float maxCollateral); - void RequestStorage(ContentId contentId, int pricePerBytePerSecond, float requiredCollateral, float minRequiredNumberOfNodes); - void AssertThatBalance(IResolveConstraint constraint, string message = ""); - decimal GetBalance(); - } - - public class MarketplaceAccess : IMarketplaceAccess - { - private readonly K8sManager k8sManager; - private readonly MarketplaceController marketplaceController; - private readonly TestLog log; - private readonly CodexNodeGroup group; - private readonly GethCompanionNodeContainer container; - - public MarketplaceAccess( - K8sManager k8sManager, - MarketplaceController marketplaceController, - TestLog log, - CodexNodeGroup group, - GethCompanionNodeContainer container) - { - this.k8sManager = k8sManager; - this.marketplaceController = marketplaceController; - this.log = log; - this.group = group; - this.container = container; - } - - public void Initialize() - { - EnsureAccount(); - - marketplaceController.AddToBalance(container.Account, group.Origin.MarketplaceConfig!.InitialBalance); - - log.Log($"Initialized Geth companion node with account '{container.Account}' and initial balance {group.Origin.MarketplaceConfig!.InitialBalance}"); - } - - public void RequestStorage(ContentId contentId, int pricePerBytePerSecond, float requiredCollateral, float minRequiredNumberOfNodes) - { - throw new NotImplementedException(); - } - - public void MakeStorageAvailable(ByteSize size, int minPricePerBytePerSecond, float maxCollateral) - { - throw new NotImplementedException(); - } - - public void AssertThatBalance(IResolveConstraint constraint, string message = "") - { - throw new NotImplementedException(); - } - - public decimal GetBalance() - { - return marketplaceController.GetBalance(container.Account); - } - - private void EnsureAccount() - { - FetchAccount(); - if (string.IsNullOrEmpty(container.Account)) - { - Thread.Sleep(TimeSpan.FromSeconds(15)); - FetchAccount(); - } - Assert.That(container.Account, Is.Not.Empty, "Unable to fetch account for geth companion node. Test infra failure."); - } - - private void FetchAccount() - { - container.Account = k8sManager.ExecuteCommand(group.GethCompanionGroup!.Pod!, container.Name, "cat", GethDockerImage.AccountFilename); - } - } - - public class MarketplaceUnavailable : IMarketplaceAccess - { - public void RequestStorage(ContentId contentId, int pricePerBytePerSecond, float requiredCollateral, float minRequiredNumberOfNodes) - { - Unavailable(); - } - - public void MakeStorageAvailable(ByteSize size, int minPricePerBytePerSecond, float maxCollateral) - { - Unavailable(); - } - - public void AssertThatBalance(IResolveConstraint constraint, string message = "") - { - Unavailable(); - } - - public decimal GetBalance() - { - Unavailable(); - return 0; - } - - private void Unavailable() - { - Assert.Fail("Incorrect test setup: Marketplace was not enabled for this group of Codex nodes. Add 'EnableMarketplace(...)' after 'SetupCodexNodes()' to enable it."); - throw new InvalidOperationException(); - } - } -} diff --git a/CodexDistTestCore/Marketplace/MarketplaceController.cs b/CodexDistTestCore/Marketplace/MarketplaceController.cs deleted file mode 100644 index 8bab9e9..0000000 --- a/CodexDistTestCore/Marketplace/MarketplaceController.cs +++ /dev/null @@ -1,24 +0,0 @@ -using CodexDistTestCore.Config; -using NUnit.Framework; -using System.Numerics; -using System.Text; - -namespace CodexDistTestCore.Marketplace -{ - public class MarketplaceController - { - private readonly TestLog log; - private readonly K8sManager k8sManager; - private readonly NumberSource companionGroupNumberSource = new NumberSource(0); - private List companionGroups = new List(); - private GethBootstrapInfo? bootstrapInfo; - - public MarketplaceController(TestLog log, K8sManager k8sManager) - { - this.log = log; - this.k8sManager = k8sManager; - } - - - } -} diff --git a/CodexDistTestCore/Marketplace/MarketplaceInitialConfig.cs b/CodexDistTestCore/Marketplace/MarketplaceInitialConfig.cs deleted file mode 100644 index 23ad25f..0000000 --- a/CodexDistTestCore/Marketplace/MarketplaceInitialConfig.cs +++ /dev/null @@ -1,4 +0,0 @@ -namespace CodexDistTestCore.Marketplace -{ - -} diff --git a/CodexDistTestCore/Metrics/K8sPrometheusSpecs.cs b/CodexDistTestCore/Metrics/K8sPrometheusSpecs.cs deleted file mode 100644 index 0fc0865..0000000 --- a/CodexDistTestCore/Metrics/K8sPrometheusSpecs.cs +++ /dev/null @@ -1,122 +0,0 @@ -using CodexDistTestCore.Config; -using k8s.Models; - -namespace CodexDistTestCore.Metrics -{ - public class K8sPrometheusSpecs - { - public const string ContainerName = "dtest-prom"; - public const string ConfigFilepath = "/etc/prometheus/prometheus.yml"; - private const string dockerImage = "thatbenbierens/prometheus-envconf:latest"; - private const string portName = "prom-1"; - private readonly string config; - - public K8sPrometheusSpecs(int servicePort, int prometheusNumber, string config) - { - ServicePort = servicePort; - PrometheusNumber = prometheusNumber; - this.config = config; - } - - public int ServicePort { get; } - public int PrometheusNumber { get; } - - public string GetDeploymentName() - { - return "test-prom" + PrometheusNumber; - } - - public V1Deployment CreatePrometheusDeployment() - { - var deploymentSpec = new V1Deployment - { - ApiVersion = "apps/v1", - Metadata = new V1ObjectMeta - { - Name = GetDeploymentName(), - NamespaceProperty = K8sCluster.K8sNamespace - }, - Spec = new V1DeploymentSpec - { - Replicas = 1, - Selector = new V1LabelSelector - { - MatchLabels = CreateSelector() - }, - Template = new V1PodTemplateSpec - { - Metadata = new V1ObjectMeta - { - Labels = CreateSelector() - }, - Spec = new V1PodSpec - { - Containers = new List - { - new V1Container - { - Name = ContainerName, - Image = dockerImage, - Ports = new List - { - new V1ContainerPort - { - ContainerPort = 9090, - Name = portName - } - }, - Env = new List - { - new V1EnvVar - { - Name = "PROM_CONFIG", - Value = config - } - } - } - } - } - } - } - }; - - return deploymentSpec; - } - - public V1Service CreatePrometheusService() - { - var serviceSpec = new V1Service - { - ApiVersion = "v1", - Metadata = new V1ObjectMeta - { - Name = "codex-prom-service" + PrometheusNumber, - NamespaceProperty = K8sCluster.K8sNamespace - }, - Spec = new V1ServiceSpec - { - Type = "NodePort", - Selector = CreateSelector(), - Ports = new List - { - new V1ServicePort - { - Name = "prom-service" + PrometheusNumber, - Protocol = "TCP", - Port = 9090, - TargetPort = portName, - NodePort = ServicePort - } - } - } - }; - - return serviceSpec; - } - - private Dictionary CreateSelector() - { - return new Dictionary { { "test-prom", "dtest-prom" } }; - } - } -} diff --git a/CodexDistTestCore/Metrics/MetricsAccess.cs b/CodexDistTestCore/Metrics/MetricsAccess.cs deleted file mode 100644 index 2f6456e..0000000 --- a/CodexDistTestCore/Metrics/MetricsAccess.cs +++ /dev/null @@ -1,63 +0,0 @@ -using NUnit.Framework; -using NUnit.Framework.Constraints; - -namespace CodexDistTestCore.Metrics -{ - public interface IMetricsAccess - { - void AssertThat(string metricName, IResolveConstraint constraint, string message = ""); - } - - public class MetricsUnavailable : IMetricsAccess - { - public void AssertThat(string metricName, IResolveConstraint constraint, string message = "") - { - Assert.Fail("Incorrect test setup: Metrics were not enabled for this group of Codex nodes. Add 'EnableMetrics()' after 'SetupCodexNodes()' to enable it."); - throw new InvalidOperationException(); - } - } - - public class MetricsAccess : IMetricsAccess - { - private readonly MetricsQuery query; - private readonly OnlineCodexNode node; - - public MetricsAccess(MetricsQuery query, OnlineCodexNode node) - { - this.query = query; - this.node = node; - } - - public void AssertThat(string metricName, IResolveConstraint constraint, string message = "") - { - var metricSet = GetMetricWithTimeout(metricName, node); - var metricValue = metricSet.Values[0].Value; - Assert.That(metricValue, constraint, message); - } - - private MetricsSet GetMetricWithTimeout(string metricName, OnlineCodexNode node) - { - var start = DateTime.UtcNow; - - while (true) - { - var mostRecent = GetMostRecent(metricName, node); - if (mostRecent != null) return mostRecent; - if (DateTime.UtcNow - start > Timing.WaitForMetricTimeout()) - { - Assert.Fail($"Timeout: Unable to get metric '{metricName}'."); - throw new TimeoutException(); - } - - Utils.Sleep(TimeSpan.FromSeconds(2)); - } - } - - private MetricsSet? GetMostRecent(string metricName, OnlineCodexNode node) - { - var result = query.GetMostRecent(metricName, node); - if (result == null) return null; - return result.Sets.LastOrDefault(); - } - } -} diff --git a/CodexDistTestCore/Metrics/MetricsAggregator.cs b/CodexDistTestCore/Metrics/MetricsAggregator.cs deleted file mode 100644 index adf0c3f..0000000 --- a/CodexDistTestCore/Metrics/MetricsAggregator.cs +++ /dev/null @@ -1,78 +0,0 @@ -using NUnit.Framework; -using System.Text; - -namespace CodexDistTestCore.Metrics -{ - public class MetricsAggregator - { - private readonly NumberSource prometheusNumberSource = new NumberSource(0); - private readonly TestLog log; - private readonly K8sManager k8sManager; - private readonly Dictionary activePrometheuses = new Dictionary(); - - public MetricsAggregator(TestLog log, K8sManager k8sManager) - { - this.log = log; - this.k8sManager = k8sManager; - } - - public void BeginCollectingMetricsFor(OnlineCodexNode[] nodes) - { - log.Log($"Starting metrics collecting for {nodes.Length} nodes..."); - - var config = GeneratePrometheusConfig(nodes); - var prometheus = k8sManager.BringOnlinePrometheus(config, prometheusNumberSource.GetNextNumber()); - var query = new MetricsQuery(prometheus); - activePrometheuses.Add(query, nodes); - - log.Log("Metrics service started."); - - foreach (var node in nodes) - { - node.Metrics = new MetricsAccess(query, node); - } - } - - public void DownloadAllMetrics() - { - var download = new MetricsDownloader(log, activePrometheuses); - download.DownloadAllMetrics(); - } - - private string GeneratePrometheusConfig(OnlineCodexNode[] nodes) - { - var config = ""; - config += "global:\n"; - config += " scrape_interval: 30s\n"; - config += " scrape_timeout: 10s\n"; - config += "\n"; - config += "scrape_configs:\n"; - config += " - job_name: services\n"; - config += " metrics_path: /metrics\n"; - config += " static_configs:\n"; - config += " - targets:\n"; - - foreach (var node in nodes) - { - var ip = node.Group.PodInfo!.Ip; - var port = node.Container.MetricsPort; - config += $" - '{ip}:{port}'\n"; - } - - var bytes = Encoding.ASCII.GetBytes(config); - return Convert.ToBase64String(bytes); - } - } - - public class PrometheusInfo - { - public PrometheusInfo(int servicePort, PodInfo podInfo) - { - ServicePort = servicePort; - PodInfo = podInfo; - } - - public int ServicePort { get; } - public PodInfo PodInfo { get; } - } -} diff --git a/CodexDistTestCore/Metrics/MetricsDownloader.cs b/CodexDistTestCore/Metrics/MetricsDownloader.cs deleted file mode 100644 index 18fd10b..0000000 --- a/CodexDistTestCore/Metrics/MetricsDownloader.cs +++ /dev/null @@ -1,97 +0,0 @@ -using System.Globalization; - -namespace CodexDistTestCore.Metrics -{ - public class MetricsDownloader - { - private readonly TestLog log; - private readonly Dictionary activePrometheuses; - - public MetricsDownloader(TestLog log, Dictionary activePrometheuses) - { - this.log = log; - this.activePrometheuses = activePrometheuses; - } - - public void DownloadAllMetrics() - { - foreach (var pair in activePrometheuses) - { - DownloadAllMetrics(pair.Key, pair.Value); - } - } - - private void DownloadAllMetrics(MetricsQuery query, OnlineCodexNode[] nodes) - { - foreach (var node in nodes) - { - DownloadAllMetricsForNode(query, node); - } - } - - private void DownloadAllMetricsForNode(MetricsQuery query, OnlineCodexNode node) - { - var metrics = query.GetAllMetricsForNode(node); - if (metrics == null || metrics.Sets.Length == 0 || metrics.Sets.All(s => s.Values.Length == 0)) return; - - var headers = new[] { "timestamp" }.Concat(metrics.Sets.Select(s => s.Name)).ToArray(); - var map = CreateValueMap(metrics); - - WriteToFile(node.GetName(), headers, map); - } - - private void WriteToFile(string nodeName, string[] headers, Dictionary> map) - { - var file = log.CreateSubfile("csv"); - log.Log($"Downloading metrics for {nodeName} to file {file.FilenameWithoutPath}"); - - file.WriteRaw(string.Join(",", headers)); - - foreach (var pair in map) - { - file.WriteRaw(string.Join(",", new[] { FormatTimestamp(pair.Key) }.Concat(pair.Value))); - } - } - - private Dictionary> CreateValueMap(Metrics metrics) - { - var map = CreateForAllTimestamps(metrics); - foreach (var metric in metrics.Sets) - { - AddToMap(map, metric); - } - return map; - - } - - private Dictionary> CreateForAllTimestamps(Metrics metrics) - { - var result = new Dictionary>(); - var timestamps = metrics.Sets.SelectMany(s => s.Values).Select(v => v.Timestamp).Distinct().ToArray(); - foreach (var timestamp in timestamps) result.Add(timestamp, new List()); - return result; - } - - private void AddToMap(Dictionary> map, MetricsSet metric) - { - foreach (var key in map.Keys) - { - map[key].Add(GetValueAtTimestamp(key, metric)); - } - } - - private string GetValueAtTimestamp(DateTime key, MetricsSet metric) - { - var value = metric.Values.SingleOrDefault(v => v.Timestamp == key); - if (value == null) return ""; - return value.Value.ToString(CultureInfo.InvariantCulture); - } - - private string FormatTimestamp(DateTime key) - { - var origin = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc); - var diff = key - origin; - return Math.Floor(diff.TotalSeconds).ToString(CultureInfo.InvariantCulture); - } - } -} diff --git a/CodexDistTestCore/Metrics/MetricsQuery.cs b/CodexDistTestCore/Metrics/MetricsQuery.cs deleted file mode 100644 index c028a6c..0000000 --- a/CodexDistTestCore/Metrics/MetricsQuery.cs +++ /dev/null @@ -1,190 +0,0 @@ -using CodexDistTestCore.Config; -using System.Globalization; - -namespace CodexDistTestCore.Metrics -{ - public class MetricsQuery - { - private readonly K8sCluster k8sCluster = new K8sCluster(); - private readonly Http http; - - public MetricsQuery(PrometheusInfo prometheusInfo) - { - http = new Http( - k8sCluster.GetIp(), - prometheusInfo.ServicePort, - "api/v1"); - } - - public Metrics? GetMostRecent(string metricName, OnlineCodexNode node) - { - var response = GetLastOverTime(metricName, GetInstanceStringForNode(node)); - if (response == null) return null; - - return new Metrics - { - Sets = response.data.result.Select(r => - { - return new MetricsSet - { - Instance = r.metric.instance, - Values = MapSingleValue(r.value) - }; - }).ToArray() - }; - } - - public Metrics? GetMetrics(string metricName) - { - var response = GetAll(metricName); - if (response == null) return null; - return MapResponseToMetrics(response); - } - - public Metrics? GetAllMetricsForNode(OnlineCodexNode node) - { - var response = http.HttpGetJson($"query?query={GetInstanceStringForNode(node)}{GetQueryTimeRange()}"); - if (response.status != "success") return null; - return MapResponseToMetrics(response); - } - - private PrometheusQueryResponse? GetLastOverTime(string metricName, string instanceString) - { - var response = http.HttpGetJson($"query?query=last_over_time({metricName}{instanceString}{GetQueryTimeRange()})"); - if (response.status != "success") return null; - return response; - } - - private PrometheusQueryResponse? GetAll(string metricName) - { - var response = http.HttpGetJson($"query?query={metricName}{GetQueryTimeRange()}"); - if (response.status != "success") return null; - return response; - } - - private Metrics MapResponseToMetrics(PrometheusQueryResponse response) - { - return new Metrics - { - Sets = response.data.result.Select(r => - { - return new MetricsSet - { - Name = r.metric.__name__, - Instance = r.metric.instance, - Values = MapMultipleValues(r.values) - }; - }).ToArray() - }; - } - - private MetricsSetValue[] MapSingleValue(object[] value) - { - if (value != null && value.Length > 0) - { - return new[] - { - MapValue(value) - }; - } - return Array.Empty(); - } - - private MetricsSetValue[] MapMultipleValues(object[][] values) - { - if (values != null && values.Length > 0) - { - return values.Select(v => MapValue(v)).ToArray(); - } - return Array.Empty(); - } - - private MetricsSetValue MapValue(object[] value) - { - if (value.Length != 2) throw new InvalidOperationException("Expected value to be [double, string]."); - - return new MetricsSetValue - { - Timestamp = ToTimestamp(value[0]), - Value = ToValue(value[1]) - }; - } - - private string GetInstanceNameForNode(OnlineCodexNode node) - { - var pod = node.Group.PodInfo!; - return $"{pod.Ip}:{node.Container.MetricsPort}"; - } - - private string GetInstanceStringForNode(OnlineCodexNode node) - { - return "{instance=\"" + GetInstanceNameForNode(node) + "\"}"; - } - - private string GetQueryTimeRange() - { - return "[12h]"; - } - - private double ToValue(object v) - { - return Convert.ToDouble(v, CultureInfo.InvariantCulture); - } - - private DateTime ToTimestamp(object v) - { - var unixSeconds = ToValue(v); - return new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc).AddSeconds(unixSeconds); - } - } - - public class Metrics - { - public MetricsSet[] Sets { get; set; } = Array.Empty(); - } - - public class MetricsSet - { - public string Name { get; set; } = string.Empty; - public string Instance { get; set; } = string.Empty; - public MetricsSetValue[] Values { get; set; } = Array.Empty(); - } - - public class MetricsSetValue - { - public DateTime Timestamp { get; set; } - public double Value { get; set; } - } - - public class PrometheusQueryResponse - { - public string status { get; set; } = string.Empty; - public PrometheusQueryResponseData data { get; set; } = new(); - } - - public class PrometheusQueryResponseData - { - public string resultType { get; set; } = string.Empty; - public PrometheusQueryResponseDataResultEntry[] result { get; set; } = Array.Empty(); - } - - public class PrometheusQueryResponseDataResultEntry - { - public ResultEntryMetric metric { get; set; } = new(); - public object[] value { get; set; } = Array.Empty(); - public object[][] values { get; set; } = Array.Empty(); - } - - public class ResultEntryMetric - { - public string __name__ { get; set; } = string.Empty; - public string instance { get; set; } = string.Empty; - public string job { get; set; } = string.Empty; - } - - public class PrometheusAllNamesResponse - { - public string status { get; set; } = string.Empty; - public string[] data { get; set; } = Array.Empty(); - } -} diff --git a/CodexDistTestCore/NumberSource.cs b/CodexDistTestCore/NumberSource.cs deleted file mode 100644 index 4338610..0000000 --- a/CodexDistTestCore/NumberSource.cs +++ /dev/null @@ -1,19 +0,0 @@ -namespace CodexDistTestCore -{ - public class NumberSource - { - private int number; - - public NumberSource(int start) - { - number = start; - } - - public int GetNextNumber() - { - var n = number; - number++; - return n; - } - } -} diff --git a/CodexDistTestCore/OfflineCodexNodes.cs b/CodexDistTestCore/OfflineCodexNodes.cs deleted file mode 100644 index 84003a6..0000000 --- a/CodexDistTestCore/OfflineCodexNodes.cs +++ /dev/null @@ -1,97 +0,0 @@ -using CodexDistTestCore.Marketplace; - -namespace CodexDistTestCore -{ - public interface IOfflineCodexNodes - { - IOfflineCodexNodes At(Location location); - IOfflineCodexNodes WithLogLevel(CodexLogLevel level); - IOfflineCodexNodes WithBootstrapNode(IOnlineCodexNode node); - IOfflineCodexNodes WithStorageQuota(ByteSize storageQuota); - IOfflineCodexNodes EnableMetrics(); - IOfflineCodexNodes EnableMarketplace(int initialBalance); - ICodexNodeGroup BringOnline(); - } - - public enum Location - { - Unspecified, - BensLaptop, - BensOldGamingMachine, - } - - public class OfflineCodexNodes : IOfflineCodexNodes - { - private readonly IK8sManager k8SManager; - - public int NumberOfNodes { get; } - public Location Location { get; private set; } - public CodexLogLevel? LogLevel { get; private set; } - public IOnlineCodexNode? BootstrapNode { get; private set; } - public ByteSize? StorageQuota { get; private set; } - public bool MetricsEnabled { get; private set; } - public MarketplaceInitialConfig? MarketplaceConfig { get; private set; } - - public OfflineCodexNodes(IK8sManager k8SManager, int numberOfNodes) - { - this.k8SManager = k8SManager; - NumberOfNodes = numberOfNodes; - Location = Location.Unspecified; - MetricsEnabled = false; - } - - public ICodexNodeGroup BringOnline() - { - return k8SManager.BringOnline(this); - } - - public IOfflineCodexNodes At(Location location) - { - Location = location; - return this; - } - - public IOfflineCodexNodes WithBootstrapNode(IOnlineCodexNode node) - { - BootstrapNode = node; - return this; - } - - public IOfflineCodexNodes WithLogLevel(CodexLogLevel level) - { - LogLevel = level; - return this; - } - - public IOfflineCodexNodes WithStorageQuota(ByteSize storageQuota) - { - StorageQuota = storageQuota; - return this; - } - - public IOfflineCodexNodes EnableMetrics() - { - MetricsEnabled = true; - return this; - } - - public IOfflineCodexNodes EnableMarketplace(int initialBalance) - { - MarketplaceConfig = new MarketplaceInitialConfig(initialBalance); - return this; - } - - public string Describe() - { - var args = string.Join(',', DescribeArgs()); - return $"{NumberOfNodes} CodexNodes with [{args}]"; - } - - private IEnumerable DescribeArgs() - { - if (LogLevel != null) yield return ($"LogLevel={LogLevel}"); - if (BootstrapNode != null) yield return ("BootstrapNode=set-not-shown-here"); - if (StorageQuota != null) yield return ($"StorageQuote={StorageQuota.SizeInBytes}"); - } - } -} diff --git a/CodexDistTestCore/OnlineCodexNode.cs b/CodexDistTestCore/OnlineCodexNode.cs deleted file mode 100644 index 64f9c6d..0000000 --- a/CodexDistTestCore/OnlineCodexNode.cs +++ /dev/null @@ -1,141 +0,0 @@ -using CodexDistTestCore.Config; -using CodexDistTestCore.Marketplace; -using CodexDistTestCore.Metrics; -using NUnit.Framework; - -namespace CodexDistTestCore -{ - public interface IOnlineCodexNode - { - CodexDebugResponse GetDebugInfo(); - ContentId UploadFile(TestFile file); - TestFile? DownloadContent(ContentId contentId); - void ConnectToPeer(IOnlineCodexNode node); - ICodexNodeLog DownloadLog(); - IMetricsAccess Metrics { get; } - IMarketplaceAccess Marketplace { get; } - } - - public class OnlineCodexNode : IOnlineCodexNode - { - private const string SuccessfullyConnectedMessage = "Successfully connected to peer"; - private const string UploadFailedMessage = "Unable to store block"; - - private readonly K8sCluster k8sCluster = new K8sCluster(); - private readonly TestLog log; - private readonly IFileManager fileManager; - - public OnlineCodexNode(TestLog log, IFileManager fileManager, CodexNodeContainer container) - { - this.log = log; - this.fileManager = fileManager; - Container = container; - } - - public CodexNodeContainer Container { get; } - public CodexNodeGroup Group { get; internal set; } = null!; - public IMetricsAccess Metrics { get; set; } = new MetricsUnavailable(); - public IMarketplaceAccess Marketplace { set; get; } = new MarketplaceUnavailable(); - - public string GetName() - { - return $"<{Container.Name}>"; - } - - public CodexDebugResponse GetDebugInfo() - { - var response = Http().HttpGetJson("debug/info"); - Log($"Got DebugInfo with id: '{response.id}'."); - return response; - } - - public ContentId UploadFile(TestFile file) - { - Log($"Uploading file of size {file.GetFileSize()}..."); - using var fileStream = File.OpenRead(file.Filename); - var response = Http().HttpPostStream("upload", fileStream); - if (response.StartsWith(UploadFailedMessage)) - { - Assert.Fail("Node failed to store block."); - } - Log($"Uploaded file. Received contentId: '{response}'."); - return new ContentId(response); - } - - public TestFile? DownloadContent(ContentId contentId) - { - Log($"Downloading for contentId: '{contentId.Id}'..."); - var file = fileManager.CreateEmptyTestFile(); - DownloadToFile(contentId.Id, file); - Log($"Downloaded file of size {file.GetFileSize()} to '{file.Filename}'."); - return file; - } - - public void ConnectToPeer(IOnlineCodexNode node) - { - var peer = (OnlineCodexNode)node; - - Log($"Connecting to peer {peer.GetName()}..."); - var peerInfo = node.GetDebugInfo(); - var peerId = peerInfo.id; - var peerMultiAddress = GetPeerMultiAddress(peer, peerInfo); - - var response = Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}"); - - Assert.That(response, Is.EqualTo(SuccessfullyConnectedMessage), "Unable to connect codex nodes."); - Log($"Successfully connected to peer {peer.GetName()}."); - } - - public ICodexNodeLog DownloadLog() - { - return Group.DownloadLog(this); - } - - public string Describe() - { - return $"{Group.Describe()} contains {GetName()}"; - } - - private string GetPeerMultiAddress(OnlineCodexNode peer, CodexDebugResponse peerInfo) - { - var multiAddress = peerInfo.addrs.First(); - // Todo: Is there a case where First address in list is not the way? - - if (Group == peer.Group) - { - return multiAddress; - } - - // The peer we want to connect is in a different pod. - // We must replace the default IP with the pod IP in the multiAddress. - return multiAddress.Replace("0.0.0.0", peer.Group.PodInfo!.Ip); - } - - private void DownloadToFile(string contentId, TestFile file) - { - using var fileStream = File.OpenWrite(file.Filename); - using var downloadStream = Http().HttpGetStream("download/" + contentId); - downloadStream.CopyTo(fileStream); - } - - private Http Http() - { - return new Http(ip: k8sCluster.GetIp(), port: Container.ServicePort, baseUrl: "/api/codex/v1"); - } - - private void Log(string msg) - { - log.Log($"{GetName()}: {msg}"); - } - } - - public class ContentId - { - public ContentId(string id) - { - Id = id; - } - - public string Id { get; } - } -} diff --git a/CodexDistTestCore/PodLogDownloader.cs b/CodexDistTestCore/PodLogDownloader.cs deleted file mode 100644 index e09a57c..0000000 --- a/CodexDistTestCore/PodLogDownloader.cs +++ /dev/null @@ -1,64 +0,0 @@ -using NUnit.Framework; - -namespace CodexDistTestCore -{ - public interface IPodLogHandler - { - void Log(Stream log); - } - - public class PodLogDownloader - { - public const string DontDownloadLogsOnFailureKey = "DontDownloadLogsOnFailure"; - - private readonly TestLog log; - private readonly IK8sManager k8SManager; - - public PodLogDownloader(TestLog log, IK8sManager k8sManager) - { - this.log = log; - k8SManager = k8sManager; - } - - public CodexNodeLog DownloadLog(OnlineCodexNode node) - { - var description = node.Describe(); - var subFile = log.CreateSubfile(); - - log.Log($"Downloading logs for {description} to file {subFile.FilenameWithoutPath}"); - var handler = new PodLogDownloadHandler(description, subFile); - k8SManager.FetchPodLog(node, handler); - return handler.CreateCodexNodeLog(); - } - } - - public class PodLogDownloadHandler : IPodLogHandler - { - private readonly string description; - private readonly LogFile log; - - public PodLogDownloadHandler(string description, LogFile log) - { - this.description = description; - this.log = log; - } - - public CodexNodeLog CreateCodexNodeLog() - { - return new CodexNodeLog(log); - } - - public void Log(Stream stream) - { - log.Write($"{description} -->> {log.FilenameWithoutPath}"); - log.WriteRaw(description); - var reader = new StreamReader(stream); - var line = reader.ReadLine(); - while (line != null) - { - log.WriteRaw(line); - line = reader.ReadLine(); - } - } - } -} diff --git a/CodexDistTestCore/Timing.cs b/CodexDistTestCore/Timing.cs deleted file mode 100644 index cfa9456..0000000 --- a/CodexDistTestCore/Timing.cs +++ /dev/null @@ -1,131 +0,0 @@ -using NUnit.Framework; - -namespace CodexDistTestCore -{ - [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] - public class UseLongTimeoutsAttribute : PropertyAttribute - { - public UseLongTimeoutsAttribute() - : base(Timing.UseLongTimeoutsKey) - { - } - } - - public static class Timing - { - public const string UseLongTimeoutsKey = "UseLongTimeouts"; - - public static TimeSpan HttpCallTimeout() - { - return GetTimes().HttpCallTimeout(); - } - - public static int HttpCallRetryCount() - { - return GetTimes().HttpCallRetryCount(); - } - - public static void HttpCallRetryDelay() - { - Utils.Sleep(GetTimes().HttpCallRetryDelay()); - } - - public static void WaitForK8sServiceDelay() - { - Utils.Sleep(GetTimes().WaitForK8sServiceDelay()); - } - - public static TimeSpan K8sOperationTimeout() - { - return GetTimes().K8sOperationTimeout(); - } - - public static TimeSpan WaitForMetricTimeout() - { - return GetTimes().WaitForMetricTimeout(); - } - - private static ITimeSet GetTimes() - { - var testProperties = TestContext.CurrentContext.Test.Properties; - if (testProperties.ContainsKey(UseLongTimeoutsKey)) return new LongTimeSet(); - return new DefaultTimeSet(); - } - } - - public interface ITimeSet - { - TimeSpan HttpCallTimeout(); - int HttpCallRetryCount(); - TimeSpan HttpCallRetryDelay(); - TimeSpan WaitForK8sServiceDelay(); - TimeSpan K8sOperationTimeout(); - TimeSpan WaitForMetricTimeout(); - } - - public class DefaultTimeSet : ITimeSet - { - public TimeSpan HttpCallTimeout() - { - return TimeSpan.FromSeconds(10); - } - - public int HttpCallRetryCount() - { - return 5; - } - - public TimeSpan HttpCallRetryDelay() - { - return TimeSpan.FromSeconds(3); - } - - public TimeSpan WaitForK8sServiceDelay() - { - return TimeSpan.FromSeconds(1); - } - - public TimeSpan K8sOperationTimeout() - { - return TimeSpan.FromMinutes(5); - } - - public TimeSpan WaitForMetricTimeout() - { - return TimeSpan.FromSeconds(30); - } - } - - public class LongTimeSet : ITimeSet - { - public TimeSpan HttpCallTimeout() - { - return TimeSpan.FromHours(2); - } - - public int HttpCallRetryCount() - { - return 2; - } - - public TimeSpan HttpCallRetryDelay() - { - return TimeSpan.FromMinutes(5); - } - - public TimeSpan WaitForK8sServiceDelay() - { - return TimeSpan.FromSeconds(10); - } - - public TimeSpan K8sOperationTimeout() - { - return TimeSpan.FromMinutes(15); - } - - public TimeSpan WaitForMetricTimeout() - { - return TimeSpan.FromMinutes(5); - } - } -} diff --git a/CodexDistTestCore/TryContract.cs b/CodexDistTestCore/TryContract.cs deleted file mode 100644 index efc9d5a..0000000 --- a/CodexDistTestCore/TryContract.cs +++ /dev/null @@ -1,101 +0,0 @@ -using Nethereum.Web3; -using Nethereum.ABI.FunctionEncoding.Attributes; -using Nethereum.Contracts.CQS; -using Nethereum.Util; -using Nethereum.Web3.Accounts; -using Nethereum.Hex.HexConvertors.Extensions; -using Nethereum.Contracts; -using Nethereum.Contracts.Extensions; -using System.Numerics; -using NUnit.Framework; - -// https://docs.nethereum.com/en/latest/nethereum-smartcontrats-gettingstarted/ - -namespace CodexDistTestCore -{ - public class TryContract - { - [Test] - [Ignore("aaa")] - public void DoThing() - { - var url = "http://testchain.nethereum.com:8545"; - var privateKey = "0x7580e7fb49df1c861f0050fae31c2224c6aba908e116b8da44ee8cd927b990b0"; - var account = new Account(privateKey); - var web3 = new Web3(account, url); - - // Deploy contract: - var deploymentMessage = new StandardTokenDeployment - { - TotalSupply = 100000 - }; - var deploymentHandler = web3.Eth.GetContractDeploymentHandler(); - var transactionReceipt = Utils.Wait(deploymentHandler.SendRequestAndWaitForReceiptAsync(deploymentMessage)); - var contractAddress = transactionReceipt.ContractAddress; - - // Get balance: - var balanceOfFunctionMessage = new BalanceOfFunction() - { - Owner = account.Address, - }; - - var balanceHandler = web3.Eth.GetContractQueryHandler(); - var balance = Utils.Wait(balanceHandler.QueryAsync(contractAddress, balanceOfFunctionMessage)); - long asInt = ((long)balance); - - // Transfer: - var receiverAddress = "0xde0B295669a9FD93d5F28D9Ec85E40f4cb697BAe"; - var transferHandler = web3.Eth.GetContractTransactionHandler(); - var transfer = new TransferFunction() - { - To = receiverAddress, - TokenAmount = 100 - }; - var transferReceipt = Utils.Wait(transferHandler.SendRequestAndWaitForReceiptAsync(contractAddress, transfer)); - - // Signing: - var signedTransaction = Utils.Wait(transferHandler.SignTransactionAsync(contractAddress, transfer)); - } - } - - public class StandardTokenDeployment : ContractDeploymentMessage - { - - public static string BYTECODE = "0x60606040526040516020806106f5833981016040528080519060200190919050505b80600160005060003373ffffffffffffffffffffffffffffffffffffffff16815260200190815260200160002060005081905550806000600050819055505b506106868061006f6000396000f360606040523615610074576000357c010000000000000000000000000000000000000000000000000000000090048063095ea7b31461008157806318160ddd146100b657806323b872dd146100d957806370a0823114610117578063a9059cbb14610143578063dd62ed3e1461017857610074565b61007f5b610002565b565b005b6100a060048080359060200190919080359060200190919050506101ad565b6040518082815260200191505060405180910390f35b6100c36004805050610674565b6040518082815260200191505060405180910390f35b6101016004808035906020019091908035906020019091908035906020019091905050610281565b6040518082815260200191505060405180910390f35b61012d600480803590602001909190505061048d565b6040518082815260200191505060405180910390f35b61016260048080359060200190919080359060200190919050506104cb565b6040518082815260200191505060405180910390f35b610197600480803590602001909190803590602001909190505061060b565b6040518082815260200191505060405180910390f35b600081600260005060003373ffffffffffffffffffffffffffffffffffffffff16815260200190815260200160002060005060008573ffffffffffffffffffffffffffffffffffffffff168152602001908152602001600020600050819055508273ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff167f8c5be1e5ebec7d5bd14f71427d1e84f3dd0314c0f7b2291e5b200ac8c7c3b925846040518082815260200191505060405180910390a36001905061027b565b92915050565b600081600160005060008673ffffffffffffffffffffffffffffffffffffffff168152602001908152602001600020600050541015801561031b575081600260005060008673ffffffffffffffffffffffffffffffffffffffff16815260200190815260200160002060005060003373ffffffffffffffffffffffffffffffffffffffff1681526020019081526020016000206000505410155b80156103275750600082115b1561047c5781600160005060008573ffffffffffffffffffffffffffffffffffffffff1681526020019081526020016000206000828282505401925050819055508273ffffffffffffffffffffffffffffffffffffffff168473ffffffffffffffffffffffffffffffffffffffff167fddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef846040518082815260200191505060405180910390a381600160005060008673ffffffffffffffffffffffffffffffffffffffff16815260200190815260200160002060008282825054039250508190555081600260005060008673ffffffffffffffffffffffffffffffffffffffff16815260200190815260200160002060005060003373ffffffffffffffffffffffffffffffffffffffff1681526020019081526020016000206000828282505403925050819055506001905061048656610485565b60009050610486565b5b9392505050565b6000600160005060008373ffffffffffffffffffffffffffffffffffffffff1681526020019081526020016000206000505490506104c6565b919050565b600081600160005060003373ffffffffffffffffffffffffffffffffffffffff168152602001908152602001600020600050541015801561050c5750600082115b156105fb5781600160005060003373ffffffffffffffffffffffffffffffffffffffff16815260200190815260200160002060008282825054039250508190555081600160005060008573ffffffffffffffffffffffffffffffffffffffff1681526020019081526020016000206000828282505401925050819055508273ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff167fddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef846040518082815260200191505060405180910390a36001905061060556610604565b60009050610605565b5b92915050565b6000600260005060008473ffffffffffffffffffffffffffffffffffffffff16815260200190815260200160002060005060008373ffffffffffffffffffffffffffffffffffffffff16815260200190815260200160002060005054905061066e565b92915050565b60006000600050549050610683565b9056"; - - public StandardTokenDeployment() : base(BYTECODE) { } - - [Parameter("uint256", "totalSupply")] - public BigInteger TotalSupply { get; set; } - } - - [Function("balanceOf", "uint256")] - public class BalanceOfFunction : FunctionMessage - { - [Parameter("address", "_owner", 1)] - public string Owner { get; set; } - } - - [Function("transfer", "bool")] - public class TransferFunction : FunctionMessage - { - [Parameter("address", "_to", 1)] - public string To { get; set; } - - [Parameter("uint256", "_value", 2)] - public BigInteger TokenAmount { get; set; } - } - - [Event("Transfer")] - public class TransferEventDTO : IEventDTO - { - [Parameter("address", "_from", 1, true)] - public string From { get; set; } - - [Parameter("address", "_to", 2, true)] - public string To { get; set; } - - [Parameter("uint256", "_value", 3, false)] - public BigInteger Value { get; set; } - } -} diff --git a/CodexDistTestCore/Utils.cs b/CodexDistTestCore/Utils.cs deleted file mode 100644 index ae3e761..0000000 --- a/CodexDistTestCore/Utils.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace CodexDistTestCore -{ - public static class Utils - { - - } -} diff --git a/LongTests/BasicTests/LargeFileTests.cs b/LongTests/BasicTests/LargeFileTests.cs index a7f0c9b..3db5d59 100644 --- a/LongTests/BasicTests/LargeFileTests.cs +++ b/LongTests/BasicTests/LargeFileTests.cs @@ -1,4 +1,5 @@ -using CodexDistTestCore; +using DistTestCore; +using DistTestCore.Codex; using NUnit.Framework; namespace TestsLong.BasicTests diff --git a/LongTests/BasicTests/TestInfraTests.cs b/LongTests/BasicTests/TestInfraTests.cs index c39069c..7acd78e 100644 --- a/LongTests/BasicTests/TestInfraTests.cs +++ b/LongTests/BasicTests/TestInfraTests.cs @@ -1,4 +1,5 @@ -using CodexDistTestCore; +using DistTestCore; +using DistTestCore.Codex; using NUnit.Framework; namespace TestsLong.BasicTests diff --git a/LongTests/TestsLong.csproj b/LongTests/TestsLong.csproj index fc5152f..136951d 100644 --- a/LongTests/TestsLong.csproj +++ b/LongTests/TestsLong.csproj @@ -13,7 +13,7 @@ - + diff --git a/cs-codex-dist-testing.sln b/cs-codex-dist-testing.sln index 1f5c04f..4bc64fc 100644 --- a/cs-codex-dist-testing.sln +++ b/cs-codex-dist-testing.sln @@ -7,8 +7,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tests", "Tests\Tests.csproj EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TestsLong", "LongTests\TestsLong.csproj", "{AFCE270E-F844-4A7C-9006-69AE622BB1F4}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CodexDistTestCore", "CodexDistTestCore\CodexDistTestCore.csproj", "{19306DE1-CEE5-4F7B-AA5D-FD91926D853D}" -EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DistTestCore", "DistTestCore\DistTestCore.csproj", "{47F31305-6E68-4827-8E39-7B41DAA1CE7A}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "KubernetesWorkflow", "KubernetesWorkflow\KubernetesWorkflow.csproj", "{359123AA-3D9B-4442-80F4-19E32E3EC9EA}" @@ -17,7 +15,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Utils", "Utils\Utils.csproj EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Logging", "Logging\Logging.csproj", "{8481A4A6-4BDD-41B0-A3EB-EF53F7BD40D1}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NethereumWorkflow", "Nethereum\NethereumWorkflow.csproj", "{D6C3555E-D52D-4993-A87B-71AB650398FD}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "NethereumWorkflow", "Nethereum\NethereumWorkflow.csproj", "{D6C3555E-D52D-4993-A87B-71AB650398FD}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -33,10 +31,6 @@ Global {AFCE270E-F844-4A7C-9006-69AE622BB1F4}.Debug|Any CPU.Build.0 = Debug|Any CPU {AFCE270E-F844-4A7C-9006-69AE622BB1F4}.Release|Any CPU.ActiveCfg = Release|Any CPU {AFCE270E-F844-4A7C-9006-69AE622BB1F4}.Release|Any CPU.Build.0 = Release|Any CPU - {19306DE1-CEE5-4F7B-AA5D-FD91926D853D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {19306DE1-CEE5-4F7B-AA5D-FD91926D853D}.Debug|Any CPU.Build.0 = Debug|Any CPU - {19306DE1-CEE5-4F7B-AA5D-FD91926D853D}.Release|Any CPU.ActiveCfg = Release|Any CPU - {19306DE1-CEE5-4F7B-AA5D-FD91926D853D}.Release|Any CPU.Build.0 = Release|Any CPU {47F31305-6E68-4827-8E39-7B41DAA1CE7A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {47F31305-6E68-4827-8E39-7B41DAA1CE7A}.Debug|Any CPU.Build.0 = Debug|Any CPU {47F31305-6E68-4827-8E39-7B41DAA1CE7A}.Release|Any CPU.ActiveCfg = Release|Any CPU From 802f3459e9764513c859445de26e0920eac01d2b Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 17 Apr 2023 09:10:39 +0200 Subject: [PATCH 34/46] Fixes issue where oneclient-test would fail because node was not ready. --- DistTestCore/Codex/CodexAccess.cs | 4 +--- DistTestCore/CodexNodeGroup.cs | 29 +++++++++++++++-------------- DistTestCore/FileManager.cs | 17 ++++++++++++----- Logging/FixtureLog.cs | 4 ++-- 4 files changed, 30 insertions(+), 24 deletions(-) diff --git a/DistTestCore/Codex/CodexAccess.cs b/DistTestCore/Codex/CodexAccess.cs index d0c0f72..c07a457 100644 --- a/DistTestCore/Codex/CodexAccess.cs +++ b/DistTestCore/Codex/CodexAccess.cs @@ -13,9 +13,7 @@ namespace DistTestCore.Codex public CodexDebugResponse GetDebugInfo() { - var response = Http().HttpGetJson("debug/info"); - //Log($"Got DebugInfo with id: '{response.id}'."); - return response; + return Http().HttpGetJson("debug/info"); } public string UploadFile(FileStream fileStream) diff --git a/DistTestCore/CodexNodeGroup.cs b/DistTestCore/CodexNodeGroup.cs index 25d46c1..4e2cd16 100644 --- a/DistTestCore/CodexNodeGroup.cs +++ b/DistTestCore/CodexNodeGroup.cs @@ -47,13 +47,6 @@ namespace DistTestCore public RunningContainers Containers { get; private set; } public OnlineCodexNode[] Nodes { get; private set; } - //public GethCompanionGroup? GethCompanionGroup { get; set; } - - //public CodexNodeContainer[] GetContainers() - //{ - // return Nodes.Select(n => n.Container).ToArray(); - //} - public IEnumerator GetEnumerator() { return Nodes.Cast().GetEnumerator(); @@ -64,13 +57,6 @@ namespace DistTestCore return Nodes.GetEnumerator(); } - //public CodexNodeLog DownloadLog(IOnlineCodexNode node) - //{ - // var logDownloader = new PodLogDownloader(log, k8SManager); - // var n = (OnlineCodexNode)node; - // return logDownloader.DownloadLog(n); - //} - public string Describe() { return $""; @@ -79,7 +65,22 @@ namespace DistTestCore private OnlineCodexNode CreateOnlineCodexNode(RunningContainer c, ICodexNodeFactory factory) { var access = new CodexAccess(c); + EnsureOnline(access); return factory.CreateOnlineCodexNode(access, this); } + + private void EnsureOnline(CodexAccess access) + { + try + { + var debugInfo = access.GetDebugInfo(); + if (debugInfo == null || string.IsNullOrEmpty(debugInfo.id)) throw new InvalidOperationException("Unable to get debug-info from codex node at startup."); + } + catch (Exception e) + { + lifecycle.Log.Error($"Failed to start codex node: {e}"); + throw; + } + } } } diff --git a/DistTestCore/FileManager.cs b/DistTestCore/FileManager.cs index 10f126b..b195e9c 100644 --- a/DistTestCore/FileManager.cs +++ b/DistTestCore/FileManager.cs @@ -14,7 +14,6 @@ namespace DistTestCore { public const int ChunkSize = 1024 * 1024; private readonly Random random = new Random(); - private readonly List activeFiles = new List(); private readonly TestLog log; private readonly string folder; @@ -22,7 +21,7 @@ namespace DistTestCore { folder = configuration.GetFileManagerFolder(); - if (!Directory.Exists(folder)) Directory.CreateDirectory(folder); + EnsureDirectory(); this.log = log; } @@ -30,7 +29,6 @@ namespace DistTestCore { var result = new TestFile(Path.Combine(folder, Guid.NewGuid().ToString() + "_test.bin")); File.Create(result.Filename).Close(); - activeFiles.Add(result); return result; } @@ -44,8 +42,7 @@ namespace DistTestCore public void DeleteAllTestFiles() { - foreach (var file in activeFiles) File.Delete(file.Filename); - activeFiles.Clear(); + DeleteDirectory(); } private void GenerateFileBytes(TestFile result, ByteSize size) @@ -66,6 +63,16 @@ namespace DistTestCore using var stream = new FileStream(result.Filename, FileMode.Append); stream.Write(bytes, 0, bytes.Length); } + + private void EnsureDirectory() + { + if (!Directory.Exists(folder)) Directory.CreateDirectory(folder); + } + + private void DeleteDirectory() + { + Directory.Delete(folder, true); + } } public class TestFile diff --git a/Logging/FixtureLog.cs b/Logging/FixtureLog.cs index e06f4bc..7bb80ab 100644 --- a/Logging/FixtureLog.cs +++ b/Logging/FixtureLog.cs @@ -10,8 +10,8 @@ namespace Logging public FixtureLog(LogConfig config) { start = DateTime.UtcNow; - var folder = DetermineFolder(config); // "root/2023-04 /14" - var fixtureName = GetFixtureName(); // "11-09-23Z_ExampleTests" + var folder = DetermineFolder(config); + var fixtureName = GetFixtureName(); fullName = Path.Combine(folder, fixtureName); } From 8880ddd2bd8fc9acfcdc38c283c3b136e96e2729 Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 17 Apr 2023 10:31:14 +0200 Subject: [PATCH 35/46] Attempting to set up geth bootstrap argument --- DistTestCore/Logs/LogDownloadHandler.cs | 19 +++----- .../Marketplace/GethBootstrapNodeInfo.cs | 6 ++- .../Marketplace/GethBootstrapNodeStarter.cs | 6 ++- .../Marketplace/GethCompanionNodeStarter.cs | 2 +- .../Marketplace/GethContainerRecipe.cs | 15 ++++-- DistTestCore/Marketplace/GethInfoExtractor.cs | 48 +++++++++++++++++-- DistTestCore/Marketplace/GethStartupConfig.cs | 4 +- KubernetesWorkflow/K8sController.cs | 2 +- KubernetesWorkflow/StartupWorkflow.cs | 20 +++++++- Tests/BasicTests/ExampleTests.cs | 2 +- Tests/BasicTests/OneClientTests.cs | 1 - 11 files changed, 96 insertions(+), 29 deletions(-) diff --git a/DistTestCore/Logs/LogDownloadHandler.cs b/DistTestCore/Logs/LogDownloadHandler.cs index a01ad9e..9cc1012 100644 --- a/DistTestCore/Logs/LogDownloadHandler.cs +++ b/DistTestCore/Logs/LogDownloadHandler.cs @@ -3,15 +3,16 @@ using Logging; namespace DistTestCore.Logs { - public class LogDownloadHandler : ILogHandler + public class LogDownloadHandler : LogHandler, ILogHandler { - private readonly string description; private readonly LogFile log; public LogDownloadHandler(string description, LogFile log) { - this.description = description; this.log = log; + + log.Write($"{description} -->> {log.FullFilename}"); + log.WriteRaw(description); } public CodexNodeLog CreateCodexNodeLog() @@ -19,17 +20,9 @@ namespace DistTestCore.Logs return new CodexNodeLog(log); } - public void Log(Stream stream) + protected override void ProcessLine(string line) { - log.Write($"{description} -->> {log.FullFilename}"); - log.WriteRaw(description); - var reader = new StreamReader(stream); - var line = reader.ReadLine(); - while (line != null) - { - log.WriteRaw(line); - line = reader.ReadLine(); - } + log.WriteRaw(line); } } } diff --git a/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs b/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs index 39b6715..46189c4 100644 --- a/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs +++ b/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs @@ -6,16 +6,20 @@ namespace DistTestCore.Marketplace { public class GethBootstrapNodeInfo { - public GethBootstrapNodeInfo(RunningContainers runningContainers, string account, string genesisJsonBase64) + public GethBootstrapNodeInfo(RunningContainers runningContainers, string account, string genesisJsonBase64, string pubKey, Port discoveryPort) { RunningContainers = runningContainers; Account = account; GenesisJsonBase64 = genesisJsonBase64; + PubKey = pubKey; + DiscoveryPort = discoveryPort; } public RunningContainers RunningContainers { get; } public string Account { get; } public string GenesisJsonBase64 { get; } + public string PubKey { get; } + public Port DiscoveryPort { get; } public NethereumInteraction StartInteraction(TestLog log) { diff --git a/DistTestCore/Marketplace/GethBootstrapNodeStarter.cs b/DistTestCore/Marketplace/GethBootstrapNodeStarter.cs index 7c114c4..2868752 100644 --- a/DistTestCore/Marketplace/GethBootstrapNodeStarter.cs +++ b/DistTestCore/Marketplace/GethBootstrapNodeStarter.cs @@ -26,16 +26,18 @@ namespace DistTestCore.Marketplace var extractor = new GethInfoExtractor(workflow, containers.Containers[0]); var account = extractor.ExtractAccount(); var genesisJsonBase64 = extractor.ExtractGenesisJsonBase64(); + var pubKey = extractor.ExtractPubKey(); + var discoveryPort = containers.Containers[0].Recipe.GetPortByTag(GethContainerRecipe.DiscoveryPortTag); Log($"Geth bootstrap node started with account '{account}'"); - return new GethBootstrapNodeInfo(containers, account, genesisJsonBase64); + return new GethBootstrapNodeInfo(containers, account, genesisJsonBase64, pubKey, discoveryPort); } private StartupConfig CreateBootstrapStartupConfig() { var config = new StartupConfig(); - config.Add(new GethStartupConfig(true, bootstrapGenesisJsonBase64)); + config.Add(new GethStartupConfig(true, bootstrapGenesisJsonBase64, null!)); return config; } diff --git a/DistTestCore/Marketplace/GethCompanionNodeStarter.cs b/DistTestCore/Marketplace/GethCompanionNodeStarter.cs index b8b178b..c746bf1 100644 --- a/DistTestCore/Marketplace/GethCompanionNodeStarter.cs +++ b/DistTestCore/Marketplace/GethCompanionNodeStarter.cs @@ -38,7 +38,7 @@ namespace DistTestCore.Marketplace private StartupConfig CreateCompanionNodeStartupConfig(GethBootstrapNodeInfo bootstrapNode) { var config = new StartupConfig(); - config.Add(new GethStartupConfig(false, bootstrapNode.GenesisJsonBase64)); + config.Add(new GethStartupConfig(false, bootstrapNode.GenesisJsonBase64, bootstrapNode)); return config; } diff --git a/DistTestCore/Marketplace/GethContainerRecipe.cs b/DistTestCore/Marketplace/GethContainerRecipe.cs index 2dbfbde..fad225f 100644 --- a/DistTestCore/Marketplace/GethContainerRecipe.cs +++ b/DistTestCore/Marketplace/GethContainerRecipe.cs @@ -6,6 +6,7 @@ namespace DistTestCore.Marketplace { public const string DockerImage = "thatbenbierens/geth-confenv:latest"; public const string HttpPortTag = "http_port"; + public const string DiscoveryPortTag = "disc_port"; public const string AccountFilename = "account_string.txt"; public const string GenesisFilename = "genesis.json"; @@ -23,18 +24,26 @@ namespace DistTestCore.Marketplace private string CreateArgs(GethStartupConfig config) { + var discovery = AddInternalPort(tag: DiscoveryPortTag); + if (config.IsBootstrapNode) { AddEnvVar("IS_BOOTSTRAP", "1"); var exposedPort = AddExposedPort(); - return $"--http.port {exposedPort.Number}"; + return $"--http.port {exposedPort.Number} --discovery.port {discovery.Number} --nodiscover"; } var port = AddInternalPort(); - var discovery = AddInternalPort(); var authRpc = AddInternalPort(); var httpPort = AddInternalPort(tag: HttpPortTag); - return $"--port {port.Number} --discovery.port {discovery.Number} --authrpc.port {authRpc.Number} --http.port {httpPort.Number}"; + + var bootPubKey = config.BootstrapNode.PubKey; + var bootIp = config.BootstrapNode.RunningContainers.Containers[0].Pod.Ip; + var bootPort = config.BootstrapNode.DiscoveryPort.Number; + var bootstrapArg = $"--bootnodes enode://{bootPubKey}@{bootIp}:{bootPort}"; + // geth --bootnodes enode://pubkey1@ip1:port1 + + return $"--port {port.Number} --discovery.port {discovery.Number} --authrpc.port {authRpc.Number} --http.port {httpPort.Number} --nodiscover {bootstrapArg}"; } } } diff --git a/DistTestCore/Marketplace/GethInfoExtractor.cs b/DistTestCore/Marketplace/GethInfoExtractor.cs index f27e396..59725ec 100644 --- a/DistTestCore/Marketplace/GethInfoExtractor.cs +++ b/DistTestCore/Marketplace/GethInfoExtractor.cs @@ -17,7 +17,6 @@ namespace DistTestCore.Marketplace public string ExtractAccount() { var account = Retry(FetchAccount); - if (string.IsNullOrEmpty(account)) throw new InvalidOperationException("Unable to fetch account for geth node. Test infra failure."); return account; @@ -26,12 +25,17 @@ namespace DistTestCore.Marketplace public string ExtractGenesisJsonBase64() { var genesisJson = Retry(FetchGenesisJson); - if (string.IsNullOrEmpty(genesisJson)) throw new InvalidOperationException("Unable to fetch genesis-json for geth node. Test infra failure."); - var encoded = Convert.ToBase64String(Encoding.ASCII.GetBytes(genesisJson)); + return Convert.ToBase64String(Encoding.ASCII.GetBytes(genesisJson)); + } - return encoded; + public string ExtractPubKey() + { + var pubKey = Retry(FetchPubKey); + if (string.IsNullOrEmpty(pubKey)) throw new InvalidOperationException("Unable to fetch enode from geth node. Test infra failure."); + + return pubKey; } private string Retry(Func fetch) @@ -54,5 +58,41 @@ namespace DistTestCore.Marketplace { return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.AccountFilename); } + + private string FetchPubKey() + { + var enodeFinder = new PubKeyFinder(); + workflow.DownloadContainerLog(container, enodeFinder); + return enodeFinder.GetPubKey(); + } + } + + public class PubKeyFinder : LogHandler, ILogHandler + { + private const string openTag = "self=\"enode://"; + private string pubKey = string.Empty; + + public string GetPubKey() + { + return pubKey; + } + + protected override void ProcessLine(string line) + { + if (line.Contains(openTag)) + { + ExtractPubKey(line); + } + } + + private void ExtractPubKey(string line) + { + var openIndex = line.IndexOf(openTag) + openTag.Length; + var closeIndex = line.IndexOf("@"); + + pubKey = line.Substring( + startIndex: openIndex, + length: closeIndex - openIndex); + } } } diff --git a/DistTestCore/Marketplace/GethStartupConfig.cs b/DistTestCore/Marketplace/GethStartupConfig.cs index 60164bb..baeb421 100644 --- a/DistTestCore/Marketplace/GethStartupConfig.cs +++ b/DistTestCore/Marketplace/GethStartupConfig.cs @@ -2,13 +2,15 @@ { public class GethStartupConfig { - public GethStartupConfig(bool isBootstrapNode, string genesisJsonBase64) + public GethStartupConfig(bool isBootstrapNode, string genesisJsonBase64, GethBootstrapNodeInfo bootstrapNode) { IsBootstrapNode = isBootstrapNode; GenesisJsonBase64 = genesisJsonBase64; + BootstrapNode = bootstrapNode; } public bool IsBootstrapNode { get; } public string GenesisJsonBase64 { get; } + public GethBootstrapNodeInfo BootstrapNode { get; } } } diff --git a/KubernetesWorkflow/K8sController.cs b/KubernetesWorkflow/K8sController.cs index 0804e8f..6ce366a 100644 --- a/KubernetesWorkflow/K8sController.cs +++ b/KubernetesWorkflow/K8sController.cs @@ -45,7 +45,7 @@ namespace KubernetesWorkflow public void DownloadPodLog(RunningPod pod, ContainerRecipe recipe, ILogHandler logHandler) { - var stream = client.ReadNamespacedPodLog(pod.Name, K8sNamespace, recipe.Name); + using var stream = client.ReadNamespacedPodLog(pod.Name, K8sNamespace, recipe.Name); logHandler.Log(stream); } diff --git a/KubernetesWorkflow/StartupWorkflow.cs b/KubernetesWorkflow/StartupWorkflow.cs index f575b15..d37699f 100644 --- a/KubernetesWorkflow/StartupWorkflow.cs +++ b/KubernetesWorkflow/StartupWorkflow.cs @@ -1,4 +1,6 @@ -namespace KubernetesWorkflow +using System.IO; + +namespace KubernetesWorkflow { public class StartupWorkflow { @@ -94,4 +96,20 @@ { void Log(Stream log); } + + public abstract class LogHandler : ILogHandler + { + public void Log(Stream log) + { + using var reader = new StreamReader(log); + var line = reader.ReadLine(); + while (line != null) + { + ProcessLine(line); + line = reader.ReadLine(); + } + } + + protected abstract void ProcessLine(string line); + } } diff --git a/Tests/BasicTests/ExampleTests.cs b/Tests/BasicTests/ExampleTests.cs index a3e3ae4..b70f3c4 100644 --- a/Tests/BasicTests/ExampleTests.cs +++ b/Tests/BasicTests/ExampleTests.cs @@ -49,7 +49,7 @@ namespace Tests.BasicTests [Test] public void MarketplaceExample() { - var group = SetupCodexNodes(4) + var group = SetupCodexNodes(2) .WithStorageQuota(10.GB()) .EnableMarketplace(initialBalance: 20) .BringOnline(); diff --git a/Tests/BasicTests/OneClientTests.cs b/Tests/BasicTests/OneClientTests.cs index 76ff79b..a2c0c57 100644 --- a/Tests/BasicTests/OneClientTests.cs +++ b/Tests/BasicTests/OneClientTests.cs @@ -15,7 +15,6 @@ namespace Tests.BasicTests } [Test] - [Ignore("Unstable.")] public void RestartTest() { var group = SetupCodexNodes(1).BringOnline(); From ca822c508d69077d5b54555816bb237dcd16dbdc Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 17 Apr 2023 11:28:07 +0200 Subject: [PATCH 36/46] Adds catch-retry to geth info extractor --- DistTestCore/Marketplace/GethInfoExtractor.cs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/DistTestCore/Marketplace/GethInfoExtractor.cs b/DistTestCore/Marketplace/GethInfoExtractor.cs index 59725ec..7151a60 100644 --- a/DistTestCore/Marketplace/GethInfoExtractor.cs +++ b/DistTestCore/Marketplace/GethInfoExtractor.cs @@ -40,7 +40,7 @@ namespace DistTestCore.Marketplace private string Retry(Func fetch) { - var result = fetch(); + var result = Catch(fetch); if (string.IsNullOrEmpty(result)) { Thread.Sleep(TimeSpan.FromSeconds(5)); @@ -49,6 +49,18 @@ namespace DistTestCore.Marketplace return result; } + private string Catch(Func fetch) + { + try + { + return fetch(); + } + catch + { + return string.Empty; + } + } + private string FetchGenesisJson() { return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.GenesisFilename); From f06216b9317cc70645079921185dda884b592a95 Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 17 Apr 2023 16:28:07 +0200 Subject: [PATCH 37/46] Deploys codex-contracts along with geth bootstrap node. --- DistTestCore/DistTest.cs | 54 ++++++--------- .../CodexContractsContainerConfig.cs | 16 +++++ .../CodexContractsContainerRecipe.cs | 23 +++++++ .../Marketplace/CodexContractsStarter.cs | 68 +++++++++++++++++++ .../Marketplace/GethBootstrapNodeStarter.cs | 14 +++- .../Marketplace/GethContainerRecipe.cs | 2 +- KubernetesWorkflow/K8sController.cs | 14 +--- Utils/Time.cs | 16 +++++ 8 files changed, 160 insertions(+), 47 deletions(-) create mode 100644 DistTestCore/Marketplace/CodexContractsContainerConfig.cs create mode 100644 DistTestCore/Marketplace/CodexContractsContainerRecipe.cs create mode 100644 DistTestCore/Marketplace/CodexContractsStarter.cs diff --git a/DistTestCore/DistTest.cs b/DistTestCore/DistTest.cs index 75711c8..972a539 100644 --- a/DistTestCore/DistTest.cs +++ b/DistTestCore/DistTest.cs @@ -35,7 +35,7 @@ namespace DistTestCore catch (Exception ex) { GlobalTestFailure.HasFailed = true; - Error($"Global setup cleanup failed with: {ex}"); + fixtureLog.Error($"Global setup cleanup failed with: {ex}"); throw; } @@ -67,7 +67,7 @@ namespace DistTestCore } catch (Exception ex) { - Error("Cleanup failed: " + ex.Message); + fixtureLog.Error("Cleanup failed: " + ex.Message); GlobalTestFailure.HasFailed = true; } } @@ -82,36 +82,6 @@ namespace DistTestCore return new CodexSetup(lifecycle.CodexStarter, numberOfNodes); } - private void IncludeLogsAndMetricsOnTestFailure() - { - var result = TestContext.CurrentContext.Result; - if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed) - { - fixtureLog.MarkAsFailed(); - - if (IsDownloadingLogsAndMetricsEnabled()) - { - Log("Downloading all CodexNode logs and metrics because of test failure..."); - DownloadAllLogs(); - DownloadAllMetrics(); - } - else - { - Log("Skipping download of all CodexNode logs and metrics due to [DontDownloadLogsAndMetricsOnFailure] attribute."); - } - } - } - - private void Log(string msg) - { - lifecycle.Log.Log(msg); - } - - private void Error(string msg) - { - lifecycle.Log.Error(msg); - } - private void CreateNewTestLifecycle() { Stopwatch.Measure(fixtureLog, $"Setup for {GetCurrentTestName()}", () => @@ -133,6 +103,26 @@ namespace DistTestCore }); } + private void IncludeLogsAndMetricsOnTestFailure() + { + var result = TestContext.CurrentContext.Result; + if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed) + { + fixtureLog.MarkAsFailed(); + + if (IsDownloadingLogsAndMetricsEnabled()) + { + lifecycle.Log.Log("Downloading all CodexNode logs and metrics because of test failure..."); + DownloadAllLogs(); + DownloadAllMetrics(); + } + else + { + lifecycle.Log.Log("Skipping download of all CodexNode logs and metrics due to [DontDownloadLogsAndMetricsOnFailure] attribute."); + } + } + } + private string GetTestDuration() { var testDuration = DateTime.UtcNow - testStart; diff --git a/DistTestCore/Marketplace/CodexContractsContainerConfig.cs b/DistTestCore/Marketplace/CodexContractsContainerConfig.cs new file mode 100644 index 0000000..3b669a4 --- /dev/null +++ b/DistTestCore/Marketplace/CodexContractsContainerConfig.cs @@ -0,0 +1,16 @@ +using KubernetesWorkflow; + +namespace DistTestCore.Marketplace +{ + public class CodexContractsContainerConfig + { + public CodexContractsContainerConfig(string bootstrapNodeIp, Port jsonRpcPort) + { + BootstrapNodeIp = bootstrapNodeIp; + JsonRpcPort = jsonRpcPort; + } + + public string BootstrapNodeIp { get; } + public Port JsonRpcPort { get; } + } +} diff --git a/DistTestCore/Marketplace/CodexContractsContainerRecipe.cs b/DistTestCore/Marketplace/CodexContractsContainerRecipe.cs new file mode 100644 index 0000000..63d6f30 --- /dev/null +++ b/DistTestCore/Marketplace/CodexContractsContainerRecipe.cs @@ -0,0 +1,23 @@ +using KubernetesWorkflow; + +namespace DistTestCore.Marketplace +{ + public class CodexContractsContainerRecipe : ContainerRecipeFactory + { + public const string DockerImage = "thatbenbierens/codex-contracts-deployment"; + + protected override string Image => DockerImage; + + protected override void Initialize(StartupConfig startupConfig) + { + var config = startupConfig.Get(); + + var ip = config.BootstrapNodeIp; + var port = config.JsonRpcPort.Number; + + AddEnvVar("DISTTEST_NETWORK_URL", $"http://{ip}:{port}"); + AddEnvVar("HARDHAT_NETWORK", "codexdisttestnetwork"); + AddEnvVar("KEEP_ALIVE", "1"); + } + } +} diff --git a/DistTestCore/Marketplace/CodexContractsStarter.cs b/DistTestCore/Marketplace/CodexContractsStarter.cs new file mode 100644 index 0000000..a6539bb --- /dev/null +++ b/DistTestCore/Marketplace/CodexContractsStarter.cs @@ -0,0 +1,68 @@ +using KubernetesWorkflow; +using Utils; + +namespace DistTestCore.Marketplace +{ + public class CodexContractsStarter + { + private const string readyString = "Done! Sleeping indefinitely..."; + private readonly TestLifecycle lifecycle; + private readonly WorkflowCreator workflowCreator; + + public CodexContractsStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator) + { + this.lifecycle = lifecycle; + this.workflowCreator = workflowCreator; + } + + public void Start(RunningContainer bootstrapContainer) + { + var workflow = workflowCreator.CreateWorkflow(); + var startupConfig = CreateStartupConfig(bootstrapContainer); + + lifecycle.Log.Log("Deploying Codex contracts..."); + var containers = workflow.Start(1, Location.Unspecified, new CodexContractsContainerRecipe(), startupConfig); + if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 Codex contracts container to be created. Test infra failure."); + var container = containers.Containers[0]; + + WaitUntil(() => + { + var logHandler = new ContractsReadyLogHandler(readyString); + workflow.DownloadContainerLog(container, logHandler); + return logHandler.Found; + }); + + lifecycle.Log.Log("Contracts deployed."); + } + + private void WaitUntil(Func predicate) + { + Time.WaitUntil(predicate, TimeSpan.FromMinutes(2), TimeSpan.FromSeconds(1)); + } + + private StartupConfig CreateStartupConfig(RunningContainer bootstrapContainer) + { + var startupConfig = new StartupConfig(); + var contractsConfig = new CodexContractsContainerConfig(bootstrapContainer.Pod.Ip, bootstrapContainer.Recipe.GetPortByTag(GethContainerRecipe.HttpPortTag)); + startupConfig.Add(contractsConfig); + return startupConfig; + } + } + + public class ContractsReadyLogHandler : LogHandler + { + private readonly string targetString; + + public ContractsReadyLogHandler(string targetString) + { + this.targetString = targetString; + } + + public bool Found { get; private set; } + + protected override void ProcessLine(string line) + { + if (line.Contains(targetString)) Found = true; + } + } +} diff --git a/DistTestCore/Marketplace/GethBootstrapNodeStarter.cs b/DistTestCore/Marketplace/GethBootstrapNodeStarter.cs index 2868752..bc5d4aa 100644 --- a/DistTestCore/Marketplace/GethBootstrapNodeStarter.cs +++ b/DistTestCore/Marketplace/GethBootstrapNodeStarter.cs @@ -7,11 +7,13 @@ namespace DistTestCore.Marketplace private const string bootstrapGenesisJsonBase64 = "ewogICAgImNvbmZpZyI6IHsKICAgICAgImNoYWluSWQiOiA3ODk5ODgsCiAgICAgICJob21lc3RlYWRCbG9jayI6IDAsCiAgICAgICJlaXAxNTBCbG9jayI6IDAsCiAgICAgICJlaXAxNTVCbG9jayI6IDAsCiAgICAgICJlaXAxNThCbG9jayI6IDAsCiAgICAgICJieXphbnRpdW1CbG9jayI6IDAsCiAgICAgICJjb25zdGFudGlub3BsZUJsb2NrIjogMCwKICAgICAgInBldGVyc2J1cmdCbG9jayI6IDAsCiAgICAgICJpc3RhbmJ1bEJsb2NrIjogMCwKICAgICAgIm11aXJHbGFjaWVyQmxvY2siOiAwLAogICAgICAiYmVybGluQmxvY2siOiAwLAogICAgICAibG9uZG9uQmxvY2siOiAwLAogICAgICAiYXJyb3dHbGFjaWVyQmxvY2siOiAwLAogICAgICAiZ3JheUdsYWNpZXJCbG9jayI6IDAsCiAgICAgICJjbGlxdWUiOiB7CiAgICAgICAgInBlcmlvZCI6IDUsCiAgICAgICAgImVwb2NoIjogMzAwMDAKICAgICAgfQogICAgfSwKICAgICJkaWZmaWN1bHR5IjogIjEiLAogICAgImdhc0xpbWl0IjogIjgwMDAwMDAwMCIsCiAgICAiZXh0cmFkYXRhIjogIjB4MDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMEFDQ09VTlRfSEVSRTAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAiLAogICAgImFsbG9jIjogewogICAgICAiMHhBQ0NPVU5UX0hFUkUiOiB7ICJiYWxhbmNlIjogIjUwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAiIH0KICAgIH0KICB9"; private readonly TestLifecycle lifecycle; private readonly WorkflowCreator workflowCreator; + private readonly CodexContractsStarter codexContractsStarter; public GethBootstrapNodeStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator) { this.lifecycle = lifecycle; this.workflowCreator = workflowCreator; + codexContractsStarter = new CodexContractsStarter(lifecycle, workflowCreator); } public GethBootstrapNodeInfo StartGethBootstrapNode() @@ -22,18 +24,26 @@ namespace DistTestCore.Marketplace var workflow = workflowCreator.CreateWorkflow(); var containers = workflow.Start(1, Location.Unspecified, new GethContainerRecipe(), startupConfig); if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 Geth bootstrap node to be created. Test infra failure."); + var bootstrapContainer = containers.Containers[0]; - var extractor = new GethInfoExtractor(workflow, containers.Containers[0]); + var extractor = new GethInfoExtractor(workflow, bootstrapContainer); var account = extractor.ExtractAccount(); var genesisJsonBase64 = extractor.ExtractGenesisJsonBase64(); var pubKey = extractor.ExtractPubKey(); - var discoveryPort = containers.Containers[0].Recipe.GetPortByTag(GethContainerRecipe.DiscoveryPortTag); + var discoveryPort = bootstrapContainer.Recipe.GetPortByTag(GethContainerRecipe.DiscoveryPortTag); Log($"Geth bootstrap node started with account '{account}'"); + DeployCodexContracts(bootstrapContainer); + return new GethBootstrapNodeInfo(containers, account, genesisJsonBase64, pubKey, discoveryPort); } + private void DeployCodexContracts(RunningContainer bootstrapContainer) + { + codexContractsStarter.Start(bootstrapContainer); + } + private StartupConfig CreateBootstrapStartupConfig() { var config = new StartupConfig(); diff --git a/DistTestCore/Marketplace/GethContainerRecipe.cs b/DistTestCore/Marketplace/GethContainerRecipe.cs index fad225f..3cb7d41 100644 --- a/DistTestCore/Marketplace/GethContainerRecipe.cs +++ b/DistTestCore/Marketplace/GethContainerRecipe.cs @@ -29,7 +29,7 @@ namespace DistTestCore.Marketplace if (config.IsBootstrapNode) { AddEnvVar("IS_BOOTSTRAP", "1"); - var exposedPort = AddExposedPort(); + var exposedPort = AddExposedPort(tag: HttpPortTag); return $"--http.port {exposedPort.Number} --discovery.port {discovery.Number} --nodiscover"; } diff --git a/KubernetesWorkflow/K8sController.cs b/KubernetesWorkflow/K8sController.cs index 6ce366a..611c8fe 100644 --- a/KubernetesWorkflow/K8sController.cs +++ b/KubernetesWorkflow/K8sController.cs @@ -1,5 +1,6 @@ using k8s; using k8s.Models; +using Utils; namespace KubernetesWorkflow { @@ -345,18 +346,7 @@ namespace KubernetesWorkflow private void WaitUntil(Func predicate) { - var start = DateTime.UtcNow; - var state = predicate(); - while (!state) - { - if (DateTime.UtcNow - start > cluster.K8sOperationTimeout()) - { - throw new TimeoutException("K8s operation timed out."); - } - - cluster.WaitForK8sServiceDelay(); - state = predicate(); - } + Time.WaitUntil(predicate, cluster.K8sOperationTimeout(), cluster.WaitForK8sServiceDelay()); } #endregion diff --git a/Utils/Time.cs b/Utils/Time.cs index afe4f29..0f4f71b 100644 --- a/Utils/Time.cs +++ b/Utils/Time.cs @@ -22,5 +22,21 @@ result += $"{d.Seconds} secs"; return result; } + + public static void WaitUntil(Func predicate, TimeSpan timeout, TimeSpan retryTime) + { + var start = DateTime.UtcNow; + var state = predicate(); + while (!state) + { + if (DateTime.UtcNow - start > timeout) + { + throw new TimeoutException("Operation timed out."); + } + + Sleep(retryTime); + state = predicate(); + } + } } } From 9b38447dbe912c9fc3681f8cc77305386ae53358 Mon Sep 17 00:00:00 2001 From: benbierens Date: Tue, 18 Apr 2023 10:22:11 +0200 Subject: [PATCH 38/46] Setting up access to marketplace address --- DistTestCore/ByteSize.cs | 2 +- DistTestCore/CodexSetup.cs | 12 +++- DistTestCore/DistTestCore.csproj | 1 + DistTestCore/GethStarter.cs | 52 ++++++++++------- .../CodexContractsContainerRecipe.cs | 1 + .../Marketplace/CodexContractsStarter.cs | 17 +++++- ...Extractor.cs => ContainerInfoExtractor.cs} | 25 +++++++- .../Marketplace/GethBootstrapNodeInfo.cs | 11 ---- .../Marketplace/GethBootstrapNodeStarter.cs | 11 +--- .../Marketplace/GethCompanionNodeStarter.cs | 2 +- DistTestCore/Marketplace/GethStartResult.cs | 6 +- DistTestCore/Marketplace/MarketplaceAccess.cs | 8 +-- .../Marketplace/MarketplaceAccessFactory.cs | 8 +-- .../Marketplace/MarketplaceInitialConfig.cs | 8 ++- .../Marketplace/MarketplaceNetwork.cs | 27 +++++++++ DistTestCore/Tokens.cs | 57 +++++++++++++++++++ Nethereum/NethereumInteraction.cs | 8 +++ Tests/BasicTests/ExampleTests.cs | 2 +- 18 files changed, 192 insertions(+), 66 deletions(-) rename DistTestCore/Marketplace/{GethInfoExtractor.cs => ContainerInfoExtractor.cs} (76%) create mode 100644 DistTestCore/Marketplace/MarketplaceNetwork.cs create mode 100644 DistTestCore/Tokens.cs diff --git a/DistTestCore/ByteSize.cs b/DistTestCore/ByteSize.cs index 4f13da8..783cce9 100644 --- a/DistTestCore/ByteSize.cs +++ b/DistTestCore/ByteSize.cs @@ -10,7 +10,7 @@ public long SizeInBytes { get; } } - public static class IntExtensions + public static class ByteSizeIntExtensions { private const long Kilo = 1024; diff --git a/DistTestCore/CodexSetup.cs b/DistTestCore/CodexSetup.cs index 612c37e..72dc103 100644 --- a/DistTestCore/CodexSetup.cs +++ b/DistTestCore/CodexSetup.cs @@ -11,7 +11,8 @@ namespace DistTestCore //ICodexStartupConfig WithBootstrapNode(IOnlineCodexNode node); ICodexSetup WithStorageQuota(ByteSize storageQuota); ICodexSetup EnableMetrics(); - ICodexSetup EnableMarketplace(int initialBalance); + ICodexSetup EnableMarketplace(TestToken initialBalance); + ICodexSetup EnableMarketplace(TestToken initialBalance, Ether initialEther); ICodexNodeGroup BringOnline(); } @@ -62,9 +63,14 @@ namespace DistTestCore return this; } - public ICodexSetup EnableMarketplace(int initialBalance) + public ICodexSetup EnableMarketplace(TestToken initialBalance) { - MarketplaceConfig = new MarketplaceInitialConfig(initialBalance); + return EnableMarketplace(initialBalance, 1000.Eth()); + } + + public ICodexSetup EnableMarketplace(TestToken initialBalance, Ether initialEther) + { + MarketplaceConfig = new MarketplaceInitialConfig(initialEther, initialBalance); return this; } diff --git a/DistTestCore/DistTestCore.csproj b/DistTestCore/DistTestCore.csproj index ccd5f66..f7fe20a 100644 --- a/DistTestCore/DistTestCore.csproj +++ b/DistTestCore/DistTestCore.csproj @@ -8,6 +8,7 @@ + diff --git a/DistTestCore/GethStarter.cs b/DistTestCore/GethStarter.cs index 3e01a5e..dbb19f9 100644 --- a/DistTestCore/GethStarter.cs +++ b/DistTestCore/GethStarter.cs @@ -5,13 +5,15 @@ namespace DistTestCore { public class GethStarter { - private readonly GethBootstrapNodeCache bootstrapNodeCache; + private readonly MarketplaceNetworkCache marketplaceNetworkCache; private readonly GethCompanionNodeStarter companionNodeStarter; private readonly TestLifecycle lifecycle; public GethStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator) { - bootstrapNodeCache = new GethBootstrapNodeCache(new GethBootstrapNodeStarter(lifecycle, workflowCreator)); + marketplaceNetworkCache = new MarketplaceNetworkCache( + new GethBootstrapNodeStarter(lifecycle, workflowCreator), + new CodexContractsStarter(lifecycle, workflowCreator)); companionNodeStarter = new GethCompanionNodeStarter(lifecycle, workflowCreator); this.lifecycle = lifecycle; } @@ -20,26 +22,28 @@ namespace DistTestCore { if (codexSetup.MarketplaceConfig == null) return CreateMarketplaceUnavailableResult(); - var bootstrapNode = bootstrapNodeCache.Get(); - var companionNodes = StartCompanionNodes(codexSetup, bootstrapNode); + var marketplaceNetwork = marketplaceNetworkCache.Get(); + var companionNodes = StartCompanionNodes(codexSetup, marketplaceNetwork); - TransferInitialBalance(bootstrapNode, codexSetup.MarketplaceConfig.InitialBalance, companionNodes); + TransferInitialBalance(marketplaceNetwork, codexSetup.MarketplaceConfig, companionNodes); - return CreateGethStartResult(bootstrapNode, companionNodes); + return CreateGethStartResult(marketplaceNetwork, companionNodes); } - private void TransferInitialBalance(GethBootstrapNodeInfo bootstrapNode, int initialBalance, GethCompanionNodeInfo[] companionNodes) + private void TransferInitialBalance(MarketplaceNetwork marketplaceNetwork, MarketplaceInitialConfig marketplaceConfig, GethCompanionNodeInfo[] companionNodes) { - var interaction = bootstrapNode.StartInteraction(lifecycle.Log); + var interaction = marketplaceNetwork.StartInteraction(lifecycle.Log); foreach (var node in companionNodes) { - interaction.TransferTo(node.Account, initialBalance); + interaction.TransferTo(node.Account, marketplaceConfig.InitialEth.Wei); + // wrong level: mintTestTokens? interactions knows nothing about contract details! + //interaction.MintTestTokens(node.Account, marketplaceConfig.InitialTestTokens.Amount); } } - private GethStartResult CreateGethStartResult(GethBootstrapNodeInfo bootstrapNode, GethCompanionNodeInfo[] companionNodes) + private GethStartResult CreateGethStartResult(MarketplaceNetwork marketplaceNetwork, GethCompanionNodeInfo[] companionNodes) { - return new GethStartResult(CreateMarketplaceAccessFactory(bootstrapNode), bootstrapNode, companionNodes); + return new GethStartResult(CreateMarketplaceAccessFactory(marketplaceNetwork), marketplaceNetwork, companionNodes); } private GethStartResult CreateMarketplaceUnavailableResult() @@ -47,34 +51,38 @@ namespace DistTestCore return new GethStartResult(new MarketplaceUnavailableAccessFactory(), null!, Array.Empty()); } - private IMarketplaceAccessFactory CreateMarketplaceAccessFactory(GethBootstrapNodeInfo bootstrapNode) + private IMarketplaceAccessFactory CreateMarketplaceAccessFactory(MarketplaceNetwork marketplaceNetwork) { - return new GethMarketplaceAccessFactory(lifecycle.Log, bootstrapNode!); + return new GethMarketplaceAccessFactory(lifecycle.Log, marketplaceNetwork); } - private GethCompanionNodeInfo[] StartCompanionNodes(CodexSetup codexSetup, GethBootstrapNodeInfo bootstrapNode) + private GethCompanionNodeInfo[] StartCompanionNodes(CodexSetup codexSetup, MarketplaceNetwork marketplaceNetwork) { - return companionNodeStarter.StartCompanionNodesFor(codexSetup, bootstrapNode); + return companionNodeStarter.StartCompanionNodesFor(codexSetup, marketplaceNetwork.Bootstrap); } } - public class GethBootstrapNodeCache + public class MarketplaceNetworkCache { private readonly GethBootstrapNodeStarter bootstrapNodeStarter; - private GethBootstrapNodeInfo? bootstrapNode; + private readonly CodexContractsStarter codexContractsStarter; + private MarketplaceNetwork? network; - public GethBootstrapNodeCache(GethBootstrapNodeStarter bootstrapNodeStarter) + public MarketplaceNetworkCache(GethBootstrapNodeStarter bootstrapNodeStarter, CodexContractsStarter codexContractsStarter) { this.bootstrapNodeStarter = bootstrapNodeStarter; + this.codexContractsStarter = codexContractsStarter; } - public GethBootstrapNodeInfo Get() + public MarketplaceNetwork Get() { - if (bootstrapNode == null) + if (network == null) { - bootstrapNode = bootstrapNodeStarter.StartGethBootstrapNode(); + var bootstrapInfo = bootstrapNodeStarter.StartGethBootstrapNode(); + var marketplaceInfo = codexContractsStarter.Start(bootstrapInfo.RunningContainers.Containers[0]); + network = new MarketplaceNetwork(bootstrapInfo, marketplaceInfo ); } - return bootstrapNode; + return network; } } } diff --git a/DistTestCore/Marketplace/CodexContractsContainerRecipe.cs b/DistTestCore/Marketplace/CodexContractsContainerRecipe.cs index 63d6f30..d2a93a7 100644 --- a/DistTestCore/Marketplace/CodexContractsContainerRecipe.cs +++ b/DistTestCore/Marketplace/CodexContractsContainerRecipe.cs @@ -5,6 +5,7 @@ namespace DistTestCore.Marketplace public class CodexContractsContainerRecipe : ContainerRecipeFactory { public const string DockerImage = "thatbenbierens/codex-contracts-deployment"; + public const string MarketplaceAddressFilename = "/usr/app/deployments/codexdisttestnetwork/Marketplace.json"; protected override string Image => DockerImage; diff --git a/DistTestCore/Marketplace/CodexContractsStarter.cs b/DistTestCore/Marketplace/CodexContractsStarter.cs index a6539bb..c013618 100644 --- a/DistTestCore/Marketplace/CodexContractsStarter.cs +++ b/DistTestCore/Marketplace/CodexContractsStarter.cs @@ -15,7 +15,7 @@ namespace DistTestCore.Marketplace this.workflowCreator = workflowCreator; } - public void Start(RunningContainer bootstrapContainer) + public MarketplaceInfo Start(RunningContainer bootstrapContainer) { var workflow = workflowCreator.CreateWorkflow(); var startupConfig = CreateStartupConfig(bootstrapContainer); @@ -32,7 +32,12 @@ namespace DistTestCore.Marketplace return logHandler.Found; }); + var extractor = new ContainerInfoExtractor(workflow, container); + var marketplaceAddress = extractor.ExtractMarketplaceAddress(); + lifecycle.Log.Log("Contracts deployed."); + + return new MarketplaceInfo(marketplaceAddress); } private void WaitUntil(Func predicate) @@ -49,6 +54,16 @@ namespace DistTestCore.Marketplace } } + public class MarketplaceInfo + { + public MarketplaceInfo(string address) + { + Address = address; + } + + public string Address { get; } + } + public class ContractsReadyLogHandler : LogHandler { private readonly string targetString; diff --git a/DistTestCore/Marketplace/GethInfoExtractor.cs b/DistTestCore/Marketplace/ContainerInfoExtractor.cs similarity index 76% rename from DistTestCore/Marketplace/GethInfoExtractor.cs rename to DistTestCore/Marketplace/ContainerInfoExtractor.cs index 7151a60..0dc5320 100644 --- a/DistTestCore/Marketplace/GethInfoExtractor.cs +++ b/DistTestCore/Marketplace/ContainerInfoExtractor.cs @@ -1,14 +1,15 @@ using KubernetesWorkflow; +using Newtonsoft.Json; using System.Text; namespace DistTestCore.Marketplace { - public class GethInfoExtractor + public class ContainerInfoExtractor { private readonly StartupWorkflow workflow; private readonly RunningContainer container; - public GethInfoExtractor(StartupWorkflow workflow, RunningContainer container) + public ContainerInfoExtractor(StartupWorkflow workflow, RunningContainer container) { this.workflow = workflow; this.container = container; @@ -38,6 +39,14 @@ namespace DistTestCore.Marketplace return pubKey; } + public string ExtractMarketplaceAddress() + { + var marketplaceAddress = Retry(FetchMarketplaceAddress); + if (string.IsNullOrEmpty(marketplaceAddress)) throw new InvalidOperationException("Unable to fetch marketplace account from codex-contracts node. Test infra failure."); + + return marketplaceAddress; + } + private string Retry(Func fetch) { var result = Catch(fetch); @@ -71,6 +80,13 @@ namespace DistTestCore.Marketplace return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.AccountFilename); } + private string FetchMarketplaceAddress() + { + var json = workflow.ExecuteCommand(container, "cat", CodexContractsContainerRecipe.MarketplaceAddressFilename); + var marketplace = JsonConvert.DeserializeObject(json); + return marketplace!.address; + } + private string FetchPubKey() { var enodeFinder = new PubKeyFinder(); @@ -107,4 +123,9 @@ namespace DistTestCore.Marketplace length: closeIndex - openIndex); } } + + public class MarketplaceJson + { + public string address { get; set; } = string.Empty; + } } diff --git a/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs b/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs index 46189c4..be73d36 100644 --- a/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs +++ b/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs @@ -1,6 +1,4 @@ using KubernetesWorkflow; -using Logging; -using NethereumWorkflow; namespace DistTestCore.Marketplace { @@ -20,14 +18,5 @@ namespace DistTestCore.Marketplace public string GenesisJsonBase64 { get; } public string PubKey { get; } public Port DiscoveryPort { get; } - - public NethereumInteraction StartInteraction(TestLog log) - { - var ip = RunningContainers.RunningPod.Cluster.IP; - var port = RunningContainers.Containers[0].ServicePorts[0].Number; - - var creator = new NethereumInteractionCreator(log, ip, port, Account); - return creator.CreateWorkflow(); - } } } diff --git a/DistTestCore/Marketplace/GethBootstrapNodeStarter.cs b/DistTestCore/Marketplace/GethBootstrapNodeStarter.cs index bc5d4aa..293555f 100644 --- a/DistTestCore/Marketplace/GethBootstrapNodeStarter.cs +++ b/DistTestCore/Marketplace/GethBootstrapNodeStarter.cs @@ -7,13 +7,11 @@ namespace DistTestCore.Marketplace private const string bootstrapGenesisJsonBase64 = "ewogICAgImNvbmZpZyI6IHsKICAgICAgImNoYWluSWQiOiA3ODk5ODgsCiAgICAgICJob21lc3RlYWRCbG9jayI6IDAsCiAgICAgICJlaXAxNTBCbG9jayI6IDAsCiAgICAgICJlaXAxNTVCbG9jayI6IDAsCiAgICAgICJlaXAxNThCbG9jayI6IDAsCiAgICAgICJieXphbnRpdW1CbG9jayI6IDAsCiAgICAgICJjb25zdGFudGlub3BsZUJsb2NrIjogMCwKICAgICAgInBldGVyc2J1cmdCbG9jayI6IDAsCiAgICAgICJpc3RhbmJ1bEJsb2NrIjogMCwKICAgICAgIm11aXJHbGFjaWVyQmxvY2siOiAwLAogICAgICAiYmVybGluQmxvY2siOiAwLAogICAgICAibG9uZG9uQmxvY2siOiAwLAogICAgICAiYXJyb3dHbGFjaWVyQmxvY2siOiAwLAogICAgICAiZ3JheUdsYWNpZXJCbG9jayI6IDAsCiAgICAgICJjbGlxdWUiOiB7CiAgICAgICAgInBlcmlvZCI6IDUsCiAgICAgICAgImVwb2NoIjogMzAwMDAKICAgICAgfQogICAgfSwKICAgICJkaWZmaWN1bHR5IjogIjEiLAogICAgImdhc0xpbWl0IjogIjgwMDAwMDAwMCIsCiAgICAiZXh0cmFkYXRhIjogIjB4MDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMEFDQ09VTlRfSEVSRTAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAiLAogICAgImFsbG9jIjogewogICAgICAiMHhBQ0NPVU5UX0hFUkUiOiB7ICJiYWxhbmNlIjogIjUwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAiIH0KICAgIH0KICB9"; private readonly TestLifecycle lifecycle; private readonly WorkflowCreator workflowCreator; - private readonly CodexContractsStarter codexContractsStarter; public GethBootstrapNodeStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator) { this.lifecycle = lifecycle; this.workflowCreator = workflowCreator; - codexContractsStarter = new CodexContractsStarter(lifecycle, workflowCreator); } public GethBootstrapNodeInfo StartGethBootstrapNode() @@ -26,7 +24,7 @@ namespace DistTestCore.Marketplace if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 Geth bootstrap node to be created. Test infra failure."); var bootstrapContainer = containers.Containers[0]; - var extractor = new GethInfoExtractor(workflow, bootstrapContainer); + var extractor = new ContainerInfoExtractor(workflow, bootstrapContainer); var account = extractor.ExtractAccount(); var genesisJsonBase64 = extractor.ExtractGenesisJsonBase64(); var pubKey = extractor.ExtractPubKey(); @@ -34,16 +32,9 @@ namespace DistTestCore.Marketplace Log($"Geth bootstrap node started with account '{account}'"); - DeployCodexContracts(bootstrapContainer); - return new GethBootstrapNodeInfo(containers, account, genesisJsonBase64, pubKey, discoveryPort); } - private void DeployCodexContracts(RunningContainer bootstrapContainer) - { - codexContractsStarter.Start(bootstrapContainer); - } - private StartupConfig CreateBootstrapStartupConfig() { var config = new StartupConfig(); diff --git a/DistTestCore/Marketplace/GethCompanionNodeStarter.cs b/DistTestCore/Marketplace/GethCompanionNodeStarter.cs index c746bf1..20844d2 100644 --- a/DistTestCore/Marketplace/GethCompanionNodeStarter.cs +++ b/DistTestCore/Marketplace/GethCompanionNodeStarter.cs @@ -30,7 +30,7 @@ namespace DistTestCore.Marketplace private GethCompanionNodeInfo CreateCompanionInfo(StartupWorkflow workflow, RunningContainer container) { - var extractor = new GethInfoExtractor(workflow, container); + var extractor = new ContainerInfoExtractor(workflow, container); var account = extractor.ExtractAccount(); return new GethCompanionNodeInfo(container, account); } diff --git a/DistTestCore/Marketplace/GethStartResult.cs b/DistTestCore/Marketplace/GethStartResult.cs index f1d9c97..2f0d24d 100644 --- a/DistTestCore/Marketplace/GethStartResult.cs +++ b/DistTestCore/Marketplace/GethStartResult.cs @@ -2,15 +2,15 @@ { public class GethStartResult { - public GethStartResult(IMarketplaceAccessFactory marketplaceAccessFactory, GethBootstrapNodeInfo bootstrapNode, GethCompanionNodeInfo[] companionNodes) + public GethStartResult(IMarketplaceAccessFactory marketplaceAccessFactory, MarketplaceNetwork marketplaceNetwork, GethCompanionNodeInfo[] companionNodes) { MarketplaceAccessFactory = marketplaceAccessFactory; - BootstrapNode = bootstrapNode; + MarketplaceNetwork = marketplaceNetwork; CompanionNodes = companionNodes; } public IMarketplaceAccessFactory MarketplaceAccessFactory { get; } - public GethBootstrapNodeInfo BootstrapNode { get; } + public MarketplaceNetwork MarketplaceNetwork { get; } public GethCompanionNodeInfo[] CompanionNodes { get; } } } diff --git a/DistTestCore/Marketplace/MarketplaceAccess.cs b/DistTestCore/Marketplace/MarketplaceAccess.cs index 8de63dd..40e8506 100644 --- a/DistTestCore/Marketplace/MarketplaceAccess.cs +++ b/DistTestCore/Marketplace/MarketplaceAccess.cs @@ -15,13 +15,13 @@ namespace DistTestCore.Marketplace public class MarketplaceAccess : IMarketplaceAccess { private readonly TestLog log; - private readonly GethBootstrapNodeInfo bootstrapNode; + private readonly MarketplaceNetwork marketplaceNetwork; private readonly GethCompanionNodeInfo companionNode; - public MarketplaceAccess(TestLog log, GethBootstrapNodeInfo bootstrapNode, GethCompanionNodeInfo companionNode) + public MarketplaceAccess(TestLog log, MarketplaceNetwork marketplaceNetwork, GethCompanionNodeInfo companionNode) { this.log = log; - this.bootstrapNode = bootstrapNode; + this.marketplaceNetwork = marketplaceNetwork; this.companionNode = companionNode; } @@ -42,7 +42,7 @@ namespace DistTestCore.Marketplace public decimal GetBalance() { - var interaction = bootstrapNode.StartInteraction(log); + var interaction = marketplaceNetwork.StartInteraction(log); return interaction.GetBalance(companionNode.Account); } } diff --git a/DistTestCore/Marketplace/MarketplaceAccessFactory.cs b/DistTestCore/Marketplace/MarketplaceAccessFactory.cs index fba1155..cf268ae 100644 --- a/DistTestCore/Marketplace/MarketplaceAccessFactory.cs +++ b/DistTestCore/Marketplace/MarketplaceAccessFactory.cs @@ -19,18 +19,18 @@ namespace DistTestCore.Marketplace public class GethMarketplaceAccessFactory : IMarketplaceAccessFactory { private readonly TestLog log; - private readonly GethBootstrapNodeInfo bootstrapNode; + private readonly MarketplaceNetwork marketplaceNetwork; - public GethMarketplaceAccessFactory(TestLog log, GethBootstrapNodeInfo bootstrapNode) + public GethMarketplaceAccessFactory(TestLog log, MarketplaceNetwork marketplaceNetwork) { this.log = log; - this.bootstrapNode = bootstrapNode; + this.marketplaceNetwork = marketplaceNetwork; } public IMarketplaceAccess CreateMarketplaceAccess(CodexAccess access) { var companionNode = GetGethCompanionNode(access); - return new MarketplaceAccess(log, bootstrapNode, companionNode); + return new MarketplaceAccess(log, marketplaceNetwork, companionNode); } private GethCompanionNodeInfo GetGethCompanionNode(CodexAccess access) diff --git a/DistTestCore/Marketplace/MarketplaceInitialConfig.cs b/DistTestCore/Marketplace/MarketplaceInitialConfig.cs index a815842..1b66199 100644 --- a/DistTestCore/Marketplace/MarketplaceInitialConfig.cs +++ b/DistTestCore/Marketplace/MarketplaceInitialConfig.cs @@ -2,11 +2,13 @@ { public class MarketplaceInitialConfig { - public MarketplaceInitialConfig(int initialBalance) + public MarketplaceInitialConfig(Ether initialEth, TestToken initialTestTokens) { - InitialBalance = initialBalance; + InitialEth = initialEth; + InitialTestTokens = initialTestTokens; } - public int InitialBalance { get; } + public Ether InitialEth { get; } + public TestToken InitialTestTokens { get; } } } diff --git a/DistTestCore/Marketplace/MarketplaceNetwork.cs b/DistTestCore/Marketplace/MarketplaceNetwork.cs new file mode 100644 index 0000000..627e6a0 --- /dev/null +++ b/DistTestCore/Marketplace/MarketplaceNetwork.cs @@ -0,0 +1,27 @@ +using Logging; +using NethereumWorkflow; + +namespace DistTestCore.Marketplace +{ + public class MarketplaceNetwork + { + public MarketplaceNetwork(GethBootstrapNodeInfo bootstrap, MarketplaceInfo marketplace) + { + Bootstrap = bootstrap; + Marketplace = marketplace; + } + + public GethBootstrapNodeInfo Bootstrap { get; } + public MarketplaceInfo Marketplace { get; } + + public NethereumInteraction StartInteraction(TestLog log) + { + var ip = Bootstrap.RunningContainers.RunningPod.Cluster.IP; + var port = Bootstrap.RunningContainers.Containers[0].ServicePorts[0].Number; + var account = Bootstrap.Account; + + var creator = new NethereumInteractionCreator(log, ip, port, account); + return creator.CreateWorkflow(); + } + } +} diff --git a/DistTestCore/Tokens.cs b/DistTestCore/Tokens.cs new file mode 100644 index 0000000..4669660 --- /dev/null +++ b/DistTestCore/Tokens.cs @@ -0,0 +1,57 @@ +namespace DistTestCore +{ + public class Ether + { + public Ether(decimal wei) + { + Wei = wei; + } + + public decimal Wei { get; } + } + + public class TestToken + { + public TestToken(decimal amount) + { + Amount = amount; + } + + public decimal Amount { get; } + } + + public static class TokensIntExtensions + { + private const decimal weiPerEth = 1000000000000000000; + + public static TestToken TestTokens(this int i) + { + return TestTokens(Convert.ToDecimal(i)); + } + + public static TestToken TestTokens(this decimal i) + { + return new TestToken(i); + } + + public static Ether Eth(this int i) + { + return Eth(Convert.ToDecimal(i)); + } + + public static Ether Wei(this int i) + { + return Wei(Convert.ToDecimal(i)); + } + + public static Ether Eth(this decimal i) + { + return new Ether(i * weiPerEth); + } + + public static Ether Wei(this decimal i) + { + return new Ether(i); + } + } +} diff --git a/Nethereum/NethereumInteraction.cs b/Nethereum/NethereumInteraction.cs index 832f32f..a135bd8 100644 --- a/Nethereum/NethereumInteraction.cs +++ b/Nethereum/NethereumInteraction.cs @@ -30,6 +30,14 @@ namespace NethereumWorkflow Log($"Transferred {amount} to {account}"); } + public void MintTestTokens(string account, decimal amount) + { + if (amount < 1 || string.IsNullOrEmpty(account)) throw new ArgumentException("Invalid arguments for MintTestTokens"); + + // - address of token? -> address of marketplace? + // - nethereum do contract call + } + public decimal GetBalance(string account) { var bigInt = Time.Wait(web3.Eth.GetBalance.SendRequestAsync(account)); diff --git a/Tests/BasicTests/ExampleTests.cs b/Tests/BasicTests/ExampleTests.cs index b70f3c4..01eee02 100644 --- a/Tests/BasicTests/ExampleTests.cs +++ b/Tests/BasicTests/ExampleTests.cs @@ -51,7 +51,7 @@ namespace Tests.BasicTests { var group = SetupCodexNodes(2) .WithStorageQuota(10.GB()) - .EnableMarketplace(initialBalance: 20) + .EnableMarketplace(20.TestTokens()) .BringOnline(); foreach (var node in group) From 98f5e481d17eeae8a7a149f22a3a2f265030a2cc Mon Sep 17 00:00:00 2001 From: benbierens Date: Tue, 18 Apr 2023 13:22:41 +0200 Subject: [PATCH 39/46] We mint test tokens! --- DistTestCore/BaseStarter.cs | 39 ++++++++++++++++ DistTestCore/CodexStarter.cs | 2 +- DistTestCore/GethStarter.cs | 8 ++-- .../Marketplace/CodexContractsStarter.cs | 12 ++--- .../Marketplace/ContainerInfoExtractor.cs | 13 ++++++ .../Marketplace/GethBootstrapNodeInfo.cs | 4 +- .../Marketplace/GethBootstrapNodeStarter.cs | 19 +++----- .../Marketplace/GethContainerRecipe.cs | 3 +- .../Marketplace/MarketplaceNetwork.cs | 3 +- DistTestCore/PrometheusStarter.cs | 2 +- DistTestCore/Stopwatch.cs | 25 ++++++++-- Nethereum/NethereumInteraction.cs | 46 +++++++++++++++++-- Nethereum/NethereumInteractionCreator.cs | 8 ++-- 13 files changed, 145 insertions(+), 39 deletions(-) create mode 100644 DistTestCore/BaseStarter.cs diff --git a/DistTestCore/BaseStarter.cs b/DistTestCore/BaseStarter.cs new file mode 100644 index 0000000..e4f7fde --- /dev/null +++ b/DistTestCore/BaseStarter.cs @@ -0,0 +1,39 @@ +using KubernetesWorkflow; + +namespace DistTestCore +{ + public class BaseStarter + { + protected readonly TestLifecycle lifecycle; + protected readonly WorkflowCreator workflowCreator; + private Stopwatch? stopwatch; + + public BaseStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator) + { + this.lifecycle = lifecycle; + this.workflowCreator = workflowCreator; + } + + protected void LogStart(string msg) + { + Log(msg); + stopwatch = Stopwatch.Begin(lifecycle.Log, GetClassName()); + } + + protected void LogEnd(string msg) + { + stopwatch!.End(msg); + stopwatch = null; + } + + protected void Log(string msg) + { + lifecycle.Log.Log($"{GetClassName} {msg}"); + } + + private string GetClassName() + { + return GetType().Name; + } + } +} diff --git a/DistTestCore/CodexStarter.cs b/DistTestCore/CodexStarter.cs index e90c0f0..9dead26 100644 --- a/DistTestCore/CodexStarter.cs +++ b/DistTestCore/CodexStarter.cs @@ -3,7 +3,7 @@ using KubernetesWorkflow; namespace DistTestCore { - public class CodexStarter + public class CodexStarter // basestarter { private readonly TestLifecycle lifecycle; private readonly WorkflowCreator workflowCreator; diff --git a/DistTestCore/GethStarter.cs b/DistTestCore/GethStarter.cs index dbb19f9..0618078 100644 --- a/DistTestCore/GethStarter.cs +++ b/DistTestCore/GethStarter.cs @@ -3,7 +3,7 @@ using KubernetesWorkflow; namespace DistTestCore { - public class GethStarter + public class GethStarter // basestarter { private readonly MarketplaceNetworkCache marketplaceNetworkCache; private readonly GethCompanionNodeStarter companionNodeStarter; @@ -36,8 +36,10 @@ namespace DistTestCore foreach (var node in companionNodes) { interaction.TransferTo(node.Account, marketplaceConfig.InitialEth.Wei); - // wrong level: mintTestTokens? interactions knows nothing about contract details! - //interaction.MintTestTokens(node.Account, marketplaceConfig.InitialTestTokens.Amount); + + var tokenAddress = interaction.GetTokenAddress(marketplaceNetwork.Marketplace.Address); + + interaction.MintTestTokens(node.Account, marketplaceConfig.InitialTestTokens.Amount, tokenAddress); } } diff --git a/DistTestCore/Marketplace/CodexContractsStarter.cs b/DistTestCore/Marketplace/CodexContractsStarter.cs index c013618..1be0df3 100644 --- a/DistTestCore/Marketplace/CodexContractsStarter.cs +++ b/DistTestCore/Marketplace/CodexContractsStarter.cs @@ -3,24 +3,22 @@ using Utils; namespace DistTestCore.Marketplace { - public class CodexContractsStarter + public class CodexContractsStarter : BaseStarter { private const string readyString = "Done! Sleeping indefinitely..."; - private readonly TestLifecycle lifecycle; - private readonly WorkflowCreator workflowCreator; public CodexContractsStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator) + : base(lifecycle, workflowCreator) { - this.lifecycle = lifecycle; - this.workflowCreator = workflowCreator; } public MarketplaceInfo Start(RunningContainer bootstrapContainer) { + LogStart("Deploying Codex contracts..."); + var workflow = workflowCreator.CreateWorkflow(); var startupConfig = CreateStartupConfig(bootstrapContainer); - lifecycle.Log.Log("Deploying Codex contracts..."); var containers = workflow.Start(1, Location.Unspecified, new CodexContractsContainerRecipe(), startupConfig); if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 Codex contracts container to be created. Test infra failure."); var container = containers.Containers[0]; @@ -35,7 +33,7 @@ namespace DistTestCore.Marketplace var extractor = new ContainerInfoExtractor(workflow, container); var marketplaceAddress = extractor.ExtractMarketplaceAddress(); - lifecycle.Log.Log("Contracts deployed."); + LogEnd("Contracts deployed."); return new MarketplaceInfo(marketplaceAddress); } diff --git a/DistTestCore/Marketplace/ContainerInfoExtractor.cs b/DistTestCore/Marketplace/ContainerInfoExtractor.cs index 0dc5320..7a9105b 100644 --- a/DistTestCore/Marketplace/ContainerInfoExtractor.cs +++ b/DistTestCore/Marketplace/ContainerInfoExtractor.cs @@ -39,6 +39,14 @@ namespace DistTestCore.Marketplace return pubKey; } + public string ExtractBootstrapPrivateKey() + { + var privKey = Retry(FetchBootstrapPrivateKey); + if (string.IsNullOrEmpty(privKey)) throw new InvalidOperationException("Unable to fetch private key from geth node. Test infra failure."); + + return privKey; + } + public string ExtractMarketplaceAddress() { var marketplaceAddress = Retry(FetchMarketplaceAddress); @@ -80,6 +88,11 @@ namespace DistTestCore.Marketplace return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.AccountFilename); } + private string FetchBootstrapPrivateKey() + { + return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.BootstrapPrivateKeyFilename); + } + private string FetchMarketplaceAddress() { var json = workflow.ExecuteCommand(container, "cat", CodexContractsContainerRecipe.MarketplaceAddressFilename); diff --git a/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs b/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs index be73d36..ab4196f 100644 --- a/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs +++ b/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs @@ -4,12 +4,13 @@ namespace DistTestCore.Marketplace { public class GethBootstrapNodeInfo { - public GethBootstrapNodeInfo(RunningContainers runningContainers, string account, string genesisJsonBase64, string pubKey, Port discoveryPort) + public GethBootstrapNodeInfo(RunningContainers runningContainers, string account, string genesisJsonBase64, string pubKey, string privateKey, Port discoveryPort) { RunningContainers = runningContainers; Account = account; GenesisJsonBase64 = genesisJsonBase64; PubKey = pubKey; + PrivateKey = privateKey; DiscoveryPort = discoveryPort; } @@ -17,6 +18,7 @@ namespace DistTestCore.Marketplace public string Account { get; } public string GenesisJsonBase64 { get; } public string PubKey { get; } + public string PrivateKey { get; } public Port DiscoveryPort { get; } } } diff --git a/DistTestCore/Marketplace/GethBootstrapNodeStarter.cs b/DistTestCore/Marketplace/GethBootstrapNodeStarter.cs index 293555f..494940b 100644 --- a/DistTestCore/Marketplace/GethBootstrapNodeStarter.cs +++ b/DistTestCore/Marketplace/GethBootstrapNodeStarter.cs @@ -2,21 +2,18 @@ namespace DistTestCore.Marketplace { - public class GethBootstrapNodeStarter + public class GethBootstrapNodeStarter : BaseStarter { private const string bootstrapGenesisJsonBase64 = "ewogICAgImNvbmZpZyI6IHsKICAgICAgImNoYWluSWQiOiA3ODk5ODgsCiAgICAgICJob21lc3RlYWRCbG9jayI6IDAsCiAgICAgICJlaXAxNTBCbG9jayI6IDAsCiAgICAgICJlaXAxNTVCbG9jayI6IDAsCiAgICAgICJlaXAxNThCbG9jayI6IDAsCiAgICAgICJieXphbnRpdW1CbG9jayI6IDAsCiAgICAgICJjb25zdGFudGlub3BsZUJsb2NrIjogMCwKICAgICAgInBldGVyc2J1cmdCbG9jayI6IDAsCiAgICAgICJpc3RhbmJ1bEJsb2NrIjogMCwKICAgICAgIm11aXJHbGFjaWVyQmxvY2siOiAwLAogICAgICAiYmVybGluQmxvY2siOiAwLAogICAgICAibG9uZG9uQmxvY2siOiAwLAogICAgICAiYXJyb3dHbGFjaWVyQmxvY2siOiAwLAogICAgICAiZ3JheUdsYWNpZXJCbG9jayI6IDAsCiAgICAgICJjbGlxdWUiOiB7CiAgICAgICAgInBlcmlvZCI6IDUsCiAgICAgICAgImVwb2NoIjogMzAwMDAKICAgICAgfQogICAgfSwKICAgICJkaWZmaWN1bHR5IjogIjEiLAogICAgImdhc0xpbWl0IjogIjgwMDAwMDAwMCIsCiAgICAiZXh0cmFkYXRhIjogIjB4MDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMEFDQ09VTlRfSEVSRTAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAiLAogICAgImFsbG9jIjogewogICAgICAiMHhBQ0NPVU5UX0hFUkUiOiB7ICJiYWxhbmNlIjogIjUwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAiIH0KICAgIH0KICB9"; - private readonly TestLifecycle lifecycle; - private readonly WorkflowCreator workflowCreator; public GethBootstrapNodeStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator) + : base(lifecycle, workflowCreator) { - this.lifecycle = lifecycle; - this.workflowCreator = workflowCreator; } public GethBootstrapNodeInfo StartGethBootstrapNode() { - Log("Starting Geth bootstrap node..."); + LogStart("Starting Geth bootstrap node..."); var startupConfig = CreateBootstrapStartupConfig(); var workflow = workflowCreator.CreateWorkflow(); @@ -28,11 +25,12 @@ namespace DistTestCore.Marketplace var account = extractor.ExtractAccount(); var genesisJsonBase64 = extractor.ExtractGenesisJsonBase64(); var pubKey = extractor.ExtractPubKey(); + var privateKey = extractor.ExtractBootstrapPrivateKey(); var discoveryPort = bootstrapContainer.Recipe.GetPortByTag(GethContainerRecipe.DiscoveryPortTag); - Log($"Geth bootstrap node started with account '{account}'"); + LogEnd($"Geth bootstrap node started with account '{account}'"); - return new GethBootstrapNodeInfo(containers, account, genesisJsonBase64, pubKey, discoveryPort); + return new GethBootstrapNodeInfo(containers, account, genesisJsonBase64, pubKey, privateKey, discoveryPort); } private StartupConfig CreateBootstrapStartupConfig() @@ -41,10 +39,5 @@ namespace DistTestCore.Marketplace config.Add(new GethStartupConfig(true, bootstrapGenesisJsonBase64, null!)); return config; } - - private void Log(string msg) - { - lifecycle.Log.Log(msg); - } } } diff --git a/DistTestCore/Marketplace/GethContainerRecipe.cs b/DistTestCore/Marketplace/GethContainerRecipe.cs index 3cb7d41..959daf8 100644 --- a/DistTestCore/Marketplace/GethContainerRecipe.cs +++ b/DistTestCore/Marketplace/GethContainerRecipe.cs @@ -9,7 +9,8 @@ namespace DistTestCore.Marketplace public const string DiscoveryPortTag = "disc_port"; public const string AccountFilename = "account_string.txt"; public const string GenesisFilename = "genesis.json"; - + public const string BootstrapPrivateKeyFilename = "bootstrap_private.key"; + protected override string Image => DockerImage; protected override void Initialize(StartupConfig startupConfig) diff --git a/DistTestCore/Marketplace/MarketplaceNetwork.cs b/DistTestCore/Marketplace/MarketplaceNetwork.cs index 627e6a0..7d27372 100644 --- a/DistTestCore/Marketplace/MarketplaceNetwork.cs +++ b/DistTestCore/Marketplace/MarketplaceNetwork.cs @@ -19,8 +19,9 @@ namespace DistTestCore.Marketplace var ip = Bootstrap.RunningContainers.RunningPod.Cluster.IP; var port = Bootstrap.RunningContainers.Containers[0].ServicePorts[0].Number; var account = Bootstrap.Account; + var privateKey = Bootstrap.PrivateKey; - var creator = new NethereumInteractionCreator(log, ip, port, account); + var creator = new NethereumInteractionCreator(log, ip, port, account, privateKey); return creator.CreateWorkflow(); } } diff --git a/DistTestCore/PrometheusStarter.cs b/DistTestCore/PrometheusStarter.cs index a58b3e4..bd375b8 100644 --- a/DistTestCore/PrometheusStarter.cs +++ b/DistTestCore/PrometheusStarter.cs @@ -5,7 +5,7 @@ using System.Text; namespace DistTestCore { - public class PrometheusStarter + public class PrometheusStarter // basestarter { private readonly TestLifecycle lifecycle; private readonly WorkflowCreator workflowCreator; diff --git a/DistTestCore/Stopwatch.cs b/DistTestCore/Stopwatch.cs index 5be09c0..878912f 100644 --- a/DistTestCore/Stopwatch.cs +++ b/DistTestCore/Stopwatch.cs @@ -5,15 +5,32 @@ namespace DistTestCore { public class Stopwatch { + private readonly DateTime start = DateTime.UtcNow; + private readonly BaseLog log; + private readonly string name; + + public Stopwatch(BaseLog log, string name) + { + this.log = log; + this.name = name; + } + public static void Measure(BaseLog log, string name, Action action) { - var start = DateTime.UtcNow; - + var sw = Begin(log, name); action(); + sw.End(); + } + public static Stopwatch Begin(BaseLog log, string name) + { + return new Stopwatch(log, name); + } + + public void End(string msg = "") + { var duration = DateTime.UtcNow - start; - - log.Log($"{name} ({Time.FormatDuration(duration)})"); + log.Log($"{name} {msg} ({Time.FormatDuration(duration)})"); } } } diff --git a/Nethereum/NethereumInteraction.cs b/Nethereum/NethereumInteraction.cs index a135bd8..9092524 100644 --- a/Nethereum/NethereumInteraction.cs +++ b/Nethereum/NethereumInteraction.cs @@ -1,4 +1,6 @@ using Logging; +using Nethereum.ABI.FunctionEncoding.Attributes; +using Nethereum.Contracts; using Nethereum.Hex.HexTypes; using Nethereum.Web3; using System.Numerics; @@ -30,12 +32,28 @@ namespace NethereumWorkflow Log($"Transferred {amount} to {account}"); } - public void MintTestTokens(string account, decimal amount) + public string GetTokenAddress(string marketplaceAddress) + { + var function = new GetTokenFunction(); + + var handler = web3.Eth.GetContractQueryHandler(); + var result = Time.Wait(handler.QueryAsync(marketplaceAddress, function)); + + return result; + } + + public void MintTestTokens(string account, decimal amount, string tokenAddress) { if (amount < 1 || string.IsNullOrEmpty(account)) throw new ArgumentException("Invalid arguments for MintTestTokens"); - // - address of token? -> address of marketplace? - // - nethereum do contract call + var function = new MintTokensFunction + { + Holder = account, + Amount = ToBig(amount) + }; + + var handler = web3.Eth.GetContractTransactionHandler(); + Time.Wait(handler.SendRequestAndWaitForReceiptAsync(tokenAddress, function)); } public decimal GetBalance(string account) @@ -48,11 +66,16 @@ namespace NethereumWorkflow private HexBigInteger ToHexBig(decimal amount) { - var bigint = new BigInteger(amount); + var bigint = ToBig(amount); var str = bigint.ToString("X"); return new HexBigInteger(str); } + private BigInteger ToBig(decimal amount) + { + return new BigInteger(amount); + } + private decimal ToDecimal(HexBigInteger hexBigInteger) { return (decimal)hexBigInteger.Value; @@ -63,4 +86,19 @@ namespace NethereumWorkflow log.Log(msg); } } + + [Function("token", "address")] + public class GetTokenFunction : FunctionMessage + { + } + + [Function("mint")] + public class MintTokensFunction : FunctionMessage + { + [Parameter("address", "holder", 1)] + public string Holder { get; set; } + + [Parameter("uint256", "amount", 2)] + public BigInteger Amount { get; set; } + } } diff --git a/Nethereum/NethereumInteractionCreator.cs b/Nethereum/NethereumInteractionCreator.cs index b0b41f7..3695fa1 100644 --- a/Nethereum/NethereumInteractionCreator.cs +++ b/Nethereum/NethereumInteractionCreator.cs @@ -9,13 +9,15 @@ namespace NethereumWorkflow private readonly string ip; private readonly int port; private readonly string rootAccount; + private readonly string privateKey; - public NethereumInteractionCreator(TestLog log, string ip, int port, string rootAccount) + public NethereumInteractionCreator(TestLog log, string ip, int port, string rootAccount, string privateKey) { this.log = log; this.ip = ip; this.port = port; this.rootAccount = rootAccount; + this.privateKey = privateKey; } public NethereumInteraction CreateWorkflow() @@ -25,8 +27,8 @@ namespace NethereumWorkflow private Web3 CreateWeb3() { - //var bootstrapaccount = new ManagedAccount(bootstrapInfo.Account, "qwerty!@#$%^"); - return new Web3($"http://{ip}:{port}"); + var account = new Nethereum.Web3.Accounts.Account(privateKey); + return new Web3(account, $"http://{ip}:{port}"); } } } From e36d910f2f40b658dcd42e0875429d5cd04c827b Mon Sep 17 00:00:00 2001 From: benbierens Date: Tue, 18 Apr 2023 13:45:48 +0200 Subject: [PATCH 40/46] Applies BaseStarter type --- DistTestCore/Codex/CodexContainerRecipe.cs | 11 +++++----- DistTestCore/CodexStarter.cs | 21 ++++++------------- DistTestCore/GethStarter.cs | 10 ++++----- .../Marketplace/GethCompanionNodeStarter.cs | 17 ++++----------- .../Marketplace/GethContainerRecipe.cs | 5 +++-- DistTestCore/PrometheusStarter.cs | 17 ++++----------- 6 files changed, 26 insertions(+), 55 deletions(-) diff --git a/DistTestCore/Codex/CodexContainerRecipe.cs b/DistTestCore/Codex/CodexContainerRecipe.cs index 50d6441..ab3dbee 100644 --- a/DistTestCore/Codex/CodexContainerRecipe.cs +++ b/DistTestCore/Codex/CodexContainerRecipe.cs @@ -41,13 +41,12 @@ namespace DistTestCore.Codex var companionNode = gethConfig.CompanionNodes[Index]; Additional(companionNode); - // Bootstrap node access from within the cluster: - //var ip = gethConfig.BootstrapNode.RunningContainers.RunningPod.Ip; - //var port = gethConfig.BootstrapNode.RunningContainers.Containers[0].Recipe.GetPortByTag(GethContainerRecipe.HttpPortTag); + var ip = companionNode.RunningContainer.Pod.Ip; + var port = companionNode.RunningContainer.Recipe.GetPortByTag(GethContainerRecipe.WsPortTag).Number; - //AddEnvVar("ETH_PROVIDER", "todo"); - //AddEnvVar("ETH_ACCOUNT", companionNode.Account); - //AddEnvVar("ETH_DEPLOYMENT", "todo"); + AddEnvVar("ETH_PROVIDER", $"ws://{ip}:{port}"); + AddEnvVar("ETH_ACCOUNT", companionNode.Account); + AddEnvVar("ETH_MARKETPLACE_ADDRESS", gethConfig.MarketplaceNetwork.Marketplace.Address); } } } diff --git a/DistTestCore/CodexStarter.cs b/DistTestCore/CodexStarter.cs index 9dead26..54342c7 100644 --- a/DistTestCore/CodexStarter.cs +++ b/DistTestCore/CodexStarter.cs @@ -3,22 +3,18 @@ using KubernetesWorkflow; namespace DistTestCore { - public class CodexStarter // basestarter + public class CodexStarter : BaseStarter { - private readonly TestLifecycle lifecycle; - private readonly WorkflowCreator workflowCreator; - public CodexStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator) + : base(lifecycle, workflowCreator) { - this.lifecycle = lifecycle; - this.workflowCreator = workflowCreator; } public List RunningGroups { get; } = new List(); public ICodexNodeGroup BringOnline(CodexSetup codexSetup) { - Log($"Starting {codexSetup.Describe()}..."); + LogStart($"Starting {codexSetup.Describe()}..."); var gethStartResult = lifecycle.GethStarter.BringOnlineMarketplaceFor(codexSetup); var startupConfig = new StartupConfig(); @@ -32,17 +28,17 @@ namespace DistTestCore var codexNodeFactory = new CodexNodeFactory(lifecycle, metricAccessFactory, gethStartResult.MarketplaceAccessFactory); var group = CreateCodexGroup(codexSetup, containers, codexNodeFactory); - Log($"Started at '{group.Containers.RunningPod.Ip}'"); + LogEnd($"Started at '{group.Containers.RunningPod.Ip}'"); return group; } public void BringOffline(CodexNodeGroup group) { - Log($"Stopping {group.Describe()}..."); + LogStart($"Stopping {group.Describe()}..."); var workflow = CreateWorkflow(); workflow.Stop(group.Containers); RunningGroups.Remove(group); - Log("Stopped."); + LogEnd("Stopped."); } public void DeleteAllResources() @@ -76,10 +72,5 @@ namespace DistTestCore { return workflowCreator.CreateWorkflow(); } - - private void Log(string msg) - { - lifecycle.Log.Log(msg); - } } } diff --git a/DistTestCore/GethStarter.cs b/DistTestCore/GethStarter.cs index 0618078..50a2d86 100644 --- a/DistTestCore/GethStarter.cs +++ b/DistTestCore/GethStarter.cs @@ -3,19 +3,18 @@ using KubernetesWorkflow; namespace DistTestCore { - public class GethStarter // basestarter + public class GethStarter : BaseStarter { private readonly MarketplaceNetworkCache marketplaceNetworkCache; private readonly GethCompanionNodeStarter companionNodeStarter; - private readonly TestLifecycle lifecycle; public GethStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator) + : base(lifecycle, workflowCreator) { marketplaceNetworkCache = new MarketplaceNetworkCache( new GethBootstrapNodeStarter(lifecycle, workflowCreator), new CodexContractsStarter(lifecycle, workflowCreator)); companionNodeStarter = new GethCompanionNodeStarter(lifecycle, workflowCreator); - this.lifecycle = lifecycle; } public GethStartResult BringOnlineMarketplaceFor(CodexSetup codexSetup) @@ -33,12 +32,11 @@ namespace DistTestCore private void TransferInitialBalance(MarketplaceNetwork marketplaceNetwork, MarketplaceInitialConfig marketplaceConfig, GethCompanionNodeInfo[] companionNodes) { var interaction = marketplaceNetwork.StartInteraction(lifecycle.Log); + var tokenAddress = interaction.GetTokenAddress(marketplaceNetwork.Marketplace.Address); + foreach (var node in companionNodes) { interaction.TransferTo(node.Account, marketplaceConfig.InitialEth.Wei); - - var tokenAddress = interaction.GetTokenAddress(marketplaceNetwork.Marketplace.Address); - interaction.MintTestTokens(node.Account, marketplaceConfig.InitialTestTokens.Amount, tokenAddress); } } diff --git a/DistTestCore/Marketplace/GethCompanionNodeStarter.cs b/DistTestCore/Marketplace/GethCompanionNodeStarter.cs index 20844d2..7e20c4b 100644 --- a/DistTestCore/Marketplace/GethCompanionNodeStarter.cs +++ b/DistTestCore/Marketplace/GethCompanionNodeStarter.cs @@ -2,20 +2,16 @@ namespace DistTestCore.Marketplace { - public class GethCompanionNodeStarter + public class GethCompanionNodeStarter : BaseStarter { - private readonly TestLifecycle lifecycle; - private readonly WorkflowCreator workflowCreator; - public GethCompanionNodeStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator) + : base(lifecycle, workflowCreator) { - this.lifecycle = lifecycle; - this.workflowCreator = workflowCreator; } public GethCompanionNodeInfo[] StartCompanionNodesFor(CodexSetup codexSetup, GethBootstrapNodeInfo bootstrapNode) { - Log($"Initializing companions for {codexSetup.NumberOfNodes} Codex nodes."); + LogStart($"Initializing companions for {codexSetup.NumberOfNodes} Codex nodes."); var startupConfig = CreateCompanionNodeStartupConfig(bootstrapNode); @@ -23,7 +19,7 @@ namespace DistTestCore.Marketplace var containers = workflow.Start(codexSetup.NumberOfNodes, Location.Unspecified, new GethContainerRecipe(), startupConfig); if (containers.Containers.Length != codexSetup.NumberOfNodes) throw new InvalidOperationException("Expected a Geth companion node to be created for each Codex node. Test infra failure."); - Log("Initialized companion nodes."); + LogEnd("Initialized companion nodes."); return containers.Containers.Select(c => CreateCompanionInfo(workflow, c)).ToArray(); } @@ -41,10 +37,5 @@ namespace DistTestCore.Marketplace config.Add(new GethStartupConfig(false, bootstrapNode.GenesisJsonBase64, bootstrapNode)); return config; } - - private void Log(string msg) - { - lifecycle.Log.Log(msg); - } } } diff --git a/DistTestCore/Marketplace/GethContainerRecipe.cs b/DistTestCore/Marketplace/GethContainerRecipe.cs index 959daf8..6706901 100644 --- a/DistTestCore/Marketplace/GethContainerRecipe.cs +++ b/DistTestCore/Marketplace/GethContainerRecipe.cs @@ -6,6 +6,7 @@ namespace DistTestCore.Marketplace { public const string DockerImage = "thatbenbierens/geth-confenv:latest"; public const string HttpPortTag = "http_port"; + public const string WsPortTag = "ws_port"; public const string DiscoveryPortTag = "disc_port"; public const string AccountFilename = "account_string.txt"; public const string GenesisFilename = "genesis.json"; @@ -37,14 +38,14 @@ namespace DistTestCore.Marketplace var port = AddInternalPort(); var authRpc = AddInternalPort(); var httpPort = AddInternalPort(tag: HttpPortTag); + var wsPort = AddInternalPort(tag: WsPortTag); var bootPubKey = config.BootstrapNode.PubKey; var bootIp = config.BootstrapNode.RunningContainers.Containers[0].Pod.Ip; var bootPort = config.BootstrapNode.DiscoveryPort.Number; var bootstrapArg = $"--bootnodes enode://{bootPubKey}@{bootIp}:{bootPort}"; - // geth --bootnodes enode://pubkey1@ip1:port1 - return $"--port {port.Number} --discovery.port {discovery.Number} --authrpc.port {authRpc.Number} --http.port {httpPort.Number} --nodiscover {bootstrapArg}"; + return $"--port {port.Number} --discovery.port {discovery.Number} --authrpc.port {authRpc.Number} --http.port {httpPort.Number} --ws --ws.port {wsPort.Number} --nodiscover {bootstrapArg}"; } } } diff --git a/DistTestCore/PrometheusStarter.cs b/DistTestCore/PrometheusStarter.cs index bd375b8..936c876 100644 --- a/DistTestCore/PrometheusStarter.cs +++ b/DistTestCore/PrometheusStarter.cs @@ -5,22 +5,18 @@ using System.Text; namespace DistTestCore { - public class PrometheusStarter // basestarter + public class PrometheusStarter : BaseStarter { - private readonly TestLifecycle lifecycle; - private readonly WorkflowCreator workflowCreator; - public PrometheusStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator) + : base(lifecycle, workflowCreator) { - this.lifecycle = lifecycle; - this.workflowCreator = workflowCreator; } public IMetricsAccessFactory CollectMetricsFor(CodexSetup codexSetup, RunningContainers containers) { if (!codexSetup.MetricsEnabled) return new MetricsUnavailableAccessFactory(); - Log($"Starting metrics server for {containers.Describe()}"); + LogStart($"Starting metrics server for {containers.Describe()}"); var startupConfig = new StartupConfig(); startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(containers.Containers))); @@ -28,7 +24,7 @@ namespace DistTestCore var runningContainers = workflow.Start(1, Location.Unspecified, new PrometheusContainerRecipe(), startupConfig); if (runningContainers.Containers.Length != 1) throw new InvalidOperationException("Expected only 1 Prometheus container to be created."); - Log("Metrics server started."); + LogEnd("Metrics server started."); return new CodexNodeMetricsAccessFactory(runningContainers); } @@ -56,10 +52,5 @@ namespace DistTestCore var bytes = Encoding.ASCII.GetBytes(config); return Convert.ToBase64String(bytes); } - - private void Log(string msg) - { - lifecycle.Log.Log(msg); - } } } From 12d122ad83e21ca93c290a917453ccdfd9a5686d Mon Sep 17 00:00:00 2001 From: benbierens Date: Tue, 18 Apr 2023 15:33:12 +0200 Subject: [PATCH 41/46] Finishes implementation of marketplace support --- DistTestCore/Codex/CodexAccess.cs | 43 +++++++++++++ DistTestCore/GethStarter.cs | 4 +- DistTestCore/Http.cs | 14 +++++ .../Marketplace/CodexContractsStarter.cs | 13 ++-- .../Marketplace/GethBootstrapNodeInfo.cs | 13 ++++ DistTestCore/Marketplace/MarketplaceAccess.cs | 62 +++++++++++++++---- .../Marketplace/MarketplaceAccessFactory.cs | 2 +- .../Marketplace/MarketplaceNetwork.cs | 8 +-- Nethereum/NethereumInteraction.cs | 24 ++++++- Tests/BasicTests/ExampleTests.cs | 53 +++++++++------- 10 files changed, 185 insertions(+), 51 deletions(-) diff --git a/DistTestCore/Codex/CodexAccess.cs b/DistTestCore/Codex/CodexAccess.cs index c07a457..ae84a91 100644 --- a/DistTestCore/Codex/CodexAccess.cs +++ b/DistTestCore/Codex/CodexAccess.cs @@ -26,6 +26,16 @@ namespace DistTestCore.Codex return Http().HttpGetStream("download/" + contentId); } + public CodexSalesAvailabilityResponse SalesAvailability(CodexSalesAvailabilityRequest request) + { + return Http().HttpPostJson("sales/availability", request); + } + + public CodexSalesRequestStorageResponse RequestStorage(CodexSalesRequestStorageRequest request, string contentId) + { + return Http().HttpPostJson($"storage/request/{contentId}", request); + } + private Http Http() { var ip = Container.Pod.Cluster.IP; @@ -53,4 +63,37 @@ namespace DistTestCore.Codex public string version { get; set; } = string.Empty; public string revision { get; set; } = string.Empty; } + + public class CodexSalesAvailabilityRequest + { + public string size { get; set; } = string.Empty; + public string duration { get; set; } = string.Empty; + public string minPrice { get; set; } = string.Empty; + public string maxCollateral { get; set; } = string.Empty; + } + + public class CodexSalesAvailabilityResponse + { + public string id { get; set; } = string.Empty; + public string size { get; set; } = string.Empty; + public string duration { get; set; } = string.Empty; + public string minPrice { get; set; } = string.Empty; + public string maxCollateral { get; set; } = string.Empty; + } + + public class CodexSalesRequestStorageRequest + { + public string duration { get; set; } = string.Empty; + public string proofProbability { get; set; } = string.Empty; + public string reward { get; set; } = string.Empty; + public string collateral { get; set; } = string.Empty; + public string? expiry { get; set; } + public uint? nodes { get; set; } + public uint? tolerance { get; set;} + } + + public class CodexSalesRequestStorageResponse + { + public string purchaseId { get; set; } = string.Empty; + } } diff --git a/DistTestCore/GethStarter.cs b/DistTestCore/GethStarter.cs index 50a2d86..2b2cbb3 100644 --- a/DistTestCore/GethStarter.cs +++ b/DistTestCore/GethStarter.cs @@ -32,7 +32,7 @@ namespace DistTestCore private void TransferInitialBalance(MarketplaceNetwork marketplaceNetwork, MarketplaceInitialConfig marketplaceConfig, GethCompanionNodeInfo[] companionNodes) { var interaction = marketplaceNetwork.StartInteraction(lifecycle.Log); - var tokenAddress = interaction.GetTokenAddress(marketplaceNetwork.Marketplace.Address); + var tokenAddress = marketplaceNetwork.Marketplace.TokenAddress; foreach (var node in companionNodes) { @@ -79,7 +79,7 @@ namespace DistTestCore if (network == null) { var bootstrapInfo = bootstrapNodeStarter.StartGethBootstrapNode(); - var marketplaceInfo = codexContractsStarter.Start(bootstrapInfo.RunningContainers.Containers[0]); + var marketplaceInfo = codexContractsStarter.Start(bootstrapInfo); network = new MarketplaceNetwork(bootstrapInfo, marketplaceInfo ); } return network; diff --git a/DistTestCore/Http.cs b/DistTestCore/Http.cs index f75768b..0d242cf 100644 --- a/DistTestCore/Http.cs +++ b/DistTestCore/Http.cs @@ -1,6 +1,7 @@ using Newtonsoft.Json; using NUnit.Framework; using System.Net.Http.Headers; +using System.Net.Http.Json; using Utils; namespace DistTestCore @@ -37,6 +38,19 @@ namespace DistTestCore return JsonConvert.DeserializeObject(HttpGetString(route))!; } + public TResponse HttpPostJson(string route, TRequest body) + { + return Retry(() => + { + using var client = GetClient(); + var url = GetUrl() + route; + using var content = JsonContent.Create(body); + var result = Time.Wait(client.PostAsync(url, content)); + var json = Time.Wait(result.Content.ReadAsStringAsync()); + return JsonConvert.DeserializeObject(json)!; + }); + } + public string HttpPostStream(string route, Stream stream) { return Retry(() => diff --git a/DistTestCore/Marketplace/CodexContractsStarter.cs b/DistTestCore/Marketplace/CodexContractsStarter.cs index 1be0df3..841e830 100644 --- a/DistTestCore/Marketplace/CodexContractsStarter.cs +++ b/DistTestCore/Marketplace/CodexContractsStarter.cs @@ -12,12 +12,12 @@ namespace DistTestCore.Marketplace { } - public MarketplaceInfo Start(RunningContainer bootstrapContainer) + public MarketplaceInfo Start(GethBootstrapNodeInfo bootstrapNode) { LogStart("Deploying Codex contracts..."); var workflow = workflowCreator.CreateWorkflow(); - var startupConfig = CreateStartupConfig(bootstrapContainer); + var startupConfig = CreateStartupConfig(bootstrapNode.RunningContainers.Containers[0]); var containers = workflow.Start(1, Location.Unspecified, new CodexContractsContainerRecipe(), startupConfig); if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 Codex contracts container to be created. Test infra failure."); @@ -33,9 +33,12 @@ namespace DistTestCore.Marketplace var extractor = new ContainerInfoExtractor(workflow, container); var marketplaceAddress = extractor.ExtractMarketplaceAddress(); + var interaction = bootstrapNode.StartInteraction(lifecycle.Log); + var tokenAddress = interaction.GetTokenAddress(marketplaceAddress); + LogEnd("Contracts deployed."); - return new MarketplaceInfo(marketplaceAddress); + return new MarketplaceInfo(marketplaceAddress, tokenAddress); } private void WaitUntil(Func predicate) @@ -54,12 +57,14 @@ namespace DistTestCore.Marketplace public class MarketplaceInfo { - public MarketplaceInfo(string address) + public MarketplaceInfo(string address, string tokenAddress) { Address = address; + TokenAddress = tokenAddress; } public string Address { get; } + public string TokenAddress { get; } } public class ContractsReadyLogHandler : LogHandler diff --git a/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs b/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs index ab4196f..68c7485 100644 --- a/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs +++ b/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs @@ -1,4 +1,6 @@ using KubernetesWorkflow; +using Logging; +using NethereumWorkflow; namespace DistTestCore.Marketplace { @@ -20,5 +22,16 @@ namespace DistTestCore.Marketplace public string PubKey { get; } public string PrivateKey { get; } public Port DiscoveryPort { get; } + + public NethereumInteraction StartInteraction(TestLog log) + { + var ip = RunningContainers.RunningPod.Cluster.IP; + var port = RunningContainers.Containers[0].ServicePorts[0].Number; + var account = Account; + var privateKey = PrivateKey; + + var creator = new NethereumInteractionCreator(log, ip, port, account, privateKey); + return creator.CreateWorkflow(); + } } } diff --git a/DistTestCore/Marketplace/MarketplaceAccess.cs b/DistTestCore/Marketplace/MarketplaceAccess.cs index 40e8506..dd784c9 100644 --- a/DistTestCore/Marketplace/MarketplaceAccess.cs +++ b/DistTestCore/Marketplace/MarketplaceAccess.cs @@ -1,13 +1,15 @@ -using Logging; +using DistTestCore.Codex; +using Logging; using NUnit.Framework; using NUnit.Framework.Constraints; +using System.Numerics; namespace DistTestCore.Marketplace { public interface IMarketplaceAccess { - void MakeStorageAvailable(ByteSize size, int minPricePerBytePerSecond, float maxCollateral); - void RequestStorage(ContentId contentId, int pricePerBytePerSecond, float requiredCollateral, float minRequiredNumberOfNodes); + string MakeStorageAvailable(ByteSize size, TestToken minPricePerBytePerSecond, TestToken maxCollateral, TimeSpan maxDuration); + string RequestStorage(ContentId contentId, TestToken pricePerBytePerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration); void AssertThatBalance(IResolveConstraint constraint, string message = ""); decimal GetBalance(); } @@ -17,22 +19,58 @@ namespace DistTestCore.Marketplace private readonly TestLog log; private readonly MarketplaceNetwork marketplaceNetwork; private readonly GethCompanionNodeInfo companionNode; + private readonly CodexAccess codexAccess; - public MarketplaceAccess(TestLog log, MarketplaceNetwork marketplaceNetwork, GethCompanionNodeInfo companionNode) + public MarketplaceAccess(TestLog log, MarketplaceNetwork marketplaceNetwork, GethCompanionNodeInfo companionNode, CodexAccess codexAccess) { this.log = log; this.marketplaceNetwork = marketplaceNetwork; this.companionNode = companionNode; + this.codexAccess = codexAccess; } - public void RequestStorage(ContentId contentId, int pricePerBytePerSecond, float requiredCollateral, float minRequiredNumberOfNodes) + public string RequestStorage(ContentId contentId, TestToken pricePerBytePerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration) { - throw new NotImplementedException(); + var request = new CodexSalesRequestStorageRequest + { + duration = ToHexBigInt(duration.TotalSeconds), + proofProbability = ToHexBigInt(proofProbability), + reward = ToHexBigInt(pricePerBytePerSecond), + collateral = ToHexBigInt(requiredCollateral), + expiry = null, + nodes = minRequiredNumberOfNodes, + tolerance = null, + }; + + var response = codexAccess.RequestStorage(request, contentId.Id); + + return response.purchaseId; } - public void MakeStorageAvailable(ByteSize size, int minPricePerBytePerSecond, float maxCollateral) + public string MakeStorageAvailable(ByteSize size, TestToken minPricePerBytePerSecond, TestToken maxCollateral, TimeSpan duration) { - throw new NotImplementedException(); + var request = new CodexSalesAvailabilityRequest + { + size = ToHexBigInt(size.SizeInBytes), + duration = ToHexBigInt(duration.TotalSeconds), + maxCollateral = ToHexBigInt(maxCollateral), + minPrice = ToHexBigInt(minPricePerBytePerSecond) + }; + + var response = codexAccess.SalesAvailability(request); + + return response.id; + } + + private string ToHexBigInt(double d) + { + return "0x" + string.Format("{0:X}", Convert.ToInt64(d)); + } + + public string ToHexBigInt(TestToken t) + { + var bigInt = new BigInteger(t.Amount); + return "0x" + bigInt.ToString("X"); } public void AssertThatBalance(IResolveConstraint constraint, string message = "") @@ -43,20 +81,22 @@ namespace DistTestCore.Marketplace public decimal GetBalance() { var interaction = marketplaceNetwork.StartInteraction(log); - return interaction.GetBalance(companionNode.Account); + return interaction.GetBalance(marketplaceNetwork.Marketplace.TokenAddress, companionNode.Account); } } public class MarketplaceUnavailable : IMarketplaceAccess { - public void RequestStorage(ContentId contentId, int pricePerBytePerSecond, float requiredCollateral, float minRequiredNumberOfNodes) + public string RequestStorage(ContentId contentId, TestToken pricePerBytePerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration) { Unavailable(); + return string.Empty; } - public void MakeStorageAvailable(ByteSize size, int minPricePerBytePerSecond, float maxCollateral) + public string MakeStorageAvailable(ByteSize size, TestToken minPricePerBytePerSecond, TestToken maxCollateral, TimeSpan duration) { Unavailable(); + return string.Empty; } public void AssertThatBalance(IResolveConstraint constraint, string message = "") diff --git a/DistTestCore/Marketplace/MarketplaceAccessFactory.cs b/DistTestCore/Marketplace/MarketplaceAccessFactory.cs index cf268ae..b0389e1 100644 --- a/DistTestCore/Marketplace/MarketplaceAccessFactory.cs +++ b/DistTestCore/Marketplace/MarketplaceAccessFactory.cs @@ -30,7 +30,7 @@ namespace DistTestCore.Marketplace public IMarketplaceAccess CreateMarketplaceAccess(CodexAccess access) { var companionNode = GetGethCompanionNode(access); - return new MarketplaceAccess(log, marketplaceNetwork, companionNode); + return new MarketplaceAccess(log, marketplaceNetwork, companionNode, access); } private GethCompanionNodeInfo GetGethCompanionNode(CodexAccess access) diff --git a/DistTestCore/Marketplace/MarketplaceNetwork.cs b/DistTestCore/Marketplace/MarketplaceNetwork.cs index 7d27372..5f43fa9 100644 --- a/DistTestCore/Marketplace/MarketplaceNetwork.cs +++ b/DistTestCore/Marketplace/MarketplaceNetwork.cs @@ -16,13 +16,7 @@ namespace DistTestCore.Marketplace public NethereumInteraction StartInteraction(TestLog log) { - var ip = Bootstrap.RunningContainers.RunningPod.Cluster.IP; - var port = Bootstrap.RunningContainers.Containers[0].ServicePorts[0].Number; - var account = Bootstrap.Account; - var privateKey = Bootstrap.PrivateKey; - - var creator = new NethereumInteractionCreator(log, ip, port, account, privateKey); - return creator.CreateWorkflow(); + return Bootstrap.StartInteraction(log); } } } diff --git a/Nethereum/NethereumInteraction.cs b/Nethereum/NethereumInteraction.cs index 9092524..0388bd0 100644 --- a/Nethereum/NethereumInteraction.cs +++ b/Nethereum/NethereumInteraction.cs @@ -56,10 +56,16 @@ namespace NethereumWorkflow Time.Wait(handler.SendRequestAndWaitForReceiptAsync(tokenAddress, function)); } - public decimal GetBalance(string account) + public decimal GetBalance(string tokenAddress, string account) { - var bigInt = Time.Wait(web3.Eth.GetBalance.SendRequestAsync(account)); - var result = ToDecimal(bigInt); + var function = new GetTokenBalanceFunction + { + Owner = account + }; + + var handler = web3.Eth.GetContractQueryHandler(); + var result = ToDecimal(Time.Wait(handler.QueryAsync(tokenAddress, function))); + Log($"Balance of {account} is {result}"); return result; } @@ -81,6 +87,11 @@ namespace NethereumWorkflow return (decimal)hexBigInteger.Value; } + private decimal ToDecimal(BigInteger bigInteger) + { + return (decimal)bigInteger; + } + private void Log(string msg) { log.Log(msg); @@ -101,4 +112,11 @@ namespace NethereumWorkflow [Parameter("uint256", "amount", 2)] public BigInteger Amount { get; set; } } + + [Function("balanceOf", "uint256")] + public class GetTokenBalanceFunction :FunctionMessage + { + [Parameter("address", "owner", 1)] + public string Owner { get; set; } + } } diff --git a/Tests/BasicTests/ExampleTests.cs b/Tests/BasicTests/ExampleTests.cs index 01eee02..3269225 100644 --- a/Tests/BasicTests/ExampleTests.cs +++ b/Tests/BasicTests/ExampleTests.cs @@ -1,6 +1,7 @@ using DistTestCore; using DistTestCore.Codex; using NUnit.Framework; +using Utils; namespace Tests.BasicTests { @@ -49,38 +50,44 @@ namespace Tests.BasicTests [Test] public void MarketplaceExample() { - var group = SetupCodexNodes(2) + var primary = SetupCodexNodes(1) .WithStorageQuota(10.GB()) - .EnableMarketplace(20.TestTokens()) - .BringOnline(); + .EnableMarketplace(initialBalance: 234.TestTokens()) + .BringOnline()[0]; - foreach (var node in group) - { - Assert.That(node.Marketplace.GetBalance(), Is.EqualTo(20)); - } + Assert.That(primary.Marketplace.GetBalance(), Is.EqualTo(234)); - // WIP: Balance is now only ETH. - // todo: All nodes should have plenty of ETH to pay for transactions. - // todo: Upload our own token, use this exclusively. ETH should be invisibile to the tests. + var secondary = SetupCodexNodes(1) + .EnableMarketplace(initialBalance: 1000.TestTokens()) + .BringOnline()[0]; + primary.ConnectToPeer(secondary); - //var secondary = SetupCodexNodes(1) - // .EnableMarketplace(initialBalance: 1000) - // .BringOnline()[0]; + // Gives 503 - Persistance not enabled in current codex image. + primary.Marketplace.MakeStorageAvailable( + size: 10.GB(), + minPricePerBytePerSecond: 1.TestTokens(), + maxCollateral: 20.TestTokens(), + maxDuration: TimeSpan.FromMinutes(3)); - //primary.ConnectToPeer(secondary); - //primary.Marketplace.MakeStorageAvailable(10.GB(), minPricePerBytePerSecond: 1, maxCollateral: 20); + var testFile = GenerateTestFile(10.MB()); + var contentId = secondary.UploadFile(testFile); - //var testFile = GenerateTestFile(10.MB()); - //var contentId = secondary.UploadFile(testFile); - //secondary.Marketplace.RequestStorage(contentId, pricePerBytePerSecond: 2, - // requiredCollateral: 10, minRequiredNumberOfNodes: 1); + // Gives 500 - Persistance not enabled in current codex image. + secondary.Marketplace.RequestStorage(contentId, + pricePerBytePerSecond: 2.TestTokens(), + requiredCollateral: 10.TestTokens(), + minRequiredNumberOfNodes: 1, + proofProbability: 5, + duration: TimeSpan.FromMinutes(2)); - //primary.Marketplace.AssertThatBalance(Is.LessThan(20), "Collateral was not placed."); - //var primaryBalance = primary.Marketplace.GetBalance(); + Time.Sleep(TimeSpan.FromMinutes(3)); - //secondary.Marketplace.AssertThatBalance(Is.LessThan(1000), "Contractor was not charged for storage."); - //primary.Marketplace.AssertThatBalance(Is.GreaterThan(primaryBalance), "Storer was not paid for storage."); + primary.Marketplace.AssertThatBalance(Is.LessThan(20), "Collateral was not placed."); + var primaryBalance = primary.Marketplace.GetBalance(); + + secondary.Marketplace.AssertThatBalance(Is.LessThan(1000), "Contractor was not charged for storage."); + primary.Marketplace.AssertThatBalance(Is.GreaterThan(primaryBalance), "Storer was not paid for storage."); } } } From 48ca6f41fa7067a6ee168d5ba32771c981b7c928 Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 19 Apr 2023 07:59:28 +0200 Subject: [PATCH 42/46] Removes genesis json. --- DistTestCore/CodexNodeGroup.cs | 4 ++-- DistTestCore/Marketplace/ContainerInfoExtractor.cs | 13 ------------- DistTestCore/Marketplace/GethBootstrapNodeInfo.cs | 4 +--- .../Marketplace/GethBootstrapNodeStarter.cs | 7 ++----- .../Marketplace/GethCompanionNodeStarter.cs | 2 +- DistTestCore/Marketplace/GethContainerRecipe.cs | 2 -- DistTestCore/Marketplace/GethStartupConfig.cs | 4 +--- 7 files changed, 7 insertions(+), 29 deletions(-) diff --git a/DistTestCore/CodexNodeGroup.cs b/DistTestCore/CodexNodeGroup.cs index 4e2cd16..4ce577c 100644 --- a/DistTestCore/CodexNodeGroup.cs +++ b/DistTestCore/CodexNodeGroup.cs @@ -78,8 +78,8 @@ namespace DistTestCore } catch (Exception e) { - lifecycle.Log.Error($"Failed to start codex node: {e}"); - throw; + lifecycle.Log.Error($"Failed to start codex node: {e}. Test infra failure."); + throw new InvalidOperationException($"Failed to start codex node. Test infra failure.", e); } } } diff --git a/DistTestCore/Marketplace/ContainerInfoExtractor.cs b/DistTestCore/Marketplace/ContainerInfoExtractor.cs index 7a9105b..09d8aef 100644 --- a/DistTestCore/Marketplace/ContainerInfoExtractor.cs +++ b/DistTestCore/Marketplace/ContainerInfoExtractor.cs @@ -23,14 +23,6 @@ namespace DistTestCore.Marketplace return account; } - public string ExtractGenesisJsonBase64() - { - var genesisJson = Retry(FetchGenesisJson); - if (string.IsNullOrEmpty(genesisJson)) throw new InvalidOperationException("Unable to fetch genesis-json for geth node. Test infra failure."); - - return Convert.ToBase64String(Encoding.ASCII.GetBytes(genesisJson)); - } - public string ExtractPubKey() { var pubKey = Retry(FetchPubKey); @@ -78,11 +70,6 @@ namespace DistTestCore.Marketplace } } - private string FetchGenesisJson() - { - return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.GenesisFilename); - } - private string FetchAccount() { return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.AccountFilename); diff --git a/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs b/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs index 68c7485..0ea4c69 100644 --- a/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs +++ b/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs @@ -6,11 +6,10 @@ namespace DistTestCore.Marketplace { public class GethBootstrapNodeInfo { - public GethBootstrapNodeInfo(RunningContainers runningContainers, string account, string genesisJsonBase64, string pubKey, string privateKey, Port discoveryPort) + public GethBootstrapNodeInfo(RunningContainers runningContainers, string account, string pubKey, string privateKey, Port discoveryPort) { RunningContainers = runningContainers; Account = account; - GenesisJsonBase64 = genesisJsonBase64; PubKey = pubKey; PrivateKey = privateKey; DiscoveryPort = discoveryPort; @@ -18,7 +17,6 @@ namespace DistTestCore.Marketplace public RunningContainers RunningContainers { get; } public string Account { get; } - public string GenesisJsonBase64 { get; } public string PubKey { get; } public string PrivateKey { get; } public Port DiscoveryPort { get; } diff --git a/DistTestCore/Marketplace/GethBootstrapNodeStarter.cs b/DistTestCore/Marketplace/GethBootstrapNodeStarter.cs index 494940b..e0efc61 100644 --- a/DistTestCore/Marketplace/GethBootstrapNodeStarter.cs +++ b/DistTestCore/Marketplace/GethBootstrapNodeStarter.cs @@ -4,8 +4,6 @@ namespace DistTestCore.Marketplace { public class GethBootstrapNodeStarter : BaseStarter { - private const string bootstrapGenesisJsonBase64 = "ewogICAgImNvbmZpZyI6IHsKICAgICAgImNoYWluSWQiOiA3ODk5ODgsCiAgICAgICJob21lc3RlYWRCbG9jayI6IDAsCiAgICAgICJlaXAxNTBCbG9jayI6IDAsCiAgICAgICJlaXAxNTVCbG9jayI6IDAsCiAgICAgICJlaXAxNThCbG9jayI6IDAsCiAgICAgICJieXphbnRpdW1CbG9jayI6IDAsCiAgICAgICJjb25zdGFudGlub3BsZUJsb2NrIjogMCwKICAgICAgInBldGVyc2J1cmdCbG9jayI6IDAsCiAgICAgICJpc3RhbmJ1bEJsb2NrIjogMCwKICAgICAgIm11aXJHbGFjaWVyQmxvY2siOiAwLAogICAgICAiYmVybGluQmxvY2siOiAwLAogICAgICAibG9uZG9uQmxvY2siOiAwLAogICAgICAiYXJyb3dHbGFjaWVyQmxvY2siOiAwLAogICAgICAiZ3JheUdsYWNpZXJCbG9jayI6IDAsCiAgICAgICJjbGlxdWUiOiB7CiAgICAgICAgInBlcmlvZCI6IDUsCiAgICAgICAgImVwb2NoIjogMzAwMDAKICAgICAgfQogICAgfSwKICAgICJkaWZmaWN1bHR5IjogIjEiLAogICAgImdhc0xpbWl0IjogIjgwMDAwMDAwMCIsCiAgICAiZXh0cmFkYXRhIjogIjB4MDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMEFDQ09VTlRfSEVSRTAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAiLAogICAgImFsbG9jIjogewogICAgICAiMHhBQ0NPVU5UX0hFUkUiOiB7ICJiYWxhbmNlIjogIjUwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAiIH0KICAgIH0KICB9"; - public GethBootstrapNodeStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator) : base(lifecycle, workflowCreator) { @@ -23,20 +21,19 @@ namespace DistTestCore.Marketplace var extractor = new ContainerInfoExtractor(workflow, bootstrapContainer); var account = extractor.ExtractAccount(); - var genesisJsonBase64 = extractor.ExtractGenesisJsonBase64(); var pubKey = extractor.ExtractPubKey(); var privateKey = extractor.ExtractBootstrapPrivateKey(); var discoveryPort = bootstrapContainer.Recipe.GetPortByTag(GethContainerRecipe.DiscoveryPortTag); LogEnd($"Geth bootstrap node started with account '{account}'"); - return new GethBootstrapNodeInfo(containers, account, genesisJsonBase64, pubKey, privateKey, discoveryPort); + return new GethBootstrapNodeInfo(containers, account, pubKey, privateKey, discoveryPort); } private StartupConfig CreateBootstrapStartupConfig() { var config = new StartupConfig(); - config.Add(new GethStartupConfig(true, bootstrapGenesisJsonBase64, null!)); + config.Add(new GethStartupConfig(true, null!)); return config; } } diff --git a/DistTestCore/Marketplace/GethCompanionNodeStarter.cs b/DistTestCore/Marketplace/GethCompanionNodeStarter.cs index 7e20c4b..e9cd37b 100644 --- a/DistTestCore/Marketplace/GethCompanionNodeStarter.cs +++ b/DistTestCore/Marketplace/GethCompanionNodeStarter.cs @@ -34,7 +34,7 @@ namespace DistTestCore.Marketplace private StartupConfig CreateCompanionNodeStartupConfig(GethBootstrapNodeInfo bootstrapNode) { var config = new StartupConfig(); - config.Add(new GethStartupConfig(false, bootstrapNode.GenesisJsonBase64, bootstrapNode)); + config.Add(new GethStartupConfig(false, bootstrapNode)); return config; } } diff --git a/DistTestCore/Marketplace/GethContainerRecipe.cs b/DistTestCore/Marketplace/GethContainerRecipe.cs index 6706901..0c0ca91 100644 --- a/DistTestCore/Marketplace/GethContainerRecipe.cs +++ b/DistTestCore/Marketplace/GethContainerRecipe.cs @@ -9,7 +9,6 @@ namespace DistTestCore.Marketplace public const string WsPortTag = "ws_port"; public const string DiscoveryPortTag = "disc_port"; public const string AccountFilename = "account_string.txt"; - public const string GenesisFilename = "genesis.json"; public const string BootstrapPrivateKeyFilename = "bootstrap_private.key"; protected override string Image => DockerImage; @@ -21,7 +20,6 @@ namespace DistTestCore.Marketplace var args = CreateArgs(config); AddEnvVar("GETH_ARGS", args); - AddEnvVar("GENESIS_JSON", config.GenesisJsonBase64); } private string CreateArgs(GethStartupConfig config) diff --git a/DistTestCore/Marketplace/GethStartupConfig.cs b/DistTestCore/Marketplace/GethStartupConfig.cs index baeb421..a8026f1 100644 --- a/DistTestCore/Marketplace/GethStartupConfig.cs +++ b/DistTestCore/Marketplace/GethStartupConfig.cs @@ -2,15 +2,13 @@ { public class GethStartupConfig { - public GethStartupConfig(bool isBootstrapNode, string genesisJsonBase64, GethBootstrapNodeInfo bootstrapNode) + public GethStartupConfig(bool isBootstrapNode, GethBootstrapNodeInfo bootstrapNode) { IsBootstrapNode = isBootstrapNode; - GenesisJsonBase64 = genesisJsonBase64; BootstrapNode = bootstrapNode; } public bool IsBootstrapNode { get; } - public string GenesisJsonBase64 { get; } public GethBootstrapNodeInfo BootstrapNode { get; } } } From 7e6de4146ed7f2e8034707e4a9f8555f7f9f05f4 Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 19 Apr 2023 09:19:06 +0200 Subject: [PATCH 43/46] Better logging and much faster initial balance transactions --- DistTestCore/BaseStarter.cs | 4 +-- DistTestCore/CodexStarter.cs | 9 ++++- DistTestCore/GethStarter.cs | 6 +++- .../Marketplace/GethCompanionNodeStarter.cs | 6 ++-- DistTestCore/OnlineCodexNode.cs | 1 + Nethereum/NethereumInteraction.cs | 35 +++++++++++-------- 6 files changed, 40 insertions(+), 21 deletions(-) diff --git a/DistTestCore/BaseStarter.cs b/DistTestCore/BaseStarter.cs index e4f7fde..7d259e7 100644 --- a/DistTestCore/BaseStarter.cs +++ b/DistTestCore/BaseStarter.cs @@ -28,12 +28,12 @@ namespace DistTestCore protected void Log(string msg) { - lifecycle.Log.Log($"{GetClassName} {msg}"); + lifecycle.Log.Log($"{GetClassName()} {msg}"); } private string GetClassName() { - return GetType().Name; + return $"({GetType().Name})"; } } } diff --git a/DistTestCore/CodexStarter.cs b/DistTestCore/CodexStarter.cs index 54342c7..84da9b9 100644 --- a/DistTestCore/CodexStarter.cs +++ b/DistTestCore/CodexStarter.cs @@ -14,6 +14,7 @@ namespace DistTestCore public ICodexNodeGroup BringOnline(CodexSetup codexSetup) { + LogSeparator(); LogStart($"Starting {codexSetup.Describe()}..."); var gethStartResult = lifecycle.GethStarter.BringOnlineMarketplaceFor(codexSetup); @@ -28,7 +29,8 @@ namespace DistTestCore var codexNodeFactory = new CodexNodeFactory(lifecycle, metricAccessFactory, gethStartResult.MarketplaceAccessFactory); var group = CreateCodexGroup(codexSetup, containers, codexNodeFactory); - LogEnd($"Started at '{group.Containers.RunningPod.Ip}'"); + LogEnd($"Started {codexSetup.NumberOfNodes} nodes at '{group.Containers.RunningPod.Ip}'. They are: [{string.Join(",", group.Select(n => n.GetName()))}]"); + LogSeparator(); return group; } @@ -72,5 +74,10 @@ namespace DistTestCore { return workflowCreator.CreateWorkflow(); } + + private void LogSeparator() + { + Log("----------------------------------------------------------------------------"); + } } } diff --git a/DistTestCore/GethStarter.cs b/DistTestCore/GethStarter.cs index 2b2cbb3..e4b5b37 100644 --- a/DistTestCore/GethStarter.cs +++ b/DistTestCore/GethStarter.cs @@ -24,7 +24,9 @@ namespace DistTestCore var marketplaceNetwork = marketplaceNetworkCache.Get(); var companionNodes = StartCompanionNodes(codexSetup, marketplaceNetwork); + LogStart("Setting up initial balance..."); TransferInitialBalance(marketplaceNetwork, codexSetup.MarketplaceConfig, companionNodes); + LogEnd($"Initial balance of {codexSetup.MarketplaceConfig.InitialTestTokens.Amount} TestTokens set for {codexSetup.NumberOfNodes} nodes."); return CreateGethStartResult(marketplaceNetwork, companionNodes); } @@ -36,9 +38,11 @@ namespace DistTestCore foreach (var node in companionNodes) { - interaction.TransferTo(node.Account, marketplaceConfig.InitialEth.Wei); + interaction.TransferWeiTo(node.Account, marketplaceConfig.InitialEth.Wei); interaction.MintTestTokens(node.Account, marketplaceConfig.InitialTestTokens.Amount, tokenAddress); } + + interaction.WaitForAllTransactions(); } private GethStartResult CreateGethStartResult(MarketplaceNetwork marketplaceNetwork, GethCompanionNodeInfo[] companionNodes) diff --git a/DistTestCore/Marketplace/GethCompanionNodeStarter.cs b/DistTestCore/Marketplace/GethCompanionNodeStarter.cs index e9cd37b..047af28 100644 --- a/DistTestCore/Marketplace/GethCompanionNodeStarter.cs +++ b/DistTestCore/Marketplace/GethCompanionNodeStarter.cs @@ -19,9 +19,11 @@ namespace DistTestCore.Marketplace var containers = workflow.Start(codexSetup.NumberOfNodes, Location.Unspecified, new GethContainerRecipe(), startupConfig); if (containers.Containers.Length != codexSetup.NumberOfNodes) throw new InvalidOperationException("Expected a Geth companion node to be created for each Codex node. Test infra failure."); - LogEnd("Initialized companion nodes."); + var result = containers.Containers.Select(c => CreateCompanionInfo(workflow, c)).ToArray(); - return containers.Containers.Select(c => CreateCompanionInfo(workflow, c)).ToArray(); + LogEnd($"Initialized {codexSetup.NumberOfNodes} companion nodes. Their accounts: [{string.Join(",", result.Select(c => c.Account))}]"); + + return result; } private GethCompanionNodeInfo CreateCompanionInfo(StartupWorkflow workflow, RunningContainer container) diff --git a/DistTestCore/OnlineCodexNode.cs b/DistTestCore/OnlineCodexNode.cs index a12de9f..0750ee3 100644 --- a/DistTestCore/OnlineCodexNode.cs +++ b/DistTestCore/OnlineCodexNode.cs @@ -8,6 +8,7 @@ namespace DistTestCore { public interface IOnlineCodexNode { + string GetName(); CodexDebugResponse GetDebugInfo(); ContentId UploadFile(TestFile file); TestFile? DownloadContent(ContentId contentId); diff --git a/Nethereum/NethereumInteraction.cs b/Nethereum/NethereumInteraction.cs index 0388bd0..adce008 100644 --- a/Nethereum/NethereumInteraction.cs +++ b/Nethereum/NethereumInteraction.cs @@ -10,6 +10,7 @@ namespace NethereumWorkflow { public class NethereumInteraction { + private readonly List openTasks = new List(); private readonly TestLog log; private readonly Web3 web3; private readonly string rootAccount; @@ -21,25 +22,21 @@ namespace NethereumWorkflow this.rootAccount = rootAccount; } - public void TransferTo(string account, decimal amount) - { - if (amount < 1 || string.IsNullOrEmpty(account)) throw new ArgumentException("Invalid arguments for AddToBalance"); - - var value = ToHexBig(amount); - var transactionId = Time.Wait(web3.Eth.TransactionManager.SendTransactionAsync(rootAccount, account, value)); - Time.Wait(web3.Eth.TransactionManager.TransactionReceiptService.PollForReceiptAsync(transactionId)); - - Log($"Transferred {amount} to {account}"); - } - public string GetTokenAddress(string marketplaceAddress) { var function = new GetTokenFunction(); var handler = web3.Eth.GetContractQueryHandler(); - var result = Time.Wait(handler.QueryAsync(marketplaceAddress, function)); + return Time.Wait(handler.QueryAsync(marketplaceAddress, function)); + } - return result; + public void TransferWeiTo(string account, decimal amount) + { + if (amount < 1 || string.IsNullOrEmpty(account)) throw new ArgumentException("Invalid arguments for AddToBalance"); + + var value = ToHexBig(amount); + var transactionId = Time.Wait(web3.Eth.TransactionManager.SendTransactionAsync(rootAccount, account, value)); + openTasks.Add(web3.Eth.TransactionManager.TransactionReceiptService.PollForReceiptAsync(transactionId)); } public void MintTestTokens(string account, decimal amount, string tokenAddress) @@ -53,7 +50,7 @@ namespace NethereumWorkflow }; var handler = web3.Eth.GetContractTransactionHandler(); - Time.Wait(handler.SendRequestAndWaitForReceiptAsync(tokenAddress, function)); + openTasks.Add(handler.SendRequestAndWaitForReceiptAsync(tokenAddress, function)); } public decimal GetBalance(string tokenAddress, string account) @@ -66,10 +63,18 @@ namespace NethereumWorkflow var handler = web3.Eth.GetContractQueryHandler(); var result = ToDecimal(Time.Wait(handler.QueryAsync(tokenAddress, function))); - Log($"Balance of {account} is {result}"); + Log($"Balance of {account} is {result} TestTokens."); return result; } + public void WaitForAllTransactions() + { + var tasks = openTasks.ToArray(); + openTasks.Clear(); + + Task.WaitAll(tasks); + } + private HexBigInteger ToHexBig(decimal amount) { var bigint = ToBig(amount); From e4e7afd5802f46429e46b3cc80722fdd0757b6bc Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 19 Apr 2023 09:57:37 +0200 Subject: [PATCH 44/46] Better logging --- DistTestCore/ByteSize.cs | 5 +++ DistTestCore/Codex/CodexAccess.cs | 1 + DistTestCore/Codex/CodexContainerRecipe.cs | 2 +- DistTestCore/CodexSetup.cs | 2 +- DistTestCore/GethStarter.cs | 2 +- DistTestCore/Marketplace/MarketplaceAccess.cs | 43 ++++++++++++++++--- DistTestCore/OnlineCodexNode.cs | 2 +- DistTestCore/Tokens.cs | 5 +++ KubernetesWorkflow/RunningContainers.cs | 5 +++ Nethereum/NethereumInteraction.cs | 15 +------ Tests/BasicTests/ExampleTests.cs | 2 - 11 files changed, 57 insertions(+), 27 deletions(-) diff --git a/DistTestCore/ByteSize.cs b/DistTestCore/ByteSize.cs index 783cce9..2a41e94 100644 --- a/DistTestCore/ByteSize.cs +++ b/DistTestCore/ByteSize.cs @@ -8,6 +8,11 @@ } public long SizeInBytes { get; } + + public override string ToString() + { + return $"{SizeInBytes} bytes"; + } } public static class ByteSizeIntExtensions diff --git a/DistTestCore/Codex/CodexAccess.cs b/DistTestCore/Codex/CodexAccess.cs index ae84a91..00ac2f2 100644 --- a/DistTestCore/Codex/CodexAccess.cs +++ b/DistTestCore/Codex/CodexAccess.cs @@ -1,4 +1,5 @@ using KubernetesWorkflow; +using Newtonsoft.Json; namespace DistTestCore.Codex { diff --git a/DistTestCore/Codex/CodexContainerRecipe.cs b/DistTestCore/Codex/CodexContainerRecipe.cs index ab3dbee..01f4933 100644 --- a/DistTestCore/Codex/CodexContainerRecipe.cs +++ b/DistTestCore/Codex/CodexContainerRecipe.cs @@ -5,7 +5,7 @@ namespace DistTestCore.Codex { public class CodexContainerRecipe : ContainerRecipeFactory { - public const string DockerImage = "thatbenbierens/nim-codex:sha-b204837"; + public const string DockerImage = "thatbenbierens/nim-codex:sha-bf5512b"; public const string MetricsPortTag = "metrics_port"; protected override string Image => DockerImage; diff --git a/DistTestCore/CodexSetup.cs b/DistTestCore/CodexSetup.cs index 72dc103..39020f6 100644 --- a/DistTestCore/CodexSetup.cs +++ b/DistTestCore/CodexSetup.cs @@ -84,7 +84,7 @@ namespace DistTestCore { if (LogLevel != null) yield return $"LogLevel={LogLevel}"; //if (BootstrapNode != null) yield return "BootstrapNode=set-not-shown-here"; - if (StorageQuota != null) yield return $"StorageQuote={StorageQuota.SizeInBytes}"; + if (StorageQuota != null) yield return $"StorageQuote={StorageQuota}"; } } } diff --git a/DistTestCore/GethStarter.cs b/DistTestCore/GethStarter.cs index e4b5b37..b3c782b 100644 --- a/DistTestCore/GethStarter.cs +++ b/DistTestCore/GethStarter.cs @@ -26,7 +26,7 @@ namespace DistTestCore LogStart("Setting up initial balance..."); TransferInitialBalance(marketplaceNetwork, codexSetup.MarketplaceConfig, companionNodes); - LogEnd($"Initial balance of {codexSetup.MarketplaceConfig.InitialTestTokens.Amount} TestTokens set for {codexSetup.NumberOfNodes} nodes."); + LogEnd($"Initial balance of {codexSetup.MarketplaceConfig.InitialTestTokens} set for {codexSetup.NumberOfNodes} nodes."); return CreateGethStartResult(marketplaceNetwork, companionNodes); } diff --git a/DistTestCore/Marketplace/MarketplaceAccess.cs b/DistTestCore/Marketplace/MarketplaceAccess.cs index dd784c9..713081e 100644 --- a/DistTestCore/Marketplace/MarketplaceAccess.cs +++ b/DistTestCore/Marketplace/MarketplaceAccess.cs @@ -3,6 +3,7 @@ using Logging; using NUnit.Framework; using NUnit.Framework.Constraints; using System.Numerics; +using Utils; namespace DistTestCore.Marketplace { @@ -11,7 +12,7 @@ namespace DistTestCore.Marketplace string MakeStorageAvailable(ByteSize size, TestToken minPricePerBytePerSecond, TestToken maxCollateral, TimeSpan maxDuration); string RequestStorage(ContentId contentId, TestToken pricePerBytePerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration); void AssertThatBalance(IResolveConstraint constraint, string message = ""); - decimal GetBalance(); + TestToken GetBalance(); } public class MarketplaceAccess : IMarketplaceAccess @@ -42,23 +43,40 @@ namespace DistTestCore.Marketplace tolerance = null, }; + Log($"Requesting storage for: {contentId.Id}... (" + + $"pricePerBytePerSecond: {pricePerBytePerSecond}," + + $"requiredCollateral: {requiredCollateral}," + + $"minRequiredNumberOfNodes: {minRequiredNumberOfNodes}," + + $"proofProbability: {proofProbability}," + + $"duration: {Time.FormatDuration(duration)})"); + var response = codexAccess.RequestStorage(request, contentId.Id); + Log($"Storage requested successfully. PurchaseId: {response.purchaseId}"); + return response.purchaseId; } - public string MakeStorageAvailable(ByteSize size, TestToken minPricePerBytePerSecond, TestToken maxCollateral, TimeSpan duration) + public string MakeStorageAvailable(ByteSize size, TestToken minPricePerBytePerSecond, TestToken maxCollateral, TimeSpan maxDuration) { var request = new CodexSalesAvailabilityRequest { size = ToHexBigInt(size.SizeInBytes), - duration = ToHexBigInt(duration.TotalSeconds), + duration = ToHexBigInt(maxDuration.TotalSeconds), maxCollateral = ToHexBigInt(maxCollateral), minPrice = ToHexBigInt(minPricePerBytePerSecond) }; + Log($"Making storage available... (" + + $"size: {size}," + + $"minPricePerBytePerSecond: {minPricePerBytePerSecond}," + + $"maxCollateral: {maxCollateral}," + + $"maxDuration: {Time.FormatDuration(maxDuration)}"); + var response = codexAccess.SalesAvailability(request); + Log($"Storage successfully made available. Id: {response.id}"); + return response.id; } @@ -78,10 +96,21 @@ namespace DistTestCore.Marketplace Assert.That(GetBalance(), constraint, message); } - public decimal GetBalance() + public TestToken GetBalance() { var interaction = marketplaceNetwork.StartInteraction(log); - return interaction.GetBalance(marketplaceNetwork.Marketplace.TokenAddress, companionNode.Account); + var account = companionNode.Account; + var amount = interaction.GetBalance(marketplaceNetwork.Marketplace.TokenAddress, account); + var balance = new TestToken(amount); + + Log($"Balance of {codexAccess.Container.GetName()}({account}) is {balance}."); + + return balance; + } + + private void Log(string msg) + { + log.Log(msg); } } @@ -104,10 +133,10 @@ namespace DistTestCore.Marketplace Unavailable(); } - public decimal GetBalance() + public TestToken GetBalance() { Unavailable(); - return 0; + return new TestToken(0); } private void Unavailable() diff --git a/DistTestCore/OnlineCodexNode.cs b/DistTestCore/OnlineCodexNode.cs index 0750ee3..bf0dfdf 100644 --- a/DistTestCore/OnlineCodexNode.cs +++ b/DistTestCore/OnlineCodexNode.cs @@ -40,7 +40,7 @@ namespace DistTestCore public string GetName() { - return $"<{CodexAccess.Container.Recipe.Name}>"; + return CodexAccess.Container.GetName(); } public CodexDebugResponse GetDebugInfo() diff --git a/DistTestCore/Tokens.cs b/DistTestCore/Tokens.cs index 4669660..3f92852 100644 --- a/DistTestCore/Tokens.cs +++ b/DistTestCore/Tokens.cs @@ -18,6 +18,11 @@ } public decimal Amount { get; } + + public override string ToString() + { + return $"{Amount} TestTokens"; + } } public static class TokensIntExtensions diff --git a/KubernetesWorkflow/RunningContainers.cs b/KubernetesWorkflow/RunningContainers.cs index 783c6f8..27a9991 100644 --- a/KubernetesWorkflow/RunningContainers.cs +++ b/KubernetesWorkflow/RunningContainers.cs @@ -28,6 +28,11 @@ ServicePorts = servicePorts; } + public string GetName() + { + return $"<{Recipe.Name}>"; + } + public RunningPod Pod { get; } public ContainerRecipe Recipe { get; } public Port[] ServicePorts { get; } diff --git a/Nethereum/NethereumInteraction.cs b/Nethereum/NethereumInteraction.cs index adce008..735bd6f 100644 --- a/Nethereum/NethereumInteraction.cs +++ b/Nethereum/NethereumInteraction.cs @@ -61,10 +61,7 @@ namespace NethereumWorkflow }; var handler = web3.Eth.GetContractQueryHandler(); - var result = ToDecimal(Time.Wait(handler.QueryAsync(tokenAddress, function))); - - Log($"Balance of {account} is {result} TestTokens."); - return result; + return ToDecimal(Time.Wait(handler.QueryAsync(tokenAddress, function))); } public void WaitForAllTransactions() @@ -87,20 +84,10 @@ namespace NethereumWorkflow return new BigInteger(amount); } - private decimal ToDecimal(HexBigInteger hexBigInteger) - { - return (decimal)hexBigInteger.Value; - } - private decimal ToDecimal(BigInteger bigInteger) { return (decimal)bigInteger; } - - private void Log(string msg) - { - log.Log(msg); - } } [Function("token", "address")] diff --git a/Tests/BasicTests/ExampleTests.cs b/Tests/BasicTests/ExampleTests.cs index 3269225..9c43ab1 100644 --- a/Tests/BasicTests/ExampleTests.cs +++ b/Tests/BasicTests/ExampleTests.cs @@ -63,7 +63,6 @@ namespace Tests.BasicTests primary.ConnectToPeer(secondary); - // Gives 503 - Persistance not enabled in current codex image. primary.Marketplace.MakeStorageAvailable( size: 10.GB(), minPricePerBytePerSecond: 1.TestTokens(), @@ -73,7 +72,6 @@ namespace Tests.BasicTests var testFile = GenerateTestFile(10.MB()); var contentId = secondary.UploadFile(testFile); - // Gives 500 - Persistance not enabled in current codex image. secondary.Marketplace.RequestStorage(contentId, pricePerBytePerSecond: 2.TestTokens(), requiredCollateral: 10.TestTokens(), From e9d84a5cf74dbf70b3cdb6825a9d2d76443589e9 Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 19 Apr 2023 10:42:08 +0200 Subject: [PATCH 45/46] Better logging for json errors and node actions. --- DistTestCore/ByteSize.cs | 10 +++++++ DistTestCore/Codex/CodexAccess.cs | 1 - DistTestCore/DistTest.cs | 6 ++--- DistTestCore/Http.cs | 26 +++++++++++++++---- DistTestCore/Logs/CodexNodeLog.cs | 6 +++-- DistTestCore/Logs/LogDownloadHandler.cs | 6 +++-- DistTestCore/Marketplace/MarketplaceAccess.cs | 20 +++++++------- DistTestCore/Metrics/MetricsAccess.cs | 26 ++++++++++++------- DistTestCore/Metrics/MetricsAccessFactory.cs | 6 +++-- DistTestCore/OnlineCodexNode.cs | 2 +- DistTestCore/PrometheusStarter.cs | 2 +- DistTestCore/TestLifecycle.cs | 4 +-- DistTestCore/Tokens.cs | 25 ++++++++++++++++++ Logging/TestLog.cs | 4 +-- Tests/BasicTests/ExampleTests.cs | 4 +-- 15 files changed, 105 insertions(+), 43 deletions(-) diff --git a/DistTestCore/ByteSize.cs b/DistTestCore/ByteSize.cs index 2a41e94..dc288bc 100644 --- a/DistTestCore/ByteSize.cs +++ b/DistTestCore/ByteSize.cs @@ -9,6 +9,16 @@ public long SizeInBytes { get; } + public override bool Equals(object? obj) + { + return obj is ByteSize size && SizeInBytes == size.SizeInBytes; + } + + public override int GetHashCode() + { + return HashCode.Combine(SizeInBytes); + } + public override string ToString() { return $"{SizeInBytes} bytes"; diff --git a/DistTestCore/Codex/CodexAccess.cs b/DistTestCore/Codex/CodexAccess.cs index 00ac2f2..ae84a91 100644 --- a/DistTestCore/Codex/CodexAccess.cs +++ b/DistTestCore/Codex/CodexAccess.cs @@ -1,5 +1,4 @@ using KubernetesWorkflow; -using Newtonsoft.Json; namespace DistTestCore.Codex { diff --git a/DistTestCore/DistTest.cs b/DistTestCore/DistTest.cs index 972a539..ca3f70b 100644 --- a/DistTestCore/DistTest.cs +++ b/DistTestCore/DistTest.cs @@ -40,9 +40,9 @@ namespace DistTestCore } fixtureLog.Log("Global setup cleanup successful"); - fixtureLog.Log($"Codex image: {CodexContainerRecipe.DockerImage}"); - fixtureLog.Log($"Prometheus image: {PrometheusContainerRecipe.DockerImage}"); - fixtureLog.Log($"Geth image: {GethContainerRecipe.DockerImage}"); + fixtureLog.Log($"Codex image: '{CodexContainerRecipe.DockerImage}'"); + fixtureLog.Log($"Prometheus image: '{PrometheusContainerRecipe.DockerImage}'"); + fixtureLog.Log($"Geth image: '{GethContainerRecipe.DockerImage}'"); } [SetUp] diff --git a/DistTestCore/Http.cs b/DistTestCore/Http.cs index 0d242cf..62ccbfb 100644 --- a/DistTestCore/Http.cs +++ b/DistTestCore/Http.cs @@ -35,20 +35,22 @@ namespace DistTestCore public T HttpGetJson(string route) { - return JsonConvert.DeserializeObject(HttpGetString(route))!; + var json = HttpGetString(route); + return TryJsonDeserialize(json); } public TResponse HttpPostJson(string route, TRequest body) { - return Retry(() => + var json = Retry(() => { using var client = GetClient(); var url = GetUrl() + route; using var content = JsonContent.Create(body); var result = Time.Wait(client.PostAsync(url, content)); - var json = Time.Wait(result.Content.ReadAsStringAsync()); - return JsonConvert.DeserializeObject(json)!; + return Time.Wait(result.Content.ReadAsStringAsync()); }); + + return TryJsonDeserialize(json); } public string HttpPostStream(string route, Stream stream) @@ -98,13 +100,27 @@ namespace DistTestCore retryCounter++; if (retryCounter > Timing.HttpCallRetryCount()) { - Assert.Fail(exception.Message); + Assert.Fail(exception.ToString()); throw; } } } } + private static T TryJsonDeserialize(string json) + { + try + { + return JsonConvert.DeserializeObject(json)!; + } + catch (Exception exception) + { + var msg = $"Failed to deserialize JSON: '{json}' with exception: {exception}"; + Assert.Fail(msg); + throw new InvalidOperationException(msg, exception); + } + } + private static HttpClient GetClient() { var client = new HttpClient(); diff --git a/DistTestCore/Logs/CodexNodeLog.cs b/DistTestCore/Logs/CodexNodeLog.cs index 3b65241..6dd658f 100644 --- a/DistTestCore/Logs/CodexNodeLog.cs +++ b/DistTestCore/Logs/CodexNodeLog.cs @@ -11,10 +11,12 @@ namespace DistTestCore.Logs public class CodexNodeLog : ICodexNodeLog { private readonly LogFile logFile; + private readonly OnlineCodexNode owner; - public CodexNodeLog(LogFile logFile) + public CodexNodeLog(LogFile logFile, OnlineCodexNode owner) { this.logFile = logFile; + this.owner = owner; } public void AssertLogContains(string expectedString) @@ -29,7 +31,7 @@ namespace DistTestCore.Logs line = streamReader.ReadLine(); } - Assert.Fail($"Unable to find string '{expectedString}' in CodexNode log file {logFile.FullFilename}"); + Assert.Fail($"{owner.GetName()} Unable to find string '{expectedString}' in CodexNode log file {logFile.FullFilename}"); } } } diff --git a/DistTestCore/Logs/LogDownloadHandler.cs b/DistTestCore/Logs/LogDownloadHandler.cs index 9cc1012..2c7dc9f 100644 --- a/DistTestCore/Logs/LogDownloadHandler.cs +++ b/DistTestCore/Logs/LogDownloadHandler.cs @@ -5,10 +5,12 @@ namespace DistTestCore.Logs { public class LogDownloadHandler : LogHandler, ILogHandler { + private readonly OnlineCodexNode node; private readonly LogFile log; - public LogDownloadHandler(string description, LogFile log) + public LogDownloadHandler(OnlineCodexNode node, string description, LogFile log) { + this.node = node; this.log = log; log.Write($"{description} -->> {log.FullFilename}"); @@ -17,7 +19,7 @@ namespace DistTestCore.Logs public CodexNodeLog CreateCodexNodeLog() { - return new CodexNodeLog(log); + return new CodexNodeLog(log, node); } protected override void ProcessLine(string line) diff --git a/DistTestCore/Marketplace/MarketplaceAccess.cs b/DistTestCore/Marketplace/MarketplaceAccess.cs index 713081e..881ce93 100644 --- a/DistTestCore/Marketplace/MarketplaceAccess.cs +++ b/DistTestCore/Marketplace/MarketplaceAccess.cs @@ -44,10 +44,10 @@ namespace DistTestCore.Marketplace }; Log($"Requesting storage for: {contentId.Id}... (" + - $"pricePerBytePerSecond: {pricePerBytePerSecond}," + - $"requiredCollateral: {requiredCollateral}," + - $"minRequiredNumberOfNodes: {minRequiredNumberOfNodes}," + - $"proofProbability: {proofProbability}," + + $"pricePerBytePerSecond: {pricePerBytePerSecond}, " + + $"requiredCollateral: {requiredCollateral}, " + + $"minRequiredNumberOfNodes: {minRequiredNumberOfNodes}, " + + $"proofProbability: {proofProbability}, " + $"duration: {Time.FormatDuration(duration)})"); var response = codexAccess.RequestStorage(request, contentId.Id); @@ -68,10 +68,10 @@ namespace DistTestCore.Marketplace }; Log($"Making storage available... (" + - $"size: {size}," + - $"minPricePerBytePerSecond: {minPricePerBytePerSecond}," + - $"maxCollateral: {maxCollateral}," + - $"maxDuration: {Time.FormatDuration(maxDuration)}"); + $"size: {size}, " + + $"minPricePerBytePerSecond: {minPricePerBytePerSecond}, " + + $"maxCollateral: {maxCollateral}, " + + $"maxDuration: {Time.FormatDuration(maxDuration)})"); var response = codexAccess.SalesAvailability(request); @@ -103,14 +103,14 @@ namespace DistTestCore.Marketplace var amount = interaction.GetBalance(marketplaceNetwork.Marketplace.TokenAddress, account); var balance = new TestToken(amount); - Log($"Balance of {codexAccess.Container.GetName()}({account}) is {balance}."); + Log($"Balance of {account} is {balance}."); return balance; } private void Log(string msg) { - log.Log(msg); + log.Log($"{codexAccess.Container.GetName()} {msg}"); } } diff --git a/DistTestCore/Metrics/MetricsAccess.cs b/DistTestCore/Metrics/MetricsAccess.cs index e5bd2b3..fffaea8 100644 --- a/DistTestCore/Metrics/MetricsAccess.cs +++ b/DistTestCore/Metrics/MetricsAccess.cs @@ -1,4 +1,5 @@ using KubernetesWorkflow; +using Logging; using NUnit.Framework; using NUnit.Framework.Constraints; using Utils; @@ -10,22 +11,15 @@ namespace DistTestCore.Metrics void AssertThat(string metricName, IResolveConstraint constraint, string message = ""); } - public class MetricsUnavailable : IMetricsAccess - { - public void AssertThat(string metricName, IResolveConstraint constraint, string message = "") - { - Assert.Fail("Incorrect test setup: Metrics were not enabled for this group of Codex nodes. Add 'EnableMetrics()' after 'SetupCodexNodes()' to enable it."); - throw new InvalidOperationException(); - } - } - public class MetricsAccess : IMetricsAccess { + private readonly TestLog log; private readonly MetricsQuery query; private readonly RunningContainer node; - public MetricsAccess(MetricsQuery query, RunningContainer node) + public MetricsAccess(TestLog log, MetricsQuery query, RunningContainer node) { + this.log = log; this.query = query; this.node = node; } @@ -34,6 +28,9 @@ namespace DistTestCore.Metrics { var metricSet = GetMetricWithTimeout(metricName); var metricValue = metricSet.Values[0].Value; + + log.Log($"{node.GetName()} metric '{metricName}' = {metricValue}"); + Assert.That(metricValue, constraint, message); } @@ -67,4 +64,13 @@ namespace DistTestCore.Metrics return result.Sets.LastOrDefault(); } } + + public class MetricsUnavailable : IMetricsAccess + { + public void AssertThat(string metricName, IResolveConstraint constraint, string message = "") + { + Assert.Fail("Incorrect test setup: Metrics were not enabled for this group of Codex nodes. Add 'EnableMetrics()' after 'SetupCodexNodes()' to enable it."); + throw new InvalidOperationException(); + } + } } diff --git a/DistTestCore/Metrics/MetricsAccessFactory.cs b/DistTestCore/Metrics/MetricsAccessFactory.cs index fcf5dfb..dad95b8 100644 --- a/DistTestCore/Metrics/MetricsAccessFactory.cs +++ b/DistTestCore/Metrics/MetricsAccessFactory.cs @@ -17,17 +17,19 @@ namespace DistTestCore.Metrics public class CodexNodeMetricsAccessFactory : IMetricsAccessFactory { + private readonly TestLifecycle lifecycle; private readonly RunningContainers prometheusContainer; - public CodexNodeMetricsAccessFactory(RunningContainers prometheusContainer) + public CodexNodeMetricsAccessFactory(TestLifecycle lifecycle, RunningContainers prometheusContainer) { + this.lifecycle = lifecycle; this.prometheusContainer = prometheusContainer; } public IMetricsAccess CreateMetricsAccess(RunningContainer codexContainer) { var query = new MetricsQuery(prometheusContainer); - return new MetricsAccess(query, codexContainer); + return new MetricsAccess(lifecycle.Log, query, codexContainer); } } } diff --git a/DistTestCore/OnlineCodexNode.cs b/DistTestCore/OnlineCodexNode.cs index bf0dfdf..a0d7542 100644 --- a/DistTestCore/OnlineCodexNode.cs +++ b/DistTestCore/OnlineCodexNode.cs @@ -91,7 +91,7 @@ namespace DistTestCore public string Describe() { - return $"({Group.Describe()} contains {GetName()})"; + return $"({GetName()} in {Group.Describe()})"; } private string GetPeerMultiAddress(OnlineCodexNode peer, CodexDebugResponse peerInfo) diff --git a/DistTestCore/PrometheusStarter.cs b/DistTestCore/PrometheusStarter.cs index 936c876..eb66efc 100644 --- a/DistTestCore/PrometheusStarter.cs +++ b/DistTestCore/PrometheusStarter.cs @@ -26,7 +26,7 @@ namespace DistTestCore LogEnd("Metrics server started."); - return new CodexNodeMetricsAccessFactory(runningContainers); + return new CodexNodeMetricsAccessFactory(lifecycle, runningContainers); } private string GeneratePrometheusConfig(RunningContainer[] nodes) diff --git a/DistTestCore/TestLifecycle.cs b/DistTestCore/TestLifecycle.cs index 2257610..3158f4e 100644 --- a/DistTestCore/TestLifecycle.cs +++ b/DistTestCore/TestLifecycle.cs @@ -35,12 +35,12 @@ namespace DistTestCore { var subFile = Log.CreateSubfile(); var description = node.Describe(); - var handler = new LogDownloadHandler(description, subFile); + var handler = new LogDownloadHandler(node, description, subFile); Log.Log($"Downloading logs for {description} to file '{subFile.FullFilename}'"); CodexStarter.DownloadLog(node.CodexAccess.Container, handler); - return new CodexNodeLog(subFile); + return new CodexNodeLog(subFile, node); } } } diff --git a/DistTestCore/Tokens.cs b/DistTestCore/Tokens.cs index 3f92852..07d1692 100644 --- a/DistTestCore/Tokens.cs +++ b/DistTestCore/Tokens.cs @@ -8,6 +8,21 @@ } public decimal Wei { get; } + + public override bool Equals(object? obj) + { + return obj is Ether ether && Wei == ether.Wei; + } + + public override int GetHashCode() + { + return HashCode.Combine(Wei); + } + + public override string ToString() + { + return $"{Wei} Wei"; + } } public class TestToken @@ -19,6 +34,16 @@ public decimal Amount { get; } + public override bool Equals(object? obj) + { + return obj is TestToken token && Amount == token.Amount; + } + + public override int GetHashCode() + { + return HashCode.Combine(Amount); + } + public override string ToString() { return $"{Amount} TestTokens"; diff --git a/Logging/TestLog.cs b/Logging/TestLog.cs index 8c4dc17..066d84a 100644 --- a/Logging/TestLog.cs +++ b/Logging/TestLog.cs @@ -14,7 +14,7 @@ namespace Logging methodName = GetMethodName(); fullName = Path.Combine(folder, methodName); - Log($"Begin: {methodName}"); + Log($"*** Begin: {methodName}"); } public LogFile CreateSubfile(string ext = "log") @@ -26,7 +26,7 @@ namespace Logging { var result = TestContext.CurrentContext.Result; - Log($"Finished: {methodName} = {result.Outcome.Status}"); + Log($"*** Finished: {methodName} = {result.Outcome.Status}"); if (!string.IsNullOrEmpty(result.Message)) { Log(result.Message); diff --git a/Tests/BasicTests/ExampleTests.cs b/Tests/BasicTests/ExampleTests.cs index 9c43ab1..dc86bfa 100644 --- a/Tests/BasicTests/ExampleTests.cs +++ b/Tests/BasicTests/ExampleTests.cs @@ -41,7 +41,7 @@ namespace Tests.BasicTests primary.ConnectToPeer(secondary); primary2.ConnectToPeer(secondary2); - Thread.Sleep(TimeSpan.FromMinutes(5)); + Thread.Sleep(TimeSpan.FromMinutes(2)); primary.Metrics.AssertThat("libp2p_peers", Is.EqualTo(1)); primary2.Metrics.AssertThat("libp2p_peers", Is.EqualTo(1)); @@ -55,7 +55,7 @@ namespace Tests.BasicTests .EnableMarketplace(initialBalance: 234.TestTokens()) .BringOnline()[0]; - Assert.That(primary.Marketplace.GetBalance(), Is.EqualTo(234)); + primary.Marketplace.AssertThatBalance(Is.EqualTo(234.TestTokens())); var secondary = SetupCodexNodes(1) .EnableMarketplace(initialBalance: 1000.TestTokens()) From 5b132bcb546f0f8c1125d1b52947bb0a64217c94 Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 19 Apr 2023 11:23:11 +0200 Subject: [PATCH 46/46] Fixes connection from codex node to its companion --- DistTestCore/Marketplace/GethContainerRecipe.cs | 2 +- Tests/BasicTests/ExampleTests.cs | 13 +++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/DistTestCore/Marketplace/GethContainerRecipe.cs b/DistTestCore/Marketplace/GethContainerRecipe.cs index 0c0ca91..5067cf7 100644 --- a/DistTestCore/Marketplace/GethContainerRecipe.cs +++ b/DistTestCore/Marketplace/GethContainerRecipe.cs @@ -43,7 +43,7 @@ namespace DistTestCore.Marketplace var bootPort = config.BootstrapNode.DiscoveryPort.Number; var bootstrapArg = $"--bootnodes enode://{bootPubKey}@{bootIp}:{bootPort}"; - return $"--port {port.Number} --discovery.port {discovery.Number} --authrpc.port {authRpc.Number} --http.port {httpPort.Number} --ws --ws.port {wsPort.Number} --nodiscover {bootstrapArg}"; + return $"--port {port.Number} --discovery.port {discovery.Number} --authrpc.port {authRpc.Number} --http.port {httpPort.Number} --ws --ws.addr 0.0.0.0 --ws.port {wsPort.Number} --nodiscover {bootstrapArg}"; } } } diff --git a/Tests/BasicTests/ExampleTests.cs b/Tests/BasicTests/ExampleTests.cs index dc86bfa..31cc56a 100644 --- a/Tests/BasicTests/ExampleTests.cs +++ b/Tests/BasicTests/ExampleTests.cs @@ -51,7 +51,7 @@ namespace Tests.BasicTests public void MarketplaceExample() { var primary = SetupCodexNodes(1) - .WithStorageQuota(10.GB()) + .WithStorageQuota(11.GB()) .EnableMarketplace(initialBalance: 234.TestTokens()) .BringOnline()[0]; @@ -79,13 +79,14 @@ namespace Tests.BasicTests proofProbability: 5, duration: TimeSpan.FromMinutes(2)); - Time.Sleep(TimeSpan.FromMinutes(3)); + Time.Sleep(TimeSpan.FromMinutes(1)); - primary.Marketplace.AssertThatBalance(Is.LessThan(20), "Collateral was not placed."); - var primaryBalance = primary.Marketplace.GetBalance(); + primary.Marketplace.AssertThatBalance(Is.LessThan(234.TestTokens()), "Collateral was not placed."); - secondary.Marketplace.AssertThatBalance(Is.LessThan(1000), "Contractor was not charged for storage."); - primary.Marketplace.AssertThatBalance(Is.GreaterThan(primaryBalance), "Storer was not paid for storage."); + Time.Sleep(TimeSpan.FromMinutes(2)); + + primary.Marketplace.AssertThatBalance(Is.GreaterThan(234.TestTokens()), "Storer was not paid for storage."); + secondary.Marketplace.AssertThatBalance(Is.LessThan(1000.TestTokens()), "Contractor was not charged for storage."); } } }