From c16055d7e477726accd61002f469e6755630e0a5 Mon Sep 17 00:00:00 2001 From: Eric Mastro Date: Tue, 2 May 2023 12:20:43 +1000 Subject: [PATCH 01/15] bump to .net 7 --- DistTestCore/DistTestCore.csproj | 2 +- KubernetesWorkflow/KubernetesWorkflow.csproj | 2 +- Logging/Logging.csproj | 2 +- LongTests/TestsLong.csproj | 2 +- Nethereum/NethereumWorkflow.csproj | 6 +++--- Tests/Tests.csproj | 2 +- Utils/Utils.csproj | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/DistTestCore/DistTestCore.csproj b/DistTestCore/DistTestCore.csproj index f7fe20a..6c4a133 100644 --- a/DistTestCore/DistTestCore.csproj +++ b/DistTestCore/DistTestCore.csproj @@ -1,7 +1,7 @@  - net6.0 + net7.0 DistTestCore enable enable diff --git a/KubernetesWorkflow/KubernetesWorkflow.csproj b/KubernetesWorkflow/KubernetesWorkflow.csproj index dbfbf96..cf95d1d 100644 --- a/KubernetesWorkflow/KubernetesWorkflow.csproj +++ b/KubernetesWorkflow/KubernetesWorkflow.csproj @@ -1,7 +1,7 @@ - net6.0 + net7.0 KubernetesWorkflow enable enable diff --git a/Logging/Logging.csproj b/Logging/Logging.csproj index fad5ec4..defbdcc 100644 --- a/Logging/Logging.csproj +++ b/Logging/Logging.csproj @@ -1,7 +1,7 @@ - net6.0 + net7.0 Logging enable enable diff --git a/LongTests/TestsLong.csproj b/LongTests/TestsLong.csproj index 136951d..90f1cd6 100644 --- a/LongTests/TestsLong.csproj +++ b/LongTests/TestsLong.csproj @@ -1,7 +1,7 @@ - net6.0 + net7.0 enable enable diff --git a/Nethereum/NethereumWorkflow.csproj b/Nethereum/NethereumWorkflow.csproj index 99771c5..fc0c5c3 100644 --- a/Nethereum/NethereumWorkflow.csproj +++ b/Nethereum/NethereumWorkflow.csproj @@ -1,16 +1,16 @@  - net6.0 + net7.0 NethereumWorkflow enable enable - + - + diff --git a/Tests/Tests.csproj b/Tests/Tests.csproj index 136951d..90f1cd6 100644 --- a/Tests/Tests.csproj +++ b/Tests/Tests.csproj @@ -1,7 +1,7 @@ - net6.0 + net7.0 enable enable diff --git a/Utils/Utils.csproj b/Utils/Utils.csproj index 42c2303..b728fc9 100644 --- a/Utils/Utils.csproj +++ b/Utils/Utils.csproj @@ -1,7 +1,7 @@ - net6.0 + net7.0 Utils enable enable From 050bb85d27a5af26d512ff48260928612d931edc Mon Sep 17 00:00:00 2001 From: Eric Mastro Date: Tue, 2 May 2023 15:16:10 +1000 Subject: [PATCH 02/15] add arch preprocessor directive Add preprocessor directive that checks if the current platform architecture is ARM64. The preprocessor directive checks for ARM64 architecture and changes which docker image to load in the recipes. # Conflicts: # DistTestCore/Codex/CodexContainerRecipe.cs --- DistTestCore/Codex/CodexContainerRecipe.cs | 11 ++++++++--- DistTestCore/DistTestCore.csproj | 7 +++++++ .../Marketplace/CodexContractsContainerRecipe.cs | 6 +++++- DistTestCore/Marketplace/GethContainerRecipe.cs | 6 +++++- 4 files changed, 25 insertions(+), 5 deletions(-) diff --git a/DistTestCore/Codex/CodexContainerRecipe.cs b/DistTestCore/Codex/CodexContainerRecipe.cs index c3095b1..ff263a7 100644 --- a/DistTestCore/Codex/CodexContainerRecipe.cs +++ b/DistTestCore/Codex/CodexContainerRecipe.cs @@ -1,12 +1,17 @@ -using DistTestCore.Marketplace; +using System.Runtime.InteropServices; +using DistTestCore.Marketplace; using KubernetesWorkflow; namespace DistTestCore.Codex { public class CodexContainerRecipe : ContainerRecipeFactory { - //public const string DockerImage = "thatbenbierens/nim-codex:sha-9716635"; - public const string DockerImage = "thatbenbierens/codexlocal:latest"; + #if Arm64 + public const string DockerImage = "emizzle/nim-codex-arm64:sha-c7af585"; + #else + //public const string DockerImage = "thatbenbierens/nim-codex:sha-9716635"; + public const string DockerImage = "thatbenbierens/codexlocal:latest"; + #endif public const string MetricsPortTag = "metrics_port"; protected override string Image => DockerImage; diff --git a/DistTestCore/DistTestCore.csproj b/DistTestCore/DistTestCore.csproj index 6c4a133..3e06fa7 100644 --- a/DistTestCore/DistTestCore.csproj +++ b/DistTestCore/DistTestCore.csproj @@ -5,6 +5,13 @@ DistTestCore enable enable + true + + + + + Arm64 diff --git a/DistTestCore/Marketplace/CodexContractsContainerRecipe.cs b/DistTestCore/Marketplace/CodexContractsContainerRecipe.cs index 42bbeca..00a9a7c 100644 --- a/DistTestCore/Marketplace/CodexContractsContainerRecipe.cs +++ b/DistTestCore/Marketplace/CodexContractsContainerRecipe.cs @@ -4,7 +4,11 @@ namespace DistTestCore.Marketplace { public class CodexContractsContainerRecipe : ContainerRecipeFactory { - public const string DockerImage = "thatbenbierens/codex-contracts-deployment"; + #if Arm64 + public const string DockerImage = "emizzle/codex-contracts-deployment:latest"; + #else + public const string DockerImage = "thatbenbierens/codex-contracts-deployment"; + #endif public const string MarketplaceAddressFilename = "/usr/app/deployments/codexdisttestnetwork/Marketplace.json"; public const string MarketplaceArtifactFilename = "/usr/app/artifacts/contracts/Marketplace.sol/Marketplace.json"; diff --git a/DistTestCore/Marketplace/GethContainerRecipe.cs b/DistTestCore/Marketplace/GethContainerRecipe.cs index f2693ac..008d293 100644 --- a/DistTestCore/Marketplace/GethContainerRecipe.cs +++ b/DistTestCore/Marketplace/GethContainerRecipe.cs @@ -4,7 +4,11 @@ namespace DistTestCore.Marketplace { public class GethContainerRecipe : ContainerRecipeFactory { - public const string DockerImage = "thatbenbierens/geth-confenv:latest"; + #if Arm64 + public const string DockerImage = "emizzle/geth-confenv:latest"; + #else + public const string DockerImage = "thatbenbierens/geth-confenv:latest"; + #endif public const string HttpPortTag = "http_port"; public const string DiscoveryPortTag = "disc_port"; private const string defaultArgs = "--ipcdisable --syncmode full"; From ea0a6908628df94163c7279af0e919a8d25eacaf Mon Sep 17 00:00:00 2001 From: Eric Mastro Date: Tue, 2 May 2023 15:29:05 +1000 Subject: [PATCH 03/15] clean up --- DistTestCore/DistTestCore.csproj | 3 --- 1 file changed, 3 deletions(-) diff --git a/DistTestCore/DistTestCore.csproj b/DistTestCore/DistTestCore.csproj index 3e06fa7..512d692 100644 --- a/DistTestCore/DistTestCore.csproj +++ b/DistTestCore/DistTestCore.csproj @@ -6,9 +6,6 @@ enable enable true - - Arm64 From 01c8238311bdb74f4784022155c7ff77e5f541cd Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 3 May 2023 10:21:15 +0200 Subject: [PATCH 04/15] Applies faster geth image --- DistTestCore/GethStarter.cs | 9 +- .../CodexContractsContainerRecipe.cs | 2 +- .../Marketplace/ContainerInfoExtractor.cs | 36 ++++--- .../Marketplace/GethBootstrapNodeInfo.cs | 23 +++-- .../Marketplace/GethBootstrapNodeStarter.cs | 10 +- .../Marketplace/GethCompanionNodeInfo.cs | 13 ++- .../Marketplace/GethCompanionNodeStarter.cs | 53 ++++++----- .../Marketplace/GethContainerRecipe.cs | 32 ++++--- DistTestCore/Marketplace/GethStartupConfig.cs | 4 +- DistTestCore/Marketplace/MarketplaceAccess.cs | 4 +- .../Marketplace/MarketplaceAccessFactory.cs | 6 +- Nethereum/NethereumInteraction.cs | 93 ++++++++----------- Nethereum/NethereumInteractionCreator.cs | 6 +- Tests/BasicTests/ExampleTests.cs | 6 +- 14 files changed, 143 insertions(+), 154 deletions(-) diff --git a/DistTestCore/GethStarter.cs b/DistTestCore/GethStarter.cs index 914b680..3ac8ce0 100644 --- a/DistTestCore/GethStarter.cs +++ b/DistTestCore/GethStarter.cs @@ -36,13 +36,8 @@ namespace DistTestCore var interaction = marketplaceNetwork.StartInteraction(lifecycle.Log); var tokenAddress = marketplaceNetwork.Marketplace.TokenAddress; - foreach (var account in companionNode.Accounts) - { - interaction.TransferWeiTo(account.Account, marketplaceConfig.InitialEth.Wei); - interaction.MintTestTokens(account.Account, marketplaceConfig.InitialTestTokens.Amount, tokenAddress); - } - - interaction.WaitForAllTransactions(); + var accounts = companionNode.Accounts.Select(a => a.Account).ToArray(); + interaction.MintTestTokens(accounts, marketplaceConfig.InitialTestTokens.Amount, tokenAddress); } private GethStartResult CreateGethStartResult(MarketplaceNetwork marketplaceNetwork, GethCompanionNodeInfo companionNode) diff --git a/DistTestCore/Marketplace/CodexContractsContainerRecipe.cs b/DistTestCore/Marketplace/CodexContractsContainerRecipe.cs index 00a9a7c..1df6d2e 100644 --- a/DistTestCore/Marketplace/CodexContractsContainerRecipe.cs +++ b/DistTestCore/Marketplace/CodexContractsContainerRecipe.cs @@ -7,7 +7,7 @@ namespace DistTestCore.Marketplace #if Arm64 public const string DockerImage = "emizzle/codex-contracts-deployment:latest"; #else - public const string DockerImage = "thatbenbierens/codex-contracts-deployment"; + public const string DockerImage = "thatbenbierens/codex-contracts-deployment:nomint"; #endif public const string MarketplaceAddressFilename = "/usr/app/deployments/codexdisttestnetwork/Marketplace.json"; public const string MarketplaceArtifactFilename = "/usr/app/artifacts/contracts/Marketplace.sol/Marketplace.json"; diff --git a/DistTestCore/Marketplace/ContainerInfoExtractor.cs b/DistTestCore/Marketplace/ContainerInfoExtractor.cs index 281a613..f99827b 100644 --- a/DistTestCore/Marketplace/ContainerInfoExtractor.cs +++ b/DistTestCore/Marketplace/ContainerInfoExtractor.cs @@ -19,13 +19,14 @@ namespace DistTestCore.Marketplace this.container = container; } - public string ExtractAccount(int? orderNumber) + public AllGethAccounts ExtractAccounts() { log.Debug(); - var account = Retry(() => FetchAccount(orderNumber)); - if (string.IsNullOrEmpty(account)) throw new InvalidOperationException("Unable to fetch account for geth node. Test infra failure."); + var accountsCsv = Retry(() => FetchAccountsCsv()); + if (string.IsNullOrEmpty(accountsCsv)) throw new InvalidOperationException("Unable to fetch accounts.csv for geth node. Test infra failure."); - return account; + var lines = accountsCsv.Split('\n'); + return new AllGethAccounts(lines.Select(ParseLineToAccount).ToArray()); } public string ExtractPubKey() @@ -37,15 +38,6 @@ namespace DistTestCore.Marketplace return pubKey; } - public string ExtractPrivateKey(int? orderNumber) - { - log.Debug(); - var privKey = Retry(() => FetchPrivateKey(orderNumber)); - if (string.IsNullOrEmpty(privKey)) throw new InvalidOperationException("Unable to fetch private key from geth node. Test infra failure."); - - return privKey; - } - public string ExtractMarketplaceAddress() { log.Debug(); @@ -88,14 +80,9 @@ namespace DistTestCore.Marketplace } } - private string FetchAccount(int? orderNumber) + private string FetchAccountsCsv() { - return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.GetAccountFilename(orderNumber)); - } - - private string FetchPrivateKey(int? orderNumber) - { - return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.GetPrivateKeyFilename(orderNumber)); + return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.AccountsFilename); } private string FetchMarketplaceAddress() @@ -120,6 +107,15 @@ namespace DistTestCore.Marketplace workflow.DownloadContainerLog(container, enodeFinder); return enodeFinder.GetPubKey(); } + + private GethAccount ParseLineToAccount(string l) + { + var tokens = l.Replace("\r", "").Split(','); + if (tokens.Length != 2) throw new InvalidOperationException(); + var account = tokens[0]; + var privateKey = tokens[1]; + return new GethAccount(account, privateKey); + } } public class PubKeyFinder : LogHandler, ILogHandler diff --git a/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs b/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs index 0c19fbf..b59fb80 100644 --- a/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs +++ b/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs @@ -6,19 +6,19 @@ namespace DistTestCore.Marketplace { public class GethBootstrapNodeInfo { - public GethBootstrapNodeInfo(RunningContainers runningContainers, string account, string pubKey, string privateKey, Port discoveryPort) + public GethBootstrapNodeInfo(RunningContainers runningContainers, AllGethAccounts allAccounts, string pubKey, Port discoveryPort) { RunningContainers = runningContainers; - Account = account; + AllAccounts = allAccounts; + Account = allAccounts.Accounts[0]; PubKey = pubKey; - PrivateKey = privateKey; DiscoveryPort = discoveryPort; } public RunningContainers RunningContainers { get; } - public string Account { get; } + public AllGethAccounts AllAccounts { get; } + public GethAccount Account { get; } public string PubKey { get; } - public string PrivateKey { get; } public Port DiscoveryPort { get; } public NethereumInteraction StartInteraction(BaseLog log) @@ -26,10 +26,19 @@ namespace DistTestCore.Marketplace var ip = RunningContainers.RunningPod.Cluster.IP; var port = RunningContainers.Containers[0].ServicePorts[0].Number; var account = Account; - var privateKey = PrivateKey; - var creator = new NethereumInteractionCreator(log, ip, port, account, privateKey); + var creator = new NethereumInteractionCreator(log, ip, port, account.PrivateKey); return creator.CreateWorkflow(); } } + + public class AllGethAccounts + { + public GethAccount[] Accounts { get; } + + public AllGethAccounts(GethAccount[] accounts) + { + Accounts = accounts; + } + } } diff --git a/DistTestCore/Marketplace/GethBootstrapNodeStarter.cs b/DistTestCore/Marketplace/GethBootstrapNodeStarter.cs index 300893b..56296ff 100644 --- a/DistTestCore/Marketplace/GethBootstrapNodeStarter.cs +++ b/DistTestCore/Marketplace/GethBootstrapNodeStarter.cs @@ -20,20 +20,20 @@ namespace DistTestCore.Marketplace var bootstrapContainer = containers.Containers[0]; var extractor = new ContainerInfoExtractor(lifecycle.Log, workflow, bootstrapContainer); - var account = extractor.ExtractAccount(null); + var accounts = extractor.ExtractAccounts(); var pubKey = extractor.ExtractPubKey(); - var privateKey = extractor.ExtractPrivateKey(null); var discoveryPort = bootstrapContainer.Recipe.GetPortByTag(GethContainerRecipe.DiscoveryPortTag); + var result = new GethBootstrapNodeInfo(containers, accounts, pubKey, discoveryPort); - LogEnd($"Geth bootstrap node started with account '{account}'"); + LogEnd($"Geth bootstrap node started with account '{result.Account.Account}'"); - return new GethBootstrapNodeInfo(containers, account, pubKey, privateKey, discoveryPort); + return result; } private StartupConfig CreateBootstrapStartupConfig() { var config = new StartupConfig(); - config.Add(new GethStartupConfig(true, null!, 0)); + config.Add(new GethStartupConfig(true, null!, 0, 0)); return config; } } diff --git a/DistTestCore/Marketplace/GethCompanionNodeInfo.cs b/DistTestCore/Marketplace/GethCompanionNodeInfo.cs index 64e8ad9..5731ab3 100644 --- a/DistTestCore/Marketplace/GethCompanionNodeInfo.cs +++ b/DistTestCore/Marketplace/GethCompanionNodeInfo.cs @@ -6,30 +6,29 @@ namespace DistTestCore.Marketplace { public class GethCompanionNodeInfo { - public GethCompanionNodeInfo(RunningContainer runningContainer, GethCompanionAccount[] accounts) + public GethCompanionNodeInfo(RunningContainer runningContainer, GethAccount[] accounts) { RunningContainer = runningContainer; Accounts = accounts; } public RunningContainer RunningContainer { get; } - public GethCompanionAccount[] Accounts { get; } + public GethAccount[] Accounts { get; } - public NethereumInteraction StartInteraction(BaseLog log, GethCompanionAccount account) + public NethereumInteraction StartInteraction(BaseLog log, GethAccount account) { var ip = RunningContainer.Pod.Cluster.IP; var port = RunningContainer.ServicePorts[0].Number; - var accountStr = account.Account; var privateKey = account.PrivateKey; - var creator = new NethereumInteractionCreator(log, ip, port, accountStr, privateKey); + var creator = new NethereumInteractionCreator(log, ip, port, privateKey); return creator.CreateWorkflow(); } } - public class GethCompanionAccount + public class GethAccount { - public GethCompanionAccount(string account, string privateKey) + public GethAccount(string account, string privateKey) { Account = account; PrivateKey = privateKey; diff --git a/DistTestCore/Marketplace/GethCompanionNodeStarter.cs b/DistTestCore/Marketplace/GethCompanionNodeStarter.cs index 80c1bcd..a71c661 100644 --- a/DistTestCore/Marketplace/GethCompanionNodeStarter.cs +++ b/DistTestCore/Marketplace/GethCompanionNodeStarter.cs @@ -5,6 +5,8 @@ namespace DistTestCore.Marketplace { public class GethCompanionNodeStarter : BaseStarter { + private int companionAccountIndex = 0; + public GethCompanionNodeStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator) : base(lifecycle, workflowCreator) { @@ -14,53 +16,43 @@ namespace DistTestCore.Marketplace { LogStart($"Initializing companion for {codexSetup.NumberOfNodes} Codex nodes."); - var startupConfig = CreateCompanionNodeStartupConfig(marketplace.Bootstrap, codexSetup.NumberOfNodes); + var config = CreateCompanionNodeStartupConfig(marketplace.Bootstrap, codexSetup.NumberOfNodes); var workflow = workflowCreator.CreateWorkflow(); - var containers = workflow.Start(1, Location.Unspecified, new GethContainerRecipe(), startupConfig); - WaitForAccountCreation(codexSetup.NumberOfNodes); + var containers = workflow.Start(1, Location.Unspecified, new GethContainerRecipe(), CreateStartupConfig(config)); if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected one Geth companion node to be created. Test infra failure."); var container = containers.Containers[0]; - var node = CreateCompanionInfo(workflow, container, codexSetup.NumberOfNodes); + var node = CreateCompanionInfo(container, marketplace, config); EnsureCompanionNodeIsSynced(node, marketplace); LogEnd($"Initialized one companion node for {codexSetup.NumberOfNodes} Codex nodes. Their accounts: [{string.Join(",", node.Accounts.Select(a => a.Account))}]"); return node; } - private void WaitForAccountCreation(int numberOfNodes) + private GethCompanionNodeInfo CreateCompanionInfo(RunningContainer container, MarketplaceNetwork marketplace, GethStartupConfig config) { - // We wait proportional to the number of account the node has to create. It takes a few seconds for each one to generate the keys and create the files - // we will be trying to read in 'ExtractAccount', later on in the start-up process. - Time.Sleep(TimeSpan.FromSeconds(4.5 * numberOfNodes)); - } - - private GethCompanionNodeInfo CreateCompanionInfo(StartupWorkflow workflow, RunningContainer container, int numberOfAccounts) - { - var extractor = new ContainerInfoExtractor(lifecycle.Log, workflow, container); - var accounts = ExtractAccounts(extractor, numberOfAccounts).ToArray(); + var accounts = ExtractAccounts(marketplace, config); return new GethCompanionNodeInfo(container, accounts); } - private IEnumerable ExtractAccounts(ContainerInfoExtractor extractor, int numberOfAccounts) + private static GethAccount[] ExtractAccounts(MarketplaceNetwork marketplace, GethStartupConfig config) { - for (int i = 0; i < numberOfAccounts; i++) yield return ExtractAccount(extractor, i + 1); - } - - private GethCompanionAccount ExtractAccount(ContainerInfoExtractor extractor, int orderNumber) - { - var account = extractor.ExtractAccount(orderNumber); - var privKey = extractor.ExtractPrivateKey(orderNumber); - return new GethCompanionAccount(account, privKey); + return marketplace.Bootstrap.AllAccounts.Accounts + .Skip(1 + config.CompanionAccountStartIndex) + .Take(config.NumberOfCompanionAccounts) + .ToArray(); } private void EnsureCompanionNodeIsSynced(GethCompanionNodeInfo node, MarketplaceNetwork marketplace) { try { - var interaction = node.StartInteraction(lifecycle.Log, node.Accounts.First()); - interaction.EnsureSynced(marketplace.Marketplace.Address, marketplace.Marketplace.Abi); + Time.WaitUntil(() => + { + var interaction = node.StartInteraction(lifecycle.Log, node.Accounts.First()); + return interaction.IsSynced(marketplace.Marketplace.Address, marketplace.Marketplace.Abi); + }, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(3)); } catch (Exception e) { @@ -68,10 +60,17 @@ namespace DistTestCore.Marketplace } } - private StartupConfig CreateCompanionNodeStartupConfig(GethBootstrapNodeInfo bootstrapNode, int numberOfAccounts) + private GethStartupConfig CreateCompanionNodeStartupConfig(GethBootstrapNodeInfo bootstrapNode, int numberOfAccounts) + { + var config = new GethStartupConfig(false, bootstrapNode, companionAccountIndex, numberOfAccounts); + companionAccountIndex += numberOfAccounts; + return config; + } + + private StartupConfig CreateStartupConfig(GethStartupConfig gethConfig) { var config = new StartupConfig(); - config.Add(new GethStartupConfig(false, bootstrapNode, numberOfAccounts)); + config.Add(gethConfig); return config; } } diff --git a/DistTestCore/Marketplace/GethContainerRecipe.cs b/DistTestCore/Marketplace/GethContainerRecipe.cs index 008d293..fa95054 100644 --- a/DistTestCore/Marketplace/GethContainerRecipe.cs +++ b/DistTestCore/Marketplace/GethContainerRecipe.cs @@ -7,23 +7,14 @@ namespace DistTestCore.Marketplace #if Arm64 public const string DockerImage = "emizzle/geth-confenv:latest"; #else - public const string DockerImage = "thatbenbierens/geth-confenv:latest"; + public const string DockerImage = "thatbenbierens/geth-confenv:onethousand"; #endif + public const string HttpPortTag = "http_port"; public const string DiscoveryPortTag = "disc_port"; private const string defaultArgs = "--ipcdisable --syncmode full"; - public static string GetAccountFilename(int? orderNumber) - { - if (orderNumber == null) return "account_string.txt"; - return $"account_string_{orderNumber.Value}.txt"; - } - - public static string GetPrivateKeyFilename(int? orderNumber) - { - if (orderNumber == null) return "private.key"; - return $"private_{orderNumber.Value}.key"; - } + public const string AccountsFilename = "accounts.csv"; protected override string Image => DockerImage; @@ -50,14 +41,17 @@ namespace DistTestCore.Marketplace private string CreateBootstapArgs(Port discovery) { - AddEnvVar("IS_BOOTSTRAP", "1"); + AddEnvVar("ENABLE_MINER", "1"); + UnlockAccounts(0, 1); var exposedPort = AddExposedPort(tag: HttpPortTag); return $"--http.port {exposedPort.Number} --port {discovery.Number} --discovery.port {discovery.Number} {defaultArgs}"; } private string CreateCompanionArgs(Port discovery, GethStartupConfig config) { - AddEnvVar("NUMBER_OF_ACCOUNTS", config.NumberOfCompanionAccounts.ToString()); + UnlockAccounts( + config.CompanionAccountStartIndex + 1, + config.NumberOfCompanionAccounts); var port = AddInternalPort(); var authRpc = AddInternalPort(); @@ -70,5 +64,15 @@ namespace DistTestCore.Marketplace return $"--port {port.Number} --discovery.port {discovery.Number} --authrpc.port {authRpc.Number} --http.addr 0.0.0.0 --http.port {httpPort.Number} --ws --ws.addr 0.0.0.0 --ws.port {httpPort.Number} {bootstrapArg} {defaultArgs}"; } + + private void UnlockAccounts(int startIndex, int numberOfAccounts) + { + if (startIndex < 0) throw new ArgumentException(); + if (numberOfAccounts < 1) throw new ArgumentException(); + if (startIndex + numberOfAccounts > 1000) throw new ArgumentException("Out of accounts!"); + + AddEnvVar("UNLOCK_START_INDEX", startIndex.ToString()); + AddEnvVar("UNLOCK_NUMBER", numberOfAccounts.ToString()); + } } } diff --git a/DistTestCore/Marketplace/GethStartupConfig.cs b/DistTestCore/Marketplace/GethStartupConfig.cs index bc9671f..7aee078 100644 --- a/DistTestCore/Marketplace/GethStartupConfig.cs +++ b/DistTestCore/Marketplace/GethStartupConfig.cs @@ -2,15 +2,17 @@ { public class GethStartupConfig { - public GethStartupConfig(bool isBootstrapNode, GethBootstrapNodeInfo bootstrapNode, int numberOfCompanionAccounts) + public GethStartupConfig(bool isBootstrapNode, GethBootstrapNodeInfo bootstrapNode, int companionAccountStartIndex, int numberOfCompanionAccounts) { IsBootstrapNode = isBootstrapNode; BootstrapNode = bootstrapNode; + CompanionAccountStartIndex = companionAccountStartIndex; NumberOfCompanionAccounts = numberOfCompanionAccounts; } public bool IsBootstrapNode { get; } public GethBootstrapNodeInfo BootstrapNode { get; } + public int CompanionAccountStartIndex { get; } public int NumberOfCompanionAccounts { get; } } } diff --git a/DistTestCore/Marketplace/MarketplaceAccess.cs b/DistTestCore/Marketplace/MarketplaceAccess.cs index b66679c..1cc19e7 100644 --- a/DistTestCore/Marketplace/MarketplaceAccess.cs +++ b/DistTestCore/Marketplace/MarketplaceAccess.cs @@ -19,10 +19,10 @@ namespace DistTestCore.Marketplace { private readonly TestLog log; private readonly MarketplaceNetwork marketplaceNetwork; - private readonly GethCompanionAccount account; + private readonly GethAccount account; private readonly CodexAccess codexAccess; - public MarketplaceAccess(TestLog log, MarketplaceNetwork marketplaceNetwork, GethCompanionAccount account, CodexAccess codexAccess) + public MarketplaceAccess(TestLog log, MarketplaceNetwork marketplaceNetwork, GethAccount account, CodexAccess codexAccess) { this.log = log; this.marketplaceNetwork = marketplaceNetwork; diff --git a/DistTestCore/Marketplace/MarketplaceAccessFactory.cs b/DistTestCore/Marketplace/MarketplaceAccessFactory.cs index ea4786c..cd37d81 100644 --- a/DistTestCore/Marketplace/MarketplaceAccessFactory.cs +++ b/DistTestCore/Marketplace/MarketplaceAccessFactory.cs @@ -33,10 +33,10 @@ namespace DistTestCore.Marketplace return new MarketplaceAccess(log, marketplaceNetwork, companionNode, access); } - private GethCompanionAccount GetGethCompanionNode(CodexAccess access) + private GethAccount GetGethCompanionNode(CodexAccess access) { - var account = access.Container.Recipe.Additionals.Single(a => a is GethCompanionAccount); - return (GethCompanionAccount)account; + var account = access.Container.Recipe.Additionals.Single(a => a is GethAccount); + return (GethAccount)account; } } } diff --git a/Nethereum/NethereumInteraction.cs b/Nethereum/NethereumInteraction.cs index 3d74e54..46d6da0 100644 --- a/Nethereum/NethereumInteraction.cs +++ b/Nethereum/NethereumInteraction.cs @@ -10,16 +10,13 @@ namespace NethereumWorkflow { public class NethereumInteraction { - private readonly List openTasks = new List(); private readonly BaseLog log; private readonly Web3 web3; - private readonly string rootAccount; - internal NethereumInteraction(BaseLog log, Web3 web3, string rootAccount) + internal NethereumInteraction(BaseLog log, Web3 web3) { this.log = log; this.web3 = web3; - this.rootAccount = rootAccount; } public string GetTokenAddress(string marketplaceAddress) @@ -31,29 +28,13 @@ namespace NethereumWorkflow return Time.Wait(handler.QueryAsync(marketplaceAddress, function)); } - public void TransferWeiTo(string account, decimal amount) + public void MintTestTokens(string[] accounts, decimal amount, string tokenAddress) { - log.Debug($"{amount} --> {account}"); - if (amount < 1 || string.IsNullOrEmpty(account)) throw new ArgumentException("Invalid arguments for AddToBalance"); + if (amount < 1 || accounts.Length < 1) throw new ArgumentException("Invalid arguments for MintTestTokens"); - var value = ToHexBig(amount); - var transactionId = Time.Wait(web3.Eth.TransactionManager.SendTransactionAsync(rootAccount, account, value)); - openTasks.Add(web3.Eth.TransactionManager.TransactionReceiptService.PollForReceiptAsync(transactionId)); - } + var tasks = accounts.Select(a => MintTokens(a, amount, tokenAddress)); - public void MintTestTokens(string account, decimal amount, string tokenAddress) - { - log.Debug($"({tokenAddress}) {amount} --> {account}"); - if (amount < 1 || string.IsNullOrEmpty(account)) throw new ArgumentException("Invalid arguments for MintTestTokens"); - - var function = new MintTokensFunction - { - Holder = account, - Amount = ToBig(amount) - }; - - var handler = web3.Eth.GetContractTransactionHandler(); - openTasks.Add(handler.SendRequestAndWaitForReceiptAsync(tokenAddress, function)); + Task.WaitAll(tasks.ToArray()); } public decimal GetBalance(string tokenAddress, string account) @@ -68,48 +49,54 @@ namespace NethereumWorkflow return ToDecimal(Time.Wait(handler.QueryAsync(tokenAddress, function))); } - public void WaitForAllTransactions() + public bool IsSynced(string marketplaceAddress, string marketplaceAbi) { - var tasks = openTasks.ToArray(); - openTasks.Clear(); - - Task.WaitAll(tasks); + try + { + return IsBlockNumberOK() && IsContractAvailable(marketplaceAddress, marketplaceAbi); + } + catch + { + return false; + } } - public void EnsureSynced(string marketplaceAddress, string marketplaceAbi) + private Task MintTokens(string account, decimal amount, string tokenAddress) { - WaitUntilSynced(); - WaitForContract(marketplaceAddress, marketplaceAbi); + log.Debug($"({tokenAddress}) {amount} --> {account}"); + if (string.IsNullOrEmpty(account)) throw new ArgumentException("Invalid arguments for MintTestTokens"); + + var function = new MintTokensFunction + { + Holder = account, + Amount = ToBig(amount) + }; + + var handler = web3.Eth.GetContractTransactionHandler(); + return handler.SendRequestAndWaitForReceiptAsync(tokenAddress, function); } - private void WaitUntilSynced() + private bool IsBlockNumberOK() { log.Debug(); - Time.WaitUntil(() => - { - var sync = Time.Wait(web3.Eth.Syncing.SendRequestAsync()); - var number = Time.Wait(web3.Eth.Blocks.GetBlockNumber.SendRequestAsync()); - var numberOfBlocks = ToDecimal(number); - return !sync.IsSyncing && numberOfBlocks > 256; - - }, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(3)); + var sync = Time.Wait(web3.Eth.Syncing.SendRequestAsync()); + var number = Time.Wait(web3.Eth.Blocks.GetBlockNumber.SendRequestAsync()); + var numberOfBlocks = ToDecimal(number); + return !sync.IsSyncing && numberOfBlocks > 256; } - private void WaitForContract(string marketplaceAddress, string marketplaceAbi) + private bool IsContractAvailable(string marketplaceAddress, string marketplaceAbi) { log.Debug(); - Time.WaitUntil(() => + try { - try - { - var contract = web3.Eth.GetContract(marketplaceAbi, marketplaceAddress); - return contract != null; - } - catch - { - return false; - } - }, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(3)); + var contract = web3.Eth.GetContract(marketplaceAbi, marketplaceAddress); + return contract != null; + } + catch + { + return false; + } } private HexBigInteger ToHexBig(decimal amount) diff --git a/Nethereum/NethereumInteractionCreator.cs b/Nethereum/NethereumInteractionCreator.cs index ff7e9fd..3ac0431 100644 --- a/Nethereum/NethereumInteractionCreator.cs +++ b/Nethereum/NethereumInteractionCreator.cs @@ -8,21 +8,19 @@ namespace NethereumWorkflow private readonly BaseLog log; private readonly string ip; private readonly int port; - private readonly string rootAccount; private readonly string privateKey; - public NethereumInteractionCreator(BaseLog log, string ip, int port, string rootAccount, string privateKey) + public NethereumInteractionCreator(BaseLog log, string ip, int port, string privateKey) { this.log = log; this.ip = ip; this.port = port; - this.rootAccount = rootAccount; this.privateKey = privateKey; } public NethereumInteraction CreateWorkflow() { - return new NethereumInteraction(log, CreateWeb3(), rootAccount); + return new NethereumInteraction(log, CreateWeb3()); } private Web3 CreateWeb3() diff --git a/Tests/BasicTests/ExampleTests.cs b/Tests/BasicTests/ExampleTests.cs index afafe9a..6951433 100644 --- a/Tests/BasicTests/ExampleTests.cs +++ b/Tests/BasicTests/ExampleTests.cs @@ -71,13 +71,13 @@ namespace Tests.BasicTests requiredCollateral: 10.TestTokens(), minRequiredNumberOfNodes: 1, proofProbability: 5, - duration: TimeSpan.FromMinutes(2)); + duration: TimeSpan.FromMinutes(1)); - Time.Sleep(TimeSpan.FromMinutes(1)); + Time.Sleep(TimeSpan.FromSeconds(10)); seller.Marketplace.AssertThatBalance(Is.LessThan(sellerInitialBalance), "Collateral was not placed."); - Time.Sleep(TimeSpan.FromMinutes(2)); + Time.Sleep(TimeSpan.FromMinutes(1)); seller.Marketplace.AssertThatBalance(Is.GreaterThan(sellerInitialBalance), "Seller was not paid for storage."); buyer.Marketplace.AssertThatBalance(Is.LessThan(buyerInitialBalance), "Buyer was not charged for storage."); From 79a40904e4c5f1a23eea0fdf82b15374e26b2f58 Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 3 May 2023 14:18:37 +0200 Subject: [PATCH 05/15] setup --- DistTestCore/CodexStarter.cs | 2 +- DistTestCore/Configuration.cs | 2 +- DistTestCore/DistTest.cs | 3 +- KubernetesWorkflow/ApplicationLifecycle.cs | 36 +++++++++ KubernetesWorkflow/Configuration.cs | 6 +- KubernetesWorkflow/K8sController.cs | 87 +++++++++++++++------- KubernetesWorkflow/StartupWorkflow.cs | 16 +++- KubernetesWorkflow/WorkflowCreator.cs | 5 +- Tests/BasicTests/TwoClientTests.cs | 1 + 9 files changed, 118 insertions(+), 40 deletions(-) create mode 100644 KubernetesWorkflow/ApplicationLifecycle.cs diff --git a/DistTestCore/CodexStarter.cs b/DistTestCore/CodexStarter.cs index 900967a..52c70ca 100644 --- a/DistTestCore/CodexStarter.cs +++ b/DistTestCore/CodexStarter.cs @@ -44,7 +44,7 @@ namespace DistTestCore public void DeleteAllResources() { var workflow = CreateWorkflow(); - workflow.DeleteAllResources(); + workflow.DeleteTestResources(); RunningGroups.Clear(); } diff --git a/DistTestCore/Configuration.cs b/DistTestCore/Configuration.cs index 5de9afc..b60a11f 100644 --- a/DistTestCore/Configuration.cs +++ b/DistTestCore/Configuration.cs @@ -7,7 +7,7 @@ namespace DistTestCore public KubernetesWorkflow.Configuration GetK8sConfiguration() { return new KubernetesWorkflow.Configuration( - k8sNamespace: "codex-test-ns", + k8sNamespacePrefix: "ct-", kubeConfigFile: null, operationTimeout: Timing.K8sOperationTimeout(), retryDelay: Timing.K8sServiceDelay(), diff --git a/DistTestCore/DistTest.cs b/DistTestCore/DistTest.cs index a85f689..b0686e1 100644 --- a/DistTestCore/DistTest.cs +++ b/DistTestCore/DistTest.cs @@ -153,7 +153,8 @@ namespace DistTestCore private void CreateNewTestLifecycle() { - Stopwatch.Measure(fixtureLog, $"Setup for {GetCurrentTestName()}", () => + var testName = GetCurrentTestName(); + Stopwatch.Measure(fixtureLog, $"Setup for {testName}", () => { lifecycle = new TestLifecycle(fixtureLog.CreateTestLog(), configuration); testStart = DateTime.UtcNow; diff --git a/KubernetesWorkflow/ApplicationLifecycle.cs b/KubernetesWorkflow/ApplicationLifecycle.cs new file mode 100644 index 0000000..5289573 --- /dev/null +++ b/KubernetesWorkflow/ApplicationLifecycle.cs @@ -0,0 +1,36 @@ +using Utils; + +namespace KubernetesWorkflow +{ + public class ApplicationLifecycle + { + private static ApplicationLifecycle? instance; + private readonly NumberSource servicePortNumberSource = new NumberSource(30001); + private readonly NumberSource namespaceNumberSource = new NumberSource(0); + + private ApplicationLifecycle() + { + } + + public static ApplicationLifecycle Instance + { + // I know singletons are quite evil. But we need to be sure this object is created only once + // and persists for the entire application lifecycle. + get + { + if (instance == null) instance = new ApplicationLifecycle(); + return instance; + } + } + + public NumberSource GetServiceNumberSource() + { + return servicePortNumberSource; + } + + public string GetTestNamespace() + { + return namespaceNumberSource.GetNextNumber().ToString("D5"); + } + } +} diff --git a/KubernetesWorkflow/Configuration.cs b/KubernetesWorkflow/Configuration.cs index b5a4779..f94924d 100644 --- a/KubernetesWorkflow/Configuration.cs +++ b/KubernetesWorkflow/Configuration.cs @@ -2,16 +2,16 @@ { public class Configuration { - public Configuration(string k8sNamespace, string? kubeConfigFile, TimeSpan operationTimeout, TimeSpan retryDelay, ConfigurationLocationEntry[] locationMap) + public Configuration(string k8sNamespacePrefix, string? kubeConfigFile, TimeSpan operationTimeout, TimeSpan retryDelay, ConfigurationLocationEntry[] locationMap) { - K8sNamespace = k8sNamespace; + K8sNamespacePrefix = k8sNamespacePrefix; KubeConfigFile = kubeConfigFile; OperationTimeout = operationTimeout; RetryDelay = retryDelay; LocationMap = locationMap; } - public string K8sNamespace { get; } + public string K8sNamespacePrefix { get; } public string? KubeConfigFile { get; } public TimeSpan OperationTimeout { get; } public TimeSpan RetryDelay { get; } diff --git a/KubernetesWorkflow/K8sController.cs b/KubernetesWorkflow/K8sController.cs index 21e8266..118101d 100644 --- a/KubernetesWorkflow/K8sController.cs +++ b/KubernetesWorkflow/K8sController.cs @@ -11,15 +11,16 @@ namespace KubernetesWorkflow private readonly K8sCluster cluster; private readonly KnownK8sPods knownPods; private readonly WorkflowNumberSource workflowNumberSource; + private readonly string testNamespace; private readonly Kubernetes client; - public K8sController(BaseLog log, K8sCluster cluster, KnownK8sPods knownPods, WorkflowNumberSource workflowNumberSource) + public K8sController(BaseLog log, K8sCluster cluster, KnownK8sPods knownPods, WorkflowNumberSource workflowNumberSource, string testNamespace) { this.log = log; this.cluster = cluster; this.knownPods = knownPods; this.workflowNumberSource = workflowNumberSource; - + this.testNamespace = testNamespace; client = new Kubernetes(cluster.GetK8sClientConfig()); } @@ -52,14 +53,14 @@ namespace KubernetesWorkflow public void DownloadPodLog(RunningPod pod, ContainerRecipe recipe, ILogHandler logHandler) { log.Debug(); - using var stream = client.ReadNamespacedPodLog(pod.Name, K8sNamespace, recipe.Name); + using var stream = client.ReadNamespacedPodLog(pod.Name, K8sTestNamespace, recipe.Name); logHandler.Log(stream); } public string ExecuteCommand(RunningPod pod, string containerName, string command, params string[] args) { log.Debug($"{containerName}: {command} ({string.Join(",", args)})"); - var runner = new CommandRunner(client, K8sNamespace, pod, containerName, command, args); + var runner = new CommandRunner(client, K8sTestNamespace, pod, containerName, command, args); runner.Run(); return runner.GetStdOut(); } @@ -67,11 +68,39 @@ namespace KubernetesWorkflow public void DeleteAllResources() { log.Debug(); - DeleteNamespace(); + var all = client.ListNamespace().Items; + var namespaces = all.Select(n => n.Name()).Where(n => n.StartsWith(cluster.Configuration.K8sNamespacePrefix)); + + foreach (var ns in namespaces) + { + DeleteNamespace(ns); + } + foreach (var ns in namespaces) + { + WaitUntilNamespaceDeleted(ns); + } + } + + public void DeleteTestNamespace() + { + log.Debug(); + if (IsTestNamespaceOnline()) + { + client.DeleteNamespace(K8sTestNamespace, null, null, gracePeriodSeconds: 0); + } WaitUntilNamespaceDeleted(); } + public void DeleteNamespace(string ns) + { + log.Debug(); + if (IsNamespaceOnline(ns)) + { + client.DeleteNamespace(ns, null, null, gracePeriodSeconds: 0); + } + } + #region Namespace management private void EnsureTestNamespace() @@ -83,30 +112,27 @@ namespace KubernetesWorkflow ApiVersion = "v1", Metadata = new V1ObjectMeta { - Name = K8sNamespace, - Labels = new Dictionary { { "name", K8sNamespace } } + Name = K8sTestNamespace, + Labels = new Dictionary { { "name", K8sTestNamespace } } } }; client.CreateNamespace(namespaceSpec); WaitUntilNamespaceCreated(); } - private void DeleteNamespace() + private string K8sTestNamespace { - if (IsTestNamespaceOnline()) - { - client.DeleteNamespace(K8sNamespace, null, null, gracePeriodSeconds: 0); - } - } - - private string K8sNamespace - { - get { return cluster.Configuration.K8sNamespace; } + get { return cluster.Configuration.K8sNamespacePrefix + testNamespace; } } private bool IsTestNamespaceOnline() { - return client.ListNamespace().Items.Any(n => n.Metadata.Name == K8sNamespace); + return IsNamespaceOnline(K8sTestNamespace); + } + + private bool IsNamespaceOnline(string name) + { + return client.ListNamespace().Items.Any(n => n.Metadata.Name == name); } #endregion @@ -141,7 +167,7 @@ namespace KubernetesWorkflow } }; - client.CreateNamespacedDeployment(deploymentSpec, K8sNamespace); + client.CreateNamespacedDeployment(deploymentSpec, K8sTestNamespace); WaitUntilDeploymentOnline(deploymentSpec.Metadata.Name); return deploymentSpec.Metadata.Name; @@ -149,7 +175,7 @@ namespace KubernetesWorkflow private void DeleteDeployment(string deploymentName) { - client.DeleteNamespacedDeployment(deploymentName, K8sNamespace); + client.DeleteNamespacedDeployment(deploymentName, K8sTestNamespace); WaitUntilDeploymentOffline(deploymentName); } @@ -173,7 +199,7 @@ namespace KubernetesWorkflow return new V1ObjectMeta { Name = "deploy-" + workflowNumberSource.WorkflowNumber, - NamespaceProperty = K8sNamespace + NamespaceProperty = K8sTestNamespace }; } @@ -257,14 +283,14 @@ namespace KubernetesWorkflow } }; - client.CreateNamespacedService(serviceSpec, K8sNamespace); + client.CreateNamespacedService(serviceSpec, K8sTestNamespace); return (serviceSpec.Metadata.Name, result); } private void DeleteService(string serviceName) { - client.DeleteNamespacedService(serviceName, K8sNamespace); + client.DeleteNamespacedService(serviceName, K8sTestNamespace); } private V1ObjectMeta CreateServiceMetadata() @@ -272,7 +298,7 @@ namespace KubernetesWorkflow return new V1ObjectMeta { Name = "service-" + workflowNumberSource.WorkflowNumber, - NamespaceProperty = K8sNamespace + NamespaceProperty = K8sTestNamespace }; } @@ -323,11 +349,16 @@ namespace KubernetesWorkflow WaitUntil(() => !IsTestNamespaceOnline()); } + private void WaitUntilNamespaceDeleted(string name) + { + WaitUntil(() => !IsNamespaceOnline(name)); + } + private void WaitUntilDeploymentOnline(string deploymentName) { WaitUntil(() => { - var deployment = client.ReadNamespacedDeployment(deploymentName, K8sNamespace); + var deployment = client.ReadNamespacedDeployment(deploymentName, K8sTestNamespace); return deployment?.Status.AvailableReplicas != null && deployment.Status.AvailableReplicas > 0; }); } @@ -336,7 +367,7 @@ namespace KubernetesWorkflow { WaitUntil(() => { - var deployments = client.ListNamespacedDeployment(K8sNamespace); + var deployments = client.ListNamespacedDeployment(K8sTestNamespace); var deployment = deployments.Items.SingleOrDefault(d => d.Metadata.Name == deploymentName); return deployment == null || deployment.Status.AvailableReplicas == 0; }); @@ -346,7 +377,7 @@ namespace KubernetesWorkflow { WaitUntil(() => { - var pods = client.ListNamespacedPod(K8sNamespace).Items; + var pods = client.ListNamespacedPod(K8sTestNamespace).Items; var pod = pods.SingleOrDefault(p => p.Metadata.Name == podName); return pod == null; }); @@ -369,7 +400,7 @@ namespace KubernetesWorkflow private (string, string) FetchNewPod() { - var pods = client.ListNamespacedPod(K8sNamespace).Items; + var pods = client.ListNamespacedPod(K8sTestNamespace).Items; var newPods = pods.Where(p => !knownPods.Contains(p.Name())).ToArray(); if (newPods.Length != 1) throw new InvalidOperationException("Expected only 1 pod to be created. Test infra failure."); diff --git a/KubernetesWorkflow/StartupWorkflow.cs b/KubernetesWorkflow/StartupWorkflow.cs index 8aaba75..cd229b0 100644 --- a/KubernetesWorkflow/StartupWorkflow.cs +++ b/KubernetesWorkflow/StartupWorkflow.cs @@ -8,14 +8,16 @@ namespace KubernetesWorkflow private readonly WorkflowNumberSource numberSource; private readonly K8sCluster cluster; private readonly KnownK8sPods knownK8SPods; + private readonly string testNamespace; private readonly RecipeComponentFactory componentFactory = new RecipeComponentFactory(); - internal StartupWorkflow(BaseLog log, WorkflowNumberSource numberSource, K8sCluster cluster, KnownK8sPods knownK8SPods) + internal StartupWorkflow(BaseLog log, WorkflowNumberSource numberSource, K8sCluster cluster, KnownK8sPods knownK8SPods, string testNamespace) { this.log = log; this.numberSource = numberSource; this.cluster = cluster; this.knownK8SPods = knownK8SPods; + this.testNamespace = testNamespace; } public RunningContainers Start(int numberOfContainers, Location location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig) @@ -62,6 +64,14 @@ namespace KubernetesWorkflow }); } + public void DeleteTestResources() + { + K8s(controller => + { + controller.DeleteTestNamespace(); + }); + } + private RunningContainer[] CreateContainers(RunningPod runningPod, ContainerRecipe[] recipes, StartupConfig startupConfig) { log.Debug(); @@ -82,14 +92,14 @@ namespace KubernetesWorkflow private void K8s(Action action) { - var controller = new K8sController(log, cluster, knownK8SPods, numberSource); + var controller = new K8sController(log, cluster, knownK8SPods, numberSource, testNamespace); action(controller); controller.Dispose(); } private T K8s(Func action) { - var controller = new K8sController(log, cluster, knownK8SPods, numberSource); + var controller = new K8sController(log, cluster, knownK8SPods, numberSource, testNamespace); var result = action(controller); controller.Dispose(); return result; diff --git a/KubernetesWorkflow/WorkflowCreator.cs b/KubernetesWorkflow/WorkflowCreator.cs index 1f54bea..e4f8032 100644 --- a/KubernetesWorkflow/WorkflowCreator.cs +++ b/KubernetesWorkflow/WorkflowCreator.cs @@ -6,7 +6,6 @@ namespace KubernetesWorkflow public class WorkflowCreator { private readonly NumberSource numberSource = new NumberSource(0); - private readonly NumberSource servicePortNumberSource = new NumberSource(30001); private readonly NumberSource containerNumberSource = new NumberSource(0); private readonly KnownK8sPods knownPods = new KnownK8sPods(); private readonly K8sCluster cluster; @@ -21,10 +20,10 @@ namespace KubernetesWorkflow public StartupWorkflow CreateWorkflow() { var workflowNumberSource = new WorkflowNumberSource(numberSource.GetNextNumber(), - servicePortNumberSource, + ApplicationLifecycle.Instance.GetServiceNumberSource(), containerNumberSource); - return new StartupWorkflow(log, workflowNumberSource, cluster, knownPods); + return new StartupWorkflow(log, workflowNumberSource, cluster, knownPods, ApplicationLifecycle.Instance.GetTestNamespace()); } } } diff --git a/Tests/BasicTests/TwoClientTests.cs b/Tests/BasicTests/TwoClientTests.cs index 90af450..14e0e21 100644 --- a/Tests/BasicTests/TwoClientTests.cs +++ b/Tests/BasicTests/TwoClientTests.cs @@ -5,6 +5,7 @@ using NUnit.Framework; namespace Tests.BasicTests { [TestFixture] + [Parallelizable(ParallelScope.All)] public class TwoClientTests : DistTest { [Test] From 532eb3d4f9e809f0e6d182396a854de54f3e65a9 Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 3 May 2023 14:55:26 +0200 Subject: [PATCH 06/15] Fixes single-instance DistTest class being used to run multiple tests in parallel --- DistTestCore/DistTest.cs | 57 +++++++++++++++------------ DistTestCore/FileManager.cs | 4 +- DistTestCore/TestLifecycle.cs | 9 +++++ KubernetesWorkflow/WorkflowCreator.cs | 4 +- 4 files changed, 46 insertions(+), 28 deletions(-) diff --git a/DistTestCore/DistTest.cs b/DistTestCore/DistTest.cs index b0686e1..175fae6 100644 --- a/DistTestCore/DistTest.cs +++ b/DistTestCore/DistTest.cs @@ -6,7 +6,6 @@ using KubernetesWorkflow; using Logging; using NUnit.Framework; using System.Reflection; -using Utils; namespace DistTestCore { @@ -15,14 +14,16 @@ namespace DistTestCore { private readonly Configuration configuration = new Configuration(); private readonly Assembly[] testAssemblies; - private FixtureLog fixtureLog = null!; - private TestLifecycle lifecycle = null!; - private DateTime testStart = DateTime.MinValue; + private readonly FixtureLog fixtureLog; + private readonly object lifecycleLock = new object(); + private readonly Dictionary lifecycles = new Dictionary(); public DistTest() { var assemblies = AppDomain.CurrentDomain.GetAssemblies(); testAssemblies = assemblies.Where(a => a.FullName!.ToLowerInvariant().Contains("test")).ToArray(); + + fixtureLog = new FixtureLog(configuration.GetLogConfig()); } [OneTimeSetUp] @@ -31,7 +32,6 @@ namespace DistTestCore // Previous test run may have been interrupted. // Begin by cleaning everything up. Timing.UseLongTimeouts = false; - fixtureLog = new FixtureLog(configuration.GetLogConfig()); try { @@ -85,7 +85,7 @@ namespace DistTestCore public TestFile GenerateTestFile(ByteSize size) { - return lifecycle.FileManager.GenerateTestFile(size); + return Get().FileManager.GenerateTestFile(size); } public IOnlineCodexNode SetupCodexBootstrapNode() @@ -128,12 +128,20 @@ namespace DistTestCore public ICodexNodeGroup BringOnline(ICodexSetup codexSetup) { - return lifecycle.CodexStarter.BringOnline((CodexSetup)codexSetup); + return Get().CodexStarter.BringOnline((CodexSetup)codexSetup); } protected BaseLog Log { - get { return lifecycle.Log; } + get { return Get().Log; } + } + + private TestLifecycle Get() + { + lock (lifecycleLock) + { + return lifecycles[GetCurrentTestName()]; + } } private bool ShouldUseLongTimeouts() @@ -156,24 +164,27 @@ namespace DistTestCore var testName = GetCurrentTestName(); Stopwatch.Measure(fixtureLog, $"Setup for {testName}", () => { - lifecycle = new TestLifecycle(fixtureLog.CreateTestLog(), configuration); - testStart = DateTime.UtcNow; + lock (lifecycleLock) + { + lifecycles.Add(testName, new TestLifecycle(fixtureLog.CreateTestLog(), configuration)); + } }); } private void DisposeTestLifecycle() { - fixtureLog.Log($"{GetCurrentTestName()} = {GetTestResult()} ({GetTestDuration()})"); + var lifecycle = Get(); + fixtureLog.Log($"{GetCurrentTestName()} = {GetTestResult()} ({lifecycle.GetTestDuration()})"); Stopwatch.Measure(fixtureLog, $"Teardown for {GetCurrentTestName()}", () => { lifecycle.Log.EndTest(); - IncludeLogsAndMetricsOnTestFailure(); + IncludeLogsAndMetricsOnTestFailure(lifecycle); lifecycle.DeleteAllResources(); lifecycle = null!; }); } - private void IncludeLogsAndMetricsOnTestFailure() + private void IncludeLogsAndMetricsOnTestFailure(TestLifecycle lifecycle) { var result = TestContext.CurrentContext.Result; if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed) @@ -183,8 +194,8 @@ namespace DistTestCore if (IsDownloadingLogsAndMetricsEnabled()) { lifecycle.Log.Log("Downloading all CodexNode logs and metrics because of test failure..."); - DownloadAllLogs(); - DownloadAllMetrics(); + DownloadAllLogs(lifecycle); + DownloadAllMetrics(lifecycle); } else { @@ -193,25 +204,19 @@ namespace DistTestCore } } - private string GetTestDuration() + private void DownloadAllLogs(TestLifecycle lifecycle) { - var testDuration = DateTime.UtcNow - testStart; - return Time.FormatDuration(testDuration); - } - - private void DownloadAllLogs() - { - OnEachCodexNode(node => + OnEachCodexNode(lifecycle, node => { lifecycle.DownloadLog(node); }); } - private void DownloadAllMetrics() + private void DownloadAllMetrics(TestLifecycle lifecycle) { var metricsDownloader = new MetricsDownloader(lifecycle.Log); - OnEachCodexNode(node => + OnEachCodexNode(lifecycle, node => { var m = node.Metrics as MetricsAccess; if (m != null) @@ -221,7 +226,7 @@ namespace DistTestCore }); } - private void OnEachCodexNode(Action action) + private void OnEachCodexNode(TestLifecycle lifecycle, Action action) { var allNodes = lifecycle.CodexStarter.RunningGroups.SelectMany(g => g.Nodes); foreach (var node in allNodes) diff --git a/DistTestCore/FileManager.cs b/DistTestCore/FileManager.cs index b195e9c..ae58cd6 100644 --- a/DistTestCore/FileManager.cs +++ b/DistTestCore/FileManager.cs @@ -1,5 +1,6 @@ using Logging; using NUnit.Framework; +using Utils; namespace DistTestCore { @@ -13,13 +14,14 @@ namespace DistTestCore public class FileManager : IFileManager { public const int ChunkSize = 1024 * 1024; + private static NumberSource folderNumberSource = new NumberSource(0); private readonly Random random = new Random(); private readonly TestLog log; private readonly string folder; public FileManager(TestLog log, Configuration configuration) { - folder = configuration.GetFileManagerFolder(); + folder = Path.Combine(configuration.GetFileManagerFolder(), folderNumberSource.GetNextNumber().ToString("D5")); EnsureDirectory(); this.log = log; diff --git a/DistTestCore/TestLifecycle.cs b/DistTestCore/TestLifecycle.cs index 4dddf34..09974e9 100644 --- a/DistTestCore/TestLifecycle.cs +++ b/DistTestCore/TestLifecycle.cs @@ -1,12 +1,14 @@ using DistTestCore.Logs; using KubernetesWorkflow; using Logging; +using Utils; namespace DistTestCore { public class TestLifecycle { private readonly WorkflowCreator workflowCreator; + private DateTime testStart = DateTime.MinValue; public TestLifecycle(TestLog log, Configuration configuration) { @@ -17,6 +19,7 @@ namespace DistTestCore CodexStarter = new CodexStarter(this, workflowCreator); PrometheusStarter = new PrometheusStarter(this, workflowCreator); GethStarter = new GethStarter(this, workflowCreator); + testStart = DateTime.UtcNow; } public TestLog Log { get; } @@ -42,5 +45,11 @@ namespace DistTestCore return new CodexNodeLog(subFile, node); } + + public string GetTestDuration() + { + var testDuration = DateTime.UtcNow - testStart; + return Time.FormatDuration(testDuration); + } } } diff --git a/KubernetesWorkflow/WorkflowCreator.cs b/KubernetesWorkflow/WorkflowCreator.cs index e4f8032..aa0a098 100644 --- a/KubernetesWorkflow/WorkflowCreator.cs +++ b/KubernetesWorkflow/WorkflowCreator.cs @@ -10,11 +10,13 @@ namespace KubernetesWorkflow private readonly KnownK8sPods knownPods = new KnownK8sPods(); private readonly K8sCluster cluster; private readonly BaseLog log; + private readonly string testNamespace; public WorkflowCreator(BaseLog log, Configuration configuration) { cluster = new K8sCluster(configuration); this.log = log; + testNamespace = ApplicationLifecycle.Instance.GetTestNamespace(); } public StartupWorkflow CreateWorkflow() @@ -23,7 +25,7 @@ namespace KubernetesWorkflow ApplicationLifecycle.Instance.GetServiceNumberSource(), containerNumberSource); - return new StartupWorkflow(log, workflowNumberSource, cluster, knownPods, ApplicationLifecycle.Instance.GetTestNamespace()); + return new StartupWorkflow(log, workflowNumberSource, cluster, knownPods, testNamespace); } } } From 2ed6993b58e7a6e1aa89fe7c21f772b50b2fe307 Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 4 May 2023 08:25:48 +0200 Subject: [PATCH 07/15] wip debugging anomalous test failures --- DistTestCore/Configuration.cs | 2 +- DistTestCore/DistTest.cs | 1 + KubernetesWorkflow/CommandRunner.cs | 8 ++-- KubernetesWorkflow/K8sClient.cs | 36 +++++++++++++++++ KubernetesWorkflow/K8sController.cs | 44 ++++++++++---------- Tests/BasicTests/DownloadTests.cs | 1 + Tests/BasicTests/NetworkIsolationTest.cs | 51 ++++++++++++++++++++++++ Tests/BasicTests/TwoClientTests.cs | 22 +++++++--- Tests/BasicTests/UploadTests.cs | 1 + Tests/Parallelism.cs | 6 +++ 10 files changed, 139 insertions(+), 33 deletions(-) create mode 100644 KubernetesWorkflow/K8sClient.cs create mode 100644 Tests/BasicTests/NetworkIsolationTest.cs create mode 100644 Tests/Parallelism.cs diff --git a/DistTestCore/Configuration.cs b/DistTestCore/Configuration.cs index b60a11f..d532fc6 100644 --- a/DistTestCore/Configuration.cs +++ b/DistTestCore/Configuration.cs @@ -21,7 +21,7 @@ namespace DistTestCore public Logging.LogConfig GetLogConfig() { - return new Logging.LogConfig("CodexTestLogs", debugEnabled: false); + return new Logging.LogConfig("CodexTestLogs", debugEnabled: true); } public string GetFileManagerFolder() diff --git a/DistTestCore/DistTest.cs b/DistTestCore/DistTest.cs index 175fae6..ed96fc5 100644 --- a/DistTestCore/DistTest.cs +++ b/DistTestCore/DistTest.cs @@ -10,6 +10,7 @@ using System.Reflection; namespace DistTestCore { [SetUpFixture] + [Parallelizable(ParallelScope.All)] public abstract class DistTest { private readonly Configuration configuration = new Configuration(); diff --git a/KubernetesWorkflow/CommandRunner.cs b/KubernetesWorkflow/CommandRunner.cs index a045500..daf8b9b 100644 --- a/KubernetesWorkflow/CommandRunner.cs +++ b/KubernetesWorkflow/CommandRunner.cs @@ -5,7 +5,7 @@ namespace KubernetesWorkflow { public class CommandRunner { - private readonly Kubernetes client; + private readonly K8sClient client; private readonly string k8sNamespace; private readonly RunningPod pod; private readonly string containerName; @@ -13,7 +13,7 @@ namespace KubernetesWorkflow private readonly string[] arguments; private readonly List lines = new List(); - public CommandRunner(Kubernetes client, string k8sNamespace, RunningPod pod, string containerName, string command, string[] arguments) + public CommandRunner(K8sClient client, string k8sNamespace, RunningPod pod, string containerName, string command, string[] arguments) { this.client = client; this.k8sNamespace = k8sNamespace; @@ -27,8 +27,8 @@ namespace KubernetesWorkflow { var input = new[] { command }.Concat(arguments).ToArray(); - Time.Wait(client.NamespacedPodExecAsync( - pod.Name, k8sNamespace, containerName, input, false, Callback, new CancellationToken())); + Time.Wait(client.Run(c => c.NamespacedPodExecAsync( + pod.Name, k8sNamespace, containerName, input, false, Callback, new CancellationToken()))); } public string GetStdOut() diff --git a/KubernetesWorkflow/K8sClient.cs b/KubernetesWorkflow/K8sClient.cs new file mode 100644 index 0000000..14b53ce --- /dev/null +++ b/KubernetesWorkflow/K8sClient.cs @@ -0,0 +1,36 @@ +using k8s; + +namespace KubernetesWorkflow +{ + public class K8sClient + { + private readonly Kubernetes client; + private static readonly object clientLock = new object(); + + public K8sClient(KubernetesClientConfiguration config) + { + client = new Kubernetes(config); + } + + public void Run(Action action) + { + lock (clientLock) + { + action(client); + } + } + + public T Run(Func action) + { + lock (clientLock) + { + return action(client); + } + } + + public void Dispose() + { + client.Dispose(); + } + } +} diff --git a/KubernetesWorkflow/K8sController.cs b/KubernetesWorkflow/K8sController.cs index 118101d..d50ac28 100644 --- a/KubernetesWorkflow/K8sController.cs +++ b/KubernetesWorkflow/K8sController.cs @@ -11,8 +11,7 @@ namespace KubernetesWorkflow private readonly K8sCluster cluster; private readonly KnownK8sPods knownPods; private readonly WorkflowNumberSource workflowNumberSource; - private readonly string testNamespace; - private readonly Kubernetes client; + private readonly K8sClient client; public K8sController(BaseLog log, K8sCluster cluster, KnownK8sPods knownPods, WorkflowNumberSource workflowNumberSource, string testNamespace) { @@ -20,8 +19,10 @@ namespace KubernetesWorkflow this.cluster = cluster; this.knownPods = knownPods; this.workflowNumberSource = workflowNumberSource; - this.testNamespace = testNamespace; - client = new Kubernetes(cluster.GetK8sClientConfig()); + client = new K8sClient(cluster.GetK8sClientConfig()); + + K8sTestNamespace = cluster.Configuration.K8sNamespacePrefix + testNamespace; + log.Debug($"'{K8sTestNamespace}'"); } public void Dispose() @@ -53,7 +54,7 @@ namespace KubernetesWorkflow public void DownloadPodLog(RunningPod pod, ContainerRecipe recipe, ILogHandler logHandler) { log.Debug(); - using var stream = client.ReadNamespacedPodLog(pod.Name, K8sTestNamespace, recipe.Name); + using var stream = client.Run(c => c.ReadNamespacedPodLog(pod.Name, K8sTestNamespace, recipe.Name)); logHandler.Log(stream); } @@ -69,7 +70,7 @@ namespace KubernetesWorkflow { log.Debug(); - var all = client.ListNamespace().Items; + var all = client.Run(c => c.ListNamespace().Items); var namespaces = all.Select(n => n.Name()).Where(n => n.StartsWith(cluster.Configuration.K8sNamespacePrefix)); foreach (var ns in namespaces) @@ -87,7 +88,7 @@ namespace KubernetesWorkflow log.Debug(); if (IsTestNamespaceOnline()) { - client.DeleteNamespace(K8sTestNamespace, null, null, gracePeriodSeconds: 0); + client.Run(c => c.DeleteNamespace(K8sTestNamespace, null, null, gracePeriodSeconds: 0)); } WaitUntilNamespaceDeleted(); } @@ -97,12 +98,14 @@ namespace KubernetesWorkflow log.Debug(); if (IsNamespaceOnline(ns)) { - client.DeleteNamespace(ns, null, null, gracePeriodSeconds: 0); + client.Run(c => c.DeleteNamespace(ns, null, null, gracePeriodSeconds: 0)); } } #region Namespace management + private string K8sTestNamespace { get; } + private void EnsureTestNamespace() { if (IsTestNamespaceOnline()) return; @@ -116,15 +119,10 @@ namespace KubernetesWorkflow Labels = new Dictionary { { "name", K8sTestNamespace } } } }; - client.CreateNamespace(namespaceSpec); + client.Run(c => c.CreateNamespace(namespaceSpec)); WaitUntilNamespaceCreated(); } - private string K8sTestNamespace - { - get { return cluster.Configuration.K8sNamespacePrefix + testNamespace; } - } - private bool IsTestNamespaceOnline() { return IsNamespaceOnline(K8sTestNamespace); @@ -132,7 +130,7 @@ namespace KubernetesWorkflow private bool IsNamespaceOnline(string name) { - return client.ListNamespace().Items.Any(n => n.Metadata.Name == name); + return client.Run(c => c.ListNamespace().Items.Any(n => n.Metadata.Name == name)); } #endregion @@ -167,7 +165,7 @@ namespace KubernetesWorkflow } }; - client.CreateNamespacedDeployment(deploymentSpec, K8sTestNamespace); + client.Run(c => c.CreateNamespacedDeployment(deploymentSpec, K8sTestNamespace)); WaitUntilDeploymentOnline(deploymentSpec.Metadata.Name); return deploymentSpec.Metadata.Name; @@ -175,7 +173,7 @@ namespace KubernetesWorkflow private void DeleteDeployment(string deploymentName) { - client.DeleteNamespacedDeployment(deploymentName, K8sTestNamespace); + client.Run(c => c.DeleteNamespacedDeployment(deploymentName, K8sTestNamespace)); WaitUntilDeploymentOffline(deploymentName); } @@ -283,14 +281,14 @@ namespace KubernetesWorkflow } }; - client.CreateNamespacedService(serviceSpec, K8sTestNamespace); + client.Run(c => c.CreateNamespacedService(serviceSpec, K8sTestNamespace)); return (serviceSpec.Metadata.Name, result); } private void DeleteService(string serviceName) { - client.DeleteNamespacedService(serviceName, K8sTestNamespace); + client.Run(c => c.DeleteNamespacedService(serviceName, K8sTestNamespace)); } private V1ObjectMeta CreateServiceMetadata() @@ -358,7 +356,7 @@ namespace KubernetesWorkflow { WaitUntil(() => { - var deployment = client.ReadNamespacedDeployment(deploymentName, K8sTestNamespace); + var deployment = client.Run(c => c.ReadNamespacedDeployment(deploymentName, K8sTestNamespace)); return deployment?.Status.AvailableReplicas != null && deployment.Status.AvailableReplicas > 0; }); } @@ -367,7 +365,7 @@ namespace KubernetesWorkflow { WaitUntil(() => { - var deployments = client.ListNamespacedDeployment(K8sTestNamespace); + var deployments = client.Run(c => c.ListNamespacedDeployment(K8sTestNamespace)); var deployment = deployments.Items.SingleOrDefault(d => d.Metadata.Name == deploymentName); return deployment == null || deployment.Status.AvailableReplicas == 0; }); @@ -377,7 +375,7 @@ namespace KubernetesWorkflow { WaitUntil(() => { - var pods = client.ListNamespacedPod(K8sTestNamespace).Items; + var pods = client.Run(c => c.ListNamespacedPod(K8sTestNamespace)).Items; var pod = pods.SingleOrDefault(p => p.Metadata.Name == podName); return pod == null; }); @@ -400,7 +398,7 @@ namespace KubernetesWorkflow private (string, string) FetchNewPod() { - var pods = client.ListNamespacedPod(K8sTestNamespace).Items; + var pods = client.Run(c => c.ListNamespacedPod(K8sTestNamespace)).Items; var newPods = pods.Where(p => !knownPods.Contains(p.Name())).ToArray(); if (newPods.Length != 1) throw new InvalidOperationException("Expected only 1 pod to be created. Test infra failure."); diff --git a/Tests/BasicTests/DownloadTests.cs b/Tests/BasicTests/DownloadTests.cs index 0f61daf..44ea4ee 100644 --- a/Tests/BasicTests/DownloadTests.cs +++ b/Tests/BasicTests/DownloadTests.cs @@ -6,6 +6,7 @@ namespace Tests.ParallelTests [TestFixture] public class DownloadTests : DistTest { + [Ignore("a")] [TestCase(3, 500)] [TestCase(5, 100)] [TestCase(10, 256)] diff --git a/Tests/BasicTests/NetworkIsolationTest.cs b/Tests/BasicTests/NetworkIsolationTest.cs new file mode 100644 index 0000000..b26dcbe --- /dev/null +++ b/Tests/BasicTests/NetworkIsolationTest.cs @@ -0,0 +1,51 @@ +using DistTestCore; +using NUnit.Framework; +using Utils; + +namespace Tests.BasicTests +{ + [Ignore("not a real test!")] + [TestFixture] + public class NetworkIsolationTest : DistTest + { + private IOnlineCodexNode? node = null; + + // net isolation: only on real cluster? + // parallel upload/download tests? + // operation times. + + [Test] + public void SetUpANodeAndWait() + { + node = SetupCodexNode(); + + while (node != null) + { + Time.Sleep(TimeSpan.FromSeconds(5)); + } + } + + [Test] + public void ForeignNodeConnects() + { + var myNode = SetupCodexNode(); + + while (node == null) + { + Time.Sleep(TimeSpan.FromSeconds(1)); + } + + myNode.ConnectToPeer(node); + + var testFile = GenerateTestFile(1.MB()); + + var contentId = node.UploadFile(testFile); + + var downloadedFile = myNode.DownloadContent(contentId); + + testFile.AssertIsEqual(downloadedFile); + + node = null; + } + } +} diff --git a/Tests/BasicTests/TwoClientTests.cs b/Tests/BasicTests/TwoClientTests.cs index 14e0e21..461dfa8 100644 --- a/Tests/BasicTests/TwoClientTests.cs +++ b/Tests/BasicTests/TwoClientTests.cs @@ -5,18 +5,25 @@ using NUnit.Framework; namespace Tests.BasicTests { [TestFixture] - [Parallelizable(ParallelScope.All)] public class TwoClientTests : DistTest { - [Test] - public void TwoClientsOnePodTest() + [TestCase(1)] + [TestCase(2)] + [TestCase(3)] + [TestCase(4)] + [TestCase(5)] + [TestCase(6)] + [TestCase(7)] + [TestCase(8)] + [TestCase(9)] + public void TwoClientsOnePodTest(int size) { var group = SetupCodexNodes(2); var primary = group[0]; var secondary = group[1]; - PerformTwoClientTest(primary, secondary); + PerformTwoClientTest(primary, secondary, size.MB()); } [Test] @@ -39,10 +46,15 @@ namespace Tests.BasicTests } private void PerformTwoClientTest(IOnlineCodexNode primary, IOnlineCodexNode secondary) + { + PerformTwoClientTest(primary, secondary, 1.MB()); + } + + private void PerformTwoClientTest(IOnlineCodexNode primary, IOnlineCodexNode secondary, ByteSize size) { primary.ConnectToPeer(secondary); - var testFile = GenerateTestFile(1.MB()); + var testFile = GenerateTestFile(size); var contentId = primary.UploadFile(testFile); diff --git a/Tests/BasicTests/UploadTests.cs b/Tests/BasicTests/UploadTests.cs index 8cbbe7a..fbe146f 100644 --- a/Tests/BasicTests/UploadTests.cs +++ b/Tests/BasicTests/UploadTests.cs @@ -6,6 +6,7 @@ namespace Tests.ParallelTests [TestFixture] public class UploadTests : DistTest { + [Ignore("a")] [TestCase(3, 50)] [TestCase(5, 75)] [TestCase(10, 25)] diff --git a/Tests/Parallelism.cs b/Tests/Parallelism.cs new file mode 100644 index 0000000..c6f8089 --- /dev/null +++ b/Tests/Parallelism.cs @@ -0,0 +1,6 @@ +using NUnit.Framework; + +[assembly: LevelOfParallelism(30)] +namespace Tests +{ +} From 5a4a5795b22b32c98d0c45ff7279918827cb1291 Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 4 May 2023 08:55:20 +0200 Subject: [PATCH 08/15] Makes timings not static and ties them to test lifecycle --- DistTestCore/Codex/CodexAccess.cs | 6 ++- DistTestCore/CodexNodeGroup.cs | 2 +- DistTestCore/Configuration.cs | 6 +-- DistTestCore/DistTest.cs | 44 +++++++++-------- DistTestCore/Http.cs | 14 +++--- DistTestCore/Metrics/MetricsAccess.cs | 6 ++- DistTestCore/Metrics/MetricsAccessFactory.cs | 4 +- DistTestCore/Metrics/MetricsQuery.cs | 3 +- DistTestCore/TestLifecycle.cs | 6 ++- DistTestCore/Timing.cs | 52 ++------------------ 10 files changed, 56 insertions(+), 87 deletions(-) diff --git a/DistTestCore/Codex/CodexAccess.cs b/DistTestCore/Codex/CodexAccess.cs index 006b66e..c4fe91f 100644 --- a/DistTestCore/Codex/CodexAccess.cs +++ b/DistTestCore/Codex/CodexAccess.cs @@ -6,10 +6,12 @@ namespace DistTestCore.Codex public class CodexAccess { private readonly BaseLog log; + private readonly ITimeSet timeSet; - public CodexAccess(BaseLog log, RunningContainer runningContainer) + public CodexAccess(BaseLog log, ITimeSet timeSet, RunningContainer runningContainer) { this.log = log; + this.timeSet = timeSet; Container = runningContainer; } @@ -44,7 +46,7 @@ namespace DistTestCore.Codex { var ip = Container.Pod.Cluster.IP; var port = Container.ServicePorts[0].Number; - return new Http(log, ip, port, baseUrl: "/api/codex/v1"); + return new Http(log, timeSet, ip, port, baseUrl: "/api/codex/v1"); } public string ConnectToPeer(string peerId, string peerMultiAddress) diff --git a/DistTestCore/CodexNodeGroup.cs b/DistTestCore/CodexNodeGroup.cs index b0930d3..d0867d7 100644 --- a/DistTestCore/CodexNodeGroup.cs +++ b/DistTestCore/CodexNodeGroup.cs @@ -64,7 +64,7 @@ namespace DistTestCore private OnlineCodexNode CreateOnlineCodexNode(RunningContainer c, ICodexNodeFactory factory) { - var access = new CodexAccess(lifecycle.Log, c); + var access = new CodexAccess(lifecycle.Log, lifecycle.TimeSet, c); EnsureOnline(access); return factory.CreateOnlineCodexNode(access, this); } diff --git a/DistTestCore/Configuration.cs b/DistTestCore/Configuration.cs index d532fc6..2120404 100644 --- a/DistTestCore/Configuration.cs +++ b/DistTestCore/Configuration.cs @@ -4,13 +4,13 @@ namespace DistTestCore { public class Configuration { - public KubernetesWorkflow.Configuration GetK8sConfiguration() + public KubernetesWorkflow.Configuration GetK8sConfiguration(ITimeSet timeSet) { return new KubernetesWorkflow.Configuration( k8sNamespacePrefix: "ct-", kubeConfigFile: null, - operationTimeout: Timing.K8sOperationTimeout(), - retryDelay: Timing.K8sServiceDelay(), + operationTimeout: timeSet.K8sOperationTimeout(), + retryDelay: timeSet.WaitForK8sServiceDelay(), locationMap: new[] { new ConfigurationLocationEntry(Location.BensOldGamingMachine, "worker01"), diff --git a/DistTestCore/DistTest.cs b/DistTestCore/DistTest.cs index ed96fc5..2629973 100644 --- a/DistTestCore/DistTest.cs +++ b/DistTestCore/DistTest.cs @@ -32,13 +32,11 @@ namespace DistTestCore { // Previous test run may have been interrupted. // Begin by cleaning everything up. - Timing.UseLongTimeouts = false; - try { Stopwatch.Measure(fixtureLog, "Global setup", () => { - var wc = new WorkflowCreator(fixtureLog, configuration.GetK8sConfiguration()); + var wc = new WorkflowCreator(fixtureLog, configuration.GetK8sConfiguration(GetTimeSet())); wc.CreateWorkflow().DeleteAllResources(); }); } @@ -58,8 +56,6 @@ namespace DistTestCore [SetUp] public void SetUpDistTest() { - Timing.UseLongTimeouts = ShouldUseLongTimeouts(); - if (GlobalTestFailure.HasFailed) { Assert.Inconclusive("Skip test: Previous test failed during clean up."); @@ -145,21 +141,6 @@ namespace DistTestCore } } - private bool ShouldUseLongTimeouts() - { - // Don't be fooled! TestContext.CurrentTest.Test allows you easy access to the attributes of the current test. - // But this doesn't work for tests making use of [TestCase]. So instead, we use reflection here to figure out - // if the attribute is present. - var currentTest = TestContext.CurrentContext.Test; - var className = currentTest.ClassName; - var methodName = currentTest.MethodName; - - var testClasses = testAssemblies.SelectMany(a => a.GetTypes()).Where(c => c.FullName == className).ToArray(); - var testMethods = testClasses.SelectMany(c => c.GetMethods()).Where(m => m.Name == methodName).ToArray(); - - return testMethods.Any(m => m.GetCustomAttribute() != null); - } - private void CreateNewTestLifecycle() { var testName = GetCurrentTestName(); @@ -167,7 +148,7 @@ namespace DistTestCore { lock (lifecycleLock) { - lifecycles.Add(testName, new TestLifecycle(fixtureLog.CreateTestLog(), configuration)); + lifecycles.Add(testName, new TestLifecycle(fixtureLog.CreateTestLog(), configuration, GetTimeSet())); } }); } @@ -185,6 +166,27 @@ namespace DistTestCore }); } + private ITimeSet GetTimeSet() + { + if (ShouldUseLongTimeouts()) return new LongTimeSet(); + return new DefaultTimeSet(); + } + + private bool ShouldUseLongTimeouts() + { + // Don't be fooled! TestContext.CurrentTest.Test allows you easy access to the attributes of the current test. + // But this doesn't work for tests making use of [TestCase]. So instead, we use reflection here to figure out + // if the attribute is present. + var currentTest = TestContext.CurrentContext.Test; + var className = currentTest.ClassName; + var methodName = currentTest.MethodName; + + var testClasses = testAssemblies.SelectMany(a => a.GetTypes()).Where(c => c.FullName == className).ToArray(); + var testMethods = testClasses.SelectMany(c => c.GetMethods()).Where(m => m.Name == methodName).ToArray(); + + return testMethods.Any(m => m.GetCustomAttribute() != null); + } + private void IncludeLogsAndMetricsOnTestFailure(TestLifecycle lifecycle) { var result = TestContext.CurrentContext.Result; diff --git a/DistTestCore/Http.cs b/DistTestCore/Http.cs index 2969596..3dfee9d 100644 --- a/DistTestCore/Http.cs +++ b/DistTestCore/Http.cs @@ -10,13 +10,15 @@ namespace DistTestCore public class Http { private readonly BaseLog log; + private readonly ITimeSet timeSet; private readonly string ip; private readonly int port; private readonly string baseUrl; - public Http(BaseLog log, string ip, int port, string baseUrl) + public Http(BaseLog log, ITimeSet timeSet, string ip, int port, string baseUrl) { this.log = log; + this.timeSet = timeSet; this.ip = ip; this.port = port; this.baseUrl = baseUrl; @@ -103,7 +105,7 @@ namespace DistTestCore log.Debug($"({url}) = '{message}'", 3); } - private static T Retry(Func operation) + private T Retry(Func operation) { var retryCounter = 0; @@ -115,9 +117,9 @@ namespace DistTestCore } catch (Exception exception) { - Timing.HttpCallRetryDelay(); + timeSet.HttpCallRetryDelay(); retryCounter++; - if (retryCounter > Timing.HttpCallRetryCount()) + if (retryCounter > timeSet.HttpCallRetryCount()) { Assert.Fail(exception.ToString()); throw; @@ -140,10 +142,10 @@ namespace DistTestCore } } - private static HttpClient GetClient() + private HttpClient GetClient() { var client = new HttpClient(); - client.Timeout = Timing.HttpCallTimeout(); + client.Timeout = timeSet.HttpCallTimeout(); return client; } } diff --git a/DistTestCore/Metrics/MetricsAccess.cs b/DistTestCore/Metrics/MetricsAccess.cs index f0d4ff9..e52c175 100644 --- a/DistTestCore/Metrics/MetricsAccess.cs +++ b/DistTestCore/Metrics/MetricsAccess.cs @@ -14,12 +14,14 @@ namespace DistTestCore.Metrics public class MetricsAccess : IMetricsAccess { private readonly TestLog log; + private readonly ITimeSet timeSet; private readonly MetricsQuery query; private readonly RunningContainer node; - public MetricsAccess(TestLog log, MetricsQuery query, RunningContainer node) + public MetricsAccess(TestLog log, ITimeSet timeSet, MetricsQuery query, RunningContainer node) { this.log = log; + this.timeSet = timeSet; this.query = query; this.node = node; } @@ -47,7 +49,7 @@ namespace DistTestCore.Metrics { var mostRecent = GetMostRecent(metricName); if (mostRecent != null) return mostRecent; - if (DateTime.UtcNow - start > Timing.WaitForMetricTimeout()) + if (DateTime.UtcNow - start > timeSet.WaitForMetricTimeout()) { Assert.Fail($"Timeout: Unable to get metric '{metricName}'."); throw new TimeoutException(); diff --git a/DistTestCore/Metrics/MetricsAccessFactory.cs b/DistTestCore/Metrics/MetricsAccessFactory.cs index 6f93886..24103ab 100644 --- a/DistTestCore/Metrics/MetricsAccessFactory.cs +++ b/DistTestCore/Metrics/MetricsAccessFactory.cs @@ -28,8 +28,8 @@ namespace DistTestCore.Metrics public IMetricsAccess CreateMetricsAccess(RunningContainer codexContainer) { - var query = new MetricsQuery(lifecycle.Log, prometheusContainer); - return new MetricsAccess(lifecycle.Log, query, codexContainer); + var query = new MetricsQuery(lifecycle.Log, lifecycle.TimeSet, prometheusContainer); + return new MetricsAccess(lifecycle.Log, lifecycle.TimeSet, query, codexContainer); } } } diff --git a/DistTestCore/Metrics/MetricsQuery.cs b/DistTestCore/Metrics/MetricsQuery.cs index cc8bd67..8c5f24f 100644 --- a/DistTestCore/Metrics/MetricsQuery.cs +++ b/DistTestCore/Metrics/MetricsQuery.cs @@ -9,12 +9,13 @@ namespace DistTestCore.Metrics { private readonly Http http; - public MetricsQuery(BaseLog log, RunningContainers runningContainers) + public MetricsQuery(BaseLog log, ITimeSet timeSet, RunningContainers runningContainers) { RunningContainers = runningContainers; http = new Http( log, + timeSet, runningContainers.RunningPod.Cluster.IP, runningContainers.Containers[0].ServicePorts[0].Number, "api/v1"); diff --git a/DistTestCore/TestLifecycle.cs b/DistTestCore/TestLifecycle.cs index 09974e9..1505745 100644 --- a/DistTestCore/TestLifecycle.cs +++ b/DistTestCore/TestLifecycle.cs @@ -10,10 +10,11 @@ namespace DistTestCore private readonly WorkflowCreator workflowCreator; private DateTime testStart = DateTime.MinValue; - public TestLifecycle(TestLog log, Configuration configuration) + public TestLifecycle(TestLog log, Configuration configuration, ITimeSet timeSet) { Log = log; - workflowCreator = new WorkflowCreator(log, configuration.GetK8sConfiguration()); + TimeSet = timeSet; + workflowCreator = new WorkflowCreator(log, configuration.GetK8sConfiguration(timeSet)); FileManager = new FileManager(Log, configuration); CodexStarter = new CodexStarter(this, workflowCreator); @@ -23,6 +24,7 @@ namespace DistTestCore } public TestLog Log { get; } + public ITimeSet TimeSet { get; } public FileManager FileManager { get; } public CodexStarter CodexStarter { get; } public PrometheusStarter PrometheusStarter { get; } diff --git a/DistTestCore/Timing.cs b/DistTestCore/Timing.cs index 3cfc1bc..9771767 100644 --- a/DistTestCore/Timing.cs +++ b/DistTestCore/Timing.cs @@ -8,53 +8,11 @@ namespace DistTestCore { } - public static class Timing - { - public static bool UseLongTimeouts { get; set; } - - - public static TimeSpan HttpCallTimeout() - { - return GetTimes().HttpCallTimeout(); - } - - public static int HttpCallRetryCount() - { - return GetTimes().HttpCallRetryCount(); - } - - public static void HttpCallRetryDelay() - { - Time.Sleep(GetTimes().HttpCallRetryDelay()); - } - - public static TimeSpan K8sServiceDelay() - { - return GetTimes().WaitForK8sServiceDelay(); - } - - public static TimeSpan K8sOperationTimeout() - { - return GetTimes().K8sOperationTimeout(); - } - - public static TimeSpan WaitForMetricTimeout() - { - return GetTimes().WaitForMetricTimeout(); - } - - private static ITimeSet GetTimes() - { - if (UseLongTimeouts) return new LongTimeSet(); - return new DefaultTimeSet(); - } - } - public interface ITimeSet { TimeSpan HttpCallTimeout(); int HttpCallRetryCount(); - TimeSpan HttpCallRetryDelay(); + void HttpCallRetryDelay(); TimeSpan WaitForK8sServiceDelay(); TimeSpan K8sOperationTimeout(); TimeSpan WaitForMetricTimeout(); @@ -72,9 +30,9 @@ namespace DistTestCore return 5; } - public TimeSpan HttpCallRetryDelay() + public void HttpCallRetryDelay() { - return TimeSpan.FromSeconds(3); + Time.Sleep(TimeSpan.FromSeconds(3)); } public TimeSpan WaitForK8sServiceDelay() @@ -105,9 +63,9 @@ namespace DistTestCore return 2; } - public TimeSpan HttpCallRetryDelay() + public void HttpCallRetryDelay() { - return TimeSpan.FromMinutes(5); + Time.Sleep(TimeSpan.FromMinutes(5)); } public TimeSpan WaitForK8sServiceDelay() From ab07ac0389e323db62c2c3114a34ec3384678e22 Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 4 May 2023 09:16:15 +0200 Subject: [PATCH 09/15] Fixes issue where multiple instances of ApplicaitonLifecycle are created. --- KubernetesWorkflow/ApplicationLifecycle.cs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/KubernetesWorkflow/ApplicationLifecycle.cs b/KubernetesWorkflow/ApplicationLifecycle.cs index 5289573..7d6fb3d 100644 --- a/KubernetesWorkflow/ApplicationLifecycle.cs +++ b/KubernetesWorkflow/ApplicationLifecycle.cs @@ -4,6 +4,7 @@ namespace KubernetesWorkflow { public class ApplicationLifecycle { + private static object instanceLock = new object(); private static ApplicationLifecycle? instance; private readonly NumberSource servicePortNumberSource = new NumberSource(30001); private readonly NumberSource namespaceNumberSource = new NumberSource(0); @@ -18,8 +19,11 @@ namespace KubernetesWorkflow // and persists for the entire application lifecycle. get { - if (instance == null) instance = new ApplicationLifecycle(); - return instance; + lock (instanceLock) + { + if (instance == null) instance = new ApplicationLifecycle(); + return instance; + } } } From 533bf325774fac3ccf365df40557b57756e26a5d Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 4 May 2023 11:34:43 +0200 Subject: [PATCH 10/15] Better logging in case codex node fails to respond to debug/info request. --- DistTestCore/Codex/CodexAccess.cs | 18 ++++++++++++++++++ DistTestCore/CodexNodeGroup.cs | 24 +++++------------------- DistTestCore/CodexStarter.cs | 1 + KubernetesWorkflow/ContainerRecipe.cs | 8 ++++++++ KubernetesWorkflow/StartupWorkflow.cs | 10 ++++++++-- Tests/BasicTests/DownloadTests.cs | 3 +-- Tests/BasicTests/NetworkIsolationTest.cs | 2 -- Tests/BasicTests/UploadTests.cs | 3 +-- 8 files changed, 42 insertions(+), 27 deletions(-) diff --git a/DistTestCore/Codex/CodexAccess.cs b/DistTestCore/Codex/CodexAccess.cs index c4fe91f..8654878 100644 --- a/DistTestCore/Codex/CodexAccess.cs +++ b/DistTestCore/Codex/CodexAccess.cs @@ -42,6 +42,24 @@ namespace DistTestCore.Codex return Http().HttpPostJson($"storage/request/{contentId}", request); } + public void EnsureOnline() + { + try + { + var debugInfo = GetDebugInfo(); + if (debugInfo == null || string.IsNullOrEmpty(debugInfo.id)) throw new InvalidOperationException("Unable to get debug-info from codex node at startup."); + + var nodePeerId = debugInfo.id; + var nodeName = Container.Name; + log.AddStringReplace(nodePeerId, $"___{nodeName}___"); + } + catch (Exception e) + { + log.Error($"Failed to start codex node: {e}. Test infra failure."); + throw new InvalidOperationException($"Failed to start codex node. Test infra failure.", e); + } + } + private Http Http() { var ip = Container.Pod.Cluster.IP; diff --git a/DistTestCore/CodexNodeGroup.cs b/DistTestCore/CodexNodeGroup.cs index d0867d7..2005410 100644 --- a/DistTestCore/CodexNodeGroup.cs +++ b/DistTestCore/CodexNodeGroup.cs @@ -62,29 +62,15 @@ namespace DistTestCore return $"group:[{Containers.Describe()}]"; } + public void EnsureOnline() + { + foreach (var node in Nodes) node.CodexAccess.EnsureOnline(); + } + private OnlineCodexNode CreateOnlineCodexNode(RunningContainer c, ICodexNodeFactory factory) { var access = new CodexAccess(lifecycle.Log, lifecycle.TimeSet, c); - EnsureOnline(access); return factory.CreateOnlineCodexNode(access, this); } - - private void EnsureOnline(CodexAccess access) - { - try - { - var debugInfo = access.GetDebugInfo(); - if (debugInfo == null || string.IsNullOrEmpty(debugInfo.id)) throw new InvalidOperationException("Unable to get debug-info from codex node at startup."); - - var nodePeerId = debugInfo.id; - var nodeName = access.Container.Name; - lifecycle.Log.AddStringReplace(nodePeerId, $"___{nodeName}___"); - } - catch (Exception e) - { - lifecycle.Log.Error($"Failed to start codex node: {e}. Test infra failure."); - throw new InvalidOperationException($"Failed to start codex node. Test infra failure.", e); - } - } } } diff --git a/DistTestCore/CodexStarter.cs b/DistTestCore/CodexStarter.cs index 52c70ca..e36ebca 100644 --- a/DistTestCore/CodexStarter.cs +++ b/DistTestCore/CodexStarter.cs @@ -74,6 +74,7 @@ namespace DistTestCore { var group = new CodexNodeGroup(lifecycle, codexSetup, runningContainers, codexNodeFactory); RunningGroups.Add(group); + group.EnsureOnline(); return group; } diff --git a/KubernetesWorkflow/ContainerRecipe.cs b/KubernetesWorkflow/ContainerRecipe.cs index 9fbbb9f..ce3252f 100644 --- a/KubernetesWorkflow/ContainerRecipe.cs +++ b/KubernetesWorkflow/ContainerRecipe.cs @@ -24,6 +24,14 @@ { return ExposedPorts.Concat(InternalPorts).Single(p => p.Tag == tag); } + + public override string ToString() + { + return $"(container-recipe: {Name}, image: {Image}, " + + $"exposedPorts: {string.Join(",", ExposedPorts.Select(p => p.Number))}, " + + $"internalPorts: {string.Join(",", InternalPorts.Select(p => p.Number))}, " + + $"envVars: {string.Join(",", EnvVars.Select(v => v.Name + ":" + v.Value))}, "; + } } public class Port diff --git a/KubernetesWorkflow/StartupWorkflow.cs b/KubernetesWorkflow/StartupWorkflow.cs index cd229b0..20cea0f 100644 --- a/KubernetesWorkflow/StartupWorkflow.cs +++ b/KubernetesWorkflow/StartupWorkflow.cs @@ -75,7 +75,13 @@ namespace KubernetesWorkflow private RunningContainer[] CreateContainers(RunningPod runningPod, ContainerRecipe[] recipes, StartupConfig startupConfig) { log.Debug(); - return recipes.Select(r => new RunningContainer(runningPod, r, runningPod.GetServicePortsForContainerRecipe(r), startupConfig)).ToArray(); + return recipes.Select(r => + { + var servicePorts = runningPod.GetServicePortsForContainerRecipe(r); + log.Debug($"{r} -> service ports: {string.Join(",", servicePorts.Select(p => p.Number))}"); + + return new RunningContainer(runningPod, r, servicePorts, startupConfig); + }).ToArray(); } private ContainerRecipe[] CreateRecipes(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig) @@ -84,7 +90,7 @@ namespace KubernetesWorkflow var result = new List(); for (var i = 0; i < numberOfContainers; i++) { - result.Add(recipeFactory.CreateRecipe(i ,numberSource.GetContainerNumber(), componentFactory, startupConfig)); + result.Add(recipeFactory.CreateRecipe(i, numberSource.GetContainerNumber(), componentFactory, startupConfig)); } return result.ToArray(); diff --git a/Tests/BasicTests/DownloadTests.cs b/Tests/BasicTests/DownloadTests.cs index 44ea4ee..36a635f 100644 --- a/Tests/BasicTests/DownloadTests.cs +++ b/Tests/BasicTests/DownloadTests.cs @@ -6,7 +6,6 @@ namespace Tests.ParallelTests [TestFixture] public class DownloadTests : DistTest { - [Ignore("a")] [TestCase(3, 500)] [TestCase(5, 100)] [TestCase(10, 256)] @@ -37,4 +36,4 @@ namespace Tests.ParallelTests } } } -} \ No newline at end of file +} diff --git a/Tests/BasicTests/NetworkIsolationTest.cs b/Tests/BasicTests/NetworkIsolationTest.cs index b26dcbe..a7b2520 100644 --- a/Tests/BasicTests/NetworkIsolationTest.cs +++ b/Tests/BasicTests/NetworkIsolationTest.cs @@ -11,8 +11,6 @@ namespace Tests.BasicTests private IOnlineCodexNode? node = null; // net isolation: only on real cluster? - // parallel upload/download tests? - // operation times. [Test] public void SetUpANodeAndWait() diff --git a/Tests/BasicTests/UploadTests.cs b/Tests/BasicTests/UploadTests.cs index fbe146f..ea3b41c 100644 --- a/Tests/BasicTests/UploadTests.cs +++ b/Tests/BasicTests/UploadTests.cs @@ -6,7 +6,6 @@ namespace Tests.ParallelTests [TestFixture] public class UploadTests : DistTest { - [Ignore("a")] [TestCase(3, 50)] [TestCase(5, 75)] [TestCase(10, 25)] @@ -43,4 +42,4 @@ namespace Tests.ParallelTests } } } -} \ No newline at end of file +} From eae138f7fda9597e8528bebc4bd95ffb82deab24 Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 4 May 2023 14:55:39 +0200 Subject: [PATCH 11/15] Adds policy and test for network isolation. This will not work on docker-desktop clusters. --- DistTestCore/DistTest.cs | 11 +++- DistTestCore/OnlineCodexNode.cs | 4 ++ KubernetesWorkflow/K8sController.cs | 73 +++++++++++++++++++++++- Tests/BasicTests/NetworkIsolationTest.cs | 36 ++++++------ 4 files changed, 100 insertions(+), 24 deletions(-) diff --git a/DistTestCore/DistTest.cs b/DistTestCore/DistTest.cs index 2629973..e5ee885 100644 --- a/DistTestCore/DistTest.cs +++ b/DistTestCore/DistTest.cs @@ -128,9 +128,16 @@ namespace DistTestCore return Get().CodexStarter.BringOnline((CodexSetup)codexSetup); } - protected BaseLog Log + protected void Log(string msg) { - get { return Get().Log; } + TestContext.Progress.WriteLine(msg); + Get().Log.Log(msg); + } + + protected void Debug(string msg) + { + TestContext.Progress.WriteLine(msg); + Get().Log.Debug(msg); } private TestLifecycle Get() diff --git a/DistTestCore/OnlineCodexNode.cs b/DistTestCore/OnlineCodexNode.cs index 91cf11a..7fb30e1 100644 --- a/DistTestCore/OnlineCodexNode.cs +++ b/DistTestCore/OnlineCodexNode.cs @@ -92,6 +92,10 @@ namespace DistTestCore public ICodexSetup BringOffline() { + if (Group.Count() > 1) throw new InvalidOperationException("Codex-nodes that are part of a group cannot be " + + "individually shut down. Use 'BringOffline()' on the group object to stop the group. This method is only " + + "available for codex-nodes in groups of 1."); + return Group.BringOffline(); } diff --git a/KubernetesWorkflow/K8sController.cs b/KubernetesWorkflow/K8sController.cs index d50ac28..fd42d07 100644 --- a/KubernetesWorkflow/K8sController.cs +++ b/KubernetesWorkflow/K8sController.cs @@ -22,7 +22,7 @@ namespace KubernetesWorkflow client = new K8sClient(cluster.GetK8sClientConfig()); K8sTestNamespace = cluster.Configuration.K8sNamespacePrefix + testNamespace; - log.Debug($"'{K8sTestNamespace}'"); + log.Debug($"Test namespace: '{K8sTestNamespace}'"); } public void Dispose() @@ -121,6 +121,8 @@ namespace KubernetesWorkflow }; client.Run(c => c.CreateNamespace(namespaceSpec)); WaitUntilNamespaceCreated(); + + CreatePolicy(); } private bool IsTestNamespaceOnline() @@ -133,6 +135,67 @@ namespace KubernetesWorkflow return client.Run(c => c.ListNamespace().Items.Any(n => n.Metadata.Name == name)); } + private void CreatePolicy() + { + client.Run(c => + { + var body = new V1NetworkPolicy + { + Metadata = new V1ObjectMeta + { + Name = "isolate-policy", + NamespaceProperty = K8sTestNamespace + }, + Spec = new V1NetworkPolicySpec + { + PodSelector = new V1LabelSelector + { + MatchLabels = GetSelector() + }, + PolicyTypes = new[] + { + "Ingress", + "Egress" + }, + Ingress = new List + { + new V1NetworkPolicyIngressRule + { + FromProperty = new List + { + new V1NetworkPolicyPeer + { + NamespaceSelector = new V1LabelSelector + { + MatchLabels = GetMyNamespaceSelector() + } + } + } + } + }, + Egress = new List + { + new V1NetworkPolicyEgressRule + { + To = new List + { + new V1NetworkPolicyPeer + { + NamespaceSelector = new V1LabelSelector + { + MatchLabels = GetMyNamespaceSelector() + } + } + } + } + } + } + }; + + c.CreateNamespacedNetworkPolicy(body, K8sTestNamespace); + }); + } + #endregion #region Deployment management @@ -192,12 +255,18 @@ namespace KubernetesWorkflow return new Dictionary { { "codex-test-node", "dist-test-" + workflowNumberSource.WorkflowNumber } }; } + private IDictionary GetMyNamespaceSelector() + { + return new Dictionary { { "name", "thatisincorrect" } }; + } + private V1ObjectMeta CreateDeploymentMetadata() { return new V1ObjectMeta { Name = "deploy-" + workflowNumberSource.WorkflowNumber, - NamespaceProperty = K8sTestNamespace + NamespaceProperty = K8sTestNamespace, + Labels = GetSelector() }; } diff --git a/Tests/BasicTests/NetworkIsolationTest.cs b/Tests/BasicTests/NetworkIsolationTest.cs index a7b2520..f3406d4 100644 --- a/Tests/BasicTests/NetworkIsolationTest.cs +++ b/Tests/BasicTests/NetworkIsolationTest.cs @@ -4,23 +4,20 @@ using Utils; namespace Tests.BasicTests { - [Ignore("not a real test!")] + // Warning! + // This is a test to check network-isolation in the test-infrastructure. + // It requires parallelism(2) or greater to run. [TestFixture] public class NetworkIsolationTest : DistTest { private IOnlineCodexNode? node = null; - // net isolation: only on real cluster? - [Test] public void SetUpANodeAndWait() { node = SetupCodexNode(); - while (node != null) - { - Time.Sleep(TimeSpan.FromSeconds(5)); - } + Time.WaitUntil(() => node == null, TimeSpan.FromMinutes(5), TimeSpan.FromSeconds(5)); } [Test] @@ -28,22 +25,21 @@ namespace Tests.BasicTests { var myNode = SetupCodexNode(); - while (node == null) + Time.WaitUntil(() => node != null, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(5)); + + try { - Time.Sleep(TimeSpan.FromSeconds(1)); + myNode.ConnectToPeer(node!); + } + catch + { + // Good! This connection should be prohibited by the network isolation policy. + node = null; + return; } - myNode.ConnectToPeer(node); - - var testFile = GenerateTestFile(1.MB()); - - var contentId = node.UploadFile(testFile); - - var downloadedFile = myNode.DownloadContent(contentId); - - testFile.AssertIsEqual(downloadedFile); - - node = null; + Assert.Fail("Connection could be established between two Codex nodes running in different namespaces. " + + "This may cause cross-test interference. Network isolation policy should be applied. Test infra failure."); } } } From 3db2217c3269d2ebf1546272f9964a18824685d6 Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 4 May 2023 15:21:36 +0200 Subject: [PATCH 12/15] oops forgot to update calls to log --- Tests/BasicTests/PeerTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tests/BasicTests/PeerTests.cs b/Tests/BasicTests/PeerTests.cs index f7fa700..714e31b 100644 --- a/Tests/BasicTests/PeerTests.cs +++ b/Tests/BasicTests/PeerTests.cs @@ -75,8 +75,8 @@ namespace Tests.BasicTests //var enginePeers = string.Join(",", a.enginePeers.Select(p => p.peerId)); var switchPeers = string.Join(",", a.switchPeers.Select(p => p.peerId)); - //Log.Debug($"Looking for {b.id} in engine-peers [{enginePeers}]"); - Log.Debug($"{a.id} is looking for {b.id} in switch-peers [{switchPeers}]"); + //Debug($"Looking for {b.id} in engine-peers [{enginePeers}]"); + Debug($"{a.id} is looking for {b.id} in switch-peers [{switchPeers}]"); //Assert.That(a.enginePeers.Any(p => p.peerId == b.id), $"{a.id} was looking for '{b.id}' in engine-peers [{enginePeers}] but it was not found."); Assert.That(a.switchPeers.Any(p => p.peerId == b.id), $"{a.id} was looking for '{b.id}' in switch-peers [{switchPeers}] but it was not found."); From d0b4e31167d2e3f65323239ff943e31c0639a27c Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 4 May 2023 15:22:11 +0200 Subject: [PATCH 13/15] Turns off parallel tests for now. --- Tests/Parallelism.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/Parallelism.cs b/Tests/Parallelism.cs index c6f8089..f45d8f2 100644 --- a/Tests/Parallelism.cs +++ b/Tests/Parallelism.cs @@ -1,6 +1,6 @@ using NUnit.Framework; -[assembly: LevelOfParallelism(30)] +[assembly: LevelOfParallelism(1)] namespace Tests { } From e96c8a2a13ba345d3aa37d4d857d0c6fc99e182a Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 5 May 2023 07:58:48 +0200 Subject: [PATCH 14/15] Enables checking for engine peers in peer tests --- Tests/BasicTests/PeerTests.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Tests/BasicTests/PeerTests.cs b/Tests/BasicTests/PeerTests.cs index 714e31b..09b41ac 100644 --- a/Tests/BasicTests/PeerTests.cs +++ b/Tests/BasicTests/PeerTests.cs @@ -72,13 +72,13 @@ namespace Tests.BasicTests private void AssertKnows(CodexDebugResponse a, CodexDebugResponse b) { - //var enginePeers = string.Join(",", a.enginePeers.Select(p => p.peerId)); + var enginePeers = string.Join(",", a.enginePeers.Select(p => p.peerId)); var switchPeers = string.Join(",", a.switchPeers.Select(p => p.peerId)); - //Debug($"Looking for {b.id} in engine-peers [{enginePeers}]"); + Debug($"{a.id} is looking for {b.id} in engine-peers [{enginePeers}]"); Debug($"{a.id} is looking for {b.id} in switch-peers [{switchPeers}]"); - //Assert.That(a.enginePeers.Any(p => p.peerId == b.id), $"{a.id} was looking for '{b.id}' in engine-peers [{enginePeers}] but it was not found."); + Assert.That(a.enginePeers.Any(p => p.peerId == b.id), $"{a.id} was looking for '{b.id}' in engine-peers [{enginePeers}] but it was not found."); Assert.That(a.switchPeers.Any(p => p.peerId == b.id), $"{a.id} was looking for '{b.id}' in switch-peers [{switchPeers}] but it was not found."); } } From 87f3b9ec00560cab3e6249273e1db3fe94dffebe Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 5 May 2023 08:33:10 +0200 Subject: [PATCH 15/15] Turns off debug logging --- DistTestCore/Configuration.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DistTestCore/Configuration.cs b/DistTestCore/Configuration.cs index 2120404..ee01559 100644 --- a/DistTestCore/Configuration.cs +++ b/DistTestCore/Configuration.cs @@ -21,7 +21,7 @@ namespace DistTestCore public Logging.LogConfig GetLogConfig() { - return new Logging.LogConfig("CodexTestLogs", debugEnabled: true); + return new Logging.LogConfig("CodexTestLogs", debugEnabled: false); } public string GetFileManagerFolder()