From 063b69ef9901dc2a20f25d1a2fbd1711ed27d9f9 Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 26 Jun 2023 16:00:16 +0200 Subject: [PATCH 01/49] Stops marketplace test on first failure --- ContinuousTests/ContinuousTests.csproj | 1 + ContinuousTests/Tests/MarketplaceTest.cs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ContinuousTests/ContinuousTests.csproj b/ContinuousTests/ContinuousTests.csproj index bddcb94..5543d01 100644 --- a/ContinuousTests/ContinuousTests.csproj +++ b/ContinuousTests/ContinuousTests.csproj @@ -14,6 +14,7 @@ + diff --git a/ContinuousTests/Tests/MarketplaceTest.cs b/ContinuousTests/Tests/MarketplaceTest.cs index 578e94d..3702732 100644 --- a/ContinuousTests/Tests/MarketplaceTest.cs +++ b/ContinuousTests/Tests/MarketplaceTest.cs @@ -11,7 +11,7 @@ namespace ContinuousTests.Tests { public override int RequiredNumberOfNodes => 1; public override TimeSpan RunTestEvery => TimeSpan.FromDays(4); - public override TestFailMode TestFailMode => TestFailMode.AlwaysRunAllMoments; + public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure; public const int EthereumAccountIndex = 200; // TODO: Check against all other account indices of all other tests. From 731ab90ce12279d4be3a0ae65a21fd74cf96048f Mon Sep 17 00:00:00 2001 From: benbierens Date: Tue, 27 Jun 2023 08:29:39 +0200 Subject: [PATCH 02/49] Easier local deployments --- CodexNetDeployer/Configuration.cs | 24 ++++++++---------------- CodexNetDeployer/Deployer.cs | 16 ++++++++++++---- CodexNetDeployer/Program.cs | 2 +- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/CodexNetDeployer/Configuration.cs b/CodexNetDeployer/Configuration.cs index 2e7f232..b25e483 100644 --- a/CodexNetDeployer/Configuration.cs +++ b/CodexNetDeployer/Configuration.cs @@ -8,16 +8,16 @@ namespace CodexNetDeployer public class Configuration { [Uniform("codex-image", "ci", "CODEXIMAGE", true, "Docker image of Codex.")] - public string CodexImage { get; set; } = string.Empty; + public string CodexImage { get; set; } = CodexContainerRecipe.DockerImage; [Uniform("geth-image", "gi", "GETHIMAGE", true, "Docker image of Geth.")] - public string GethImage { get; set; } = string.Empty; + public string GethImage { get; set; } = GethContainerRecipe.DockerImage; [Uniform("contracts-image", "oi", "CONTRACTSIMAGE", true, "Docker image of Codex Contracts.")] - public string ContractsImage { get; set; } = string.Empty; + public string ContractsImage { get; set; } = CodexContractsContainerRecipe.DockerImage; - [Uniform("kube-config", "kc", "KUBECONFIG", true, "Path to Kubeconfig file.")] - public string KubeConfigFile { get; set; } = string.Empty; + [Uniform("kube-config", "kc", "KUBECONFIG", false, "Path to Kubeconfig file. Use 'null' (default) to use local cluster.")] + public string KubeConfigFile { get; set; } = "null"; [Uniform("kube-namespace", "kn", "KUBENAMESPACE", true, "Kubernetes namespace to be used for deployment.")] public string KubeNamespace { get; set; } = string.Empty; @@ -32,18 +32,10 @@ namespace CodexNetDeployer public int? StorageQuota { get; set; } [Uniform("log-level", "l", "LOGLEVEL", true, "Log level used by each Codex node. [Trace, Debug*, Info, Warn, Error]")] - public CodexLogLevel CodexLogLevel { get; set; } + public CodexLogLevel CodexLogLevel { get; set; } = CodexLogLevel.Debug; public TestRunnerLocation RunnerLocation { get; set; } = TestRunnerLocation.InternalToCluster; - public class Defaults - { - public string CodexImage { get; set; } = CodexContainerRecipe.DockerImage; - public string GethImage { get; set; } = GethContainerRecipe.DockerImage; - public string ContractsImage { get; set; } = CodexContractsContainerRecipe.DockerImage; - public CodexLogLevel CodexLogLevel { get; set; } = CodexLogLevel.Debug; - } - public List Validate() { var errors = new List(); @@ -74,7 +66,7 @@ namespace CodexNetDeployer { if (value == null || value.Value < 1) { - errors.Add($"{variable} is must be set and must be greater than 0."); + errors.Add($"{variable} must be set and must be greater than 0."); } } @@ -82,7 +74,7 @@ namespace CodexNetDeployer { if (string.IsNullOrWhiteSpace(value)) { - errors.Add($"{variable} is must be set."); + errors.Add($"{variable} must be set."); } } } diff --git a/CodexNetDeployer/Deployer.cs b/CodexNetDeployer/Deployer.cs index 5745dd9..ba0ecc5 100644 --- a/CodexNetDeployer/Deployer.cs +++ b/CodexNetDeployer/Deployer.cs @@ -51,9 +51,11 @@ namespace CodexNetDeployer private (WorkflowCreator, TestLifecycle) CreateFacilities() { + var kubeConfig = GetKubeConfig(config.KubeConfigFile); + var lifecycleConfig = new DistTestCore.Configuration ( - kubeConfigFile: config.KubeConfigFile, + kubeConfigFile: kubeConfig, logPath: "null", logDebug: false, dataFilesPath: "notUsed", @@ -61,18 +63,24 @@ namespace CodexNetDeployer runnerLocation: config.RunnerLocation ); - var kubeConfig = new KubernetesWorkflow.Configuration( + var kubeFlowConfig = new KubernetesWorkflow.Configuration( k8sNamespacePrefix: config.KubeNamespace, - kubeConfigFile: config.KubeConfigFile, + kubeConfigFile: kubeConfig, operationTimeout: timeset.K8sOperationTimeout(), retryDelay: timeset.WaitForK8sServiceDelay()); - var workflowCreator = new WorkflowCreator(log, kubeConfig, testNamespacePostfix: string.Empty); + var workflowCreator = new WorkflowCreator(log, kubeFlowConfig, testNamespacePostfix: string.Empty); var lifecycle = new TestLifecycle(log, lifecycleConfig, timeset, workflowCreator); return (workflowCreator, lifecycle); } + private string? GetKubeConfig(string kubeConfigFile) + { + if (string.IsNullOrEmpty(kubeConfigFile) || kubeConfigFile.ToLowerInvariant() == "null") return null; + return kubeConfigFile; + } + private DeploymentMetadata CreateMetadata() { return new DeploymentMetadata( diff --git a/CodexNetDeployer/Program.cs b/CodexNetDeployer/Program.cs index bc38c80..4ac3a2f 100644 --- a/CodexNetDeployer/Program.cs +++ b/CodexNetDeployer/Program.cs @@ -17,7 +17,7 @@ public class Program return; } - var uniformArgs = new ArgsUniform(new Configuration.Defaults(), args); + var uniformArgs = new ArgsUniform(args); var config = uniformArgs.Parse(true); if (args.Any(a => a == "--external")) From 86d954e1032db6db9c6ab59c8631813faa6e4584 Mon Sep 17 00:00:00 2001 From: benbierens Date: Tue, 27 Jun 2023 10:16:59 +0200 Subject: [PATCH 03/49] Debugging continuous test runner --- ContinuousTests/Configuration.cs | 6 +- ContinuousTests/ContinuousTestRunner.cs | 9 ++- ContinuousTests/SingleTestRun.cs | 55 +++++++++++++++---- .../{TestStarter.cs => TestLoop.cs} | 22 ++++++-- 4 files changed, 70 insertions(+), 22 deletions(-) rename ContinuousTests/{TestStarter.cs => TestLoop.cs} (58%) diff --git a/ContinuousTests/Configuration.cs b/ContinuousTests/Configuration.cs index c4d5b7a..1d6b635 100644 --- a/ContinuousTests/Configuration.cs +++ b/ContinuousTests/Configuration.cs @@ -18,8 +18,8 @@ namespace ContinuousTests [Uniform("keep", "k", "KEEP", false, "Set to '1' to retain logs of successful tests.")] public bool KeepPassedTestLogs { get; set; } = false; - [Uniform("kube-config", "kc", "KUBECONFIG", true, "Path to Kubeconfig file.")] - public string KubeConfigFile { get; set; } = string.Empty; + [Uniform("kube-config", "kc", "KUBECONFIG", true, "Path to Kubeconfig file. Use 'null' (default) to use local cluster.")] + public string KubeConfigFile { get; set; } = "null"; public CodexDeployment CodexDeployment { get; set; } = null!; } @@ -30,7 +30,7 @@ namespace ContinuousTests { var uniformArgs = new ArgsUniform(args); - var result = uniformArgs.Parse(); + var result = uniformArgs.Parse(true); result.CodexDeployment = ParseCodexDeploymentJson(result.CodexDeploymentJson); diff --git a/ContinuousTests/ContinuousTestRunner.cs b/ContinuousTests/ContinuousTestRunner.cs index 2d75c80..8d7fe44 100644 --- a/ContinuousTests/ContinuousTestRunner.cs +++ b/ContinuousTests/ContinuousTestRunner.cs @@ -20,15 +20,18 @@ namespace ContinuousTests startupChecker.Check(); var overviewLog = new FixtureLog(new LogConfig(config.LogPath, false), "Overview"); + overviewLog.Log("Continuous tests starting..."); var allTests = testFactory.CreateTests(); - var testStarters = allTests.Select(t => new TestStarter(config, overviewLog, t.GetType(), t.RunTestEvery)).ToArray(); + var testLoop = allTests.Select(t => new TestLoop(config, overviewLog, t.GetType(), t.RunTestEvery)).ToArray(); - foreach (var t in testStarters) + foreach (var t in testLoop) { + overviewLog.Log("Launching test-loop for " + t.Name); t.Begin(); Thread.Sleep(TimeSpan.FromMinutes(5)); } - + + overviewLog.Log("All test-loops launched."); while (true) Thread.Sleep((2 ^ 31) - 1); } } diff --git a/ContinuousTests/SingleTestRun.cs b/ContinuousTests/SingleTestRun.cs index 7e8d0a4..5d3d98d 100644 --- a/ContinuousTests/SingleTestRun.cs +++ b/ContinuousTests/SingleTestRun.cs @@ -4,6 +4,7 @@ using Logging; using Utils; using KubernetesWorkflow; using NUnit.Framework.Internal; +using System.Reflection; namespace ContinuousTests { @@ -41,20 +42,33 @@ namespace ContinuousTests try { RunTest(); - - if (!config.KeepPassedTestLogs) fixtureLog.Delete(); + fileManager.DeleteAllTestFiles(); + Directory.Delete(dataFolder, true); } catch (Exception ex) { - fixtureLog.Error("Test run failed with exception: " + ex); - fixtureLog.MarkAsFailed(); + overviewLog.Error("Test infra failure: SingleTestRun failed with " + ex); + Environment.Exit(-1); } - fileManager.DeleteAllTestFiles(); - Directory.Delete(dataFolder, true); }); } private void RunTest() + { + try + { + RunTestMoments(); + + if (!config.KeepPassedTestLogs) fixtureLog.Delete(); + } + catch (Exception ex) + { + fixtureLog.Error("Test run failed with exception: " + ex); + fixtureLog.MarkAsFailed(); + } + } + + private void RunTestMoments() { var earliestMoment = handle.GetEarliestMoment(); @@ -66,7 +80,7 @@ namespace ContinuousTests if (handle.Test.TestFailMode == TestFailMode.StopAfterFirstFailure && exceptions.Any()) { Log("Exception detected. TestFailMode = StopAfterFirstFailure. Stopping..."); - throw exceptions.Single(); + ThrowFailTest(); } var nextMoment = handle.GetNextMoment(t); @@ -80,9 +94,7 @@ namespace ContinuousTests { if (exceptions.Any()) { - var ex = exceptions.First(); - OverviewLog(" > Test failed: " + ex); - throw ex; + ThrowFailTest(); } OverviewLog(" > Test passed."); return; @@ -90,6 +102,28 @@ namespace ContinuousTests } } + private void ThrowFailTest() + { + var ex = UnpackException(exceptions.First()); + Log(ex.ToString()); + OverviewLog(" > Test failed: " + ex.Message); + throw ex; + } + + private Exception UnpackException(Exception exception) + { + if (exception is AggregateException a) + { + return UnpackException(a.InnerExceptions.First()); + } + if (exception is TargetInvocationException t) + { + return UnpackException(t.InnerException!); + } + + return exception; + } + private void RunMoment(int t) { using (var context = new TestExecutionContext.IsolatedContext()) @@ -100,7 +134,6 @@ namespace ContinuousTests } catch (Exception ex) { - Log($" > TestMoment yielded exception: " + ex); exceptions.Add(ex); } } diff --git a/ContinuousTests/TestStarter.cs b/ContinuousTests/TestLoop.cs similarity index 58% rename from ContinuousTests/TestStarter.cs rename to ContinuousTests/TestLoop.cs index 6253f64..8943735 100644 --- a/ContinuousTests/TestStarter.cs +++ b/ContinuousTests/TestLoop.cs @@ -2,29 +2,41 @@ namespace ContinuousTests { - public class TestStarter + public class TestLoop { private readonly Configuration config; private readonly BaseLog overviewLog; private readonly Type testType; private readonly TimeSpan runsEvery; - public TestStarter(Configuration config, BaseLog overviewLog, Type testType, TimeSpan runsEvery) + public TestLoop(Configuration config, BaseLog overviewLog, Type testType, TimeSpan runsEvery) { this.config = config; this.overviewLog = overviewLog; this.testType = testType; this.runsEvery = runsEvery; + + Name = testType.Name; } + public string Name { get; } + public void Begin() { Task.Run(() => { - while (true) + try { - StartTest(); - Thread.Sleep(runsEvery); + while (true) + { + StartTest(); + Thread.Sleep(runsEvery); + } + } + catch(Exception ex) + { + overviewLog.Error("Test infra failure: TestLoop failed with " + ex); + Environment.Exit(-1); } }); } From 63068aae1d71772516927e67cc41243e55c66147 Mon Sep 17 00:00:00 2001 From: benbierens Date: Tue, 27 Jun 2023 15:28:00 +0200 Subject: [PATCH 04/49] Debugging marketplace test --- ContinuousTests/Tests/MarketplaceTest.cs | 80 ++++++++++++------- DistTestCore/Codex/CodexNode.cs | 11 +++ DistTestCore/DistTest.cs | 2 +- .../{CodexNodeLog.cs => DownloadedLog.cs} | 10 +-- DistTestCore/Logs/LogDownloadHandler.cs | 10 +-- DistTestCore/Marketplace/MarketplaceAccess.cs | 8 +- DistTestCore/OnlineCodexNode.cs | 6 +- DistTestCore/TestLifecycle.cs | 10 +-- KubernetesWorkflow/K8sController.cs | 11 +-- KubernetesWorkflow/RunningPod.cs | 33 +++++--- Tests/BasicTests/ExampleTests.cs | 2 +- 11 files changed, 115 insertions(+), 68 deletions(-) rename DistTestCore/Logs/{CodexNodeLog.cs => DownloadedLog.cs} (68%) diff --git a/ContinuousTests/Tests/MarketplaceTest.cs b/ContinuousTests/Tests/MarketplaceTest.cs index 3702732..c99bcd1 100644 --- a/ContinuousTests/Tests/MarketplaceTest.cs +++ b/ContinuousTests/Tests/MarketplaceTest.cs @@ -3,6 +3,7 @@ using DistTestCore.Codex; using DistTestCore.Marketplace; using KubernetesWorkflow; using Logging; +using Newtonsoft.Json; using NUnit.Framework; namespace ContinuousTests.Tests @@ -14,15 +15,14 @@ namespace ContinuousTests.Tests public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure; public const int EthereumAccountIndex = 200; // TODO: Check against all other account indices of all other tests. + public const string MarketplaceTestNamespace = "codex-continuous-marketplace"; // prevent clashes too - private const string MarketplaceTestNamespace = "codex-continuous-marketplace"; - - private readonly ByteSize fileSize = 100.MB(); - private readonly TestToken pricePerBytePerSecond = 1.TestTokens(); + private readonly uint numberOfSlots = 3; + private readonly ByteSize fileSize = 10.MB(); + private readonly TestToken pricePerSlotPerSecond = 10.TestTokens(); private TestFile file = null!; private ContentId? cid; - private TestToken startingBalance = null!; private string purchaseId = string.Empty; [TestMoment(t: Zero)] @@ -30,22 +30,28 @@ namespace ContinuousTests.Tests { var contractDuration = TimeSpan.FromDays(3) + TimeSpan.FromHours(1); decimal totalDurationSeconds = Convert.ToDecimal(contractDuration.TotalSeconds); - var expectedTotalCost = pricePerBytePerSecond.Amount * totalDurationSeconds; + var expectedTotalCost = numberOfSlots * pricePerSlotPerSecond.Amount * (totalDurationSeconds + 1); + Log.Log("expected total cost: " + expectedTotalCost); file = FileManager.GenerateTestFile(fileSize); var (workflowCreator, lifecycle) = CreateFacilities(); var flow = workflowCreator.CreateWorkflow(); - var startupConfig = new StartupConfig(); - var codexStartConfig = new CodexStartupConfig(CodexLogLevel.Debug); - codexStartConfig.MarketplaceConfig = new MarketplaceInitialConfig(0.Eth(), 0.TestTokens(), false); - codexStartConfig.MarketplaceConfig.AccountIndexOverride = EthereumAccountIndex; - startupConfig.Add(codexStartConfig); - startupConfig.Add(Configuration.CodexDeployment.GethStartResult); - var rc = flow.Start(1, Location.Unspecified, new CodexContainerRecipe(), startupConfig); try { + var debugInfo = Nodes[0].GetDebugInfo(); + Assert.That(!string.IsNullOrEmpty(debugInfo.spr)); + + var startupConfig = new StartupConfig(); + var codexStartConfig = new CodexStartupConfig(CodexLogLevel.Debug); + codexStartConfig.MarketplaceConfig = new MarketplaceInitialConfig(0.Eth(), 0.TestTokens(), false); + codexStartConfig.MarketplaceConfig.AccountIndexOverride = EthereumAccountIndex; + codexStartConfig.BootstrapSpr = debugInfo.spr; + startupConfig.Add(codexStartConfig); + startupConfig.Add(Configuration.CodexDeployment.GethStartResult); + var rc = flow.Start(1, Location.Unspecified, new CodexContainerRecipe(), startupConfig); + var account = Configuration.CodexDeployment.GethStartResult.MarketplaceNetwork.Bootstrap.AllAccounts.Accounts[EthereumAccountIndex]; var tokenAddress = Configuration.CodexDeployment.GethStartResult.MarketplaceNetwork.Marketplace.TokenAddress; @@ -60,21 +66,24 @@ namespace ContinuousTests.Tests cid = UploadFile(codexAccess.Node, file); Assert.That(cid, Is.Not.Null); - startingBalance = marketAccess.GetBalance(); + var balance = marketAccess.GetBalance(); + Log.Log("Account: " + account.Account); + Log.Log("Balance: " + balance); purchaseId = marketAccess.RequestStorage( contentId: cid!, - pricePerBytePerSecond: pricePerBytePerSecond, + pricePerSlotPerSecond: pricePerSlotPerSecond, requiredCollateral: 100.TestTokens(), - minRequiredNumberOfNodes: 3, + minRequiredNumberOfNodes: numberOfSlots, proofProbability: 10, duration: contractDuration); + Log.Log($"PurchaseId: '{purchaseId}'"); Assert.That(!string.IsNullOrEmpty(purchaseId)); } finally { - flow.Stop(rc); + flow.DeleteTestResources(); } } @@ -83,13 +92,17 @@ namespace ContinuousTests.Tests { var (workflowCreator, lifecycle) = CreateFacilities(); var flow = workflowCreator.CreateWorkflow(); - var startupConfig = new StartupConfig(); - var codexStartConfig = new CodexStartupConfig(CodexLogLevel.Debug); - startupConfig.Add(codexStartConfig); - var rc = flow.Start(1, Location.Unspecified, new CodexContainerRecipe(), startupConfig); - + try { + var debugInfo = Nodes[0].GetDebugInfo(); + Assert.That(!string.IsNullOrEmpty(debugInfo.spr)); + + var startupConfig = new StartupConfig(); + var codexStartConfig = new CodexStartupConfig(CodexLogLevel.Debug); + codexStartConfig.BootstrapSpr = debugInfo.spr; + startupConfig.Add(codexStartConfig); + var rc = flow.Start(1, Location.Unspecified, new CodexContainerRecipe(), startupConfig); var container = rc.Containers[0]; var codexAccess = new CodexAccess(lifecycle, container); @@ -99,32 +112,39 @@ namespace ContinuousTests.Tests } finally { - flow.Stop(rc); + flow.DeleteTestResources(); } } private (WorkflowCreator, TestLifecycle) CreateFacilities() { + var kubeConfig = GetKubeConfig(Configuration.KubeConfigFile); var lifecycleConfig = new DistTestCore.Configuration ( - kubeConfigFile: Configuration.KubeConfigFile, + kubeConfigFile: kubeConfig, logPath: "null", - logDebug: false, - dataFilesPath: "notUsed", + logDebug: false, + dataFilesPath: Configuration.LogPath, codexLogLevel: CodexLogLevel.Debug, - runnerLocation: TestRunnerLocation.InternalToCluster + runnerLocation: TestRunnerLocation.ExternalToCluster ); - var kubeConfig = new KubernetesWorkflow.Configuration( + var kubeFlowConfig = new KubernetesWorkflow.Configuration( k8sNamespacePrefix: MarketplaceTestNamespace, - kubeConfigFile: Configuration.KubeConfigFile, + kubeConfigFile: kubeConfig, operationTimeout: TimeSet.K8sOperationTimeout(), retryDelay: TimeSet.WaitForK8sServiceDelay()); - var workflowCreator = new WorkflowCreator(Log, kubeConfig, testNamespacePostfix: string.Empty); + var workflowCreator = new WorkflowCreator(Log, kubeFlowConfig, testNamespacePostfix: string.Empty); var lifecycle = new TestLifecycle(new NullLog(), lifecycleConfig, TimeSet, workflowCreator); return (workflowCreator, lifecycle); } + + private string? GetKubeConfig(string kubeConfigFile) + { + if (string.IsNullOrEmpty(kubeConfigFile) || kubeConfigFile.ToLowerInvariant() == "null") return null; + return kubeConfigFile; + } } } diff --git a/DistTestCore/Codex/CodexNode.cs b/DistTestCore/Codex/CodexNode.cs index 510c2fb..bb2220e 100644 --- a/DistTestCore/Codex/CodexNode.cs +++ b/DistTestCore/Codex/CodexNode.cs @@ -65,6 +65,11 @@ namespace DistTestCore.Codex return Http().HttpPostJson($"storage/request/{contentId}", request); } + public CodexStoragePurchase GetPurchaseStatus(string purchaseId) + { + return Http().HttpGetJson($"storage/purchases/{purchaseId}"); + } + public string ConnectToPeer(string peerId, string peerMultiAddress) { return Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}"); @@ -170,4 +175,10 @@ namespace DistTestCore.Codex public uint? nodes { get; set; } public uint? tolerance { get; set; } } + + public class CodexStoragePurchase + { + public string state { get; set; } = string.Empty; + public string error { get; set; } = string.Empty; + } } diff --git a/DistTestCore/DistTest.cs b/DistTestCore/DistTest.cs index a93c180..e2a5c77 100644 --- a/DistTestCore/DistTest.cs +++ b/DistTestCore/DistTest.cs @@ -250,7 +250,7 @@ namespace DistTestCore { OnEachCodexNode(lifecycle, node => { - lifecycle.DownloadLog(node); + lifecycle.DownloadLog(node.CodexAccess.Container); }); } diff --git a/DistTestCore/Logs/CodexNodeLog.cs b/DistTestCore/Logs/DownloadedLog.cs similarity index 68% rename from DistTestCore/Logs/CodexNodeLog.cs rename to DistTestCore/Logs/DownloadedLog.cs index 6dd658f..9d22c81 100644 --- a/DistTestCore/Logs/CodexNodeLog.cs +++ b/DistTestCore/Logs/DownloadedLog.cs @@ -3,17 +3,17 @@ using NUnit.Framework; namespace DistTestCore.Logs { - public interface ICodexNodeLog + public interface IDownloadedLog { void AssertLogContains(string expectedString); } - public class CodexNodeLog : ICodexNodeLog + public class DownloadedLog : IDownloadedLog { private readonly LogFile logFile; - private readonly OnlineCodexNode owner; + private readonly string owner; - public CodexNodeLog(LogFile logFile, OnlineCodexNode owner) + public DownloadedLog(LogFile logFile, string owner) { this.logFile = logFile; this.owner = owner; @@ -31,7 +31,7 @@ namespace DistTestCore.Logs line = streamReader.ReadLine(); } - Assert.Fail($"{owner.GetName()} Unable to find string '{expectedString}' in CodexNode log file {logFile.FullFilename}"); + Assert.Fail($"{owner} Unable to find string '{expectedString}' in CodexNode log file {logFile.FullFilename}"); } } } diff --git a/DistTestCore/Logs/LogDownloadHandler.cs b/DistTestCore/Logs/LogDownloadHandler.cs index 2c7dc9f..483e46b 100644 --- a/DistTestCore/Logs/LogDownloadHandler.cs +++ b/DistTestCore/Logs/LogDownloadHandler.cs @@ -5,21 +5,21 @@ namespace DistTestCore.Logs { public class LogDownloadHandler : LogHandler, ILogHandler { - private readonly OnlineCodexNode node; + private readonly RunningContainer container; private readonly LogFile log; - public LogDownloadHandler(OnlineCodexNode node, string description, LogFile log) + public LogDownloadHandler(RunningContainer container, string description, LogFile log) { - this.node = node; + this.container = container; this.log = log; log.Write($"{description} -->> {log.FullFilename}"); log.WriteRaw(description); } - public CodexNodeLog CreateCodexNodeLog() + public DownloadedLog DownloadLog() { - return new CodexNodeLog(log, node); + return new DownloadedLog(log, container.Name); } protected override void ProcessLine(string line) diff --git a/DistTestCore/Marketplace/MarketplaceAccess.cs b/DistTestCore/Marketplace/MarketplaceAccess.cs index 29730a6..f64e271 100644 --- a/DistTestCore/Marketplace/MarketplaceAccess.cs +++ b/DistTestCore/Marketplace/MarketplaceAccess.cs @@ -10,7 +10,7 @@ namespace DistTestCore.Marketplace public interface IMarketplaceAccess { string MakeStorageAvailable(ByteSize size, TestToken minPricePerBytePerSecond, TestToken maxCollateral, TimeSpan maxDuration); - string RequestStorage(ContentId contentId, TestToken pricePerBytePerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration); + string RequestStorage(ContentId contentId, TestToken pricePerSlotPerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration); void AssertThatBalance(IResolveConstraint constraint, string message = ""); TestToken GetBalance(); } @@ -30,13 +30,13 @@ namespace DistTestCore.Marketplace this.codexAccess = codexAccess; } - public string RequestStorage(ContentId contentId, TestToken pricePerBytePerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration) + public string RequestStorage(ContentId contentId, TestToken pricePerSlotPerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration) { var request = new CodexSalesRequestStorageRequest { duration = ToHexBigInt(duration.TotalSeconds), proofProbability = ToHexBigInt(proofProbability), - reward = ToHexBigInt(pricePerBytePerSecond), + reward = ToHexBigInt(pricePerSlotPerSecond), collateral = ToHexBigInt(requiredCollateral), expiry = null, nodes = minRequiredNumberOfNodes, @@ -44,7 +44,7 @@ namespace DistTestCore.Marketplace }; Log($"Requesting storage for: {contentId.Id}... (" + - $"pricePerBytePerSecond: {pricePerBytePerSecond}, " + + $"pricePerSlotPerSecond: {pricePerSlotPerSecond}, " + $"requiredCollateral: {requiredCollateral}, " + $"minRequiredNumberOfNodes: {minRequiredNumberOfNodes}, " + $"proofProbability: {proofProbability}, " + diff --git a/DistTestCore/OnlineCodexNode.cs b/DistTestCore/OnlineCodexNode.cs index adc7dfc..d1e5301 100644 --- a/DistTestCore/OnlineCodexNode.cs +++ b/DistTestCore/OnlineCodexNode.cs @@ -16,7 +16,7 @@ namespace DistTestCore ContentId UploadFile(TestFile file); TestFile? DownloadContent(ContentId contentId, string fileLabel = ""); void ConnectToPeer(IOnlineCodexNode node); - ICodexNodeLog DownloadLog(); + IDownloadedLog DownloadLog(); IMetricsAccess Metrics { get; } IMarketplaceAccess Marketplace { get; } ICodexSetup BringOffline(); @@ -107,9 +107,9 @@ namespace DistTestCore Log($"Successfully connected to peer {peer.GetName()}."); } - public ICodexNodeLog DownloadLog() + public IDownloadedLog DownloadLog() { - return lifecycle.DownloadLog(this); + return lifecycle.DownloadLog(CodexAccess.Container); } public ICodexSetup BringOffline() diff --git a/DistTestCore/TestLifecycle.cs b/DistTestCore/TestLifecycle.cs index 667b96c..6ea8441 100644 --- a/DistTestCore/TestLifecycle.cs +++ b/DistTestCore/TestLifecycle.cs @@ -41,16 +41,16 @@ namespace DistTestCore FileManager.DeleteAllTestFiles(); } - public ICodexNodeLog DownloadLog(OnlineCodexNode node) + public IDownloadedLog DownloadLog(RunningContainer container) { var subFile = Log.CreateSubfile(); - var description = node.GetName(); - var handler = new LogDownloadHandler(node, description, subFile); + var description = container.Name; + var handler = new LogDownloadHandler(container, description, subFile); Log.Log($"Downloading logs for {description} to file '{subFile.FullFilename}'"); - CodexStarter.DownloadLog(node.CodexAccess.Container, handler); + CodexStarter.DownloadLog(container, handler); - return new CodexNodeLog(subFile, node); + return new DownloadedLog(subFile, description); } public string GetTestDuration() diff --git a/KubernetesWorkflow/K8sController.cs b/KubernetesWorkflow/K8sController.cs index 6174291..d679c8c 100644 --- a/KubernetesWorkflow/K8sController.cs +++ b/KubernetesWorkflow/K8sController.cs @@ -39,7 +39,7 @@ namespace KubernetesWorkflow var (serviceName, servicePortsMap) = CreateService(containerRecipes); var podInfo = FetchNewPod(); - return new RunningPod(cluster, podInfo, deploymentName, serviceName, servicePortsMap); + return new RunningPod(cluster, podInfo, deploymentName, serviceName, servicePortsMap.ToArray()); } public void Stop(RunningPod pod) @@ -436,9 +436,9 @@ namespace KubernetesWorkflow #region Service management - private (string, Dictionary) CreateService(ContainerRecipe[] containerRecipes) + private (string, List) CreateService(ContainerRecipe[] containerRecipes) { - var result = new Dictionary(); + var result = new List(); var ports = CreateServicePorts(containerRecipes); @@ -468,7 +468,7 @@ namespace KubernetesWorkflow return (serviceSpec.Metadata.Name, result); } - private void ReadBackServiceAndMapPorts(V1Service serviceSpec, ContainerRecipe[] containerRecipes, Dictionary result) + private void ReadBackServiceAndMapPorts(V1Service serviceSpec, ContainerRecipe[] containerRecipes, List result) { // For each container-recipe, we need to figure out which service-ports it was assigned by K8s. var readback = client.Run(c => c.ReadNamespacedService(serviceSpec.Metadata.Name, K8sTestNamespace)); @@ -485,7 +485,8 @@ namespace KubernetesWorkflow // These service ports belongs to this recipe. var optionals = matchingServicePorts.Select(p => MapNodePortIfAble(p, portName)); var ports = optionals.Where(p => p != null).Select(p => p!).ToArray(); - result.Add(r, ports); + + result.Add(new ContainerRecipePortMapEntry(r.Number, ports)); } } } diff --git a/KubernetesWorkflow/RunningPod.cs b/KubernetesWorkflow/RunningPod.cs index 1618410..ca8c9c1 100644 --- a/KubernetesWorkflow/RunningPod.cs +++ b/KubernetesWorkflow/RunningPod.cs @@ -2,35 +2,50 @@ { public class RunningPod { - private readonly Dictionary servicePortMap; - - public RunningPod(K8sCluster cluster, PodInfo podInfo, string deploymentName, string serviceName, Dictionary servicePortMap) + public RunningPod(K8sCluster cluster, PodInfo podInfo, string deploymentName, string serviceName, ContainerRecipePortMapEntry[] portMapEntries) { Cluster = cluster; PodInfo = podInfo; DeploymentName = deploymentName; ServiceName = serviceName; - this.servicePortMap = servicePortMap; + PortMapEntries = portMapEntries; } public K8sCluster Cluster { get; } public PodInfo PodInfo { get; } + public ContainerRecipePortMapEntry[] PortMapEntries { get; } internal string DeploymentName { get; } internal string ServiceName { get; } public Port[] GetServicePortsForContainerRecipe(ContainerRecipe containerRecipe) { - if (!servicePortMap.ContainsKey(containerRecipe)) return Array.Empty(); - return servicePortMap[containerRecipe]; + if (PortMapEntries.Any(p => p.ContainerNumber == containerRecipe.Number)) + { + return PortMapEntries.Single(p => p.ContainerNumber == containerRecipe.Number).Ports; + } + + return Array.Empty(); } } + public class ContainerRecipePortMapEntry + { + public ContainerRecipePortMapEntry(int containerNumber, Port[] ports) + { + ContainerNumber = containerNumber; + Ports = ports; + } + + public int ContainerNumber { get; } + public Port[] Ports { get; } + } + public class PodInfo { - public PodInfo(string podName, string podIp, string k8sNodeName) + public PodInfo(string name, string ip, string k8sNodeName) { - Name = podName; - Ip = podIp; + Name = name; + Ip = ip; K8SNodeName = k8sNodeName; } diff --git a/Tests/BasicTests/ExampleTests.cs b/Tests/BasicTests/ExampleTests.cs index 976ccc3..3007ae4 100644 --- a/Tests/BasicTests/ExampleTests.cs +++ b/Tests/BasicTests/ExampleTests.cs @@ -64,7 +64,7 @@ namespace Tests.BasicTests var contentId = buyer.UploadFile(testFile); buyer.Marketplace.RequestStorage(contentId, - pricePerBytePerSecond: 2.TestTokens(), + pricePerSlotPerSecond: 2.TestTokens(), requiredCollateral: 10.TestTokens(), minRequiredNumberOfNodes: 1, proofProbability: 5, From 2f5bf4b76e85cc8b6c3be95c86f7705f39a14145 Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 28 Jun 2023 08:48:46 +0200 Subject: [PATCH 05/49] Adds configuration for selling of storage space --- CodexNetDeployer/CodexNodeStarter.cs | 48 +++++++++++-------- CodexNetDeployer/Configuration.cs | 13 +++++ CodexNetDeployer/Deployer.cs | 8 +++- DistTestCore/Codex/CodexDeployment.cs | 12 ++++- DistTestCore/Marketplace/MarketplaceAccess.cs | 10 ++-- 5 files changed, 63 insertions(+), 28 deletions(-) diff --git a/CodexNetDeployer/CodexNodeStarter.cs b/CodexNetDeployer/CodexNodeStarter.cs index ee071db..7c33892 100644 --- a/CodexNetDeployer/CodexNodeStarter.cs +++ b/CodexNetDeployer/CodexNodeStarter.cs @@ -2,7 +2,6 @@ using DistTestCore.Codex; using DistTestCore.Marketplace; using KubernetesWorkflow; -using Logging; namespace CodexNetDeployer { @@ -11,19 +10,15 @@ namespace CodexNetDeployer private readonly Configuration config; private readonly WorkflowCreator workflowCreator; private readonly TestLifecycle lifecycle; - private readonly BaseLog log; - private readonly ITimeSet timeSet; private readonly GethStartResult gethResult; private string bootstrapSpr = ""; private int validatorsLeft; - public CodexNodeStarter(Configuration config, WorkflowCreator workflowCreator, TestLifecycle lifecycle, BaseLog log, ITimeSet timeSet, GethStartResult gethResult, int numberOfValidators) + public CodexNodeStarter(Configuration config, WorkflowCreator workflowCreator, TestLifecycle lifecycle, GethStartResult gethResult, int numberOfValidators) { this.config = config; this.workflowCreator = workflowCreator; this.lifecycle = lifecycle; - this.log = log; - this.timeSet = timeSet; this.gethResult = gethResult; this.validatorsLeft = numberOfValidators; } @@ -39,24 +34,39 @@ namespace CodexNetDeployer var containers = workflow.Start(1, Location.Unspecified, new CodexContainerRecipe(), workflowStartup); var container = containers.Containers.First(); - var address = lifecycle.Configuration.GetAddress(container); - var codexNode = new CodexNode(log, timeSet, address); - var debugInfo = codexNode.GetDebugInfo(); + var codexAccess = new CodexAccess(lifecycle, container); + var account = gethResult.MarketplaceNetwork.Bootstrap.AllAccounts.Accounts[i]; + var tokenAddress = gethResult.MarketplaceNetwork.Marketplace.TokenAddress; + var marketAccess = new MarketplaceAccess(lifecycle, gethResult.MarketplaceNetwork, account, codexAccess); + + var debugInfo = codexAccess.Node.GetDebugInfo(); if (!string.IsNullOrWhiteSpace(debugInfo.spr)) { - var pod = container.Pod.PodInfo; - Console.Write($"Online ({pod.Name} at {pod.Ip} on '{pod.K8SNodeName}')" + Environment.NewLine); + Console.Write("Online\t"); - if (string.IsNullOrEmpty(bootstrapSpr)) bootstrapSpr = debugInfo.spr; - validatorsLeft--; - return container; - } - else - { - Console.Write("Unknown failure." + Environment.NewLine); - return null; + var interaction = gethResult.MarketplaceNetwork.Bootstrap.StartInteraction(lifecycle); + interaction.MintTestTokens(new[] { account.Account }, config.InitialTestTokens, tokenAddress); + Console.Write("Tokens minted\t"); + + var response = marketAccess.MakeStorageAvailable( + totalSpace: (config.StorageQuota!.Value - 1).MB(), + minPriceForTotalSpace: config.MinPrice.TestTokens(), + maxCollateral: config.MaxCollateral.TestTokens(), + maxDuration: TimeSpan.FromSeconds(config.MaxDuration)); + + if (!string.IsNullOrEmpty(response)) + { + Console.Write("Storage available\tOK" + Environment.NewLine); + + if (string.IsNullOrEmpty(bootstrapSpr)) bootstrapSpr = debugInfo.spr; + validatorsLeft--; + return container; + } } + + Console.Write("Unknown failure." + Environment.NewLine); + return null; } private CodexStartupConfig CreateCodexStartupConfig(string bootstrapSpr, int i, int validatorsLeft) diff --git a/CodexNetDeployer/Configuration.cs b/CodexNetDeployer/Configuration.cs index b25e483..d58e233 100644 --- a/CodexNetDeployer/Configuration.cs +++ b/CodexNetDeployer/Configuration.cs @@ -34,6 +34,18 @@ namespace CodexNetDeployer [Uniform("log-level", "l", "LOGLEVEL", true, "Log level used by each Codex node. [Trace, Debug*, Info, Warn, Error]")] public CodexLogLevel CodexLogLevel { get; set; } = CodexLogLevel.Debug; + [Uniform("test-tokens", "tt", "TESTTOKENS", true, "Initial amount of test-tokens minted for each Codex node.")] + public int InitialTestTokens { get; set; } = int.MaxValue; + + [Uniform("min-price", "mp", "MINPRICE", true, "Minimum price for the storage space for which contracts will be accepted.")] + public int MinPrice { get; set; } + + [Uniform("max-collateral", "mc", "MAXCOLLATERAL", true, "Maximum collateral that will be placed for the total storage space.")] + public int MaxCollateral { get; set; } + + [Uniform("max-duration", "md", "MAXDURATION", true, "Maximum duration in seconds for contracts which will be accepted.")] + public int MaxDuration { get; set; } + public TestRunnerLocation RunnerLocation { get; set; } = TestRunnerLocation.InternalToCluster; public List Validate() @@ -59,6 +71,7 @@ namespace CodexNetDeployer { if (p.PropertyType == typeof(string)) onString(p.Name, (string)p.GetValue(this)!); if (p.PropertyType == typeof(int?)) onInt(p.Name, (int?)p.GetValue(this)!); + if (p.PropertyType == typeof(int)) onInt(p.Name, (int)p.GetValue(this)!); } } diff --git a/CodexNetDeployer/Deployer.cs b/CodexNetDeployer/Deployer.cs index ba0ecc5..b4efd3f 100644 --- a/CodexNetDeployer/Deployer.cs +++ b/CodexNetDeployer/Deployer.cs @@ -38,7 +38,7 @@ namespace CodexNetDeployer Log("Starting Codex nodes..."); // Each node must have its own IP, so it needs it own pod. Start them 1 at a time. - var codexStarter = new CodexNodeStarter(config, workflowCreator, lifecycle, log, timeset, gethResults, config.NumberOfValidators!.Value); + var codexStarter = new CodexNodeStarter(config, workflowCreator, lifecycle, gethResults, config.NumberOfValidators!.Value); var codexContainers = new List(); for (var i = 0; i < config.NumberOfCodexNodes; i++) { @@ -91,7 +91,11 @@ namespace CodexNetDeployer numberOfCodexNodes: config.NumberOfCodexNodes!.Value, numberOfValidators: config.NumberOfValidators!.Value, storageQuotaMB: config.StorageQuota!.Value, - codexLogLevel: config.CodexLogLevel); + codexLogLevel: config.CodexLogLevel, + initialTestTokens: config.InitialTestTokens, + minPrice: config.MinPrice, + maxCollateral: config.MaxCollateral, + maxDuration: config.MaxDuration); } private void Log(string msg) diff --git a/DistTestCore/Codex/CodexDeployment.cs b/DistTestCore/Codex/CodexDeployment.cs index cc4246e..37db831 100644 --- a/DistTestCore/Codex/CodexDeployment.cs +++ b/DistTestCore/Codex/CodexDeployment.cs @@ -19,7 +19,7 @@ namespace DistTestCore.Codex public class DeploymentMetadata { - public DeploymentMetadata(string codexImage, string gethImage, string contractsImage, string kubeNamespace, int numberOfCodexNodes, int numberOfValidators, int storageQuotaMB, CodexLogLevel codexLogLevel) + public DeploymentMetadata(string codexImage, string gethImage, string contractsImage, string kubeNamespace, int numberOfCodexNodes, int numberOfValidators, int storageQuotaMB, CodexLogLevel codexLogLevel, int initialTestTokens, int minPrice, int maxCollateral, int maxDuration) { DeployDateTimeUtc = DateTime.UtcNow; CodexImage = codexImage; @@ -30,8 +30,12 @@ namespace DistTestCore.Codex NumberOfValidators = numberOfValidators; StorageQuotaMB = storageQuotaMB; CodexLogLevel = codexLogLevel; + InitialTestTokens = initialTestTokens; + MinPrice = minPrice; + MaxCollateral = maxCollateral; + MaxDuration = maxDuration; } - + public string CodexImage { get; } public DateTime DeployDateTimeUtc { get; } public string GethImage { get; } @@ -41,5 +45,9 @@ namespace DistTestCore.Codex public int NumberOfValidators { get; } public int StorageQuotaMB { get; } public CodexLogLevel CodexLogLevel { get; } + public int InitialTestTokens { get; } + public int MinPrice { get; } + public int MaxCollateral { get; } + public int MaxDuration { get; } } } diff --git a/DistTestCore/Marketplace/MarketplaceAccess.cs b/DistTestCore/Marketplace/MarketplaceAccess.cs index f64e271..858717d 100644 --- a/DistTestCore/Marketplace/MarketplaceAccess.cs +++ b/DistTestCore/Marketplace/MarketplaceAccess.cs @@ -62,19 +62,19 @@ namespace DistTestCore.Marketplace return response; } - public string MakeStorageAvailable(ByteSize size, TestToken minPricePerBytePerSecond, TestToken maxCollateral, TimeSpan maxDuration) + public string MakeStorageAvailable(ByteSize totalSpace, TestToken minPriceForTotalSpace, TestToken maxCollateral, TimeSpan maxDuration) { var request = new CodexSalesAvailabilityRequest { - size = ToHexBigInt(size.SizeInBytes), + size = ToHexBigInt(totalSpace.SizeInBytes), duration = ToHexBigInt(maxDuration.TotalSeconds), maxCollateral = ToHexBigInt(maxCollateral), - minPrice = ToHexBigInt(minPricePerBytePerSecond) + minPrice = ToHexBigInt(minPriceForTotalSpace) }; Log($"Making storage available... (" + - $"size: {size}, " + - $"minPricePerBytePerSecond: {minPricePerBytePerSecond}, " + + $"size: {totalSpace}, " + + $"minPricePerBytePerSecond: {minPriceForTotalSpace}, " + $"maxCollateral: {maxCollateral}, " + $"maxDuration: {Time.FormatDuration(maxDuration)})"); From fe1ee45775725de4b5c5ee90bd629f5c6982e998 Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 28 Jun 2023 10:41:04 +0200 Subject: [PATCH 06/49] Debugging and cleanin up marketplace test --- ContinuousTests/Tests/MarketplaceTest.cs | 53 ++++++++++++++----- DistTestCore/Marketplace/MarketplaceAccess.cs | 27 +++++----- 2 files changed, 54 insertions(+), 26 deletions(-) diff --git a/ContinuousTests/Tests/MarketplaceTest.cs b/ContinuousTests/Tests/MarketplaceTest.cs index c99bcd1..9c9a6a9 100644 --- a/ContinuousTests/Tests/MarketplaceTest.cs +++ b/ContinuousTests/Tests/MarketplaceTest.cs @@ -11,7 +11,7 @@ namespace ContinuousTests.Tests public class MarketplaceTest : ContinuousTest { public override int RequiredNumberOfNodes => 1; - public override TimeSpan RunTestEvery => TimeSpan.FromDays(4); + public override TimeSpan RunTestEvery => TimeSpan.FromMinutes(15); public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure; public const int EthereumAccountIndex = 200; // TODO: Check against all other account indices of all other tests. @@ -28,10 +28,9 @@ namespace ContinuousTests.Tests [TestMoment(t: Zero)] public void NodePostsStorageRequest() { - var contractDuration = TimeSpan.FromDays(3) + TimeSpan.FromHours(1); + var contractDuration = TimeSpan.FromMinutes(11); //TimeSpan.FromDays(3) + TimeSpan.FromHours(1); decimal totalDurationSeconds = Convert.ToDecimal(contractDuration.TotalSeconds); - var expectedTotalCost = numberOfSlots * pricePerSlotPerSecond.Amount * (totalDurationSeconds + 1); - Log.Log("expected total cost: " + expectedTotalCost); + var expectedTotalCost = numberOfSlots * pricePerSlotPerSecond.Amount * (totalDurationSeconds + 1) * 1000000; file = FileManager.GenerateTestFile(fileSize); @@ -44,7 +43,7 @@ namespace ContinuousTests.Tests Assert.That(!string.IsNullOrEmpty(debugInfo.spr)); var startupConfig = new StartupConfig(); - var codexStartConfig = new CodexStartupConfig(CodexLogLevel.Debug); + var codexStartConfig = new CodexStartupConfig(CodexLogLevel.Trace); codexStartConfig.MarketplaceConfig = new MarketplaceInitialConfig(0.Eth(), 0.TestTokens(), false); codexStartConfig.MarketplaceConfig.AccountIndexOverride = EthereumAccountIndex; codexStartConfig.BootstrapSpr = debugInfo.spr; @@ -52,7 +51,7 @@ namespace ContinuousTests.Tests startupConfig.Add(Configuration.CodexDeployment.GethStartResult); var rc = flow.Start(1, Location.Unspecified, new CodexContainerRecipe(), startupConfig); - var account = Configuration.CodexDeployment.GethStartResult.MarketplaceNetwork.Bootstrap.AllAccounts.Accounts[EthereumAccountIndex]; + var account = Configuration.CodexDeployment.GethStartResult.CompanionNode.Accounts[EthereumAccountIndex]; var tokenAddress = Configuration.CodexDeployment.GethStartResult.MarketplaceNetwork.Marketplace.TokenAddress; var interaction = Configuration.CodexDeployment.GethStartResult.MarketplaceNetwork.Bootstrap.StartInteraction(lifecycle); @@ -61,15 +60,13 @@ namespace ContinuousTests.Tests var container = rc.Containers[0]; var marketplaceNetwork = Configuration.CodexDeployment.GethStartResult.MarketplaceNetwork; var codexAccess = new CodexAccess(lifecycle, container); + var myNodeInfo = codexAccess.Node.GetDebugInfo(); + var marketAccess = new MarketplaceAccess(lifecycle, marketplaceNetwork, account, codexAccess); cid = UploadFile(codexAccess.Node, file); Assert.That(cid, Is.Not.Null); - var balance = marketAccess.GetBalance(); - Log.Log("Account: " + account.Account); - Log.Log("Balance: " + balance); - purchaseId = marketAccess.RequestStorage( contentId: cid!, pricePerSlotPerSecond: pricePerSlotPerSecond, @@ -78,8 +75,33 @@ namespace ContinuousTests.Tests proofProbability: 10, duration: contractDuration); - Log.Log($"PurchaseId: '{purchaseId}'"); + Log($"PurchaseId: '{purchaseId}'"); Assert.That(!string.IsNullOrEmpty(purchaseId)); + + var lastState = ""; + var waitStart = DateTime.UtcNow; + var filesizeInMb = fileSize.SizeInBytes / 1024; + var maxWaitTime = TimeSpan.FromSeconds(filesizeInMb * 10.0); + while (lastState != "started") + { + var purchaseStatus = codexAccess.Node.GetPurchaseStatus(purchaseId); + if (purchaseStatus != null && purchaseStatus.state != lastState) + { + lastState = purchaseStatus.state; + } + + Thread.Sleep(2000); + + if (lastState == "errored") + { + Assert.Fail("Contract start failed: " + JsonConvert.SerializeObject(purchaseStatus)); + } + + if (DateTime.UtcNow - waitStart > maxWaitTime) + { + Assert.Fail($"Contract was not picked up within {maxWaitTime.TotalSeconds} seconds timeout: " + JsonConvert.SerializeObject(purchaseStatus)); + } + } } finally { @@ -87,7 +109,7 @@ namespace ContinuousTests.Tests } } - [TestMoment(t: DayThree)] + [TestMoment(t: MinuteFive * 2)] public void StoredDataIsAvailableAfterThreeDays() { var (workflowCreator, lifecycle) = CreateFacilities(); @@ -135,7 +157,7 @@ namespace ContinuousTests.Tests operationTimeout: TimeSet.K8sOperationTimeout(), retryDelay: TimeSet.WaitForK8sServiceDelay()); - var workflowCreator = new WorkflowCreator(Log, kubeFlowConfig, testNamespacePostfix: string.Empty); + var workflowCreator = new WorkflowCreator(base.Log, kubeFlowConfig, testNamespacePostfix: string.Empty); var lifecycle = new TestLifecycle(new NullLog(), lifecycleConfig, TimeSet, workflowCreator); return (workflowCreator, lifecycle); @@ -146,5 +168,10 @@ namespace ContinuousTests.Tests if (string.IsNullOrEmpty(kubeConfigFile) || kubeConfigFile.ToLowerInvariant() == "null") return null; return kubeConfigFile; } + + private new void Log(string msg) + { + base.Log.Log(msg); + } } } diff --git a/DistTestCore/Marketplace/MarketplaceAccess.cs b/DistTestCore/Marketplace/MarketplaceAccess.cs index f64e271..8a67564 100644 --- a/DistTestCore/Marketplace/MarketplaceAccess.cs +++ b/DistTestCore/Marketplace/MarketplaceAccess.cs @@ -34,10 +34,10 @@ namespace DistTestCore.Marketplace { var request = new CodexSalesRequestStorageRequest { - duration = ToHexBigInt(duration.TotalSeconds), - proofProbability = ToHexBigInt(proofProbability), - reward = ToHexBigInt(pricePerSlotPerSecond), - collateral = ToHexBigInt(requiredCollateral), + duration = ToDecInt(duration.TotalSeconds), + proofProbability = ToDecInt(proofProbability), + reward = ToDecInt(pricePerSlotPerSecond), + collateral = ToDecInt(requiredCollateral), expiry = null, nodes = minRequiredNumberOfNodes, tolerance = null, @@ -66,10 +66,10 @@ namespace DistTestCore.Marketplace { var request = new CodexSalesAvailabilityRequest { - size = ToHexBigInt(size.SizeInBytes), - duration = ToHexBigInt(maxDuration.TotalSeconds), - maxCollateral = ToHexBigInt(maxCollateral), - minPrice = ToHexBigInt(minPricePerBytePerSecond) + size = ToDecInt(size.SizeInBytes), + duration = ToDecInt(maxDuration.TotalSeconds), + maxCollateral = ToDecInt(maxCollateral), + minPrice = ToDecInt(minPricePerBytePerSecond) }; Log($"Making storage available... (" + @@ -85,15 +85,16 @@ namespace DistTestCore.Marketplace return response.id; } - private string ToHexBigInt(double d) + private string ToDecInt(double d) { - return "0x" + string.Format("{0:X}", Convert.ToInt64(d)); + var i = new BigInteger(d); + return i.ToString("D"); } - public string ToHexBigInt(TestToken t) + public string ToDecInt(TestToken t) { - var bigInt = new BigInteger(t.Amount); - return "0x" + bigInt.ToString("X"); + var i = new BigInteger(t.Amount); + return i.ToString("D"); } public void AssertThatBalance(IResolveConstraint constraint, string message = "") From f7edfd4eee7cb6928356af59c75774f8c25dcb4f Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 28 Jun 2023 11:01:10 +0200 Subject: [PATCH 07/49] Checking for collisions on custom k8s namespaces and account indices --- ContinuousTests/ContinuousTest.cs | 2 + ContinuousTests/StartupChecker.cs | 75 +++++++++++++++++++++--- ContinuousTests/Tests/MarketplaceTest.cs | 5 +- 3 files changed, 70 insertions(+), 12 deletions(-) diff --git a/ContinuousTests/ContinuousTest.cs b/ContinuousTests/ContinuousTest.cs index 3f0c90e..1d4fb23 100644 --- a/ContinuousTests/ContinuousTest.cs +++ b/ContinuousTests/ContinuousTest.cs @@ -38,6 +38,8 @@ namespace ContinuousTests public abstract int RequiredNumberOfNodes { get; } public abstract TimeSpan RunTestEvery { get; } public abstract TestFailMode TestFailMode { get; } + public virtual int EthereumAccountIndex { get { return -1; } } + public virtual string CustomK8sNamespace { get { return string.Empty; } } public string Name { diff --git a/ContinuousTests/StartupChecker.cs b/ContinuousTests/StartupChecker.cs index 8000b5b..12e34cc 100644 --- a/ContinuousTests/StartupChecker.cs +++ b/ContinuousTests/StartupChecker.cs @@ -1,6 +1,7 @@ using DistTestCore.Codex; using DistTestCore; using Logging; +using NUnit.Framework.Internal; namespace ContinuousTests { @@ -40,20 +41,12 @@ namespace ContinuousTests handle.GetLastMoment(); } - var errors = new List(); - foreach (var test in tests) - { - if (test.RequiredNumberOfNodes > config.CodexDeployment.CodexContainers.Length) - { - errors.Add($"Test '{test.Name}' requires {test.RequiredNumberOfNodes} nodes. Deployment only has {config.CodexDeployment.CodexContainers.Length}"); - } - } - if (!Directory.Exists(config.LogPath)) { Directory.CreateDirectory(config.LogPath); } + var errors = CheckTests(tests); if (errors.Any()) { throw new Exception("Prerun check failed: " + string.Join(", ", errors)); @@ -98,5 +91,69 @@ namespace ContinuousTests return true; } + private List CheckTests(ContinuousTest[] tests) + { + var errors = new List(); + CheckRequiredNumberOfNodes(tests, errors); + CheckCustomNamespaceClashes(tests, errors); + CheckEthereumIndexClashes(tests, errors); + return errors; + } + + private void CheckEthereumIndexClashes(ContinuousTest[] tests, List errors) + { + var offLimits = config.CodexDeployment.CodexContainers.Length; + foreach (var test in tests) + { + if (test.EthereumAccountIndex != -1) + { + if (test.EthereumAccountIndex <= offLimits) + { + errors.Add($"Test '{test.Name}' has selected 'EthereumAccountIndex' = {test.EthereumAccountIndex}. All accounts up to and including {offLimits} are being used by the targetted Codex net. Select a different 'EthereumAccountIndex'."); + } + } + } + + DuplicatesCheck(tests, errors, + considerCondition: t => t.EthereumAccountIndex != -1, + getValue: t => t.EthereumAccountIndex, + propertyName: nameof(ContinuousTest.EthereumAccountIndex)); + } + + private void CheckCustomNamespaceClashes(ContinuousTest[] tests, List errors) + { + DuplicatesCheck(tests, errors, + considerCondition: t => !string.IsNullOrEmpty(t.CustomK8sNamespace), + getValue: t => t.CustomK8sNamespace, + propertyName: nameof(ContinuousTest.CustomK8sNamespace)); + } + + private void DuplicatesCheck(ContinuousTest[] tests, List errors, Func considerCondition, Func getValue, string propertyName) + { + foreach (var test in tests) + { + if (considerCondition(test)) + { + var duplicates = tests.Where(t => t != test && getValue(t) == getValue(test)).ToList(); + if (duplicates.Any()) + { + duplicates.Add(test); + errors.Add($"Tests '{string.Join(",", duplicates.Select(d => d.Name))}' have the same '{propertyName}'. These must be unique."); + return; + } + } + } + } + + private void CheckRequiredNumberOfNodes(ContinuousTest[] tests, List errors) + { + foreach (var test in tests) + { + if (test.RequiredNumberOfNodes > config.CodexDeployment.CodexContainers.Length) + { + errors.Add($"Test '{test.Name}' requires {test.RequiredNumberOfNodes} nodes. Deployment only has {config.CodexDeployment.CodexContainers.Length}"); + } + } + } } } diff --git a/ContinuousTests/Tests/MarketplaceTest.cs b/ContinuousTests/Tests/MarketplaceTest.cs index 9c9a6a9..5728068 100644 --- a/ContinuousTests/Tests/MarketplaceTest.cs +++ b/ContinuousTests/Tests/MarketplaceTest.cs @@ -13,9 +13,8 @@ namespace ContinuousTests.Tests public override int RequiredNumberOfNodes => 1; public override TimeSpan RunTestEvery => TimeSpan.FromMinutes(15); public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure; - - public const int EthereumAccountIndex = 200; // TODO: Check against all other account indices of all other tests. - public const string MarketplaceTestNamespace = "codex-continuous-marketplace"; // prevent clashes too + public override int EthereumAccountIndex => 200; + public override string CustomK8sNamespace => "codex-continuous-marketplace"; private readonly uint numberOfSlots = 3; private readonly ByteSize fileSize = 10.MB(); From 9a3f45e60d9e4b76783437fefd18d3b2e9219319 Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 28 Jun 2023 11:48:05 +0200 Subject: [PATCH 08/49] Extracts node-running logic from marketplace test --- ContinuousTests/ContinuousTest.cs | 11 ++ ContinuousTests/NodeRunner.cs | 106 ++++++++++++++++ ContinuousTests/Tests/MarketplaceTest.cs | 150 +++++------------------ 3 files changed, 150 insertions(+), 117 deletions(-) create mode 100644 ContinuousTests/NodeRunner.cs diff --git a/ContinuousTests/ContinuousTest.cs b/ContinuousTests/ContinuousTest.cs index 1d4fb23..cdbf6a8 100644 --- a/ContinuousTests/ContinuousTest.cs +++ b/ContinuousTests/ContinuousTest.cs @@ -1,6 +1,7 @@ using DistTestCore; using DistTestCore.Codex; using Logging; +using Utils; namespace ContinuousTests { @@ -27,6 +28,15 @@ namespace ContinuousTests Log = log; FileManager = fileManager; Configuration = configuration; + + if (nodes != null) + { + NodeRunner = new NodeRunner(Nodes.ToList().PickOneRandom(), configuration, TimeSet, Log, CustomK8sNamespace, EthereumAccountIndex); + } + else + { + NodeRunner = null!; + } } public CodexNode[] Nodes { get; private set; } = null!; @@ -34,6 +44,7 @@ namespace ContinuousTests public IFileManager FileManager { get; private set; } = null!; public Configuration Configuration { get; private set; } = null!; public virtual ITimeSet TimeSet { get { return new DefaultTimeSet(); } } + public NodeRunner NodeRunner { get; private set; } = null!; public abstract int RequiredNumberOfNodes { get; } public abstract TimeSpan RunTestEvery { get; } diff --git a/ContinuousTests/NodeRunner.cs b/ContinuousTests/NodeRunner.cs new file mode 100644 index 0000000..5663817 --- /dev/null +++ b/ContinuousTests/NodeRunner.cs @@ -0,0 +1,106 @@ +using DistTestCore.Codex; +using DistTestCore.Marketplace; +using DistTestCore; +using KubernetesWorkflow; +using NUnit.Framework; +using Logging; + +namespace ContinuousTests +{ + public class NodeRunner + { + private readonly CodexNode bootstrapNode; + private readonly Configuration config; + private readonly ITimeSet timeSet; + private readonly BaseLog log; + private readonly string customNamespace; + private readonly int ethereumAccountIndex; + + public NodeRunner(CodexNode bootstrapNode, Configuration config, ITimeSet timeSet, BaseLog log, string customNamespace, int ethereumAccountIndex) + { + this.bootstrapNode = bootstrapNode; + this.config = config; + this.timeSet = timeSet; + this.log = log; + this.customNamespace = customNamespace; + this.ethereumAccountIndex = ethereumAccountIndex; + } + + public void RunNode(Action operation) + { + RunNode(operation, 0.TestTokens()); + } + + public void RunNode(Action operation, TestToken mintTestTokens) + { + var (workflowCreator, lifecycle) = CreateFacilities(); + var flow = workflowCreator.CreateWorkflow(); + + try + { + var debugInfo = bootstrapNode.GetDebugInfo(); + Assert.That(!string.IsNullOrEmpty(debugInfo.spr)); + + var startupConfig = new StartupConfig(); + var codexStartConfig = new CodexStartupConfig(CodexLogLevel.Trace); + codexStartConfig.MarketplaceConfig = new MarketplaceInitialConfig(0.Eth(), 0.TestTokens(), false); + codexStartConfig.MarketplaceConfig.AccountIndexOverride = ethereumAccountIndex; + codexStartConfig.BootstrapSpr = debugInfo.spr; + startupConfig.Add(codexStartConfig); + startupConfig.Add(config.CodexDeployment.GethStartResult); + var rc = flow.Start(1, Location.Unspecified, new CodexContainerRecipe(), startupConfig); + + var account = config.CodexDeployment.GethStartResult.CompanionNode.Accounts[ethereumAccountIndex]; + + var marketplaceNetwork = config.CodexDeployment.GethStartResult.MarketplaceNetwork; + if (mintTestTokens.Amount > 0) + { + var tokenAddress = marketplaceNetwork.Marketplace.TokenAddress; + var interaction = marketplaceNetwork.Bootstrap.StartInteraction(lifecycle); + interaction.MintTestTokens(new[] { account.Account }, mintTestTokens.Amount, tokenAddress); + } + + var container = rc.Containers[0]; + var codexAccess = new CodexAccess(lifecycle, container); + var marketAccess = new MarketplaceAccess(lifecycle, marketplaceNetwork, account, codexAccess); + + operation(codexAccess, marketAccess); + } + finally + { + flow.DeleteTestResources(); + } + } + + private (WorkflowCreator, TestLifecycle) CreateFacilities() + { + var kubeConfig = GetKubeConfig(config.KubeConfigFile); + var lifecycleConfig = new DistTestCore.Configuration + ( + kubeConfigFile: kubeConfig, + logPath: "null", + logDebug: false, + dataFilesPath: config.LogPath, + codexLogLevel: CodexLogLevel.Debug, + runnerLocation: TestRunnerLocation.ExternalToCluster + ); + + var kubeFlowConfig = new KubernetesWorkflow.Configuration( + k8sNamespacePrefix: customNamespace, + kubeConfigFile: kubeConfig, + operationTimeout: timeSet.K8sOperationTimeout(), + retryDelay: timeSet.WaitForK8sServiceDelay()); + + var workflowCreator = new WorkflowCreator(log, kubeFlowConfig, testNamespacePostfix: string.Empty); + var lifecycle = new TestLifecycle(new NullLog(), lifecycleConfig, timeSet, workflowCreator); + + return (workflowCreator, lifecycle); + } + + private static string? GetKubeConfig(string kubeConfigFile) + { + if (string.IsNullOrEmpty(kubeConfigFile) || kubeConfigFile.ToLowerInvariant() == "null") return null; + return kubeConfigFile; + } + } +} diff --git a/ContinuousTests/Tests/MarketplaceTest.cs b/ContinuousTests/Tests/MarketplaceTest.cs index 5728068..7dd1a26 100644 --- a/ContinuousTests/Tests/MarketplaceTest.cs +++ b/ContinuousTests/Tests/MarketplaceTest.cs @@ -1,8 +1,5 @@ using DistTestCore; using DistTestCore.Codex; -using DistTestCore.Marketplace; -using KubernetesWorkflow; -using Logging; using Newtonsoft.Json; using NUnit.Framework; @@ -33,40 +30,12 @@ namespace ContinuousTests.Tests file = FileManager.GenerateTestFile(fileSize); - var (workflowCreator, lifecycle) = CreateFacilities(); - var flow = workflowCreator.CreateWorkflow(); - - try + NodeRunner.RunNode((codexAccess, marketplaceAccess) => { - var debugInfo = Nodes[0].GetDebugInfo(); - Assert.That(!string.IsNullOrEmpty(debugInfo.spr)); - - var startupConfig = new StartupConfig(); - var codexStartConfig = new CodexStartupConfig(CodexLogLevel.Trace); - codexStartConfig.MarketplaceConfig = new MarketplaceInitialConfig(0.Eth(), 0.TestTokens(), false); - codexStartConfig.MarketplaceConfig.AccountIndexOverride = EthereumAccountIndex; - codexStartConfig.BootstrapSpr = debugInfo.spr; - startupConfig.Add(codexStartConfig); - startupConfig.Add(Configuration.CodexDeployment.GethStartResult); - var rc = flow.Start(1, Location.Unspecified, new CodexContainerRecipe(), startupConfig); - - var account = Configuration.CodexDeployment.GethStartResult.CompanionNode.Accounts[EthereumAccountIndex]; - var tokenAddress = Configuration.CodexDeployment.GethStartResult.MarketplaceNetwork.Marketplace.TokenAddress; - - var interaction = Configuration.CodexDeployment.GethStartResult.MarketplaceNetwork.Bootstrap.StartInteraction(lifecycle); - interaction.MintTestTokens(new[] { account.Account }, expectedTotalCost, tokenAddress); - - var container = rc.Containers[0]; - var marketplaceNetwork = Configuration.CodexDeployment.GethStartResult.MarketplaceNetwork; - var codexAccess = new CodexAccess(lifecycle, container); - var myNodeInfo = codexAccess.Node.GetDebugInfo(); - - var marketAccess = new MarketplaceAccess(lifecycle, marketplaceNetwork, account, codexAccess); - cid = UploadFile(codexAccess.Node, file); Assert.That(cid, Is.Not.Null); - purchaseId = marketAccess.RequestStorage( + purchaseId = marketplaceAccess.RequestStorage( contentId: cid!, pricePerSlotPerSecond: pricePerSlotPerSecond, requiredCollateral: 100.TestTokens(), @@ -74,103 +43,50 @@ namespace ContinuousTests.Tests proofProbability: 10, duration: contractDuration); - Log($"PurchaseId: '{purchaseId}'"); Assert.That(!string.IsNullOrEmpty(purchaseId)); - var lastState = ""; - var waitStart = DateTime.UtcNow; - var filesizeInMb = fileSize.SizeInBytes / 1024; - var maxWaitTime = TimeSpan.FromSeconds(filesizeInMb * 10.0); - while (lastState != "started") - { - var purchaseStatus = codexAccess.Node.GetPurchaseStatus(purchaseId); - if (purchaseStatus != null && purchaseStatus.state != lastState) - { - lastState = purchaseStatus.state; - } - - Thread.Sleep(2000); - - if (lastState == "errored") - { - Assert.Fail("Contract start failed: " + JsonConvert.SerializeObject(purchaseStatus)); - } - - if (DateTime.UtcNow - waitStart > maxWaitTime) - { - Assert.Fail($"Contract was not picked up within {maxWaitTime.TotalSeconds} seconds timeout: " + JsonConvert.SerializeObject(purchaseStatus)); - } - } - } - finally - { - flow.DeleteTestResources(); - } + WaitForContractToStart(codexAccess, purchaseId); + }); } [TestMoment(t: MinuteFive * 2)] public void StoredDataIsAvailableAfterThreeDays() { - var (workflowCreator, lifecycle) = CreateFacilities(); - var flow = workflowCreator.CreateWorkflow(); - - try + NodeRunner.RunNode((codexAccess, marketplaceAccess) => { - var debugInfo = Nodes[0].GetDebugInfo(); - Assert.That(!string.IsNullOrEmpty(debugInfo.spr)); - - var startupConfig = new StartupConfig(); - var codexStartConfig = new CodexStartupConfig(CodexLogLevel.Debug); - codexStartConfig.BootstrapSpr = debugInfo.spr; - startupConfig.Add(codexStartConfig); - var rc = flow.Start(1, Location.Unspecified, new CodexContainerRecipe(), startupConfig); - var container = rc.Containers[0]; - var codexAccess = new CodexAccess(lifecycle, container); - var result = DownloadContent(codexAccess.Node, cid!); file.AssertIsEqual(result); - } - finally + }); + } + + private void WaitForContractToStart(CodexAccess codexAccess, string purchaseId) + { + var lastState = ""; + var waitStart = DateTime.UtcNow; + var filesizeInMb = fileSize.SizeInBytes / 1024; + var maxWaitTime = TimeSpan.FromSeconds(filesizeInMb * 10.0); + + while (lastState != "started") { - flow.DeleteTestResources(); + var purchaseStatus = codexAccess.Node.GetPurchaseStatus(purchaseId); + if (purchaseStatus != null && purchaseStatus.state != lastState) + { + lastState = purchaseStatus.state; + } + + Thread.Sleep(2000); + + if (lastState == "errored") + { + Assert.Fail("Contract start failed: " + JsonConvert.SerializeObject(purchaseStatus)); + } + + if (DateTime.UtcNow - waitStart > maxWaitTime) + { + Assert.Fail($"Contract was not picked up within {maxWaitTime.TotalSeconds} seconds timeout: " + JsonConvert.SerializeObject(purchaseStatus)); + } } } - - private (WorkflowCreator, TestLifecycle) CreateFacilities() - { - var kubeConfig = GetKubeConfig(Configuration.KubeConfigFile); - var lifecycleConfig = new DistTestCore.Configuration - ( - kubeConfigFile: kubeConfig, - logPath: "null", - logDebug: false, - dataFilesPath: Configuration.LogPath, - codexLogLevel: CodexLogLevel.Debug, - runnerLocation: TestRunnerLocation.ExternalToCluster - ); - - var kubeFlowConfig = new KubernetesWorkflow.Configuration( - k8sNamespacePrefix: MarketplaceTestNamespace, - kubeConfigFile: kubeConfig, - operationTimeout: TimeSet.K8sOperationTimeout(), - retryDelay: TimeSet.WaitForK8sServiceDelay()); - - var workflowCreator = new WorkflowCreator(base.Log, kubeFlowConfig, testNamespacePostfix: string.Empty); - var lifecycle = new TestLifecycle(new NullLog(), lifecycleConfig, TimeSet, workflowCreator); - - return (workflowCreator, lifecycle); - } - - private string? GetKubeConfig(string kubeConfigFile) - { - if (string.IsNullOrEmpty(kubeConfigFile) || kubeConfigFile.ToLowerInvariant() == "null") return null; - return kubeConfigFile; - } - - private new void Log(string msg) - { - base.Log.Log(msg); - } } } From 11269d1c211240af78bbaa887e2662fd9844ba9d Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 28 Jun 2023 12:01:20 +0200 Subject: [PATCH 09/49] Sets up transient node test --- ContinuousTests/ContinuousTest.cs | 3 +- ContinuousTests/ContinuousTestRunner.cs | 2 +- ContinuousTests/NodeRunner.cs | 16 +++++--- ContinuousTests/Tests/TransientNodeTest.cs | 47 ++++++++++++++++++++++ 4 files changed, 60 insertions(+), 8 deletions(-) create mode 100644 ContinuousTests/Tests/TransientNodeTest.cs diff --git a/ContinuousTests/ContinuousTest.cs b/ContinuousTests/ContinuousTest.cs index cdbf6a8..4ac0da1 100644 --- a/ContinuousTests/ContinuousTest.cs +++ b/ContinuousTests/ContinuousTest.cs @@ -1,7 +1,6 @@ using DistTestCore; using DistTestCore.Codex; using Logging; -using Utils; namespace ContinuousTests { @@ -31,7 +30,7 @@ namespace ContinuousTests if (nodes != null) { - NodeRunner = new NodeRunner(Nodes.ToList().PickOneRandom(), configuration, TimeSet, Log, CustomK8sNamespace, EthereumAccountIndex); + NodeRunner = new NodeRunner(Nodes, configuration, TimeSet, Log, CustomK8sNamespace, EthereumAccountIndex); } else { diff --git a/ContinuousTests/ContinuousTestRunner.cs b/ContinuousTests/ContinuousTestRunner.cs index 8d7fe44..d9cd276 100644 --- a/ContinuousTests/ContinuousTestRunner.cs +++ b/ContinuousTests/ContinuousTestRunner.cs @@ -28,7 +28,7 @@ namespace ContinuousTests { overviewLog.Log("Launching test-loop for " + t.Name); t.Begin(); - Thread.Sleep(TimeSpan.FromMinutes(5)); + Thread.Sleep(TimeSpan.FromSeconds(15)); } overviewLog.Log("All test-loops launched."); diff --git a/ContinuousTests/NodeRunner.cs b/ContinuousTests/NodeRunner.cs index 5663817..92f17dc 100644 --- a/ContinuousTests/NodeRunner.cs +++ b/ContinuousTests/NodeRunner.cs @@ -4,21 +4,22 @@ using DistTestCore; using KubernetesWorkflow; using NUnit.Framework; using Logging; +using Utils; namespace ContinuousTests { public class NodeRunner { - private readonly CodexNode bootstrapNode; + private readonly CodexNode[] nodes; private readonly Configuration config; private readonly ITimeSet timeSet; private readonly BaseLog log; private readonly string customNamespace; private readonly int ethereumAccountIndex; - public NodeRunner(CodexNode bootstrapNode, Configuration config, ITimeSet timeSet, BaseLog log, string customNamespace, int ethereumAccountIndex) + public NodeRunner(CodexNode[] nodes, Configuration config, ITimeSet timeSet, BaseLog log, string customNamespace, int ethereumAccountIndex) { - this.bootstrapNode = bootstrapNode; + this.nodes = nodes; this.config = config; this.timeSet = timeSet; this.log = log; @@ -28,10 +29,15 @@ namespace ContinuousTests public void RunNode(Action operation) { - RunNode(operation, 0.TestTokens()); + RunNode(nodes.ToList().PickOneRandom(), operation, 0.TestTokens()); } - public void RunNode(Action operation, TestToken mintTestTokens) + public void RunNode(CodexNode bootstrapNode, Action operation) + { + RunNode(bootstrapNode, operation, 0.TestTokens()); + } + + public void RunNode(CodexNode bootstrapNode, Action operation, TestToken mintTestTokens) { var (workflowCreator, lifecycle) = CreateFacilities(); var flow = workflowCreator.CreateWorkflow(); diff --git a/ContinuousTests/Tests/TransientNodeTest.cs b/ContinuousTests/Tests/TransientNodeTest.cs new file mode 100644 index 0000000..24971e2 --- /dev/null +++ b/ContinuousTests/Tests/TransientNodeTest.cs @@ -0,0 +1,47 @@ +using DistTestCore; +using DistTestCore.Codex; +using NUnit.Framework; + +namespace ContinuousTests.Tests +{ + public class TransientNodeTest : ContinuousTest + { + public override int RequiredNumberOfNodes => 3; + public override TimeSpan RunTestEvery => TimeSpan.FromMinutes(10); + public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure; + public override string CustomK8sNamespace => nameof(TransientNodeTest).ToLowerInvariant(); + public override int EthereumAccountIndex => 201; + + private TestFile file = null!; + private ContentId cid = null!; + + private CodexNode UploadBootstapNode { get { return Nodes[0]; } } + private CodexNode DownloadBootstapNode { get { return Nodes[1]; } } + private CodexNode IntermediateNode { get { return Nodes[2]; } } + + [TestMoment(t: 0)] + public void UploadWithTransientNode() + { + file = FileManager.GenerateTestFile(10.MB()); + + NodeRunner.RunNode(UploadBootstapNode, (codexAccess, marketplaceAccess) => + { + cid = UploadFile(codexAccess.Node, file)!; + Assert.That(cid, Is.Not.Null); + + var resultFile = DownloadContent(IntermediateNode, cid); + file.AssertIsEqual(resultFile); + }); + } + + [TestMoment(t: MinuteFive)] + public void DownloadWithTransientNode() + { + NodeRunner.RunNode(DownloadBootstapNode, (codexAccess, marketplaceAccess) => + { + var resultFile = DownloadContent(codexAccess.Node, cid); + file.AssertIsEqual(resultFile); + }); + } + } +} From efc638a0f9a2156d92439144b7323022a2457887 Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 28 Jun 2023 15:11:20 +0200 Subject: [PATCH 10/49] Debugging logging output --- ContinuousTests/ContinuousTest.cs | 2 +- ContinuousTests/ContinuousTestRunner.cs | 21 ++++++++++- ContinuousTests/K8sFactory.cs | 41 ++++++++++++++++++++++ ContinuousTests/NodeRunner.cs | 29 ++------------- ContinuousTests/Tests/MarketplaceTest.cs | 5 ++- ContinuousTests/Tests/PerformanceTests.cs | 2 +- ContinuousTests/Tests/TransientNodeTest.cs | 4 +-- ContinuousTests/Tests/TwoClientTest.cs | 2 +- DistTestCore/FileManager.cs | 1 - DistTestCore/Metrics/MetricsAccess.cs | 4 +-- DistTestCore/Metrics/MetricsDownloader.cs | 4 +-- DistTestCore/TestLifecycle.cs | 6 ++-- Logging/BaseLog.cs | 21 +++++++++-- Logging/FixtureLog.cs | 4 +-- Logging/NullLog.cs | 4 +-- Logging/TestLog.cs | 18 ++-------- 16 files changed, 104 insertions(+), 64 deletions(-) create mode 100644 ContinuousTests/K8sFactory.cs diff --git a/ContinuousTests/ContinuousTest.cs b/ContinuousTests/ContinuousTest.cs index 4ac0da1..9570dc1 100644 --- a/ContinuousTests/ContinuousTest.cs +++ b/ContinuousTests/ContinuousTest.cs @@ -77,7 +77,7 @@ namespace ContinuousTests return new ContentId(response); } - public TestFile DownloadContent(CodexNode node, ContentId contentId, string fileLabel = "") + public TestFile DownloadFile(CodexNode node, ContentId contentId, string fileLabel = "") { var logMessage = $"Downloading for contentId: '{contentId.Id}'..."; var file = FileManager.CreateEmptyTestFile(fileLabel); diff --git a/ContinuousTests/ContinuousTestRunner.cs b/ContinuousTests/ContinuousTestRunner.cs index d9cd276..85db06f 100644 --- a/ContinuousTests/ContinuousTestRunner.cs +++ b/ContinuousTests/ContinuousTestRunner.cs @@ -1,9 +1,11 @@ -using Logging; +using DistTestCore; +using Logging; namespace ContinuousTests { public class ContinuousTestRunner { + private readonly K8sFactory k8SFactory = new K8sFactory(); private readonly ConfigLoader configLoader = new ConfigLoader(); private readonly TestFactory testFactory = new TestFactory(); private readonly Configuration config; @@ -22,6 +24,9 @@ namespace ContinuousTests var overviewLog = new FixtureLog(new LogConfig(config.LogPath, false), "Overview"); overviewLog.Log("Continuous tests starting..."); var allTests = testFactory.CreateTests(); + + ClearAllCustomNamespaces(allTests, overviewLog); + var testLoop = allTests.Select(t => new TestLoop(config, overviewLog, t.GetType(), t.RunTestEvery)).ToArray(); foreach (var t in testLoop) @@ -34,5 +39,19 @@ namespace ContinuousTests overviewLog.Log("All test-loops launched."); while (true) Thread.Sleep((2 ^ 31) - 1); } + + private void ClearAllCustomNamespaces(ContinuousTest[] allTests, FixtureLog log) + { + foreach (var test in allTests) ClearAllCustomNamespaces(test, log); + } + + private void ClearAllCustomNamespaces(ContinuousTest test, FixtureLog log) + { + if (string.IsNullOrEmpty(test.CustomK8sNamespace)) return; + + log.Log($"Clearing namespace '{test.CustomK8sNamespace}'..."); + var (workflowCreator, _) = k8SFactory.CreateFacilities(config, test.CustomK8sNamespace, new DefaultTimeSet(), log); + workflowCreator.CreateWorkflow().DeleteTestResources(); + } } } diff --git a/ContinuousTests/K8sFactory.cs b/ContinuousTests/K8sFactory.cs new file mode 100644 index 0000000..040a4a3 --- /dev/null +++ b/ContinuousTests/K8sFactory.cs @@ -0,0 +1,41 @@ +using DistTestCore.Codex; +using DistTestCore; +using KubernetesWorkflow; +using Logging; + +namespace ContinuousTests +{ + public class K8sFactory + { + public (WorkflowCreator, TestLifecycle) CreateFacilities(Configuration config, string customNamespace, ITimeSet timeSet, BaseLog log) + { + var kubeConfig = GetKubeConfig(config.KubeConfigFile); + var lifecycleConfig = new DistTestCore.Configuration + ( + kubeConfigFile: kubeConfig, + logPath: "null", + logDebug: false, + dataFilesPath: config.LogPath, + codexLogLevel: CodexLogLevel.Debug, + runnerLocation: TestRunnerLocation.ExternalToCluster + ); + + var kubeFlowConfig = new KubernetesWorkflow.Configuration( + k8sNamespacePrefix: customNamespace, + kubeConfigFile: kubeConfig, + operationTimeout: timeSet.K8sOperationTimeout(), + retryDelay: timeSet.WaitForK8sServiceDelay()); + + var workflowCreator = new WorkflowCreator(log, kubeFlowConfig, testNamespacePostfix: string.Empty); + var lifecycle = new TestLifecycle(log, lifecycleConfig, timeSet, workflowCreator); + + return (workflowCreator, lifecycle); + } + + private static string? GetKubeConfig(string kubeConfigFile) + { + if (string.IsNullOrEmpty(kubeConfigFile) || kubeConfigFile.ToLowerInvariant() == "null") return null; + return kubeConfigFile; + } + } +} diff --git a/ContinuousTests/NodeRunner.cs b/ContinuousTests/NodeRunner.cs index 92f17dc..1a36d62 100644 --- a/ContinuousTests/NodeRunner.cs +++ b/ContinuousTests/NodeRunner.cs @@ -10,6 +10,7 @@ namespace ContinuousTests { public class NodeRunner { + private readonly K8sFactory k8SFactory = new K8sFactory(); private readonly CodexNode[] nodes; private readonly Configuration config; private readonly ITimeSet timeSet; @@ -80,33 +81,7 @@ namespace ContinuousTests private (WorkflowCreator, TestLifecycle) CreateFacilities() { - var kubeConfig = GetKubeConfig(config.KubeConfigFile); - var lifecycleConfig = new DistTestCore.Configuration - ( - kubeConfigFile: kubeConfig, - logPath: "null", - logDebug: false, - dataFilesPath: config.LogPath, - codexLogLevel: CodexLogLevel.Debug, - runnerLocation: TestRunnerLocation.ExternalToCluster - ); - - var kubeFlowConfig = new KubernetesWorkflow.Configuration( - k8sNamespacePrefix: customNamespace, - kubeConfigFile: kubeConfig, - operationTimeout: timeSet.K8sOperationTimeout(), - retryDelay: timeSet.WaitForK8sServiceDelay()); - - var workflowCreator = new WorkflowCreator(log, kubeFlowConfig, testNamespacePostfix: string.Empty); - var lifecycle = new TestLifecycle(new NullLog(), lifecycleConfig, timeSet, workflowCreator); - - return (workflowCreator, lifecycle); - } - - private static string? GetKubeConfig(string kubeConfigFile) - { - if (string.IsNullOrEmpty(kubeConfigFile) || kubeConfigFile.ToLowerInvariant() == "null") return null; - return kubeConfigFile; + return k8SFactory.CreateFacilities(config, customNamespace, timeSet, log); } } } diff --git a/ContinuousTests/Tests/MarketplaceTest.cs b/ContinuousTests/Tests/MarketplaceTest.cs index 7dd1a26..9582b66 100644 --- a/ContinuousTests/Tests/MarketplaceTest.cs +++ b/ContinuousTests/Tests/MarketplaceTest.cs @@ -2,6 +2,7 @@ using DistTestCore.Codex; using Newtonsoft.Json; using NUnit.Framework; +using Utils; namespace ContinuousTests.Tests { @@ -54,7 +55,7 @@ namespace ContinuousTests.Tests { NodeRunner.RunNode((codexAccess, marketplaceAccess) => { - var result = DownloadContent(codexAccess.Node, cid!); + var result = DownloadFile(codexAccess.Node, cid!); file.AssertIsEqual(result); }); @@ -67,6 +68,7 @@ namespace ContinuousTests.Tests var filesizeInMb = fileSize.SizeInBytes / 1024; var maxWaitTime = TimeSpan.FromSeconds(filesizeInMb * 10.0); + Log.Log(nameof(WaitForContractToStart) + " for " + Time.FormatDuration(maxWaitTime)); while (lastState != "started") { var purchaseStatus = codexAccess.Node.GetPurchaseStatus(purchaseId); @@ -87,6 +89,7 @@ namespace ContinuousTests.Tests Assert.Fail($"Contract was not picked up within {maxWaitTime.TotalSeconds} seconds timeout: " + JsonConvert.SerializeObject(purchaseStatus)); } } + Log.Log("Contract started."); } } } diff --git a/ContinuousTests/Tests/PerformanceTests.cs b/ContinuousTests/Tests/PerformanceTests.cs index 53af7be..032dcfa 100644 --- a/ContinuousTests/Tests/PerformanceTests.cs +++ b/ContinuousTests/Tests/PerformanceTests.cs @@ -66,7 +66,7 @@ namespace ContinuousTests.Tests TestFile? result = null; var time = Measure(() => { - result = DownloadContent(downloadNode, cid!); + result = DownloadFile(downloadNode, cid!); }); file.AssertIsEqual(result); diff --git a/ContinuousTests/Tests/TransientNodeTest.cs b/ContinuousTests/Tests/TransientNodeTest.cs index 24971e2..5c49e1b 100644 --- a/ContinuousTests/Tests/TransientNodeTest.cs +++ b/ContinuousTests/Tests/TransientNodeTest.cs @@ -29,7 +29,7 @@ namespace ContinuousTests.Tests cid = UploadFile(codexAccess.Node, file)!; Assert.That(cid, Is.Not.Null); - var resultFile = DownloadContent(IntermediateNode, cid); + var resultFile = DownloadFile(IntermediateNode, cid); file.AssertIsEqual(resultFile); }); } @@ -39,7 +39,7 @@ namespace ContinuousTests.Tests { NodeRunner.RunNode(DownloadBootstapNode, (codexAccess, marketplaceAccess) => { - var resultFile = DownloadContent(codexAccess.Node, cid); + var resultFile = DownloadFile(codexAccess.Node, cid); file.AssertIsEqual(resultFile); }); } diff --git a/ContinuousTests/Tests/TwoClientTest.cs b/ContinuousTests/Tests/TwoClientTest.cs index e6cbc1a..9d23d77 100644 --- a/ContinuousTests/Tests/TwoClientTest.cs +++ b/ContinuousTests/Tests/TwoClientTest.cs @@ -24,7 +24,7 @@ namespace ContinuousTests.Tests [TestMoment(t: MinuteFive)] public void DownloadTestFile() { - var dl = DownloadContent(Nodes[1], cid!); + var dl = DownloadFile(Nodes[1], cid!); file.AssertIsEqual(dl); } diff --git a/DistTestCore/FileManager.cs b/DistTestCore/FileManager.cs index 8d8c55f..ab86473 100644 --- a/DistTestCore/FileManager.cs +++ b/DistTestCore/FileManager.cs @@ -1,6 +1,5 @@ using Logging; using NUnit.Framework; -using System.Runtime.InteropServices; using Utils; namespace DistTestCore diff --git a/DistTestCore/Metrics/MetricsAccess.cs b/DistTestCore/Metrics/MetricsAccess.cs index 5ea0940..23b6522 100644 --- a/DistTestCore/Metrics/MetricsAccess.cs +++ b/DistTestCore/Metrics/MetricsAccess.cs @@ -14,12 +14,12 @@ namespace DistTestCore.Metrics public class MetricsAccess : IMetricsAccess { - private readonly TestLog log; + private readonly BaseLog log; private readonly ITimeSet timeSet; private readonly MetricsQuery query; private readonly RunningContainer node; - public MetricsAccess(TestLog log, ITimeSet timeSet, MetricsQuery query, RunningContainer node) + public MetricsAccess(BaseLog log, ITimeSet timeSet, MetricsQuery query, RunningContainer node) { this.log = log; this.timeSet = timeSet; diff --git a/DistTestCore/Metrics/MetricsDownloader.cs b/DistTestCore/Metrics/MetricsDownloader.cs index 4a458dd..d0d11cd 100644 --- a/DistTestCore/Metrics/MetricsDownloader.cs +++ b/DistTestCore/Metrics/MetricsDownloader.cs @@ -5,9 +5,9 @@ namespace DistTestCore.Metrics { public class MetricsDownloader { - private readonly TestLog log; + private readonly BaseLog log; - public MetricsDownloader(TestLog log) + public MetricsDownloader(BaseLog log) { this.log = log; } diff --git a/DistTestCore/TestLifecycle.cs b/DistTestCore/TestLifecycle.cs index 6ea8441..5249960 100644 --- a/DistTestCore/TestLifecycle.cs +++ b/DistTestCore/TestLifecycle.cs @@ -9,12 +9,12 @@ namespace DistTestCore { private DateTime testStart = DateTime.MinValue; - public TestLifecycle(TestLog log, Configuration configuration, ITimeSet timeSet) + public TestLifecycle(BaseLog log, Configuration configuration, ITimeSet timeSet) : this(log, configuration, timeSet, new WorkflowCreator(log, configuration.GetK8sConfiguration(timeSet))) { } - public TestLifecycle(TestLog log, Configuration configuration, ITimeSet timeSet, WorkflowCreator workflowCreator) + public TestLifecycle(BaseLog log, Configuration configuration, ITimeSet timeSet, WorkflowCreator workflowCreator) { Log = log; Configuration = configuration; @@ -27,7 +27,7 @@ namespace DistTestCore testStart = DateTime.UtcNow; } - public TestLog Log { get; } + public BaseLog Log { get; } public Configuration Configuration { get; } public ITimeSet TimeSet { get; } public FileManager FileManager { get; } diff --git a/Logging/BaseLog.cs b/Logging/BaseLog.cs index c312256..14656e3 100644 --- a/Logging/BaseLog.cs +++ b/Logging/BaseLog.cs @@ -4,6 +4,7 @@ namespace Logging { public abstract class BaseLog { + private readonly NumberSource subfileNumberSource = new NumberSource(0); private readonly bool debug; private readonly List replacements = new List(); private bool hasFailed; @@ -14,17 +15,21 @@ namespace Logging this.debug = debug; } - protected abstract LogFile CreateLogFile(); + protected abstract string GetFullName(); - protected LogFile LogFile + public LogFile LogFile { get { - if (logFile == null) logFile = CreateLogFile(); + if (logFile == null) logFile = new LogFile(GetFullName(), "log"); return logFile; } } + public virtual void EndTest() + { + } + public virtual void Log(string message) { LogFile.Write(ApplyReplacements(message)); @@ -63,6 +68,11 @@ namespace Logging File.Delete(LogFile.FullFilename); } + public LogFile CreateSubfile(string ext = "log") + { + return new LogFile($"{GetFullName()}_{GetSubfileNumber()}", ext); + } + private string ApplyReplacements(string str) { foreach (var replacement in replacements) @@ -71,6 +81,11 @@ namespace Logging } return str; } + + private string GetSubfileNumber() + { + return subfileNumberSource.GetNextNumber().ToString().PadLeft(6, '0'); + } } public class BaseLogStringReplacement diff --git a/Logging/FixtureLog.cs b/Logging/FixtureLog.cs index cc5ea87..e386f01 100644 --- a/Logging/FixtureLog.cs +++ b/Logging/FixtureLog.cs @@ -28,9 +28,9 @@ namespace Logging Directory.Delete(fullName, true); } - protected override LogFile CreateLogFile() + protected override string GetFullName() { - return new LogFile(fullName, "log"); + return fullName; } private string DetermineFolder(LogConfig config) diff --git a/Logging/NullLog.cs b/Logging/NullLog.cs index 5323508..06113b1 100644 --- a/Logging/NullLog.cs +++ b/Logging/NullLog.cs @@ -6,9 +6,9 @@ { } - protected override LogFile CreateLogFile() + protected override string GetFullName() { - return null!; + return "NULL"; } public override void Log(string message) diff --git a/Logging/TestLog.cs b/Logging/TestLog.cs index 6ac0c99..ed2cdfd 100644 --- a/Logging/TestLog.cs +++ b/Logging/TestLog.cs @@ -1,11 +1,9 @@ using NUnit.Framework; -using Utils; namespace Logging { public class TestLog : BaseLog { - private readonly NumberSource subfileNumberSource = new NumberSource(0); private readonly string methodName; private readonly string fullName; @@ -18,12 +16,7 @@ namespace Logging Log($"*** Begin: {methodName}"); } - public LogFile CreateSubfile(string ext = "log") - { - return new LogFile($"{fullName}_{GetSubfileNumber()}", ext); - } - - public void EndTest() + public override void EndTest() { var result = TestContext.CurrentContext.Result; @@ -40,9 +33,9 @@ namespace Logging } } - protected override LogFile CreateLogFile() + protected override string GetFullName() { - return new LogFile(fullName, "log"); + return fullName; } private string GetMethodName(string name) @@ -53,11 +46,6 @@ namespace Logging return $"{test.MethodName}{args}"; } - private string GetSubfileNumber() - { - return subfileNumberSource.GetNextNumber().ToString().PadLeft(6, '0'); - } - private static string FormatArguments(TestContext.TestAdapter test) { if (test.Arguments == null || !test.Arguments.Any()) return ""; From eaa218f8e054955fc3941a47e07d4a7ffcf10ae5 Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 28 Jun 2023 16:19:37 +0200 Subject: [PATCH 11/49] Setting up cancelling --- ContinuousTests/ContinuousTest.cs | 4 ++- ContinuousTests/ContinuousTestRunner.cs | 19 ++++++++---- ContinuousTests/Program.cs | 14 ++++++++- ContinuousTests/SingleTestRun.cs | 22 +++++++++----- ContinuousTests/TaskFactory.cs | 39 +++++++++++++++++++++++++ ContinuousTests/TestLoop.cs | 21 +++++++++---- 6 files changed, 97 insertions(+), 22 deletions(-) create mode 100644 ContinuousTests/TaskFactory.cs diff --git a/ContinuousTests/ContinuousTest.cs b/ContinuousTests/ContinuousTest.cs index 9570dc1..8d3deb8 100644 --- a/ContinuousTests/ContinuousTest.cs +++ b/ContinuousTests/ContinuousTest.cs @@ -21,12 +21,13 @@ namespace ContinuousTests private const string UploadFailedMessage = "Unable to store block"; - public void Initialize(CodexNode[] nodes, BaseLog log, FileManager fileManager, Configuration configuration) + public void Initialize(CodexNode[] nodes, BaseLog log, FileManager fileManager, Configuration configuration, CancellationToken cancelToken) { Nodes = nodes; Log = log; FileManager = fileManager; Configuration = configuration; + CancelToken = cancelToken; if (nodes != null) { @@ -43,6 +44,7 @@ namespace ContinuousTests public IFileManager FileManager { get; private set; } = null!; public Configuration Configuration { get; private set; } = null!; public virtual ITimeSet TimeSet { get { return new DefaultTimeSet(); } } + public CancellationToken CancelToken { get; private set; } = null; public NodeRunner NodeRunner { get; private set; } = null!; public abstract int RequiredNumberOfNodes { get; } diff --git a/ContinuousTests/ContinuousTestRunner.cs b/ContinuousTests/ContinuousTestRunner.cs index 85db06f..a6dcfda 100644 --- a/ContinuousTests/ContinuousTestRunner.cs +++ b/ContinuousTests/ContinuousTestRunner.cs @@ -10,34 +10,41 @@ namespace ContinuousTests private readonly TestFactory testFactory = new TestFactory(); private readonly Configuration config; private readonly StartupChecker startupChecker; + private readonly CancellationToken cancelToken; - public ContinuousTestRunner(string[] args) + public ContinuousTestRunner(string[] args, CancellationToken cancelToken) { config = configLoader.Load(args); startupChecker = new StartupChecker(config); + this.cancelToken = cancelToken; } public void Run() { startupChecker.Check(); + var taskFactory = new TaskFactory(); var overviewLog = new FixtureLog(new LogConfig(config.LogPath, false), "Overview"); overviewLog.Log("Continuous tests starting..."); var allTests = testFactory.CreateTests(); ClearAllCustomNamespaces(allTests, overviewLog); - var testLoop = allTests.Select(t => new TestLoop(config, overviewLog, t.GetType(), t.RunTestEvery)).ToArray(); + var testLoops = allTests.Select(t => new TestLoop(taskFactory, config, overviewLog, t.GetType(), t.RunTestEvery, cancelToken)).ToArray(); - foreach (var t in testLoop) + foreach (var testLoop in testLoops) { - overviewLog.Log("Launching test-loop for " + t.Name); - t.Begin(); + cancelToken.ThrowIfCancellationRequested(); + + overviewLog.Log("Launching test-loop for " + testLoop.Name); + testLoop.Begin(); Thread.Sleep(TimeSpan.FromSeconds(15)); } overviewLog.Log("All test-loops launched."); - while (true) Thread.Sleep((2 ^ 31) - 1); + cancelToken.WaitHandle.WaitOne(); + overviewLog.Log("Cancelling all test-loops..."); + taskFactory.WaitAll(); } private void ClearAllCustomNamespaces(ContinuousTest[] allTests, FixtureLog log) diff --git a/ContinuousTests/Program.cs b/ContinuousTests/Program.cs index 1e90a44..848e02c 100644 --- a/ContinuousTests/Program.cs +++ b/ContinuousTests/Program.cs @@ -6,7 +6,19 @@ public class Program { Console.WriteLine("Codex Continous-Test-Runner."); Console.WriteLine("Running..."); - var runner = new ContinuousTestRunner(args); + + var cts = new CancellationTokenSource(); + var runner = new ContinuousTestRunner(args, cts.Token); + + Console.CancelKeyPress += (sender, e) => + { + Console.WriteLine("Stopping..."); + e.Cancel = true; + + cts.Cancel(); + }; + runner.Run(); + Console.WriteLine("Done."); } } diff --git a/ContinuousTests/SingleTestRun.cs b/ContinuousTests/SingleTestRun.cs index 5d3d98d..ac15e74 100644 --- a/ContinuousTests/SingleTestRun.cs +++ b/ContinuousTests/SingleTestRun.cs @@ -12,21 +12,24 @@ namespace ContinuousTests { private readonly CodexNodeFactory codexNodeFactory = new CodexNodeFactory(); private readonly List exceptions = new List(); + private readonly TaskFactory taskFactory; private readonly Configuration config; private readonly BaseLog overviewLog; private readonly TestHandle handle; + private readonly CancellationToken cancelToken; private readonly CodexNode[] nodes; private readonly FileManager fileManager; private readonly FixtureLog fixtureLog; private readonly string testName; private readonly string dataFolder; - public SingleTestRun(Configuration config, BaseLog overviewLog, TestHandle handle) + public SingleTestRun(TaskFactory taskFactory, Configuration config, BaseLog overviewLog, TestHandle handle, CancellationToken cancelToken) { + this.taskFactory = taskFactory; this.config = config; this.overviewLog = overviewLog; this.handle = handle; - + this.cancelToken = cancelToken; testName = handle.Test.GetType().Name; fixtureLog = new FixtureLog(new LogConfig(config.LogPath, false), testName); @@ -37,7 +40,7 @@ namespace ContinuousTests public void Run() { - Task.Run(() => + taskFactory.Run(() => { try { @@ -75,6 +78,8 @@ namespace ContinuousTests var t = earliestMoment; while (true) { + cancelToken.ThrowIfCancellationRequested(); + RunMoment(t); if (handle.Test.TestFailMode == TestFailMode.StopAfterFirstFailure && exceptions.Any()) @@ -86,9 +91,10 @@ namespace ContinuousTests var nextMoment = handle.GetNextMoment(t); if (nextMoment != null) { - Log($" > Next TestMoment in {nextMoment.Value} seconds..."); - t += nextMoment.Value; - Thread.Sleep(nextMoment.Value * 1000); + var delta = TimeSpan.FromSeconds(nextMoment.Value - t); + Log($" > Next TestMoment in {Time.FormatDuration(delta)} seconds..."); + cancelToken.WaitHandle.WaitOne(delta); + t = nextMoment.Value; } else { @@ -144,12 +150,12 @@ namespace ContinuousTests private void InitializeTest(string name) { Log($" > Running TestMoment '{name}'"); - handle.Test.Initialize(nodes, fixtureLog, fileManager, config); + handle.Test.Initialize(nodes, fixtureLog, fileManager, config, cancelToken); } private void DecommissionTest() { - handle.Test.Initialize(null!, null!, null!, null!); + handle.Test.Initialize(null!, null!, null!, null!, cancelToken); } private void Log(string msg) diff --git a/ContinuousTests/TaskFactory.cs b/ContinuousTests/TaskFactory.cs new file mode 100644 index 0000000..749cab7 --- /dev/null +++ b/ContinuousTests/TaskFactory.cs @@ -0,0 +1,39 @@ +namespace ContinuousTests +{ + public class TaskFactory + { + private readonly object taskLock = new(); + private readonly List activeTasks = new List(); + + public void Run(Action action) + { + lock (taskLock) + { + activeTasks.Add(Task.Run(action).ContinueWith(CleanupTask, null)); + } + } + + public void WaitAll() + { + var tasks = activeTasks.ToArray(); + Task.WaitAll(tasks); + + var moreTasks = false; + lock (taskLock) + { + activeTasks.RemoveAll(task => task.IsCompleted); + moreTasks = activeTasks.Any(); + } + + if (moreTasks) WaitAll(); + } + + private void CleanupTask(Task completedTask, object? arg) + { + lock (taskLock) + { + activeTasks.Remove(completedTask); + } + } + } +} diff --git a/ContinuousTests/TestLoop.cs b/ContinuousTests/TestLoop.cs index 8943735..4de12a1 100644 --- a/ContinuousTests/TestLoop.cs +++ b/ContinuousTests/TestLoop.cs @@ -4,18 +4,21 @@ namespace ContinuousTests { public class TestLoop { + private readonly TaskFactory taskFactory; private readonly Configuration config; private readonly BaseLog overviewLog; private readonly Type testType; private readonly TimeSpan runsEvery; + private readonly CancellationToken cancelToken; - public TestLoop(Configuration config, BaseLog overviewLog, Type testType, TimeSpan runsEvery) + public TestLoop(TaskFactory taskFactory, Configuration config, BaseLog overviewLog, Type testType, TimeSpan runsEvery, CancellationToken cancelToken) { + this.taskFactory = taskFactory; this.config = config; this.overviewLog = overviewLog; this.testType = testType; this.runsEvery = runsEvery; - + this.cancelToken = cancelToken; Name = testType.Name; } @@ -23,17 +26,23 @@ namespace ContinuousTests public void Begin() { - Task.Run(() => + taskFactory.Run(() => { try { while (true) { + cancelToken.ThrowIfCancellationRequested(); StartTest(); - Thread.Sleep(runsEvery); + + cancelToken.WaitHandle.WaitOne(runsEvery); } } - catch(Exception ex) + catch (OperationCanceledException) + { + overviewLog.Log("Test-loop " + testType.Name + " is cancelled."); + } + catch (Exception ex) { overviewLog.Error("Test infra failure: TestLoop failed with " + ex); Environment.Exit(-1); @@ -45,7 +54,7 @@ namespace ContinuousTests { var test = (ContinuousTest)Activator.CreateInstance(testType)!; var handle = new TestHandle(test); - var run = new SingleTestRun(config, overviewLog, handle); + var run = new SingleTestRun(taskFactory, config, overviewLog, handle, cancelToken); run.Run(); } } From dc38797ee7212aa352ac59aa0300c8f86fb6d5b3 Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 29 Jun 2023 10:23:04 +0200 Subject: [PATCH 12/49] Correct cancelling behavior --- CodexNetDeployer/CodexNodeStarter.cs | 49 ++++++++++++++---------- ContinuousTests/ContinuousTest.cs | 2 +- ContinuousTests/ContinuousTestRunner.cs | 5 ++- ContinuousTests/Tests/MarketplaceTest.cs | 12 ++++-- 4 files changed, 41 insertions(+), 27 deletions(-) diff --git a/CodexNetDeployer/CodexNodeStarter.cs b/CodexNetDeployer/CodexNodeStarter.cs index 7c33892..898afb6 100644 --- a/CodexNetDeployer/CodexNodeStarter.cs +++ b/CodexNetDeployer/CodexNodeStarter.cs @@ -40,32 +40,41 @@ namespace CodexNetDeployer var tokenAddress = gethResult.MarketplaceNetwork.Marketplace.TokenAddress; var marketAccess = new MarketplaceAccess(lifecycle, gethResult.MarketplaceNetwork, account, codexAccess); - var debugInfo = codexAccess.Node.GetDebugInfo(); - if (!string.IsNullOrWhiteSpace(debugInfo.spr)) + try { - Console.Write("Online\t"); - - var interaction = gethResult.MarketplaceNetwork.Bootstrap.StartInteraction(lifecycle); - interaction.MintTestTokens(new[] { account.Account }, config.InitialTestTokens, tokenAddress); - Console.Write("Tokens minted\t"); - - var response = marketAccess.MakeStorageAvailable( - totalSpace: (config.StorageQuota!.Value - 1).MB(), - minPriceForTotalSpace: config.MinPrice.TestTokens(), - maxCollateral: config.MaxCollateral.TestTokens(), - maxDuration: TimeSpan.FromSeconds(config.MaxDuration)); - - if (!string.IsNullOrEmpty(response)) + var debugInfo = codexAccess.Node.GetDebugInfo(); + if (!string.IsNullOrWhiteSpace(debugInfo.spr)) { - Console.Write("Storage available\tOK" + Environment.NewLine); + Console.Write("Online\t"); - if (string.IsNullOrEmpty(bootstrapSpr)) bootstrapSpr = debugInfo.spr; - validatorsLeft--; - return container; + var interaction = gethResult.MarketplaceNetwork.Bootstrap.StartInteraction(lifecycle); + interaction.MintTestTokens(new[] { account.Account }, config.InitialTestTokens, tokenAddress); + Console.Write("Tokens minted\t"); + + var response = marketAccess.MakeStorageAvailable( + totalSpace: (config.StorageQuota!.Value - 1).MB(), + minPriceForTotalSpace: config.MinPrice.TestTokens(), + maxCollateral: config.MaxCollateral.TestTokens(), + maxDuration: TimeSpan.FromSeconds(config.MaxDuration)); + + if (!string.IsNullOrEmpty(response)) + { + Console.Write("Storage available\tOK" + Environment.NewLine); + + if (string.IsNullOrEmpty(bootstrapSpr)) bootstrapSpr = debugInfo.spr; + validatorsLeft--; + return container; + } } } + catch (Exception ex) + { + Console.WriteLine("Exception:" + ex.ToString()); + } + + Console.Write("Unknown failure. Downloading container log." + Environment.NewLine); + lifecycle.DownloadLog(container); - Console.Write("Unknown failure." + Environment.NewLine); return null; } diff --git a/ContinuousTests/ContinuousTest.cs b/ContinuousTests/ContinuousTest.cs index 8d3deb8..8bfdf1e 100644 --- a/ContinuousTests/ContinuousTest.cs +++ b/ContinuousTests/ContinuousTest.cs @@ -44,7 +44,7 @@ namespace ContinuousTests public IFileManager FileManager { get; private set; } = null!; public Configuration Configuration { get; private set; } = null!; public virtual ITimeSet TimeSet { get { return new DefaultTimeSet(); } } - public CancellationToken CancelToken { get; private set; } = null; + public CancellationToken CancelToken { get; private set; } = new CancellationToken(); public NodeRunner NodeRunner { get; private set; } = null!; public abstract int RequiredNumberOfNodes { get; } diff --git a/ContinuousTests/ContinuousTestRunner.cs b/ContinuousTests/ContinuousTestRunner.cs index a6dcfda..a0f15ab 100644 --- a/ContinuousTests/ContinuousTestRunner.cs +++ b/ContinuousTests/ContinuousTestRunner.cs @@ -34,17 +34,18 @@ namespace ContinuousTests foreach (var testLoop in testLoops) { - cancelToken.ThrowIfCancellationRequested(); + if (cancelToken.IsCancellationRequested) break; overviewLog.Log("Launching test-loop for " + testLoop.Name); testLoop.Begin(); Thread.Sleep(TimeSpan.FromSeconds(15)); } - overviewLog.Log("All test-loops launched."); + overviewLog.Log("Finished launching test-loops."); cancelToken.WaitHandle.WaitOne(); overviewLog.Log("Cancelling all test-loops..."); taskFactory.WaitAll(); + overviewLog.Log("All tasks cancelled."); } private void ClearAllCustomNamespaces(ContinuousTest[] allTests, FixtureLog log) diff --git a/ContinuousTests/Tests/MarketplaceTest.cs b/ContinuousTests/Tests/MarketplaceTest.cs index 9582b66..02c3dcc 100644 --- a/ContinuousTests/Tests/MarketplaceTest.cs +++ b/ContinuousTests/Tests/MarketplaceTest.cs @@ -65,28 +65,32 @@ namespace ContinuousTests.Tests { var lastState = ""; var waitStart = DateTime.UtcNow; - var filesizeInMb = fileSize.SizeInBytes / 1024; + var filesizeInMb = fileSize.SizeInBytes / (1024 * 1024); var maxWaitTime = TimeSpan.FromSeconds(filesizeInMb * 10.0); - Log.Log(nameof(WaitForContractToStart) + " for " + Time.FormatDuration(maxWaitTime)); + Log.Log($"{nameof(WaitForContractToStart)} for {Time.FormatDuration(maxWaitTime)}"); while (lastState != "started") { + CancelToken.ThrowIfCancellationRequested(); + var purchaseStatus = codexAccess.Node.GetPurchaseStatus(purchaseId); + var statusJson = JsonConvert.SerializeObject(purchaseStatus); if (purchaseStatus != null && purchaseStatus.state != lastState) { lastState = purchaseStatus.state; + Log.Log("Purchase status: " + statusJson); } Thread.Sleep(2000); if (lastState == "errored") { - Assert.Fail("Contract start failed: " + JsonConvert.SerializeObject(purchaseStatus)); + Assert.Fail("Contract start failed: " + statusJson); } if (DateTime.UtcNow - waitStart > maxWaitTime) { - Assert.Fail($"Contract was not picked up within {maxWaitTime.TotalSeconds} seconds timeout: " + JsonConvert.SerializeObject(purchaseStatus)); + Assert.Fail($"Contract was not picked up within {maxWaitTime.TotalSeconds} seconds timeout: {statusJson}"); } } Log.Log("Contract started."); From 4b6c8a919188a8e35ff004fbf401350e0e221469 Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 29 Jun 2023 10:45:29 +0200 Subject: [PATCH 13/49] Sets up codex net downloader for log inspection --- CodexNetDownloader/CodexNetDownloader.csproj | 16 +++++ CodexNetDownloader/Configuration.cs | 22 +++++++ CodexNetDownloader/Program.cs | 62 ++++++++++++++++++++ ContinuousTests/ContinuousTestRunner.cs | 2 +- ContinuousTests/K8sFactory.cs | 10 ++-- ContinuousTests/NodeRunner.cs | 2 +- cs-codex-dist-testing.sln | 8 ++- 7 files changed, 114 insertions(+), 8 deletions(-) create mode 100644 CodexNetDownloader/CodexNetDownloader.csproj create mode 100644 CodexNetDownloader/Configuration.cs create mode 100644 CodexNetDownloader/Program.cs diff --git a/CodexNetDownloader/CodexNetDownloader.csproj b/CodexNetDownloader/CodexNetDownloader.csproj new file mode 100644 index 0000000..0df1223 --- /dev/null +++ b/CodexNetDownloader/CodexNetDownloader.csproj @@ -0,0 +1,16 @@ + + + + Exe + net7.0 + enable + enable + + + + + + + + + diff --git a/CodexNetDownloader/Configuration.cs b/CodexNetDownloader/Configuration.cs new file mode 100644 index 0000000..008dfbf --- /dev/null +++ b/CodexNetDownloader/Configuration.cs @@ -0,0 +1,22 @@ +using ArgsUniform; +using DistTestCore; +using DistTestCore.Codex; + +namespace CodexNetDownloader +{ + public class Configuration + { + [Uniform("output-path", "o", "OUTPUT", true, "Path where files will be written.")] + public string OutputPath { get; set; } = "output"; + + [Uniform("codex-deployment", "c", "CODEXDEPLOYMENT", true, "Path to codex-deployment JSON file.")] + public string CodexDeploymentJson { get; set; } = string.Empty; + + [Uniform("kube-config", "kc", "KUBECONFIG", true, "Path to Kubeconfig file. Use 'null' (default) to use local cluster.")] + public string KubeConfigFile { get; set; } = "null"; + + public CodexDeployment CodexDeployment { get; set; } = null!; + + public TestRunnerLocation RunnerLocation { get; set; } = TestRunnerLocation.InternalToCluster; + } +} diff --git a/CodexNetDownloader/Program.cs b/CodexNetDownloader/Program.cs new file mode 100644 index 0000000..d17d76a --- /dev/null +++ b/CodexNetDownloader/Program.cs @@ -0,0 +1,62 @@ +using ArgsUniform; +using ContinuousTests; +using DistTestCore; +using DistTestCore.Codex; +using Logging; +using Newtonsoft.Json; + +public class Program +{ + public static void Main(string[] args) + { + var nl = Environment.NewLine; + Console.WriteLine("CodexNetDownloader" + nl); + + if (args.Any(a => a == "-h" || a == "--help" || a == "-?")) + { + PrintHelp(); + return; + } + + var uniformArgs = new ArgsUniform(args); + var config = uniformArgs.Parse(true); + + if (args.Any(a => a == "--external")) + { + config.RunnerLocation = TestRunnerLocation.ExternalToCluster; + } + + config.CodexDeployment = ParseCodexDeploymentJson(config.CodexDeploymentJson); + + if (!Directory.Exists(config.OutputPath)) Directory.CreateDirectory(config.OutputPath); + + var k8sFactory = new K8sFactory(); + var (_, lifecycle) = k8sFactory.CreateFacilities(config.KubeConfigFile, config.OutputPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog()); + + foreach (var container in config.CodexDeployment.CodexContainers) + { + lifecycle.DownloadLog(container); + } + + Console.WriteLine("Done!"); + } + + private static CodexDeployment ParseCodexDeploymentJson(string filename) + { + var d = JsonConvert.DeserializeObject(File.ReadAllText(filename))!; + if (d == null) throw new Exception("Unable to parse " + filename); + return d; + } + + private static void PrintHelp() + { + var nl = Environment.NewLine; + Console.WriteLine("CodexNetDownloader lets you download all container logs given a codex-deployment.json file." + nl); + + Console.WriteLine("CodexNetDownloader assumes you are running this tool from *inside* the Kubernetes cluster. " + + "If you are not running this from a container inside the cluster, add the argument '--external'." + nl); + + var uniformArgs = new ArgsUniform(); + uniformArgs.PrintHelp(); + } +} diff --git a/ContinuousTests/ContinuousTestRunner.cs b/ContinuousTests/ContinuousTestRunner.cs index a0f15ab..e465e28 100644 --- a/ContinuousTests/ContinuousTestRunner.cs +++ b/ContinuousTests/ContinuousTestRunner.cs @@ -58,7 +58,7 @@ namespace ContinuousTests if (string.IsNullOrEmpty(test.CustomK8sNamespace)) return; log.Log($"Clearing namespace '{test.CustomK8sNamespace}'..."); - var (workflowCreator, _) = k8SFactory.CreateFacilities(config, test.CustomK8sNamespace, new DefaultTimeSet(), log); + var (workflowCreator, _) = k8SFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, config.DataPath, test.CustomK8sNamespace, new DefaultTimeSet(), log); workflowCreator.CreateWorkflow().DeleteTestResources(); } } diff --git a/ContinuousTests/K8sFactory.cs b/ContinuousTests/K8sFactory.cs index 040a4a3..221e112 100644 --- a/ContinuousTests/K8sFactory.cs +++ b/ContinuousTests/K8sFactory.cs @@ -7,22 +7,22 @@ namespace ContinuousTests { public class K8sFactory { - public (WorkflowCreator, TestLifecycle) CreateFacilities(Configuration config, string customNamespace, ITimeSet timeSet, BaseLog log) + public (WorkflowCreator, TestLifecycle) CreateFacilities(string kubeConfigFile, string logPath, string dataFilePath, string customNamespace, ITimeSet timeSet, BaseLog log) { - var kubeConfig = GetKubeConfig(config.KubeConfigFile); + var kubeConfig = GetKubeConfig(kubeConfigFile); var lifecycleConfig = new DistTestCore.Configuration ( kubeConfigFile: kubeConfig, - logPath: "null", + logPath: logPath, logDebug: false, - dataFilesPath: config.LogPath, + dataFilesPath: dataFilePath, codexLogLevel: CodexLogLevel.Debug, runnerLocation: TestRunnerLocation.ExternalToCluster ); var kubeFlowConfig = new KubernetesWorkflow.Configuration( k8sNamespacePrefix: customNamespace, - kubeConfigFile: kubeConfig, + kubeConfigFile: kubeConfig, operationTimeout: timeSet.K8sOperationTimeout(), retryDelay: timeSet.WaitForK8sServiceDelay()); diff --git a/ContinuousTests/NodeRunner.cs b/ContinuousTests/NodeRunner.cs index 1a36d62..be6d416 100644 --- a/ContinuousTests/NodeRunner.cs +++ b/ContinuousTests/NodeRunner.cs @@ -81,7 +81,7 @@ namespace ContinuousTests private (WorkflowCreator, TestLifecycle) CreateFacilities() { - return k8SFactory.CreateFacilities(config, customNamespace, timeSet, log); + return k8SFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, config.DataPath, customNamespace, timeSet, log); } } } diff --git a/cs-codex-dist-testing.sln b/cs-codex-dist-testing.sln index d72ecae..8e696b3 100644 --- a/cs-codex-dist-testing.sln +++ b/cs-codex-dist-testing.sln @@ -21,7 +21,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ContinuousTests", "Continuo EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CodexNetDeployer", "CodexNetDeployer\CodexNetDeployer.csproj", "{871CAF12-14BE-4509-BC6E-20FDF0B1083A}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ArgsUniform", "ArgsUniform\ArgsUniform.csproj", "{634324B1-E359-42B4-A269-BDC429936B3C}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ArgsUniform", "ArgsUniform\ArgsUniform.csproj", "{634324B1-E359-42B4-A269-BDC429936B3C}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CodexNetDownloader", "CodexNetDownloader\CodexNetDownloader.csproj", "{6CDF35D2-906A-4285-8E1F-4794588B948B}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -69,6 +71,10 @@ Global {634324B1-E359-42B4-A269-BDC429936B3C}.Debug|Any CPU.Build.0 = Debug|Any CPU {634324B1-E359-42B4-A269-BDC429936B3C}.Release|Any CPU.ActiveCfg = Release|Any CPU {634324B1-E359-42B4-A269-BDC429936B3C}.Release|Any CPU.Build.0 = Release|Any CPU + {6CDF35D2-906A-4285-8E1F-4794588B948B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {6CDF35D2-906A-4285-8E1F-4794588B948B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {6CDF35D2-906A-4285-8E1F-4794588B948B}.Release|Any CPU.ActiveCfg = Release|Any CPU + {6CDF35D2-906A-4285-8E1F-4794588B948B}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE From 6a5cde2b9183fa4e9ddd393af364e88860f20ff5 Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 29 Jun 2023 11:05:58 +0200 Subject: [PATCH 14/49] Separate config option for storage space to sell. --- CodexNetDeployer/CodexNodeStarter.cs | 2 +- CodexNetDeployer/Configuration.cs | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/CodexNetDeployer/CodexNodeStarter.cs b/CodexNetDeployer/CodexNodeStarter.cs index 898afb6..f6faafa 100644 --- a/CodexNetDeployer/CodexNodeStarter.cs +++ b/CodexNetDeployer/CodexNodeStarter.cs @@ -52,7 +52,7 @@ namespace CodexNetDeployer Console.Write("Tokens minted\t"); var response = marketAccess.MakeStorageAvailable( - totalSpace: (config.StorageQuota!.Value - 1).MB(), + totalSpace: config.StorageSell!.Value.MB(), minPriceForTotalSpace: config.MinPrice.TestTokens(), maxCollateral: config.MaxCollateral.TestTokens(), maxDuration: TimeSpan.FromSeconds(config.MaxDuration)); diff --git a/CodexNetDeployer/Configuration.cs b/CodexNetDeployer/Configuration.cs index d58e233..3d79b0e 100644 --- a/CodexNetDeployer/Configuration.cs +++ b/CodexNetDeployer/Configuration.cs @@ -28,9 +28,12 @@ namespace CodexNetDeployer [Uniform("validators", "v", "VALIDATORS", true, "Number of Codex nodes that will be validating.")] public int? NumberOfValidators { get; set; } - [Uniform("storage-quota", "s", "STORAGEQUOTA", true, "Storage quota in megabytes used by each Codex node.")] + [Uniform("storage-quota", "sq", "STORAGEQUOTA", true, "Storage quota in megabytes used by each Codex node.")] public int? StorageQuota { get; set; } + [Uniform("storage-sell", "ss", "STORAGESELL", true, "Number of megabytes of storage quota to make available for selling.")] + public int? StorageSell { get; set; } + [Uniform("log-level", "l", "LOGLEVEL", true, "Log level used by each Codex node. [Trace, Debug*, Info, Warn, Error]")] public CodexLogLevel CodexLogLevel { get; set; } = CodexLogLevel.Debug; @@ -60,6 +63,10 @@ namespace CodexNetDeployer { errors.Add($"{nameof(NumberOfValidators)} ({NumberOfValidators}) may not be greater than {nameof(NumberOfCodexNodes)} ({NumberOfCodexNodes})."); } + if (StorageSell.HasValue && StorageQuota.HasValue && StorageSell.Value >= StorageQuota.Value) + { + errors.Add("StorageSell cannot be greater than or equal to StorageQuota."); + } return errors; } From caf5de678a14970c92424b4d76867c064a2499ad Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 29 Jun 2023 13:39:05 +0200 Subject: [PATCH 15/49] Prevents starting a test-run when one is already started. --- ContinuousTests/ContinuousTestRunner.cs | 2 +- ContinuousTests/SingleTestRun.cs | 5 +++-- ContinuousTests/TestLoop.cs | 8 +++++++- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/ContinuousTests/ContinuousTestRunner.cs b/ContinuousTests/ContinuousTestRunner.cs index e465e28..7ad16fa 100644 --- a/ContinuousTests/ContinuousTestRunner.cs +++ b/ContinuousTests/ContinuousTestRunner.cs @@ -38,7 +38,7 @@ namespace ContinuousTests overviewLog.Log("Launching test-loop for " + testLoop.Name); testLoop.Begin(); - Thread.Sleep(TimeSpan.FromSeconds(15)); + Thread.Sleep(TimeSpan.FromSeconds(5)); } overviewLog.Log("Finished launching test-loops."); diff --git a/ContinuousTests/SingleTestRun.cs b/ContinuousTests/SingleTestRun.cs index ac15e74..585971a 100644 --- a/ContinuousTests/SingleTestRun.cs +++ b/ContinuousTests/SingleTestRun.cs @@ -31,14 +31,14 @@ namespace ContinuousTests this.handle = handle; this.cancelToken = cancelToken; testName = handle.Test.GetType().Name; - fixtureLog = new FixtureLog(new LogConfig(config.LogPath, false), testName); + fixtureLog = new FixtureLog(new LogConfig(config.LogPath, true), testName); nodes = CreateRandomNodes(handle.Test.RequiredNumberOfNodes); dataFolder = config.DataPath + "-" + Guid.NewGuid(); fileManager = new FileManager(fixtureLog, CreateFileManagerConfiguration()); } - public void Run() + public void Run(EventWaitHandle runFinishedHandle) { taskFactory.Run(() => { @@ -47,6 +47,7 @@ namespace ContinuousTests RunTest(); fileManager.DeleteAllTestFiles(); Directory.Delete(dataFolder, true); + runFinishedHandle.Set(); } catch (Exception ex) { diff --git a/ContinuousTests/TestLoop.cs b/ContinuousTests/TestLoop.cs index 4de12a1..e57fee8 100644 --- a/ContinuousTests/TestLoop.cs +++ b/ContinuousTests/TestLoop.cs @@ -10,6 +10,7 @@ namespace ContinuousTests private readonly Type testType; private readonly TimeSpan runsEvery; private readonly CancellationToken cancelToken; + private readonly EventWaitHandle runFinishedHandle = new EventWaitHandle(true, EventResetMode.ManualReset); public TestLoop(TaskFactory taskFactory, Configuration config, BaseLog overviewLog, Type testType, TimeSpan runsEvery, CancellationToken cancelToken) { @@ -32,7 +33,10 @@ namespace ContinuousTests { while (true) { + WaitHandle.WaitAny(new[] { runFinishedHandle, cancelToken.WaitHandle }); + cancelToken.ThrowIfCancellationRequested(); + StartTest(); cancelToken.WaitHandle.WaitOne(runsEvery); @@ -55,7 +59,9 @@ namespace ContinuousTests var test = (ContinuousTest)Activator.CreateInstance(testType)!; var handle = new TestHandle(test); var run = new SingleTestRun(taskFactory, config, overviewLog, handle, cancelToken); - run.Run(); + + runFinishedHandle.Reset(); + run.Run(runFinishedHandle); } } } From d985e3191a2e4c49613a66cf061dacb62bf9737f Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 29 Jun 2023 16:03:45 +0200 Subject: [PATCH 16/49] Adds blockTTL to configuration of codex containers --- CodexNetDeployer/CodexNodeStarter.cs | 4 ++++ CodexNetDeployer/Configuration.cs | 5 +++++ ContinuousTests/NodeRunner.cs | 10 +++++++++- ContinuousTests/SingleTestRun.cs | 3 ++- DistTestCore/Codex/CodexContainerRecipe.cs | 4 ++++ DistTestCore/Codex/CodexNode.cs | 7 +++++-- DistTestCore/Codex/CodexStartupConfig.cs | 1 + 7 files changed, 30 insertions(+), 4 deletions(-) diff --git a/CodexNetDeployer/CodexNodeStarter.cs b/CodexNetDeployer/CodexNodeStarter.cs index f6faafa..a370615 100644 --- a/CodexNetDeployer/CodexNodeStarter.cs +++ b/CodexNetDeployer/CodexNodeStarter.cs @@ -87,6 +87,10 @@ namespace CodexNetDeployer var marketplaceConfig = new MarketplaceInitialConfig(100000.Eth(), 0.TestTokens(), validatorsLeft > 0); marketplaceConfig.AccountIndexOverride = i; codexStart.MarketplaceConfig = marketplaceConfig; + if (config.BlockTTL != Configuration.SecondsIn1Day) + { + codexStart.BlockTTL = config.BlockTTL; + } return codexStart; } diff --git a/CodexNetDeployer/Configuration.cs b/CodexNetDeployer/Configuration.cs index 3d79b0e..648e5bb 100644 --- a/CodexNetDeployer/Configuration.cs +++ b/CodexNetDeployer/Configuration.cs @@ -7,6 +7,8 @@ namespace CodexNetDeployer { public class Configuration { + public const int SecondsIn1Day = 24 * 60 * 60; + [Uniform("codex-image", "ci", "CODEXIMAGE", true, "Docker image of Codex.")] public string CodexImage { get; set; } = CodexContainerRecipe.DockerImage; @@ -49,6 +51,9 @@ namespace CodexNetDeployer [Uniform("max-duration", "md", "MAXDURATION", true, "Maximum duration in seconds for contracts which will be accepted.")] public int MaxDuration { get; set; } + [Uniform("block-ttl", "bt", "BLOCKTTL", false, "Block timeout in seconds. Default is 24 hours.")] + public int BlockTTL { get; set; } = SecondsIn1Day; + public TestRunnerLocation RunnerLocation { get; set; } = TestRunnerLocation.InternalToCluster; public List Validate() diff --git a/ContinuousTests/NodeRunner.cs b/ContinuousTests/NodeRunner.cs index be6d416..ee4a554 100644 --- a/ContinuousTests/NodeRunner.cs +++ b/ContinuousTests/NodeRunner.cs @@ -71,7 +71,15 @@ namespace ContinuousTests var codexAccess = new CodexAccess(lifecycle, container); var marketAccess = new MarketplaceAccess(lifecycle, marketplaceNetwork, account, codexAccess); - operation(codexAccess, marketAccess); + try + { + operation(codexAccess, marketAccess); + } + catch + { + lifecycle.DownloadLog(container); + throw; + } } finally { diff --git a/ContinuousTests/SingleTestRun.cs b/ContinuousTests/SingleTestRun.cs index 585971a..8f29c34 100644 --- a/ContinuousTests/SingleTestRun.cs +++ b/ContinuousTests/SingleTestRun.cs @@ -167,7 +167,8 @@ namespace ContinuousTests private void OverviewLog(string msg) { Log(msg); - overviewLog.Log(testName + ": " + msg); + var containerNames = $"({string.Join(",", nodes.Select(n => n.Container.Name))})"; + overviewLog.Log( testName + ": " + msg); } private CodexNode[] CreateRandomNodes(int number) diff --git a/DistTestCore/Codex/CodexContainerRecipe.cs b/DistTestCore/Codex/CodexContainerRecipe.cs index b7c3b94..83e1948 100644 --- a/DistTestCore/Codex/CodexContainerRecipe.cs +++ b/DistTestCore/Codex/CodexContainerRecipe.cs @@ -40,6 +40,10 @@ namespace DistTestCore.Codex { AddEnvVar("STORAGE_QUOTA", config.StorageQuota.SizeInBytes.ToString()!); } + if (config.BlockTTL != null) + { + AddEnvVar("BLOCK_TTL", config.BlockTTL.ToString()!); + } if (config.MetricsEnabled) { AddEnvVar("METRICS_ADDR", "0.0.0.0"); diff --git a/DistTestCore/Codex/CodexNode.cs b/DistTestCore/Codex/CodexNode.cs index bb2220e..3e33225 100644 --- a/DistTestCore/Codex/CodexNode.cs +++ b/DistTestCore/Codex/CodexNode.cs @@ -1,4 +1,5 @@ -using Logging; +using KubernetesWorkflow; +using Logging; using Utils; namespace DistTestCore.Codex @@ -8,13 +9,15 @@ namespace DistTestCore.Codex private readonly BaseLog log; private readonly ITimeSet timeSet; - public CodexNode(BaseLog log, ITimeSet timeSet, Address address) + public CodexNode(BaseLog log, RunningContainer container, ITimeSet timeSet, Address address) { this.log = log; + Container = container; this.timeSet = timeSet; Address = address; } + public RunningContainer Container { get; } public Address Address { get; } public CodexDebugResponse GetDebugInfo() diff --git a/DistTestCore/Codex/CodexStartupConfig.cs b/DistTestCore/Codex/CodexStartupConfig.cs index b13512b..2a17334 100644 --- a/DistTestCore/Codex/CodexStartupConfig.cs +++ b/DistTestCore/Codex/CodexStartupConfig.cs @@ -17,5 +17,6 @@ namespace DistTestCore.Codex public bool MetricsEnabled { get; set; } public MarketplaceInitialConfig? MarketplaceConfig { get; set; } public string? BootstrapSpr { get; set; } + public int? BlockTTL { get; set; } } } From 66e6cdc027568640d834f1f9989301c9f9e19376 Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 29 Jun 2023 16:07:49 +0200 Subject: [PATCH 17/49] Flattens CodexNode into CodexAccess --- CodexNetDeployer/CodexNodeStarter.cs | 5 +- ...exNodeFactory.cs => CodexAccessFactory.cs} | 6 +- ContinuousTests/ContinuousTest.cs | 10 +-- ContinuousTests/NodeRunner.cs | 11 +-- ContinuousTests/SingleTestRun.cs | 6 +- ContinuousTests/StartupChecker.cs | 4 +- ContinuousTests/Tests/PerformanceTests.cs | 4 +- ContinuousTests/Tests/TransientNodeTest.cs | 10 +-- DistTestCore/Codex/CodexAccess.cs | 88 ++++++++++++++----- .../Codex/{CodexNode.cs => CodexApiTypes.cs} | 80 ----------------- DistTestCore/CodexNodeGroup.cs | 11 ++- DistTestCore/Marketplace/MarketplaceAccess.cs | 4 +- DistTestCore/OnlineCodexNode.cs | 12 +-- 13 files changed, 112 insertions(+), 139 deletions(-) rename ContinuousTests/{CodexNodeFactory.cs => CodexAccessFactory.cs} (59%) rename DistTestCore/Codex/{CodexNode.cs => CodexApiTypes.cs} (58%) diff --git a/CodexNetDeployer/CodexNodeStarter.cs b/CodexNetDeployer/CodexNodeStarter.cs index a370615..999ba14 100644 --- a/CodexNetDeployer/CodexNodeStarter.cs +++ b/CodexNetDeployer/CodexNodeStarter.cs @@ -34,15 +34,14 @@ namespace CodexNetDeployer var containers = workflow.Start(1, Location.Unspecified, new CodexContainerRecipe(), workflowStartup); var container = containers.Containers.First(); - var codexAccess = new CodexAccess(lifecycle, container); - + var codexAccess = new CodexAccess(lifecycle.Log, container, lifecycle.TimeSet, lifecycle.Configuration.GetAddress(container)); var account = gethResult.MarketplaceNetwork.Bootstrap.AllAccounts.Accounts[i]; var tokenAddress = gethResult.MarketplaceNetwork.Marketplace.TokenAddress; var marketAccess = new MarketplaceAccess(lifecycle, gethResult.MarketplaceNetwork, account, codexAccess); try { - var debugInfo = codexAccess.Node.GetDebugInfo(); + var debugInfo = codexAccess.GetDebugInfo(); if (!string.IsNullOrWhiteSpace(debugInfo.spr)) { Console.Write("Online\t"); diff --git a/ContinuousTests/CodexNodeFactory.cs b/ContinuousTests/CodexAccessFactory.cs similarity index 59% rename from ContinuousTests/CodexNodeFactory.cs rename to ContinuousTests/CodexAccessFactory.cs index 43b1fea..08eb815 100644 --- a/ContinuousTests/CodexNodeFactory.cs +++ b/ContinuousTests/CodexAccessFactory.cs @@ -5,14 +5,14 @@ using Logging; namespace ContinuousTests { - public class CodexNodeFactory + public class CodexAccessFactory { - public CodexNode[] Create(RunningContainer[] containers, BaseLog log, ITimeSet timeSet) + public CodexAccess[] Create(RunningContainer[] containers, BaseLog log, ITimeSet timeSet) { return containers.Select(container => { var address = container.ClusterExternalAddress; - return new CodexNode(log, timeSet, address); + return new CodexAccess(log, container, timeSet, address); }).ToArray(); } } diff --git a/ContinuousTests/ContinuousTest.cs b/ContinuousTests/ContinuousTest.cs index 8bfdf1e..80b914e 100644 --- a/ContinuousTests/ContinuousTest.cs +++ b/ContinuousTests/ContinuousTest.cs @@ -21,7 +21,7 @@ namespace ContinuousTests private const string UploadFailedMessage = "Unable to store block"; - public void Initialize(CodexNode[] nodes, BaseLog log, FileManager fileManager, Configuration configuration, CancellationToken cancelToken) + public void Initialize(CodexAccess[] nodes, BaseLog log, FileManager fileManager, Configuration configuration, CancellationToken cancelToken) { Nodes = nodes; Log = log; @@ -39,7 +39,7 @@ namespace ContinuousTests } } - public CodexNode[] Nodes { get; private set; } = null!; + public CodexAccess[] Nodes { get; private set; } = null!; public BaseLog Log { get; private set; } = null!; public IFileManager FileManager { get; private set; } = null!; public Configuration Configuration { get; private set; } = null!; @@ -61,7 +61,7 @@ namespace ContinuousTests } } - public ContentId? UploadFile(CodexNode node, TestFile file) + public ContentId? UploadFile(CodexAccess node, TestFile file) { using var fileStream = File.OpenRead(file.Filename); @@ -79,7 +79,7 @@ namespace ContinuousTests return new ContentId(response); } - public TestFile DownloadFile(CodexNode node, ContentId contentId, string fileLabel = "") + public TestFile DownloadFile(CodexAccess node, ContentId contentId, string fileLabel = "") { var logMessage = $"Downloading for contentId: '{contentId.Id}'..."; var file = FileManager.CreateEmptyTestFile(fileLabel); @@ -88,7 +88,7 @@ namespace ContinuousTests return file; } - private void DownloadToFile(CodexNode node, string contentId, TestFile file) + private void DownloadToFile(CodexAccess node, string contentId, TestFile file) { using var fileStream = File.OpenWrite(file.Filename); try diff --git a/ContinuousTests/NodeRunner.cs b/ContinuousTests/NodeRunner.cs index ee4a554..9551cc0 100644 --- a/ContinuousTests/NodeRunner.cs +++ b/ContinuousTests/NodeRunner.cs @@ -11,14 +11,14 @@ namespace ContinuousTests public class NodeRunner { private readonly K8sFactory k8SFactory = new K8sFactory(); - private readonly CodexNode[] nodes; + private readonly CodexAccess[] nodes; private readonly Configuration config; private readonly ITimeSet timeSet; private readonly BaseLog log; private readonly string customNamespace; private readonly int ethereumAccountIndex; - public NodeRunner(CodexNode[] nodes, Configuration config, ITimeSet timeSet, BaseLog log, string customNamespace, int ethereumAccountIndex) + public NodeRunner(CodexAccess[] nodes, Configuration config, ITimeSet timeSet, BaseLog log, string customNamespace, int ethereumAccountIndex) { this.nodes = nodes; this.config = config; @@ -33,12 +33,12 @@ namespace ContinuousTests RunNode(nodes.ToList().PickOneRandom(), operation, 0.TestTokens()); } - public void RunNode(CodexNode bootstrapNode, Action operation) + public void RunNode(CodexAccess bootstrapNode, Action operation) { RunNode(bootstrapNode, operation, 0.TestTokens()); } - public void RunNode(CodexNode bootstrapNode, Action operation, TestToken mintTestTokens) + public void RunNode(CodexAccess bootstrapNode, Action operation, TestToken mintTestTokens) { var (workflowCreator, lifecycle) = CreateFacilities(); var flow = workflowCreator.CreateWorkflow(); @@ -68,7 +68,8 @@ namespace ContinuousTests } var container = rc.Containers[0]; - var codexAccess = new CodexAccess(lifecycle, container); + var address = lifecycle.Configuration.GetAddress(container); + var codexAccess = new CodexAccess(log, container, lifecycle.TimeSet, address); var marketAccess = new MarketplaceAccess(lifecycle, marketplaceNetwork, account, codexAccess); try diff --git a/ContinuousTests/SingleTestRun.cs b/ContinuousTests/SingleTestRun.cs index 8f29c34..17752d2 100644 --- a/ContinuousTests/SingleTestRun.cs +++ b/ContinuousTests/SingleTestRun.cs @@ -10,14 +10,14 @@ namespace ContinuousTests { public class SingleTestRun { - private readonly CodexNodeFactory codexNodeFactory = new CodexNodeFactory(); + private readonly CodexAccessFactory codexNodeFactory = new CodexAccessFactory(); private readonly List exceptions = new List(); private readonly TaskFactory taskFactory; private readonly Configuration config; private readonly BaseLog overviewLog; private readonly TestHandle handle; private readonly CancellationToken cancelToken; - private readonly CodexNode[] nodes; + private readonly CodexAccess[] nodes; private readonly FileManager fileManager; private readonly FixtureLog fixtureLog; private readonly string testName; @@ -171,7 +171,7 @@ namespace ContinuousTests overviewLog.Log( testName + ": " + msg); } - private CodexNode[] CreateRandomNodes(int number) + private CodexAccess[] CreateRandomNodes(int number) { var containers = SelectRandomContainers(number); fixtureLog.Log("Selected nodes: " + string.Join(",", containers.Select(c => c.Name))); diff --git a/ContinuousTests/StartupChecker.cs b/ContinuousTests/StartupChecker.cs index 12e34cc..e4889b9 100644 --- a/ContinuousTests/StartupChecker.cs +++ b/ContinuousTests/StartupChecker.cs @@ -8,7 +8,7 @@ namespace ContinuousTests public class StartupChecker { private readonly TestFactory testFactory = new TestFactory(); - private readonly CodexNodeFactory codexNodeFactory = new CodexNodeFactory(); + private readonly CodexAccessFactory codexNodeFactory = new CodexAccessFactory(); private readonly Configuration config; public StartupChecker(Configuration config) @@ -77,7 +77,7 @@ namespace ContinuousTests } } - private bool EnsureOnline(CodexNode n) + private bool EnsureOnline(CodexAccess n) { try { diff --git a/ContinuousTests/Tests/PerformanceTests.cs b/ContinuousTests/Tests/PerformanceTests.cs index 032dcfa..2df3052 100644 --- a/ContinuousTests/Tests/PerformanceTests.cs +++ b/ContinuousTests/Tests/PerformanceTests.cs @@ -42,7 +42,7 @@ namespace ContinuousTests.Tests public override TimeSpan RunTestEvery => TimeSpan.FromHours(1); public override TestFailMode TestFailMode => TestFailMode.AlwaysRunAllMoments; - public void UploadTest(int megabytes, CodexNode uploadNode) + public void UploadTest(int megabytes, CodexAccess uploadNode) { var file = FileManager.GenerateTestFile(megabytes.MB()); @@ -56,7 +56,7 @@ namespace ContinuousTests.Tests Assert.That(timePerMB, Is.LessThan(CodexContainerRecipe.MaxUploadTimePerMegabyte), "MaxUploadTimePerMegabyte performance threshold breached."); } - public void DownloadTest(int megabytes, CodexNode uploadNode, CodexNode downloadNode) + public void DownloadTest(int megabytes, CodexAccess uploadNode, CodexAccess downloadNode) { var file = FileManager.GenerateTestFile(megabytes.MB()); diff --git a/ContinuousTests/Tests/TransientNodeTest.cs b/ContinuousTests/Tests/TransientNodeTest.cs index 5c49e1b..37eb194 100644 --- a/ContinuousTests/Tests/TransientNodeTest.cs +++ b/ContinuousTests/Tests/TransientNodeTest.cs @@ -15,9 +15,9 @@ namespace ContinuousTests.Tests private TestFile file = null!; private ContentId cid = null!; - private CodexNode UploadBootstapNode { get { return Nodes[0]; } } - private CodexNode DownloadBootstapNode { get { return Nodes[1]; } } - private CodexNode IntermediateNode { get { return Nodes[2]; } } + private CodexAccess UploadBootstapNode { get { return Nodes[0]; } } + private CodexAccess DownloadBootstapNode { get { return Nodes[1]; } } + private CodexAccess IntermediateNode { get { return Nodes[2]; } } [TestMoment(t: 0)] public void UploadWithTransientNode() @@ -26,7 +26,7 @@ namespace ContinuousTests.Tests NodeRunner.RunNode(UploadBootstapNode, (codexAccess, marketplaceAccess) => { - cid = UploadFile(codexAccess.Node, file)!; + cid = UploadFile(codexAccess, file)!; Assert.That(cid, Is.Not.Null); var resultFile = DownloadFile(IntermediateNode, cid); @@ -39,7 +39,7 @@ namespace ContinuousTests.Tests { NodeRunner.RunNode(DownloadBootstapNode, (codexAccess, marketplaceAccess) => { - var resultFile = DownloadFile(codexAccess.Node, cid); + var resultFile = DownloadFile(codexAccess, cid); file.AssertIsEqual(resultFile); }); } diff --git a/DistTestCore/Codex/CodexAccess.cs b/DistTestCore/Codex/CodexAccess.cs index ee0e2a5..d0cb1fc 100644 --- a/DistTestCore/Codex/CodexAccess.cs +++ b/DistTestCore/Codex/CodexAccess.cs @@ -1,40 +1,86 @@ using KubernetesWorkflow; +using Logging; +using Utils; namespace DistTestCore.Codex { public class CodexAccess { - private readonly TestLifecycle lifecycle; + private readonly BaseLog log; + private readonly ITimeSet timeSet; - public CodexAccess(TestLifecycle lifecycle, RunningContainer runningContainer) + public CodexAccess(BaseLog log, RunningContainer container, ITimeSet timeSet, Address address) { - this.lifecycle = lifecycle; - Container = runningContainer; - - var address = lifecycle.Configuration.GetAddress(Container); - Node = new CodexNode(lifecycle.Log, lifecycle.TimeSet, address); + this.log = log; + Container = container; + this.timeSet = timeSet; + Address = address; } public RunningContainer Container { get; } - public CodexNode Node { get; } + public Address Address { get; } - public void EnsureOnline() + public CodexDebugResponse GetDebugInfo() { - try - { - var debugInfo = Node.GetDebugInfo(); - if (debugInfo == null || string.IsNullOrEmpty(debugInfo.id)) throw new InvalidOperationException("Unable to get debug-info from codex node at startup."); + return Http(TimeSpan.FromSeconds(2)).HttpGetJson("debug/info"); + } - var nodePeerId = debugInfo.id; - var nodeName = Container.Name; - lifecycle.Log.AddStringReplace(nodePeerId, nodeName); - lifecycle.Log.AddStringReplace(debugInfo.table.localNode.nodeId, nodeName); - } - catch (Exception e) + public CodexDebugPeerResponse GetDebugPeer(string peerId) + { + return GetDebugPeer(peerId, TimeSpan.FromSeconds(2)); + } + + public CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout) + { + var http = Http(timeout); + var str = http.HttpGetString($"debug/peer/{peerId}"); + + if (str.ToLowerInvariant() == "unable to find peer!") { - lifecycle.Log.Error($"Failed to start codex node: {e}. Test infra failure."); - throw new InvalidOperationException($"Failed to start codex node. Test infra failure.", e); + return new CodexDebugPeerResponse + { + IsPeerFound = false + }; } + + var result = http.TryJsonDeserialize(str); + result.IsPeerFound = true; + return result; + } + + public string UploadFile(FileStream fileStream) + { + return Http().HttpPostStream("upload", fileStream); + } + + public Stream DownloadFile(string contentId) + { + return Http().HttpGetStream("download/" + contentId); + } + + public CodexSalesAvailabilityResponse SalesAvailability(CodexSalesAvailabilityRequest request) + { + return Http().HttpPostJson("sales/availability", request); + } + + public string RequestStorage(CodexSalesRequestStorageRequest request, string contentId) + { + return Http().HttpPostJson($"storage/request/{contentId}", request); + } + + public CodexStoragePurchase GetPurchaseStatus(string purchaseId) + { + return Http().HttpGetJson($"storage/purchases/{purchaseId}"); + } + + public string ConnectToPeer(string peerId, string peerMultiAddress) + { + return Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}"); + } + + private Http Http(TimeSpan? timeoutOverride = null) + { + return new Http(log, timeSet, Address, baseUrl: "/api/codex/v1", timeoutOverride); } } } diff --git a/DistTestCore/Codex/CodexNode.cs b/DistTestCore/Codex/CodexApiTypes.cs similarity index 58% rename from DistTestCore/Codex/CodexNode.cs rename to DistTestCore/Codex/CodexApiTypes.cs index 3e33225..885540e 100644 --- a/DistTestCore/Codex/CodexNode.cs +++ b/DistTestCore/Codex/CodexApiTypes.cs @@ -4,86 +4,6 @@ using Utils; namespace DistTestCore.Codex { - public class CodexNode - { - private readonly BaseLog log; - private readonly ITimeSet timeSet; - - public CodexNode(BaseLog log, RunningContainer container, ITimeSet timeSet, Address address) - { - this.log = log; - Container = container; - this.timeSet = timeSet; - Address = address; - } - - public RunningContainer Container { get; } - public Address Address { get; } - - public CodexDebugResponse GetDebugInfo() - { - return Http(TimeSpan.FromSeconds(2)).HttpGetJson("debug/info"); - } - - public CodexDebugPeerResponse GetDebugPeer(string peerId) - { - return GetDebugPeer(peerId, TimeSpan.FromSeconds(2)); - } - - public CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout) - { - var http = Http(timeout); - var str = http.HttpGetString($"debug/peer/{peerId}"); - - if (str.ToLowerInvariant() == "unable to find peer!") - { - return new CodexDebugPeerResponse - { - IsPeerFound = false - }; - } - - var result = http.TryJsonDeserialize(str); - result.IsPeerFound = true; - return result; - } - - public string UploadFile(FileStream fileStream) - { - return Http().HttpPostStream("upload", fileStream); - } - - public Stream DownloadFile(string contentId) - { - return Http().HttpGetStream("download/" + contentId); - } - - public CodexSalesAvailabilityResponse SalesAvailability(CodexSalesAvailabilityRequest request) - { - return Http().HttpPostJson("sales/availability", request); - } - - public string RequestStorage(CodexSalesRequestStorageRequest request, string contentId) - { - return Http().HttpPostJson($"storage/request/{contentId}", request); - } - - public CodexStoragePurchase GetPurchaseStatus(string purchaseId) - { - return Http().HttpGetJson($"storage/purchases/{purchaseId}"); - } - - public string ConnectToPeer(string peerId, string peerMultiAddress) - { - return Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}"); - } - - private Http Http(TimeSpan? timeoutOverride = null) - { - return new Http(log, timeSet, Address, baseUrl: "/api/codex/v1", timeoutOverride); - } - } - public class CodexDebugResponse { public string id { get; set; } = string.Empty; diff --git a/DistTestCore/CodexNodeGroup.cs b/DistTestCore/CodexNodeGroup.cs index 77d46e6..41dab1f 100644 --- a/DistTestCore/CodexNodeGroup.cs +++ b/DistTestCore/CodexNodeGroup.cs @@ -64,12 +64,19 @@ namespace DistTestCore public void EnsureOnline() { - foreach (var node in Nodes) node.CodexAccess.EnsureOnline(); + foreach (var node in Nodes) + { + var debugInfo = node.CodexAccess.GetDebugInfo(); + var nodePeerId = debugInfo.id; + var nodeName = node.CodexAccess.Container.Name; + lifecycle.Log.AddStringReplace(nodePeerId, nodeName); + lifecycle.Log.AddStringReplace(debugInfo.table.localNode.nodeId, nodeName); + } } private OnlineCodexNode CreateOnlineCodexNode(RunningContainer c, ICodexNodeFactory factory) { - var access = new CodexAccess(lifecycle, c); + var access = new CodexAccess(lifecycle.Log, c, lifecycle.TimeSet, lifecycle.Configuration.GetAddress(c)); return factory.CreateOnlineCodexNode(access, this); } } diff --git a/DistTestCore/Marketplace/MarketplaceAccess.cs b/DistTestCore/Marketplace/MarketplaceAccess.cs index 852ee6d..bdf03c5 100644 --- a/DistTestCore/Marketplace/MarketplaceAccess.cs +++ b/DistTestCore/Marketplace/MarketplaceAccess.cs @@ -50,7 +50,7 @@ namespace DistTestCore.Marketplace $"proofProbability: {proofProbability}, " + $"duration: {Time.FormatDuration(duration)})"); - var response = codexAccess.Node.RequestStorage(request, contentId.Id); + var response = codexAccess.RequestStorage(request, contentId.Id); if (response == "Purchasing not available") { @@ -78,7 +78,7 @@ namespace DistTestCore.Marketplace $"maxCollateral: {maxCollateral}, " + $"maxDuration: {Time.FormatDuration(maxDuration)})"); - var response = codexAccess.Node.SalesAvailability(request); + var response = codexAccess.SalesAvailability(request); Log($"Storage successfully made available. Id: {response.id}"); diff --git a/DistTestCore/OnlineCodexNode.cs b/DistTestCore/OnlineCodexNode.cs index d1e5301..5a66fce 100644 --- a/DistTestCore/OnlineCodexNode.cs +++ b/DistTestCore/OnlineCodexNode.cs @@ -49,7 +49,7 @@ namespace DistTestCore public CodexDebugResponse GetDebugInfo() { - var debugInfo = CodexAccess.Node.GetDebugInfo(); + var debugInfo = CodexAccess.GetDebugInfo(); var known = string.Join(",", debugInfo.table.nodes.Select(n => n.peerId)); Log($"Got DebugInfo with id: '{debugInfo.id}'. This node knows: {known}"); return debugInfo; @@ -57,12 +57,12 @@ namespace DistTestCore public CodexDebugPeerResponse GetDebugPeer(string peerId) { - return CodexAccess.Node.GetDebugPeer(peerId); + return CodexAccess.GetDebugPeer(peerId); } public CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout) { - return CodexAccess.Node.GetDebugPeer(peerId, timeout); + return CodexAccess.GetDebugPeer(peerId, timeout); } public ContentId UploadFile(TestFile file) @@ -72,7 +72,7 @@ namespace DistTestCore var logMessage = $"Uploading file {file.Describe()}..."; var response = Stopwatch.Measure(lifecycle.Log, logMessage, () => { - return CodexAccess.Node.UploadFile(fileStream); + return CodexAccess.UploadFile(fileStream); }); if (response.StartsWith(UploadFailedMessage)) @@ -101,7 +101,7 @@ namespace DistTestCore Log($"Connecting to peer {peer.GetName()}..."); var peerInfo = node.GetDebugInfo(); - var response = CodexAccess.Node.ConnectToPeer(peerInfo.id, GetPeerMultiAddress(peer, peerInfo)); + var response = CodexAccess.ConnectToPeer(peerInfo.id, GetPeerMultiAddress(peer, peerInfo)); Assert.That(response, Is.EqualTo(SuccessfullyConnectedMessage), "Unable to connect codex nodes."); Log($"Successfully connected to peer {peer.GetName()}."); @@ -141,7 +141,7 @@ namespace DistTestCore using var fileStream = File.OpenWrite(file.Filename); try { - using var downloadStream = CodexAccess.Node.DownloadFile(contentId); + using var downloadStream = CodexAccess.DownloadFile(contentId); downloadStream.CopyTo(fileStream); } catch From e7e464b4fa445a9a7a69c3f33e8272ff522af187 Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 30 Jun 2023 08:39:18 +0200 Subject: [PATCH 18/49] Adds container alias to log. Useful for transient nodes --- ContinuousTests/NodeRunner.cs | 9 +++++---- ContinuousTests/SingleTestRun.cs | 2 +- DistTestCore/Codex/CodexAccess.cs | 2 +- DistTestCore/Http.cs | 13 +++++++++++-- 4 files changed, 18 insertions(+), 8 deletions(-) diff --git a/ContinuousTests/NodeRunner.cs b/ContinuousTests/NodeRunner.cs index 9551cc0..b970a08 100644 --- a/ContinuousTests/NodeRunner.cs +++ b/ContinuousTests/NodeRunner.cs @@ -28,17 +28,17 @@ namespace ContinuousTests this.ethereumAccountIndex = ethereumAccountIndex; } - public void RunNode(Action operation) + public void RunNode(Action operation) { RunNode(nodes.ToList().PickOneRandom(), operation, 0.TestTokens()); } - public void RunNode(CodexAccess bootstrapNode, Action operation) + public void RunNode(CodexAccess bootstrapNode, Action operation) { RunNode(bootstrapNode, operation, 0.TestTokens()); } - public void RunNode(CodexAccess bootstrapNode, Action operation, TestToken mintTestTokens) + public void RunNode(CodexAccess bootstrapNode, Action operation, TestToken mintTestTokens) { var (workflowCreator, lifecycle) = CreateFacilities(); var flow = workflowCreator.CreateWorkflow(); @@ -49,6 +49,7 @@ namespace ContinuousTests Assert.That(!string.IsNullOrEmpty(debugInfo.spr)); var startupConfig = new StartupConfig(); + startupConfig.NameOverride = "TransientNode"; var codexStartConfig = new CodexStartupConfig(CodexLogLevel.Trace); codexStartConfig.MarketplaceConfig = new MarketplaceInitialConfig(0.Eth(), 0.TestTokens(), false); codexStartConfig.MarketplaceConfig.AccountIndexOverride = ethereumAccountIndex; @@ -74,7 +75,7 @@ namespace ContinuousTests try { - operation(codexAccess, marketAccess); + operation(codexAccess, marketAccess, lifecycle); } catch { diff --git a/ContinuousTests/SingleTestRun.cs b/ContinuousTests/SingleTestRun.cs index 17752d2..b67a0f7 100644 --- a/ContinuousTests/SingleTestRun.cs +++ b/ContinuousTests/SingleTestRun.cs @@ -168,7 +168,7 @@ namespace ContinuousTests { Log(msg); var containerNames = $"({string.Join(",", nodes.Select(n => n.Container.Name))})"; - overviewLog.Log( testName + ": " + msg); + overviewLog.Log($"{containerNames} {testName}: {msg}"); } private CodexAccess[] CreateRandomNodes(int number) diff --git a/DistTestCore/Codex/CodexAccess.cs b/DistTestCore/Codex/CodexAccess.cs index d0cb1fc..2aea3b6 100644 --- a/DistTestCore/Codex/CodexAccess.cs +++ b/DistTestCore/Codex/CodexAccess.cs @@ -80,7 +80,7 @@ namespace DistTestCore.Codex private Http Http(TimeSpan? timeoutOverride = null) { - return new Http(log, timeSet, Address, baseUrl: "/api/codex/v1", timeoutOverride); + return new Http(log, timeSet, Address, baseUrl: "/api/codex/v1", Container.Name, timeoutOverride); } } } diff --git a/DistTestCore/Http.cs b/DistTestCore/Http.cs index 2aa5085..d4b716a 100644 --- a/DistTestCore/Http.cs +++ b/DistTestCore/Http.cs @@ -12,14 +12,16 @@ namespace DistTestCore private readonly ITimeSet timeSet; private readonly Address address; private readonly string baseUrl; + private readonly string? logAlias; private readonly TimeSpan? timeoutOverride; - public Http(BaseLog log, ITimeSet timeSet, Address address, string baseUrl, TimeSpan? timeoutOverride = null) + public Http(BaseLog log, ITimeSet timeSet, Address address, string baseUrl, string? logAlias = null, TimeSpan? timeoutOverride = null) { this.log = log; this.timeSet = timeSet; this.address = address; this.baseUrl = baseUrl; + this.logAlias = logAlias; this.timeoutOverride = timeoutOverride; if (!this.baseUrl.StartsWith("/")) this.baseUrl = "/" + this.baseUrl; if (!this.baseUrl.EndsWith("/")) this.baseUrl += "/"; @@ -113,7 +115,14 @@ namespace DistTestCore private void Log(string url, string message) { - log.Debug($"({url}) = '{message}'", 3); + if (logAlias != null) + { + log.Debug($"({logAlias})({url}) = '{message}'", 3); + } + else + { + log.Debug($"({url}) = '{message}'", 3); + } } private T Retry(Func operation, string description) From 0d4960f3ff3217a37a1b07d5bac441730e8237ff Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 30 Jun 2023 09:09:59 +0200 Subject: [PATCH 19/49] cleanup help cli args. Adds option to stop tests on first failure and download cluster logs. --- ArgsUniform/ArgsUniform.cs | 23 ++++++++++++++++------- ArgsUniform/ExampleUser.cs | 7 ++++++- CodexNetDeployer/Program.cs | 11 +---------- CodexNetDownloader/Program.cs | 13 ++----------- ContinuousTests/Configuration.cs | 20 +++++++++++++++++++- ContinuousTests/ContinuousTestRunner.cs | 2 +- ContinuousTests/K8sFactory.cs | 4 ++-- ContinuousTests/NodeRunner.cs | 2 +- ContinuousTests/Program.cs | 15 ++++++++++++--- ContinuousTests/SingleTestRun.cs | 22 +++++++++++++++++++++- 10 files changed, 81 insertions(+), 38 deletions(-) diff --git a/ArgsUniform/ArgsUniform.cs b/ArgsUniform/ArgsUniform.cs index 7d58cf1..88e539a 100644 --- a/ArgsUniform/ArgsUniform.cs +++ b/ArgsUniform/ArgsUniform.cs @@ -4,6 +4,7 @@ namespace ArgsUniform { public class ArgsUniform { + private readonly Action printAppInfo; private readonly object? defaultsProvider; private readonly IEnv.IEnv env; private readonly string[] args; @@ -12,23 +13,24 @@ namespace ArgsUniform private const int envStart = 48; private const int descStart = 80; - public ArgsUniform(params string[] args) - : this(new IEnv.Env(), args) + public ArgsUniform(Action printAppInfo, params string[] args) + : this(printAppInfo, new IEnv.Env(), args) { } - public ArgsUniform(object defaultsProvider, params string[] args) - : this(defaultsProvider, new IEnv.Env(), args) + public ArgsUniform(Action printAppInfo, object defaultsProvider, params string[] args) + : this(printAppInfo, defaultsProvider, new IEnv.Env(), args) { } - public ArgsUniform(IEnv.IEnv env, params string[] args) - : this(null!, env, args) + public ArgsUniform(Action printAppInfo, IEnv.IEnv env, params string[] args) + : this(printAppInfo, null!, env, args) { } - public ArgsUniform(object defaultsProvider, IEnv.IEnv env, params string[] args) + public ArgsUniform(Action printAppInfo, object defaultsProvider, IEnv.IEnv env, params string[] args) { + this.printAppInfo = printAppInfo; this.defaultsProvider = defaultsProvider; this.env = env; this.args = args; @@ -36,6 +38,13 @@ namespace ArgsUniform public T Parse(bool printResult = false) { + if (args.Any(a => a == "-h" || a == "--help" || a == "-?")) + { + printAppInfo(); + PrintHelp(); + throw new Exception(); + } + var result = Activator.CreateInstance(); var uniformProperties = typeof(T).GetProperties().Where(m => m.GetCustomAttributes(typeof(UniformAttribute), false).Length == 1).ToArray(); var missingRequired = new List(); diff --git a/ArgsUniform/ExampleUser.cs b/ArgsUniform/ExampleUser.cs index f8b5787..31db9ab 100644 --- a/ArgsUniform/ExampleUser.cs +++ b/ArgsUniform/ExampleUser.cs @@ -21,9 +21,14 @@ // env var: "AAA=BBB" var args = "--ccc=ddd"; - var uniform = new ArgsUniform(new DefaultsProvider(), args); + var uniform = new ArgsUniform(PrintHelp, new DefaultsProvider(), args); var aaa = uniform.Parse(); } + + private static void PrintHelp() + { + Console.WriteLine("Help text!"); + } } } diff --git a/CodexNetDeployer/Program.cs b/CodexNetDeployer/Program.cs index 4ac3a2f..b50c159 100644 --- a/CodexNetDeployer/Program.cs +++ b/CodexNetDeployer/Program.cs @@ -11,13 +11,7 @@ public class Program var nl = Environment.NewLine; Console.WriteLine("CodexNetDeployer" + nl); - if (args.Any(a => a == "-h" || a == "--help" || a == "-?")) - { - PrintHelp(); - return; - } - - var uniformArgs = new ArgsUniform(args); + var uniformArgs = new ArgsUniform(PrintHelp, args); var config = uniformArgs.Parse(true); if (args.Any(a => a == "--external")) @@ -54,8 +48,5 @@ public class Program Console.WriteLine("CodexNetDeployer assumes you are running this tool from *inside* the Kubernetes cluster you want to deploy to. " + "If you are not running this from a container inside the cluster, add the argument '--external'." + nl); - - var uniformArgs = new ArgsUniform(); - uniformArgs.PrintHelp(); } } diff --git a/CodexNetDownloader/Program.cs b/CodexNetDownloader/Program.cs index d17d76a..f52a1d2 100644 --- a/CodexNetDownloader/Program.cs +++ b/CodexNetDownloader/Program.cs @@ -12,13 +12,7 @@ public class Program var nl = Environment.NewLine; Console.WriteLine("CodexNetDownloader" + nl); - if (args.Any(a => a == "-h" || a == "--help" || a == "-?")) - { - PrintHelp(); - return; - } - - var uniformArgs = new ArgsUniform(args); + var uniformArgs = new ArgsUniform(PrintHelp, args); var config = uniformArgs.Parse(true); if (args.Any(a => a == "--external")) @@ -31,7 +25,7 @@ public class Program if (!Directory.Exists(config.OutputPath)) Directory.CreateDirectory(config.OutputPath); var k8sFactory = new K8sFactory(); - var (_, lifecycle) = k8sFactory.CreateFacilities(config.KubeConfigFile, config.OutputPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog()); + var (_, lifecycle) = k8sFactory.CreateFacilities(config.KubeConfigFile, config.OutputPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation); foreach (var container in config.CodexDeployment.CodexContainers) { @@ -55,8 +49,5 @@ public class Program Console.WriteLine("CodexNetDownloader assumes you are running this tool from *inside* the Kubernetes cluster. " + "If you are not running this from a container inside the cluster, add the argument '--external'." + nl); - - var uniformArgs = new ArgsUniform(); - uniformArgs.PrintHelp(); } } diff --git a/ContinuousTests/Configuration.cs b/ContinuousTests/Configuration.cs index 1d6b635..19a2bf6 100644 --- a/ContinuousTests/Configuration.cs +++ b/ContinuousTests/Configuration.cs @@ -21,18 +21,27 @@ namespace ContinuousTests [Uniform("kube-config", "kc", "KUBECONFIG", true, "Path to Kubeconfig file. Use 'null' (default) to use local cluster.")] public string KubeConfigFile { get; set; } = "null"; + [Uniform("stop", "s", "STOPONFAIL", false, "If true, runner will stop on first test failure and download all cluster container logs. False by default.")] + public bool StopOnFailure { get; set; } = false; + public CodexDeployment CodexDeployment { get; set; } = null!; + + public TestRunnerLocation RunnerLocation { get; set; } = TestRunnerLocation.InternalToCluster; } public class ConfigLoader { public Configuration Load(string[] args) { - var uniformArgs = new ArgsUniform(args); + var uniformArgs = new ArgsUniform(PrintHelp, args); var result = uniformArgs.Parse(true); result.CodexDeployment = ParseCodexDeploymentJson(result.CodexDeploymentJson); + if (args.Any(a => a == "--external")) + { + result.RunnerLocation = TestRunnerLocation.ExternalToCluster; + } return result; } @@ -43,5 +52,14 @@ namespace ContinuousTests if (d == null) throw new Exception("Unable to parse " + filename); return d; } + + private static void PrintHelp() + { + var nl = Environment.NewLine; + Console.WriteLine("CodexNetDownloader lets you download all container logs given a codex-deployment.json file." + nl); + + Console.WriteLine("CodexNetDownloader assumes you are running this tool from *inside* the Kubernetes cluster. " + + "If you are not running this from a container inside the cluster, add the argument '--external'." + nl); + } } } diff --git a/ContinuousTests/ContinuousTestRunner.cs b/ContinuousTests/ContinuousTestRunner.cs index 7ad16fa..0e699e1 100644 --- a/ContinuousTests/ContinuousTestRunner.cs +++ b/ContinuousTests/ContinuousTestRunner.cs @@ -58,7 +58,7 @@ namespace ContinuousTests if (string.IsNullOrEmpty(test.CustomK8sNamespace)) return; log.Log($"Clearing namespace '{test.CustomK8sNamespace}'..."); - var (workflowCreator, _) = k8SFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, config.DataPath, test.CustomK8sNamespace, new DefaultTimeSet(), log); + var (workflowCreator, _) = k8SFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, config.DataPath, test.CustomK8sNamespace, new DefaultTimeSet(), log, config.RunnerLocation); workflowCreator.CreateWorkflow().DeleteTestResources(); } } diff --git a/ContinuousTests/K8sFactory.cs b/ContinuousTests/K8sFactory.cs index 221e112..0704e9f 100644 --- a/ContinuousTests/K8sFactory.cs +++ b/ContinuousTests/K8sFactory.cs @@ -7,7 +7,7 @@ namespace ContinuousTests { public class K8sFactory { - public (WorkflowCreator, TestLifecycle) CreateFacilities(string kubeConfigFile, string logPath, string dataFilePath, string customNamespace, ITimeSet timeSet, BaseLog log) + public (WorkflowCreator, TestLifecycle) CreateFacilities(string kubeConfigFile, string logPath, string dataFilePath, string customNamespace, ITimeSet timeSet, BaseLog log, TestRunnerLocation runnerLocation) { var kubeConfig = GetKubeConfig(kubeConfigFile); var lifecycleConfig = new DistTestCore.Configuration @@ -17,7 +17,7 @@ namespace ContinuousTests logDebug: false, dataFilesPath: dataFilePath, codexLogLevel: CodexLogLevel.Debug, - runnerLocation: TestRunnerLocation.ExternalToCluster + runnerLocation: runnerLocation ); var kubeFlowConfig = new KubernetesWorkflow.Configuration( diff --git a/ContinuousTests/NodeRunner.cs b/ContinuousTests/NodeRunner.cs index b970a08..02116ee 100644 --- a/ContinuousTests/NodeRunner.cs +++ b/ContinuousTests/NodeRunner.cs @@ -91,7 +91,7 @@ namespace ContinuousTests private (WorkflowCreator, TestLifecycle) CreateFacilities() { - return k8SFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, config.DataPath, customNamespace, timeSet, log); + return k8SFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, config.DataPath, customNamespace, timeSet, log, config.RunnerLocation); } } } diff --git a/ContinuousTests/Program.cs b/ContinuousTests/Program.cs index 848e02c..273f13a 100644 --- a/ContinuousTests/Program.cs +++ b/ContinuousTests/Program.cs @@ -7,18 +7,27 @@ public class Program Console.WriteLine("Codex Continous-Test-Runner."); Console.WriteLine("Running..."); - var cts = new CancellationTokenSource(); - var runner = new ContinuousTestRunner(args, cts.Token); + var runner = new ContinuousTestRunner(args, Cancellation.Cts.Token); Console.CancelKeyPress += (sender, e) => { Console.WriteLine("Stopping..."); e.Cancel = true; - cts.Cancel(); + Cancellation.Cts.Cancel(); }; runner.Run(); Console.WriteLine("Done."); } + + public static class Cancellation + { + static Cancellation() + { + Cts = new CancellationTokenSource(); + } + + public static CancellationTokenSource Cts { get; } + } } diff --git a/ContinuousTests/SingleTestRun.cs b/ContinuousTests/SingleTestRun.cs index b67a0f7..2accc79 100644 --- a/ContinuousTests/SingleTestRun.cs +++ b/ContinuousTests/SingleTestRun.cs @@ -5,6 +5,7 @@ using Utils; using KubernetesWorkflow; using NUnit.Framework.Internal; using System.Reflection; +using static Program; namespace ContinuousTests { @@ -69,6 +70,14 @@ namespace ContinuousTests { fixtureLog.Error("Test run failed with exception: " + ex); fixtureLog.MarkAsFailed(); + + if (config.StopOnFailure) + { + OverviewLog("Configured to stop on first failure. Downloading cluster logs..."); + DownloadClusterLogs(); + OverviewLog("Log download finished. Cancelling test runner..."); + Cancellation.Cts.Cancel(); + } } } @@ -117,6 +126,17 @@ namespace ContinuousTests throw ex; } + private void DownloadClusterLogs() + { + var k8sFactory = new K8sFactory(); + var (_, lifecycle) = k8sFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation); + + foreach (var container in config.CodexDeployment.CodexContainers) + { + lifecycle.DownloadLog(container); + } + } + private Exception UnpackException(Exception exception) { if (exception is AggregateException a) @@ -192,7 +212,7 @@ namespace ContinuousTests private DistTestCore.Configuration CreateFileManagerConfiguration() { return new DistTestCore.Configuration(null, string.Empty, false, dataFolder, - CodexLogLevel.Error, TestRunnerLocation.ExternalToCluster); + CodexLogLevel.Error, config.RunnerLocation); } } } From 0caa53bf8ca72fee1f4f5c08828e74ca831912a7 Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 30 Jun 2023 09:14:54 +0200 Subject: [PATCH 20/49] support for bool in argsuniform --- ArgsUniform/ArgsUniform.cs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/ArgsUniform/ArgsUniform.cs b/ArgsUniform/ArgsUniform.cs index 88e539a..1cb0792 100644 --- a/ArgsUniform/ArgsUniform.cs +++ b/ArgsUniform/ArgsUniform.cs @@ -203,6 +203,7 @@ namespace ArgsUniform { if (uniformProperty.PropertyType == typeof(int?)) return AssignOptionalInt(result, uniformProperty, value); if (uniformProperty.PropertyType.IsEnum) return AssignEnum(result, uniformProperty, value); + if (uniformProperty.PropertyType == typeof(bool)) return AssignBool(result, uniformProperty, value); throw new NotSupportedException(); } @@ -230,6 +231,15 @@ namespace ArgsUniform return false; } + private static bool AssignBool(T result, PropertyInfo uniformProperty, object value) + { + if (value != null) + { + uniformProperty.SetValue(result, true); + } + return true; + } + private string? GetFromArgs(string key) { var argKey = $"--{key}="; From 0987a71f1e0c2ce7d84ae18087796ff7b72c3c77 Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 30 Jun 2023 09:33:31 +0200 Subject: [PATCH 21/49] deploy script for easy use in the cluster --- CodexNetDeployer/deploy-continuous-testnet.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 CodexNetDeployer/deploy-continuous-testnet.sh diff --git a/CodexNetDeployer/deploy-continuous-testnet.sh b/CodexNetDeployer/deploy-continuous-testnet.sh new file mode 100644 index 0000000..2f02e45 --- /dev/null +++ b/CodexNetDeployer/deploy-continuous-testnet.sh @@ -0,0 +1,12 @@ +dotnet run \ + --kube-config=/opt/kubeconfig.yaml \ + --kube-namespace=codex-continuous-tests \ + --nodes=5 \ + --validators=3 \ + --storage-quota=2048 \ + --storage-sell=1024 \ + --min-price=1024 \ + --max-collateral=1024 \ + --max-duration=3600000 \ + --block-ttl=120 + From 6e03301b7ac10bfa492054da200b52e357841095 Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 30 Jun 2023 09:43:20 +0200 Subject: [PATCH 22/49] Missing include --- ContinuousTests/Configuration.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/ContinuousTests/Configuration.cs b/ContinuousTests/Configuration.cs index 19a2bf6..dd122cb 100644 --- a/ContinuousTests/Configuration.cs +++ b/ContinuousTests/Configuration.cs @@ -1,4 +1,5 @@ using ArgsUniform; +using DistTestCore; using DistTestCore.Codex; using Newtonsoft.Json; From 4dab688b6f672096225adc45a9fb1d77d5c0c19e Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 30 Jun 2023 09:54:13 +0200 Subject: [PATCH 23/49] Adds delay in deployer to ensure accounts are unlocked on time. --- CodexNetDeployer/Deployer.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CodexNetDeployer/Deployer.cs b/CodexNetDeployer/Deployer.cs index b4efd3f..65cbb06 100644 --- a/CodexNetDeployer/Deployer.cs +++ b/CodexNetDeployer/Deployer.cs @@ -35,6 +35,11 @@ namespace CodexNetDeployer Log("Geth started. Codex contracts deployed."); Log("Warning: It can take up to 45 minutes for the Geth node to finish unlocking all if its 1000 preconfigured accounts."); + // It takes a second for the geth node to unlock a single account. Let's assume 3. + // We can't start the codex nodes until their accounts are definitely unlocked. So + // We wait: + Thread.Sleep(TimeSpan.FromSeconds(3.0 * config.NumberOfCodexNodes!.Value)); + Log("Starting Codex nodes..."); // Each node must have its own IP, so it needs it own pod. Start them 1 at a time. From b0a4ed93d8f9abe7956f224f7ff7fa30594656e5 Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 30 Jun 2023 09:58:59 +0200 Subject: [PATCH 24/49] script for easy running of continuous tests in cluster --- ContinuousTests/run-continuous-tests.sh | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 ContinuousTests/run-continuous-tests.sh diff --git a/ContinuousTests/run-continuous-tests.sh b/ContinuousTests/run-continuous-tests.sh new file mode 100644 index 0000000..6586ca7 --- /dev/null +++ b/ContinuousTests/run-continuous-tests.sh @@ -0,0 +1,4 @@ +dotnet run \ + --kube-config=/opt/kubeconfig.yaml \ + --codex-deployment=codex-deployment.json \ + --stop=1 From 9970f225ccc25f5b5b6ca3d1e5bbd8e40d62cf23 Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 30 Jun 2023 10:08:51 +0200 Subject: [PATCH 25/49] Makes codexAccessFactory follow configuration-runner location --- ContinuousTests/CodexAccessFactory.cs | 3 ++- ContinuousTests/SingleTestRun.cs | 2 +- ContinuousTests/StartupChecker.cs | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/ContinuousTests/CodexAccessFactory.cs b/ContinuousTests/CodexAccessFactory.cs index 08eb815..a149017 100644 --- a/ContinuousTests/CodexAccessFactory.cs +++ b/ContinuousTests/CodexAccessFactory.cs @@ -7,11 +7,12 @@ namespace ContinuousTests { public class CodexAccessFactory { - public CodexAccess[] Create(RunningContainer[] containers, BaseLog log, ITimeSet timeSet) + public CodexAccess[] Create(Configuration config, RunningContainer[] containers, BaseLog log, ITimeSet timeSet) { return containers.Select(container => { var address = container.ClusterExternalAddress; + if (config.RunnerLocation == TestRunnerLocation.InternalToCluster) address = container.ClusterInternalAddress; return new CodexAccess(log, container, timeSet, address); }).ToArray(); } diff --git a/ContinuousTests/SingleTestRun.cs b/ContinuousTests/SingleTestRun.cs index 2accc79..cb1c801 100644 --- a/ContinuousTests/SingleTestRun.cs +++ b/ContinuousTests/SingleTestRun.cs @@ -195,7 +195,7 @@ namespace ContinuousTests { var containers = SelectRandomContainers(number); fixtureLog.Log("Selected nodes: " + string.Join(",", containers.Select(c => c.Name))); - return codexNodeFactory.Create(containers, fixtureLog, handle.Test.TimeSet); + return codexNodeFactory.Create(config, containers, fixtureLog, handle.Test.TimeSet); } private RunningContainer[] SelectRandomContainers(int number) diff --git a/ContinuousTests/StartupChecker.cs b/ContinuousTests/StartupChecker.cs index e4889b9..4c33ccd 100644 --- a/ContinuousTests/StartupChecker.cs +++ b/ContinuousTests/StartupChecker.cs @@ -55,7 +55,7 @@ namespace ContinuousTests private void CheckCodexNodes(BaseLog log, Configuration config) { - var nodes = codexNodeFactory.Create(config.CodexDeployment.CodexContainers, log, new DefaultTimeSet()); + var nodes = codexNodeFactory.Create(config, config.CodexDeployment.CodexContainers, log, new DefaultTimeSet()); var pass = true; foreach (var n in nodes) { From 05e60a6b747cfbae7f75aac4525d9ff852beb167 Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 3 Jul 2023 08:39:11 +0200 Subject: [PATCH 26/49] Future-counting test --- ContinuousTests/SingleTestRun.cs | 14 +- ContinuousTests/Tests/MarketplaceTest.cs | 164 ++++++++++----------- ContinuousTests/Tests/PerformanceTests.cs | 136 ++++++++--------- ContinuousTests/Tests/TransientNodeTest.cs | 87 ++++++----- ContinuousTests/Tests/TwoClientTest.cs | 4 +- DistTestCore/Codex/CodexAccess.cs | 5 + DistTestCore/Codex/CodexApiTypes.cs | 5 + 7 files changed, 221 insertions(+), 194 deletions(-) diff --git a/ContinuousTests/SingleTestRun.cs b/ContinuousTests/SingleTestRun.cs index cb1c801..9316789 100644 --- a/ContinuousTests/SingleTestRun.cs +++ b/ContinuousTests/SingleTestRun.cs @@ -62,6 +62,7 @@ namespace ContinuousTests { try { + OverviewLog(" > Starting test. " + FuturesInfo()); RunTestMoments(); if (!config.KeepPassedTestLogs) fixtureLog.Delete(); @@ -112,7 +113,7 @@ namespace ContinuousTests { ThrowFailTest(); } - OverviewLog(" > Test passed."); + OverviewLog(" > Test passed. " + FuturesInfo()); return; } } @@ -122,10 +123,19 @@ namespace ContinuousTests { var ex = UnpackException(exceptions.First()); Log(ex.ToString()); - OverviewLog(" > Test failed: " + ex.Message); + OverviewLog($" > Test failed {FuturesInfo()}: " + ex.Message); throw ex; } + private string FuturesInfo() + { + var containers = config.CodexDeployment.CodexContainers; + var nodes = codexNodeFactory.Create(config, containers, fixtureLog, handle.Test.TimeSet); + var f = nodes.Select(n => n.GetDebugFutures().ToString()); + var msg = $"(Futures: [{string.Join(", ", f)}])"; + return msg; + } + private void DownloadClusterLogs() { var k8sFactory = new K8sFactory(); diff --git a/ContinuousTests/Tests/MarketplaceTest.cs b/ContinuousTests/Tests/MarketplaceTest.cs index 02c3dcc..86500a2 100644 --- a/ContinuousTests/Tests/MarketplaceTest.cs +++ b/ContinuousTests/Tests/MarketplaceTest.cs @@ -1,99 +1,99 @@ -using DistTestCore; -using DistTestCore.Codex; -using Newtonsoft.Json; -using NUnit.Framework; -using Utils; +//using DistTestCore; +//using DistTestCore.Codex; +//using Newtonsoft.Json; +//using NUnit.Framework; +//using Utils; -namespace ContinuousTests.Tests -{ - public class MarketplaceTest : ContinuousTest - { - public override int RequiredNumberOfNodes => 1; - public override TimeSpan RunTestEvery => TimeSpan.FromMinutes(15); - public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure; - public override int EthereumAccountIndex => 200; - public override string CustomK8sNamespace => "codex-continuous-marketplace"; +//namespace ContinuousTests.Tests +//{ +// public class MarketplaceTest : ContinuousTest +// { +// public override int RequiredNumberOfNodes => 1; +// public override TimeSpan RunTestEvery => TimeSpan.FromMinutes(10); +// public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure; +// public override int EthereumAccountIndex => 200; +// public override string CustomK8sNamespace => "codex-continuous-marketplace"; - private readonly uint numberOfSlots = 3; - private readonly ByteSize fileSize = 10.MB(); - private readonly TestToken pricePerSlotPerSecond = 10.TestTokens(); +// private readonly uint numberOfSlots = 3; +// private readonly ByteSize fileSize = 10.MB(); +// private readonly TestToken pricePerSlotPerSecond = 10.TestTokens(); - private TestFile file = null!; - private ContentId? cid; - private string purchaseId = string.Empty; +// private TestFile file = null!; +// private ContentId? cid; +// private string purchaseId = string.Empty; - [TestMoment(t: Zero)] - public void NodePostsStorageRequest() - { - var contractDuration = TimeSpan.FromMinutes(11); //TimeSpan.FromDays(3) + TimeSpan.FromHours(1); - decimal totalDurationSeconds = Convert.ToDecimal(contractDuration.TotalSeconds); - var expectedTotalCost = numberOfSlots * pricePerSlotPerSecond.Amount * (totalDurationSeconds + 1) * 1000000; +// [TestMoment(t: Zero)] +// public void NodePostsStorageRequest() +// { +// var contractDuration = TimeSpan.FromMinutes(8); +// decimal totalDurationSeconds = Convert.ToDecimal(contractDuration.TotalSeconds); +// var expectedTotalCost = numberOfSlots * pricePerSlotPerSecond.Amount * (totalDurationSeconds + 1) * 1000000; - file = FileManager.GenerateTestFile(fileSize); +// file = FileManager.GenerateTestFile(fileSize); - NodeRunner.RunNode((codexAccess, marketplaceAccess) => - { - cid = UploadFile(codexAccess.Node, file); - Assert.That(cid, Is.Not.Null); +// NodeRunner.RunNode((codexAccess, marketplaceAccess) => +// { +// cid = UploadFile(codexAccess.Node, file); +// Assert.That(cid, Is.Not.Null); - purchaseId = marketplaceAccess.RequestStorage( - contentId: cid!, - pricePerSlotPerSecond: pricePerSlotPerSecond, - requiredCollateral: 100.TestTokens(), - minRequiredNumberOfNodes: numberOfSlots, - proofProbability: 10, - duration: contractDuration); +// purchaseId = marketplaceAccess.RequestStorage( +// contentId: cid!, +// pricePerSlotPerSecond: pricePerSlotPerSecond, +// requiredCollateral: 100.TestTokens(), +// minRequiredNumberOfNodes: numberOfSlots, +// proofProbability: 10, +// duration: contractDuration); - Assert.That(!string.IsNullOrEmpty(purchaseId)); +// Assert.That(!string.IsNullOrEmpty(purchaseId)); - WaitForContractToStart(codexAccess, purchaseId); - }); - } +// WaitForContractToStart(codexAccess, purchaseId); +// }); +// } - [TestMoment(t: MinuteFive * 2)] - public void StoredDataIsAvailableAfterThreeDays() - { - NodeRunner.RunNode((codexAccess, marketplaceAccess) => - { - var result = DownloadFile(codexAccess.Node, cid!); +// [TestMoment(t: MinuteFive + MinuteOne)] +// public void StoredDataIsAvailableAfterThreeDays() +// { +// NodeRunner.RunNode((codexAccess, marketplaceAccess) => +// { +// var result = DownloadFile(codexAccess.Node, cid!); - file.AssertIsEqual(result); - }); - } +// file.AssertIsEqual(result); +// }); +// } - private void WaitForContractToStart(CodexAccess codexAccess, string purchaseId) - { - var lastState = ""; - var waitStart = DateTime.UtcNow; - var filesizeInMb = fileSize.SizeInBytes / (1024 * 1024); - var maxWaitTime = TimeSpan.FromSeconds(filesizeInMb * 10.0); +// private void WaitForContractToStart(CodexAccess codexAccess, string purchaseId) +// { +// var lastState = ""; +// var waitStart = DateTime.UtcNow; +// var filesizeInMb = fileSize.SizeInBytes / (1024 * 1024); +// var maxWaitTime = TimeSpan.FromSeconds(filesizeInMb * 10.0); - Log.Log($"{nameof(WaitForContractToStart)} for {Time.FormatDuration(maxWaitTime)}"); - while (lastState != "started") - { - CancelToken.ThrowIfCancellationRequested(); +// Log.Log($"{nameof(WaitForContractToStart)} for {Time.FormatDuration(maxWaitTime)}"); +// while (lastState != "started") +// { +// CancelToken.ThrowIfCancellationRequested(); - var purchaseStatus = codexAccess.Node.GetPurchaseStatus(purchaseId); - var statusJson = JsonConvert.SerializeObject(purchaseStatus); - if (purchaseStatus != null && purchaseStatus.state != lastState) - { - lastState = purchaseStatus.state; - Log.Log("Purchase status: " + statusJson); - } +// var purchaseStatus = codexAccess.Node.GetPurchaseStatus(purchaseId); +// var statusJson = JsonConvert.SerializeObject(purchaseStatus); +// if (purchaseStatus != null && purchaseStatus.state != lastState) +// { +// lastState = purchaseStatus.state; +// Log.Log("Purchase status: " + statusJson); +// } - Thread.Sleep(2000); +// Thread.Sleep(2000); - if (lastState == "errored") - { - Assert.Fail("Contract start failed: " + statusJson); - } +// if (lastState == "errored") +// { +// Assert.Fail("Contract start failed: " + statusJson); +// } - if (DateTime.UtcNow - waitStart > maxWaitTime) - { - Assert.Fail($"Contract was not picked up within {maxWaitTime.TotalSeconds} seconds timeout: {statusJson}"); - } - } - Log.Log("Contract started."); - } - } -} +// if (DateTime.UtcNow - waitStart > maxWaitTime) +// { +// Assert.Fail($"Contract was not picked up within {maxWaitTime.TotalSeconds} seconds timeout: {statusJson}"); +// } +// } +// Log.Log("Contract started."); +// } +// } +//} diff --git a/ContinuousTests/Tests/PerformanceTests.cs b/ContinuousTests/Tests/PerformanceTests.cs index 2df3052..b6094a0 100644 --- a/ContinuousTests/Tests/PerformanceTests.cs +++ b/ContinuousTests/Tests/PerformanceTests.cs @@ -1,86 +1,86 @@ -using DistTestCore; -using DistTestCore.Codex; -using NUnit.Framework; +//using DistTestCore; +//using DistTestCore.Codex; +//using NUnit.Framework; -namespace ContinuousTests.Tests -{ - public class UploadPerformanceTest : PerformanceTest - { - public override int RequiredNumberOfNodes => 1; +//namespace ContinuousTests.Tests +//{ +// public class UploadPerformanceTest : PerformanceTest +// { +// public override int RequiredNumberOfNodes => 1; - [TestMoment(t: Zero)] - public void UploadTest() - { - UploadTest(100, Nodes[0]); - } - } +// [TestMoment(t: Zero)] +// public void UploadTest() +// { +// UploadTest(100, Nodes[0]); +// } +// } - public class DownloadLocalPerformanceTest : PerformanceTest - { - public override int RequiredNumberOfNodes => 1; +// public class DownloadLocalPerformanceTest : PerformanceTest +// { +// public override int RequiredNumberOfNodes => 1; - [TestMoment(t: Zero)] - public void DownloadTest() - { - DownloadTest(100, Nodes[0], Nodes[0]); - } - } +// [TestMoment(t: Zero)] +// public void DownloadTest() +// { +// DownloadTest(100, Nodes[0], Nodes[0]); +// } +// } - public class DownloadRemotePerformanceTest : PerformanceTest - { - public override int RequiredNumberOfNodes => 2; +// public class DownloadRemotePerformanceTest : PerformanceTest +// { +// public override int RequiredNumberOfNodes => 2; - [TestMoment(t: Zero)] - public void DownloadTest() - { - DownloadTest(100, Nodes[0], Nodes[1]); - } - } +// [TestMoment(t: Zero)] +// public void DownloadTest() +// { +// DownloadTest(100, Nodes[0], Nodes[1]); +// } +// } - public abstract class PerformanceTest : ContinuousTest - { - public override TimeSpan RunTestEvery => TimeSpan.FromHours(1); - public override TestFailMode TestFailMode => TestFailMode.AlwaysRunAllMoments; +// public abstract class PerformanceTest : ContinuousTest +// { +// public override TimeSpan RunTestEvery => TimeSpan.FromMinutes(10); +// public override TestFailMode TestFailMode => TestFailMode.AlwaysRunAllMoments; - public void UploadTest(int megabytes, CodexAccess uploadNode) - { - var file = FileManager.GenerateTestFile(megabytes.MB()); +// public void UploadTest(int megabytes, CodexAccess uploadNode) +// { +// var file = FileManager.GenerateTestFile(megabytes.MB()); - var time = Measure(() => - { - UploadFile(uploadNode, file); - }); +// var time = Measure(() => +// { +// UploadFile(uploadNode, file); +// }); - var timePerMB = time / megabytes; +// var timePerMB = time / megabytes; - Assert.That(timePerMB, Is.LessThan(CodexContainerRecipe.MaxUploadTimePerMegabyte), "MaxUploadTimePerMegabyte performance threshold breached."); - } +// Assert.That(timePerMB, Is.LessThan(CodexContainerRecipe.MaxUploadTimePerMegabyte), "MaxUploadTimePerMegabyte performance threshold breached."); +// } - public void DownloadTest(int megabytes, CodexAccess uploadNode, CodexAccess downloadNode) - { - var file = FileManager.GenerateTestFile(megabytes.MB()); +// public void DownloadTest(int megabytes, CodexAccess uploadNode, CodexAccess downloadNode) +// { +// var file = FileManager.GenerateTestFile(megabytes.MB()); - var cid = UploadFile(uploadNode, file); - Assert.That(cid, Is.Not.Null); +// var cid = UploadFile(uploadNode, file); +// Assert.That(cid, Is.Not.Null); - TestFile? result = null; - var time = Measure(() => - { - result = DownloadFile(downloadNode, cid!); - }); +// TestFile? result = null; +// var time = Measure(() => +// { +// result = DownloadFile(downloadNode, cid!); +// }); - file.AssertIsEqual(result); +// file.AssertIsEqual(result); - var timePerMB = time / megabytes; +// var timePerMB = time / megabytes; - Assert.That(timePerMB, Is.LessThan(CodexContainerRecipe.MaxDownloadTimePerMegabyte), "MaxDownloadTimePerMegabyte performance threshold breached."); - } +// Assert.That(timePerMB, Is.LessThan(CodexContainerRecipe.MaxDownloadTimePerMegabyte), "MaxDownloadTimePerMegabyte performance threshold breached."); +// } - private static TimeSpan Measure(Action action) - { - var start = DateTime.UtcNow; - action(); - return DateTime.UtcNow - start; - } - } -} +// private static TimeSpan Measure(Action action) +// { +// var start = DateTime.UtcNow; +// action(); +// return DateTime.UtcNow - start; +// } +// } +//} diff --git a/ContinuousTests/Tests/TransientNodeTest.cs b/ContinuousTests/Tests/TransientNodeTest.cs index 37eb194..fe2ae34 100644 --- a/ContinuousTests/Tests/TransientNodeTest.cs +++ b/ContinuousTests/Tests/TransientNodeTest.cs @@ -1,47 +1,54 @@ -using DistTestCore; -using DistTestCore.Codex; -using NUnit.Framework; +//using DistTestCore; +//using DistTestCore.Codex; +//using NUnit.Framework; -namespace ContinuousTests.Tests -{ - public class TransientNodeTest : ContinuousTest - { - public override int RequiredNumberOfNodes => 3; - public override TimeSpan RunTestEvery => TimeSpan.FromMinutes(10); - public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure; - public override string CustomK8sNamespace => nameof(TransientNodeTest).ToLowerInvariant(); - public override int EthereumAccountIndex => 201; +//namespace ContinuousTests.Tests +//{ +// public class TransientNodeTest : ContinuousTest +// { +// public override int RequiredNumberOfNodes => 3; +// public override TimeSpan RunTestEvery => TimeSpan.FromMinutes(1); +// public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure; +// public override string CustomK8sNamespace => nameof(TransientNodeTest).ToLowerInvariant(); +// public override int EthereumAccountIndex => 201; - private TestFile file = null!; - private ContentId cid = null!; +// private TestFile file = null!; +// private ContentId cid = null!; - private CodexAccess UploadBootstapNode { get { return Nodes[0]; } } - private CodexAccess DownloadBootstapNode { get { return Nodes[1]; } } - private CodexAccess IntermediateNode { get { return Nodes[2]; } } +// private CodexAccess UploadBootstapNode { get { return Nodes[0]; } } +// private CodexAccess DownloadBootstapNode { get { return Nodes[1]; } } +// private CodexAccess IntermediateNode { get { return Nodes[2]; } } - [TestMoment(t: 0)] - public void UploadWithTransientNode() - { - file = FileManager.GenerateTestFile(10.MB()); +// [TestMoment(t: 0)] +// public void UploadWithTransientNode() +// { +// file = FileManager.GenerateTestFile(10.MB()); - NodeRunner.RunNode(UploadBootstapNode, (codexAccess, marketplaceAccess) => - { - cid = UploadFile(codexAccess, file)!; - Assert.That(cid, Is.Not.Null); +// NodeRunner.RunNode(UploadBootstapNode, (codexAccess, marketplaceAccess, lifecycle) => +// { +// cid = UploadFile(codexAccess, file)!; +// Assert.That(cid, Is.Not.Null); - var resultFile = DownloadFile(IntermediateNode, cid); - file.AssertIsEqual(resultFile); - }); - } +// var dlt = Task.Run(() => +// { +// Thread.Sleep(10000); +// lifecycle.DownloadLog(codexAccess.Container); +// }); - [TestMoment(t: MinuteFive)] - public void DownloadWithTransientNode() - { - NodeRunner.RunNode(DownloadBootstapNode, (codexAccess, marketplaceAccess) => - { - var resultFile = DownloadFile(codexAccess, cid); - file.AssertIsEqual(resultFile); - }); - } - } -} +// var resultFile = DownloadFile(IntermediateNode, cid); +// dlt.Wait(); +// file.AssertIsEqual(resultFile); +// }); +// } + +// [TestMoment(t: 30)] +// public void DownloadWithTransientNode() +// { +// NodeRunner.RunNode(DownloadBootstapNode, (codexAccess, marketplaceAccess, lifecycle) => +// { +// var resultFile = DownloadFile(codexAccess, cid); +// file.AssertIsEqual(resultFile); +// }); +// } +// } +//} diff --git a/ContinuousTests/Tests/TwoClientTest.cs b/ContinuousTests/Tests/TwoClientTest.cs index 9d23d77..53a26ba 100644 --- a/ContinuousTests/Tests/TwoClientTest.cs +++ b/ContinuousTests/Tests/TwoClientTest.cs @@ -6,7 +6,7 @@ namespace ContinuousTests.Tests public class TwoClientTest : ContinuousTest { public override int RequiredNumberOfNodes => 2; - public override TimeSpan RunTestEvery => TimeSpan.FromHours(1); + public override TimeSpan RunTestEvery => TimeSpan.FromSeconds(30); public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure; private ContentId? cid; @@ -21,7 +21,7 @@ namespace ContinuousTests.Tests Assert.That(cid, Is.Not.Null); } - [TestMoment(t: MinuteFive)] + [TestMoment(t: 10)] public void DownloadTestFile() { var dl = DownloadFile(Nodes[1], cid!); diff --git a/DistTestCore/Codex/CodexAccess.cs b/DistTestCore/Codex/CodexAccess.cs index 2aea3b6..0ec046c 100644 --- a/DistTestCore/Codex/CodexAccess.cs +++ b/DistTestCore/Codex/CodexAccess.cs @@ -48,6 +48,11 @@ namespace DistTestCore.Codex return result; } + public int GetDebugFutures() + { + return Http().HttpGetJson("debug/futures").futures; + } + public string UploadFile(FileStream fileStream) { return Http().HttpPostStream("upload", fileStream); diff --git a/DistTestCore/Codex/CodexApiTypes.cs b/DistTestCore/Codex/CodexApiTypes.cs index 885540e..835df10 100644 --- a/DistTestCore/Codex/CodexApiTypes.cs +++ b/DistTestCore/Codex/CodexApiTypes.cs @@ -16,6 +16,11 @@ namespace DistTestCore.Codex public CodexDebugTableResponse table { get; set; } = new(); } + public class CodexDebugFutures + { + public int futures { get; set; } + } + public class CodexDebugTableResponse { public CodexDebugTableNodeResponse localNode { get; set; } = new(); From 52c9ec9211b40be66be43d9d4aab3dfb5425f6f4 Mon Sep 17 00:00:00 2001 From: benbierens Date: Tue, 4 Jul 2023 16:04:18 +0200 Subject: [PATCH 27/49] Restores two-client test --- DistTestCore/Codex/CodexContainerRecipe.cs | 41 +++++++++++++--------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/DistTestCore/Codex/CodexContainerRecipe.cs b/DistTestCore/Codex/CodexContainerRecipe.cs index b7c3b94..2500ef5 100644 --- a/DistTestCore/Codex/CodexContainerRecipe.cs +++ b/DistTestCore/Codex/CodexContainerRecipe.cs @@ -1,15 +1,16 @@ using DistTestCore.Marketplace; using KubernetesWorkflow; +using System.Net.Sockets; namespace DistTestCore.Codex { public class CodexContainerRecipe : ContainerRecipeFactory { #if Arm64 - public const string DockerImage = "codexstorage/nim-codex:sha-7b88ea0"; + public const string DockerImage = "codexstorage/nim-codex:sha-f053135"; #else - public const string DockerImage = "thatbenbierens/nim-codex:dhting"; - //public const string DockerImage = "codexstorage/nim-codex:sha-7b88ea0"; + //public const string DockerImage = "thatbenbierens/nim-codex:dhting"; + public const string DockerImage = "codexstorage/nim-codex:sha-f053135"; #endif public const string MetricsPortTag = "metrics_port"; public const string DiscoveryPortTag = "discovery-port"; @@ -24,26 +25,32 @@ namespace DistTestCore.Codex { var config = startupConfig.Get(); - AddExposedPortAndVar("API_PORT"); - AddEnvVar("DATA_DIR", $"datadir{ContainerNumber}"); - AddInternalPortAndVar("DISC_PORT", DiscoveryPortTag); - AddEnvVar("LOG_LEVEL", config.LogLevel.ToString()!.ToUpperInvariant()); + AddExposedPortAndVar("CODEX_API_PORT"); + AddEnvVar("CODEX_API_BINDADDR", "0.0.0.0"); + + AddEnvVar("CODEX_DATA_DIR", $"datadir{ContainerNumber}"); + AddInternalPortAndVar("CODEX_DISC_PORT", DiscoveryPortTag); + AddEnvVar("CODEX_LOG_LEVEL", config.LogLevel.ToString()!.ToUpperInvariant()); + + // This makes the node announce itself to its local (pod) IP address. + AddEnvVar("CODEX_NAT_ADDR", "$(hostname --ip-address)"); var listenPort = AddInternalPort(); - AddEnvVar("LISTEN_ADDRS", $"/ip4/0.0.0.0/tcp/{listenPort.Number}"); + AddEnvVar("CODEX_LISTEN_ADDRS", $"/ip4/0.0.0.0/tcp/{listenPort.Number}"); if (!string.IsNullOrEmpty(config.BootstrapSpr)) { - AddEnvVar("BOOTSTRAP_SPR", config.BootstrapSpr); + AddEnvVar("CODEX_BOOTSTRAP_NODE", config.BootstrapSpr); } if (config.StorageQuota != null) { - AddEnvVar("STORAGE_QUOTA", config.StorageQuota.SizeInBytes.ToString()!); + AddEnvVar("CODEX_STORAGE_QUOTA", config.StorageQuota.SizeInBytes.ToString()!); } if (config.MetricsEnabled) { - AddEnvVar("METRICS_ADDR", "0.0.0.0"); - AddInternalPortAndVar("METRICS_PORT", tag: MetricsPortTag); + AddEnvVar("CODEX_METRICS", "true"); + AddEnvVar("CODEX_METRICS_ADDRESS", "0.0.0.0"); + AddInternalPortAndVar("CODEX_METRICS_PORT", tag: MetricsPortTag); } if (config.MarketplaceConfig != null) @@ -56,14 +63,14 @@ namespace DistTestCore.Codex var ip = companionNode.RunningContainer.Pod.PodInfo.Ip; var port = companionNode.RunningContainer.Recipe.GetPortByTag(GethContainerRecipe.HttpPortTag).Number; - AddEnvVar("ETH_PROVIDER", $"ws://{ip}:{port}"); - AddEnvVar("ETH_ACCOUNT", companionNodeAccount.Account); - AddEnvVar("ETH_MARKETPLACE_ADDRESS", gethConfig.MarketplaceNetwork.Marketplace.Address); - AddEnvVar("PERSISTENCE", "1"); + AddEnvVar("CODEX_ETH_PROVIDER", $"ws://{ip}:{port}"); + AddEnvVar("CODEX_ETH_ACCOUNT", companionNodeAccount.Account); + AddEnvVar("CODEX_MARKETPLACE_ADDRESS", gethConfig.MarketplaceNetwork.Marketplace.Address); + AddEnvVar("CODEX_PERSISTENCE", "true"); if (config.MarketplaceConfig.IsValidator) { - AddEnvVar("VALIDATOR", "1"); + AddEnvVar("CODEX_VALIDATOR", "true"); } } } From d982f398704ce8fa6048b10a7eaacf58c8e146dc Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 7 Jul 2023 08:52:53 +0200 Subject: [PATCH 28/49] Adds threshold checking test --- ContinuousTests/ContinuousTestRunner.cs | 2 +- ContinuousTests/SingleTestRun.cs | 1 - ContinuousTests/StartupChecker.cs | 9 +++-- ContinuousTests/Tests/ThresholdChecks.cs | 43 ++++++++++++++++++++++++ DistTestCore/Codex/CodexAccess.cs | 5 +++ DistTestCore/Codex/CodexApiTypes.cs | 5 +++ 6 files changed, 61 insertions(+), 4 deletions(-) create mode 100644 ContinuousTests/Tests/ThresholdChecks.cs diff --git a/ContinuousTests/ContinuousTestRunner.cs b/ContinuousTests/ContinuousTestRunner.cs index 0e699e1..5be4ff8 100644 --- a/ContinuousTests/ContinuousTestRunner.cs +++ b/ContinuousTests/ContinuousTestRunner.cs @@ -15,7 +15,7 @@ namespace ContinuousTests public ContinuousTestRunner(string[] args, CancellationToken cancelToken) { config = configLoader.Load(args); - startupChecker = new StartupChecker(config); + startupChecker = new StartupChecker(config, cancelToken); this.cancelToken = cancelToken; } diff --git a/ContinuousTests/SingleTestRun.cs b/ContinuousTests/SingleTestRun.cs index 9316789..f8e6583 100644 --- a/ContinuousTests/SingleTestRun.cs +++ b/ContinuousTests/SingleTestRun.cs @@ -62,7 +62,6 @@ namespace ContinuousTests { try { - OverviewLog(" > Starting test. " + FuturesInfo()); RunTestMoments(); if (!config.KeepPassedTestLogs) fixtureLog.Delete(); diff --git a/ContinuousTests/StartupChecker.cs b/ContinuousTests/StartupChecker.cs index 4c33ccd..8e9357a 100644 --- a/ContinuousTests/StartupChecker.cs +++ b/ContinuousTests/StartupChecker.cs @@ -1,7 +1,6 @@ using DistTestCore.Codex; using DistTestCore; using Logging; -using NUnit.Framework.Internal; namespace ContinuousTests { @@ -10,10 +9,12 @@ namespace ContinuousTests private readonly TestFactory testFactory = new TestFactory(); private readonly CodexAccessFactory codexNodeFactory = new CodexAccessFactory(); private readonly Configuration config; + private readonly CancellationToken cancelToken; - public StartupChecker(Configuration config) + public StartupChecker(Configuration config, CancellationToken cancelToken) { this.config = config; + this.cancelToken = cancelToken; } public void Check() @@ -36,6 +37,8 @@ namespace ContinuousTests } foreach (var test in tests) { + cancelToken.ThrowIfCancellationRequested(); + var handle = new TestHandle(test); handle.GetEarliestMoment(); handle.GetLastMoment(); @@ -59,6 +62,8 @@ namespace ContinuousTests var pass = true; foreach (var n in nodes) { + cancelToken.ThrowIfCancellationRequested(); + log.Log($"Checking '{n.Address.Host}'..."); if (EnsureOnline(n)) diff --git a/ContinuousTests/Tests/ThresholdChecks.cs b/ContinuousTests/Tests/ThresholdChecks.cs new file mode 100644 index 0000000..f1b8167 --- /dev/null +++ b/ContinuousTests/Tests/ThresholdChecks.cs @@ -0,0 +1,43 @@ +using DistTestCore; +using DistTestCore.Codex; +using NUnit.Framework; + +namespace ContinuousTests.Tests +{ + public class ThresholdChecks : ContinuousTest + { + public override int RequiredNumberOfNodes => 1; + public override TimeSpan RunTestEvery => TimeSpan.FromSeconds(30); + public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure; + + [TestMoment(t: 0)] + public void CheckAllThresholds() + { + var allNodes = CreateAccessToAllNodes(); + foreach (var n in allNodes) CheckThresholds(n); + } + + private void CheckThresholds(CodexAccess n) + { + var breaches = n.GetDebugThresholdBreaches(); + if (breaches.breaches.Any()) + { + Assert.Fail(string.Join(",", breaches.breaches.Select(b => FormatBreach(n, b)))); + } + } + + private string FormatBreach(CodexAccess n, string breach) + { + return $"{n.Container.Name} = '{breach}'"; + } + + private CodexAccess[] CreateAccessToAllNodes() + { + // Normally, a continuous test accesses only a subset of the nodes in the deployment. + // This time, we want to check all of them. + var factory = new CodexAccessFactory(); + var allContainers = Configuration.CodexDeployment.CodexContainers; + return factory.Create(Configuration, allContainers, Log, new DefaultTimeSet()); + } + } +} diff --git a/DistTestCore/Codex/CodexAccess.cs b/DistTestCore/Codex/CodexAccess.cs index 0ec046c..fde9997 100644 --- a/DistTestCore/Codex/CodexAccess.cs +++ b/DistTestCore/Codex/CodexAccess.cs @@ -53,6 +53,11 @@ namespace DistTestCore.Codex return Http().HttpGetJson("debug/futures").futures; } + public CodexDebugThresholdBreaches GetDebugThresholdBreaches() + { + return Http().HttpGetJson("debug/loop"); + } + public string UploadFile(FileStream fileStream) { return Http().HttpPostStream("upload", fileStream); diff --git a/DistTestCore/Codex/CodexApiTypes.cs b/DistTestCore/Codex/CodexApiTypes.cs index 835df10..cde05fd 100644 --- a/DistTestCore/Codex/CodexApiTypes.cs +++ b/DistTestCore/Codex/CodexApiTypes.cs @@ -76,6 +76,11 @@ namespace DistTestCore.Codex public string address { get; set; } = string.Empty; } + public class CodexDebugThresholdBreaches + { + public string[] breaches { get; set; } = Array.Empty(); + } + public class CodexSalesAvailabilityRequest { public string size { get; set; } = string.Empty; From 7d856c276b515e77bd8434cb924fb62f6b874afc Mon Sep 17 00:00:00 2001 From: benbierens Date: Tue, 11 Jul 2023 08:19:14 +0200 Subject: [PATCH 29/49] Only display new threshold breaches --- ContinuousTests/ContinuousTest.cs | 7 +++---- ContinuousTests/Tests/ThresholdChecks.cs | 19 ++++++++++++++++++- DistTestCore/OnlineCodexNode.cs | 7 +++---- 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/ContinuousTests/ContinuousTest.cs b/ContinuousTests/ContinuousTest.cs index 80b914e..6818cc4 100644 --- a/ContinuousTests/ContinuousTest.cs +++ b/ContinuousTests/ContinuousTest.cs @@ -71,10 +71,9 @@ namespace ContinuousTests return node.UploadFile(fileStream); }); - if (response.StartsWith(UploadFailedMessage)) - { - return null; - } + if (string.IsNullOrEmpty(response)) return null; + if (response.StartsWith(UploadFailedMessage)) return null; + Log.Log($"Uploaded file. Received contentId: '{response}'."); return new ContentId(response); } diff --git a/ContinuousTests/Tests/ThresholdChecks.cs b/ContinuousTests/Tests/ThresholdChecks.cs index f1b8167..6ed6ea2 100644 --- a/ContinuousTests/Tests/ThresholdChecks.cs +++ b/ContinuousTests/Tests/ThresholdChecks.cs @@ -10,6 +10,8 @@ namespace ContinuousTests.Tests public override TimeSpan RunTestEvery => TimeSpan.FromSeconds(30); public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure; + private static readonly List previousBreaches = new List(); + [TestMoment(t: 0)] public void CheckAllThresholds() { @@ -22,7 +24,22 @@ namespace ContinuousTests.Tests var breaches = n.GetDebugThresholdBreaches(); if (breaches.breaches.Any()) { - Assert.Fail(string.Join(",", breaches.breaches.Select(b => FormatBreach(n, b)))); + var newBreaches = new List(); + foreach (var b in breaches.breaches) + { + if (!previousBreaches.Contains(b)) + { + newBreaches.Add(b); + previousBreaches.Add(b); + } + } + + if (newBreaches.Any()) + { + Assert.Fail(string.Join(",", newBreaches.Select(b => FormatBreach(n, b)))); + + Program.Cancellation.Cts.Cancel(); + } } } diff --git a/DistTestCore/OnlineCodexNode.cs b/DistTestCore/OnlineCodexNode.cs index 5a66fce..cd37fb4 100644 --- a/DistTestCore/OnlineCodexNode.cs +++ b/DistTestCore/OnlineCodexNode.cs @@ -75,10 +75,9 @@ namespace DistTestCore return CodexAccess.UploadFile(fileStream); }); - if (response.StartsWith(UploadFailedMessage)) - { - Assert.Fail("Node failed to store block."); - } + if (string.IsNullOrEmpty(response)) Assert.Fail("Received empty response."); + if (response.StartsWith(UploadFailedMessage)) Assert.Fail("Node failed to store block."); + var logReplacement = $"(CID:{file.Describe()})"; Log($"ContentId '{response}' is {logReplacement}"); lifecycle.Log.AddStringReplace(response, logReplacement); From cccf9e426c6acd7b0c39105bbcf2fa2a1b661008 Mon Sep 17 00:00:00 2001 From: benbierens Date: Tue, 11 Jul 2023 10:05:00 +0200 Subject: [PATCH 30/49] Adds deploy confirm interaction --- CodexNetDeployer/Program.cs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CodexNetDeployer/Program.cs b/CodexNetDeployer/Program.cs index 4ac3a2f..13a4fd9 100644 --- a/CodexNetDeployer/Program.cs +++ b/CodexNetDeployer/Program.cs @@ -35,6 +35,13 @@ public class Program return; } + if (!args.Any(a => a == "-y")) + { + Console.WriteLine("Does the above config look good? [y/n]"); + if (Console.ReadLine()!.ToLowerInvariant() != "y") return; + Console.WriteLine("I think so too."); + } + var deployer = new Deployer(config); var deployment = deployer.Deploy(); From 76dfd7a86ca4c06cf92069b1a847b786e4bb1553 Mon Sep 17 00:00:00 2001 From: benbierens Date: Tue, 11 Jul 2023 10:10:47 +0200 Subject: [PATCH 31/49] Support for boolean uniform-args --- ArgsUniform/ArgsUniform.cs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/ArgsUniform/ArgsUniform.cs b/ArgsUniform/ArgsUniform.cs index 7d58cf1..a53eee6 100644 --- a/ArgsUniform/ArgsUniform.cs +++ b/ArgsUniform/ArgsUniform.cs @@ -194,12 +194,21 @@ namespace ArgsUniform { if (uniformProperty.PropertyType == typeof(int?)) return AssignOptionalInt(result, uniformProperty, value); if (uniformProperty.PropertyType.IsEnum) return AssignEnum(result, uniformProperty, value); + if (uniformProperty.PropertyType == typeof(bool)) return AssignBool(result, uniformProperty, value); throw new NotSupportedException(); } } } + private bool AssignBool(T result, PropertyInfo uniformProperty, object value) + { + var s = value.ToString()!.ToLowerInvariant(); + var isTrue = (s == "1" || s == "true"); + uniformProperty.SetValue(result, isTrue); + return true; + } + private static bool AssignEnum(T result, PropertyInfo uniformProperty, object value) { var s = value.ToString(); From 9778ef51a7ea9a1fb91291b70092c2c17b7afc61 Mon Sep 17 00:00:00 2001 From: benbierens Date: Tue, 11 Jul 2023 10:59:41 +0200 Subject: [PATCH 32/49] Loops in metrics deployment --- CodexNetDeployer/CodexNodeStarter.cs | 2 +- CodexNetDeployer/Configuration.cs | 3 +++ CodexNetDeployer/Deployer.cs | 13 +++++++++++++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/CodexNetDeployer/CodexNodeStarter.cs b/CodexNetDeployer/CodexNodeStarter.cs index 7c33892..2896ea7 100644 --- a/CodexNetDeployer/CodexNodeStarter.cs +++ b/CodexNetDeployer/CodexNodeStarter.cs @@ -20,7 +20,7 @@ namespace CodexNetDeployer this.workflowCreator = workflowCreator; this.lifecycle = lifecycle; this.gethResult = gethResult; - this.validatorsLeft = numberOfValidators; + validatorsLeft = numberOfValidators; } public RunningContainer? Start(int i) diff --git a/CodexNetDeployer/Configuration.cs b/CodexNetDeployer/Configuration.cs index d58e233..4d4c735 100644 --- a/CodexNetDeployer/Configuration.cs +++ b/CodexNetDeployer/Configuration.cs @@ -46,6 +46,9 @@ namespace CodexNetDeployer [Uniform("max-duration", "md", "MAXDURATION", true, "Maximum duration in seconds for contracts which will be accepted.")] public int MaxDuration { get; set; } + [Uniform("record-metrics", "rm", "RECORDMETRICS", false, "If true, metrics will be collected for all Codex nodes.")] + public bool RecordMetrics { get; set; } = false; + public TestRunnerLocation RunnerLocation { get; set; } = TestRunnerLocation.InternalToCluster; public List Validate() diff --git a/CodexNetDeployer/Deployer.cs b/CodexNetDeployer/Deployer.cs index b4efd3f..64e46aa 100644 --- a/CodexNetDeployer/Deployer.cs +++ b/CodexNetDeployer/Deployer.cs @@ -27,6 +27,7 @@ namespace CodexNetDeployer // We trick the Geth companion node into unlocking all of its accounts, by saying we want to start 999 codex nodes. var setup = new CodexSetup(999, config.CodexLogLevel); setup.WithStorageQuota(config.StorageQuota!.Value.MB()).EnableMarketplace(0.TestTokens()); + setup.MetricsEnabled = config.RecordMetrics; Log("Creating Geth instance and deploying contracts..."); var gethStarter = new GethStarter(lifecycle, workflowCreator); @@ -46,6 +47,11 @@ namespace CodexNetDeployer if (container != null) codexContainers.Add(container); } + if (setup.MetricsEnabled) + { + StartMetricsService(lifecycle, setup, codexContainers); + } + return new CodexDeployment(gethResults, codexContainers.ToArray(), CreateMetadata()); } @@ -75,6 +81,13 @@ namespace CodexNetDeployer return (workflowCreator, lifecycle); } + private void StartMetricsService(TestLifecycle lifecycle, CodexSetup setup, List codexContainers) + { + Log("Starting metrics service..."); + var runningContainers = new RunningContainers(null!, null!, codexContainers.ToArray()); + lifecycle.PrometheusStarter.CollectMetricsFor(setup, runningContainers); + } + private string? GetKubeConfig(string kubeConfigFile) { if (string.IsNullOrEmpty(kubeConfigFile) || kubeConfigFile.ToLowerInvariant() == "null") return null; From 17935f4c9e7749ec240544dddb6279d51758065e Mon Sep 17 00:00:00 2001 From: benbierens Date: Tue, 11 Jul 2023 12:21:48 +0200 Subject: [PATCH 33/49] Adds metrics container to deployment json --- CodexNetDeployer/CodexNodeStarter.cs | 2 ++ CodexNetDeployer/Configuration.cs | 4 ++++ CodexNetDeployer/Deployer.cs | 14 +++++++------- DistTestCore/Codex/CodexDeployment.cs | 8 ++++++-- DistTestCore/CodexStarter.cs | 11 ++++++++++- DistTestCore/PrometheusStarter.cs | 6 ++---- 6 files changed, 31 insertions(+), 14 deletions(-) diff --git a/CodexNetDeployer/CodexNodeStarter.cs b/CodexNetDeployer/CodexNodeStarter.cs index 65db0e7..9478a98 100644 --- a/CodexNetDeployer/CodexNodeStarter.cs +++ b/CodexNetDeployer/CodexNodeStarter.cs @@ -86,6 +86,8 @@ namespace CodexNetDeployer var marketplaceConfig = new MarketplaceInitialConfig(100000.Eth(), 0.TestTokens(), validatorsLeft > 0); marketplaceConfig.AccountIndexOverride = i; codexStart.MarketplaceConfig = marketplaceConfig; + codexStart.MetricsEnabled = config.RecordMetrics; + if (config.BlockTTL != Configuration.SecondsIn1Day) { codexStart.BlockTTL = config.BlockTTL; diff --git a/CodexNetDeployer/Configuration.cs b/CodexNetDeployer/Configuration.cs index 89fce7e..f3cc793 100644 --- a/CodexNetDeployer/Configuration.cs +++ b/CodexNetDeployer/Configuration.cs @@ -2,6 +2,7 @@ using DistTestCore; using DistTestCore.Codex; using DistTestCore.Marketplace; +using DistTestCore.Metrics; namespace CodexNetDeployer { @@ -18,6 +19,9 @@ namespace CodexNetDeployer [Uniform("contracts-image", "oi", "CONTRACTSIMAGE", true, "Docker image of Codex Contracts.")] public string ContractsImage { get; set; } = CodexContractsContainerRecipe.DockerImage; + [Uniform("metrics-image", "mi", "METRICSIMAGE", true, "Docker image of Prometheus.")] + public string MetricsImage { get; set; } = PrometheusContainerRecipe.DockerImage; + [Uniform("kube-config", "kc", "KUBECONFIG", false, "Path to Kubeconfig file. Use 'null' (default) to use local cluster.")] public string KubeConfigFile { get; set; } = "null"; diff --git a/CodexNetDeployer/Deployer.cs b/CodexNetDeployer/Deployer.cs index 04461e9..04c39d1 100644 --- a/CodexNetDeployer/Deployer.cs +++ b/CodexNetDeployer/Deployer.cs @@ -52,12 +52,9 @@ namespace CodexNetDeployer if (container != null) codexContainers.Add(container); } - if (setup.MetricsEnabled) - { - StartMetricsService(lifecycle, setup, codexContainers); - } + var prometheusContainer = StartMetricsService(lifecycle, setup, codexContainers); - return new CodexDeployment(gethResults, codexContainers.ToArray(), CreateMetadata()); + return new CodexDeployment(gethResults, codexContainers.ToArray(), prometheusContainer, CreateMetadata()); } private (WorkflowCreator, TestLifecycle) CreateFacilities() @@ -86,11 +83,13 @@ namespace CodexNetDeployer return (workflowCreator, lifecycle); } - private void StartMetricsService(TestLifecycle lifecycle, CodexSetup setup, List codexContainers) + private RunningContainer? StartMetricsService(TestLifecycle lifecycle, CodexSetup setup, List codexContainers) { + if (!setup.MetricsEnabled) return null; + Log("Starting metrics service..."); var runningContainers = new RunningContainers(null!, null!, codexContainers.ToArray()); - lifecycle.PrometheusStarter.CollectMetricsFor(setup, runningContainers); + return lifecycle.PrometheusStarter.CollectMetricsFor(runningContainers).Containers.Single(); } private string? GetKubeConfig(string kubeConfigFile) @@ -105,6 +104,7 @@ namespace CodexNetDeployer codexImage: config.CodexImage, gethImage: config.GethImage, contractsImage: config.ContractsImage, + prometheusImage: config.MetricsImage, kubeNamespace: config.KubeNamespace, numberOfCodexNodes: config.NumberOfCodexNodes!.Value, numberOfValidators: config.NumberOfValidators!.Value, diff --git a/DistTestCore/Codex/CodexDeployment.cs b/DistTestCore/Codex/CodexDeployment.cs index 37db831..39aa689 100644 --- a/DistTestCore/Codex/CodexDeployment.cs +++ b/DistTestCore/Codex/CodexDeployment.cs @@ -5,26 +5,29 @@ namespace DistTestCore.Codex { public class CodexDeployment { - public CodexDeployment(GethStartResult gethStartResult, RunningContainer[] codexContainers, DeploymentMetadata metadata) + public CodexDeployment(GethStartResult gethStartResult, RunningContainer[] codexContainers, RunningContainer? prometheusContainer, DeploymentMetadata metadata) { GethStartResult = gethStartResult; CodexContainers = codexContainers; + PrometheusContainer = prometheusContainer; Metadata = metadata; } public GethStartResult GethStartResult { get; } public RunningContainer[] CodexContainers { get; } + public RunningContainer? PrometheusContainer { get; } public DeploymentMetadata Metadata { get; } } public class DeploymentMetadata { - public DeploymentMetadata(string codexImage, string gethImage, string contractsImage, string kubeNamespace, int numberOfCodexNodes, int numberOfValidators, int storageQuotaMB, CodexLogLevel codexLogLevel, int initialTestTokens, int minPrice, int maxCollateral, int maxDuration) + public DeploymentMetadata(string codexImage, string gethImage, string contractsImage, string prometheusImage, string kubeNamespace, int numberOfCodexNodes, int numberOfValidators, int storageQuotaMB, CodexLogLevel codexLogLevel, int initialTestTokens, int minPrice, int maxCollateral, int maxDuration) { DeployDateTimeUtc = DateTime.UtcNow; CodexImage = codexImage; GethImage = gethImage; ContractsImage = contractsImage; + PrometheusImage = prometheusImage; KubeNamespace = kubeNamespace; NumberOfCodexNodes = numberOfCodexNodes; NumberOfValidators = numberOfValidators; @@ -40,6 +43,7 @@ namespace DistTestCore.Codex public DateTime DeployDateTimeUtc { get; } public string GethImage { get; } public string ContractsImage { get; } + public string PrometheusImage { get; } public string KubeNamespace { get; } public int NumberOfCodexNodes { get; } public int NumberOfValidators { get; } diff --git a/DistTestCore/CodexStarter.cs b/DistTestCore/CodexStarter.cs index 627bfb9..144d55f 100644 --- a/DistTestCore/CodexStarter.cs +++ b/DistTestCore/CodexStarter.cs @@ -1,5 +1,6 @@ using DistTestCore.Codex; using DistTestCore.Marketplace; +using DistTestCore.Metrics; using KubernetesWorkflow; using Logging; @@ -23,7 +24,7 @@ namespace DistTestCore var startupConfig = CreateStartupConfig(gethStartResult, codexSetup); var containers = StartCodexContainers(startupConfig, codexSetup.NumberOfNodes, codexSetup.Location); - var metricAccessFactory = lifecycle.PrometheusStarter.CollectMetricsFor(codexSetup, containers); + var metricAccessFactory = CollectMetrics(codexSetup, containers); var codexNodeFactory = new CodexNodeFactory(lifecycle, metricAccessFactory, gethStartResult.MarketplaceAccessFactory); @@ -57,6 +58,14 @@ namespace DistTestCore workflow.DownloadContainerLog(container, logHandler); } + private IMetricsAccessFactory CollectMetrics(CodexSetup codexSetup, RunningContainers containers) + { + if (!codexSetup.MetricsEnabled) return new MetricsUnavailableAccessFactory(); + + var runningContainers = lifecycle.PrometheusStarter.CollectMetricsFor(containers); + return new CodexNodeMetricsAccessFactory(lifecycle, runningContainers); + } + private StartupConfig CreateStartupConfig(GethStartResult gethStartResult, CodexSetup codexSetup) { var startupConfig = new StartupConfig(); diff --git a/DistTestCore/PrometheusStarter.cs b/DistTestCore/PrometheusStarter.cs index 64edae9..a0fc9a4 100644 --- a/DistTestCore/PrometheusStarter.cs +++ b/DistTestCore/PrometheusStarter.cs @@ -12,10 +12,8 @@ namespace DistTestCore { } - public IMetricsAccessFactory CollectMetricsFor(CodexSetup codexSetup, RunningContainers containers) + public RunningContainers CollectMetricsFor(RunningContainers containers) { - if (!codexSetup.MetricsEnabled) return new MetricsUnavailableAccessFactory(); - LogStart($"Starting metrics server for {containers.Describe()}"); var startupConfig = new StartupConfig(); startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(containers.Containers))); @@ -26,7 +24,7 @@ namespace DistTestCore LogEnd("Metrics server started."); - return new CodexNodeMetricsAccessFactory(lifecycle, runningContainers); + return runningContainers; } private string GeneratePrometheusConfig(RunningContainer[] nodes) From 42acb862b68cc83f672d0c8e4fb6c0f3c77a2e00 Mon Sep 17 00:00:00 2001 From: benbierens Date: Tue, 11 Jul 2023 14:56:40 +0200 Subject: [PATCH 34/49] Removes docker images from config because they aren't configurable this way. --- CodexNetDeployer/Configuration.cs | 20 +++----------------- CodexNetDeployer/Deployer.cs | 4 ---- DistTestCore/Codex/CodexDeployment.cs | 10 +--------- 3 files changed, 4 insertions(+), 30 deletions(-) diff --git a/CodexNetDeployer/Configuration.cs b/CodexNetDeployer/Configuration.cs index f3cc793..b030618 100644 --- a/CodexNetDeployer/Configuration.cs +++ b/CodexNetDeployer/Configuration.cs @@ -1,8 +1,6 @@ using ArgsUniform; using DistTestCore; using DistTestCore.Codex; -using DistTestCore.Marketplace; -using DistTestCore.Metrics; namespace CodexNetDeployer { @@ -10,18 +8,6 @@ namespace CodexNetDeployer { public const int SecondsIn1Day = 24 * 60 * 60; - [Uniform("codex-image", "ci", "CODEXIMAGE", true, "Docker image of Codex.")] - public string CodexImage { get; set; } = CodexContainerRecipe.DockerImage; - - [Uniform("geth-image", "gi", "GETHIMAGE", true, "Docker image of Geth.")] - public string GethImage { get; set; } = GethContainerRecipe.DockerImage; - - [Uniform("contracts-image", "oi", "CONTRACTSIMAGE", true, "Docker image of Codex Contracts.")] - public string ContractsImage { get; set; } = CodexContractsContainerRecipe.DockerImage; - - [Uniform("metrics-image", "mi", "METRICSIMAGE", true, "Docker image of Prometheus.")] - public string MetricsImage { get; set; } = PrometheusContainerRecipe.DockerImage; - [Uniform("kube-config", "kc", "KUBECONFIG", false, "Path to Kubeconfig file. Use 'null' (default) to use local cluster.")] public string KubeConfigFile { get; set; } = "null"; @@ -55,12 +41,12 @@ namespace CodexNetDeployer [Uniform("max-duration", "md", "MAXDURATION", true, "Maximum duration in seconds for contracts which will be accepted.")] public int MaxDuration { get; set; } - [Uniform("record-metrics", "rm", "RECORDMETRICS", false, "If true, metrics will be collected for all Codex nodes.")] - public bool RecordMetrics { get; set; } = false; - [Uniform("block-ttl", "bt", "BLOCKTTL", false, "Block timeout in seconds. Default is 24 hours.")] public int BlockTTL { get; set; } = SecondsIn1Day; + [Uniform("record-metrics", "rm", "RECORDMETRICS", false, "If true, metrics will be collected for all Codex nodes.")] + public bool RecordMetrics { get; set; } = false; + public TestRunnerLocation RunnerLocation { get; set; } = TestRunnerLocation.InternalToCluster; public List Validate() diff --git a/CodexNetDeployer/Deployer.cs b/CodexNetDeployer/Deployer.cs index 04c39d1..e7180a6 100644 --- a/CodexNetDeployer/Deployer.cs +++ b/CodexNetDeployer/Deployer.cs @@ -101,10 +101,6 @@ namespace CodexNetDeployer private DeploymentMetadata CreateMetadata() { return new DeploymentMetadata( - codexImage: config.CodexImage, - gethImage: config.GethImage, - contractsImage: config.ContractsImage, - prometheusImage: config.MetricsImage, kubeNamespace: config.KubeNamespace, numberOfCodexNodes: config.NumberOfCodexNodes!.Value, numberOfValidators: config.NumberOfValidators!.Value, diff --git a/DistTestCore/Codex/CodexDeployment.cs b/DistTestCore/Codex/CodexDeployment.cs index 39aa689..52192dc 100644 --- a/DistTestCore/Codex/CodexDeployment.cs +++ b/DistTestCore/Codex/CodexDeployment.cs @@ -21,13 +21,9 @@ namespace DistTestCore.Codex public class DeploymentMetadata { - public DeploymentMetadata(string codexImage, string gethImage, string contractsImage, string prometheusImage, string kubeNamespace, int numberOfCodexNodes, int numberOfValidators, int storageQuotaMB, CodexLogLevel codexLogLevel, int initialTestTokens, int minPrice, int maxCollateral, int maxDuration) + public DeploymentMetadata(string kubeNamespace, int numberOfCodexNodes, int numberOfValidators, int storageQuotaMB, CodexLogLevel codexLogLevel, int initialTestTokens, int minPrice, int maxCollateral, int maxDuration) { DeployDateTimeUtc = DateTime.UtcNow; - CodexImage = codexImage; - GethImage = gethImage; - ContractsImage = contractsImage; - PrometheusImage = prometheusImage; KubeNamespace = kubeNamespace; NumberOfCodexNodes = numberOfCodexNodes; NumberOfValidators = numberOfValidators; @@ -39,11 +35,7 @@ namespace DistTestCore.Codex MaxDuration = maxDuration; } - public string CodexImage { get; } public DateTime DeployDateTimeUtc { get; } - public string GethImage { get; } - public string ContractsImage { get; } - public string PrometheusImage { get; } public string KubeNamespace { get; } public int NumberOfCodexNodes { get; } public int NumberOfValidators { get; } From 671ee4ea624f837e5197f0743dba421471d3d057 Mon Sep 17 00:00:00 2001 From: Slava <20563034+veaceslavdoina@users.noreply.github.com> Date: Tue, 11 Jul 2023 19:01:42 +0300 Subject: [PATCH 35/49] Update Docker workflow (#28) --- .github/workflows/docker.yml | 27 +-------------------------- 1 file changed, 1 insertion(+), 26 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 5bd7114..5b7e459 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -78,31 +78,7 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Docker - Build and export to Docker - uses: docker/build-push-action@v4 - with: - context: . - file: ${{ env.DOCKER_FILE }} - platforms: ${{ env.PLATFORM }} - load: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - - - name: Docker - Minify image - uses: kitabisa/docker-slim-action@v1 - id: slim - env: - DSLIM_HTTP_PROBE: false - with: - target: ${{ steps.meta.outputs.tags }} - overwrite: true - - - name: Docker - Show slim report - run: echo "${REPORT}" | jq -r - env: - REPORT: ${{ steps.slim.outputs.report }} - - - name: Docker - Push to Docker registry + - name: Docker - Build and Push uses: docker/build-push-action@v4 with: context: . @@ -145,4 +121,3 @@ jobs: inputs: ${{ env.TAGS }} images: ${{ needs.build.outputs.tags-linux-amd64 }},${{ needs.build.outputs.tags-linux-arm64 }} push: true - From 1f2635ea7d3ac74f75cb5b90c5af6cfada80ca08 Mon Sep 17 00:00:00 2001 From: Slava <20563034+veaceslavdoina@users.noreply.github.com> Date: Tue, 11 Jul 2023 21:54:58 +0300 Subject: [PATCH 36/49] Update docker docs (#29) * Update Docker docs * Update example manifest --- docker/README.md | 183 +++++++++++++++++++++++++++++++++-------------- 1 file changed, 131 insertions(+), 52 deletions(-) diff --git a/docker/README.md b/docker/README.md index 020d028..c0d9657 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,59 +1,138 @@ # Run tests with Docker in Kubernetes -We may [run tests localy](../LOCALSETUP.MD) using installed Dotnet and inside Kubernetes we may use a [prepared Docker images](https://hub.docker.com/r/codexstorage/cs-codex-dist-tests/tags). + We may [run tests on local](../LOCALSETUP.MD) or remote Kubernetes cluster. Local cluster flow uses direct access to nodes ports and this is why we introduced a different way to check services ports, for more information please see [Tests run modes](../../../issues/20). Configuration option `RUNNERLOCATION` is responsible for that. -Custom [entrypoint](docker-entrypoint.sh) will do the following + For local run it is easier to install .Net and run tests on Docker Desktop Kubernetes cluster. In case of remote run we do not expose services via Ingress Controller and we can't access cluster nodes, this is why we should run tests only inside the Kubernetes. + + We can run tests on remote cluster in the following ways + +#### Run pod inside the cluster using generic .Net image + +
+steps + +1. Create dist-tests-runner.yaml + ```yaml + -- + apiVersion: v1 + kind: Pod + metadata: + name: dist-tests-runner + namespace: default + spec: + containers: + - name: dotnet + image: mcr.microsoft.com/dotnet/sdk:7.0 + command: ["sleep", "infinity"] + ``` +2. Deploy pod in the cluster + ```shell + kubectl apply -f dist-tests-runner.yaml + ``` + +3. Copy kubeconfig to the pod + ```shell + kubectl cp kubeconfig.yaml dist-tests-runner:/opt + ``` + +4. Exec into the pod via kubectl or [OpenLens](https://github.com/MuhammedKalkan/OpenLens) + ```shell + kubectl exec -it dist-tests-runner -- bash + ``` + +5. Clone repository inside the pod + ```shell + git clone https://github.com/codex-storage/cs-codex-dist-tests.git + ``` + +6. Update kubeconfig option in config file + ```shell + cd cs-codex-dist-tests + vi DistTestCore/Configuration.cs + ``` + ```dotnet + GetNullableEnvVarOrDefault("KUBECONFIG", "/opt/kubeconfig.yaml") + ``` + +7. Run tests + ```shell + dotnet test Tests + ``` + +8. Check the results and analyze the logs +
+ +#### Run pod inside the cluster using [prepared Docker images](https://hub.docker.com/r/codexstorage/cs-codex-dist-tests/tags) + + Before the run we should create some objects inside the cluster + 1. Namespace where we will run the image + 2. [Service Account to run tests inside the cluster](https://github.com/codex-storage/cs-codex-dist-tests/issues/21) + 3. Secret with kubeconfig for created SA + 4. Configmap with custom app config if required + + For more information please see [Manual run inside Kubernetes via Job](../../../issues/7) + + Then we need to create a manifest to run the pod +
+ runner.yaml + + ```yaml + --- + apiVersion: v1 + kind: Pod + metadata: + name: dist-tests-runner + namespace: cs-codex-dist-tests + labels: + name: cs-codex-dist-tests + spec: + containers: + - name: cs-codex-dist-tests + image: codexstorage/cs-codex-dist-tests:sha-671ee4e + env: + - name: RUNNERLOCATION + value: InternalToCluster + - name: KUBECONFIG + value: /opt/kubeconfig.yaml + - name: CONFIG + value: "/opt/Configuration.cs" + - name: CONFIG_SHOW + value: "true" + volumeMounts: + - name: kubeconfig + mountPath: /opt/kubeconfig.yaml + subPath: kubeconfig.yaml + - name: config + mountPath: /opt/Configuration.cs + subPath: Configuration.cs + - name: logs + mountPath: /var/log/cs-codex-dist-tests + # command: + # - "dotnet" + # - "test" + # - "Tests" + restartPolicy: Never + volumes: + - name: kubeconfig + secret: + secretName: cs-codex-dist-tests-app-kubeconfig + - name: config + configMap: + name: cs-codex-dist-tests + - name: logs + hostPath: + path: /var/log/cs-codex-dist-tests + ``` + For more information about pod variables please see [job.yaml](job.yaml). +
+ + And then apply it + ```shell + kubectl apply -f runner.yaml + ``` + + After the pod run, custom [entrypoint](docker-entrypoint.sh) will do the following 1. Clone repository 2. Switch to the specific branch - `master` by default 3. Run all tests - `dotnet test` - -**Run with defaults** -```bash -docker run \ - --rm \ - --name cs-codex-dist-tests \ - codexstorage/cs-codex-dist-tests:sha-686757e -``` - -**Just short tests** -```bash -docker run \ - --rm \ - --name cs-codex-dist-tests \ - codexstorage/cs-codex-dist-tests:sha-686757e \ - dotnet test Tests -``` - -**Custom branch** -```bash -docker run \ - --rm \ - --name cs-codex-dist-tests \ - --env BRANCH=feature/tests \ - codexstorage/cs-codex-dist-tests:sha-686757e -``` - -**Custom local config** -```bash -docker run \ - --rm \ - --name cs-codex-dist-tests \ - --env CONFIG=/opt/Configuration.cs \ - --env CONFIG_SHOW=true \ - --volume $PWD/DistTestCore/Configuration.cs:/opt/Configuration.cs \ - codexstorage/cs-codex-dist-tests:sha-686757e -``` - -**Local kubeconfig with custom local config** -```bash -docker run \ - --rm \ - --name cs-codex-dist-tests \ - --env CONFIG=/opt/Configuration.cs \ - --env CONFIG_SHOW=true \ - --env SOURCE=https://github.com/codex-storage/cs-codex-dist-tests.git \ - --volume $PWD/DistTestCore/Configuration.cs:/opt/Configuration.cs \ - --volume $PWD/kubeconfig.yml:/opt/kubeconfig.yml \ - codexstorage/cs-codex-dist-tests:sha-686757e -``` From 11502717c38ba2fcb285d35537904727e5c14149 Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 12 Jul 2023 14:53:27 +0200 Subject: [PATCH 37/49] Fixes hexint interface to marketplace API --- DistTestCore/Marketplace/MarketplaceAccess.cs | 33 ++++++++++--------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/DistTestCore/Marketplace/MarketplaceAccess.cs b/DistTestCore/Marketplace/MarketplaceAccess.cs index 858717d..8a67564 100644 --- a/DistTestCore/Marketplace/MarketplaceAccess.cs +++ b/DistTestCore/Marketplace/MarketplaceAccess.cs @@ -34,10 +34,10 @@ namespace DistTestCore.Marketplace { var request = new CodexSalesRequestStorageRequest { - duration = ToHexBigInt(duration.TotalSeconds), - proofProbability = ToHexBigInt(proofProbability), - reward = ToHexBigInt(pricePerSlotPerSecond), - collateral = ToHexBigInt(requiredCollateral), + duration = ToDecInt(duration.TotalSeconds), + proofProbability = ToDecInt(proofProbability), + reward = ToDecInt(pricePerSlotPerSecond), + collateral = ToDecInt(requiredCollateral), expiry = null, nodes = minRequiredNumberOfNodes, tolerance = null, @@ -62,19 +62,19 @@ namespace DistTestCore.Marketplace return response; } - public string MakeStorageAvailable(ByteSize totalSpace, TestToken minPriceForTotalSpace, TestToken maxCollateral, TimeSpan maxDuration) + public string MakeStorageAvailable(ByteSize size, TestToken minPricePerBytePerSecond, TestToken maxCollateral, TimeSpan maxDuration) { var request = new CodexSalesAvailabilityRequest { - size = ToHexBigInt(totalSpace.SizeInBytes), - duration = ToHexBigInt(maxDuration.TotalSeconds), - maxCollateral = ToHexBigInt(maxCollateral), - minPrice = ToHexBigInt(minPriceForTotalSpace) + size = ToDecInt(size.SizeInBytes), + duration = ToDecInt(maxDuration.TotalSeconds), + maxCollateral = ToDecInt(maxCollateral), + minPrice = ToDecInt(minPricePerBytePerSecond) }; Log($"Making storage available... (" + - $"size: {totalSpace}, " + - $"minPricePerBytePerSecond: {minPriceForTotalSpace}, " + + $"size: {size}, " + + $"minPricePerBytePerSecond: {minPricePerBytePerSecond}, " + $"maxCollateral: {maxCollateral}, " + $"maxDuration: {Time.FormatDuration(maxDuration)})"); @@ -85,15 +85,16 @@ namespace DistTestCore.Marketplace return response.id; } - private string ToHexBigInt(double d) + private string ToDecInt(double d) { - return "0x" + string.Format("{0:X}", Convert.ToInt64(d)); + var i = new BigInteger(d); + return i.ToString("D"); } - public string ToHexBigInt(TestToken t) + public string ToDecInt(TestToken t) { - var bigInt = new BigInteger(t.Amount); - return "0x" + bigInt.ToString("X"); + var i = new BigInteger(t.Amount); + return i.ToString("D"); } public void AssertThatBalance(IResolveConstraint constraint, string message = "") From 1f8fc52cb1265abb7d30f6433e8bc6606d6d69a4 Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 13 Jul 2023 09:20:24 +0200 Subject: [PATCH 38/49] Adds three-client test to check discovery relaying --- ContinuousTests/StartupChecker.cs | 8 +++++--- Tests/BasicTests/ThreeClientTest.cs | 24 ++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 3 deletions(-) create mode 100644 Tests/BasicTests/ThreeClientTest.cs diff --git a/ContinuousTests/StartupChecker.cs b/ContinuousTests/StartupChecker.cs index 8e9357a..a2e1e1f 100644 --- a/ContinuousTests/StartupChecker.cs +++ b/ContinuousTests/StartupChecker.cs @@ -64,9 +64,9 @@ namespace ContinuousTests { cancelToken.ThrowIfCancellationRequested(); - log.Log($"Checking '{n.Address.Host}'..."); + log.Log($"Checking {n.Container.Name} @ '{n.Address.Host}:{n.Address.Port}'..."); - if (EnsureOnline(n)) + if (EnsureOnline(log, n)) { log.Log("OK"); } @@ -82,12 +82,14 @@ namespace ContinuousTests } } - private bool EnsureOnline(CodexAccess n) + private bool EnsureOnline(BaseLog log, CodexAccess n) { try { var info = n.GetDebugInfo(); if (info == null || string.IsNullOrEmpty(info.id)) return false; + + log.Log($"Codex version: '{info.codex.version}' revision: '{info.codex.revision}'"); } catch { diff --git a/Tests/BasicTests/ThreeClientTest.cs b/Tests/BasicTests/ThreeClientTest.cs new file mode 100644 index 0000000..78ef25e --- /dev/null +++ b/Tests/BasicTests/ThreeClientTest.cs @@ -0,0 +1,24 @@ +using DistTestCore; +using NUnit.Framework; + +namespace Tests.BasicTests +{ + [TestFixture] + public class ThreeClientTest : AutoBootstrapDistTest + { + [Test] + public void ThreeClient() + { + var primary = SetupCodexNode(); + var secondary = SetupCodexNode(); + + var testFile = GenerateTestFile(10.MB()); + + var contentId = primary.UploadFile(testFile); + + var downloadedFile = secondary.DownloadContent(contentId); + + testFile.AssertIsEqual(downloadedFile); + } + } +} From 61099218490950e6f95afe1921fcb9db32761ccc Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 13 Jul 2023 10:05:40 +0200 Subject: [PATCH 39/49] Enables auto-NAT option --- DistTestCore/Codex/CodexContainerRecipe.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DistTestCore/Codex/CodexContainerRecipe.cs b/DistTestCore/Codex/CodexContainerRecipe.cs index 64f2177..480503b 100644 --- a/DistTestCore/Codex/CodexContainerRecipe.cs +++ b/DistTestCore/Codex/CodexContainerRecipe.cs @@ -33,7 +33,7 @@ namespace DistTestCore.Codex AddEnvVar("CODEX_LOG_LEVEL", config.LogLevel.ToString()!.ToUpperInvariant()); // This makes the node announce itself to its local (pod) IP address. - AddEnvVar("CODEX_NAT_ADDR", "$(hostname --ip-address)"); + AddEnvVar("NAT_IP_AUTO", "true"); var listenPort = AddInternalPort(); AddEnvVar("CODEX_LISTEN_ADDRS", $"/ip4/0.0.0.0/tcp/{listenPort.Number}"); From 2de7fc7f2083adff6ba34575fd3f461d7ed84825 Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 13 Jul 2023 10:36:43 +0200 Subject: [PATCH 40/49] Bumps to latest official codex image. --- DistTestCore/Codex/CodexContainerRecipe.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DistTestCore/Codex/CodexContainerRecipe.cs b/DistTestCore/Codex/CodexContainerRecipe.cs index 480503b..b94e5ba 100644 --- a/DistTestCore/Codex/CodexContainerRecipe.cs +++ b/DistTestCore/Codex/CodexContainerRecipe.cs @@ -7,10 +7,10 @@ namespace DistTestCore.Codex public class CodexContainerRecipe : ContainerRecipeFactory { #if Arm64 - public const string DockerImage = "codexstorage/nim-codex:sha-f053135"; + public const string DockerImage = "codexstorage/nim-codex:sha-6dd7e55"; #else //public const string DockerImage = "thatbenbierens/nim-codex:dhting"; - public const string DockerImage = "codexstorage/nim-codex:sha-f053135"; + public const string DockerImage = "codexstorage/nim-codex:sha-6dd7e55"; #endif public const string MetricsPortTag = "metrics_port"; public const string DiscoveryPortTag = "discovery-port"; From f23926636c1023024e5ad794933dcf0fb90ec030 Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 14 Jul 2023 10:18:37 +0200 Subject: [PATCH 41/49] Setting up way to run different codex images --- DistTestCore/AutoBootstrapDistTest.cs | 5 ++-- DistTestCore/Codex/CodexAccess.cs | 4 +-- DistTestCore/Codex/CodexContainerRecipe.cs | 16 +++++++--- DistTestCore/CodexStarter.cs | 2 +- .../Helpers/PeerConnectionTestHelpers.cs | 23 ++++++++++---- DistTestCore/Http.cs | 14 +++++---- DistTestCore/Timing.cs | 2 +- Logging/TestLog.cs | 7 ++++- .../PeerDiscoveryTests/PeerDiscoveryTests.cs | 1 - .../PeerDiscoveryTests/VariableImageTests.cs | 30 +++++++++++++++++++ 10 files changed, 80 insertions(+), 24 deletions(-) create mode 100644 Tests/PeerDiscoveryTests/VariableImageTests.cs diff --git a/DistTestCore/AutoBootstrapDistTest.cs b/DistTestCore/AutoBootstrapDistTest.cs index edf182f..25bb155 100644 --- a/DistTestCore/AutoBootstrapDistTest.cs +++ b/DistTestCore/AutoBootstrapDistTest.cs @@ -6,7 +6,7 @@ namespace DistTestCore { public override IOnlineCodexNode SetupCodexBootstrapNode(Action setup) { - throw new Exception("AutoBootstrapDistTest creates and attaches a single boostrap node for you. " + + throw new Exception("AutoBootstrapDistTest creates and attaches a single bootstrap node for you. " + "If you want to control the bootstrap node from your test, please use DistTest instead."); } @@ -21,7 +21,8 @@ namespace DistTestCore [SetUp] public void SetUpBootstrapNode() { - BootstrapNode = BringOnline(CreateCodexSetup(1))[0]; + var setup = CreateCodexSetup(1).WithName("BOOTSTRAP"); + BootstrapNode = BringOnline(setup)[0]; } protected IOnlineCodexNode BootstrapNode { get; private set; } = null!; diff --git a/DistTestCore/Codex/CodexAccess.cs b/DistTestCore/Codex/CodexAccess.cs index fde9997..b6ae9fe 100644 --- a/DistTestCore/Codex/CodexAccess.cs +++ b/DistTestCore/Codex/CodexAccess.cs @@ -22,12 +22,12 @@ namespace DistTestCore.Codex public CodexDebugResponse GetDebugInfo() { - return Http(TimeSpan.FromSeconds(2)).HttpGetJson("debug/info"); + return Http(TimeSpan.FromSeconds(60)).HttpGetJson("debug/info"); } public CodexDebugPeerResponse GetDebugPeer(string peerId) { - return GetDebugPeer(peerId, TimeSpan.FromSeconds(2)); + return GetDebugPeer(peerId, TimeSpan.FromSeconds(10)); } public CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout) diff --git a/DistTestCore/Codex/CodexContainerRecipe.cs b/DistTestCore/Codex/CodexContainerRecipe.cs index b94e5ba..0bbfc87 100644 --- a/DistTestCore/Codex/CodexContainerRecipe.cs +++ b/DistTestCore/Codex/CodexContainerRecipe.cs @@ -1,6 +1,5 @@ using DistTestCore.Marketplace; using KubernetesWorkflow; -using System.Net.Sockets; namespace DistTestCore.Codex { @@ -9,8 +8,8 @@ namespace DistTestCore.Codex #if Arm64 public const string DockerImage = "codexstorage/nim-codex:sha-6dd7e55"; #else - //public const string DockerImage = "thatbenbierens/nim-codex:dhting"; - public const string DockerImage = "codexstorage/nim-codex:sha-6dd7e55"; + public const string DockerImage = "thatbenbierens/nim-codex:loopingyeah"; + //public const string DockerImage = "codexstorage/nim-codex:sha-6dd7e55"; #endif public const string MetricsPortTag = "metrics_port"; public const string DiscoveryPortTag = "discovery-port"; @@ -19,7 +18,16 @@ namespace DistTestCore.Codex public static readonly TimeSpan MaxUploadTimePerMegabyte = TimeSpan.FromSeconds(2.0); public static readonly TimeSpan MaxDownloadTimePerMegabyte = TimeSpan.FromSeconds(2.0); - protected override string Image => DockerImage; + public static string DockerImageOverride = string.Empty; + + protected override string Image + { + get + { + if (!string.IsNullOrEmpty(DockerImageOverride)) return DockerImageOverride; + return DockerImage; + } + } protected override void Initialize(StartupConfig startupConfig) { diff --git a/DistTestCore/CodexStarter.cs b/DistTestCore/CodexStarter.cs index 144d55f..9f86836 100644 --- a/DistTestCore/CodexStarter.cs +++ b/DistTestCore/CodexStarter.cs @@ -30,7 +30,7 @@ namespace DistTestCore var group = CreateCodexGroup(codexSetup, containers, codexNodeFactory); var podInfo = group.Containers.RunningPod.PodInfo; - LogEnd($"Started {codexSetup.NumberOfNodes} nodes at location '{podInfo.K8SNodeName}'={podInfo.Ip}. They are: {group.Describe()}"); + LogEnd($"Started {codexSetup.NumberOfNodes} nodes of image '{containers.Containers.First().Recipe.Image}' at location '{podInfo.K8SNodeName}'={podInfo.Ip}. They are: {group.Describe()}"); LogSeparator(); return group; } diff --git a/DistTestCore/Helpers/PeerConnectionTestHelpers.cs b/DistTestCore/Helpers/PeerConnectionTestHelpers.cs index 5eb0cdf..4320aa0 100644 --- a/DistTestCore/Helpers/PeerConnectionTestHelpers.cs +++ b/DistTestCore/Helpers/PeerConnectionTestHelpers.cs @@ -52,12 +52,12 @@ namespace DistTestCore.Helpers private static void RetryWhilePairs(List pairs, Action action) { - var timeout = DateTime.UtcNow + TimeSpan.FromMinutes(10); + var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(30); while (pairs.Any() && timeout > DateTime.UtcNow) { action(); - if (pairs.Any()) Time.Sleep(TimeSpan.FromSeconds(5)); + if (pairs.Any()) Time.Sleep(TimeSpan.FromSeconds(2)); } } @@ -140,6 +140,12 @@ namespace DistTestCore.Helpers } } + public override string ToString() + { + if (Response == null || string.IsNullOrEmpty(Response.id)) return "UNKNOWN"; + return Response.id; + } + private static string GetExpectedDiscoveryEndpoint(Entry[] allEntries, CodexDebugTableNodeResponse node) { var peer = allEntries.SingleOrDefault(e => e.Response.table.localNode.peerId == node.peerId); @@ -188,10 +194,15 @@ namespace DistTestCore.Helpers return GetResultMessage() + GetTimePostfix(); } + public override string ToString() + { + return $"[{GetMessage()}]"; + } + private string GetResultMessage() { - var aName = A.Response.id; - var bName = B.Response.id; + var aName = A.ToString(); + var bName = B.ToString(); if (Success) { @@ -203,8 +214,8 @@ namespace DistTestCore.Helpers private string GetTimePostfix() { - var aName = A.Response.id; - var bName = B.Response.id; + var aName = A.ToString(); + var bName = B.ToString(); return $" ({aName}->{bName}: {aToBTime.TotalMinutes} seconds, {bName}->{aName}: {bToATime.TotalSeconds} seconds)"; } diff --git a/DistTestCore/Http.cs b/DistTestCore/Http.cs index d4b716a..f20c315 100644 --- a/DistTestCore/Http.cs +++ b/DistTestCore/Http.cs @@ -127,16 +127,18 @@ namespace DistTestCore private T Retry(Func operation, string description) { - return Time.Retry(operation, timeSet.HttpCallRetryTimeout(), timeSet.HttpCallRetryDelay(), description); + return Time.Retry(operation, GetTimeout(), timeSet.HttpCallRetryDelay(), description); } private HttpClient GetClient() { - if (timeoutOverride.HasValue) - { - return GetClient(timeoutOverride.Value); - } - return GetClient(timeSet.HttpCallTimeout()); + return GetClient(GetTimeout()); + } + + private TimeSpan GetTimeout() + { + if (timeoutOverride.HasValue) return timeoutOverride.Value; + return timeSet.HttpCallTimeout(); } private HttpClient GetClient(TimeSpan timeout) diff --git a/DistTestCore/Timing.cs b/DistTestCore/Timing.cs index c395d07..a653ce2 100644 --- a/DistTestCore/Timing.cs +++ b/DistTestCore/Timing.cs @@ -41,7 +41,7 @@ namespace DistTestCore public TimeSpan K8sOperationTimeout() { - return TimeSpan.FromMinutes(5); + return TimeSpan.FromMinutes(1); } public TimeSpan WaitForMetricTimeout() diff --git a/Logging/TestLog.cs b/Logging/TestLog.cs index ed2cdfd..1bf5d41 100644 --- a/Logging/TestLog.cs +++ b/Logging/TestLog.cs @@ -43,7 +43,7 @@ namespace Logging if (!string.IsNullOrEmpty(name)) return name; var test = TestContext.CurrentContext.Test; var args = FormatArguments(test); - return $"{test.MethodName}{args}"; + return ReplaceInvalidCharacters($"{test.MethodName}{args}"); } private static string FormatArguments(TestContext.TestAdapter test) @@ -51,5 +51,10 @@ namespace Logging if (test.Arguments == null || !test.Arguments.Any()) return ""; return $"[{string.Join(',', test.Arguments)}]"; } + + private static string ReplaceInvalidCharacters(string name) + { + return name.Replace(":", "_"); + } } } diff --git a/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs b/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs index 57b79cf..002fe79 100644 --- a/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs +++ b/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs @@ -1,7 +1,6 @@ using DistTestCore; using DistTestCore.Helpers; using NUnit.Framework; -using Utils; namespace Tests.PeerDiscoveryTests { diff --git a/Tests/PeerDiscoveryTests/VariableImageTests.cs b/Tests/PeerDiscoveryTests/VariableImageTests.cs new file mode 100644 index 0000000..54af262 --- /dev/null +++ b/Tests/PeerDiscoveryTests/VariableImageTests.cs @@ -0,0 +1,30 @@ +using DistTestCore; +using DistTestCore.Codex; +using NUnit.Framework; + +namespace Tests.PeerDiscoveryTests +{ + [TestFixture] + public class VariableImageTests : DistTest + { + [TestCase("nim-codex:sha-a899384")] + [TestCase("nim-codex:sha-3879ec8")] + [TestCase("nim-codex:sha-6dd7e55")] + [TestCase("nim-codex:sha-3f2b417")] + [TestCase("nim-codex:sha-00f6554")] + [TestCase("nim-codex:sha-f053135")] + public void ThreeNodes(string dockerImage) + { + var img = "codexstorage/" + dockerImage; + Log("Image override: " + img); + CodexContainerRecipe.DockerImageOverride = img; + + var boot = SetupCodexBootstrapNode(); + SetupCodexNode(c => c.WithBootstrapNode(boot)); + SetupCodexNode(c => c.WithBootstrapNode(boot)); + + PeerConnectionTestHelpers.AssertFullyConnected(GetAllOnlineCodexNodes()); + } + } +} + From b295314bde164c399df1e49308c40e0f6ef8ec52 Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 14 Jul 2023 10:45:26 +0200 Subject: [PATCH 42/49] fixes blockTTL envvar --- DistTestCore/Codex/CodexContainerRecipe.cs | 2 +- Tests/PeerDiscoveryTests/VariableImageTests.cs | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/DistTestCore/Codex/CodexContainerRecipe.cs b/DistTestCore/Codex/CodexContainerRecipe.cs index 0bbfc87..67e59ef 100644 --- a/DistTestCore/Codex/CodexContainerRecipe.cs +++ b/DistTestCore/Codex/CodexContainerRecipe.cs @@ -56,7 +56,7 @@ namespace DistTestCore.Codex } if (config.BlockTTL != null) { - AddEnvVar("BLOCK_TTL", config.BlockTTL.ToString()!); + AddEnvVar("CODEX_BLOCK_TTL", config.BlockTTL.ToString()!); } if (config.MetricsEnabled) { diff --git a/Tests/PeerDiscoveryTests/VariableImageTests.cs b/Tests/PeerDiscoveryTests/VariableImageTests.cs index 54af262..95a8dee 100644 --- a/Tests/PeerDiscoveryTests/VariableImageTests.cs +++ b/Tests/PeerDiscoveryTests/VariableImageTests.cs @@ -13,6 +13,7 @@ namespace Tests.PeerDiscoveryTests [TestCase("nim-codex:sha-3f2b417")] [TestCase("nim-codex:sha-00f6554")] [TestCase("nim-codex:sha-f053135")] + [TestCase("nim-codex:sha-711e5e0")] public void ThreeNodes(string dockerImage) { var img = "codexstorage/" + dockerImage; From adeba216653c9eb4e729948e9868159407d1601e Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 17 Jul 2023 08:54:07 +0200 Subject: [PATCH 43/49] Disables debug output by default --- DistTestCore/Configuration.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DistTestCore/Configuration.cs b/DistTestCore/Configuration.cs index fceb4e0..03514cd 100644 --- a/DistTestCore/Configuration.cs +++ b/DistTestCore/Configuration.cs @@ -17,7 +17,7 @@ namespace DistTestCore { kubeConfigFile = GetNullableEnvVarOrDefault("KUBECONFIG", null); logPath = GetEnvVarOrDefault("LOGPATH", "CodexTestLogs"); - logDebug = GetEnvVarOrDefault("LOGDEBUG", "true").ToLowerInvariant() == "true"; + logDebug = GetEnvVarOrDefault("LOGDEBUG", "false").ToLowerInvariant() == "true"; dataFilesPath = GetEnvVarOrDefault("DATAFILEPATH", "TestDataFiles"); codexLogLevel = ParseEnum.Parse(GetEnvVarOrDefault("LOGLEVEL", nameof(CodexLogLevel.Trace))); runnerLocation = ParseEnum.Parse(GetEnvVarOrDefault("RUNNERLOCATION", nameof(TestRunnerLocation.ExternalToCluster))); From 4815194db007280167288296755ac8aa9294ebd3 Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 17 Jul 2023 08:56:18 +0200 Subject: [PATCH 44/49] Removes variable image tests. --- .../PeerDiscoveryTests/VariableImageTests.cs | 31 ------------------- 1 file changed, 31 deletions(-) delete mode 100644 Tests/PeerDiscoveryTests/VariableImageTests.cs diff --git a/Tests/PeerDiscoveryTests/VariableImageTests.cs b/Tests/PeerDiscoveryTests/VariableImageTests.cs deleted file mode 100644 index 95a8dee..0000000 --- a/Tests/PeerDiscoveryTests/VariableImageTests.cs +++ /dev/null @@ -1,31 +0,0 @@ -using DistTestCore; -using DistTestCore.Codex; -using NUnit.Framework; - -namespace Tests.PeerDiscoveryTests -{ - [TestFixture] - public class VariableImageTests : DistTest - { - [TestCase("nim-codex:sha-a899384")] - [TestCase("nim-codex:sha-3879ec8")] - [TestCase("nim-codex:sha-6dd7e55")] - [TestCase("nim-codex:sha-3f2b417")] - [TestCase("nim-codex:sha-00f6554")] - [TestCase("nim-codex:sha-f053135")] - [TestCase("nim-codex:sha-711e5e0")] - public void ThreeNodes(string dockerImage) - { - var img = "codexstorage/" + dockerImage; - Log("Image override: " + img); - CodexContainerRecipe.DockerImageOverride = img; - - var boot = SetupCodexBootstrapNode(); - SetupCodexNode(c => c.WithBootstrapNode(boot)); - SetupCodexNode(c => c.WithBootstrapNode(boot)); - - PeerConnectionTestHelpers.AssertFullyConnected(GetAllOnlineCodexNodes()); - } - } -} - From ac33d3c7be717a66aee8fc9e5ea18ceaa15bb549 Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 17 Jul 2023 13:47:41 +0200 Subject: [PATCH 45/49] Moves some of the fully-connected download tests to the long-tests --- .../LongFullyConnectedDownloadTests.cs | 22 +++++++++++++++++++ .../FullyConnectedDownloadTests.cs | 4 ++-- 2 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 LongTests/DownloadConnectivityTests/LongFullyConnectedDownloadTests.cs diff --git a/LongTests/DownloadConnectivityTests/LongFullyConnectedDownloadTests.cs b/LongTests/DownloadConnectivityTests/LongFullyConnectedDownloadTests.cs new file mode 100644 index 0000000..5bb7323 --- /dev/null +++ b/LongTests/DownloadConnectivityTests/LongFullyConnectedDownloadTests.cs @@ -0,0 +1,22 @@ +using DistTestCore.Helpers; +using DistTestCore; +using NUnit.Framework; + +namespace TestsLong.DownloadConnectivityTests +{ + [TestFixture] + public class LongFullyConnectedDownloadTests : AutoBootstrapDistTest + { + [Test] + [UseLongTimeouts] + [Combinatorial] + public void FullyConnectedDownloadTest( + [Values(10, 15, 20)] int numberOfNodes, + [Values(10, 100)] int sizeMBs) + { + for (var i = 0; i < numberOfNodes; i++) SetupCodexNode(); + + PeerDownloadTestHelpers.AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes(), sizeMBs.MB()); + } + } +} diff --git a/Tests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs b/Tests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs index ff3f62e..a1f2c5f 100644 --- a/Tests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs +++ b/Tests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs @@ -9,8 +9,8 @@ namespace Tests.DownloadConnectivityTests [Test] [Combinatorial] public void FullyConnectedDownloadTest( - [Values(3, 10, 20)] int numberOfNodes, - [Values(1, 10, 100)] int sizeMBs) + [Values(1, 3, 5)] int numberOfNodes, + [Values(1, 10)] int sizeMBs) { for (var i = 0; i < numberOfNodes; i++) SetupCodexNode(); From 1154544e99e2b7582adad71aed8bf8cef6cf41f8 Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 17 Jul 2023 13:57:56 +0200 Subject: [PATCH 46/49] prints images before confirmation --- CodexNetDeployer/Program.cs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CodexNetDeployer/Program.cs b/CodexNetDeployer/Program.cs index 234ce78..fc4d5f6 100644 --- a/CodexNetDeployer/Program.cs +++ b/CodexNetDeployer/Program.cs @@ -1,6 +1,9 @@ using ArgsUniform; using CodexNetDeployer; using DistTestCore; +using DistTestCore.Codex; +using DistTestCore.Marketplace; +using DistTestCore.Metrics; using Newtonsoft.Json; using Configuration = CodexNetDeployer.Configuration; @@ -29,6 +32,12 @@ public class Program return; } + Console.WriteLine("Using images:" + nl + + $"\tCodex image: '{CodexContainerRecipe.DockerImage}'" + nl + + $"\tCodex Contracts image: '{CodexContractsContainerRecipe.DockerImage}'" + nl + + $"\tPrometheus image: '{PrometheusContainerRecipe.DockerImage}'" + nl + + $"\tGeth image: '{GethContainerRecipe.DockerImage}'" + nl); + if (!args.Any(a => a == "-y")) { Console.WriteLine("Does the above config look good? [y/n]"); From fe25166478cfa03873d02aad5262b8b107abad66 Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 17 Jul 2023 15:21:10 +0200 Subject: [PATCH 47/49] Better http timeout and retrying values --- DistTestCore/Codex/CodexAccess.cs | 13 ++++--------- .../Helpers/PeerConnectionTestHelpers.cs | 3 +-- DistTestCore/Http.cs | 19 +++---------------- DistTestCore/OnlineCodexNode.cs | 6 ------ DistTestCore/Timing.cs | 6 +++--- 5 files changed, 11 insertions(+), 36 deletions(-) diff --git a/DistTestCore/Codex/CodexAccess.cs b/DistTestCore/Codex/CodexAccess.cs index b6ae9fe..a38aa5d 100644 --- a/DistTestCore/Codex/CodexAccess.cs +++ b/DistTestCore/Codex/CodexAccess.cs @@ -22,17 +22,12 @@ namespace DistTestCore.Codex public CodexDebugResponse GetDebugInfo() { - return Http(TimeSpan.FromSeconds(60)).HttpGetJson("debug/info"); + return Http().HttpGetJson("debug/info"); } public CodexDebugPeerResponse GetDebugPeer(string peerId) { - return GetDebugPeer(peerId, TimeSpan.FromSeconds(10)); - } - - public CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout) - { - var http = Http(timeout); + var http = Http(); var str = http.HttpGetString($"debug/peer/{peerId}"); if (str.ToLowerInvariant() == "unable to find peer!") @@ -88,9 +83,9 @@ namespace DistTestCore.Codex return Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}"); } - private Http Http(TimeSpan? timeoutOverride = null) + private Http Http() { - return new Http(log, timeSet, Address, baseUrl: "/api/codex/v1", Container.Name, timeoutOverride); + return new Http(log, timeSet, Address, baseUrl: "/api/codex/v1", Container.Name); } } } diff --git a/DistTestCore/Helpers/PeerConnectionTestHelpers.cs b/DistTestCore/Helpers/PeerConnectionTestHelpers.cs index 4320aa0..93c8f21 100644 --- a/DistTestCore/Helpers/PeerConnectionTestHelpers.cs +++ b/DistTestCore/Helpers/PeerConnectionTestHelpers.cs @@ -167,7 +167,6 @@ namespace DistTestCore.Helpers public class Pair { - private readonly TimeSpan timeout = TimeSpan.FromSeconds(60); private TimeSpan aToBTime = TimeSpan.FromSeconds(0); private TimeSpan bToATime = TimeSpan.FromSeconds(0); @@ -235,7 +234,7 @@ namespace DistTestCore.Helpers try { - var response = a.Node.GetDebugPeer(peerId, timeout); + var response = a.Node.GetDebugPeer(peerId); if (!response.IsPeerFound) { return PeerConnectionState.NoConnection; diff --git a/DistTestCore/Http.cs b/DistTestCore/Http.cs index f20c315..391dbc7 100644 --- a/DistTestCore/Http.cs +++ b/DistTestCore/Http.cs @@ -13,16 +13,14 @@ namespace DistTestCore private readonly Address address; private readonly string baseUrl; private readonly string? logAlias; - private readonly TimeSpan? timeoutOverride; - public Http(BaseLog log, ITimeSet timeSet, Address address, string baseUrl, string? logAlias = null, TimeSpan? timeoutOverride = null) + public Http(BaseLog log, ITimeSet timeSet, Address address, string baseUrl, string? logAlias = null) { this.log = log; this.timeSet = timeSet; this.address = address; this.baseUrl = baseUrl; this.logAlias = logAlias; - this.timeoutOverride = timeoutOverride; if (!this.baseUrl.StartsWith("/")) this.baseUrl = "/" + this.baseUrl; if (!this.baseUrl.EndsWith("/")) this.baseUrl += "/"; } @@ -127,24 +125,13 @@ namespace DistTestCore private T Retry(Func operation, string description) { - return Time.Retry(operation, GetTimeout(), timeSet.HttpCallRetryDelay(), description); + return Time.Retry(operation, timeSet.HttpCallRetryTime(), timeSet.HttpCallRetryDelay(), description); } private HttpClient GetClient() - { - return GetClient(GetTimeout()); - } - - private TimeSpan GetTimeout() - { - if (timeoutOverride.HasValue) return timeoutOverride.Value; - return timeSet.HttpCallTimeout(); - } - - private HttpClient GetClient(TimeSpan timeout) { var client = new HttpClient(); - client.Timeout = timeout; + client.Timeout = timeSet.HttpCallTimeout(); return client; } } diff --git a/DistTestCore/OnlineCodexNode.cs b/DistTestCore/OnlineCodexNode.cs index cd37fb4..1054d3e 100644 --- a/DistTestCore/OnlineCodexNode.cs +++ b/DistTestCore/OnlineCodexNode.cs @@ -12,7 +12,6 @@ namespace DistTestCore string GetName(); CodexDebugResponse GetDebugInfo(); CodexDebugPeerResponse GetDebugPeer(string peerId); - CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout); ContentId UploadFile(TestFile file); TestFile? DownloadContent(ContentId contentId, string fileLabel = ""); void ConnectToPeer(IOnlineCodexNode node); @@ -60,11 +59,6 @@ namespace DistTestCore return CodexAccess.GetDebugPeer(peerId); } - public CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout) - { - return CodexAccess.GetDebugPeer(peerId, timeout); - } - public ContentId UploadFile(TestFile file) { using var fileStream = File.OpenRead(file.Filename); diff --git a/DistTestCore/Timing.cs b/DistTestCore/Timing.cs index a653ce2..bd385ae 100644 --- a/DistTestCore/Timing.cs +++ b/DistTestCore/Timing.cs @@ -10,7 +10,7 @@ namespace DistTestCore public interface ITimeSet { TimeSpan HttpCallTimeout(); - TimeSpan HttpCallRetryTimeout(); + TimeSpan HttpCallRetryTime(); TimeSpan HttpCallRetryDelay(); TimeSpan WaitForK8sServiceDelay(); TimeSpan K8sOperationTimeout(); @@ -24,7 +24,7 @@ namespace DistTestCore return TimeSpan.FromSeconds(10); } - public TimeSpan HttpCallRetryTimeout() + public TimeSpan HttpCallRetryTime() { return TimeSpan.FromMinutes(1); } @@ -57,7 +57,7 @@ namespace DistTestCore return TimeSpan.FromHours(2); } - public TimeSpan HttpCallRetryTimeout() + public TimeSpan HttpCallRetryTime() { return TimeSpan.FromHours(5); } From 6892080128f86fff9d6d6271bbb9eeec4bebc7d0 Mon Sep 17 00:00:00 2001 From: benbierens Date: Tue, 18 Jul 2023 08:03:16 +0200 Subject: [PATCH 48/49] Disables future-counting API call --- DistTestCore/Codex/CodexAccess.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/DistTestCore/Codex/CodexAccess.cs b/DistTestCore/Codex/CodexAccess.cs index a38aa5d..e2158ee 100644 --- a/DistTestCore/Codex/CodexAccess.cs +++ b/DistTestCore/Codex/CodexAccess.cs @@ -45,7 +45,8 @@ namespace DistTestCore.Codex public int GetDebugFutures() { - return Http().HttpGetJson("debug/futures").futures; + // Some Codex images support debug/futures to count the number of open futures. + return 0; // Http().HttpGetJson("debug/futures").futures; } public CodexDebugThresholdBreaches GetDebugThresholdBreaches() From 5f89db4012bb1ca7e5d5cbccb23fde6330fdc352 Mon Sep 17 00:00:00 2001 From: benbierens Date: Tue, 18 Jul 2023 09:11:25 +0200 Subject: [PATCH 49/49] Bumps to latest codex image --- DistTestCore/Codex/CodexContainerRecipe.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/DistTestCore/Codex/CodexContainerRecipe.cs b/DistTestCore/Codex/CodexContainerRecipe.cs index 67e59ef..29ee032 100644 --- a/DistTestCore/Codex/CodexContainerRecipe.cs +++ b/DistTestCore/Codex/CodexContainerRecipe.cs @@ -6,10 +6,10 @@ namespace DistTestCore.Codex public class CodexContainerRecipe : ContainerRecipeFactory { #if Arm64 - public const string DockerImage = "codexstorage/nim-codex:sha-6dd7e55"; + public const string DockerImage = "codexstorage/nim-codex:sha-7227a4a"; #else - public const string DockerImage = "thatbenbierens/nim-codex:loopingyeah"; - //public const string DockerImage = "codexstorage/nim-codex:sha-6dd7e55"; + //public const string DockerImage = "thatbenbierens/nim-codex:loopingyeah"; + public const string DockerImage = "codexstorage/nim-codex:sha-7227a4a"; #endif public const string MetricsPortTag = "metrics_port"; public const string DiscoveryPortTag = "discovery-port";