From 0a8928b5aafcfe0f0db143af5455091b267f2241 Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 30 Aug 2023 08:33:38 +0200 Subject: [PATCH 01/11] Adds logging and connectivity check. --- ContinuousTests/Tests/PeersTest.cs | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/ContinuousTests/Tests/PeersTest.cs b/ContinuousTests/Tests/PeersTest.cs index 13c54d9..978c31c 100644 --- a/ContinuousTests/Tests/PeersTest.cs +++ b/ContinuousTests/Tests/PeersTest.cs @@ -1,4 +1,5 @@ using DistTestCore.Codex; +using DistTestCore.Helpers; using NUnit.Framework; namespace ContinuousTests.Tests @@ -10,10 +11,23 @@ namespace ContinuousTests.Tests public override TestFailMode TestFailMode => TestFailMode.AlwaysRunAllMoments; [TestMoment(t: 0)] + public void CheckConnectivity() + { + var checker = new PeerConnectionTestHelpers(Log); + checker.AssertFullyConnected(Nodes); + } + + [TestMoment(t: 10)] public void CheckRoutingTables() { - var allIds = Nodes.Select(n => n.GetDebugInfo().table.localNode.nodeId).ToArray(); + var allInfos = Nodes.Select(n => + { + var info = n.GetDebugInfo(); + Log.Log($"{n.GetName()} = {info.table.localNode.nodeId}"); + return info; + }).ToArray(); + var allIds = allInfos.Select(i => i.table.localNode.nodeId).ToArray(); var errors = Nodes.Select(n => AreAllPresent(n, allIds)).Where(s => !string.IsNullOrEmpty(s)).ToArray(); if (errors.Any()) @@ -30,7 +44,8 @@ namespace ContinuousTests.Tests if (!expected.All(ex => known.Contains(ex))) { - return $"Not all of '{string.Join(",", expected)}' were present in routing table: '{string.Join(",", known)}'"; + var nl = Environment.NewLine; + return $"Not all of{nl}'{string.Join(",", expected)}'{nl}were present in routing table:{nl}'{string.Join(",", known)}'"; } return string.Empty; From a5773bf7109de882e043e6c82d2c68a6335ef8a9 Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 30 Aug 2023 08:53:09 +0200 Subject: [PATCH 02/11] More logging. Set container names for codex deployment. --- CodexNetDeployer/CodexNodeStarter.cs | 7 +++++++ ContinuousTests/Tests/PeersTest.cs | 5 ++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CodexNetDeployer/CodexNodeStarter.cs b/CodexNetDeployer/CodexNodeStarter.cs index 1bc4af9..8e861f3 100644 --- a/CodexNetDeployer/CodexNodeStarter.cs +++ b/CodexNetDeployer/CodexNodeStarter.cs @@ -28,6 +28,7 @@ namespace CodexNetDeployer var workflowStartup = new StartupConfig(); workflowStartup.Add(gethResult); workflowStartup.Add(CreateCodexStartupConfig(bootstrapSpr, i, validatorsLeft)); + workflowStartup.NameOverride = GetCodexContainerName(i); var containers = workflow.Start(1, Location.Unspecified, new CodexContainerRecipe(), workflowStartup); @@ -75,6 +76,12 @@ namespace CodexNetDeployer return null; } + private string GetCodexContainerName(int i) + { + if (i == 0) return "BOOTSTRAP"; + return "CODEX" + i; + } + private CodexStartupConfig CreateCodexStartupConfig(string bootstrapSpr, int i, int validatorsLeft) { var codexStart = new CodexStartupConfig(config.CodexLogLevel); diff --git a/ContinuousTests/Tests/PeersTest.cs b/ContinuousTests/Tests/PeersTest.cs index 978c31c..5726cff 100644 --- a/ContinuousTests/Tests/PeersTest.cs +++ b/ContinuousTests/Tests/PeersTest.cs @@ -24,6 +24,7 @@ namespace ContinuousTests.Tests { var info = n.GetDebugInfo(); Log.Log($"{n.GetName()} = {info.table.localNode.nodeId}"); + Log.AddStringReplace(info.table.localNode.nodeId, n.GetName()); return info; }).ToArray(); @@ -45,7 +46,9 @@ namespace ContinuousTests.Tests if (!expected.All(ex => known.Contains(ex))) { var nl = Environment.NewLine; - return $"Not all of{nl}'{string.Join(",", expected)}'{nl}were present in routing table:{nl}'{string.Join(",", known)}'"; + return $"{nl}At node '{info.table.localNode.nodeId}'{nl}" + + $"Not all of{nl}'{string.Join(",", expected)}'{nl}" + + $"were present in routing table:{nl}'{string.Join(",", known)}'"; } return string.Empty; From a1ec187919defeb14301e9aedc6d497593343150 Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 30 Aug 2023 09:23:13 +0200 Subject: [PATCH 03/11] Configured runner to stop after 10 failures --- ContinuousTests/Configuration.cs | 4 ++-- ContinuousTests/SingleTestRun.cs | 16 +++++++++++----- ContinuousTests/run.sh | 2 +- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/ContinuousTests/Configuration.cs b/ContinuousTests/Configuration.cs index da1eaeb..04491a9 100644 --- a/ContinuousTests/Configuration.cs +++ b/ContinuousTests/Configuration.cs @@ -22,8 +22,8 @@ namespace ContinuousTests [Uniform("kube-config", "kc", "KUBECONFIG", true, "Path to Kubeconfig file. Use 'null' (default) to use local cluster.")] public string KubeConfigFile { get; set; } = "null"; - [Uniform("stop", "s", "STOPONFAIL", false, "If true, runner will stop on first test failure and download all cluster container logs. False by default.")] - public bool StopOnFailure { get; set; } = false; + [Uniform("stop", "s", "STOPONFAIL", false, "If greater than zero, runner will stop after this many test failures and download all cluster container logs. 0 by default.")] + public int StopOnFailure { get; set; } = 0; [Uniform("dl-logs", "dl", "DLLOGS", false, "If true, runner will periodically download and save/append container logs to the log path.")] public bool DownloadContainerLogs { get; set; } = false; diff --git a/ContinuousTests/SingleTestRun.cs b/ContinuousTests/SingleTestRun.cs index e268064..a43dc2f 100644 --- a/ContinuousTests/SingleTestRun.cs +++ b/ContinuousTests/SingleTestRun.cs @@ -23,6 +23,7 @@ namespace ContinuousTests private readonly FixtureLog fixtureLog; private readonly string testName; private readonly string dataFolder; + private static int failureCount = 0; public SingleTestRun(TaskFactory taskFactory, Configuration config, BaseLog overviewLog, TestHandle handle, CancellationToken cancelToken) { @@ -71,12 +72,17 @@ namespace ContinuousTests fixtureLog.Error("Test run failed with exception: " + ex); fixtureLog.MarkAsFailed(); - if (config.StopOnFailure) + failureCount++; + if (config.StopOnFailure > 0) { - OverviewLog("Configured to stop on first failure. Downloading cluster logs..."); - DownloadClusterLogs(); - OverviewLog("Log download finished. Cancelling test runner..."); - Cancellation.Cts.Cancel(); + OverviewLog($"Failures: {failureCount} / {config.StopOnFailure}"); + if (failureCount >= config.StopOnFailure) + { + OverviewLog($"Configured to stop after {config.StopOnFailure} failures. Downloading cluster logs..."); + DownloadClusterLogs(); + OverviewLog("Log download finished. Cancelling test runner..."); + Cancellation.Cts.Cancel(); + } } } } diff --git a/ContinuousTests/run.sh b/ContinuousTests/run.sh index 67c9419..6b281d7 100644 --- a/ContinuousTests/run.sh +++ b/ContinuousTests/run.sh @@ -2,5 +2,5 @@ dotnet run \ --kube-config=/opt/kubeconfig.yaml \ --codex-deployment=codex-deployment.json \ --keep=1 \ - --stop=1 \ + --stop=10 \ --dl-logs=1 From 8d7a7b4d1b43f15e0fc4c53c8ccb1de0fc312152 Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 30 Aug 2023 10:13:48 +0200 Subject: [PATCH 04/11] Better exception logging for tests with multiple time-moments. --- ContinuousTests/SingleTestRun.cs | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/ContinuousTests/SingleTestRun.cs b/ContinuousTests/SingleTestRun.cs index a43dc2f..9115fdf 100644 --- a/ContinuousTests/SingleTestRun.cs +++ b/ContinuousTests/SingleTestRun.cs @@ -126,10 +126,11 @@ namespace ContinuousTests private void ThrowFailTest() { - var ex = UnpackException(exceptions.First()); - Log(ex.ToString()); - OverviewLog($" > Test failed {FuturesInfo()}: " + ex.Message); - throw ex; + var exs = UnpackExceptions(exceptions); + var exceptionsMessage = GetCombinedExceptionsMessage(exs); + Log(exceptionsMessage); + OverviewLog($" > Test failed {FuturesInfo()}: " + exceptionsMessage); + throw new Exception(exceptionsMessage); } private string FuturesInfo() @@ -152,6 +153,16 @@ namespace ContinuousTests } } + private string GetCombinedExceptionsMessage(Exception[] exceptions) + { + return string.Join(Environment.NewLine, exceptions.Select(ex => ex.ToString())); + } + + private Exception[] UnpackExceptions(List exceptions) + { + return exceptions.Select(UnpackException).ToArray(); + } + private Exception UnpackException(Exception exception) { if (exception is AggregateException a) From 7d042d0a1c4a35385c61efec7597c28dcbe13062 Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 30 Aug 2023 10:57:20 +0200 Subject: [PATCH 05/11] Log replacements for node ids in continuous test fixture logs. --- ContinuousTests/ContinuousTestRunner.cs | 2 +- ContinuousTests/SingleTestRun.cs | 8 +++++++- ContinuousTests/StartupChecker.cs | 4 ++++ ContinuousTests/TestLoop.cs | 6 ++++-- DistTestCore/Codex/CodexApiTypes.cs | 5 +---- Logging/BaseLog.cs | 13 +++++++------ 6 files changed, 24 insertions(+), 14 deletions(-) diff --git a/ContinuousTests/ContinuousTestRunner.cs b/ContinuousTests/ContinuousTestRunner.cs index bfe6000..fb69e05 100644 --- a/ContinuousTests/ContinuousTestRunner.cs +++ b/ContinuousTests/ContinuousTestRunner.cs @@ -30,7 +30,7 @@ namespace ContinuousTests ClearAllCustomNamespaces(allTests, overviewLog); - var testLoops = allTests.Select(t => new TestLoop(taskFactory, config, overviewLog, t.GetType(), t.RunTestEvery, cancelToken)).ToArray(); + var testLoops = allTests.Select(t => new TestLoop(taskFactory, config, overviewLog, t.GetType(), t.RunTestEvery, startupChecker, cancelToken)).ToArray(); foreach (var testLoop in testLoops) { diff --git a/ContinuousTests/SingleTestRun.cs b/ContinuousTests/SingleTestRun.cs index 9115fdf..4d9a041 100644 --- a/ContinuousTests/SingleTestRun.cs +++ b/ContinuousTests/SingleTestRun.cs @@ -25,7 +25,7 @@ namespace ContinuousTests private readonly string dataFolder; private static int failureCount = 0; - public SingleTestRun(TaskFactory taskFactory, Configuration config, BaseLog overviewLog, TestHandle handle, CancellationToken cancelToken) + public SingleTestRun(TaskFactory taskFactory, Configuration config, BaseLog overviewLog, TestHandle handle, StartupChecker startupChecker, CancellationToken cancelToken) { this.taskFactory = taskFactory; this.config = config; @@ -34,6 +34,7 @@ namespace ContinuousTests this.cancelToken = cancelToken; testName = handle.Test.GetType().Name; fixtureLog = new FixtureLog(new LogConfig(config.LogPath, true), DateTime.UtcNow, testName); + ApplyLogReplacements(fixtureLog, startupChecker); nodes = CreateRandomNodes(handle.Test.RequiredNumberOfNodes); dataFolder = config.DataPath + "-" + Guid.NewGuid(); @@ -87,6 +88,11 @@ namespace ContinuousTests } } + private void ApplyLogReplacements(FixtureLog fixtureLog, StartupChecker startupChecker) + { + foreach (var replacement in startupChecker.LogReplacements) fixtureLog.AddStringReplace(replacement.From, replacement.To); + } + private void RunTestMoments() { var earliestMoment = handle.GetEarliestMoment(); diff --git a/ContinuousTests/StartupChecker.cs b/ContinuousTests/StartupChecker.cs index 5986fa5..2fee5eb 100644 --- a/ContinuousTests/StartupChecker.cs +++ b/ContinuousTests/StartupChecker.cs @@ -15,6 +15,7 @@ namespace ContinuousTests { this.config = config; this.cancelToken = cancelToken; + LogReplacements = new List(); } public void Check() @@ -28,6 +29,8 @@ namespace ContinuousTests log.Log("All OK."); } + public List LogReplacements { get; } + private void PreflightCheck(Configuration config) { var tests = testFactory.CreateTests(); @@ -90,6 +93,7 @@ namespace ContinuousTests if (info == null || string.IsNullOrEmpty(info.id)) return false; log.Log($"Codex version: '{info.codex.version}' revision: '{info.codex.revision}'"); + LogReplacements.Add(new BaseLogStringReplacement(info.id, n.GetName())); } catch { diff --git a/ContinuousTests/TestLoop.cs b/ContinuousTests/TestLoop.cs index e57fee8..79ae999 100644 --- a/ContinuousTests/TestLoop.cs +++ b/ContinuousTests/TestLoop.cs @@ -9,16 +9,18 @@ namespace ContinuousTests private readonly BaseLog overviewLog; private readonly Type testType; private readonly TimeSpan runsEvery; + private readonly StartupChecker startupChecker; private readonly CancellationToken cancelToken; private readonly EventWaitHandle runFinishedHandle = new EventWaitHandle(true, EventResetMode.ManualReset); - public TestLoop(TaskFactory taskFactory, Configuration config, BaseLog overviewLog, Type testType, TimeSpan runsEvery, CancellationToken cancelToken) + public TestLoop(TaskFactory taskFactory, Configuration config, BaseLog overviewLog, Type testType, TimeSpan runsEvery, StartupChecker startupChecker, CancellationToken cancelToken) { this.taskFactory = taskFactory; this.config = config; this.overviewLog = overviewLog; this.testType = testType; this.runsEvery = runsEvery; + this.startupChecker = startupChecker; this.cancelToken = cancelToken; Name = testType.Name; } @@ -58,7 +60,7 @@ namespace ContinuousTests { var test = (ContinuousTest)Activator.CreateInstance(testType)!; var handle = new TestHandle(test); - var run = new SingleTestRun(taskFactory, config, overviewLog, handle, cancelToken); + var run = new SingleTestRun(taskFactory, config, overviewLog, handle, startupChecker, cancelToken); runFinishedHandle.Reset(); run.Run(runFinishedHandle); diff --git a/DistTestCore/Codex/CodexApiTypes.cs b/DistTestCore/Codex/CodexApiTypes.cs index 32bf353..5944b84 100644 --- a/DistTestCore/Codex/CodexApiTypes.cs +++ b/DistTestCore/Codex/CodexApiTypes.cs @@ -1,7 +1,4 @@ -using KubernetesWorkflow; -using Logging; -using Newtonsoft.Json; -using Utils; +using Newtonsoft.Json; namespace DistTestCore.Codex { diff --git a/Logging/BaseLog.cs b/Logging/BaseLog.cs index 1164d1b..d11ecc1 100644 --- a/Logging/BaseLog.cs +++ b/Logging/BaseLog.cs @@ -60,6 +60,7 @@ namespace Logging public virtual void AddStringReplace(string from, string to) { if (string.IsNullOrWhiteSpace(from)) return; + if (replacements.Any(r => r.From == from)) return; replacements.Add(new BaseLogStringReplacement(from, to)); } @@ -98,20 +99,20 @@ namespace Logging public class BaseLogStringReplacement { - private readonly string from; - private readonly string to; - public BaseLogStringReplacement(string from, string to) { - this.from = from; - this.to = to; + From = from; + To = to; if (string.IsNullOrEmpty(from) || string.IsNullOrEmpty(to) || from == to) throw new ArgumentException(); } + public string From { get; } + public string To { get; } + public string Apply(string msg) { - return msg.Replace(from, to); + return msg.Replace(From, To); } } } From 647022a27eae2d100c166122daccae647f4ef22b Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 31 Aug 2023 09:50:34 +0200 Subject: [PATCH 06/11] Updates overview log for all-nodes tests. --- ContinuousTests/SingleTestRun.cs | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/ContinuousTests/SingleTestRun.cs b/ContinuousTests/SingleTestRun.cs index 4d9a041..3d9c03d 100644 --- a/ContinuousTests/SingleTestRun.cs +++ b/ContinuousTests/SingleTestRun.cs @@ -36,7 +36,7 @@ namespace ContinuousTests fixtureLog = new FixtureLog(new LogConfig(config.LogPath, true), DateTime.UtcNow, testName); ApplyLogReplacements(fixtureLog, startupChecker); - nodes = CreateRandomNodes(handle.Test.RequiredNumberOfNodes); + nodes = CreateRandomNodes(); dataFolder = config.DataPath + "-" + Guid.NewGuid(); fileManager = new FileManager(fixtureLog, CreateFileManagerConfiguration()); } @@ -219,19 +219,26 @@ namespace ContinuousTests private void OverviewLog(string msg) { Log(msg); - var containerNames = $"({string.Join(",", nodes.Select(n => n.Container.Name))})"; + var containerNames = GetContainerNames(); overviewLog.Log($"{containerNames} {testName}: {msg}"); } - private CodexAccess[] CreateRandomNodes(int number) + private string GetContainerNames() { - var containers = SelectRandomContainers(number); + if (handle.Test.RequiredNumberOfNodes == -1) return "(All Nodes)"; + return $"({string.Join(",", nodes.Select(n => n.Container.Name))})"; + } + + private CodexAccess[] CreateRandomNodes() + { + var containers = SelectRandomContainers(); fixtureLog.Log("Selected nodes: " + string.Join(",", containers.Select(c => c.Name))); return codexNodeFactory.Create(config, containers, fixtureLog, handle.Test.TimeSet); } - private RunningContainer[] SelectRandomContainers(int number) + private RunningContainer[] SelectRandomContainers() { + var number = handle.Test.RequiredNumberOfNodes; if (number == -1) return config.CodexDeployment.CodexContainers; var containers = config.CodexDeployment.CodexContainers.ToList(); From cf9d2ffe19e13cd7f1fd993f3ef636a3c20d45f1 Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 31 Aug 2023 11:19:53 +0200 Subject: [PATCH 07/11] Fix for marketplace request deserialization --- DistTestCore/Codex/CodexAccess.cs | 2 +- DistTestCore/Http.cs | 38 +++++++++++++++++++------------ 2 files changed, 25 insertions(+), 15 deletions(-) diff --git a/DistTestCore/Codex/CodexAccess.cs b/DistTestCore/Codex/CodexAccess.cs index f88b390..cf6ac19 100644 --- a/DistTestCore/Codex/CodexAccess.cs +++ b/DistTestCore/Codex/CodexAccess.cs @@ -75,7 +75,7 @@ namespace DistTestCore.Codex public string RequestStorage(CodexSalesRequestStorageRequest request, string contentId) { - return Http().HttpPostJson($"storage/request/{contentId}", request); + return Http().HttpPostJson($"storage/request/{contentId}", request); } public CodexStoragePurchase GetPurchaseStatus(string purchaseId) diff --git a/DistTestCore/Http.cs b/DistTestCore/Http.cs index 94d73f2..773f9a5 100644 --- a/DistTestCore/Http.cs +++ b/DistTestCore/Http.cs @@ -55,25 +55,20 @@ namespace DistTestCore public TResponse HttpPostJson(string route, TRequest body) { - var response = HttpPostJson(route, body); + var response = PostJson(route, body); var json = Time.Wait(response.Content.ReadAsStringAsync()); - if(!response.IsSuccessStatusCode) { + if (!response.IsSuccessStatusCode) + { throw new HttpRequestException(json); } Log(GetUrl() + route, json); return TryJsonDeserialize(json); } - public HttpResponseMessage HttpPostJson(string route, TRequest body) + public string HttpPostJson(string route, TRequest body) { - return Retry(() => - { - using var client = GetClient(); - var url = GetUrl() + route; - using var content = JsonContent.Create(body); - Log(url, JsonConvert.SerializeObject(body)); - return Time.Wait(client.PostAsync(url, content)); - }, $"HTTP-POST-JSON: {route}"); + var response = PostJson(route, body); + return Time.Wait(response.Content.ReadAsStringAsync()); } public string HttpPostString(string route, string body) @@ -122,7 +117,8 @@ namespace DistTestCore public T TryJsonDeserialize(string json) { var errors = new List(); - var deserialized = JsonConvert.DeserializeObject(json, new JsonSerializerSettings(){ + var deserialized = JsonConvert.DeserializeObject(json, new JsonSerializerSettings() + { Error = delegate(object? sender, Serialization.ErrorEventArgs args) { if (args.CurrentObject == args.ErrorContext.OriginalObject) @@ -136,15 +132,29 @@ namespace DistTestCore } } }); - if (errors.Count() > 0) { + if (errors.Count() > 0) + { throw new JsonSerializationException($"Failed to deserialize JSON '{json}' with exception(s): \n{string.Join("\n", errors)}"); } - else if (deserialized == null) { + else if (deserialized == null) + { throw new JsonSerializationException($"Failed to deserialize JSON '{json}': resulting deserialized object is null"); } return deserialized; } + private HttpResponseMessage PostJson(string route, TRequest body) + { + return Retry(() => + { + using var client = GetClient(); + var url = GetUrl() + route; + using var content = JsonContent.Create(body); + Log(url, JsonConvert.SerializeObject(body)); + return Time.Wait(client.PostAsync(url, content)); + }, $"HTTP-POST-JSON: {route}"); + } + private string GetUrl() { return $"{address.Host}:{address.Port}{baseUrl}"; From a29d646027287060f05784333eaa20831e04756e Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 1 Sep 2023 08:45:23 +0200 Subject: [PATCH 08/11] Removes future-numbers. --- ContinuousTests/SingleTestRun.cs | 13 ++----------- DistTestCore/Codex/CodexAccess.cs | 6 ------ 2 files changed, 2 insertions(+), 17 deletions(-) diff --git a/ContinuousTests/SingleTestRun.cs b/ContinuousTests/SingleTestRun.cs index 3d9c03d..ff66db1 100644 --- a/ContinuousTests/SingleTestRun.cs +++ b/ContinuousTests/SingleTestRun.cs @@ -124,7 +124,7 @@ namespace ContinuousTests { ThrowFailTest(); } - OverviewLog(" > Test passed. " + FuturesInfo()); + OverviewLog(" > Test passed."); return; } } @@ -135,19 +135,10 @@ namespace ContinuousTests var exs = UnpackExceptions(exceptions); var exceptionsMessage = GetCombinedExceptionsMessage(exs); Log(exceptionsMessage); - OverviewLog($" > Test failed {FuturesInfo()}: " + exceptionsMessage); + OverviewLog($" > Test failed: " + exceptionsMessage); throw new Exception(exceptionsMessage); } - private string FuturesInfo() - { - var containers = config.CodexDeployment.CodexContainers; - var nodes = codexNodeFactory.Create(config, containers, fixtureLog, handle.Test.TimeSet); - var f = nodes.Select(n => n.GetDebugFutures().ToString()); - var msg = $"(Futures: [{string.Join(", ", f)}])"; - return msg; - } - private void DownloadClusterLogs() { var k8sFactory = new K8sFactory(); diff --git a/DistTestCore/Codex/CodexAccess.cs b/DistTestCore/Codex/CodexAccess.cs index cf6ac19..67f4263 100644 --- a/DistTestCore/Codex/CodexAccess.cs +++ b/DistTestCore/Codex/CodexAccess.cs @@ -47,12 +47,6 @@ namespace DistTestCore.Codex return result; } - public int GetDebugFutures() - { - // Some Codex images support debug/futures to count the number of open futures. - return 0; // Http().HttpGetJson("debug/futures").futures; - } - public CodexDebugThresholdBreaches GetDebugThresholdBreaches() { return Http().HttpGetJson("debug/loop"); From a26d38b74995872cfdb16aa84665ba8e0c8d3349 Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 1 Sep 2023 09:28:04 +0200 Subject: [PATCH 09/11] Moves container logs into log folder. --- ContinuousTests/SingleTestRun.cs | 4 +++- Logging/NullLog.cs | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/ContinuousTests/SingleTestRun.cs b/ContinuousTests/SingleTestRun.cs index ff66db1..1e2b3a4 100644 --- a/ContinuousTests/SingleTestRun.cs +++ b/ContinuousTests/SingleTestRun.cs @@ -142,7 +142,9 @@ namespace ContinuousTests private void DownloadClusterLogs() { var k8sFactory = new K8sFactory(); - var lifecycle = k8sFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog()); + var log = new NullLog(); + log.FullFilename = Path.Combine(config.LogPath, "NODE"); + var lifecycle = k8sFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), log); foreach (var container in config.CodexDeployment.CodexContainers) { diff --git a/Logging/NullLog.cs b/Logging/NullLog.cs index 06113b1..75d43ff 100644 --- a/Logging/NullLog.cs +++ b/Logging/NullLog.cs @@ -6,9 +6,11 @@ { } + public string FullFilename { get; set; } = "NULL"; + protected override string GetFullName() { - return "NULL"; + return FullFilename; } public override void Log(string message) From 3df4a97bba3d0039ff337ef34a970a69ed9c4d14 Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 1 Sep 2023 10:39:41 +0200 Subject: [PATCH 10/11] debugging local --- ContinuousTests/Tests/HoldMyBeerTest.cs | 12 +----------- Tests/BasicTests/ContinuousSubstitute.cs | 13 ++++++++++--- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/ContinuousTests/Tests/HoldMyBeerTest.cs b/ContinuousTests/Tests/HoldMyBeerTest.cs index 18e72f1..f54b658 100644 --- a/ContinuousTests/Tests/HoldMyBeerTest.cs +++ b/ContinuousTests/Tests/HoldMyBeerTest.cs @@ -15,17 +15,7 @@ namespace ContinuousTests.Tests [TestMoment(t: Zero)] public void UploadTestFile() { - var metadata = Configuration.CodexDeployment.Metadata; - var maxQuotaUseMb = metadata.StorageQuotaMB / 2; - var safeTTL = Math.Max(metadata.BlockTTL, metadata.BlockMI) + 30; - var runsPerTtl = Convert.ToInt32(safeTTL / RunTestEvery.TotalSeconds); - var filesizePerUploadMb = Math.Min(80, maxQuotaUseMb / runsPerTtl); - // This filesize should keep the quota below 50% of the node's max. - - var filesize = filesizePerUploadMb.MB(); - double codexDefaultBlockSize = 31 * 64 * 33; - var numberOfBlocks = Convert.ToInt64(Math.Ceiling(filesize.SizeInBytes / codexDefaultBlockSize)); - Assert.That(numberOfBlocks, Is.EqualTo(1282)); + var filesize = 80.MB(); file = FileManager.GenerateTestFile(filesize); diff --git a/Tests/BasicTests/ContinuousSubstitute.cs b/Tests/BasicTests/ContinuousSubstitute.cs index 80f77f9..0f4570a 100644 --- a/Tests/BasicTests/ContinuousSubstitute.cs +++ b/Tests/BasicTests/ContinuousSubstitute.cs @@ -4,6 +4,7 @@ using Utils; namespace Tests.BasicTests { + [Ignore("Used for debugging continuous tests")] [TestFixture] public class ContinuousSubstitute : AutoBootstrapDistTest { @@ -60,16 +61,22 @@ namespace Tests.BasicTests var checkTime = DateTime.UtcNow + TimeSpan.FromMinutes(1); var endTime = DateTime.UtcNow + TimeSpan.FromHours(10); + var uploadInterval = 0; while (DateTime.UtcNow < endTime) { CreatePeerConnectionTestHelpers().AssertFullyConnected(GetAllOnlineCodexNodes()); + CheckRoutingTables(GetAllOnlineCodexNodes()); - if (DateTime.UtcNow > checkTime) + if (uploadInterval == 0) { - CheckRoutingTables(GetAllOnlineCodexNodes()); + uploadInterval = 2; + var node = RandomUtils.PickOneRandom(nodes.ToList()); + var file = GenerateTestFile(50.MB()); + node.UploadFile(file); } + else uploadInterval--; - Thread.Sleep(5000); + Thread.Sleep(30000); } } From 23f5318dfc218a7551930dc4533d99110b0a41b3 Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 1 Sep 2023 14:40:48 +0200 Subject: [PATCH 11/11] More uploads --- Tests/BasicTests/ContinuousSubstitute.cs | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/Tests/BasicTests/ContinuousSubstitute.cs b/Tests/BasicTests/ContinuousSubstitute.cs index 0f4570a..6f4ba16 100644 --- a/Tests/BasicTests/ContinuousSubstitute.cs +++ b/Tests/BasicTests/ContinuousSubstitute.cs @@ -61,22 +61,16 @@ namespace Tests.BasicTests var checkTime = DateTime.UtcNow + TimeSpan.FromMinutes(1); var endTime = DateTime.UtcNow + TimeSpan.FromHours(10); - var uploadInterval = 0; while (DateTime.UtcNow < endTime) { CreatePeerConnectionTestHelpers().AssertFullyConnected(GetAllOnlineCodexNodes()); CheckRoutingTables(GetAllOnlineCodexNodes()); - if (uploadInterval == 0) - { - uploadInterval = 2; - var node = RandomUtils.PickOneRandom(nodes.ToList()); - var file = GenerateTestFile(50.MB()); - node.UploadFile(file); - } - else uploadInterval--; + var node = RandomUtils.PickOneRandom(nodes.ToList()); + var file = GenerateTestFile(50.MB()); + node.UploadFile(file); - Thread.Sleep(30000); + Thread.Sleep(20000); } }