From 2dd5775296d00e9087a9b8dcd7cce6caca0364ad Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 20 Jul 2023 10:52:19 +0200 Subject: [PATCH 1/5] Very simple log appender --- CodexNetDownloader/Configuration.cs | 3 + CodexNetDownloader/ContinuousLogDownloader.cs | 84 +++++++++++++++++++ CodexNetDownloader/Program.cs | 15 +++- 3 files changed, 99 insertions(+), 3 deletions(-) create mode 100644 CodexNetDownloader/ContinuousLogDownloader.cs diff --git a/CodexNetDownloader/Configuration.cs b/CodexNetDownloader/Configuration.cs index 008dfbf..feecf24 100644 --- a/CodexNetDownloader/Configuration.cs +++ b/CodexNetDownloader/Configuration.cs @@ -15,6 +15,9 @@ namespace CodexNetDownloader [Uniform("kube-config", "kc", "KUBECONFIG", true, "Path to Kubeconfig file. Use 'null' (default) to use local cluster.")] public string KubeConfigFile { get; set; } = "null"; + [Uniform("continuous", "c", "CONTINUOUS", false, "If true, will continuously download and append log files.")] + public bool Continuous { get; set; } = false; + public CodexDeployment CodexDeployment { get; set; } = null!; public TestRunnerLocation RunnerLocation { get; set; } = TestRunnerLocation.InternalToCluster; diff --git a/CodexNetDownloader/ContinuousLogDownloader.cs b/CodexNetDownloader/ContinuousLogDownloader.cs new file mode 100644 index 0000000..074e197 --- /dev/null +++ b/CodexNetDownloader/ContinuousLogDownloader.cs @@ -0,0 +1,84 @@ +using DistTestCore; +using KubernetesWorkflow; + +namespace CodexNetDownloader +{ + public class ContinuousLogDownloader + { + private readonly TestLifecycle lifecycle; + private readonly Configuration config; + + public ContinuousLogDownloader(TestLifecycle lifecycle, Configuration config) + { + this.lifecycle = lifecycle; + this.config = config; + } + + public void Run() + { + while (true) + { + UpdateLogs(); + + Thread.Sleep(TimeSpan.FromSeconds(30)); + } + } + + private void UpdateLogs() + { + Console.WriteLine("Updating logs..."); + foreach (var container in config.CodexDeployment.CodexContainers) + { + UpdateLog(container); + } + } + + private void UpdateLog(RunningContainer container) + { + var filepath = Path.Combine(config.OutputPath, GetLogName(container)); + if (!File.Exists(filepath)) File.WriteAllLines(filepath, new[] { "" }); + + var appender = new LogAppender(filepath); + + lifecycle.CodexStarter.DownloadLog(container, appender); + } + + private static string GetLogName(RunningContainer container) + { + return container.Name + .Replace("<","") + .Replace(">", "") + + ".log"; + } + } + + public class LogAppender : ILogHandler + { + private readonly string filename; + + public LogAppender(string filename) + { + this.filename = filename; + } + + public void Log(Stream log) + { + using var reader = new StreamReader(log); + var currentLines = File.ReadAllLines(filename); + var line = reader.ReadLine(); + while (line != null) + { + AppendLineIfNew(line, currentLines); + line = reader.ReadLine(); + } + } + + private void AppendLineIfNew(string line, string[] currentLines) + { + if (!currentLines.Contains(line)) + { + File.AppendAllLines(filename, new[] { line }); + } + } + } +} diff --git a/CodexNetDownloader/Program.cs b/CodexNetDownloader/Program.cs index f52a1d2..9a631cc 100644 --- a/CodexNetDownloader/Program.cs +++ b/CodexNetDownloader/Program.cs @@ -1,4 +1,5 @@ using ArgsUniform; +using CodexNetDownloader; using ContinuousTests; using DistTestCore; using DistTestCore.Codex; @@ -27,12 +28,20 @@ public class Program var k8sFactory = new K8sFactory(); var (_, lifecycle) = k8sFactory.CreateFacilities(config.KubeConfigFile, config.OutputPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation); - foreach (var container in config.CodexDeployment.CodexContainers) + if (config.Continuous) { - lifecycle.DownloadLog(container); + var dl = new ContinuousLogDownloader(lifecycle, config); + dl.Run(); } + else + { + foreach (var container in config.CodexDeployment.CodexContainers) + { + lifecycle.DownloadLog(container); + } - Console.WriteLine("Done!"); + Console.WriteLine("Done!"); + } } private static CodexDeployment ParseCodexDeploymentJson(string filename) From 9de7e1d3208d972adcfb7a4768b1b15e46586575 Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 20 Jul 2023 13:37:11 +0200 Subject: [PATCH 2/5] Moves log downloader into continous test runner --- CodexNetDownloader/Configuration.cs | 9 ++-- CodexNetDownloader/Program.cs | 15 ++---- ContinuousTests/Configuration.cs | 3 ++ .../ContinuousLogDownloader.cs | 51 +++++++++++-------- ContinuousTests/ContinuousTestRunner.cs | 15 ++++++ ContinuousTests/run.sh | 3 +- 6 files changed, 56 insertions(+), 40 deletions(-) rename {CodexNetDownloader => ContinuousTests}/ContinuousLogDownloader.cs (53%) diff --git a/CodexNetDownloader/Configuration.cs b/CodexNetDownloader/Configuration.cs index feecf24..4073e9a 100644 --- a/CodexNetDownloader/Configuration.cs +++ b/CodexNetDownloader/Configuration.cs @@ -10,16 +10,13 @@ namespace CodexNetDownloader public string OutputPath { get; set; } = "output"; [Uniform("codex-deployment", "c", "CODEXDEPLOYMENT", true, "Path to codex-deployment JSON file.")] - public string CodexDeploymentJson { get; set; } = string.Empty; + public string CodexDeploymentJson { get; set; } = string.Empty; // @"d:\Projects\cs-codex-dist-tests\CodexNetDownloader\codex-deployment.json"; [Uniform("kube-config", "kc", "KUBECONFIG", true, "Path to Kubeconfig file. Use 'null' (default) to use local cluster.")] - public string KubeConfigFile { get; set; } = "null"; - - [Uniform("continuous", "c", "CONTINUOUS", false, "If true, will continuously download and append log files.")] - public bool Continuous { get; set; } = false; + public string KubeConfigFile { get; set; } = string.Empty;// @"c:\Users\Ben\.kube\codex-tests-ams3-dev-kubeconfig.yaml"; public CodexDeployment CodexDeployment { get; set; } = null!; - public TestRunnerLocation RunnerLocation { get; set; } = TestRunnerLocation.InternalToCluster; + public TestRunnerLocation RunnerLocation { get; set; } = TestRunnerLocation.ExternalToCluster; } } diff --git a/CodexNetDownloader/Program.cs b/CodexNetDownloader/Program.cs index 9a631cc..f52a1d2 100644 --- a/CodexNetDownloader/Program.cs +++ b/CodexNetDownloader/Program.cs @@ -1,5 +1,4 @@ using ArgsUniform; -using CodexNetDownloader; using ContinuousTests; using DistTestCore; using DistTestCore.Codex; @@ -28,20 +27,12 @@ public class Program var k8sFactory = new K8sFactory(); var (_, lifecycle) = k8sFactory.CreateFacilities(config.KubeConfigFile, config.OutputPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation); - if (config.Continuous) + foreach (var container in config.CodexDeployment.CodexContainers) { - var dl = new ContinuousLogDownloader(lifecycle, config); - dl.Run(); + lifecycle.DownloadLog(container); } - else - { - foreach (var container in config.CodexDeployment.CodexContainers) - { - lifecycle.DownloadLog(container); - } - Console.WriteLine("Done!"); - } + Console.WriteLine("Done!"); } private static CodexDeployment ParseCodexDeploymentJson(string filename) diff --git a/ContinuousTests/Configuration.cs b/ContinuousTests/Configuration.cs index dd122cb..ee93984 100644 --- a/ContinuousTests/Configuration.cs +++ b/ContinuousTests/Configuration.cs @@ -25,6 +25,9 @@ namespace ContinuousTests [Uniform("stop", "s", "STOPONFAIL", false, "If true, runner will stop on first test failure and download all cluster container logs. False by default.")] public bool StopOnFailure { get; set; } = false; + [Uniform("dl-logs", "dl", "DLLOGS", false, "If true, runner will periodically download and save/append container logs to the log path.")] + public bool DownloadContainerLogs { get; set; } = false; + public CodexDeployment CodexDeployment { get; set; } = null!; public TestRunnerLocation RunnerLocation { get; set; } = TestRunnerLocation.InternalToCluster; diff --git a/CodexNetDownloader/ContinuousLogDownloader.cs b/ContinuousTests/ContinuousLogDownloader.cs similarity index 53% rename from CodexNetDownloader/ContinuousLogDownloader.cs rename to ContinuousTests/ContinuousLogDownloader.cs index 074e197..7fdc880 100644 --- a/CodexNetDownloader/ContinuousLogDownloader.cs +++ b/ContinuousTests/ContinuousLogDownloader.cs @@ -1,33 +1,37 @@ using DistTestCore; +using DistTestCore.Codex; using KubernetesWorkflow; -namespace CodexNetDownloader +namespace ContinuousTests { public class ContinuousLogDownloader { private readonly TestLifecycle lifecycle; - private readonly Configuration config; + private readonly CodexDeployment deployment; + private readonly string outputPath; + private readonly CancellationToken cancelToken; - public ContinuousLogDownloader(TestLifecycle lifecycle, Configuration config) + public ContinuousLogDownloader(TestLifecycle lifecycle, CodexDeployment deployment, string outputPath, CancellationToken cancelToken) { this.lifecycle = lifecycle; - this.config = config; + this.deployment = deployment; + this.outputPath = outputPath; + this.cancelToken = cancelToken; } public void Run() { - while (true) + while (!cancelToken.IsCancellationRequested) { UpdateLogs(); - Thread.Sleep(TimeSpan.FromSeconds(30)); + cancelToken.WaitHandle.WaitOne(TimeSpan.FromSeconds(15)); } } private void UpdateLogs() { - Console.WriteLine("Updating logs..."); - foreach (var container in config.CodexDeployment.CodexContainers) + foreach (var container in deployment.CodexContainers) { UpdateLog(container); } @@ -35,8 +39,11 @@ namespace CodexNetDownloader private void UpdateLog(RunningContainer container) { - var filepath = Path.Combine(config.OutputPath, GetLogName(container)); - if (!File.Exists(filepath)) File.WriteAllLines(filepath, new[] { "" }); + var filepath = Path.Combine(outputPath, GetLogName(container)); + if (!File.Exists(filepath)) + { + File.WriteAllLines(filepath, new[] { container.Name }); + } var appender = new LogAppender(filepath); @@ -46,7 +53,7 @@ namespace CodexNetDownloader private static string GetLogName(RunningContainer container) { return container.Name - .Replace("<","") + .Replace("<", "") .Replace(">", "") + ".log"; } @@ -64,20 +71,22 @@ namespace CodexNetDownloader public void Log(Stream log) { using var reader = new StreamReader(log); - var currentLines = File.ReadAllLines(filename); + var lines = File.ReadAllLines(filename); + var lastLine = lines.Last(); + var recording = lines.Length < 3; var line = reader.ReadLine(); while (line != null) { - AppendLineIfNew(line, currentLines); - line = reader.ReadLine(); - } - } + if (recording) + { + File.AppendAllLines(filename, new[] { line }); + } + else + { + recording = line == lastLine; + } - private void AppendLineIfNew(string line, string[] currentLines) - { - if (!currentLines.Contains(line)) - { - File.AppendAllLines(filename, new[] { line }); + line = reader.ReadLine(); } } } diff --git a/ContinuousTests/ContinuousTestRunner.cs b/ContinuousTests/ContinuousTestRunner.cs index 5be4ff8..9028090 100644 --- a/ContinuousTests/ContinuousTestRunner.cs +++ b/ContinuousTests/ContinuousTestRunner.cs @@ -30,6 +30,8 @@ namespace ContinuousTests ClearAllCustomNamespaces(allTests, overviewLog); + StartLogDownloader(taskFactory); + var testLoops = allTests.Select(t => new TestLoop(taskFactory, config, overviewLog, t.GetType(), t.RunTestEvery, cancelToken)).ToArray(); foreach (var testLoop in testLoops) @@ -61,5 +63,18 @@ namespace ContinuousTests var (workflowCreator, _) = k8SFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, config.DataPath, test.CustomK8sNamespace, new DefaultTimeSet(), log, config.RunnerLocation); workflowCreator.CreateWorkflow().DeleteTestResources(); } + + private void StartLogDownloader(TaskFactory taskFactory) + { + if (!config.DownloadContainerLogs) return; + + var path = Path.Combine(config.LogPath, "containers"); + if (!Directory.Exists(path)) Directory.CreateDirectory(path); + + var (_, lifecycle) = k8SFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, config.DataPath, config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation); + var downloader = new ContinuousLogDownloader(lifecycle, config.CodexDeployment, path, cancelToken); + + taskFactory.Run(downloader.Run); + } } } diff --git a/ContinuousTests/run.sh b/ContinuousTests/run.sh index 643bffd..67c9419 100644 --- a/ContinuousTests/run.sh +++ b/ContinuousTests/run.sh @@ -2,4 +2,5 @@ dotnet run \ --kube-config=/opt/kubeconfig.yaml \ --codex-deployment=codex-deployment.json \ --keep=1 \ - --stop=1 + --stop=1 \ + --dl-logs=1 From c762e303311be7a4f3d76705926a4ad91d45c8f6 Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 21 Jul 2023 07:58:10 +0200 Subject: [PATCH 3/5] autoconfirms deploy --- CodexNetDeployer/deploy-continuous-testnet.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CodexNetDeployer/deploy-continuous-testnet.sh b/CodexNetDeployer/deploy-continuous-testnet.sh index 65d5666..5337922 100644 --- a/CodexNetDeployer/deploy-continuous-testnet.sh +++ b/CodexNetDeployer/deploy-continuous-testnet.sh @@ -9,4 +9,5 @@ dotnet run \ --min-price=1024 \ --max-collateral=1024 \ --max-duration=3600000 \ - --block-ttl=120 + --block-ttl=120 \ + -y From 8b3589a48903f1366e5e62f8f41f08bc7d46d51a Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 21 Jul 2023 09:00:07 +0200 Subject: [PATCH 4/5] Adds one post-test log download --- ContinuousTests/ContinuousLogDownloader.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ContinuousTests/ContinuousLogDownloader.cs b/ContinuousTests/ContinuousLogDownloader.cs index 7fdc880..f5e0161 100644 --- a/ContinuousTests/ContinuousLogDownloader.cs +++ b/ContinuousTests/ContinuousLogDownloader.cs @@ -27,6 +27,12 @@ namespace ContinuousTests cancelToken.WaitHandle.WaitOne(TimeSpan.FromSeconds(15)); } + + // After testing has stopped, we wait a little bit and fetch the logs one more time. + // If our latest fetch was not recent, interesting test-related log activity might + // not have been captured yet. + Thread.Sleep(TimeSpan.FromSeconds(10)); + UpdateLogs(); } private void UpdateLogs() From ce7ad1ba706d91fd805369fdb75f205a1b23f5dc Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 21 Jul 2023 09:29:14 +0200 Subject: [PATCH 5/5] restores downloader default config --- CodexNetDownloader/Configuration.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CodexNetDownloader/Configuration.cs b/CodexNetDownloader/Configuration.cs index 4073e9a..008dfbf 100644 --- a/CodexNetDownloader/Configuration.cs +++ b/CodexNetDownloader/Configuration.cs @@ -10,13 +10,13 @@ namespace CodexNetDownloader public string OutputPath { get; set; } = "output"; [Uniform("codex-deployment", "c", "CODEXDEPLOYMENT", true, "Path to codex-deployment JSON file.")] - public string CodexDeploymentJson { get; set; } = string.Empty; // @"d:\Projects\cs-codex-dist-tests\CodexNetDownloader\codex-deployment.json"; + public string CodexDeploymentJson { get; set; } = string.Empty; [Uniform("kube-config", "kc", "KUBECONFIG", true, "Path to Kubeconfig file. Use 'null' (default) to use local cluster.")] - public string KubeConfigFile { get; set; } = string.Empty;// @"c:\Users\Ben\.kube\codex-tests-ams3-dev-kubeconfig.yaml"; + public string KubeConfigFile { get; set; } = "null"; public CodexDeployment CodexDeployment { get; set; } = null!; - public TestRunnerLocation RunnerLocation { get; set; } = TestRunnerLocation.ExternalToCluster; + public TestRunnerLocation RunnerLocation { get; set; } = TestRunnerLocation.InternalToCluster; } }