diff --git a/CodexNetDeployer/deploy-continuous-testnet.sh b/CodexNetDeployer/deploy-continuous-testnet.sh index 65d5666..5337922 100644 --- a/CodexNetDeployer/deploy-continuous-testnet.sh +++ b/CodexNetDeployer/deploy-continuous-testnet.sh @@ -9,4 +9,5 @@ dotnet run \ --min-price=1024 \ --max-collateral=1024 \ --max-duration=3600000 \ - --block-ttl=120 + --block-ttl=120 \ + -y diff --git a/ContinuousTests/Configuration.cs b/ContinuousTests/Configuration.cs index 1cfb697..b0cebbd 100644 --- a/ContinuousTests/Configuration.cs +++ b/ContinuousTests/Configuration.cs @@ -25,6 +25,9 @@ namespace ContinuousTests [Uniform("stop", "s", "STOPONFAIL", false, "If true, runner will stop on first test failure and download all cluster container logs. False by default.")] public bool StopOnFailure { get; set; } = false; + [Uniform("dl-logs", "dl", "DLLOGS", false, "If true, runner will periodically download and save/append container logs to the log path.")] + public bool DownloadContainerLogs { get; set; } = false; + public CodexDeployment CodexDeployment { get; set; } = null!; public TestRunnerLocation RunnerLocation { get; set; } = TestRunnerLocation.InternalToCluster; diff --git a/ContinuousTests/ContinuousLogDownloader.cs b/ContinuousTests/ContinuousLogDownloader.cs new file mode 100644 index 0000000..f5e0161 --- /dev/null +++ b/ContinuousTests/ContinuousLogDownloader.cs @@ -0,0 +1,99 @@ +using DistTestCore; +using DistTestCore.Codex; +using KubernetesWorkflow; + +namespace ContinuousTests +{ + public class ContinuousLogDownloader + { + private readonly TestLifecycle lifecycle; + private readonly CodexDeployment deployment; + private readonly string outputPath; + private readonly CancellationToken cancelToken; + + public ContinuousLogDownloader(TestLifecycle lifecycle, CodexDeployment deployment, string outputPath, CancellationToken cancelToken) + { + this.lifecycle = lifecycle; + this.deployment = deployment; + this.outputPath = outputPath; + this.cancelToken = cancelToken; + } + + public void Run() + { + while (!cancelToken.IsCancellationRequested) + { + UpdateLogs(); + + cancelToken.WaitHandle.WaitOne(TimeSpan.FromSeconds(15)); + } + + // After testing has stopped, we wait a little bit and fetch the logs one more time. + // If our latest fetch was not recent, interesting test-related log activity might + // not have been captured yet. + Thread.Sleep(TimeSpan.FromSeconds(10)); + UpdateLogs(); + } + + private void UpdateLogs() + { + foreach (var container in deployment.CodexContainers) + { + UpdateLog(container); + } + } + + private void UpdateLog(RunningContainer container) + { + var filepath = Path.Combine(outputPath, GetLogName(container)); + if (!File.Exists(filepath)) + { + File.WriteAllLines(filepath, new[] { container.Name }); + } + + var appender = new LogAppender(filepath); + + lifecycle.CodexStarter.DownloadLog(container, appender); + } + + private static string GetLogName(RunningContainer container) + { + return container.Name + .Replace("<", "") + .Replace(">", "") + + ".log"; + } + } + + public class LogAppender : ILogHandler + { + private readonly string filename; + + public LogAppender(string filename) + { + this.filename = filename; + } + + public void Log(Stream log) + { + using var reader = new StreamReader(log); + var lines = File.ReadAllLines(filename); + var lastLine = lines.Last(); + var recording = lines.Length < 3; + var line = reader.ReadLine(); + while (line != null) + { + if (recording) + { + File.AppendAllLines(filename, new[] { line }); + } + else + { + recording = line == lastLine; + } + + line = reader.ReadLine(); + } + } + } +} diff --git a/ContinuousTests/ContinuousTestRunner.cs b/ContinuousTests/ContinuousTestRunner.cs index d7ac7d9..985707a 100644 --- a/ContinuousTests/ContinuousTestRunner.cs +++ b/ContinuousTests/ContinuousTestRunner.cs @@ -30,6 +30,8 @@ namespace ContinuousTests ClearAllCustomNamespaces(allTests, overviewLog); + StartLogDownloader(taskFactory); + var testLoops = allTests.Select(t => new TestLoop(taskFactory, config, overviewLog, t.GetType(), t.RunTestEvery, cancelToken)).ToArray(); foreach (var testLoop in testLoops) @@ -61,5 +63,18 @@ namespace ContinuousTests var (workflowCreator, _) = k8SFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, config.DataPath, test.CustomK8sNamespace, new DefaultTimeSet(), log, config.RunnerLocation); workflowCreator.CreateWorkflow().DeleteTestResources(); } + + private void StartLogDownloader(TaskFactory taskFactory) + { + if (!config.DownloadContainerLogs) return; + + var path = Path.Combine(config.LogPath, "containers"); + if (!Directory.Exists(path)) Directory.CreateDirectory(path); + + var (_, lifecycle) = k8SFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, config.DataPath, config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation); + var downloader = new ContinuousLogDownloader(lifecycle, config.CodexDeployment, path, cancelToken); + + taskFactory.Run(downloader.Run); + } } } diff --git a/ContinuousTests/run.sh b/ContinuousTests/run.sh index 643bffd..67c9419 100644 --- a/ContinuousTests/run.sh +++ b/ContinuousTests/run.sh @@ -2,4 +2,5 @@ dotnet run \ --kube-config=/opt/kubeconfig.yaml \ --codex-deployment=codex-deployment.json \ --keep=1 \ - --stop=1 + --stop=1 \ + --dl-logs=1