Merge branch 'feature/log-continuous-downloader'

This commit is contained in:
benbierens 2023-07-21 09:31:28 +02:00
commit dc4749aca6
No known key found for this signature in database
GPG Key ID: FE44815D96D0A1AA
5 changed files with 121 additions and 2 deletions

View File

@ -9,4 +9,5 @@ dotnet run \
--min-price=1024 \
--max-collateral=1024 \
--max-duration=3600000 \
--block-ttl=120
--block-ttl=120 \
-y

View File

@ -25,6 +25,9 @@ namespace ContinuousTests
[Uniform("stop", "s", "STOPONFAIL", false, "If true, runner will stop on first test failure and download all cluster container logs. False by default.")]
public bool StopOnFailure { get; set; } = false;
[Uniform("dl-logs", "dl", "DLLOGS", false, "If true, runner will periodically download and save/append container logs to the log path.")]
public bool DownloadContainerLogs { get; set; } = false;
public CodexDeployment CodexDeployment { get; set; } = null!;
public TestRunnerLocation RunnerLocation { get; set; } = TestRunnerLocation.InternalToCluster;

View File

@ -0,0 +1,99 @@
using DistTestCore;
using DistTestCore.Codex;
using KubernetesWorkflow;
namespace ContinuousTests
{
public class ContinuousLogDownloader
{
private readonly TestLifecycle lifecycle;
private readonly CodexDeployment deployment;
private readonly string outputPath;
private readonly CancellationToken cancelToken;
public ContinuousLogDownloader(TestLifecycle lifecycle, CodexDeployment deployment, string outputPath, CancellationToken cancelToken)
{
this.lifecycle = lifecycle;
this.deployment = deployment;
this.outputPath = outputPath;
this.cancelToken = cancelToken;
}
public void Run()
{
while (!cancelToken.IsCancellationRequested)
{
UpdateLogs();
cancelToken.WaitHandle.WaitOne(TimeSpan.FromSeconds(15));
}
// After testing has stopped, we wait a little bit and fetch the logs one more time.
// If our latest fetch was not recent, interesting test-related log activity might
// not have been captured yet.
Thread.Sleep(TimeSpan.FromSeconds(10));
UpdateLogs();
}
private void UpdateLogs()
{
foreach (var container in deployment.CodexContainers)
{
UpdateLog(container);
}
}
private void UpdateLog(RunningContainer container)
{
var filepath = Path.Combine(outputPath, GetLogName(container));
if (!File.Exists(filepath))
{
File.WriteAllLines(filepath, new[] { container.Name });
}
var appender = new LogAppender(filepath);
lifecycle.CodexStarter.DownloadLog(container, appender);
}
private static string GetLogName(RunningContainer container)
{
return container.Name
.Replace("<", "")
.Replace(">", "")
+ ".log";
}
}
public class LogAppender : ILogHandler
{
private readonly string filename;
public LogAppender(string filename)
{
this.filename = filename;
}
public void Log(Stream log)
{
using var reader = new StreamReader(log);
var lines = File.ReadAllLines(filename);
var lastLine = lines.Last();
var recording = lines.Length < 3;
var line = reader.ReadLine();
while (line != null)
{
if (recording)
{
File.AppendAllLines(filename, new[] { line });
}
else
{
recording = line == lastLine;
}
line = reader.ReadLine();
}
}
}
}

View File

@ -30,6 +30,8 @@ namespace ContinuousTests
ClearAllCustomNamespaces(allTests, overviewLog);
StartLogDownloader(taskFactory);
var testLoops = allTests.Select(t => new TestLoop(taskFactory, config, overviewLog, t.GetType(), t.RunTestEvery, cancelToken)).ToArray();
foreach (var testLoop in testLoops)
@ -61,5 +63,18 @@ namespace ContinuousTests
var (workflowCreator, _) = k8SFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, config.DataPath, test.CustomK8sNamespace, new DefaultTimeSet(), log, config.RunnerLocation);
workflowCreator.CreateWorkflow().DeleteTestResources();
}
private void StartLogDownloader(TaskFactory taskFactory)
{
if (!config.DownloadContainerLogs) return;
var path = Path.Combine(config.LogPath, "containers");
if (!Directory.Exists(path)) Directory.CreateDirectory(path);
var (_, lifecycle) = k8SFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, config.DataPath, config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation);
var downloader = new ContinuousLogDownloader(lifecycle, config.CodexDeployment, path, cancelToken);
taskFactory.Run(downloader.Run);
}
}
}

View File

@ -2,4 +2,5 @@ dotnet run \
--kube-config=/opt/kubeconfig.yaml \
--codex-deployment=codex-deployment.json \
--keep=1 \
--stop=1
--stop=1 \
--dl-logs=1