status-im/cs-codex-dist-tests
2
0
Fork 0
mirror of synced 2025-02-02 11:47:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Container crash detection during start-up

Browse Source
This commit is contained in:
Ben 2024-06-19 10:39:14 +02:00
parent e1da38b6e8
commit 5ca135646a
No known key found for this signature in database
GPG Key ID: 541B9D8C9F1426A1
8 changed files with 70 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
Framework/Core/CoreInterface.cs
View File

@ -30,11 +30,11 @@ namespace Core
public IDownloadedLog DownloadLog(RunningContainer container, int? tailLines = null)
{
var workflow = entryPoint.Tools.CreateWorkflow();
var file = entryPoint.Tools.GetLog().CreateSubfile();
entryPoint.Tools.GetLog().Log($"Downloading container log for '{container.Name}' to file '{file.FullFilename}'...");
var logHandler = new LogDownloadHandler(container.Name, file);
var msg = $"Downloading container log for '{container.Name}'";
entryPoint.Tools.GetLog().Log(msg);
var logHandler = new WriteToFileLogHandler(entryPoint.Tools.GetLog(), msg);
workflow.DownloadContainerLog(container, logHandler, tailLines);
return logHandler.DownloadLog();
return new DownloadedLog(logHandler);
}
public string ExecuteContainerCommand(IHasContainer containerSource, string command, params string[] args)

7
Framework/Core/DownloadedLog.cs
View File

@ -1,4 +1,5 @@
using Logging;
using KubernetesWorkflow;
using Logging;
namespace Core
{
@ -14,9 +15,9 @@ namespace Core
{
private readonly LogFile logFile;
internal DownloadedLog(LogFile logFile)
internal DownloadedLog(WriteToFileLogHandler logHandler)
{
this.logFile = logFile;
logFile = logHandler.LogFile;
}
public void IterateLines(Action<string> action)

28
Framework/Core/LogDownloadHandler.cs
View File

@ -1,28 +0,0 @@
using KubernetesWorkflow;
using Logging;
namespace Core
{
internal class LogDownloadHandler : LogHandler, ILogHandler
{
private readonly LogFile log;
internal LogDownloadHandler(string description, LogFile log)
{
this.log = log;
log.Write($"{description} -->> {log.FullFilename}");
log.WriteRaw(description);
}
internal IDownloadedLog DownloadLog()
{
return new DownloadedLog(log);
}
protected override void ProcessLine(string line)
{
log.WriteRaw(line);
}
}
}

7
Framework/KubernetesWorkflow/CrashWatcher.cs
View File

@ -11,7 +11,6 @@ namespace KubernetesWorkflow
private readonly string podName;
private readonly string recipeName;
private readonly string k8sNamespace;
private ILogHandler? logHandler;
private CancellationTokenSource cts;
private Task? worker;
private Exception? workerException;
@ -27,11 +26,10 @@ namespace KubernetesWorkflow
cts = new CancellationTokenSource();
}
public void Start(ILogHandler logHandler)
public void Start()
{
if (worker != null) throw new InvalidOperationException();
this.logHandler = logHandler;
cts = new CancellationTokenSource();
worker = Task.Run(Worker);
}
@ -93,7 +91,8 @@ namespace KubernetesWorkflow
private void DownloadCrashedContainerLogs(Kubernetes client)
{
using var stream = client.ReadNamespacedPodLog(podName, k8sNamespace, recipeName, previous: true);
logHandler!.Log(stream);
var handler = new WriteToFileLogHandler(log, "Crash detected for " + containerName);
handler.Log(stream);
}
}
}

34
Framework/KubernetesWorkflow/K8sController.cs
View File

@ -45,7 +45,7 @@ namespace KubernetesWorkflow
public void WaitUntilOnline(RunningContainer container)
{
WaitUntilDeploymentOnline(container.Recipe.Name);
WaitUntilDeploymentOnline(container);
}
public PodInfo GetPodInfo(RunningDeployment deployment)
@ -64,14 +64,14 @@ namespace KubernetesWorkflow
if (waitTillStopped) WaitUntilPodsForDeploymentAreOffline(startResult.Deployment);
}
public void DownloadPodLog(RunningContainer container, ILogHandler logHandler, int? tailLines)
public void DownloadPodLog(RunningContainer container, ILogHandler logHandler, int? tailLines, bool? previous)
{
log.Debug();
var podName = GetPodName(container);
var recipeName = container.Recipe.Name;
using var stream = client.Run(c => c.ReadNamespacedPodLog(podName, K8sNamespace, recipeName, tailLines: tailLines));
using var stream = client.Run(c => c.ReadNamespacedPodLog(podName, K8sNamespace, recipeName, tailLines: tailLines, previous: previous));
logHandler.Log(stream);
}
@ -879,15 +879,39 @@ namespace KubernetesWorkflow
WaitUntil(() => !IsNamespaceOnline(@namespace), nameof(WaitUntilNamespaceDeleted));
}
private void WaitUntilDeploymentOnline(string deploymentName)
private void WaitUntilDeploymentOnline(RunningContainer container)
{
WaitUntil(() =>
{
var deployment = client.Run(c => c.ReadNamespacedDeployment(deploymentName, K8sNamespace));
CheckForCrash(container);
var deployment = client.Run(c => c.ReadNamespacedDeployment(container.Recipe.Name, K8sNamespace));
return deployment?.Status.AvailableReplicas != null && deployment.Status.AvailableReplicas > 0;
}, nameof(WaitUntilDeploymentOnline));
}
private void CheckForCrash(RunningContainer container)
{
var deploymentName = container.Recipe.Name;
var podName = GetPodName(container);
var podInfo = client.Run(c => c.ReadNamespacedPod(podName, K8sNamespace));
if (podInfo == null) return;
if (podInfo.Status == null) return;
if (podInfo.Status.ContainerStatuses == null) return;
var result = podInfo.Status.ContainerStatuses.Any(c => c.RestartCount > 0);
if (result)
{
var msg = $"Pod crash detected for deployment {deploymentName} (pod:{podName})";
log.Error(msg);
DownloadPodLog(container, new WriteToFileLogHandler(log, msg), tailLines: null, previous: true);
throw new Exception(msg);
}
}
private void WaitUntilDeploymentOffline(string deploymentName)
{
WaitUntil(() =>

25
Framework/KubernetesWorkflow/LogHandler.cs
View File

@ -1,4 +1,6 @@
namespace KubernetesWorkflow
using Logging;
namespace KubernetesWorkflow
{
public interface ILogHandler
{
@ -20,4 +22,25 @@
protected abstract void ProcessLine(string line);
}
public class WriteToFileLogHandler : LogHandler, ILogHandler
{
public WriteToFileLogHandler(ILog sourceLog, string description)
{
LogFile = sourceLog.CreateSubfile();
var msg = $"{description} -->> {LogFile.FullFilename}";
sourceLog.Log(msg);
LogFile.Write(msg);
LogFile.WriteRaw(description);
}
public LogFile LogFile { get; }
protected override void ProcessLine(string line)
{
LogFile.WriteRaw(line);
}
}
}

6
Framework/KubernetesWorkflow/StartupWorkflow.cs
View File

@ -15,7 +15,7 @@ namespace KubernetesWorkflow
PodInfo GetPodInfo(RunningPod pod);
CrashWatcher CreateCrashWatcher(RunningContainer container);
void Stop(RunningPod pod, bool waitTillStopped);
void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null);
void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null, bool? previous = null);
string ExecuteCommand(RunningContainer container, string command, params string[] args);
void DeleteNamespace(bool wait);
void DeleteNamespacesStartingWith(string namespacePrefix, bool wait);
@ -106,11 +106,11 @@ namespace KubernetesWorkflow
});
}
public void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null)
public void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null, bool? previous = null)
{
K8s(controller =>
{
controller.DownloadPodLog(container, logHandler, tailLines);
controller.DownloadPodLog(container, logHandler, tailLines, previous);
});
}

26
ProjectPlugins/CodexPlugin/CodexAccess.cs
View File

@ -8,12 +8,11 @@ using Utils;
namespace CodexPlugin
{
public class CodexAccess : ILogHandler
public class CodexAccess
{
private readonly ILog log;
private readonly IPluginTools tools;
private readonly Mapper mapper = new Mapper();
private bool hasContainerCrashed;
public CodexAccess(IPluginTools tools, RunningPod container, CrashWatcher crashWatcher)
{
@ -21,9 +20,8 @@ namespace CodexPlugin
log = tools.GetLog();
Container = container;
CrashWatcher = crashWatcher;
hasContainerCrashed = false;
CrashWatcher.Start(this);
CrashWatcher.Start();
}
public RunningPod Container { get; }
@ -209,25 +207,7 @@ namespace CodexPlugin
private void CheckContainerCrashed(HttpClient client)
{
if (hasContainerCrashed) throw new Exception($"Container {GetName()} has crashed.");
}
void ILogHandler.Log(Stream crashLog)
{
var file = log.CreateSubfile();
Log($"Downloading log to '{file.FullFilename}'...");
file.Write($"Container log for {Container.Name}.");
using var reader = new StreamReader(crashLog);
var line = reader.ReadLine();
while (line != null)
{
file.Write(line);
line = reader.ReadLine();
}
Log("Container log successfully downloaded.");
hasContainerCrashed = true;
if (CrashWatcher.HasContainerCrashed()) throw new Exception($"Container {GetName()} has crashed.");
}
private Retry CreateRetryConfig(string description, Action<Failure> onFailure)