diff --git a/Framework/Core/CoreInterface.cs b/Framework/Core/CoreInterface.cs index b43c1c2..81483a8 100644 --- a/Framework/Core/CoreInterface.cs +++ b/Framework/Core/CoreInterface.cs @@ -30,11 +30,11 @@ namespace Core public IDownloadedLog DownloadLog(RunningContainer container, int? tailLines = null) { var workflow = entryPoint.Tools.CreateWorkflow(); - var file = entryPoint.Tools.GetLog().CreateSubfile(); - entryPoint.Tools.GetLog().Log($"Downloading container log for '{container.Name}' to file '{file.FullFilename}'..."); - var logHandler = new LogDownloadHandler(container.Name, file); + var msg = $"Downloading container log for '{container.Name}'"; + entryPoint.Tools.GetLog().Log(msg); + var logHandler = new WriteToFileLogHandler(entryPoint.Tools.GetLog(), msg); workflow.DownloadContainerLog(container, logHandler, tailLines); - return logHandler.DownloadLog(); + return new DownloadedLog(logHandler); } public string ExecuteContainerCommand(IHasContainer containerSource, string command, params string[] args) diff --git a/Framework/Core/DownloadedLog.cs b/Framework/Core/DownloadedLog.cs index fa4d559..3979f3e 100644 --- a/Framework/Core/DownloadedLog.cs +++ b/Framework/Core/DownloadedLog.cs @@ -1,4 +1,5 @@ -using Logging; +using KubernetesWorkflow; +using Logging; namespace Core { @@ -14,9 +15,9 @@ namespace Core { private readonly LogFile logFile; - internal DownloadedLog(LogFile logFile) + internal DownloadedLog(WriteToFileLogHandler logHandler) { - this.logFile = logFile; + logFile = logHandler.LogFile; } public void IterateLines(Action action) diff --git a/Framework/Core/LogDownloadHandler.cs b/Framework/Core/LogDownloadHandler.cs deleted file mode 100644 index e1736ed..0000000 --- a/Framework/Core/LogDownloadHandler.cs +++ /dev/null @@ -1,28 +0,0 @@ -using KubernetesWorkflow; -using Logging; - -namespace Core -{ - internal class LogDownloadHandler : LogHandler, ILogHandler - { - private readonly LogFile log; - - internal LogDownloadHandler(string description, LogFile log) - { - this.log = log; - - log.Write($"{description} -->> {log.FullFilename}"); - log.WriteRaw(description); - } - - internal IDownloadedLog DownloadLog() - { - return new DownloadedLog(log); - } - - protected override void ProcessLine(string line) - { - log.WriteRaw(line); - } - } -} diff --git a/Framework/KubernetesWorkflow/CrashWatcher.cs b/Framework/KubernetesWorkflow/CrashWatcher.cs index 57e30eb..7f38cfb 100644 --- a/Framework/KubernetesWorkflow/CrashWatcher.cs +++ b/Framework/KubernetesWorkflow/CrashWatcher.cs @@ -11,7 +11,6 @@ namespace KubernetesWorkflow private readonly string podName; private readonly string recipeName; private readonly string k8sNamespace; - private ILogHandler? logHandler; private CancellationTokenSource cts; private Task? worker; private Exception? workerException; @@ -27,11 +26,10 @@ namespace KubernetesWorkflow cts = new CancellationTokenSource(); } - public void Start(ILogHandler logHandler) + public void Start() { if (worker != null) throw new InvalidOperationException(); - this.logHandler = logHandler; cts = new CancellationTokenSource(); worker = Task.Run(Worker); } @@ -93,7 +91,8 @@ namespace KubernetesWorkflow private void DownloadCrashedContainerLogs(Kubernetes client) { using var stream = client.ReadNamespacedPodLog(podName, k8sNamespace, recipeName, previous: true); - logHandler!.Log(stream); + var handler = new WriteToFileLogHandler(log, "Crash detected for " + containerName); + handler.Log(stream); } } } diff --git a/Framework/KubernetesWorkflow/K8sController.cs b/Framework/KubernetesWorkflow/K8sController.cs index 00c7563..e5fac7c 100644 --- a/Framework/KubernetesWorkflow/K8sController.cs +++ b/Framework/KubernetesWorkflow/K8sController.cs @@ -45,7 +45,7 @@ namespace KubernetesWorkflow public void WaitUntilOnline(RunningContainer container) { - WaitUntilDeploymentOnline(container.Recipe.Name); + WaitUntilDeploymentOnline(container); } public PodInfo GetPodInfo(RunningDeployment deployment) @@ -64,14 +64,14 @@ namespace KubernetesWorkflow if (waitTillStopped) WaitUntilPodsForDeploymentAreOffline(startResult.Deployment); } - public void DownloadPodLog(RunningContainer container, ILogHandler logHandler, int? tailLines) + public void DownloadPodLog(RunningContainer container, ILogHandler logHandler, int? tailLines, bool? previous) { log.Debug(); var podName = GetPodName(container); var recipeName = container.Recipe.Name; - using var stream = client.Run(c => c.ReadNamespacedPodLog(podName, K8sNamespace, recipeName, tailLines: tailLines)); + using var stream = client.Run(c => c.ReadNamespacedPodLog(podName, K8sNamespace, recipeName, tailLines: tailLines, previous: previous)); logHandler.Log(stream); } @@ -879,15 +879,39 @@ namespace KubernetesWorkflow WaitUntil(() => !IsNamespaceOnline(@namespace), nameof(WaitUntilNamespaceDeleted)); } - private void WaitUntilDeploymentOnline(string deploymentName) + private void WaitUntilDeploymentOnline(RunningContainer container) { WaitUntil(() => { - var deployment = client.Run(c => c.ReadNamespacedDeployment(deploymentName, K8sNamespace)); + CheckForCrash(container); + + var deployment = client.Run(c => c.ReadNamespacedDeployment(container.Recipe.Name, K8sNamespace)); return deployment?.Status.AvailableReplicas != null && deployment.Status.AvailableReplicas > 0; }, nameof(WaitUntilDeploymentOnline)); } + private void CheckForCrash(RunningContainer container) + { + var deploymentName = container.Recipe.Name; + var podName = GetPodName(container); + + var podInfo = client.Run(c => c.ReadNamespacedPod(podName, K8sNamespace)); + if (podInfo == null) return; + if (podInfo.Status == null) return; + if (podInfo.Status.ContainerStatuses == null) return; + + var result = podInfo.Status.ContainerStatuses.Any(c => c.RestartCount > 0); + if (result) + { + var msg = $"Pod crash detected for deployment {deploymentName} (pod:{podName})"; + log.Error(msg); + + DownloadPodLog(container, new WriteToFileLogHandler(log, msg), tailLines: null, previous: true); + + throw new Exception(msg); + } + } + private void WaitUntilDeploymentOffline(string deploymentName) { WaitUntil(() => diff --git a/Framework/KubernetesWorkflow/LogHandler.cs b/Framework/KubernetesWorkflow/LogHandler.cs index 77e5746..f185ba1 100644 --- a/Framework/KubernetesWorkflow/LogHandler.cs +++ b/Framework/KubernetesWorkflow/LogHandler.cs @@ -1,4 +1,6 @@ -namespace KubernetesWorkflow +using Logging; + +namespace KubernetesWorkflow { public interface ILogHandler { @@ -20,4 +22,25 @@ protected abstract void ProcessLine(string line); } + + public class WriteToFileLogHandler : LogHandler, ILogHandler + { + public WriteToFileLogHandler(ILog sourceLog, string description) + { + LogFile = sourceLog.CreateSubfile(); + + var msg = $"{description} -->> {LogFile.FullFilename}"; + sourceLog.Log(msg); + + LogFile.Write(msg); + LogFile.WriteRaw(description); + } + + public LogFile LogFile { get; } + + protected override void ProcessLine(string line) + { + LogFile.WriteRaw(line); + } + } } diff --git a/Framework/KubernetesWorkflow/StartupWorkflow.cs b/Framework/KubernetesWorkflow/StartupWorkflow.cs index 3a7326d..6a4d50c 100644 --- a/Framework/KubernetesWorkflow/StartupWorkflow.cs +++ b/Framework/KubernetesWorkflow/StartupWorkflow.cs @@ -15,7 +15,7 @@ namespace KubernetesWorkflow PodInfo GetPodInfo(RunningPod pod); CrashWatcher CreateCrashWatcher(RunningContainer container); void Stop(RunningPod pod, bool waitTillStopped); - void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null); + void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null, bool? previous = null); string ExecuteCommand(RunningContainer container, string command, params string[] args); void DeleteNamespace(bool wait); void DeleteNamespacesStartingWith(string namespacePrefix, bool wait); @@ -106,11 +106,11 @@ namespace KubernetesWorkflow }); } - public void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null) + public void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null, bool? previous = null) { K8s(controller => { - controller.DownloadPodLog(container, logHandler, tailLines); + controller.DownloadPodLog(container, logHandler, tailLines, previous); }); } diff --git a/ProjectPlugins/CodexPlugin/CodexAccess.cs b/ProjectPlugins/CodexPlugin/CodexAccess.cs index 80f1061..110a386 100644 --- a/ProjectPlugins/CodexPlugin/CodexAccess.cs +++ b/ProjectPlugins/CodexPlugin/CodexAccess.cs @@ -8,12 +8,11 @@ using Utils; namespace CodexPlugin { - public class CodexAccess : ILogHandler + public class CodexAccess { private readonly ILog log; private readonly IPluginTools tools; private readonly Mapper mapper = new Mapper(); - private bool hasContainerCrashed; public CodexAccess(IPluginTools tools, RunningPod container, CrashWatcher crashWatcher) { @@ -21,9 +20,8 @@ namespace CodexPlugin log = tools.GetLog(); Container = container; CrashWatcher = crashWatcher; - hasContainerCrashed = false; - CrashWatcher.Start(this); + CrashWatcher.Start(); } public RunningPod Container { get; } @@ -209,25 +207,7 @@ namespace CodexPlugin private void CheckContainerCrashed(HttpClient client) { - if (hasContainerCrashed) throw new Exception($"Container {GetName()} has crashed."); - } - - void ILogHandler.Log(Stream crashLog) - { - var file = log.CreateSubfile(); - Log($"Downloading log to '{file.FullFilename}'..."); - file.Write($"Container log for {Container.Name}."); - - using var reader = new StreamReader(crashLog); - var line = reader.ReadLine(); - while (line != null) - { - file.Write(line); - line = reader.ReadLine(); - } - - Log("Container log successfully downloaded."); - hasContainerCrashed = true; + if (CrashWatcher.HasContainerCrashed()) throw new Exception($"Container {GetName()} has crashed."); } private Retry CreateRetryConfig(string description, Action onFailure)