From d16b8cb0119df1f67beebc65a49d3ad215d2b2e1 Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 1 Aug 2024 10:39:06 +0200 Subject: [PATCH] Fixes identity issue for runningpod/runningcontainer and log saving for stopped containers --- Framework/Core/CoreInterface.cs | 6 +--- .../DownloadedLog.cs | 7 ++-- .../KubernetesWorkflow/StartupWorkflow.cs | 29 +++++++++++++++-- .../Types/RunningContainer.cs | 21 +++++++++++- .../KubernetesWorkflow/Types/RunningPod.cs | 24 +++++++++++++- ProjectPlugins/CodexPlugin/CodexNode.cs | 24 +++++++------- .../OverwatchSupport/CodexLogConverter.cs | 2 +- .../OverwatchSupport/CodexTranscriptWriter.cs | 23 +++++++++---- Tests/CodexContinuousTests/NodeRunner.cs | 1 + Tests/CodexTests/CodexDistTest.cs | 3 +- Tests/DistTestCore/DownloadedLogExtensions.cs | 2 +- Tests/DistTestCore/TestLifecycle.cs | 32 +++++++++++++------ 12 files changed, 130 insertions(+), 44 deletions(-) rename Framework/{Core => KubernetesWorkflow}/DownloadedLog.cs (97%) diff --git a/Framework/Core/CoreInterface.cs b/Framework/Core/CoreInterface.cs index e48b6de..f368293 100644 --- a/Framework/Core/CoreInterface.cs +++ b/Framework/Core/CoreInterface.cs @@ -30,11 +30,7 @@ namespace Core public IDownloadedLog DownloadLog(RunningContainer container, int? tailLines = null) { var workflow = entryPoint.Tools.CreateWorkflow(); - var msg = $"Downloading container log for '{container.Name}'"; - entryPoint.Tools.GetLog().Log(msg); - var logHandler = new WriteToFileLogHandler(entryPoint.Tools.GetLog(), msg); - workflow.DownloadContainerLog(container, logHandler, tailLines); - return new DownloadedLog(logHandler, container.Name); + return workflow.DownloadContainerLog(container, tailLines); } public string ExecuteContainerCommand(IHasContainer containerSource, string command, params string[] args) diff --git a/Framework/Core/DownloadedLog.cs b/Framework/KubernetesWorkflow/DownloadedLog.cs similarity index 97% rename from Framework/Core/DownloadedLog.cs rename to Framework/KubernetesWorkflow/DownloadedLog.cs index 381ef3d..a3f0809 100644 --- a/Framework/Core/DownloadedLog.cs +++ b/Framework/KubernetesWorkflow/DownloadedLog.cs @@ -1,7 +1,6 @@ -using KubernetesWorkflow; -using Logging; +using Logging; -namespace Core +namespace KubernetesWorkflow { public interface IDownloadedLog { @@ -23,7 +22,7 @@ namespace Core logFile = logHandler.LogFile; ContainerName = containerName; } - + public string ContainerName { get; } public void IterateLines(Action action, params string[] thatContain) diff --git a/Framework/KubernetesWorkflow/StartupWorkflow.cs b/Framework/KubernetesWorkflow/StartupWorkflow.cs index 6a4d50c..529a48f 100644 --- a/Framework/KubernetesWorkflow/StartupWorkflow.cs +++ b/Framework/KubernetesWorkflow/StartupWorkflow.cs @@ -16,6 +16,7 @@ namespace KubernetesWorkflow CrashWatcher CreateCrashWatcher(RunningContainer container); void Stop(RunningPod pod, bool waitTillStopped); void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null, bool? previous = null); + IDownloadedLog DownloadContainerLog(RunningContainer container, int? tailLines = null, bool? previous = null); string ExecuteCommand(RunningContainer container, string command, params string[] args); void DeleteNamespace(bool wait); void DeleteNamespacesStartingWith(string namespacePrefix, bool wait); @@ -60,7 +61,7 @@ namespace KubernetesWorkflow var startResult = controller.BringOnline(recipes, location); var containers = CreateContainers(startResult, recipes, startupConfig); - var rc = new RunningPod(startupConfig, startResult, containers); + var rc = new RunningPod(Guid.NewGuid().ToString(), startupConfig, startResult, containers); cluster.Configuration.Hooks.OnContainersStarted(rc); if (startResult.ExternalService != null) @@ -99,11 +100,19 @@ namespace KubernetesWorkflow public void Stop(RunningPod runningPod, bool waitTillStopped) { + if (runningPod.IsStopped) return; + foreach (var c in runningPod.Containers) + { + c.StopLog = DownloadContainerLog(c); + } + runningPod.IsStopped = true; + K8s(controller => { controller.Stop(runningPod.StartResult, waitTillStopped); - cluster.Configuration.Hooks.OnContainersStopped(runningPod); }); + + cluster.Configuration.Hooks.OnContainersStopped(runningPod); } public void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null, bool? previous = null) @@ -114,6 +123,20 @@ namespace KubernetesWorkflow }); } + public IDownloadedLog DownloadContainerLog(RunningContainer container, int? tailLines = null, bool? previous = null) + { + var msg = $"Downloading container log for '{container.Name}'"; + log.Log(msg); + var logHandler = new WriteToFileLogHandler(log, msg); + + K8s(controller => + { + controller.DownloadPodLog(container, logHandler, tailLines, previous); + }); + + return new DownloadedLog(logHandler, container.Name); + } + public string ExecuteCommand(RunningContainer container, string command, params string[] args) { return K8s(controller => @@ -147,7 +170,7 @@ namespace KubernetesWorkflow var addresses = CreateContainerAddresses(startResult, r); log.Debug($"{r}={name} -> container addresses: {string.Join(Environment.NewLine, addresses.Select(a => a.ToString()))}"); - return new RunningContainer(name, r, addresses); + return new RunningContainer(Guid.NewGuid().ToString(), name, r, addresses); }).ToArray(); } diff --git a/Framework/KubernetesWorkflow/Types/RunningContainer.cs b/Framework/KubernetesWorkflow/Types/RunningContainer.cs index b0fbc02..ad51265 100644 --- a/Framework/KubernetesWorkflow/Types/RunningContainer.cs +++ b/Framework/KubernetesWorkflow/Types/RunningContainer.cs @@ -7,16 +7,19 @@ namespace KubernetesWorkflow.Types { public class RunningContainer { - public RunningContainer(string name, ContainerRecipe recipe, ContainerAddress[] addresses) + public RunningContainer(string id, string name, ContainerRecipe recipe, ContainerAddress[] addresses) { + Id = id; Name = name; Recipe = recipe; Addresses = addresses; } + public string Id { get; } public string Name { get; } public ContainerRecipe Recipe { get; } public ContainerAddress[] Addresses { get; } + public IDownloadedLog? StopLog { get; internal set; } [JsonIgnore] public RunningPod RunningPod { get; internal set; } = null!; @@ -50,5 +53,21 @@ namespace KubernetesWorkflow.Types } throw new Exception("Running location not known."); } + + public override string ToString() + { + return Name; + } + + public override bool Equals(object? obj) + { + return obj is RunningContainer container && + Id == container.Id; + } + + public override int GetHashCode() + { + return HashCode.Combine(Id); + } } } diff --git a/Framework/KubernetesWorkflow/Types/RunningPod.cs b/Framework/KubernetesWorkflow/Types/RunningPod.cs index 4d03cf3..b278181 100644 --- a/Framework/KubernetesWorkflow/Types/RunningPod.cs +++ b/Framework/KubernetesWorkflow/Types/RunningPod.cs @@ -4,8 +4,9 @@ namespace KubernetesWorkflow.Types { public class RunningPod { - public RunningPod(StartupConfig startupConfig, StartResult startResult, RunningContainer[] containers) + public RunningPod(string id, StartupConfig startupConfig, StartResult startResult, RunningContainer[] containers) { + Id = id; StartupConfig = startupConfig; StartResult = startResult; Containers = containers; @@ -13,6 +14,7 @@ namespace KubernetesWorkflow.Types foreach (var c in containers) c.RunningPod = this; } + public string Id { get; } public StartupConfig StartupConfig { get; } public StartResult StartResult { get; } public RunningContainer[] Containers { get; } @@ -23,10 +25,30 @@ namespace KubernetesWorkflow.Types get { return $"'{string.Join("&", Containers.Select(c => c.Name).ToArray())}'"; } } + [JsonIgnore] + public bool IsStopped { get; internal set; } + public string Describe() { return string.Join(",", Containers.Select(c => c.Name)); } + + public override bool Equals(object? obj) + { + return obj is RunningPod pod && + Id == pod.Id; + } + + public override int GetHashCode() + { + return HashCode.Combine(Id); + } + + public override string ToString() + { + if (IsStopped) return Name + " (*)"; + return Name; + } } public static class RunningContainersExtensions diff --git a/ProjectPlugins/CodexPlugin/CodexNode.cs b/ProjectPlugins/CodexPlugin/CodexNode.cs index f8a65b0..cfea972 100644 --- a/ProjectPlugins/CodexPlugin/CodexNode.cs +++ b/ProjectPlugins/CodexPlugin/CodexNode.cs @@ -41,6 +41,7 @@ namespace CodexPlugin public class CodexNode : ICodexNode { private const string UploadFailedMessage = "Unable to store block"; + private readonly ILog log; private readonly IPluginTools tools; private readonly ICodexNodeHooks hooks; private readonly EthAccount? ethAccount; @@ -57,6 +58,8 @@ namespace CodexPlugin this.hooks = hooks; Version = new DebugInfoVersion(); transferSpeeds = new TransferSpeeds(); + + log = new LogPrefixer(tools.GetLog(), $"{GetName()} "); } public void Awake() @@ -141,9 +144,8 @@ namespace CodexPlugin hooks.OnFileUploading(uniqueId, size); - var logMessage = $"Uploading file {file.Describe()}..."; - Log(logMessage); - var measurement = Stopwatch.Measure(tools.GetLog(), logMessage, () => + var logMessage = $"Uploading file '{file.Describe()}'..."; + var measurement = Stopwatch.Measure(log, logMessage, () => { return CodexAccess.UploadFile(fileStream, onFailure); }); @@ -154,7 +156,7 @@ namespace CodexPlugin if (string.IsNullOrEmpty(response)) FrameworkAssert.Fail("Received empty response."); if (response.StartsWith(UploadFailedMessage)) FrameworkAssert.Fail("Node failed to store block."); - Log($"Uploaded file. Received contentId: '{response}'."); + Log($"Uploaded file '{file.Describe()}'. Received contentId: '{response}'."); var cid = new ContentId(response); hooks.OnFileUploaded(uniqueId, size, cid); @@ -168,15 +170,16 @@ namespace CodexPlugin public TrackedFile? DownloadContent(ContentId contentId, Action onFailure, string fileLabel = "") { - var logMessage = $"Downloading for contentId: '{contentId.Id}'..."; - hooks.OnFileDownloading(contentId); - Log(logMessage); var file = tools.GetFileManager().CreateEmptyFile(fileLabel); - var measurement = Stopwatch.Measure(tools.GetLog(), logMessage, () => DownloadToFile(contentId.Id, file, onFailure)); + var logMessage = $"Downloading '{contentId.Id}' to '{file.Filename}'"; + hooks.OnFileDownloading(contentId); + + var measurement = Stopwatch.Measure(log, logMessage, () => DownloadToFile(contentId.Id, file, onFailure)); + var size = file.GetFilesize(); transferSpeeds.AddDownloadSample(size, measurement); - Log($"Downloaded file {file.Describe()} to '{file.Filename}'."); hooks.OnFileDownloaded(size, contentId); + return file; } @@ -231,7 +234,6 @@ namespace CodexPlugin throw new Exception($"Invalid version information received from Codex node {GetName()}: {debugInfo.Version}"); } - var log = tools.GetLog(); log.AddStringReplace(peerId, nodeName); log.AddStringReplace(CodexUtils.ToShortId(peerId), nodeName); log.AddStringReplace(debugInfo.Table.LocalNode.NodeId, nodeName); @@ -273,7 +275,7 @@ namespace CodexPlugin private void Log(string msg) { - tools.GetLog().Log($"{GetName()}: {msg}"); + log.Log(msg); } private void DoNothing(Failure failure) diff --git a/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexLogConverter.cs b/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexLogConverter.cs index a5d6749..0f62c53 100644 --- a/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexLogConverter.cs +++ b/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexLogConverter.cs @@ -1,5 +1,5 @@ using CodexPlugin.OverwatchSupport.LineConverters; -using Core; +using KubernetesWorkflow; using OverwatchTranscript; using Utils; diff --git a/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexTranscriptWriter.cs b/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexTranscriptWriter.cs index 4965592..8bfbb02 100644 --- a/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexTranscriptWriter.cs +++ b/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexTranscriptWriter.cs @@ -1,5 +1,6 @@ using CodexPlugin.Hooks; -using Core; +using KubernetesWorkflow; +using Logging; using OverwatchTranscript; using Utils; @@ -8,21 +9,26 @@ namespace CodexPlugin.OverwatchSupport public class CodexTranscriptWriter : ICodexHooksProvider { private const string CodexHeaderKey = "cdx_h"; + private readonly ILog log; private readonly ITranscriptWriter writer; private readonly CodexLogConverter converter; private readonly NameIdMap nameIdMap = new NameIdMap(); - public CodexTranscriptWriter(ITranscriptWriter transcriptWriter) + public CodexTranscriptWriter(ILog log, ITranscriptWriter transcriptWriter) { + this.log = log; writer = transcriptWriter; converter = new CodexLogConverter(writer, nameIdMap); } public void Finalize(string outputFilepath) { - writer.AddHeader(CodexHeaderKey, CreateCodexHeader()); + log.Log("Finalizing Codex transcript..."); + writer.AddHeader(CodexHeaderKey, CreateCodexHeader()); writer.Write(outputFilepath); + + log.Log("Done"); } public ICodexNodeHooks CreateHooks(string nodeName) @@ -38,14 +44,17 @@ namespace CodexPlugin.OverwatchSupport public void ProcessLogs(IDownloadedLog[] downloadedLogs) { - foreach (var log in downloadedLogs) + foreach (var l in downloadedLogs) { - writer.IncludeArtifact(log.GetFilepath()); + log.Log("Include artifact: " + l.GetFilepath()); + writer.IncludeArtifact(l.GetFilepath()); + // Not all of these logs are necessarily Codex logs. // Check, and process only the Codex ones. - if (IsCodexLog(log)) + if (IsCodexLog(l)) { - converter.ProcessLog(log); + log.Log("Processing Codex log: " + l.GetFilepath()); + converter.ProcessLog(l); } } } diff --git a/Tests/CodexContinuousTests/NodeRunner.cs b/Tests/CodexContinuousTests/NodeRunner.cs index e58facc..77a383e 100644 --- a/Tests/CodexContinuousTests/NodeRunner.cs +++ b/Tests/CodexContinuousTests/NodeRunner.cs @@ -4,6 +4,7 @@ using Utils; using Core; using CodexPlugin; using KubernetesWorkflow.Types; +using KubernetesWorkflow; namespace ContinuousTests { diff --git a/Tests/CodexTests/CodexDistTest.cs b/Tests/CodexTests/CodexDistTest.cs index 9ba4c6c..8c25c29 100644 --- a/Tests/CodexTests/CodexDistTest.cs +++ b/Tests/CodexTests/CodexDistTest.cs @@ -138,7 +138,8 @@ namespace CodexTests { if (GetTranscriptAttributeOfCurrentTest() == null) return; - var writer = new CodexTranscriptWriter(Transcript.NewWriter()); + var log = new LogPrefixer(lifecycle.Log, "(Transcript) "); + var writer = new CodexTranscriptWriter(log, Transcript.NewWriter()); Ci.SetCodexHooksProvider(writer); writers.Add(lifecycle, writer); } diff --git a/Tests/DistTestCore/DownloadedLogExtensions.cs b/Tests/DistTestCore/DownloadedLogExtensions.cs index 51ee96f..91eeb08 100644 --- a/Tests/DistTestCore/DownloadedLogExtensions.cs +++ b/Tests/DistTestCore/DownloadedLogExtensions.cs @@ -1,4 +1,4 @@ -using Core; +using KubernetesWorkflow; using NUnit.Framework; namespace DistTestCore diff --git a/Tests/DistTestCore/TestLifecycle.cs b/Tests/DistTestCore/TestLifecycle.cs index b67b409..7af09c2 100644 --- a/Tests/DistTestCore/TestLifecycle.cs +++ b/Tests/DistTestCore/TestLifecycle.cs @@ -15,6 +15,7 @@ namespace DistTestCore private readonly Dictionary metadata; private readonly List runningContainers = new(); private readonly string deployId; + private readonly List stoppedContainerLogs = new List(); public TestLifecycle(TestLog log, Configuration configuration, ITimeSet timeSet, string testNamespace, string deployId, bool waitForCleanup) { @@ -80,6 +81,12 @@ namespace DistTestCore public void OnContainersStopped(RunningPod rc) { runningContainers.Remove(rc); + + stoppedContainerLogs.AddRange(rc.Containers.Select(c => + { + if (c.StopLog == null) throw new Exception("Expected StopLog for stopped container " + c.Name); + return c.StopLog; + })); } public void OnContainerRecipeCreated(ContainerRecipe recipe) @@ -98,24 +105,31 @@ namespace DistTestCore } } - private IDownloadedLog[] allLogs = Array.Empty(); - public IDownloadedLog[] DownloadAllLogs() { - if (allLogs.Any()) return allLogs; - try { - var result = new List(); + var result = new List(); + result.AddRange(stoppedContainerLogs); foreach (var rc in runningContainers) { - foreach (var c in rc.Containers) + if (rc.IsStopped) { - result.Add(CoreInterface.DownloadLog(c)); + foreach (var c in rc.Containers) + { + if (c.StopLog == null) throw new Exception("No stop-log was downloaded for container."); + result.Add(c.StopLog); + } + } + else + { + foreach (var c in rc.Containers) + { + result.Add(CoreInterface.DownloadLog(c)); + } } } - allLogs = result.ToArray(); - return allLogs; + return result.ToArray(); } catch (Exception ex) {