Fixes identity issue for runningpod/runningcontainer and log saving for stopped containers

Framework/Core/CoreInterface.cs

@@ -30,11 +30,7 @@ namespace Core
         public IDownloadedLog DownloadLog(RunningContainer container, int? tailLines = null)
         {
             var workflow = entryPoint.Tools.CreateWorkflow();
-            var msg = $"Downloading container log for '{container.Name}'";
+            return workflow.DownloadContainerLog(container, tailLines);
-            entryPoint.Tools.GetLog().Log(msg);
-            var logHandler = new WriteToFileLogHandler(entryPoint.Tools.GetLog(), msg);
-            workflow.DownloadContainerLog(container, logHandler, tailLines);
-            return new DownloadedLog(logHandler, container.Name);
         }
 
         public string ExecuteContainerCommand(IHasContainer containerSource, string command, params string[] args)

Framework/KubernetesWorkflow/DownloadedLog.cs

@@ -1,7 +1,6 @@
-using KubernetesWorkflow;
+using Logging;
-using Logging;
 
-namespace Core
+namespace KubernetesWorkflow
 {
     public interface IDownloadedLog
     {

Framework/KubernetesWorkflow/StartupWorkflow.cs

@@ -16,6 +16,7 @@ namespace KubernetesWorkflow
         CrashWatcher CreateCrashWatcher(RunningContainer container);
         void Stop(RunningPod pod, bool waitTillStopped);
         void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null, bool? previous = null);
+        IDownloadedLog DownloadContainerLog(RunningContainer container, int? tailLines = null, bool? previous = null);
         string ExecuteCommand(RunningContainer container, string command, params string[] args);
         void DeleteNamespace(bool wait);
         void DeleteNamespacesStartingWith(string namespacePrefix, bool wait);
@@ -60,7 +61,7 @@ namespace KubernetesWorkflow
                 var startResult = controller.BringOnline(recipes, location);
                 var containers = CreateContainers(startResult, recipes, startupConfig);
 
-                var rc = new RunningPod(startupConfig, startResult, containers);
+                var rc = new RunningPod(Guid.NewGuid().ToString(), startupConfig, startResult, containers);
                 cluster.Configuration.Hooks.OnContainersStarted(rc);
 
                 if (startResult.ExternalService != null)
@@ -99,11 +100,19 @@ namespace KubernetesWorkflow
 
         public void Stop(RunningPod runningPod, bool waitTillStopped)
         {
+            if (runningPod.IsStopped) return;
+            foreach (var c in runningPod.Containers)
+            {
+                c.StopLog = DownloadContainerLog(c);
+            }
+            runningPod.IsStopped = true;
+
             K8s(controller =>
             {
                 controller.Stop(runningPod.StartResult, waitTillStopped);
-                cluster.Configuration.Hooks.OnContainersStopped(runningPod);
             });
+
+            cluster.Configuration.Hooks.OnContainersStopped(runningPod);
         }
 
         public void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null, bool? previous = null)
@@ -114,6 +123,20 @@ namespace KubernetesWorkflow
             });
         }
 
+        public IDownloadedLog DownloadContainerLog(RunningContainer container, int? tailLines = null, bool? previous = null)
+        {
+            var msg = $"Downloading container log for '{container.Name}'";
+            log.Log(msg);
+            var logHandler = new WriteToFileLogHandler(log, msg);
+
+            K8s(controller =>
+            {
+                controller.DownloadPodLog(container, logHandler, tailLines, previous);
+            });
+
+            return new DownloadedLog(logHandler, container.Name);
+        }
+
         public string ExecuteCommand(RunningContainer container, string command, params string[] args)
         {
             return K8s(controller =>
@@ -147,7 +170,7 @@ namespace KubernetesWorkflow
                 var addresses = CreateContainerAddresses(startResult, r);
                 log.Debug($"{r}={name} -> container addresses: {string.Join(Environment.NewLine, addresses.Select(a => a.ToString()))}");
 
-                return new RunningContainer(name, r, addresses);
+                return new RunningContainer(Guid.NewGuid().ToString(), name, r, addresses);
 
             }).ToArray();
         }

Framework/KubernetesWorkflow/Types/RunningContainer.cs

@@ -7,16 +7,19 @@ namespace KubernetesWorkflow.Types
 {
     public class RunningContainer
     {
-        public RunningContainer(string name, ContainerRecipe recipe, ContainerAddress[] addresses)
+        public RunningContainer(string id, string name, ContainerRecipe recipe, ContainerAddress[] addresses)
         {
+            Id = id;
             Name = name;
             Recipe = recipe;
             Addresses = addresses;
         }
 
+        public string Id { get; }
         public string Name { get; }
         public ContainerRecipe Recipe { get; }
         public ContainerAddress[] Addresses { get; }
+        public IDownloadedLog? StopLog { get; internal set; }
 
         [JsonIgnore]
         public RunningPod RunningPod { get; internal set; } = null!;
@@ -50,5 +53,21 @@ namespace KubernetesWorkflow.Types
             }
             throw new Exception("Running location not known.");
         }
+
+        public override string ToString()
+        {
+            return Name;
+        }
+
+        public override bool Equals(object? obj)
+        {
+            return obj is RunningContainer container &&
+                   Id == container.Id;
+        }
+
+        public override int GetHashCode()
+        {
+            return HashCode.Combine(Id);
+        }
     }
 }

Framework/KubernetesWorkflow/Types/RunningPod.cs

@@ -4,8 +4,9 @@ namespace KubernetesWorkflow.Types
 {
     public class RunningPod
     {
-        public RunningPod(StartupConfig startupConfig, StartResult startResult, RunningContainer[] containers)
+        public RunningPod(string id, StartupConfig startupConfig, StartResult startResult, RunningContainer[] containers)
         {
+            Id = id;
             StartupConfig = startupConfig;
             StartResult = startResult;
             Containers = containers;
@@ -13,6 +14,7 @@ namespace KubernetesWorkflow.Types
             foreach (var c in containers) c.RunningPod = this;
         }
 
+        public string Id { get; }
         public StartupConfig StartupConfig { get; }
         public StartResult StartResult { get; }
         public RunningContainer[] Containers { get; }
@@ -23,10 +25,30 @@ namespace KubernetesWorkflow.Types
             get { return $"'{string.Join("&", Containers.Select(c => c.Name).ToArray())}'"; }
         }
 
+        [JsonIgnore]
+        public bool IsStopped { get; internal set; }
+
         public string Describe()
         {
             return string.Join(",", Containers.Select(c => c.Name));
         }
+
+        public override bool Equals(object? obj)
+        {
+            return obj is RunningPod pod &&
+                   Id == pod.Id;
+        }
+
+        public override int GetHashCode()
+        {
+            return HashCode.Combine(Id);
+        }
+
+        public override string ToString()
+        {
+            if (IsStopped) return Name + " (*)";
+            return Name;
+        }
     }
 
     public static class RunningContainersExtensions

ProjectPlugins/CodexPlugin/CodexNode.cs

@@ -41,6 +41,7 @@ namespace CodexPlugin
     public class CodexNode : ICodexNode
     {
         private const string UploadFailedMessage = "Unable to store block";
+        private readonly ILog log;
         private readonly IPluginTools tools;
         private readonly ICodexNodeHooks hooks;
         private readonly EthAccount? ethAccount;
@@ -57,6 +58,8 @@ namespace CodexPlugin
             this.hooks = hooks;
             Version = new DebugInfoVersion();
             transferSpeeds = new TransferSpeeds();
+
+            log = new LogPrefixer(tools.GetLog(), $"{GetName()} ");
         }
 
         public void Awake()
@@ -141,9 +144,8 @@ namespace CodexPlugin
 
             hooks.OnFileUploading(uniqueId, size);
 
-            var logMessage = $"Uploading file {file.Describe()}...";
+            var logMessage = $"Uploading file '{file.Describe()}'...";
-            Log(logMessage);
+            var measurement = Stopwatch.Measure(log, logMessage, () =>
-            var measurement = Stopwatch.Measure(tools.GetLog(), logMessage, () =>
             {
                 return CodexAccess.UploadFile(fileStream, onFailure);
             });
@@ -154,7 +156,7 @@ namespace CodexPlugin
             if (string.IsNullOrEmpty(response)) FrameworkAssert.Fail("Received empty response.");
             if (response.StartsWith(UploadFailedMessage)) FrameworkAssert.Fail("Node failed to store block.");
 
-            Log($"Uploaded file. Received contentId: '{response}'.");
+            Log($"Uploaded file '{file.Describe()}'. Received contentId: '{response}'.");
 
             var cid = new ContentId(response);
             hooks.OnFileUploaded(uniqueId, size, cid);
@@ -168,15 +170,16 @@ namespace CodexPlugin
 
         public TrackedFile? DownloadContent(ContentId contentId, Action<Failure> onFailure, string fileLabel = "")
         {
-            var logMessage = $"Downloading for contentId: '{contentId.Id}'...";
-            hooks.OnFileDownloading(contentId);
-            Log(logMessage);
             var file = tools.GetFileManager().CreateEmptyFile(fileLabel);
-            var measurement = Stopwatch.Measure(tools.GetLog(), logMessage, () => DownloadToFile(contentId.Id, file, onFailure));
+            var logMessage = $"Downloading '{contentId.Id}' to '{file.Filename}'";
+            hooks.OnFileDownloading(contentId);
+
+            var measurement = Stopwatch.Measure(log, logMessage, () => DownloadToFile(contentId.Id, file, onFailure));
+
             var size = file.GetFilesize();
             transferSpeeds.AddDownloadSample(size, measurement);
-            Log($"Downloaded file {file.Describe()} to '{file.Filename}'.");
             hooks.OnFileDownloaded(size, contentId);
+
             return file;
         }
 
@@ -231,7 +234,6 @@ namespace CodexPlugin
                 throw new Exception($"Invalid version information received from Codex node {GetName()}: {debugInfo.Version}");
             }
 
-            var log = tools.GetLog();
             log.AddStringReplace(peerId, nodeName);
             log.AddStringReplace(CodexUtils.ToShortId(peerId), nodeName);
             log.AddStringReplace(debugInfo.Table.LocalNode.NodeId, nodeName);
@@ -273,7 +275,7 @@ namespace CodexPlugin
 
         private void Log(string msg)
         {
-            tools.GetLog().Log($"{GetName()}: {msg}");
+            log.Log(msg);
         }
 
         private void DoNothing(Failure failure)

ProjectPlugins/CodexPlugin/OverwatchSupport/CodexLogConverter.cs

@@ -1,5 +1,5 @@
 using CodexPlugin.OverwatchSupport.LineConverters;
-using Core;
+using KubernetesWorkflow;
 using OverwatchTranscript;
 using Utils;
 

ProjectPlugins/CodexPlugin/OverwatchSupport/CodexTranscriptWriter.cs

@@ -1,5 +1,6 @@
 using CodexPlugin.Hooks;
-using Core;
+using KubernetesWorkflow;
+using Logging;
 using OverwatchTranscript;
 using Utils;
 
@@ -8,21 +9,26 @@ namespace CodexPlugin.OverwatchSupport
     public class CodexTranscriptWriter : ICodexHooksProvider
     {
         private const string CodexHeaderKey = "cdx_h";
+        private readonly ILog log;
         private readonly ITranscriptWriter writer;
         private readonly CodexLogConverter converter;
         private readonly NameIdMap nameIdMap = new NameIdMap();
 
-        public CodexTranscriptWriter(ITranscriptWriter transcriptWriter)
+        public CodexTranscriptWriter(ILog log, ITranscriptWriter transcriptWriter)
         {
+            this.log = log;
             writer = transcriptWriter;
             converter = new CodexLogConverter(writer, nameIdMap);
         }
 
         public void Finalize(string outputFilepath)
         {
-            writer.AddHeader(CodexHeaderKey, CreateCodexHeader());
+            log.Log("Finalizing Codex transcript...");
 
+            writer.AddHeader(CodexHeaderKey, CreateCodexHeader());
             writer.Write(outputFilepath);
+
+            log.Log("Done");
         }
 
         public ICodexNodeHooks CreateHooks(string nodeName)
@@ -38,14 +44,17 @@ namespace CodexPlugin.OverwatchSupport
 
         public void ProcessLogs(IDownloadedLog[] downloadedLogs)
         {
-            foreach (var log in downloadedLogs)
+            foreach (var l in downloadedLogs)
             {
-                writer.IncludeArtifact(log.GetFilepath());
+                log.Log("Include artifact: " + l.GetFilepath());
+                writer.IncludeArtifact(l.GetFilepath());
+
                 // Not all of these logs are necessarily Codex logs.
                 // Check, and process only the Codex ones.
-                if (IsCodexLog(log))
+                if (IsCodexLog(l))
                 {
-                    converter.ProcessLog(log);
+                    log.Log("Processing Codex log: " + l.GetFilepath());
+                    converter.ProcessLog(l);
                 }
             }
         }

Tests/CodexContinuousTests/NodeRunner.cs

@@ -4,6 +4,7 @@ using Utils;
 using Core;
 using CodexPlugin;
 using KubernetesWorkflow.Types;
+using KubernetesWorkflow;
 
 namespace ContinuousTests
 {

Tests/CodexTests/CodexDistTest.cs

@@ -138,7 +138,8 @@ namespace CodexTests
         {
             if (GetTranscriptAttributeOfCurrentTest() == null) return;
 
-            var writer = new CodexTranscriptWriter(Transcript.NewWriter());
+            var log = new LogPrefixer(lifecycle.Log, "(Transcript) ");
+            var writer = new CodexTranscriptWriter(log, Transcript.NewWriter());
             Ci.SetCodexHooksProvider(writer);
             writers.Add(lifecycle, writer);
         }

Tests/DistTestCore/DownloadedLogExtensions.cs

@@ -1,4 +1,4 @@
-using Core;
+using KubernetesWorkflow;
 using NUnit.Framework;
 
 namespace DistTestCore

Tests/DistTestCore/TestLifecycle.cs

@@ -15,6 +15,7 @@ namespace DistTestCore
         private readonly Dictionary<string, string> metadata; 
         private readonly List<RunningPod> runningContainers = new();
         private readonly string deployId;
+        private readonly List<IDownloadedLog> stoppedContainerLogs = new List<IDownloadedLog>();
 
         public TestLifecycle(TestLog log, Configuration configuration, ITimeSet timeSet, string testNamespace, string deployId, bool waitForCleanup)
         {
@@ -80,6 +81,12 @@ namespace DistTestCore
         public void OnContainersStopped(RunningPod rc)
         {
             runningContainers.Remove(rc);
+
+            stoppedContainerLogs.AddRange(rc.Containers.Select(c =>
+            {
+                if (c.StopLog == null) throw new Exception("Expected StopLog for stopped container " + c.Name);
+                return c.StopLog;
+            }));
         }
 
         public void OnContainerRecipeCreated(ContainerRecipe recipe)
@@ -98,24 +105,31 @@ namespace DistTestCore
             }
         }
 
-        private IDownloadedLog[] allLogs = Array.Empty<IDownloadedLog>();
-
         public IDownloadedLog[] DownloadAllLogs()
         {
-            if (allLogs.Any()) return allLogs;
-
             try
             {
                 var result = new List<IDownloadedLog>();
+                result.AddRange(stoppedContainerLogs);
                 foreach (var rc in runningContainers)
                 {
-                    foreach (var c in rc.Containers)
+                    if (rc.IsStopped)
                     {
-                        result.Add(CoreInterface.DownloadLog(c));
+                        foreach (var c in rc.Containers)
+                        {
+                            if (c.StopLog == null) throw new Exception("No stop-log was downloaded for container.");
+                            result.Add(c.StopLog);
+                        }
+                    }
+                    else
+                    {
+                        foreach (var c in rc.Containers)
+                        {
+                            result.Add(CoreInterface.DownloadLog(c));
+                        }
                     }
                 }
-                allLogs = result.ToArray();
+                return result.ToArray();
-                return allLogs;
             }
             catch (Exception ex)
             {