Fixes identity issue for runningpod/runningcontainer and log saving for stopped containers

This commit is contained in:
benbierens 2024-08-01 10:39:06 +02:00
parent 37611bdc66
commit d16b8cb011
No known key found for this signature in database
GPG Key ID: 877D2C2E09A22F3A
12 changed files with 130 additions and 44 deletions

View File

@ -30,11 +30,7 @@ namespace Core
public IDownloadedLog DownloadLog(RunningContainer container, int? tailLines = null) public IDownloadedLog DownloadLog(RunningContainer container, int? tailLines = null)
{ {
var workflow = entryPoint.Tools.CreateWorkflow(); var workflow = entryPoint.Tools.CreateWorkflow();
var msg = $"Downloading container log for '{container.Name}'"; return workflow.DownloadContainerLog(container, tailLines);
entryPoint.Tools.GetLog().Log(msg);
var logHandler = new WriteToFileLogHandler(entryPoint.Tools.GetLog(), msg);
workflow.DownloadContainerLog(container, logHandler, tailLines);
return new DownloadedLog(logHandler, container.Name);
} }
public string ExecuteContainerCommand(IHasContainer containerSource, string command, params string[] args) public string ExecuteContainerCommand(IHasContainer containerSource, string command, params string[] args)

View File

@ -1,7 +1,6 @@
using KubernetesWorkflow; using Logging;
using Logging;
namespace Core namespace KubernetesWorkflow
{ {
public interface IDownloadedLog public interface IDownloadedLog
{ {

View File

@ -16,6 +16,7 @@ namespace KubernetesWorkflow
CrashWatcher CreateCrashWatcher(RunningContainer container); CrashWatcher CreateCrashWatcher(RunningContainer container);
void Stop(RunningPod pod, bool waitTillStopped); void Stop(RunningPod pod, bool waitTillStopped);
void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null, bool? previous = null); void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null, bool? previous = null);
IDownloadedLog DownloadContainerLog(RunningContainer container, int? tailLines = null, bool? previous = null);
string ExecuteCommand(RunningContainer container, string command, params string[] args); string ExecuteCommand(RunningContainer container, string command, params string[] args);
void DeleteNamespace(bool wait); void DeleteNamespace(bool wait);
void DeleteNamespacesStartingWith(string namespacePrefix, bool wait); void DeleteNamespacesStartingWith(string namespacePrefix, bool wait);
@ -60,7 +61,7 @@ namespace KubernetesWorkflow
var startResult = controller.BringOnline(recipes, location); var startResult = controller.BringOnline(recipes, location);
var containers = CreateContainers(startResult, recipes, startupConfig); var containers = CreateContainers(startResult, recipes, startupConfig);
var rc = new RunningPod(startupConfig, startResult, containers); var rc = new RunningPod(Guid.NewGuid().ToString(), startupConfig, startResult, containers);
cluster.Configuration.Hooks.OnContainersStarted(rc); cluster.Configuration.Hooks.OnContainersStarted(rc);
if (startResult.ExternalService != null) if (startResult.ExternalService != null)
@ -99,11 +100,19 @@ namespace KubernetesWorkflow
public void Stop(RunningPod runningPod, bool waitTillStopped) public void Stop(RunningPod runningPod, bool waitTillStopped)
{ {
if (runningPod.IsStopped) return;
foreach (var c in runningPod.Containers)
{
c.StopLog = DownloadContainerLog(c);
}
runningPod.IsStopped = true;
K8s(controller => K8s(controller =>
{ {
controller.Stop(runningPod.StartResult, waitTillStopped); controller.Stop(runningPod.StartResult, waitTillStopped);
cluster.Configuration.Hooks.OnContainersStopped(runningPod);
}); });
cluster.Configuration.Hooks.OnContainersStopped(runningPod);
} }
public void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null, bool? previous = null) public void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null, bool? previous = null)
@ -114,6 +123,20 @@ namespace KubernetesWorkflow
}); });
} }
public IDownloadedLog DownloadContainerLog(RunningContainer container, int? tailLines = null, bool? previous = null)
{
var msg = $"Downloading container log for '{container.Name}'";
log.Log(msg);
var logHandler = new WriteToFileLogHandler(log, msg);
K8s(controller =>
{
controller.DownloadPodLog(container, logHandler, tailLines, previous);
});
return new DownloadedLog(logHandler, container.Name);
}
public string ExecuteCommand(RunningContainer container, string command, params string[] args) public string ExecuteCommand(RunningContainer container, string command, params string[] args)
{ {
return K8s(controller => return K8s(controller =>
@ -147,7 +170,7 @@ namespace KubernetesWorkflow
var addresses = CreateContainerAddresses(startResult, r); var addresses = CreateContainerAddresses(startResult, r);
log.Debug($"{r}={name} -> container addresses: {string.Join(Environment.NewLine, addresses.Select(a => a.ToString()))}"); log.Debug($"{r}={name} -> container addresses: {string.Join(Environment.NewLine, addresses.Select(a => a.ToString()))}");
return new RunningContainer(name, r, addresses); return new RunningContainer(Guid.NewGuid().ToString(), name, r, addresses);
}).ToArray(); }).ToArray();
} }

View File

@ -7,16 +7,19 @@ namespace KubernetesWorkflow.Types
{ {
public class RunningContainer public class RunningContainer
{ {
public RunningContainer(string name, ContainerRecipe recipe, ContainerAddress[] addresses) public RunningContainer(string id, string name, ContainerRecipe recipe, ContainerAddress[] addresses)
{ {
Id = id;
Name = name; Name = name;
Recipe = recipe; Recipe = recipe;
Addresses = addresses; Addresses = addresses;
} }
public string Id { get; }
public string Name { get; } public string Name { get; }
public ContainerRecipe Recipe { get; } public ContainerRecipe Recipe { get; }
public ContainerAddress[] Addresses { get; } public ContainerAddress[] Addresses { get; }
public IDownloadedLog? StopLog { get; internal set; }
[JsonIgnore] [JsonIgnore]
public RunningPod RunningPod { get; internal set; } = null!; public RunningPod RunningPod { get; internal set; } = null!;
@ -50,5 +53,21 @@ namespace KubernetesWorkflow.Types
} }
throw new Exception("Running location not known."); throw new Exception("Running location not known.");
} }
public override string ToString()
{
return Name;
}
public override bool Equals(object? obj)
{
return obj is RunningContainer container &&
Id == container.Id;
}
public override int GetHashCode()
{
return HashCode.Combine(Id);
}
} }
} }

View File

@ -4,8 +4,9 @@ namespace KubernetesWorkflow.Types
{ {
public class RunningPod public class RunningPod
{ {
public RunningPod(StartupConfig startupConfig, StartResult startResult, RunningContainer[] containers) public RunningPod(string id, StartupConfig startupConfig, StartResult startResult, RunningContainer[] containers)
{ {
Id = id;
StartupConfig = startupConfig; StartupConfig = startupConfig;
StartResult = startResult; StartResult = startResult;
Containers = containers; Containers = containers;
@ -13,6 +14,7 @@ namespace KubernetesWorkflow.Types
foreach (var c in containers) c.RunningPod = this; foreach (var c in containers) c.RunningPod = this;
} }
public string Id { get; }
public StartupConfig StartupConfig { get; } public StartupConfig StartupConfig { get; }
public StartResult StartResult { get; } public StartResult StartResult { get; }
public RunningContainer[] Containers { get; } public RunningContainer[] Containers { get; }
@ -23,10 +25,30 @@ namespace KubernetesWorkflow.Types
get { return $"'{string.Join("&", Containers.Select(c => c.Name).ToArray())}'"; } get { return $"'{string.Join("&", Containers.Select(c => c.Name).ToArray())}'"; }
} }
[JsonIgnore]
public bool IsStopped { get; internal set; }
public string Describe() public string Describe()
{ {
return string.Join(",", Containers.Select(c => c.Name)); return string.Join(",", Containers.Select(c => c.Name));
} }
public override bool Equals(object? obj)
{
return obj is RunningPod pod &&
Id == pod.Id;
}
public override int GetHashCode()
{
return HashCode.Combine(Id);
}
public override string ToString()
{
if (IsStopped) return Name + " (*)";
return Name;
}
} }
public static class RunningContainersExtensions public static class RunningContainersExtensions

View File

@ -41,6 +41,7 @@ namespace CodexPlugin
public class CodexNode : ICodexNode public class CodexNode : ICodexNode
{ {
private const string UploadFailedMessage = "Unable to store block"; private const string UploadFailedMessage = "Unable to store block";
private readonly ILog log;
private readonly IPluginTools tools; private readonly IPluginTools tools;
private readonly ICodexNodeHooks hooks; private readonly ICodexNodeHooks hooks;
private readonly EthAccount? ethAccount; private readonly EthAccount? ethAccount;
@ -57,6 +58,8 @@ namespace CodexPlugin
this.hooks = hooks; this.hooks = hooks;
Version = new DebugInfoVersion(); Version = new DebugInfoVersion();
transferSpeeds = new TransferSpeeds(); transferSpeeds = new TransferSpeeds();
log = new LogPrefixer(tools.GetLog(), $"{GetName()} ");
} }
public void Awake() public void Awake()
@ -141,9 +144,8 @@ namespace CodexPlugin
hooks.OnFileUploading(uniqueId, size); hooks.OnFileUploading(uniqueId, size);
var logMessage = $"Uploading file {file.Describe()}..."; var logMessage = $"Uploading file '{file.Describe()}'...";
Log(logMessage); var measurement = Stopwatch.Measure(log, logMessage, () =>
var measurement = Stopwatch.Measure(tools.GetLog(), logMessage, () =>
{ {
return CodexAccess.UploadFile(fileStream, onFailure); return CodexAccess.UploadFile(fileStream, onFailure);
}); });
@ -154,7 +156,7 @@ namespace CodexPlugin
if (string.IsNullOrEmpty(response)) FrameworkAssert.Fail("Received empty response."); if (string.IsNullOrEmpty(response)) FrameworkAssert.Fail("Received empty response.");
if (response.StartsWith(UploadFailedMessage)) FrameworkAssert.Fail("Node failed to store block."); if (response.StartsWith(UploadFailedMessage)) FrameworkAssert.Fail("Node failed to store block.");
Log($"Uploaded file. Received contentId: '{response}'."); Log($"Uploaded file '{file.Describe()}'. Received contentId: '{response}'.");
var cid = new ContentId(response); var cid = new ContentId(response);
hooks.OnFileUploaded(uniqueId, size, cid); hooks.OnFileUploaded(uniqueId, size, cid);
@ -168,15 +170,16 @@ namespace CodexPlugin
public TrackedFile? DownloadContent(ContentId contentId, Action<Failure> onFailure, string fileLabel = "") public TrackedFile? DownloadContent(ContentId contentId, Action<Failure> onFailure, string fileLabel = "")
{ {
var logMessage = $"Downloading for contentId: '{contentId.Id}'...";
hooks.OnFileDownloading(contentId);
Log(logMessage);
var file = tools.GetFileManager().CreateEmptyFile(fileLabel); var file = tools.GetFileManager().CreateEmptyFile(fileLabel);
var measurement = Stopwatch.Measure(tools.GetLog(), logMessage, () => DownloadToFile(contentId.Id, file, onFailure)); var logMessage = $"Downloading '{contentId.Id}' to '{file.Filename}'";
hooks.OnFileDownloading(contentId);
var measurement = Stopwatch.Measure(log, logMessage, () => DownloadToFile(contentId.Id, file, onFailure));
var size = file.GetFilesize(); var size = file.GetFilesize();
transferSpeeds.AddDownloadSample(size, measurement); transferSpeeds.AddDownloadSample(size, measurement);
Log($"Downloaded file {file.Describe()} to '{file.Filename}'.");
hooks.OnFileDownloaded(size, contentId); hooks.OnFileDownloaded(size, contentId);
return file; return file;
} }
@ -231,7 +234,6 @@ namespace CodexPlugin
throw new Exception($"Invalid version information received from Codex node {GetName()}: {debugInfo.Version}"); throw new Exception($"Invalid version information received from Codex node {GetName()}: {debugInfo.Version}");
} }
var log = tools.GetLog();
log.AddStringReplace(peerId, nodeName); log.AddStringReplace(peerId, nodeName);
log.AddStringReplace(CodexUtils.ToShortId(peerId), nodeName); log.AddStringReplace(CodexUtils.ToShortId(peerId), nodeName);
log.AddStringReplace(debugInfo.Table.LocalNode.NodeId, nodeName); log.AddStringReplace(debugInfo.Table.LocalNode.NodeId, nodeName);
@ -273,7 +275,7 @@ namespace CodexPlugin
private void Log(string msg) private void Log(string msg)
{ {
tools.GetLog().Log($"{GetName()}: {msg}"); log.Log(msg);
} }
private void DoNothing(Failure failure) private void DoNothing(Failure failure)

View File

@ -1,5 +1,5 @@
using CodexPlugin.OverwatchSupport.LineConverters; using CodexPlugin.OverwatchSupport.LineConverters;
using Core; using KubernetesWorkflow;
using OverwatchTranscript; using OverwatchTranscript;
using Utils; using Utils;

View File

@ -1,5 +1,6 @@
using CodexPlugin.Hooks; using CodexPlugin.Hooks;
using Core; using KubernetesWorkflow;
using Logging;
using OverwatchTranscript; using OverwatchTranscript;
using Utils; using Utils;
@ -8,21 +9,26 @@ namespace CodexPlugin.OverwatchSupport
public class CodexTranscriptWriter : ICodexHooksProvider public class CodexTranscriptWriter : ICodexHooksProvider
{ {
private const string CodexHeaderKey = "cdx_h"; private const string CodexHeaderKey = "cdx_h";
private readonly ILog log;
private readonly ITranscriptWriter writer; private readonly ITranscriptWriter writer;
private readonly CodexLogConverter converter; private readonly CodexLogConverter converter;
private readonly NameIdMap nameIdMap = new NameIdMap(); private readonly NameIdMap nameIdMap = new NameIdMap();
public CodexTranscriptWriter(ITranscriptWriter transcriptWriter) public CodexTranscriptWriter(ILog log, ITranscriptWriter transcriptWriter)
{ {
this.log = log;
writer = transcriptWriter; writer = transcriptWriter;
converter = new CodexLogConverter(writer, nameIdMap); converter = new CodexLogConverter(writer, nameIdMap);
} }
public void Finalize(string outputFilepath) public void Finalize(string outputFilepath)
{ {
writer.AddHeader(CodexHeaderKey, CreateCodexHeader()); log.Log("Finalizing Codex transcript...");
writer.AddHeader(CodexHeaderKey, CreateCodexHeader());
writer.Write(outputFilepath); writer.Write(outputFilepath);
log.Log("Done");
} }
public ICodexNodeHooks CreateHooks(string nodeName) public ICodexNodeHooks CreateHooks(string nodeName)
@ -38,14 +44,17 @@ namespace CodexPlugin.OverwatchSupport
public void ProcessLogs(IDownloadedLog[] downloadedLogs) public void ProcessLogs(IDownloadedLog[] downloadedLogs)
{ {
foreach (var log in downloadedLogs) foreach (var l in downloadedLogs)
{ {
writer.IncludeArtifact(log.GetFilepath()); log.Log("Include artifact: " + l.GetFilepath());
writer.IncludeArtifact(l.GetFilepath());
// Not all of these logs are necessarily Codex logs. // Not all of these logs are necessarily Codex logs.
// Check, and process only the Codex ones. // Check, and process only the Codex ones.
if (IsCodexLog(log)) if (IsCodexLog(l))
{ {
converter.ProcessLog(log); log.Log("Processing Codex log: " + l.GetFilepath());
converter.ProcessLog(l);
} }
} }
} }

View File

@ -4,6 +4,7 @@ using Utils;
using Core; using Core;
using CodexPlugin; using CodexPlugin;
using KubernetesWorkflow.Types; using KubernetesWorkflow.Types;
using KubernetesWorkflow;
namespace ContinuousTests namespace ContinuousTests
{ {

View File

@ -138,7 +138,8 @@ namespace CodexTests
{ {
if (GetTranscriptAttributeOfCurrentTest() == null) return; if (GetTranscriptAttributeOfCurrentTest() == null) return;
var writer = new CodexTranscriptWriter(Transcript.NewWriter()); var log = new LogPrefixer(lifecycle.Log, "(Transcript) ");
var writer = new CodexTranscriptWriter(log, Transcript.NewWriter());
Ci.SetCodexHooksProvider(writer); Ci.SetCodexHooksProvider(writer);
writers.Add(lifecycle, writer); writers.Add(lifecycle, writer);
} }

View File

@ -1,4 +1,4 @@
using Core; using KubernetesWorkflow;
using NUnit.Framework; using NUnit.Framework;
namespace DistTestCore namespace DistTestCore

View File

@ -15,6 +15,7 @@ namespace DistTestCore
private readonly Dictionary<string, string> metadata; private readonly Dictionary<string, string> metadata;
private readonly List<RunningPod> runningContainers = new(); private readonly List<RunningPod> runningContainers = new();
private readonly string deployId; private readonly string deployId;
private readonly List<IDownloadedLog> stoppedContainerLogs = new List<IDownloadedLog>();
public TestLifecycle(TestLog log, Configuration configuration, ITimeSet timeSet, string testNamespace, string deployId, bool waitForCleanup) public TestLifecycle(TestLog log, Configuration configuration, ITimeSet timeSet, string testNamespace, string deployId, bool waitForCleanup)
{ {
@ -80,6 +81,12 @@ namespace DistTestCore
public void OnContainersStopped(RunningPod rc) public void OnContainersStopped(RunningPod rc)
{ {
runningContainers.Remove(rc); runningContainers.Remove(rc);
stoppedContainerLogs.AddRange(rc.Containers.Select(c =>
{
if (c.StopLog == null) throw new Exception("Expected StopLog for stopped container " + c.Name);
return c.StopLog;
}));
} }
public void OnContainerRecipeCreated(ContainerRecipe recipe) public void OnContainerRecipeCreated(ContainerRecipe recipe)
@ -98,24 +105,31 @@ namespace DistTestCore
} }
} }
private IDownloadedLog[] allLogs = Array.Empty<IDownloadedLog>();
public IDownloadedLog[] DownloadAllLogs() public IDownloadedLog[] DownloadAllLogs()
{ {
if (allLogs.Any()) return allLogs;
try try
{ {
var result = new List<IDownloadedLog>(); var result = new List<IDownloadedLog>();
result.AddRange(stoppedContainerLogs);
foreach (var rc in runningContainers) foreach (var rc in runningContainers)
{
if (rc.IsStopped)
{
foreach (var c in rc.Containers)
{
if (c.StopLog == null) throw new Exception("No stop-log was downloaded for container.");
result.Add(c.StopLog);
}
}
else
{ {
foreach (var c in rc.Containers) foreach (var c in rc.Containers)
{ {
result.Add(CoreInterface.DownloadLog(c)); result.Add(CoreInterface.DownloadLog(c));
} }
} }
allLogs = result.ToArray(); }
return allLogs; return result.ToArray();
} }
catch (Exception ex) catch (Exception ex)
{ {