status-im
/
cs-codex-dist-tests
mirror of
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fixes identity issue for runningpod/runningcontainer and log saving for stopped containers

This commit is contained in:
benbierens 2024-08-01 10:39:06 +02:00
parent 37611bdc66
commit d16b8cb011
No known key found for this signature in database
GPG Key ID: 877D2C2E09A22F3A
12 changed files with 130 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Framework/Core/CoreInterface.cs
View File

@ -30,11 +30,7 @@ namespace Core
public IDownloadedLog DownloadLog(RunningContainer container, int? tailLines = null)
{
var workflow = entryPoint.Tools.CreateWorkflow();
var msg = $"Downloading container log for '{container.Name}'";
entryPoint.Tools.GetLog().Log(msg);
var logHandler = new WriteToFileLogHandler(entryPoint.Tools.GetLog(), msg);
workflow.DownloadContainerLog(container, logHandler, tailLines);
return new DownloadedLog(logHandler, container.Name);
return workflow.DownloadContainerLog(container, tailLines);
}
public string ExecuteContainerCommand(IHasContainer containerSource, string command, params string[] args)

7
Framework/Core/DownloadedLog.cs → Framework/KubernetesWorkflow/DownloadedLog.cs
View File

@ -1,7 +1,6 @@
using KubernetesWorkflow;
using Logging;
using Logging;
namespace Core
namespace KubernetesWorkflow
{
public interface IDownloadedLog
{
@ -23,7 +22,7 @@ namespace Core
logFile = logHandler.LogFile;
ContainerName = containerName;
}
public string ContainerName { get; }
public void IterateLines(Action<string> action, params string[] thatContain)

29
Framework/KubernetesWorkflow/StartupWorkflow.cs
View File

@ -16,6 +16,7 @@ namespace KubernetesWorkflow
CrashWatcher CreateCrashWatcher(RunningContainer container);
void Stop(RunningPod pod, bool waitTillStopped);
void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null, bool? previous = null);
IDownloadedLog DownloadContainerLog(RunningContainer container, int? tailLines = null, bool? previous = null);
string ExecuteCommand(RunningContainer container, string command, params string[] args);
void DeleteNamespace(bool wait);
void DeleteNamespacesStartingWith(string namespacePrefix, bool wait);
@ -60,7 +61,7 @@ namespace KubernetesWorkflow
var startResult = controller.BringOnline(recipes, location);
var containers = CreateContainers(startResult, recipes, startupConfig);
var rc = new RunningPod(startupConfig, startResult, containers);
var rc = new RunningPod(Guid.NewGuid().ToString(), startupConfig, startResult, containers);
cluster.Configuration.Hooks.OnContainersStarted(rc);
if (startResult.ExternalService != null)
@ -99,11 +100,19 @@ namespace KubernetesWorkflow
public void Stop(RunningPod runningPod, bool waitTillStopped)
{
if (runningPod.IsStopped) return;
foreach (var c in runningPod.Containers)
{
c.StopLog = DownloadContainerLog(c);
}
runningPod.IsStopped = true;
K8s(controller =>
{
controller.Stop(runningPod.StartResult, waitTillStopped);
cluster.Configuration.Hooks.OnContainersStopped(runningPod);
});
cluster.Configuration.Hooks.OnContainersStopped(runningPod);
}
public void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null, bool? previous = null)
@ -114,6 +123,20 @@ namespace KubernetesWorkflow
});
}
public IDownloadedLog DownloadContainerLog(RunningContainer container, int? tailLines = null, bool? previous = null)
{
var msg = $"Downloading container log for '{container.Name}'";
log.Log(msg);
var logHandler = new WriteToFileLogHandler(log, msg);
K8s(controller =>
{
controller.DownloadPodLog(container, logHandler, tailLines, previous);
});
return new DownloadedLog(logHandler, container.Name);
}
public string ExecuteCommand(RunningContainer container, string command, params string[] args)
{
return K8s(controller =>
@ -147,7 +170,7 @@ namespace KubernetesWorkflow
var addresses = CreateContainerAddresses(startResult, r);
log.Debug($"{r}={name} -> container addresses: {string.Join(Environment.NewLine, addresses.Select(a => a.ToString()))}");
return new RunningContainer(name, r, addresses);
return new RunningContainer(Guid.NewGuid().ToString(), name, r, addresses);
}).ToArray();
}

21
Framework/KubernetesWorkflow/Types/RunningContainer.cs
View File

@ -7,16 +7,19 @@ namespace KubernetesWorkflow.Types
{
public class RunningContainer
{
public RunningContainer(string name, ContainerRecipe recipe, ContainerAddress[] addresses)
public RunningContainer(string id, string name, ContainerRecipe recipe, ContainerAddress[] addresses)
{
Id = id;
Name = name;
Recipe = recipe;
Addresses = addresses;
}
public string Id { get; }
public string Name { get; }
public ContainerRecipe Recipe { get; }
public ContainerAddress[] Addresses { get; }
public IDownloadedLog? StopLog { get; internal set; }
[JsonIgnore]
public RunningPod RunningPod { get; internal set; } = null!;
@ -50,5 +53,21 @@ namespace KubernetesWorkflow.Types
}
throw new Exception("Running location not known.");
}
public override string ToString()
{
return Name;
}
public override bool Equals(object? obj)
{
return obj is RunningContainer container &&
Id == container.Id;
}
public override int GetHashCode()
{
return HashCode.Combine(Id);
}
}
}

24
Framework/KubernetesWorkflow/Types/RunningPod.cs
View File

@ -4,8 +4,9 @@ namespace KubernetesWorkflow.Types
{
public class RunningPod
{
public RunningPod(StartupConfig startupConfig, StartResult startResult, RunningContainer[] containers)
public RunningPod(string id, StartupConfig startupConfig, StartResult startResult, RunningContainer[] containers)
{
Id = id;
StartupConfig = startupConfig;
StartResult = startResult;
Containers = containers;
@ -13,6 +14,7 @@ namespace KubernetesWorkflow.Types
foreach (var c in containers) c.RunningPod = this;
}
public string Id { get; }
public StartupConfig StartupConfig { get; }
public StartResult StartResult { get; }
public RunningContainer[] Containers { get; }
@ -23,10 +25,30 @@ namespace KubernetesWorkflow.Types
get { return $"'{string.Join("&", Containers.Select(c => c.Name).ToArray())}'"; }
}
[JsonIgnore]
public bool IsStopped { get; internal set; }
public string Describe()
{
return string.Join(",", Containers.Select(c => c.Name));
}
public override bool Equals(object? obj)
{
return obj is RunningPod pod &&
Id == pod.Id;
}
public override int GetHashCode()
{
return HashCode.Combine(Id);
}
public override string ToString()
{
if (IsStopped) return Name + " (*)";
return Name;
}
}
public static class RunningContainersExtensions

24
ProjectPlugins/CodexPlugin/CodexNode.cs
View File

@ -41,6 +41,7 @@ namespace CodexPlugin
public class CodexNode : ICodexNode
{
private const string UploadFailedMessage = "Unable to store block";
private readonly ILog log;
private readonly IPluginTools tools;
private readonly ICodexNodeHooks hooks;
private readonly EthAccount? ethAccount;
@ -57,6 +58,8 @@ namespace CodexPlugin
this.hooks = hooks;
Version = new DebugInfoVersion();
transferSpeeds = new TransferSpeeds();
log = new LogPrefixer(tools.GetLog(), $"{GetName()} ");
}
public void Awake()
@ -141,9 +144,8 @@ namespace CodexPlugin
hooks.OnFileUploading(uniqueId, size);
var logMessage = $"Uploading file {file.Describe()}...";
Log(logMessage);
var measurement = Stopwatch.Measure(tools.GetLog(), logMessage, () =>
var logMessage = $"Uploading file '{file.Describe()}'...";
var measurement = Stopwatch.Measure(log, logMessage, () =>
{
return CodexAccess.UploadFile(fileStream, onFailure);
});
@ -154,7 +156,7 @@ namespace CodexPlugin
if (string.IsNullOrEmpty(response)) FrameworkAssert.Fail("Received empty response.");
if (response.StartsWith(UploadFailedMessage)) FrameworkAssert.Fail("Node failed to store block.");
Log($"Uploaded file. Received contentId: '{response}'.");
Log($"Uploaded file '{file.Describe()}'. Received contentId: '{response}'.");
var cid = new ContentId(response);
hooks.OnFileUploaded(uniqueId, size, cid);
@ -168,15 +170,16 @@ namespace CodexPlugin
public TrackedFile? DownloadContent(ContentId contentId, Action<Failure> onFailure, string fileLabel = "")
{
var logMessage = $"Downloading for contentId: '{contentId.Id}'...";
hooks.OnFileDownloading(contentId);
Log(logMessage);
var file = tools.GetFileManager().CreateEmptyFile(fileLabel);
var measurement = Stopwatch.Measure(tools.GetLog(), logMessage, () => DownloadToFile(contentId.Id, file, onFailure));
var logMessage = $"Downloading '{contentId.Id}' to '{file.Filename}'";
hooks.OnFileDownloading(contentId);
var measurement = Stopwatch.Measure(log, logMessage, () => DownloadToFile(contentId.Id, file, onFailure));
var size = file.GetFilesize();
transferSpeeds.AddDownloadSample(size, measurement);
Log($"Downloaded file {file.Describe()} to '{file.Filename}'.");
hooks.OnFileDownloaded(size, contentId);
return file;
}
@ -231,7 +234,6 @@ namespace CodexPlugin
throw new Exception($"Invalid version information received from Codex node {GetName()}: {debugInfo.Version}");
}
var log = tools.GetLog();
log.AddStringReplace(peerId, nodeName);
log.AddStringReplace(CodexUtils.ToShortId(peerId), nodeName);
log.AddStringReplace(debugInfo.Table.LocalNode.NodeId, nodeName);
@ -273,7 +275,7 @@ namespace CodexPlugin
private void Log(string msg)
{
tools.GetLog().Log($"{GetName()}: {msg}");
log.Log(msg);
}
private void DoNothing(Failure failure)

2
ProjectPlugins/CodexPlugin/OverwatchSupport/CodexLogConverter.cs
View File

@ -1,5 +1,5 @@
using CodexPlugin.OverwatchSupport.LineConverters;
using Core;
using KubernetesWorkflow;
using OverwatchTranscript;
using Utils;

23
ProjectPlugins/CodexPlugin/OverwatchSupport/CodexTranscriptWriter.cs
View File

@ -1,5 +1,6 @@
using CodexPlugin.Hooks;
using Core;
using KubernetesWorkflow;
using Logging;
using OverwatchTranscript;
using Utils;
@ -8,21 +9,26 @@ namespace CodexPlugin.OverwatchSupport
public class CodexTranscriptWriter : ICodexHooksProvider
{
private const string CodexHeaderKey = "cdx_h";
private readonly ILog log;
private readonly ITranscriptWriter writer;
private readonly CodexLogConverter converter;
private readonly NameIdMap nameIdMap = new NameIdMap();
public CodexTranscriptWriter(ITranscriptWriter transcriptWriter)
public CodexTranscriptWriter(ILog log, ITranscriptWriter transcriptWriter)
{
this.log = log;
writer = transcriptWriter;
converter = new CodexLogConverter(writer, nameIdMap);
}
public void Finalize(string outputFilepath)
{
writer.AddHeader(CodexHeaderKey, CreateCodexHeader());
log.Log("Finalizing Codex transcript...");
writer.AddHeader(CodexHeaderKey, CreateCodexHeader());
writer.Write(outputFilepath);
log.Log("Done");
}
public ICodexNodeHooks CreateHooks(string nodeName)
@ -38,14 +44,17 @@ namespace CodexPlugin.OverwatchSupport
public void ProcessLogs(IDownloadedLog[] downloadedLogs)
{
foreach (var log in downloadedLogs)
foreach (var l in downloadedLogs)
{
writer.IncludeArtifact(log.GetFilepath());
log.Log("Include artifact: " + l.GetFilepath());
writer.IncludeArtifact(l.GetFilepath());
// Not all of these logs are necessarily Codex logs.
// Check, and process only the Codex ones.
if (IsCodexLog(log))
if (IsCodexLog(l))
{
converter.ProcessLog(log);
log.Log("Processing Codex log: " + l.GetFilepath());
converter.ProcessLog(l);
}
}
}

1
Tests/CodexContinuousTests/NodeRunner.cs
View File

@ -4,6 +4,7 @@ using Utils;
using Core;
using CodexPlugin;
using KubernetesWorkflow.Types;
using KubernetesWorkflow;
namespace ContinuousTests
{

3
Tests/CodexTests/CodexDistTest.cs
View File

@ -138,7 +138,8 @@ namespace CodexTests
{
if (GetTranscriptAttributeOfCurrentTest() == null) return;
var writer = new CodexTranscriptWriter(Transcript.NewWriter());
var log = new LogPrefixer(lifecycle.Log, "(Transcript) ");
var writer = new CodexTranscriptWriter(log, Transcript.NewWriter());
Ci.SetCodexHooksProvider(writer);
writers.Add(lifecycle, writer);
}

2
Tests/DistTestCore/DownloadedLogExtensions.cs
View File

@ -1,4 +1,4 @@
using Core;
using KubernetesWorkflow;
using NUnit.Framework;
namespace DistTestCore

32
Tests/DistTestCore/TestLifecycle.cs
View File

@ -15,6 +15,7 @@ namespace DistTestCore
private readonly Dictionary<string, string> metadata;
private readonly List<RunningPod> runningContainers = new();
private readonly string deployId;
private readonly List<IDownloadedLog> stoppedContainerLogs = new List<IDownloadedLog>();
public TestLifecycle(TestLog log, Configuration configuration, ITimeSet timeSet, string testNamespace, string deployId, bool waitForCleanup)
{
@ -80,6 +81,12 @@ namespace DistTestCore
public void OnContainersStopped(RunningPod rc)
{
runningContainers.Remove(rc);
stoppedContainerLogs.AddRange(rc.Containers.Select(c =>
{
if (c.StopLog == null) throw new Exception("Expected StopLog for stopped container " + c.Name);
return c.StopLog;
}));
}
public void OnContainerRecipeCreated(ContainerRecipe recipe)
@ -98,24 +105,31 @@ namespace DistTestCore
}
}
private IDownloadedLog[] allLogs = Array.Empty<IDownloadedLog>();
public IDownloadedLog[] DownloadAllLogs()
{
if (allLogs.Any()) return allLogs;
try
{
var result = new List<IDownloadedLog>();
var result = new List<IDownloadedLog>();
result.AddRange(stoppedContainerLogs);
foreach (var rc in runningContainers)
{
foreach (var c in rc.Containers)
if (rc.IsStopped)
{
result.Add(CoreInterface.DownloadLog(c));
foreach (var c in rc.Containers)
{
if (c.StopLog == null) throw new Exception("No stop-log was downloaded for container.");
result.Add(c.StopLog);
}
}
else
{
foreach (var c in rc.Containers)
{
result.Add(CoreInterface.DownloadLog(c));
}
}
}
allLogs = result.ToArray();
return allLogs;
return result.ToArray();
}
catch (Exception ex)
{