logos-storage/logos-storage-nim-cs-dist-tests
1
0
Fork 0
mirror of https://github.com/logos-storage/logos-storage-nim-cs-dist-tests.git synced 2026-01-04 22:43:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Moving downloadedlog to logging module

Browse Source
This commit is contained in:
ThatBen 2025-01-16 10:15:02 +01:00
parent 0c7b8fb33e
commit ec644eed4a
No known key found for this signature in database
GPG Key ID: 62C543548433D43E
27 changed files with 164 additions and 124 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Framework/Core/CoreInterface.cs
View File

@ -1,5 +1,6 @@
using KubernetesWorkflow;
using KubernetesWorkflow.Types;
using Logging;
namespace Core
{

2
Framework/KubernetesWorkflow/LogHandler.cs
View File

@ -29,7 +29,7 @@ namespace KubernetesWorkflow
{
LogFile = sourceLog.CreateSubfile(addFileName);
var msg = $"{description} -->> {LogFile.FullFilename}";
var msg = $"{description} -->> {LogFile.Filename}";
sourceLog.Log(msg);
LogFile.Write(msg);

2
Framework/KubernetesWorkflow/StartupWorkflow.cs
View File

@ -134,7 +134,7 @@ namespace KubernetesWorkflow
controller.DownloadPodLog(container, logHandler, tailLines, previous);
});
return new DownloadedLog(logHandler, container.Name);
return new DownloadedLog(logHandler.LogFile, container.Name);
}
public string ExecuteCommand(RunningContainer container, string command, params string[] args)

6
Framework/Logging/BaseLog.cs
View File

@ -34,7 +34,7 @@ namespace Logging
{
get
{
if (logFile == null) logFile = new LogFile(GetFullName(), "log");
if (logFile == null) logFile = new LogFile(GetFullName() + ".log");
return logFile;
}
}
@ -69,7 +69,7 @@ namespace Logging
public virtual void Delete()
{
File.Delete(LogFile.FullFilename);
File.Delete(LogFile.Filename);
}
public LogFile CreateSubfile(string addName, string ext = "log")
@ -78,7 +78,7 @@ namespace Logging
.Replace("<", "")
.Replace(">", "");
return new LogFile($"{GetFullName()}_{GetSubfileNumber()}_{addName}", ext);
return new LogFile($"{GetFullName()}_{GetSubfileNumber()}_{addName}.{ext}");
}
protected string ApplyReplacements(string str)

28
Framework/KubernetesWorkflow/DownloadedLog.cs → Framework/Logging/DownloadedLog.cs
View File

@ -1,10 +1,8 @@
using Logging;
namespace KubernetesWorkflow
namespace Logging
{
public interface IDownloadedLog
{
string ContainerName { get; }
string SourceName { get; }
void IterateLines(Action<string> action);
void IterateLines(Action<string> action, params string[] thatContain);
@ -14,21 +12,27 @@ namespace KubernetesWorkflow
void DeleteFile();
}
internal class DownloadedLog : IDownloadedLog
public class DownloadedLog : IDownloadedLog
{
private readonly LogFile logFile;
internal DownloadedLog(WriteToFileLogHandler logHandler, string containerName)
public DownloadedLog(string filepath, string sourceName)
{
logFile = logHandler.LogFile;
ContainerName = containerName;
logFile = new LogFile(filepath);
SourceName = sourceName;
}
public string ContainerName { get; }
public DownloadedLog(LogFile logFile, string sourceName)
{
this.logFile = logFile;
SourceName = sourceName;
}
public string SourceName { get; }
public void IterateLines(Action<string> action)
{
using var file = File.OpenRead(logFile.FullFilename);
using var file = File.OpenRead(logFile.Filename);
using var streamReader = new StreamReader(file);
var line = streamReader.ReadLine();
@ -64,12 +68,12 @@ namespace KubernetesWorkflow
public string GetFilepath()
{
return logFile.FullFilename;
return logFile.Filename;
}
public void DeleteFile()
{
File.Delete(logFile.FullFilename);
File.Delete(logFile.Filename);
}
}
}

22
Framework/Logging/LogFile.cs
View File

@ -4,20 +4,16 @@ namespace Logging
{
public class LogFile
{
private readonly string extension;
private readonly object fileLock = new object();
private string filename;
public LogFile(string filename, string extension)
public LogFile(string filename)
{
this.filename = filename;
this.extension = extension;
FullFilename = filename + "." + extension;
Filename = filename;
EnsurePathExists(filename);
}
public string FullFilename { get; private set; }
public string Filename { get; private set; }
public void Write(string message)
{
@ -30,7 +26,7 @@ namespace Logging
{
lock (fileLock)
{
File.AppendAllLines(FullFilename, new[] { message });
File.AppendAllLines(Filename, new[] { message });
}
}
catch (Exception ex)
@ -39,16 +35,6 @@ namespace Logging
}
}
public void ConcatToFilename(string toAdd)
{
var oldFullName = FullFilename;
filename += toAdd;
FullFilename = filename + "." + extension;
File.Move(oldFullName, FullFilename);
}
private static string GetTimestamp()
{
return $"[{Time.FormatTimestamp(DateTime.UtcNow)}]";

17
ProjectPlugins/CodexPlugin/CodexAccess.cs
View File

@ -36,6 +36,11 @@ namespace CodexPlugin
instance = null!;
}
public IDownloadedLog DownloadLog(string additionalName = "")
{
return processControl.DownloadLog(instance, log.CreateSubfile(GetName() + additionalName));
}
public string GetImageName()
{
return instance.ImageName;
@ -195,12 +200,6 @@ namespace CodexPlugin
public Address GetDiscoveryEndpoint()
{
return instance.DiscoveryEndpoint;
//var info = codexAccess.GetPodInfo();
//return new Address(
// logName: $"{GetName()}:DiscoveryPort",
// host: info.Ip,
// port: Container.Recipe.GetPortByTag(CodexContainerRecipe.DiscoveryPortTag)!.Number
//);
}
public Address GetApiEndpoint()
@ -215,17 +214,17 @@ namespace CodexPlugin
public Address? GetMetricsEndpoint()
{
return instance.GetMetricsEndpoint();
return instance.MetricsEndpoint;
}
public EthAccount? GetEthAccount()
{
return instance.GetEthAccount();
return instance.EthAccount;
}
public void DeleteDataDirFolder()
{
instance.DeleteDataDirFolder();
processControl.DeleteDataDirFolder(instance);
}
private T OnCodex<T>(Func<CodexApi, Task<T>> action)

20
ProjectPlugins/CodexPlugin/CodexDeployment.cs
View File

@ -29,17 +29,17 @@ namespace CodexPlugin
public DeploymentMetadata Metadata { get; }
}
public class CodexInstance
{
public CodexInstance(RunningPod pod, DebugInfo info)
{
Pod = pod;
Info = info;
}
//public class CodexInstance
//{
// public CodexInstance(RunningPod pod, DebugInfo info)
// {
// Pod = pod;
// Info = info;
// }
public RunningPod Pod { get; }
public DebugInfo Info { get; }
}
// public RunningPod Pod { get; }
// public DebugInfo Info { get; }
//}
public class DeploymentMetadata
{

77
ProjectPlugins/CodexPlugin/CodexInstance.cs
View File

@ -14,31 +14,22 @@ namespace CodexPlugin
Address DiscoveryEndpoint { get; }
Address ApiEndpoint { get; }
Address ListenEndpoint { get; }
void DeleteDataDirFolder();
EthAccount? GetEthAccount();
Address? GetMetricsEndpoint();
EthAccount? EthAccount { get; }
Address? MetricsEndpoint { get; }
}
public class CodexContainerInstance : ICodexInstance
public class CodexInstance : ICodexInstance
{
private readonly RunningContainer container;
private readonly IPluginTools tools;
private readonly ILog log;
private readonly EthAccount? ethAccount = null;
public CodexContainerInstance(IPluginTools tools, ILog log, RunningPod pod)
public CodexInstance(string name, string imageName, DateTime startUtc, Address discoveryEndpoint, Address apiEndpoint, Address listenEndpoint, EthAccount? ethAccount, Address? metricsEndpoint)
{
container = pod.Containers.Single();
this.tools = tools;
this.log = log;
Name = container.Name;
ImageName = container.Recipe.Image;
StartUtc = container.Recipe.RecipeCreatedUtc;
DiscoveryEndpoint = container.GetInternalAddress(CodexContainerRecipe.DiscoveryPortTag);
ApiEndpoint = container.GetAddress(CodexContainerRecipe.ApiPortTag);
ListenEndpoint = container.GetInternalAddress(CodexContainerRecipe.ListenPortTag);
ethAccount = container.Recipe.Additionals.Get<EthAccount>();
Name = name;
ImageName = imageName;
StartUtc = startUtc;
DiscoveryEndpoint = discoveryEndpoint;
ApiEndpoint = apiEndpoint;
ListenEndpoint = listenEndpoint;
EthAccount = ethAccount;
MetricsEndpoint = metricsEndpoint;
}
public string Name { get; }
@ -47,29 +38,37 @@ namespace CodexPlugin
public Address DiscoveryEndpoint { get; }
public Address ApiEndpoint { get; }
public Address ListenEndpoint { get; }
public EthAccount? EthAccount { get; }
public Address? MetricsEndpoint { get; }
}
public void DeleteDataDirFolder()
public static class CodexInstanceContainerExtension
{
public static ICodexInstance CreateFromPod(RunningPod pod)
{
try
{
var dataDirVar = container.Recipe.EnvVars.Single(e => e.Name == "CODEX_DATA_DIR");
var dataDir = dataDirVar.Value;
var workflow = tools.CreateWorkflow();
workflow.ExecuteCommand(container, "rm", "-Rfv", $"/codex/{dataDir}/repo");
log.Log("Deleted repo folder.");
}
catch (Exception e)
{
log.Log("Unable to delete repo folder: " + e);
}
var container = pod.Containers.Single();
return new CodexInstance(
name: container.Name,
imageName: container.Recipe.Image,
startUtc: container.Recipe.RecipeCreatedUtc,
discoveryEndpoint: container.GetInternalAddress(CodexContainerRecipe.DiscoveryPortTag),
apiEndpoint: container.GetAddress(CodexContainerRecipe.ApiPortTag),
listenEndpoint: container.GetInternalAddress(CodexContainerRecipe.ListenPortTag),
ethAccount: container.Recipe.Additionals.Get<EthAccount>(),
metricsEndpoint: GetMetricsEndpoint(container)
);
}
public EthAccount? GetEthAccount()
{
return ethAccount;
}
// todo: is this needed for the discovery address??
//var info = codexAccess.GetPodInfo();
//return new Address(
// logName: $"{GetName()}:DiscoveryPort",
// host: info.Ip,
// port: Container.Recipe.GetPortByTag(CodexContainerRecipe.DiscoveryPortTag)!.Number
//);
public Address? GetMetricsEndpoint()
private static Address? GetMetricsEndpoint(RunningContainer container)
{
try
{

6
ProjectPlugins/CodexPlugin/CodexNode.cs
View File

@ -46,6 +46,7 @@ namespace CodexPlugin
/// </summary>
void DeleteDataDirFolder();
void Stop(bool waitTillStopped);
IDownloadedLog DownloadLog(string additionalName = "");
bool HasCrashed();
}
@ -272,6 +273,11 @@ namespace CodexPlugin
codexAccess.Stop(waitTillStopped);
}
public IDownloadedLog DownloadLog(string additionalName = "")
{
return codexAccess.DownloadLog(additionalName);
}
public Address GetDiscoveryEndpoint()
{
return codexAccess.GetDiscoveryEndpoint();

28
ProjectPlugins/CodexPlugin/CodexStarter.cs
View File

@ -59,6 +59,32 @@ namespace CodexPlugin
Log("Stopped.");
}
public IDownloadedLog DownloadLog(ICodexInstance instance, LogFile file)
{
var workflow = pluginTools.CreateWorkflow();
var pod = podMap[instance];
return workflow.DownloadContainerLog(pod.Containers.Single());
}
public void DeleteDataDirFolder(ICodexInstance instance)
{
var pod = podMap[instance];
var container = pod.Containers.Single();
try
{
var dataDirVar = container.Recipe.EnvVars.Single(e => e.Name == "CODEX_DATA_DIR");
var dataDir = dataDirVar.Value;
var workflow = pluginTools.CreateWorkflow();
workflow.ExecuteCommand(container, "rm", "-Rfv", $"/codex/{dataDir}/repo");
Log("Deleted repo folder.");
}
catch (Exception e)
{
Log("Unable to delete repo folder: " + e);
}
}
public ICodexNodeGroup WrapCodexContainers(CoreInterface coreInterface, RunningPod[] containers)
{
var codexNodeFactory = new CodexNodeFactory(pluginTools, HooksFactory);
@ -145,7 +171,7 @@ namespace CodexPlugin
private ICodexInstance CreateInstance(RunningPod pod)
{
var instance = new CodexContainerInstance(pluginTools, pluginTools.GetLog(), pod);
var instance = CodexInstanceContainerExtension.CreateFromPod(pod);
podMap.Add(instance, pod);
return instance;
}

1
ProjectPlugins/CodexPlugin/OverwatchSupport/CodexLogConverter.cs
View File

@ -1,5 +1,6 @@
using CodexPlugin.OverwatchSupport.LineConverters;
using KubernetesWorkflow;
using Logging;
using OverwatchTranscript;
using Utils;

24
ProjectPlugins/CodexPlugin/ProcessControl.cs
View File

@ -1,7 +1,29 @@
namespace CodexPlugin
using Logging;
namespace CodexPlugin
{
public interface IProcessControl
{
void Stop(ICodexInstance instance, bool waitTillStopped);
IDownloadedLog DownloadLog(ICodexInstance instance, LogFile file);
void DeleteDataDirFolder(ICodexInstance instance);
}
//public void DeleteDataDirFolder()
//{
// try
// {
// var dataDirVar = container.Recipe.EnvVars.Single(e => e.Name == "CODEX_DATA_DIR");
// var dataDir = dataDirVar.Value;
// var workflow = tools.CreateWorkflow();
// workflow.ExecuteCommand(container, "rm", "-Rfv", $"/codex/{dataDir}/repo");
// log.Log("Deleted repo folder.");
// }
// catch (Exception e)
// {
// log.Log("Unable to delete repo folder: " + e);
// }
//}
}

2
ProjectPlugins/MetricsPlugin/MetricsDownloader.cs
View File

@ -26,7 +26,7 @@ namespace MetricsPlugin
private LogFile WriteToFile(string nodeName, string[] headers, Dictionary<DateTime, List<string>> map)
{
var file = log.CreateSubfile("csv");
log.Log($"Downloading metrics for {nodeName} to file {file.FullFilename}");
log.Log($"Downloading metrics for {nodeName} to file {file.Filename}");
file.WriteRaw(string.Join(",", headers));

2
Tests/CodexContinuousTests/ElasticSearchLogDownloader.cs
View File

@ -37,7 +37,7 @@ namespace ContinuousTests
var endpoint = CreateElasticSearchEndpoint();
var queryTemplate = CreateQueryTemplate(container, startUtc, endUtc);
targetFile.Write($"Downloading '{container.Name}' to '{targetFile.FullFilename}'.");
targetFile.Write($"Downloading '{container.Name}' to '{targetFile.Filename}'.");
var reconstructor = new LogReconstructor(targetFile, endpoint, queryTemplate);
reconstructor.DownloadFullLog();

8
Tests/CodexContinuousTests/NodeRunner.cs
View File

@ -24,12 +24,6 @@ namespace ContinuousTests
this.customNamespace = customNamespace;
}
public IDownloadedLog DownloadLog(RunningContainer container, int? tailLines = null)
{
var entryPoint = CreateEntryPoint();
return entryPoint.CreateInterface().DownloadLog(container, tailLines);
}
public void RunNode(Action<ICodexSetup> setup, Action<ICodexNode> operation)
{
RunNode(nodes.ToList().PickOneRandom(), setup, operation);
@ -59,7 +53,7 @@ namespace ContinuousTests
}
catch
{
DownloadLog(node.Container);
node.DownloadLog();
throw;
}
}

15
Tests/CodexContinuousTests/SingleTestRun.cs
View File

@ -125,13 +125,14 @@ namespace ContinuousTests
foreach (var node in nodes)
{
var container = node.Container;
var deploymentName = container.RunningPod.StartResult.Deployment.Name;
var namespaceName = container.RunningPod.StartResult.Cluster.Configuration.KubernetesNamespace;
var openingLine =
$"{namespaceName} - {deploymentName} = {node.Container.Name} = {node.GetDebugInfo().Id}";
elasticSearchLogDownloader.Download(fixtureLog.CreateSubfile(node.GetName()), node.Container, effectiveStart,
effectiveEnd, openingLine);
//var container = node.Container;
//var deploymentName = container.RunningPod.StartResult.Deployment.Name;
//var namespaceName = container.RunningPod.StartResult.Cluster.Configuration.KubernetesNamespace;
//var openingLine =
// $"{namespaceName} - {deploymentName} = {node.Container.Name} = {node.GetDebugInfo().Id}";
//elasticSearchLogDownloader.Download(fixtureLog.CreateSubfile(node.GetName()), node.Container, effectiveStart,
// effectiveEnd, openingLine);
throw new NotImplementedException("access to container info is unavilable.");
}
}

4
Tests/CodexLongTests/ScalabilityTests/MultiPeerDownloadTests.cs
View File

@ -31,7 +31,7 @@ namespace CodexTests.ScalabilityTests
Task.WaitAll(uploadTasks);
var cid = new ContentId(uploadTasks.Select(t => t.Result.Id).Distinct().Single());
var uploadLog = Ci.DownloadLog(hosts[0]);
var uploadLog = hosts[0].DownloadLog();
var expectedNumberOfBlocks = RoundUp(fileSize.MB().SizeInBytes, 64.KB().SizeInBytes) + 1; // +1 for manifest block.
var blockCids = uploadLog
.FindLinesThatContain("Block Stored")
@ -50,7 +50,7 @@ namespace CodexTests.ScalabilityTests
var resultFile = client.DownloadContent(cid);
resultFile!.AssertIsEqual(file);
var downloadLog = Ci.DownloadLog(client);
var downloadLog = client.DownloadLog();
var host = string.Empty;
var blockAddressHostMap = new Dictionary<string, List<string>>();
downloadLog

2
Tests/CodexReleaseTests/DataTests/InterruptUploadTest.cs
View File

@ -32,7 +32,7 @@ namespace CodexReleaseTests.DataTests
process.Kill();
Thread.Sleep(1000);
var log = Ci.DownloadLog(node);
var log = node.DownloadLog();
return !log.GetLinesContaining("Unhandled exception in async proc, aborting").Any();
}

2
Tests/DistTestCore/DownloadedLogExtensions.cs
View File

@ -1,4 +1,4 @@
using KubernetesWorkflow;
using Logging;
using NUnit.Framework;
namespace DistTestCore

1
Tests/DistTestCore/Logs/BaseTestLog.cs
View File

@ -23,7 +23,6 @@ namespace DistTestCore.Logs
{
if (hasFailed) return;
hasFailed = true;
LogFile.ConcatToFilename("_FAILED");
}
}
}

1
Tests/DistTestCore/TestLifecycle.cs
View File

@ -4,6 +4,7 @@ using FileUtils;
using KubernetesWorkflow;
using KubernetesWorkflow.Recipe;
using KubernetesWorkflow.Types;
using Logging;
using Utils;
namespace DistTestCore

2
Tests/ExperimentalTests/BasicTests/ExampleTests.cs
View File

@ -20,7 +20,7 @@ namespace CodexTests.BasicTests
var localDatasets = primary.LocalFiles();
CollectionAssert.Contains(localDatasets.Content.Select(c => c.Cid), cid);
var log = Ci.DownloadLog(primary);
var log = primary.DownloadLog();
log.AssertLogContains("Uploaded file");
}

8
Tests/ExperimentalTests/CodexDistTest.cs
View File

@ -105,7 +105,7 @@ namespace CodexTests
public void CheckLogForErrors(ICodexNode node)
{
Log($"Checking {node.GetName()} log for errors.");
var log = Ci.DownloadLog(node);
var log = node.DownloadLog();
log.AssertLogDoesNotContain("Block validation failed");
log.AssertLogDoesNotContainLinesStartingWith("ERR ");
@ -203,7 +203,7 @@ namespace CodexTests
Stopwatch.Measure(lifecycle.Log, $"Transcript.Finalize: {outputFilepath}", () =>
{
writer.IncludeFile(lifecycle.Log.LogFile.FullFilename);
writer.IncludeFile(lifecycle.Log.LogFile.Filename);
writer.Finalize(outputFilepath);
});
}
@ -215,9 +215,9 @@ namespace CodexTests
private string GetOutputFullPath(TestLifecycle lifecycle, CreateTranscriptAttribute attr)
{
var outputPath = Path.GetDirectoryName(lifecycle.Log.LogFile.FullFilename);
var outputPath = Path.GetDirectoryName(lifecycle.Log.LogFile.Filename);
if (outputPath == null) throw new Exception("Logfile path is null");
var filename = Path.GetFileNameWithoutExtension(lifecycle.Log.LogFile.FullFilename);
var filename = Path.GetFileNameWithoutExtension(lifecycle.Log.LogFile.Filename);
if (string.IsNullOrEmpty(filename)) throw new Exception("Logfile name is null or empty");
var outputFile = Path.Combine(outputPath, filename + "_" + attr.OutputFilename);
if (!outputFile.EndsWith(".owts")) outputFile += ".owts";

2
Tests/ExperimentalTests/UtilityTests/LogHelperTests.cs
View File

@ -35,7 +35,7 @@ namespace CodexTests.UtilityTests
private Dictionary<string, int> GetLogMap(ICodexNode node, DateTime? startUtc = null)
{
var log = Ci.DownloadLog(node);
var log = node.DownloadLog();
var map = new Dictionary<string, int>();
log.IterateLines(line =>
{

2
Tools/CodexNetDeployer/CodexNodeStarter.cs
View File

@ -100,7 +100,7 @@ namespace CodexNetDeployer
if (codexNode != null)
{
Console.WriteLine("Downloading container log.");
ci.DownloadLog(codexNode);
codexNode.DownloadLog();
}
return null;

3
Tools/CodexNetDeployer/Deployer.cs
View File

@ -181,7 +181,8 @@ namespace CodexNetDeployer
private CodexInstance CreateCodexInstance(ICodexNode node)
{
return new CodexInstance(node.Container.RunningPod, node.GetDebugInfo());
//return new CodexInstance(node.Container.RunningPod, node.GetDebugInfo());
throw new NotImplementedException();
}
private string? GetKubeConfig(string kubeConfigFile)