codex-storage
/
cs-codex-dist-tests
mirror of https://github.com/codex-storage/cs-codex-dist-tests.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Successful access to codex log from within test

This commit is contained in:
benbierens 2023-03-26 10:45:01 +02:00
parent b088763c3d
commit ce963a228c
No known key found for this signature in database
GPG Key ID: FE44815D96D0A1AA
10 changed files with 211 additions and 132 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
CodexDistTestCore/CodexNodeGroup.cs
View File

@ -12,10 +12,12 @@ namespace CodexDistTestCore
public class CodexNodeGroup : ICodexNodeGroup
{
private readonly TestLog log;
private readonly IK8sManager k8SManager;
public CodexNodeGroup(int orderNumber, OfflineCodexNodes origin, IK8sManager k8SManager, OnlineCodexNode[] nodes)
public CodexNodeGroup(TestLog log, int orderNumber, OfflineCodexNodes origin, IK8sManager k8SManager, OnlineCodexNode[] nodes)
{
this.log = log;
OrderNumber = orderNumber;
Origin = origin;
this.k8SManager = k8SManager;
@ -77,6 +79,13 @@ namespace CodexDistTestCore
};
}
public CodexNodeLog DownloadLog(IOnlineCodexNode node)
{
var logDownloader = new PodLogDownloader(log, k8SManager);
var n = (OnlineCodexNode)node;
return logDownloader.DownloadLog(n);
}
public Dictionary<string, string> GetSelector()
{
return new Dictionary<string, string> { { "codex-test-node", "dist-test-" + OrderNumber } };

29
CodexDistTestCore/CodexNodeLog.cs Normal file
View File

@ -0,0 +1,29 @@
using NUnit.Framework;
namespace CodexDistTestCore
{
public class CodexNodeLog
{
private readonly LogFile logFile;
public CodexNodeLog(LogFile logFile)
{
this.logFile = logFile;
}
public void AssertLogContains(string expectedString)
{
using var file = File.OpenRead(logFile.FullFilename);
using var streamReader = new StreamReader(file);
var line = streamReader.ReadLine();
while (line != null)
{
if (line.Contains(expectedString)) return;
line = streamReader.ReadLine();
}
Assert.Fail($"Unable to find string '{expectedString}' in CodexNode log file {logFile.FilenameWithoutPath}");
}
}
}

22
CodexDistTestCore/DistTest.cs
View File

@ -56,7 +56,8 @@ namespace CodexDistTestCore
{
try
{
log.EndTest(k8sManager);
log.EndTest();
IncludeLogsOnTestFailure();
k8sManager.DeleteAllResources();
fileManager.DeleteAllTestFiles();
}
@ -76,6 +77,25 @@ namespace CodexDistTestCore
{
return new OfflineCodexNodes(k8sManager, numberOfNodes);
}
private void IncludeLogsOnTestFailure()
{
var result = TestContext.CurrentContext.Result;
if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed)
{
k8sManager.ForEachOnlineGroup(DownloadLogs);
}
}
private void DownloadLogs(CodexNodeGroup group)
{
foreach (var node in group)
{
var downloader = new PodLogDownloader(log, k8sManager);
var n = (OnlineCodexNode)node;
downloader.DownloadLog(n);
}
}
}
public static class GlobalTestFailure

18
CodexDistTestCore/K8sManager.cs
View File

@ -4,12 +4,13 @@
{
ICodexNodeGroup BringOnline(OfflineCodexNodes node);
IOfflineCodexNodes BringOffline(ICodexNodeGroup node);
void FetchPodLog(OnlineCodexNode node, IPodLogHandler logHandler);
}
public class K8sManager : IK8sManager
{
private readonly CodexGroupNumberSource codexGroupNumberSource = new CodexGroupNumberSource();
private readonly List<CodexNodeGroup> onlineCodexNodes = new List<CodexNodeGroup>();
private readonly List<CodexNodeGroup> onlineCodexNodeGroups = new List<CodexNodeGroup>();
private readonly KnownK8sPods knownPods = new KnownK8sPods();
private readonly TestLog log;
private readonly IFileManager fileManager;
@ -47,17 +48,22 @@
K8s(k => k.DeleteAllResources());
}
public void FetchAllPodsLogs(IPodLogsHandler logHandler)
public void ForEachOnlineGroup(Action<CodexNodeGroup> action)
{
K8s(k => k.FetchAllPodsLogs(onlineCodexNodes.ToArray(), logHandler));
foreach (var group in onlineCodexNodeGroups) action(group);
}
public void FetchPodLog(OnlineCodexNode node, IPodLogHandler logHandler)
{
K8s(k => k.FetchPodLog(node, logHandler));
}
private CodexNodeGroup CreateOnlineCodexNodes(OfflineCodexNodes offline)
{
var containers = CreateContainers(offline.NumberOfNodes);
var online = containers.Select(c => new OnlineCodexNode(log, fileManager, c)).ToArray();
var result = new CodexNodeGroup(codexGroupNumberSource.GetNextCodexNodeGroupNumber(), offline, this, online);
onlineCodexNodes.Add(result);
var result = new CodexNodeGroup(log, codexGroupNumberSource.GetNextCodexNodeGroupNumber(), offline, this, online);
onlineCodexNodeGroups.Add(result);
return result;
}
@ -72,7 +78,7 @@
private CodexNodeGroup GetAndRemoveActiveNodeFor(ICodexNodeGroup node)
{
var n = (CodexNodeGroup)node;
onlineCodexNodes.Remove(n);
onlineCodexNodeGroups.Remove(n);
return n;
}

21
CodexDistTestCore/K8sOperations.cs
View File

@ -51,25 +51,10 @@ namespace CodexDistTestCore
WaitUntilNamespaceDeleted();
}
public void FetchAllPodsLogs(CodexNodeGroup[] onlines, IPodLogsHandler logHandler)
public void FetchPodLog(OnlineCodexNode node, IPodLogHandler logHandler)
{
var logNumberSource = new NumberSource(0);
foreach (var online in onlines)
{
foreach (var node in online)
{
WritePodLogs(online, node, logHandler, logNumberSource);
}
}
}
private void WritePodLogs(CodexNodeGroup online, IOnlineCodexNode node, IPodLogsHandler logHandler, NumberSource logNumberSource)
{
var n = (OnlineCodexNode)node;
var nodeDescription = $"{online.Describe()} contains {n.GetName()}";
var stream = client.ReadNamespacedPodLog(online.PodInfo!.Name, K8sNamespace, n.Container.Name);
logHandler.Log(logNumberSource.GetNextNumber(), nodeDescription, stream);
var stream = client.ReadNamespacedPodLog(node.Group.PodInfo!.Name, K8sNamespace, node.Container.Name);
logHandler.Log(stream);
}
private void FetchPodInfo(CodexNodeGroup online)

11
CodexDistTestCore/OnlineCodexNode.cs
View File

@ -9,6 +9,7 @@ namespace CodexDistTestCore
ContentId UploadFile(TestFile file);
TestFile? DownloadContent(ContentId contentId);
void ConnectToPeer(IOnlineCodexNode node);
CodexNodeLog DownloadLog();
}
public class OnlineCodexNode : IOnlineCodexNode
@ -79,6 +80,16 @@ namespace CodexDistTestCore
Log($"Successfully connected to peer {peer.GetName()}.");
}
public CodexNodeLog DownloadLog()
{
return Group.DownloadLog(this);
}
public string Describe()
{
return $"{Group.Describe()} contains {GetName()}";
}
private string GetPeerMultiAddress(OnlineCodexNode peer, CodexDebugResponse peerInfo)
{
var multiAddress = peerInfo.addrs.First();

60
CodexDistTestCore/PodLogDownloader.cs Normal file
View File

@ -0,0 +1,60 @@
namespace CodexDistTestCore
{
public interface IPodLogHandler
{
void Log(Stream log);
}
public class PodLogDownloader
{
private readonly TestLog log;
private readonly IK8sManager k8SManager;
public PodLogDownloader(TestLog log, IK8sManager k8sManager)
{
this.log = log;
k8SManager = k8sManager;
}
public CodexNodeLog DownloadLog(OnlineCodexNode node)
{
var description = node.Describe();
var subFile = log.CreateSubfile();
log.Log($"Downloading logs for {description} to file {subFile.FilenameWithoutPath}");
var handler = new PodLogDownloadHandler(description, subFile);
k8SManager.FetchPodLog(node, handler);
return handler.CreateCodexNodeLog();
}
}
public class PodLogDownloadHandler : IPodLogHandler
{
private readonly string description;
private readonly LogFile log;
public PodLogDownloadHandler(string description, LogFile log)
{
this.description = description;
this.log = log;
}
public CodexNodeLog CreateCodexNodeLog()
{
return new CodexNodeLog(log);
}
public void Log(Stream stream)
{
log.Write($"{description} -->> {log.FilenameWithoutPath}");
log.WriteRaw(description);
var reader = new StreamReader(stream);
var line = reader.ReadLine();
while (line != null)
{
log.WriteRaw(line);
line = reader.ReadLine();
}
}
}
}

7
CodexDistTestCore/PodLogsHandler.cs
View File

@ -1,7 +0,0 @@
namespace CodexDistTestCore
{
public interface IPodLogsHandler
{
void Log(int id, string podDescription, Stream log);
}
}

51
CodexDistTestCore/TestLog.cs
View File

@ -29,23 +29,15 @@ namespace CodexDistTestCore
Log($"[ERROR] {message}");
}
public void EndTest(K8sManager k8sManager)
public void EndTest()
{
var result = TestContext.CurrentContext.Result;
Log($"Finished: {GetTestName()} = {result.Outcome.Status}");
if (!string.IsNullOrEmpty(result.Message))
{
Log(result.Message);
}
if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed)
{
Log($"{result.StackTrace}");
var logWriter = new PodLogWriter(CreateSubfile());
logWriter.IncludeFullPodLogging(k8sManager);
}
}
@ -69,44 +61,9 @@ namespace CodexDistTestCore
}
}
public class PodLogWriter : IPodLogsHandler
{
private readonly LogFile file;
public PodLogWriter(LogFile file)
{
this.file = file;
}
public void IncludeFullPodLogging(K8sManager k8sManager)
{
file.Write("Full pod logging:");
k8sManager.FetchAllPodsLogs(this);
}
public void Log(int id, string podDescription, Stream log)
{
file.Write($"{podDescription} -->> {file.FilenameWithoutPath}");
LogRaw(podDescription);
var reader = new StreamReader(log);
var line = reader.ReadLine();
while (line != null)
{
LogRaw(line);
line = reader.ReadLine();
}
}
private void LogRaw(string message)
{
file!.WriteRaw(message);
}
}
public class LogFile
{
private readonly string filepath;
private readonly string filename;
public LogFile(DateTime now, string name)
{
@ -118,10 +75,10 @@ namespace CodexDistTestCore
Directory.CreateDirectory(filepath);
FilenameWithoutPath = $"{Pad(now.Hour)}-{Pad(now.Minute)}-{Pad(now.Second)}Z_{name.Replace('.', '-')}.log";
filename = Path.Combine(filepath, FilenameWithoutPath);
FullFilename = Path.Combine(filepath, FilenameWithoutPath);
}
public string FullFilename { get; }
public string FilenameWithoutPath { get; }
public void Write(string message)
@ -133,7 +90,7 @@ namespace CodexDistTestCore
{
try
{
File.AppendAllLines(filename, new[] { message });
File.AppendAllLines(FullFilename, new[] { message });
}
catch (Exception ex)
{

113
Tests/BasicTests/SimpleTests.cs
View File

@ -1,5 +1,4 @@
using CodexDistTestCore;
using CodexDistTestCore.Config;
using NUnit.Framework;
namespace Tests.BasicTests
@ -8,77 +7,87 @@ namespace Tests.BasicTests
public class SimpleTests : DistTest
{
[Test]
public void GetDebugInfo()
public void CanAccessLogs()
{
var dockerImage = new CodexDockerImage();
var group = SetupCodexNodes(2).BringOnline();
var node = SetupCodexNodes(1)
.BringOnline()[0];
var log = group[0].DownloadLog();
var debugInfo = node.GetDebugInfo();
Assert.That(debugInfo.spr, Is.Not.Empty);
Assert.That(debugInfo.codex.revision, Is.EqualTo(dockerImage.GetExpectedImageRevision()));
log.AssertLogContains("topics=\"discv5\"");
}
[Test]
public void OneClientTest()
{
var primary = SetupCodexNodes(1)
.BringOnline()[0];
//[Test]
//public void GetDebugInfo()
//{
// var dockerImage = new CodexDockerImage();
var testFile = GenerateTestFile(1.MB());
// var node = SetupCodexNodes(1)
// .BringOnline()[0];
var contentId = primary.UploadFile(testFile);
// var debugInfo = node.GetDebugInfo();
var downloadedFile = primary.DownloadContent(contentId);
// Assert.That(debugInfo.spr, Is.Not.Empty);
// Assert.That(debugInfo.codex.revision, Is.EqualTo(dockerImage.GetExpectedImageRevision()));
//}
testFile.AssertIsEqual(downloadedFile);
}
//[Test]
//public void OneClientTest()
//{
// var primary = SetupCodexNodes(1)
// .BringOnline()[0];
[Test]
public void TwoClientsOnePodTest()
{
var group = SetupCodexNodes(2)
.BringOnline();
// var testFile = GenerateTestFile(1.MB());
var primary = group[0];
var secondary = group[1];
// var contentId = primary.UploadFile(testFile);
PerformTwoClientTest(primary, secondary);
}
// var downloadedFile = primary.DownloadContent(contentId);
[Test]
public void TwoClientsTwoPodsTest()
{
var primary = SetupCodexNodes(1)
.BringOnline()[0];
// testFile.AssertIsEqual(downloadedFile);
//}
var secondary = SetupCodexNodes(1)
.BringOnline()[0];
//[Test]
//public void TwoClientsOnePodTest()
//{
// var group = SetupCodexNodes(2)
// .BringOnline();
PerformTwoClientTest(primary, secondary);
}
// var primary = group[0];
// var secondary = group[1];
[Test]
public void TwoClientsTwoLocationsTest()
{
var primary = SetupCodexNodes(1)
.At(Location.BensLaptop)
.BringOnline()[0];
// PerformTwoClientTest(primary, secondary);
//}
var secondary = SetupCodexNodes(1)
.At(Location.BensOldGamingMachine)
.BringOnline()[0];
//[Test]
//public void TwoClientsTwoPodsTest()
//{
// var primary = SetupCodexNodes(1)
// .BringOnline()[0];
var debugInfo = primary.GetDebugInfo();
Assert.That(debugInfo.spr, Is.Not.Empty);
// var secondary = SetupCodexNodes(1)
// .BringOnline()[0];
var debugInfo2 = secondary.GetDebugInfo();
Assert.That(debugInfo2.spr, Is.Not.Empty);
// PerformTwoClientTest(primary, secondary);
//}
PerformTwoClientTest(primary, secondary);
}
//[Test]
//public void TwoClientsTwoLocationsTest()
//{
// var primary = SetupCodexNodes(1)
// .At(Location.BensLaptop)
// .BringOnline()[0];
// var secondary = SetupCodexNodes(1)
// .At(Location.BensOldGamingMachine)
// .BringOnline()[0];
// var debugInfo = primary.GetDebugInfo();
// Assert.That(debugInfo.spr, Is.Not.Empty);
// var debugInfo2 = secondary.GetDebugInfo();
// Assert.That(debugInfo2.spr, Is.Not.Empty);
// PerformTwoClientTest(primary, secondary);
//}
private void PerformTwoClientTest(IOnlineCodexNode primary, IOnlineCodexNode secondary)
{