codex-storage
/
cs-codex-dist-tests
mirror of https://github.com/codex-storage/cs-codex-dist-tests.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Successful access to codex log from within test

This commit is contained in:
benbierens 2023-03-26 10:45:01 +02:00
parent b088763c3d
commit ce963a228c
No known key found for this signature in database
GPG Key ID: FE44815D96D0A1AA
10 changed files with 211 additions and 132 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
CodexDistTestCore/CodexNodeGroup.cs
View File

@ -12,10 +12,12 @@ namespace CodexDistTestCore
public class CodexNodeGroup : ICodexNodeGroup public class CodexNodeGroup : ICodexNodeGroup
{ {
private readonly TestLog log;
private readonly IK8sManager k8SManager; private readonly IK8sManager k8SManager;
public CodexNodeGroup(int orderNumber, OfflineCodexNodes origin, IK8sManager k8SManager, OnlineCodexNode[] nodes) public CodexNodeGroup(TestLog log, int orderNumber, OfflineCodexNodes origin, IK8sManager k8SManager, OnlineCodexNode[] nodes)
{ {
this.log = log;
OrderNumber = orderNumber; OrderNumber = orderNumber;
Origin = origin; Origin = origin;
this.k8SManager = k8SManager; this.k8SManager = k8SManager;
@ -77,6 +79,13 @@ namespace CodexDistTestCore
}; };
} }
public CodexNodeLog DownloadLog(IOnlineCodexNode node)
{
var logDownloader = new PodLogDownloader(log, k8SManager);
var n = (OnlineCodexNode)node;
return logDownloader.DownloadLog(n);
}
public Dictionary<string, string> GetSelector() public Dictionary<string, string> GetSelector()
{ {
return new Dictionary<string, string> { { "codex-test-node", "dist-test-" + OrderNumber } }; return new Dictionary<string, string> { { "codex-test-node", "dist-test-" + OrderNumber } };

29
CodexDistTestCore/CodexNodeLog.cs Normal file
View File

@ -0,0 +1,29 @@
using NUnit.Framework;
namespace CodexDistTestCore
{
public class CodexNodeLog
{
private readonly LogFile logFile;
public CodexNodeLog(LogFile logFile)
{
this.logFile = logFile;
}
public void AssertLogContains(string expectedString)
{
using var file = File.OpenRead(logFile.FullFilename);
using var streamReader = new StreamReader(file);
var line = streamReader.ReadLine();
while (line != null)
{
if (line.Contains(expectedString)) return;
line = streamReader.ReadLine();
}
Assert.Fail($"Unable to find string '{expectedString}' in CodexNode log file {logFile.FilenameWithoutPath}");
}
}
}

22
CodexDistTestCore/DistTest.cs
View File

@ -56,7 +56,8 @@ namespace CodexDistTestCore
{ {
try try
{ {
log.EndTest(k8sManager); log.EndTest();
IncludeLogsOnTestFailure();
k8sManager.DeleteAllResources(); k8sManager.DeleteAllResources();
fileManager.DeleteAllTestFiles(); fileManager.DeleteAllTestFiles();
} }
@ -76,6 +77,25 @@ namespace CodexDistTestCore
{ {
return new OfflineCodexNodes(k8sManager, numberOfNodes); return new OfflineCodexNodes(k8sManager, numberOfNodes);
} }
private void IncludeLogsOnTestFailure()
{
var result = TestContext.CurrentContext.Result;
if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed)
{
k8sManager.ForEachOnlineGroup(DownloadLogs);
}
}
private void DownloadLogs(CodexNodeGroup group)
{
foreach (var node in group)
{
var downloader = new PodLogDownloader(log, k8sManager);
var n = (OnlineCodexNode)node;
downloader.DownloadLog(n);
}
}
} }
public static class GlobalTestFailure public static class GlobalTestFailure

18
CodexDistTestCore/K8sManager.cs
View File

@ -4,12 +4,13 @@
{ {
ICodexNodeGroup BringOnline(OfflineCodexNodes node); ICodexNodeGroup BringOnline(OfflineCodexNodes node);
IOfflineCodexNodes BringOffline(ICodexNodeGroup node); IOfflineCodexNodes BringOffline(ICodexNodeGroup node);
void FetchPodLog(OnlineCodexNode node, IPodLogHandler logHandler);
} }
public class K8sManager : IK8sManager public class K8sManager : IK8sManager
{ {
private readonly CodexGroupNumberSource codexGroupNumberSource = new CodexGroupNumberSource(); private readonly CodexGroupNumberSource codexGroupNumberSource = new CodexGroupNumberSource();
private readonly List<CodexNodeGroup> onlineCodexNodes = new List<CodexNodeGroup>(); private readonly List<CodexNodeGroup> onlineCodexNodeGroups = new List<CodexNodeGroup>();
private readonly KnownK8sPods knownPods = new KnownK8sPods(); private readonly KnownK8sPods knownPods = new KnownK8sPods();
private readonly TestLog log; private readonly TestLog log;
private readonly IFileManager fileManager; private readonly IFileManager fileManager;
@ -47,17 +48,22 @@
K8s(k => k.DeleteAllResources()); K8s(k => k.DeleteAllResources());
} }
public void FetchAllPodsLogs(IPodLogsHandler logHandler) public void ForEachOnlineGroup(Action<CodexNodeGroup> action)
{ {
K8s(k => k.FetchAllPodsLogs(onlineCodexNodes.ToArray(), logHandler)); foreach (var group in onlineCodexNodeGroups) action(group);
}
public void FetchPodLog(OnlineCodexNode node, IPodLogHandler logHandler)
{
K8s(k => k.FetchPodLog(node, logHandler));
} }
private CodexNodeGroup CreateOnlineCodexNodes(OfflineCodexNodes offline) private CodexNodeGroup CreateOnlineCodexNodes(OfflineCodexNodes offline)
{ {
var containers = CreateContainers(offline.NumberOfNodes); var containers = CreateContainers(offline.NumberOfNodes);
var online = containers.Select(c => new OnlineCodexNode(log, fileManager, c)).ToArray(); var online = containers.Select(c => new OnlineCodexNode(log, fileManager, c)).ToArray();
var result = new CodexNodeGroup(codexGroupNumberSource.GetNextCodexNodeGroupNumber(), offline, this, online); var result = new CodexNodeGroup(log, codexGroupNumberSource.GetNextCodexNodeGroupNumber(), offline, this, online);
onlineCodexNodes.Add(result); onlineCodexNodeGroups.Add(result);
return result; return result;
} }
@ -72,7 +78,7 @@
private CodexNodeGroup GetAndRemoveActiveNodeFor(ICodexNodeGroup node) private CodexNodeGroup GetAndRemoveActiveNodeFor(ICodexNodeGroup node)
{ {
var n = (CodexNodeGroup)node; var n = (CodexNodeGroup)node;
onlineCodexNodes.Remove(n); onlineCodexNodeGroups.Remove(n);
return n; return n;
} }

21
CodexDistTestCore/K8sOperations.cs
View File

@ -51,25 +51,10 @@ namespace CodexDistTestCore
WaitUntilNamespaceDeleted(); WaitUntilNamespaceDeleted();
} }
public void FetchAllPodsLogs(CodexNodeGroup[] onlines, IPodLogsHandler logHandler) public void FetchPodLog(OnlineCodexNode node, IPodLogHandler logHandler)
{ {
var logNumberSource = new NumberSource(0); var stream = client.ReadNamespacedPodLog(node.Group.PodInfo!.Name, K8sNamespace, node.Container.Name);
foreach (var online in onlines) logHandler.Log(stream);
{
foreach (var node in online)
{
WritePodLogs(online, node, logHandler, logNumberSource);
}
}
}
private void WritePodLogs(CodexNodeGroup online, IOnlineCodexNode node, IPodLogsHandler logHandler, NumberSource logNumberSource)
{
var n = (OnlineCodexNode)node;
var nodeDescription = $"{online.Describe()} contains {n.GetName()}";
var stream = client.ReadNamespacedPodLog(online.PodInfo!.Name, K8sNamespace, n.Container.Name);
logHandler.Log(logNumberSource.GetNextNumber(), nodeDescription, stream);
} }
private void FetchPodInfo(CodexNodeGroup online) private void FetchPodInfo(CodexNodeGroup online)

11
CodexDistTestCore/OnlineCodexNode.cs
View File

@ -9,6 +9,7 @@ namespace CodexDistTestCore
ContentId UploadFile(TestFile file); ContentId UploadFile(TestFile file);
TestFile? DownloadContent(ContentId contentId); TestFile? DownloadContent(ContentId contentId);
void ConnectToPeer(IOnlineCodexNode node); void ConnectToPeer(IOnlineCodexNode node);
CodexNodeLog DownloadLog();
} }
public class OnlineCodexNode : IOnlineCodexNode public class OnlineCodexNode : IOnlineCodexNode
@ -79,6 +80,16 @@ namespace CodexDistTestCore
Log($"Successfully connected to peer {peer.GetName()}."); Log($"Successfully connected to peer {peer.GetName()}.");
} }
public CodexNodeLog DownloadLog()
{
return Group.DownloadLog(this);
}
public string Describe()
{
return $"{Group.Describe()} contains {GetName()}";
}
private string GetPeerMultiAddress(OnlineCodexNode peer, CodexDebugResponse peerInfo) private string GetPeerMultiAddress(OnlineCodexNode peer, CodexDebugResponse peerInfo)
{ {
var multiAddress = peerInfo.addrs.First(); var multiAddress = peerInfo.addrs.First();

60
CodexDistTestCore/PodLogDownloader.cs Normal file
View File

@ -0,0 +1,60 @@
namespace CodexDistTestCore
{
public interface IPodLogHandler
{
void Log(Stream log);
}
public class PodLogDownloader
{
private readonly TestLog log;
private readonly IK8sManager k8SManager;
public PodLogDownloader(TestLog log, IK8sManager k8sManager)
{
this.log = log;
k8SManager = k8sManager;
}
public CodexNodeLog DownloadLog(OnlineCodexNode node)
{
var description = node.Describe();
var subFile = log.CreateSubfile();
log.Log($"Downloading logs for {description} to file {subFile.FilenameWithoutPath}");
var handler = new PodLogDownloadHandler(description, subFile);
k8SManager.FetchPodLog(node, handler);
return handler.CreateCodexNodeLog();
}
}
public class PodLogDownloadHandler : IPodLogHandler
{
private readonly string description;
private readonly LogFile log;
public PodLogDownloadHandler(string description, LogFile log)
{
this.description = description;
this.log = log;
}
public CodexNodeLog CreateCodexNodeLog()
{
return new CodexNodeLog(log);
}
public void Log(Stream stream)
{
log.Write($"{description} -->> {log.FilenameWithoutPath}");
log.WriteRaw(description);
var reader = new StreamReader(stream);
var line = reader.ReadLine();
while (line != null)
{
log.WriteRaw(line);
line = reader.ReadLine();
}
}
}
}

7
CodexDistTestCore/PodLogsHandler.cs
View File

@ -1,7 +0,0 @@
namespace CodexDistTestCore
{
public interface IPodLogsHandler
{
void Log(int id, string podDescription, Stream log);
}
}

51
CodexDistTestCore/TestLog.cs
View File

@ -29,23 +29,15 @@ namespace CodexDistTestCore
Log($"[ERROR] {message}"); Log($"[ERROR] {message}");
} }
public void EndTest(K8sManager k8sManager) public void EndTest()
{ {
var result = TestContext.CurrentContext.Result; var result = TestContext.CurrentContext.Result;
Log($"Finished: {GetTestName()} = {result.Outcome.Status}"); Log($"Finished: {GetTestName()} = {result.Outcome.Status}");
if (!string.IsNullOrEmpty(result.Message)) if (!string.IsNullOrEmpty(result.Message))
{ {
Log(result.Message); Log(result.Message);
}
if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed)
{
Log($"{result.StackTrace}"); Log($"{result.StackTrace}");
var logWriter = new PodLogWriter(CreateSubfile());
logWriter.IncludeFullPodLogging(k8sManager);
} }
} }
@ -69,44 +61,9 @@ namespace CodexDistTestCore
} }
} }
public class PodLogWriter : IPodLogsHandler
{
private readonly LogFile file;
public PodLogWriter(LogFile file)
{
this.file = file;
}
public void IncludeFullPodLogging(K8sManager k8sManager)
{
file.Write("Full pod logging:");
k8sManager.FetchAllPodsLogs(this);
}
public void Log(int id, string podDescription, Stream log)
{
file.Write($"{podDescription} -->> {file.FilenameWithoutPath}");
LogRaw(podDescription);
var reader = new StreamReader(log);
var line = reader.ReadLine();
while (line != null)
{
LogRaw(line);
line = reader.ReadLine();
}
}
private void LogRaw(string message)
{
file!.WriteRaw(message);
}
}
public class LogFile public class LogFile
{ {
private readonly string filepath; private readonly string filepath;
private readonly string filename;
public LogFile(DateTime now, string name) public LogFile(DateTime now, string name)
{ {
@ -118,10 +75,10 @@ namespace CodexDistTestCore
Directory.CreateDirectory(filepath); Directory.CreateDirectory(filepath);
FilenameWithoutPath = $"{Pad(now.Hour)}-{Pad(now.Minute)}-{Pad(now.Second)}Z_{name.Replace('.', '-')}.log"; FilenameWithoutPath = $"{Pad(now.Hour)}-{Pad(now.Minute)}-{Pad(now.Second)}Z_{name.Replace('.', '-')}.log";
FullFilename = Path.Combine(filepath, FilenameWithoutPath);
filename = Path.Combine(filepath, FilenameWithoutPath);
} }
public string FullFilename { get; }
public string FilenameWithoutPath { get; } public string FilenameWithoutPath { get; }
public void Write(string message) public void Write(string message)
@ -133,7 +90,7 @@ namespace CodexDistTestCore
{ {
try try
{ {
File.AppendAllLines(filename, new[] { message }); File.AppendAllLines(FullFilename, new[] { message });
} }
catch (Exception ex) catch (Exception ex)
{ {

113
Tests/BasicTests/SimpleTests.cs
View File

@ -1,5 +1,4 @@
using CodexDistTestCore; using CodexDistTestCore;
using CodexDistTestCore.Config;
using NUnit.Framework; using NUnit.Framework;
namespace Tests.BasicTests namespace Tests.BasicTests
@ -8,77 +7,87 @@ namespace Tests.BasicTests
public class SimpleTests : DistTest public class SimpleTests : DistTest
{ {
[Test] [Test]
public void GetDebugInfo() public void CanAccessLogs()
{ {
var dockerImage = new CodexDockerImage(); var group = SetupCodexNodes(2).BringOnline();
var node = SetupCodexNodes(1) var log = group[0].DownloadLog();
.BringOnline()[0];
var debugInfo = node.GetDebugInfo(); log.AssertLogContains("topics=\"discv5\"");
Assert.That(debugInfo.spr, Is.Not.Empty);
Assert.That(debugInfo.codex.revision, Is.EqualTo(dockerImage.GetExpectedImageRevision()));
} }
[Test] //[Test]
public void OneClientTest() //public void GetDebugInfo()
{ //{
var primary = SetupCodexNodes(1) // var dockerImage = new CodexDockerImage();
.BringOnline()[0];
var testFile = GenerateTestFile(1.MB()); // var node = SetupCodexNodes(1)
// .BringOnline()[0];
var contentId = primary.UploadFile(testFile); // var debugInfo = node.GetDebugInfo();
var downloadedFile = primary.DownloadContent(contentId); // Assert.That(debugInfo.spr, Is.Not.Empty);
// Assert.That(debugInfo.codex.revision, Is.EqualTo(dockerImage.GetExpectedImageRevision()));
//}
testFile.AssertIsEqual(downloadedFile); //[Test]
} //public void OneClientTest()
//{
// var primary = SetupCodexNodes(1)
// .BringOnline()[0];
[Test] // var testFile = GenerateTestFile(1.MB());
public void TwoClientsOnePodTest()
{
var group = SetupCodexNodes(2)
.BringOnline();
var primary = group[0]; // var contentId = primary.UploadFile(testFile);
var secondary = group[1];
PerformTwoClientTest(primary, secondary); // var downloadedFile = primary.DownloadContent(contentId);
}
[Test] // testFile.AssertIsEqual(downloadedFile);
public void TwoClientsTwoPodsTest() //}
{
var primary = SetupCodexNodes(1)
.BringOnline()[0];
var secondary = SetupCodexNodes(1) //[Test]
.BringOnline()[0]; //public void TwoClientsOnePodTest()
//{
// var group = SetupCodexNodes(2)
// .BringOnline();
PerformTwoClientTest(primary, secondary); // var primary = group[0];
} // var secondary = group[1];
[Test] // PerformTwoClientTest(primary, secondary);
public void TwoClientsTwoLocationsTest() //}
{
var primary = SetupCodexNodes(1)
.At(Location.BensLaptop)
.BringOnline()[0];
var secondary = SetupCodexNodes(1) //[Test]
.At(Location.BensOldGamingMachine) //public void TwoClientsTwoPodsTest()
.BringOnline()[0]; //{
// var primary = SetupCodexNodes(1)
// .BringOnline()[0];
var debugInfo = primary.GetDebugInfo(); // var secondary = SetupCodexNodes(1)
Assert.That(debugInfo.spr, Is.Not.Empty); // .BringOnline()[0];
var debugInfo2 = secondary.GetDebugInfo(); // PerformTwoClientTest(primary, secondary);
Assert.That(debugInfo2.spr, Is.Not.Empty); //}
PerformTwoClientTest(primary, secondary); //[Test]
} //public void TwoClientsTwoLocationsTest()
//{
// var primary = SetupCodexNodes(1)
// .At(Location.BensLaptop)
// .BringOnline()[0];
// var secondary = SetupCodexNodes(1)
// .At(Location.BensOldGamingMachine)
// .BringOnline()[0];
// var debugInfo = primary.GetDebugInfo();
// Assert.That(debugInfo.spr, Is.Not.Empty);
// var debugInfo2 = secondary.GetDebugInfo();
// Assert.That(debugInfo2.spr, Is.Not.Empty);
// PerformTwoClientTest(primary, secondary);
//}
private void PerformTwoClientTest(IOnlineCodexNode primary, IOnlineCodexNode secondary) private void PerformTwoClientTest(IOnlineCodexNode primary, IOnlineCodexNode secondary)
{ {