codex-storage
/
cs-codex-dist-tests
mirror of https://github.com/codex-storage/cs-codex-dist-tests.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Better logging

This commit is contained in:
benbierens 2023-03-22 09:22:18 +01:00
parent a36e364996
commit a60124239c
No known key found for this signature in database
GPG Key ID: FE44815D96D0A1AA
8 changed files with 104 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
CodexDistTestCore/OnlineCodexNodes.cs → CodexDistTestCore/CodexNodeGroup.cs
View File

@ -2,17 +2,17 @@
namespace CodexDistTestCore
{
public interface IOnlineCodexNodes
public interface ICodexNodeGroup
{
IOfflineCodexNodes BringOffline();
IOnlineCodexNode this[int index] { get; }
}
public class OnlineCodexNodes : IOnlineCodexNodes
public class CodexNodeGroup : ICodexNodeGroup
{
private readonly IK8sManager k8SManager;
public OnlineCodexNodes(int orderNumber, OfflineCodexNodes origin, IK8sManager k8SManager, OnlineCodexNode[] nodes)
public CodexNodeGroup(int orderNumber, OfflineCodexNodes origin, IK8sManager k8SManager, OnlineCodexNode[] nodes)
{
OrderNumber = orderNumber;
Origin = origin;
@ -70,7 +70,7 @@ namespace CodexDistTestCore
public string Describe()
{
return $"CodexNode{OrderNumber}-{Origin.Describe()}";
return $"CodexNodeGroup#{OrderNumber}-{Origin.Describe()}";
}
}
}

30
CodexDistTestCore/DistTest.cs
View File

@ -5,6 +5,7 @@ namespace CodexDistTestCore
[SetUpFixture]
public abstract class DistTest
{
private TestLog log = null!;
private FileManager fileManager = null!;
private K8sManager k8sManager = null!;
@ -13,11 +14,22 @@ namespace CodexDistTestCore
{
// Previous test run may have been interrupted.
// Begin by cleaning everything up.
fileManager = new FileManager();
k8sManager = new K8sManager(fileManager);
log = new TestLog();
fileManager = new FileManager(log);
k8sManager = new K8sManager(log, fileManager);
k8sManager.DeleteAllResources();
fileManager.DeleteAllTestFiles();
try
{
k8sManager.DeleteAllResources();
fileManager.DeleteAllTestFiles();
}
catch (Exception ex)
{
GlobalTestFailure.HasFailed = true;
log.Error($"Global setup cleanup failed with: {ex}");
throw;
}
log.Log("Global setup cleanup successful");
}
[SetUp]
@ -29,9 +41,9 @@ namespace CodexDistTestCore
}
else
{
TestLog.BeginTest();
fileManager = new FileManager();
k8sManager = new K8sManager(fileManager);
log = new TestLog();
fileManager = new FileManager(log);
k8sManager = new K8sManager(log, fileManager);
}
}
@ -40,13 +52,13 @@ namespace CodexDistTestCore
{
try
{
TestLog.EndTest(k8sManager);
log.EndTest(k8sManager);
k8sManager.DeleteAllResources();
fileManager.DeleteAllTestFiles();
}
catch (Exception ex)
{
TestLog.Error("Cleanup failed: " + ex.Message);
log.Error("Cleanup failed: " + ex.Message);
GlobalTestFailure.HasFailed = true;
}
}

16
CodexDistTestCore/FileManager.cs
View File

@ -15,10 +15,12 @@ namespace CodexDistTestCore
private const string Folder = "TestDataFiles";
private readonly Random random = new Random();
private readonly List<TestFile> activeFiles = new List<TestFile>();
private readonly TestLog log;
public FileManager()
public FileManager(TestLog log)
{
if (!Directory.Exists(Folder)) Directory.CreateDirectory(Folder);
this.log = log;
}
public TestFile CreateEmptyTestFile()
@ -26,7 +28,7 @@ namespace CodexDistTestCore
var result = new TestFile(Path.Combine(Folder, Guid.NewGuid().ToString() + "_test.bin"));
File.Create(result.Filename).Close();
activeFiles.Add(result);
TestLog.Log($"Created test file '{result.Filename}'.");
log.Log($"Created test file '{result.Filename}'.");
return result;
}
@ -34,7 +36,7 @@ namespace CodexDistTestCore
{
var result = CreateEmptyTestFile();
GenerateFileBytes(result, size);
TestLog.Log($"Generated {size.SizeInBytes} bytes of content for file '{result.Filename}'.");
log.Log($"Generated {size.SizeInBytes} bytes of content for file '{result.Filename}'.");
return result;
}
@ -73,11 +75,19 @@ namespace CodexDistTestCore
public string Filename { get; }
public long GetFileSize()
{
var info = new FileInfo(Filename);
return info.Length;
}
public void AssertIsEqual(TestFile? other)
{
if (other == null) Assert.Fail("TestFile is null.");
if (other == this || other!.Filename == Filename) Assert.Fail("TestFile is compared to itself.");
if (GetFileSize() != other.GetFileSize()) Assert.Fail("file sizes unequal?");
using var stream1 = new FileStream(Filename, FileMode.Open, FileAccess.Read);
using var stream2 = new FileStream(other.Filename, FileMode.Open, FileAccess.Read);

28
CodexDistTestCore/K8sManager.cs
View File

@ -2,40 +2,42 @@
{
public interface IK8sManager
{
IOnlineCodexNodes BringOnline(OfflineCodexNodes node);
IOfflineCodexNodes BringOffline(IOnlineCodexNodes node);
ICodexNodeGroup BringOnline(OfflineCodexNodes node);
IOfflineCodexNodes BringOffline(ICodexNodeGroup node);
}
public class K8sManager : IK8sManager
{
private readonly NumberSource onlineCodexNodeOrderNumberSource = new NumberSource(0);
private readonly List<OnlineCodexNodes> onlineCodexNodes = new List<OnlineCodexNodes>();
private readonly List<CodexNodeGroup> onlineCodexNodes = new List<CodexNodeGroup>();
private readonly KnownK8sPods knownPods = new KnownK8sPods();
private readonly TestLog log;
private readonly IFileManager fileManager;
public K8sManager(IFileManager fileManager)
public K8sManager(TestLog log, IFileManager fileManager)
{
this.log = log;
this.fileManager = fileManager;
}
public IOnlineCodexNodes BringOnline(OfflineCodexNodes offline)
public ICodexNodeGroup BringOnline(OfflineCodexNodes offline)
{
var online = CreateOnlineCodexNodes(offline);
K8s().BringOnline(online, offline);
TestLog.Log($"{offline.NumberOfNodes} Codex nodes online.");
log.Log($"{online.Describe()} online.");
return online;
}
public IOfflineCodexNodes BringOffline(IOnlineCodexNodes node)
public IOfflineCodexNodes BringOffline(ICodexNodeGroup node)
{
var online = GetAndRemoveActiveNodeFor(node);
K8s().BringOffline(online);
TestLog.Log($"{online.Describe()} offline.");
log.Log($"{online.Describe()} offline.");
return online.Origin;
}
@ -50,11 +52,11 @@
K8s().FetchAllPodsLogs(onlineCodexNodes.ToArray(), logHandler);
}
private OnlineCodexNodes CreateOnlineCodexNodes(OfflineCodexNodes offline)
private CodexNodeGroup CreateOnlineCodexNodes(OfflineCodexNodes offline)
{
var containers = CreateContainers(offline.NumberOfNodes);
var online = containers.Select(c => new OnlineCodexNode(fileManager, c)).ToArray();
var result = new OnlineCodexNodes(onlineCodexNodeOrderNumberSource.GetNextNumber(), offline, this, online);
var online = containers.Select(c => new OnlineCodexNode(log, fileManager, c)).ToArray();
var result = new CodexNodeGroup(onlineCodexNodeOrderNumberSource.GetNextNumber(), offline, this, online);
onlineCodexNodes.Add(result);
return result;
}
@ -67,9 +69,9 @@
return containers.ToArray();
}
private OnlineCodexNodes GetAndRemoveActiveNodeFor(IOnlineCodexNodes node)
private CodexNodeGroup GetAndRemoveActiveNodeFor(ICodexNodeGroup node)
{
var n = (OnlineCodexNodes)node;
var n = (CodexNodeGroup)node;
onlineCodexNodes.Remove(n);
return n;
}

22
CodexDistTestCore/K8sOperations.cs
View File

@ -21,7 +21,7 @@ namespace CodexDistTestCore
client = new Kubernetes(config);
}
public void BringOnline(OnlineCodexNodes online, OfflineCodexNodes offline)
public void BringOnline(CodexNodeGroup online, OfflineCodexNodes offline)
{
EnsureTestNamespace();
@ -32,7 +32,7 @@ namespace CodexDistTestCore
AssignActivePodNames(online);
}
public void BringOffline(OnlineCodexNodes online)
public void BringOffline(CodexNodeGroup online)
{
var deploymentName = online.Deployment.Name();
DeleteDeployment(online);
@ -48,7 +48,7 @@ namespace CodexDistTestCore
WaitUntilNamespaceDeleted();
}
public void FetchAllPodsLogs(OnlineCodexNodes[] onlines, IPodLogsHandler logHandler)
public void FetchAllPodsLogs(CodexNodeGroup[] onlines, IPodLogsHandler logHandler)
{
foreach (var online in onlines)
{
@ -61,7 +61,7 @@ namespace CodexDistTestCore
}
}
private void AssignActivePodNames(OnlineCodexNodes online)
private void AssignActivePodNames(CodexNodeGroup online)
{
var pods = client.ListNamespacedPod(K8sNamespace);
var podNames = pods.Items.Select(p => p.Name());
@ -77,7 +77,7 @@ namespace CodexDistTestCore
#region Waiting
private void WaitUntilOnline(OnlineCodexNodes online)
private void WaitUntilOnline(CodexNodeGroup online)
{
WaitUntil(() =>
{
@ -126,7 +126,7 @@ namespace CodexDistTestCore
#region Service management
private void CreateService(OnlineCodexNodes online)
private void CreateService(CodexNodeGroup online)
{
var serviceSpec = new V1Service
{
@ -143,7 +143,7 @@ namespace CodexDistTestCore
online.Service = client.CreateNamespacedService(serviceSpec, K8sNamespace);
}
private List<V1ServicePort> CreateServicePorts(OnlineCodexNodes online)
private List<V1ServicePort> CreateServicePorts(CodexNodeGroup online)
{
var result = new List<V1ServicePort>();
var containers = online.GetContainers();
@ -160,7 +160,7 @@ namespace CodexDistTestCore
return result;
}
private void DeleteService(OnlineCodexNodes online)
private void DeleteService(CodexNodeGroup online)
{
if (online.Service == null) return;
client.DeleteNamespacedService(online.Service.Name(), K8sNamespace);
@ -171,7 +171,7 @@ namespace CodexDistTestCore
#region Deployment management
private void CreateDeployment(OnlineCodexNodes online, OfflineCodexNodes offline)
private void CreateDeployment(CodexNodeGroup online, OfflineCodexNodes offline)
{
var deploymentSpec = new V1Deployment
{
@ -201,7 +201,7 @@ namespace CodexDistTestCore
online.Deployment = client.CreateNamespacedDeployment(deploymentSpec, K8sNamespace);
}
private List<V1Container> CreateDeploymentContainers(OnlineCodexNodes online, OfflineCodexNodes offline)
private List<V1Container> CreateDeploymentContainers(CodexNodeGroup online, OfflineCodexNodes offline)
{
var result = new List<V1Container>();
var containers = online.GetContainers();
@ -225,7 +225,7 @@ namespace CodexDistTestCore
return result;
}
private void DeleteDeployment(OnlineCodexNodes online)
private void DeleteDeployment(CodexNodeGroup online)
{
if (online.Deployment == null) return;
client.DeleteNamespacedDeployment(online.Deployment.Name(), K8sNamespace);

18
CodexDistTestCore/OfflineCodexNodes.cs
View File

@ -5,7 +5,7 @@
IOfflineCodexNodes WithLogLevel(CodexLogLevel level);
IOfflineCodexNodes WithBootstrapNode(IOnlineCodexNode node);
IOfflineCodexNodes WithStorageQuota(ByteSize storageQuota);
IOnlineCodexNodes BringOnline();
ICodexNodeGroup BringOnline();
}
public enum CodexLogLevel
@ -32,7 +32,7 @@
NumberOfNodes = numberOfNodes;
}
public IOnlineCodexNodes BringOnline()
public ICodexNodeGroup BringOnline()
{
return k8SManager.BringOnline(this);
}
@ -57,11 +57,15 @@
public string Describe()
{
var result = "";
if (LogLevel != null) result += $"LogLevel={LogLevel},";
if (BootstrapNode != null) result += "BootstrapNode=set,";
if (StorageQuota != null) result += $"StorageQuote={StorageQuota.SizeInBytes},";
return result;
var args = string.Join(',', DescribeArgs());
return $"{NumberOfNodes} CodexNodes with [{args}]";
}
private IEnumerable<string> DescribeArgs()
{
if (LogLevel != null) yield return ($"LogLevel={LogLevel}");
if (BootstrapNode != null) yield return ("BootstrapNode=set");
if (StorageQuota != null) yield return ($"StorageQuote={StorageQuota.SizeInBytes}");
}
}
}

19
CodexDistTestCore/OnlineCodexNode.cs
View File

@ -11,38 +11,46 @@ namespace CodexDistTestCore
public class OnlineCodexNode : IOnlineCodexNode
{
private readonly TestLog log;
private readonly IFileManager fileManager;
public OnlineCodexNode(IFileManager fileManager, CodexNodeContainer environment)
public OnlineCodexNode(TestLog log, IFileManager fileManager, CodexNodeContainer container)
{
this.log = log;
this.fileManager = fileManager;
Container = environment;
Container = container;
}
public CodexNodeContainer Container { get; }
public CodexDebugResponse GetDebugInfo()
{
return Http().HttpGetJson<CodexDebugResponse>("debug/info");
var response = Http().HttpGetJson<CodexDebugResponse>("debug/info");
Log("Got DebugInfo with id: " + response.id);
return response;
}
public ContentId UploadFile(TestFile file)
{
Log($"Uploading file of size {file.GetFileSize()}");
using var fileStream = File.OpenRead(file.Filename);
var response = Http().HttpPostStream("upload", fileStream);
if (response.StartsWith("Unable to store block"))
{
Assert.Fail("Node failed to store block.");
}
Log($"Uploaded file. Received contentId: {response}");
return new ContentId(response);
}
public TestFile? DownloadContent(ContentId contentId)
{
Log($"Downloading for contentId: {contentId}");
var file = fileManager.CreateEmptyTestFile();
using var fileStream = File.OpenWrite(file.Filename);
using var downloadStream = Http().HttpGetStream("download/" + contentId.Id);
downloadStream.CopyTo(fileStream);
Log($"Downloaded file of size {file.GetFileSize()}");
return file;
}
@ -50,6 +58,11 @@ namespace CodexDistTestCore
{
return new Http(ip: "127.0.0.1", port: Container.ServicePort, baseUrl: "/api/codex/v1");
}
private void Log(string msg)
{
log.Log($"Node {Container.Name}: {msg}");
}
}
public class ContentId

35
CodexDistTestCore/TestLog.cs
View File

@ -5,35 +5,28 @@ namespace CodexDistTestCore
public class TestLog
{
public const string LogRoot = "D:/CodexTestLogs";
private readonly LogFile file;
private static LogFile? file = null;
// This is all way too static. It needs to be cleaned up.
public static void Log(string message)
public TestLog()
{
file!.Write(message);
}
public static void Error(string message)
{
Log($"[ERROR] {message}");
}
public static void BeginTest()
{
if (file != null) throw new InvalidOperationException("Test is already started!");
var name = GetTestName();
file = new LogFile(name);
Log($"Begin: {name}");
}
public static void EndTest(K8sManager k8sManager)
public void Log(string message)
{
if (file == null) throw new InvalidOperationException("No test is started!");
file.Write(message);
}
public void Error(string message)
{
Log($"[ERROR] {message}");
}
public void EndTest(K8sManager k8sManager)
{
var result = TestContext.CurrentContext.Result;
Log($"Finished: {GetTestName()} = {result.Outcome.Status}");
@ -42,8 +35,6 @@ namespace CodexDistTestCore
var logWriter = new PodLogWriter(file);
logWriter.IncludeFullPodLogging(k8sManager);
}
file = null;
}
private static string GetTestName()
@ -65,14 +56,14 @@ namespace CodexDistTestCore
public void IncludeFullPodLogging(K8sManager k8sManager)
{
TestLog.Log("Full pod logging:");
file.Write("Full pod logging:");
k8sManager.FetchAllPodsLogs(this);
}
public void Log(int id, string podDescription, Stream log)
{
var logFile = id.ToString().PadLeft(6, '0');
TestLog.Log($"{podDescription} -->> {logFile}");
file.Write($"{podDescription} -->> {logFile}");
LogRaw(podDescription, logFile);
var reader = new StreamReader(log);
var line = reader.ReadLine();