From d95318f2aa607d457c37fec79e6701f139d1ad40 Mon Sep 17 00:00:00 2001 From: E M <5089238+emizzle@users.noreply.github.com> Date: Wed, 27 May 2026 12:44:38 +1000 Subject: [PATCH] Make release tests use json log sink for structured logs --- .../LogosStorageLogFormat.cs | 10 ++++++++ .../LogosStorageProcessRecipe.cs | 1 + .../StoragePlugin/LogosStorageSetup.cs | 7 ++++++ .../LogosStorageStartupConfig.cs | 1 + .../DataTests/DataExpiryTest.cs | 5 +++- .../DataTests/DhtTest.cs | 7 ++++-- .../DataTests/InterruptUploadTest.cs | 2 +- .../DataTests/LocalFilesTest.cs | 5 ++-- .../DataTests/ManifestOnlyDownloadTest.cs | 9 ++++--- .../DataTests/OneClientTest.cs | 2 +- .../DataTests/StreamlessDownloadTest.cs | 9 ++++--- .../DataTests/SwarmTest.cs | 10 ++++++-- .../DataTests/TheseusTest.cs | 10 ++++++-- .../DataTests/ThreeClientTest.cs | 7 +++--- .../DataTests/TwoClientTest.cs | 25 +++++++++++++++---- .../DataTests/UnknownCidTest.cs | 2 +- .../NodeTests/BasicInfoTests.cs | 15 ++++++----- .../NodeTests/PeerTableTests.cs | 2 +- 18 files changed, 96 insertions(+), 33 deletions(-) create mode 100644 ProjectPlugins/LogosStorageClient/LogosStorageLogFormat.cs diff --git a/ProjectPlugins/LogosStorageClient/LogosStorageLogFormat.cs b/ProjectPlugins/LogosStorageClient/LogosStorageLogFormat.cs new file mode 100644 index 00000000..91e51cb7 --- /dev/null +++ b/ProjectPlugins/LogosStorageClient/LogosStorageLogFormat.cs @@ -0,0 +1,10 @@ +namespace LogosStorageClient +{ + public enum LogosStorageLogFormat + { + Auto, + Colors, + NoColors, + Json + } +} diff --git a/ProjectPlugins/StoragePlugin/LogosStorageProcessRecipe.cs b/ProjectPlugins/StoragePlugin/LogosStorageProcessRecipe.cs index 1dca2dfb..ec0da0b0 100644 --- a/ProjectPlugins/StoragePlugin/LogosStorageProcessRecipe.cs +++ b/ProjectPlugins/StoragePlugin/LogosStorageProcessRecipe.cs @@ -60,6 +60,7 @@ namespace StoragePlugin AddArg("--disc-port", pc.DiscPort); AddArg("--log-level", config.LogLevelWithTopics()); + AddArg("--log-format", config.LogFormat.ToString().ToLowerInvariant()); // This makes the node announce itself to its local IP address. AddArg("--nat", $"extip:{pc.LocalIpAddrs}"); diff --git a/ProjectPlugins/StoragePlugin/LogosStorageSetup.cs b/ProjectPlugins/StoragePlugin/LogosStorageSetup.cs index 43cbb364..5c1d9eac 100644 --- a/ProjectPlugins/StoragePlugin/LogosStorageSetup.cs +++ b/ProjectPlugins/StoragePlugin/LogosStorageSetup.cs @@ -11,6 +11,7 @@ namespace StoragePlugin ILogosStorageSetup WithBootstrapNode(IStorageNode node); ILogosStorageSetup WithLogLevel(LogosStorageLogLevel level); ILogosStorageSetup WithLogLevel(LogosStorageLogLevel level, LogosStorageLogCustomTopics customTopics); + ILogosStorageSetup WithLogFormat(LogosStorageLogFormat format); ILogosStorageSetup WithStorageQuota(ByteSize storageQuota); ILogosStorageSetup WithBlockTTL(TimeSpan duration); ILogosStorageSetup WithBlockMaintenanceInterval(TimeSpan duration); @@ -82,6 +83,12 @@ namespace StoragePlugin return this; } + public ILogosStorageSetup WithLogFormat(LogosStorageLogFormat format) + { + LogFormat = format; + return this; + } + public ILogosStorageSetup WithStorageQuota(ByteSize storageQuota) { StorageQuota = storageQuota; diff --git a/ProjectPlugins/StoragePlugin/LogosStorageStartupConfig.cs b/ProjectPlugins/StoragePlugin/LogosStorageStartupConfig.cs index 0336e802..24dbf018 100644 --- a/ProjectPlugins/StoragePlugin/LogosStorageStartupConfig.cs +++ b/ProjectPlugins/StoragePlugin/LogosStorageStartupConfig.cs @@ -9,6 +9,7 @@ namespace StoragePlugin public string? NameOverride { get; set; } public ILocation Location { get; set; } = KnownLocations.UnspecifiedLocation; public LogosStorageLogLevel LogLevel { get; set; } + public LogosStorageLogFormat LogFormat { get; set; } public LogosStorageLogCustomTopics? CustomTopics { get; set; } = new LogosStorageLogCustomTopics(LogosStorageLogLevel.Info, LogosStorageLogLevel.Warn); public ByteSize? StorageQuota { get; set; } public bool MetricsEnabled { get; set; } diff --git a/Tests/LogosStorageReleaseTests/DataTests/DataExpiryTest.cs b/Tests/LogosStorageReleaseTests/DataTests/DataExpiryTest.cs index 0c4f8424..dfc303a5 100644 --- a/Tests/LogosStorageReleaseTests/DataTests/DataExpiryTest.cs +++ b/Tests/LogosStorageReleaseTests/DataTests/DataExpiryTest.cs @@ -2,6 +2,7 @@ using StoragePlugin; using LogosStorageTests; using NUnit.Framework; using Utils; +using LogosStorageClient; namespace LogosStorageReleaseTests.DataTests { @@ -24,7 +25,9 @@ namespace LogosStorageReleaseTests.DataTests public void DeletesExpiredData() { var fileSize = 3.MB(); - var node = StartLogosStorage(s => WithFastBlockExpiry(s)); + var node = StartLogosStorage(s => + WithFastBlockExpiry(s).WithLogFormat(LogosStorageLogFormat.Json) + ); var startSpace = node.Space(); Assert.That(startSpace.QuotaUsedBytes, Is.EqualTo(0)); diff --git a/Tests/LogosStorageReleaseTests/DataTests/DhtTest.cs b/Tests/LogosStorageReleaseTests/DataTests/DhtTest.cs index 5246204a..4d96920c 100644 --- a/Tests/LogosStorageReleaseTests/DataTests/DhtTest.cs +++ b/Tests/LogosStorageReleaseTests/DataTests/DhtTest.cs @@ -31,7 +31,10 @@ namespace LogosStorageReleaseTests.DataTests.DHT public void PressureTest() { var cids = new List(); - var nodes = StartLogosStorage(numNodes); + var nodes = StartLogosStorage( + numNodes, + s => s.WithLogFormat(LogosStorageLogFormat.Json) + ); for (var i = 0; i < numFilesPerNode; i++) { @@ -45,7 +48,7 @@ namespace LogosStorageReleaseTests.DataTests.DHT var estimate = numNodes * numFilesPerNode * 2; Log($"Estimate of DHT records: {estimate}"); - var node = StartLogosStorage(); + var node = StartLogosStorage(s => s.WithLogFormat(LogosStorageLogFormat.Json)); for (var i = 0; i < numToFetch; i++) { var timing = Stopwatch.Measure(GetTestLog(), nameof(PressureTest) + i, () => diff --git a/Tests/LogosStorageReleaseTests/DataTests/InterruptUploadTest.cs b/Tests/LogosStorageReleaseTests/DataTests/InterruptUploadTest.cs index fea8b06c..c5083324 100644 --- a/Tests/LogosStorageReleaseTests/DataTests/InterruptUploadTest.cs +++ b/Tests/LogosStorageReleaseTests/DataTests/InterruptUploadTest.cs @@ -12,7 +12,7 @@ namespace LogosStorageReleaseTests.DataTests [Test] public void UploadInterruptTest() { - var nodes = StartLogosStorage(10); + var nodes = StartLogosStorage(10, s => s.WithLogFormat(LogosStorageLogFormat.Json)); var tasks = nodes.Select(n => Task.Run(() => RunInterruptUploadTest(n))); Task.WaitAll(tasks.ToArray()); diff --git a/Tests/LogosStorageReleaseTests/DataTests/LocalFilesTest.cs b/Tests/LogosStorageReleaseTests/DataTests/LocalFilesTest.cs index 05e47d45..ee5bdfd6 100644 --- a/Tests/LogosStorageReleaseTests/DataTests/LocalFilesTest.cs +++ b/Tests/LogosStorageReleaseTests/DataTests/LocalFilesTest.cs @@ -1,4 +1,5 @@ -using LogosStorageTests; +using LogosStorageClient; +using LogosStorageTests; using NUnit.Framework; using System; using System.Collections.Generic; @@ -15,7 +16,7 @@ namespace LogosStorageReleaseTests.DataTests [Test] public void ShouldShowLocalFiles() { - var node = StartLogosStorage(); + var node = StartLogosStorage(s => s.WithLogFormat(LogosStorageLogFormat.Json)); var size1 = 123.KB(); var size2 = 23.MB(); diff --git a/Tests/LogosStorageReleaseTests/DataTests/ManifestOnlyDownloadTest.cs b/Tests/LogosStorageReleaseTests/DataTests/ManifestOnlyDownloadTest.cs index af4a21ea..a06f4a6b 100644 --- a/Tests/LogosStorageReleaseTests/DataTests/ManifestOnlyDownloadTest.cs +++ b/Tests/LogosStorageReleaseTests/DataTests/ManifestOnlyDownloadTest.cs @@ -1,4 +1,5 @@ -using LogosStorageTests; +using LogosStorageClient; +using LogosStorageTests; using NUnit.Framework; using Utils; @@ -10,8 +11,10 @@ namespace LogosStorageReleaseTests.DataTests [Test] public void ManifestOnlyTest() { - var uploader = StartLogosStorage(); - var downloader = StartLogosStorage(s => s.WithBootstrapNode(uploader)); + var uploader = StartLogosStorage(s => s.WithLogFormat(LogosStorageLogFormat.Json)); + var downloader = StartLogosStorage(s => + s.WithBootstrapNode(uploader).WithLogFormat(LogosStorageLogFormat.Json) + ); var file = GenerateTestFile(2.GB()); var size = file.GetFilesize().SizeInBytes; diff --git a/Tests/LogosStorageReleaseTests/DataTests/OneClientTest.cs b/Tests/LogosStorageReleaseTests/DataTests/OneClientTest.cs index 2bd057c7..cc5af068 100644 --- a/Tests/LogosStorageReleaseTests/DataTests/OneClientTest.cs +++ b/Tests/LogosStorageReleaseTests/DataTests/OneClientTest.cs @@ -11,7 +11,7 @@ namespace LogosStorageReleaseTests.DataTests [Test] public void OneClient() { - var node = StartLogosStorage(); + var node = StartLogosStorage(s => s.WithLogFormat(LogosStorageLogFormat.Json)); PerformOneClientTest(node); diff --git a/Tests/LogosStorageReleaseTests/DataTests/StreamlessDownloadTest.cs b/Tests/LogosStorageReleaseTests/DataTests/StreamlessDownloadTest.cs index ad377258..b75dd554 100644 --- a/Tests/LogosStorageReleaseTests/DataTests/StreamlessDownloadTest.cs +++ b/Tests/LogosStorageReleaseTests/DataTests/StreamlessDownloadTest.cs @@ -1,4 +1,5 @@ -using LogosStorageTests; +using LogosStorageClient; +using LogosStorageTests; using NUnit.Framework; using Utils; @@ -10,8 +11,10 @@ namespace LogosStorageReleaseTests.DataTests [Test] public void StreamlessTest() { - var uploader = StartLogosStorage(); - var downloader = StartLogosStorage(s => s.WithBootstrapNode(uploader)); + var uploader = StartLogosStorage(s => s.WithLogFormat(LogosStorageLogFormat.Json)); + var downloader = StartLogosStorage(s => + s.WithBootstrapNode(uploader).WithLogFormat(LogosStorageLogFormat.Json) + ); var size = 10.MB(); var file = GenerateTestFile(size); diff --git a/Tests/LogosStorageReleaseTests/DataTests/SwarmTest.cs b/Tests/LogosStorageReleaseTests/DataTests/SwarmTest.cs index 9e084994..b62573d9 100644 --- a/Tests/LogosStorageReleaseTests/DataTests/SwarmTest.cs +++ b/Tests/LogosStorageReleaseTests/DataTests/SwarmTest.cs @@ -40,7 +40,10 @@ namespace LogosStorageReleaseTests.DataTests public void Stream() { var filesize = filesizeMb.MB(); - nodes = StartLogosStorage(numberOfNodes); + nodes = StartLogosStorage( + numberOfNodes, + s => s.WithLogFormat(LogosStorageLogFormat.Json) + ); var files = nodes.Select(n => UploadUniqueFilePerNode(n, filesize)).ToArray(); var tasks = ParallelDownloadEachFile(files); @@ -53,7 +56,10 @@ namespace LogosStorageReleaseTests.DataTests public void Streamless() { var filesize = filesizeMb.MB(); - nodes = StartLogosStorage(numberOfNodes); + nodes = StartLogosStorage( + numberOfNodes, + s => s.WithLogFormat(LogosStorageLogFormat.Json) + ); var files = nodes.Select(n => UploadUniqueFilePerNode(n, filesize)).ToArray(); var tasks = ParallelStreamlessDownloadEachFile(files); diff --git a/Tests/LogosStorageReleaseTests/DataTests/TheseusTest.cs b/Tests/LogosStorageReleaseTests/DataTests/TheseusTest.cs index c444dedf..3e52fa9d 100644 --- a/Tests/LogosStorageReleaseTests/DataTests/TheseusTest.cs +++ b/Tests/LogosStorageReleaseTests/DataTests/TheseusTest.cs @@ -29,7 +29,13 @@ namespace LogosStorageReleaseTests.DataTests Assert.That(remainingNodes, Is.GreaterThan(0)); Assert.That(steps, Is.GreaterThan(remainingNodes + 1)); - nodes.AddRange(StartLogosStorage(remainingNodes + 1)); + nodes.AddRange( + StartLogosStorage( + remainingNodes + 1, + s => s.WithLogFormat(LogosStorageLogFormat.Json) + ) + ); + cid = nodes.First().UploadFile(file); AllNodesHaveFile(); @@ -40,7 +46,7 @@ namespace LogosStorageReleaseTests.DataTests nodes[0].Stop(waitTillStopped: true); nodes.RemoveAt(0); - nodes.Add(StartLogosStorage()); + nodes.Add(StartLogosStorage(s => s.WithLogFormat(LogosStorageLogFormat.Json))); AllNodesHaveFile(); } diff --git a/Tests/LogosStorageReleaseTests/DataTests/ThreeClientTest.cs b/Tests/LogosStorageReleaseTests/DataTests/ThreeClientTest.cs index 69de0fbf..51754620 100644 --- a/Tests/LogosStorageReleaseTests/DataTests/ThreeClientTest.cs +++ b/Tests/LogosStorageReleaseTests/DataTests/ThreeClientTest.cs @@ -1,4 +1,5 @@ -using LogosStorageTests; +using LogosStorageClient; +using LogosStorageTests; using NUnit.Framework; using Utils; @@ -9,8 +10,8 @@ namespace LogosStorageReleaseTests.DataTests [Test] public void ThreeClient() { - var primary = StartLogosStorage(); - var secondary = StartLogosStorage(); + var primary = StartLogosStorage(s => s.WithLogFormat(LogosStorageLogFormat.Json)); + var secondary = StartLogosStorage(s => s.WithLogFormat(LogosStorageLogFormat.Json)); var testFile = GenerateTestFile(10.MB()); diff --git a/Tests/LogosStorageReleaseTests/DataTests/TwoClientTest.cs b/Tests/LogosStorageReleaseTests/DataTests/TwoClientTest.cs index 71e6d4ad..45911f0d 100644 --- a/Tests/LogosStorageReleaseTests/DataTests/TwoClientTest.cs +++ b/Tests/LogosStorageReleaseTests/DataTests/TwoClientTest.cs @@ -1,7 +1,7 @@ using LogosStorageClient; -using StoragePlugin; using LogosStorageTests; using NUnit.Framework; +using StoragePlugin; using Utils; namespace LogosStorageReleaseTests.DataTests @@ -12,8 +12,14 @@ namespace LogosStorageReleaseTests.DataTests [Test] public void TwoClientTest() { - var uploader = StartLogosStorage(s => s.WithName("Uploader")); - var downloader = StartLogosStorage(s => s.WithName("Downloader").WithBootstrapNode(uploader)); + var uploader = StartLogosStorage(s => + s.WithName("Uploader").WithLogFormat(LogosStorageLogFormat.Json) + ); + var downloader = StartLogosStorage(s => + s.WithName("Downloader") + .WithBootstrapNode(uploader) + .WithLogFormat(LogosStorageLogFormat.Json) + ); PerformTwoClientTest(uploader, downloader); } @@ -29,8 +35,17 @@ namespace LogosStorageReleaseTests.DataTests return; } - var uploader = Ci.StartStorageNode(s => s.WithName("Uploader").At(locations.Get(0))); - var downloader = Ci.StartStorageNode(s => s.WithName("Downloader").WithBootstrapNode(uploader).At(locations.Get(1))); + var uploader = Ci.StartStorageNode(s => + s.WithName("Uploader") + .At(locations.Get(0)) + .WithLogFormat(LogosStorageLogFormat.Json) + ); + var downloader = Ci.StartStorageNode(s => + s.WithName("Downloader") + .WithBootstrapNode(uploader) + .At(locations.Get(1)) + .WithLogFormat(LogosStorageLogFormat.Json) + ); PerformTwoClientTest(uploader, downloader); } diff --git a/Tests/LogosStorageReleaseTests/DataTests/UnknownCidTest.cs b/Tests/LogosStorageReleaseTests/DataTests/UnknownCidTest.cs index b0d29b25..04c3199d 100644 --- a/Tests/LogosStorageReleaseTests/DataTests/UnknownCidTest.cs +++ b/Tests/LogosStorageReleaseTests/DataTests/UnknownCidTest.cs @@ -10,7 +10,7 @@ namespace LogosStorageReleaseTests.DataTests [Test] public void DownloadingUnknownCidDoesNotCauseCrash() { - var node = StartLogosStorage(); + var node = StartLogosStorage(s => s.WithLogFormat(LogosStorageLogFormat.Json)); var unknownCid = new ContentId("zDvZRwzkzHsok3Z8yMoiXE9EDBFwgr8WygB8s4ddcLzzSwwXAxLZ"); diff --git a/Tests/LogosStorageReleaseTests/NodeTests/BasicInfoTests.cs b/Tests/LogosStorageReleaseTests/NodeTests/BasicInfoTests.cs index 139e709d..5eb93f79 100644 --- a/Tests/LogosStorageReleaseTests/NodeTests/BasicInfoTests.cs +++ b/Tests/LogosStorageReleaseTests/NodeTests/BasicInfoTests.cs @@ -1,4 +1,5 @@ -using LogosStorageTests; +using LogosStorageClient; +using LogosStorageTests; using NUnit.Framework; using Utils; @@ -11,7 +12,9 @@ namespace LogosStorageReleaseTests.NodeTests public void QuotaTest() { var size = 3.GB(); - var node = StartLogosStorage(s => s.WithStorageQuota(size)); + var node = StartLogosStorage(s => + s.WithStorageQuota(size).WithLogFormat(LogosStorageLogFormat.Json) + ); var space = node.Space(); Assert.That(space.QuotaMaxBytes, Is.EqualTo(size.SizeInBytes)); @@ -20,8 +23,8 @@ namespace LogosStorageReleaseTests.NodeTests [Test] public void Spr() { - var node = StartLogosStorage(); - + var node = StartLogosStorage(s => s.WithLogFormat(LogosStorageLogFormat.Json)); + var info = node.GetDebugInfo(); Assert.That(!string.IsNullOrEmpty(info.Spr)); @@ -34,7 +37,7 @@ namespace LogosStorageReleaseTests.NodeTests [Test] public void VersionInfo() { - var node = StartLogosStorage(); + var node = StartLogosStorage(s => s.WithLogFormat(LogosStorageLogFormat.Json)); var info = node.GetDebugInfo(); Assert.That(!string.IsNullOrEmpty(info.Version.Version)); @@ -44,7 +47,7 @@ namespace LogosStorageReleaseTests.NodeTests [Test] public void AnnounceAddress() { - var node = StartLogosStorage(); + var node = StartLogosStorage(s => s.WithLogFormat(LogosStorageLogFormat.Json)); var addr = node.GetListenEndpoint(); var info = node.GetDebugInfo(); diff --git a/Tests/LogosStorageReleaseTests/NodeTests/PeerTableTests.cs b/Tests/LogosStorageReleaseTests/NodeTests/PeerTableTests.cs index 81acfa7e..3900fe50 100644 --- a/Tests/LogosStorageReleaseTests/NodeTests/PeerTableTests.cs +++ b/Tests/LogosStorageReleaseTests/NodeTests/PeerTableTests.cs @@ -11,7 +11,7 @@ namespace LogosStorageReleaseTests.NodeTests [Test] public void PeerTableCompleteness() { - var nodes = StartLogosStorage(10); + var nodes = StartLogosStorage(10, s => s.WithLogFormat(LogosStorageLogFormat.Json)); AssertAllNodesSeeEachOther(nodes.Concat([BootstrapNode!])); }