From 6486dba2895900d1181e5992deba2fb7bdfe6258 Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 16 Aug 2023 10:52:44 +0200 Subject: [PATCH] Asserting correct filesize and block numbers in the codex logs --- DistTestCore/Logs/DownloadedLog.cs | 27 +++++++++ DistTestCore/Timing.cs | 6 +- Tests/BasicTests/ContinuousSubstitute.cs | 75 +++++++++++++++++++++++- 3 files changed, 103 insertions(+), 5 deletions(-) diff --git a/DistTestCore/Logs/DownloadedLog.cs b/DistTestCore/Logs/DownloadedLog.cs index 9d22c81..606c411 100644 --- a/DistTestCore/Logs/DownloadedLog.cs +++ b/DistTestCore/Logs/DownloadedLog.cs @@ -6,6 +6,8 @@ namespace DistTestCore.Logs public interface IDownloadedLog { void AssertLogContains(string expectedString); + string[] FindLinesThatContain(params string[] tags); + void DeleteFile(); } public class DownloadedLog : IDownloadedLog @@ -33,5 +35,30 @@ namespace DistTestCore.Logs Assert.Fail($"{owner} Unable to find string '{expectedString}' in CodexNode log file {logFile.FullFilename}"); } + + public string[] FindLinesThatContain(params string[] tags) + { + var result = new List(); + using var file = File.OpenRead(logFile.FullFilename); + using var streamReader = new StreamReader(file); + + var line = streamReader.ReadLine(); + while (line != null) + { + if (tags.All(line.Contains)) + { + result.Add(line); + } + + line = streamReader.ReadLine(); + } + + return result.ToArray(); + } + + public void DeleteFile() + { + File.Delete(logFile.FullFilename); + } } } diff --git a/DistTestCore/Timing.cs b/DistTestCore/Timing.cs index bd385ae..38df6d8 100644 --- a/DistTestCore/Timing.cs +++ b/DistTestCore/Timing.cs @@ -21,7 +21,7 @@ namespace DistTestCore { public TimeSpan HttpCallTimeout() { - return TimeSpan.FromSeconds(10); + return TimeSpan.FromMinutes(5); } public TimeSpan HttpCallRetryTime() @@ -36,12 +36,12 @@ namespace DistTestCore public TimeSpan WaitForK8sServiceDelay() { - return TimeSpan.FromSeconds(1); + return TimeSpan.FromSeconds(10); } public TimeSpan K8sOperationTimeout() { - return TimeSpan.FromMinutes(1); + return TimeSpan.FromMinutes(30); } public TimeSpan WaitForMetricTimeout() diff --git a/Tests/BasicTests/ContinuousSubstitute.cs b/Tests/BasicTests/ContinuousSubstitute.cs index 0436bcf..4325d57 100644 --- a/Tests/BasicTests/ContinuousSubstitute.cs +++ b/Tests/BasicTests/ContinuousSubstitute.cs @@ -1,5 +1,6 @@ using DistTestCore; using NUnit.Framework; +using NUnit.Framework.Constraints; using Utils; namespace Tests.BasicTests @@ -59,7 +60,6 @@ namespace Tests.BasicTests } [Test] - [UseLongTimeouts] public void HoldMyBeerTest() { var group = SetupCodexNodes(5, o => o @@ -72,17 +72,52 @@ namespace Tests.BasicTests var nodes = group.Cast().ToArray(); var endTime = DateTime.UtcNow + TimeSpan.FromHours(1); + + var filesize = 80.MB(); + double codexDefaultBlockSize = 31 * 64 * 33; + var numberOfBlocks = Convert.ToInt64(Math.Ceiling(filesize.SizeInBytes / codexDefaultBlockSize)); + var sizeInBytes = filesize.SizeInBytes; + Assert.That(numberOfBlocks, Is.EqualTo(1282)); + while (DateTime.UtcNow < endTime) { foreach (var node in nodes) { try { - var file = GenerateTestFile(80.MB()); + var file = GenerateTestFile(filesize); var cid = node.UploadFile(file); + var cidTag = cid.Id.Substring(cid.Id.Length - (1 + 6)); + var uploadLog = node.DownloadLog(); + + var storeLines = uploadLog.FindLinesThatContain("Stored data", "topics=\"codex node\""); + uploadLog.DeleteFile(); + + var storeLine = GetLineForCidTag(storeLines, cidTag); + if (storeLine == null) + { + Assert.Fail("Storeline not found for cid" + cidTag); + return; + } + AssertStoreLineContains(storeLine, numberOfBlocks, sizeInBytes); + + var dl = node.DownloadContent(cid); file.AssertIsEqual(dl); + var downloadLog = node.DownloadLog(); + + var sentLines = downloadLog.FindLinesThatContain("Sent bytes", "topics=\"codex restapi\""); + downloadLog.DeleteFile(); + + var sentLine = GetLineForCidTag(sentLines, cidTag); + if (sentLine == null) + { + Assert.Fail("Sentline not found for cid" + cidTag); + return; + } + AssertSentLineContains(sentLine, sizeInBytes); + } catch { @@ -95,5 +130,41 @@ namespace Tests.BasicTests Thread.Sleep(TimeSpan.FromSeconds(3)); } } + + private void AssertSentLineContains(string sentLine, long sizeInBytes) + { + var tag = "bytes="; + var token = sentLine.Substring(sentLine.IndexOf(tag) + tag.Length); + var bytes = Convert.ToInt64(token); + Assert.AreEqual(sizeInBytes, bytes, "Sent bytes: Number of bytes incorrect"); + } + + private void AssertStoreLineContains(string storeLine, long numberOfBlocks, long sizeInBytes) + { + var tokens = storeLine.Split(" "); + + var blocksToken = GetToken(tokens, "blocks="); + var sizeToken = GetToken(tokens, "size="); + if (blocksToken == null) Assert.Fail("blockToken not found in " + storeLine); + if (sizeToken == null) Assert.Fail("sizeToken not found in " + storeLine); + + var blocks = Convert.ToInt64(blocksToken); + var size = Convert.ToInt64(sizeToken); + + Assert.AreEqual(numberOfBlocks, blocks, "Stored data: Number of blocks incorrect"); + Assert.AreEqual(sizeInBytes, size, "Stored data: Number of blocks incorrect"); + } + + private string? GetLineForCidTag(string[] lines, string cidTag) + { + return lines.SingleOrDefault(l => l.Contains(cidTag)); + } + + private string? GetToken(string[] tokens, string tag) + { + var token = tokens.SingleOrDefault(t => t.StartsWith(tag)); + if (token == null) return null; + return token.Substring(tag.Length); + } } }