diff --git a/ContinuousTests/NodeRunner.cs b/ContinuousTests/NodeRunner.cs index 5b79bf5..d599c4d 100644 --- a/ContinuousTests/NodeRunner.cs +++ b/ContinuousTests/NodeRunner.cs @@ -5,6 +5,7 @@ using KubernetesWorkflow; using NUnit.Framework; using Logging; using Utils; +using DistTestCore.Logs; namespace ContinuousTests { @@ -38,6 +39,21 @@ namespace ContinuousTests RunNode(bootstrapNode, operation, 0.TestTokens()); } + public IDownloadedLog DownloadLog(RunningContainer container, int? tailLines = null) + { + var subFile = log.CreateSubfile(); + var description = container.Name; + var handler = new LogDownloadHandler(container, description, subFile); + + log.Log($"Downloading logs for {description} to file '{subFile.FullFilename}'"); + + var lifecycle = CreateTestLifecycle(); + var flow = lifecycle.WorkflowCreator.CreateWorkflow(); + flow.DownloadContainerLog(container, handler, tailLines); + + return new DownloadedLog(subFile, description); + } + public void RunNode(CodexAccess bootstrapNode, Action operation, TestToken mintTestTokens) { var lifecycle = CreateTestLifecycle(); diff --git a/ContinuousTests/Tests/HoldMyBeerTest.cs b/ContinuousTests/Tests/HoldMyBeerTest.cs new file mode 100644 index 0000000..0f5e422 --- /dev/null +++ b/ContinuousTests/Tests/HoldMyBeerTest.cs @@ -0,0 +1,103 @@ +using DistTestCore; +using Logging; +using NUnit.Framework; + +namespace ContinuousTests.Tests +{ + public class HoldMyBeerTest : ContinuousTest + { + public override int RequiredNumberOfNodes => 1; + public override TimeSpan RunTestEvery => TimeSpan.FromSeconds(30); + public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure; + + private ContentId? cid; + private TestFile file = null!; + + [TestMoment(t: Zero)] + public void UploadTestFile() + { + var filesize = 80.MB(); + double codexDefaultBlockSize = 31 * 64 * 33; + var numberOfBlocks = Convert.ToInt64(Math.Ceiling(filesize.SizeInBytes / codexDefaultBlockSize)); + var sizeInBytes = filesize.SizeInBytes; + Assert.That(numberOfBlocks, Is.EqualTo(1282)); + + + file = FileManager.GenerateTestFile(filesize); + + cid = UploadFile(Nodes[0], file); + Assert.That(cid, Is.Not.Null); + + var cidTag = cid!.Id.Substring(cid.Id.Length - 6); + Stopwatch.Measure(Log, "upload-log-asserts", () => + { + var uploadLog = NodeRunner.DownloadLog(Nodes[0].Container, 50000); + + var storeLines = uploadLog.FindLinesThatContain("Stored data", "topics=\"codex node\""); + uploadLog.DeleteFile(); + + var storeLine = GetLineForCidTag(storeLines, cidTag); + AssertStoreLineContains(storeLine, numberOfBlocks, sizeInBytes); + }); + + + var dl = DownloadFile(Nodes[1], cid!); + file.AssertIsEqual(dl); + + Stopwatch.Measure(Log, "download-log-asserts", () => + { + var downloadLog = NodeRunner.DownloadLog(Nodes[0].Container, 50000); + + var sentLines = downloadLog.FindLinesThatContain("Sent bytes", "topics=\"codex restapi\""); + downloadLog.DeleteFile(); + + var sentLine = GetLineForCidTag(sentLines, cidTag); + AssertSentLineContains(sentLine, sizeInBytes); + }); + } + + private void AssertSentLineContains(string sentLine, long sizeInBytes) + { + var tag = "bytes="; + var token = sentLine.Substring(sentLine.IndexOf(tag) + tag.Length); + var bytes = Convert.ToInt64(token); + Assert.AreEqual(sizeInBytes, bytes, $"Sent bytes: Number of bytes incorrect. Line: '{sentLine}'"); + } + + private void AssertStoreLineContains(string storeLine, long numberOfBlocks, long sizeInBytes) + { + var tokens = storeLine.Split(" "); + + var blocksToken = GetToken(tokens, "blocks="); + var sizeToken = GetToken(tokens, "size="); + if (blocksToken == null) Assert.Fail("blockToken not found in " + storeLine); + if (sizeToken == null) Assert.Fail("sizeToken not found in " + storeLine); + + var blocks = Convert.ToInt64(blocksToken); + var size = Convert.ToInt64(sizeToken?.Replace("'NByte", "")); + + var lineLog = $" Line: '{storeLine}'"; + Assert.AreEqual(numberOfBlocks, blocks, "Stored data: Number of blocks incorrect." + lineLog); + Assert.AreEqual(sizeInBytes, size, "Stored data: Number of blocks incorrect." + lineLog); + } + + private string GetLineForCidTag(string[] lines, string cidTag) + { + var result = lines.SingleOrDefault(l => l.Contains(cidTag)); + if (result == null) + { + Assert.Fail($"Failed to find '{cidTag}' in lines: '{string.Join(",", lines)}'"); + throw new Exception(); + } + + return result; + } + + private string? GetToken(string[] tokens, string tag) + { + var token = tokens.SingleOrDefault(t => t.StartsWith(tag)); + if (token == null) return null; + return token.Substring(tag.Length); + } + } +} diff --git a/Tests/BasicTests/ContinuousSubstitute.cs b/Tests/BasicTests/ContinuousSubstitute.cs index 36f071e..0dc0113 100644 --- a/Tests/BasicTests/ContinuousSubstitute.cs +++ b/Tests/BasicTests/ContinuousSubstitute.cs @@ -80,6 +80,7 @@ namespace Tests.BasicTests var sizeInBytes = filesize.SizeInBytes; Assert.That(numberOfBlocks, Is.EqualTo(1282)); + var startTime = DateTime.UtcNow; var successfulUploads = 0; var successfulDownloads = 0; @@ -129,8 +130,9 @@ namespace Tests.BasicTests } catch { + var testDuration = DateTime.UtcNow - startTime; Log("Test failed. Delaying shut-down by 30 seconds to collect metrics."); - Log($"Test failed after {successfulUploads} successful uploads and {successfulDownloads} successful downloads"); + Log($"Test failed after {Time.FormatDuration(testDuration)} and {successfulUploads} successful uploads and {successfulDownloads} successful downloads"); Thread.Sleep(TimeSpan.FromSeconds(30)); throw; }