cs-codex-dist-tests/Tests/BasicTests/ContinuousSubstitute.cs

195 lines
7.5 KiB
C#
Raw Normal View History

2023-08-14 08:26:04 +00:00
using DistTestCore;
2023-08-16 14:13:29 +00:00
using Logging;
2023-08-02 13:11:27 +00:00
using NUnit.Framework;
using Utils;
namespace Tests.BasicTests
{
[TestFixture]
2023-08-15 10:55:30 +00:00
public class ContinuousSubstitute : AutoBootstrapDistTest
2023-08-02 13:11:27 +00:00
{
[Test]
2023-08-08 12:42:59 +00:00
[UseLongTimeouts]
2023-08-02 13:11:27 +00:00
public void ContinuousTestSubstitute()
{
2023-08-14 08:26:04 +00:00
var group = SetupCodexNodes(5, o => o
.EnableMetrics()
.EnableMarketplace(100000.TestTokens(), 0.Eth(), isValidator: true)
.WithBlockTTL(TimeSpan.FromMinutes(2))
.WithStorageQuota(3.GB()));
2023-08-02 13:11:27 +00:00
2023-08-14 08:26:04 +00:00
var nodes = group.Cast<OnlineCodexNode>().ToArray();
2023-08-02 13:11:27 +00:00
2023-08-14 08:26:04 +00:00
foreach (var node in nodes)
2023-08-02 13:11:27 +00:00
{
2023-08-14 08:26:04 +00:00
node.Marketplace.MakeStorageAvailable(
size: 1.GB(),
minPricePerBytePerSecond: 1.TestTokens(),
maxCollateral: 1024.TestTokens(),
maxDuration: TimeSpan.FromMinutes(5));
}
2023-08-02 13:11:27 +00:00
2023-08-14 08:26:04 +00:00
var endTime = DateTime.UtcNow + TimeSpan.FromHours(10);
while (DateTime.UtcNow < endTime)
{
var allNodes = nodes.ToList();
var primary = allNodes.PickOneRandom();
var secondary = allNodes.PickOneRandom();
2023-08-02 13:11:27 +00:00
2023-08-14 08:26:04 +00:00
Log("Run Test");
PerformTest(primary, secondary);
2023-08-02 13:11:27 +00:00
2023-08-14 08:26:04 +00:00
Thread.Sleep(TimeSpan.FromSeconds(5));
2023-08-02 13:11:27 +00:00
}
}
2023-08-08 12:42:59 +00:00
private ByteSize fileSize = 80.MB();
2023-08-08 08:45:16 +00:00
2023-08-02 13:11:27 +00:00
private void PerformTest(IOnlineCodexNode primary, IOnlineCodexNode secondary)
{
ScopedTestFiles(() =>
{
2023-08-08 08:45:16 +00:00
var testFile = GenerateTestFile(fileSize);
2023-08-02 13:11:27 +00:00
var contentId = primary.UploadFile(testFile);
var downloadedFile = secondary.DownloadContent(contentId);
testFile.AssertIsEqual(downloadedFile);
});
}
2023-08-14 13:10:36 +00:00
2023-08-14 08:26:04 +00:00
[Test]
public void HoldMyBeerTest()
{
2023-08-16 14:13:29 +00:00
var blockExpirationTime = TimeSpan.FromMinutes(3);
var group = SetupCodexNodes(3, o => o
2023-08-22 07:06:44 +00:00
.EnableMetrics()
2023-08-16 14:13:29 +00:00
.WithBlockTTL(blockExpirationTime)
2023-08-22 07:06:44 +00:00
.WithBlockMaintenanceInterval(TimeSpan.FromMinutes(2))
.WithBlockMaintenanceNumber(10000)
2023-08-16 14:13:29 +00:00
.WithStorageQuota(2000.MB()));
2023-08-14 08:26:04 +00:00
var nodes = group.Cast<OnlineCodexNode>().ToArray();
var endTime = DateTime.UtcNow + TimeSpan.FromHours(24);
var filesize = 80.MB();
double codexDefaultBlockSize = 31 * 64 * 33;
var numberOfBlocks = Convert.ToInt64(Math.Ceiling(filesize.SizeInBytes / codexDefaultBlockSize));
var sizeInBytes = filesize.SizeInBytes;
Assert.That(numberOfBlocks, Is.EqualTo(1282));
var startTime = DateTime.UtcNow;
2023-08-16 14:13:29 +00:00
var successfulUploads = 0;
var successfulDownloads = 0;
2023-08-15 10:55:30 +00:00
while (DateTime.UtcNow < endTime)
{
foreach (var node in nodes)
{
2023-08-15 13:00:47 +00:00
try
{
Thread.Sleep(TimeSpan.FromSeconds(5));
2023-08-16 14:13:29 +00:00
ScopedTestFiles(() =>
2023-08-16 14:13:29 +00:00
{
var uploadStartTime = DateTime.UtcNow;
var file = GenerateTestFile(filesize);
var cid = node.UploadFile(file);
var cidTag = cid.Id.Substring(cid.Id.Length - 6);
2023-08-22 13:51:39 +00:00
Measure("upload-log-asserts", () =>
{
var uploadLog = node.DownloadLog(tailLines: 50000);
var storeLines = uploadLog.FindLinesThatContain("Stored data", "topics=\"codex node\"");
uploadLog.DeleteFile();
var storeLine = GetLineForCidTag(storeLines, cidTag);
AssertStoreLineContains(storeLine, numberOfBlocks, sizeInBytes);
});
successfulUploads++;
var uploadTimeTaken = DateTime.UtcNow - uploadStartTime;
if (uploadTimeTaken >= blockExpirationTime.Subtract(TimeSpan.FromSeconds(10)))
{
Assert.Fail("Upload took too long. Blocks already expired.");
}
var dl = node.DownloadContent(cid);
file.AssertIsEqual(dl);
2023-08-22 13:51:39 +00:00
Measure("download-log-asserts", () =>
{
var downloadLog = node.DownloadLog(tailLines: 50000);
var sentLines = downloadLog.FindLinesThatContain("Sent bytes", "topics=\"codex restapi\"");
downloadLog.DeleteFile();
var sentLine = GetLineForCidTag(sentLines, cidTag);
AssertSentLineContains(sentLine, sizeInBytes);
});
successfulDownloads++;
2023-08-16 14:13:29 +00:00
});
2023-08-15 13:00:47 +00:00
}
catch
{
var testDuration = DateTime.UtcNow - startTime;
2023-08-15 13:00:47 +00:00
Log("Test failed. Delaying shut-down by 30 seconds to collect metrics.");
Log($"Test failed after {Time.FormatDuration(testDuration)} and {successfulUploads} successful uploads and {successfulDownloads} successful downloads");
2023-08-15 13:00:47 +00:00
Thread.Sleep(TimeSpan.FromSeconds(30));
throw;
}
2023-08-15 10:55:30 +00:00
}
2023-08-14 13:10:36 +00:00
2023-08-16 14:13:29 +00:00
Thread.Sleep(TimeSpan.FromSeconds(5));
2023-08-14 08:26:04 +00:00
}
}
private void AssertSentLineContains(string sentLine, long sizeInBytes)
{
var tag = "bytes=";
var token = sentLine.Substring(sentLine.IndexOf(tag) + tag.Length);
var bytes = Convert.ToInt64(token);
Assert.AreEqual(sizeInBytes, bytes, $"Sent bytes: Number of bytes incorrect. Line: '{sentLine}'");
}
private void AssertStoreLineContains(string storeLine, long numberOfBlocks, long sizeInBytes)
{
var tokens = storeLine.Split(" ");
var blocksToken = GetToken(tokens, "blocks=");
var sizeToken = GetToken(tokens, "size=");
if (blocksToken == null) Assert.Fail("blockToken not found in " + storeLine);
if (sizeToken == null) Assert.Fail("sizeToken not found in " + storeLine);
var blocks = Convert.ToInt64(blocksToken);
var size = Convert.ToInt64(sizeToken?.Replace("'NByte", ""));
var lineLog = $" Line: '{storeLine}'";
Assert.AreEqual(numberOfBlocks, blocks, "Stored data: Number of blocks incorrect." + lineLog);
Assert.AreEqual(sizeInBytes, size, "Stored data: Number of blocks incorrect." + lineLog);
}
private string GetLineForCidTag(string[] lines, string cidTag)
{
var result = lines.SingleOrDefault(l => l.Contains(cidTag));
if (result == null)
{
Assert.Fail($"Failed to find '{cidTag}' in lines: '{string.Join(",", lines)}'");
throw new Exception();
}
return result;
}
private string? GetToken(string[] tokens, string tag)
{
var token = tokens.SingleOrDefault(t => t.StartsWith(tag));
if (token == null) return null;
return token.Substring(tag.Length);
}
2023-08-02 13:11:27 +00:00
}
}