2023-08-14 08:26:04 +00:00
using DistTestCore ;
2023-08-16 14:13:29 +00:00
using Logging ;
2023-08-02 13:11:27 +00:00
using NUnit.Framework ;
using Utils ;
namespace Tests.BasicTests
{
[TestFixture]
2023-08-15 10:55:30 +00:00
public class ContinuousSubstitute : AutoBootstrapDistTest
2023-08-02 13:11:27 +00:00
{
[Test]
2023-08-08 12:42:59 +00:00
[UseLongTimeouts]
2023-08-02 13:11:27 +00:00
public void ContinuousTestSubstitute ( )
{
2023-08-14 08:26:04 +00:00
var group = SetupCodexNodes ( 5 , o = > o
. EnableMetrics ( )
. EnableMarketplace ( 100000. TestTokens ( ) , 0. Eth ( ) , isValidator : true )
. WithBlockTTL ( TimeSpan . FromMinutes ( 2 ) )
. WithStorageQuota ( 3. GB ( ) ) ) ;
2023-08-02 13:11:27 +00:00
2023-08-14 08:26:04 +00:00
var nodes = group . Cast < OnlineCodexNode > ( ) . ToArray ( ) ;
2023-08-02 13:11:27 +00:00
2023-08-14 08:26:04 +00:00
foreach ( var node in nodes )
2023-08-02 13:11:27 +00:00
{
2023-08-14 08:26:04 +00:00
node . Marketplace . MakeStorageAvailable (
size : 1. GB ( ) ,
minPricePerBytePerSecond : 1. TestTokens ( ) ,
maxCollateral : 1024. TestTokens ( ) ,
maxDuration : TimeSpan . FromMinutes ( 5 ) ) ;
}
2023-08-02 13:11:27 +00:00
2023-08-14 08:26:04 +00:00
var endTime = DateTime . UtcNow + TimeSpan . FromHours ( 10 ) ;
while ( DateTime . UtcNow < endTime )
{
var allNodes = nodes . ToList ( ) ;
var primary = allNodes . PickOneRandom ( ) ;
var secondary = allNodes . PickOneRandom ( ) ;
2023-08-02 13:11:27 +00:00
2023-08-14 08:26:04 +00:00
Log ( "Run Test" ) ;
PerformTest ( primary , secondary ) ;
2023-08-02 13:11:27 +00:00
2023-08-14 08:26:04 +00:00
Thread . Sleep ( TimeSpan . FromSeconds ( 5 ) ) ;
2023-08-02 13:11:27 +00:00
}
}
2023-08-08 12:42:59 +00:00
private ByteSize fileSize = 80. MB ( ) ;
2023-08-08 08:45:16 +00:00
2023-08-02 13:11:27 +00:00
private void PerformTest ( IOnlineCodexNode primary , IOnlineCodexNode secondary )
{
ScopedTestFiles ( ( ) = >
{
2023-08-08 08:45:16 +00:00
var testFile = GenerateTestFile ( fileSize ) ;
2023-08-02 13:11:27 +00:00
var contentId = primary . UploadFile ( testFile ) ;
var downloadedFile = secondary . DownloadContent ( contentId ) ;
testFile . AssertIsEqual ( downloadedFile ) ;
} ) ;
}
2023-08-14 13:10:36 +00:00
2023-08-14 08:26:04 +00:00
[Test]
public void HoldMyBeerTest ( )
{
2023-08-16 14:13:29 +00:00
var blockExpirationTime = TimeSpan . FromMinutes ( 3 ) ;
2023-08-21 05:58:58 +00:00
var group = SetupCodexNodes ( 3 , o = > o
2023-08-22 07:06:44 +00:00
. EnableMetrics ( )
2023-08-16 14:13:29 +00:00
. WithBlockTTL ( blockExpirationTime )
2023-08-22 07:06:44 +00:00
. WithBlockMaintenanceInterval ( TimeSpan . FromMinutes ( 2 ) )
2023-08-14 14:37:31 +00:00
. WithBlockMaintenanceNumber ( 10000 )
2023-08-16 14:13:29 +00:00
. WithStorageQuota ( 2000. MB ( ) ) ) ;
2023-08-14 08:26:04 +00:00
var nodes = group . Cast < OnlineCodexNode > ( ) . ToArray ( ) ;
2023-08-21 05:58:58 +00:00
var endTime = DateTime . UtcNow + TimeSpan . FromHours ( 24 ) ;
2023-08-16 08:52:44 +00:00
var filesize = 80. MB ( ) ;
double codexDefaultBlockSize = 31 * 64 * 33 ;
var numberOfBlocks = Convert . ToInt64 ( Math . Ceiling ( filesize . SizeInBytes / codexDefaultBlockSize ) ) ;
var sizeInBytes = filesize . SizeInBytes ;
Assert . That ( numberOfBlocks , Is . EqualTo ( 1282 ) ) ;
2023-08-18 08:31:20 +00:00
var startTime = DateTime . UtcNow ;
2023-08-16 14:13:29 +00:00
var successfulUploads = 0 ;
var successfulDownloads = 0 ;
2023-08-15 10:55:30 +00:00
while ( DateTime . UtcNow < endTime )
{
foreach ( var node in nodes )
{
2023-08-15 13:00:47 +00:00
try
{
2023-08-21 05:58:58 +00:00
Thread . Sleep ( TimeSpan . FromSeconds ( 5 ) ) ;
2023-08-16 14:13:29 +00:00
2023-08-21 05:58:58 +00:00
ScopedTestFiles ( ( ) = >
2023-08-16 14:13:29 +00:00
{
2023-08-21 05:58:58 +00:00
var uploadStartTime = DateTime . UtcNow ;
var file = GenerateTestFile ( filesize ) ;
var cid = node . UploadFile ( file ) ;
var cidTag = cid . Id . Substring ( cid . Id . Length - 6 ) ;
2023-08-22 13:51:39 +00:00
Measure ( "upload-log-asserts" , ( ) = >
2023-08-21 05:58:58 +00:00
{
var uploadLog = node . DownloadLog ( tailLines : 50000 ) ;
var storeLines = uploadLog . FindLinesThatContain ( "Stored data" , "topics=\"codex node\"" ) ;
uploadLog . DeleteFile ( ) ;
var storeLine = GetLineForCidTag ( storeLines , cidTag ) ;
AssertStoreLineContains ( storeLine , numberOfBlocks , sizeInBytes ) ;
} ) ;
successfulUploads + + ;
var uploadTimeTaken = DateTime . UtcNow - uploadStartTime ;
if ( uploadTimeTaken > = blockExpirationTime . Subtract ( TimeSpan . FromSeconds ( 10 ) ) )
{
Assert . Fail ( "Upload took too long. Blocks already expired." ) ;
}
var dl = node . DownloadContent ( cid ) ;
file . AssertIsEqual ( dl ) ;
2023-08-22 13:51:39 +00:00
Measure ( "download-log-asserts" , ( ) = >
2023-08-21 05:58:58 +00:00
{
var downloadLog = node . DownloadLog ( tailLines : 50000 ) ;
var sentLines = downloadLog . FindLinesThatContain ( "Sent bytes" , "topics=\"codex restapi\"" ) ;
downloadLog . DeleteFile ( ) ;
var sentLine = GetLineForCidTag ( sentLines , cidTag ) ;
AssertSentLineContains ( sentLine , sizeInBytes ) ;
} ) ;
successfulDownloads + + ;
2023-08-16 14:13:29 +00:00
} ) ;
2023-08-15 13:00:47 +00:00
}
catch
{
2023-08-18 08:31:20 +00:00
var testDuration = DateTime . UtcNow - startTime ;
2023-08-15 13:00:47 +00:00
Log ( "Test failed. Delaying shut-down by 30 seconds to collect metrics." ) ;
2023-08-18 08:31:20 +00:00
Log ( $"Test failed after {Time.FormatDuration(testDuration)} and {successfulUploads} successful uploads and {successfulDownloads} successful downloads" ) ;
2023-08-15 13:00:47 +00:00
Thread . Sleep ( TimeSpan . FromSeconds ( 30 ) ) ;
throw ;
}
2023-08-15 10:55:30 +00:00
}
2023-08-14 13:10:36 +00:00
2023-08-16 14:13:29 +00:00
Thread . Sleep ( TimeSpan . FromSeconds ( 5 ) ) ;
2023-08-14 08:26:04 +00:00
}
}
2023-08-16 08:52:44 +00:00
private void AssertSentLineContains ( string sentLine , long sizeInBytes )
{
var tag = "bytes=" ;
var token = sentLine . Substring ( sentLine . IndexOf ( tag ) + tag . Length ) ;
var bytes = Convert . ToInt64 ( token ) ;
2023-08-16 09:39:24 +00:00
Assert . AreEqual ( sizeInBytes , bytes , $"Sent bytes: Number of bytes incorrect. Line: '{sentLine}'" ) ;
2023-08-16 08:52:44 +00:00
}
private void AssertStoreLineContains ( string storeLine , long numberOfBlocks , long sizeInBytes )
{
var tokens = storeLine . Split ( " " ) ;
var blocksToken = GetToken ( tokens , "blocks=" ) ;
var sizeToken = GetToken ( tokens , "size=" ) ;
if ( blocksToken = = null ) Assert . Fail ( "blockToken not found in " + storeLine ) ;
if ( sizeToken = = null ) Assert . Fail ( "sizeToken not found in " + storeLine ) ;
var blocks = Convert . ToInt64 ( blocksToken ) ;
2023-08-16 09:39:24 +00:00
var size = Convert . ToInt64 ( sizeToken ? . Replace ( "'NByte" , "" ) ) ;
2023-08-16 08:52:44 +00:00
2023-08-16 09:39:24 +00:00
var lineLog = $" Line: '{storeLine}'" ;
Assert . AreEqual ( numberOfBlocks , blocks , "Stored data: Number of blocks incorrect." + lineLog ) ;
Assert . AreEqual ( sizeInBytes , size , "Stored data: Number of blocks incorrect." + lineLog ) ;
2023-08-16 08:52:44 +00:00
}
2023-08-16 09:39:24 +00:00
private string GetLineForCidTag ( string [ ] lines , string cidTag )
2023-08-16 08:52:44 +00:00
{
2023-08-16 09:39:24 +00:00
var result = lines . SingleOrDefault ( l = > l . Contains ( cidTag ) ) ;
if ( result = = null )
{
Assert . Fail ( $"Failed to find '{cidTag}' in lines: '{string.Join(" , ", lines)}'" ) ;
throw new Exception ( ) ;
}
return result ;
2023-08-16 08:52:44 +00:00
}
private string? GetToken ( string [ ] tokens , string tag )
{
var token = tokens . SingleOrDefault ( t = > t . StartsWith ( tag ) ) ;
if ( token = = null ) return null ;
return token . Substring ( tag . Length ) ;
}
2023-08-02 13:11:27 +00:00
}
}