status-im/cs-codex-dist-tests
2
0
Fork 0
mirror of synced 2025-01-11 17:14:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'feature/fully-connected-download-tests'

Browse Source
This commit is contained in:
benbierens 2023-06-20 13:39:07 +02:00
commit 62d646836d
No known key found for this signature in database
GPG Key ID: FE44815D96D0A1AA
18 changed files with 285 additions and 124 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
DistTestCore/ByteSize.cs
View File

@ -1,14 +1,23 @@
namespace DistTestCore
using Utils;
namespace DistTestCore
{
public class ByteSize
{
public ByteSize(long sizeInBytes)
{
if (sizeInBytes < 0) throw new ArgumentException("Cannot create ByteSize object with size less than 0. Was: " + sizeInBytes);
SizeInBytes = sizeInBytes;
}
public long SizeInBytes { get; }
public long ToMB()
{
return SizeInBytes / (1024 * 1024);
}
public override bool Equals(object? obj)
{
return obj is ByteSize size && SizeInBytes == size.SizeInBytes;
@ -21,7 +30,7 @@
public override string ToString()
{
return $"{SizeInBytes} bytes";
return Formatter.FormatByteSize(SizeInBytes);
}
}

14
DistTestCore/Codex/CodexContainerRecipe.cs
View File

@ -5,15 +5,19 @@ namespace DistTestCore.Codex
{
public class CodexContainerRecipe : ContainerRecipeFactory
{
#if Arm64
#if Arm64
public const string DockerImage = "codexstorage/nim-codex:sha-7b88ea0";
#else
//public const string DockerImage = "codexstorage/nim-codex:sha-7b88ea0";
public const string DockerImage = "codexstorage/nim-codex:sha-7b88ea0";
#endif
#else
public const string DockerImage = "thatbenbierens/nim-codex:dhting";
//public const string DockerImage = "codexstorage/nim-codex:sha-7b88ea0";
#endif
public const string MetricsPortTag = "metrics_port";
public const string DiscoveryPortTag = "discovery-port";
// Used by tests for time-constraint assersions.
public static readonly TimeSpan MaxUploadTimePerMegabyte = TimeSpan.FromSeconds(2.0);
public static readonly TimeSpan MaxDownloadTimePerMegabyte = TimeSpan.FromSeconds(2.0);
protected override string Image => DockerImage;
protected override void Initialize(StartupConfig startupConfig)

5
DistTestCore/DistTest.cs
View File

@ -10,7 +10,6 @@ using System.Reflection;
namespace DistTestCore
{
[SetUpFixture]
[Parallelizable(ParallelScope.All)]
public abstract class DistTest
{
@ -87,9 +86,9 @@ namespace DistTestCore
}
}
public TestFile GenerateTestFile(ByteSize size)
public TestFile GenerateTestFile(ByteSize size, string label = "")
{
return Get().FileManager.GenerateTestFile(size);
return Get().FileManager.GenerateTestFile(size, label);
}
/// <summary>

114
DistTestCore/FileManager.cs
View File

@ -1,13 +1,14 @@
using Logging;
using NUnit.Framework;
using System.Runtime.InteropServices;
using Utils;
namespace DistTestCore
{
public interface IFileManager
{
TestFile CreateEmptyTestFile();
TestFile GenerateTestFile(ByteSize size);
TestFile CreateEmptyTestFile(string label = "");
TestFile GenerateTestFile(ByteSize size, string label = "");
void DeleteAllTestFiles();
void PushFileSet();
void PopFileSet();
@ -15,7 +16,7 @@ namespace DistTestCore
public class FileManager : IFileManager
{
public const int ChunkSize = 1024 * 1024;
public const int ChunkSize = 1024 * 1024 * 100;
private static NumberSource folderNumberSource = new NumberSource(0);
private readonly Random random = new Random();
private readonly TestLog log;
@ -30,19 +31,20 @@ namespace DistTestCore
this.log = log;
}
public TestFile CreateEmptyTestFile()
public TestFile CreateEmptyTestFile(string label = "")
{
var result = new TestFile(Path.Combine(folder, Guid.NewGuid().ToString() + "_test.bin"));
var path = Path.Combine(folder, Guid.NewGuid().ToString() + "_test.bin");
var result = new TestFile(log, path, label);
File.Create(result.Filename).Close();
if (fileSetStack.Any()) fileSetStack.Last().Add(result);
return result;
}
public TestFile GenerateTestFile(ByteSize size)
public TestFile GenerateTestFile(ByteSize size, string label)
{
var result = CreateEmptyTestFile();
GenerateFileBytes(result, size);
log.Log($"Generated {size.SizeInBytes} bytes of content for file '{result.Filename}'.");
var sw = Stopwatch.Begin(log);
var result = GenerateFile(size, label);
sw.End($"Generated file '{result.Describe()}'.");
return result;
}
@ -72,14 +74,50 @@ namespace DistTestCore
}
}
private TestFile GenerateFile(ByteSize size, string label)
{
var result = CreateEmptyTestFile(label);
CheckSpaceAvailable(result, size);
GenerateFileBytes(result, size);
return result;
}
private void CheckSpaceAvailable(TestFile testFile, ByteSize size)
{
var file = new FileInfo(testFile.Filename);
var drive = new DriveInfo(file.Directory!.Root.FullName);
var spaceAvailable = drive.TotalFreeSpace;
if (spaceAvailable < size.SizeInBytes)
{
var msg = $"Inconclusive: Not enough disk space to perform test. " +
$"{Formatter.FormatByteSize(size.SizeInBytes)} required. " +
$"{Formatter.FormatByteSize(spaceAvailable)} available.";
log.Log(msg);
Assert.Inconclusive(msg);
}
}
private void GenerateFileBytes(TestFile result, ByteSize size)
{
long bytesLeft = size.SizeInBytes;
int chunkSize = ChunkSize;
while (bytesLeft > 0)
{
var length = Math.Min(bytesLeft, ChunkSize);
AppendRandomBytesToFile(result, length);
bytesLeft -= length;
try
{
var length = Math.Min(bytesLeft, chunkSize);
AppendRandomBytesToFile(result, length);
bytesLeft -= length;
}
catch
{
chunkSize = chunkSize / 2;
if (chunkSize < 1024) throw;
}
}
}
@ -104,20 +142,39 @@ namespace DistTestCore
public class TestFile
{
public TestFile(string filename)
private readonly TestLog log;
public TestFile(TestLog log, string filename, string label)
{
this.log = log;
Filename = filename;
Label = label;
}
public string Filename { get; }
public long GetFileSize()
{
var info = new FileInfo(Filename);
return info.Length;
}
public string Label { get; }
public void AssertIsEqual(TestFile? actual)
{
var sw = Stopwatch.Begin(log);
try
{
AssertEqual(actual);
}
finally
{
sw.End($"{nameof(TestFile)}.{nameof(AssertIsEqual)}");
}
}
public string Describe()
{
var sizePostfix = $" ({Formatter.FormatByteSize(GetFileSize())})";
if (!string.IsNullOrEmpty(Label)) return Label + sizePostfix;
return $"'{Filename}'{sizePostfix}";
}
private void AssertEqual(TestFile? actual)
{
if (actual == null) Assert.Fail("TestFile is null.");
if (actual == this || actual!.Filename == Filename) Assert.Fail("TestFile is compared to itself.");
@ -138,10 +195,25 @@ namespace DistTestCore
readExpected = streamExpected.Read(bytesExpected, 0, FileManager.ChunkSize);
readActual = streamActual.Read(bytesActual, 0, FileManager.ChunkSize);
if (readExpected == 0 && readActual == 0) return;
if (readExpected == 0 && readActual == 0)
{
log.Log($"OK: '{Describe()}' is equal to '{actual.Describe()}'.");
return;
}
Assert.That(readActual, Is.EqualTo(readExpected), "Unable to read buffers of equal length.");
CollectionAssert.AreEqual(bytesExpected, bytesActual, "Files are not binary-equal.");
for (var i = 0; i < readActual; i++)
{
if (bytesExpected[i] != bytesActual[i]) Assert.Fail("File contents not equal.");
}
}
}
private long GetFileSize()
{
var info = new FileInfo(Filename);
return info.Length;
}
}
}

12
DistTestCore/Helpers/PeerConnectionTestHelpers.cs
View File

@ -16,10 +16,18 @@ namespace DistTestCore.Helpers
public void AssertFullyConnected(IEnumerable<IOnlineCodexNode> nodes)
{
AssertFullyConnected(nodes.ToArray());
var n = nodes.ToArray();
AssertFullyConnected(n);
for (int i = 0; i < 5; i++)
{
Time.Sleep(TimeSpan.FromSeconds(30));
AssertFullyConnected(n);
}
}
public void AssertFullyConnected(params IOnlineCodexNode[] nodes)
private void AssertFullyConnected(IOnlineCodexNode[] nodes)
{
test.Log($"Asserting peers are fully-connected for nodes: '{string.Join(",", nodes.Select(n => n.GetName()))}'...");
var entries = CreateEntries(nodes);

41
DistTestCore/Helpers/PeerDownloadTestHelpers.cs
View File

@ -1,4 +1,7 @@
namespace DistTestCore.Helpers
using DistTestCore.Codex;
using NUnit.Framework;
namespace DistTestCore.Helpers
{
public class PeerDownloadTestHelpers
{
@ -9,14 +12,11 @@
this.test = test;
}
public void AssertFullDownloadInterconnectivity(IEnumerable<IOnlineCodexNode> nodes)
{
AssertFullDownloadInterconnectivity(nodes, 1.MB());
}
public void AssertFullDownloadInterconnectivity(IEnumerable<IOnlineCodexNode> nodes, ByteSize testFileSize)
{
test.Log($"Asserting full download interconnectivity for nodes: '{string.Join(",", nodes.Select(n => n.GetName()))}'...");
var start = DateTime.UtcNow;
foreach (var node in nodes)
{
var uploader = node;
@ -29,12 +29,27 @@
}
test.Log($"Success! Full download interconnectivity for nodes: {string.Join(",", nodes.Select(n => n.GetName()))}");
var timeTaken = DateTime.UtcNow - start;
AssertTimePerMB(timeTaken, nodes.Count(), testFileSize);
}
private void AssertTimePerMB(TimeSpan timeTaken, int numberOfNodes, ByteSize size)
{
var numberOfDownloads = numberOfNodes * (numberOfNodes - 1);
var timePerDownload = timeTaken / numberOfDownloads;
float sizeInMB = size.ToMB();
var timePerMB = timePerDownload / sizeInMB;
test.Log($"Performed {numberOfDownloads} downloads of {size} in {timeTaken.TotalSeconds} seconds, for an average of {timePerMB.TotalSeconds} seconds per MB.");
Assert.That(timePerMB, Is.LessThan(CodexContainerRecipe.MaxDownloadTimePerMegabyte), "MaxDownloadTimePerMegabyte performance threshold breached.");
}
private void PerformTest(IOnlineCodexNode uploader, IOnlineCodexNode[] downloaders, ByteSize testFileSize)
{
// 1 test file per downloader.
var files = downloaders.Select(d => test.GenerateTestFile(testFileSize)).ToArray();
// Generate 1 test file per downloader.
var files = downloaders.Select(d => GenerateTestFile(uploader, d, testFileSize)).ToArray();
// Upload all the test files to the uploader.
var contentIds = files.Select(uploader.UploadFile).ToArray();
@ -43,10 +58,18 @@
for (var i = 0; i < downloaders.Length; i++)
{
var expectedFile = files[i];
var downloadedFile = downloaders[i].DownloadContent(contentIds[i]);
var downloadedFile = downloaders[i].DownloadContent(contentIds[i], $"{expectedFile.Label}DOWNLOADED");
expectedFile.AssertIsEqual(downloadedFile);
}
}
private TestFile GenerateTestFile(IOnlineCodexNode uploader, IOnlineCodexNode downloader, ByteSize testFileSize)
{
var up = uploader.GetName().Replace("<", "").Replace(">", "");
var down = downloader.GetName().Replace("<", "").Replace(">", "");
var label = $"FROM{up}TO{down}";
return test.GenerateTestFile(testFileSize, label);
}
}
}

1
DistTestCore/Marketplace/MarketplaceAccessFactory.cs
View File

@ -1,5 +1,4 @@
using DistTestCore.Codex;
using Logging;
namespace DistTestCore.Marketplace
{

25
DistTestCore/OnlineCodexNode.cs
View File

@ -2,6 +2,7 @@
using DistTestCore.Logs;
using DistTestCore.Marketplace;
using DistTestCore.Metrics;
using Logging;
using NUnit.Framework;
namespace DistTestCore
@ -13,7 +14,7 @@ namespace DistTestCore
CodexDebugPeerResponse GetDebugPeer(string peerId);
CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout);
ContentId UploadFile(TestFile file);
TestFile? DownloadContent(ContentId contentId);
TestFile? DownloadContent(ContentId contentId, string fileLabel = "");
void ConnectToPeer(IOnlineCodexNode node);
ICodexNodeLog DownloadLog();
IMetricsAccess Metrics { get; }
@ -66,23 +67,31 @@ namespace DistTestCore
public ContentId UploadFile(TestFile file)
{
Log($"Uploading file of size {file.GetFileSize()}...");
using var fileStream = File.OpenRead(file.Filename);
var response = CodexAccess.UploadFile(fileStream);
var logMessage = $"Uploading file {file.Describe()}...";
var response = Stopwatch.Measure(lifecycle.Log, logMessage, () =>
{
return CodexAccess.UploadFile(fileStream);
});
if (response.StartsWith(UploadFailedMessage))
{
Assert.Fail("Node failed to store block.");
}
var logReplacement = $"(CID:{file.Describe()})";
Log($"ContentId '{response}' is {logReplacement}");
lifecycle.Log.AddStringReplace(response, logReplacement);
Log($"Uploaded file. Received contentId: '{response}'.");
return new ContentId(response);
}
public TestFile? DownloadContent(ContentId contentId)
public TestFile? DownloadContent(ContentId contentId, string fileLabel = "")
{
Log($"Downloading for contentId: '{contentId.Id}'...");
var file = lifecycle.FileManager.CreateEmptyTestFile();
DownloadToFile(contentId.Id, file);
Log($"Downloaded file of size {file.GetFileSize()} to '{file.Filename}'.");
var logMessage = $"Downloading for contentId: '{contentId.Id}'...";
var file = lifecycle.FileManager.CreateEmptyTestFile(fileLabel);
Stopwatch.Measure(lifecycle.Log, logMessage, () => DownloadToFile(contentId.Id, file));
Log($"Downloaded file {file.Describe()} to '{file.Filename}'.");
return file;
}

4
DistTestCore/Timing.cs
View File

@ -31,7 +31,7 @@ namespace DistTestCore
public TimeSpan HttpCallRetryDelay()
{
return TimeSpan.FromSeconds(3);
return TimeSpan.FromSeconds(1);
}
public TimeSpan WaitForK8sServiceDelay()
@ -64,7 +64,7 @@ namespace DistTestCore
public TimeSpan HttpCallRetryDelay()
{
return TimeSpan.FromMinutes(5);
return TimeSpan.FromSeconds(2);
}
public TimeSpan WaitForK8sServiceDelay()

1
KubernetesWorkflow/K8sController.cs
View File

@ -391,6 +391,7 @@ namespace KubernetesWorkflow
{
Name = recipe.Name,
Image = recipe.Image,
ImagePullPolicy = "Always",
Ports = CreateContainerPorts(recipe),
Env = CreateEnv(recipe)
};

8
Logging/Stopwatch.cs
View File

@ -23,6 +23,14 @@ namespace Logging
sw.End();
}
public static T Measure<T>(BaseLog log, string name, Func<T> action, bool debug = false)
{
var sw = Begin(log, name, debug);
var result = action();
sw.End();
return result;
}
public static Stopwatch Begin(BaseLog log)
{
return Begin(log, "");

66
LongTests/BasicTests/LargeFileTests.cs
View File

@ -1,24 +1,74 @@
using DistTestCore;
using DistTestCore.Codex;
using NUnit.Framework;
using NUnit.Framework.Interfaces;
namespace TestsLong.BasicTests
{
[TestFixture]
public class LargeFileTests : DistTest
{
[Test, UseLongTimeouts]
public void OneClientLargeFileTest()
#region Abort test run after first failure
private bool stop;
[SetUp]
public void SetUp()
{
var primary = SetupCodexNode(s => s
.WithStorageQuota(20.GB()));
if (stop)
{
Assert.Inconclusive("Previous test failed");
}
}
var testFile = GenerateTestFile(10.GB());
[TearDown]
public void TearDown()
{
if (TestContext.CurrentContext.Result.Outcome.Status == TestStatus.Failed)
{
stop = true;
}
}
var contentId = primary.UploadFile(testFile);
#endregion
var downloadedFile = primary.DownloadContent(contentId);
[TestCase( 1 * 1)] // 1 MB
[TestCase( 1 * 10)]
[TestCase( 1 * 100)]
[TestCase( 1 * 1024)] // 1 GB
[TestCase( 1024 * 10)]
[TestCase( 1024 * 100)]
[TestCase( 1024 * 1024)] // 1 TB :O
[UseLongTimeouts]
public void DownloadCorrectnessTest(long size)
{
var sizeMB = size.MB();
testFile.AssertIsEqual(downloadedFile);
var expectedFile = GenerateTestFile(sizeMB);
var node = SetupCodexNode(s => s.WithStorageQuota((size + 10).MB()));
var uploadStart = DateTime.UtcNow;
var cid = node.UploadFile(expectedFile);
var downloadStart = DateTime.UtcNow;
var actualFile = node.DownloadContent(cid);
var downloadFinished = DateTime.UtcNow;
expectedFile.AssertIsEqual(actualFile);
AssertTimeConstraint(uploadStart, downloadStart, downloadFinished, size);
}
private void AssertTimeConstraint(DateTime uploadStart, DateTime downloadStart, DateTime downloadFinished, long size)
{
float sizeInMB = size;
var uploadTimePerMB = (uploadStart - downloadStart) / sizeInMB;
var downloadTimePerMB = (downloadStart - downloadFinished) / sizeInMB;
Assert.That(uploadTimePerMB, Is.LessThan(CodexContainerRecipe.MaxUploadTimePerMegabyte),
"MaxUploadTimePerMegabyte performance threshold breached.");
Assert.That(downloadTimePerMB, Is.LessThan(CodexContainerRecipe.MaxDownloadTimePerMegabyte),
"MaxDownloadTimePerMegabyte performance threshold breached.");
}
}
}

21
LongTests/BasicTests/TestInfraTests.cs
View File

@ -26,26 +26,5 @@ namespace TestsLong.BasicTests
Assert.That(!string.IsNullOrEmpty(n.GetDebugInfo().id));
}
}
[Test, UseLongTimeouts]
public void DownloadConsistencyTest()
{
var primary = SetupCodexNode(s => s
.WithStorageQuota(2.MB()));
var testFile = GenerateTestFile(1.MB());
var contentId = primary.UploadFile(testFile);
var files = new List<TestFile?>();
for (var i = 0; i < 100; i++)
{
files.Add(primary.DownloadContent(contentId));
}
Assert.That(files.All(f => f != null));
Assert.That(files.All(f => f!.GetFileSize() == testFile.GetFileSize()));
foreach (var file in files) file!.AssertIsEqual(testFile);
}
}
}

6
LongTests/Parallelism.cs Normal file
View File

@ -0,0 +1,6 @@
using NUnit.Framework;
[assembly: LevelOfParallelism(1)]
namespace Tests
{
}

20
Tests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs Normal file
View File

@ -0,0 +1,20 @@
using DistTestCore;
using NUnit.Framework;
namespace Tests.DownloadConnectivityTests
{
[TestFixture]
public class FullyConnectedDownloadTests : AutoBootstrapDistTest
{
[Test]
[Combinatorial]
public void FullyConnectedDownloadTest(
[Values(3, 10, 20)] int numberOfNodes,
[Values(1, 10, 100)] int sizeMBs)
{
for (var i = 0; i < numberOfNodes; i++) SetupCodexNode();
PeerDownloadTestHelpers.AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes(), sizeMBs.MB());
}
}
}

9
Tests/PeerDiscoveryTests/LayeredDiscoveryTests.cs
View File

@ -1,7 +1,6 @@
using DistTestCore;
using DistTestCore.Helpers;
using NUnit.Framework;
using Utils;
namespace Tests.PeerDiscoveryTests
{
@ -35,7 +34,6 @@ namespace Tests.PeerDiscoveryTests
[TestCase(5)]
[TestCase(10)]
[TestCase(20)]
[TestCase(50)]
public void NodeChainTest(int chainLength)
{
var node = SetupCodexNode();
@ -45,18 +43,11 @@ namespace Tests.PeerDiscoveryTests
}
AssertAllNodesConnected();
for (int i = 0; i < 5; i++)
{
Time.Sleep(TimeSpan.FromSeconds(30));
AssertAllNodesConnected();
}
}
private void AssertAllNodesConnected()
{
PeerConnectionTestHelpers.AssertFullyConnected(GetAllOnlineCodexNodes());
//PeerDownloadTestHelpers.AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes());
}
}
}

33
Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs
View File

@ -18,16 +18,6 @@ namespace Tests.PeerDiscoveryTests
Assert.That(result.IsPeerFound, Is.False);
}
[TestCase(2)]
[TestCase(3)]
[TestCase(10)]
public void VariableNodes(int number)
{
SetupCodexNodes(number);
AssertAllNodesConnected();
}
[TestCase(2)]
[TestCase(3)]
[TestCase(10)]
@ -42,32 +32,9 @@ namespace Tests.PeerDiscoveryTests
AssertAllNodesConnected();
}
[TestCase(3, 3)]
[TestCase(3, 5)]
[TestCase(3, 10)]
[TestCase(5, 10)]
[TestCase(3, 20)]
[TestCase(5, 20)]
public void StagedVariableNodes(int numberOfNodes, int numberOfStages)
{
for (var i = 0; i < numberOfStages; i++)
{
SetupCodexNodes(numberOfNodes);
AssertAllNodesConnected();
}
for (int i = 0; i < 5; i++)
{
Time.Sleep(TimeSpan.FromSeconds(30));
AssertAllNodesConnected();
}
}
private void AssertAllNodesConnected()
{
PeerConnectionTestHelpers.AssertFullyConnected(GetAllOnlineCodexNodes());
//PeerDownloadTestHelpers.AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes());
}
}
}

16
Utils/Formatter.cs Normal file
View File

@ -0,0 +1,16 @@
namespace Utils
{
public static class Formatter
{
private static readonly string[] sizeSuffixes = { "B", "KB", "MB", "GB", "TB", "PB", "EB" };
public static string FormatByteSize(long bytes)
{
if (bytes == 0) return "0" + sizeSuffixes[0];
var sizeOrder = Convert.ToInt32(Math.Floor(Math.Log(bytes, 1024)));
var digit = Math.Round(bytes / Math.Pow(1024, sizeOrder), 1);
return digit.ToString() + sizeSuffixes[sizeOrder];
}
}
}