logos-storage/logos-storage-nim-cs-dist-tests
1
0
Fork 0
mirror of https://github.com/logos-storage/logos-storage-nim-cs-dist-tests.git synced 2026-01-05 23:13:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

Sets up data tests

Browse Source
This commit is contained in:
Ben 2024-11-21 10:46:11 +01:00
parent f4c622a1d3
commit 704847001d
No known key found for this signature in database
GPG Key ID: 0F16E812E736C24B
11 changed files with 281 additions and 138 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
Tests/CodexReleaseTests/CodexReleaseTests.csproj
View File

@ -6,4 +6,12 @@
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="NUnit" Version="3.13.3" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\CodexTests\CodexTests.csproj" />
</ItemGroup>
</Project>

31
Tests/CodexTests/BasicTests/OneClientTests.cs → Tests/CodexReleaseTests/DataTests/InterruptUploadTest.cs
View File

@ -1,26 +1,16 @@
using CodexPlugin;
using CodexTests;
using FileUtils;
using NUnit.Framework;
using System.Diagnostics;
using Utils;
namespace CodexTests.BasicTests
namespace CodexReleaseTests.DataTests
{
[TestFixture]
public class OneClientTests : CodexDistTest
public class InterruptUploadTest : CodexDistTest
{
[Test]
public void OneClientTest()
{
var node = StartCodex();
PerformOneClientTest(node);
LogNodeStatus(node);
}
[Test]
public void InterruptUploadTest()
public void UploadInterruptTest()
{
var nodes = StartCodex(10);
@ -28,6 +18,8 @@ namespace CodexTests.BasicTests
Task.WaitAll(tasks.ToArray());
Assert.That(tasks.Select(t => t.Result).All(r => r == true));
WaitAndCheckNodesStaysAlive(TimeSpan.FromMinutes(2), nodes);
}
private bool RunInterruptUploadTest(ICodexNode node)
@ -51,16 +43,5 @@ namespace CodexTests.BasicTests
var filePath = file.Filename;
return Process.Start("curl", $"-X POST {codexUrl} -H \"Content-Type: application/octet-stream\" -T {filePath}");
}
private void PerformOneClientTest(ICodexNode primary)
{
var testFile = GenerateTestFile(1.MB());
var contentId = primary.UploadFile(testFile);
var downloadedFile = primary.DownloadContent(contentId);
testFile.AssertIsEqual(downloadedFile);
}
}
}

39
Tests/CodexReleaseTests/DataTests/OneClientTest.cs Normal file
View File

@ -0,0 +1,39 @@
using CodexPlugin;
using CodexTests;
using NUnit.Framework;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Utils;
namespace CodexReleaseTests.DataTests
{
[TestFixture]
public class OneClientTest : CodexDistTest
{
[Test]
public void OneClient()
{
var node = StartCodex();
PerformOneClientTest(node);
LogNodeStatus(node);
}
private void PerformOneClientTest(ICodexNode primary)
{
var testFile = GenerateTestFile(1.MB());
var contentId = primary.UploadFile(testFile);
AssertNodesContainFile(contentId, primary);
var downloadedFile = primary.DownloadContent(contentId);
testFile.AssertIsEqual(downloadedFile);
}
}
}

84
Tests/CodexReleaseTests/DataTests/SwarmTest.cs
View File

@ -1,12 +1,92 @@
using System;
using CodexPlugin;
using CodexTests;
using FileUtils;
using NUnit.Framework;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Utils;
namespace CodexReleaseTests.DataTests
{
internal class SwarmTest
[TestFixture]
public class SwarmTests : AutoBootstrapDistTest
{
private const int NumberOfNodes = 10;
private const int FileSizeMb = 100;
[Test]
public void SmallSwarm()
{
var nodes = StartCodex(NumberOfNodes);
var files = nodes.Select(UploadUniqueFilePerNode).ToArray();
var tasks = ParallelDownloadEachFile(nodes, files);
Task.WaitAll(tasks);
AssertAllFilesDownloadedCorrectly(files);
}
private SwarmTestNetworkFile UploadUniqueFilePerNode(ICodexNode node)
{
var file = GenerateTestFile(FileSizeMb.MB());
var cid = node.UploadFile(file);
return new SwarmTestNetworkFile(file, cid);
}
private Task[] ParallelDownloadEachFile(ICodexNodeGroup nodes, SwarmTestNetworkFile[] files)
{
var tasks = new List<Task>();
foreach (var node in nodes)
{
foreach (var file in files)
{
tasks.Add(StartDownload(node, file));
}
}
return tasks.ToArray();
}
private Task StartDownload(ICodexNode node, SwarmTestNetworkFile file)
{
return Task.Run(() =>
{
try
{
file.Downloaded = node.DownloadContent(file.Cid);
}
catch (Exception ex)
{
file.Error = ex;
}
});
}
private void AssertAllFilesDownloadedCorrectly(SwarmTestNetworkFile[] files)
{
foreach (var file in files)
{
if (file.Error != null) throw file.Error;
file.Original.AssertIsEqual(file.Downloaded);
}
}
private class SwarmTestNetworkFile
{
public SwarmTestNetworkFile(TrackedFile original, ContentId cid)
{
Original = original;
Cid = cid;
}
public TrackedFile Original { get; }
public ContentId Cid { get; }
public TrackedFile? Downloaded { get; set; }
public Exception? Error { get; set; } = null;
}
}
}

24
Tests/CodexReleaseTests/DataTests/ThreeClientTest.cs
View File

@ -1,12 +1,32 @@
using System;
using CodexTests;
using NUnit.Framework;
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Utils;
namespace CodexReleaseTests.DataTests
{
internal class ThreeClientTest
public class ThreeClientTest : AutoBootstrapDistTest
{
[Test]
public void ThreeClient()
{
var primary = StartCodex();
var secondary = StartCodex();
var testFile = GenerateTestFile(10.MB());
var contentId = primary.UploadFile(testFile);
AssertNodesContainFile(contentId, primary);
var downloadedFile = secondary.DownloadContent(contentId);
AssertNodesContainFile(contentId, primary, secondary);
testFile.AssertIsEqual(downloadedFile);
}
}
}

52
Tests/CodexReleaseTests/DataTests/TwoClientTest.cs
View File

@ -1,12 +1,60 @@
using System;
using CodexPlugin;
using CodexTests;
using NUnit.Framework;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Utils;
namespace CodexReleaseTests.DataTests
{
internal class TwoClientTest
[TestFixture]
public class TwoClientTests : CodexDistTest
{
[Test]
public void TwoClientTest()
{
var uploader = StartCodex(s => s.WithName("Uploader"));
var downloader = StartCodex(s => s.WithName("Downloader").WithBootstrapNode(uploader));
PerformTwoClientTest(uploader, downloader);
}
[Test]
public void TwoClientsTwoLocationsTest()
{
var locations = Ci.GetKnownLocations();
if (locations.NumberOfLocations < 2)
{
Assert.Inconclusive("Two-locations test requires 2 nodes to be available in the cluster.");
return;
}
var uploader = Ci.StartCodexNode(s => s.WithName("Uploader").At(locations.Get(0)));
var downloader = Ci.StartCodexNode(s => s.WithName("Downloader").WithBootstrapNode(uploader).At(locations.Get(1)));
PerformTwoClientTest(uploader, downloader);
}
private void PerformTwoClientTest(ICodexNode uploader, ICodexNode downloader)
{
PerformTwoClientTest(uploader, downloader, 10.MB());
}
private void PerformTwoClientTest(ICodexNode uploader, ICodexNode downloader, ByteSize size)
{
var testFile = GenerateTestFile(size);
var contentId = uploader.UploadFile(testFile);
AssertNodesContainFile(contentId, uploader);
var downloadedFile = downloader.DownloadContent(contentId);
AssertNodesContainFile(contentId, uploader, downloader);
testFile.AssertIsEqual(downloadedFile);
CheckLogForErrors(uploader, downloader);
}
}
}

40
Tests/CodexReleaseTests/DataTests/UnknownCidTest.cs Normal file
View File

@ -0,0 +1,40 @@
using CodexPlugin;
using CodexTests;
using NUnit.Framework;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace CodexReleaseTests.DataTests
{
[TestFixture]
public class UnknownCidTest : CodexDistTest
{
[Test]
public void DownloadingUnknownCidDoesNotCauseCrash()
{
var node = StartCodex();
var unknownCid = new ContentId("zDvZRwzkzHsok3Z8yMoiXE9EDBFwgr8WygB8s4ddcLzzSwwXAxLZ");
var localFiles = node.LocalFiles().Content;
CollectionAssert.DoesNotContain(localFiles.Select(f => f.Cid), unknownCid);
try
{
node.DownloadContent(unknownCid);
}
catch (Exception ex)
{
if (!ex.Message.StartsWith("Retry 'DownloadFile' timed out"))
{
throw;
}
}
WaitAndCheckNodesStaysAlive(TimeSpan.FromMinutes(2), node);
}
}
}

54
Tests/CodexTests/BasicTests/ThreeClientTest.cs
View File

@ -1,54 +0,0 @@
using CodexPlugin;
using NUnit.Framework;
using Utils;
namespace CodexTests.BasicTests
{
[TestFixture]
public class ThreeClientTest : AutoBootstrapDistTest
{
[Test]
public void ThreeClient()
{
var primary = StartCodex();
var secondary = StartCodex();
var testFile = GenerateTestFile(10.MB());
var contentId = primary.UploadFile(testFile);
var downloadedFile = secondary.DownloadContent(contentId);
testFile.AssertIsEqual(downloadedFile);
}
[Test]
public void DownloadingUnknownCidDoesNotCauseCrash()
{
var node = StartCodex(2).First();
var unknownCid = new ContentId("zDvZRwzkzHsok3Z8yMoiXE9EDBFwgr8WygB8s4ddcLzzSwwXAxLZ");
try
{
node.DownloadContent(unknownCid);
}
catch (Exception ex)
{
if (!ex.Message.StartsWith("Retry 'DownloadFile' timed out"))
{
throw;
}
}
// Check that the node stays alive for at least another 5 minutes.
var start = DateTime.UtcNow;
while ((DateTime.UtcNow - start) < TimeSpan.FromMinutes(5))
{
Thread.Sleep(5000);
var info = node.GetDebugInfo();
Assert.That(!string.IsNullOrEmpty(info.Id));
}
}
}
}

52
Tests/CodexTests/BasicTests/TwoClientTests.cs
View File

@ -1,52 +0,0 @@
using CodexPlugin;
using NUnit.Framework;
using Utils;
namespace CodexTests.BasicTests
{
[TestFixture]
public class TwoClientTests : CodexDistTest
{
[Test]
public void TwoClientTest()
{
var uploader = StartCodex(s => s.WithName("Uploader"));
var downloader = StartCodex(s => s.WithName("Downloader").WithBootstrapNode(uploader));
PerformTwoClientTest(uploader, downloader);
}
[Test]
public void TwoClientsTwoLocationsTest()
{
var locations = Ci.GetKnownLocations();
if (locations.NumberOfLocations < 2)
{
Assert.Inconclusive("Two-locations test requires 2 nodes to be available in the cluster.");
return;
}
var uploader = Ci.StartCodexNode(s => s.WithName("Uploader").At(locations.Get(0)));
var downloader = Ci.StartCodexNode(s => s.WithName("Downloader").WithBootstrapNode(uploader).At(locations.Get(1)));
PerformTwoClientTest(uploader, downloader);
}
private void PerformTwoClientTest(ICodexNode uploader, ICodexNode downloader)
{
PerformTwoClientTest(uploader, downloader, 10.MB());
}
private void PerformTwoClientTest(ICodexNode uploader, ICodexNode downloader, ByteSize size)
{
var testFile = GenerateTestFile(size);
var contentId = uploader.UploadFile(testFile);
var downloadedFile = downloader.DownloadContent(contentId);
testFile.AssertIsEqual(downloadedFile);
CheckLogForErrors(uploader, downloader);
}
}
}

33
Tests/CodexTests/CodexDistTest.cs
View File

@ -108,6 +108,39 @@ namespace CodexTests
GetBasicNodeStatus(node));
}
public void WaitAndCheckNodesStaysAlive(TimeSpan duration, ICodexNodeGroup nodes)
{
WaitAndCheckNodesStaysAlive(duration, nodes.ToArray());
}
public void WaitAndCheckNodesStaysAlive(TimeSpan duration, params ICodexNode[] nodes)
{
var start = DateTime.UtcNow;
while ((DateTime.UtcNow - start) < duration)
{
Thread.Sleep(5000);
foreach (var node in nodes)
{
var info = node.GetDebugInfo();
Assert.That(!string.IsNullOrEmpty(info.Id));
}
}
}
public void AssertNodesContainFile(ContentId cid, ICodexNodeGroup nodes)
{
AssertNodesContainFile(cid, nodes.ToArray());
}
public void AssertNodesContainFile(ContentId cid, params ICodexNode[] nodes)
{
foreach (var node in nodes)
{
var localDatasets = node.LocalFiles();
CollectionAssert.Contains(localDatasets.Content.Select(c => c.Cid), cid);
}
}
private string GetBasicNodeStatus(ICodexNode node)
{
return JsonConvert.SerializeObject(node.GetDebugInfo(), Formatting.Indented) + Environment.NewLine +

2
Tests/CodexTests/DownloadConnectivityTests/SwarmTests.cs → Tests/CodexTests/DownloadConnectivityTests/DetectBlockRetransmitTest.cs
View File

@ -4,7 +4,7 @@ using Utils;
namespace CodexTests.DownloadConnectivityTests
{
[TestFixture]
public class SwarmTests : AutoBootstrapDistTest
public class DetectBlockRetransmitTest : AutoBootstrapDistTest
{
[Test]
[Combinatorial]