diff --git a/Tests/ExperimentalTests/DownloadConnectivityTests/DatalayerReliabilityTests.cs b/Tests/ExperimentalTests/DownloadConnectivityTests/DatalayerReliabilityTests.cs new file mode 100644 index 0000000..6c61996 --- /dev/null +++ b/Tests/ExperimentalTests/DownloadConnectivityTests/DatalayerReliabilityTests.cs @@ -0,0 +1,150 @@ +using CodexPlugin; +using CodexTests; +using NUnit.Framework; +using Utils; + +namespace ExperimentalTests.DownloadConnectivityTests +{ + /// + /// https://hackmd.io/rwPtPJ7KTw6cGjhN0zNYig + /// + [TestFixture] + public class DatalayerReliabilityTests : AutoBootstrapDistTest + { + [Test] + [Combinatorial] + public void SingleSetTest( + [Values(1, 10, 100, 1000)] int fileSizeMb, + [Values(5, 10, 20, 30)] int numDownloaders + ) + { + var file = GenerateTestFile(fileSizeMb.MB()); + var uploaders = StartCodex(n => n.WithName("uploader")); + var downloaders = StartCodex(numDownloaders, n => n.WithName("downloader")); + + var cid = uploaders.UploadFile(file); + + var downloadTasks = new List(); + foreach (var dl in downloaders) + { + downloadTasks.Add(Task.Run(() => + { + dl.DownloadContent(cid); + })); + } + + Task.WaitAll(downloadTasks.ToArray()); + + Assert.That(downloadTasks.All(t => !t.IsFaulted)); + } + + public class TransferPlan + { + public ICodexNode Uploader { get; set; } = null!; + public ContentId Cid { get; set; } = null!; + public List Downloaders { get; } = new List(); + } + + public class DownloaderPlan + { + public ICodexNode Node { get; set; } = null!; + public List TransferPlans { get; } = new List(); + } + + public class AvailableDownloaders + { + private readonly List all = new List(); + private readonly List available = new List(); + private readonly int maxUsagePerDownloader; + private readonly int numDownloadersPerPlan; + + public AvailableDownloaders(int maxUsagePerDownloader, int numDownloadersPerPlan) + { + this.maxUsagePerDownloader = maxUsagePerDownloader; + this.numDownloadersPerPlan = numDownloadersPerPlan; + } + + public void Assign(TransferPlan plan) + { + while (plan.Downloaders.Count < numDownloadersPerPlan) + { + var open = available.Where(a => !plan.Downloaders.Contains(a)).ToArray(); + if (!open.Any()) + { + var dl = new DownloaderPlan(); + all.Add(dl); + available.Add(dl); + } + else + { + var dl = RandomUtils.GetOneRandom(open); + dl.TransferPlans.Add(plan); + plan.Downloaders.Add(dl); + + if (dl.TransferPlans.Count == maxUsagePerDownloader) available.Remove(dl); + } + } + } + + public DownloaderPlan[] GetAll() + { + return all.ToArray(); + } + } + + [Test] + [Combinatorial] + public void MultiSetTest( + [Values(1, 3, 5, 10)] int numDatasets, + [Values(1, 10, 100, 1000)] int fileSizeMb, + [Values(5, 10, 20, 30)] int numDownloadersPerDataset, + [Values(3, 5)] int maxDatasetsPerDownloader + ) + { + var plans = new List(); + var uploaders = StartCodex(numDatasets, n => n.WithName("uploader")); + foreach (var n in uploaders) + { + plans.Add(new TransferPlan + { + Uploader = n, + Cid = n.UploadFile(GenerateTestFile(fileSizeMb.MB())) + }); + } + + Assert.That(plans.Count, Is.GreaterThan(0)); + + var available = new AvailableDownloaders(maxDatasetsPerDownloader, numDownloadersPerDataset); + foreach (var plan in plans) + { + available.Assign(plan); + } + var allDownloaderPlans = available.GetAll(); + Assert.That(allDownloaderPlans.Length, Is.LessThan(100)); + Log($"Using {allDownloaderPlans.Length} downloaders..."); + var nodes = StartCodex(allDownloaderPlans.Length, n => n.WithName("downloader")); + for (var i = 0; i < allDownloaderPlans.Length; i++) + { + allDownloaderPlans[i].Node = nodes[i]; + } + + var downloadTasks = new List(); + foreach (var dlPlan in allDownloaderPlans) + { + downloadTasks.Add(Task.Run(() => + { + var tf = dlPlan.TransferPlans.ToList(); + while (tf.Count > 0) + { + var t = tf.PickOneRandom(); + dlPlan.Node.DownloadContent(t.Cid); + } + })); + } + + Task.WaitAll(downloadTasks.ToArray()); + + Assert.That(downloadTasks.All(t => !t.IsFaulted)); + } + } +}