From f344facb642a86f357496dc9ad6ace66eb7ec884 Mon Sep 17 00:00:00 2001 From: ThatBen Date: Mon, 2 Jun 2025 13:32:03 +0200 Subject: [PATCH] Applies purchase-paramters type to ensure slot sizes --- Framework/Utils/ByteSize.cs | 15 +++++ .../CodexPlugin/CodexDockerImage.cs | 2 +- .../MarketTests/FinishTest.cs | 15 ++--- .../MarketTests/RepairTest.cs | 33 +++------ .../MarketTests/SequentialContracts.cs | 17 ++--- .../MarketTests/StartTest.cs | 14 ++-- .../CodexReleaseTests/Utils/PurchaseParams.cs | 67 +++++++++++++++++++ 7 files changed, 115 insertions(+), 48 deletions(-) create mode 100644 Tests/CodexReleaseTests/Utils/PurchaseParams.cs diff --git a/Framework/Utils/ByteSize.cs b/Framework/Utils/ByteSize.cs index 9b0f7c8a..b66c952c 100644 --- a/Framework/Utils/ByteSize.cs +++ b/Framework/Utils/ByteSize.cs @@ -25,6 +25,21 @@ return new ByteSize(Convert.ToInt64(result)); } + public int DivUp(ByteSize div) + { + var d = div.SizeInBytes; + var remaining = SizeInBytes; + var result = 0; + while (remaining > d) + { + remaining -= d; + result++; + } + + if (remaining > 0) result++; + return result; + } + public override bool Equals(object? obj) { return obj is ByteSize size && SizeInBytes == size.SizeInBytes; diff --git a/ProjectPlugins/CodexPlugin/CodexDockerImage.cs b/ProjectPlugins/CodexPlugin/CodexDockerImage.cs index 01ffd4d7..57619f0b 100644 --- a/ProjectPlugins/CodexPlugin/CodexDockerImage.cs +++ b/ProjectPlugins/CodexPlugin/CodexDockerImage.cs @@ -2,7 +2,7 @@ { public class CodexDockerImage { - private const string DefaultDockerImage = "codexstorage/nim-codex:sha-28a83db-dist-tests"; + private const string DefaultDockerImage = "codexstorage/nim-codex:0.2.3-dist-tests"; public static string Override { get; set; } = string.Empty; diff --git a/Tests/CodexReleaseTests/MarketTests/FinishTest.cs b/Tests/CodexReleaseTests/MarketTests/FinishTest.cs index 0d10cc93..4f465c7f 100644 --- a/Tests/CodexReleaseTests/MarketTests/FinishTest.cs +++ b/Tests/CodexReleaseTests/MarketTests/FinishTest.cs @@ -12,19 +12,16 @@ namespace CodexReleaseTests.MarketTests public FinishTest(int hosts, int slots, int tolerance) { this.hosts = hosts; - this.slots = slots; - this.tolerance = tolerance; + purchaseParams = new PurchaseParams(slots, tolerance, uploadFilesize: 10.MB()); } - private const int FilesizeMb = 10; private readonly TestToken pricePerBytePerSecond = 10.TstWei(); private readonly int hosts; - private readonly int slots; - private readonly int tolerance; + private readonly PurchaseParams purchaseParams; protected override int NumberOfHosts => hosts; protected override int NumberOfClients => 1; - protected override ByteSize HostAvailabilitySize => (5 * FilesizeMb).MB(); + protected override ByteSize HostAvailabilitySize => purchaseParams.SlotSize.Multiply(5.1); protected override TimeSpan HostAvailabilityMaxDuration => Get8TimesConfiguredPeriodDuration() * 12; [Test] @@ -53,14 +50,14 @@ namespace CodexReleaseTests.MarketTests private IStoragePurchaseContract CreateStorageRequest(ICodexNode client) { - var cid = client.UploadFile(GenerateTestFile(FilesizeMb.MB())); + var cid = client.UploadFile(GenerateTestFile(purchaseParams.UploadFilesize)); var config = GetContracts().Deployment.Config; return client.Marketplace.RequestStorage(new StoragePurchaseRequest(cid) { Duration = GetContractDuration(), Expiry = GetContractExpiry(), - MinRequiredNumberOfNodes = (uint)slots, - NodeFailureTolerance = (uint)tolerance, + MinRequiredNumberOfNodes = (uint)purchaseParams.Nodes, + NodeFailureTolerance = (uint)purchaseParams.Tolerance, PricePerBytePerSecond = pricePerBytePerSecond, ProofProbability = 20, CollateralPerByte = 100.TstWei() diff --git a/Tests/CodexReleaseTests/MarketTests/RepairTest.cs b/Tests/CodexReleaseTests/MarketTests/RepairTest.cs index 2d8fa054..c515f639 100644 --- a/Tests/CodexReleaseTests/MarketTests/RepairTest.cs +++ b/Tests/CodexReleaseTests/MarketTests/RepairTest.cs @@ -11,35 +11,22 @@ namespace CodexReleaseTests.MarketTests { #region Setup - private readonly ByteSize Filesize; - private readonly uint Slots; - private readonly uint Tolerance; - private readonly ByteSize EncodedFilesize; - private readonly ByteSize SlotSize; + private readonly PurchaseParams purchaseParams = new PurchaseParams( + nodes: 4, + tolerance: 2, + uploadFilesize: 32.MB() + ); public RepairTest() { - Filesize = 32.MB(); - Slots = 4; - Tolerance = 2; - - EncodedFilesize = new ByteSize(Filesize.SizeInBytes * (Slots / Tolerance)); - SlotSize = new ByteSize(EncodedFilesize.SizeInBytes / Slots); - Assert.That(IsPowerOfTwo(SlotSize)); - Assert.That(Slots, Is.LessThan(NumberOfHosts)); + Assert.That(purchaseParams.Nodes, Is.LessThan(NumberOfHosts)); } protected override int NumberOfHosts => 5; protected override int NumberOfClients => 1; - protected override ByteSize HostAvailabilitySize => SlotSize.Multiply(1.1); // Each host can hold 1 slot. + protected override ByteSize HostAvailabilitySize => purchaseParams.SlotSize.Multiply(1.1); // Each host can hold 1 slot. protected override TimeSpan HostAvailabilityMaxDuration => TimeSpan.FromDays(5.0); - private static bool IsPowerOfTwo(ByteSize size) - { - var x = size.SizeInBytes; - return (x != 0) && ((x & (x - 1)) == 0); - } - #endregion [Ignore("Test is ready. Waiting for repair implementation. " + @@ -173,14 +160,14 @@ namespace CodexReleaseTests.MarketTests private IStoragePurchaseContract CreateStorageRequest(ICodexNode client) { - var cid = client.UploadFile(GenerateTestFile(Filesize)); + var cid = client.UploadFile(GenerateTestFile(purchaseParams.UploadFilesize)); var config = GetContracts().Deployment.Config; return client.Marketplace.RequestStorage(new StoragePurchaseRequest(cid) { Duration = HostAvailabilityMaxDuration / 2, Expiry = TimeSpan.FromMinutes(10.0), - MinRequiredNumberOfNodes = Slots, - NodeFailureTolerance = Tolerance, + MinRequiredNumberOfNodes = (uint)purchaseParams.Nodes, + NodeFailureTolerance = (uint)purchaseParams.Tolerance, PricePerBytePerSecond = 10.TstWei(), ProofProbability = 1, // One proof every period. Free slot as quickly as possible. CollateralPerByte = 1.TstWei() diff --git a/Tests/CodexReleaseTests/MarketTests/SequentialContracts.cs b/Tests/CodexReleaseTests/MarketTests/SequentialContracts.cs index b10e34ca..f8724f3d 100644 --- a/Tests/CodexReleaseTests/MarketTests/SequentialContracts.cs +++ b/Tests/CodexReleaseTests/MarketTests/SequentialContracts.cs @@ -12,18 +12,15 @@ namespace CodexReleaseTests.MarketTests public SequentialContracts(int hosts, int slots, int tolerance) { this.hosts = hosts; - this.slots = slots; - this.tolerance = tolerance; + purchaseParams = new PurchaseParams(slots, tolerance, 10.MB()); } - private const int FilesizeMb = 10; private readonly int hosts; - private readonly int slots; - private readonly int tolerance; + private readonly PurchaseParams purchaseParams; protected override int NumberOfHosts => hosts; protected override int NumberOfClients => 8; - protected override ByteSize HostAvailabilitySize => (1000 * FilesizeMb).MB(); + protected override ByteSize HostAvailabilitySize => purchaseParams.SlotSize.Multiply(100.0); protected override TimeSpan HostAvailabilityMaxDuration => Get8TimesConfiguredPeriodDuration() * 12; private readonly TestToken pricePerBytePerSecond = 10.TstWei(); @@ -80,14 +77,14 @@ namespace CodexReleaseTests.MarketTests private IStoragePurchaseContract CreateStorageRequest(ICodexNode client) { - var cid = client.UploadFile(GenerateTestFile(FilesizeMb.MB())); + var cid = client.UploadFile(GenerateTestFile(purchaseParams.UploadFilesize)); var config = GetContracts().Deployment.Config; return client.Marketplace.RequestStorage(new StoragePurchaseRequest(cid) { Duration = GetContractDuration(), Expiry = GetContractExpiry(), - MinRequiredNumberOfNodes = (uint)slots, - NodeFailureTolerance = (uint)tolerance, + MinRequiredNumberOfNodes = (uint)purchaseParams.Nodes, + NodeFailureTolerance = (uint)purchaseParams.Tolerance, PricePerBytePerSecond = pricePerBytePerSecond, ProofProbability = 10000, CollateralPerByte = 1.TstWei() @@ -107,7 +104,7 @@ namespace CodexReleaseTests.MarketTests private TimeSpan Get8TimesConfiguredPeriodDuration() { var config = GetContracts().Deployment.Config; - return TimeSpan.FromSeconds(((double)config.Proofs.Period) * 8.0); + return TimeSpan.FromSeconds(config.Proofs.Period * 8.0); } } } diff --git a/Tests/CodexReleaseTests/MarketTests/StartTest.cs b/Tests/CodexReleaseTests/MarketTests/StartTest.cs index 2ba12d6a..c64aef2a 100644 --- a/Tests/CodexReleaseTests/MarketTests/StartTest.cs +++ b/Tests/CodexReleaseTests/MarketTests/StartTest.cs @@ -8,12 +8,16 @@ namespace CodexReleaseTests.MarketTests [TestFixture] public class StartTest : MarketplaceAutoBootstrapDistTest { - private const int FilesizeMb = 10; + private readonly PurchaseParams purchaseParams = new PurchaseParams( + nodes: 3, + tolerance: 1, + uploadFilesize: 10.MB() + ); private readonly TestToken pricePerBytePerSecond = 10.TstWei(); protected override int NumberOfHosts => 5; protected override int NumberOfClients => 1; - protected override ByteSize HostAvailabilitySize => (5 * FilesizeMb).MB(); + protected override ByteSize HostAvailabilitySize => purchaseParams.SlotSize.Multiply(10.0); protected override TimeSpan HostAvailabilityMaxDuration => Get8TimesConfiguredPeriodDuration() * 12; [Test] @@ -36,14 +40,14 @@ namespace CodexReleaseTests.MarketTests private IStoragePurchaseContract CreateStorageRequest(ICodexNode client) { - var cid = client.UploadFile(GenerateTestFile(FilesizeMb.MB())); + var cid = client.UploadFile(GenerateTestFile(purchaseParams.UploadFilesize)); var config = GetContracts().Deployment.Config; return client.Marketplace.RequestStorage(new StoragePurchaseRequest(cid) { Duration = GetContractDuration(), Expiry = GetContractExpiry(), - MinRequiredNumberOfNodes = 3, - NodeFailureTolerance = 1, + MinRequiredNumberOfNodes = (uint)purchaseParams.Nodes, + NodeFailureTolerance = (uint)purchaseParams.Tolerance, PricePerBytePerSecond = pricePerBytePerSecond, ProofProbability = 20, CollateralPerByte = 100.TstWei() diff --git a/Tests/CodexReleaseTests/Utils/PurchaseParams.cs b/Tests/CodexReleaseTests/Utils/PurchaseParams.cs new file mode 100644 index 00000000..483d6f36 --- /dev/null +++ b/Tests/CodexReleaseTests/Utils/PurchaseParams.cs @@ -0,0 +1,67 @@ +using NUnit.Framework; +using Utils; + +namespace CodexReleaseTests.Utils +{ + public class PurchaseParams + { + private readonly ByteSize blockSize = 64.KB(); + + public PurchaseParams(int nodes, int tolerance, ByteSize uploadFilesize) + { + Nodes = nodes; + Tolerance = tolerance; + UploadFilesize = uploadFilesize; + + EncodedDatasetSize = CalculateEncodedDatasetSize(); + SlotSize = CalculateSlotSize(); + + Assert.That(IsPowerOfTwo(SlotSize)); + } + + public int Nodes { get; } + public int Tolerance { get; } + public ByteSize UploadFilesize { get; } + public ByteSize EncodedDatasetSize { get; } + public ByteSize SlotSize { get; } + + private ByteSize CalculateSlotSize() + { + // encoded dataset is divided over the nodes. + // then each slot is rounded up to the nearest power-of-two blocks. + var numBlocks = EncodedDatasetSize.DivUp(blockSize); + var numSlotBlocks = 1 + ((numBlocks - 1) / Nodes); // round-up div. + + // Next power of two: + var numSlotBlocksPow2 = NextPowerOf2(numSlotBlocks); + return new ByteSize(blockSize.SizeInBytes * numSlotBlocksPow2); + } + + private ByteSize CalculateEncodedDatasetSize() + { + var numBlocks = UploadFilesize.DivUp(blockSize); + + var ecK = Nodes - Tolerance; + var ecM = Tolerance; + + // for each K blocks, we generate M parity blocks + var numParityBlocks = (numBlocks / ecK) * ecM; + var totalBlocks = numBlocks + numParityBlocks; + + return new ByteSize(blockSize.SizeInBytes * totalBlocks); + } + + private int NextPowerOf2(int n) + { + n = n - 1; + var lg = Convert.ToInt32(Math.Round(Math.Log2(Convert.ToDouble(n)))); + return 1 << (lg + 1); + } + + private static bool IsPowerOfTwo(ByteSize size) + { + var x = size.SizeInBytes; + return (x != 0) && ((x & (x - 1)) == 0); + } + } +}