From aa4576c97de727593faf69a609c04d425cc92ab6 Mon Sep 17 00:00:00 2001 From: ThatBen Date: Fri, 6 Jun 2025 08:23:35 +0200 Subject: [PATCH 1/6] Fixes nullref in marketplace config extension --- .../Marketplace/Customizations.cs | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/ProjectPlugins/CodexContractsPlugin/Marketplace/Customizations.cs b/ProjectPlugins/CodexContractsPlugin/Marketplace/Customizations.cs index 1ac64325..638cb9a7 100644 --- a/ProjectPlugins/CodexContractsPlugin/Marketplace/Customizations.cs +++ b/ProjectPlugins/CodexContractsPlugin/Marketplace/Customizations.cs @@ -96,8 +96,23 @@ namespace CodexContractsPlugin.Marketplace public partial class MarketplaceConfig : IMarketplaceConfigInput { - public int MaxNumberOfSlashes => this.Collateral.MaxNumberOfSlashes; - public TimeSpan PeriodDuration => TimeSpan.FromSeconds(this.Proofs.Period); + public int MaxNumberOfSlashes + { + get + { + if (Collateral == null) return -1; + return Collateral.MaxNumberOfSlashes; + } + } + + public TimeSpan PeriodDuration + { + get + { + if (Proofs == null) return TimeSpan.MinValue; + return TimeSpan.FromSeconds(this.Proofs.Period); + } + } } } #pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable. From 0e4a6774ef71554001c34ddc1ae5dd4742aba4d6 Mon Sep 17 00:00:00 2001 From: ThatBen Date: Fri, 6 Jun 2025 09:49:55 +0200 Subject: [PATCH 2/6] Convenient rerun attribute --- Tests/CodexReleaseTests/MarketTests/FailTest.cs | 2 +- .../CodexReleaseTests/MarketTests/FinishTest.cs | 2 +- .../CodexReleaseTests/MarketTests/RepairTest.cs | 2 +- Tests/CodexReleaseTests/MarketTests/StartTest.cs | 2 +- Tests/CodexReleaseTests/RerunAttribute.cs | 16 ++++++++++++++++ 5 files changed, 20 insertions(+), 4 deletions(-) create mode 100644 Tests/CodexReleaseTests/RerunAttribute.cs diff --git a/Tests/CodexReleaseTests/MarketTests/FailTest.cs b/Tests/CodexReleaseTests/MarketTests/FailTest.cs index cbd39600..dc278bcd 100644 --- a/Tests/CodexReleaseTests/MarketTests/FailTest.cs +++ b/Tests/CodexReleaseTests/MarketTests/FailTest.cs @@ -21,7 +21,7 @@ namespace CodexReleaseTests.MarketTests [Test] [Combinatorial] public void Fail( - [Values([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16])] int rerun + [Rerun] int rerun ) { var hosts = StartHosts(); diff --git a/Tests/CodexReleaseTests/MarketTests/FinishTest.cs b/Tests/CodexReleaseTests/MarketTests/FinishTest.cs index 6f97773b..8ab3ea4b 100644 --- a/Tests/CodexReleaseTests/MarketTests/FinishTest.cs +++ b/Tests/CodexReleaseTests/MarketTests/FinishTest.cs @@ -27,7 +27,7 @@ namespace CodexReleaseTests.MarketTests [Test] [Combinatorial] public void Finish( - [Values([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16])] int rerun + [Rerun] int rerun ) { var hosts = StartHosts(); diff --git a/Tests/CodexReleaseTests/MarketTests/RepairTest.cs b/Tests/CodexReleaseTests/MarketTests/RepairTest.cs index c515f639..90f87f77 100644 --- a/Tests/CodexReleaseTests/MarketTests/RepairTest.cs +++ b/Tests/CodexReleaseTests/MarketTests/RepairTest.cs @@ -34,7 +34,7 @@ namespace CodexReleaseTests.MarketTests [Test] [Combinatorial] public void RollingRepairSingleFailure( - [Values([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16])] int rerun, + [Rerun] int rerun, [Values(10)] int numFailures) { var hosts = StartHosts().ToList(); diff --git a/Tests/CodexReleaseTests/MarketTests/StartTest.cs b/Tests/CodexReleaseTests/MarketTests/StartTest.cs index c64aef2a..18008138 100644 --- a/Tests/CodexReleaseTests/MarketTests/StartTest.cs +++ b/Tests/CodexReleaseTests/MarketTests/StartTest.cs @@ -23,7 +23,7 @@ namespace CodexReleaseTests.MarketTests [Test] [Combinatorial] public void Start( - [Values([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16])] int rerun + [Rerun] int rerun ) { var hosts = StartHosts(); diff --git a/Tests/CodexReleaseTests/RerunAttribute.cs b/Tests/CodexReleaseTests/RerunAttribute.cs new file mode 100644 index 00000000..6af13d4f --- /dev/null +++ b/Tests/CodexReleaseTests/RerunAttribute.cs @@ -0,0 +1,16 @@ +using NUnit.Framework; + +namespace CodexReleaseTests +{ + public class RerunAttribute : ValuesAttribute + { + private const int NumberOfReRuns = 1; + + public RerunAttribute() + { + var list = new List(); + for (var i = 0; i < NumberOfReRuns; i++) list.Add(i); + data = list.ToArray(); + } + } +} From a0cc09d9efd879f75c7bc80fe48bc33ed67e9421 Mon Sep 17 00:00:00 2001 From: ThatBen Date: Fri, 6 Jun 2025 10:47:54 +0200 Subject: [PATCH 3/6] Adds test where each host must fill two slots. --- .../MarketTests/FinishTest.cs | 2 +- .../MarketTests/MaxCapacityTest.cs | 72 +++++++++++++++++++ .../MarketTests/SequentialContracts.cs | 2 +- 3 files changed, 74 insertions(+), 2 deletions(-) create mode 100644 Tests/CodexReleaseTests/MarketTests/MaxCapacityTest.cs diff --git a/Tests/CodexReleaseTests/MarketTests/FinishTest.cs b/Tests/CodexReleaseTests/MarketTests/FinishTest.cs index 8ab3ea4b..6e591929 100644 --- a/Tests/CodexReleaseTests/MarketTests/FinishTest.cs +++ b/Tests/CodexReleaseTests/MarketTests/FinishTest.cs @@ -22,7 +22,7 @@ namespace CodexReleaseTests.MarketTests protected override int NumberOfHosts => hosts; protected override int NumberOfClients => 1; protected override ByteSize HostAvailabilitySize => purchaseParams.SlotSize.Multiply(5.1); - protected override TimeSpan HostAvailabilityMaxDuration => Get8TimesConfiguredPeriodDuration() * 12; + protected override TimeSpan HostAvailabilityMaxDuration => GetContractDuration() * 2; [Test] [Combinatorial] diff --git a/Tests/CodexReleaseTests/MarketTests/MaxCapacityTest.cs b/Tests/CodexReleaseTests/MarketTests/MaxCapacityTest.cs new file mode 100644 index 00000000..cd7306d6 --- /dev/null +++ b/Tests/CodexReleaseTests/MarketTests/MaxCapacityTest.cs @@ -0,0 +1,72 @@ +using CodexClient; +using CodexReleaseTests.Utils; +using NUnit.Framework; +using Utils; + +namespace CodexReleaseTests.MarketTests +{ + public class MaxCapacityTest : MarketplaceAutoBootstrapDistTest + { + private readonly TestToken pricePerBytePerSecond = 10.TstWei(); + private readonly PurchaseParams purchaseParams = new PurchaseParams( + nodes: 10, + tolerance: 5, + uploadFilesize: 10.MB() + ); + + protected override int NumberOfHosts => purchaseParams.Nodes / 2; + protected override int NumberOfClients => 1; + protected override ByteSize HostAvailabilitySize => purchaseParams.SlotSize.Multiply(2.1); + protected override TimeSpan HostAvailabilityMaxDuration => GetContractDuration() * 2; + + [Test] + [Combinatorial] + public void TwoSlotsEach( + [Rerun] int rerun + ) + { + var hosts = StartHosts(); + var client = StartClients().Single(); + AssertHostAvailabilitiesAreEmpty(hosts); + + var request = CreateStorageRequest(client); + + request.WaitForStorageContractSubmitted(); + AssertContractIsOnChain(request); + + WaitForContractStarted(request); + AssertContractSlotsAreFilledByHosts(request, hosts); + } + + private IStoragePurchaseContract CreateStorageRequest(ICodexNode client) + { + var cid = client.UploadFile(GenerateTestFile(purchaseParams.UploadFilesize)); + var config = GetContracts().Deployment.Config; + return client.Marketplace.RequestStorage(new StoragePurchaseRequest(cid) + { + Duration = GetContractDuration(), + Expiry = GetContractExpiry(), + MinRequiredNumberOfNodes = (uint)purchaseParams.Nodes, + NodeFailureTolerance = (uint)purchaseParams.Tolerance, + PricePerBytePerSecond = pricePerBytePerSecond, + ProofProbability = 20, + CollateralPerByte = 100.TstWei() + }); + } + + private TimeSpan GetContractExpiry() + { + return GetContractDuration() / 2; + } + + private TimeSpan GetContractDuration() + { + return Get8TimesConfiguredPeriodDuration(); + } + + private TimeSpan Get8TimesConfiguredPeriodDuration() + { + return GetPeriodDuration() * 8.0; + } + } +} diff --git a/Tests/CodexReleaseTests/MarketTests/SequentialContracts.cs b/Tests/CodexReleaseTests/MarketTests/SequentialContracts.cs index 383093cf..199ff470 100644 --- a/Tests/CodexReleaseTests/MarketTests/SequentialContracts.cs +++ b/Tests/CodexReleaseTests/MarketTests/SequentialContracts.cs @@ -21,7 +21,7 @@ namespace CodexReleaseTests.MarketTests protected override int NumberOfHosts => hosts; protected override int NumberOfClients => 6; protected override ByteSize HostAvailabilitySize => purchaseParams.SlotSize.Multiply(100.0); - protected override TimeSpan HostAvailabilityMaxDuration => Get8TimesConfiguredPeriodDuration() * 12; + protected override TimeSpan HostAvailabilityMaxDuration => GetContractDuration() * 2; private readonly TestToken pricePerBytePerSecond = 10.TstWei(); [Test] From 668b812b80a44f2b66aa5f37f5f1222b621ee182 Mon Sep 17 00:00:00 2001 From: ThatBen Date: Fri, 6 Jun 2025 10:48:28 +0200 Subject: [PATCH 4/6] 8 reruns for all marketplace tests --- Tests/CodexReleaseTests/RerunAttribute.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/CodexReleaseTests/RerunAttribute.cs b/Tests/CodexReleaseTests/RerunAttribute.cs index 6af13d4f..d30b010d 100644 --- a/Tests/CodexReleaseTests/RerunAttribute.cs +++ b/Tests/CodexReleaseTests/RerunAttribute.cs @@ -4,7 +4,7 @@ namespace CodexReleaseTests { public class RerunAttribute : ValuesAttribute { - private const int NumberOfReRuns = 1; + private const int NumberOfReRuns = 8; public RerunAttribute() { From 6b2ca648e4ac3713a835aa6c7531d68ac89e279b Mon Sep 17 00:00:00 2001 From: ThatBen Date: Mon, 9 Jun 2025 09:52:12 +0200 Subject: [PATCH 5/6] cleanup --- Tools/TraceContract/Output.cs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Tools/TraceContract/Output.cs b/Tools/TraceContract/Output.cs index be89d3f6..6365037b 100644 --- a/Tools/TraceContract/Output.cs +++ b/Tools/TraceContract/Output.cs @@ -86,11 +86,6 @@ namespace TraceContract Add(requestEvent.Block.Utc, $"Slot reservations full. Index: {slotIndex}"); } - public void LogReserveSlotCalls(ReserveSlotFunction[] reserveSlotFunctions) - { - foreach (var call in reserveSlotFunctions) LogReserveSlotCall(call); - } - public void WriteContractEvents() { var sorted = entries.OrderBy(e => e.Utc).ToArray(); From 0c74acab1187e0cc00655cde0576af79e95f119d Mon Sep 17 00:00:00 2001 From: ThatBen Date: Mon, 9 Jun 2025 10:14:30 +0200 Subject: [PATCH 6/6] Updates dataset size name --- ProjectPlugins/CodexClient/CodexTypes.cs | 2 +- ProjectPlugins/CodexClient/Mapper.cs | 2 +- Tests/CodexReleaseTests/DataTests/LocalFilesTest.cs | 4 ++-- Tests/CodexReleaseTests/DataTests/ManifestOnlyDownloadTest.cs | 2 +- Tests/CodexReleaseTests/DataTests/StreamlessDownloadTest.cs | 2 +- Tools/BiblioTech/CodexChecking/CodexTwoWayChecker.cs | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/ProjectPlugins/CodexClient/CodexTypes.cs b/ProjectPlugins/CodexClient/CodexTypes.cs index f9a0d9d2..12e7c92c 100644 --- a/ProjectPlugins/CodexClient/CodexTypes.cs +++ b/ProjectPlugins/CodexClient/CodexTypes.cs @@ -66,7 +66,7 @@ namespace CodexClient public class Manifest { public string RootHash { get; set; } = string.Empty; - public ByteSize OriginalBytes { get; set; } = ByteSize.Zero; + public ByteSize DatasetSize { get; set; } = ByteSize.Zero; public ByteSize BlockSize { get; set; } = ByteSize.Zero; public bool Protected { get; set; } } diff --git a/ProjectPlugins/CodexClient/Mapper.cs b/ProjectPlugins/CodexClient/Mapper.cs index 35f872ef..69d586f8 100644 --- a/ProjectPlugins/CodexClient/Mapper.cs +++ b/ProjectPlugins/CodexClient/Mapper.cs @@ -208,7 +208,7 @@ namespace CodexClient return new Manifest { BlockSize = new ByteSize(Convert.ToInt64(manifest.BlockSize)), - OriginalBytes = new ByteSize(Convert.ToInt64(manifest.DatasetSize)), + DatasetSize = new ByteSize(Convert.ToInt64(manifest.DatasetSize)), RootHash = manifest.TreeCid, Protected = manifest.Protected }; diff --git a/Tests/CodexReleaseTests/DataTests/LocalFilesTest.cs b/Tests/CodexReleaseTests/DataTests/LocalFilesTest.cs index 4069aa49..8a69c834 100644 --- a/Tests/CodexReleaseTests/DataTests/LocalFilesTest.cs +++ b/Tests/CodexReleaseTests/DataTests/LocalFilesTest.cs @@ -33,9 +33,9 @@ namespace CodexReleaseTests.DataTests var local2 = localFiles.Content.Single(f => f.Cid == cid2); Assert.That(local1.Manifest.Protected, Is.False); - Assert.That(local1.Manifest.OriginalBytes, Is.EqualTo(size1)); + Assert.That(local1.Manifest.DatasetSize, Is.EqualTo(size1)); Assert.That(local2.Manifest.Protected, Is.False); - Assert.That(local2.Manifest.OriginalBytes, Is.EqualTo(size2)); + Assert.That(local2.Manifest.DatasetSize, Is.EqualTo(size2)); } } } diff --git a/Tests/CodexReleaseTests/DataTests/ManifestOnlyDownloadTest.cs b/Tests/CodexReleaseTests/DataTests/ManifestOnlyDownloadTest.cs index c55b3f8f..f1fcc84c 100644 --- a/Tests/CodexReleaseTests/DataTests/ManifestOnlyDownloadTest.cs +++ b/Tests/CodexReleaseTests/DataTests/ManifestOnlyDownloadTest.cs @@ -27,7 +27,7 @@ namespace CodexReleaseTests.DataTests Assert.That(spaceDiff, Is.LessThan(64.KB().SizeInBytes)); Assert.That(localDataset.Cid, Is.EqualTo(cid)); - Assert.That(localDataset.Manifest.OriginalBytes.SizeInBytes, Is.EqualTo(file.GetFilesize().SizeInBytes)); + Assert.That(localDataset.Manifest.DatasetSize.SizeInBytes, Is.EqualTo(file.GetFilesize().SizeInBytes)); } } } diff --git a/Tests/CodexReleaseTests/DataTests/StreamlessDownloadTest.cs b/Tests/CodexReleaseTests/DataTests/StreamlessDownloadTest.cs index d53d16dc..93b89f19 100644 --- a/Tests/CodexReleaseTests/DataTests/StreamlessDownloadTest.cs +++ b/Tests/CodexReleaseTests/DataTests/StreamlessDownloadTest.cs @@ -22,7 +22,7 @@ namespace CodexReleaseTests.DataTests var localDataset = downloader.DownloadStreamlessWait(cid, size); Assert.That(localDataset.Cid, Is.EqualTo(cid)); - Assert.That(localDataset.Manifest.OriginalBytes.SizeInBytes, Is.EqualTo(file.GetFilesize().SizeInBytes)); + Assert.That(localDataset.Manifest.DatasetSize.SizeInBytes, Is.EqualTo(file.GetFilesize().SizeInBytes)); // Stop the uploader node and verify that the downloader has the data. uploader.Stop(waitTillStopped: true); diff --git a/Tools/BiblioTech/CodexChecking/CodexTwoWayChecker.cs b/Tools/BiblioTech/CodexChecking/CodexTwoWayChecker.cs index ac83ccd9..1dc6b681 100644 --- a/Tools/BiblioTech/CodexChecking/CodexTwoWayChecker.cs +++ b/Tools/BiblioTech/CodexChecking/CodexTwoWayChecker.cs @@ -163,7 +163,7 @@ namespace BiblioTech.CodexChecking private bool IsManifestLengthCompatible(ICheckResponseHandler handler, TransferCheck check, Manifest manifest) { var dataLength = check.UniqueData.Length; - var manifestLength = manifest.OriginalBytes.SizeInBytes; + var manifestLength = manifest.DatasetSize.SizeInBytes; Log($"Checking manifest length: dataLength={dataLength},manifestLength={manifestLength}");