From 8138ef5afdcaca7782f497b54b0b63a4886c62a7 Mon Sep 17 00:00:00 2001 From: Giuliano Mega Date: Fri, 21 Jun 2024 19:50:56 -0300 Subject: [PATCH] Fix verifiable manifest initialization (#839) * fix verifiable manifest initialization * fix linearstrategy, use verifiableStrategy to select blocks for slots * check for both strategies in attribute inheritance test --- codex/indexingstrategy.nim | 6 ++--- codex/manifest/manifest.nim | 7 +++--- codex/node.nim | 8 ++++--- tests/codex/node/testcontracts.nim | 2 +- tests/codex/testindexingstrategy.nim | 8 +++++++ tests/codex/testmanifest.nim | 33 ++++++++++++++++++++++++++++ 6 files changed, 54 insertions(+), 10 deletions(-) diff --git a/codex/indexingstrategy.nim b/codex/indexingstrategy.nim index 27444522..d8eeba58 100644 --- a/codex/indexingstrategy.nim +++ b/codex/indexingstrategy.nim @@ -49,8 +49,8 @@ func getLinearIndicies( self.checkIteration(iteration) let - first = self.firstIndex + iteration * (self.step + 1) - last = min(first + self.step, self.lastIndex) + first = self.firstIndex + iteration * self.step + last = min(first + self.step - 1, self.lastIndex) getIter(first, last, 1) @@ -94,4 +94,4 @@ func init*( firstIndex: firstIndex, lastIndex: lastIndex, iterations: iterations, - step: divUp((lastIndex - firstIndex), iterations)) + step: divUp((lastIndex - firstIndex + 1), iterations)) diff --git a/codex/manifest/manifest.nim b/codex/manifest/manifest.nim index b9d40945..abfd5b4c 100644 --- a/codex/manifest/manifest.nim +++ b/codex/manifest/manifest.nim @@ -231,7 +231,7 @@ func new*( treeCid: Cid, datasetSize: NBytes, ecK, ecM: int, - strategy: StrategyType): Manifest = + strategy = SteppedStrategy): Manifest = ## Create an erasure protected dataset from an ## unprotected one ## @@ -277,7 +277,7 @@ func new*( ecM: int, originalTreeCid: Cid, originalDatasetSize: NBytes, - strategy: StrategyType): Manifest = + strategy = SteppedStrategy): Manifest = Manifest( treeCid: treeCid, @@ -299,7 +299,7 @@ func new*( verifyRoot: Cid, slotRoots: openArray[Cid], cellSize = DefaultCellSize, - strategy = SteppedStrategy): ?!Manifest = + strategy = LinearStrategy): ?!Manifest = ## Create a verifiable dataset from an ## protected one ## @@ -324,6 +324,7 @@ func new*( ecM: manifest.ecM, originalTreeCid: manifest.treeCid, originalDatasetSize: manifest.originalDatasetSize, + protectedStrategy: manifest.protectedStrategy, verifiable: true, verifyRoot: verifyRoot, slotRoots: @slotRoots, diff --git a/codex/node.nim b/codex/node.nim index dd76a717..e5156a69 100644 --- a/codex/node.nim +++ b/codex/node.nim @@ -534,7 +534,9 @@ proc onStore( trace "Unable to fetch manifest for cid", cid, err = err.msg return failure(err) - without builder =? Poseidon2Builder.new(self.networkStore, manifest), err: + without builder =? Poseidon2Builder.new( + self.networkStore, manifest, manifest.verifiableStrategy + ), err: trace "Unable to create slots builder", err = err.msg return failure(err) @@ -559,8 +561,8 @@ proc onStore( return success() - without indexer =? manifest.protectedStrategy.init( - 0, manifest.numSlotBlocks() - 1, manifest.numSlots).catch, err: + without indexer =? manifest.verifiableStrategy.init( + 0, manifest.blocksCount - 1, manifest.numSlots).catch, err: trace "Unable to create indexing strategy from protected manifest", err = err.msg return failure(err) diff --git a/tests/codex/node/testcontracts.nim b/tests/codex/node/testcontracts.nim index 362a3ae8..49557f2c 100644 --- a/tests/codex/node/testcontracts.nim +++ b/tests/codex/node/testcontracts.nim @@ -134,7 +134,7 @@ asyncchecksuite "Test Node - Host contracts": return success() (await onStore(request, 1.u256, onBlocks)).tryGet() - check fetchedBytes == 262144 + check fetchedBytes == 12 * DefaultBlockSize.uint let indexer = verifiable.protectedStrategy.init( 0, verifiable.numSlotBlocks() - 1, verifiable.numSlots) diff --git a/tests/codex/testindexingstrategy.nim b/tests/codex/testindexingstrategy.nim index 8ca428e7..7a30259f 100644 --- a/tests/codex/testindexingstrategy.nim +++ b/tests/codex/testindexingstrategy.nim @@ -58,6 +58,14 @@ suite "Indexing strategies": expect IndexingWrongIterationsError: discard LinearStrategy.init(0, 10, 0) + test "should split elements evenly when possible": + let + l = LinearStrategy.init(0, 11, 3) + check: + toSeq(l.getIndicies(0)) == @[0, 1, 2, 3].mapIt(it) + toSeq(l.getIndicies(1)) == @[4, 5, 6, 7].mapIt(it) + toSeq(l.getIndicies(2)) == @[8, 9, 10, 11].mapIt(it) + test "linear - oob": expect IndexingError: discard linear.getIndicies(3) diff --git a/tests/codex/testmanifest.nim b/tests/codex/testmanifest.nim index 188cc957..3393fa08 100644 --- a/tests/codex/testmanifest.nim +++ b/tests/codex/testmanifest.nim @@ -74,3 +74,36 @@ checksuite "Manifest": test "Should encode/decode to/from verifiable manifest": check: encodeDecode(verifiableManifest) == verifiableManifest + + +suite "Manifest - Attribute Inheritance": + proc makeProtectedManifest(strategy: StrategyType): Manifest = + Manifest.new( + manifest = Manifest.new( + treeCid = Cid.example, + blockSize = 1.MiBs, + datasetSize = 100.MiBs, + ), + treeCid = Cid.example, + datasetSize = 200.MiBs, + ecK = 1, + ecM = 1, + strategy = strategy + ) + + test "Should preserve interleaving strategy for protected manifest in verifiable manifest": + var verifiable = Manifest.new( + manifest = makeProtectedManifest(SteppedStrategy), + verifyRoot = Cid.example, + slotRoots = @[Cid.example, Cid.example] + ).tryGet() + + check verifiable.protectedStrategy == SteppedStrategy + + verifiable = Manifest.new( + manifest = makeProtectedManifest(LinearStrategy), + verifyRoot = Cid.example, + slotRoots = @[Cid.example, Cid.example] + ).tryGet() + + check verifiable.protectedStrategy == LinearStrategy