diff --git a/codex/indexingstrategy.nim b/codex/indexingstrategy.nim index 27444522..d8eeba58 100644 --- a/codex/indexingstrategy.nim +++ b/codex/indexingstrategy.nim @@ -49,8 +49,8 @@ func getLinearIndicies( self.checkIteration(iteration) let - first = self.firstIndex + iteration * (self.step + 1) - last = min(first + self.step, self.lastIndex) + first = self.firstIndex + iteration * self.step + last = min(first + self.step - 1, self.lastIndex) getIter(first, last, 1) @@ -94,4 +94,4 @@ func init*( firstIndex: firstIndex, lastIndex: lastIndex, iterations: iterations, - step: divUp((lastIndex - firstIndex), iterations)) + step: divUp((lastIndex - firstIndex + 1), iterations)) diff --git a/codex/manifest/manifest.nim b/codex/manifest/manifest.nim index 486a8fc3..abfd5b4c 100644 --- a/codex/manifest/manifest.nim +++ b/codex/manifest/manifest.nim @@ -135,13 +135,6 @@ func isManifest*(mc: MultiCodec): ?!bool = # Various sizes and verification ############################################################ -func bytes*(self: Manifest, pad = true): NBytes = - ## Compute how many bytes corresponding StoreStream(Manifest, pad) will return - if pad or self.protected: - self.blocksCount.NBytes * self.blockSize - else: - self.datasetSize - func rounded*(self: Manifest): int = ## Number of data blocks in *protected* manifest including padding at the end roundUp(self.originalBlocksCount, self.ecK) @@ -238,7 +231,7 @@ func new*( treeCid: Cid, datasetSize: NBytes, ecK, ecM: int, - strategy: StrategyType): Manifest = + strategy = SteppedStrategy): Manifest = ## Create an erasure protected dataset from an ## unprotected one ## @@ -284,7 +277,7 @@ func new*( ecM: int, originalTreeCid: Cid, originalDatasetSize: NBytes, - strategy: StrategyType): Manifest = + strategy = SteppedStrategy): Manifest = Manifest( treeCid: treeCid, @@ -306,7 +299,7 @@ func new*( verifyRoot: Cid, slotRoots: openArray[Cid], cellSize = DefaultCellSize, - strategy = SteppedStrategy): ?!Manifest = + strategy = LinearStrategy): ?!Manifest = ## Create a verifiable dataset from an ## protected one ## @@ -331,6 +324,7 @@ func new*( ecM: manifest.ecM, originalTreeCid: manifest.treeCid, originalDatasetSize: manifest.originalDatasetSize, + protectedStrategy: manifest.protectedStrategy, verifiable: true, verifyRoot: verifyRoot, slotRoots: @slotRoots, diff --git a/codex/node.nim b/codex/node.nim index b206ea64..34578698 100644 --- a/codex/node.nim +++ b/codex/node.nim @@ -534,7 +534,9 @@ proc onStore( trace "Unable to fetch manifest for cid", cid, err = err.msg return failure(err) - without builder =? Poseidon2Builder.new(self.networkStore, manifest), err: + without builder =? Poseidon2Builder.new( + self.networkStore, manifest, manifest.verifiableStrategy + ), err: trace "Unable to create slots builder", err = err.msg return failure(err) @@ -559,8 +561,8 @@ proc onStore( return success() - without indexer =? manifest.protectedStrategy.init( - 0, manifest.numSlotBlocks() - 1, manifest.numSlots).catch, err: + without indexer =? manifest.verifiableStrategy.init( + 0, manifest.blocksCount - 1, manifest.numSlots).catch, err: trace "Unable to create indexing strategy from protected manifest", err = err.msg return failure(err) diff --git a/codex/streams/storestream.nim b/codex/streams/storestream.nim index ce89171c..8a3b1a3c 100644 --- a/codex/streams/storestream.nim +++ b/codex/streams/storestream.nim @@ -38,7 +38,6 @@ type StoreStream* = ref object of SeekableStream store*: BlockStore # Store where to lookup block contents manifest*: Manifest # List of block CIDs - pad*: bool # Pad last block to manifest.blockSize? method initStream*(s: StoreStream) = if s.objName.len == 0: @@ -57,13 +56,15 @@ proc new*( result = StoreStream( store: store, manifest: manifest, - pad: pad, offset: 0) result.initStream() method `size`*(self: StoreStream): int = - bytes(self.manifest, self.pad).int + ## The size of a StoreStream is the size of the original dataset, without + ## padding or parity blocks. + let m = self.manifest + (if m.protected: m.originalDatasetSize else: m.datasetSize).int proc `size=`*(self: StoreStream, size: int) {.error: "Setting the size is forbidden".} = diff --git a/tests/codex/node/testcontracts.nim b/tests/codex/node/testcontracts.nim index 362a3ae8..49557f2c 100644 --- a/tests/codex/node/testcontracts.nim +++ b/tests/codex/node/testcontracts.nim @@ -134,7 +134,7 @@ asyncchecksuite "Test Node - Host contracts": return success() (await onStore(request, 1.u256, onBlocks)).tryGet() - check fetchedBytes == 262144 + check fetchedBytes == 12 * DefaultBlockSize.uint let indexer = verifiable.protectedStrategy.init( 0, verifiable.numSlotBlocks() - 1, verifiable.numSlots) diff --git a/tests/codex/testindexingstrategy.nim b/tests/codex/testindexingstrategy.nim index 8ca428e7..7a30259f 100644 --- a/tests/codex/testindexingstrategy.nim +++ b/tests/codex/testindexingstrategy.nim @@ -58,6 +58,14 @@ suite "Indexing strategies": expect IndexingWrongIterationsError: discard LinearStrategy.init(0, 10, 0) + test "should split elements evenly when possible": + let + l = LinearStrategy.init(0, 11, 3) + check: + toSeq(l.getIndicies(0)) == @[0, 1, 2, 3].mapIt(it) + toSeq(l.getIndicies(1)) == @[4, 5, 6, 7].mapIt(it) + toSeq(l.getIndicies(2)) == @[8, 9, 10, 11].mapIt(it) + test "linear - oob": expect IndexingError: discard linear.getIndicies(3) diff --git a/tests/codex/testmanifest.nim b/tests/codex/testmanifest.nim index 188cc957..3393fa08 100644 --- a/tests/codex/testmanifest.nim +++ b/tests/codex/testmanifest.nim @@ -74,3 +74,36 @@ checksuite "Manifest": test "Should encode/decode to/from verifiable manifest": check: encodeDecode(verifiableManifest) == verifiableManifest + + +suite "Manifest - Attribute Inheritance": + proc makeProtectedManifest(strategy: StrategyType): Manifest = + Manifest.new( + manifest = Manifest.new( + treeCid = Cid.example, + blockSize = 1.MiBs, + datasetSize = 100.MiBs, + ), + treeCid = Cid.example, + datasetSize = 200.MiBs, + ecK = 1, + ecM = 1, + strategy = strategy + ) + + test "Should preserve interleaving strategy for protected manifest in verifiable manifest": + var verifiable = Manifest.new( + manifest = makeProtectedManifest(SteppedStrategy), + verifyRoot = Cid.example, + slotRoots = @[Cid.example, Cid.example] + ).tryGet() + + check verifiable.protectedStrategy == SteppedStrategy + + verifiable = Manifest.new( + manifest = makeProtectedManifest(LinearStrategy), + verifyRoot = Cid.example, + slotRoots = @[Cid.example, Cid.example] + ).tryGet() + + check verifiable.protectedStrategy == LinearStrategy diff --git a/tests/codex/teststorestream.nim b/tests/codex/teststorestream.nim index eaf92c1d..b717a8ec 100644 --- a/tests/codex/teststorestream.nim +++ b/tests/codex/teststorestream.nim @@ -1,12 +1,15 @@ import pkg/chronos import pkg/questionable/results -import pkg/codex/streams -import pkg/codex/stores -import pkg/codex/manifest -import pkg/codex/blocktype as bt +import pkg/codex/[ + streams, + stores, + indexingstrategy, + manifest, + blocktype as bt] import ../asynctest +import ./examples import ./helpers asyncchecksuite "StoreStream": @@ -99,3 +102,40 @@ asyncchecksuite "StoreStream": await stream.readExactly(addr buf[0], 15) check sequentialBytes(buf,15,0) + +suite "StoreStream - Size Tests": + + var stream: StoreStream + + teardown: + await stream.close() + + test "Should return dataset size as stream size": + let manifest = Manifest.new( + treeCid = Cid.example, + datasetSize = 80.NBytes, + blockSize = 10.NBytes + ) + + stream = StoreStream.new(CacheStore.new(), manifest) + + check stream.size == 80 + + test "Should not count parity/padding bytes as part of stream size": + let protectedManifest = Manifest.new( + treeCid = Cid.example, + datasetSize = 120.NBytes, # size including parity bytes + blockSize = 10.NBytes, + version = CIDv1, + hcodec = Sha256HashCodec, + codec = BlockCodec, + ecK = 2, + ecM = 1, + originalTreeCid = Cid.example, + originalDatasetSize = 80.NBytes, # size without parity bytes + strategy = StrategyType.SteppedStrategy + ) + + stream = StoreStream.new(CacheStore.new(), protectedManifest) + + check stream.size == 80