Merge branch 'master' into feature/ceremony-files

This commit is contained in:
Ben 2024-06-24 09:57:51 +02:00
commit 305b80aedb
No known key found for this signature in database
GPG Key ID: 541B9D8C9F1426A1
8 changed files with 102 additions and 24 deletions

View File

@ -49,8 +49,8 @@ func getLinearIndicies(
self.checkIteration(iteration) self.checkIteration(iteration)
let let
first = self.firstIndex + iteration * (self.step + 1) first = self.firstIndex + iteration * self.step
last = min(first + self.step, self.lastIndex) last = min(first + self.step - 1, self.lastIndex)
getIter(first, last, 1) getIter(first, last, 1)
@ -94,4 +94,4 @@ func init*(
firstIndex: firstIndex, firstIndex: firstIndex,
lastIndex: lastIndex, lastIndex: lastIndex,
iterations: iterations, iterations: iterations,
step: divUp((lastIndex - firstIndex), iterations)) step: divUp((lastIndex - firstIndex + 1), iterations))

View File

@ -135,13 +135,6 @@ func isManifest*(mc: MultiCodec): ?!bool =
# Various sizes and verification # Various sizes and verification
############################################################ ############################################################
func bytes*(self: Manifest, pad = true): NBytes =
## Compute how many bytes corresponding StoreStream(Manifest, pad) will return
if pad or self.protected:
self.blocksCount.NBytes * self.blockSize
else:
self.datasetSize
func rounded*(self: Manifest): int = func rounded*(self: Manifest): int =
## Number of data blocks in *protected* manifest including padding at the end ## Number of data blocks in *protected* manifest including padding at the end
roundUp(self.originalBlocksCount, self.ecK) roundUp(self.originalBlocksCount, self.ecK)
@ -238,7 +231,7 @@ func new*(
treeCid: Cid, treeCid: Cid,
datasetSize: NBytes, datasetSize: NBytes,
ecK, ecM: int, ecK, ecM: int,
strategy: StrategyType): Manifest = strategy = SteppedStrategy): Manifest =
## Create an erasure protected dataset from an ## Create an erasure protected dataset from an
## unprotected one ## unprotected one
## ##
@ -284,7 +277,7 @@ func new*(
ecM: int, ecM: int,
originalTreeCid: Cid, originalTreeCid: Cid,
originalDatasetSize: NBytes, originalDatasetSize: NBytes,
strategy: StrategyType): Manifest = strategy = SteppedStrategy): Manifest =
Manifest( Manifest(
treeCid: treeCid, treeCid: treeCid,
@ -306,7 +299,7 @@ func new*(
verifyRoot: Cid, verifyRoot: Cid,
slotRoots: openArray[Cid], slotRoots: openArray[Cid],
cellSize = DefaultCellSize, cellSize = DefaultCellSize,
strategy = SteppedStrategy): ?!Manifest = strategy = LinearStrategy): ?!Manifest =
## Create a verifiable dataset from an ## Create a verifiable dataset from an
## protected one ## protected one
## ##
@ -331,6 +324,7 @@ func new*(
ecM: manifest.ecM, ecM: manifest.ecM,
originalTreeCid: manifest.treeCid, originalTreeCid: manifest.treeCid,
originalDatasetSize: manifest.originalDatasetSize, originalDatasetSize: manifest.originalDatasetSize,
protectedStrategy: manifest.protectedStrategy,
verifiable: true, verifiable: true,
verifyRoot: verifyRoot, verifyRoot: verifyRoot,
slotRoots: @slotRoots, slotRoots: @slotRoots,

View File

@ -534,7 +534,9 @@ proc onStore(
trace "Unable to fetch manifest for cid", cid, err = err.msg trace "Unable to fetch manifest for cid", cid, err = err.msg
return failure(err) return failure(err)
without builder =? Poseidon2Builder.new(self.networkStore, manifest), err: without builder =? Poseidon2Builder.new(
self.networkStore, manifest, manifest.verifiableStrategy
), err:
trace "Unable to create slots builder", err = err.msg trace "Unable to create slots builder", err = err.msg
return failure(err) return failure(err)
@ -559,8 +561,8 @@ proc onStore(
return success() return success()
without indexer =? manifest.protectedStrategy.init( without indexer =? manifest.verifiableStrategy.init(
0, manifest.numSlotBlocks() - 1, manifest.numSlots).catch, err: 0, manifest.blocksCount - 1, manifest.numSlots).catch, err:
trace "Unable to create indexing strategy from protected manifest", err = err.msg trace "Unable to create indexing strategy from protected manifest", err = err.msg
return failure(err) return failure(err)

View File

@ -38,7 +38,6 @@ type
StoreStream* = ref object of SeekableStream StoreStream* = ref object of SeekableStream
store*: BlockStore # Store where to lookup block contents store*: BlockStore # Store where to lookup block contents
manifest*: Manifest # List of block CIDs manifest*: Manifest # List of block CIDs
pad*: bool # Pad last block to manifest.blockSize?
method initStream*(s: StoreStream) = method initStream*(s: StoreStream) =
if s.objName.len == 0: if s.objName.len == 0:
@ -57,13 +56,15 @@ proc new*(
result = StoreStream( result = StoreStream(
store: store, store: store,
manifest: manifest, manifest: manifest,
pad: pad,
offset: 0) offset: 0)
result.initStream() result.initStream()
method `size`*(self: StoreStream): int = method `size`*(self: StoreStream): int =
bytes(self.manifest, self.pad).int ## The size of a StoreStream is the size of the original dataset, without
## padding or parity blocks.
let m = self.manifest
(if m.protected: m.originalDatasetSize else: m.datasetSize).int
proc `size=`*(self: StoreStream, size: int) proc `size=`*(self: StoreStream, size: int)
{.error: "Setting the size is forbidden".} = {.error: "Setting the size is forbidden".} =

View File

@ -134,7 +134,7 @@ asyncchecksuite "Test Node - Host contracts":
return success() return success()
(await onStore(request, 1.u256, onBlocks)).tryGet() (await onStore(request, 1.u256, onBlocks)).tryGet()
check fetchedBytes == 262144 check fetchedBytes == 12 * DefaultBlockSize.uint
let indexer = verifiable.protectedStrategy.init( let indexer = verifiable.protectedStrategy.init(
0, verifiable.numSlotBlocks() - 1, verifiable.numSlots) 0, verifiable.numSlotBlocks() - 1, verifiable.numSlots)

View File

@ -58,6 +58,14 @@ suite "Indexing strategies":
expect IndexingWrongIterationsError: expect IndexingWrongIterationsError:
discard LinearStrategy.init(0, 10, 0) discard LinearStrategy.init(0, 10, 0)
test "should split elements evenly when possible":
let
l = LinearStrategy.init(0, 11, 3)
check:
toSeq(l.getIndicies(0)) == @[0, 1, 2, 3].mapIt(it)
toSeq(l.getIndicies(1)) == @[4, 5, 6, 7].mapIt(it)
toSeq(l.getIndicies(2)) == @[8, 9, 10, 11].mapIt(it)
test "linear - oob": test "linear - oob":
expect IndexingError: expect IndexingError:
discard linear.getIndicies(3) discard linear.getIndicies(3)

View File

@ -74,3 +74,36 @@ checksuite "Manifest":
test "Should encode/decode to/from verifiable manifest": test "Should encode/decode to/from verifiable manifest":
check: check:
encodeDecode(verifiableManifest) == verifiableManifest encodeDecode(verifiableManifest) == verifiableManifest
suite "Manifest - Attribute Inheritance":
proc makeProtectedManifest(strategy: StrategyType): Manifest =
Manifest.new(
manifest = Manifest.new(
treeCid = Cid.example,
blockSize = 1.MiBs,
datasetSize = 100.MiBs,
),
treeCid = Cid.example,
datasetSize = 200.MiBs,
ecK = 1,
ecM = 1,
strategy = strategy
)
test "Should preserve interleaving strategy for protected manifest in verifiable manifest":
var verifiable = Manifest.new(
manifest = makeProtectedManifest(SteppedStrategy),
verifyRoot = Cid.example,
slotRoots = @[Cid.example, Cid.example]
).tryGet()
check verifiable.protectedStrategy == SteppedStrategy
verifiable = Manifest.new(
manifest = makeProtectedManifest(LinearStrategy),
verifyRoot = Cid.example,
slotRoots = @[Cid.example, Cid.example]
).tryGet()
check verifiable.protectedStrategy == LinearStrategy

View File

@ -1,12 +1,15 @@
import pkg/chronos import pkg/chronos
import pkg/questionable/results import pkg/questionable/results
import pkg/codex/streams import pkg/codex/[
import pkg/codex/stores streams,
import pkg/codex/manifest stores,
import pkg/codex/blocktype as bt indexingstrategy,
manifest,
blocktype as bt]
import ../asynctest import ../asynctest
import ./examples
import ./helpers import ./helpers
asyncchecksuite "StoreStream": asyncchecksuite "StoreStream":
@ -99,3 +102,40 @@ asyncchecksuite "StoreStream":
await stream.readExactly(addr buf[0], 15) await stream.readExactly(addr buf[0], 15)
check sequentialBytes(buf,15,0) check sequentialBytes(buf,15,0)
suite "StoreStream - Size Tests":
var stream: StoreStream
teardown:
await stream.close()
test "Should return dataset size as stream size":
let manifest = Manifest.new(
treeCid = Cid.example,
datasetSize = 80.NBytes,
blockSize = 10.NBytes
)
stream = StoreStream.new(CacheStore.new(), manifest)
check stream.size == 80
test "Should not count parity/padding bytes as part of stream size":
let protectedManifest = Manifest.new(
treeCid = Cid.example,
datasetSize = 120.NBytes, # size including parity bytes
blockSize = 10.NBytes,
version = CIDv1,
hcodec = Sha256HashCodec,
codec = BlockCodec,
ecK = 2,
ecM = 1,
originalTreeCid = Cid.example,
originalDatasetSize = 80.NBytes, # size without parity bytes
strategy = StrategyType.SteppedStrategy
)
stream = StoreStream.new(CacheStore.new(), protectedManifest)
check stream.size == 80