Integrate slot builder (#666)
* rework merkle tree support * rename merkletree -> codexmerkletree * treed and proof encoding/decoding * style * adding codex merkle and coders tests * use default hash codec * proof size changed * add from nodes test * shorte file names * wip poseidon tree * shorten file names * root returns a result * import poseidon tests * fix merge issues and cleanup a few warnings * setting up slot builder * Getting cids in slot * ensures blocks are devisable by number of slots * wip * Implements indexing strategies * Swaps in indexing strategy into erasure. * wires slot and indexing tests up * Fixes issue where indexing strategy stepped gives wrong values for smallest of ranges * debugs indexing strategies * Can select slot blocks * finding number of pad cells * Implements building slot tree * finishes implementing slot builder * Adds check that block size is a multiple of cell size * Cleanup slotbuilder * Review comments by Tomasz * Fixes issue where ecK was used as numberOfSlots. * rework merkle tree support * deps * rename merkletree -> codexmerkletree * treed and proof encoding/decoding * style * adding codex merkle and coders tests * remove new codecs for now * proof size changed * add from nodes test * shorte file names * wip poseidon tree * shorten file names * fix bad `elements` iter * bump * bump * wip * reworking slotbuilder * move out of manifest * expose getCidAndProof * import index strat... * remove getMHash * remove unused artifacts * alias zero * add digest for multihash * merge issues * remove unused hashes * add option to result converter * misc * fix tests * add helper to derive EC block count * rename method * misc * bump * extract slot root building into own proc * revert to manifest to accessor --------- Co-authored-by: benbierens <thatbenbierens@gmail.com>
This commit is contained in:
Dmitriy Ryajov
GitHub
parent
b8ee2ac71e
commit
fffb674bba
|
|
@ -361,7 +361,12 @@ proc blocksDeliveryHandler*(
|
|||
without proof =? bd.proof:
|
||||
error "Proof expected for a leaf block delivery"
|
||||
continue
|
||||
if err =? (await b.localStore.putBlockCidAndProof(bd.address.treeCid, bd.address.index, bd.blk.cid, proof)).errorOption:
|
||||
if err =? (await b.localStore.putCidAndProof(
|
||||
bd.address.treeCid,
|
||||
bd.address.index,
|
||||
bd.blk.cid,
|
||||
proof)).errorOption:
|
||||
|
||||
error "Unable to store proof and cid for a block"
|
||||
continue
|
||||
|
||||
|
|
|
|||
|
|
@ -39,6 +39,7 @@ const
|
|||
BlockCodec* = multiCodec("codex-block")
|
||||
SlotRootCodec* = multiCodec("codex-slot-root")
|
||||
SlotProvingRootCodec* = multiCodec("codex-proving-root")
|
||||
CodexSlotCell* = multiCodec("codex-slot-cell")
|
||||
|
||||
CodexHashesCodecs* = [
|
||||
Sha256HashCodec,
|
||||
|
|
@ -52,15 +53,15 @@ const
|
|||
BlockCodec,
|
||||
SlotRootCodec,
|
||||
SlotProvingRootCodec,
|
||||
CodexSlotCell,
|
||||
]
|
||||
|
||||
|
||||
proc initEmptyCidTable(): ?!Table[(CidVersion, MultiCodec, MultiCodec), Cid] =
|
||||
## Initialize padding blocks table
|
||||
##
|
||||
## TODO: Ideally this is done at compile time, but for now
|
||||
## we do it at runtime because of an `importc` error that is
|
||||
## coming from somwhere in MultiHash that I can't track down.
|
||||
## coming from somewhere in MultiHash that I can't track down.
|
||||
##
|
||||
|
||||
let
|
||||
|
|
@ -68,8 +69,6 @@ proc initEmptyCidTable(): ?!Table[(CidVersion, MultiCodec, MultiCodec), Cid] =
|
|||
PadHashes = {
|
||||
Sha256HashCodec: ? MultiHash.digest($Sha256HashCodec, emptyData).mapFailure,
|
||||
Sha512HashCodec: ? MultiHash.digest($Sha512HashCodec, emptyData).mapFailure,
|
||||
Pos2Bn128SpngCodec: ? MultiHash.digest($Pos2Bn128SpngCodec, emptyData).mapFailure,
|
||||
Pos2Bn128MrklCodec: ? MultiHash.digest($Pos2Bn128SpngCodec, emptyData).mapFailure,
|
||||
}.toTable
|
||||
|
||||
var
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ import ../stores
|
|||
import ../blocktype as bt
|
||||
import ../utils
|
||||
import ../utils/asynciter
|
||||
import ../indexingstrategy
|
||||
|
||||
import pkg/stew/byteutils
|
||||
|
||||
|
|
@ -127,7 +128,12 @@ proc prepareEncodingData(
|
|||
##
|
||||
|
||||
let
|
||||
indicies = toSeq(countup(step, params.rounded - 1, params.steps))
|
||||
strategy = SteppedIndexingStrategy.new(
|
||||
firstIndex = 0,
|
||||
lastIndex = params.rounded - 1,
|
||||
numberOfIterations = params.steps
|
||||
)
|
||||
indicies = strategy.getIndicies(step)
|
||||
pendingBlocksIter = self.getPendingBlocks(manifest, indicies.filterIt(it < manifest.blocksCount))
|
||||
|
||||
var resolved = 0
|
||||
|
|
@ -171,7 +177,12 @@ proc prepareDecodingData(
|
|||
##
|
||||
|
||||
let
|
||||
indicies = toSeq(countup(step, encoded.blocksCount - 1, encoded.steps))
|
||||
strategy = SteppedIndexingStrategy.new(
|
||||
firstIndex = 0,
|
||||
lastIndex = encoded.blocksCount - 1,
|
||||
numberOfIterations = encoded.steps
|
||||
)
|
||||
indicies = strategy.getIndicies(step)
|
||||
pendingBlocksIter = self.getPendingBlocks(encoded, indicies)
|
||||
|
||||
var
|
||||
|
|
@ -213,7 +224,10 @@ proc prepareDecodingData(
|
|||
|
||||
return success (dataPieces, parityPieces)
|
||||
|
||||
proc init(_: type EncodingParams, manifest: Manifest, ecK: int, ecM: int): ?!EncodingParams =
|
||||
proc init*(
|
||||
_: type EncodingParams,
|
||||
manifest: Manifest,
|
||||
ecK: int, ecM: int): ?!EncodingParams =
|
||||
if ecK > manifest.blocksCount:
|
||||
return failure("Unable to encode manifest, not enough blocks, ecK = " & $ecK & ", blocksCount = " & $manifest.blocksCount)
|
||||
|
||||
|
|
|
|||
|
|
@ -7,6 +7,8 @@
|
|||
## This file may not be copied, modified, or distributed except according to
|
||||
## those terms.
|
||||
|
||||
import std/options
|
||||
|
||||
import pkg/stew/results
|
||||
import pkg/chronos
|
||||
import pkg/questionable/results
|
||||
|
|
@ -29,6 +31,12 @@ template mapFailure*[T, V, E](
|
|||
template mapFailure*[T, V](exp: Result[T, V]): Result[T, ref CatchableError] =
|
||||
mapFailure(exp, CodexError)
|
||||
|
||||
template toResult*[T](exp: Option[T]): Result[T, ref CatchableError] =
|
||||
if exp.isSome:
|
||||
success exp.get
|
||||
else:
|
||||
T.failure("Option is None")
|
||||
|
||||
proc allFutureResult*[T](fut: seq[Future[T]]): Future[?!void] {.async.} =
|
||||
try:
|
||||
await allFuturesThrowing(fut)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,61 @@
|
|||
import std/sequtils
|
||||
import ./utils
|
||||
|
||||
# I'm choosing to use an assert here because:
|
||||
# 1. These are a programmer errors and *should not* happen during application runtime.
|
||||
# 2. Users don't have to deal with Result types.
|
||||
|
||||
type
|
||||
# Representing a strategy for grouping indices (of blocks usually)
|
||||
# Given an interation-count as input, will produce a seq of
|
||||
# selected indices.
|
||||
IndexingStrategy* = ref object of RootObj
|
||||
firstIndex*: int # Lowest index that can be returned
|
||||
lastIndex*: int # Highest index that can be returned
|
||||
numberOfIterations*: int # getIndices(iteration) will run from 0 ..< numberOfIterations
|
||||
step*: int
|
||||
|
||||
# Simplest approach:
|
||||
# 0 => 0, 1, 2
|
||||
# 1 => 3, 4, 5
|
||||
# 2 => 6, 7, 8
|
||||
LinearIndexingStrategy* = ref object of IndexingStrategy
|
||||
|
||||
# Stepped indexing:
|
||||
# 0 => 0, 3, 6
|
||||
# 1 => 1, 4, 7
|
||||
# 2 => 2, 5, 8
|
||||
SteppedIndexingStrategy* = ref object of IndexingStrategy
|
||||
|
||||
proc assertIteration(self: IndexingStrategy, iteration: int): void =
|
||||
if iteration >= self.numberOfIterations:
|
||||
raiseAssert("Indexing iteration can't be greater than or equal to numberOfIterations.")
|
||||
|
||||
method getIndicies*(self: IndexingStrategy, iteration: int): seq[int] {.base.} =
|
||||
raiseAssert("Not implemented")
|
||||
|
||||
proc new*(T: type IndexingStrategy, firstIndex, lastIndex, numberOfIterations: int): T =
|
||||
if firstIndex > lastIndex:
|
||||
raiseAssert("firstIndex (" & $firstIndex & ") can't be greater than lastIndex (" & $lastIndex & ")")
|
||||
if numberOfIterations <= 0:
|
||||
raiseAssert("numberOfIteration (" & $numberOfIterations & ") must be greater than zero.")
|
||||
|
||||
T(
|
||||
firstIndex: firstIndex,
|
||||
lastIndex: lastIndex,
|
||||
numberOfIterations: numberOfIterations,
|
||||
step: divUp((lastIndex - firstIndex), numberOfIterations)
|
||||
)
|
||||
|
||||
method getIndicies*(self: LinearIndexingStrategy, iteration: int): seq[int] =
|
||||
self.assertIteration(iteration)
|
||||
|
||||
let
|
||||
first = self.firstIndex + iteration * (self.step + 1)
|
||||
last = min(first + self.step, self.lastIndex)
|
||||
|
||||
toSeq(countup(first, last, 1))
|
||||
|
||||
method getIndicies*(self: SteppedIndexingStrategy, iteration: int): seq[int] =
|
||||
self.assertIteration(iteration)
|
||||
toSeq(countup(self.firstIndex + iteration, self.lastIndex, self.numberOfIterations))
|
||||
|
|
@ -98,6 +98,11 @@ proc verificationRoot*(self: Manifest): Cid =
|
|||
proc slotRoots*(self: Manifest): seq[Cid] =
|
||||
self.slotRoots
|
||||
|
||||
proc numSlots*(self: Manifest): int =
|
||||
if not self.protected:
|
||||
0
|
||||
else:
|
||||
self.ecK + self.ecM
|
||||
############################################################
|
||||
# Operations on block list
|
||||
############################################################
|
||||
|
|
|
|||
|
|
@ -13,7 +13,6 @@ push: {.upraises: [].}
|
|||
|
||||
import std/bitops
|
||||
import std/sequtils
|
||||
|
||||
import pkg/questionable
|
||||
import pkg/questionable/results
|
||||
import pkg/libp2p/[cid, multicodec, multihash]
|
||||
|
|
@ -40,15 +39,15 @@ type
|
|||
|
||||
ByteHash* = seq[byte]
|
||||
ByteTree* = MerkleTree[ByteHash, ByteTreeKey]
|
||||
ByteTreeProof* = MerkleProof[ByteHash, ByteTreeKey]
|
||||
ByteProof* = MerkleProof[ByteHash, ByteTreeKey]
|
||||
|
||||
CodexTree* = ref object of ByteTree
|
||||
mhash: MHash
|
||||
mcodec*: MultiCodec
|
||||
|
||||
CodexProof* = ref object of ByteTreeProof
|
||||
mhash: MHash
|
||||
CodexProof* = ref object of ByteProof
|
||||
mcodec*: MultiCodec
|
||||
|
||||
func getMhash*(mcodec: MultiCodec): ?!MHash =
|
||||
func mhash*(mcodec: MultiCodec): ?!MHash =
|
||||
let
|
||||
mhash = CodeHashes.getOrDefault(mcodec)
|
||||
|
||||
|
|
@ -63,21 +62,15 @@ func digestSize*(self: (CodexTree or CodexProof)): int =
|
|||
|
||||
self.mhash.size
|
||||
|
||||
func mcodec*(self: (CodexTree or CodexProof)): MultiCodec =
|
||||
## Multicodec
|
||||
##
|
||||
|
||||
self.mhash.mcodec
|
||||
|
||||
func bytes*(mhash: MultiHash): seq[byte] =
|
||||
## Extract hash bytes
|
||||
func digestBytes*(mhash: MultiHash): seq[byte] =
|
||||
## Extract hash digestBytes
|
||||
##
|
||||
|
||||
mhash.data.buffer[mhash.dpos..<mhash.dpos + mhash.size]
|
||||
|
||||
func getProof*(self: CodexTree, index: int): ?!CodexProof =
|
||||
var
|
||||
proof = CodexProof(mhash: self.mhash)
|
||||
proof = CodexProof(mcodec: self.mcodec)
|
||||
|
||||
? self.getProof(index, proof)
|
||||
|
||||
|
|
@ -88,8 +81,8 @@ func verify*(self: CodexProof, leaf: MultiHash, root: MultiHash): ?!void =
|
|||
##
|
||||
|
||||
let
|
||||
rootBytes = root.bytes
|
||||
leafBytes = leaf.bytes
|
||||
rootBytes = root.digestBytes
|
||||
leafBytes = leaf.digestBytes
|
||||
|
||||
if self.mcodec != root.mcodec or
|
||||
self.mcodec != leaf.mcodec:
|
||||
|
|
@ -166,7 +159,7 @@ func init*(
|
|||
return failure "Empty leaves"
|
||||
|
||||
let
|
||||
mhash = ? mcodec.getMhash()
|
||||
mhash = ? mcodec.mhash()
|
||||
compressor = proc(x, y: seq[byte], key: ByteTreeKey): ?!ByteHash {.noSideEffect.} =
|
||||
compress(x, y, key, mhash)
|
||||
Zero: ByteHash = newSeq[byte](mhash.size)
|
||||
|
|
@ -175,7 +168,7 @@ func init*(
|
|||
return failure "Invalid hash length"
|
||||
|
||||
var
|
||||
self = CodexTree(mhash: mhash, compress: compressor, zero: Zero)
|
||||
self = CodexTree(mcodec: mcodec, compress: compressor, zero: Zero)
|
||||
|
||||
self.layers = ? merkleTreeWorker(self, leaves, isBottomLayer = true)
|
||||
success self
|
||||
|
|
@ -189,7 +182,7 @@ func init*(
|
|||
|
||||
let
|
||||
mcodec = leaves[0].mcodec
|
||||
leaves = leaves.mapIt( it.bytes )
|
||||
leaves = leaves.mapIt( it.digestBytes )
|
||||
|
||||
CodexTree.init(mcodec, leaves)
|
||||
|
||||
|
|
@ -201,7 +194,7 @@ func init*(
|
|||
|
||||
let
|
||||
mcodec = (? leaves[0].mhash.mapFailure).mcodec
|
||||
leaves = leaves.mapIt( (? it.mhash.mapFailure).bytes )
|
||||
leaves = leaves.mapIt( (? it.mhash.mapFailure).digestBytes )
|
||||
|
||||
CodexTree.init(mcodec, leaves)
|
||||
|
||||
|
|
@ -215,7 +208,7 @@ proc fromNodes*(
|
|||
return failure "Empty nodes"
|
||||
|
||||
let
|
||||
mhash = ? mcodec.getMhash()
|
||||
mhash = ? mcodec.mhash()
|
||||
Zero = newSeq[byte](mhash.size)
|
||||
compressor = proc(x, y: seq[byte], key: ByteTreeKey): ?!ByteHash {.noSideEffect.} =
|
||||
compress(x, y, key, mhash)
|
||||
|
|
@ -224,7 +217,7 @@ proc fromNodes*(
|
|||
return failure "Invalid hash length"
|
||||
|
||||
var
|
||||
self = CodexTree(compress: compressor, zero: Zero, mhash: mhash)
|
||||
self = CodexTree(compress: compressor, zero: Zero, mcodec: mcodec)
|
||||
layer = nleaves
|
||||
pos = 0
|
||||
|
||||
|
|
@ -251,16 +244,15 @@ func init*(
|
|||
return failure "Empty nodes"
|
||||
|
||||
let
|
||||
mhash = ? mcodec.getMhash()
|
||||
mhash = ? mcodec.mhash()
|
||||
Zero = newSeq[byte](mhash.size)
|
||||
compressor = proc(x, y: seq[byte], key: ByteTreeKey): ?!seq[byte] {.noSideEffect.} =
|
||||
compress(x, y, key, mhash)
|
||||
|
||||
|
||||
success CodexProof(
|
||||
compress: compressor,
|
||||
zero: Zero,
|
||||
mhash: mhash,
|
||||
mcodec: mcodec,
|
||||
index: index,
|
||||
nleaves: nleaves,
|
||||
path: @nodes)
|
||||
|
|
|
|||
|
|
@ -16,13 +16,6 @@ import pkg/questionable/results
|
|||
import ../errors
|
||||
|
||||
type
|
||||
PutFn*[H] = proc(i: Natural, x: H): ?!void {.noSideEffect, raises: [].}
|
||||
GetFn*[H] = proc(i: Natural): ?!H {.noSideEffect, raises: [].}
|
||||
|
||||
StoreBackend*[H] = object
|
||||
put: PutFn[H]
|
||||
get: GetFn[H]
|
||||
|
||||
CompressFn*[H, K] = proc (x, y: H, key: K): ?!H {.noSideEffect, raises: [].}
|
||||
|
||||
MerkleTree*[H, K] = ref object of RootObj
|
||||
|
|
|
|||
|
|
@ -27,6 +27,8 @@ const
|
|||
KeyOddF = F.fromhex("0x2")
|
||||
KeyOddAndBottomLayerF = F.fromhex("0x3")
|
||||
|
||||
Poseidon2Zero* = zero
|
||||
|
||||
type
|
||||
Poseidon2Hash* = F
|
||||
|
||||
|
|
@ -39,6 +41,9 @@ type
|
|||
Poseidon2Tree* = MerkleTree[Poseidon2Hash, PoseidonKeysEnum]
|
||||
Poseidon2Proof* = MerkleProof[Poseidon2Hash, PoseidonKeysEnum]
|
||||
|
||||
func toArray32*(bytes: openArray[byte]): array[32, byte] =
|
||||
result[0..<bytes.len] = bytes[0..<bytes.len]
|
||||
|
||||
converter toKey*(key: PoseidonKeysEnum): Poseidon2Hash =
|
||||
case key:
|
||||
of KeyNone: KeyNoneF
|
||||
|
|
@ -60,7 +65,7 @@ func init*(
|
|||
success compress( x, y, key.toKey )
|
||||
|
||||
var
|
||||
self = Poseidon2Tree(compress: compressor, zero: zero)
|
||||
self = Poseidon2Tree(compress: compressor, zero: Poseidon2Zero)
|
||||
|
||||
self.layers = ? merkleTreeWorker(self, leaves, isBottomLayer = true)
|
||||
success self
|
||||
|
|
|
|||
|
|
@ -251,7 +251,7 @@ proc store*(
|
|||
for index, cid in cids:
|
||||
without proof =? tree.getProof(index), err:
|
||||
return failure(err)
|
||||
if err =? (await self.blockStore.putBlockCidAndProof(treeCid, index, cid, proof)).errorOption:
|
||||
if err =? (await self.blockStore.putCidAndProof(treeCid, index, cid, proof)).errorOption:
|
||||
# TODO add log here
|
||||
return failure(err)
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,275 @@
|
|||
## Nim-Codex
|
||||
## Copyright (c) 2023 Status Research & Development GmbH
|
||||
## Licensed under either of
|
||||
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
|
||||
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
|
||||
## at your option.
|
||||
## This file may not be copied, modified, or distributed except according to
|
||||
## those terms.
|
||||
|
||||
{.push raises: [].}
|
||||
|
||||
import std/math
|
||||
import std/sequtils
|
||||
import std/sugar
|
||||
|
||||
import pkg/libp2p
|
||||
import pkg/chronos
|
||||
import pkg/chronicles
|
||||
import pkg/questionable/results
|
||||
import pkg/poseidon2
|
||||
import pkg/poseidon2/io
|
||||
|
||||
import ../indexingstrategy
|
||||
import ../merkletree
|
||||
import ../stores
|
||||
import ../manifest
|
||||
import ../utils
|
||||
import ../utils/digest
|
||||
|
||||
const
|
||||
# TODO: Unified with the CellSize specified in branch "data-sampler"
|
||||
# Number of bytes in a cell. A cell is the smallest unit of data used
|
||||
# in the proving circuit.
|
||||
CellSize* = 2048
|
||||
|
||||
type
|
||||
SlotBuilder* = object of RootObj
|
||||
store: BlockStore
|
||||
manifest: Manifest
|
||||
strategy: IndexingStrategy
|
||||
cellSize: int
|
||||
blockPadBytes: seq[byte]
|
||||
slotsPadLeafs: seq[Poseidon2Hash]
|
||||
rootsPadLeafs: seq[Poseidon2Hash]
|
||||
|
||||
func numBlockPadBytes*(self: SlotBuilder): Natural =
|
||||
## Number of padding bytes required for a pow2
|
||||
## merkle tree for each block.
|
||||
##
|
||||
|
||||
self.blockPadBytes.len
|
||||
|
||||
func numSlotsPadLeafs*(self: SlotBuilder): Natural =
|
||||
## Number of padding field elements required for a pow2
|
||||
## merkle tree for each slot.
|
||||
##
|
||||
|
||||
self.slotsPadLeafs.len
|
||||
|
||||
func numRootsPadLeafs*(self: SlotBuilder): Natural =
|
||||
## Number of padding field elements required for a pow2
|
||||
## merkle tree for the slot roots.
|
||||
##
|
||||
|
||||
self.rootsPadLeafs.len
|
||||
|
||||
func numSlotBlocks*(self: SlotBuilder): Natural =
|
||||
## Number of blocks per slot.
|
||||
##
|
||||
|
||||
self.manifest.blocksCount div self.manifest.numSlots
|
||||
|
||||
func numBlockRoots*(self: SlotBuilder): Natural =
|
||||
## Number of cells per block.
|
||||
##
|
||||
|
||||
self.manifest.blockSize.int div self.cellSize
|
||||
|
||||
func toCellCid*(cell: Poseidon2Hash): ?!Cid =
|
||||
let
|
||||
cellMhash = ? MultiHash.init(Pos2Bn128MrklCodec, cell.toBytes).mapFailure
|
||||
cellCid = ? Cid.init(CIDv1, CodexSlotCell, cellMhash).mapFailure
|
||||
|
||||
success cellCid
|
||||
|
||||
func toSlotCid*(root: Poseidon2Hash): ?!Cid =
|
||||
let
|
||||
mhash = ? MultiHash.init($multiCodec("identity"), root.toBytes).mapFailure
|
||||
treeCid = ? Cid.init(CIDv1, SlotRootCodec, mhash).mapFailure
|
||||
|
||||
success treeCid
|
||||
|
||||
func toProvingCid*(root: Poseidon2Hash): ?!Cid =
|
||||
let
|
||||
mhash = ? MultiHash.init($multiCodec("identity"), root.toBytes).mapFailure
|
||||
treeCid = ? Cid.init(CIDv1, SlotProvingRootCodec, mhash).mapFailure
|
||||
|
||||
success treeCid
|
||||
|
||||
func mapToSlotCids*(slotRoots: seq[Poseidon2Hash]): ?!seq[Cid] =
|
||||
success slotRoots.mapIt( ? it.toSlotCid )
|
||||
|
||||
func toEncodableProof*(
|
||||
proof: Poseidon2Proof): ?!CodexProof =
|
||||
|
||||
let
|
||||
encodableProof = CodexProof(
|
||||
mcodec: multiCodec("identity"), # copy bytes as is
|
||||
index: proof.index,
|
||||
nleaves: proof.nleaves,
|
||||
path: proof.path.mapIt( @(it.toBytes) ))
|
||||
|
||||
success encodableProof
|
||||
|
||||
func toVerifiableProof*(
|
||||
proof: CodexProof): ?!Poseidon2Proof =
|
||||
|
||||
let
|
||||
verifiableProof = Poseidon2Proof(
|
||||
index: proof.index,
|
||||
nleaves: proof.nleaves,
|
||||
path: proof.path.mapIt(
|
||||
? Poseidon2Hash.fromBytes(it.toArray32).toResult
|
||||
))
|
||||
|
||||
success verifiableProof
|
||||
|
||||
proc getCellHashes*(
|
||||
self: SlotBuilder,
|
||||
slotIndex: int): Future[?!seq[Poseidon2Hash]] {.async.} =
|
||||
|
||||
let
|
||||
treeCid = self.manifest.treeCid
|
||||
blockCount = self.manifest.blocksCount
|
||||
numberOfSlots = self.manifest.numSlots
|
||||
|
||||
logScope:
|
||||
treeCid = treeCid
|
||||
blockCount = blockCount
|
||||
numberOfSlots = numberOfSlots
|
||||
index = blockIndex
|
||||
slotIndex = slotIndex
|
||||
|
||||
let
|
||||
hashes: seq[Poseidon2Hash] = collect(newSeq):
|
||||
for blockIndex in self.strategy.getIndicies(slotIndex):
|
||||
trace "Getting block CID for tree at index"
|
||||
|
||||
without blk =? (await self.store.getBlock(treeCid, blockIndex)), err:
|
||||
error "Failed to get block CID for tree at index"
|
||||
return failure(err)
|
||||
|
||||
without digest =? Poseidon2Tree.digest(blk.data & self.blockPadBytes, self.cellSize), err:
|
||||
error "Failed to create digest for block"
|
||||
return failure(err)
|
||||
|
||||
digest
|
||||
|
||||
success hashes
|
||||
|
||||
proc buildSlotTree*(
|
||||
self: SlotBuilder,
|
||||
slotIndex: int): Future[?!Poseidon2Tree] {.async.} =
|
||||
without cellHashes =? (await self.getCellHashes(slotIndex)), err:
|
||||
error "Failed to select slot blocks", err = err.msg
|
||||
return failure(err)
|
||||
|
||||
Poseidon2Tree.init(cellHashes & self.slotsPadLeafs)
|
||||
|
||||
proc buildSlot*(
|
||||
self: SlotBuilder,
|
||||
slotIndex: int): Future[?!Poseidon2Hash] {.async.} =
|
||||
## Build a slot tree and store it in the block store.
|
||||
##
|
||||
|
||||
without tree =? (await self.buildSlotTree(slotIndex)) and
|
||||
treeCid =? tree.root.?toSlotCid, err:
|
||||
error "Failed to build slot tree", err = err.msg
|
||||
return failure(err)
|
||||
|
||||
trace "Storing slot tree", treeCid, slotIndex, leaves = tree.leavesCount
|
||||
for i, leaf in tree.leaves:
|
||||
without cellCid =? leaf.toCellCid, err:
|
||||
error "Failed to get CID for slot cell", err = err.msg
|
||||
return failure(err)
|
||||
|
||||
without proof =? tree.getProof(i) and
|
||||
encodableProof =? proof.toEncodableProof, err:
|
||||
error "Failed to get proof for slot tree", err = err.msg
|
||||
return failure(err)
|
||||
|
||||
if err =? (await self.store.putCidAndProof(
|
||||
treeCid, i, cellCid, encodableProof)).errorOption:
|
||||
error "Failed to store slot tree", err = err.msg
|
||||
return failure(err)
|
||||
|
||||
tree.root()
|
||||
|
||||
proc buildSlots*(self: SlotBuilder): Future[?!seq[Poseidon2Hash]] {.async.} =
|
||||
let
|
||||
slotRoots: seq[Poseidon2Hash] = collect(newSeq):
|
||||
for i in 0..<self.manifest.numSlots:
|
||||
without root =? (await self.buildSlot(i)), err:
|
||||
error "Failed to build slot", err = err.msg, index = i
|
||||
return failure(err)
|
||||
root
|
||||
|
||||
success slotRoots
|
||||
|
||||
func buildRootsTree*(
|
||||
self: SlotBuilder,
|
||||
slotRoots: seq[Poseidon2Hash]): ?!Poseidon2Tree =
|
||||
Poseidon2Tree.init(slotRoots & self.rootsPadLeafs)
|
||||
|
||||
proc buildManifest*(self: SlotBuilder): Future[?!Manifest] {.async.} =
|
||||
|
||||
without slotRoots =? await self.buildSlots(), err:
|
||||
error "Failed to build slot roots", err = err.msg
|
||||
return failure(err)
|
||||
|
||||
without provingRootCid =? self.buildRootsTree(slotRoots).?root.?toProvingCid, err:
|
||||
error "Failed to build proving tree", err = err.msg
|
||||
return failure(err)
|
||||
|
||||
without rootCids =? slotRoots.mapToSlotCids(), err:
|
||||
error "Failed to map slot roots to CIDs", err = err.msg
|
||||
return failure(err)
|
||||
|
||||
Manifest.new(self.manifest, provingRootCid, rootCids)
|
||||
|
||||
func nextPowerOfTwoPad*(a: int): int =
|
||||
## Returns the difference between the original
|
||||
## value and the next power of two.
|
||||
##
|
||||
|
||||
nextPowerOfTwo(a) - a
|
||||
|
||||
proc new*(
|
||||
T: type SlotBuilder,
|
||||
store: BlockStore,
|
||||
manifest: Manifest,
|
||||
strategy: IndexingStrategy = nil,
|
||||
cellSize = CellSize): ?!SlotBuilder =
|
||||
|
||||
if not manifest.protected:
|
||||
return failure("Can only create SlotBuilder using protected manifests.")
|
||||
|
||||
if (manifest.blocksCount mod manifest.numSlots) != 0:
|
||||
return failure("Number of blocks must be divisable by number of slots.")
|
||||
|
||||
if (manifest.blockSize.int mod cellSize) != 0:
|
||||
return failure("Block size must be divisable by cell size.")
|
||||
|
||||
let
|
||||
strategy = if strategy == nil:
|
||||
SteppedIndexingStrategy.new(
|
||||
0, manifest.blocksCount - 1, manifest.numSlots)
|
||||
else:
|
||||
strategy
|
||||
|
||||
# all trees have to be padded to power of two
|
||||
numBlockCells = manifest.blockSize.int div cellSize # number of cells per block
|
||||
blockPadBytes = newSeq[byte](numBlockCells.nextPowerOfTwoPad * cellSize) # power of two padding for blocks
|
||||
numSlotLeafs = (manifest.blocksCount div manifest.numSlots)
|
||||
slotsPadLeafs = newSeqWith(numSlotLeafs.nextPowerOfTwoPad, Poseidon2Zero) # power of two padding for block roots
|
||||
rootsPadLeafs = newSeqWith(manifest.numSlots.nextPowerOfTwoPad, Poseidon2Zero)
|
||||
|
||||
success SlotBuilder(
|
||||
store: store,
|
||||
manifest: manifest,
|
||||
strategy: strategy,
|
||||
cellSize: cellSize,
|
||||
blockPadBytes: blockPadBytes,
|
||||
slotsPadLeafs: slotsPadLeafs,
|
||||
rootsPadLeafs: rootsPadLeafs)
|
||||
|
|
@ -43,6 +43,11 @@ method getBlock*(self: BlockStore, treeCid: Cid, index: Natural): Future[?!Block
|
|||
|
||||
raiseAssert("getBlock by treecid not implemented!")
|
||||
|
||||
method getCid*(self: BlockStore, treeCid: Cid, index: Natural): Future[?!Cid] {.base.} =
|
||||
## Get a cid given a tree and index
|
||||
##
|
||||
raiseAssert("getCid by treecid not implemented!")
|
||||
|
||||
method getBlock*(self: BlockStore, address: BlockAddress): Future[?!Block] {.base.} =
|
||||
## Get a block from the blockstore
|
||||
##
|
||||
|
|
@ -65,17 +70,26 @@ method putBlock*(
|
|||
|
||||
raiseAssert("putBlock not implemented!")
|
||||
|
||||
method putBlockCidAndProof*(
|
||||
method putCidAndProof*(
|
||||
self: BlockStore,
|
||||
treeCid: Cid,
|
||||
index: Natural,
|
||||
blockCid: Cid,
|
||||
proof: CodexProof
|
||||
): Future[?!void] {.base.} =
|
||||
## Put a block to the blockstore
|
||||
## Put a block proof to the blockstore
|
||||
##
|
||||
|
||||
raiseAssert("putBlockCidAndProof not implemented!")
|
||||
raiseAssert("putCidAndProof not implemented!")
|
||||
|
||||
method getCidAndProof*(
|
||||
self: BlockStore,
|
||||
treeCid: Cid,
|
||||
index: Natural): Future[?!(Cid, CodexProof)] {.base.} =
|
||||
## Get a block proof from the blockstore
|
||||
##
|
||||
|
||||
raiseAssert("putCidAndProof not implemented!")
|
||||
|
||||
method ensureExpiry*(
|
||||
self: BlockStore,
|
||||
|
|
|
|||
|
|
@ -65,20 +65,24 @@ method getBlock*(self: CacheStore, cid: Cid): Future[?!Block] {.async.} =
|
|||
trace "Error requesting block from cache", cid, error = exc.msg
|
||||
return failure exc
|
||||
|
||||
proc getCidAndProof(self: CacheStore, treeCid: Cid, index: Natural): ?!(Cid, CodexProof) =
|
||||
method getCidAndProof*(
|
||||
self: CacheStore,
|
||||
treeCid: Cid,
|
||||
index: Natural): Future[?!(Cid, CodexProof)] {.async.} =
|
||||
|
||||
if cidAndProof =? self.cidAndProofCache.getOption((treeCid, index)):
|
||||
success(cidAndProof)
|
||||
else:
|
||||
failure(newException(BlockNotFoundError, "Block not in cache: " & $BlockAddress.init(treeCid, index)))
|
||||
|
||||
method getBlock*(self: CacheStore, treeCid: Cid, index: Natural): Future[?!Block] {.async.} =
|
||||
without cidAndProof =? self.getCidAndProof(treeCid, index), err:
|
||||
without cidAndProof =? (await self.getCidAndProof(treeCid, index)), err:
|
||||
return failure(err)
|
||||
|
||||
await self.getBlock(cidAndProof[0])
|
||||
|
||||
method getBlockAndProof*(self: CacheStore, treeCid: Cid, index: Natural): Future[?!(Block, CodexProof)] {.async.} =
|
||||
without cidAndProof =? self.getCidAndProof(treeCid, index), err:
|
||||
without cidAndProof =? (await self.getCidAndProof(treeCid, index)), err:
|
||||
return failure(err)
|
||||
|
||||
let (cid, proof) = cidAndProof
|
||||
|
|
@ -106,7 +110,7 @@ method hasBlock*(self: CacheStore, cid: Cid): Future[?!bool] {.async.} =
|
|||
return (cid in self.cache).success
|
||||
|
||||
method hasBlock*(self: CacheStore, treeCid: Cid, index: Natural): Future[?!bool] {.async.} =
|
||||
without cidAndProof =? self.getCidAndProof(treeCid, index), err:
|
||||
without cidAndProof =? (await self.getCidAndProof(treeCid, index)), err:
|
||||
if err of BlockNotFoundError:
|
||||
return success(false)
|
||||
else:
|
||||
|
|
@ -114,7 +118,6 @@ method hasBlock*(self: CacheStore, treeCid: Cid, index: Natural): Future[?!bool]
|
|||
|
||||
await self.hasBlock(cidAndProof[0])
|
||||
|
||||
|
||||
func cids(self: CacheStore): (iterator: Cid {.gcsafe.}) =
|
||||
return iterator(): Cid =
|
||||
for cid in self.cache.keys:
|
||||
|
|
@ -210,7 +213,7 @@ method putBlock*(
|
|||
discard self.putBlockSync(blk)
|
||||
return success()
|
||||
|
||||
method putBlockCidAndProof*(
|
||||
method putCidAndProof*(
|
||||
self: CacheStore,
|
||||
treeCid: Cid,
|
||||
index: Natural,
|
||||
|
|
|
|||
|
|
@ -78,13 +78,13 @@ method putBlock*(
|
|||
await self.engine.resolveBlocks(@[blk])
|
||||
return success()
|
||||
|
||||
method putBlockCidAndProof*(
|
||||
method putCidAndProof*(
|
||||
self: NetworkStore,
|
||||
treeCid: Cid,
|
||||
index: Natural,
|
||||
blockCid: Cid,
|
||||
proof: CodexProof): Future[?!void] =
|
||||
self.localStore.putBlockCidAndProof(treeCid, index, blockCid, proof)
|
||||
self.localStore.putCidAndProof(treeCid, index, blockCid, proof)
|
||||
|
||||
method ensureExpiry*(
|
||||
self: NetworkStore,
|
||||
|
|
|
|||
|
|
@ -106,7 +106,7 @@ proc decodeCid(_: type (Cid, CodexProof), data: seq[byte]): ?!Cid =
|
|||
cid = ? Cid.init(data[sizeof(uint64)..<sizeof(uint64) + n]).mapFailure
|
||||
success(cid)
|
||||
|
||||
method putBlockCidAndProof*(
|
||||
method putCidAndProof*(
|
||||
self: RepoStore,
|
||||
treeCid: Cid,
|
||||
index: Natural,
|
||||
|
|
@ -125,11 +125,10 @@ method putBlockCidAndProof*(
|
|||
|
||||
await self.metaDs.put(key, value)
|
||||
|
||||
proc getCidAndProof(
|
||||
method getCidAndProof*(
|
||||
self: RepoStore,
|
||||
treeCid: Cid,
|
||||
index: Natural
|
||||
): Future[?!(Cid, CodexProof)] {.async.} =
|
||||
index: Natural): Future[?!(Cid, CodexProof)] {.async.} =
|
||||
without key =? createBlockCidAndProofMetadataKey(treeCid, index), err:
|
||||
return failure(err)
|
||||
|
||||
|
|
@ -146,7 +145,7 @@ proc getCidAndProof(
|
|||
trace "Got cid and proof for block", cid, proof = $proof
|
||||
return success (cid, proof)
|
||||
|
||||
proc getCid(
|
||||
method getCid*(
|
||||
self: RepoStore,
|
||||
treeCid: Cid,
|
||||
index: Natural): Future[?!Cid] {.async.} =
|
||||
|
|
@ -541,7 +540,8 @@ method close*(self: RepoStore): Future[void] {.async.} =
|
|||
## For some implementations this may be a no-op
|
||||
##
|
||||
|
||||
(await self.repoDs.close()).expect("Should close datastore")
|
||||
(await self.metaDs.close()).expect("Should meta datastore")
|
||||
(await self.repoDs.close()).expect("Should repo datastore")
|
||||
|
||||
proc reserve*(self: RepoStore, bytes: uint): Future[?!void] {.async.} =
|
||||
## Reserve bytes
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@ proc putSomeProofs*(store: BlockStore, tree: CodexTree, iter: Iter[int]): Future
|
|||
without proof =? tree.getProof(i), err:
|
||||
return failure(err)
|
||||
|
||||
let res = await store.putBlockCidAndProof(treeCid, i, blkCid, proof)
|
||||
let res = await store.putCidAndProof(treeCid, i, blkCid, proof)
|
||||
|
||||
if err =? res.errorOption:
|
||||
return failure(err)
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@
|
|||
import pkg/poseidon2
|
||||
import pkg/poseidon2/io
|
||||
import pkg/questionable/results
|
||||
import pkg/libp2p/multihash
|
||||
|
||||
import ../merkletree
|
||||
|
||||
|
|
@ -38,3 +39,15 @@ func digest*(
|
|||
##
|
||||
|
||||
(? Poseidon2Tree.digestTree(bytes, chunkSize)).root
|
||||
|
||||
func digestMhash*(
|
||||
_: type Poseidon2Tree,
|
||||
bytes: openArray[byte], chunkSize: int): ?!MultiHash =
|
||||
## Hashes chunks of data with a sponge of rate 2 and
|
||||
## returns the multihash of the root
|
||||
##
|
||||
|
||||
let
|
||||
hash = ? Poseidon2Tree.digest(bytes, chunkSize)
|
||||
|
||||
? MultiHash.init(Pos2Bn128MrklCodec, hash).mapFailure
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import pkg/codex/manifest
|
|||
import pkg/codex/merkletree
|
||||
import pkg/codex/blockexchange
|
||||
import pkg/codex/rng
|
||||
import pkg/codex/utils
|
||||
|
||||
import ./helpers/nodeutils
|
||||
import ./helpers/randomchunker
|
||||
|
|
@ -27,6 +28,13 @@ export libp2p except setup, eventually
|
|||
func `==`*(a, b: Block): bool =
|
||||
(a.cid == b.cid) and (a.data == b.data)
|
||||
|
||||
proc calcEcBlocksCount*(blocksCount: int, ecK, ecM: int): int =
|
||||
let
|
||||
rounded = roundUp(blocksCount, ecK)
|
||||
steps = divUp(blocksCount, ecK)
|
||||
|
||||
rounded + (steps * ecM)
|
||||
|
||||
proc lenPrefix*(msg: openArray[byte]): seq[byte] =
|
||||
## Write `msg` with a varint-encoded length prefix
|
||||
##
|
||||
|
|
@ -94,7 +102,7 @@ proc storeDataGetManifest*(store: BlockStore, chunker: Chunker): Future[Manifest
|
|||
|
||||
for i in 0..<tree.leavesCount:
|
||||
let proof = tree.getProof(i).tryGet()
|
||||
(await store.putBlockCidAndProof(treeCid, i, cids[i], proof)).tryGet()
|
||||
(await store.putCidAndProof(treeCid, i, cids[i], proof)).tryGet()
|
||||
|
||||
return manifest
|
||||
|
||||
|
|
|
|||
|
|
@ -51,7 +51,7 @@ suite "Test CodexTree":
|
|||
var tree = CodexTree.init(leaves = expectedLeaves)
|
||||
check:
|
||||
tree.isOk
|
||||
tree.get().leaves == expectedLeaves.mapIt( it.bytes )
|
||||
tree.get().leaves == expectedLeaves.mapIt( it.digestBytes )
|
||||
tree.get().mcodec == sha256
|
||||
|
||||
test "Should build tree from cid leaves":
|
||||
|
|
@ -68,10 +68,10 @@ suite "Test CodexTree":
|
|||
|
||||
check:
|
||||
tree.isOk
|
||||
tree.get().leaves == expectedLeaves.mapIt( it.mhash.tryGet.bytes )
|
||||
tree.get().leaves == expectedLeaves.mapIt( it.mhash.tryGet.digestBytes )
|
||||
tree.get().mcodec == sha256
|
||||
|
||||
test "Should build from raw bytes (should not hash leaves)":
|
||||
test "Should build from raw digestbytes (should not hash leaves)":
|
||||
let
|
||||
tree = CodexTree.init(sha256, leaves = data).tryGet
|
||||
|
||||
|
|
@ -91,7 +91,7 @@ suite "Test CodexTree":
|
|||
tree == fromNodes
|
||||
|
||||
let
|
||||
mhash = sha256.getMhash().tryGet
|
||||
mhash = sha256.mhash().tryGet
|
||||
zero: seq[byte] = newSeq[byte](mhash.size)
|
||||
compress = proc(x, y: seq[byte], key: ByteTreeKey): seq[byte] =
|
||||
compress(x, y, key, mhash).tryGet
|
||||
|
|
|
|||
|
|
@ -29,11 +29,10 @@ const
|
|||
"0000000000000000000000000000006".toBytes,
|
||||
"0000000000000000000000000000007".toBytes,
|
||||
"0000000000000000000000000000008".toBytes,
|
||||
"0000000000000000000000000000009".toBytes,
|
||||
"0000000000000000000000000000010".toBytes,
|
||||
"0000000000000000000000000000009".toBytes, # note one less to account for padding of field elements
|
||||
]
|
||||
|
||||
suite "Test CodexTree":
|
||||
suite "Test Poseidon2Tree":
|
||||
var
|
||||
expectedLeaves: seq[Poseidon2Hash]
|
||||
|
||||
|
|
@ -54,8 +53,8 @@ suite "Test CodexTree":
|
|||
test "Init tree from byte leaves":
|
||||
let
|
||||
tree = Poseidon2Tree.init(
|
||||
leaves = data.mapIt(
|
||||
array[31, byte].initCopyFrom( it )
|
||||
leaves = expectedLeaves.mapIt(
|
||||
array[31, byte].initCopyFrom( it.toBytes )
|
||||
)).tryGet
|
||||
|
||||
check:
|
||||
|
|
|
|||
|
|
@ -0,0 +1,324 @@
|
|||
import std/sequtils
|
||||
import std/math
|
||||
import std/importutils
|
||||
import std/sugar
|
||||
|
||||
import pkg/chronos
|
||||
import pkg/asynctest
|
||||
import pkg/questionable/results
|
||||
import pkg/codex/blocktype as bt
|
||||
import pkg/codex/rng
|
||||
import pkg/codex/stores
|
||||
import pkg/codex/chunker
|
||||
import pkg/codex/merkletree
|
||||
import pkg/codex/utils
|
||||
import pkg/codex/utils/digest
|
||||
import pkg/datastore
|
||||
import pkg/poseidon2
|
||||
import pkg/poseidon2/io
|
||||
import constantine/math/io/io_fields
|
||||
|
||||
import ../helpers
|
||||
import ../examples
|
||||
import ../merkletree/helpers
|
||||
|
||||
import pkg/codex/indexingstrategy {.all.}
|
||||
import pkg/codex/slots/slotbuilder {.all.}
|
||||
|
||||
suite "Slot builder":
|
||||
let
|
||||
blockSize = 1024
|
||||
cellSize = 64
|
||||
ecK = 3
|
||||
ecM = 2
|
||||
|
||||
numSlots = ecK + ecM
|
||||
numDatasetBlocks = 100
|
||||
numBlockCells = blockSize div cellSize
|
||||
|
||||
numTotalBlocks = calcEcBlocksCount(numDatasetBlocks, ecK, ecM) # total number of blocks in the dataset after
|
||||
# EC (should will match number of slots)
|
||||
originalDatasetSize = numDatasetBlocks * blockSize # size of the dataset before EC
|
||||
totalDatasetSize = numTotalBlocks * blockSize # size of the dataset after EC
|
||||
numTotalSlotBlocks = nextPowerOfTwo(numTotalBlocks div numSlots)
|
||||
|
||||
blockPadBytes =
|
||||
newSeq[byte](numBlockCells.nextPowerOfTwoPad * cellSize) # power of two padding for blocks
|
||||
|
||||
slotsPadLeafs =
|
||||
newSeqWith((numTotalBlocks div numSlots).nextPowerOfTwoPad, Poseidon2Zero) # power of two padding for block roots
|
||||
|
||||
rootsPadLeafs =
|
||||
newSeqWith(numSlots.nextPowerOfTwoPad, Poseidon2Zero)
|
||||
|
||||
var
|
||||
datasetBlocks: seq[bt.Block]
|
||||
localStore: BlockStore
|
||||
manifest: Manifest
|
||||
protectedManifest: Manifest
|
||||
expectedEmptyCid: Cid
|
||||
slotBuilder: SlotBuilder
|
||||
chunker: Chunker
|
||||
|
||||
proc createBlocks(): Future[void] {.async.} =
|
||||
while true:
|
||||
let chunk = await chunker.getBytes()
|
||||
if chunk.len <= 0:
|
||||
break
|
||||
let blk = bt.Block.new(chunk).tryGet()
|
||||
datasetBlocks.add(blk)
|
||||
discard await localStore.putBlock(blk)
|
||||
|
||||
proc createProtectedManifest(): Future[void] {.async.} =
|
||||
let
|
||||
cids = datasetBlocks.mapIt(it.cid)
|
||||
datasetTree = CodexTree.init(cids[0..<numDatasetBlocks]).tryGet()
|
||||
datasetTreeCid = datasetTree.rootCid().tryGet()
|
||||
|
||||
protectedTree = CodexTree.init(cids).tryGet()
|
||||
protectedTreeCid = protectedTree.rootCid().tryGet()
|
||||
|
||||
for index, cid in cids[0..<numDatasetBlocks]:
|
||||
let proof = datasetTree.getProof(index).tryget()
|
||||
(await localStore.putCidAndProof(datasetTreeCid, index, cid, proof)).tryGet
|
||||
|
||||
for index, cid in cids:
|
||||
let proof = protectedTree.getProof(index).tryget()
|
||||
(await localStore.putCidAndProof(protectedTreeCid, index, cid, proof)).tryGet
|
||||
|
||||
manifest = Manifest.new(
|
||||
treeCid = datasetTreeCid,
|
||||
blockSize = blockSize.NBytes,
|
||||
datasetSize = originalDatasetSize.NBytes)
|
||||
|
||||
protectedManifest = Manifest.new(
|
||||
manifest = manifest,
|
||||
treeCid = protectedTreeCid,
|
||||
datasetSize = totalDatasetSize.NBytes,
|
||||
ecK = ecK,
|
||||
ecM = ecM)
|
||||
|
||||
let
|
||||
manifestBlock = bt.Block.new(
|
||||
manifest.encode().tryGet(),
|
||||
codec = ManifestCodec).tryGet()
|
||||
|
||||
protectedManifestBlock = bt.Block.new(
|
||||
protectedManifest.encode().tryGet(),
|
||||
codec = ManifestCodec).tryGet()
|
||||
|
||||
(await localStore.putBlock(manifestBlock)).tryGet()
|
||||
(await localStore.putBlock(protectedManifestBlock)).tryGet()
|
||||
|
||||
expectedEmptyCid = emptyCid(
|
||||
protectedManifest.version,
|
||||
protectedManifest.hcodec,
|
||||
protectedManifest.codec).tryGet()
|
||||
|
||||
privateAccess(SlotBuilder) # enable access to private fields
|
||||
|
||||
setup:
|
||||
let
|
||||
repoDs = SQLiteDatastore.new(Memory).tryGet()
|
||||
metaDs = SQLiteDatastore.new(Memory).tryGet()
|
||||
localStore = RepoStore.new(repoDs, metaDs)
|
||||
|
||||
chunker = RandomChunker.new(Rng.instance(), size = totalDatasetSize, chunkSize = blockSize)
|
||||
await createBlocks()
|
||||
await createProtectedManifest()
|
||||
|
||||
teardown:
|
||||
await localStore.close()
|
||||
|
||||
# Need to reset all objects because otherwise they get
|
||||
# captured by the test runner closures, not good!
|
||||
reset(datasetBlocks)
|
||||
reset(localStore)
|
||||
reset(manifest)
|
||||
reset(protectedManifest)
|
||||
reset(expectedEmptyCid)
|
||||
reset(slotBuilder)
|
||||
reset(chunker)
|
||||
|
||||
test "Can only create slotBuilder with protected manifest":
|
||||
let
|
||||
unprotectedManifest = Manifest.new(
|
||||
treeCid = Cid.example,
|
||||
blockSize = blockSize.NBytes,
|
||||
datasetSize = originalDatasetSize.NBytes)
|
||||
|
||||
check:
|
||||
SlotBuilder.new(localStore, unprotectedManifest, cellSize = cellSize)
|
||||
.error.msg == "Can only create SlotBuilder using protected manifests."
|
||||
|
||||
test "Number of blocks must be devisable by number of slots":
|
||||
let
|
||||
mismatchManifest = Manifest.new(
|
||||
manifest = Manifest.new(
|
||||
treeCid = Cid.example,
|
||||
blockSize = blockSize.NBytes,
|
||||
datasetSize = originalDatasetSize.NBytes),
|
||||
treeCid = Cid.example,
|
||||
datasetSize = totalDatasetSize.NBytes,
|
||||
ecK = ecK - 1,
|
||||
ecM = ecM)
|
||||
|
||||
check:
|
||||
SlotBuilder.new(localStore, mismatchManifest, cellSize = cellSize)
|
||||
.error.msg == "Number of blocks must be divisable by number of slots."
|
||||
|
||||
test "Block size must be divisable by cell size":
|
||||
let
|
||||
mismatchManifest = Manifest.new(
|
||||
manifest = Manifest.new(
|
||||
treeCid = Cid.example,
|
||||
blockSize = (blockSize + 1).NBytes,
|
||||
datasetSize = (originalDatasetSize - 1).NBytes),
|
||||
treeCid = Cid.example,
|
||||
datasetSize = (totalDatasetSize - 1).NBytes,
|
||||
ecK = ecK,
|
||||
ecM = ecM)
|
||||
|
||||
check:
|
||||
SlotBuilder.new(localStore, mismatchManifest, cellSize = cellSize)
|
||||
.error.msg == "Block size must be divisable by cell size."
|
||||
|
||||
test "Should build correct slot builder":
|
||||
slotBuilder = SlotBuilder.new(
|
||||
localStore,
|
||||
protectedManifest,
|
||||
cellSize = cellSize).tryGet()
|
||||
|
||||
check:
|
||||
slotBuilder.numBlockPadBytes == blockPadBytes.len
|
||||
slotBuilder.numSlotsPadLeafs == slotsPadLeafs.len
|
||||
slotBuilder.numRootsPadLeafs == rootsPadLeafs.len
|
||||
|
||||
test "Should build slot hashes for all slots":
|
||||
let
|
||||
steppedStrategy = SteppedIndexingStrategy.new(0, numTotalBlocks - 1, numSlots)
|
||||
slotBuilder = SlotBuilder.new(
|
||||
localStore,
|
||||
protectedManifest,
|
||||
cellSize = cellSize).tryGet()
|
||||
|
||||
for i in 0 ..< numSlots:
|
||||
let
|
||||
expectedBlock = steppedStrategy
|
||||
.getIndicies(i)
|
||||
.mapIt( datasetBlocks[it] )
|
||||
|
||||
expectedHashes: seq[Poseidon2Hash] = collect(newSeq):
|
||||
for blk in expectedBlock:
|
||||
SpongeMerkle.digest(blk.data & blockPadBytes, cellSize)
|
||||
|
||||
cellHashes = (await slotBuilder.getCellHashes(i)).tryGet()
|
||||
|
||||
check:
|
||||
expectedHashes == cellHashes
|
||||
|
||||
test "Should build slot trees for all slots":
|
||||
let
|
||||
steppedStrategy = SteppedIndexingStrategy.new(0, numTotalBlocks - 1, numSlots)
|
||||
slotBuilder = SlotBuilder.new(
|
||||
localStore,
|
||||
protectedManifest,
|
||||
cellSize = cellSize).tryGet()
|
||||
|
||||
for i in 0 ..< numSlots:
|
||||
let
|
||||
expectedBlock = steppedStrategy
|
||||
.getIndicies(i)
|
||||
.mapIt( datasetBlocks[it] )
|
||||
|
||||
expectedHashes: seq[Poseidon2Hash] = collect(newSeq):
|
||||
for blk in expectedBlock:
|
||||
SpongeMerkle.digest(blk.data & blockPadBytes, cellSize)
|
||||
expectedRoot = Merkle.digest(expectedHashes & slotsPadLeafs)
|
||||
|
||||
slotTree = (await slotBuilder.buildSlotTree(i)).tryGet()
|
||||
|
||||
check:
|
||||
expectedRoot == slotTree.root().tryGet()
|
||||
|
||||
test "Should persist trees for all slots":
|
||||
let
|
||||
slotBuilder = SlotBuilder.new(
|
||||
localStore,
|
||||
protectedManifest,
|
||||
cellSize = cellSize).tryGet()
|
||||
|
||||
for i in 0 ..< numSlots:
|
||||
let
|
||||
slotTree = (await slotBuilder.buildSlotTree(i)).tryGet()
|
||||
slotRoot = (await slotBuilder.buildSlot(i)).tryGet()
|
||||
slotCid = slotRoot.toSlotCid().tryGet()
|
||||
|
||||
for cellIndex in 0..<numTotalSlotBlocks:
|
||||
let
|
||||
(cellCid, proof) = (await localStore.getCidAndProof(slotCid, cellIndex)).tryGet()
|
||||
verifiableProof = proof.toVerifiableProof().tryGet()
|
||||
posProof = slotTree.getProof(cellIndex).tryGet
|
||||
|
||||
check:
|
||||
verifiableProof.index == posProof.index
|
||||
verifiableProof.nleaves == posProof.nleaves
|
||||
verifiableProof.path == posProof.path
|
||||
|
||||
test "Should build correct verification root":
|
||||
let
|
||||
steppedStrategy = SteppedIndexingStrategy.new(0, numTotalBlocks - 1, numSlots)
|
||||
slotBuilder = SlotBuilder.new(
|
||||
localStore,
|
||||
protectedManifest,
|
||||
cellSize = cellSize).tryGet()
|
||||
slotRoots = (await slotBuilder.buildSlots()).tryGet
|
||||
|
||||
slotsHashes = collect(newSeq):
|
||||
for i in 0 ..< numSlots:
|
||||
let
|
||||
expectedBlocks = steppedStrategy
|
||||
.getIndicies(i)
|
||||
.mapIt( datasetBlocks[it] )
|
||||
|
||||
slotHashes: seq[Poseidon2Hash] = collect(newSeq):
|
||||
for blk in expectedBlocks:
|
||||
SpongeMerkle.digest(blk.data & blockPadBytes, cellSize)
|
||||
|
||||
Merkle.digest(slotHashes & slotsPadLeafs)
|
||||
|
||||
expectedRoot = Merkle.digest(slotsHashes & rootsPadLeafs)
|
||||
rootHash = slotBuilder.buildRootsTree(slotRoots).tryGet().root.tryGet()
|
||||
|
||||
check:
|
||||
expectedRoot == rootHash
|
||||
|
||||
test "Should build correct verification root manifest":
|
||||
let
|
||||
steppedStrategy = SteppedIndexingStrategy.new(0, numTotalBlocks - 1, numSlots)
|
||||
slotBuilder = SlotBuilder.new(
|
||||
localStore,
|
||||
protectedManifest,
|
||||
cellSize = cellSize).tryGet()
|
||||
|
||||
slotsHashes = collect(newSeq):
|
||||
for i in 0 ..< numSlots:
|
||||
let
|
||||
expectedBlocks = steppedStrategy
|
||||
.getIndicies(i)
|
||||
.mapIt( datasetBlocks[it] )
|
||||
|
||||
slotHashes: seq[Poseidon2Hash] = collect(newSeq):
|
||||
for blk in expectedBlocks:
|
||||
SpongeMerkle.digest(blk.data & blockPadBytes, cellSize)
|
||||
|
||||
Merkle.digest(slotHashes & slotsPadLeafs)
|
||||
|
||||
expectedRoot = Merkle.digest(slotsHashes & rootsPadLeafs)
|
||||
manifest = (await slotBuilder.buildManifest()).tryGet()
|
||||
mhash = manifest.verificationRoot.mhash.tryGet()
|
||||
mhashBytes = mhash.digestBytes
|
||||
rootHash = Poseidon2Hash.fromBytes(mhashBytes.toArray32).toResult.tryGet()
|
||||
|
||||
check:
|
||||
expectedRoot == rootHash
|
||||
|
|
@ -15,7 +15,7 @@ import pkg/codex/utils
|
|||
|
||||
import ./helpers
|
||||
|
||||
asyncchecksuite "Erasure encode/decode":
|
||||
suite "Erasure encode/decode":
|
||||
const BlockSize = 1024'nb
|
||||
const dataSetSize = BlockSize * 123 # weird geometry
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,65 @@
|
|||
import std/sequtils
|
||||
import pkg/chronos
|
||||
import pkg/asynctest
|
||||
|
||||
import ./helpers
|
||||
|
||||
import pkg/codex/indexingstrategy
|
||||
|
||||
for offset in @[0, 1, 2, 100]:
|
||||
suite "Indexing strategies (Offset: " & $offset & ")":
|
||||
let
|
||||
firstIndex = 0 + offset
|
||||
lastIndex = 12 + offset
|
||||
nIters = 3
|
||||
linear = LinearIndexingStrategy.new(firstIndex, lastIndex, nIters)
|
||||
stepped = SteppedIndexingStrategy.new(firstIndex, lastIndex, nIters)
|
||||
|
||||
test "linear":
|
||||
check:
|
||||
linear.getIndicies(0) == @[0, 1, 2, 3, 4].mapIt(it + offset)
|
||||
linear.getIndicies(1) == @[5, 6, 7, 8, 9].mapIt(it + offset)
|
||||
linear.getIndicies(2) == @[10, 11, 12].mapIt(it + offset)
|
||||
|
||||
test "stepped":
|
||||
check:
|
||||
stepped.getIndicies(0) == @[0, 3, 6, 9, 12].mapIt(it + offset)
|
||||
stepped.getIndicies(1) == @[1, 4, 7, 10].mapIt(it + offset)
|
||||
stepped.getIndicies(2) == @[2, 5, 8, 11].mapIt(it + offset)
|
||||
|
||||
suite "Indexing strategies":
|
||||
let
|
||||
linear = LinearIndexingStrategy.new(0, 10, 3)
|
||||
stepped = SteppedIndexingStrategy.new(0, 10, 3)
|
||||
|
||||
test "smallest range 0":
|
||||
let
|
||||
l = LinearIndexingStrategy.new(0, 0, 1)
|
||||
s = SteppedIndexingStrategy.new(0, 0, 1)
|
||||
check:
|
||||
l.getIndicies(0) == @[0]
|
||||
s.getIndicies(0) == @[0]
|
||||
|
||||
test "smallest range 1":
|
||||
let
|
||||
l = LinearIndexingStrategy.new(0, 1, 1)
|
||||
s = SteppedIndexingStrategy.new(0, 1, 1)
|
||||
check:
|
||||
l.getIndicies(0) == @[0, 1]
|
||||
s.getIndicies(0) == @[0, 1]
|
||||
|
||||
test "first index must be smaller than last index":
|
||||
expect AssertionDefect:
|
||||
discard LinearIndexingStrategy.new(10, 0, 1)
|
||||
|
||||
test "numberOfIterations must be greater than zero":
|
||||
expect AssertionDefect:
|
||||
discard LinearIndexingStrategy.new(0, 10, 0)
|
||||
|
||||
test "linear - oob":
|
||||
expect AssertionDefect:
|
||||
discard linear.getIndicies(3)
|
||||
|
||||
test "stepped - oob":
|
||||
expect AssertionDefect:
|
||||
discard stepped.getIndicies(3)
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
import ./slotbuilder/testslotbuilder
|
||||
|
||||
{.warning[UnusedImport]: off.}
|
||||
|
|
@ -14,5 +14,7 @@ import ./codex/testsystemclock
|
|||
import ./codex/testvalidation
|
||||
import ./codex/testasyncstreamwrapper
|
||||
import ./codex/testmerkletree
|
||||
import ./codex/testslotbuilder
|
||||
import ./codex/testindexingstrategy
|
||||
|
||||
{.warning[UnusedImport]: off.}
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
Subproject commit 4f2259e1cef65085d092b2b713bb67f5aac55626
|
||||
Subproject commit b239791c568d9f9a76fd66d2322b2754700b6cc5
|
||||
|
|
@ -1 +1 @@
|
|||
Subproject commit 0cfecf7d780b8c3295d11251d64a49f3c7258fbf
|
||||
Subproject commit 3b403b0752790438bed6342b431412cc05474acb
|
||||
Loading…
Reference in New Issue