Integrate slot builder (#666)

* rework merkle tree support

* rename merkletree -> codexmerkletree

* treed and proof encoding/decoding

* style

* adding codex merkle and coders tests

* use default hash codec

* proof size changed

* add from nodes test

* shorte file names

* wip poseidon tree

* shorten file names

* root returns a result

* import poseidon tests

* fix merge issues and cleanup a few warnings

* setting up slot builder

* Getting cids in slot

* ensures blocks are devisable by number of slots

* wip

* Implements indexing strategies

* Swaps in indexing strategy into erasure.

* wires slot and indexing tests up

* Fixes issue where indexing strategy stepped gives wrong values for smallest of ranges

* debugs indexing strategies

* Can select slot blocks

* finding number of pad cells

* Implements building slot tree

* finishes implementing slot builder

* Adds check that block size is a multiple of cell size

* Cleanup slotbuilder

* Review comments by Tomasz

* Fixes issue where ecK was used as numberOfSlots.

* rework merkle tree support

* deps

* rename merkletree -> codexmerkletree

* treed and proof encoding/decoding

* style

* adding codex merkle and coders tests

* remove new codecs for now

* proof size changed

* add from nodes test

* shorte file names

* wip poseidon tree

* shorten file names

* fix bad `elements` iter

* bump

* bump

* wip

* reworking slotbuilder

* move out of manifest

* expose getCidAndProof

* import index strat...

* remove getMHash

* remove unused artifacts

* alias zero

* add digest for multihash

* merge issues

* remove unused hashes

* add option to result converter

* misc

* fix tests

* add helper to derive EC block count

* rename method

* misc

* bump

* extract slot root building into own proc

* revert to manifest to accessor

---------

Co-authored-by: benbierens <thatbenbierens@gmail.com>

codex/blockexchange/engine/engine.nim

@@ -361,7 +361,12 @@ proc blocksDeliveryHandler*(
       without proof =? bd.proof:
         error "Proof expected for a leaf block delivery"
         continue
-      if err =? (await b.localStore.putBlockCidAndProof(bd.address.treeCid, bd.address.index, bd.blk.cid, proof)).errorOption:
+      if err =? (await b.localStore.putCidAndProof(
+          bd.address.treeCid,
+          bd.address.index,
+          bd.blk.cid,
+          proof)).errorOption:
+
         error "Unable to store proof and cid for a block"
         continue
 

codex/codextypes.nim

@@ -39,6 +39,7 @@ const
   BlockCodec* = multiCodec("codex-block")
   SlotRootCodec* = multiCodec("codex-slot-root")
   SlotProvingRootCodec* = multiCodec("codex-proving-root")
+  CodexSlotCell* = multiCodec("codex-slot-cell")
 
   CodexHashesCodecs* = [
     Sha256HashCodec,
@@ -52,15 +53,15 @@ const
     BlockCodec,
     SlotRootCodec,
     SlotProvingRootCodec,
+    CodexSlotCell,
   ]
 
-
 proc initEmptyCidTable(): ?!Table[(CidVersion, MultiCodec, MultiCodec), Cid] =
   ## Initialize padding blocks table
   ##
   ## TODO: Ideally this is done at compile time, but for now
   ## we do it at runtime because of an `importc` error that is
-  ## coming from somwhere in MultiHash that I can't track down.
+  ## coming from somewhere in MultiHash that I can't track down.
   ##
 
   let
@@ -68,8 +69,6 @@ proc initEmptyCidTable(): ?!Table[(CidVersion, MultiCodec, MultiCodec), Cid] =
     PadHashes = {
       Sha256HashCodec: ? MultiHash.digest($Sha256HashCodec, emptyData).mapFailure,
       Sha512HashCodec: ? MultiHash.digest($Sha512HashCodec, emptyData).mapFailure,
-      Pos2Bn128SpngCodec: ? MultiHash.digest($Pos2Bn128SpngCodec, emptyData).mapFailure,
-      Pos2Bn128MrklCodec: ? MultiHash.digest($Pos2Bn128SpngCodec, emptyData).mapFailure,
     }.toTable
 
   var

codex/erasure/erasure.nim

@@ -25,6 +25,7 @@ import ../stores
 import ../blocktype as bt
 import ../utils
 import ../utils/asynciter
+import ../indexingstrategy
 
 import pkg/stew/byteutils
 
@@ -127,7 +128,12 @@ proc prepareEncodingData(
   ##
 
   let
-    indicies = toSeq(countup(step, params.rounded - 1, params.steps))
+    strategy = SteppedIndexingStrategy.new(
+      firstIndex = 0,
+      lastIndex = params.rounded - 1,
+      numberOfIterations = params.steps
+    )
+    indicies = strategy.getIndicies(step)
     pendingBlocksIter = self.getPendingBlocks(manifest, indicies.filterIt(it < manifest.blocksCount))
 
   var resolved = 0
@@ -171,7 +177,12 @@ proc prepareDecodingData(
   ##
 
   let
-    indicies = toSeq(countup(step, encoded.blocksCount - 1, encoded.steps))
+    strategy = SteppedIndexingStrategy.new(
+      firstIndex = 0,
+      lastIndex = encoded.blocksCount - 1,
+      numberOfIterations = encoded.steps
+    )
+    indicies = strategy.getIndicies(step)
     pendingBlocksIter = self.getPendingBlocks(encoded, indicies)
 
   var
@@ -213,7 +224,10 @@ proc prepareDecodingData(
 
   return success (dataPieces, parityPieces)
 
-proc init(_: type EncodingParams, manifest: Manifest, ecK: int, ecM: int): ?!EncodingParams =
+proc init*(
+  _: type EncodingParams,
+  manifest: Manifest,
+  ecK: int, ecM: int): ?!EncodingParams =
   if ecK > manifest.blocksCount:
     return failure("Unable to encode manifest, not enough blocks, ecK = " & $ecK & ", blocksCount = " & $manifest.blocksCount)
 

codex/errors.nim

@@ -7,6 +7,8 @@
 ## This file may not be copied, modified, or distributed except according to
 ## those terms.
 
+import std/options
+
 import pkg/stew/results
 import pkg/chronos
 import pkg/questionable/results
@@ -29,6 +31,12 @@ template mapFailure*[T, V, E](
 template mapFailure*[T, V](exp: Result[T, V]): Result[T, ref CatchableError] =
   mapFailure(exp, CodexError)
 
+template toResult*[T](exp: Option[T]): Result[T, ref CatchableError] =
+  if exp.isSome:
+    success exp.get
+  else:
+    T.failure("Option is None")
+
 proc allFutureResult*[T](fut: seq[Future[T]]): Future[?!void] {.async.} =
   try:
     await allFuturesThrowing(fut)

codex/indexingstrategy.nim

@@ -0,0 +1,61 @@
+import std/sequtils
+import ./utils
+
+# I'm choosing to use an assert here because:
+# 1. These are a programmer errors and *should not* happen during application runtime.
+# 2. Users don't have to deal with Result types.
+
+type
+  # Representing a strategy for grouping indices (of blocks usually)
+  # Given an interation-count as input, will produce a seq of
+  # selected indices.
+  IndexingStrategy* = ref object of RootObj
+    firstIndex*: int             # Lowest index that can be returned
+    lastIndex*: int              # Highest index that can be returned
+    numberOfIterations*: int     # getIndices(iteration) will run from 0 ..< numberOfIterations
+    step*: int
+
+  # Simplest approach:
+  # 0 => 0, 1, 2
+  # 1 => 3, 4, 5
+  # 2 => 6, 7, 8
+  LinearIndexingStrategy* = ref object of IndexingStrategy
+
+  # Stepped indexing:
+  # 0 => 0, 3, 6
+  # 1 => 1, 4, 7
+  # 2 => 2, 5, 8
+  SteppedIndexingStrategy* = ref object of IndexingStrategy
+
+proc assertIteration(self: IndexingStrategy, iteration: int): void =
+  if iteration >= self.numberOfIterations:
+    raiseAssert("Indexing iteration can't be greater than or equal to numberOfIterations.")
+
+method getIndicies*(self: IndexingStrategy, iteration: int): seq[int] {.base.} =
+  raiseAssert("Not implemented")
+
+proc new*(T: type IndexingStrategy, firstIndex, lastIndex, numberOfIterations: int): T =
+  if firstIndex > lastIndex:
+    raiseAssert("firstIndex (" & $firstIndex & ") can't be greater than lastIndex (" & $lastIndex & ")")
+  if numberOfIterations <= 0:
+    raiseAssert("numberOfIteration (" & $numberOfIterations & ") must be greater than zero.")
+
+  T(
+    firstIndex: firstIndex,
+    lastIndex: lastIndex,
+    numberOfIterations: numberOfIterations,
+    step: divUp((lastIndex - firstIndex), numberOfIterations)
+  )
+
+method getIndicies*(self: LinearIndexingStrategy, iteration: int): seq[int] =
+  self.assertIteration(iteration)
+
+  let
+    first = self.firstIndex + iteration * (self.step + 1)
+    last = min(first + self.step, self.lastIndex)
+
+  toSeq(countup(first, last, 1))
+
+method getIndicies*(self: SteppedIndexingStrategy, iteration: int): seq[int] =
+  self.assertIteration(iteration)
+  toSeq(countup(self.firstIndex + iteration, self.lastIndex, self.numberOfIterations))

codex/manifest/manifest.nim

@@ -98,6 +98,11 @@ proc verificationRoot*(self: Manifest): Cid =
 proc slotRoots*(self: Manifest): seq[Cid] =
   self.slotRoots
 
+proc numSlots*(self: Manifest): int =
+  if not self.protected:
+    0
+  else:
+    self.ecK + self.ecM
 ############################################################
 # Operations on block list
 ############################################################

codex/merkletree/codex/codex.nim

@@ -13,7 +13,6 @@ push: {.upraises: [].}
 
 import std/bitops
 import std/sequtils
-
 import pkg/questionable
 import pkg/questionable/results
 import pkg/libp2p/[cid, multicodec, multihash]
@@ -40,15 +39,15 @@ type
 
   ByteHash* = seq[byte]
   ByteTree* = MerkleTree[ByteHash, ByteTreeKey]
-  ByteTreeProof* = MerkleProof[ByteHash, ByteTreeKey]
+  ByteProof* = MerkleProof[ByteHash, ByteTreeKey]
 
   CodexTree* = ref object of ByteTree
-    mhash: MHash
+    mcodec*: MultiCodec
 
-  CodexProof* = ref object of ByteTreeProof
+  CodexProof* = ref object of ByteProof
-    mhash: MHash
+    mcodec*: MultiCodec
 
-func getMhash*(mcodec: MultiCodec): ?!MHash =
+func mhash*(mcodec: MultiCodec): ?!MHash =
   let
     mhash = CodeHashes.getOrDefault(mcodec)
 
@@ -63,21 +62,15 @@ func digestSize*(self: (CodexTree or CodexProof)): int =
 
   self.mhash.size
 
-func mcodec*(self: (CodexTree or CodexProof)): MultiCodec =
+func digestBytes*(mhash: MultiHash): seq[byte] =
-  ## Multicodec
+  ## Extract hash digestBytes
-  ##
-
-  self.mhash.mcodec
-
-func bytes*(mhash: MultiHash): seq[byte] =
-  ## Extract hash bytes
   ##
 
   mhash.data.buffer[mhash.dpos..<mhash.dpos + mhash.size]
 
 func getProof*(self: CodexTree, index: int): ?!CodexProof =
   var
-    proof = CodexProof(mhash: self.mhash)
+    proof = CodexProof(mcodec: self.mcodec)
 
   ? self.getProof(index, proof)
 
@@ -88,8 +81,8 @@ func verify*(self: CodexProof, leaf: MultiHash, root: MultiHash): ?!void =
   ##
 
   let
-    rootBytes = root.bytes
+    rootBytes = root.digestBytes
-    leafBytes = leaf.bytes
+    leafBytes = leaf.digestBytes
 
   if self.mcodec != root.mcodec or
     self.mcodec != leaf.mcodec:
@@ -166,7 +159,7 @@ func init*(
     return failure "Empty leaves"
 
   let
-    mhash = ? mcodec.getMhash()
+    mhash = ? mcodec.mhash()
     compressor = proc(x, y: seq[byte], key: ByteTreeKey): ?!ByteHash {.noSideEffect.} =
       compress(x, y, key, mhash)
     Zero: ByteHash = newSeq[byte](mhash.size)
@@ -175,7 +168,7 @@ func init*(
     return failure "Invalid hash length"
 
   var
-    self = CodexTree(mhash: mhash, compress: compressor, zero: Zero)
+    self = CodexTree(mcodec: mcodec, compress: compressor, zero: Zero)
 
   self.layers = ? merkleTreeWorker(self, leaves, isBottomLayer = true)
   success self
@@ -189,7 +182,7 @@ func init*(
 
   let
     mcodec = leaves[0].mcodec
-    leaves = leaves.mapIt( it.bytes )
+    leaves = leaves.mapIt( it.digestBytes )
 
   CodexTree.init(mcodec, leaves)
 
@@ -201,7 +194,7 @@ func init*(
 
   let
     mcodec = (? leaves[0].mhash.mapFailure).mcodec
-    leaves = leaves.mapIt( (? it.mhash.mapFailure).bytes )
+    leaves = leaves.mapIt( (? it.mhash.mapFailure).digestBytes )
 
   CodexTree.init(mcodec, leaves)
 
@@ -215,7 +208,7 @@ proc fromNodes*(
     return failure "Empty nodes"
 
   let
-    mhash = ? mcodec.getMhash()
+    mhash = ? mcodec.mhash()
     Zero = newSeq[byte](mhash.size)
     compressor = proc(x, y: seq[byte], key: ByteTreeKey): ?!ByteHash {.noSideEffect.} =
       compress(x, y, key, mhash)
@@ -224,7 +217,7 @@ proc fromNodes*(
     return failure "Invalid hash length"
 
   var
-    self = CodexTree(compress: compressor, zero: Zero, mhash: mhash)
+    self = CodexTree(compress: compressor, zero: Zero, mcodec: mcodec)
     layer = nleaves
     pos = 0
 
@@ -251,16 +244,15 @@ func init*(
     return failure "Empty nodes"
 
   let
-    mhash = ? mcodec.getMhash()
+    mhash = ? mcodec.mhash()
     Zero = newSeq[byte](mhash.size)
     compressor = proc(x, y: seq[byte], key: ByteTreeKey): ?!seq[byte] {.noSideEffect.} =
       compress(x, y, key, mhash)
 
-
   success CodexProof(
     compress: compressor,
     zero: Zero,
-    mhash: mhash,
+    mcodec: mcodec,
     index: index,
     nleaves: nleaves,
     path: @nodes)

codex/merkletree/merkletree.nim

@@ -16,13 +16,6 @@ import pkg/questionable/results
 import ../errors
 
 type
-  PutFn*[H] = proc(i: Natural, x: H): ?!void {.noSideEffect, raises: [].}
-  GetFn*[H] = proc(i: Natural): ?!H {.noSideEffect, raises: [].}
-
-  StoreBackend*[H] = object
-    put: PutFn[H]
-    get: GetFn[H]
-
   CompressFn*[H, K] = proc (x, y: H, key: K): ?!H {.noSideEffect, raises: [].}
 
   MerkleTree*[H, K] = ref object of RootObj

codex/merkletree/poseidon2.nim

@@ -27,6 +27,8 @@ const
   KeyOddF               = F.fromhex("0x2")
   KeyOddAndBottomLayerF = F.fromhex("0x3")
 
+  Poseidon2Zero* = zero
+
 type
   Poseidon2Hash* = F
 
@@ -39,6 +41,9 @@ type
   Poseidon2Tree* = MerkleTree[Poseidon2Hash, PoseidonKeysEnum]
   Poseidon2Proof* = MerkleProof[Poseidon2Hash, PoseidonKeysEnum]
 
+func toArray32*(bytes: openArray[byte]): array[32, byte] =
+  result[0..<bytes.len] = bytes[0..<bytes.len]
+
 converter toKey*(key: PoseidonKeysEnum): Poseidon2Hash =
   case key:
   of KeyNone: KeyNoneF
@@ -60,7 +65,7 @@ func init*(
       success compress( x, y, key.toKey )
 
   var
-    self = Poseidon2Tree(compress: compressor, zero: zero)
+    self = Poseidon2Tree(compress: compressor, zero: Poseidon2Zero)
 
   self.layers = ? merkleTreeWorker(self, leaves, isBottomLayer = true)
   success self

codex/node.nim

@@ -251,7 +251,7 @@ proc store*(
   for index, cid in cids:
     without proof =? tree.getProof(index), err:
       return failure(err)
-    if err =? (await self.blockStore.putBlockCidAndProof(treeCid, index, cid, proof)).errorOption:
+    if err =? (await self.blockStore.putCidAndProof(treeCid, index, cid, proof)).errorOption:
       # TODO add log here
       return failure(err)
 

codex/slots/slotbuilder.nim

@@ -0,0 +1,275 @@
+## Nim-Codex
+## Copyright (c) 2023 Status Research & Development GmbH
+## Licensed under either of
+##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+## at your option.
+## This file may not be copied, modified, or distributed except according to
+## those terms.
+
+{.push raises: [].}
+
+import std/math
+import std/sequtils
+import std/sugar
+
+import pkg/libp2p
+import pkg/chronos
+import pkg/chronicles
+import pkg/questionable/results
+import pkg/poseidon2
+import pkg/poseidon2/io
+
+import ../indexingstrategy
+import ../merkletree
+import ../stores
+import ../manifest
+import ../utils
+import ../utils/digest
+
+const
+  # TODO: Unified with the CellSize specified in branch "data-sampler"
+  # Number of bytes in a cell. A cell is the smallest unit of data used
+  # in the proving circuit.
+  CellSize* = 2048
+
+type
+  SlotBuilder* = object of RootObj
+    store: BlockStore
+    manifest: Manifest
+    strategy: IndexingStrategy
+    cellSize: int
+    blockPadBytes: seq[byte]
+    slotsPadLeafs: seq[Poseidon2Hash]
+    rootsPadLeafs: seq[Poseidon2Hash]
+
+func numBlockPadBytes*(self: SlotBuilder): Natural =
+  ## Number of padding bytes required for a pow2
+  ## merkle tree for each block.
+  ##
+
+  self.blockPadBytes.len
+
+func numSlotsPadLeafs*(self: SlotBuilder): Natural =
+  ## Number of padding field elements required for a pow2
+  ## merkle tree for each slot.
+  ##
+
+  self.slotsPadLeafs.len
+
+func numRootsPadLeafs*(self: SlotBuilder): Natural =
+  ## Number of padding field elements required for a pow2
+  ## merkle tree for the slot roots.
+  ##
+
+  self.rootsPadLeafs.len
+
+func numSlotBlocks*(self: SlotBuilder): Natural =
+  ## Number of blocks per slot.
+  ##
+
+  self.manifest.blocksCount div self.manifest.numSlots
+
+func numBlockRoots*(self: SlotBuilder): Natural =
+  ## Number of cells per block.
+  ##
+
+  self.manifest.blockSize.int div self.cellSize
+
+func toCellCid*(cell: Poseidon2Hash): ?!Cid =
+  let
+    cellMhash = ? MultiHash.init(Pos2Bn128MrklCodec, cell.toBytes).mapFailure
+    cellCid = ? Cid.init(CIDv1, CodexSlotCell, cellMhash).mapFailure
+
+  success cellCid
+
+func toSlotCid*(root: Poseidon2Hash): ?!Cid =
+  let
+    mhash = ? MultiHash.init($multiCodec("identity"), root.toBytes).mapFailure
+    treeCid = ? Cid.init(CIDv1, SlotRootCodec, mhash).mapFailure
+
+  success treeCid
+
+func toProvingCid*(root: Poseidon2Hash): ?!Cid =
+  let
+    mhash = ? MultiHash.init($multiCodec("identity"), root.toBytes).mapFailure
+    treeCid = ? Cid.init(CIDv1, SlotProvingRootCodec, mhash).mapFailure
+
+  success treeCid
+
+func mapToSlotCids*(slotRoots: seq[Poseidon2Hash]): ?!seq[Cid] =
+  success slotRoots.mapIt( ? it.toSlotCid )
+
+func toEncodableProof*(
+  proof: Poseidon2Proof): ?!CodexProof =
+
+  let
+    encodableProof = CodexProof(
+      mcodec: multiCodec("identity"), # copy bytes as is
+      index: proof.index,
+      nleaves: proof.nleaves,
+      path: proof.path.mapIt( @(it.toBytes) ))
+
+  success encodableProof
+
+func toVerifiableProof*(
+  proof: CodexProof): ?!Poseidon2Proof =
+
+  let
+    verifiableProof = Poseidon2Proof(
+      index: proof.index,
+      nleaves: proof.nleaves,
+      path: proof.path.mapIt(
+        ? Poseidon2Hash.fromBytes(it.toArray32).toResult
+      ))
+
+  success verifiableProof
+
+proc getCellHashes*(
+  self: SlotBuilder,
+  slotIndex: int): Future[?!seq[Poseidon2Hash]] {.async.} =
+
+  let
+    treeCid = self.manifest.treeCid
+    blockCount = self.manifest.blocksCount
+    numberOfSlots = self.manifest.numSlots
+
+  logScope:
+    treeCid = treeCid
+    blockCount = blockCount
+    numberOfSlots = numberOfSlots
+    index = blockIndex
+    slotIndex = slotIndex
+
+  let
+    hashes: seq[Poseidon2Hash] = collect(newSeq):
+      for blockIndex in self.strategy.getIndicies(slotIndex):
+        trace "Getting block CID for tree at index"
+
+        without blk =? (await self.store.getBlock(treeCid, blockIndex)), err:
+          error "Failed to get block CID for tree at index"
+          return failure(err)
+
+        without digest =? Poseidon2Tree.digest(blk.data & self.blockPadBytes, self.cellSize), err:
+          error "Failed to create digest for block"
+          return failure(err)
+
+        digest
+
+  success hashes
+
+proc buildSlotTree*(
+  self: SlotBuilder,
+  slotIndex: int): Future[?!Poseidon2Tree] {.async.} =
+  without cellHashes =? (await self.getCellHashes(slotIndex)), err:
+    error "Failed to select slot blocks", err = err.msg
+    return failure(err)
+
+  Poseidon2Tree.init(cellHashes & self.slotsPadLeafs)
+
+proc buildSlot*(
+  self: SlotBuilder,
+  slotIndex: int): Future[?!Poseidon2Hash] {.async.} =
+  ## Build a slot tree and store it in the block store.
+  ##
+
+  without tree =? (await self.buildSlotTree(slotIndex)) and
+    treeCid =? tree.root.?toSlotCid, err:
+    error "Failed to build slot tree", err = err.msg
+    return failure(err)
+
+  trace "Storing slot tree", treeCid, slotIndex, leaves = tree.leavesCount
+  for i, leaf in tree.leaves:
+    without cellCid =? leaf.toCellCid, err:
+      error "Failed to get CID for slot cell", err = err.msg
+      return failure(err)
+
+    without proof =? tree.getProof(i) and
+      encodableProof =? proof.toEncodableProof, err:
+      error "Failed to get proof for slot tree", err = err.msg
+      return failure(err)
+
+    if err =? (await self.store.putCidAndProof(
+      treeCid, i, cellCid, encodableProof)).errorOption:
+      error "Failed to store slot tree", err = err.msg
+      return failure(err)
+
+  tree.root()
+
+proc buildSlots*(self: SlotBuilder): Future[?!seq[Poseidon2Hash]] {.async.} =
+  let
+    slotRoots: seq[Poseidon2Hash] = collect(newSeq):
+      for i in 0..<self.manifest.numSlots:
+        without root =? (await self.buildSlot(i)), err:
+          error "Failed to build slot", err = err.msg, index = i
+          return failure(err)
+        root
+
+  success slotRoots
+
+func buildRootsTree*(
+  self: SlotBuilder,
+  slotRoots: seq[Poseidon2Hash]): ?!Poseidon2Tree =
+  Poseidon2Tree.init(slotRoots & self.rootsPadLeafs)
+
+proc buildManifest*(self: SlotBuilder): Future[?!Manifest] {.async.} =
+
+  without slotRoots =? await self.buildSlots(), err:
+    error "Failed to build slot roots", err = err.msg
+    return failure(err)
+
+  without provingRootCid =? self.buildRootsTree(slotRoots).?root.?toProvingCid, err:
+    error "Failed to build proving tree", err = err.msg
+    return failure(err)
+
+  without rootCids =? slotRoots.mapToSlotCids(), err:
+    error "Failed to map slot roots to CIDs", err = err.msg
+    return failure(err)
+
+  Manifest.new(self.manifest, provingRootCid, rootCids)
+
+func nextPowerOfTwoPad*(a: int): int =
+  ## Returns the difference between the original
+  ## value and the next power of two.
+  ##
+
+  nextPowerOfTwo(a) - a
+
+proc new*(
+  T: type SlotBuilder,
+  store: BlockStore,
+  manifest: Manifest,
+  strategy: IndexingStrategy = nil,
+  cellSize = CellSize): ?!SlotBuilder =
+
+  if not manifest.protected:
+    return failure("Can only create SlotBuilder using protected manifests.")
+
+  if (manifest.blocksCount mod manifest.numSlots) != 0:
+    return failure("Number of blocks must be divisable by number of slots.")
+
+  if (manifest.blockSize.int mod cellSize) != 0:
+    return failure("Block size must be divisable by cell size.")
+
+  let
+    strategy = if strategy == nil:
+      SteppedIndexingStrategy.new(
+        0, manifest.blocksCount - 1, manifest.numSlots)
+      else:
+        strategy
+
+    # all trees have to be padded to power of two
+    numBlockCells = manifest.blockSize.int div cellSize                       # number of cells per block
+    blockPadBytes = newSeq[byte](numBlockCells.nextPowerOfTwoPad * cellSize)  # power of two padding for blocks
+    numSlotLeafs = (manifest.blocksCount div manifest.numSlots)
+    slotsPadLeafs = newSeqWith(numSlotLeafs.nextPowerOfTwoPad, Poseidon2Zero) # power of two padding for block roots
+    rootsPadLeafs = newSeqWith(manifest.numSlots.nextPowerOfTwoPad, Poseidon2Zero)
+
+  success SlotBuilder(
+    store: store,
+    manifest: manifest,
+    strategy: strategy,
+    cellSize: cellSize,
+    blockPadBytes: blockPadBytes,
+    slotsPadLeafs: slotsPadLeafs,
+    rootsPadLeafs: rootsPadLeafs)

codex/stores/blockstore.nim

@@ -43,6 +43,11 @@ method getBlock*(self: BlockStore, treeCid: Cid, index: Natural): Future[?!Block
 
   raiseAssert("getBlock by treecid not implemented!")
 
+method getCid*(self: BlockStore, treeCid: Cid, index: Natural): Future[?!Cid] {.base.} =
+  ## Get a cid given a tree and index
+  ##
+  raiseAssert("getCid by treecid not implemented!")
+
 method getBlock*(self: BlockStore, address: BlockAddress): Future[?!Block] {.base.} =
   ## Get a block from the blockstore
   ##
@@ -65,17 +70,26 @@ method putBlock*(
 
   raiseAssert("putBlock not implemented!")
 
-method putBlockCidAndProof*(
+method putCidAndProof*(
   self: BlockStore,
   treeCid: Cid,
   index: Natural,
   blockCid: Cid,
   proof: CodexProof
 ): Future[?!void] {.base.} =
-  ## Put a block to the blockstore
+  ## Put a block proof to the blockstore
   ##
 
-  raiseAssert("putBlockCidAndProof not implemented!")
+  raiseAssert("putCidAndProof not implemented!")
+
+method getCidAndProof*(
+  self: BlockStore,
+  treeCid: Cid,
+  index: Natural): Future[?!(Cid, CodexProof)] {.base.} =
+  ## Get a block proof from the blockstore
+  ##
+
+  raiseAssert("putCidAndProof not implemented!")
 
 method ensureExpiry*(
     self: BlockStore,

codex/stores/cachestore.nim

@@ -65,20 +65,24 @@ method getBlock*(self: CacheStore, cid: Cid): Future[?!Block] {.async.} =
     trace "Error requesting block from cache", cid, error = exc.msg
     return failure exc
 
-proc getCidAndProof(self: CacheStore, treeCid: Cid, index: Natural): ?!(Cid, CodexProof) =
+method getCidAndProof*(
+  self: CacheStore,
+  treeCid: Cid,
+  index: Natural): Future[?!(Cid, CodexProof)] {.async.} =
+
   if cidAndProof =? self.cidAndProofCache.getOption((treeCid, index)):
     success(cidAndProof)
   else:
     failure(newException(BlockNotFoundError, "Block not in cache: " & $BlockAddress.init(treeCid, index)))
 
 method getBlock*(self: CacheStore, treeCid: Cid, index: Natural): Future[?!Block] {.async.} =
-  without cidAndProof =? self.getCidAndProof(treeCid, index), err:
+  without cidAndProof =? (await self.getCidAndProof(treeCid, index)), err:
     return failure(err)
 
   await self.getBlock(cidAndProof[0])
 
 method getBlockAndProof*(self: CacheStore, treeCid: Cid, index: Natural): Future[?!(Block, CodexProof)] {.async.} =
-  without cidAndProof =? self.getCidAndProof(treeCid, index), err:
+  without cidAndProof =? (await self.getCidAndProof(treeCid, index)), err:
     return failure(err)
 
   let (cid, proof) = cidAndProof
@@ -106,7 +110,7 @@ method hasBlock*(self: CacheStore, cid: Cid): Future[?!bool] {.async.} =
   return (cid in self.cache).success
 
 method hasBlock*(self: CacheStore, treeCid: Cid, index: Natural): Future[?!bool] {.async.} =
-  without cidAndProof =? self.getCidAndProof(treeCid, index), err:
+  without cidAndProof =? (await self.getCidAndProof(treeCid, index)), err:
     if err of BlockNotFoundError:
       return success(false)
     else:
@@ -114,7 +118,6 @@ method hasBlock*(self: CacheStore, treeCid: Cid, index: Natural): Future[?!bool]
 
   await self.hasBlock(cidAndProof[0])
 
-
 func cids(self: CacheStore): (iterator: Cid {.gcsafe.}) =
   return iterator(): Cid =
     for cid in self.cache.keys:
@@ -210,7 +213,7 @@ method putBlock*(
   discard self.putBlockSync(blk)
   return success()
 
-method putBlockCidAndProof*(
+method putCidAndProof*(
   self: CacheStore,
   treeCid: Cid,
   index: Natural,

codex/stores/networkstore.nim

@@ -78,13 +78,13 @@ method putBlock*(
   await self.engine.resolveBlocks(@[blk])
   return success()
 
-method putBlockCidAndProof*(
+method putCidAndProof*(
   self: NetworkStore,
   treeCid: Cid,
   index: Natural,
   blockCid: Cid,
   proof: CodexProof): Future[?!void] =
-  self.localStore.putBlockCidAndProof(treeCid, index, blockCid, proof)
+  self.localStore.putCidAndProof(treeCid, index, blockCid, proof)
 
 method ensureExpiry*(
   self: NetworkStore,

codex/stores/repostore.nim

@@ -106,7 +106,7 @@ proc decodeCid(_: type (Cid, CodexProof), data: seq[byte]): ?!Cid =
     cid = ? Cid.init(data[sizeof(uint64)..<sizeof(uint64) + n]).mapFailure
   success(cid)
 
-method putBlockCidAndProof*(
+method putCidAndProof*(
   self: RepoStore,
   treeCid: Cid,
   index: Natural,
@@ -125,11 +125,10 @@ method putBlockCidAndProof*(
 
   await self.metaDs.put(key, value)
 
-proc getCidAndProof(
+method getCidAndProof*(
   self: RepoStore,
   treeCid: Cid,
-  index: Natural
+  index: Natural): Future[?!(Cid, CodexProof)] {.async.} =
-): Future[?!(Cid, CodexProof)] {.async.} =
   without key =? createBlockCidAndProofMetadataKey(treeCid, index), err:
     return failure(err)
 
@@ -146,7 +145,7 @@ proc getCidAndProof(
   trace "Got cid and proof for block", cid, proof = $proof
   return success (cid, proof)
 
-proc getCid(
+method getCid*(
   self: RepoStore,
   treeCid: Cid,
   index: Natural): Future[?!Cid] {.async.} =
@@ -541,7 +540,8 @@ method close*(self: RepoStore): Future[void] {.async.} =
   ## For some implementations this may be a no-op
   ##
 
-  (await self.repoDs.close()).expect("Should close datastore")
+  (await self.metaDs.close()).expect("Should meta datastore")
+  (await self.repoDs.close()).expect("Should repo datastore")
 
 proc reserve*(self: RepoStore, bytes: uint): Future[?!void] {.async.} =
   ## Reserve bytes

codex/stores/treehelper.nim

@@ -36,7 +36,7 @@ proc putSomeProofs*(store: BlockStore, tree: CodexTree, iter: Iter[int]): Future
     without proof =? tree.getProof(i), err:
       return failure(err)
 
-    let res = await store.putBlockCidAndProof(treeCid, i, blkCid, proof)
+    let res = await store.putCidAndProof(treeCid, i, blkCid, proof)
 
     if err =? res.errorOption:
       return failure(err)

codex/utils/digest.nim

@@ -10,6 +10,7 @@
 import pkg/poseidon2
 import pkg/poseidon2/io
 import pkg/questionable/results
+import pkg/libp2p/multihash
 
 import ../merkletree
 
@@ -38,3 +39,15 @@ func digest*(
   ##
 
   (? Poseidon2Tree.digestTree(bytes, chunkSize)).root
+
+func digestMhash*(
+  _: type Poseidon2Tree,
+  bytes: openArray[byte], chunkSize: int): ?!MultiHash =
+  ## Hashes chunks of data with a sponge of rate 2 and
+  ## returns the multihash of the root
+  ##
+
+  let
+    hash = ? Poseidon2Tree.digest(bytes, chunkSize)
+
+  ? MultiHash.init(Pos2Bn128MrklCodec, hash).mapFailure

tests/codex/helpers.nim

@@ -9,6 +9,7 @@ import pkg/codex/manifest
 import pkg/codex/merkletree
 import pkg/codex/blockexchange
 import pkg/codex/rng
+import pkg/codex/utils
 
 import ./helpers/nodeutils
 import ./helpers/randomchunker
@@ -27,6 +28,13 @@ export libp2p except setup, eventually
 func `==`*(a, b: Block): bool =
   (a.cid == b.cid) and (a.data == b.data)
 
+proc calcEcBlocksCount*(blocksCount: int, ecK, ecM: int): int =
+  let
+    rounded = roundUp(blocksCount, ecK)
+    steps = divUp(blocksCount, ecK)
+
+  rounded + (steps * ecM)
+
 proc lenPrefix*(msg: openArray[byte]): seq[byte] =
   ## Write `msg` with a varint-encoded length prefix
   ##
@@ -94,7 +102,7 @@ proc storeDataGetManifest*(store: BlockStore, chunker: Chunker): Future[Manifest
 
   for i in 0..<tree.leavesCount:
     let proof = tree.getProof(i).tryGet()
-    (await store.putBlockCidAndProof(treeCid, i, cids[i], proof)).tryGet()
+    (await store.putCidAndProof(treeCid, i, cids[i], proof)).tryGet()
 
   return manifest
 

tests/codex/merkletree/testcodextree.nim

@@ -51,7 +51,7 @@ suite "Test CodexTree":
     var tree = CodexTree.init(leaves = expectedLeaves)
     check:
       tree.isOk
-      tree.get().leaves == expectedLeaves.mapIt( it.bytes )
+      tree.get().leaves == expectedLeaves.mapIt( it.digestBytes )
       tree.get().mcodec == sha256
 
   test "Should build tree from cid leaves":
@@ -68,10 +68,10 @@ suite "Test CodexTree":
 
     check:
       tree.isOk
-      tree.get().leaves == expectedLeaves.mapIt( it.mhash.tryGet.bytes )
+      tree.get().leaves == expectedLeaves.mapIt( it.mhash.tryGet.digestBytes )
       tree.get().mcodec == sha256
 
-  test "Should build from raw bytes (should not hash leaves)":
+  test "Should build from raw digestbytes (should not hash leaves)":
     let
       tree = CodexTree.init(sha256, leaves = data).tryGet
 
@@ -91,7 +91,7 @@ suite "Test CodexTree":
       tree == fromNodes
 
 let
-  mhash = sha256.getMhash().tryGet
+  mhash = sha256.mhash().tryGet
   zero: seq[byte] = newSeq[byte](mhash.size)
   compress = proc(x, y: seq[byte], key: ByteTreeKey): seq[byte] =
     compress(x, y, key, mhash).tryGet

tests/codex/merkletree/testposeidon2tree.nim

@@ -29,11 +29,10 @@ const
       "0000000000000000000000000000006".toBytes,
       "0000000000000000000000000000007".toBytes,
       "0000000000000000000000000000008".toBytes,
-      "0000000000000000000000000000009".toBytes,
+      "0000000000000000000000000000009".toBytes, # note one less to account for padding of field elements
-      "0000000000000000000000000000010".toBytes,
     ]
 
-suite "Test CodexTree":
+suite "Test Poseidon2Tree":
   var
     expectedLeaves: seq[Poseidon2Hash]
 
@@ -54,8 +53,8 @@ suite "Test CodexTree":
   test "Init tree from byte leaves":
     let
       tree = Poseidon2Tree.init(
-        leaves = data.mapIt(
+        leaves = expectedLeaves.mapIt(
-          array[31, byte].initCopyFrom( it )
+          array[31, byte].initCopyFrom( it.toBytes )
         )).tryGet
 
     check:

tests/codex/slotbuilder/testslotbuilder.nim

@@ -0,0 +1,324 @@
+import std/sequtils
+import std/math
+import std/importutils
+import std/sugar
+
+import pkg/chronos
+import pkg/asynctest
+import pkg/questionable/results
+import pkg/codex/blocktype as bt
+import pkg/codex/rng
+import pkg/codex/stores
+import pkg/codex/chunker
+import pkg/codex/merkletree
+import pkg/codex/utils
+import pkg/codex/utils/digest
+import pkg/datastore
+import pkg/poseidon2
+import pkg/poseidon2/io
+import constantine/math/io/io_fields
+
+import ../helpers
+import ../examples
+import ../merkletree/helpers
+
+import pkg/codex/indexingstrategy {.all.}
+import pkg/codex/slots/slotbuilder {.all.}
+
+suite "Slot builder":
+  let
+    blockSize = 1024
+    cellSize = 64
+    ecK = 3
+    ecM = 2
+
+    numSlots = ecK + ecM
+    numDatasetBlocks = 100
+    numBlockCells = blockSize div cellSize
+
+    numTotalBlocks = calcEcBlocksCount(numDatasetBlocks, ecK, ecM)                # total number of blocks in the dataset after
+                                                                                  # EC (should will match number of slots)
+    originalDatasetSize = numDatasetBlocks * blockSize                            # size of the dataset before EC
+    totalDatasetSize = numTotalBlocks * blockSize                                 # size of the dataset after EC
+    numTotalSlotBlocks = nextPowerOfTwo(numTotalBlocks div numSlots)
+
+    blockPadBytes =
+      newSeq[byte](numBlockCells.nextPowerOfTwoPad * cellSize)                    # power of two padding for blocks
+
+    slotsPadLeafs =
+      newSeqWith((numTotalBlocks div numSlots).nextPowerOfTwoPad, Poseidon2Zero)  # power of two padding for block roots
+
+    rootsPadLeafs =
+      newSeqWith(numSlots.nextPowerOfTwoPad, Poseidon2Zero)
+
+  var
+    datasetBlocks: seq[bt.Block]
+    localStore: BlockStore
+    manifest: Manifest
+    protectedManifest: Manifest
+    expectedEmptyCid: Cid
+    slotBuilder: SlotBuilder
+    chunker: Chunker
+
+  proc createBlocks(): Future[void] {.async.} =
+    while true:
+      let chunk = await chunker.getBytes()
+      if chunk.len <= 0:
+        break
+      let blk = bt.Block.new(chunk).tryGet()
+      datasetBlocks.add(blk)
+      discard await localStore.putBlock(blk)
+
+  proc createProtectedManifest(): Future[void] {.async.} =
+    let
+      cids = datasetBlocks.mapIt(it.cid)
+      datasetTree = CodexTree.init(cids[0..<numDatasetBlocks]).tryGet()
+      datasetTreeCid = datasetTree.rootCid().tryGet()
+
+      protectedTree = CodexTree.init(cids).tryGet()
+      protectedTreeCid = protectedTree.rootCid().tryGet()
+
+    for index, cid in cids[0..<numDatasetBlocks]:
+      let proof = datasetTree.getProof(index).tryget()
+      (await localStore.putCidAndProof(datasetTreeCid, index, cid, proof)).tryGet
+
+    for index, cid in cids:
+      let proof = protectedTree.getProof(index).tryget()
+      (await localStore.putCidAndProof(protectedTreeCid, index, cid, proof)).tryGet
+
+    manifest = Manifest.new(
+      treeCid = datasetTreeCid,
+      blockSize = blockSize.NBytes,
+      datasetSize = originalDatasetSize.NBytes)
+
+    protectedManifest = Manifest.new(
+      manifest = manifest,
+      treeCid = protectedTreeCid,
+      datasetSize = totalDatasetSize.NBytes,
+      ecK = ecK,
+      ecM = ecM)
+
+    let
+      manifestBlock = bt.Block.new(
+        manifest.encode().tryGet(),
+        codec = ManifestCodec).tryGet()
+
+      protectedManifestBlock = bt.Block.new(
+        protectedManifest.encode().tryGet(),
+        codec = ManifestCodec).tryGet()
+
+    (await localStore.putBlock(manifestBlock)).tryGet()
+    (await localStore.putBlock(protectedManifestBlock)).tryGet()
+
+    expectedEmptyCid = emptyCid(
+      protectedManifest.version,
+      protectedManifest.hcodec,
+      protectedManifest.codec).tryGet()
+
+  privateAccess(SlotBuilder) # enable access to private fields
+
+  setup:
+    let
+      repoDs = SQLiteDatastore.new(Memory).tryGet()
+      metaDs = SQLiteDatastore.new(Memory).tryGet()
+    localStore = RepoStore.new(repoDs, metaDs)
+
+    chunker = RandomChunker.new(Rng.instance(), size = totalDatasetSize, chunkSize = blockSize)
+    await createBlocks()
+    await createProtectedManifest()
+
+  teardown:
+    await localStore.close()
+
+    # Need to reset all objects because otherwise they get
+    # captured by the test runner closures, not good!
+    reset(datasetBlocks)
+    reset(localStore)
+    reset(manifest)
+    reset(protectedManifest)
+    reset(expectedEmptyCid)
+    reset(slotBuilder)
+    reset(chunker)
+
+  test "Can only create slotBuilder with protected manifest":
+    let
+      unprotectedManifest = Manifest.new(
+        treeCid = Cid.example,
+        blockSize = blockSize.NBytes,
+        datasetSize = originalDatasetSize.NBytes)
+
+    check:
+      SlotBuilder.new(localStore, unprotectedManifest, cellSize = cellSize)
+        .error.msg == "Can only create SlotBuilder using protected manifests."
+
+  test "Number of blocks must be devisable by number of slots":
+    let
+      mismatchManifest = Manifest.new(
+        manifest = Manifest.new(
+          treeCid = Cid.example,
+          blockSize = blockSize.NBytes,
+          datasetSize = originalDatasetSize.NBytes),
+        treeCid = Cid.example,
+        datasetSize = totalDatasetSize.NBytes,
+        ecK = ecK - 1,
+        ecM = ecM)
+
+    check:
+      SlotBuilder.new(localStore, mismatchManifest, cellSize = cellSize)
+        .error.msg == "Number of blocks must be divisable by number of slots."
+
+  test "Block size must be divisable by cell size":
+    let
+      mismatchManifest = Manifest.new(
+        manifest = Manifest.new(
+          treeCid = Cid.example,
+          blockSize = (blockSize + 1).NBytes,
+          datasetSize = (originalDatasetSize - 1).NBytes),
+        treeCid = Cid.example,
+        datasetSize = (totalDatasetSize - 1).NBytes,
+        ecK = ecK,
+        ecM = ecM)
+
+    check:
+      SlotBuilder.new(localStore, mismatchManifest, cellSize = cellSize)
+        .error.msg == "Block size must be divisable by cell size."
+
+  test "Should build correct slot builder":
+    slotBuilder = SlotBuilder.new(
+      localStore,
+      protectedManifest,
+      cellSize = cellSize).tryGet()
+
+    check:
+      slotBuilder.numBlockPadBytes == blockPadBytes.len
+      slotBuilder.numSlotsPadLeafs == slotsPadLeafs.len
+      slotBuilder.numRootsPadLeafs == rootsPadLeafs.len
+
+  test "Should build slot hashes for all slots":
+    let
+      steppedStrategy = SteppedIndexingStrategy.new(0, numTotalBlocks - 1, numSlots)
+      slotBuilder = SlotBuilder.new(
+        localStore,
+        protectedManifest,
+        cellSize = cellSize).tryGet()
+
+    for i in 0 ..< numSlots:
+      let
+        expectedBlock = steppedStrategy
+          .getIndicies(i)
+          .mapIt( datasetBlocks[it] )
+
+        expectedHashes: seq[Poseidon2Hash] = collect(newSeq):
+          for blk in expectedBlock:
+            SpongeMerkle.digest(blk.data & blockPadBytes, cellSize)
+
+        cellHashes = (await slotBuilder.getCellHashes(i)).tryGet()
+
+      check:
+        expectedHashes == cellHashes
+
+  test "Should build slot trees for all slots":
+    let
+      steppedStrategy = SteppedIndexingStrategy.new(0, numTotalBlocks - 1, numSlots)
+      slotBuilder = SlotBuilder.new(
+        localStore,
+        protectedManifest,
+        cellSize = cellSize).tryGet()
+
+    for i in 0 ..< numSlots:
+      let
+        expectedBlock = steppedStrategy
+          .getIndicies(i)
+          .mapIt( datasetBlocks[it] )
+
+        expectedHashes: seq[Poseidon2Hash] = collect(newSeq):
+          for blk in expectedBlock:
+            SpongeMerkle.digest(blk.data & blockPadBytes, cellSize)
+        expectedRoot = Merkle.digest(expectedHashes & slotsPadLeafs)
+
+        slotTree = (await slotBuilder.buildSlotTree(i)).tryGet()
+
+      check:
+        expectedRoot == slotTree.root().tryGet()
+
+  test "Should persist trees for all slots":
+    let
+      slotBuilder = SlotBuilder.new(
+        localStore,
+        protectedManifest,
+        cellSize = cellSize).tryGet()
+
+    for i in 0 ..< numSlots:
+      let
+        slotTree = (await slotBuilder.buildSlotTree(i)).tryGet()
+        slotRoot = (await slotBuilder.buildSlot(i)).tryGet()
+        slotCid = slotRoot.toSlotCid().tryGet()
+
+      for cellIndex in 0..<numTotalSlotBlocks:
+        let
+          (cellCid, proof) = (await localStore.getCidAndProof(slotCid, cellIndex)).tryGet()
+          verifiableProof = proof.toVerifiableProof().tryGet()
+          posProof = slotTree.getProof(cellIndex).tryGet
+
+        check:
+          verifiableProof.index == posProof.index
+          verifiableProof.nleaves == posProof.nleaves
+          verifiableProof.path == posProof.path
+
+  test "Should build correct verification root":
+    let
+      steppedStrategy = SteppedIndexingStrategy.new(0, numTotalBlocks - 1, numSlots)
+      slotBuilder = SlotBuilder.new(
+        localStore,
+        protectedManifest,
+        cellSize = cellSize).tryGet()
+      slotRoots = (await slotBuilder.buildSlots()).tryGet
+
+      slotsHashes = collect(newSeq):
+        for i in 0 ..< numSlots:
+          let
+            expectedBlocks = steppedStrategy
+              .getIndicies(i)
+              .mapIt( datasetBlocks[it] )
+
+            slotHashes: seq[Poseidon2Hash] = collect(newSeq):
+              for blk in expectedBlocks:
+                SpongeMerkle.digest(blk.data & blockPadBytes, cellSize)
+
+          Merkle.digest(slotHashes & slotsPadLeafs)
+
+      expectedRoot = Merkle.digest(slotsHashes & rootsPadLeafs)
+      rootHash = slotBuilder.buildRootsTree(slotRoots).tryGet().root.tryGet()
+
+    check:
+      expectedRoot == rootHash
+
+  test "Should build correct verification root manifest":
+    let
+      steppedStrategy = SteppedIndexingStrategy.new(0, numTotalBlocks - 1, numSlots)
+      slotBuilder = SlotBuilder.new(
+        localStore,
+        protectedManifest,
+        cellSize = cellSize).tryGet()
+
+      slotsHashes = collect(newSeq):
+        for i in 0 ..< numSlots:
+          let
+            expectedBlocks = steppedStrategy
+              .getIndicies(i)
+              .mapIt( datasetBlocks[it] )
+
+            slotHashes: seq[Poseidon2Hash] = collect(newSeq):
+              for blk in expectedBlocks:
+                SpongeMerkle.digest(blk.data & blockPadBytes, cellSize)
+
+          Merkle.digest(slotHashes & slotsPadLeafs)
+
+      expectedRoot = Merkle.digest(slotsHashes & rootsPadLeafs)
+      manifest = (await slotBuilder.buildManifest()).tryGet()
+      mhash = manifest.verificationRoot.mhash.tryGet()
+      mhashBytes = mhash.digestBytes
+      rootHash = Poseidon2Hash.fromBytes(mhashBytes.toArray32).toResult.tryGet()
+
+    check:
+      expectedRoot == rootHash

tests/codex/testerasure.nim

@@ -15,7 +15,7 @@ import pkg/codex/utils
 
 import ./helpers
 
-asyncchecksuite "Erasure encode/decode":
+suite "Erasure encode/decode":
   const BlockSize = 1024'nb
   const dataSetSize = BlockSize * 123 # weird geometry
 

tests/codex/testindexingstrategy.nim

@@ -0,0 +1,65 @@
+import std/sequtils
+import pkg/chronos
+import pkg/asynctest
+
+import ./helpers
+
+import pkg/codex/indexingstrategy
+
+for offset in @[0, 1, 2, 100]:
+  suite "Indexing strategies (Offset: " & $offset & ")":
+    let
+      firstIndex = 0 + offset
+      lastIndex = 12 + offset
+      nIters = 3
+      linear = LinearIndexingStrategy.new(firstIndex, lastIndex, nIters)
+      stepped = SteppedIndexingStrategy.new(firstIndex, lastIndex, nIters)
+
+    test "linear":
+      check:
+        linear.getIndicies(0) == @[0, 1, 2, 3, 4].mapIt(it + offset)
+        linear.getIndicies(1) == @[5, 6, 7, 8, 9].mapIt(it + offset)
+        linear.getIndicies(2) == @[10, 11, 12].mapIt(it + offset)
+
+    test "stepped":
+      check:
+        stepped.getIndicies(0) == @[0, 3, 6, 9, 12].mapIt(it + offset)
+        stepped.getIndicies(1) == @[1, 4, 7, 10].mapIt(it + offset)
+        stepped.getIndicies(2) == @[2, 5, 8, 11].mapIt(it + offset)
+
+suite "Indexing strategies":
+  let
+    linear = LinearIndexingStrategy.new(0, 10, 3)
+    stepped = SteppedIndexingStrategy.new(0, 10, 3)
+
+  test "smallest range 0":
+    let
+      l = LinearIndexingStrategy.new(0, 0, 1)
+      s = SteppedIndexingStrategy.new(0, 0, 1)
+    check:
+      l.getIndicies(0) == @[0]
+      s.getIndicies(0) == @[0]
+
+  test "smallest range 1":
+    let
+      l = LinearIndexingStrategy.new(0, 1, 1)
+      s = SteppedIndexingStrategy.new(0, 1, 1)
+    check:
+      l.getIndicies(0) == @[0, 1]
+      s.getIndicies(0) == @[0, 1]
+
+  test "first index must be smaller than last index":
+    expect AssertionDefect:
+      discard LinearIndexingStrategy.new(10, 0, 1)
+
+  test "numberOfIterations must be greater than zero":
+    expect AssertionDefect:
+      discard LinearIndexingStrategy.new(0, 10, 0)
+
+  test "linear - oob":
+    expect AssertionDefect:
+      discard linear.getIndicies(3)
+
+  test "stepped - oob":
+    expect AssertionDefect:
+      discard stepped.getIndicies(3)

tests/codex/testslotbuilder.nim

@@ -0,0 +1,3 @@
+import ./slotbuilder/testslotbuilder
+
+{.warning[UnusedImport]: off.}

tests/testCodex.nim

@@ -14,5 +14,7 @@ import ./codex/testsystemclock
 import ./codex/testvalidation
 import ./codex/testasyncstreamwrapper
 import ./codex/testmerkletree
+import ./codex/testslotbuilder
+import ./codex/testindexingstrategy
 
 {.warning[UnusedImport]: off.}

vendor/nim-libp2p

@@ -1 +1 @@
-Subproject commit 4f2259e1cef65085d092b2b713bb67f5aac55626
+Subproject commit b239791c568d9f9a76fd66d2322b2754700b6cc5

vendor/nim-poseidon2

@@ -1 +1 @@
-Subproject commit 0cfecf7d780b8c3295d11251d64a49f3c7258fbf
+Subproject commit 3b403b0752790438bed6342b431412cc05474acb