Rework builder & sampler (#697)

* add proof initializer for poseidon2 and `$`

* fix padding and block selection

* fix sample selection and input construction

* fix sample selection & more descriptive names

* add concrete types for sampler & builder

* add missing digest calls

* use concrete types

* add sample test fixtures

* use concrete types and don't fetch dummy blocks

codex/merkletree/codex/codex.nim

@@ -7,16 +7,14 @@
 ## This file may not be copied, modified, or distributed except according to
 ## those terms.
 
-import pkg/upraises
+{.push raises: [].}
-
-push: {.upraises: [].}
 
 import std/bitops
 import std/sequtils
+
 import pkg/questionable
 import pkg/questionable/results
 import pkg/libp2p/[cid, multicodec, multihash]
-import pkg/stew/byteutils
 
 import ../../utils
 import ../../rng
@@ -72,7 +70,7 @@ func getProof*(self: CodexTree, index: int): ?!CodexProof =
 
   success proof
 
-func verify*(self: CodexProof, leaf: MultiHash, root: MultiHash): ?!void =
+func verify*(self: CodexProof, leaf: MultiHash, root: MultiHash): ?!bool =
   ## Verify hash
   ##
 
@@ -88,11 +86,9 @@ func verify*(self: CodexProof, leaf: MultiHash, root: MultiHash): ?!void =
     leafBytes.len != leaf.size:
     return failure "Invalid hash length"
 
-  ? self.verify(leafBytes, rootBytes)
+  self.verify(leafBytes, rootBytes)
 
-  success()
+func verify*(self: CodexProof, leaf: Cid, root: Cid): ?!bool =
-
-func verify*(self: CodexProof, leaf: Cid, root: Cid): ?!void =
   self.verify(? leaf.mhash.mapFailure, ? leaf.mhash.mapFailure)
 
 proc rootCid*(
@@ -124,16 +120,19 @@ func getLeafCid*(
   Cid.init(version, dataCodec, mhash).mapFailure
 
 proc `$`*(self: CodexTree): string =
-  "CodexTree( mcodec: " &
+  let root = if self.root.isOk: byteutils.toHex(self.root.get) else: "none"
-    $self.mcodec &
+  "CodexTree(" &
-    ", leavesCount: " &
+    " root: " & root &
-    $self.leavesCount & " )"
+    ", leavesCount: " & $self.leavesCount &
+    ", levels: " & $self.levels &
+    ", mcodec: " & $self.mcodec & " )"
 
 proc `$`*(self: CodexProof): string =
-  "CodexProof( mcodec: " &
+  "CodexProof(" &
-    $self.mcodec & ", nleaves: " &
+  " nleaves: " & $self.nleaves &
-    $self.nleaves & ", index: " &
+  ", index: " & $self.index &
-    $self.index & " )"
+  ", path: " & $self.path.mapIt( byteutils.toHex(it) ) &
+  ", mcodec: " & $self.mcodec & " )"
 
 func compress*(
   x, y: openArray[byte],
@@ -226,7 +225,9 @@ proc fromNodes*(
     index = Rng.instance.rand(nleaves - 1)
     proof = ? self.getProof(index)
 
-  ? proof.verify(self.leaves[index], ? self.root) # sanity check
+  if not ? proof.verify(self.leaves[index], ? self.root): # sanity check
+    return failure "Unable to verify tree built from nodes"
+
   success self
 
 func init*(

codex/merkletree/merkletree.nim

@@ -117,11 +117,8 @@ func reconstructRoot*[H, K](proof: MerkleProof[H, K], leaf: H): ?!H =
 
   return success h
 
-func verify*[H, K](proof: MerkleProof[H, K], leaf: H, root: H): ?!void =
+func verify*[H, K](proof: MerkleProof[H, K], leaf: H, root: H): ?!bool =
-  return if bool(root == ? proof.reconstructRoot(leaf)):
+  success bool(root == ? proof.reconstructRoot(leaf))
-      success()
-    else:
-      failure("invalid proof")
 
 func merkleTreeWorker*[H, K](
   self: MerkleTree[H, K],

codex/merkletree/poseidon2.nim

@@ -7,6 +7,8 @@
 ## This file may not be copied, modified, or distributed except according to
 ## those terms.
 
+{.push raises: [].}
+
 import std/sequtils
 
 import pkg/poseidon2
@@ -30,7 +32,8 @@ const
   Poseidon2Zero* = zero
 
 type
-  Poseidon2Hash* = F
+  Bn254Fr* = F
+  Poseidon2Hash* = Bn254Fr
 
   PoseidonKeysEnum* = enum  # can't use non-ordinals as enum values
     KeyNone
@@ -41,6 +44,19 @@ type
   Poseidon2Tree* = MerkleTree[Poseidon2Hash, PoseidonKeysEnum]
   Poseidon2Proof* = MerkleProof[Poseidon2Hash, PoseidonKeysEnum]
 
+proc `$`*(self: Poseidon2Tree): string =
+  let root = if self.root.isOk: self.root.get.toHex else: "none"
+  "Poseidon2Tree(" &
+    " root: " & root &
+    ", leavesCount: " & $self.leavesCount &
+    ", levels: " & $self.levels & " )"
+
+proc `$`*(self: Poseidon2Proof): string =
+  "Poseidon2Proof(" &
+  " nleaves: " & $self.nleaves &
+  ", index: " & $self.index &
+  ", path: " & $self.path.mapIt( it.toHex ) & " )"
+
 func toArray32*(bytes: openArray[byte]): array[32, byte] =
   result[0..<bytes.len] = bytes[0..<bytes.len]
 
@@ -104,6 +120,29 @@ proc fromNodes*(
     index = Rng.instance.rand(nleaves - 1)
     proof = ? self.getProof(index)
 
-  ? proof.verify(self.leaves[index], ? self.root) # sanity check
+  if not ? proof.verify(self.leaves[index], ? self.root): # sanity check
+    return failure "Unable to verify tree built from nodes"
 
   success self
+
+func init*(
+  _: type Poseidon2Proof,
+  index: int,
+  nleaves: int,
+  nodes: openArray[Poseidon2Hash]): ?!Poseidon2Proof =
+
+  if nodes.len == 0:
+    return failure "Empty nodes"
+
+  let
+    compressor = proc(
+      x, y: Poseidon2Hash,
+      key: PoseidonKeysEnum): ?!Poseidon2Hash {.noSideEffect.} =
+      success compress( x, y, key.toKey )
+
+  success Poseidon2Proof(
+    compress: compressor,
+    zero: Poseidon2Zero,
+    index: index,
+    nleaves: nleaves,
+    path: @nodes)

codex/node.nim

@@ -38,9 +38,11 @@ import ./streams
 import ./erasure
 import ./discovery
 import ./contracts
+import ./indexingstrategy
 import ./utils
 import ./errors
 import ./logutils
+import ./utils/poseidon2digest
 
 export logutils
 
@@ -390,7 +392,7 @@ proc setupRequest(
     trace "Unable to erasure code dataset"
     return failure(error)
 
-  without builder =? SlotsBuilder.new(self.blockStore, encoded), err:
+  without builder =? Poseidon2Builder.new(self.blockStore, encoded), err:
     trace "Unable to create slot builder"
     return failure(err)
 
@@ -503,7 +505,7 @@ proc onStore(
     trace "Unable to fetch manifest for cid", cid, err = err.msg
     return failure(err)
 
-  without builder =? SlotsBuilder.new(self.blockStore, manifest), err:
+  without builder =? Poseidon2Builder.new(self.blockStore, manifest), err:
     trace "Unable to create slots builder", err = err.msg
     return failure(err)
 
@@ -528,8 +530,13 @@ proc onStore(
 
     return success()
 
-  if blksIter =? builder.slotIndiciesIter(slotIdx) and
+  without indexer =? manifest.protectedStrategy.init(
-    err =? (await self.fetchBatched(
+    0, manifest.blocksCount() - 1, manifest.numSlots).catch and
+    blksIter =? indexer.getIndicies(slotIdx).catch, err:
+      trace "Unable to create indexing strategy from protected manifest", err = err.msg
+      return failure(err)
+
+  if err =? (await self.fetchBatched(
       manifest.treeCid,
       blksIter,
       onBatch = updateExpiry)).errorOption:
@@ -572,7 +579,7 @@ proc onProve(
     error "Unable to fetch manifest for cid", err = err.msg
     return failure(err)
 
-  without builder =? SlotsBuilder.new(self.blockStore, manifest), err:
+  without builder =? Poseidon2Builder.new(self.blockStore, manifest), err:
     error "Unable to create slots builder", err = err.msg
     return failure(err)
 

codex/slots/builder.nim

@@ -1,4 +1,9 @@
 import ./builder/builder
 import ./converters
 
+import ../merkletree
+
 export builder, converters
+
+type
+  Poseidon2Builder* = SlotsBuilder[Poseidon2Tree, Poseidon2Hash]

codex/slots/builder/builder.nim

@@ -15,192 +15,165 @@ import std/sugar
 
 import pkg/libp2p
 import pkg/chronos
+import pkg/chronicles
 import pkg/questionable
 import pkg/questionable/results
-import pkg/poseidon2
+import pkg/constantine/math/io/io_fields
-import pkg/poseidon2/io
-import pkg/constantine/math/arithmetic/finite_fields
 
-import ../../logutils
+import ../../utils
-import ../../indexingstrategy
-import ../../merkletree
 import ../../stores
 import ../../manifest
-import ../../utils
+import ../../merkletree
-import ../../utils/asynciter
 import ../../utils/digest
-import ../../utils/poseidon2digest
+import ../../utils/asynciter
+import ../../indexingstrategy
+
 import ../converters
 
-export converters
+export converters, asynciter
 
 logScope:
   topics = "codex slotsbuilder"
 
-const
-  # TODO: Unified with the DefaultCellSize specified in branch "data-sampler"
-  # in the proving circuit.
-
-  DefaultEmptyBlock* = newSeq[byte](DefaultBlockSize.int)
-  DefaultEmptyCell* = newSeq[byte](DefaultCellSize.int)
-
 type
-  # TODO: should be a generic type that
+  SlotsBuilder*[T, H] = ref object of RootObj
-  # supports all merkle trees
-  SlotsBuilder* = ref object of RootObj
     store: BlockStore
-    manifest: Manifest
+    manifest: Manifest            # current manifest
-    strategy: IndexingStrategy
+    strategy: IndexingStrategy    # indexing strategy
-    cellSize: NBytes
+    cellSize: NBytes              # cell size
-    emptyDigestTree: Poseidon2Tree
+    numSlotBlocks: Natural        # number of blocks per slot (should yield a power of two number of cells)
-    blockPadBytes: seq[byte]
+    slotRoots: seq[H]             # roots of the slots
-    slotsPadLeafs: seq[Poseidon2Hash]
+    emptyBlock: seq[byte]         # empty block
-    rootsPadLeafs: seq[Poseidon2Hash]
+    verifiableTree: ?T            # verification tree (dataset tree)
-    slotRoots: seq[Poseidon2Hash]
+    emptyDigestTree: T            # empty digest tree for empty blocks
-    verifyTree: ?Poseidon2Tree
 
-func slotRoots*(self: SlotsBuilder): seq[Poseidon2Hash] {.inline.} =
+func verifiable*[T, H](self: SlotsBuilder[T, H]): bool {.inline.} =
+  ## Returns true if the slots are verifiable.
+  ##
+
+  self.manifest.verifiable
+
+func slotRoots*[T, H](self: SlotsBuilder[T, H]): seq[H] {.inline.} =
   ## Returns the slot roots.
   ##
 
   self.slotRoots
 
-func verifyTree*(self: SlotsBuilder): ?Poseidon2Tree {.inline.} =
+func verifyTree*[T, H](self: SlotsBuilder[T, H]): ?T {.inline.} =
   ## Returns the slots tree (verification tree).
   ##
 
-  self.verifyTree
+  self.verifiableTree
 
-func verifyRoot*(self: SlotsBuilder): ?Poseidon2Hash {.inline.} =
+func verifyRoot*[T, H](self: SlotsBuilder[T, H]): ?H {.inline.} =
   ## Returns the slots root (verification root).
   ##
 
-  self.verifyTree.?root().?toOption
+  if tree =? self.verifyTree and root =? tree.root:
+    return some root
 
-func nextPowerOfTwoPad*(a: int): int =
+func numSlots*[T, H](self: SlotsBuilder[T, H]): Natural =
-  ## Returns the difference between the original
-  ## value and the next power of two.
-  ##
-
-  nextPowerOfTwo(a) - a
-
-func numBlockPadBytes*(self: SlotsBuilder): Natural =
-  ## Number of padding bytes required for a pow2
-  ## merkle tree for each block.
-  ##
-
-  self.blockPadBytes.len
-
-func numSlotsPadLeafs*(self: SlotsBuilder): Natural =
-  ## Number of padding field elements required for a pow2
-  ## merkle tree for each slot.
-  ##
-
-  self.slotsPadLeafs.len
-
-func numRootsPadLeafs*(self: SlotsBuilder): Natural =
-  ## Number of padding field elements required for a pow2
-  ## merkle tree for the slot roots.
-  ##
-
-  self.rootsPadLeafs.len
-
-func numSlots*(self: SlotsBuilder): Natural =
   ## Number of slots.
   ##
 
   self.manifest.numSlots
 
-func numSlotBlocks*(self: SlotsBuilder): Natural =
+func numSlotBlocks*[T, H](self: SlotsBuilder[T, H]): Natural =
   ## Number of blocks per slot.
   ##
 
-  self.manifest.blocksCount div self.manifest.numSlots
+  self.numSlotBlocks
 
-func slotBytes*(self: SlotsBuilder): NBytes =
+func numBlocks*[T, H](self: SlotsBuilder[T, H]): Natural =
+  ## Number of blocks.
+  ##
+
+  self.numSlotBlocks * self.manifest.numSlots
+
+func slotBytes*[T, H](self: SlotsBuilder[T, H]): NBytes =
   ## Number of bytes per slot.
   ##
 
   (self.manifest.blockSize.int * self.numSlotBlocks).NBytes
 
-func numBlockCells*(self: SlotsBuilder): Natural =
+func numBlockCells*[T, H](self: SlotsBuilder[T, H]): Natural =
   ## Number of cells per block.
   ##
 
   (self.manifest.blockSize div self.cellSize).Natural
 
-func numBlockCellsPadded*(self: SlotsBuilder): Natural =
+func cellSize*[T, H](self: SlotsBuilder[T, H]): NBytes =
-  ## Number of cells per block including padding.
-  ##
-
-  nextPowerOfTwo(self.numBlockCells.int).Natural
-
-func cellSize*(self: SlotsBuilder): NBytes =
   ## Cell size.
   ##
 
   self.cellSize
 
-func numSlotCells*(self: SlotsBuilder): Natural =
+func numSlotCells*[T, H](self: SlotsBuilder[T, H]): Natural =
   ## Number of cells per slot.
   ##
 
   self.numBlockCells * self.numSlotBlocks
 
-func numSlotCellsPadded*(self: SlotsBuilder): Natural =
+func slotIndiciesIter*[T, H](self: SlotsBuilder[T, H], slot: Natural): ?!Iter[int] =
-  ## Number of cells per slot including padding.
-  ##
-
-  nextPowerOfTwo(self.numBlockCellsPadded.int * self.numSlotBlocks.int).Natural
-
-func emptyDigestTree*(self: SlotsBuilder): Poseidon2Tree {.inline.} =
-  ## Returns the tree of a padding block.
-  ##
-
-  self.emptyDigestTree
-
-func slotIndiciesIter*(self: SlotsBuilder, slot: Natural): ?!Iter[int] =
   ## Returns the slot indices.
   ##
 
   self.strategy.getIndicies(slot).catch
 
-func slotIndicies*(self: SlotsBuilder, slot: Natural): seq[int] =
+func slotIndicies*[T, H](self: SlotsBuilder[T, H], slot: Natural): seq[int] =
   ## Returns the slot indices.
   ##
 
   if iter =? self.strategy.getIndicies(slot).catch:
-    toSeq(iter)
+    return toSeq(iter)
-  else:
-    trace "Failed to get slot indicies"
-    newSeq[int]()
 
-func manifest*(self: SlotsBuilder): Manifest =
+func manifest*[T, H](self: SlotsBuilder[T, H]): Manifest =
   ## Returns the manifest.
   ##
 
   self.manifest
 
-proc buildBlockTree*(
+proc buildBlockTree*[T, H](
-  self: SlotsBuilder,
+  self: SlotsBuilder[T, H],
-  blkIdx: Natural): Future[?!(seq[byte], Poseidon2Tree)] {.async.} =
+  blkIdx: Natural,
+  slotPos: Natural): Future[?!(seq[byte], T)] {.async.} =
+  ## Build the block digest tree and return a tuple with the
+  ## block data and the tree.
+  ##
+
+  logScope:
+    blkIdx = blkIdx
+    slotPos = slotPos
+    numSlotBlocks = self.manifest.numSlotBlocks
+    cellSize = self.cellSize
+
+  trace "Building block tree"
+
+  if slotPos > (self.manifest.numSlotBlocks - 1):
+    # pad blocks are 0 byte blocks
+    trace "Returning empty digest tree for pad block"
+    return success (self.emptyBlock, self.emptyDigestTree)
+
   without blk =? await self.store.getBlock(self.manifest.treeCid, blkIdx), err:
-    error "Failed to get block CID for tree at index"
+    error "Failed to get block CID for tree at index", err = err.msg
     return failure(err)
 
   if blk.isEmpty:
-    success (DefaultEmptyBlock & self.blockPadBytes, self.emptyDigestTree)
+    success (self.emptyBlock, self.emptyDigestTree)
   else:
     without tree =?
-      Poseidon2Tree.digestTree(blk.data & self.blockPadBytes, self.cellSize.int), err:
+      T.digestTree(blk.data, self.cellSize.int), err:
-      error "Failed to create digest for block"
+      error "Failed to create digest for block", err = err.msg
       return failure(err)
 
     success (blk.data, tree)
 
-proc getBlockHashes*(
+proc getCellHashes*[T, H](
-  self: SlotsBuilder,
+  self: SlotsBuilder[T, H],
-  slotIndex: Natural): Future[?!seq[Poseidon2Hash]] {.async.} =
+  slotIndex: Natural): Future[?!seq[H]] {.async.} =
+  ## Collect all the cells from a block and return
+  ## their hashes.
+  ##
 
   let
     treeCid = self.manifest.treeCid
@@ -209,38 +182,44 @@ proc getBlockHashes*(
 
   logScope:
     treeCid = treeCid
-    blockCount = blockCount
+    origBlockCount = blockCount
     numberOfSlots = numberOfSlots
-    index = blkIdx
     slotIndex = slotIndex
 
-  let
+  let hashes = collect(newSeq):
-    hashes: seq[Poseidon2Hash] = collect(newSeq):
+    for i, blkIdx in self.strategy.getIndicies(slotIndex):
-      for blkIdx in self.strategy.getIndicies(slotIndex):
+      logScope:
-        trace "Getting block CID for tree at index"
+        blkIdx = blkIdx
+        pos = i
 
-        without (_, blockTree) =? (await self.buildBlockTree(blkIdx)) and
+      trace "Getting block CID for tree at index"
-          digest =? blockTree.root, err:
+      without (_, tree) =? (await self.buildBlockTree(blkIdx, i)) and
-          error "Failed to get block CID for block tree at index", err = err.msg
+        digest =? tree.root, err:
-          return failure(err)
+        error "Failed to get block CID for tree at index", err = err.msg
+        return failure(err)
 
-        digest
+      trace "Get block digest", digest = digest.toHex
+      digest
 
   success hashes
 
-proc buildSlotTree*(
+proc buildSlotTree*[T, H](
-  self: SlotsBuilder,
+  self: SlotsBuilder[T, H],
-  slotIndex: Natural): Future[?!Poseidon2Tree] {.async.} =
+  slotIndex: Natural): Future[?!T] {.async.} =
-  without blockHashes =? (await self.getBlockHashes(slotIndex)), err:
+  ## Build the slot tree from the block digest hashes
+  ## and return the tree.
+
+  without cellHashes =? (await self.getCellHashes(slotIndex)), err:
     error "Failed to select slot blocks", err = err.msg
     return failure(err)
 
-  Poseidon2Tree.init(blockHashes & self.slotsPadLeafs)
+  T.init(cellHashes)
 
-proc buildSlot*(
+proc buildSlot*[T, H](
-  self: SlotsBuilder,
+  self: SlotsBuilder[T, H],
-  slotIndex: Natural): Future[?!Poseidon2Hash] {.async.} =
+  slotIndex: Natural): Future[?!H] {.async.} =
-  ## Build a slot tree and store it in the block store.
+  ## Build a slot tree and store the proofs in
+  ## the block store.
   ##
 
   logScope:
@@ -272,12 +251,10 @@ proc buildSlot*(
 
   tree.root()
 
-func buildVerifyTree*(
+func buildVerifyTree*[T, H](self: SlotsBuilder[T, H], slotRoots: openArray[H]): ?!T =
-  self: SlotsBuilder,
+  T.init(@slotRoots)
-  slotRoots: openArray[Poseidon2Hash]): ?!Poseidon2Tree =
-  Poseidon2Tree.init(@slotRoots & self.rootsPadLeafs)
 
-proc buildSlots*(self: SlotsBuilder): Future[?!void] {.async.} =
+proc buildSlots*[T, H](self: SlotsBuilder[T, H]): Future[?!void] {.async.} =
   ## Build all slot trees and store them in the block store.
   ##
 
@@ -300,14 +277,14 @@ proc buildSlots*(self: SlotsBuilder): Future[?!void] {.async.} =
     return failure(err)
 
   if verifyTree =? self.verifyTree and verifyRoot =? verifyTree.root:
-    if verifyRoot != root: # TODO: `!=` doesn't work for SecretBool
+    if not bool(verifyRoot == root): # TODO: `!=` doesn't work for SecretBool
         return failure "Existing slots root doesn't match reconstructed root."
 
-  self.verifyTree = some tree
+  self.verifiableTree = some tree
 
   success()
 
-proc buildManifest*(self: SlotsBuilder): Future[?!Manifest] {.async.} =
+proc buildManifest*[T, H](self: SlotsBuilder[T, H]): Future[?!Manifest] {.async.} =
   if err =? (await self.buildSlots()).errorOption:
     error "Failed to build slot roots", err = err.msg
     return failure(err)
@@ -321,68 +298,88 @@ proc buildManifest*(self: SlotsBuilder): Future[?!Manifest] {.async.} =
     error "Failed to map slot roots to CIDs", err = err.msg
     return failure(err)
 
-  Manifest.new(self.manifest, rootProvingCid, rootCids)
+  Manifest.new(
+    self.manifest,
+    rootProvingCid,
+    rootCids,
+    self.cellSize,
+    self.strategy.strategyType)
 
-proc new*(
+proc new*[T, H](
-  T: type SlotsBuilder,
+  _: type SlotsBuilder[T, H],
   store: BlockStore,
   manifest: Manifest,
-  strategy: ?IndexingStrategy = none IndexingStrategy,
+  strategy = SteppedStrategy,
-  cellSize = DefaultCellSize): ?!SlotsBuilder =
+  cellSize = DefaultCellSize): ?!SlotsBuilder[T, H] =
 
   if not manifest.protected:
-    return failure("Can only create SlotsBuilder using protected manifests.")
+    trace "Manifest is not protected."
+    return failure("Manifest is not protected.")
+
+  logScope:
+    blockSize = manifest.blockSize
+    strategy = strategy
+    cellSize = cellSize
 
   if (manifest.blocksCount mod manifest.numSlots) != 0:
+    trace "Number of blocks must be divisable by number of slots."
     return failure("Number of blocks must be divisable by number of slots.")
 
+  let cellSize = if manifest.verifiable: manifest.cellSize else: cellSize
   if (manifest.blockSize mod cellSize) != 0.NBytes:
+    trace "Block size must be divisable by cell size."
     return failure("Block size must be divisable by cell size.")
 
   let
-    strategy = if strategy.isNone:
+    numBlockCells     = (manifest.blockSize div cellSize).int         # number of cells per block
-      ? SteppedStrategy.init(
+    numSlotCells      = manifest.numSlotBlocks * numBlockCells        # number of uncorrected slot cells
-        0, manifest.blocksCount - 1, manifest.numSlots).catch
+    pow2SlotCells     = nextPowerOfTwo(numSlotCells)                  # pow2 cells per slot
-      else:
+    numPadSlotBlocks  = pow2SlotCells div numBlockCells               # pow2 blocks per slot
-        strategy.get
+    numPadBlocksTotal = numPadSlotBlocks * manifest.numSlots          # total number of pad blocks
 
-    # all trees have to be padded to power of two
+    emptyBlock        = newSeq[byte](manifest.blockSize.int)
-    numBlockCells = (manifest.blockSize div cellSize).int                         # number of cells per block
+    emptyDigestTree   = ? T.digestTree(emptyBlock, cellSize.int)
-    blockPadBytes = newSeq[byte](numBlockCells.nextPowerOfTwoPad * cellSize.int)  # power of two padding for blocks
-    numSlotBlocks = (manifest.blocksCount div manifest.numSlots)
-    slotsPadLeafs = newSeqWith(numSlotBlocks.nextPowerOfTwoPad, Poseidon2Zero)     # power of two padding for block roots
-    rootsPadLeafs = newSeqWith(manifest.numSlots.nextPowerOfTwoPad, Poseidon2Zero)
-    emptyDigestTree = ? Poseidon2Tree.digestTree(DefaultEmptyBlock & blockPadBytes, DefaultCellSize.int)
 
-  var self = SlotsBuilder(
+    strategy = ? strategy.init(
-    store: store,
+      0,
-    manifest: manifest,
+      numPadBlocksTotal - 1,
-    strategy: strategy,
+      manifest.numSlots).catch
-    cellSize: cellSize,
+
-    blockPadBytes: blockPadBytes,
+  logScope:
-    slotsPadLeafs: slotsPadLeafs,
+    numBlockCells     = numBlockCells
-    rootsPadLeafs: rootsPadLeafs,
+    numSlotCells      = numSlotCells
-    emptyDigestTree: emptyDigestTree)
+    pow2SlotCells     = pow2SlotCells
+    numPadSlotBlocks  = numPadSlotBlocks
+    numPadBlocksTotal = numPadBlocksTotal
+    strategy          = strategy.strategyType
+
+  trace "Creating slots builder"
+
+  var
+    self = SlotsBuilder[T, H](
+      store: store,
+      manifest: manifest,
+      strategy: strategy,
+      cellSize: cellSize,
+      emptyBlock: emptyBlock,
+      numSlotBlocks: numPadSlotBlocks,
+      emptyDigestTree: emptyDigestTree)
 
   if manifest.verifiable:
-    if manifest.slotRoots.len == 0 or manifest.slotRoots.len != manifest.numSlots:
+    if manifest.slotRoots.len == 0 or
+      manifest.slotRoots.len != manifest.numSlots:
       return failure "Manifest is verifiable but slot roots are missing or invalid."
 
-    let slotRoots = manifest.slotRoots.mapIt( (? it.fromSlotCid()))
+    let
+      slotRoots = manifest.slotRoots.mapIt( (? it.fromSlotCid() ))
+      tree = ? self.buildVerifyTree(slotRoots)
+      expectedRoot = ? manifest.verifyRoot.fromVerifyCid()
+      verifyRoot = ? tree.root
 
-    without tree =? self.buildVerifyTree(slotRoots), err:
+    if verifyRoot != expectedRoot:
-      error "Failed to build slot roots tree", err = err.msg
+      return failure "Existing slots root doesn't match reconstructed root."
-      return failure(err)
-
-    without expectedRoot =? manifest.verifyRoot.fromVerifyCid(), err:
-      error "Unable to convert manifest verifyRoot to hash", error = err.msg
-      return failure(err)
-
-    if verifyRoot =? tree.root:
-      if verifyRoot != expectedRoot:
-        return failure "Existing slots root doesn't match reconstructed root."
 
     self.slotRoots = slotRoots
-    self.verifyTree = some tree
+    self.verifiableTree = some tree
 
   success self

codex/slots/sampler.nim

@@ -1,4 +1,9 @@
 import ./sampler/sampler
 import ./sampler/utils
 
+import ../merkletree
+
 export sampler, utils
+
+type
+  Poseidon2Sampler* = DataSampler[Poseidon2Tree, Poseidon2Hash]

codex/slots/sampler/sampler.nim

@@ -8,182 +8,146 @@
 ## those terms.
 
 import std/sugar
-import std/sequtils
 
+import pkg/chronicles
 import pkg/chronos
 import pkg/questionable
 import pkg/questionable/results
-import pkg/constantine/math/arithmetic
-import pkg/poseidon2
-import pkg/poseidon2/types
-import pkg/poseidon2/io
 import pkg/stew/arrayops
 
-import ../../logutils
 import ../../market
 import ../../blocktype as bt
 import ../../merkletree
 import ../../manifest
 import ../../stores
 
+import ../converters
 import ../builder
-
+import ../types
 import ./utils
 
 logScope:
   topics = "codex datasampler"
 
 type
-  Cell* = seq[byte]
+  DataSampler*[T, H] = ref object of RootObj
-
-  Sample* = object
-    data*: Cell
-    slotProof*: Poseidon2Proof
-    cellProof*: Poseidon2Proof
-    slotBlockIdx*: Natural
-    blockCellIdx*: Natural
-
-  ProofInput* = object
-    entropy*: Poseidon2Hash
-    verifyRoot*: Poseidon2Hash
-    verifyProof*: Poseidon2Proof
-    numSlots*: Natural
-    numCells*: Natural
-    slotIndex*: Natural
-    samples*: seq[Sample]
-
-  DataSampler* = ref object of RootObj
     index: Natural
     blockStore: BlockStore
-    # The following data is invariant over time for a given slot:
+    builder: SlotsBuilder[T, H]
-    builder: SlotsBuilder
 
-proc new*(
+func getCell*[T, H](
-    T: type DataSampler,
+  self: DataSampler[T, H],
+  blkBytes: seq[byte],
+  blkCellIdx: Natural): seq[byte] =
+
+  let
+    cellSize = self.builder.cellSize.uint64
+    dataStart = cellSize * blkCellIdx.uint64
+    dataEnd = dataStart + cellSize
+
+  doAssert (dataEnd - dataStart) == cellSize, "Invalid cell size"
+
+  toInputData[H](blkBytes[dataStart ..< dataEnd])
+
+proc getSample*[T, H](
+  self: DataSampler[T, H],
+  cellIdx: int,
+  slotTreeCid: Cid,
+  slotRoot: H): Future[?!Sample[H]] {.async.} =
+
+  let
+    cellsPerBlock = self.builder.numBlockCells
+    blkCellIdx    = cellIdx.toCellInBlk(cellsPerBlock) # block cell index
+    blkSlotIdx    = cellIdx.toBlkInSlot(cellsPerBlock) # slot tree index
+    origBlockIdx  = self.builder.slotIndicies(self.index)[blkSlotIdx]  # convert to original dataset block index
+
+  logScope:
+    cellIdx       = cellIdx
+    blkSlotIdx    = blkSlotIdx
+    blkCellIdx    = blkCellIdx
+    origBlockIdx  = origBlockIdx
+
+  trace "Retrieving sample from block tree"
+  let
+    (_, proof) = (await self.blockStore.getCidAndProof(
+      slotTreeCid, blkSlotIdx.Natural)).valueOr:
+      return failure("Failed to get slot tree CID and proof")
+
+    slotProof = proof.toVerifiableProof().valueOr:
+      return failure("Failed to get verifiable proof")
+
+    (bytes, blkTree) = (await self.builder.buildBlockTree(
+      origBlockIdx, blkSlotIdx)).valueOr:
+      return failure("Failed to build block tree")
+
+    cellData = self.getCell(bytes, blkCellIdx)
+    cellProof = blkTree.getProof(blkCellIdx).valueOr:
+      return failure("Failed to get proof from block tree")
+
+  success Sample[H](
+    cellData: cellData,
+    merklePaths: (cellProof.path & slotProof.path))
+
+proc getProofInput*[T, H](
+  self: DataSampler[T, H],
+  entropy: ProofChallenge,
+  nSamples: Natural): Future[?!ProofInput[H]] {.async.} =
+  ## Generate proofs as input to the proving circuit.
+  ##
+
+  let
+    entropy = H.fromBytes(
+      array[31, byte].initCopyFrom(entropy[0..30])) # truncate to 31 bytes, otherwise it _might_ be greater than mod
+
+    verifyTree = self.builder.verifyTree.toFailure.valueOr:
+      return failure("Failed to get verify tree")
+
+    slotProof = verifyTree.getProof(self.index).valueOr:
+      return failure("Failed to get slot proof")
+
+    datasetRoot = verifyTree.root().valueOr:
+      return failure("Failed to get dataset root")
+
+    slotTreeCid = self.builder.manifest.slotRoots[self.index]
+    slotRoot    = self.builder.slotRoots[self.index]
+    cellIdxs    = entropy.cellIndices(
+      slotRoot,
+      self.builder.numSlotCells,
+      nSamples)
+
+  logScope:
+    cells = cellIdxs
+
+  trace "Collecting input for proof"
+  let samples = collect(newSeq):
+    for cellIdx in cellIdxs:
+      (await self.getSample(cellIdx, slotTreeCid, slotRoot)).valueOr:
+        return failure("Failed to get sample")
+
+  success ProofInput[H](
+    entropy: entropy,
+    datasetRoot: datasetRoot,
+    slotProof: slotProof.path,
+    nSlotsPerDataSet: self.builder.numSlots,
+    nCellsPerSlot: self.builder.numSlotCells,
+    slotRoot: slotRoot,
+    slotIndex: self.index,
+    samples: samples)
+
+proc new*[T, H](
+    _: type DataSampler[T, H],
     index: Natural,
     blockStore: BlockStore,
-    builder: SlotsBuilder): ?!DataSampler =
+    builder: SlotsBuilder[T, H]): ?!DataSampler[T, H] =
 
   if index > builder.slotRoots.high:
     error "Slot index is out of range"
     return failure("Slot index is out of range")
 
-  success DataSampler(
+  if not builder.verifiable:
+    return failure("Cannot instantiate DataSampler for non-verifiable builder")
+
+  success DataSampler[T, H](
     index: index,
     blockStore: blockStore,
     builder: builder)
-
-proc getCell*(self: DataSampler, blkBytes: seq[byte], blkCellIdx: Natural): Cell =
-  let
-    cellSize = self.builder.cellSize.uint64
-    dataStart = cellSize * blkCellIdx.uint64
-    dataEnd = dataStart + cellSize
-  return blkBytes[dataStart ..< dataEnd]
-
-proc createProofSample(self: DataSampler, slotTreeCid: Cid, cellIdx: Natural): Future[?!Sample] {.async.} =
-  let
-    cellsPerBlock = self.builder.numBlockCells
-    blkCellIdx = cellIdx.toBlockCellIdx(cellsPerBlock)
-    slotBlkIdx = cellIdx.toBlockIdx(cellsPerBlock)
-
-  logScope:
-    cellIdx = cellIdx
-    slotBlkIdx = slotBlkIdx
-    blkCellIdx = blkCellIdx
-
-  without (cid, proof) =? await self.blockStore.getCidAndProof(
-    slotTreeCid,
-    slotBlkIdx.Natural), err:
-    error "Failed to get block from block store", err = err.msg
-    return failure(err)
-
-  without slotProof =? proof.toVerifiableProof(), err:
-    error "Unable to convert slot proof to poseidon proof", error = err.msg
-    return failure(err)
-
-  # If the cell index is greater than or equal to the UNPADDED number of slot cells,
-  # Then we're sampling inside a padded block.
-  # In this case, we use the pre-generated zero-data and pre-generated padding-proof for this cell index.
-  if cellIdx >= self.builder.numSlotCells:
-    trace "Sampling a padded block"
-
-    without blockProof =? self.builder.emptyDigestTree.getProof(blkCellIdx), err:
-      error "Failed to get proof from empty block tree", err = err.msg
-      return failure(err)
-
-    success(Sample(
-      data: newSeq[byte](self.builder.cellSize.int),
-      slotProof: slotProof,
-      cellProof: blockProof,
-      slotBlockIdx: slotBlkIdx.Natural,
-      blockCellIdx: blkCellIdx.Natural))
-
-  else:
-    trace "Sampling a dataset block"
-    # This converts our slotBlockIndex to a datasetBlockIndex using the
-    # indexing-strategy used by the builder.
-    # We need this to fetch the block data. We can't do it by slotTree + slotBlkIdx.
-    let datasetBlockIndex = self.builder.slotIndicies(self.index)[slotBlkIdx]
-
-    without (bytes, blkTree) =? await self.builder.buildBlockTree(datasetBlockIndex), err:
-      error "Failed to build block tree", err = err.msg
-      return failure(err)
-
-    without blockProof =? blkTree.getProof(blkCellIdx), err:
-      error "Failed to get proof from block tree", err = err.msg
-      return failure(err)
-
-    success(Sample(
-      data: self.getCell(bytes, blkCellIdx),
-      slotProof: slotProof,
-      cellProof: blockProof,
-      slotBlockIdx: slotBlkIdx.Natural,
-      blockCellIdx: blkCellIdx.Natural))
-
-proc getProofInput*(
-  self: DataSampler,
-  entropy: ProofChallenge,
-  nSamples: Natural): Future[?!ProofInput] {.async.} =
-  ## Generate proofs as input to the proving circuit.
-  ##
-
-  let
-    entropy = Poseidon2Hash.fromBytes(
-      array[31, byte].initCopyFrom(entropy[0..30])) # truncate to 31 bytes, otherwise it _might_ be greater than mod
-
-  without verifyTree =? self.builder.verifyTree and
-    verifyProof =? verifyTree.getProof(self.index) and
-    verifyRoot =? verifyTree.root(), err:
-    error "Failed to get slot proof from verify tree", err = err.msg
-    return failure(err)
-
-  let slotTreeCid = self.builder.manifest.slotRoots[self.index]
-
-  logScope:
-    index = self.index
-    samples = nSamples
-    slotTreeCid = slotTreeCid
-
-  trace "Collecting input for proof"
-
-  let cellIdxs = entropy.cellIndices(
-      self.builder.slotRoots[self.index],
-      self.builder.numSlotCellsPadded,
-      nSamples)
-
-  trace "Found cell indices", cellIdxs
-  let samples = collect(newSeq):
-    for cellIdx in cellIdxs:
-      without sample =? (await self.createProofSample(slotTreeCid, cellIdx)), err:
-        error "Failed to create proof sample", error = err.msg
-        return failure(err)
-      sample
-
-  success ProofInput(
-    entropy: entropy,
-    verifyRoot: verifyRoot,
-    verifyProof: verifyProof,
-    numSlots: self.builder.numSlots,
-    numCells: self.builder.numSlotCells,
-    slotIndex: self.index,
-    samples: samples)

codex/slots/sampler/utils.nim

@@ -9,8 +9,9 @@
 
 import std/sugar
 import std/bitops
+import std/sequtils
 
-import pkg/chronicles
+import pkg/questionable/results
 import pkg/poseidon2
 import pkg/poseidon2/io
 
@@ -20,8 +21,11 @@ import pkg/constantine/math/io/io_fields
 
 import ../../merkletree
 
+func toInputData*[H](data: seq[byte]): seq[byte] =
+  return toSeq(data.elements(H)).mapIt( @(it.toBytes) ).concat
+
 func extractLowBits*[n: static int](elm: BigInt[n], k: int): uint64 =
-  assert( k > 0 and k <= 64 )
+  doAssert( k > 0 and k <= 64 )
   var r  = 0'u64
   for i in 0..<k:
     let b = bit[n](elm, i)
@@ -48,13 +52,13 @@ func ceilingLog2*(x : int) : int =
   else:
     return (floorLog2(x-1) + 1)
 
-func toBlockIdx*(cell: Natural, numCells: Natural): Natural =
+func toBlkInSlot*(cell: Natural, numCells: Natural): Natural =
   let log2 = ceilingLog2(numCells)
   doAssert( 1 shl log2 == numCells , "`numCells` is assumed to be a power of two" )
 
   return cell div numCells
 
-func toBlockCellIdx*(cell: Natural, numCells: Natural): Natural =
+func toCellInBlk*(cell: Natural, numCells: Natural): Natural =
   let log2 = ceilingLog2(numCells)
   doAssert( 1 shl log2 == numCells , "`numCells` is assumed to be a power of two" )
 
@@ -67,7 +71,7 @@ func cellIndex*(
   let log2 = ceilingLog2(numCells)
   doAssert( 1 shl log2 == numCells , "`numCells` is assumed to be a power of two" )
 
-  let hash = Sponge.digest( @[ slotRoot, entropy, counter.toF ], rate = 2 )
+  let hash = Sponge.digest( @[ entropy, slotRoot, counter.toF ], rate = 2 )
   return int( extractLowBits(hash, log2) )
 
 func cellIndices*(
@@ -75,11 +79,8 @@ func cellIndices*(
   slotRoot: Poseidon2Hash,
   numCells: Natural, nSamples: Natural): seq[Natural] =
 
-  trace "Calculating cell indices", numCells, nSamples
-
   var indices: seq[Natural]
   while (indices.len < nSamples):
     let idx = cellIndex(entropy, slotRoot, numCells, indices.len + 1)
     indices.add(idx.Natural)
   indices
-

codex/slots/types.nim

@@ -0,0 +1,28 @@
+## Nim-Codex
+## Copyright (c) 2024 Status Research & Development GmbH
+## Licensed under either of
+##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+## at your option.
+## This file may not be copied, modified, or distributed except according to
+## those terms.
+
+type
+  Sample*[H] = object
+    cellData*: seq[byte]
+    merklePaths*: seq[H]
+
+  PublicInputs*[H] = object
+    slotIndex*: int
+    datasetRoot*: H
+    entropy*: H
+
+  ProofInput*[H] = object
+    entropy*: H
+    datasetRoot*: H
+    slotIndex*: Natural
+    slotRoot*: H
+    nCellsPerSlot*: Natural
+    nSlotsPerDataSet*: Natural
+    slotProof*: seq[H]
+    samples*: seq[Sample[H]]

codex/utils/poseidon2digest.nim

@@ -10,29 +10,59 @@
 import pkg/poseidon2
 import pkg/questionable/results
 import pkg/libp2p/multihash
+import pkg/stew/byteutils
+
+import pkg/constantine/math/arithmetic
+import pkg/constantine/math/io/io_bigints
+import pkg/constantine/math/io/io_fields
 
 import ../merkletree
 
+func spongeDigest*(
+  _: type Poseidon2Hash,
+  bytes: openArray[byte],
+  rate: static int = 2): ?!Poseidon2Hash =
+  ## Hashes chunks of data with a sponge of rate 1 or 2.
+  ##
+
+  success Sponge.digest(bytes, rate)
+
+func spongeDigest*(
+  _: type Poseidon2Hash,
+  bytes: openArray[Bn254Fr],
+  rate: static int = 2): ?!Poseidon2Hash =
+  ## Hashes chunks of elements with a sponge of rate 1 or 2.
+  ##
+
+  success Sponge.digest(bytes, rate)
+
 func digestTree*(
   _: type Poseidon2Tree,
-  bytes: openArray[byte], chunkSize: int): ?!Poseidon2Tree =
+  bytes: openArray[byte],
+  chunkSize: int): ?!Poseidon2Tree =
   ## Hashes chunks of data with a sponge of rate 2, and combines the
   ## resulting chunk hashes in a merkle root.
   ##
 
+  # doAssert not(rate == 1 or rate == 2), "rate can only be 1 or 2"
+
+  if not chunkSize > 0:
+    return failure("chunkSize must be greater than 0")
+
   var index = 0
   var leaves: seq[Poseidon2Hash]
   while index < bytes.len:
     let start = index
     let finish = min(index + chunkSize, bytes.len)
-    let digest = Sponge.digest(bytes.toOpenArray(start, finish - 1), rate = 2)
+    let digest = ? Poseidon2Hash.spongeDigest(bytes.toOpenArray(start, finish - 1), 2)
     leaves.add(digest)
     index += chunkSize
   return Poseidon2Tree.init(leaves)
 
 func digest*(
   _: type Poseidon2Tree,
-  bytes: openArray[byte], chunkSize: int): ?!Poseidon2Hash =
+  bytes: openArray[byte],
+  chunkSize: int): ?!Poseidon2Hash =
   ## Hashes chunks of data with a sponge of rate 2, and combines the
   ## resulting chunk hashes in a merkle root.
   ##
@@ -41,7 +71,8 @@ func digest*(
 
 func digestMhash*(
   _: type Poseidon2Tree,
-  bytes: openArray[byte], chunkSize: int): ?!MultiHash =
+  bytes: openArray[byte],
+  chunkSize: int): ?!MultiHash =
   ## Hashes chunks of data with a sponge of rate 2 and
   ## returns the multihash of the root
   ##

tests/circuits/fixtures/input.json

@@ -0,0 +1,527 @@
+{
+  "dataSetRoot":      "16074246370508166450132968585287196391860062495017081813239200574579640171677"
+, "entropy":          "1234567"
+, "nCellsPerSlot":    512
+, "nSlotsPerDataSet": 11
+, "slotIndex":        3
+, "slotRoot":         "20744935707483803411869804102043283881376973626291244537230284476834672019997"
+, "slotProof":
+    [ "14279309641024220656349577745390262299143357053971618723978902485113885925133"
+    , "17350220251883387610715080716935498684002984280929482268590417788651882821293"
+    , "3614172556528990402229172918446087216573760062512459539027101853103043539066"
+    , "9593656216696187567506330076677122799107266567595923589824071605501987205034"
+    , "0"
+    , "0"
+    , "0"
+    , "0"
+    ]
+, "cellData":
+    [ [ "211066599696340205996365563960462032209214145564176017965177408819390441927"
+      , "256399834032317991719099034134771774537377713676282398278615627599320306708"
+      , "40526956212941839024868120947422067322935297516255336725720469887577875470"
+      , "369406626072040375689238003388146123438765868500054546379159741776926336393"
+      , "333671948877941061129138970028865844558745266314244224980179694307884999701"
+      , "12844191661993811401197260475054004253686019503294245287625435635597431176"
+      , "103242505924551040184986153577926463300854479121445566984435820844932904541"
+      , "357267598134410031301285377503939045679462829143562392803919346036584141082"
+      , "162961530392479745020607594774288130869393650464001426863668385541077786641"
+      , "426633666684068667053061341108297711407154520752589357873877687002123230254"
+      , "131217200633679697678620573903653316618851166977399592012996945149000115543"
+      , "347806146382240882424437318028620747118202052207615339887983883245341422889"
+      , "373560143578047669373240014805175743607410583882636003337120578008437374619"
+      , "188643112726610812698950916978832639394489168469865816132268079928321994342"
+      , "261643693073361806247543578456646407192325702146752104760117340650316255422"
+      , "260425332276127964154119199351807753107358504026338378706506655351595199132"
+      , "374895089121103633563000003194929058314955925591115574099805066048344387554"
+      , "251251687633538360627384151887287228926166832908633974282160065378311171354"
+      , "72870348025150463132527129203816383011245664502016571773976961034605631401"
+      , "234969517550818492643515432666311755833657223377594670026839098818269671638"
+      , "250704662734070531273640113211555977086430125469327371276827055724111014200"
+      , "85287059658255939741261887818611116570376488685014337052369839580946343903"
+      , "148959658976765873884541474400081762855732313547557243964921157967555048302"
+      , "402116967301520959272239788104745348344918207829380669065019055837727389479"
+      , "440503687192139964066297025050080823601005280790824714962651368433530759519"
+      , "149064344353438643307231355233484617851197634669308957706338957575177745645"
+      , "249140840255377018814348914718469942883867200392561109698520706525194687651"
+      , "108796851176515124780842490199733462942992752881710253277179665118758071359"
+      , "168245155425564161902686596247453762364112240129335852645432169468767513906"
+      , "129930759770229612264501691941454447321585806001002088749773920103899362070"
+      , "26204732465033162738545933008873662562758704699080684615280202127289894343"
+      , "434343986187775289860542785529657444690073100436887472033336117760907652966"
+      , "361202432740487795596808128962740911600093857340619816047190021218849540225"
+      , "100616813001075101816809823591611435583084916492802624686700765550893945525"
+      , "262383566766515164611427346701355047932794351290691325516723194829671679460"
+      , "223966317895663049002893008659178463136086169222436544014405848127792334099"
+      , "416071800998333357259662686053338495720384342746822618737948080251761863079"
+      , "402582378631671531909245563300554883898015688826468151075059467077182712018"
+      , "271682867846395286938993638577506552857925968097084028550962231439839229096"
+      , "447239701195120035004067146414333903010840427278848992921567785105217019890"
+      , "275718367441702584521943480326858762208121719038684001399322597215141179102"
+      , "86424132534636958411139699704035287483951667275825975536719519441147485752"
+      , "149332313586183975744469174256094358432564607635406143904268565140988616920"
+      , "431284330776421588418608279008210842837281123158951642164956884286883748089"
+      , "328694410745471749523135644660455669483988686888634622076863114197617693825"
+      , "112671940998917362968156144648543607958275336559773039070338509488400224090"
+      , "40612851250697989627190554726382690498263128439797780029697069621854862060"
+      , "235047914228675997216342104196597257178021277585839376175077878186492271543"
+      , "169718735151210135199527910197065439221144015957220768545119706561079163228"
+      , "345850109040121443591415752965486014695272086901102608769402892906795715635"
+      , "107916794601837835951508003838161872232087225679609623116098837565956752373"
+      , "415195406060373162425374246348197423165008252112453298469572523506488563795"
+      , "18574536720926634955170276058049993354780447816096707123565164996905722992"
+      , "77316964375201096801231570737992491072607736322255206149311341101525354519"
+      , "198566211140075666401818378905444403588640345933666108724809349396921957675"
+      , "71926050707400318807625168942501384117254391471312636171205913503928815127"
+      , "303403754792341398295052586417779761162194818661412867094096550880325459639"
+      , "444796230931706624375881141460151785952798771079111017071124833045933389733"
+      , "430832727519144759265424205289522948157007336118070755365887670730658782114"
+      , "75431213985866235726407973044434444984663930761207296437571668004273515965"
+      , "9242635103653159191249730220870735855877366952081272723035956668095954838"
+      , "93770678769846326584848478152412123903909949977598807336203128684179492141"
+      , "438043261966019084676377174988310087831395864304423411701911688757793135582"
+      , "175357918416563657734972138036003712114814934655676571874083109097142591069"
+      , "301619681954194702458985259161884119574424456150215738560639417824784261940"
+      , "376627771252167062559065889174364784087884871999807562320457079200311413098"
+      , "77407"
+      ]
+    , [ "424838222746158988229624788694939151178615656210585621868910231014323837551"
+      , "113188486246955346418956949679485575685258850346101035778277727456423482970"
+      , "275449978206132565019222655023969430014622832597654643418394602485645803413"
+      , "407856757018138010439232009766252068440920591566039673660874626878413077905"
+      , "433369046493777496016384947949950877133856528218602671493669395706908819748"
+      , "258364166531180422149545015891786023981872586904946376136311798402581278793"
+      , "111997719221028147522803956659709775148434180015507797582340706052412284571"
+      , "370086597976426508280413876667101417393598181708677558733730556109327409076"
+      , "394139601979449259075910238117153992797849553309541269624848742084205563806"
+      , "224088276319080487199395482893988152025671468715318212801266537036640477323"
+      , "412710245119809501914481942088314642684754542082140451180970198371889738885"
+      , "353872602359946553306242341162348980635834907495492814598834657175405697176"
+      , "252575199506028475372678621140654219936768774012823764047177692104580452933"
+      , "259093153824033122452869556249315839899366493071746219770487886456301642099"
+      , "433829976798312333371154167497560676982294392632725612538640639617101218872"
+      , "69918581382122563555200898078544150952625715196904114153232367538572342772"
+      , "337937520623192257595352158476909569245988839238498098464935654555688460123"
+      , "264739973232292969253318643276671532055689422253184191167449284055983944338"
+      , "326116252818829775096345069850111970510714050346103409479803743342138260656"
+      , "236666502377413649728378889488706275212721356921124776708954261777813709815"
+      , "211625935799984260567718228446525455893664313064841539301444509150157287163"
+      , "60213206239417039999880027112341976360540689886703427811513517396638607512"
+      , "68310118105957780876770075529546844404225720757669797609686816545988561625"
+      , "423863085551351065136684030270731679105571943009795949621903966660399419936"
+      , "388914614294393005039878123500859325222684672184567792659076815268598434245"
+      , "449456790291560508709069826219925144971979653209755565240911568965768874382"
+      , "448810363770763694447869940916735951256986784286793489549428379909616059117"
+      , "93646909783664049092056237949587618925209622020026157405117796611689551192"
+      , "352210795298632954574896499649181574584074853828419384742874364724522457331"
+      , "37455517056393404525863484733101879886413925183061645520768912552476716150"
+      , "386617357584684336812125385078476270301738184058813703112840991226785114117"
+      , "309940292044597334261558429176136686101590729982259514472573656131579113438"
+      , "375815246167575100319857872432879650174355611853064771241069582477717074415"
+      , "332214507344122806007757734266883566559371568252728459951766124888176633706"
+      , "148990259460952914990881100852534318351247069504848477833147446514732789712"
+      , "328669527889838880414072022433859139004058211332184916573516704632073044118"
+      , "39278026039348543645873027549112998051601664395028652771103624299930924528"
+      , "147717660530589785119644237145092759103012624422229579698752386490700965238"
+      , "374018518701362017594752095877197725242352803195413267746619111489936980685"
+      , "19185486483883210730969367354195688373879769005461272861759636600984416877"
+      , "61866046506558157021682973167090213780467780519546382332208868591026703563"
+      , "186854966504766517012887726614015646154225796572138017810371160981778288347"
+      , "87813550250328892091332566928942844770632705056120813488729800874811845697"
+      , "207775163424060266085108794048371834145545842567796157378282772999548202308"
+      , "369987573847786237689249538753881486995686208870889713177975415012214427429"
+      , "240880979016395044518849230927615466120209140082149273390921042537474853143"
+      , "174902051454932913375934735427101804474275543418199101786687925733405159872"
+      , "342217255652950822372803598682842961053537267723988087801275319754065261308"
+      , "403207518067666945448161377960706451817747922771285796668778802535227939962"
+      , "407191459999036791052261270163259267557900498930952521056725210031161568230"
+      , "338216472523551728793268225845396218561132966393725938551091882807069657206"
+      , "118364222678576550870375112494142500603091119946985793934499723872824782886"
+      , "269721611028665323587192624288165848310917029418261851436925829954710472436"
+      , "227424498125745236125352117206136621428869662458452610254773560636280935711"
+      , "334380807433339401906359555583987757411855090694162252260781648609761248049"
+      , "42470806516174819075107446234449247453971524726021445768611797530804156161"
+      , "418994916402918322951830716592888390611524984156817012683478842068581638820"
+      , "363263142412048420546716019774090729399081311227606555141174736853886128407"
+      , "192292715923468025058557166341646729623133127372303791236447550026886802680"
+      , "450253878713722337767292128303865371116770532625906889925779639839924402495"
+      , "412596147086332805611392200560087191411541689130482740065137300252639590489"
+      , "264059866105067484811456793906835462997849523506366903974724979936196358724"
+      , "80922384260325792825608274004101667366364502159441286540209512108302572137"
+      , "69261112192907071823876894642934902051254647002357333297635511793652550535"
+      , "342810644068354896385837929029331085645928473943142618800192452300937098227"
+      , "228361826362950356801549793202622774869100858404479869989505302905936946659"
+      , "89244"
+      ]
+    , [ "359926778925154809567585559910064420821311221299937568759183366972901588855"
+      , "128688825421597555299113005875649976344934035310192572516197551102045945994"
+      , "225379354482053502673695304662016054746658452775856157938886424152134693969"
+      , "321872319934407904743844034025550739031781361848065513098922085967524180784"
+      , "250375637325405951645070615947051799520095842815922754899017741501395744611"
+      , "97676493052788080672307677749501730337551225267342432472194527468634722352"
+      , "140101187036396881926000630022834247990512766860086619783437252676747730662"
+      , "428833039353549335335605804240430918631639449874968791377641834408506136850"
+      , "418359203734539413977740838354554804415161215624809316660001820037711273005"
+      , "197411877418795659129213175603102238904459737200167987167255995825203749339"
+      , "221646252100316559257470803343058462853641953756978011126414924636869625612"
+      , "106393540293584181037890192557883541231531964825708650997196071036779482686"
+      , "121473330828208543539643554911190528237124877123871673169194452404939833883"
+      , "234055622144947293638512253368547046093971383516706577723613696162225606040"
+      , "68307451767502390304445005915787226559811433450721625085741437522802389574"
+      , "446891883436763112014492564462451523127134145501201571918449345324780633462"
+      , "83718652783543018019599555197511164121642363321504039439786358267060414949"
+      , "90267297500929836073049162292104311427365986272517761342871530827272320168"
+      , "398425606698859520268856768787424690872952910789754531465894080258173664751"
+      , "323570139379118444589557840594603212198136718240764273769210492735883659788"
+      , "318597103584099056378057647488068323974418467250708490151864712850204121402"
+      , "6299083430773359277240726214182464517380839990956316267425262319606033077"
+      , "27638206326925436960316131682014727983280820447721477666884742925275976240"
+      , "434344186848869917381375812528446841024676181532946456237439060027443649574"
+      , "64735754118644738348599761514174981324344130078598038275246522384474432918"
+      , "53068717269762105498508401788249005564862415051761175636612434108259085043"
+      , "35813044996911619267309099508360887777226716179396659295580849861836012116"
+      , "67751791392924142809580984450371772015056060429352159361446914484238646676"
+      , "68534949135677447506316576616742207938855454921330757052467057435206318183"
+      , "98510151949547604999069864337574320742530406479752176012935179772005228326"
+      , "342190252152505345443004241184891966319091967630257822491352072978326623645"
+      , "362701658859425316334005554473186516818256386066010799465369887406035738447"
+      , "266999116654850467726292928465517542818678046748008340458185725047959981772"
+      , "227089355966197874086821090531951502393729872265201602128464978982907992285"
+      , "240800343500959216904535208047288234867926058830277460630902462914796702354"
+      , "447956858573680756485556898469710354624642653441041335815079102198306530583"
+      , "89422712944117481549242421245588048728782658978853365197341587057196539094"
+      , "72610343179362050463955730204044877712105879926618304878262944723464870506"
+      , "8676698500519447254981838968537883138182811064381378248657969913325524054"
+      , "180453700216061196739413267121764366438386031039311941313645439527087166894"
+      , "63346784016053253727621352882430122335280702556586808389293772066791583857"
+      , "400031453850139978805133735852120986024101930860924735862305444817383365395"
+      , "230104622290558570218036071349472289358926019290368625724986905348610140188"
+      , "175689489221336091369196327293045073133701056385414159213338224521167050830"
+      , "73310331103509697419315970265031228794034932318600293733074730812482185479"
+      , "371383255255842707875498538452907102684511927320158672347778293876877893808"
+      , "165319345890230193939972313953881372394171342391835626454759321320435952720"
+      , "184753541001210613115361457830691571384268642766106935428207829332011259768"
+      , "378810733004878432563271790625801205570962762395854282745166380274493181314"
+      , "86321674336629444862383262780020871828941143514651008200999725989892879714"
+      , "332634533993388248915777870988529817692938793120418377552247997050250349749"
+      , "41742010257820712511267405684781534740157292266212120171929853763272599516"
+      , "224101330592139734390658213442359402546038743346294438455635537496290117560"
+      , "204363902046137087420878796391135987770418514008394389852388361468850216359"
+      , "296526036888463156867153847102999031430641220536577556044554029990778763710"
+      , "137568796227115931047082828377698464185467276723279763304686078869351280509"
+      , "147456720891843338735232645318045656082153590170441596326500204393398792771"
+      , "297291342309626392877004635010131510068476699687818509485687346071074942006"
+      , "20748013593486831233186810444485136836664689496258849465507672301203832324"
+      , "335431726883875036252568773858744219410188302724423071283643221351691013313"
+      , "50487384098835523033417523223562560857744547945136829388944024807752630716"
+      , "425952679139710019732649156938676226492714486376811103362264146658191708598"
+      , "439787938069461539508728805881194071103269524405653997968488488049426387373"
+      , "279863410796988495259178322026394289028023166886112128504434877538089779477"
+      , "398941099058270093463626870965433502581301413791423667994257456160377865247"
+      , "5759692644185723491187478313102540786562625675495805072053262277490225012"
+      , "115176"
+      ]
+    , [ "199440901482393381753657315848210955092644686957900737971520778206058989647"
+      , "339215657660349719251187938243949119362753238126598750470832739830379601048"
+      , "17957417011314891245400567671664723859427624136284133790923936126779445290"
+      , "294761585889095249928492042608516765584723814657871392207964321318076158536"
+      , "367304199921887970655898837985855454346911090426896946930048519042744277770"
+      , "173405546837606747721292792526074597519538685230485266741646923399938591491"
+      , "13202798104529529703580600642858924379886936325402696094624200032343206719"
+      , "28211272278315691894282764239983301742024079691520980592618486887749800025"
+      , "73792448247120972778500624350664849847034095641998271809779791788652649022"
+      , "386961947078838359430674078072441680475090687247027225632133013772954043342"
+      , "247859266401821616700765969075270662915024391205665146199401830650793676517"
+      , "243938047874995926342875119559105623088213951205962677439167259642163766960"
+      , "14909501249861872673329370269106359532506159818320693170564856401208688898"
+      , "200331653478898243177761429526240803993101536716611440775588088625522029071"
+      , "127891684617049394579738365914860024509007913559921966744972525605976847919"
+      , "202912167983786187592861727924433749852786012202809544200943965898118027816"
+      , "176370650316309755425558132466370508977563252342126855874617990006444464573"
+      , "179490319446297562731655155074196383539396893457024237113284509223965454107"
+      , "118703379899134287650980989454755985628620830085932176414465582081598659194"
+      , "102025594191113707886629488454876652037341696284939671367279050169222988689"
+      , "421132375430104331136732058548913808473025321492255503838896123601628815453"
+      , "328334791815856213206267892694535121226673194052951034497930366807851111845"
+      , "83012322813281668737061895967682970093636853452224812061110092135287899376"
+      , "329204708391107220275172926348002826875172581958734129645882445887919889321"
+      , "410748869385474485539728045765785256294532845070137964737879849265390625591"
+      , "197274807717335387012872999914232051341799797613667869923402386359379902675"
+      , "235713095185988155553500217595661312303861624791720350423698435045853678746"
+      , "150631584359141913552843813384841153102535679219851913018874172414598353488"
+      , "207783836843813911284913666774420250626019971129676431904589416081127483900"
+      , "15728034718954665549174230921445077500399069880555428849958014406422697976"
+      , "69799423545177501667748653663121504155622623013014583690766769624482972893"
+      , "265665371394145560256710553779588458846778960884758560117805511822299802326"
+      , "149195925869584039415331014261414953602245337159354350672030141190471260449"
+      , "162328395279114064180857718453973759378615891406692054752029241989300597156"
+      , "104643123291849369328362168243087456326274773811561124607060302871149280568"
+      , "320704123383295724141970902124509237736831907552238934395000039155541269937"
+      , "77914486216152383968400591704791756847610018757857965758408455442143531631"
+      , "238365259321088298905088250444146071545398991768186471689605160523129613763"
+      , "279409375422154593510552116666741774885392805494152031820287626934209978908"
+      , "195118776021452609708543280509123101181249086555819844047203390788132717252"
+      , "197977884437087886153482042453896079316138251415359773453987232734849953584"
+      , "168185043240980087638006965666857387510828226074470344841850764460631595331"
+      , "231157923359356077977363679818678437836536420121744865399935742538602805912"
+      , "177903771863742191900138437188329108771172098110036075491750018158192424072"
+      , "313552174443290416730632310997197097951229162137491595709733420111980331403"
+      , "273253450712049988786741336540196077743997302924525995219038781650977490211"
+      , "421908030281055821389377531613150504859996607596444776050212044919345332385"
+      , "180108184992593746898140529947178182204857361841304042854173884504394805936"
+      , "37075272799330399065679301151342697855905822543084867570322173216259074746"
+      , "364885615491975468180698816037289079302391684470748470356247509218051645743"
+      , "397482868106190800111749908137311511782294652288655656060379563261618687603"
+      , "192853269627895017416381451198403197013798085262867793715522216458130791820"
+      , "450480853450142791928572953497280890976328598410525218090104787739409705079"
+      , "40278654070502961330170439514434669768416784968274956021022493731441898222"
+      , "251277143131769020481025315216040765839561111684608785317366609258959942695"
+      , "95094468748825454459610961968601800404132682484160170977941285794444806916"
+      , "160586633865113902389134480029183924655750088163491531655080014223476604929"
+      , "211661229493873434581423107377168478957907088187044576032505407590783850232"
+      , "409651293631434750272174456674594508340952460788494035327119354167465019826"
+      , "233213211946836553080627522409887799285199986120567245752841080275284294566"
+      , "143182900674482411759481361336063079267405785923487419697568907351386146653"
+      , "430050085956999990041799366370428972519385994997821389120583306252090911051"
+      , "241257468571530133762629460194062384921386438426367660851087853915892684115"
+      , "106478922860328643074356032194102718325829441005019365153538120054339275205"
+      , "252933430690486626644000908036895289326526510137514385824014300035301154822"
+      , "242924628511152862437189415942615812459145003511499320946981326550434266392"
+      , "107566"
+      ]
+    , [ "10892488375325920610152294701785926476935321890453222549428070867493882259"
+      , "230776541958253414701326765204413805639078570664616139597900921490475143840"
+      , "162235550819840758141721889536295480113278275911087429090017541814695333320"
+      , "318634611531007856220026646570492630940047240387334221027051366284225674524"
+      , "347695480420330337439096561080975864031317110001559084872978387417651465445"
+      , "243301070227446762125488369714708670219289121859111929877695012393609726208"
+      , "312153141205681954392441579373652470334482191990502803524039048245142338874"
+      , "243769659456658813016931656268556893289414122189859136776671772112510762644"
+      , "235510946617019540983239794402008700611014664578713646813426116406215382253"
+      , "394638234040056271265534896894991100558052611842099314808878257003754175212"
+      , "112730195097163222179949980634599934392634120069300673310070655800491242211"
+      , "112545144551723145227061757291353149296490850338535641681237178578430772288"
+      , "399161925498018051746424503488168548076537557369822821644005390567188305750"
+      , "291823556779130095044851238536860577785281148245623338113991004798525195947"
+      , "443006765181360772964201996695219533539991249361936300010158089981376772939"
+      , "74018417655448362012716269253153545524112649147848318337218250865231619883"
+      , "361038295627629757439073080656763633087074408866229383288288831546300069767"
+      , "269655542872834422597617091710145830035767834433114567250727497135451412216"
+      , "58289072717559976781527037332370898163550414718655391280446986067842922181"
+      , "365399954331278626941053447122605263207706937018525782908294459265663426953"
+      , "83576501872896181822963149907518188809522341045703363171261203985876068484"
+      , "203403783686919490357886887779316222746544665267595235714814853282937072937"
+      , "226090172488558641139782927103632452136731207206963592401497570070700221117"
+      , "249813560776802008219945831355163226098682741553305882699128946352901227282"
+      , "236586835155013316983057471119105606475813711035522306026673814184519826069"
+      , "420611449257527132395951061920868895981487081726689195426673433565077012458"
+      , "414979562418422585161189066520079202660418565693776869373399931253524536378"
+      , "115851377630895049619958829726730418778383692593973233290077769966938018584"
+      , "248071158447148977966329235335508229928647083339924308951291087789494073866"
+      , "8254100651607835318906499132096926759832050649688561036854000785129084907"
+      , "91385483239630999205401307763890943367451672856478206313711901403403429289"
+      , "369346641925770698935046191632374874762045839747680407985443471202350286304"
+      , "236809023553698844817613980395588655518336237009075203833474757804664254158"
+      , "8367847400805682648908349286699722579431227561083498361702537964306290078"
+      , "599241730770400067632779114435635342549228985534229813617556932580328166"
+      , "347112528350917448294202348829052279076907614831011498643025223836596915573"
+      , "384244379244118003891043669466323943736726794634167201471569326059716944701"
+      , "118013777197672343498581960057939216208494837962825017767101107204031333144"
+      , "27234916267695376599463409893017377196853108034589808756909998459364893467"
+      , "443519198016088819704735929590164254445884637317806160485888215392818578737"
+      , "396780482611567392375183169345153676737175342167284140440545202776279411157"
+      , "420351155303051883480975795437743307852799862858964108014000673383502660760"
+      , "17379377743250873773932440622865094720355292220433235366224143179854831702"
+      , "299671454782683147939928632170233327590769402224392134648893444626929909373"
+      , "143062753141414050359792615867774312517100868919516205179025540179759009492"
+      , "79497692490953838158801094558761984613913564034406069659969793097043605498"
+      , "422748645389700647011491406944374966856916994331478229959954030359911549565"
+      , "101802829812014644970197499895811874607753186302439171072935333706660468030"
+      , "376428369998893026519415315112012919906032811618495880392785036762185101192"
+      , "193969030999254195249242252871597931610859408264053152789041067245597391073"
+      , "262277607928686742238285487190873200602833495734085188071246746209841324139"
+      , "154099884960502807271641574310268486840763221700920893692135020347157046386"
+      , "155875061164585018658671842995328931296342883770473498362059838106632382461"
+      , "248574435283666782825705601259695525637993294311364397935499480206725256362"
+      , "171325185063038052248966557755722232979612702743265316145563443527135798688"
+      , "19982746887818897250405980185937061235439217109294376948752373205830077881"
+      , "363719103724181291346130833008745948141602173373912337315631878022251200824"
+      , "174596812883797666354579747966720458118607233660798323559531788300018084931"
+      , "296611197821867585469311917529661698312828606304722408477045992233526328708"
+      , "115884038550627840768260751168697204665962542002024023842649383174518895165"
+      , "265597417366164841889730505737916646261040505851159477903649773521314216810"
+      , "59890857222664166616144499778264545115438678877460924559608721809777680238"
+      , "150275344313515259978222149421664752546204516114655782705875535407472455999"
+      , "119762211657733951640135703777940013374803447228667976561992857374027112851"
+      , "124750313254270944205036764607428674226393862430770588148329786501973600535"
+      , "223562415856611692667255745278751292230311455355383541649772348032933666931"
+      , "70851"
+      ]
+    ]
+, "merklePaths":
+    [ [ "12330511756602656312909435206144412037562550923861053314147193494605624608532"
+      , "11626412651279744307149389430094868876817959605147995292836995419396445628874"
+      , "5992799448428980485292103987999399446160713735809250167288027256759377161164"
+      , "19665782623633007009046423286205585606029554995684266199914536255425604862856"
+      , "16487082247902739801276638140231448150060608377815926330854457096549924699346"
+      , "13757776896542890425183206586238760231244468647794123671233758868377423038254"
+      , "5689382212199289790852870716648320118282913659439556386010574440270030991956"
+      , "19397819444005071538172083272808000152189643623980212079104170950073560541073"
+      , "13602141253349313166682170066159161155345481788270338632198566205283140117430"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      ]
+    , [ "20475873274412005481064639175076772466269555202404881992985142339142752174247"
+      , "16346160910918020455716115524174351294558871582658635761456705803199256575588"
+      , "2853750013041818757293788273269978233503065226276819540991640203844566736443"
+      , "9192572535522846104757923561847416578417599996904208474460700268961782558170"
+      , "11041850361074018956732434352769265303294549935255362322653487210796196161858"
+      , "20835509643844784930831626622202658364476409300598072395494952478408974334325"
+      , "15426115581767819720710837762133134950520914636073261355445708100826108573907"
+      , "7565353224987902191368863653499353764559862741092477570130316358454603122676"
+      , "2622681935585012630617774892501744551457568716225188460692779556142778732663"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      ]
+    , [ "13099868869639061574800067722436547911616384523753701384396275064543709640456"
+      , "353757809120595355213201328586632712724405232919181040928026587840976194078"
+      , "17653300914565730132855106316678548541847283141888106932466281385199556950861"
+      , "15467462085462582082877261755656498905479817107855355753427299990712166382496"
+      , "8291733777946446853018893495264026584437749231931118771866890345692346711355"
+      , "15510790697317206014779022286261864844918915222875014882833700758879700055506"
+      , "5689382212199289790852870716648320118282913659439556386010574440270030991956"
+      , "19397819444005071538172083272808000152189643623980212079104170950073560541073"
+      , "13602141253349313166682170066159161155345481788270338632198566205283140117430"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      ]
+    , [ "123238869181525326412236116167249816873084559218151119452851092131080991962"
+      , "9610314342084317647296061595824603740114670828357751076517430572434680540425"
+      , "16802740554584732104294972716558962567961331277692246600846665155168171370476"
+      , "151083360419914122898584757765086723506432610661508069194962432698872036623"
+      , "10357032992337239725601662829902169825217513617307319193581711776597892496381"
+      , "10120699018002766520605012835043517238241846918467244955580419060582311503402"
+      , "21149604008153751948441881526949680605328007895979738537313721955134548786062"
+      , "5720106921878932614189421948890362637585879521377362100104826996201092964473"
+      , "2622681935585012630617774892501744551457568716225188460692779556142778732663"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      ]
+    , [ "19153513112714782931111012244694922243101677748840856395814929006033044311081"
+      , "21046138187228318287277629107063936540039891592394801899272249280765102572688"
+      , "18057980437430910028015917806534512217725128031222973066601095455076586015436"
+      , "5766677914654397407881589917461473873246279171605373166264025525757502238061"
+      , "12019967669236656188577515900815533059046454955207846938479617973037184411021"
+      , "14504305765289705714959523666100275156034056689367568164630385862257567596209"
+      , "7152002871325824138073253423783370852632926621899161541618248808716037342022"
+      , "9714587356194206699401761190093056901650105401919163689816999407566849779455"
+      , "13602141253349313166682170066159161155345481788270338632198566205283140117430"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      , "0"
+      ]
+    ]
+}

tests/codex/node/testcontracts.nim

@@ -31,6 +31,7 @@ import pkg/codex/erasure
 import pkg/codex/merkletree
 import pkg/codex/blocktype as bt
 import pkg/codex/utils/asynciter
+import pkg/codex/indexingstrategy
 
 import pkg/codex/node {.all.}
 
@@ -54,7 +55,7 @@ asyncchecksuite "Test Node - Host contracts":
     manifestCidStr: string
     manifestCid: Cid
     market: MockMarket
-    builder: SlotsBuilder
+    builder: Poseidon2Builder
     verifiable: Manifest
     verifiableBlock: bt.Block
     protected: Manifest
@@ -84,7 +85,7 @@ asyncchecksuite "Test Node - Host contracts":
     (await localStore.putBlock(manifestBlock)).tryGet()
 
     protected = (await erasure.encode(manifest, 3, 2)).tryGet()
-    builder = SlotsBuilder.new(localStore, protected).tryGet()
+    builder = Poseidon2Builder.new(localStore, protected).tryGet()
     verifiable = (await builder.buildManifest()).tryGet()
     verifiableBlock = bt.Block.new(
       verifiable.encode().tryGet(),
@@ -127,9 +128,12 @@ asyncchecksuite "Test Node - Host contracts":
       return success()
 
     (await onStore(request, 1.u256, onBlocks)).tryGet()
-    check fetchedBytes == 786432
+    check fetchedBytes == 851968
 
-    for index in builder.slotIndicies(1):
+    let indexer = verifiable.protectedStrategy.init(
+      0, verifiable.blocksCount - 1, verifiable.numSlots)
+
+    for index in indexer.getIndicies(1):
       let
         blk = (await localStore.getBlock(verifiable.treeCid, index)).tryGet
         expiryKey = (createBlockExpirationMetadataKey(blk.cid)).tryGet

tests/codex/node/testnode.nim

@@ -126,7 +126,7 @@ asyncchecksuite "Test Node - Basic":
         codec = ManifestCodec).tryGet()
 
       protected = (await erasure.encode(manifest, 3, 2)).tryGet()
-      builder = SlotsBuilder.new(localStore, protected).tryGet()
+      builder = Poseidon2Builder.new(localStore, protected).tryGet()
       verifiable = (await builder.buildManifest()).tryGet()
       verifiableBlock = bt.Block.new(
         verifiable.encode().tryGet(),

tests/codex/slots/backends/helpers.nim

@@ -0,0 +1,143 @@
+
+import std/sequtils
+import std/sugar
+import std/strutils
+
+import pkg/poseidon2
+import pkg/constantine/math/arithmetic
+import pkg/constantine/math/io/io_bigints
+import pkg/constantine/math/io/io_fields
+
+import pkg/codex/merkletree
+import pkg/codex/slots
+import pkg/codex/slots/types
+import pkg/codex/utils/json
+
+export types
+
+func fromCircomData*[H](cellData: seq[byte]): seq[H] =
+  var
+    pos = 0
+    cellElms: seq[Bn254Fr]
+  while pos < cellData.len:
+    var
+      step = 32
+      offset = min(pos + step, cellData.len)
+      data = cellData[pos..<offset]
+    let ff = Bn254Fr.fromBytes(data.toArray32).get
+    cellElms.add(ff)
+    pos += data.len
+
+  cellElms
+
+func toPublicInputs*[H](input: ProofInput[H]): PublicInputs[H] =
+  PublicInputs[H](
+    slotIndex: input.slotIndex,
+    datasetRoot: input.datasetRoot,
+    entropy: input.entropy
+  )
+
+func toJsonDecimal*(big: BigInt[254]): string =
+  let s = big.toDecimal.strip( leading = true, trailing = false, chars = {'0'} )
+  if s.len == 0: "0" else: s
+
+func toJson*[H](input: ProofInput[H]): JsonNode =
+  var
+    input = input
+
+  %* {
+    "dataSetRoot": input.datasetRoot.toBig.toJsonDecimal,
+    "entropy": input.entropy.toBig.toJsonDecimal,
+    "nCellsPerSlot": input.nCellsPerSlot,
+    "nSlotsPerDataSet": input.nSlotsPerDataSet,
+    "slotIndex": input.slotIndex,
+    "slotRoot": input.slotRoot.toDecimal,
+    "slotProof": input.slotProof.mapIt( it.toBig.toJsonDecimal ),
+    "cellData": input.samples.mapIt(
+      toSeq( it.cellData.elements(H) ).mapIt( it.toBig.toJsonDecimal )
+    ),
+    "merklePaths": input.samples.mapIt(
+      it.merklePaths.mapIt( it.toBig.toJsonDecimal )
+    )
+  }
+
+func jsonToProofInput*[H](inputJson: JsonNode): ProofInput[H] =
+  let
+    cellData =
+      inputJson["cellData"].mapIt(
+          it.mapIt(
+            block:
+              var
+                big: BigInt[256]
+                data = newSeq[byte](big.bits div 8)
+              assert bool(big.fromDecimal( it.str ))
+              data.marshal(big, littleEndian)
+              data
+          ).concat # flatten out elements
+        )
+
+    merklePaths =
+        inputJson["merklePaths"].mapIt(
+          it.mapIt(
+            block:
+              var
+                big: BigInt[254]
+                hash: H
+              assert bool(big.fromDecimal( it.getStr ))
+              hash.fromBig( big )
+              hash
+          )
+        )
+
+    slotProof = inputJson["slotProof"].mapIt(
+      block:
+        var
+          big: BigInt[254]
+          hash: H
+        assert bool(big.fromDecimal( it.str ))
+        hash.fromBig( big )
+        hash
+      )
+
+    datasetRoot = block:
+      var
+        big: BigInt[254]
+        hash: H
+      assert bool(big.fromDecimal( inputJson["dataSetRoot"].str ))
+      hash.fromBig( big )
+      hash
+
+    slotRoot = block:
+      var
+        big: BigInt[254]
+        hash: H
+      assert bool(big.fromDecimal( inputJson["slotRoot"].str ))
+      hash.fromBig( big )
+      hash
+
+    entropy = block:
+      var
+        big: BigInt[254]
+        hash: H
+      assert bool(big.fromDecimal( inputJson["entropy"].str ))
+      hash.fromBig( big )
+      hash
+
+    nCellsPerSlot = inputJson["nCellsPerSlot"].getInt
+    nSlotsPerDataSet = inputJson["nSlotsPerDataSet"].getInt
+    slotIndex = inputJson["slotIndex"].getInt
+
+  ProofInput[H](
+    entropy: entropy,
+    slotIndex: slotIndex,
+    datasetRoot: datasetRoot,
+    slotProof: slotProof,
+    slotRoot: slotRoot,
+    nCellsPerSlot: nCellsPerSlot,
+    nSlotsPerDataSet: nSlotsPerDataSet,
+    samples: zip(cellData, merklePaths)
+      .mapIt(Sample[H](
+        cellData: it[0],
+        merklePaths: it[1]
+      ))
+  )

tests/codex/slots/helpers.nim

@@ -0,0 +1,166 @@
+
+import std/sugar
+
+import pkg/chronos
+import pkg/libp2p/cid
+
+import pkg/codex/codextypes
+import pkg/codex/stores
+import pkg/codex/merkletree
+import pkg/codex/manifest
+import pkg/codex/blocktype as bt
+import pkg/codex/chunker
+import pkg/codex/indexingstrategy
+import pkg/codex/slots
+import pkg/codex/rng
+import pkg/codex/utils/poseidon2digest
+
+import ../helpers
+
+proc storeManifest*(store: BlockStore, manifest: Manifest): Future[?!bt.Block] {.async.} =
+  without encodedVerifiable =? manifest.encode(), err:
+    trace "Unable to encode manifest"
+    return failure(err)
+
+  without blk =? bt.Block.new(data = encodedVerifiable, codec = ManifestCodec), error:
+    trace "Unable to create block from manifest"
+    return failure(error)
+
+  if err =? (await store.putBlock(blk)).errorOption:
+    trace "Unable to store manifest block", cid = blk.cid, err = err.msg
+    return failure(err)
+
+  success blk
+
+proc makeManifest*(
+  cids: seq[Cid],
+  datasetSize: NBytes,
+  blockSize: NBytes,
+  store: BlockStore,
+  hcodec = Sha256HashCodec,
+  dataCodec = BlockCodec): Future[?!Manifest] {.async.} =
+  without tree =? CodexTree.init(cids), err:
+    return failure(err)
+
+  without treeCid =? tree.rootCid(CIDv1, dataCodec), err:
+    return failure(err)
+
+  for index, cid in cids:
+    without proof =? tree.getProof(index), err:
+      return failure(err)
+
+    if err =? (await store.putCidAndProof(treeCid, index, cid, proof)).errorOption:
+      # TODO add log here
+      return failure(err)
+
+  let
+    manifest = Manifest.new(
+      treeCid = treeCid,
+      blockSize = blockSize,
+      datasetSize = datasetSize,
+      version = CIDv1,
+      hcodec = hcodec,
+      codec = dataCodec)
+
+  without manifestBlk =? await store.storeManifest(manifest), err:
+    trace "Unable to store manifest"
+    return failure(err)
+
+  success manifest
+
+proc createBlocks*(
+  chunker: Chunker,
+  store: BlockStore): Future[seq[bt.Block]] {.async.} =
+
+  collect(newSeq):
+    while (let chunk = await chunker.getBytes(); chunk.len > 0):
+      let blk = bt.Block.new(chunk).tryGet()
+      discard await store.putBlock(blk)
+      blk
+
+proc createProtectedManifest*(
+  datasetBlocks: seq[bt.Block],
+  store: BlockStore,
+  numDatasetBlocks: int,
+  ecK: int, ecM: int,
+  blockSize: NBytes,
+  originalDatasetSize: int,
+  totalDatasetSize: int):
+  Future[tuple[manifest: Manifest, protected: Manifest]] {.async.} =
+
+  let
+    cids = datasetBlocks.mapIt(it.cid)
+    datasetTree = CodexTree.init(cids[0..<numDatasetBlocks]).tryGet()
+    datasetTreeCid = datasetTree.rootCid().tryGet()
+
+    protectedTree = CodexTree.init(cids).tryGet()
+    protectedTreeCid = protectedTree.rootCid().tryGet()
+
+  for index, cid in cids[0..<numDatasetBlocks]:
+    let proof = datasetTree.getProof(index).tryget()
+    (await store.putCidAndProof(datasetTreeCid, index, cid, proof)).tryGet
+
+  for index, cid in cids:
+    let proof = protectedTree.getProof(index).tryget()
+    (await store.putCidAndProof(protectedTreeCid, index, cid, proof)).tryGet
+
+  let
+    manifest = Manifest.new(
+      treeCid = datasetTreeCid,
+      blockSize = blockSize,
+      datasetSize = originalDatasetSize.NBytes)
+
+    protectedManifest = Manifest.new(
+      manifest = manifest,
+      treeCid = protectedTreeCid,
+      datasetSize = totalDatasetSize.NBytes,
+      ecK = ecK,
+      ecM = ecM,
+      strategy = SteppedStrategy)
+
+    manifestBlock = bt.Block.new(
+      manifest.encode().tryGet(),
+      codec = ManifestCodec).tryGet()
+
+    protectedManifestBlock = bt.Block.new(
+      protectedManifest.encode().tryGet(),
+      codec = ManifestCodec).tryGet()
+
+  (await store.putBlock(manifestBlock)).tryGet()
+  (await store.putBlock(protectedManifestBlock)).tryGet()
+
+  (manifest, protectedManifest)
+
+proc createVerifiableManifest*(
+  store: BlockStore,
+  numDatasetBlocks: int,
+  ecK: int, ecM: int,
+  blockSize: NBytes,
+  cellSize: NBytes):
+  Future[tuple[manifest: Manifest, protected: Manifest, verifiable: Manifest]] {.async.} =
+
+  let
+    numSlots = ecK + ecM
+    numTotalBlocks = calcEcBlocksCount(numDatasetBlocks, ecK, ecM)  # total number of blocks in the dataset after
+                                                                    # EC (should will match number of slots)
+    originalDatasetSize = numDatasetBlocks * blockSize.int
+    totalDatasetSize    = numTotalBlocks * blockSize.int
+
+    chunker = RandomChunker.new(Rng.instance(), size = totalDatasetSize, chunkSize = blockSize)
+    datasetBlocks = await chunker.createBlocks(store)
+
+    (manifest, protectedManifest) =
+        await createProtectedManifest(
+          datasetBlocks,
+          store,
+          numDatasetBlocks,
+          ecK, ecM,
+          blockSize,
+          originalDatasetSize,
+          totalDatasetSize)
+
+    builder = Poseidon2Builder.new(store, protectedManifest, cellSize = cellSize).tryGet
+    verifiableManifest = (await builder.buildManifest()).tryGet
+
+  # build the slots and manifest
+  (manifest, protectedManifest, verifiableManifest)

tests/codex/slots/provingtestenv.nim

@@ -1,194 +0,0 @@
-import std/sequtils
-import std/math
-
-import pkg/questionable/results
-import pkg/poseidon2/io
-import pkg/poseidon2
-import pkg/chronos
-import pkg/codex/stores/cachestore
-import pkg/codex/chunker
-import pkg/codex/stores
-import pkg/codex/blocktype as bt
-import pkg/codex/contracts/requests
-import pkg/codex/contracts
-import pkg/codex/merkletree
-import pkg/codex/stores/cachestore
-import pkg/codex/indexingstrategy
-
-import pkg/codex/slots/converters
-import pkg/codex/slots/builder/builder
-import pkg/codex/utils/poseidon2digest
-import pkg/codex/utils/asynciter
-
-import ../helpers
-import ../merkletree/helpers
-
-const
-  # The number of slot blocks and number of slots, combined with
-  # the bytes per block, make it so that there are exactly 256 cells
-  # in the dataset.
-  bytesPerBlock* = 64 * 1024
-  cellsPerBlock* = bytesPerBlock div DefaultCellSize.int
-  numberOfSlotBlocks* = 3
-  numberOfSlotBlocksPadded* = numberOfSlotBlocks.nextPowerOfTwo
-  totalNumberOfSlots* = 2
-  datasetSlotIndex* = 1
-  cellsPerSlot* = (bytesPerBlock * numberOfSlotBlocks) div DefaultCellSize.int
-  totalNumCells = ((numberOfSlotBlocks * totalNumberOfSlots * bytesPerBlock) div DefaultCellSize.int)
-
-type
-  ProvingTestEnvironment* = ref object
-    # Invariant:
-    # These challenges are chosen such that with the testenv default values
-    # and nSamples=3, they will land on [3x data cells + 0x padded cell],
-    # and [2x data cells + 1x padded cell] respectively:
-    challengeNoPad*: Poseidon2Hash
-    challengeOnePad*: Poseidon2Hash
-    blockPadBytes*: seq[byte]
-    emptyBlockTree*: Poseidon2Tree
-    emptyBlockCid*: Cid
-    # Variant:
-    localStore*: CacheStore
-    manifest*: Manifest
-    manifestBlock*: bt.Block
-    slot*: Slot
-    datasetBlocks*: seq[bt.Block]
-    slotTree*: Poseidon2Tree
-    slotRootCid*: Cid
-    slotRoots*: seq[Poseidon2Hash]
-    datasetToSlotTree*: Poseidon2Tree
-    datasetRootHash*: Poseidon2Hash
-
-proc createDatasetBlocks(self: ProvingTestEnvironment): Future[void] {.async.} =
-  var data: seq[byte] = @[]
-
-  # This generates a number of blocks that have different data, such that
-  # Each cell in each block is unique, but nothing is random.
-  for i in 0 ..< totalNumCells:
-    data = data & (i.byte).repeat(DefaultCellSize.uint64)
-
-  let chunker = MockChunker.new(
-    dataset = data,
-    chunkSize = bytesPerBlock)
-
-  while true:
-    let chunk = await chunker.getBytes()
-    if chunk.len <= 0:
-      break
-    let b = bt.Block.new(chunk).tryGet()
-    self.datasetBlocks.add(b)
-    discard await self.localStore.putBlock(b)
-
-proc createSlotTree(self: ProvingTestEnvironment, dSlotIndex: uint64): Future[Poseidon2Tree] {.async.} =
-  let
-    slotSize = (bytesPerBlock * numberOfSlotBlocks).uint64
-    blocksInSlot = slotSize div bytesPerBlock.uint64
-    datasetBlockIndexingStrategy = SteppedStrategy.init(0, self.datasetBlocks.len - 1, totalNumberOfSlots)
-    datasetBlockIndices = toSeq(datasetBlockIndexingStrategy.getIndicies(dSlotIndex.int))
-
-  let
-    slotBlocks = datasetBlockIndices.mapIt(self.datasetBlocks[it])
-    slotBlockRoots = slotBlocks.mapIt(Poseidon2Tree.digest(it.data & self.blockPadBytes, DefaultCellSize.int).tryGet())
-    slotBlockRootPads = newSeqWith((slotBlockRoots.len).nextPowerOfTwoPad, Poseidon2Zero)
-    tree = Poseidon2Tree.init(slotBlockRoots & slotBlockRootPads).tryGet()
-    treeCid = tree.root().tryGet().toSlotCid().tryGet()
-
-  for i in 0 ..< numberOfSlotBlocksPadded:
-    var blkCid: Cid
-    if i < slotBlockRoots.len:
-      blkCid = slotBlockRoots[i].toCellCid().tryGet()
-    else:
-      blkCid = self.emptyBlockCid
-
-    let proof = tree.getProof(i).tryGet().toEncodableProof().tryGet()
-    discard await self.localStore.putCidAndProof(treeCid, i, blkCid, proof)
-
-  return tree
-
-proc createDatasetRootHashAndSlotTree(self: ProvingTestEnvironment): Future[void] {.async.} =
-  var slotTrees = newSeq[Poseidon2Tree]()
-  for i in 0 ..< totalNumberOfSlots:
-    slotTrees.add(await self.createSlotTree(i.uint64))
-  self.slotTree = slotTrees[datasetSlotIndex]
-  self.slotRootCid = slotTrees[datasetSlotIndex].root().tryGet().toSlotCid().tryGet()
-  self.slotRoots = slotTrees.mapIt(it.root().tryGet())
-  let rootsPadLeafs = newSeqWith(totalNumberOfSlots.nextPowerOfTwoPad, Poseidon2Zero)
-  self.datasetToSlotTree = Poseidon2Tree.init(self.slotRoots & rootsPadLeafs).tryGet()
-  self.datasetRootHash = self.datasetToSlotTree.root().tryGet()
-
-proc createManifest(self: ProvingTestEnvironment): Future[void] {.async.} =
-  let
-    cids = self.datasetBlocks.mapIt(it.cid)
-    tree = CodexTree.init(cids).tryGet()
-    treeCid = tree.rootCid(CIDv1, BlockCodec).tryGet()
-
-  for i in 0 ..< self.datasetBlocks.len:
-    let
-      blk = self.datasetBlocks[i]
-      leafCid = blk.cid
-      proof = tree.getProof(i).tryGet()
-    discard await self.localStore.putBlock(blk)
-    discard await self.localStore.putCidAndProof(treeCid, i, leafCid, proof)
-
-  # Basic manifest:
-  self.manifest = Manifest.new(
-    treeCid = treeCid,
-    blockSize = bytesPerBlock.NBytes,
-    datasetSize = (bytesPerBlock * numberOfSlotBlocks * totalNumberOfSlots).NBytes)
-
-  # Protected manifest:
-  self.manifest = Manifest.new(
-    manifest = self.manifest,
-    treeCid = treeCid,
-    datasetSize = self.manifest.datasetSize,
-    ecK = totalNumberOfSlots,
-    ecM = 0,
-    strategy = StrategyType.SteppedStrategy
-  )
-
-  # Verifiable manifest:
-  self.manifest = Manifest.new(
-    manifest = self.manifest,
-    verifyRoot = self.datasetRootHash.toVerifyCid().tryGet(),
-    slotRoots = self.slotRoots.mapIt(it.toSlotCid().tryGet())
-  ).tryGet()
-
-  self.manifestBlock = bt.Block.new(self.manifest.encode().tryGet(), codec = ManifestCodec).tryGet()
-  discard await self.localStore.putBlock(self.manifestBlock)
-
-proc createSlot(self: ProvingTestEnvironment): void =
-  self.slot = Slot(
-    request: StorageRequest(
-      ask: StorageAsk(
-        slots: totalNumberOfSlots.uint64,
-        slotSize: u256(bytesPerBlock * numberOfSlotBlocks)
-      ),
-      content: StorageContent(
-        cid: $self.manifestBlock.cid
-      ),
-    ),
-    slotIndex: u256(datasetSlotIndex)
-  )
-
-proc createProvingTestEnvironment*(): Future[ProvingTestEnvironment] {.async.} =
-  let
-    numBlockCells = bytesPerBlock.int div DefaultCellSize.int
-    blockPadBytes = newSeq[byte](numBlockCells.nextPowerOfTwoPad * DefaultCellSize.int)
-    emptyBlockTree = Poseidon2Tree.digestTree(DefaultEmptyBlock & blockPadBytes, DefaultCellSize.int).tryGet()
-    emptyBlockCid = emptyBlockTree.root.tryGet().toCellCid().tryGet()
-
-  var testEnv = ProvingTestEnvironment(
-    challengeNoPad: toF(6),
-    challengeOnePad: toF(9),
-    blockPadBytes: blockPadBytes,
-    emptyBlockTree: emptyBlockTree,
-    emptyBlockCid: emptyBlockCid
-  )
-
-  testEnv.localStore = CacheStore.new()
-  await testEnv.createDatasetBlocks()
-  await testEnv.createDatasetRootHashAndSlotTree()
-  await testEnv.createManifest()
-  testEnv.createSlot()
-
-  return testEnv

tests/codex/slots/sampler/testsampler.nim

@@ -0,0 +1,173 @@
+import std/sequtils
+import std/options
+import std/importutils
+
+import ../../../asynctest
+
+import pkg/questionable
+import pkg/questionable/results
+import pkg/datastore
+
+import pkg/codex/rng
+import pkg/codex/stores
+import pkg/codex/merkletree
+import pkg/codex/utils/json
+import pkg/codex/codextypes
+import pkg/codex/slots
+import pkg/codex/slots/builder
+import pkg/codex/utils/poseidon2digest
+import pkg/codex/slots/sampler/utils
+
+import pkg/constantine/math/arithmetic
+import pkg/constantine/math/io/io_bigints
+import pkg/constantine/math/io/io_fields
+
+import ../backends/helpers
+import ../helpers
+import ../../helpers
+
+suite "Test control samples":
+
+  var
+    inputData: string
+    inputJson: JsonNode
+    proofInput: ProofInput[Poseidon2Hash]
+
+  setup:
+    inputData = readFile("tests/circuits/fixtures/input.json")
+    inputJson = parseJson(inputData)
+    proofInput = jsonToProofInput[Poseidon2Hash](inputJson)
+
+  test "Should verify control samples":
+    let
+      blockCells = 32
+      cellIdxs = proofInput.entropy.cellIndices(proofInput.slotRoot, proofInput.nCellsPerSlot, 5)
+
+    for i, cellIdx in cellIdxs:
+      let
+        sample = proofInput.samples[i]
+        cellIdx = cellIdxs[i]
+
+        cellProof = Poseidon2Proof.init(
+          cellIdx.toCellInBlk(blockCells),
+          proofInput.nCellsPerSlot,
+          sample.merklePaths[0..<5]).tryGet
+
+        slotProof = Poseidon2Proof.init(
+          cellIdx.toBlkInSlot(blockCells),
+          proofInput.nCellsPerSlot,
+          sample.merklePaths[5..<9]).tryGet
+
+        cellData = fromCircomData[Poseidon2Hash](sample.cellData)
+        cellLeaf = Poseidon2Hash.spongeDigest(cellData, rate = 2).tryGet
+        slotLeaf = cellProof.reconstructRoot(cellLeaf).tryGet
+
+      check slotProof.verify(slotLeaf, proofInput.slotRoot).tryGet
+
+  test "Should verify control dataset root":
+    let
+      datasetProof = Poseidon2Proof.init(
+        proofInput.slotIndex,
+        proofInput.nSlotsPerDataSet,
+        proofInput.slotProof[0..<4]).tryGet
+
+    check datasetProof.verify(proofInput.slotRoot, proofInput.datasetRoot).tryGet
+
+suite "Test sampler samples":
+
+  let
+    slotIndex     = 3
+    nSamples      = 5
+    ecK           = 3
+    ecM           = 2
+    datasetBlocks = 8
+    entropy       = 1234567.toF
+    blockSize     = DefaultBlockSize
+    cellSize      = DefaultCellSize
+
+  var
+    store: RepoStore
+    builder: Poseidon2Builder
+    manifest: Manifest
+    protected: Manifest
+    verifiable: Manifest
+
+  setup:
+    let
+      repoDs = SQLiteDatastore.new(Memory).tryGet()
+      metaDs = SQLiteDatastore.new(Memory).tryGet()
+
+    store = RepoStore.new(repoDs, metaDs)
+
+    (manifest, protected, verifiable) =
+        await createVerifiableManifest(
+          store,
+          datasetBlocks,
+          ecK, ecM,
+          blockSize,
+          cellSize)
+
+    # create sampler
+    builder = Poseidon2Builder.new(store, verifiable).tryGet
+
+  teardown:
+    await store.close()
+
+  test "Should fail instantiating for invalid slot index":
+    let
+      sampler = Poseidon2Sampler.new(builder.slotRoots.len, store, builder)
+
+    check sampler.isErr
+
+  test "Should fail instantiating for non verifiable builder":
+    let
+      nonVerifiableBuilder = Poseidon2Builder.new(store, protected).tryGet
+      sampler = Poseidon2Sampler.new(slotIndex, store, nonVerifiableBuilder)
+
+    check sampler.isErr
+
+  test "Should verify samples":
+    let
+      sampler = Poseidon2Sampler.new(slotIndex, store, builder).tryGet
+
+      verifyTree  = builder.verifyTree.get                  # get the dataset tree
+      slotProof   = verifyTree.getProof(slotIndex).tryGet   # get slot proof for index
+      datasetRoot = verifyTree.root().tryGet                # get dataset root
+      slotTreeCid = verifiable.slotRoots[slotIndex]         # get slot tree cid to retrieve proof from storage
+      slotRoot    = builder.slotRoots[slotIndex]            # get slot root hash
+      cellIdxs    = entropy.cellIndices(slotRoot, builder.numSlotCells, nSamples)
+
+      nBlockCells = builder.numBlockCells
+      nSlotCells  = builder.numSlotCells
+
+    for i, cellIdx in cellIdxs:
+      let
+        sample = (await sampler.getSample(cellIdx, slotTreeCid, slotRoot)).tryGet
+
+        cellProof = Poseidon2Proof.init(
+          cellIdx.toCellInBlk(nBlockCells),
+          nSlotCells,
+          sample.merklePaths[0..<5]).tryGet
+
+        slotProof = Poseidon2Proof.init(
+          cellIdx.toBlkInSlot(nBlockCells),
+          nSlotCells,
+          sample.merklePaths[5..<sample.merklePaths.len]).tryGet
+
+        cellData = fromCircomData[Poseidon2Hash](sample.cellData)
+        cellLeaf = Poseidon2Hash.spongeDigest(cellData, rate = 2).tryGet
+        slotLeaf = cellProof.reconstructRoot(cellLeaf).tryGet
+
+      check slotProof.verify(slotLeaf, slotRoot).tryGet
+
+  test "Should verify dataset root":
+    let
+      sampler = Poseidon2Sampler.new(slotIndex, store, builder).tryGet
+      proofInput = (await sampler.getProofInput(entropy.toBytes.toArray32, nSamples)).tryGet
+
+      datasetProof = Poseidon2Proof.init(
+          proofInput.slotIndex,
+          builder.slotRoots.len,
+          proofInput.slotProof).tryGet
+
+    check datasetProof.verify(builder.slotRoots[slotIndex], builder.verifyRoot.get).tryGet

tests/codex/slots/sampler/testutils.nim

@@ -2,7 +2,8 @@ import std/sequtils
 import std/sugar
 import std/random
 import std/strutils
-import std/math
+
+import ../../../asynctest
 
 import pkg/questionable/results
 import pkg/constantine/math/arithmetic
@@ -10,7 +11,6 @@ import pkg/constantine/math/io/io_fields
 import pkg/poseidon2/io
 import pkg/poseidon2
 import pkg/chronos
-import pkg/codex/stores/cachestore
 import pkg/codex/chunker
 import pkg/codex/stores
 import pkg/codex/blocktype as bt
@@ -19,31 +19,27 @@ import pkg/codex/contracts
 import pkg/codex/merkletree
 import pkg/codex/stores/cachestore
 
+import pkg/codex/slots/types
 import pkg/codex/slots/sampler/utils
 
-import ../../asynctest
+import ../backends/helpers
-import ../helpers
+import ../../helpers
-import ../examples
+import ../../examples
-import ../merkletree/helpers
+import ../../merkletree/helpers
-import ./provingtestenv
 
 asyncchecksuite "Test proof sampler utils":
-  let knownIndices: seq[Natural] = @[57, 82, 49]
+  let
+    cellsPerBlock = DefaultBlockSize div DefaultCellSize
 
   var
-    env: ProvingTestEnvironment
+    inputData: string
-    slotRoot: Poseidon2Hash
+    inputJson: JsonNode
-    numCells: Natural
+    proofInput: ProofInput[Poseidon2Hash]
-    numCellsPadded: Natural
 
   setup:
-    env = await createProvingTestEnvironment()
+    inputData = readFile("tests/circuits/fixtures/input.json")
-    slotRoot = env.slotRoots[datasetSlotIndex]
+    inputJson = parseJson(inputData)
-    numCells = cellsPerSlot
+    proofInput = jsonToProofInput[Poseidon2Hash](inputJson)
-    numCellsPadded = numCells.nextPowerOfTwo
-
-  teardown:
-    reset(env)
 
   test "Extract low bits":
     proc extract(value: uint64, nBits: int): uint64 =
@@ -64,45 +60,39 @@ asyncchecksuite "Test proof sampler utils":
 
   test "Can find single slot-cell index":
     proc slotCellIndex(i: Natural): Natural =
-      return cellIndex(env.challengeNoPad, slotRoot, numCellsPadded, i)
+      return cellIndex(proofInput.entropy, proofInput.slotRoot, proofInput.nCellsPerSlot, i)
 
     proc getExpectedIndex(i: int): Natural =
       let
-        numberOfCellsInSlot = (bytesPerBlock * numberOfSlotBlocks) div DefaultCellSize.uint64.int
+        hash = Sponge.digest(@[proofInput.entropy, proofInput.slotRoot, toF(i)], rate = 2)
-        numberOfCellsInSlotPadded = numberOfCellsInSlot.nextPowerOfTwo
-        hash = Sponge.digest(@[slotRoot, env.challengeNoPad, toF(i)], rate = 2)
 
-      return int(extractLowBits(hash.toBig(), ceilingLog2(numberOfCellsInSlotPadded)))
+      return int(extractLowBits(hash.toBig(), ceilingLog2(proofInput.nCellsPerSlot)))
 
     check:
       slotCellIndex(1) == getExpectedIndex(1)
-      slotCellIndex(1) == knownIndices[0]
       slotCellIndex(2) == getExpectedIndex(2)
-      slotCellIndex(2) == knownIndices[1]
       slotCellIndex(3) == getExpectedIndex(3)
-      slotCellIndex(3) == knownIndices[2]
 
   test "Can find sequence of slot-cell indices":
     proc slotCellIndices(n: int): seq[Natural]  =
-      cellIndices(env.challengeNoPad, slotRoot, numCellsPadded, n)
+      cellIndices(proofInput.entropy, proofInput.slotRoot, numCells = proofInput.nCellsPerSlot, n)
 
     proc getExpectedIndices(n: int): seq[Natural]  =
-      return collect(newSeq, (for i in 1..n: cellIndex(env.challengeNoPad, slotRoot, numCellsPadded, i)))
+      return collect(newSeq, (for i in 1..n: cellIndex(proofInput.entropy, proofInput.slotRoot, proofInput.nCellsPerSlot, i)))
 
     check:
       slotCellIndices(3) == getExpectedIndices(3)
-      slotCellIndices(3) == knownIndices
 
   for (input, expected) in [(10, 0), (31, 0), (32, 1), (63, 1), (64, 2)]:
     test "Can get slotBlockIndex from slotCellIndex (" & $input & " -> " & $expected & ")":
-      let slotBlockIndex = toBlockIdx(input, numCells = 32)
+      let slotBlockIndex = toBlkInSlot(input, numCells = cellsPerBlock)
 
       check:
         slotBlockIndex == expected
 
   for (input, expected) in [(10, 10), (31, 31), (32, 0), (63, 31), (64, 0)]:
     test "Can get blockCellIndex from slotCellIndex (" & $input & " -> " & $expected & ")":
-      let blockCellIndex = toBlockCellIdx(input, numCells = 32)
+      let blockCellIndex = toCellInBlk(input, numCells = cellsPerBlock)
 
       check:
         blockCellIndex == expected

tests/codex/slots/testsampler.nim

@@ -1,155 +1,4 @@
-import std/sequtils
+import ./sampler/testsampler
-import std/sugar
+import ./sampler/testutils
-import std/random
-import std/strutils
 
-import pkg/questionable/results
+{.warning[UnusedImport]: off.}
-import pkg/constantine/math/arithmetic
-import pkg/constantine/math/io/io_fields
-import pkg/poseidon2/io
-import pkg/poseidon2
-import pkg/chronos
-import pkg/nimcrypto
-import pkg/codex/stores/cachestore
-import pkg/codex/chunker
-import pkg/codex/stores
-import pkg/codex/blocktype as bt
-import pkg/codex/contracts/requests
-import pkg/codex/contracts
-import pkg/codex/merkletree
-import pkg/codex/stores/cachestore
-
-import pkg/codex/slots/sampler
-import pkg/codex/slots/builder/builder
-
-import ../helpers
-import ../examples
-import ../merkletree/helpers
-import ../../asynctest
-import ./testsampler_expected
-import ./provingtestenv
-
-asyncchecksuite "Test DataSampler":
-  var
-    env: ProvingTestEnvironment
-    dataSampler: DataSampler
-    blk: bt.Block
-    cell0Bytes: seq[byte]
-    cell1Bytes: seq[byte]
-    cell2Bytes: seq[byte]
-
-  proc createDataSampler(): Future[void] {.async.} =
-    dataSampler = DataSampler.new(
-      datasetSlotIndex,
-      env.localStore,
-      SlotsBuilder.new(env.localStore, env.manifest).tryGet()).tryGet()
-
-  setup:
-    randomize()
-    env = await createProvingTestEnvironment()
-    let bytes = newSeqWith(bytesPerBlock, rand(uint8))
-    blk = bt.Block.new(bytes).tryGet()
-    cell0Bytes = bytes[0..<DefaultCellSize.uint64]
-    cell1Bytes = bytes[DefaultCellSize.uint64..<(DefaultCellSize.uint64*2)]
-    cell2Bytes = bytes[(DefaultCellSize.uint64*2)..<(DefaultCellSize.uint64*3)]
-
-    await createDataSampler()
-
-  teardown:
-    reset(env)
-    reset(dataSampler)
-
-  test "Can get cell from block":
-    let
-      sample0 = dataSampler.getCell(blk.data, 0)
-      sample1 = dataSampler.getCell(blk.data, 1)
-      sample2 = dataSampler.getCell(blk.data, 2)
-
-    check:
-      sample0 == cell0Bytes
-      sample1 == cell1Bytes
-      sample2 == cell2Bytes
-
-  test "Can gather proof input":
-    let
-      nSamples = 3
-      challengeBytes = env.challengeNoPad.toBytes()
-      input = (await dataSampler.getProofInput(challengeBytes, nSamples)).tryget()
-
-    proc equal(a: Poseidon2Hash, b: Poseidon2Hash): bool =
-      a.toDecimal() == b.toDecimal()
-
-    proc toStr(proof: Poseidon2Proof): string =
-      let a = proof.path.mapIt(toHex(it))
-      join(a)
-
-    let
-      expectedBlockSlotProofs = getExpectedBlockSlotProofs()
-      expectedCellBlockProofs = getExpectedCellBlockProofs()
-      expectedCellData = getExpectedCellData()
-      expectedProof = env.datasetToSlotTree.getProof(datasetSlotIndex).tryGet()
-
-    check:
-      equal(input.verifyRoot, env.datasetRootHash)
-      equal(input.entropy, env.challengeNoPad)
-      input.numCells == ((bytesPerBlock * numberOfSlotBlocks) div DefaultCellSize.int).Natural
-      input.numSlots == totalNumberOfSlots.Natural
-      input.slotIndex == env.slot.slotIndex.truncate(Natural)
-      input.verifyProof == expectedProof
-
-      # block-slot proofs
-      input.samples[0].slotBlockIdx == 1
-      input.samples[1].slotBlockIdx == 2
-      input.samples[2].slotBlockIdx == 1
-      toStr(input.samples[0].slotProof) == expectedBlockSlotProofs[0]
-      toStr(input.samples[1].slotProof) == expectedBlockSlotProofs[1]
-      toStr(input.samples[2].slotProof) == expectedBlockSlotProofs[2]
-
-      # cell-block proofs
-      input.samples[0].blockCellIdx == 25
-      input.samples[1].blockCellIdx == 18
-      input.samples[2].blockCellIdx == 17
-      toStr(input.samples[0].cellProof) == expectedCellBlockProofs[0]
-      toStr(input.samples[1].cellProof) == expectedCellBlockProofs[1]
-      toStr(input.samples[2].cellProof) == expectedCellBlockProofs[2]
-
-      # cell data
-      nimcrypto.toHex(input.samples[0].data) == expectedCellData[0]
-      nimcrypto.toHex(input.samples[1].data) == expectedCellData[1]
-      nimcrypto.toHex(input.samples[2].data) == expectedCellData[2]
-
-  test "Can select samples in padded cells":
-    let
-      nSamples = 3
-      challengeBytes = env.challengeOnePad.toBytes()
-      input = (await dataSampler.getProofInput(challengeBytes, nSamples)).tryget()
-
-    proc equal(a: Poseidon2Hash, b: Poseidon2Hash): bool =
-      a.toDecimal() == b.toDecimal()
-
-    proc toStr(proof: Poseidon2Proof): string =
-      let a = proof.path.mapIt(toHex(it))
-      join(a)
-
-    let expectedProof = env.datasetToSlotTree.getProof(datasetSlotIndex).tryGet()
-
-    check:
-      equal(input.verifyRoot, env.datasetRootHash)
-      equal(input.entropy, env.challengeOnePad)
-      input.numCells == ((bytesPerBlock * numberOfSlotBlocks) div DefaultCellSize.int).Natural
-      input.numSlots == totalNumberOfSlots.Natural
-      input.slotIndex == env.slot.slotIndex.truncate(Natural)
-      input.verifyProof == expectedProof
-
-      input.samples[0].slotBlockIdx == 2
-      input.samples[1].slotBlockIdx == 2
-      input.samples[2].slotBlockIdx == 3
-      # The third sample is a padded sample
-      toStr(input.samples[2].slotProof) == toStr(env.slotTree.getProof(3).tryGet())
-
-      input.samples[0].blockCellIdx == 29
-      input.samples[1].blockCellIdx == 26
-      input.samples[2].blockCellIdx == 30
-      toStr(input.samples[2].cellProof) == toStr(env.emptyBlockTree.getProof(30).tryGet())
-
-      input.samples[2].data == newSeq[byte](DefaultCellSize.int)

tests/codex/slots/testsampler_expected.nim

@@ -1,25 +0,0 @@
-# Snapshot of expected values for testsampler.
-
-import std/strutils
-import pkg/codex/codextypes
-
-proc getExpectedCellBlockProofs*(): seq[string] =
-  @[
-    "0x13935d7f73028fd425e557b8e0b737fcba3f8bcda0e07a9047868bc7c5a2519e0x17d36c9bcef9febb77b129e8bbe971b754d7b3e1df8c7d881b64bf0aa055f5570x02610a45fa39d4859e5c0c53bda9cc6bd627ae4b5aa462f9e86f47f3db96b2700x27a362c0f92c887d74ee77df2df5e51f0f66b0564a2af8b1c11b02dab0a34f780x095eb3c0d166c19f9cac8ea0e6ba274dfeef802cf7d01c90378585fd4d788e56",
-    "0x128b43cb1f87736d5b826c5a440d373c0e4c147a81a0edc40b106abd2c10af960x2160ba1c1ed893685b849a8f1be6d32bb11a6ebb8690c5af95e0ce0e9d72b6580x1332185d0ef7b76e55c7464f37df96d41f256249d74678d8191030912036b7aa0x0522383db3f9cb9c75d5b209bd795566e9cffa632819628194416a479998a4270x1decd3fb7ff458261149731115657cecd7eb2fe4a6cf84f3c6761aa8b0dd6b9a",
-    "0x2a2c8663e354dbc1857a654134de3b690d73ffedcc7c0ba770a39f3e48675ee10x01914d98d4103889ea59ce6e970dae95f7e463350184c5f7f3806997e10748520x10e8f6235ac7e95af5aea6fc77ebe60e1028b48eb84027ea46d403d809852b6b0x04bd51be7ff42db27a7605124d3d2d1c4cbca43f93d1997060ac8a1009fa48b40x095eb3c0d166c19f9cac8ea0e6ba274dfeef802cf7d01c90378585fd4d788e56"
-  ]
-
-proc getExpectedBlockSlotProofs*(): seq[string] =
-  @[
-    "0x008fccffc5d2e5010a44bc718fed80d4d891e6517c022ba1bff65763dd79dbb90x117662f365386ee6784274dfa3a3b35126291cdf013c5e90d07bee79f32b2f40",
-    "0x00000000000000000000000000000000000000000000000000000000000000000x2a66917fa49371e835376fcece0d854c77008ac1195740963b1ac4491ee1aaf1",
-    "0x008fccffc5d2e5010a44bc718fed80d4d891e6517c022ba1bff65763dd79dbb90x117662f365386ee6784274dfa3a3b35126291cdf013c5e90d07bee79f32b2f40"
-  ]
-
-proc getExpectedCellData*(): seq[string] =
-  @[
-    "79".repeat(DefaultCellSize.int),
-    "B2".repeat(DefaultCellSize.int),
-    "71".repeat(DefaultCellSize.int)
-  ]

tests/codex/slots/testslotbuilder.nim

@@ -3,6 +3,8 @@ import std/math
 import std/importutils
 import std/sugar
 
+import ../../asynctest
+
 import pkg/chronos
 import pkg/questionable/results
 import pkg/codex/blocktype as bt
@@ -13,12 +15,13 @@ import pkg/codex/merkletree
 import pkg/codex/manifest {.all.}
 import pkg/codex/utils
 import pkg/codex/utils/digest
+import pkg/codex/utils/poseidon2digest
 import pkg/datastore
 import pkg/poseidon2
 import pkg/poseidon2/io
-import constantine/math/io/io_fields
+import pkg/constantine/math/io/io_fields
 
-import ../../asynctest
+import ./helpers
 import ../helpers
 import ../examples
 import ../merkletree/helpers
@@ -26,9 +29,12 @@ import ../merkletree/helpers
 import pkg/codex/indexingstrategy {.all.}
 import pkg/codex/slots {.all.}
 
-privateAccess(SlotsBuilder) # enable access to private fields
+privateAccess(Poseidon2Builder) # enable access to private fields
 privateAccess(Manifest) # enable access to private fields
 
+const
+  Strategy = SteppedStrategy
+
 suite "Slot builder":
   let
     blockSize = NBytes 1024
@@ -37,98 +43,49 @@ suite "Slot builder":
     ecM = 2
 
     numSlots = ecK + ecM
-    numDatasetBlocks = 100
+    numDatasetBlocks = 50
-    numBlockCells = (blockSize div cellSize).int
+    numTotalBlocks = calcEcBlocksCount(numDatasetBlocks, ecK, ecM)  # total number of blocks in the dataset after
+                                                                    # EC (should will match number of slots)
+    originalDatasetSize = numDatasetBlocks * blockSize.int
+    totalDatasetSize    = numTotalBlocks * blockSize.int
 
-    numTotalBlocks = calcEcBlocksCount(numDatasetBlocks, ecK, ecM)                # total number of blocks in the dataset after
+    numBlockCells     = (blockSize div cellSize).int                # number of cells per block
-                                                                                  # EC (should will match number of slots)
+    numSlotBlocks     = numTotalBlocks div numSlots                 # number of blocks per slot
-    originalDatasetSize = numDatasetBlocks * blockSize.int                        # size of the dataset before EC
+    numSlotCells      = numSlotBlocks * numBlockCells               # number of uncorrected slot cells
-    totalDatasetSize = numTotalBlocks * blockSize.int                             # size of the dataset after EC
+    pow2SlotCells     = nextPowerOfTwo(numSlotCells)                # pow2 cells per slot
-    numTotalSlotBlocks = nextPowerOfTwo(numTotalBlocks div numSlots)
+    numPadSlotBlocks  = pow2SlotCells div numBlockCells             # pow2 blocks per slot
+    numPadBlocksTotal = numPadSlotBlocks * numSlots                 # total number of pad blocks
 
-    blockPadBytes =
+    # empty digest
-      newSeq[byte](numBlockCells.nextPowerOfTwoPad * cellSize.int)                # power of two padding for blocks
+    emptyDigest = SpongeMerkle.digest(newSeq[byte](blockSize.int), cellSize.int)
-
-    slotsPadLeafs =
-      newSeqWith((numTotalBlocks div numSlots).nextPowerOfTwoPad, Poseidon2Zero)  # power of two padding for block roots
-
-    rootsPadLeafs =
-      newSeqWith(numSlots.nextPowerOfTwoPad, Poseidon2Zero)
 
   var
     datasetBlocks: seq[bt.Block]
+    padBlocks: seq[bt.Block]
     localStore: BlockStore
     manifest: Manifest
     protectedManifest: Manifest
-    expectedEmptyCid: Cid
+    builder: Poseidon2Builder
-    slotBuilder: SlotsBuilder
     chunker: Chunker
 
-  proc createBlocks(): Future[void] {.async.} =
-    while true:
-      let chunk = await chunker.getBytes()
-      if chunk.len <= 0:
-        break
-      let blk = bt.Block.new(chunk).tryGet()
-      datasetBlocks.add(blk)
-      discard await localStore.putBlock(blk)
-
-  proc createProtectedManifest(): Future[void] {.async.} =
-    let
-      cids = datasetBlocks.mapIt(it.cid)
-      datasetTree = CodexTree.init(cids[0..<numDatasetBlocks]).tryGet()
-      datasetTreeCid = datasetTree.rootCid().tryGet()
-
-      protectedTree = CodexTree.init(cids).tryGet()
-      protectedTreeCid = protectedTree.rootCid().tryGet()
-
-    for index, cid in cids[0..<numDatasetBlocks]:
-      let proof = datasetTree.getProof(index).tryget()
-      (await localStore.putCidAndProof(datasetTreeCid, index, cid, proof)).tryGet
-
-    for index, cid in cids:
-      let proof = protectedTree.getProof(index).tryget()
-      (await localStore.putCidAndProof(protectedTreeCid, index, cid, proof)).tryGet
-
-    manifest = Manifest.new(
-      treeCid = datasetTreeCid,
-      blockSize = blockSize.NBytes,
-      datasetSize = originalDatasetSize.NBytes)
-
-    protectedManifest = Manifest.new(
-      manifest = manifest,
-      treeCid = protectedTreeCid,
-      datasetSize = totalDatasetSize.NBytes,
-      ecK = ecK,
-      ecM = ecM,
-      strategy = StrategyType.SteppedStrategy)
-
-    let
-      manifestBlock = bt.Block.new(
-        manifest.encode().tryGet(),
-        codec = ManifestCodec).tryGet()
-
-      protectedManifestBlock = bt.Block.new(
-        protectedManifest.encode().tryGet(),
-        codec = ManifestCodec).tryGet()
-
-    (await localStore.putBlock(manifestBlock)).tryGet()
-    (await localStore.putBlock(protectedManifestBlock)).tryGet()
-
-    expectedEmptyCid = emptyCid(
-      protectedManifest.version,
-      protectedManifest.hcodec,
-      protectedManifest.codec).tryGet()
-
   setup:
     let
       repoDs = SQLiteDatastore.new(Memory).tryGet()
       metaDs = SQLiteDatastore.new(Memory).tryGet()
-    localStore = RepoStore.new(repoDs, metaDs)
 
+    localStore = RepoStore.new(repoDs, metaDs)
     chunker = RandomChunker.new(Rng.instance(), size = totalDatasetSize, chunkSize = blockSize)
-    await createBlocks()
+    datasetBlocks = await chunker.createBlocks(localStore)
-    await createProtectedManifest()
+
+    (manifest, protectedManifest) =
+        await createProtectedManifest(
+          datasetBlocks,
+          localStore,
+          numDatasetBlocks,
+          ecK, ecM,
+          blockSize,
+          originalDatasetSize,
+          totalDatasetSize)
 
   teardown:
     await localStore.close()
@@ -142,11 +99,10 @@ suite "Slot builder":
     reset(localStore)
     reset(manifest)
     reset(protectedManifest)
-    reset(expectedEmptyCid)
+    reset(builder)
-    reset(slotBuilder)
     reset(chunker)
 
-  test "Can only create slotBuilder with protected manifest":
+  test "Can only create builder with protected manifest":
     let
       unprotectedManifest = Manifest.new(
         treeCid = Cid.example,
@@ -154,8 +110,8 @@ suite "Slot builder":
         datasetSize = originalDatasetSize.NBytes)
 
     check:
-      SlotsBuilder.new(localStore, unprotectedManifest, cellSize = cellSize)
+      Poseidon2Builder.new(localStore, unprotectedManifest, cellSize = cellSize)
-        .error.msg == "Can only create SlotsBuilder using protected manifests."
+        .error.msg == "Manifest is not protected."
 
   test "Number of blocks must be devisable by number of slots":
     let
@@ -168,10 +124,10 @@ suite "Slot builder":
         datasetSize = totalDatasetSize.NBytes,
         ecK = ecK - 1,
         ecM = ecM,
-        strategy = StrategyType.SteppedStrategy)
+        strategy = Strategy)
 
     check:
-      SlotsBuilder.new(localStore, mismatchManifest, cellSize = cellSize)
+      Poseidon2Builder.new(localStore, mismatchManifest, cellSize = cellSize)
         .error.msg == "Number of blocks must be divisable by number of slots."
 
   test "Block size must be divisable by cell size":
@@ -185,146 +141,150 @@ suite "Slot builder":
         datasetSize = (totalDatasetSize - 1).NBytes,
         ecK = ecK,
         ecM = ecM,
-        strategy = StrategyType.SteppedStrategy)
+        strategy = Strategy)
 
     check:
-      SlotsBuilder.new(localStore, mismatchManifest, cellSize = cellSize)
+      Poseidon2Builder.new(localStore, mismatchManifest, cellSize = cellSize)
         .error.msg == "Block size must be divisable by cell size."
 
   test "Should build correct slot builder":
-    slotBuilder = SlotsBuilder.new(
+    builder = Poseidon2Builder.new(
       localStore,
       protectedManifest,
       cellSize = cellSize).tryGet()
 
     check:
-      slotBuilder.numBlockPadBytes == blockPadBytes.len
+      builder.cellSize == cellSize
-      slotBuilder.numSlotsPadLeafs == slotsPadLeafs.len
+      builder.numSlots == numSlots
-      slotBuilder.numRootsPadLeafs == rootsPadLeafs.len
+      builder.numBlockCells == numBlockCells
+      builder.numSlotBlocks == numPadSlotBlocks
+      builder.numSlotCells == pow2SlotCells
+      builder.numBlocks == numPadBlocksTotal
 
   test "Should build slot hashes for all slots":
     let
-      steppedStrategy = SteppedStrategy.init(0, numTotalBlocks - 1, numSlots)
+      steppedStrategy = Strategy.init(
-      slotBuilder = SlotsBuilder.new(
+        0, numPadBlocksTotal - 1, numSlots)
+
+      builder = Poseidon2Builder.new(
         localStore,
         protectedManifest,
         cellSize = cellSize).tryGet()
 
-    for i in 0 ..< numSlots:
+    # for i in 0..<numSlots:
-      let
+    let
-        expectedBlock = steppedStrategy
+      expectedHashes = collect(newSeq):
-          .getIndicies(i)
+        for j, idx in steppedStrategy.getIndicies(0):
-          .mapIt( datasetBlocks[it] )
+          if j > (protectedManifest.numSlotBlocks - 1):
+            emptyDigest
+          else:
+            SpongeMerkle.digest(datasetBlocks[idx].data, cellSize.int)
 
-        expectedHashes: seq[Poseidon2Hash] = collect(newSeq):
+      cellHashes = (await builder.getCellHashes(0)).tryGet()
-          for blk in expectedBlock:
-            SpongeMerkle.digest(blk.data & blockPadBytes, cellSize.int)
 
-        cellHashes = (await slotBuilder.getBlockHashes(i)).tryGet()
+    check:
-
+      cellHashes.len == expectedHashes.len
-      check:
+      cellHashes == expectedHashes
-        expectedHashes == cellHashes
 
   test "Should build slot trees for all slots":
     let
-      steppedStrategy = SteppedStrategy.init(0, numTotalBlocks - 1, numSlots)
+      steppedStrategy = Strategy.init(
-      slotBuilder = SlotsBuilder.new(
+        0, numPadBlocksTotal - 1, numSlots)
+
+      builder = Poseidon2Builder.new(
         localStore,
         protectedManifest,
         cellSize = cellSize).tryGet()
 
-    for i in 0 ..< numSlots:
+    for i in 0..<numSlots:
       let
-        expectedBlock = steppedStrategy
+        expectedHashes = collect(newSeq):
-          .getIndicies(i)
+          for j, idx in steppedStrategy.getIndicies(i):
-          .mapIt( datasetBlocks[it] )
+            if j > (protectedManifest.numSlotBlocks - 1):
+              emptyDigest
+            else:
+              SpongeMerkle.digest(datasetBlocks[idx].data, cellSize.int)
 
-        expectedHashes: seq[Poseidon2Hash] = collect(newSeq):
+        expectedRoot = Merkle.digest(expectedHashes)
-          for blk in expectedBlock:
+        slotTree = (await builder.buildSlotTree(i)).tryGet()
-            SpongeMerkle.digest(blk.data & blockPadBytes, cellSize.int)
-        expectedRoot = Merkle.digest(expectedHashes & slotsPadLeafs)
-
-        slotTree = (await slotBuilder.buildSlotTree(i)).tryGet()
 
       check:
-        expectedRoot == slotTree.root().tryGet()
+        slotTree.root().tryGet() == expectedRoot
 
   test "Should persist trees for all slots":
     let
-      slotBuilder = SlotsBuilder.new(
+      builder = Poseidon2Builder.new(
         localStore,
         protectedManifest,
         cellSize = cellSize).tryGet()
 
-    for i in 0 ..< numSlots:
+    for i in 0..<numSlots:
       let
-        slotTree = (await slotBuilder.buildSlotTree(i)).tryGet()
+        slotTree = (await builder.buildSlotTree(i)).tryGet()
-        slotRoot = (await slotBuilder.buildSlot(i)).tryGet()
+        slotRoot = (await builder.buildSlot(i)).tryGet()
         slotCid = slotRoot.toSlotCid().tryGet()
 
-      for cellIndex in 0..<numTotalSlotBlocks:
+      for cellIndex in 0..<numPadSlotBlocks:
         let
           (cellCid, proof) = (await localStore.getCidAndProof(slotCid, cellIndex)).tryGet()
           verifiableProof = proof.toVerifiableProof().tryGet()
-          posProof = slotTree.getProof(cellIndex).tryGet
+          posProof = slotTree.getProof(cellIndex).tryGet()
 
         check:
+          verifiableProof.path == posProof.path
           verifiableProof.index == posProof.index
           verifiableProof.nleaves == posProof.nleaves
-          verifiableProof.path == posProof.path
 
   test "Should build correct verification root":
     let
-      steppedStrategy = SteppedStrategy.init(0, numTotalBlocks - 1, numSlots)
+      steppedStrategy = Strategy.init(0, numPadBlocksTotal - 1, numSlots)
-      slotBuilder = SlotsBuilder.new(
+      builder = Poseidon2Builder.new(
         localStore,
         protectedManifest,
         cellSize = cellSize).tryGet()
 
-    (await slotBuilder.buildSlots()).tryGet
+    (await builder.buildSlots()).tryGet
     let
       slotsHashes = collect(newSeq):
-        for i in 0 ..< numSlots:
+        for i in 0..<numSlots:
           let
-            expectedBlocks = steppedStrategy
+            slotHashes = collect(newSeq):
-              .getIndicies(i)
+              for j, idx in steppedStrategy.getIndicies(i):
-              .mapIt( datasetBlocks[it] )
+                if j > (protectedManifest.numSlotBlocks - 1):
+                  emptyDigest
+                else:
+                  SpongeMerkle.digest(datasetBlocks[idx].data, cellSize.int)
 
-            slotHashes: seq[Poseidon2Hash] = collect(newSeq):
+          Merkle.digest(slotHashes)
-              for blk in expectedBlocks:
-                SpongeMerkle.digest(blk.data & blockPadBytes, cellSize.int)
 
-          Merkle.digest(slotHashes & slotsPadLeafs)
+      expectedRoot = Merkle.digest(slotsHashes)
-
+      rootHash = builder.buildVerifyTree(builder.slotRoots).tryGet().root.tryGet()
-      expectedRoot = Merkle.digest(slotsHashes & rootsPadLeafs)
-      rootHash = slotBuilder.buildVerifyTree(slotBuilder.slotRoots).tryGet().root.tryGet()
 
     check:
       expectedRoot == rootHash
 
   test "Should build correct verification root manifest":
     let
-      steppedStrategy = SteppedStrategy.init(0, numTotalBlocks - 1, numSlots)
+      steppedStrategy = Strategy.init(0, numPadBlocksTotal - 1, numSlots)
-      slotBuilder = SlotsBuilder.new(
+      builder = Poseidon2Builder.new(
         localStore,
         protectedManifest,
         cellSize = cellSize).tryGet()
 
       slotsHashes = collect(newSeq):
-        for i in 0 ..< numSlots:
+        for i in 0..<numSlots:
           let
-            expectedBlocks = steppedStrategy
+            slotHashes = collect(newSeq):
-              .getIndicies(i)
+              for j, idx in steppedStrategy.getIndicies(i):
-              .mapIt( datasetBlocks[it] )
+                if j > (protectedManifest.numSlotBlocks - 1):
+                  emptyDigest
+                else:
+                  SpongeMerkle.digest(datasetBlocks[idx].data, cellSize.int)
 
-            slotHashes: seq[Poseidon2Hash] = collect(newSeq):
+          Merkle.digest(slotHashes)
-              for blk in expectedBlocks:
-                SpongeMerkle.digest(blk.data & blockPadBytes, cellSize.int)
 
-          Merkle.digest(slotHashes & slotsPadLeafs)
+      expectedRoot = Merkle.digest(slotsHashes)
-
+      manifest = (await builder.buildManifest()).tryGet()
-      expectedRoot = Merkle.digest(slotsHashes & rootsPadLeafs)
-      manifest = (await slotBuilder.buildManifest()).tryGet()
       mhash = manifest.verifyRoot.mhash.tryGet()
       mhashBytes = mhash.digestBytes
       rootHash = Poseidon2Hash.fromBytes(mhashBytes.toArray32).get
@@ -334,69 +294,69 @@ suite "Slot builder":
 
   test "Should not build from verifiable manifest with 0 slots":
     var
-      slotBuilder = SlotsBuilder.new(
+      builder = Poseidon2Builder.new(
         localStore,
         protectedManifest,
         cellSize = cellSize).tryGet()
-      verifyManifest = (await slotBuilder.buildManifest()).tryGet()
+      verifyManifest = (await builder.buildManifest()).tryGet()
 
     verifyManifest.slotRoots = @[]
-    check SlotsBuilder.new(
+    check Poseidon2Builder.new(
         localStore,
         verifyManifest,
         cellSize = cellSize).isErr
 
   test "Should not build from verifiable manifest with incorrect number of slots":
     var
-      slotBuilder = SlotsBuilder.new(
+      builder = Poseidon2Builder.new(
         localStore,
         protectedManifest,
         cellSize = cellSize).tryGet()
 
-      verifyManifest = (await slotBuilder.buildManifest()).tryGet()
+      verifyManifest = (await builder.buildManifest()).tryGet()
 
     verifyManifest.slotRoots.del(
       verifyManifest.slotRoots.len - 1
     )
 
-    check SlotsBuilder.new(
+    check Poseidon2Builder.new(
         localStore,
         verifyManifest,
         cellSize = cellSize).isErr
 
   test "Should not build from verifiable manifest with invalid verify root":
     let
-      slotBuilder = SlotsBuilder.new(
+      builder = Poseidon2Builder.new(
         localStore,
         protectedManifest,
         cellSize = cellSize).tryGet()
 
-      verifyManifest = (await slotBuilder.buildManifest()).tryGet()
+      verifyManifest = (await builder.buildManifest()).tryGet()
       offset = verifyManifest.verifyRoot.data.buffer.len div 2
 
     rng.shuffle(
       Rng.instance,
       verifyManifest.verifyRoot.data.buffer)
 
-    check SlotsBuilder.new(
+    check Poseidon2Builder.new(
         localStore,
         verifyManifest,
         cellSize = cellSize).isErr
 
   test "Should build from verifiable manifest":
     let
-      slotBuilder = SlotsBuilder.new(
+      builder = Poseidon2Builder.new(
         localStore,
         protectedManifest,
         cellSize = cellSize).tryGet()
 
-      verifyManifest = (await slotBuilder.buildManifest()).tryGet()
+      verifyManifest = (await builder.buildManifest()).tryGet()
 
-      verificationBuilder = SlotsBuilder.new(
+      verificationBuilder = Poseidon2Builder.new(
         localStore,
         verifyManifest,
         cellSize = cellSize).tryGet()
 
     check:
-      slotBuilder.slotRoots == verificationBuilder.slotRoots
+      builder.slotRoots == verificationBuilder.slotRoots
-      slotBuilder.verifyRoot == verificationBuilder.verifyRoot
+      builder.verifyRoot == verificationBuilder.verifyRoot

tests/codex/testslots.nim

@@ -1,5 +1,4 @@
 import ./slots/testslotbuilder
-import ./slots/testutils
 import ./slots/testsampler
 import ./slots/testconverters