Rework builder & sampler (#697)

* add proof initializer for poseidon2 and `$`

* fix padding and block selection

* fix sample selection and input construction

* fix sample selection & more descriptive names

* add concrete types for sampler & builder

* add missing digest calls

* use concrete types

* add sample test fixtures

* use concrete types and don't fetch dummy blocks
This commit is contained in:
Dmitriy Ryajov 2024-02-07 20:27:11 -06:00 committed by GitHub
parent afec86b3cf
commit 825766eea0
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
24 changed files with 1608 additions and 941 deletions

View File

@ -10,7 +10,7 @@ on:
env:
cache_nonce: 0 # Allows for easily busting actions/cache caches
nim_version: v1.6.14
concurrency:
group: ${{ github.workflow }}-${{ github.ref || github.run_id }}

View File

@ -7,16 +7,14 @@
## This file may not be copied, modified, or distributed except according to
## those terms.
import pkg/upraises
push: {.upraises: [].}
{.push raises: [].}
import std/bitops
import std/sequtils
import pkg/questionable
import pkg/questionable/results
import pkg/libp2p/[cid, multicodec, multihash]
import pkg/stew/byteutils
import ../../utils
import ../../rng
@ -72,7 +70,7 @@ func getProof*(self: CodexTree, index: int): ?!CodexProof =
success proof
func verify*(self: CodexProof, leaf: MultiHash, root: MultiHash): ?!void =
func verify*(self: CodexProof, leaf: MultiHash, root: MultiHash): ?!bool =
## Verify hash
##
@ -88,11 +86,9 @@ func verify*(self: CodexProof, leaf: MultiHash, root: MultiHash): ?!void =
leafBytes.len != leaf.size:
return failure "Invalid hash length"
? self.verify(leafBytes, rootBytes)
self.verify(leafBytes, rootBytes)
success()
func verify*(self: CodexProof, leaf: Cid, root: Cid): ?!void =
func verify*(self: CodexProof, leaf: Cid, root: Cid): ?!bool =
self.verify(? leaf.mhash.mapFailure, ? leaf.mhash.mapFailure)
proc rootCid*(
@ -124,16 +120,19 @@ func getLeafCid*(
Cid.init(version, dataCodec, mhash).mapFailure
proc `$`*(self: CodexTree): string =
"CodexTree( mcodec: " &
$self.mcodec &
", leavesCount: " &
$self.leavesCount & " )"
let root = if self.root.isOk: byteutils.toHex(self.root.get) else: "none"
"CodexTree(" &
" root: " & root &
", leavesCount: " & $self.leavesCount &
", levels: " & $self.levels &
", mcodec: " & $self.mcodec & " )"
proc `$`*(self: CodexProof): string =
"CodexProof( mcodec: " &
$self.mcodec & ", nleaves: " &
$self.nleaves & ", index: " &
$self.index & " )"
"CodexProof(" &
" nleaves: " & $self.nleaves &
", index: " & $self.index &
", path: " & $self.path.mapIt( byteutils.toHex(it) ) &
", mcodec: " & $self.mcodec & " )"
func compress*(
x, y: openArray[byte],
@ -226,7 +225,9 @@ proc fromNodes*(
index = Rng.instance.rand(nleaves - 1)
proof = ? self.getProof(index)
? proof.verify(self.leaves[index], ? self.root) # sanity check
if not ? proof.verify(self.leaves[index], ? self.root): # sanity check
return failure "Unable to verify tree built from nodes"
success self
func init*(

View File

@ -117,11 +117,8 @@ func reconstructRoot*[H, K](proof: MerkleProof[H, K], leaf: H): ?!H =
return success h
func verify*[H, K](proof: MerkleProof[H, K], leaf: H, root: H): ?!void =
return if bool(root == ? proof.reconstructRoot(leaf)):
success()
else:
failure("invalid proof")
func verify*[H, K](proof: MerkleProof[H, K], leaf: H, root: H): ?!bool =
success bool(root == ? proof.reconstructRoot(leaf))
func merkleTreeWorker*[H, K](
self: MerkleTree[H, K],

View File

@ -7,6 +7,8 @@
## This file may not be copied, modified, or distributed except according to
## those terms.
{.push raises: [].}
import std/sequtils
import pkg/poseidon2
@ -30,7 +32,8 @@ const
Poseidon2Zero* = zero
type
Poseidon2Hash* = F
Bn254Fr* = F
Poseidon2Hash* = Bn254Fr
PoseidonKeysEnum* = enum # can't use non-ordinals as enum values
KeyNone
@ -41,6 +44,19 @@ type
Poseidon2Tree* = MerkleTree[Poseidon2Hash, PoseidonKeysEnum]
Poseidon2Proof* = MerkleProof[Poseidon2Hash, PoseidonKeysEnum]
proc `$`*(self: Poseidon2Tree): string =
let root = if self.root.isOk: self.root.get.toHex else: "none"
"Poseidon2Tree(" &
" root: " & root &
", leavesCount: " & $self.leavesCount &
", levels: " & $self.levels & " )"
proc `$`*(self: Poseidon2Proof): string =
"Poseidon2Proof(" &
" nleaves: " & $self.nleaves &
", index: " & $self.index &
", path: " & $self.path.mapIt( it.toHex ) & " )"
func toArray32*(bytes: openArray[byte]): array[32, byte] =
result[0..<bytes.len] = bytes[0..<bytes.len]
@ -104,6 +120,29 @@ proc fromNodes*(
index = Rng.instance.rand(nleaves - 1)
proof = ? self.getProof(index)
? proof.verify(self.leaves[index], ? self.root) # sanity check
if not ? proof.verify(self.leaves[index], ? self.root): # sanity check
return failure "Unable to verify tree built from nodes"
success self
func init*(
_: type Poseidon2Proof,
index: int,
nleaves: int,
nodes: openArray[Poseidon2Hash]): ?!Poseidon2Proof =
if nodes.len == 0:
return failure "Empty nodes"
let
compressor = proc(
x, y: Poseidon2Hash,
key: PoseidonKeysEnum): ?!Poseidon2Hash {.noSideEffect.} =
success compress( x, y, key.toKey )
success Poseidon2Proof(
compress: compressor,
zero: Poseidon2Zero,
index: index,
nleaves: nleaves,
path: @nodes)

View File

@ -38,9 +38,11 @@ import ./streams
import ./erasure
import ./discovery
import ./contracts
import ./indexingstrategy
import ./utils
import ./errors
import ./logutils
import ./utils/poseidon2digest
export logutils
@ -390,7 +392,7 @@ proc setupRequest(
trace "Unable to erasure code dataset"
return failure(error)
without builder =? SlotsBuilder.new(self.blockStore, encoded), err:
without builder =? Poseidon2Builder.new(self.blockStore, encoded), err:
trace "Unable to create slot builder"
return failure(err)
@ -503,7 +505,7 @@ proc onStore(
trace "Unable to fetch manifest for cid", cid, err = err.msg
return failure(err)
without builder =? SlotsBuilder.new(self.blockStore, manifest), err:
without builder =? Poseidon2Builder.new(self.blockStore, manifest), err:
trace "Unable to create slots builder", err = err.msg
return failure(err)
@ -528,8 +530,13 @@ proc onStore(
return success()
if blksIter =? builder.slotIndiciesIter(slotIdx) and
err =? (await self.fetchBatched(
without indexer =? manifest.protectedStrategy.init(
0, manifest.blocksCount() - 1, manifest.numSlots).catch and
blksIter =? indexer.getIndicies(slotIdx).catch, err:
trace "Unable to create indexing strategy from protected manifest", err = err.msg
return failure(err)
if err =? (await self.fetchBatched(
manifest.treeCid,
blksIter,
onBatch = updateExpiry)).errorOption:
@ -572,7 +579,7 @@ proc onProve(
error "Unable to fetch manifest for cid", err = err.msg
return failure(err)
without builder =? SlotsBuilder.new(self.blockStore, manifest), err:
without builder =? Poseidon2Builder.new(self.blockStore, manifest), err:
error "Unable to create slots builder", err = err.msg
return failure(err)

View File

@ -1,4 +1,9 @@
import ./builder/builder
import ./converters
import ../merkletree
export builder, converters
type
Poseidon2Builder* = SlotsBuilder[Poseidon2Tree, Poseidon2Hash]

View File

@ -15,192 +15,165 @@ import std/sugar
import pkg/libp2p
import pkg/chronos
import pkg/chronicles
import pkg/questionable
import pkg/questionable/results
import pkg/poseidon2
import pkg/poseidon2/io
import pkg/constantine/math/arithmetic/finite_fields
import pkg/constantine/math/io/io_fields
import ../../logutils
import ../../indexingstrategy
import ../../merkletree
import ../../utils
import ../../stores
import ../../manifest
import ../../utils
import ../../utils/asynciter
import ../../merkletree
import ../../utils/digest
import ../../utils/poseidon2digest
import ../../utils/asynciter
import ../../indexingstrategy
import ../converters
export converters
export converters, asynciter
logScope:
topics = "codex slotsbuilder"
const
# TODO: Unified with the DefaultCellSize specified in branch "data-sampler"
# in the proving circuit.
DefaultEmptyBlock* = newSeq[byte](DefaultBlockSize.int)
DefaultEmptyCell* = newSeq[byte](DefaultCellSize.int)
type
# TODO: should be a generic type that
# supports all merkle trees
SlotsBuilder* = ref object of RootObj
SlotsBuilder*[T, H] = ref object of RootObj
store: BlockStore
manifest: Manifest
strategy: IndexingStrategy
cellSize: NBytes
emptyDigestTree: Poseidon2Tree
blockPadBytes: seq[byte]
slotsPadLeafs: seq[Poseidon2Hash]
rootsPadLeafs: seq[Poseidon2Hash]
slotRoots: seq[Poseidon2Hash]
verifyTree: ?Poseidon2Tree
manifest: Manifest # current manifest
strategy: IndexingStrategy # indexing strategy
cellSize: NBytes # cell size
numSlotBlocks: Natural # number of blocks per slot (should yield a power of two number of cells)
slotRoots: seq[H] # roots of the slots
emptyBlock: seq[byte] # empty block
verifiableTree: ?T # verification tree (dataset tree)
emptyDigestTree: T # empty digest tree for empty blocks
func slotRoots*(self: SlotsBuilder): seq[Poseidon2Hash] {.inline.} =
func verifiable*[T, H](self: SlotsBuilder[T, H]): bool {.inline.} =
## Returns true if the slots are verifiable.
##
self.manifest.verifiable
func slotRoots*[T, H](self: SlotsBuilder[T, H]): seq[H] {.inline.} =
## Returns the slot roots.
##
self.slotRoots
func verifyTree*(self: SlotsBuilder): ?Poseidon2Tree {.inline.} =
func verifyTree*[T, H](self: SlotsBuilder[T, H]): ?T {.inline.} =
## Returns the slots tree (verification tree).
##
self.verifyTree
self.verifiableTree
func verifyRoot*(self: SlotsBuilder): ?Poseidon2Hash {.inline.} =
func verifyRoot*[T, H](self: SlotsBuilder[T, H]): ?H {.inline.} =
## Returns the slots root (verification root).
##
self.verifyTree.?root().?toOption
if tree =? self.verifyTree and root =? tree.root:
return some root
func nextPowerOfTwoPad*(a: int): int =
## Returns the difference between the original
## value and the next power of two.
##
nextPowerOfTwo(a) - a
func numBlockPadBytes*(self: SlotsBuilder): Natural =
## Number of padding bytes required for a pow2
## merkle tree for each block.
##
self.blockPadBytes.len
func numSlotsPadLeafs*(self: SlotsBuilder): Natural =
## Number of padding field elements required for a pow2
## merkle tree for each slot.
##
self.slotsPadLeafs.len
func numRootsPadLeafs*(self: SlotsBuilder): Natural =
## Number of padding field elements required for a pow2
## merkle tree for the slot roots.
##
self.rootsPadLeafs.len
func numSlots*(self: SlotsBuilder): Natural =
func numSlots*[T, H](self: SlotsBuilder[T, H]): Natural =
## Number of slots.
##
self.manifest.numSlots
func numSlotBlocks*(self: SlotsBuilder): Natural =
func numSlotBlocks*[T, H](self: SlotsBuilder[T, H]): Natural =
## Number of blocks per slot.
##
self.manifest.blocksCount div self.manifest.numSlots
self.numSlotBlocks
func slotBytes*(self: SlotsBuilder): NBytes =
func numBlocks*[T, H](self: SlotsBuilder[T, H]): Natural =
## Number of blocks.
##
self.numSlotBlocks * self.manifest.numSlots
func slotBytes*[T, H](self: SlotsBuilder[T, H]): NBytes =
## Number of bytes per slot.
##
(self.manifest.blockSize.int * self.numSlotBlocks).NBytes
func numBlockCells*(self: SlotsBuilder): Natural =
func numBlockCells*[T, H](self: SlotsBuilder[T, H]): Natural =
## Number of cells per block.
##
(self.manifest.blockSize div self.cellSize).Natural
func numBlockCellsPadded*(self: SlotsBuilder): Natural =
## Number of cells per block including padding.
##
nextPowerOfTwo(self.numBlockCells.int).Natural
func cellSize*(self: SlotsBuilder): NBytes =
func cellSize*[T, H](self: SlotsBuilder[T, H]): NBytes =
## Cell size.
##
self.cellSize
func numSlotCells*(self: SlotsBuilder): Natural =
func numSlotCells*[T, H](self: SlotsBuilder[T, H]): Natural =
## Number of cells per slot.
##
self.numBlockCells * self.numSlotBlocks
func numSlotCellsPadded*(self: SlotsBuilder): Natural =
## Number of cells per slot including padding.
##
nextPowerOfTwo(self.numBlockCellsPadded.int * self.numSlotBlocks.int).Natural
func emptyDigestTree*(self: SlotsBuilder): Poseidon2Tree {.inline.} =
## Returns the tree of a padding block.
##
self.emptyDigestTree
func slotIndiciesIter*(self: SlotsBuilder, slot: Natural): ?!Iter[int] =
func slotIndiciesIter*[T, H](self: SlotsBuilder[T, H], slot: Natural): ?!Iter[int] =
## Returns the slot indices.
##
self.strategy.getIndicies(slot).catch
func slotIndicies*(self: SlotsBuilder, slot: Natural): seq[int] =
func slotIndicies*[T, H](self: SlotsBuilder[T, H], slot: Natural): seq[int] =
## Returns the slot indices.
##
if iter =? self.strategy.getIndicies(slot).catch:
toSeq(iter)
else:
trace "Failed to get slot indicies"
newSeq[int]()
return toSeq(iter)
func manifest*(self: SlotsBuilder): Manifest =
func manifest*[T, H](self: SlotsBuilder[T, H]): Manifest =
## Returns the manifest.
##
self.manifest
proc buildBlockTree*(
self: SlotsBuilder,
blkIdx: Natural): Future[?!(seq[byte], Poseidon2Tree)] {.async.} =
proc buildBlockTree*[T, H](
self: SlotsBuilder[T, H],
blkIdx: Natural,
slotPos: Natural): Future[?!(seq[byte], T)] {.async.} =
## Build the block digest tree and return a tuple with the
## block data and the tree.
##
logScope:
blkIdx = blkIdx
slotPos = slotPos
numSlotBlocks = self.manifest.numSlotBlocks
cellSize = self.cellSize
trace "Building block tree"
if slotPos > (self.manifest.numSlotBlocks - 1):
# pad blocks are 0 byte blocks
trace "Returning empty digest tree for pad block"
return success (self.emptyBlock, self.emptyDigestTree)
without blk =? await self.store.getBlock(self.manifest.treeCid, blkIdx), err:
error "Failed to get block CID for tree at index"
error "Failed to get block CID for tree at index", err = err.msg
return failure(err)
if blk.isEmpty:
success (DefaultEmptyBlock & self.blockPadBytes, self.emptyDigestTree)
success (self.emptyBlock, self.emptyDigestTree)
else:
without tree =?
Poseidon2Tree.digestTree(blk.data & self.blockPadBytes, self.cellSize.int), err:
error "Failed to create digest for block"
T.digestTree(blk.data, self.cellSize.int), err:
error "Failed to create digest for block", err = err.msg
return failure(err)
success (blk.data, tree)
proc getBlockHashes*(
self: SlotsBuilder,
slotIndex: Natural): Future[?!seq[Poseidon2Hash]] {.async.} =
proc getCellHashes*[T, H](
self: SlotsBuilder[T, H],
slotIndex: Natural): Future[?!seq[H]] {.async.} =
## Collect all the cells from a block and return
## their hashes.
##
let
treeCid = self.manifest.treeCid
@ -209,38 +182,44 @@ proc getBlockHashes*(
logScope:
treeCid = treeCid
blockCount = blockCount
origBlockCount = blockCount
numberOfSlots = numberOfSlots
index = blkIdx
slotIndex = slotIndex
let
hashes: seq[Poseidon2Hash] = collect(newSeq):
for blkIdx in self.strategy.getIndicies(slotIndex):
trace "Getting block CID for tree at index"
let hashes = collect(newSeq):
for i, blkIdx in self.strategy.getIndicies(slotIndex):
logScope:
blkIdx = blkIdx
pos = i
without (_, blockTree) =? (await self.buildBlockTree(blkIdx)) and
digest =? blockTree.root, err:
error "Failed to get block CID for block tree at index", err = err.msg
return failure(err)
trace "Getting block CID for tree at index"
without (_, tree) =? (await self.buildBlockTree(blkIdx, i)) and
digest =? tree.root, err:
error "Failed to get block CID for tree at index", err = err.msg
return failure(err)
digest
trace "Get block digest", digest = digest.toHex
digest
success hashes
proc buildSlotTree*(
self: SlotsBuilder,
slotIndex: Natural): Future[?!Poseidon2Tree] {.async.} =
without blockHashes =? (await self.getBlockHashes(slotIndex)), err:
proc buildSlotTree*[T, H](
self: SlotsBuilder[T, H],
slotIndex: Natural): Future[?!T] {.async.} =
## Build the slot tree from the block digest hashes
## and return the tree.
without cellHashes =? (await self.getCellHashes(slotIndex)), err:
error "Failed to select slot blocks", err = err.msg
return failure(err)
Poseidon2Tree.init(blockHashes & self.slotsPadLeafs)
T.init(cellHashes)
proc buildSlot*(
self: SlotsBuilder,
slotIndex: Natural): Future[?!Poseidon2Hash] {.async.} =
## Build a slot tree and store it in the block store.
proc buildSlot*[T, H](
self: SlotsBuilder[T, H],
slotIndex: Natural): Future[?!H] {.async.} =
## Build a slot tree and store the proofs in
## the block store.
##
logScope:
@ -272,12 +251,10 @@ proc buildSlot*(
tree.root()
func buildVerifyTree*(
self: SlotsBuilder,
slotRoots: openArray[Poseidon2Hash]): ?!Poseidon2Tree =
Poseidon2Tree.init(@slotRoots & self.rootsPadLeafs)
func buildVerifyTree*[T, H](self: SlotsBuilder[T, H], slotRoots: openArray[H]): ?!T =
T.init(@slotRoots)
proc buildSlots*(self: SlotsBuilder): Future[?!void] {.async.} =
proc buildSlots*[T, H](self: SlotsBuilder[T, H]): Future[?!void] {.async.} =
## Build all slot trees and store them in the block store.
##
@ -300,14 +277,14 @@ proc buildSlots*(self: SlotsBuilder): Future[?!void] {.async.} =
return failure(err)
if verifyTree =? self.verifyTree and verifyRoot =? verifyTree.root:
if verifyRoot != root: # TODO: `!=` doesn't work for SecretBool
if not bool(verifyRoot == root): # TODO: `!=` doesn't work for SecretBool
return failure "Existing slots root doesn't match reconstructed root."
self.verifyTree = some tree
self.verifiableTree = some tree
success()
proc buildManifest*(self: SlotsBuilder): Future[?!Manifest] {.async.} =
proc buildManifest*[T, H](self: SlotsBuilder[T, H]): Future[?!Manifest] {.async.} =
if err =? (await self.buildSlots()).errorOption:
error "Failed to build slot roots", err = err.msg
return failure(err)
@ -321,68 +298,88 @@ proc buildManifest*(self: SlotsBuilder): Future[?!Manifest] {.async.} =
error "Failed to map slot roots to CIDs", err = err.msg
return failure(err)
Manifest.new(self.manifest, rootProvingCid, rootCids)
Manifest.new(
self.manifest,
rootProvingCid,
rootCids,
self.cellSize,
self.strategy.strategyType)
proc new*(
T: type SlotsBuilder,
proc new*[T, H](
_: type SlotsBuilder[T, H],
store: BlockStore,
manifest: Manifest,
strategy: ?IndexingStrategy = none IndexingStrategy,
cellSize = DefaultCellSize): ?!SlotsBuilder =
strategy = SteppedStrategy,
cellSize = DefaultCellSize): ?!SlotsBuilder[T, H] =
if not manifest.protected:
return failure("Can only create SlotsBuilder using protected manifests.")
trace "Manifest is not protected."
return failure("Manifest is not protected.")
logScope:
blockSize = manifest.blockSize
strategy = strategy
cellSize = cellSize
if (manifest.blocksCount mod manifest.numSlots) != 0:
trace "Number of blocks must be divisable by number of slots."
return failure("Number of blocks must be divisable by number of slots.")
let cellSize = if manifest.verifiable: manifest.cellSize else: cellSize
if (manifest.blockSize mod cellSize) != 0.NBytes:
trace "Block size must be divisable by cell size."
return failure("Block size must be divisable by cell size.")
let
strategy = if strategy.isNone:
? SteppedStrategy.init(
0, manifest.blocksCount - 1, manifest.numSlots).catch
else:
strategy.get
numBlockCells = (manifest.blockSize div cellSize).int # number of cells per block
numSlotCells = manifest.numSlotBlocks * numBlockCells # number of uncorrected slot cells
pow2SlotCells = nextPowerOfTwo(numSlotCells) # pow2 cells per slot
numPadSlotBlocks = pow2SlotCells div numBlockCells # pow2 blocks per slot
numPadBlocksTotal = numPadSlotBlocks * manifest.numSlots # total number of pad blocks
# all trees have to be padded to power of two
numBlockCells = (manifest.blockSize div cellSize).int # number of cells per block
blockPadBytes = newSeq[byte](numBlockCells.nextPowerOfTwoPad * cellSize.int) # power of two padding for blocks
numSlotBlocks = (manifest.blocksCount div manifest.numSlots)
slotsPadLeafs = newSeqWith(numSlotBlocks.nextPowerOfTwoPad, Poseidon2Zero) # power of two padding for block roots
rootsPadLeafs = newSeqWith(manifest.numSlots.nextPowerOfTwoPad, Poseidon2Zero)
emptyDigestTree = ? Poseidon2Tree.digestTree(DefaultEmptyBlock & blockPadBytes, DefaultCellSize.int)
emptyBlock = newSeq[byte](manifest.blockSize.int)
emptyDigestTree = ? T.digestTree(emptyBlock, cellSize.int)
var self = SlotsBuilder(
store: store,
manifest: manifest,
strategy: strategy,
cellSize: cellSize,
blockPadBytes: blockPadBytes,
slotsPadLeafs: slotsPadLeafs,
rootsPadLeafs: rootsPadLeafs,
emptyDigestTree: emptyDigestTree)
strategy = ? strategy.init(
0,
numPadBlocksTotal - 1,
manifest.numSlots).catch
logScope:
numBlockCells = numBlockCells
numSlotCells = numSlotCells
pow2SlotCells = pow2SlotCells
numPadSlotBlocks = numPadSlotBlocks
numPadBlocksTotal = numPadBlocksTotal
strategy = strategy.strategyType
trace "Creating slots builder"
var
self = SlotsBuilder[T, H](
store: store,
manifest: manifest,
strategy: strategy,
cellSize: cellSize,
emptyBlock: emptyBlock,
numSlotBlocks: numPadSlotBlocks,
emptyDigestTree: emptyDigestTree)
if manifest.verifiable:
if manifest.slotRoots.len == 0 or manifest.slotRoots.len != manifest.numSlots:
if manifest.slotRoots.len == 0 or
manifest.slotRoots.len != manifest.numSlots:
return failure "Manifest is verifiable but slot roots are missing or invalid."
let slotRoots = manifest.slotRoots.mapIt( (? it.fromSlotCid()))
let
slotRoots = manifest.slotRoots.mapIt( (? it.fromSlotCid() ))
tree = ? self.buildVerifyTree(slotRoots)
expectedRoot = ? manifest.verifyRoot.fromVerifyCid()
verifyRoot = ? tree.root
without tree =? self.buildVerifyTree(slotRoots), err:
error "Failed to build slot roots tree", err = err.msg
return failure(err)
without expectedRoot =? manifest.verifyRoot.fromVerifyCid(), err:
error "Unable to convert manifest verifyRoot to hash", error = err.msg
return failure(err)
if verifyRoot =? tree.root:
if verifyRoot != expectedRoot:
return failure "Existing slots root doesn't match reconstructed root."
if verifyRoot != expectedRoot:
return failure "Existing slots root doesn't match reconstructed root."
self.slotRoots = slotRoots
self.verifyTree = some tree
self.verifiableTree = some tree
success self

View File

@ -1,4 +1,9 @@
import ./sampler/sampler
import ./sampler/utils
import ../merkletree
export sampler, utils
type
Poseidon2Sampler* = DataSampler[Poseidon2Tree, Poseidon2Hash]

View File

@ -8,182 +8,146 @@
## those terms.
import std/sugar
import std/sequtils
import pkg/chronicles
import pkg/chronos
import pkg/questionable
import pkg/questionable/results
import pkg/constantine/math/arithmetic
import pkg/poseidon2
import pkg/poseidon2/types
import pkg/poseidon2/io
import pkg/stew/arrayops
import ../../logutils
import ../../market
import ../../blocktype as bt
import ../../merkletree
import ../../manifest
import ../../stores
import ../converters
import ../builder
import ../types
import ./utils
logScope:
topics = "codex datasampler"
type
Cell* = seq[byte]
Sample* = object
data*: Cell
slotProof*: Poseidon2Proof
cellProof*: Poseidon2Proof
slotBlockIdx*: Natural
blockCellIdx*: Natural
ProofInput* = object
entropy*: Poseidon2Hash
verifyRoot*: Poseidon2Hash
verifyProof*: Poseidon2Proof
numSlots*: Natural
numCells*: Natural
slotIndex*: Natural
samples*: seq[Sample]
DataSampler* = ref object of RootObj
DataSampler*[T, H] = ref object of RootObj
index: Natural
blockStore: BlockStore
# The following data is invariant over time for a given slot:
builder: SlotsBuilder
builder: SlotsBuilder[T, H]
proc new*(
T: type DataSampler,
func getCell*[T, H](
self: DataSampler[T, H],
blkBytes: seq[byte],
blkCellIdx: Natural): seq[byte] =
let
cellSize = self.builder.cellSize.uint64
dataStart = cellSize * blkCellIdx.uint64
dataEnd = dataStart + cellSize
doAssert (dataEnd - dataStart) == cellSize, "Invalid cell size"
toInputData[H](blkBytes[dataStart ..< dataEnd])
proc getSample*[T, H](
self: DataSampler[T, H],
cellIdx: int,
slotTreeCid: Cid,
slotRoot: H): Future[?!Sample[H]] {.async.} =
let
cellsPerBlock = self.builder.numBlockCells
blkCellIdx = cellIdx.toCellInBlk(cellsPerBlock) # block cell index
blkSlotIdx = cellIdx.toBlkInSlot(cellsPerBlock) # slot tree index
origBlockIdx = self.builder.slotIndicies(self.index)[blkSlotIdx] # convert to original dataset block index
logScope:
cellIdx = cellIdx
blkSlotIdx = blkSlotIdx
blkCellIdx = blkCellIdx
origBlockIdx = origBlockIdx
trace "Retrieving sample from block tree"
let
(_, proof) = (await self.blockStore.getCidAndProof(
slotTreeCid, blkSlotIdx.Natural)).valueOr:
return failure("Failed to get slot tree CID and proof")
slotProof = proof.toVerifiableProof().valueOr:
return failure("Failed to get verifiable proof")
(bytes, blkTree) = (await self.builder.buildBlockTree(
origBlockIdx, blkSlotIdx)).valueOr:
return failure("Failed to build block tree")
cellData = self.getCell(bytes, blkCellIdx)
cellProof = blkTree.getProof(blkCellIdx).valueOr:
return failure("Failed to get proof from block tree")
success Sample[H](
cellData: cellData,
merklePaths: (cellProof.path & slotProof.path))
proc getProofInput*[T, H](
self: DataSampler[T, H],
entropy: ProofChallenge,
nSamples: Natural): Future[?!ProofInput[H]] {.async.} =
## Generate proofs as input to the proving circuit.
##
let
entropy = H.fromBytes(
array[31, byte].initCopyFrom(entropy[0..30])) # truncate to 31 bytes, otherwise it _might_ be greater than mod
verifyTree = self.builder.verifyTree.toFailure.valueOr:
return failure("Failed to get verify tree")
slotProof = verifyTree.getProof(self.index).valueOr:
return failure("Failed to get slot proof")
datasetRoot = verifyTree.root().valueOr:
return failure("Failed to get dataset root")
slotTreeCid = self.builder.manifest.slotRoots[self.index]
slotRoot = self.builder.slotRoots[self.index]
cellIdxs = entropy.cellIndices(
slotRoot,
self.builder.numSlotCells,
nSamples)
logScope:
cells = cellIdxs
trace "Collecting input for proof"
let samples = collect(newSeq):
for cellIdx in cellIdxs:
(await self.getSample(cellIdx, slotTreeCid, slotRoot)).valueOr:
return failure("Failed to get sample")
success ProofInput[H](
entropy: entropy,
datasetRoot: datasetRoot,
slotProof: slotProof.path,
nSlotsPerDataSet: self.builder.numSlots,
nCellsPerSlot: self.builder.numSlotCells,
slotRoot: slotRoot,
slotIndex: self.index,
samples: samples)
proc new*[T, H](
_: type DataSampler[T, H],
index: Natural,
blockStore: BlockStore,
builder: SlotsBuilder): ?!DataSampler =
builder: SlotsBuilder[T, H]): ?!DataSampler[T, H] =
if index > builder.slotRoots.high:
error "Slot index is out of range"
return failure("Slot index is out of range")
success DataSampler(
if not builder.verifiable:
return failure("Cannot instantiate DataSampler for non-verifiable builder")
success DataSampler[T, H](
index: index,
blockStore: blockStore,
builder: builder)
proc getCell*(self: DataSampler, blkBytes: seq[byte], blkCellIdx: Natural): Cell =
let
cellSize = self.builder.cellSize.uint64
dataStart = cellSize * blkCellIdx.uint64
dataEnd = dataStart + cellSize
return blkBytes[dataStart ..< dataEnd]
proc createProofSample(self: DataSampler, slotTreeCid: Cid, cellIdx: Natural): Future[?!Sample] {.async.} =
let
cellsPerBlock = self.builder.numBlockCells
blkCellIdx = cellIdx.toBlockCellIdx(cellsPerBlock)
slotBlkIdx = cellIdx.toBlockIdx(cellsPerBlock)
logScope:
cellIdx = cellIdx
slotBlkIdx = slotBlkIdx
blkCellIdx = blkCellIdx
without (cid, proof) =? await self.blockStore.getCidAndProof(
slotTreeCid,
slotBlkIdx.Natural), err:
error "Failed to get block from block store", err = err.msg
return failure(err)
without slotProof =? proof.toVerifiableProof(), err:
error "Unable to convert slot proof to poseidon proof", error = err.msg
return failure(err)
# If the cell index is greater than or equal to the UNPADDED number of slot cells,
# Then we're sampling inside a padded block.
# In this case, we use the pre-generated zero-data and pre-generated padding-proof for this cell index.
if cellIdx >= self.builder.numSlotCells:
trace "Sampling a padded block"
without blockProof =? self.builder.emptyDigestTree.getProof(blkCellIdx), err:
error "Failed to get proof from empty block tree", err = err.msg
return failure(err)
success(Sample(
data: newSeq[byte](self.builder.cellSize.int),
slotProof: slotProof,
cellProof: blockProof,
slotBlockIdx: slotBlkIdx.Natural,
blockCellIdx: blkCellIdx.Natural))
else:
trace "Sampling a dataset block"
# This converts our slotBlockIndex to a datasetBlockIndex using the
# indexing-strategy used by the builder.
# We need this to fetch the block data. We can't do it by slotTree + slotBlkIdx.
let datasetBlockIndex = self.builder.slotIndicies(self.index)[slotBlkIdx]
without (bytes, blkTree) =? await self.builder.buildBlockTree(datasetBlockIndex), err:
error "Failed to build block tree", err = err.msg
return failure(err)
without blockProof =? blkTree.getProof(blkCellIdx), err:
error "Failed to get proof from block tree", err = err.msg
return failure(err)
success(Sample(
data: self.getCell(bytes, blkCellIdx),
slotProof: slotProof,
cellProof: blockProof,
slotBlockIdx: slotBlkIdx.Natural,
blockCellIdx: blkCellIdx.Natural))
proc getProofInput*(
self: DataSampler,
entropy: ProofChallenge,
nSamples: Natural): Future[?!ProofInput] {.async.} =
## Generate proofs as input to the proving circuit.
##
let
entropy = Poseidon2Hash.fromBytes(
array[31, byte].initCopyFrom(entropy[0..30])) # truncate to 31 bytes, otherwise it _might_ be greater than mod
without verifyTree =? self.builder.verifyTree and
verifyProof =? verifyTree.getProof(self.index) and
verifyRoot =? verifyTree.root(), err:
error "Failed to get slot proof from verify tree", err = err.msg
return failure(err)
let slotTreeCid = self.builder.manifest.slotRoots[self.index]
logScope:
index = self.index
samples = nSamples
slotTreeCid = slotTreeCid
trace "Collecting input for proof"
let cellIdxs = entropy.cellIndices(
self.builder.slotRoots[self.index],
self.builder.numSlotCellsPadded,
nSamples)
trace "Found cell indices", cellIdxs
let samples = collect(newSeq):
for cellIdx in cellIdxs:
without sample =? (await self.createProofSample(slotTreeCid, cellIdx)), err:
error "Failed to create proof sample", error = err.msg
return failure(err)
sample
success ProofInput(
entropy: entropy,
verifyRoot: verifyRoot,
verifyProof: verifyProof,
numSlots: self.builder.numSlots,
numCells: self.builder.numSlotCells,
slotIndex: self.index,
samples: samples)

View File

@ -9,8 +9,9 @@
import std/sugar
import std/bitops
import std/sequtils
import pkg/chronicles
import pkg/questionable/results
import pkg/poseidon2
import pkg/poseidon2/io
@ -20,8 +21,11 @@ import pkg/constantine/math/io/io_fields
import ../../merkletree
func toInputData*[H](data: seq[byte]): seq[byte] =
return toSeq(data.elements(H)).mapIt( @(it.toBytes) ).concat
func extractLowBits*[n: static int](elm: BigInt[n], k: int): uint64 =
assert( k > 0 and k <= 64 )
doAssert( k > 0 and k <= 64 )
var r = 0'u64
for i in 0..<k:
let b = bit[n](elm, i)
@ -48,13 +52,13 @@ func ceilingLog2*(x : int) : int =
else:
return (floorLog2(x-1) + 1)
func toBlockIdx*(cell: Natural, numCells: Natural): Natural =
func toBlkInSlot*(cell: Natural, numCells: Natural): Natural =
let log2 = ceilingLog2(numCells)
doAssert( 1 shl log2 == numCells , "`numCells` is assumed to be a power of two" )
return cell div numCells
func toBlockCellIdx*(cell: Natural, numCells: Natural): Natural =
func toCellInBlk*(cell: Natural, numCells: Natural): Natural =
let log2 = ceilingLog2(numCells)
doAssert( 1 shl log2 == numCells , "`numCells` is assumed to be a power of two" )
@ -67,7 +71,7 @@ func cellIndex*(
let log2 = ceilingLog2(numCells)
doAssert( 1 shl log2 == numCells , "`numCells` is assumed to be a power of two" )
let hash = Sponge.digest( @[ slotRoot, entropy, counter.toF ], rate = 2 )
let hash = Sponge.digest( @[ entropy, slotRoot, counter.toF ], rate = 2 )
return int( extractLowBits(hash, log2) )
func cellIndices*(
@ -75,11 +79,8 @@ func cellIndices*(
slotRoot: Poseidon2Hash,
numCells: Natural, nSamples: Natural): seq[Natural] =
trace "Calculating cell indices", numCells, nSamples
var indices: seq[Natural]
while (indices.len < nSamples):
let idx = cellIndex(entropy, slotRoot, numCells, indices.len + 1)
indices.add(idx.Natural)
indices

28
codex/slots/types.nim Normal file
View File

@ -0,0 +1,28 @@
## Nim-Codex
## Copyright (c) 2024 Status Research & Development GmbH
## Licensed under either of
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
## at your option.
## This file may not be copied, modified, or distributed except according to
## those terms.
type
Sample*[H] = object
cellData*: seq[byte]
merklePaths*: seq[H]
PublicInputs*[H] = object
slotIndex*: int
datasetRoot*: H
entropy*: H
ProofInput*[H] = object
entropy*: H
datasetRoot*: H
slotIndex*: Natural
slotRoot*: H
nCellsPerSlot*: Natural
nSlotsPerDataSet*: Natural
slotProof*: seq[H]
samples*: seq[Sample[H]]

View File

@ -10,29 +10,59 @@
import pkg/poseidon2
import pkg/questionable/results
import pkg/libp2p/multihash
import pkg/stew/byteutils
import pkg/constantine/math/arithmetic
import pkg/constantine/math/io/io_bigints
import pkg/constantine/math/io/io_fields
import ../merkletree
func spongeDigest*(
_: type Poseidon2Hash,
bytes: openArray[byte],
rate: static int = 2): ?!Poseidon2Hash =
## Hashes chunks of data with a sponge of rate 1 or 2.
##
success Sponge.digest(bytes, rate)
func spongeDigest*(
_: type Poseidon2Hash,
bytes: openArray[Bn254Fr],
rate: static int = 2): ?!Poseidon2Hash =
## Hashes chunks of elements with a sponge of rate 1 or 2.
##
success Sponge.digest(bytes, rate)
func digestTree*(
_: type Poseidon2Tree,
bytes: openArray[byte], chunkSize: int): ?!Poseidon2Tree =
bytes: openArray[byte],
chunkSize: int): ?!Poseidon2Tree =
## Hashes chunks of data with a sponge of rate 2, and combines the
## resulting chunk hashes in a merkle root.
##
# doAssert not(rate == 1 or rate == 2), "rate can only be 1 or 2"
if not chunkSize > 0:
return failure("chunkSize must be greater than 0")
var index = 0
var leaves: seq[Poseidon2Hash]
while index < bytes.len:
let start = index
let finish = min(index + chunkSize, bytes.len)
let digest = Sponge.digest(bytes.toOpenArray(start, finish - 1), rate = 2)
let digest = ? Poseidon2Hash.spongeDigest(bytes.toOpenArray(start, finish - 1), 2)
leaves.add(digest)
index += chunkSize
return Poseidon2Tree.init(leaves)
func digest*(
_: type Poseidon2Tree,
bytes: openArray[byte], chunkSize: int): ?!Poseidon2Hash =
bytes: openArray[byte],
chunkSize: int): ?!Poseidon2Hash =
## Hashes chunks of data with a sponge of rate 2, and combines the
## resulting chunk hashes in a merkle root.
##
@ -41,7 +71,8 @@ func digest*(
func digestMhash*(
_: type Poseidon2Tree,
bytes: openArray[byte], chunkSize: int): ?!MultiHash =
bytes: openArray[byte],
chunkSize: int): ?!MultiHash =
## Hashes chunks of data with a sponge of rate 2 and
## returns the multihash of the root
##

View File

@ -0,0 +1,527 @@
{
"dataSetRoot": "16074246370508166450132968585287196391860062495017081813239200574579640171677"
, "entropy": "1234567"
, "nCellsPerSlot": 512
, "nSlotsPerDataSet": 11
, "slotIndex": 3
, "slotRoot": "20744935707483803411869804102043283881376973626291244537230284476834672019997"
, "slotProof":
[ "14279309641024220656349577745390262299143357053971618723978902485113885925133"
, "17350220251883387610715080716935498684002984280929482268590417788651882821293"
, "3614172556528990402229172918446087216573760062512459539027101853103043539066"
, "9593656216696187567506330076677122799107266567595923589824071605501987205034"
, "0"
, "0"
, "0"
, "0"
]
, "cellData":
[ [ "211066599696340205996365563960462032209214145564176017965177408819390441927"
, "256399834032317991719099034134771774537377713676282398278615627599320306708"
, "40526956212941839024868120947422067322935297516255336725720469887577875470"
, "369406626072040375689238003388146123438765868500054546379159741776926336393"
, "333671948877941061129138970028865844558745266314244224980179694307884999701"
, "12844191661993811401197260475054004253686019503294245287625435635597431176"
, "103242505924551040184986153577926463300854479121445566984435820844932904541"
, "357267598134410031301285377503939045679462829143562392803919346036584141082"
, "162961530392479745020607594774288130869393650464001426863668385541077786641"
, "426633666684068667053061341108297711407154520752589357873877687002123230254"
, "131217200633679697678620573903653316618851166977399592012996945149000115543"
, "347806146382240882424437318028620747118202052207615339887983883245341422889"
, "373560143578047669373240014805175743607410583882636003337120578008437374619"
, "188643112726610812698950916978832639394489168469865816132268079928321994342"
, "261643693073361806247543578456646407192325702146752104760117340650316255422"
, "260425332276127964154119199351807753107358504026338378706506655351595199132"
, "374895089121103633563000003194929058314955925591115574099805066048344387554"
, "251251687633538360627384151887287228926166832908633974282160065378311171354"
, "72870348025150463132527129203816383011245664502016571773976961034605631401"
, "234969517550818492643515432666311755833657223377594670026839098818269671638"
, "250704662734070531273640113211555977086430125469327371276827055724111014200"
, "85287059658255939741261887818611116570376488685014337052369839580946343903"
, "148959658976765873884541474400081762855732313547557243964921157967555048302"
, "402116967301520959272239788104745348344918207829380669065019055837727389479"
, "440503687192139964066297025050080823601005280790824714962651368433530759519"
, "149064344353438643307231355233484617851197634669308957706338957575177745645"
, "249140840255377018814348914718469942883867200392561109698520706525194687651"
, "108796851176515124780842490199733462942992752881710253277179665118758071359"
, "168245155425564161902686596247453762364112240129335852645432169468767513906"
, "129930759770229612264501691941454447321585806001002088749773920103899362070"
, "26204732465033162738545933008873662562758704699080684615280202127289894343"
, "434343986187775289860542785529657444690073100436887472033336117760907652966"
, "361202432740487795596808128962740911600093857340619816047190021218849540225"
, "100616813001075101816809823591611435583084916492802624686700765550893945525"
, "262383566766515164611427346701355047932794351290691325516723194829671679460"
, "223966317895663049002893008659178463136086169222436544014405848127792334099"
, "416071800998333357259662686053338495720384342746822618737948080251761863079"
, "402582378631671531909245563300554883898015688826468151075059467077182712018"
, "271682867846395286938993638577506552857925968097084028550962231439839229096"
, "447239701195120035004067146414333903010840427278848992921567785105217019890"
, "275718367441702584521943480326858762208121719038684001399322597215141179102"
, "86424132534636958411139699704035287483951667275825975536719519441147485752"
, "149332313586183975744469174256094358432564607635406143904268565140988616920"
, "431284330776421588418608279008210842837281123158951642164956884286883748089"
, "328694410745471749523135644660455669483988686888634622076863114197617693825"
, "112671940998917362968156144648543607958275336559773039070338509488400224090"
, "40612851250697989627190554726382690498263128439797780029697069621854862060"
, "235047914228675997216342104196597257178021277585839376175077878186492271543"
, "169718735151210135199527910197065439221144015957220768545119706561079163228"
, "345850109040121443591415752965486014695272086901102608769402892906795715635"
, "107916794601837835951508003838161872232087225679609623116098837565956752373"
, "415195406060373162425374246348197423165008252112453298469572523506488563795"
, "18574536720926634955170276058049993354780447816096707123565164996905722992"
, "77316964375201096801231570737992491072607736322255206149311341101525354519"
, "198566211140075666401818378905444403588640345933666108724809349396921957675"
, "71926050707400318807625168942501384117254391471312636171205913503928815127"
, "303403754792341398295052586417779761162194818661412867094096550880325459639"
, "444796230931706624375881141460151785952798771079111017071124833045933389733"
, "430832727519144759265424205289522948157007336118070755365887670730658782114"
, "75431213985866235726407973044434444984663930761207296437571668004273515965"
, "9242635103653159191249730220870735855877366952081272723035956668095954838"
, "93770678769846326584848478152412123903909949977598807336203128684179492141"
, "438043261966019084676377174988310087831395864304423411701911688757793135582"
, "175357918416563657734972138036003712114814934655676571874083109097142591069"
, "301619681954194702458985259161884119574424456150215738560639417824784261940"
, "376627771252167062559065889174364784087884871999807562320457079200311413098"
, "77407"
]
, [ "424838222746158988229624788694939151178615656210585621868910231014323837551"
, "113188486246955346418956949679485575685258850346101035778277727456423482970"
, "275449978206132565019222655023969430014622832597654643418394602485645803413"
, "407856757018138010439232009766252068440920591566039673660874626878413077905"
, "433369046493777496016384947949950877133856528218602671493669395706908819748"
, "258364166531180422149545015891786023981872586904946376136311798402581278793"
, "111997719221028147522803956659709775148434180015507797582340706052412284571"
, "370086597976426508280413876667101417393598181708677558733730556109327409076"
, "394139601979449259075910238117153992797849553309541269624848742084205563806"
, "224088276319080487199395482893988152025671468715318212801266537036640477323"
, "412710245119809501914481942088314642684754542082140451180970198371889738885"
, "353872602359946553306242341162348980635834907495492814598834657175405697176"
, "252575199506028475372678621140654219936768774012823764047177692104580452933"
, "259093153824033122452869556249315839899366493071746219770487886456301642099"
, "433829976798312333371154167497560676982294392632725612538640639617101218872"
, "69918581382122563555200898078544150952625715196904114153232367538572342772"
, "337937520623192257595352158476909569245988839238498098464935654555688460123"
, "264739973232292969253318643276671532055689422253184191167449284055983944338"
, "326116252818829775096345069850111970510714050346103409479803743342138260656"
, "236666502377413649728378889488706275212721356921124776708954261777813709815"
, "211625935799984260567718228446525455893664313064841539301444509150157287163"
, "60213206239417039999880027112341976360540689886703427811513517396638607512"
, "68310118105957780876770075529546844404225720757669797609686816545988561625"
, "423863085551351065136684030270731679105571943009795949621903966660399419936"
, "388914614294393005039878123500859325222684672184567792659076815268598434245"
, "449456790291560508709069826219925144971979653209755565240911568965768874382"
, "448810363770763694447869940916735951256986784286793489549428379909616059117"
, "93646909783664049092056237949587618925209622020026157405117796611689551192"
, "352210795298632954574896499649181574584074853828419384742874364724522457331"
, "37455517056393404525863484733101879886413925183061645520768912552476716150"
, "386617357584684336812125385078476270301738184058813703112840991226785114117"
, "309940292044597334261558429176136686101590729982259514472573656131579113438"
, "375815246167575100319857872432879650174355611853064771241069582477717074415"
, "332214507344122806007757734266883566559371568252728459951766124888176633706"
, "148990259460952914990881100852534318351247069504848477833147446514732789712"
, "328669527889838880414072022433859139004058211332184916573516704632073044118"
, "39278026039348543645873027549112998051601664395028652771103624299930924528"
, "147717660530589785119644237145092759103012624422229579698752386490700965238"
, "374018518701362017594752095877197725242352803195413267746619111489936980685"
, "19185486483883210730969367354195688373879769005461272861759636600984416877"
, "61866046506558157021682973167090213780467780519546382332208868591026703563"
, "186854966504766517012887726614015646154225796572138017810371160981778288347"
, "87813550250328892091332566928942844770632705056120813488729800874811845697"
, "207775163424060266085108794048371834145545842567796157378282772999548202308"
, "369987573847786237689249538753881486995686208870889713177975415012214427429"
, "240880979016395044518849230927615466120209140082149273390921042537474853143"
, "174902051454932913375934735427101804474275543418199101786687925733405159872"
, "342217255652950822372803598682842961053537267723988087801275319754065261308"
, "403207518067666945448161377960706451817747922771285796668778802535227939962"
, "407191459999036791052261270163259267557900498930952521056725210031161568230"
, "338216472523551728793268225845396218561132966393725938551091882807069657206"
, "118364222678576550870375112494142500603091119946985793934499723872824782886"
, "269721611028665323587192624288165848310917029418261851436925829954710472436"
, "227424498125745236125352117206136621428869662458452610254773560636280935711"
, "334380807433339401906359555583987757411855090694162252260781648609761248049"
, "42470806516174819075107446234449247453971524726021445768611797530804156161"
, "418994916402918322951830716592888390611524984156817012683478842068581638820"
, "363263142412048420546716019774090729399081311227606555141174736853886128407"
, "192292715923468025058557166341646729623133127372303791236447550026886802680"
, "450253878713722337767292128303865371116770532625906889925779639839924402495"
, "412596147086332805611392200560087191411541689130482740065137300252639590489"
, "264059866105067484811456793906835462997849523506366903974724979936196358724"
, "80922384260325792825608274004101667366364502159441286540209512108302572137"
, "69261112192907071823876894642934902051254647002357333297635511793652550535"
, "342810644068354896385837929029331085645928473943142618800192452300937098227"
, "228361826362950356801549793202622774869100858404479869989505302905936946659"
, "89244"
]
, [ "359926778925154809567585559910064420821311221299937568759183366972901588855"
, "128688825421597555299113005875649976344934035310192572516197551102045945994"
, "225379354482053502673695304662016054746658452775856157938886424152134693969"
, "321872319934407904743844034025550739031781361848065513098922085967524180784"
, "250375637325405951645070615947051799520095842815922754899017741501395744611"
, "97676493052788080672307677749501730337551225267342432472194527468634722352"
, "140101187036396881926000630022834247990512766860086619783437252676747730662"
, "428833039353549335335605804240430918631639449874968791377641834408506136850"
, "418359203734539413977740838354554804415161215624809316660001820037711273005"
, "197411877418795659129213175603102238904459737200167987167255995825203749339"
, "221646252100316559257470803343058462853641953756978011126414924636869625612"
, "106393540293584181037890192557883541231531964825708650997196071036779482686"
, "121473330828208543539643554911190528237124877123871673169194452404939833883"
, "234055622144947293638512253368547046093971383516706577723613696162225606040"
, "68307451767502390304445005915787226559811433450721625085741437522802389574"
, "446891883436763112014492564462451523127134145501201571918449345324780633462"
, "83718652783543018019599555197511164121642363321504039439786358267060414949"
, "90267297500929836073049162292104311427365986272517761342871530827272320168"
, "398425606698859520268856768787424690872952910789754531465894080258173664751"
, "323570139379118444589557840594603212198136718240764273769210492735883659788"
, "318597103584099056378057647488068323974418467250708490151864712850204121402"
, "6299083430773359277240726214182464517380839990956316267425262319606033077"
, "27638206326925436960316131682014727983280820447721477666884742925275976240"
, "434344186848869917381375812528446841024676181532946456237439060027443649574"
, "64735754118644738348599761514174981324344130078598038275246522384474432918"
, "53068717269762105498508401788249005564862415051761175636612434108259085043"
, "35813044996911619267309099508360887777226716179396659295580849861836012116"
, "67751791392924142809580984450371772015056060429352159361446914484238646676"
, "68534949135677447506316576616742207938855454921330757052467057435206318183"
, "98510151949547604999069864337574320742530406479752176012935179772005228326"
, "342190252152505345443004241184891966319091967630257822491352072978326623645"
, "362701658859425316334005554473186516818256386066010799465369887406035738447"
, "266999116654850467726292928465517542818678046748008340458185725047959981772"
, "227089355966197874086821090531951502393729872265201602128464978982907992285"
, "240800343500959216904535208047288234867926058830277460630902462914796702354"
, "447956858573680756485556898469710354624642653441041335815079102198306530583"
, "89422712944117481549242421245588048728782658978853365197341587057196539094"
, "72610343179362050463955730204044877712105879926618304878262944723464870506"
, "8676698500519447254981838968537883138182811064381378248657969913325524054"
, "180453700216061196739413267121764366438386031039311941313645439527087166894"
, "63346784016053253727621352882430122335280702556586808389293772066791583857"
, "400031453850139978805133735852120986024101930860924735862305444817383365395"
, "230104622290558570218036071349472289358926019290368625724986905348610140188"
, "175689489221336091369196327293045073133701056385414159213338224521167050830"
, "73310331103509697419315970265031228794034932318600293733074730812482185479"
, "371383255255842707875498538452907102684511927320158672347778293876877893808"
, "165319345890230193939972313953881372394171342391835626454759321320435952720"
, "184753541001210613115361457830691571384268642766106935428207829332011259768"
, "378810733004878432563271790625801205570962762395854282745166380274493181314"
, "86321674336629444862383262780020871828941143514651008200999725989892879714"
, "332634533993388248915777870988529817692938793120418377552247997050250349749"
, "41742010257820712511267405684781534740157292266212120171929853763272599516"
, "224101330592139734390658213442359402546038743346294438455635537496290117560"
, "204363902046137087420878796391135987770418514008394389852388361468850216359"
, "296526036888463156867153847102999031430641220536577556044554029990778763710"
, "137568796227115931047082828377698464185467276723279763304686078869351280509"
, "147456720891843338735232645318045656082153590170441596326500204393398792771"
, "297291342309626392877004635010131510068476699687818509485687346071074942006"
, "20748013593486831233186810444485136836664689496258849465507672301203832324"
, "335431726883875036252568773858744219410188302724423071283643221351691013313"
, "50487384098835523033417523223562560857744547945136829388944024807752630716"
, "425952679139710019732649156938676226492714486376811103362264146658191708598"
, "439787938069461539508728805881194071103269524405653997968488488049426387373"
, "279863410796988495259178322026394289028023166886112128504434877538089779477"
, "398941099058270093463626870965433502581301413791423667994257456160377865247"
, "5759692644185723491187478313102540786562625675495805072053262277490225012"
, "115176"
]
, [ "199440901482393381753657315848210955092644686957900737971520778206058989647"
, "339215657660349719251187938243949119362753238126598750470832739830379601048"
, "17957417011314891245400567671664723859427624136284133790923936126779445290"
, "294761585889095249928492042608516765584723814657871392207964321318076158536"
, "367304199921887970655898837985855454346911090426896946930048519042744277770"
, "173405546837606747721292792526074597519538685230485266741646923399938591491"
, "13202798104529529703580600642858924379886936325402696094624200032343206719"
, "28211272278315691894282764239983301742024079691520980592618486887749800025"
, "73792448247120972778500624350664849847034095641998271809779791788652649022"
, "386961947078838359430674078072441680475090687247027225632133013772954043342"
, "247859266401821616700765969075270662915024391205665146199401830650793676517"
, "243938047874995926342875119559105623088213951205962677439167259642163766960"
, "14909501249861872673329370269106359532506159818320693170564856401208688898"
, "200331653478898243177761429526240803993101536716611440775588088625522029071"
, "127891684617049394579738365914860024509007913559921966744972525605976847919"
, "202912167983786187592861727924433749852786012202809544200943965898118027816"
, "176370650316309755425558132466370508977563252342126855874617990006444464573"
, "179490319446297562731655155074196383539396893457024237113284509223965454107"
, "118703379899134287650980989454755985628620830085932176414465582081598659194"
, "102025594191113707886629488454876652037341696284939671367279050169222988689"
, "421132375430104331136732058548913808473025321492255503838896123601628815453"
, "328334791815856213206267892694535121226673194052951034497930366807851111845"
, "83012322813281668737061895967682970093636853452224812061110092135287899376"
, "329204708391107220275172926348002826875172581958734129645882445887919889321"
, "410748869385474485539728045765785256294532845070137964737879849265390625591"
, "197274807717335387012872999914232051341799797613667869923402386359379902675"
, "235713095185988155553500217595661312303861624791720350423698435045853678746"
, "150631584359141913552843813384841153102535679219851913018874172414598353488"
, "207783836843813911284913666774420250626019971129676431904589416081127483900"
, "15728034718954665549174230921445077500399069880555428849958014406422697976"
, "69799423545177501667748653663121504155622623013014583690766769624482972893"
, "265665371394145560256710553779588458846778960884758560117805511822299802326"
, "149195925869584039415331014261414953602245337159354350672030141190471260449"
, "162328395279114064180857718453973759378615891406692054752029241989300597156"
, "104643123291849369328362168243087456326274773811561124607060302871149280568"
, "320704123383295724141970902124509237736831907552238934395000039155541269937"
, "77914486216152383968400591704791756847610018757857965758408455442143531631"
, "238365259321088298905088250444146071545398991768186471689605160523129613763"
, "279409375422154593510552116666741774885392805494152031820287626934209978908"
, "195118776021452609708543280509123101181249086555819844047203390788132717252"
, "197977884437087886153482042453896079316138251415359773453987232734849953584"
, "168185043240980087638006965666857387510828226074470344841850764460631595331"
, "231157923359356077977363679818678437836536420121744865399935742538602805912"
, "177903771863742191900138437188329108771172098110036075491750018158192424072"
, "313552174443290416730632310997197097951229162137491595709733420111980331403"
, "273253450712049988786741336540196077743997302924525995219038781650977490211"
, "421908030281055821389377531613150504859996607596444776050212044919345332385"
, "180108184992593746898140529947178182204857361841304042854173884504394805936"
, "37075272799330399065679301151342697855905822543084867570322173216259074746"
, "364885615491975468180698816037289079302391684470748470356247509218051645743"
, "397482868106190800111749908137311511782294652288655656060379563261618687603"
, "192853269627895017416381451198403197013798085262867793715522216458130791820"
, "450480853450142791928572953497280890976328598410525218090104787739409705079"
, "40278654070502961330170439514434669768416784968274956021022493731441898222"
, "251277143131769020481025315216040765839561111684608785317366609258959942695"
, "95094468748825454459610961968601800404132682484160170977941285794444806916"
, "160586633865113902389134480029183924655750088163491531655080014223476604929"
, "211661229493873434581423107377168478957907088187044576032505407590783850232"
, "409651293631434750272174456674594508340952460788494035327119354167465019826"
, "233213211946836553080627522409887799285199986120567245752841080275284294566"
, "143182900674482411759481361336063079267405785923487419697568907351386146653"
, "430050085956999990041799366370428972519385994997821389120583306252090911051"
, "241257468571530133762629460194062384921386438426367660851087853915892684115"
, "106478922860328643074356032194102718325829441005019365153538120054339275205"
, "252933430690486626644000908036895289326526510137514385824014300035301154822"
, "242924628511152862437189415942615812459145003511499320946981326550434266392"
, "107566"
]
, [ "10892488375325920610152294701785926476935321890453222549428070867493882259"
, "230776541958253414701326765204413805639078570664616139597900921490475143840"
, "162235550819840758141721889536295480113278275911087429090017541814695333320"
, "318634611531007856220026646570492630940047240387334221027051366284225674524"
, "347695480420330337439096561080975864031317110001559084872978387417651465445"
, "243301070227446762125488369714708670219289121859111929877695012393609726208"
, "312153141205681954392441579373652470334482191990502803524039048245142338874"
, "243769659456658813016931656268556893289414122189859136776671772112510762644"
, "235510946617019540983239794402008700611014664578713646813426116406215382253"
, "394638234040056271265534896894991100558052611842099314808878257003754175212"
, "112730195097163222179949980634599934392634120069300673310070655800491242211"
, "112545144551723145227061757291353149296490850338535641681237178578430772288"
, "399161925498018051746424503488168548076537557369822821644005390567188305750"
, "291823556779130095044851238536860577785281148245623338113991004798525195947"
, "443006765181360772964201996695219533539991249361936300010158089981376772939"
, "74018417655448362012716269253153545524112649147848318337218250865231619883"
, "361038295627629757439073080656763633087074408866229383288288831546300069767"
, "269655542872834422597617091710145830035767834433114567250727497135451412216"
, "58289072717559976781527037332370898163550414718655391280446986067842922181"
, "365399954331278626941053447122605263207706937018525782908294459265663426953"
, "83576501872896181822963149907518188809522341045703363171261203985876068484"
, "203403783686919490357886887779316222746544665267595235714814853282937072937"
, "226090172488558641139782927103632452136731207206963592401497570070700221117"
, "249813560776802008219945831355163226098682741553305882699128946352901227282"
, "236586835155013316983057471119105606475813711035522306026673814184519826069"
, "420611449257527132395951061920868895981487081726689195426673433565077012458"
, "414979562418422585161189066520079202660418565693776869373399931253524536378"
, "115851377630895049619958829726730418778383692593973233290077769966938018584"
, "248071158447148977966329235335508229928647083339924308951291087789494073866"
, "8254100651607835318906499132096926759832050649688561036854000785129084907"
, "91385483239630999205401307763890943367451672856478206313711901403403429289"
, "369346641925770698935046191632374874762045839747680407985443471202350286304"
, "236809023553698844817613980395588655518336237009075203833474757804664254158"
, "8367847400805682648908349286699722579431227561083498361702537964306290078"
, "599241730770400067632779114435635342549228985534229813617556932580328166"
, "347112528350917448294202348829052279076907614831011498643025223836596915573"
, "384244379244118003891043669466323943736726794634167201471569326059716944701"
, "118013777197672343498581960057939216208494837962825017767101107204031333144"
, "27234916267695376599463409893017377196853108034589808756909998459364893467"
, "443519198016088819704735929590164254445884637317806160485888215392818578737"
, "396780482611567392375183169345153676737175342167284140440545202776279411157"
, "420351155303051883480975795437743307852799862858964108014000673383502660760"
, "17379377743250873773932440622865094720355292220433235366224143179854831702"
, "299671454782683147939928632170233327590769402224392134648893444626929909373"
, "143062753141414050359792615867774312517100868919516205179025540179759009492"
, "79497692490953838158801094558761984613913564034406069659969793097043605498"
, "422748645389700647011491406944374966856916994331478229959954030359911549565"
, "101802829812014644970197499895811874607753186302439171072935333706660468030"
, "376428369998893026519415315112012919906032811618495880392785036762185101192"
, "193969030999254195249242252871597931610859408264053152789041067245597391073"
, "262277607928686742238285487190873200602833495734085188071246746209841324139"
, "154099884960502807271641574310268486840763221700920893692135020347157046386"
, "155875061164585018658671842995328931296342883770473498362059838106632382461"
, "248574435283666782825705601259695525637993294311364397935499480206725256362"
, "171325185063038052248966557755722232979612702743265316145563443527135798688"
, "19982746887818897250405980185937061235439217109294376948752373205830077881"
, "363719103724181291346130833008745948141602173373912337315631878022251200824"
, "174596812883797666354579747966720458118607233660798323559531788300018084931"
, "296611197821867585469311917529661698312828606304722408477045992233526328708"
, "115884038550627840768260751168697204665962542002024023842649383174518895165"
, "265597417366164841889730505737916646261040505851159477903649773521314216810"
, "59890857222664166616144499778264545115438678877460924559608721809777680238"
, "150275344313515259978222149421664752546204516114655782705875535407472455999"
, "119762211657733951640135703777940013374803447228667976561992857374027112851"
, "124750313254270944205036764607428674226393862430770588148329786501973600535"
, "223562415856611692667255745278751292230311455355383541649772348032933666931"
, "70851"
]
]
, "merklePaths":
[ [ "12330511756602656312909435206144412037562550923861053314147193494605624608532"
, "11626412651279744307149389430094868876817959605147995292836995419396445628874"
, "5992799448428980485292103987999399446160713735809250167288027256759377161164"
, "19665782623633007009046423286205585606029554995684266199914536255425604862856"
, "16487082247902739801276638140231448150060608377815926330854457096549924699346"
, "13757776896542890425183206586238760231244468647794123671233758868377423038254"
, "5689382212199289790852870716648320118282913659439556386010574440270030991956"
, "19397819444005071538172083272808000152189643623980212079104170950073560541073"
, "13602141253349313166682170066159161155345481788270338632198566205283140117430"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
]
, [ "20475873274412005481064639175076772466269555202404881992985142339142752174247"
, "16346160910918020455716115524174351294558871582658635761456705803199256575588"
, "2853750013041818757293788273269978233503065226276819540991640203844566736443"
, "9192572535522846104757923561847416578417599996904208474460700268961782558170"
, "11041850361074018956732434352769265303294549935255362322653487210796196161858"
, "20835509643844784930831626622202658364476409300598072395494952478408974334325"
, "15426115581767819720710837762133134950520914636073261355445708100826108573907"
, "7565353224987902191368863653499353764559862741092477570130316358454603122676"
, "2622681935585012630617774892501744551457568716225188460692779556142778732663"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
]
, [ "13099868869639061574800067722436547911616384523753701384396275064543709640456"
, "353757809120595355213201328586632712724405232919181040928026587840976194078"
, "17653300914565730132855106316678548541847283141888106932466281385199556950861"
, "15467462085462582082877261755656498905479817107855355753427299990712166382496"
, "8291733777946446853018893495264026584437749231931118771866890345692346711355"
, "15510790697317206014779022286261864844918915222875014882833700758879700055506"
, "5689382212199289790852870716648320118282913659439556386010574440270030991956"
, "19397819444005071538172083272808000152189643623980212079104170950073560541073"
, "13602141253349313166682170066159161155345481788270338632198566205283140117430"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
]
, [ "123238869181525326412236116167249816873084559218151119452851092131080991962"
, "9610314342084317647296061595824603740114670828357751076517430572434680540425"
, "16802740554584732104294972716558962567961331277692246600846665155168171370476"
, "151083360419914122898584757765086723506432610661508069194962432698872036623"
, "10357032992337239725601662829902169825217513617307319193581711776597892496381"
, "10120699018002766520605012835043517238241846918467244955580419060582311503402"
, "21149604008153751948441881526949680605328007895979738537313721955134548786062"
, "5720106921878932614189421948890362637585879521377362100104826996201092964473"
, "2622681935585012630617774892501744551457568716225188460692779556142778732663"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
]
, [ "19153513112714782931111012244694922243101677748840856395814929006033044311081"
, "21046138187228318287277629107063936540039891592394801899272249280765102572688"
, "18057980437430910028015917806534512217725128031222973066601095455076586015436"
, "5766677914654397407881589917461473873246279171605373166264025525757502238061"
, "12019967669236656188577515900815533059046454955207846938479617973037184411021"
, "14504305765289705714959523666100275156034056689367568164630385862257567596209"
, "7152002871325824138073253423783370852632926621899161541618248808716037342022"
, "9714587356194206699401761190093056901650105401919163689816999407566849779455"
, "13602141253349313166682170066159161155345481788270338632198566205283140117430"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
, "0"
]
]
}

View File

@ -31,6 +31,7 @@ import pkg/codex/erasure
import pkg/codex/merkletree
import pkg/codex/blocktype as bt
import pkg/codex/utils/asynciter
import pkg/codex/indexingstrategy
import pkg/codex/node {.all.}
@ -54,7 +55,7 @@ asyncchecksuite "Test Node - Host contracts":
manifestCidStr: string
manifestCid: Cid
market: MockMarket
builder: SlotsBuilder
builder: Poseidon2Builder
verifiable: Manifest
verifiableBlock: bt.Block
protected: Manifest
@ -84,7 +85,7 @@ asyncchecksuite "Test Node - Host contracts":
(await localStore.putBlock(manifestBlock)).tryGet()
protected = (await erasure.encode(manifest, 3, 2)).tryGet()
builder = SlotsBuilder.new(localStore, protected).tryGet()
builder = Poseidon2Builder.new(localStore, protected).tryGet()
verifiable = (await builder.buildManifest()).tryGet()
verifiableBlock = bt.Block.new(
verifiable.encode().tryGet(),
@ -127,9 +128,12 @@ asyncchecksuite "Test Node - Host contracts":
return success()
(await onStore(request, 1.u256, onBlocks)).tryGet()
check fetchedBytes == 786432
check fetchedBytes == 851968
for index in builder.slotIndicies(1):
let indexer = verifiable.protectedStrategy.init(
0, verifiable.blocksCount - 1, verifiable.numSlots)
for index in indexer.getIndicies(1):
let
blk = (await localStore.getBlock(verifiable.treeCid, index)).tryGet
expiryKey = (createBlockExpirationMetadataKey(blk.cid)).tryGet

View File

@ -126,7 +126,7 @@ asyncchecksuite "Test Node - Basic":
codec = ManifestCodec).tryGet()
protected = (await erasure.encode(manifest, 3, 2)).tryGet()
builder = SlotsBuilder.new(localStore, protected).tryGet()
builder = Poseidon2Builder.new(localStore, protected).tryGet()
verifiable = (await builder.buildManifest()).tryGet()
verifiableBlock = bt.Block.new(
verifiable.encode().tryGet(),

View File

@ -0,0 +1,143 @@
import std/sequtils
import std/sugar
import std/strutils
import pkg/poseidon2
import pkg/constantine/math/arithmetic
import pkg/constantine/math/io/io_bigints
import pkg/constantine/math/io/io_fields
import pkg/codex/merkletree
import pkg/codex/slots
import pkg/codex/slots/types
import pkg/codex/utils/json
export types
func fromCircomData*[H](cellData: seq[byte]): seq[H] =
var
pos = 0
cellElms: seq[Bn254Fr]
while pos < cellData.len:
var
step = 32
offset = min(pos + step, cellData.len)
data = cellData[pos..<offset]
let ff = Bn254Fr.fromBytes(data.toArray32).get
cellElms.add(ff)
pos += data.len
cellElms
func toPublicInputs*[H](input: ProofInput[H]): PublicInputs[H] =
PublicInputs[H](
slotIndex: input.slotIndex,
datasetRoot: input.datasetRoot,
entropy: input.entropy
)
func toJsonDecimal*(big: BigInt[254]): string =
let s = big.toDecimal.strip( leading = true, trailing = false, chars = {'0'} )
if s.len == 0: "0" else: s
func toJson*[H](input: ProofInput[H]): JsonNode =
var
input = input
%* {
"dataSetRoot": input.datasetRoot.toBig.toJsonDecimal,
"entropy": input.entropy.toBig.toJsonDecimal,
"nCellsPerSlot": input.nCellsPerSlot,
"nSlotsPerDataSet": input.nSlotsPerDataSet,
"slotIndex": input.slotIndex,
"slotRoot": input.slotRoot.toDecimal,
"slotProof": input.slotProof.mapIt( it.toBig.toJsonDecimal ),
"cellData": input.samples.mapIt(
toSeq( it.cellData.elements(H) ).mapIt( it.toBig.toJsonDecimal )
),
"merklePaths": input.samples.mapIt(
it.merklePaths.mapIt( it.toBig.toJsonDecimal )
)
}
func jsonToProofInput*[H](inputJson: JsonNode): ProofInput[H] =
let
cellData =
inputJson["cellData"].mapIt(
it.mapIt(
block:
var
big: BigInt[256]
data = newSeq[byte](big.bits div 8)
assert bool(big.fromDecimal( it.str ))
data.marshal(big, littleEndian)
data
).concat # flatten out elements
)
merklePaths =
inputJson["merklePaths"].mapIt(
it.mapIt(
block:
var
big: BigInt[254]
hash: H
assert bool(big.fromDecimal( it.getStr ))
hash.fromBig( big )
hash
)
)
slotProof = inputJson["slotProof"].mapIt(
block:
var
big: BigInt[254]
hash: H
assert bool(big.fromDecimal( it.str ))
hash.fromBig( big )
hash
)
datasetRoot = block:
var
big: BigInt[254]
hash: H
assert bool(big.fromDecimal( inputJson["dataSetRoot"].str ))
hash.fromBig( big )
hash
slotRoot = block:
var
big: BigInt[254]
hash: H
assert bool(big.fromDecimal( inputJson["slotRoot"].str ))
hash.fromBig( big )
hash
entropy = block:
var
big: BigInt[254]
hash: H
assert bool(big.fromDecimal( inputJson["entropy"].str ))
hash.fromBig( big )
hash
nCellsPerSlot = inputJson["nCellsPerSlot"].getInt
nSlotsPerDataSet = inputJson["nSlotsPerDataSet"].getInt
slotIndex = inputJson["slotIndex"].getInt
ProofInput[H](
entropy: entropy,
slotIndex: slotIndex,
datasetRoot: datasetRoot,
slotProof: slotProof,
slotRoot: slotRoot,
nCellsPerSlot: nCellsPerSlot,
nSlotsPerDataSet: nSlotsPerDataSet,
samples: zip(cellData, merklePaths)
.mapIt(Sample[H](
cellData: it[0],
merklePaths: it[1]
))
)

View File

@ -0,0 +1,166 @@
import std/sugar
import pkg/chronos
import pkg/libp2p/cid
import pkg/codex/codextypes
import pkg/codex/stores
import pkg/codex/merkletree
import pkg/codex/manifest
import pkg/codex/blocktype as bt
import pkg/codex/chunker
import pkg/codex/indexingstrategy
import pkg/codex/slots
import pkg/codex/rng
import pkg/codex/utils/poseidon2digest
import ../helpers
proc storeManifest*(store: BlockStore, manifest: Manifest): Future[?!bt.Block] {.async.} =
without encodedVerifiable =? manifest.encode(), err:
trace "Unable to encode manifest"
return failure(err)
without blk =? bt.Block.new(data = encodedVerifiable, codec = ManifestCodec), error:
trace "Unable to create block from manifest"
return failure(error)
if err =? (await store.putBlock(blk)).errorOption:
trace "Unable to store manifest block", cid = blk.cid, err = err.msg
return failure(err)
success blk
proc makeManifest*(
cids: seq[Cid],
datasetSize: NBytes,
blockSize: NBytes,
store: BlockStore,
hcodec = Sha256HashCodec,
dataCodec = BlockCodec): Future[?!Manifest] {.async.} =
without tree =? CodexTree.init(cids), err:
return failure(err)
without treeCid =? tree.rootCid(CIDv1, dataCodec), err:
return failure(err)
for index, cid in cids:
without proof =? tree.getProof(index), err:
return failure(err)
if err =? (await store.putCidAndProof(treeCid, index, cid, proof)).errorOption:
# TODO add log here
return failure(err)
let
manifest = Manifest.new(
treeCid = treeCid,
blockSize = blockSize,
datasetSize = datasetSize,
version = CIDv1,
hcodec = hcodec,
codec = dataCodec)
without manifestBlk =? await store.storeManifest(manifest), err:
trace "Unable to store manifest"
return failure(err)
success manifest
proc createBlocks*(
chunker: Chunker,
store: BlockStore): Future[seq[bt.Block]] {.async.} =
collect(newSeq):
while (let chunk = await chunker.getBytes(); chunk.len > 0):
let blk = bt.Block.new(chunk).tryGet()
discard await store.putBlock(blk)
blk
proc createProtectedManifest*(
datasetBlocks: seq[bt.Block],
store: BlockStore,
numDatasetBlocks: int,
ecK: int, ecM: int,
blockSize: NBytes,
originalDatasetSize: int,
totalDatasetSize: int):
Future[tuple[manifest: Manifest, protected: Manifest]] {.async.} =
let
cids = datasetBlocks.mapIt(it.cid)
datasetTree = CodexTree.init(cids[0..<numDatasetBlocks]).tryGet()
datasetTreeCid = datasetTree.rootCid().tryGet()
protectedTree = CodexTree.init(cids).tryGet()
protectedTreeCid = protectedTree.rootCid().tryGet()
for index, cid in cids[0..<numDatasetBlocks]:
let proof = datasetTree.getProof(index).tryget()
(await store.putCidAndProof(datasetTreeCid, index, cid, proof)).tryGet
for index, cid in cids:
let proof = protectedTree.getProof(index).tryget()
(await store.putCidAndProof(protectedTreeCid, index, cid, proof)).tryGet
let
manifest = Manifest.new(
treeCid = datasetTreeCid,
blockSize = blockSize,
datasetSize = originalDatasetSize.NBytes)
protectedManifest = Manifest.new(
manifest = manifest,
treeCid = protectedTreeCid,
datasetSize = totalDatasetSize.NBytes,
ecK = ecK,
ecM = ecM,
strategy = SteppedStrategy)
manifestBlock = bt.Block.new(
manifest.encode().tryGet(),
codec = ManifestCodec).tryGet()
protectedManifestBlock = bt.Block.new(
protectedManifest.encode().tryGet(),
codec = ManifestCodec).tryGet()
(await store.putBlock(manifestBlock)).tryGet()
(await store.putBlock(protectedManifestBlock)).tryGet()
(manifest, protectedManifest)
proc createVerifiableManifest*(
store: BlockStore,
numDatasetBlocks: int,
ecK: int, ecM: int,
blockSize: NBytes,
cellSize: NBytes):
Future[tuple[manifest: Manifest, protected: Manifest, verifiable: Manifest]] {.async.} =
let
numSlots = ecK + ecM
numTotalBlocks = calcEcBlocksCount(numDatasetBlocks, ecK, ecM) # total number of blocks in the dataset after
# EC (should will match number of slots)
originalDatasetSize = numDatasetBlocks * blockSize.int
totalDatasetSize = numTotalBlocks * blockSize.int
chunker = RandomChunker.new(Rng.instance(), size = totalDatasetSize, chunkSize = blockSize)
datasetBlocks = await chunker.createBlocks(store)
(manifest, protectedManifest) =
await createProtectedManifest(
datasetBlocks,
store,
numDatasetBlocks,
ecK, ecM,
blockSize,
originalDatasetSize,
totalDatasetSize)
builder = Poseidon2Builder.new(store, protectedManifest, cellSize = cellSize).tryGet
verifiableManifest = (await builder.buildManifest()).tryGet
# build the slots and manifest
(manifest, protectedManifest, verifiableManifest)

View File

@ -1,194 +0,0 @@
import std/sequtils
import std/math
import pkg/questionable/results
import pkg/poseidon2/io
import pkg/poseidon2
import pkg/chronos
import pkg/codex/stores/cachestore
import pkg/codex/chunker
import pkg/codex/stores
import pkg/codex/blocktype as bt
import pkg/codex/contracts/requests
import pkg/codex/contracts
import pkg/codex/merkletree
import pkg/codex/stores/cachestore
import pkg/codex/indexingstrategy
import pkg/codex/slots/converters
import pkg/codex/slots/builder/builder
import pkg/codex/utils/poseidon2digest
import pkg/codex/utils/asynciter
import ../helpers
import ../merkletree/helpers
const
# The number of slot blocks and number of slots, combined with
# the bytes per block, make it so that there are exactly 256 cells
# in the dataset.
bytesPerBlock* = 64 * 1024
cellsPerBlock* = bytesPerBlock div DefaultCellSize.int
numberOfSlotBlocks* = 3
numberOfSlotBlocksPadded* = numberOfSlotBlocks.nextPowerOfTwo
totalNumberOfSlots* = 2
datasetSlotIndex* = 1
cellsPerSlot* = (bytesPerBlock * numberOfSlotBlocks) div DefaultCellSize.int
totalNumCells = ((numberOfSlotBlocks * totalNumberOfSlots * bytesPerBlock) div DefaultCellSize.int)
type
ProvingTestEnvironment* = ref object
# Invariant:
# These challenges are chosen such that with the testenv default values
# and nSamples=3, they will land on [3x data cells + 0x padded cell],
# and [2x data cells + 1x padded cell] respectively:
challengeNoPad*: Poseidon2Hash
challengeOnePad*: Poseidon2Hash
blockPadBytes*: seq[byte]
emptyBlockTree*: Poseidon2Tree
emptyBlockCid*: Cid
# Variant:
localStore*: CacheStore
manifest*: Manifest
manifestBlock*: bt.Block
slot*: Slot
datasetBlocks*: seq[bt.Block]
slotTree*: Poseidon2Tree
slotRootCid*: Cid
slotRoots*: seq[Poseidon2Hash]
datasetToSlotTree*: Poseidon2Tree
datasetRootHash*: Poseidon2Hash
proc createDatasetBlocks(self: ProvingTestEnvironment): Future[void] {.async.} =
var data: seq[byte] = @[]
# This generates a number of blocks that have different data, such that
# Each cell in each block is unique, but nothing is random.
for i in 0 ..< totalNumCells:
data = data & (i.byte).repeat(DefaultCellSize.uint64)
let chunker = MockChunker.new(
dataset = data,
chunkSize = bytesPerBlock)
while true:
let chunk = await chunker.getBytes()
if chunk.len <= 0:
break
let b = bt.Block.new(chunk).tryGet()
self.datasetBlocks.add(b)
discard await self.localStore.putBlock(b)
proc createSlotTree(self: ProvingTestEnvironment, dSlotIndex: uint64): Future[Poseidon2Tree] {.async.} =
let
slotSize = (bytesPerBlock * numberOfSlotBlocks).uint64
blocksInSlot = slotSize div bytesPerBlock.uint64
datasetBlockIndexingStrategy = SteppedStrategy.init(0, self.datasetBlocks.len - 1, totalNumberOfSlots)
datasetBlockIndices = toSeq(datasetBlockIndexingStrategy.getIndicies(dSlotIndex.int))
let
slotBlocks = datasetBlockIndices.mapIt(self.datasetBlocks[it])
slotBlockRoots = slotBlocks.mapIt(Poseidon2Tree.digest(it.data & self.blockPadBytes, DefaultCellSize.int).tryGet())
slotBlockRootPads = newSeqWith((slotBlockRoots.len).nextPowerOfTwoPad, Poseidon2Zero)
tree = Poseidon2Tree.init(slotBlockRoots & slotBlockRootPads).tryGet()
treeCid = tree.root().tryGet().toSlotCid().tryGet()
for i in 0 ..< numberOfSlotBlocksPadded:
var blkCid: Cid
if i < slotBlockRoots.len:
blkCid = slotBlockRoots[i].toCellCid().tryGet()
else:
blkCid = self.emptyBlockCid
let proof = tree.getProof(i).tryGet().toEncodableProof().tryGet()
discard await self.localStore.putCidAndProof(treeCid, i, blkCid, proof)
return tree
proc createDatasetRootHashAndSlotTree(self: ProvingTestEnvironment): Future[void] {.async.} =
var slotTrees = newSeq[Poseidon2Tree]()
for i in 0 ..< totalNumberOfSlots:
slotTrees.add(await self.createSlotTree(i.uint64))
self.slotTree = slotTrees[datasetSlotIndex]
self.slotRootCid = slotTrees[datasetSlotIndex].root().tryGet().toSlotCid().tryGet()
self.slotRoots = slotTrees.mapIt(it.root().tryGet())
let rootsPadLeafs = newSeqWith(totalNumberOfSlots.nextPowerOfTwoPad, Poseidon2Zero)
self.datasetToSlotTree = Poseidon2Tree.init(self.slotRoots & rootsPadLeafs).tryGet()
self.datasetRootHash = self.datasetToSlotTree.root().tryGet()
proc createManifest(self: ProvingTestEnvironment): Future[void] {.async.} =
let
cids = self.datasetBlocks.mapIt(it.cid)
tree = CodexTree.init(cids).tryGet()
treeCid = tree.rootCid(CIDv1, BlockCodec).tryGet()
for i in 0 ..< self.datasetBlocks.len:
let
blk = self.datasetBlocks[i]
leafCid = blk.cid
proof = tree.getProof(i).tryGet()
discard await self.localStore.putBlock(blk)
discard await self.localStore.putCidAndProof(treeCid, i, leafCid, proof)
# Basic manifest:
self.manifest = Manifest.new(
treeCid = treeCid,
blockSize = bytesPerBlock.NBytes,
datasetSize = (bytesPerBlock * numberOfSlotBlocks * totalNumberOfSlots).NBytes)
# Protected manifest:
self.manifest = Manifest.new(
manifest = self.manifest,
treeCid = treeCid,
datasetSize = self.manifest.datasetSize,
ecK = totalNumberOfSlots,
ecM = 0,
strategy = StrategyType.SteppedStrategy
)
# Verifiable manifest:
self.manifest = Manifest.new(
manifest = self.manifest,
verifyRoot = self.datasetRootHash.toVerifyCid().tryGet(),
slotRoots = self.slotRoots.mapIt(it.toSlotCid().tryGet())
).tryGet()
self.manifestBlock = bt.Block.new(self.manifest.encode().tryGet(), codec = ManifestCodec).tryGet()
discard await self.localStore.putBlock(self.manifestBlock)
proc createSlot(self: ProvingTestEnvironment): void =
self.slot = Slot(
request: StorageRequest(
ask: StorageAsk(
slots: totalNumberOfSlots.uint64,
slotSize: u256(bytesPerBlock * numberOfSlotBlocks)
),
content: StorageContent(
cid: $self.manifestBlock.cid
),
),
slotIndex: u256(datasetSlotIndex)
)
proc createProvingTestEnvironment*(): Future[ProvingTestEnvironment] {.async.} =
let
numBlockCells = bytesPerBlock.int div DefaultCellSize.int
blockPadBytes = newSeq[byte](numBlockCells.nextPowerOfTwoPad * DefaultCellSize.int)
emptyBlockTree = Poseidon2Tree.digestTree(DefaultEmptyBlock & blockPadBytes, DefaultCellSize.int).tryGet()
emptyBlockCid = emptyBlockTree.root.tryGet().toCellCid().tryGet()
var testEnv = ProvingTestEnvironment(
challengeNoPad: toF(6),
challengeOnePad: toF(9),
blockPadBytes: blockPadBytes,
emptyBlockTree: emptyBlockTree,
emptyBlockCid: emptyBlockCid
)
testEnv.localStore = CacheStore.new()
await testEnv.createDatasetBlocks()
await testEnv.createDatasetRootHashAndSlotTree()
await testEnv.createManifest()
testEnv.createSlot()
return testEnv

View File

@ -0,0 +1,173 @@
import std/sequtils
import std/options
import std/importutils
import ../../../asynctest
import pkg/questionable
import pkg/questionable/results
import pkg/datastore
import pkg/codex/rng
import pkg/codex/stores
import pkg/codex/merkletree
import pkg/codex/utils/json
import pkg/codex/codextypes
import pkg/codex/slots
import pkg/codex/slots/builder
import pkg/codex/utils/poseidon2digest
import pkg/codex/slots/sampler/utils
import pkg/constantine/math/arithmetic
import pkg/constantine/math/io/io_bigints
import pkg/constantine/math/io/io_fields
import ../backends/helpers
import ../helpers
import ../../helpers
suite "Test control samples":
var
inputData: string
inputJson: JsonNode
proofInput: ProofInput[Poseidon2Hash]
setup:
inputData = readFile("tests/circuits/fixtures/input.json")
inputJson = parseJson(inputData)
proofInput = jsonToProofInput[Poseidon2Hash](inputJson)
test "Should verify control samples":
let
blockCells = 32
cellIdxs = proofInput.entropy.cellIndices(proofInput.slotRoot, proofInput.nCellsPerSlot, 5)
for i, cellIdx in cellIdxs:
let
sample = proofInput.samples[i]
cellIdx = cellIdxs[i]
cellProof = Poseidon2Proof.init(
cellIdx.toCellInBlk(blockCells),
proofInput.nCellsPerSlot,
sample.merklePaths[0..<5]).tryGet
slotProof = Poseidon2Proof.init(
cellIdx.toBlkInSlot(blockCells),
proofInput.nCellsPerSlot,
sample.merklePaths[5..<9]).tryGet
cellData = fromCircomData[Poseidon2Hash](sample.cellData)
cellLeaf = Poseidon2Hash.spongeDigest(cellData, rate = 2).tryGet
slotLeaf = cellProof.reconstructRoot(cellLeaf).tryGet
check slotProof.verify(slotLeaf, proofInput.slotRoot).tryGet
test "Should verify control dataset root":
let
datasetProof = Poseidon2Proof.init(
proofInput.slotIndex,
proofInput.nSlotsPerDataSet,
proofInput.slotProof[0..<4]).tryGet
check datasetProof.verify(proofInput.slotRoot, proofInput.datasetRoot).tryGet
suite "Test sampler samples":
let
slotIndex = 3
nSamples = 5
ecK = 3
ecM = 2
datasetBlocks = 8
entropy = 1234567.toF
blockSize = DefaultBlockSize
cellSize = DefaultCellSize
var
store: RepoStore
builder: Poseidon2Builder
manifest: Manifest
protected: Manifest
verifiable: Manifest
setup:
let
repoDs = SQLiteDatastore.new(Memory).tryGet()
metaDs = SQLiteDatastore.new(Memory).tryGet()
store = RepoStore.new(repoDs, metaDs)
(manifest, protected, verifiable) =
await createVerifiableManifest(
store,
datasetBlocks,
ecK, ecM,
blockSize,
cellSize)
# create sampler
builder = Poseidon2Builder.new(store, verifiable).tryGet
teardown:
await store.close()
test "Should fail instantiating for invalid slot index":
let
sampler = Poseidon2Sampler.new(builder.slotRoots.len, store, builder)
check sampler.isErr
test "Should fail instantiating for non verifiable builder":
let
nonVerifiableBuilder = Poseidon2Builder.new(store, protected).tryGet
sampler = Poseidon2Sampler.new(slotIndex, store, nonVerifiableBuilder)
check sampler.isErr
test "Should verify samples":
let
sampler = Poseidon2Sampler.new(slotIndex, store, builder).tryGet
verifyTree = builder.verifyTree.get # get the dataset tree
slotProof = verifyTree.getProof(slotIndex).tryGet # get slot proof for index
datasetRoot = verifyTree.root().tryGet # get dataset root
slotTreeCid = verifiable.slotRoots[slotIndex] # get slot tree cid to retrieve proof from storage
slotRoot = builder.slotRoots[slotIndex] # get slot root hash
cellIdxs = entropy.cellIndices(slotRoot, builder.numSlotCells, nSamples)
nBlockCells = builder.numBlockCells
nSlotCells = builder.numSlotCells
for i, cellIdx in cellIdxs:
let
sample = (await sampler.getSample(cellIdx, slotTreeCid, slotRoot)).tryGet
cellProof = Poseidon2Proof.init(
cellIdx.toCellInBlk(nBlockCells),
nSlotCells,
sample.merklePaths[0..<5]).tryGet
slotProof = Poseidon2Proof.init(
cellIdx.toBlkInSlot(nBlockCells),
nSlotCells,
sample.merklePaths[5..<sample.merklePaths.len]).tryGet
cellData = fromCircomData[Poseidon2Hash](sample.cellData)
cellLeaf = Poseidon2Hash.spongeDigest(cellData, rate = 2).tryGet
slotLeaf = cellProof.reconstructRoot(cellLeaf).tryGet
check slotProof.verify(slotLeaf, slotRoot).tryGet
test "Should verify dataset root":
let
sampler = Poseidon2Sampler.new(slotIndex, store, builder).tryGet
proofInput = (await sampler.getProofInput(entropy.toBytes.toArray32, nSamples)).tryGet
datasetProof = Poseidon2Proof.init(
proofInput.slotIndex,
builder.slotRoots.len,
proofInput.slotProof).tryGet
check datasetProof.verify(builder.slotRoots[slotIndex], builder.verifyRoot.get).tryGet

View File

@ -2,7 +2,8 @@ import std/sequtils
import std/sugar
import std/random
import std/strutils
import std/math
import ../../../asynctest
import pkg/questionable/results
import pkg/constantine/math/arithmetic
@ -10,7 +11,6 @@ import pkg/constantine/math/io/io_fields
import pkg/poseidon2/io
import pkg/poseidon2
import pkg/chronos
import pkg/codex/stores/cachestore
import pkg/codex/chunker
import pkg/codex/stores
import pkg/codex/blocktype as bt
@ -19,31 +19,27 @@ import pkg/codex/contracts
import pkg/codex/merkletree
import pkg/codex/stores/cachestore
import pkg/codex/slots/types
import pkg/codex/slots/sampler/utils
import ../../asynctest
import ../helpers
import ../examples
import ../merkletree/helpers
import ./provingtestenv
import ../backends/helpers
import ../../helpers
import ../../examples
import ../../merkletree/helpers
asyncchecksuite "Test proof sampler utils":
let knownIndices: seq[Natural] = @[57, 82, 49]
let
cellsPerBlock = DefaultBlockSize div DefaultCellSize
var
env: ProvingTestEnvironment
slotRoot: Poseidon2Hash
numCells: Natural
numCellsPadded: Natural
inputData: string
inputJson: JsonNode
proofInput: ProofInput[Poseidon2Hash]
setup:
env = await createProvingTestEnvironment()
slotRoot = env.slotRoots[datasetSlotIndex]
numCells = cellsPerSlot
numCellsPadded = numCells.nextPowerOfTwo
teardown:
reset(env)
inputData = readFile("tests/circuits/fixtures/input.json")
inputJson = parseJson(inputData)
proofInput = jsonToProofInput[Poseidon2Hash](inputJson)
test "Extract low bits":
proc extract(value: uint64, nBits: int): uint64 =
@ -64,45 +60,39 @@ asyncchecksuite "Test proof sampler utils":
test "Can find single slot-cell index":
proc slotCellIndex(i: Natural): Natural =
return cellIndex(env.challengeNoPad, slotRoot, numCellsPadded, i)
return cellIndex(proofInput.entropy, proofInput.slotRoot, proofInput.nCellsPerSlot, i)
proc getExpectedIndex(i: int): Natural =
let
numberOfCellsInSlot = (bytesPerBlock * numberOfSlotBlocks) div DefaultCellSize.uint64.int
numberOfCellsInSlotPadded = numberOfCellsInSlot.nextPowerOfTwo
hash = Sponge.digest(@[slotRoot, env.challengeNoPad, toF(i)], rate = 2)
hash = Sponge.digest(@[proofInput.entropy, proofInput.slotRoot, toF(i)], rate = 2)
return int(extractLowBits(hash.toBig(), ceilingLog2(numberOfCellsInSlotPadded)))
return int(extractLowBits(hash.toBig(), ceilingLog2(proofInput.nCellsPerSlot)))
check:
slotCellIndex(1) == getExpectedIndex(1)
slotCellIndex(1) == knownIndices[0]
slotCellIndex(2) == getExpectedIndex(2)
slotCellIndex(2) == knownIndices[1]
slotCellIndex(3) == getExpectedIndex(3)
slotCellIndex(3) == knownIndices[2]
test "Can find sequence of slot-cell indices":
proc slotCellIndices(n: int): seq[Natural] =
cellIndices(env.challengeNoPad, slotRoot, numCellsPadded, n)
cellIndices(proofInput.entropy, proofInput.slotRoot, numCells = proofInput.nCellsPerSlot, n)
proc getExpectedIndices(n: int): seq[Natural] =
return collect(newSeq, (for i in 1..n: cellIndex(env.challengeNoPad, slotRoot, numCellsPadded, i)))
return collect(newSeq, (for i in 1..n: cellIndex(proofInput.entropy, proofInput.slotRoot, proofInput.nCellsPerSlot, i)))
check:
slotCellIndices(3) == getExpectedIndices(3)
slotCellIndices(3) == knownIndices
for (input, expected) in [(10, 0), (31, 0), (32, 1), (63, 1), (64, 2)]:
test "Can get slotBlockIndex from slotCellIndex (" & $input & " -> " & $expected & ")":
let slotBlockIndex = toBlockIdx(input, numCells = 32)
let slotBlockIndex = toBlkInSlot(input, numCells = cellsPerBlock)
check:
slotBlockIndex == expected
for (input, expected) in [(10, 10), (31, 31), (32, 0), (63, 31), (64, 0)]:
test "Can get blockCellIndex from slotCellIndex (" & $input & " -> " & $expected & ")":
let blockCellIndex = toBlockCellIdx(input, numCells = 32)
let blockCellIndex = toCellInBlk(input, numCells = cellsPerBlock)
check:
blockCellIndex == expected

View File

@ -1,155 +1,4 @@
import std/sequtils
import std/sugar
import std/random
import std/strutils
import ./sampler/testsampler
import ./sampler/testutils
import pkg/questionable/results
import pkg/constantine/math/arithmetic
import pkg/constantine/math/io/io_fields
import pkg/poseidon2/io
import pkg/poseidon2
import pkg/chronos
import pkg/nimcrypto
import pkg/codex/stores/cachestore
import pkg/codex/chunker
import pkg/codex/stores
import pkg/codex/blocktype as bt
import pkg/codex/contracts/requests
import pkg/codex/contracts
import pkg/codex/merkletree
import pkg/codex/stores/cachestore
import pkg/codex/slots/sampler
import pkg/codex/slots/builder/builder
import ../helpers
import ../examples
import ../merkletree/helpers
import ../../asynctest
import ./testsampler_expected
import ./provingtestenv
asyncchecksuite "Test DataSampler":
var
env: ProvingTestEnvironment
dataSampler: DataSampler
blk: bt.Block
cell0Bytes: seq[byte]
cell1Bytes: seq[byte]
cell2Bytes: seq[byte]
proc createDataSampler(): Future[void] {.async.} =
dataSampler = DataSampler.new(
datasetSlotIndex,
env.localStore,
SlotsBuilder.new(env.localStore, env.manifest).tryGet()).tryGet()
setup:
randomize()
env = await createProvingTestEnvironment()
let bytes = newSeqWith(bytesPerBlock, rand(uint8))
blk = bt.Block.new(bytes).tryGet()
cell0Bytes = bytes[0..<DefaultCellSize.uint64]
cell1Bytes = bytes[DefaultCellSize.uint64..<(DefaultCellSize.uint64*2)]
cell2Bytes = bytes[(DefaultCellSize.uint64*2)..<(DefaultCellSize.uint64*3)]
await createDataSampler()
teardown:
reset(env)
reset(dataSampler)
test "Can get cell from block":
let
sample0 = dataSampler.getCell(blk.data, 0)
sample1 = dataSampler.getCell(blk.data, 1)
sample2 = dataSampler.getCell(blk.data, 2)
check:
sample0 == cell0Bytes
sample1 == cell1Bytes
sample2 == cell2Bytes
test "Can gather proof input":
let
nSamples = 3
challengeBytes = env.challengeNoPad.toBytes()
input = (await dataSampler.getProofInput(challengeBytes, nSamples)).tryget()
proc equal(a: Poseidon2Hash, b: Poseidon2Hash): bool =
a.toDecimal() == b.toDecimal()
proc toStr(proof: Poseidon2Proof): string =
let a = proof.path.mapIt(toHex(it))
join(a)
let
expectedBlockSlotProofs = getExpectedBlockSlotProofs()
expectedCellBlockProofs = getExpectedCellBlockProofs()
expectedCellData = getExpectedCellData()
expectedProof = env.datasetToSlotTree.getProof(datasetSlotIndex).tryGet()
check:
equal(input.verifyRoot, env.datasetRootHash)
equal(input.entropy, env.challengeNoPad)
input.numCells == ((bytesPerBlock * numberOfSlotBlocks) div DefaultCellSize.int).Natural
input.numSlots == totalNumberOfSlots.Natural
input.slotIndex == env.slot.slotIndex.truncate(Natural)
input.verifyProof == expectedProof
# block-slot proofs
input.samples[0].slotBlockIdx == 1
input.samples[1].slotBlockIdx == 2
input.samples[2].slotBlockIdx == 1
toStr(input.samples[0].slotProof) == expectedBlockSlotProofs[0]
toStr(input.samples[1].slotProof) == expectedBlockSlotProofs[1]
toStr(input.samples[2].slotProof) == expectedBlockSlotProofs[2]
# cell-block proofs
input.samples[0].blockCellIdx == 25
input.samples[1].blockCellIdx == 18
input.samples[2].blockCellIdx == 17
toStr(input.samples[0].cellProof) == expectedCellBlockProofs[0]
toStr(input.samples[1].cellProof) == expectedCellBlockProofs[1]
toStr(input.samples[2].cellProof) == expectedCellBlockProofs[2]
# cell data
nimcrypto.toHex(input.samples[0].data) == expectedCellData[0]
nimcrypto.toHex(input.samples[1].data) == expectedCellData[1]
nimcrypto.toHex(input.samples[2].data) == expectedCellData[2]
test "Can select samples in padded cells":
let
nSamples = 3
challengeBytes = env.challengeOnePad.toBytes()
input = (await dataSampler.getProofInput(challengeBytes, nSamples)).tryget()
proc equal(a: Poseidon2Hash, b: Poseidon2Hash): bool =
a.toDecimal() == b.toDecimal()
proc toStr(proof: Poseidon2Proof): string =
let a = proof.path.mapIt(toHex(it))
join(a)
let expectedProof = env.datasetToSlotTree.getProof(datasetSlotIndex).tryGet()
check:
equal(input.verifyRoot, env.datasetRootHash)
equal(input.entropy, env.challengeOnePad)
input.numCells == ((bytesPerBlock * numberOfSlotBlocks) div DefaultCellSize.int).Natural
input.numSlots == totalNumberOfSlots.Natural
input.slotIndex == env.slot.slotIndex.truncate(Natural)
input.verifyProof == expectedProof
input.samples[0].slotBlockIdx == 2
input.samples[1].slotBlockIdx == 2
input.samples[2].slotBlockIdx == 3
# The third sample is a padded sample
toStr(input.samples[2].slotProof) == toStr(env.slotTree.getProof(3).tryGet())
input.samples[0].blockCellIdx == 29
input.samples[1].blockCellIdx == 26
input.samples[2].blockCellIdx == 30
toStr(input.samples[2].cellProof) == toStr(env.emptyBlockTree.getProof(30).tryGet())
input.samples[2].data == newSeq[byte](DefaultCellSize.int)
{.warning[UnusedImport]: off.}

View File

@ -1,25 +0,0 @@
# Snapshot of expected values for testsampler.
import std/strutils
import pkg/codex/codextypes
proc getExpectedCellBlockProofs*(): seq[string] =
@[
"0x13935d7f73028fd425e557b8e0b737fcba3f8bcda0e07a9047868bc7c5a2519e0x17d36c9bcef9febb77b129e8bbe971b754d7b3e1df8c7d881b64bf0aa055f5570x02610a45fa39d4859e5c0c53bda9cc6bd627ae4b5aa462f9e86f47f3db96b2700x27a362c0f92c887d74ee77df2df5e51f0f66b0564a2af8b1c11b02dab0a34f780x095eb3c0d166c19f9cac8ea0e6ba274dfeef802cf7d01c90378585fd4d788e56",
"0x128b43cb1f87736d5b826c5a440d373c0e4c147a81a0edc40b106abd2c10af960x2160ba1c1ed893685b849a8f1be6d32bb11a6ebb8690c5af95e0ce0e9d72b6580x1332185d0ef7b76e55c7464f37df96d41f256249d74678d8191030912036b7aa0x0522383db3f9cb9c75d5b209bd795566e9cffa632819628194416a479998a4270x1decd3fb7ff458261149731115657cecd7eb2fe4a6cf84f3c6761aa8b0dd6b9a",
"0x2a2c8663e354dbc1857a654134de3b690d73ffedcc7c0ba770a39f3e48675ee10x01914d98d4103889ea59ce6e970dae95f7e463350184c5f7f3806997e10748520x10e8f6235ac7e95af5aea6fc77ebe60e1028b48eb84027ea46d403d809852b6b0x04bd51be7ff42db27a7605124d3d2d1c4cbca43f93d1997060ac8a1009fa48b40x095eb3c0d166c19f9cac8ea0e6ba274dfeef802cf7d01c90378585fd4d788e56"
]
proc getExpectedBlockSlotProofs*(): seq[string] =
@[
"0x008fccffc5d2e5010a44bc718fed80d4d891e6517c022ba1bff65763dd79dbb90x117662f365386ee6784274dfa3a3b35126291cdf013c5e90d07bee79f32b2f40",
"0x00000000000000000000000000000000000000000000000000000000000000000x2a66917fa49371e835376fcece0d854c77008ac1195740963b1ac4491ee1aaf1",
"0x008fccffc5d2e5010a44bc718fed80d4d891e6517c022ba1bff65763dd79dbb90x117662f365386ee6784274dfa3a3b35126291cdf013c5e90d07bee79f32b2f40"
]
proc getExpectedCellData*(): seq[string] =
@[
"79".repeat(DefaultCellSize.int),
"B2".repeat(DefaultCellSize.int),
"71".repeat(DefaultCellSize.int)
]

View File

@ -3,6 +3,8 @@ import std/math
import std/importutils
import std/sugar
import ../../asynctest
import pkg/chronos
import pkg/questionable/results
import pkg/codex/blocktype as bt
@ -13,12 +15,13 @@ import pkg/codex/merkletree
import pkg/codex/manifest {.all.}
import pkg/codex/utils
import pkg/codex/utils/digest
import pkg/codex/utils/poseidon2digest
import pkg/datastore
import pkg/poseidon2
import pkg/poseidon2/io
import constantine/math/io/io_fields
import pkg/constantine/math/io/io_fields
import ../../asynctest
import ./helpers
import ../helpers
import ../examples
import ../merkletree/helpers
@ -26,9 +29,12 @@ import ../merkletree/helpers
import pkg/codex/indexingstrategy {.all.}
import pkg/codex/slots {.all.}
privateAccess(SlotsBuilder) # enable access to private fields
privateAccess(Poseidon2Builder) # enable access to private fields
privateAccess(Manifest) # enable access to private fields
const
Strategy = SteppedStrategy
suite "Slot builder":
let
blockSize = NBytes 1024
@ -37,98 +43,49 @@ suite "Slot builder":
ecM = 2
numSlots = ecK + ecM
numDatasetBlocks = 100
numBlockCells = (blockSize div cellSize).int
numDatasetBlocks = 50
numTotalBlocks = calcEcBlocksCount(numDatasetBlocks, ecK, ecM) # total number of blocks in the dataset after
# EC (should will match number of slots)
originalDatasetSize = numDatasetBlocks * blockSize.int
totalDatasetSize = numTotalBlocks * blockSize.int
numTotalBlocks = calcEcBlocksCount(numDatasetBlocks, ecK, ecM) # total number of blocks in the dataset after
# EC (should will match number of slots)
originalDatasetSize = numDatasetBlocks * blockSize.int # size of the dataset before EC
totalDatasetSize = numTotalBlocks * blockSize.int # size of the dataset after EC
numTotalSlotBlocks = nextPowerOfTwo(numTotalBlocks div numSlots)
numBlockCells = (blockSize div cellSize).int # number of cells per block
numSlotBlocks = numTotalBlocks div numSlots # number of blocks per slot
numSlotCells = numSlotBlocks * numBlockCells # number of uncorrected slot cells
pow2SlotCells = nextPowerOfTwo(numSlotCells) # pow2 cells per slot
numPadSlotBlocks = pow2SlotCells div numBlockCells # pow2 blocks per slot
numPadBlocksTotal = numPadSlotBlocks * numSlots # total number of pad blocks
blockPadBytes =
newSeq[byte](numBlockCells.nextPowerOfTwoPad * cellSize.int) # power of two padding for blocks
slotsPadLeafs =
newSeqWith((numTotalBlocks div numSlots).nextPowerOfTwoPad, Poseidon2Zero) # power of two padding for block roots
rootsPadLeafs =
newSeqWith(numSlots.nextPowerOfTwoPad, Poseidon2Zero)
# empty digest
emptyDigest = SpongeMerkle.digest(newSeq[byte](blockSize.int), cellSize.int)
var
datasetBlocks: seq[bt.Block]
padBlocks: seq[bt.Block]
localStore: BlockStore
manifest: Manifest
protectedManifest: Manifest
expectedEmptyCid: Cid
slotBuilder: SlotsBuilder
builder: Poseidon2Builder
chunker: Chunker
proc createBlocks(): Future[void] {.async.} =
while true:
let chunk = await chunker.getBytes()
if chunk.len <= 0:
break
let blk = bt.Block.new(chunk).tryGet()
datasetBlocks.add(blk)
discard await localStore.putBlock(blk)
proc createProtectedManifest(): Future[void] {.async.} =
let
cids = datasetBlocks.mapIt(it.cid)
datasetTree = CodexTree.init(cids[0..<numDatasetBlocks]).tryGet()
datasetTreeCid = datasetTree.rootCid().tryGet()
protectedTree = CodexTree.init(cids).tryGet()
protectedTreeCid = protectedTree.rootCid().tryGet()
for index, cid in cids[0..<numDatasetBlocks]:
let proof = datasetTree.getProof(index).tryget()
(await localStore.putCidAndProof(datasetTreeCid, index, cid, proof)).tryGet
for index, cid in cids:
let proof = protectedTree.getProof(index).tryget()
(await localStore.putCidAndProof(protectedTreeCid, index, cid, proof)).tryGet
manifest = Manifest.new(
treeCid = datasetTreeCid,
blockSize = blockSize.NBytes,
datasetSize = originalDatasetSize.NBytes)
protectedManifest = Manifest.new(
manifest = manifest,
treeCid = protectedTreeCid,
datasetSize = totalDatasetSize.NBytes,
ecK = ecK,
ecM = ecM,
strategy = StrategyType.SteppedStrategy)
let
manifestBlock = bt.Block.new(
manifest.encode().tryGet(),
codec = ManifestCodec).tryGet()
protectedManifestBlock = bt.Block.new(
protectedManifest.encode().tryGet(),
codec = ManifestCodec).tryGet()
(await localStore.putBlock(manifestBlock)).tryGet()
(await localStore.putBlock(protectedManifestBlock)).tryGet()
expectedEmptyCid = emptyCid(
protectedManifest.version,
protectedManifest.hcodec,
protectedManifest.codec).tryGet()
setup:
let
repoDs = SQLiteDatastore.new(Memory).tryGet()
metaDs = SQLiteDatastore.new(Memory).tryGet()
localStore = RepoStore.new(repoDs, metaDs)
localStore = RepoStore.new(repoDs, metaDs)
chunker = RandomChunker.new(Rng.instance(), size = totalDatasetSize, chunkSize = blockSize)
await createBlocks()
await createProtectedManifest()
datasetBlocks = await chunker.createBlocks(localStore)
(manifest, protectedManifest) =
await createProtectedManifest(
datasetBlocks,
localStore,
numDatasetBlocks,
ecK, ecM,
blockSize,
originalDatasetSize,
totalDatasetSize)
teardown:
await localStore.close()
@ -142,11 +99,10 @@ suite "Slot builder":
reset(localStore)
reset(manifest)
reset(protectedManifest)
reset(expectedEmptyCid)
reset(slotBuilder)
reset(builder)
reset(chunker)
test "Can only create slotBuilder with protected manifest":
test "Can only create builder with protected manifest":
let
unprotectedManifest = Manifest.new(
treeCid = Cid.example,
@ -154,8 +110,8 @@ suite "Slot builder":
datasetSize = originalDatasetSize.NBytes)
check:
SlotsBuilder.new(localStore, unprotectedManifest, cellSize = cellSize)
.error.msg == "Can only create SlotsBuilder using protected manifests."
Poseidon2Builder.new(localStore, unprotectedManifest, cellSize = cellSize)
.error.msg == "Manifest is not protected."
test "Number of blocks must be devisable by number of slots":
let
@ -168,10 +124,10 @@ suite "Slot builder":
datasetSize = totalDatasetSize.NBytes,
ecK = ecK - 1,
ecM = ecM,
strategy = StrategyType.SteppedStrategy)
strategy = Strategy)
check:
SlotsBuilder.new(localStore, mismatchManifest, cellSize = cellSize)
Poseidon2Builder.new(localStore, mismatchManifest, cellSize = cellSize)
.error.msg == "Number of blocks must be divisable by number of slots."
test "Block size must be divisable by cell size":
@ -185,146 +141,150 @@ suite "Slot builder":
datasetSize = (totalDatasetSize - 1).NBytes,
ecK = ecK,
ecM = ecM,
strategy = StrategyType.SteppedStrategy)
strategy = Strategy)
check:
SlotsBuilder.new(localStore, mismatchManifest, cellSize = cellSize)
Poseidon2Builder.new(localStore, mismatchManifest, cellSize = cellSize)
.error.msg == "Block size must be divisable by cell size."
test "Should build correct slot builder":
slotBuilder = SlotsBuilder.new(
builder = Poseidon2Builder.new(
localStore,
protectedManifest,
cellSize = cellSize).tryGet()
check:
slotBuilder.numBlockPadBytes == blockPadBytes.len
slotBuilder.numSlotsPadLeafs == slotsPadLeafs.len
slotBuilder.numRootsPadLeafs == rootsPadLeafs.len
builder.cellSize == cellSize
builder.numSlots == numSlots
builder.numBlockCells == numBlockCells
builder.numSlotBlocks == numPadSlotBlocks
builder.numSlotCells == pow2SlotCells
builder.numBlocks == numPadBlocksTotal
test "Should build slot hashes for all slots":
let
steppedStrategy = SteppedStrategy.init(0, numTotalBlocks - 1, numSlots)
slotBuilder = SlotsBuilder.new(
steppedStrategy = Strategy.init(
0, numPadBlocksTotal - 1, numSlots)
builder = Poseidon2Builder.new(
localStore,
protectedManifest,
cellSize = cellSize).tryGet()
for i in 0 ..< numSlots:
let
expectedBlock = steppedStrategy
.getIndicies(i)
.mapIt( datasetBlocks[it] )
# for i in 0..<numSlots:
let
expectedHashes = collect(newSeq):
for j, idx in steppedStrategy.getIndicies(0):
if j > (protectedManifest.numSlotBlocks - 1):
emptyDigest
else:
SpongeMerkle.digest(datasetBlocks[idx].data, cellSize.int)
expectedHashes: seq[Poseidon2Hash] = collect(newSeq):
for blk in expectedBlock:
SpongeMerkle.digest(blk.data & blockPadBytes, cellSize.int)
cellHashes = (await builder.getCellHashes(0)).tryGet()
cellHashes = (await slotBuilder.getBlockHashes(i)).tryGet()
check:
expectedHashes == cellHashes
check:
cellHashes.len == expectedHashes.len
cellHashes == expectedHashes
test "Should build slot trees for all slots":
let
steppedStrategy = SteppedStrategy.init(0, numTotalBlocks - 1, numSlots)
slotBuilder = SlotsBuilder.new(
steppedStrategy = Strategy.init(
0, numPadBlocksTotal - 1, numSlots)
builder = Poseidon2Builder.new(
localStore,
protectedManifest,
cellSize = cellSize).tryGet()
for i in 0 ..< numSlots:
for i in 0..<numSlots:
let
expectedBlock = steppedStrategy
.getIndicies(i)
.mapIt( datasetBlocks[it] )
expectedHashes = collect(newSeq):
for j, idx in steppedStrategy.getIndicies(i):
if j > (protectedManifest.numSlotBlocks - 1):
emptyDigest
else:
SpongeMerkle.digest(datasetBlocks[idx].data, cellSize.int)
expectedHashes: seq[Poseidon2Hash] = collect(newSeq):
for blk in expectedBlock:
SpongeMerkle.digest(blk.data & blockPadBytes, cellSize.int)
expectedRoot = Merkle.digest(expectedHashes & slotsPadLeafs)
slotTree = (await slotBuilder.buildSlotTree(i)).tryGet()
expectedRoot = Merkle.digest(expectedHashes)
slotTree = (await builder.buildSlotTree(i)).tryGet()
check:
expectedRoot == slotTree.root().tryGet()
slotTree.root().tryGet() == expectedRoot
test "Should persist trees for all slots":
let
slotBuilder = SlotsBuilder.new(
builder = Poseidon2Builder.new(
localStore,
protectedManifest,
cellSize = cellSize).tryGet()
for i in 0 ..< numSlots:
for i in 0..<numSlots:
let
slotTree = (await slotBuilder.buildSlotTree(i)).tryGet()
slotRoot = (await slotBuilder.buildSlot(i)).tryGet()
slotTree = (await builder.buildSlotTree(i)).tryGet()
slotRoot = (await builder.buildSlot(i)).tryGet()
slotCid = slotRoot.toSlotCid().tryGet()
for cellIndex in 0..<numTotalSlotBlocks:
for cellIndex in 0..<numPadSlotBlocks:
let
(cellCid, proof) = (await localStore.getCidAndProof(slotCid, cellIndex)).tryGet()
verifiableProof = proof.toVerifiableProof().tryGet()
posProof = slotTree.getProof(cellIndex).tryGet
posProof = slotTree.getProof(cellIndex).tryGet()
check:
verifiableProof.path == posProof.path
verifiableProof.index == posProof.index
verifiableProof.nleaves == posProof.nleaves
verifiableProof.path == posProof.path
test "Should build correct verification root":
let
steppedStrategy = SteppedStrategy.init(0, numTotalBlocks - 1, numSlots)
slotBuilder = SlotsBuilder.new(
steppedStrategy = Strategy.init(0, numPadBlocksTotal - 1, numSlots)
builder = Poseidon2Builder.new(
localStore,
protectedManifest,
cellSize = cellSize).tryGet()
(await slotBuilder.buildSlots()).tryGet
(await builder.buildSlots()).tryGet
let
slotsHashes = collect(newSeq):
for i in 0 ..< numSlots:
for i in 0..<numSlots:
let
expectedBlocks = steppedStrategy
.getIndicies(i)
.mapIt( datasetBlocks[it] )
slotHashes = collect(newSeq):
for j, idx in steppedStrategy.getIndicies(i):
if j > (protectedManifest.numSlotBlocks - 1):
emptyDigest
else:
SpongeMerkle.digest(datasetBlocks[idx].data, cellSize.int)
slotHashes: seq[Poseidon2Hash] = collect(newSeq):
for blk in expectedBlocks:
SpongeMerkle.digest(blk.data & blockPadBytes, cellSize.int)
Merkle.digest(slotHashes)
Merkle.digest(slotHashes & slotsPadLeafs)
expectedRoot = Merkle.digest(slotsHashes & rootsPadLeafs)
rootHash = slotBuilder.buildVerifyTree(slotBuilder.slotRoots).tryGet().root.tryGet()
expectedRoot = Merkle.digest(slotsHashes)
rootHash = builder.buildVerifyTree(builder.slotRoots).tryGet().root.tryGet()
check:
expectedRoot == rootHash
test "Should build correct verification root manifest":
let
steppedStrategy = SteppedStrategy.init(0, numTotalBlocks - 1, numSlots)
slotBuilder = SlotsBuilder.new(
steppedStrategy = Strategy.init(0, numPadBlocksTotal - 1, numSlots)
builder = Poseidon2Builder.new(
localStore,
protectedManifest,
cellSize = cellSize).tryGet()
slotsHashes = collect(newSeq):
for i in 0 ..< numSlots:
for i in 0..<numSlots:
let
expectedBlocks = steppedStrategy
.getIndicies(i)
.mapIt( datasetBlocks[it] )
slotHashes = collect(newSeq):
for j, idx in steppedStrategy.getIndicies(i):
if j > (protectedManifest.numSlotBlocks - 1):
emptyDigest
else:
SpongeMerkle.digest(datasetBlocks[idx].data, cellSize.int)
slotHashes: seq[Poseidon2Hash] = collect(newSeq):
for blk in expectedBlocks:
SpongeMerkle.digest(blk.data & blockPadBytes, cellSize.int)
Merkle.digest(slotHashes)
Merkle.digest(slotHashes & slotsPadLeafs)
expectedRoot = Merkle.digest(slotsHashes & rootsPadLeafs)
manifest = (await slotBuilder.buildManifest()).tryGet()
expectedRoot = Merkle.digest(slotsHashes)
manifest = (await builder.buildManifest()).tryGet()
mhash = manifest.verifyRoot.mhash.tryGet()
mhashBytes = mhash.digestBytes
rootHash = Poseidon2Hash.fromBytes(mhashBytes.toArray32).get
@ -334,69 +294,69 @@ suite "Slot builder":
test "Should not build from verifiable manifest with 0 slots":
var
slotBuilder = SlotsBuilder.new(
builder = Poseidon2Builder.new(
localStore,
protectedManifest,
cellSize = cellSize).tryGet()
verifyManifest = (await slotBuilder.buildManifest()).tryGet()
verifyManifest = (await builder.buildManifest()).tryGet()
verifyManifest.slotRoots = @[]
check SlotsBuilder.new(
check Poseidon2Builder.new(
localStore,
verifyManifest,
cellSize = cellSize).isErr
test "Should not build from verifiable manifest with incorrect number of slots":
var
slotBuilder = SlotsBuilder.new(
builder = Poseidon2Builder.new(
localStore,
protectedManifest,
cellSize = cellSize).tryGet()
verifyManifest = (await slotBuilder.buildManifest()).tryGet()
verifyManifest = (await builder.buildManifest()).tryGet()
verifyManifest.slotRoots.del(
verifyManifest.slotRoots.len - 1
)
check SlotsBuilder.new(
check Poseidon2Builder.new(
localStore,
verifyManifest,
cellSize = cellSize).isErr
test "Should not build from verifiable manifest with invalid verify root":
let
slotBuilder = SlotsBuilder.new(
builder = Poseidon2Builder.new(
localStore,
protectedManifest,
cellSize = cellSize).tryGet()
verifyManifest = (await slotBuilder.buildManifest()).tryGet()
verifyManifest = (await builder.buildManifest()).tryGet()
offset = verifyManifest.verifyRoot.data.buffer.len div 2
rng.shuffle(
Rng.instance,
verifyManifest.verifyRoot.data.buffer)
check SlotsBuilder.new(
check Poseidon2Builder.new(
localStore,
verifyManifest,
cellSize = cellSize).isErr
test "Should build from verifiable manifest":
let
slotBuilder = SlotsBuilder.new(
builder = Poseidon2Builder.new(
localStore,
protectedManifest,
cellSize = cellSize).tryGet()
verifyManifest = (await slotBuilder.buildManifest()).tryGet()
verifyManifest = (await builder.buildManifest()).tryGet()
verificationBuilder = SlotsBuilder.new(
verificationBuilder = Poseidon2Builder.new(
localStore,
verifyManifest,
cellSize = cellSize).tryGet()
check:
slotBuilder.slotRoots == verificationBuilder.slotRoots
slotBuilder.verifyRoot == verificationBuilder.verifyRoot
builder.slotRoots == verificationBuilder.slotRoots
builder.verifyRoot == verificationBuilder.verifyRoot

View File

@ -1,5 +1,4 @@
import ./slots/testslotbuilder
import ./slots/testutils
import ./slots/testsampler
import ./slots/testconverters