Rework builder & sampler (#697)
* add proof initializer for poseidon2 and `$` * fix padding and block selection * fix sample selection and input construction * fix sample selection & more descriptive names * add concrete types for sampler & builder * add missing digest calls * use concrete types * add sample test fixtures * use concrete types and don't fetch dummy blocks
This commit is contained in:
Dmitriy Ryajov
GitHub
parent
afec86b3cf
commit
825766eea0
|
|
@ -7,16 +7,14 @@
|
|||
## This file may not be copied, modified, or distributed except according to
|
||||
## those terms.
|
||||
|
||||
import pkg/upraises
|
||||
|
||||
push: {.upraises: [].}
|
||||
{.push raises: [].}
|
||||
|
||||
import std/bitops
|
||||
import std/sequtils
|
||||
|
||||
import pkg/questionable
|
||||
import pkg/questionable/results
|
||||
import pkg/libp2p/[cid, multicodec, multihash]
|
||||
import pkg/stew/byteutils
|
||||
|
||||
import ../../utils
|
||||
import ../../rng
|
||||
|
|
@ -72,7 +70,7 @@ func getProof*(self: CodexTree, index: int): ?!CodexProof =
|
|||
|
||||
success proof
|
||||
|
||||
func verify*(self: CodexProof, leaf: MultiHash, root: MultiHash): ?!void =
|
||||
func verify*(self: CodexProof, leaf: MultiHash, root: MultiHash): ?!bool =
|
||||
## Verify hash
|
||||
##
|
||||
|
||||
|
|
@ -88,11 +86,9 @@ func verify*(self: CodexProof, leaf: MultiHash, root: MultiHash): ?!void =
|
|||
leafBytes.len != leaf.size:
|
||||
return failure "Invalid hash length"
|
||||
|
||||
? self.verify(leafBytes, rootBytes)
|
||||
self.verify(leafBytes, rootBytes)
|
||||
|
||||
success()
|
||||
|
||||
func verify*(self: CodexProof, leaf: Cid, root: Cid): ?!void =
|
||||
func verify*(self: CodexProof, leaf: Cid, root: Cid): ?!bool =
|
||||
self.verify(? leaf.mhash.mapFailure, ? leaf.mhash.mapFailure)
|
||||
|
||||
proc rootCid*(
|
||||
|
|
@ -124,16 +120,19 @@ func getLeafCid*(
|
|||
Cid.init(version, dataCodec, mhash).mapFailure
|
||||
|
||||
proc `$`*(self: CodexTree): string =
|
||||
"CodexTree( mcodec: " &
|
||||
$self.mcodec &
|
||||
", leavesCount: " &
|
||||
$self.leavesCount & " )"
|
||||
let root = if self.root.isOk: byteutils.toHex(self.root.get) else: "none"
|
||||
"CodexTree(" &
|
||||
" root: " & root &
|
||||
", leavesCount: " & $self.leavesCount &
|
||||
", levels: " & $self.levels &
|
||||
", mcodec: " & $self.mcodec & " )"
|
||||
|
||||
proc `$`*(self: CodexProof): string =
|
||||
"CodexProof( mcodec: " &
|
||||
$self.mcodec & ", nleaves: " &
|
||||
$self.nleaves & ", index: " &
|
||||
$self.index & " )"
|
||||
"CodexProof(" &
|
||||
" nleaves: " & $self.nleaves &
|
||||
", index: " & $self.index &
|
||||
", path: " & $self.path.mapIt( byteutils.toHex(it) ) &
|
||||
", mcodec: " & $self.mcodec & " )"
|
||||
|
||||
func compress*(
|
||||
x, y: openArray[byte],
|
||||
|
|
@ -226,7 +225,9 @@ proc fromNodes*(
|
|||
index = Rng.instance.rand(nleaves - 1)
|
||||
proof = ? self.getProof(index)
|
||||
|
||||
? proof.verify(self.leaves[index], ? self.root) # sanity check
|
||||
if not ? proof.verify(self.leaves[index], ? self.root): # sanity check
|
||||
return failure "Unable to verify tree built from nodes"
|
||||
|
||||
success self
|
||||
|
||||
func init*(
|
||||
|
|
|
|||
|
|
@ -117,11 +117,8 @@ func reconstructRoot*[H, K](proof: MerkleProof[H, K], leaf: H): ?!H =
|
|||
|
||||
return success h
|
||||
|
||||
func verify*[H, K](proof: MerkleProof[H, K], leaf: H, root: H): ?!void =
|
||||
return if bool(root == ? proof.reconstructRoot(leaf)):
|
||||
success()
|
||||
else:
|
||||
failure("invalid proof")
|
||||
func verify*[H, K](proof: MerkleProof[H, K], leaf: H, root: H): ?!bool =
|
||||
success bool(root == ? proof.reconstructRoot(leaf))
|
||||
|
||||
func merkleTreeWorker*[H, K](
|
||||
self: MerkleTree[H, K],
|
||||
|
|
|
|||
|
|
@ -7,6 +7,8 @@
|
|||
## This file may not be copied, modified, or distributed except according to
|
||||
## those terms.
|
||||
|
||||
{.push raises: [].}
|
||||
|
||||
import std/sequtils
|
||||
|
||||
import pkg/poseidon2
|
||||
|
|
@ -30,7 +32,8 @@ const
|
|||
Poseidon2Zero* = zero
|
||||
|
||||
type
|
||||
Poseidon2Hash* = F
|
||||
Bn254Fr* = F
|
||||
Poseidon2Hash* = Bn254Fr
|
||||
|
||||
PoseidonKeysEnum* = enum # can't use non-ordinals as enum values
|
||||
KeyNone
|
||||
|
|
@ -41,6 +44,19 @@ type
|
|||
Poseidon2Tree* = MerkleTree[Poseidon2Hash, PoseidonKeysEnum]
|
||||
Poseidon2Proof* = MerkleProof[Poseidon2Hash, PoseidonKeysEnum]
|
||||
|
||||
proc `$`*(self: Poseidon2Tree): string =
|
||||
let root = if self.root.isOk: self.root.get.toHex else: "none"
|
||||
"Poseidon2Tree(" &
|
||||
" root: " & root &
|
||||
", leavesCount: " & $self.leavesCount &
|
||||
", levels: " & $self.levels & " )"
|
||||
|
||||
proc `$`*(self: Poseidon2Proof): string =
|
||||
"Poseidon2Proof(" &
|
||||
" nleaves: " & $self.nleaves &
|
||||
", index: " & $self.index &
|
||||
", path: " & $self.path.mapIt( it.toHex ) & " )"
|
||||
|
||||
func toArray32*(bytes: openArray[byte]): array[32, byte] =
|
||||
result[0..<bytes.len] = bytes[0..<bytes.len]
|
||||
|
||||
|
|
@ -104,6 +120,29 @@ proc fromNodes*(
|
|||
index = Rng.instance.rand(nleaves - 1)
|
||||
proof = ? self.getProof(index)
|
||||
|
||||
? proof.verify(self.leaves[index], ? self.root) # sanity check
|
||||
if not ? proof.verify(self.leaves[index], ? self.root): # sanity check
|
||||
return failure "Unable to verify tree built from nodes"
|
||||
|
||||
success self
|
||||
|
||||
func init*(
|
||||
_: type Poseidon2Proof,
|
||||
index: int,
|
||||
nleaves: int,
|
||||
nodes: openArray[Poseidon2Hash]): ?!Poseidon2Proof =
|
||||
|
||||
if nodes.len == 0:
|
||||
return failure "Empty nodes"
|
||||
|
||||
let
|
||||
compressor = proc(
|
||||
x, y: Poseidon2Hash,
|
||||
key: PoseidonKeysEnum): ?!Poseidon2Hash {.noSideEffect.} =
|
||||
success compress( x, y, key.toKey )
|
||||
|
||||
success Poseidon2Proof(
|
||||
compress: compressor,
|
||||
zero: Poseidon2Zero,
|
||||
index: index,
|
||||
nleaves: nleaves,
|
||||
path: @nodes)
|
||||
|
|
|
|||
|
|
@ -38,9 +38,11 @@ import ./streams
|
|||
import ./erasure
|
||||
import ./discovery
|
||||
import ./contracts
|
||||
import ./indexingstrategy
|
||||
import ./utils
|
||||
import ./errors
|
||||
import ./logutils
|
||||
import ./utils/poseidon2digest
|
||||
|
||||
export logutils
|
||||
|
||||
|
|
@ -390,7 +392,7 @@ proc setupRequest(
|
|||
trace "Unable to erasure code dataset"
|
||||
return failure(error)
|
||||
|
||||
without builder =? SlotsBuilder.new(self.blockStore, encoded), err:
|
||||
without builder =? Poseidon2Builder.new(self.blockStore, encoded), err:
|
||||
trace "Unable to create slot builder"
|
||||
return failure(err)
|
||||
|
||||
|
|
@ -503,7 +505,7 @@ proc onStore(
|
|||
trace "Unable to fetch manifest for cid", cid, err = err.msg
|
||||
return failure(err)
|
||||
|
||||
without builder =? SlotsBuilder.new(self.blockStore, manifest), err:
|
||||
without builder =? Poseidon2Builder.new(self.blockStore, manifest), err:
|
||||
trace "Unable to create slots builder", err = err.msg
|
||||
return failure(err)
|
||||
|
||||
|
|
@ -528,8 +530,13 @@ proc onStore(
|
|||
|
||||
return success()
|
||||
|
||||
if blksIter =? builder.slotIndiciesIter(slotIdx) and
|
||||
err =? (await self.fetchBatched(
|
||||
without indexer =? manifest.protectedStrategy.init(
|
||||
0, manifest.blocksCount() - 1, manifest.numSlots).catch and
|
||||
blksIter =? indexer.getIndicies(slotIdx).catch, err:
|
||||
trace "Unable to create indexing strategy from protected manifest", err = err.msg
|
||||
return failure(err)
|
||||
|
||||
if err =? (await self.fetchBatched(
|
||||
manifest.treeCid,
|
||||
blksIter,
|
||||
onBatch = updateExpiry)).errorOption:
|
||||
|
|
@ -572,7 +579,7 @@ proc onProve(
|
|||
error "Unable to fetch manifest for cid", err = err.msg
|
||||
return failure(err)
|
||||
|
||||
without builder =? SlotsBuilder.new(self.blockStore, manifest), err:
|
||||
without builder =? Poseidon2Builder.new(self.blockStore, manifest), err:
|
||||
error "Unable to create slots builder", err = err.msg
|
||||
return failure(err)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,9 @@
|
|||
import ./builder/builder
|
||||
import ./converters
|
||||
|
||||
import ../merkletree
|
||||
|
||||
export builder, converters
|
||||
|
||||
type
|
||||
Poseidon2Builder* = SlotsBuilder[Poseidon2Tree, Poseidon2Hash]
|
||||
|
|
|
|||
|
|
@ -15,192 +15,165 @@ import std/sugar
|
|||
|
||||
import pkg/libp2p
|
||||
import pkg/chronos
|
||||
import pkg/chronicles
|
||||
import pkg/questionable
|
||||
import pkg/questionable/results
|
||||
import pkg/poseidon2
|
||||
import pkg/poseidon2/io
|
||||
import pkg/constantine/math/arithmetic/finite_fields
|
||||
import pkg/constantine/math/io/io_fields
|
||||
|
||||
import ../../logutils
|
||||
import ../../indexingstrategy
|
||||
import ../../merkletree
|
||||
import ../../utils
|
||||
import ../../stores
|
||||
import ../../manifest
|
||||
import ../../utils
|
||||
import ../../utils/asynciter
|
||||
import ../../merkletree
|
||||
import ../../utils/digest
|
||||
import ../../utils/poseidon2digest
|
||||
import ../../utils/asynciter
|
||||
import ../../indexingstrategy
|
||||
|
||||
import ../converters
|
||||
|
||||
export converters
|
||||
export converters, asynciter
|
||||
|
||||
logScope:
|
||||
topics = "codex slotsbuilder"
|
||||
|
||||
const
|
||||
# TODO: Unified with the DefaultCellSize specified in branch "data-sampler"
|
||||
# in the proving circuit.
|
||||
|
||||
DefaultEmptyBlock* = newSeq[byte](DefaultBlockSize.int)
|
||||
DefaultEmptyCell* = newSeq[byte](DefaultCellSize.int)
|
||||
|
||||
type
|
||||
# TODO: should be a generic type that
|
||||
# supports all merkle trees
|
||||
SlotsBuilder* = ref object of RootObj
|
||||
SlotsBuilder*[T, H] = ref object of RootObj
|
||||
store: BlockStore
|
||||
manifest: Manifest
|
||||
strategy: IndexingStrategy
|
||||
cellSize: NBytes
|
||||
emptyDigestTree: Poseidon2Tree
|
||||
blockPadBytes: seq[byte]
|
||||
slotsPadLeafs: seq[Poseidon2Hash]
|
||||
rootsPadLeafs: seq[Poseidon2Hash]
|
||||
slotRoots: seq[Poseidon2Hash]
|
||||
verifyTree: ?Poseidon2Tree
|
||||
manifest: Manifest # current manifest
|
||||
strategy: IndexingStrategy # indexing strategy
|
||||
cellSize: NBytes # cell size
|
||||
numSlotBlocks: Natural # number of blocks per slot (should yield a power of two number of cells)
|
||||
slotRoots: seq[H] # roots of the slots
|
||||
emptyBlock: seq[byte] # empty block
|
||||
verifiableTree: ?T # verification tree (dataset tree)
|
||||
emptyDigestTree: T # empty digest tree for empty blocks
|
||||
|
||||
func slotRoots*(self: SlotsBuilder): seq[Poseidon2Hash] {.inline.} =
|
||||
func verifiable*[T, H](self: SlotsBuilder[T, H]): bool {.inline.} =
|
||||
## Returns true if the slots are verifiable.
|
||||
##
|
||||
|
||||
self.manifest.verifiable
|
||||
|
||||
func slotRoots*[T, H](self: SlotsBuilder[T, H]): seq[H] {.inline.} =
|
||||
## Returns the slot roots.
|
||||
##
|
||||
|
||||
self.slotRoots
|
||||
|
||||
func verifyTree*(self: SlotsBuilder): ?Poseidon2Tree {.inline.} =
|
||||
func verifyTree*[T, H](self: SlotsBuilder[T, H]): ?T {.inline.} =
|
||||
## Returns the slots tree (verification tree).
|
||||
##
|
||||
|
||||
self.verifyTree
|
||||
self.verifiableTree
|
||||
|
||||
func verifyRoot*(self: SlotsBuilder): ?Poseidon2Hash {.inline.} =
|
||||
func verifyRoot*[T, H](self: SlotsBuilder[T, H]): ?H {.inline.} =
|
||||
## Returns the slots root (verification root).
|
||||
##
|
||||
|
||||
self.verifyTree.?root().?toOption
|
||||
if tree =? self.verifyTree and root =? tree.root:
|
||||
return some root
|
||||
|
||||
func nextPowerOfTwoPad*(a: int): int =
|
||||
## Returns the difference between the original
|
||||
## value and the next power of two.
|
||||
##
|
||||
|
||||
nextPowerOfTwo(a) - a
|
||||
|
||||
func numBlockPadBytes*(self: SlotsBuilder): Natural =
|
||||
## Number of padding bytes required for a pow2
|
||||
## merkle tree for each block.
|
||||
##
|
||||
|
||||
self.blockPadBytes.len
|
||||
|
||||
func numSlotsPadLeafs*(self: SlotsBuilder): Natural =
|
||||
## Number of padding field elements required for a pow2
|
||||
## merkle tree for each slot.
|
||||
##
|
||||
|
||||
self.slotsPadLeafs.len
|
||||
|
||||
func numRootsPadLeafs*(self: SlotsBuilder): Natural =
|
||||
## Number of padding field elements required for a pow2
|
||||
## merkle tree for the slot roots.
|
||||
##
|
||||
|
||||
self.rootsPadLeafs.len
|
||||
|
||||
func numSlots*(self: SlotsBuilder): Natural =
|
||||
func numSlots*[T, H](self: SlotsBuilder[T, H]): Natural =
|
||||
## Number of slots.
|
||||
##
|
||||
|
||||
self.manifest.numSlots
|
||||
|
||||
func numSlotBlocks*(self: SlotsBuilder): Natural =
|
||||
func numSlotBlocks*[T, H](self: SlotsBuilder[T, H]): Natural =
|
||||
## Number of blocks per slot.
|
||||
##
|
||||
|
||||
self.manifest.blocksCount div self.manifest.numSlots
|
||||
self.numSlotBlocks
|
||||
|
||||
func slotBytes*(self: SlotsBuilder): NBytes =
|
||||
func numBlocks*[T, H](self: SlotsBuilder[T, H]): Natural =
|
||||
## Number of blocks.
|
||||
##
|
||||
|
||||
self.numSlotBlocks * self.manifest.numSlots
|
||||
|
||||
func slotBytes*[T, H](self: SlotsBuilder[T, H]): NBytes =
|
||||
## Number of bytes per slot.
|
||||
##
|
||||
|
||||
(self.manifest.blockSize.int * self.numSlotBlocks).NBytes
|
||||
|
||||
func numBlockCells*(self: SlotsBuilder): Natural =
|
||||
func numBlockCells*[T, H](self: SlotsBuilder[T, H]): Natural =
|
||||
## Number of cells per block.
|
||||
##
|
||||
|
||||
(self.manifest.blockSize div self.cellSize).Natural
|
||||
|
||||
func numBlockCellsPadded*(self: SlotsBuilder): Natural =
|
||||
## Number of cells per block including padding.
|
||||
##
|
||||
|
||||
nextPowerOfTwo(self.numBlockCells.int).Natural
|
||||
|
||||
func cellSize*(self: SlotsBuilder): NBytes =
|
||||
func cellSize*[T, H](self: SlotsBuilder[T, H]): NBytes =
|
||||
## Cell size.
|
||||
##
|
||||
|
||||
self.cellSize
|
||||
|
||||
func numSlotCells*(self: SlotsBuilder): Natural =
|
||||
func numSlotCells*[T, H](self: SlotsBuilder[T, H]): Natural =
|
||||
## Number of cells per slot.
|
||||
##
|
||||
|
||||
self.numBlockCells * self.numSlotBlocks
|
||||
|
||||
func numSlotCellsPadded*(self: SlotsBuilder): Natural =
|
||||
## Number of cells per slot including padding.
|
||||
##
|
||||
|
||||
nextPowerOfTwo(self.numBlockCellsPadded.int * self.numSlotBlocks.int).Natural
|
||||
|
||||
func emptyDigestTree*(self: SlotsBuilder): Poseidon2Tree {.inline.} =
|
||||
## Returns the tree of a padding block.
|
||||
##
|
||||
|
||||
self.emptyDigestTree
|
||||
|
||||
func slotIndiciesIter*(self: SlotsBuilder, slot: Natural): ?!Iter[int] =
|
||||
func slotIndiciesIter*[T, H](self: SlotsBuilder[T, H], slot: Natural): ?!Iter[int] =
|
||||
## Returns the slot indices.
|
||||
##
|
||||
|
||||
self.strategy.getIndicies(slot).catch
|
||||
|
||||
func slotIndicies*(self: SlotsBuilder, slot: Natural): seq[int] =
|
||||
func slotIndicies*[T, H](self: SlotsBuilder[T, H], slot: Natural): seq[int] =
|
||||
## Returns the slot indices.
|
||||
##
|
||||
|
||||
if iter =? self.strategy.getIndicies(slot).catch:
|
||||
toSeq(iter)
|
||||
else:
|
||||
trace "Failed to get slot indicies"
|
||||
newSeq[int]()
|
||||
return toSeq(iter)
|
||||
|
||||
func manifest*(self: SlotsBuilder): Manifest =
|
||||
func manifest*[T, H](self: SlotsBuilder[T, H]): Manifest =
|
||||
## Returns the manifest.
|
||||
##
|
||||
|
||||
self.manifest
|
||||
|
||||
proc buildBlockTree*(
|
||||
self: SlotsBuilder,
|
||||
blkIdx: Natural): Future[?!(seq[byte], Poseidon2Tree)] {.async.} =
|
||||
proc buildBlockTree*[T, H](
|
||||
self: SlotsBuilder[T, H],
|
||||
blkIdx: Natural,
|
||||
slotPos: Natural): Future[?!(seq[byte], T)] {.async.} =
|
||||
## Build the block digest tree and return a tuple with the
|
||||
## block data and the tree.
|
||||
##
|
||||
|
||||
logScope:
|
||||
blkIdx = blkIdx
|
||||
slotPos = slotPos
|
||||
numSlotBlocks = self.manifest.numSlotBlocks
|
||||
cellSize = self.cellSize
|
||||
|
||||
trace "Building block tree"
|
||||
|
||||
if slotPos > (self.manifest.numSlotBlocks - 1):
|
||||
# pad blocks are 0 byte blocks
|
||||
trace "Returning empty digest tree for pad block"
|
||||
return success (self.emptyBlock, self.emptyDigestTree)
|
||||
|
||||
without blk =? await self.store.getBlock(self.manifest.treeCid, blkIdx), err:
|
||||
error "Failed to get block CID for tree at index"
|
||||
error "Failed to get block CID for tree at index", err = err.msg
|
||||
return failure(err)
|
||||
|
||||
if blk.isEmpty:
|
||||
success (DefaultEmptyBlock & self.blockPadBytes, self.emptyDigestTree)
|
||||
success (self.emptyBlock, self.emptyDigestTree)
|
||||
else:
|
||||
without tree =?
|
||||
Poseidon2Tree.digestTree(blk.data & self.blockPadBytes, self.cellSize.int), err:
|
||||
error "Failed to create digest for block"
|
||||
T.digestTree(blk.data, self.cellSize.int), err:
|
||||
error "Failed to create digest for block", err = err.msg
|
||||
return failure(err)
|
||||
|
||||
success (blk.data, tree)
|
||||
|
||||
proc getBlockHashes*(
|
||||
self: SlotsBuilder,
|
||||
slotIndex: Natural): Future[?!seq[Poseidon2Hash]] {.async.} =
|
||||
proc getCellHashes*[T, H](
|
||||
self: SlotsBuilder[T, H],
|
||||
slotIndex: Natural): Future[?!seq[H]] {.async.} =
|
||||
## Collect all the cells from a block and return
|
||||
## their hashes.
|
||||
##
|
||||
|
||||
let
|
||||
treeCid = self.manifest.treeCid
|
||||
|
|
@ -209,38 +182,44 @@ proc getBlockHashes*(
|
|||
|
||||
logScope:
|
||||
treeCid = treeCid
|
||||
blockCount = blockCount
|
||||
origBlockCount = blockCount
|
||||
numberOfSlots = numberOfSlots
|
||||
index = blkIdx
|
||||
slotIndex = slotIndex
|
||||
|
||||
let
|
||||
hashes: seq[Poseidon2Hash] = collect(newSeq):
|
||||
for blkIdx in self.strategy.getIndicies(slotIndex):
|
||||
trace "Getting block CID for tree at index"
|
||||
let hashes = collect(newSeq):
|
||||
for i, blkIdx in self.strategy.getIndicies(slotIndex):
|
||||
logScope:
|
||||
blkIdx = blkIdx
|
||||
pos = i
|
||||
|
||||
without (_, blockTree) =? (await self.buildBlockTree(blkIdx)) and
|
||||
digest =? blockTree.root, err:
|
||||
error "Failed to get block CID for block tree at index", err = err.msg
|
||||
return failure(err)
|
||||
trace "Getting block CID for tree at index"
|
||||
without (_, tree) =? (await self.buildBlockTree(blkIdx, i)) and
|
||||
digest =? tree.root, err:
|
||||
error "Failed to get block CID for tree at index", err = err.msg
|
||||
return failure(err)
|
||||
|
||||
digest
|
||||
trace "Get block digest", digest = digest.toHex
|
||||
digest
|
||||
|
||||
success hashes
|
||||
|
||||
proc buildSlotTree*(
|
||||
self: SlotsBuilder,
|
||||
slotIndex: Natural): Future[?!Poseidon2Tree] {.async.} =
|
||||
without blockHashes =? (await self.getBlockHashes(slotIndex)), err:
|
||||
proc buildSlotTree*[T, H](
|
||||
self: SlotsBuilder[T, H],
|
||||
slotIndex: Natural): Future[?!T] {.async.} =
|
||||
## Build the slot tree from the block digest hashes
|
||||
## and return the tree.
|
||||
|
||||
without cellHashes =? (await self.getCellHashes(slotIndex)), err:
|
||||
error "Failed to select slot blocks", err = err.msg
|
||||
return failure(err)
|
||||
|
||||
Poseidon2Tree.init(blockHashes & self.slotsPadLeafs)
|
||||
T.init(cellHashes)
|
||||
|
||||
proc buildSlot*(
|
||||
self: SlotsBuilder,
|
||||
slotIndex: Natural): Future[?!Poseidon2Hash] {.async.} =
|
||||
## Build a slot tree and store it in the block store.
|
||||
proc buildSlot*[T, H](
|
||||
self: SlotsBuilder[T, H],
|
||||
slotIndex: Natural): Future[?!H] {.async.} =
|
||||
## Build a slot tree and store the proofs in
|
||||
## the block store.
|
||||
##
|
||||
|
||||
logScope:
|
||||
|
|
@ -272,12 +251,10 @@ proc buildSlot*(
|
|||
|
||||
tree.root()
|
||||
|
||||
func buildVerifyTree*(
|
||||
self: SlotsBuilder,
|
||||
slotRoots: openArray[Poseidon2Hash]): ?!Poseidon2Tree =
|
||||
Poseidon2Tree.init(@slotRoots & self.rootsPadLeafs)
|
||||
func buildVerifyTree*[T, H](self: SlotsBuilder[T, H], slotRoots: openArray[H]): ?!T =
|
||||
T.init(@slotRoots)
|
||||
|
||||
proc buildSlots*(self: SlotsBuilder): Future[?!void] {.async.} =
|
||||
proc buildSlots*[T, H](self: SlotsBuilder[T, H]): Future[?!void] {.async.} =
|
||||
## Build all slot trees and store them in the block store.
|
||||
##
|
||||
|
||||
|
|
@ -300,14 +277,14 @@ proc buildSlots*(self: SlotsBuilder): Future[?!void] {.async.} =
|
|||
return failure(err)
|
||||
|
||||
if verifyTree =? self.verifyTree and verifyRoot =? verifyTree.root:
|
||||
if verifyRoot != root: # TODO: `!=` doesn't work for SecretBool
|
||||
if not bool(verifyRoot == root): # TODO: `!=` doesn't work for SecretBool
|
||||
return failure "Existing slots root doesn't match reconstructed root."
|
||||
|
||||
self.verifyTree = some tree
|
||||
self.verifiableTree = some tree
|
||||
|
||||
success()
|
||||
|
||||
proc buildManifest*(self: SlotsBuilder): Future[?!Manifest] {.async.} =
|
||||
proc buildManifest*[T, H](self: SlotsBuilder[T, H]): Future[?!Manifest] {.async.} =
|
||||
if err =? (await self.buildSlots()).errorOption:
|
||||
error "Failed to build slot roots", err = err.msg
|
||||
return failure(err)
|
||||
|
|
@ -321,68 +298,88 @@ proc buildManifest*(self: SlotsBuilder): Future[?!Manifest] {.async.} =
|
|||
error "Failed to map slot roots to CIDs", err = err.msg
|
||||
return failure(err)
|
||||
|
||||
Manifest.new(self.manifest, rootProvingCid, rootCids)
|
||||
Manifest.new(
|
||||
self.manifest,
|
||||
rootProvingCid,
|
||||
rootCids,
|
||||
self.cellSize,
|
||||
self.strategy.strategyType)
|
||||
|
||||
proc new*(
|
||||
T: type SlotsBuilder,
|
||||
proc new*[T, H](
|
||||
_: type SlotsBuilder[T, H],
|
||||
store: BlockStore,
|
||||
manifest: Manifest,
|
||||
strategy: ?IndexingStrategy = none IndexingStrategy,
|
||||
cellSize = DefaultCellSize): ?!SlotsBuilder =
|
||||
strategy = SteppedStrategy,
|
||||
cellSize = DefaultCellSize): ?!SlotsBuilder[T, H] =
|
||||
|
||||
if not manifest.protected:
|
||||
return failure("Can only create SlotsBuilder using protected manifests.")
|
||||
trace "Manifest is not protected."
|
||||
return failure("Manifest is not protected.")
|
||||
|
||||
logScope:
|
||||
blockSize = manifest.blockSize
|
||||
strategy = strategy
|
||||
cellSize = cellSize
|
||||
|
||||
if (manifest.blocksCount mod manifest.numSlots) != 0:
|
||||
trace "Number of blocks must be divisable by number of slots."
|
||||
return failure("Number of blocks must be divisable by number of slots.")
|
||||
|
||||
let cellSize = if manifest.verifiable: manifest.cellSize else: cellSize
|
||||
if (manifest.blockSize mod cellSize) != 0.NBytes:
|
||||
trace "Block size must be divisable by cell size."
|
||||
return failure("Block size must be divisable by cell size.")
|
||||
|
||||
let
|
||||
strategy = if strategy.isNone:
|
||||
? SteppedStrategy.init(
|
||||
0, manifest.blocksCount - 1, manifest.numSlots).catch
|
||||
else:
|
||||
strategy.get
|
||||
numBlockCells = (manifest.blockSize div cellSize).int # number of cells per block
|
||||
numSlotCells = manifest.numSlotBlocks * numBlockCells # number of uncorrected slot cells
|
||||
pow2SlotCells = nextPowerOfTwo(numSlotCells) # pow2 cells per slot
|
||||
numPadSlotBlocks = pow2SlotCells div numBlockCells # pow2 blocks per slot
|
||||
numPadBlocksTotal = numPadSlotBlocks * manifest.numSlots # total number of pad blocks
|
||||
|
||||
# all trees have to be padded to power of two
|
||||
numBlockCells = (manifest.blockSize div cellSize).int # number of cells per block
|
||||
blockPadBytes = newSeq[byte](numBlockCells.nextPowerOfTwoPad * cellSize.int) # power of two padding for blocks
|
||||
numSlotBlocks = (manifest.blocksCount div manifest.numSlots)
|
||||
slotsPadLeafs = newSeqWith(numSlotBlocks.nextPowerOfTwoPad, Poseidon2Zero) # power of two padding for block roots
|
||||
rootsPadLeafs = newSeqWith(manifest.numSlots.nextPowerOfTwoPad, Poseidon2Zero)
|
||||
emptyDigestTree = ? Poseidon2Tree.digestTree(DefaultEmptyBlock & blockPadBytes, DefaultCellSize.int)
|
||||
emptyBlock = newSeq[byte](manifest.blockSize.int)
|
||||
emptyDigestTree = ? T.digestTree(emptyBlock, cellSize.int)
|
||||
|
||||
var self = SlotsBuilder(
|
||||
store: store,
|
||||
manifest: manifest,
|
||||
strategy: strategy,
|
||||
cellSize: cellSize,
|
||||
blockPadBytes: blockPadBytes,
|
||||
slotsPadLeafs: slotsPadLeafs,
|
||||
rootsPadLeafs: rootsPadLeafs,
|
||||
emptyDigestTree: emptyDigestTree)
|
||||
strategy = ? strategy.init(
|
||||
0,
|
||||
numPadBlocksTotal - 1,
|
||||
manifest.numSlots).catch
|
||||
|
||||
logScope:
|
||||
numBlockCells = numBlockCells
|
||||
numSlotCells = numSlotCells
|
||||
pow2SlotCells = pow2SlotCells
|
||||
numPadSlotBlocks = numPadSlotBlocks
|
||||
numPadBlocksTotal = numPadBlocksTotal
|
||||
strategy = strategy.strategyType
|
||||
|
||||
trace "Creating slots builder"
|
||||
|
||||
var
|
||||
self = SlotsBuilder[T, H](
|
||||
store: store,
|
||||
manifest: manifest,
|
||||
strategy: strategy,
|
||||
cellSize: cellSize,
|
||||
emptyBlock: emptyBlock,
|
||||
numSlotBlocks: numPadSlotBlocks,
|
||||
emptyDigestTree: emptyDigestTree)
|
||||
|
||||
if manifest.verifiable:
|
||||
if manifest.slotRoots.len == 0 or manifest.slotRoots.len != manifest.numSlots:
|
||||
if manifest.slotRoots.len == 0 or
|
||||
manifest.slotRoots.len != manifest.numSlots:
|
||||
return failure "Manifest is verifiable but slot roots are missing or invalid."
|
||||
|
||||
let slotRoots = manifest.slotRoots.mapIt( (? it.fromSlotCid()))
|
||||
let
|
||||
slotRoots = manifest.slotRoots.mapIt( (? it.fromSlotCid() ))
|
||||
tree = ? self.buildVerifyTree(slotRoots)
|
||||
expectedRoot = ? manifest.verifyRoot.fromVerifyCid()
|
||||
verifyRoot = ? tree.root
|
||||
|
||||
without tree =? self.buildVerifyTree(slotRoots), err:
|
||||
error "Failed to build slot roots tree", err = err.msg
|
||||
return failure(err)
|
||||
|
||||
without expectedRoot =? manifest.verifyRoot.fromVerifyCid(), err:
|
||||
error "Unable to convert manifest verifyRoot to hash", error = err.msg
|
||||
return failure(err)
|
||||
|
||||
if verifyRoot =? tree.root:
|
||||
if verifyRoot != expectedRoot:
|
||||
return failure "Existing slots root doesn't match reconstructed root."
|
||||
if verifyRoot != expectedRoot:
|
||||
return failure "Existing slots root doesn't match reconstructed root."
|
||||
|
||||
self.slotRoots = slotRoots
|
||||
self.verifyTree = some tree
|
||||
self.verifiableTree = some tree
|
||||
|
||||
success self
|
||||
|
|
|
|||
|
|
@ -1,4 +1,9 @@
|
|||
import ./sampler/sampler
|
||||
import ./sampler/utils
|
||||
|
||||
import ../merkletree
|
||||
|
||||
export sampler, utils
|
||||
|
||||
type
|
||||
Poseidon2Sampler* = DataSampler[Poseidon2Tree, Poseidon2Hash]
|
||||
|
|
|
|||
|
|
@ -8,182 +8,146 @@
|
|||
## those terms.
|
||||
|
||||
import std/sugar
|
||||
import std/sequtils
|
||||
|
||||
import pkg/chronicles
|
||||
import pkg/chronos
|
||||
import pkg/questionable
|
||||
import pkg/questionable/results
|
||||
import pkg/constantine/math/arithmetic
|
||||
import pkg/poseidon2
|
||||
import pkg/poseidon2/types
|
||||
import pkg/poseidon2/io
|
||||
import pkg/stew/arrayops
|
||||
|
||||
import ../../logutils
|
||||
import ../../market
|
||||
import ../../blocktype as bt
|
||||
import ../../merkletree
|
||||
import ../../manifest
|
||||
import ../../stores
|
||||
|
||||
import ../converters
|
||||
import ../builder
|
||||
|
||||
import ../types
|
||||
import ./utils
|
||||
|
||||
logScope:
|
||||
topics = "codex datasampler"
|
||||
|
||||
type
|
||||
Cell* = seq[byte]
|
||||
|
||||
Sample* = object
|
||||
data*: Cell
|
||||
slotProof*: Poseidon2Proof
|
||||
cellProof*: Poseidon2Proof
|
||||
slotBlockIdx*: Natural
|
||||
blockCellIdx*: Natural
|
||||
|
||||
ProofInput* = object
|
||||
entropy*: Poseidon2Hash
|
||||
verifyRoot*: Poseidon2Hash
|
||||
verifyProof*: Poseidon2Proof
|
||||
numSlots*: Natural
|
||||
numCells*: Natural
|
||||
slotIndex*: Natural
|
||||
samples*: seq[Sample]
|
||||
|
||||
DataSampler* = ref object of RootObj
|
||||
DataSampler*[T, H] = ref object of RootObj
|
||||
index: Natural
|
||||
blockStore: BlockStore
|
||||
# The following data is invariant over time for a given slot:
|
||||
builder: SlotsBuilder
|
||||
builder: SlotsBuilder[T, H]
|
||||
|
||||
proc new*(
|
||||
T: type DataSampler,
|
||||
func getCell*[T, H](
|
||||
self: DataSampler[T, H],
|
||||
blkBytes: seq[byte],
|
||||
blkCellIdx: Natural): seq[byte] =
|
||||
|
||||
let
|
||||
cellSize = self.builder.cellSize.uint64
|
||||
dataStart = cellSize * blkCellIdx.uint64
|
||||
dataEnd = dataStart + cellSize
|
||||
|
||||
doAssert (dataEnd - dataStart) == cellSize, "Invalid cell size"
|
||||
|
||||
toInputData[H](blkBytes[dataStart ..< dataEnd])
|
||||
|
||||
proc getSample*[T, H](
|
||||
self: DataSampler[T, H],
|
||||
cellIdx: int,
|
||||
slotTreeCid: Cid,
|
||||
slotRoot: H): Future[?!Sample[H]] {.async.} =
|
||||
|
||||
let
|
||||
cellsPerBlock = self.builder.numBlockCells
|
||||
blkCellIdx = cellIdx.toCellInBlk(cellsPerBlock) # block cell index
|
||||
blkSlotIdx = cellIdx.toBlkInSlot(cellsPerBlock) # slot tree index
|
||||
origBlockIdx = self.builder.slotIndicies(self.index)[blkSlotIdx] # convert to original dataset block index
|
||||
|
||||
logScope:
|
||||
cellIdx = cellIdx
|
||||
blkSlotIdx = blkSlotIdx
|
||||
blkCellIdx = blkCellIdx
|
||||
origBlockIdx = origBlockIdx
|
||||
|
||||
trace "Retrieving sample from block tree"
|
||||
let
|
||||
(_, proof) = (await self.blockStore.getCidAndProof(
|
||||
slotTreeCid, blkSlotIdx.Natural)).valueOr:
|
||||
return failure("Failed to get slot tree CID and proof")
|
||||
|
||||
slotProof = proof.toVerifiableProof().valueOr:
|
||||
return failure("Failed to get verifiable proof")
|
||||
|
||||
(bytes, blkTree) = (await self.builder.buildBlockTree(
|
||||
origBlockIdx, blkSlotIdx)).valueOr:
|
||||
return failure("Failed to build block tree")
|
||||
|
||||
cellData = self.getCell(bytes, blkCellIdx)
|
||||
cellProof = blkTree.getProof(blkCellIdx).valueOr:
|
||||
return failure("Failed to get proof from block tree")
|
||||
|
||||
success Sample[H](
|
||||
cellData: cellData,
|
||||
merklePaths: (cellProof.path & slotProof.path))
|
||||
|
||||
proc getProofInput*[T, H](
|
||||
self: DataSampler[T, H],
|
||||
entropy: ProofChallenge,
|
||||
nSamples: Natural): Future[?!ProofInput[H]] {.async.} =
|
||||
## Generate proofs as input to the proving circuit.
|
||||
##
|
||||
|
||||
let
|
||||
entropy = H.fromBytes(
|
||||
array[31, byte].initCopyFrom(entropy[0..30])) # truncate to 31 bytes, otherwise it _might_ be greater than mod
|
||||
|
||||
verifyTree = self.builder.verifyTree.toFailure.valueOr:
|
||||
return failure("Failed to get verify tree")
|
||||
|
||||
slotProof = verifyTree.getProof(self.index).valueOr:
|
||||
return failure("Failed to get slot proof")
|
||||
|
||||
datasetRoot = verifyTree.root().valueOr:
|
||||
return failure("Failed to get dataset root")
|
||||
|
||||
slotTreeCid = self.builder.manifest.slotRoots[self.index]
|
||||
slotRoot = self.builder.slotRoots[self.index]
|
||||
cellIdxs = entropy.cellIndices(
|
||||
slotRoot,
|
||||
self.builder.numSlotCells,
|
||||
nSamples)
|
||||
|
||||
logScope:
|
||||
cells = cellIdxs
|
||||
|
||||
trace "Collecting input for proof"
|
||||
let samples = collect(newSeq):
|
||||
for cellIdx in cellIdxs:
|
||||
(await self.getSample(cellIdx, slotTreeCid, slotRoot)).valueOr:
|
||||
return failure("Failed to get sample")
|
||||
|
||||
success ProofInput[H](
|
||||
entropy: entropy,
|
||||
datasetRoot: datasetRoot,
|
||||
slotProof: slotProof.path,
|
||||
nSlotsPerDataSet: self.builder.numSlots,
|
||||
nCellsPerSlot: self.builder.numSlotCells,
|
||||
slotRoot: slotRoot,
|
||||
slotIndex: self.index,
|
||||
samples: samples)
|
||||
|
||||
proc new*[T, H](
|
||||
_: type DataSampler[T, H],
|
||||
index: Natural,
|
||||
blockStore: BlockStore,
|
||||
builder: SlotsBuilder): ?!DataSampler =
|
||||
builder: SlotsBuilder[T, H]): ?!DataSampler[T, H] =
|
||||
|
||||
if index > builder.slotRoots.high:
|
||||
error "Slot index is out of range"
|
||||
return failure("Slot index is out of range")
|
||||
|
||||
success DataSampler(
|
||||
if not builder.verifiable:
|
||||
return failure("Cannot instantiate DataSampler for non-verifiable builder")
|
||||
|
||||
success DataSampler[T, H](
|
||||
index: index,
|
||||
blockStore: blockStore,
|
||||
builder: builder)
|
||||
|
||||
proc getCell*(self: DataSampler, blkBytes: seq[byte], blkCellIdx: Natural): Cell =
|
||||
let
|
||||
cellSize = self.builder.cellSize.uint64
|
||||
dataStart = cellSize * blkCellIdx.uint64
|
||||
dataEnd = dataStart + cellSize
|
||||
return blkBytes[dataStart ..< dataEnd]
|
||||
|
||||
proc createProofSample(self: DataSampler, slotTreeCid: Cid, cellIdx: Natural): Future[?!Sample] {.async.} =
|
||||
let
|
||||
cellsPerBlock = self.builder.numBlockCells
|
||||
blkCellIdx = cellIdx.toBlockCellIdx(cellsPerBlock)
|
||||
slotBlkIdx = cellIdx.toBlockIdx(cellsPerBlock)
|
||||
|
||||
logScope:
|
||||
cellIdx = cellIdx
|
||||
slotBlkIdx = slotBlkIdx
|
||||
blkCellIdx = blkCellIdx
|
||||
|
||||
without (cid, proof) =? await self.blockStore.getCidAndProof(
|
||||
slotTreeCid,
|
||||
slotBlkIdx.Natural), err:
|
||||
error "Failed to get block from block store", err = err.msg
|
||||
return failure(err)
|
||||
|
||||
without slotProof =? proof.toVerifiableProof(), err:
|
||||
error "Unable to convert slot proof to poseidon proof", error = err.msg
|
||||
return failure(err)
|
||||
|
||||
# If the cell index is greater than or equal to the UNPADDED number of slot cells,
|
||||
# Then we're sampling inside a padded block.
|
||||
# In this case, we use the pre-generated zero-data and pre-generated padding-proof for this cell index.
|
||||
if cellIdx >= self.builder.numSlotCells:
|
||||
trace "Sampling a padded block"
|
||||
|
||||
without blockProof =? self.builder.emptyDigestTree.getProof(blkCellIdx), err:
|
||||
error "Failed to get proof from empty block tree", err = err.msg
|
||||
return failure(err)
|
||||
|
||||
success(Sample(
|
||||
data: newSeq[byte](self.builder.cellSize.int),
|
||||
slotProof: slotProof,
|
||||
cellProof: blockProof,
|
||||
slotBlockIdx: slotBlkIdx.Natural,
|
||||
blockCellIdx: blkCellIdx.Natural))
|
||||
|
||||
else:
|
||||
trace "Sampling a dataset block"
|
||||
# This converts our slotBlockIndex to a datasetBlockIndex using the
|
||||
# indexing-strategy used by the builder.
|
||||
# We need this to fetch the block data. We can't do it by slotTree + slotBlkIdx.
|
||||
let datasetBlockIndex = self.builder.slotIndicies(self.index)[slotBlkIdx]
|
||||
|
||||
without (bytes, blkTree) =? await self.builder.buildBlockTree(datasetBlockIndex), err:
|
||||
error "Failed to build block tree", err = err.msg
|
||||
return failure(err)
|
||||
|
||||
without blockProof =? blkTree.getProof(blkCellIdx), err:
|
||||
error "Failed to get proof from block tree", err = err.msg
|
||||
return failure(err)
|
||||
|
||||
success(Sample(
|
||||
data: self.getCell(bytes, blkCellIdx),
|
||||
slotProof: slotProof,
|
||||
cellProof: blockProof,
|
||||
slotBlockIdx: slotBlkIdx.Natural,
|
||||
blockCellIdx: blkCellIdx.Natural))
|
||||
|
||||
proc getProofInput*(
|
||||
self: DataSampler,
|
||||
entropy: ProofChallenge,
|
||||
nSamples: Natural): Future[?!ProofInput] {.async.} =
|
||||
## Generate proofs as input to the proving circuit.
|
||||
##
|
||||
|
||||
let
|
||||
entropy = Poseidon2Hash.fromBytes(
|
||||
array[31, byte].initCopyFrom(entropy[0..30])) # truncate to 31 bytes, otherwise it _might_ be greater than mod
|
||||
|
||||
without verifyTree =? self.builder.verifyTree and
|
||||
verifyProof =? verifyTree.getProof(self.index) and
|
||||
verifyRoot =? verifyTree.root(), err:
|
||||
error "Failed to get slot proof from verify tree", err = err.msg
|
||||
return failure(err)
|
||||
|
||||
let slotTreeCid = self.builder.manifest.slotRoots[self.index]
|
||||
|
||||
logScope:
|
||||
index = self.index
|
||||
samples = nSamples
|
||||
slotTreeCid = slotTreeCid
|
||||
|
||||
trace "Collecting input for proof"
|
||||
|
||||
let cellIdxs = entropy.cellIndices(
|
||||
self.builder.slotRoots[self.index],
|
||||
self.builder.numSlotCellsPadded,
|
||||
nSamples)
|
||||
|
||||
trace "Found cell indices", cellIdxs
|
||||
let samples = collect(newSeq):
|
||||
for cellIdx in cellIdxs:
|
||||
without sample =? (await self.createProofSample(slotTreeCid, cellIdx)), err:
|
||||
error "Failed to create proof sample", error = err.msg
|
||||
return failure(err)
|
||||
sample
|
||||
|
||||
success ProofInput(
|
||||
entropy: entropy,
|
||||
verifyRoot: verifyRoot,
|
||||
verifyProof: verifyProof,
|
||||
numSlots: self.builder.numSlots,
|
||||
numCells: self.builder.numSlotCells,
|
||||
slotIndex: self.index,
|
||||
samples: samples)
|
||||
|
|
|
|||
|
|
@ -9,8 +9,9 @@
|
|||
|
||||
import std/sugar
|
||||
import std/bitops
|
||||
import std/sequtils
|
||||
|
||||
import pkg/chronicles
|
||||
import pkg/questionable/results
|
||||
import pkg/poseidon2
|
||||
import pkg/poseidon2/io
|
||||
|
||||
|
|
@ -20,8 +21,11 @@ import pkg/constantine/math/io/io_fields
|
|||
|
||||
import ../../merkletree
|
||||
|
||||
func toInputData*[H](data: seq[byte]): seq[byte] =
|
||||
return toSeq(data.elements(H)).mapIt( @(it.toBytes) ).concat
|
||||
|
||||
func extractLowBits*[n: static int](elm: BigInt[n], k: int): uint64 =
|
||||
assert( k > 0 and k <= 64 )
|
||||
doAssert( k > 0 and k <= 64 )
|
||||
var r = 0'u64
|
||||
for i in 0..<k:
|
||||
let b = bit[n](elm, i)
|
||||
|
|
@ -48,13 +52,13 @@ func ceilingLog2*(x : int) : int =
|
|||
else:
|
||||
return (floorLog2(x-1) + 1)
|
||||
|
||||
func toBlockIdx*(cell: Natural, numCells: Natural): Natural =
|
||||
func toBlkInSlot*(cell: Natural, numCells: Natural): Natural =
|
||||
let log2 = ceilingLog2(numCells)
|
||||
doAssert( 1 shl log2 == numCells , "`numCells` is assumed to be a power of two" )
|
||||
|
||||
return cell div numCells
|
||||
|
||||
func toBlockCellIdx*(cell: Natural, numCells: Natural): Natural =
|
||||
func toCellInBlk*(cell: Natural, numCells: Natural): Natural =
|
||||
let log2 = ceilingLog2(numCells)
|
||||
doAssert( 1 shl log2 == numCells , "`numCells` is assumed to be a power of two" )
|
||||
|
||||
|
|
@ -67,7 +71,7 @@ func cellIndex*(
|
|||
let log2 = ceilingLog2(numCells)
|
||||
doAssert( 1 shl log2 == numCells , "`numCells` is assumed to be a power of two" )
|
||||
|
||||
let hash = Sponge.digest( @[ slotRoot, entropy, counter.toF ], rate = 2 )
|
||||
let hash = Sponge.digest( @[ entropy, slotRoot, counter.toF ], rate = 2 )
|
||||
return int( extractLowBits(hash, log2) )
|
||||
|
||||
func cellIndices*(
|
||||
|
|
@ -75,11 +79,8 @@ func cellIndices*(
|
|||
slotRoot: Poseidon2Hash,
|
||||
numCells: Natural, nSamples: Natural): seq[Natural] =
|
||||
|
||||
trace "Calculating cell indices", numCells, nSamples
|
||||
|
||||
var indices: seq[Natural]
|
||||
while (indices.len < nSamples):
|
||||
let idx = cellIndex(entropy, slotRoot, numCells, indices.len + 1)
|
||||
indices.add(idx.Natural)
|
||||
indices
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,28 @@
|
|||
## Nim-Codex
|
||||
## Copyright (c) 2024 Status Research & Development GmbH
|
||||
## Licensed under either of
|
||||
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
|
||||
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
|
||||
## at your option.
|
||||
## This file may not be copied, modified, or distributed except according to
|
||||
## those terms.
|
||||
|
||||
type
|
||||
Sample*[H] = object
|
||||
cellData*: seq[byte]
|
||||
merklePaths*: seq[H]
|
||||
|
||||
PublicInputs*[H] = object
|
||||
slotIndex*: int
|
||||
datasetRoot*: H
|
||||
entropy*: H
|
||||
|
||||
ProofInput*[H] = object
|
||||
entropy*: H
|
||||
datasetRoot*: H
|
||||
slotIndex*: Natural
|
||||
slotRoot*: H
|
||||
nCellsPerSlot*: Natural
|
||||
nSlotsPerDataSet*: Natural
|
||||
slotProof*: seq[H]
|
||||
samples*: seq[Sample[H]]
|
||||
|
|
@ -10,29 +10,59 @@
|
|||
import pkg/poseidon2
|
||||
import pkg/questionable/results
|
||||
import pkg/libp2p/multihash
|
||||
import pkg/stew/byteutils
|
||||
|
||||
import pkg/constantine/math/arithmetic
|
||||
import pkg/constantine/math/io/io_bigints
|
||||
import pkg/constantine/math/io/io_fields
|
||||
|
||||
import ../merkletree
|
||||
|
||||
func spongeDigest*(
|
||||
_: type Poseidon2Hash,
|
||||
bytes: openArray[byte],
|
||||
rate: static int = 2): ?!Poseidon2Hash =
|
||||
## Hashes chunks of data with a sponge of rate 1 or 2.
|
||||
##
|
||||
|
||||
success Sponge.digest(bytes, rate)
|
||||
|
||||
func spongeDigest*(
|
||||
_: type Poseidon2Hash,
|
||||
bytes: openArray[Bn254Fr],
|
||||
rate: static int = 2): ?!Poseidon2Hash =
|
||||
## Hashes chunks of elements with a sponge of rate 1 or 2.
|
||||
##
|
||||
|
||||
success Sponge.digest(bytes, rate)
|
||||
|
||||
func digestTree*(
|
||||
_: type Poseidon2Tree,
|
||||
bytes: openArray[byte], chunkSize: int): ?!Poseidon2Tree =
|
||||
bytes: openArray[byte],
|
||||
chunkSize: int): ?!Poseidon2Tree =
|
||||
## Hashes chunks of data with a sponge of rate 2, and combines the
|
||||
## resulting chunk hashes in a merkle root.
|
||||
##
|
||||
|
||||
# doAssert not(rate == 1 or rate == 2), "rate can only be 1 or 2"
|
||||
|
||||
if not chunkSize > 0:
|
||||
return failure("chunkSize must be greater than 0")
|
||||
|
||||
var index = 0
|
||||
var leaves: seq[Poseidon2Hash]
|
||||
while index < bytes.len:
|
||||
let start = index
|
||||
let finish = min(index + chunkSize, bytes.len)
|
||||
let digest = Sponge.digest(bytes.toOpenArray(start, finish - 1), rate = 2)
|
||||
let digest = ? Poseidon2Hash.spongeDigest(bytes.toOpenArray(start, finish - 1), 2)
|
||||
leaves.add(digest)
|
||||
index += chunkSize
|
||||
return Poseidon2Tree.init(leaves)
|
||||
|
||||
func digest*(
|
||||
_: type Poseidon2Tree,
|
||||
bytes: openArray[byte], chunkSize: int): ?!Poseidon2Hash =
|
||||
bytes: openArray[byte],
|
||||
chunkSize: int): ?!Poseidon2Hash =
|
||||
## Hashes chunks of data with a sponge of rate 2, and combines the
|
||||
## resulting chunk hashes in a merkle root.
|
||||
##
|
||||
|
|
@ -41,7 +71,8 @@ func digest*(
|
|||
|
||||
func digestMhash*(
|
||||
_: type Poseidon2Tree,
|
||||
bytes: openArray[byte], chunkSize: int): ?!MultiHash =
|
||||
bytes: openArray[byte],
|
||||
chunkSize: int): ?!MultiHash =
|
||||
## Hashes chunks of data with a sponge of rate 2 and
|
||||
## returns the multihash of the root
|
||||
##
|
||||
|
|
|
|||
|
|
@ -0,0 +1,527 @@
|
|||
{
|
||||
"dataSetRoot": "16074246370508166450132968585287196391860062495017081813239200574579640171677"
|
||||
, "entropy": "1234567"
|
||||
, "nCellsPerSlot": 512
|
||||
, "nSlotsPerDataSet": 11
|
||||
, "slotIndex": 3
|
||||
, "slotRoot": "20744935707483803411869804102043283881376973626291244537230284476834672019997"
|
||||
, "slotProof":
|
||||
[ "14279309641024220656349577745390262299143357053971618723978902485113885925133"
|
||||
, "17350220251883387610715080716935498684002984280929482268590417788651882821293"
|
||||
, "3614172556528990402229172918446087216573760062512459539027101853103043539066"
|
||||
, "9593656216696187567506330076677122799107266567595923589824071605501987205034"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
]
|
||||
, "cellData":
|
||||
[ [ "211066599696340205996365563960462032209214145564176017965177408819390441927"
|
||||
, "256399834032317991719099034134771774537377713676282398278615627599320306708"
|
||||
, "40526956212941839024868120947422067322935297516255336725720469887577875470"
|
||||
, "369406626072040375689238003388146123438765868500054546379159741776926336393"
|
||||
, "333671948877941061129138970028865844558745266314244224980179694307884999701"
|
||||
, "12844191661993811401197260475054004253686019503294245287625435635597431176"
|
||||
, "103242505924551040184986153577926463300854479121445566984435820844932904541"
|
||||
, "357267598134410031301285377503939045679462829143562392803919346036584141082"
|
||||
, "162961530392479745020607594774288130869393650464001426863668385541077786641"
|
||||
, "426633666684068667053061341108297711407154520752589357873877687002123230254"
|
||||
, "131217200633679697678620573903653316618851166977399592012996945149000115543"
|
||||
, "347806146382240882424437318028620747118202052207615339887983883245341422889"
|
||||
, "373560143578047669373240014805175743607410583882636003337120578008437374619"
|
||||
, "188643112726610812698950916978832639394489168469865816132268079928321994342"
|
||||
, "261643693073361806247543578456646407192325702146752104760117340650316255422"
|
||||
, "260425332276127964154119199351807753107358504026338378706506655351595199132"
|
||||
, "374895089121103633563000003194929058314955925591115574099805066048344387554"
|
||||
, "251251687633538360627384151887287228926166832908633974282160065378311171354"
|
||||
, "72870348025150463132527129203816383011245664502016571773976961034605631401"
|
||||
, "234969517550818492643515432666311755833657223377594670026839098818269671638"
|
||||
, "250704662734070531273640113211555977086430125469327371276827055724111014200"
|
||||
, "85287059658255939741261887818611116570376488685014337052369839580946343903"
|
||||
, "148959658976765873884541474400081762855732313547557243964921157967555048302"
|
||||
, "402116967301520959272239788104745348344918207829380669065019055837727389479"
|
||||
, "440503687192139964066297025050080823601005280790824714962651368433530759519"
|
||||
, "149064344353438643307231355233484617851197634669308957706338957575177745645"
|
||||
, "249140840255377018814348914718469942883867200392561109698520706525194687651"
|
||||
, "108796851176515124780842490199733462942992752881710253277179665118758071359"
|
||||
, "168245155425564161902686596247453762364112240129335852645432169468767513906"
|
||||
, "129930759770229612264501691941454447321585806001002088749773920103899362070"
|
||||
, "26204732465033162738545933008873662562758704699080684615280202127289894343"
|
||||
, "434343986187775289860542785529657444690073100436887472033336117760907652966"
|
||||
, "361202432740487795596808128962740911600093857340619816047190021218849540225"
|
||||
, "100616813001075101816809823591611435583084916492802624686700765550893945525"
|
||||
, "262383566766515164611427346701355047932794351290691325516723194829671679460"
|
||||
, "223966317895663049002893008659178463136086169222436544014405848127792334099"
|
||||
, "416071800998333357259662686053338495720384342746822618737948080251761863079"
|
||||
, "402582378631671531909245563300554883898015688826468151075059467077182712018"
|
||||
, "271682867846395286938993638577506552857925968097084028550962231439839229096"
|
||||
, "447239701195120035004067146414333903010840427278848992921567785105217019890"
|
||||
, "275718367441702584521943480326858762208121719038684001399322597215141179102"
|
||||
, "86424132534636958411139699704035287483951667275825975536719519441147485752"
|
||||
, "149332313586183975744469174256094358432564607635406143904268565140988616920"
|
||||
, "431284330776421588418608279008210842837281123158951642164956884286883748089"
|
||||
, "328694410745471749523135644660455669483988686888634622076863114197617693825"
|
||||
, "112671940998917362968156144648543607958275336559773039070338509488400224090"
|
||||
, "40612851250697989627190554726382690498263128439797780029697069621854862060"
|
||||
, "235047914228675997216342104196597257178021277585839376175077878186492271543"
|
||||
, "169718735151210135199527910197065439221144015957220768545119706561079163228"
|
||||
, "345850109040121443591415752965486014695272086901102608769402892906795715635"
|
||||
, "107916794601837835951508003838161872232087225679609623116098837565956752373"
|
||||
, "415195406060373162425374246348197423165008252112453298469572523506488563795"
|
||||
, "18574536720926634955170276058049993354780447816096707123565164996905722992"
|
||||
, "77316964375201096801231570737992491072607736322255206149311341101525354519"
|
||||
, "198566211140075666401818378905444403588640345933666108724809349396921957675"
|
||||
, "71926050707400318807625168942501384117254391471312636171205913503928815127"
|
||||
, "303403754792341398295052586417779761162194818661412867094096550880325459639"
|
||||
, "444796230931706624375881141460151785952798771079111017071124833045933389733"
|
||||
, "430832727519144759265424205289522948157007336118070755365887670730658782114"
|
||||
, "75431213985866235726407973044434444984663930761207296437571668004273515965"
|
||||
, "9242635103653159191249730220870735855877366952081272723035956668095954838"
|
||||
, "93770678769846326584848478152412123903909949977598807336203128684179492141"
|
||||
, "438043261966019084676377174988310087831395864304423411701911688757793135582"
|
||||
, "175357918416563657734972138036003712114814934655676571874083109097142591069"
|
||||
, "301619681954194702458985259161884119574424456150215738560639417824784261940"
|
||||
, "376627771252167062559065889174364784087884871999807562320457079200311413098"
|
||||
, "77407"
|
||||
]
|
||||
, [ "424838222746158988229624788694939151178615656210585621868910231014323837551"
|
||||
, "113188486246955346418956949679485575685258850346101035778277727456423482970"
|
||||
, "275449978206132565019222655023969430014622832597654643418394602485645803413"
|
||||
, "407856757018138010439232009766252068440920591566039673660874626878413077905"
|
||||
, "433369046493777496016384947949950877133856528218602671493669395706908819748"
|
||||
, "258364166531180422149545015891786023981872586904946376136311798402581278793"
|
||||
, "111997719221028147522803956659709775148434180015507797582340706052412284571"
|
||||
, "370086597976426508280413876667101417393598181708677558733730556109327409076"
|
||||
, "394139601979449259075910238117153992797849553309541269624848742084205563806"
|
||||
, "224088276319080487199395482893988152025671468715318212801266537036640477323"
|
||||
, "412710245119809501914481942088314642684754542082140451180970198371889738885"
|
||||
, "353872602359946553306242341162348980635834907495492814598834657175405697176"
|
||||
, "252575199506028475372678621140654219936768774012823764047177692104580452933"
|
||||
, "259093153824033122452869556249315839899366493071746219770487886456301642099"
|
||||
, "433829976798312333371154167497560676982294392632725612538640639617101218872"
|
||||
, "69918581382122563555200898078544150952625715196904114153232367538572342772"
|
||||
, "337937520623192257595352158476909569245988839238498098464935654555688460123"
|
||||
, "264739973232292969253318643276671532055689422253184191167449284055983944338"
|
||||
, "326116252818829775096345069850111970510714050346103409479803743342138260656"
|
||||
, "236666502377413649728378889488706275212721356921124776708954261777813709815"
|
||||
, "211625935799984260567718228446525455893664313064841539301444509150157287163"
|
||||
, "60213206239417039999880027112341976360540689886703427811513517396638607512"
|
||||
, "68310118105957780876770075529546844404225720757669797609686816545988561625"
|
||||
, "423863085551351065136684030270731679105571943009795949621903966660399419936"
|
||||
, "388914614294393005039878123500859325222684672184567792659076815268598434245"
|
||||
, "449456790291560508709069826219925144971979653209755565240911568965768874382"
|
||||
, "448810363770763694447869940916735951256986784286793489549428379909616059117"
|
||||
, "93646909783664049092056237949587618925209622020026157405117796611689551192"
|
||||
, "352210795298632954574896499649181574584074853828419384742874364724522457331"
|
||||
, "37455517056393404525863484733101879886413925183061645520768912552476716150"
|
||||
, "386617357584684336812125385078476270301738184058813703112840991226785114117"
|
||||
, "309940292044597334261558429176136686101590729982259514472573656131579113438"
|
||||
, "375815246167575100319857872432879650174355611853064771241069582477717074415"
|
||||
, "332214507344122806007757734266883566559371568252728459951766124888176633706"
|
||||
, "148990259460952914990881100852534318351247069504848477833147446514732789712"
|
||||
, "328669527889838880414072022433859139004058211332184916573516704632073044118"
|
||||
, "39278026039348543645873027549112998051601664395028652771103624299930924528"
|
||||
, "147717660530589785119644237145092759103012624422229579698752386490700965238"
|
||||
, "374018518701362017594752095877197725242352803195413267746619111489936980685"
|
||||
, "19185486483883210730969367354195688373879769005461272861759636600984416877"
|
||||
, "61866046506558157021682973167090213780467780519546382332208868591026703563"
|
||||
, "186854966504766517012887726614015646154225796572138017810371160981778288347"
|
||||
, "87813550250328892091332566928942844770632705056120813488729800874811845697"
|
||||
, "207775163424060266085108794048371834145545842567796157378282772999548202308"
|
||||
, "369987573847786237689249538753881486995686208870889713177975415012214427429"
|
||||
, "240880979016395044518849230927615466120209140082149273390921042537474853143"
|
||||
, "174902051454932913375934735427101804474275543418199101786687925733405159872"
|
||||
, "342217255652950822372803598682842961053537267723988087801275319754065261308"
|
||||
, "403207518067666945448161377960706451817747922771285796668778802535227939962"
|
||||
, "407191459999036791052261270163259267557900498930952521056725210031161568230"
|
||||
, "338216472523551728793268225845396218561132966393725938551091882807069657206"
|
||||
, "118364222678576550870375112494142500603091119946985793934499723872824782886"
|
||||
, "269721611028665323587192624288165848310917029418261851436925829954710472436"
|
||||
, "227424498125745236125352117206136621428869662458452610254773560636280935711"
|
||||
, "334380807433339401906359555583987757411855090694162252260781648609761248049"
|
||||
, "42470806516174819075107446234449247453971524726021445768611797530804156161"
|
||||
, "418994916402918322951830716592888390611524984156817012683478842068581638820"
|
||||
, "363263142412048420546716019774090729399081311227606555141174736853886128407"
|
||||
, "192292715923468025058557166341646729623133127372303791236447550026886802680"
|
||||
, "450253878713722337767292128303865371116770532625906889925779639839924402495"
|
||||
, "412596147086332805611392200560087191411541689130482740065137300252639590489"
|
||||
, "264059866105067484811456793906835462997849523506366903974724979936196358724"
|
||||
, "80922384260325792825608274004101667366364502159441286540209512108302572137"
|
||||
, "69261112192907071823876894642934902051254647002357333297635511793652550535"
|
||||
, "342810644068354896385837929029331085645928473943142618800192452300937098227"
|
||||
, "228361826362950356801549793202622774869100858404479869989505302905936946659"
|
||||
, "89244"
|
||||
]
|
||||
, [ "359926778925154809567585559910064420821311221299937568759183366972901588855"
|
||||
, "128688825421597555299113005875649976344934035310192572516197551102045945994"
|
||||
, "225379354482053502673695304662016054746658452775856157938886424152134693969"
|
||||
, "321872319934407904743844034025550739031781361848065513098922085967524180784"
|
||||
, "250375637325405951645070615947051799520095842815922754899017741501395744611"
|
||||
, "97676493052788080672307677749501730337551225267342432472194527468634722352"
|
||||
, "140101187036396881926000630022834247990512766860086619783437252676747730662"
|
||||
, "428833039353549335335605804240430918631639449874968791377641834408506136850"
|
||||
, "418359203734539413977740838354554804415161215624809316660001820037711273005"
|
||||
, "197411877418795659129213175603102238904459737200167987167255995825203749339"
|
||||
, "221646252100316559257470803343058462853641953756978011126414924636869625612"
|
||||
, "106393540293584181037890192557883541231531964825708650997196071036779482686"
|
||||
, "121473330828208543539643554911190528237124877123871673169194452404939833883"
|
||||
, "234055622144947293638512253368547046093971383516706577723613696162225606040"
|
||||
, "68307451767502390304445005915787226559811433450721625085741437522802389574"
|
||||
, "446891883436763112014492564462451523127134145501201571918449345324780633462"
|
||||
, "83718652783543018019599555197511164121642363321504039439786358267060414949"
|
||||
, "90267297500929836073049162292104311427365986272517761342871530827272320168"
|
||||
, "398425606698859520268856768787424690872952910789754531465894080258173664751"
|
||||
, "323570139379118444589557840594603212198136718240764273769210492735883659788"
|
||||
, "318597103584099056378057647488068323974418467250708490151864712850204121402"
|
||||
, "6299083430773359277240726214182464517380839990956316267425262319606033077"
|
||||
, "27638206326925436960316131682014727983280820447721477666884742925275976240"
|
||||
, "434344186848869917381375812528446841024676181532946456237439060027443649574"
|
||||
, "64735754118644738348599761514174981324344130078598038275246522384474432918"
|
||||
, "53068717269762105498508401788249005564862415051761175636612434108259085043"
|
||||
, "35813044996911619267309099508360887777226716179396659295580849861836012116"
|
||||
, "67751791392924142809580984450371772015056060429352159361446914484238646676"
|
||||
, "68534949135677447506316576616742207938855454921330757052467057435206318183"
|
||||
, "98510151949547604999069864337574320742530406479752176012935179772005228326"
|
||||
, "342190252152505345443004241184891966319091967630257822491352072978326623645"
|
||||
, "362701658859425316334005554473186516818256386066010799465369887406035738447"
|
||||
, "266999116654850467726292928465517542818678046748008340458185725047959981772"
|
||||
, "227089355966197874086821090531951502393729872265201602128464978982907992285"
|
||||
, "240800343500959216904535208047288234867926058830277460630902462914796702354"
|
||||
, "447956858573680756485556898469710354624642653441041335815079102198306530583"
|
||||
, "89422712944117481549242421245588048728782658978853365197341587057196539094"
|
||||
, "72610343179362050463955730204044877712105879926618304878262944723464870506"
|
||||
, "8676698500519447254981838968537883138182811064381378248657969913325524054"
|
||||
, "180453700216061196739413267121764366438386031039311941313645439527087166894"
|
||||
, "63346784016053253727621352882430122335280702556586808389293772066791583857"
|
||||
, "400031453850139978805133735852120986024101930860924735862305444817383365395"
|
||||
, "230104622290558570218036071349472289358926019290368625724986905348610140188"
|
||||
, "175689489221336091369196327293045073133701056385414159213338224521167050830"
|
||||
, "73310331103509697419315970265031228794034932318600293733074730812482185479"
|
||||
, "371383255255842707875498538452907102684511927320158672347778293876877893808"
|
||||
, "165319345890230193939972313953881372394171342391835626454759321320435952720"
|
||||
, "184753541001210613115361457830691571384268642766106935428207829332011259768"
|
||||
, "378810733004878432563271790625801205570962762395854282745166380274493181314"
|
||||
, "86321674336629444862383262780020871828941143514651008200999725989892879714"
|
||||
, "332634533993388248915777870988529817692938793120418377552247997050250349749"
|
||||
, "41742010257820712511267405684781534740157292266212120171929853763272599516"
|
||||
, "224101330592139734390658213442359402546038743346294438455635537496290117560"
|
||||
, "204363902046137087420878796391135987770418514008394389852388361468850216359"
|
||||
, "296526036888463156867153847102999031430641220536577556044554029990778763710"
|
||||
, "137568796227115931047082828377698464185467276723279763304686078869351280509"
|
||||
, "147456720891843338735232645318045656082153590170441596326500204393398792771"
|
||||
, "297291342309626392877004635010131510068476699687818509485687346071074942006"
|
||||
, "20748013593486831233186810444485136836664689496258849465507672301203832324"
|
||||
, "335431726883875036252568773858744219410188302724423071283643221351691013313"
|
||||
, "50487384098835523033417523223562560857744547945136829388944024807752630716"
|
||||
, "425952679139710019732649156938676226492714486376811103362264146658191708598"
|
||||
, "439787938069461539508728805881194071103269524405653997968488488049426387373"
|
||||
, "279863410796988495259178322026394289028023166886112128504434877538089779477"
|
||||
, "398941099058270093463626870965433502581301413791423667994257456160377865247"
|
||||
, "5759692644185723491187478313102540786562625675495805072053262277490225012"
|
||||
, "115176"
|
||||
]
|
||||
, [ "199440901482393381753657315848210955092644686957900737971520778206058989647"
|
||||
, "339215657660349719251187938243949119362753238126598750470832739830379601048"
|
||||
, "17957417011314891245400567671664723859427624136284133790923936126779445290"
|
||||
, "294761585889095249928492042608516765584723814657871392207964321318076158536"
|
||||
, "367304199921887970655898837985855454346911090426896946930048519042744277770"
|
||||
, "173405546837606747721292792526074597519538685230485266741646923399938591491"
|
||||
, "13202798104529529703580600642858924379886936325402696094624200032343206719"
|
||||
, "28211272278315691894282764239983301742024079691520980592618486887749800025"
|
||||
, "73792448247120972778500624350664849847034095641998271809779791788652649022"
|
||||
, "386961947078838359430674078072441680475090687247027225632133013772954043342"
|
||||
, "247859266401821616700765969075270662915024391205665146199401830650793676517"
|
||||
, "243938047874995926342875119559105623088213951205962677439167259642163766960"
|
||||
, "14909501249861872673329370269106359532506159818320693170564856401208688898"
|
||||
, "200331653478898243177761429526240803993101536716611440775588088625522029071"
|
||||
, "127891684617049394579738365914860024509007913559921966744972525605976847919"
|
||||
, "202912167983786187592861727924433749852786012202809544200943965898118027816"
|
||||
, "176370650316309755425558132466370508977563252342126855874617990006444464573"
|
||||
, "179490319446297562731655155074196383539396893457024237113284509223965454107"
|
||||
, "118703379899134287650980989454755985628620830085932176414465582081598659194"
|
||||
, "102025594191113707886629488454876652037341696284939671367279050169222988689"
|
||||
, "421132375430104331136732058548913808473025321492255503838896123601628815453"
|
||||
, "328334791815856213206267892694535121226673194052951034497930366807851111845"
|
||||
, "83012322813281668737061895967682970093636853452224812061110092135287899376"
|
||||
, "329204708391107220275172926348002826875172581958734129645882445887919889321"
|
||||
, "410748869385474485539728045765785256294532845070137964737879849265390625591"
|
||||
, "197274807717335387012872999914232051341799797613667869923402386359379902675"
|
||||
, "235713095185988155553500217595661312303861624791720350423698435045853678746"
|
||||
, "150631584359141913552843813384841153102535679219851913018874172414598353488"
|
||||
, "207783836843813911284913666774420250626019971129676431904589416081127483900"
|
||||
, "15728034718954665549174230921445077500399069880555428849958014406422697976"
|
||||
, "69799423545177501667748653663121504155622623013014583690766769624482972893"
|
||||
, "265665371394145560256710553779588458846778960884758560117805511822299802326"
|
||||
, "149195925869584039415331014261414953602245337159354350672030141190471260449"
|
||||
, "162328395279114064180857718453973759378615891406692054752029241989300597156"
|
||||
, "104643123291849369328362168243087456326274773811561124607060302871149280568"
|
||||
, "320704123383295724141970902124509237736831907552238934395000039155541269937"
|
||||
, "77914486216152383968400591704791756847610018757857965758408455442143531631"
|
||||
, "238365259321088298905088250444146071545398991768186471689605160523129613763"
|
||||
, "279409375422154593510552116666741774885392805494152031820287626934209978908"
|
||||
, "195118776021452609708543280509123101181249086555819844047203390788132717252"
|
||||
, "197977884437087886153482042453896079316138251415359773453987232734849953584"
|
||||
, "168185043240980087638006965666857387510828226074470344841850764460631595331"
|
||||
, "231157923359356077977363679818678437836536420121744865399935742538602805912"
|
||||
, "177903771863742191900138437188329108771172098110036075491750018158192424072"
|
||||
, "313552174443290416730632310997197097951229162137491595709733420111980331403"
|
||||
, "273253450712049988786741336540196077743997302924525995219038781650977490211"
|
||||
, "421908030281055821389377531613150504859996607596444776050212044919345332385"
|
||||
, "180108184992593746898140529947178182204857361841304042854173884504394805936"
|
||||
, "37075272799330399065679301151342697855905822543084867570322173216259074746"
|
||||
, "364885615491975468180698816037289079302391684470748470356247509218051645743"
|
||||
, "397482868106190800111749908137311511782294652288655656060379563261618687603"
|
||||
, "192853269627895017416381451198403197013798085262867793715522216458130791820"
|
||||
, "450480853450142791928572953497280890976328598410525218090104787739409705079"
|
||||
, "40278654070502961330170439514434669768416784968274956021022493731441898222"
|
||||
, "251277143131769020481025315216040765839561111684608785317366609258959942695"
|
||||
, "95094468748825454459610961968601800404132682484160170977941285794444806916"
|
||||
, "160586633865113902389134480029183924655750088163491531655080014223476604929"
|
||||
, "211661229493873434581423107377168478957907088187044576032505407590783850232"
|
||||
, "409651293631434750272174456674594508340952460788494035327119354167465019826"
|
||||
, "233213211946836553080627522409887799285199986120567245752841080275284294566"
|
||||
, "143182900674482411759481361336063079267405785923487419697568907351386146653"
|
||||
, "430050085956999990041799366370428972519385994997821389120583306252090911051"
|
||||
, "241257468571530133762629460194062384921386438426367660851087853915892684115"
|
||||
, "106478922860328643074356032194102718325829441005019365153538120054339275205"
|
||||
, "252933430690486626644000908036895289326526510137514385824014300035301154822"
|
||||
, "242924628511152862437189415942615812459145003511499320946981326550434266392"
|
||||
, "107566"
|
||||
]
|
||||
, [ "10892488375325920610152294701785926476935321890453222549428070867493882259"
|
||||
, "230776541958253414701326765204413805639078570664616139597900921490475143840"
|
||||
, "162235550819840758141721889536295480113278275911087429090017541814695333320"
|
||||
, "318634611531007856220026646570492630940047240387334221027051366284225674524"
|
||||
, "347695480420330337439096561080975864031317110001559084872978387417651465445"
|
||||
, "243301070227446762125488369714708670219289121859111929877695012393609726208"
|
||||
, "312153141205681954392441579373652470334482191990502803524039048245142338874"
|
||||
, "243769659456658813016931656268556893289414122189859136776671772112510762644"
|
||||
, "235510946617019540983239794402008700611014664578713646813426116406215382253"
|
||||
, "394638234040056271265534896894991100558052611842099314808878257003754175212"
|
||||
, "112730195097163222179949980634599934392634120069300673310070655800491242211"
|
||||
, "112545144551723145227061757291353149296490850338535641681237178578430772288"
|
||||
, "399161925498018051746424503488168548076537557369822821644005390567188305750"
|
||||
, "291823556779130095044851238536860577785281148245623338113991004798525195947"
|
||||
, "443006765181360772964201996695219533539991249361936300010158089981376772939"
|
||||
, "74018417655448362012716269253153545524112649147848318337218250865231619883"
|
||||
, "361038295627629757439073080656763633087074408866229383288288831546300069767"
|
||||
, "269655542872834422597617091710145830035767834433114567250727497135451412216"
|
||||
, "58289072717559976781527037332370898163550414718655391280446986067842922181"
|
||||
, "365399954331278626941053447122605263207706937018525782908294459265663426953"
|
||||
, "83576501872896181822963149907518188809522341045703363171261203985876068484"
|
||||
, "203403783686919490357886887779316222746544665267595235714814853282937072937"
|
||||
, "226090172488558641139782927103632452136731207206963592401497570070700221117"
|
||||
, "249813560776802008219945831355163226098682741553305882699128946352901227282"
|
||||
, "236586835155013316983057471119105606475813711035522306026673814184519826069"
|
||||
, "420611449257527132395951061920868895981487081726689195426673433565077012458"
|
||||
, "414979562418422585161189066520079202660418565693776869373399931253524536378"
|
||||
, "115851377630895049619958829726730418778383692593973233290077769966938018584"
|
||||
, "248071158447148977966329235335508229928647083339924308951291087789494073866"
|
||||
, "8254100651607835318906499132096926759832050649688561036854000785129084907"
|
||||
, "91385483239630999205401307763890943367451672856478206313711901403403429289"
|
||||
, "369346641925770698935046191632374874762045839747680407985443471202350286304"
|
||||
, "236809023553698844817613980395588655518336237009075203833474757804664254158"
|
||||
, "8367847400805682648908349286699722579431227561083498361702537964306290078"
|
||||
, "599241730770400067632779114435635342549228985534229813617556932580328166"
|
||||
, "347112528350917448294202348829052279076907614831011498643025223836596915573"
|
||||
, "384244379244118003891043669466323943736726794634167201471569326059716944701"
|
||||
, "118013777197672343498581960057939216208494837962825017767101107204031333144"
|
||||
, "27234916267695376599463409893017377196853108034589808756909998459364893467"
|
||||
, "443519198016088819704735929590164254445884637317806160485888215392818578737"
|
||||
, "396780482611567392375183169345153676737175342167284140440545202776279411157"
|
||||
, "420351155303051883480975795437743307852799862858964108014000673383502660760"
|
||||
, "17379377743250873773932440622865094720355292220433235366224143179854831702"
|
||||
, "299671454782683147939928632170233327590769402224392134648893444626929909373"
|
||||
, "143062753141414050359792615867774312517100868919516205179025540179759009492"
|
||||
, "79497692490953838158801094558761984613913564034406069659969793097043605498"
|
||||
, "422748645389700647011491406944374966856916994331478229959954030359911549565"
|
||||
, "101802829812014644970197499895811874607753186302439171072935333706660468030"
|
||||
, "376428369998893026519415315112012919906032811618495880392785036762185101192"
|
||||
, "193969030999254195249242252871597931610859408264053152789041067245597391073"
|
||||
, "262277607928686742238285487190873200602833495734085188071246746209841324139"
|
||||
, "154099884960502807271641574310268486840763221700920893692135020347157046386"
|
||||
, "155875061164585018658671842995328931296342883770473498362059838106632382461"
|
||||
, "248574435283666782825705601259695525637993294311364397935499480206725256362"
|
||||
, "171325185063038052248966557755722232979612702743265316145563443527135798688"
|
||||
, "19982746887818897250405980185937061235439217109294376948752373205830077881"
|
||||
, "363719103724181291346130833008745948141602173373912337315631878022251200824"
|
||||
, "174596812883797666354579747966720458118607233660798323559531788300018084931"
|
||||
, "296611197821867585469311917529661698312828606304722408477045992233526328708"
|
||||
, "115884038550627840768260751168697204665962542002024023842649383174518895165"
|
||||
, "265597417366164841889730505737916646261040505851159477903649773521314216810"
|
||||
, "59890857222664166616144499778264545115438678877460924559608721809777680238"
|
||||
, "150275344313515259978222149421664752546204516114655782705875535407472455999"
|
||||
, "119762211657733951640135703777940013374803447228667976561992857374027112851"
|
||||
, "124750313254270944205036764607428674226393862430770588148329786501973600535"
|
||||
, "223562415856611692667255745278751292230311455355383541649772348032933666931"
|
||||
, "70851"
|
||||
]
|
||||
]
|
||||
, "merklePaths":
|
||||
[ [ "12330511756602656312909435206144412037562550923861053314147193494605624608532"
|
||||
, "11626412651279744307149389430094868876817959605147995292836995419396445628874"
|
||||
, "5992799448428980485292103987999399446160713735809250167288027256759377161164"
|
||||
, "19665782623633007009046423286205585606029554995684266199914536255425604862856"
|
||||
, "16487082247902739801276638140231448150060608377815926330854457096549924699346"
|
||||
, "13757776896542890425183206586238760231244468647794123671233758868377423038254"
|
||||
, "5689382212199289790852870716648320118282913659439556386010574440270030991956"
|
||||
, "19397819444005071538172083272808000152189643623980212079104170950073560541073"
|
||||
, "13602141253349313166682170066159161155345481788270338632198566205283140117430"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
]
|
||||
, [ "20475873274412005481064639175076772466269555202404881992985142339142752174247"
|
||||
, "16346160910918020455716115524174351294558871582658635761456705803199256575588"
|
||||
, "2853750013041818757293788273269978233503065226276819540991640203844566736443"
|
||||
, "9192572535522846104757923561847416578417599996904208474460700268961782558170"
|
||||
, "11041850361074018956732434352769265303294549935255362322653487210796196161858"
|
||||
, "20835509643844784930831626622202658364476409300598072395494952478408974334325"
|
||||
, "15426115581767819720710837762133134950520914636073261355445708100826108573907"
|
||||
, "7565353224987902191368863653499353764559862741092477570130316358454603122676"
|
||||
, "2622681935585012630617774892501744551457568716225188460692779556142778732663"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
]
|
||||
, [ "13099868869639061574800067722436547911616384523753701384396275064543709640456"
|
||||
, "353757809120595355213201328586632712724405232919181040928026587840976194078"
|
||||
, "17653300914565730132855106316678548541847283141888106932466281385199556950861"
|
||||
, "15467462085462582082877261755656498905479817107855355753427299990712166382496"
|
||||
, "8291733777946446853018893495264026584437749231931118771866890345692346711355"
|
||||
, "15510790697317206014779022286261864844918915222875014882833700758879700055506"
|
||||
, "5689382212199289790852870716648320118282913659439556386010574440270030991956"
|
||||
, "19397819444005071538172083272808000152189643623980212079104170950073560541073"
|
||||
, "13602141253349313166682170066159161155345481788270338632198566205283140117430"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
]
|
||||
, [ "123238869181525326412236116167249816873084559218151119452851092131080991962"
|
||||
, "9610314342084317647296061595824603740114670828357751076517430572434680540425"
|
||||
, "16802740554584732104294972716558962567961331277692246600846665155168171370476"
|
||||
, "151083360419914122898584757765086723506432610661508069194962432698872036623"
|
||||
, "10357032992337239725601662829902169825217513617307319193581711776597892496381"
|
||||
, "10120699018002766520605012835043517238241846918467244955580419060582311503402"
|
||||
, "21149604008153751948441881526949680605328007895979738537313721955134548786062"
|
||||
, "5720106921878932614189421948890362637585879521377362100104826996201092964473"
|
||||
, "2622681935585012630617774892501744551457568716225188460692779556142778732663"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
]
|
||||
, [ "19153513112714782931111012244694922243101677748840856395814929006033044311081"
|
||||
, "21046138187228318287277629107063936540039891592394801899272249280765102572688"
|
||||
, "18057980437430910028015917806534512217725128031222973066601095455076586015436"
|
||||
, "5766677914654397407881589917461473873246279171605373166264025525757502238061"
|
||||
, "12019967669236656188577515900815533059046454955207846938479617973037184411021"
|
||||
, "14504305765289705714959523666100275156034056689367568164630385862257567596209"
|
||||
, "7152002871325824138073253423783370852632926621899161541618248808716037342022"
|
||||
, "9714587356194206699401761190093056901650105401919163689816999407566849779455"
|
||||
, "13602141253349313166682170066159161155345481788270338632198566205283140117430"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
, "0"
|
||||
]
|
||||
]
|
||||
}
|
||||
|
|
@ -31,6 +31,7 @@ import pkg/codex/erasure
|
|||
import pkg/codex/merkletree
|
||||
import pkg/codex/blocktype as bt
|
||||
import pkg/codex/utils/asynciter
|
||||
import pkg/codex/indexingstrategy
|
||||
|
||||
import pkg/codex/node {.all.}
|
||||
|
||||
|
|
@ -54,7 +55,7 @@ asyncchecksuite "Test Node - Host contracts":
|
|||
manifestCidStr: string
|
||||
manifestCid: Cid
|
||||
market: MockMarket
|
||||
builder: SlotsBuilder
|
||||
builder: Poseidon2Builder
|
||||
verifiable: Manifest
|
||||
verifiableBlock: bt.Block
|
||||
protected: Manifest
|
||||
|
|
@ -84,7 +85,7 @@ asyncchecksuite "Test Node - Host contracts":
|
|||
(await localStore.putBlock(manifestBlock)).tryGet()
|
||||
|
||||
protected = (await erasure.encode(manifest, 3, 2)).tryGet()
|
||||
builder = SlotsBuilder.new(localStore, protected).tryGet()
|
||||
builder = Poseidon2Builder.new(localStore, protected).tryGet()
|
||||
verifiable = (await builder.buildManifest()).tryGet()
|
||||
verifiableBlock = bt.Block.new(
|
||||
verifiable.encode().tryGet(),
|
||||
|
|
@ -127,9 +128,12 @@ asyncchecksuite "Test Node - Host contracts":
|
|||
return success()
|
||||
|
||||
(await onStore(request, 1.u256, onBlocks)).tryGet()
|
||||
check fetchedBytes == 786432
|
||||
check fetchedBytes == 851968
|
||||
|
||||
for index in builder.slotIndicies(1):
|
||||
let indexer = verifiable.protectedStrategy.init(
|
||||
0, verifiable.blocksCount - 1, verifiable.numSlots)
|
||||
|
||||
for index in indexer.getIndicies(1):
|
||||
let
|
||||
blk = (await localStore.getBlock(verifiable.treeCid, index)).tryGet
|
||||
expiryKey = (createBlockExpirationMetadataKey(blk.cid)).tryGet
|
||||
|
|
|
|||
|
|
@ -126,7 +126,7 @@ asyncchecksuite "Test Node - Basic":
|
|||
codec = ManifestCodec).tryGet()
|
||||
|
||||
protected = (await erasure.encode(manifest, 3, 2)).tryGet()
|
||||
builder = SlotsBuilder.new(localStore, protected).tryGet()
|
||||
builder = Poseidon2Builder.new(localStore, protected).tryGet()
|
||||
verifiable = (await builder.buildManifest()).tryGet()
|
||||
verifiableBlock = bt.Block.new(
|
||||
verifiable.encode().tryGet(),
|
||||
|
|
|
|||
|
|
@ -0,0 +1,143 @@
|
|||
|
||||
import std/sequtils
|
||||
import std/sugar
|
||||
import std/strutils
|
||||
|
||||
import pkg/poseidon2
|
||||
import pkg/constantine/math/arithmetic
|
||||
import pkg/constantine/math/io/io_bigints
|
||||
import pkg/constantine/math/io/io_fields
|
||||
|
||||
import pkg/codex/merkletree
|
||||
import pkg/codex/slots
|
||||
import pkg/codex/slots/types
|
||||
import pkg/codex/utils/json
|
||||
|
||||
export types
|
||||
|
||||
func fromCircomData*[H](cellData: seq[byte]): seq[H] =
|
||||
var
|
||||
pos = 0
|
||||
cellElms: seq[Bn254Fr]
|
||||
while pos < cellData.len:
|
||||
var
|
||||
step = 32
|
||||
offset = min(pos + step, cellData.len)
|
||||
data = cellData[pos..<offset]
|
||||
let ff = Bn254Fr.fromBytes(data.toArray32).get
|
||||
cellElms.add(ff)
|
||||
pos += data.len
|
||||
|
||||
cellElms
|
||||
|
||||
func toPublicInputs*[H](input: ProofInput[H]): PublicInputs[H] =
|
||||
PublicInputs[H](
|
||||
slotIndex: input.slotIndex,
|
||||
datasetRoot: input.datasetRoot,
|
||||
entropy: input.entropy
|
||||
)
|
||||
|
||||
func toJsonDecimal*(big: BigInt[254]): string =
|
||||
let s = big.toDecimal.strip( leading = true, trailing = false, chars = {'0'} )
|
||||
if s.len == 0: "0" else: s
|
||||
|
||||
func toJson*[H](input: ProofInput[H]): JsonNode =
|
||||
var
|
||||
input = input
|
||||
|
||||
%* {
|
||||
"dataSetRoot": input.datasetRoot.toBig.toJsonDecimal,
|
||||
"entropy": input.entropy.toBig.toJsonDecimal,
|
||||
"nCellsPerSlot": input.nCellsPerSlot,
|
||||
"nSlotsPerDataSet": input.nSlotsPerDataSet,
|
||||
"slotIndex": input.slotIndex,
|
||||
"slotRoot": input.slotRoot.toDecimal,
|
||||
"slotProof": input.slotProof.mapIt( it.toBig.toJsonDecimal ),
|
||||
"cellData": input.samples.mapIt(
|
||||
toSeq( it.cellData.elements(H) ).mapIt( it.toBig.toJsonDecimal )
|
||||
),
|
||||
"merklePaths": input.samples.mapIt(
|
||||
it.merklePaths.mapIt( it.toBig.toJsonDecimal )
|
||||
)
|
||||
}
|
||||
|
||||
func jsonToProofInput*[H](inputJson: JsonNode): ProofInput[H] =
|
||||
let
|
||||
cellData =
|
||||
inputJson["cellData"].mapIt(
|
||||
it.mapIt(
|
||||
block:
|
||||
var
|
||||
big: BigInt[256]
|
||||
data = newSeq[byte](big.bits div 8)
|
||||
assert bool(big.fromDecimal( it.str ))
|
||||
data.marshal(big, littleEndian)
|
||||
data
|
||||
).concat # flatten out elements
|
||||
)
|
||||
|
||||
merklePaths =
|
||||
inputJson["merklePaths"].mapIt(
|
||||
it.mapIt(
|
||||
block:
|
||||
var
|
||||
big: BigInt[254]
|
||||
hash: H
|
||||
assert bool(big.fromDecimal( it.getStr ))
|
||||
hash.fromBig( big )
|
||||
hash
|
||||
)
|
||||
)
|
||||
|
||||
slotProof = inputJson["slotProof"].mapIt(
|
||||
block:
|
||||
var
|
||||
big: BigInt[254]
|
||||
hash: H
|
||||
assert bool(big.fromDecimal( it.str ))
|
||||
hash.fromBig( big )
|
||||
hash
|
||||
)
|
||||
|
||||
datasetRoot = block:
|
||||
var
|
||||
big: BigInt[254]
|
||||
hash: H
|
||||
assert bool(big.fromDecimal( inputJson["dataSetRoot"].str ))
|
||||
hash.fromBig( big )
|
||||
hash
|
||||
|
||||
slotRoot = block:
|
||||
var
|
||||
big: BigInt[254]
|
||||
hash: H
|
||||
assert bool(big.fromDecimal( inputJson["slotRoot"].str ))
|
||||
hash.fromBig( big )
|
||||
hash
|
||||
|
||||
entropy = block:
|
||||
var
|
||||
big: BigInt[254]
|
||||
hash: H
|
||||
assert bool(big.fromDecimal( inputJson["entropy"].str ))
|
||||
hash.fromBig( big )
|
||||
hash
|
||||
|
||||
nCellsPerSlot = inputJson["nCellsPerSlot"].getInt
|
||||
nSlotsPerDataSet = inputJson["nSlotsPerDataSet"].getInt
|
||||
slotIndex = inputJson["slotIndex"].getInt
|
||||
|
||||
ProofInput[H](
|
||||
entropy: entropy,
|
||||
slotIndex: slotIndex,
|
||||
datasetRoot: datasetRoot,
|
||||
slotProof: slotProof,
|
||||
slotRoot: slotRoot,
|
||||
nCellsPerSlot: nCellsPerSlot,
|
||||
nSlotsPerDataSet: nSlotsPerDataSet,
|
||||
samples: zip(cellData, merklePaths)
|
||||
.mapIt(Sample[H](
|
||||
cellData: it[0],
|
||||
merklePaths: it[1]
|
||||
))
|
||||
)
|
||||
|
|
@ -0,0 +1,166 @@
|
|||
|
||||
import std/sugar
|
||||
|
||||
import pkg/chronos
|
||||
import pkg/libp2p/cid
|
||||
|
||||
import pkg/codex/codextypes
|
||||
import pkg/codex/stores
|
||||
import pkg/codex/merkletree
|
||||
import pkg/codex/manifest
|
||||
import pkg/codex/blocktype as bt
|
||||
import pkg/codex/chunker
|
||||
import pkg/codex/indexingstrategy
|
||||
import pkg/codex/slots
|
||||
import pkg/codex/rng
|
||||
import pkg/codex/utils/poseidon2digest
|
||||
|
||||
import ../helpers
|
||||
|
||||
proc storeManifest*(store: BlockStore, manifest: Manifest): Future[?!bt.Block] {.async.} =
|
||||
without encodedVerifiable =? manifest.encode(), err:
|
||||
trace "Unable to encode manifest"
|
||||
return failure(err)
|
||||
|
||||
without blk =? bt.Block.new(data = encodedVerifiable, codec = ManifestCodec), error:
|
||||
trace "Unable to create block from manifest"
|
||||
return failure(error)
|
||||
|
||||
if err =? (await store.putBlock(blk)).errorOption:
|
||||
trace "Unable to store manifest block", cid = blk.cid, err = err.msg
|
||||
return failure(err)
|
||||
|
||||
success blk
|
||||
|
||||
proc makeManifest*(
|
||||
cids: seq[Cid],
|
||||
datasetSize: NBytes,
|
||||
blockSize: NBytes,
|
||||
store: BlockStore,
|
||||
hcodec = Sha256HashCodec,
|
||||
dataCodec = BlockCodec): Future[?!Manifest] {.async.} =
|
||||
without tree =? CodexTree.init(cids), err:
|
||||
return failure(err)
|
||||
|
||||
without treeCid =? tree.rootCid(CIDv1, dataCodec), err:
|
||||
return failure(err)
|
||||
|
||||
for index, cid in cids:
|
||||
without proof =? tree.getProof(index), err:
|
||||
return failure(err)
|
||||
|
||||
if err =? (await store.putCidAndProof(treeCid, index, cid, proof)).errorOption:
|
||||
# TODO add log here
|
||||
return failure(err)
|
||||
|
||||
let
|
||||
manifest = Manifest.new(
|
||||
treeCid = treeCid,
|
||||
blockSize = blockSize,
|
||||
datasetSize = datasetSize,
|
||||
version = CIDv1,
|
||||
hcodec = hcodec,
|
||||
codec = dataCodec)
|
||||
|
||||
without manifestBlk =? await store.storeManifest(manifest), err:
|
||||
trace "Unable to store manifest"
|
||||
return failure(err)
|
||||
|
||||
success manifest
|
||||
|
||||
proc createBlocks*(
|
||||
chunker: Chunker,
|
||||
store: BlockStore): Future[seq[bt.Block]] {.async.} =
|
||||
|
||||
collect(newSeq):
|
||||
while (let chunk = await chunker.getBytes(); chunk.len > 0):
|
||||
let blk = bt.Block.new(chunk).tryGet()
|
||||
discard await store.putBlock(blk)
|
||||
blk
|
||||
|
||||
proc createProtectedManifest*(
|
||||
datasetBlocks: seq[bt.Block],
|
||||
store: BlockStore,
|
||||
numDatasetBlocks: int,
|
||||
ecK: int, ecM: int,
|
||||
blockSize: NBytes,
|
||||
originalDatasetSize: int,
|
||||
totalDatasetSize: int):
|
||||
Future[tuple[manifest: Manifest, protected: Manifest]] {.async.} =
|
||||
|
||||
let
|
||||
cids = datasetBlocks.mapIt(it.cid)
|
||||
datasetTree = CodexTree.init(cids[0..<numDatasetBlocks]).tryGet()
|
||||
datasetTreeCid = datasetTree.rootCid().tryGet()
|
||||
|
||||
protectedTree = CodexTree.init(cids).tryGet()
|
||||
protectedTreeCid = protectedTree.rootCid().tryGet()
|
||||
|
||||
for index, cid in cids[0..<numDatasetBlocks]:
|
||||
let proof = datasetTree.getProof(index).tryget()
|
||||
(await store.putCidAndProof(datasetTreeCid, index, cid, proof)).tryGet
|
||||
|
||||
for index, cid in cids:
|
||||
let proof = protectedTree.getProof(index).tryget()
|
||||
(await store.putCidAndProof(protectedTreeCid, index, cid, proof)).tryGet
|
||||
|
||||
let
|
||||
manifest = Manifest.new(
|
||||
treeCid = datasetTreeCid,
|
||||
blockSize = blockSize,
|
||||
datasetSize = originalDatasetSize.NBytes)
|
||||
|
||||
protectedManifest = Manifest.new(
|
||||
manifest = manifest,
|
||||
treeCid = protectedTreeCid,
|
||||
datasetSize = totalDatasetSize.NBytes,
|
||||
ecK = ecK,
|
||||
ecM = ecM,
|
||||
strategy = SteppedStrategy)
|
||||
|
||||
manifestBlock = bt.Block.new(
|
||||
manifest.encode().tryGet(),
|
||||
codec = ManifestCodec).tryGet()
|
||||
|
||||
protectedManifestBlock = bt.Block.new(
|
||||
protectedManifest.encode().tryGet(),
|
||||
codec = ManifestCodec).tryGet()
|
||||
|
||||
(await store.putBlock(manifestBlock)).tryGet()
|
||||
(await store.putBlock(protectedManifestBlock)).tryGet()
|
||||
|
||||
(manifest, protectedManifest)
|
||||
|
||||
proc createVerifiableManifest*(
|
||||
store: BlockStore,
|
||||
numDatasetBlocks: int,
|
||||
ecK: int, ecM: int,
|
||||
blockSize: NBytes,
|
||||
cellSize: NBytes):
|
||||
Future[tuple[manifest: Manifest, protected: Manifest, verifiable: Manifest]] {.async.} =
|
||||
|
||||
let
|
||||
numSlots = ecK + ecM
|
||||
numTotalBlocks = calcEcBlocksCount(numDatasetBlocks, ecK, ecM) # total number of blocks in the dataset after
|
||||
# EC (should will match number of slots)
|
||||
originalDatasetSize = numDatasetBlocks * blockSize.int
|
||||
totalDatasetSize = numTotalBlocks * blockSize.int
|
||||
|
||||
chunker = RandomChunker.new(Rng.instance(), size = totalDatasetSize, chunkSize = blockSize)
|
||||
datasetBlocks = await chunker.createBlocks(store)
|
||||
|
||||
(manifest, protectedManifest) =
|
||||
await createProtectedManifest(
|
||||
datasetBlocks,
|
||||
store,
|
||||
numDatasetBlocks,
|
||||
ecK, ecM,
|
||||
blockSize,
|
||||
originalDatasetSize,
|
||||
totalDatasetSize)
|
||||
|
||||
builder = Poseidon2Builder.new(store, protectedManifest, cellSize = cellSize).tryGet
|
||||
verifiableManifest = (await builder.buildManifest()).tryGet
|
||||
|
||||
# build the slots and manifest
|
||||
(manifest, protectedManifest, verifiableManifest)
|
||||
|
|
@ -1,194 +0,0 @@
|
|||
import std/sequtils
|
||||
import std/math
|
||||
|
||||
import pkg/questionable/results
|
||||
import pkg/poseidon2/io
|
||||
import pkg/poseidon2
|
||||
import pkg/chronos
|
||||
import pkg/codex/stores/cachestore
|
||||
import pkg/codex/chunker
|
||||
import pkg/codex/stores
|
||||
import pkg/codex/blocktype as bt
|
||||
import pkg/codex/contracts/requests
|
||||
import pkg/codex/contracts
|
||||
import pkg/codex/merkletree
|
||||
import pkg/codex/stores/cachestore
|
||||
import pkg/codex/indexingstrategy
|
||||
|
||||
import pkg/codex/slots/converters
|
||||
import pkg/codex/slots/builder/builder
|
||||
import pkg/codex/utils/poseidon2digest
|
||||
import pkg/codex/utils/asynciter
|
||||
|
||||
import ../helpers
|
||||
import ../merkletree/helpers
|
||||
|
||||
const
|
||||
# The number of slot blocks and number of slots, combined with
|
||||
# the bytes per block, make it so that there are exactly 256 cells
|
||||
# in the dataset.
|
||||
bytesPerBlock* = 64 * 1024
|
||||
cellsPerBlock* = bytesPerBlock div DefaultCellSize.int
|
||||
numberOfSlotBlocks* = 3
|
||||
numberOfSlotBlocksPadded* = numberOfSlotBlocks.nextPowerOfTwo
|
||||
totalNumberOfSlots* = 2
|
||||
datasetSlotIndex* = 1
|
||||
cellsPerSlot* = (bytesPerBlock * numberOfSlotBlocks) div DefaultCellSize.int
|
||||
totalNumCells = ((numberOfSlotBlocks * totalNumberOfSlots * bytesPerBlock) div DefaultCellSize.int)
|
||||
|
||||
type
|
||||
ProvingTestEnvironment* = ref object
|
||||
# Invariant:
|
||||
# These challenges are chosen such that with the testenv default values
|
||||
# and nSamples=3, they will land on [3x data cells + 0x padded cell],
|
||||
# and [2x data cells + 1x padded cell] respectively:
|
||||
challengeNoPad*: Poseidon2Hash
|
||||
challengeOnePad*: Poseidon2Hash
|
||||
blockPadBytes*: seq[byte]
|
||||
emptyBlockTree*: Poseidon2Tree
|
||||
emptyBlockCid*: Cid
|
||||
# Variant:
|
||||
localStore*: CacheStore
|
||||
manifest*: Manifest
|
||||
manifestBlock*: bt.Block
|
||||
slot*: Slot
|
||||
datasetBlocks*: seq[bt.Block]
|
||||
slotTree*: Poseidon2Tree
|
||||
slotRootCid*: Cid
|
||||
slotRoots*: seq[Poseidon2Hash]
|
||||
datasetToSlotTree*: Poseidon2Tree
|
||||
datasetRootHash*: Poseidon2Hash
|
||||
|
||||
proc createDatasetBlocks(self: ProvingTestEnvironment): Future[void] {.async.} =
|
||||
var data: seq[byte] = @[]
|
||||
|
||||
# This generates a number of blocks that have different data, such that
|
||||
# Each cell in each block is unique, but nothing is random.
|
||||
for i in 0 ..< totalNumCells:
|
||||
data = data & (i.byte).repeat(DefaultCellSize.uint64)
|
||||
|
||||
let chunker = MockChunker.new(
|
||||
dataset = data,
|
||||
chunkSize = bytesPerBlock)
|
||||
|
||||
while true:
|
||||
let chunk = await chunker.getBytes()
|
||||
if chunk.len <= 0:
|
||||
break
|
||||
let b = bt.Block.new(chunk).tryGet()
|
||||
self.datasetBlocks.add(b)
|
||||
discard await self.localStore.putBlock(b)
|
||||
|
||||
proc createSlotTree(self: ProvingTestEnvironment, dSlotIndex: uint64): Future[Poseidon2Tree] {.async.} =
|
||||
let
|
||||
slotSize = (bytesPerBlock * numberOfSlotBlocks).uint64
|
||||
blocksInSlot = slotSize div bytesPerBlock.uint64
|
||||
datasetBlockIndexingStrategy = SteppedStrategy.init(0, self.datasetBlocks.len - 1, totalNumberOfSlots)
|
||||
datasetBlockIndices = toSeq(datasetBlockIndexingStrategy.getIndicies(dSlotIndex.int))
|
||||
|
||||
let
|
||||
slotBlocks = datasetBlockIndices.mapIt(self.datasetBlocks[it])
|
||||
slotBlockRoots = slotBlocks.mapIt(Poseidon2Tree.digest(it.data & self.blockPadBytes, DefaultCellSize.int).tryGet())
|
||||
slotBlockRootPads = newSeqWith((slotBlockRoots.len).nextPowerOfTwoPad, Poseidon2Zero)
|
||||
tree = Poseidon2Tree.init(slotBlockRoots & slotBlockRootPads).tryGet()
|
||||
treeCid = tree.root().tryGet().toSlotCid().tryGet()
|
||||
|
||||
for i in 0 ..< numberOfSlotBlocksPadded:
|
||||
var blkCid: Cid
|
||||
if i < slotBlockRoots.len:
|
||||
blkCid = slotBlockRoots[i].toCellCid().tryGet()
|
||||
else:
|
||||
blkCid = self.emptyBlockCid
|
||||
|
||||
let proof = tree.getProof(i).tryGet().toEncodableProof().tryGet()
|
||||
discard await self.localStore.putCidAndProof(treeCid, i, blkCid, proof)
|
||||
|
||||
return tree
|
||||
|
||||
proc createDatasetRootHashAndSlotTree(self: ProvingTestEnvironment): Future[void] {.async.} =
|
||||
var slotTrees = newSeq[Poseidon2Tree]()
|
||||
for i in 0 ..< totalNumberOfSlots:
|
||||
slotTrees.add(await self.createSlotTree(i.uint64))
|
||||
self.slotTree = slotTrees[datasetSlotIndex]
|
||||
self.slotRootCid = slotTrees[datasetSlotIndex].root().tryGet().toSlotCid().tryGet()
|
||||
self.slotRoots = slotTrees.mapIt(it.root().tryGet())
|
||||
let rootsPadLeafs = newSeqWith(totalNumberOfSlots.nextPowerOfTwoPad, Poseidon2Zero)
|
||||
self.datasetToSlotTree = Poseidon2Tree.init(self.slotRoots & rootsPadLeafs).tryGet()
|
||||
self.datasetRootHash = self.datasetToSlotTree.root().tryGet()
|
||||
|
||||
proc createManifest(self: ProvingTestEnvironment): Future[void] {.async.} =
|
||||
let
|
||||
cids = self.datasetBlocks.mapIt(it.cid)
|
||||
tree = CodexTree.init(cids).tryGet()
|
||||
treeCid = tree.rootCid(CIDv1, BlockCodec).tryGet()
|
||||
|
||||
for i in 0 ..< self.datasetBlocks.len:
|
||||
let
|
||||
blk = self.datasetBlocks[i]
|
||||
leafCid = blk.cid
|
||||
proof = tree.getProof(i).tryGet()
|
||||
discard await self.localStore.putBlock(blk)
|
||||
discard await self.localStore.putCidAndProof(treeCid, i, leafCid, proof)
|
||||
|
||||
# Basic manifest:
|
||||
self.manifest = Manifest.new(
|
||||
treeCid = treeCid,
|
||||
blockSize = bytesPerBlock.NBytes,
|
||||
datasetSize = (bytesPerBlock * numberOfSlotBlocks * totalNumberOfSlots).NBytes)
|
||||
|
||||
# Protected manifest:
|
||||
self.manifest = Manifest.new(
|
||||
manifest = self.manifest,
|
||||
treeCid = treeCid,
|
||||
datasetSize = self.manifest.datasetSize,
|
||||
ecK = totalNumberOfSlots,
|
||||
ecM = 0,
|
||||
strategy = StrategyType.SteppedStrategy
|
||||
)
|
||||
|
||||
# Verifiable manifest:
|
||||
self.manifest = Manifest.new(
|
||||
manifest = self.manifest,
|
||||
verifyRoot = self.datasetRootHash.toVerifyCid().tryGet(),
|
||||
slotRoots = self.slotRoots.mapIt(it.toSlotCid().tryGet())
|
||||
).tryGet()
|
||||
|
||||
self.manifestBlock = bt.Block.new(self.manifest.encode().tryGet(), codec = ManifestCodec).tryGet()
|
||||
discard await self.localStore.putBlock(self.manifestBlock)
|
||||
|
||||
proc createSlot(self: ProvingTestEnvironment): void =
|
||||
self.slot = Slot(
|
||||
request: StorageRequest(
|
||||
ask: StorageAsk(
|
||||
slots: totalNumberOfSlots.uint64,
|
||||
slotSize: u256(bytesPerBlock * numberOfSlotBlocks)
|
||||
),
|
||||
content: StorageContent(
|
||||
cid: $self.manifestBlock.cid
|
||||
),
|
||||
),
|
||||
slotIndex: u256(datasetSlotIndex)
|
||||
)
|
||||
|
||||
proc createProvingTestEnvironment*(): Future[ProvingTestEnvironment] {.async.} =
|
||||
let
|
||||
numBlockCells = bytesPerBlock.int div DefaultCellSize.int
|
||||
blockPadBytes = newSeq[byte](numBlockCells.nextPowerOfTwoPad * DefaultCellSize.int)
|
||||
emptyBlockTree = Poseidon2Tree.digestTree(DefaultEmptyBlock & blockPadBytes, DefaultCellSize.int).tryGet()
|
||||
emptyBlockCid = emptyBlockTree.root.tryGet().toCellCid().tryGet()
|
||||
|
||||
var testEnv = ProvingTestEnvironment(
|
||||
challengeNoPad: toF(6),
|
||||
challengeOnePad: toF(9),
|
||||
blockPadBytes: blockPadBytes,
|
||||
emptyBlockTree: emptyBlockTree,
|
||||
emptyBlockCid: emptyBlockCid
|
||||
)
|
||||
|
||||
testEnv.localStore = CacheStore.new()
|
||||
await testEnv.createDatasetBlocks()
|
||||
await testEnv.createDatasetRootHashAndSlotTree()
|
||||
await testEnv.createManifest()
|
||||
testEnv.createSlot()
|
||||
|
||||
return testEnv
|
||||
|
|
@ -0,0 +1,173 @@
|
|||
import std/sequtils
|
||||
import std/options
|
||||
import std/importutils
|
||||
|
||||
import ../../../asynctest
|
||||
|
||||
import pkg/questionable
|
||||
import pkg/questionable/results
|
||||
import pkg/datastore
|
||||
|
||||
import pkg/codex/rng
|
||||
import pkg/codex/stores
|
||||
import pkg/codex/merkletree
|
||||
import pkg/codex/utils/json
|
||||
import pkg/codex/codextypes
|
||||
import pkg/codex/slots
|
||||
import pkg/codex/slots/builder
|
||||
import pkg/codex/utils/poseidon2digest
|
||||
import pkg/codex/slots/sampler/utils
|
||||
|
||||
import pkg/constantine/math/arithmetic
|
||||
import pkg/constantine/math/io/io_bigints
|
||||
import pkg/constantine/math/io/io_fields
|
||||
|
||||
import ../backends/helpers
|
||||
import ../helpers
|
||||
import ../../helpers
|
||||
|
||||
suite "Test control samples":
|
||||
|
||||
var
|
||||
inputData: string
|
||||
inputJson: JsonNode
|
||||
proofInput: ProofInput[Poseidon2Hash]
|
||||
|
||||
setup:
|
||||
inputData = readFile("tests/circuits/fixtures/input.json")
|
||||
inputJson = parseJson(inputData)
|
||||
proofInput = jsonToProofInput[Poseidon2Hash](inputJson)
|
||||
|
||||
test "Should verify control samples":
|
||||
let
|
||||
blockCells = 32
|
||||
cellIdxs = proofInput.entropy.cellIndices(proofInput.slotRoot, proofInput.nCellsPerSlot, 5)
|
||||
|
||||
for i, cellIdx in cellIdxs:
|
||||
let
|
||||
sample = proofInput.samples[i]
|
||||
cellIdx = cellIdxs[i]
|
||||
|
||||
cellProof = Poseidon2Proof.init(
|
||||
cellIdx.toCellInBlk(blockCells),
|
||||
proofInput.nCellsPerSlot,
|
||||
sample.merklePaths[0..<5]).tryGet
|
||||
|
||||
slotProof = Poseidon2Proof.init(
|
||||
cellIdx.toBlkInSlot(blockCells),
|
||||
proofInput.nCellsPerSlot,
|
||||
sample.merklePaths[5..<9]).tryGet
|
||||
|
||||
cellData = fromCircomData[Poseidon2Hash](sample.cellData)
|
||||
cellLeaf = Poseidon2Hash.spongeDigest(cellData, rate = 2).tryGet
|
||||
slotLeaf = cellProof.reconstructRoot(cellLeaf).tryGet
|
||||
|
||||
check slotProof.verify(slotLeaf, proofInput.slotRoot).tryGet
|
||||
|
||||
test "Should verify control dataset root":
|
||||
let
|
||||
datasetProof = Poseidon2Proof.init(
|
||||
proofInput.slotIndex,
|
||||
proofInput.nSlotsPerDataSet,
|
||||
proofInput.slotProof[0..<4]).tryGet
|
||||
|
||||
check datasetProof.verify(proofInput.slotRoot, proofInput.datasetRoot).tryGet
|
||||
|
||||
suite "Test sampler samples":
|
||||
|
||||
let
|
||||
slotIndex = 3
|
||||
nSamples = 5
|
||||
ecK = 3
|
||||
ecM = 2
|
||||
datasetBlocks = 8
|
||||
entropy = 1234567.toF
|
||||
blockSize = DefaultBlockSize
|
||||
cellSize = DefaultCellSize
|
||||
|
||||
var
|
||||
store: RepoStore
|
||||
builder: Poseidon2Builder
|
||||
manifest: Manifest
|
||||
protected: Manifest
|
||||
verifiable: Manifest
|
||||
|
||||
setup:
|
||||
let
|
||||
repoDs = SQLiteDatastore.new(Memory).tryGet()
|
||||
metaDs = SQLiteDatastore.new(Memory).tryGet()
|
||||
|
||||
store = RepoStore.new(repoDs, metaDs)
|
||||
|
||||
(manifest, protected, verifiable) =
|
||||
await createVerifiableManifest(
|
||||
store,
|
||||
datasetBlocks,
|
||||
ecK, ecM,
|
||||
blockSize,
|
||||
cellSize)
|
||||
|
||||
# create sampler
|
||||
builder = Poseidon2Builder.new(store, verifiable).tryGet
|
||||
|
||||
teardown:
|
||||
await store.close()
|
||||
|
||||
test "Should fail instantiating for invalid slot index":
|
||||
let
|
||||
sampler = Poseidon2Sampler.new(builder.slotRoots.len, store, builder)
|
||||
|
||||
check sampler.isErr
|
||||
|
||||
test "Should fail instantiating for non verifiable builder":
|
||||
let
|
||||
nonVerifiableBuilder = Poseidon2Builder.new(store, protected).tryGet
|
||||
sampler = Poseidon2Sampler.new(slotIndex, store, nonVerifiableBuilder)
|
||||
|
||||
check sampler.isErr
|
||||
|
||||
test "Should verify samples":
|
||||
let
|
||||
sampler = Poseidon2Sampler.new(slotIndex, store, builder).tryGet
|
||||
|
||||
verifyTree = builder.verifyTree.get # get the dataset tree
|
||||
slotProof = verifyTree.getProof(slotIndex).tryGet # get slot proof for index
|
||||
datasetRoot = verifyTree.root().tryGet # get dataset root
|
||||
slotTreeCid = verifiable.slotRoots[slotIndex] # get slot tree cid to retrieve proof from storage
|
||||
slotRoot = builder.slotRoots[slotIndex] # get slot root hash
|
||||
cellIdxs = entropy.cellIndices(slotRoot, builder.numSlotCells, nSamples)
|
||||
|
||||
nBlockCells = builder.numBlockCells
|
||||
nSlotCells = builder.numSlotCells
|
||||
|
||||
for i, cellIdx in cellIdxs:
|
||||
let
|
||||
sample = (await sampler.getSample(cellIdx, slotTreeCid, slotRoot)).tryGet
|
||||
|
||||
cellProof = Poseidon2Proof.init(
|
||||
cellIdx.toCellInBlk(nBlockCells),
|
||||
nSlotCells,
|
||||
sample.merklePaths[0..<5]).tryGet
|
||||
|
||||
slotProof = Poseidon2Proof.init(
|
||||
cellIdx.toBlkInSlot(nBlockCells),
|
||||
nSlotCells,
|
||||
sample.merklePaths[5..<sample.merklePaths.len]).tryGet
|
||||
|
||||
cellData = fromCircomData[Poseidon2Hash](sample.cellData)
|
||||
cellLeaf = Poseidon2Hash.spongeDigest(cellData, rate = 2).tryGet
|
||||
slotLeaf = cellProof.reconstructRoot(cellLeaf).tryGet
|
||||
|
||||
check slotProof.verify(slotLeaf, slotRoot).tryGet
|
||||
|
||||
test "Should verify dataset root":
|
||||
let
|
||||
sampler = Poseidon2Sampler.new(slotIndex, store, builder).tryGet
|
||||
proofInput = (await sampler.getProofInput(entropy.toBytes.toArray32, nSamples)).tryGet
|
||||
|
||||
datasetProof = Poseidon2Proof.init(
|
||||
proofInput.slotIndex,
|
||||
builder.slotRoots.len,
|
||||
proofInput.slotProof).tryGet
|
||||
|
||||
check datasetProof.verify(builder.slotRoots[slotIndex], builder.verifyRoot.get).tryGet
|
||||
|
|
@ -2,7 +2,8 @@ import std/sequtils
|
|||
import std/sugar
|
||||
import std/random
|
||||
import std/strutils
|
||||
import std/math
|
||||
|
||||
import ../../../asynctest
|
||||
|
||||
import pkg/questionable/results
|
||||
import pkg/constantine/math/arithmetic
|
||||
|
|
@ -10,7 +11,6 @@ import pkg/constantine/math/io/io_fields
|
|||
import pkg/poseidon2/io
|
||||
import pkg/poseidon2
|
||||
import pkg/chronos
|
||||
import pkg/codex/stores/cachestore
|
||||
import pkg/codex/chunker
|
||||
import pkg/codex/stores
|
||||
import pkg/codex/blocktype as bt
|
||||
|
|
@ -19,31 +19,27 @@ import pkg/codex/contracts
|
|||
import pkg/codex/merkletree
|
||||
import pkg/codex/stores/cachestore
|
||||
|
||||
import pkg/codex/slots/types
|
||||
import pkg/codex/slots/sampler/utils
|
||||
|
||||
import ../../asynctest
|
||||
import ../helpers
|
||||
import ../examples
|
||||
import ../merkletree/helpers
|
||||
import ./provingtestenv
|
||||
import ../backends/helpers
|
||||
import ../../helpers
|
||||
import ../../examples
|
||||
import ../../merkletree/helpers
|
||||
|
||||
asyncchecksuite "Test proof sampler utils":
|
||||
let knownIndices: seq[Natural] = @[57, 82, 49]
|
||||
let
|
||||
cellsPerBlock = DefaultBlockSize div DefaultCellSize
|
||||
|
||||
var
|
||||
env: ProvingTestEnvironment
|
||||
slotRoot: Poseidon2Hash
|
||||
numCells: Natural
|
||||
numCellsPadded: Natural
|
||||
inputData: string
|
||||
inputJson: JsonNode
|
||||
proofInput: ProofInput[Poseidon2Hash]
|
||||
|
||||
setup:
|
||||
env = await createProvingTestEnvironment()
|
||||
slotRoot = env.slotRoots[datasetSlotIndex]
|
||||
numCells = cellsPerSlot
|
||||
numCellsPadded = numCells.nextPowerOfTwo
|
||||
|
||||
teardown:
|
||||
reset(env)
|
||||
inputData = readFile("tests/circuits/fixtures/input.json")
|
||||
inputJson = parseJson(inputData)
|
||||
proofInput = jsonToProofInput[Poseidon2Hash](inputJson)
|
||||
|
||||
test "Extract low bits":
|
||||
proc extract(value: uint64, nBits: int): uint64 =
|
||||
|
|
@ -64,45 +60,39 @@ asyncchecksuite "Test proof sampler utils":
|
|||
|
||||
test "Can find single slot-cell index":
|
||||
proc slotCellIndex(i: Natural): Natural =
|
||||
return cellIndex(env.challengeNoPad, slotRoot, numCellsPadded, i)
|
||||
return cellIndex(proofInput.entropy, proofInput.slotRoot, proofInput.nCellsPerSlot, i)
|
||||
|
||||
proc getExpectedIndex(i: int): Natural =
|
||||
let
|
||||
numberOfCellsInSlot = (bytesPerBlock * numberOfSlotBlocks) div DefaultCellSize.uint64.int
|
||||
numberOfCellsInSlotPadded = numberOfCellsInSlot.nextPowerOfTwo
|
||||
hash = Sponge.digest(@[slotRoot, env.challengeNoPad, toF(i)], rate = 2)
|
||||
hash = Sponge.digest(@[proofInput.entropy, proofInput.slotRoot, toF(i)], rate = 2)
|
||||
|
||||
return int(extractLowBits(hash.toBig(), ceilingLog2(numberOfCellsInSlotPadded)))
|
||||
return int(extractLowBits(hash.toBig(), ceilingLog2(proofInput.nCellsPerSlot)))
|
||||
|
||||
check:
|
||||
slotCellIndex(1) == getExpectedIndex(1)
|
||||
slotCellIndex(1) == knownIndices[0]
|
||||
slotCellIndex(2) == getExpectedIndex(2)
|
||||
slotCellIndex(2) == knownIndices[1]
|
||||
slotCellIndex(3) == getExpectedIndex(3)
|
||||
slotCellIndex(3) == knownIndices[2]
|
||||
|
||||
test "Can find sequence of slot-cell indices":
|
||||
proc slotCellIndices(n: int): seq[Natural] =
|
||||
cellIndices(env.challengeNoPad, slotRoot, numCellsPadded, n)
|
||||
cellIndices(proofInput.entropy, proofInput.slotRoot, numCells = proofInput.nCellsPerSlot, n)
|
||||
|
||||
proc getExpectedIndices(n: int): seq[Natural] =
|
||||
return collect(newSeq, (for i in 1..n: cellIndex(env.challengeNoPad, slotRoot, numCellsPadded, i)))
|
||||
return collect(newSeq, (for i in 1..n: cellIndex(proofInput.entropy, proofInput.slotRoot, proofInput.nCellsPerSlot, i)))
|
||||
|
||||
check:
|
||||
slotCellIndices(3) == getExpectedIndices(3)
|
||||
slotCellIndices(3) == knownIndices
|
||||
|
||||
for (input, expected) in [(10, 0), (31, 0), (32, 1), (63, 1), (64, 2)]:
|
||||
test "Can get slotBlockIndex from slotCellIndex (" & $input & " -> " & $expected & ")":
|
||||
let slotBlockIndex = toBlockIdx(input, numCells = 32)
|
||||
let slotBlockIndex = toBlkInSlot(input, numCells = cellsPerBlock)
|
||||
|
||||
check:
|
||||
slotBlockIndex == expected
|
||||
|
||||
for (input, expected) in [(10, 10), (31, 31), (32, 0), (63, 31), (64, 0)]:
|
||||
test "Can get blockCellIndex from slotCellIndex (" & $input & " -> " & $expected & ")":
|
||||
let blockCellIndex = toBlockCellIdx(input, numCells = 32)
|
||||
let blockCellIndex = toCellInBlk(input, numCells = cellsPerBlock)
|
||||
|
||||
check:
|
||||
blockCellIndex == expected
|
||||
|
|
@ -1,155 +1,4 @@
|
|||
import std/sequtils
|
||||
import std/sugar
|
||||
import std/random
|
||||
import std/strutils
|
||||
import ./sampler/testsampler
|
||||
import ./sampler/testutils
|
||||
|
||||
import pkg/questionable/results
|
||||
import pkg/constantine/math/arithmetic
|
||||
import pkg/constantine/math/io/io_fields
|
||||
import pkg/poseidon2/io
|
||||
import pkg/poseidon2
|
||||
import pkg/chronos
|
||||
import pkg/nimcrypto
|
||||
import pkg/codex/stores/cachestore
|
||||
import pkg/codex/chunker
|
||||
import pkg/codex/stores
|
||||
import pkg/codex/blocktype as bt
|
||||
import pkg/codex/contracts/requests
|
||||
import pkg/codex/contracts
|
||||
import pkg/codex/merkletree
|
||||
import pkg/codex/stores/cachestore
|
||||
|
||||
import pkg/codex/slots/sampler
|
||||
import pkg/codex/slots/builder/builder
|
||||
|
||||
import ../helpers
|
||||
import ../examples
|
||||
import ../merkletree/helpers
|
||||
import ../../asynctest
|
||||
import ./testsampler_expected
|
||||
import ./provingtestenv
|
||||
|
||||
asyncchecksuite "Test DataSampler":
|
||||
var
|
||||
env: ProvingTestEnvironment
|
||||
dataSampler: DataSampler
|
||||
blk: bt.Block
|
||||
cell0Bytes: seq[byte]
|
||||
cell1Bytes: seq[byte]
|
||||
cell2Bytes: seq[byte]
|
||||
|
||||
proc createDataSampler(): Future[void] {.async.} =
|
||||
dataSampler = DataSampler.new(
|
||||
datasetSlotIndex,
|
||||
env.localStore,
|
||||
SlotsBuilder.new(env.localStore, env.manifest).tryGet()).tryGet()
|
||||
|
||||
setup:
|
||||
randomize()
|
||||
env = await createProvingTestEnvironment()
|
||||
let bytes = newSeqWith(bytesPerBlock, rand(uint8))
|
||||
blk = bt.Block.new(bytes).tryGet()
|
||||
cell0Bytes = bytes[0..<DefaultCellSize.uint64]
|
||||
cell1Bytes = bytes[DefaultCellSize.uint64..<(DefaultCellSize.uint64*2)]
|
||||
cell2Bytes = bytes[(DefaultCellSize.uint64*2)..<(DefaultCellSize.uint64*3)]
|
||||
|
||||
await createDataSampler()
|
||||
|
||||
teardown:
|
||||
reset(env)
|
||||
reset(dataSampler)
|
||||
|
||||
test "Can get cell from block":
|
||||
let
|
||||
sample0 = dataSampler.getCell(blk.data, 0)
|
||||
sample1 = dataSampler.getCell(blk.data, 1)
|
||||
sample2 = dataSampler.getCell(blk.data, 2)
|
||||
|
||||
check:
|
||||
sample0 == cell0Bytes
|
||||
sample1 == cell1Bytes
|
||||
sample2 == cell2Bytes
|
||||
|
||||
test "Can gather proof input":
|
||||
let
|
||||
nSamples = 3
|
||||
challengeBytes = env.challengeNoPad.toBytes()
|
||||
input = (await dataSampler.getProofInput(challengeBytes, nSamples)).tryget()
|
||||
|
||||
proc equal(a: Poseidon2Hash, b: Poseidon2Hash): bool =
|
||||
a.toDecimal() == b.toDecimal()
|
||||
|
||||
proc toStr(proof: Poseidon2Proof): string =
|
||||
let a = proof.path.mapIt(toHex(it))
|
||||
join(a)
|
||||
|
||||
let
|
||||
expectedBlockSlotProofs = getExpectedBlockSlotProofs()
|
||||
expectedCellBlockProofs = getExpectedCellBlockProofs()
|
||||
expectedCellData = getExpectedCellData()
|
||||
expectedProof = env.datasetToSlotTree.getProof(datasetSlotIndex).tryGet()
|
||||
|
||||
check:
|
||||
equal(input.verifyRoot, env.datasetRootHash)
|
||||
equal(input.entropy, env.challengeNoPad)
|
||||
input.numCells == ((bytesPerBlock * numberOfSlotBlocks) div DefaultCellSize.int).Natural
|
||||
input.numSlots == totalNumberOfSlots.Natural
|
||||
input.slotIndex == env.slot.slotIndex.truncate(Natural)
|
||||
input.verifyProof == expectedProof
|
||||
|
||||
# block-slot proofs
|
||||
input.samples[0].slotBlockIdx == 1
|
||||
input.samples[1].slotBlockIdx == 2
|
||||
input.samples[2].slotBlockIdx == 1
|
||||
toStr(input.samples[0].slotProof) == expectedBlockSlotProofs[0]
|
||||
toStr(input.samples[1].slotProof) == expectedBlockSlotProofs[1]
|
||||
toStr(input.samples[2].slotProof) == expectedBlockSlotProofs[2]
|
||||
|
||||
# cell-block proofs
|
||||
input.samples[0].blockCellIdx == 25
|
||||
input.samples[1].blockCellIdx == 18
|
||||
input.samples[2].blockCellIdx == 17
|
||||
toStr(input.samples[0].cellProof) == expectedCellBlockProofs[0]
|
||||
toStr(input.samples[1].cellProof) == expectedCellBlockProofs[1]
|
||||
toStr(input.samples[2].cellProof) == expectedCellBlockProofs[2]
|
||||
|
||||
# cell data
|
||||
nimcrypto.toHex(input.samples[0].data) == expectedCellData[0]
|
||||
nimcrypto.toHex(input.samples[1].data) == expectedCellData[1]
|
||||
nimcrypto.toHex(input.samples[2].data) == expectedCellData[2]
|
||||
|
||||
test "Can select samples in padded cells":
|
||||
let
|
||||
nSamples = 3
|
||||
challengeBytes = env.challengeOnePad.toBytes()
|
||||
input = (await dataSampler.getProofInput(challengeBytes, nSamples)).tryget()
|
||||
|
||||
proc equal(a: Poseidon2Hash, b: Poseidon2Hash): bool =
|
||||
a.toDecimal() == b.toDecimal()
|
||||
|
||||
proc toStr(proof: Poseidon2Proof): string =
|
||||
let a = proof.path.mapIt(toHex(it))
|
||||
join(a)
|
||||
|
||||
let expectedProof = env.datasetToSlotTree.getProof(datasetSlotIndex).tryGet()
|
||||
|
||||
check:
|
||||
equal(input.verifyRoot, env.datasetRootHash)
|
||||
equal(input.entropy, env.challengeOnePad)
|
||||
input.numCells == ((bytesPerBlock * numberOfSlotBlocks) div DefaultCellSize.int).Natural
|
||||
input.numSlots == totalNumberOfSlots.Natural
|
||||
input.slotIndex == env.slot.slotIndex.truncate(Natural)
|
||||
input.verifyProof == expectedProof
|
||||
|
||||
input.samples[0].slotBlockIdx == 2
|
||||
input.samples[1].slotBlockIdx == 2
|
||||
input.samples[2].slotBlockIdx == 3
|
||||
# The third sample is a padded sample
|
||||
toStr(input.samples[2].slotProof) == toStr(env.slotTree.getProof(3).tryGet())
|
||||
|
||||
input.samples[0].blockCellIdx == 29
|
||||
input.samples[1].blockCellIdx == 26
|
||||
input.samples[2].blockCellIdx == 30
|
||||
toStr(input.samples[2].cellProof) == toStr(env.emptyBlockTree.getProof(30).tryGet())
|
||||
|
||||
input.samples[2].data == newSeq[byte](DefaultCellSize.int)
|
||||
{.warning[UnusedImport]: off.}
|
||||
|
|
|
|||
|
|
@ -1,25 +0,0 @@
|
|||
# Snapshot of expected values for testsampler.
|
||||
|
||||
import std/strutils
|
||||
import pkg/codex/codextypes
|
||||
|
||||
proc getExpectedCellBlockProofs*(): seq[string] =
|
||||
@[
|
||||
"0x13935d7f73028fd425e557b8e0b737fcba3f8bcda0e07a9047868bc7c5a2519e0x17d36c9bcef9febb77b129e8bbe971b754d7b3e1df8c7d881b64bf0aa055f5570x02610a45fa39d4859e5c0c53bda9cc6bd627ae4b5aa462f9e86f47f3db96b2700x27a362c0f92c887d74ee77df2df5e51f0f66b0564a2af8b1c11b02dab0a34f780x095eb3c0d166c19f9cac8ea0e6ba274dfeef802cf7d01c90378585fd4d788e56",
|
||||
"0x128b43cb1f87736d5b826c5a440d373c0e4c147a81a0edc40b106abd2c10af960x2160ba1c1ed893685b849a8f1be6d32bb11a6ebb8690c5af95e0ce0e9d72b6580x1332185d0ef7b76e55c7464f37df96d41f256249d74678d8191030912036b7aa0x0522383db3f9cb9c75d5b209bd795566e9cffa632819628194416a479998a4270x1decd3fb7ff458261149731115657cecd7eb2fe4a6cf84f3c6761aa8b0dd6b9a",
|
||||
"0x2a2c8663e354dbc1857a654134de3b690d73ffedcc7c0ba770a39f3e48675ee10x01914d98d4103889ea59ce6e970dae95f7e463350184c5f7f3806997e10748520x10e8f6235ac7e95af5aea6fc77ebe60e1028b48eb84027ea46d403d809852b6b0x04bd51be7ff42db27a7605124d3d2d1c4cbca43f93d1997060ac8a1009fa48b40x095eb3c0d166c19f9cac8ea0e6ba274dfeef802cf7d01c90378585fd4d788e56"
|
||||
]
|
||||
|
||||
proc getExpectedBlockSlotProofs*(): seq[string] =
|
||||
@[
|
||||
"0x008fccffc5d2e5010a44bc718fed80d4d891e6517c022ba1bff65763dd79dbb90x117662f365386ee6784274dfa3a3b35126291cdf013c5e90d07bee79f32b2f40",
|
||||
"0x00000000000000000000000000000000000000000000000000000000000000000x2a66917fa49371e835376fcece0d854c77008ac1195740963b1ac4491ee1aaf1",
|
||||
"0x008fccffc5d2e5010a44bc718fed80d4d891e6517c022ba1bff65763dd79dbb90x117662f365386ee6784274dfa3a3b35126291cdf013c5e90d07bee79f32b2f40"
|
||||
]
|
||||
|
||||
proc getExpectedCellData*(): seq[string] =
|
||||
@[
|
||||
"79".repeat(DefaultCellSize.int),
|
||||
"B2".repeat(DefaultCellSize.int),
|
||||
"71".repeat(DefaultCellSize.int)
|
||||
]
|
||||
|
|
@ -3,6 +3,8 @@ import std/math
|
|||
import std/importutils
|
||||
import std/sugar
|
||||
|
||||
import ../../asynctest
|
||||
|
||||
import pkg/chronos
|
||||
import pkg/questionable/results
|
||||
import pkg/codex/blocktype as bt
|
||||
|
|
@ -13,12 +15,13 @@ import pkg/codex/merkletree
|
|||
import pkg/codex/manifest {.all.}
|
||||
import pkg/codex/utils
|
||||
import pkg/codex/utils/digest
|
||||
import pkg/codex/utils/poseidon2digest
|
||||
import pkg/datastore
|
||||
import pkg/poseidon2
|
||||
import pkg/poseidon2/io
|
||||
import constantine/math/io/io_fields
|
||||
import pkg/constantine/math/io/io_fields
|
||||
|
||||
import ../../asynctest
|
||||
import ./helpers
|
||||
import ../helpers
|
||||
import ../examples
|
||||
import ../merkletree/helpers
|
||||
|
|
@ -26,9 +29,12 @@ import ../merkletree/helpers
|
|||
import pkg/codex/indexingstrategy {.all.}
|
||||
import pkg/codex/slots {.all.}
|
||||
|
||||
privateAccess(SlotsBuilder) # enable access to private fields
|
||||
privateAccess(Poseidon2Builder) # enable access to private fields
|
||||
privateAccess(Manifest) # enable access to private fields
|
||||
|
||||
const
|
||||
Strategy = SteppedStrategy
|
||||
|
||||
suite "Slot builder":
|
||||
let
|
||||
blockSize = NBytes 1024
|
||||
|
|
@ -37,98 +43,49 @@ suite "Slot builder":
|
|||
ecM = 2
|
||||
|
||||
numSlots = ecK + ecM
|
||||
numDatasetBlocks = 100
|
||||
numBlockCells = (blockSize div cellSize).int
|
||||
numDatasetBlocks = 50
|
||||
numTotalBlocks = calcEcBlocksCount(numDatasetBlocks, ecK, ecM) # total number of blocks in the dataset after
|
||||
# EC (should will match number of slots)
|
||||
originalDatasetSize = numDatasetBlocks * blockSize.int
|
||||
totalDatasetSize = numTotalBlocks * blockSize.int
|
||||
|
||||
numTotalBlocks = calcEcBlocksCount(numDatasetBlocks, ecK, ecM) # total number of blocks in the dataset after
|
||||
# EC (should will match number of slots)
|
||||
originalDatasetSize = numDatasetBlocks * blockSize.int # size of the dataset before EC
|
||||
totalDatasetSize = numTotalBlocks * blockSize.int # size of the dataset after EC
|
||||
numTotalSlotBlocks = nextPowerOfTwo(numTotalBlocks div numSlots)
|
||||
numBlockCells = (blockSize div cellSize).int # number of cells per block
|
||||
numSlotBlocks = numTotalBlocks div numSlots # number of blocks per slot
|
||||
numSlotCells = numSlotBlocks * numBlockCells # number of uncorrected slot cells
|
||||
pow2SlotCells = nextPowerOfTwo(numSlotCells) # pow2 cells per slot
|
||||
numPadSlotBlocks = pow2SlotCells div numBlockCells # pow2 blocks per slot
|
||||
numPadBlocksTotal = numPadSlotBlocks * numSlots # total number of pad blocks
|
||||
|
||||
blockPadBytes =
|
||||
newSeq[byte](numBlockCells.nextPowerOfTwoPad * cellSize.int) # power of two padding for blocks
|
||||
|
||||
slotsPadLeafs =
|
||||
newSeqWith((numTotalBlocks div numSlots).nextPowerOfTwoPad, Poseidon2Zero) # power of two padding for block roots
|
||||
|
||||
rootsPadLeafs =
|
||||
newSeqWith(numSlots.nextPowerOfTwoPad, Poseidon2Zero)
|
||||
# empty digest
|
||||
emptyDigest = SpongeMerkle.digest(newSeq[byte](blockSize.int), cellSize.int)
|
||||
|
||||
var
|
||||
datasetBlocks: seq[bt.Block]
|
||||
padBlocks: seq[bt.Block]
|
||||
localStore: BlockStore
|
||||
manifest: Manifest
|
||||
protectedManifest: Manifest
|
||||
expectedEmptyCid: Cid
|
||||
slotBuilder: SlotsBuilder
|
||||
builder: Poseidon2Builder
|
||||
chunker: Chunker
|
||||
|
||||
proc createBlocks(): Future[void] {.async.} =
|
||||
while true:
|
||||
let chunk = await chunker.getBytes()
|
||||
if chunk.len <= 0:
|
||||
break
|
||||
let blk = bt.Block.new(chunk).tryGet()
|
||||
datasetBlocks.add(blk)
|
||||
discard await localStore.putBlock(blk)
|
||||
|
||||
proc createProtectedManifest(): Future[void] {.async.} =
|
||||
let
|
||||
cids = datasetBlocks.mapIt(it.cid)
|
||||
datasetTree = CodexTree.init(cids[0..<numDatasetBlocks]).tryGet()
|
||||
datasetTreeCid = datasetTree.rootCid().tryGet()
|
||||
|
||||
protectedTree = CodexTree.init(cids).tryGet()
|
||||
protectedTreeCid = protectedTree.rootCid().tryGet()
|
||||
|
||||
for index, cid in cids[0..<numDatasetBlocks]:
|
||||
let proof = datasetTree.getProof(index).tryget()
|
||||
(await localStore.putCidAndProof(datasetTreeCid, index, cid, proof)).tryGet
|
||||
|
||||
for index, cid in cids:
|
||||
let proof = protectedTree.getProof(index).tryget()
|
||||
(await localStore.putCidAndProof(protectedTreeCid, index, cid, proof)).tryGet
|
||||
|
||||
manifest = Manifest.new(
|
||||
treeCid = datasetTreeCid,
|
||||
blockSize = blockSize.NBytes,
|
||||
datasetSize = originalDatasetSize.NBytes)
|
||||
|
||||
protectedManifest = Manifest.new(
|
||||
manifest = manifest,
|
||||
treeCid = protectedTreeCid,
|
||||
datasetSize = totalDatasetSize.NBytes,
|
||||
ecK = ecK,
|
||||
ecM = ecM,
|
||||
strategy = StrategyType.SteppedStrategy)
|
||||
|
||||
let
|
||||
manifestBlock = bt.Block.new(
|
||||
manifest.encode().tryGet(),
|
||||
codec = ManifestCodec).tryGet()
|
||||
|
||||
protectedManifestBlock = bt.Block.new(
|
||||
protectedManifest.encode().tryGet(),
|
||||
codec = ManifestCodec).tryGet()
|
||||
|
||||
(await localStore.putBlock(manifestBlock)).tryGet()
|
||||
(await localStore.putBlock(protectedManifestBlock)).tryGet()
|
||||
|
||||
expectedEmptyCid = emptyCid(
|
||||
protectedManifest.version,
|
||||
protectedManifest.hcodec,
|
||||
protectedManifest.codec).tryGet()
|
||||
|
||||
setup:
|
||||
let
|
||||
repoDs = SQLiteDatastore.new(Memory).tryGet()
|
||||
metaDs = SQLiteDatastore.new(Memory).tryGet()
|
||||
localStore = RepoStore.new(repoDs, metaDs)
|
||||
|
||||
localStore = RepoStore.new(repoDs, metaDs)
|
||||
chunker = RandomChunker.new(Rng.instance(), size = totalDatasetSize, chunkSize = blockSize)
|
||||
await createBlocks()
|
||||
await createProtectedManifest()
|
||||
datasetBlocks = await chunker.createBlocks(localStore)
|
||||
|
||||
(manifest, protectedManifest) =
|
||||
await createProtectedManifest(
|
||||
datasetBlocks,
|
||||
localStore,
|
||||
numDatasetBlocks,
|
||||
ecK, ecM,
|
||||
blockSize,
|
||||
originalDatasetSize,
|
||||
totalDatasetSize)
|
||||
|
||||
teardown:
|
||||
await localStore.close()
|
||||
|
|
@ -142,11 +99,10 @@ suite "Slot builder":
|
|||
reset(localStore)
|
||||
reset(manifest)
|
||||
reset(protectedManifest)
|
||||
reset(expectedEmptyCid)
|
||||
reset(slotBuilder)
|
||||
reset(builder)
|
||||
reset(chunker)
|
||||
|
||||
test "Can only create slotBuilder with protected manifest":
|
||||
test "Can only create builder with protected manifest":
|
||||
let
|
||||
unprotectedManifest = Manifest.new(
|
||||
treeCid = Cid.example,
|
||||
|
|
@ -154,8 +110,8 @@ suite "Slot builder":
|
|||
datasetSize = originalDatasetSize.NBytes)
|
||||
|
||||
check:
|
||||
SlotsBuilder.new(localStore, unprotectedManifest, cellSize = cellSize)
|
||||
.error.msg == "Can only create SlotsBuilder using protected manifests."
|
||||
Poseidon2Builder.new(localStore, unprotectedManifest, cellSize = cellSize)
|
||||
.error.msg == "Manifest is not protected."
|
||||
|
||||
test "Number of blocks must be devisable by number of slots":
|
||||
let
|
||||
|
|
@ -168,10 +124,10 @@ suite "Slot builder":
|
|||
datasetSize = totalDatasetSize.NBytes,
|
||||
ecK = ecK - 1,
|
||||
ecM = ecM,
|
||||
strategy = StrategyType.SteppedStrategy)
|
||||
strategy = Strategy)
|
||||
|
||||
check:
|
||||
SlotsBuilder.new(localStore, mismatchManifest, cellSize = cellSize)
|
||||
Poseidon2Builder.new(localStore, mismatchManifest, cellSize = cellSize)
|
||||
.error.msg == "Number of blocks must be divisable by number of slots."
|
||||
|
||||
test "Block size must be divisable by cell size":
|
||||
|
|
@ -185,146 +141,150 @@ suite "Slot builder":
|
|||
datasetSize = (totalDatasetSize - 1).NBytes,
|
||||
ecK = ecK,
|
||||
ecM = ecM,
|
||||
strategy = StrategyType.SteppedStrategy)
|
||||
strategy = Strategy)
|
||||
|
||||
check:
|
||||
SlotsBuilder.new(localStore, mismatchManifest, cellSize = cellSize)
|
||||
Poseidon2Builder.new(localStore, mismatchManifest, cellSize = cellSize)
|
||||
.error.msg == "Block size must be divisable by cell size."
|
||||
|
||||
test "Should build correct slot builder":
|
||||
slotBuilder = SlotsBuilder.new(
|
||||
builder = Poseidon2Builder.new(
|
||||
localStore,
|
||||
protectedManifest,
|
||||
cellSize = cellSize).tryGet()
|
||||
|
||||
check:
|
||||
slotBuilder.numBlockPadBytes == blockPadBytes.len
|
||||
slotBuilder.numSlotsPadLeafs == slotsPadLeafs.len
|
||||
slotBuilder.numRootsPadLeafs == rootsPadLeafs.len
|
||||
builder.cellSize == cellSize
|
||||
builder.numSlots == numSlots
|
||||
builder.numBlockCells == numBlockCells
|
||||
builder.numSlotBlocks == numPadSlotBlocks
|
||||
builder.numSlotCells == pow2SlotCells
|
||||
builder.numBlocks == numPadBlocksTotal
|
||||
|
||||
test "Should build slot hashes for all slots":
|
||||
let
|
||||
steppedStrategy = SteppedStrategy.init(0, numTotalBlocks - 1, numSlots)
|
||||
slotBuilder = SlotsBuilder.new(
|
||||
steppedStrategy = Strategy.init(
|
||||
0, numPadBlocksTotal - 1, numSlots)
|
||||
|
||||
builder = Poseidon2Builder.new(
|
||||
localStore,
|
||||
protectedManifest,
|
||||
cellSize = cellSize).tryGet()
|
||||
|
||||
for i in 0 ..< numSlots:
|
||||
let
|
||||
expectedBlock = steppedStrategy
|
||||
.getIndicies(i)
|
||||
.mapIt( datasetBlocks[it] )
|
||||
# for i in 0..<numSlots:
|
||||
let
|
||||
expectedHashes = collect(newSeq):
|
||||
for j, idx in steppedStrategy.getIndicies(0):
|
||||
if j > (protectedManifest.numSlotBlocks - 1):
|
||||
emptyDigest
|
||||
else:
|
||||
SpongeMerkle.digest(datasetBlocks[idx].data, cellSize.int)
|
||||
|
||||
expectedHashes: seq[Poseidon2Hash] = collect(newSeq):
|
||||
for blk in expectedBlock:
|
||||
SpongeMerkle.digest(blk.data & blockPadBytes, cellSize.int)
|
||||
cellHashes = (await builder.getCellHashes(0)).tryGet()
|
||||
|
||||
cellHashes = (await slotBuilder.getBlockHashes(i)).tryGet()
|
||||
|
||||
check:
|
||||
expectedHashes == cellHashes
|
||||
check:
|
||||
cellHashes.len == expectedHashes.len
|
||||
cellHashes == expectedHashes
|
||||
|
||||
test "Should build slot trees for all slots":
|
||||
let
|
||||
steppedStrategy = SteppedStrategy.init(0, numTotalBlocks - 1, numSlots)
|
||||
slotBuilder = SlotsBuilder.new(
|
||||
steppedStrategy = Strategy.init(
|
||||
0, numPadBlocksTotal - 1, numSlots)
|
||||
|
||||
builder = Poseidon2Builder.new(
|
||||
localStore,
|
||||
protectedManifest,
|
||||
cellSize = cellSize).tryGet()
|
||||
|
||||
for i in 0 ..< numSlots:
|
||||
for i in 0..<numSlots:
|
||||
let
|
||||
expectedBlock = steppedStrategy
|
||||
.getIndicies(i)
|
||||
.mapIt( datasetBlocks[it] )
|
||||
expectedHashes = collect(newSeq):
|
||||
for j, idx in steppedStrategy.getIndicies(i):
|
||||
if j > (protectedManifest.numSlotBlocks - 1):
|
||||
emptyDigest
|
||||
else:
|
||||
SpongeMerkle.digest(datasetBlocks[idx].data, cellSize.int)
|
||||
|
||||
expectedHashes: seq[Poseidon2Hash] = collect(newSeq):
|
||||
for blk in expectedBlock:
|
||||
SpongeMerkle.digest(blk.data & blockPadBytes, cellSize.int)
|
||||
expectedRoot = Merkle.digest(expectedHashes & slotsPadLeafs)
|
||||
|
||||
slotTree = (await slotBuilder.buildSlotTree(i)).tryGet()
|
||||
expectedRoot = Merkle.digest(expectedHashes)
|
||||
slotTree = (await builder.buildSlotTree(i)).tryGet()
|
||||
|
||||
check:
|
||||
expectedRoot == slotTree.root().tryGet()
|
||||
slotTree.root().tryGet() == expectedRoot
|
||||
|
||||
test "Should persist trees for all slots":
|
||||
let
|
||||
slotBuilder = SlotsBuilder.new(
|
||||
builder = Poseidon2Builder.new(
|
||||
localStore,
|
||||
protectedManifest,
|
||||
cellSize = cellSize).tryGet()
|
||||
|
||||
for i in 0 ..< numSlots:
|
||||
for i in 0..<numSlots:
|
||||
let
|
||||
slotTree = (await slotBuilder.buildSlotTree(i)).tryGet()
|
||||
slotRoot = (await slotBuilder.buildSlot(i)).tryGet()
|
||||
slotTree = (await builder.buildSlotTree(i)).tryGet()
|
||||
slotRoot = (await builder.buildSlot(i)).tryGet()
|
||||
slotCid = slotRoot.toSlotCid().tryGet()
|
||||
|
||||
for cellIndex in 0..<numTotalSlotBlocks:
|
||||
for cellIndex in 0..<numPadSlotBlocks:
|
||||
let
|
||||
(cellCid, proof) = (await localStore.getCidAndProof(slotCid, cellIndex)).tryGet()
|
||||
verifiableProof = proof.toVerifiableProof().tryGet()
|
||||
posProof = slotTree.getProof(cellIndex).tryGet
|
||||
posProof = slotTree.getProof(cellIndex).tryGet()
|
||||
|
||||
check:
|
||||
verifiableProof.path == posProof.path
|
||||
verifiableProof.index == posProof.index
|
||||
verifiableProof.nleaves == posProof.nleaves
|
||||
verifiableProof.path == posProof.path
|
||||
|
||||
test "Should build correct verification root":
|
||||
let
|
||||
steppedStrategy = SteppedStrategy.init(0, numTotalBlocks - 1, numSlots)
|
||||
slotBuilder = SlotsBuilder.new(
|
||||
steppedStrategy = Strategy.init(0, numPadBlocksTotal - 1, numSlots)
|
||||
builder = Poseidon2Builder.new(
|
||||
localStore,
|
||||
protectedManifest,
|
||||
cellSize = cellSize).tryGet()
|
||||
|
||||
(await slotBuilder.buildSlots()).tryGet
|
||||
(await builder.buildSlots()).tryGet
|
||||
let
|
||||
slotsHashes = collect(newSeq):
|
||||
for i in 0 ..< numSlots:
|
||||
for i in 0..<numSlots:
|
||||
let
|
||||
expectedBlocks = steppedStrategy
|
||||
.getIndicies(i)
|
||||
.mapIt( datasetBlocks[it] )
|
||||
slotHashes = collect(newSeq):
|
||||
for j, idx in steppedStrategy.getIndicies(i):
|
||||
if j > (protectedManifest.numSlotBlocks - 1):
|
||||
emptyDigest
|
||||
else:
|
||||
SpongeMerkle.digest(datasetBlocks[idx].data, cellSize.int)
|
||||
|
||||
slotHashes: seq[Poseidon2Hash] = collect(newSeq):
|
||||
for blk in expectedBlocks:
|
||||
SpongeMerkle.digest(blk.data & blockPadBytes, cellSize.int)
|
||||
Merkle.digest(slotHashes)
|
||||
|
||||
Merkle.digest(slotHashes & slotsPadLeafs)
|
||||
|
||||
expectedRoot = Merkle.digest(slotsHashes & rootsPadLeafs)
|
||||
rootHash = slotBuilder.buildVerifyTree(slotBuilder.slotRoots).tryGet().root.tryGet()
|
||||
expectedRoot = Merkle.digest(slotsHashes)
|
||||
rootHash = builder.buildVerifyTree(builder.slotRoots).tryGet().root.tryGet()
|
||||
|
||||
check:
|
||||
expectedRoot == rootHash
|
||||
|
||||
test "Should build correct verification root manifest":
|
||||
let
|
||||
steppedStrategy = SteppedStrategy.init(0, numTotalBlocks - 1, numSlots)
|
||||
slotBuilder = SlotsBuilder.new(
|
||||
steppedStrategy = Strategy.init(0, numPadBlocksTotal - 1, numSlots)
|
||||
builder = Poseidon2Builder.new(
|
||||
localStore,
|
||||
protectedManifest,
|
||||
cellSize = cellSize).tryGet()
|
||||
|
||||
slotsHashes = collect(newSeq):
|
||||
for i in 0 ..< numSlots:
|
||||
for i in 0..<numSlots:
|
||||
let
|
||||
expectedBlocks = steppedStrategy
|
||||
.getIndicies(i)
|
||||
.mapIt( datasetBlocks[it] )
|
||||
slotHashes = collect(newSeq):
|
||||
for j, idx in steppedStrategy.getIndicies(i):
|
||||
if j > (protectedManifest.numSlotBlocks - 1):
|
||||
emptyDigest
|
||||
else:
|
||||
SpongeMerkle.digest(datasetBlocks[idx].data, cellSize.int)
|
||||
|
||||
slotHashes: seq[Poseidon2Hash] = collect(newSeq):
|
||||
for blk in expectedBlocks:
|
||||
SpongeMerkle.digest(blk.data & blockPadBytes, cellSize.int)
|
||||
Merkle.digest(slotHashes)
|
||||
|
||||
Merkle.digest(slotHashes & slotsPadLeafs)
|
||||
|
||||
expectedRoot = Merkle.digest(slotsHashes & rootsPadLeafs)
|
||||
manifest = (await slotBuilder.buildManifest()).tryGet()
|
||||
expectedRoot = Merkle.digest(slotsHashes)
|
||||
manifest = (await builder.buildManifest()).tryGet()
|
||||
mhash = manifest.verifyRoot.mhash.tryGet()
|
||||
mhashBytes = mhash.digestBytes
|
||||
rootHash = Poseidon2Hash.fromBytes(mhashBytes.toArray32).get
|
||||
|
|
@ -334,69 +294,69 @@ suite "Slot builder":
|
|||
|
||||
test "Should not build from verifiable manifest with 0 slots":
|
||||
var
|
||||
slotBuilder = SlotsBuilder.new(
|
||||
builder = Poseidon2Builder.new(
|
||||
localStore,
|
||||
protectedManifest,
|
||||
cellSize = cellSize).tryGet()
|
||||
verifyManifest = (await slotBuilder.buildManifest()).tryGet()
|
||||
verifyManifest = (await builder.buildManifest()).tryGet()
|
||||
|
||||
verifyManifest.slotRoots = @[]
|
||||
check SlotsBuilder.new(
|
||||
check Poseidon2Builder.new(
|
||||
localStore,
|
||||
verifyManifest,
|
||||
cellSize = cellSize).isErr
|
||||
|
||||
test "Should not build from verifiable manifest with incorrect number of slots":
|
||||
var
|
||||
slotBuilder = SlotsBuilder.new(
|
||||
builder = Poseidon2Builder.new(
|
||||
localStore,
|
||||
protectedManifest,
|
||||
cellSize = cellSize).tryGet()
|
||||
|
||||
verifyManifest = (await slotBuilder.buildManifest()).tryGet()
|
||||
verifyManifest = (await builder.buildManifest()).tryGet()
|
||||
|
||||
verifyManifest.slotRoots.del(
|
||||
verifyManifest.slotRoots.len - 1
|
||||
)
|
||||
|
||||
check SlotsBuilder.new(
|
||||
check Poseidon2Builder.new(
|
||||
localStore,
|
||||
verifyManifest,
|
||||
cellSize = cellSize).isErr
|
||||
|
||||
test "Should not build from verifiable manifest with invalid verify root":
|
||||
let
|
||||
slotBuilder = SlotsBuilder.new(
|
||||
builder = Poseidon2Builder.new(
|
||||
localStore,
|
||||
protectedManifest,
|
||||
cellSize = cellSize).tryGet()
|
||||
|
||||
verifyManifest = (await slotBuilder.buildManifest()).tryGet()
|
||||
verifyManifest = (await builder.buildManifest()).tryGet()
|
||||
offset = verifyManifest.verifyRoot.data.buffer.len div 2
|
||||
|
||||
rng.shuffle(
|
||||
Rng.instance,
|
||||
verifyManifest.verifyRoot.data.buffer)
|
||||
|
||||
check SlotsBuilder.new(
|
||||
check Poseidon2Builder.new(
|
||||
localStore,
|
||||
verifyManifest,
|
||||
cellSize = cellSize).isErr
|
||||
|
||||
test "Should build from verifiable manifest":
|
||||
let
|
||||
slotBuilder = SlotsBuilder.new(
|
||||
builder = Poseidon2Builder.new(
|
||||
localStore,
|
||||
protectedManifest,
|
||||
cellSize = cellSize).tryGet()
|
||||
|
||||
verifyManifest = (await slotBuilder.buildManifest()).tryGet()
|
||||
verifyManifest = (await builder.buildManifest()).tryGet()
|
||||
|
||||
verificationBuilder = SlotsBuilder.new(
|
||||
verificationBuilder = Poseidon2Builder.new(
|
||||
localStore,
|
||||
verifyManifest,
|
||||
cellSize = cellSize).tryGet()
|
||||
|
||||
check:
|
||||
slotBuilder.slotRoots == verificationBuilder.slotRoots
|
||||
slotBuilder.verifyRoot == verificationBuilder.verifyRoot
|
||||
builder.slotRoots == verificationBuilder.slotRoots
|
||||
builder.verifyRoot == verificationBuilder.verifyRoot
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
import ./slots/testslotbuilder
|
||||
import ./slots/testutils
|
||||
import ./slots/testsampler
|
||||
import ./slots/testconverters
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue