From 825766eea0f90088409a458b01b9bc85c52bd748 Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Wed, 7 Feb 2024 20:27:11 -0600 Subject: [PATCH] Rework builder & sampler (#697) * add proof initializer for poseidon2 and `$` * fix padding and block selection * fix sample selection and input construction * fix sample selection & more descriptive names * add concrete types for sampler & builder * add missing digest calls * use concrete types * add sample test fixtures * use concrete types and don't fetch dummy blocks --- .github/workflows/ci.yml | 2 +- codex/merkletree/codex/codex.nim | 37 +- codex/merkletree/merkletree.nim | 7 +- codex/merkletree/poseidon2.nim | 43 +- codex/node.nim | 17 +- codex/slots/builder.nim | 5 + codex/slots/builder/builder.nim | 343 ++++++------ codex/slots/sampler.nim | 5 + codex/slots/sampler/sampler.nim | 258 ++++----- codex/slots/sampler/utils.nim | 17 +- codex/slots/types.nim | 28 + codex/utils/poseidon2digest.nim | 39 +- tests/circuits/fixtures/input.json | 527 ++++++++++++++++++ tests/codex/node/testcontracts.nim | 12 +- tests/codex/node/testnode.nim | 2 +- tests/codex/slots/backends/helpers.nim | 143 +++++ tests/codex/slots/helpers.nim | 166 ++++++ tests/codex/slots/provingtestenv.nim | 194 ------- tests/codex/slots/sampler/testsampler.nim | 173 ++++++ tests/codex/slots/{ => sampler}/testutils.nim | 54 +- tests/codex/slots/testsampler.nim | 157 +----- tests/codex/slots/testsampler_expected.nim | 25 - tests/codex/slots/testslotbuilder.nim | 294 +++++----- tests/codex/testslots.nim | 1 - 24 files changed, 1608 insertions(+), 941 deletions(-) create mode 100644 codex/slots/types.nim create mode 100644 tests/circuits/fixtures/input.json create mode 100644 tests/codex/slots/backends/helpers.nim create mode 100644 tests/codex/slots/helpers.nim delete mode 100644 tests/codex/slots/provingtestenv.nim create mode 100644 tests/codex/slots/sampler/testsampler.nim rename tests/codex/slots/{ => sampler}/testutils.nim (61%) delete mode 100644 tests/codex/slots/testsampler_expected.nim diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 377cb85c..c2721f9d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,7 +10,7 @@ on: env: cache_nonce: 0 # Allows for easily busting actions/cache caches nim_version: v1.6.14 - + concurrency: group: ${{ github.workflow }}-${{ github.ref || github.run_id }} diff --git a/codex/merkletree/codex/codex.nim b/codex/merkletree/codex/codex.nim index 49439eb3..974123f2 100644 --- a/codex/merkletree/codex/codex.nim +++ b/codex/merkletree/codex/codex.nim @@ -7,16 +7,14 @@ ## This file may not be copied, modified, or distributed except according to ## those terms. -import pkg/upraises - -push: {.upraises: [].} +{.push raises: [].} import std/bitops import std/sequtils + import pkg/questionable import pkg/questionable/results import pkg/libp2p/[cid, multicodec, multihash] -import pkg/stew/byteutils import ../../utils import ../../rng @@ -72,7 +70,7 @@ func getProof*(self: CodexTree, index: int): ?!CodexProof = success proof -func verify*(self: CodexProof, leaf: MultiHash, root: MultiHash): ?!void = +func verify*(self: CodexProof, leaf: MultiHash, root: MultiHash): ?!bool = ## Verify hash ## @@ -88,11 +86,9 @@ func verify*(self: CodexProof, leaf: MultiHash, root: MultiHash): ?!void = leafBytes.len != leaf.size: return failure "Invalid hash length" - ? self.verify(leafBytes, rootBytes) + self.verify(leafBytes, rootBytes) - success() - -func verify*(self: CodexProof, leaf: Cid, root: Cid): ?!void = +func verify*(self: CodexProof, leaf: Cid, root: Cid): ?!bool = self.verify(? leaf.mhash.mapFailure, ? leaf.mhash.mapFailure) proc rootCid*( @@ -124,16 +120,19 @@ func getLeafCid*( Cid.init(version, dataCodec, mhash).mapFailure proc `$`*(self: CodexTree): string = - "CodexTree( mcodec: " & - $self.mcodec & - ", leavesCount: " & - $self.leavesCount & " )" + let root = if self.root.isOk: byteutils.toHex(self.root.get) else: "none" + "CodexTree(" & + " root: " & root & + ", leavesCount: " & $self.leavesCount & + ", levels: " & $self.levels & + ", mcodec: " & $self.mcodec & " )" proc `$`*(self: CodexProof): string = - "CodexProof( mcodec: " & - $self.mcodec & ", nleaves: " & - $self.nleaves & ", index: " & - $self.index & " )" + "CodexProof(" & + " nleaves: " & $self.nleaves & + ", index: " & $self.index & + ", path: " & $self.path.mapIt( byteutils.toHex(it) ) & + ", mcodec: " & $self.mcodec & " )" func compress*( x, y: openArray[byte], @@ -226,7 +225,9 @@ proc fromNodes*( index = Rng.instance.rand(nleaves - 1) proof = ? self.getProof(index) - ? proof.verify(self.leaves[index], ? self.root) # sanity check + if not ? proof.verify(self.leaves[index], ? self.root): # sanity check + return failure "Unable to verify tree built from nodes" + success self func init*( diff --git a/codex/merkletree/merkletree.nim b/codex/merkletree/merkletree.nim index 9f70c543..2f46b93d 100644 --- a/codex/merkletree/merkletree.nim +++ b/codex/merkletree/merkletree.nim @@ -117,11 +117,8 @@ func reconstructRoot*[H, K](proof: MerkleProof[H, K], leaf: H): ?!H = return success h -func verify*[H, K](proof: MerkleProof[H, K], leaf: H, root: H): ?!void = - return if bool(root == ? proof.reconstructRoot(leaf)): - success() - else: - failure("invalid proof") +func verify*[H, K](proof: MerkleProof[H, K], leaf: H, root: H): ?!bool = + success bool(root == ? proof.reconstructRoot(leaf)) func merkleTreeWorker*[H, K]( self: MerkleTree[H, K], diff --git a/codex/merkletree/poseidon2.nim b/codex/merkletree/poseidon2.nim index 3d3bd2a2..2e56a4b9 100644 --- a/codex/merkletree/poseidon2.nim +++ b/codex/merkletree/poseidon2.nim @@ -7,6 +7,8 @@ ## This file may not be copied, modified, or distributed except according to ## those terms. +{.push raises: [].} + import std/sequtils import pkg/poseidon2 @@ -30,7 +32,8 @@ const Poseidon2Zero* = zero type - Poseidon2Hash* = F + Bn254Fr* = F + Poseidon2Hash* = Bn254Fr PoseidonKeysEnum* = enum # can't use non-ordinals as enum values KeyNone @@ -41,6 +44,19 @@ type Poseidon2Tree* = MerkleTree[Poseidon2Hash, PoseidonKeysEnum] Poseidon2Proof* = MerkleProof[Poseidon2Hash, PoseidonKeysEnum] +proc `$`*(self: Poseidon2Tree): string = + let root = if self.root.isOk: self.root.get.toHex else: "none" + "Poseidon2Tree(" & + " root: " & root & + ", leavesCount: " & $self.leavesCount & + ", levels: " & $self.levels & " )" + +proc `$`*(self: Poseidon2Proof): string = + "Poseidon2Proof(" & + " nleaves: " & $self.nleaves & + ", index: " & $self.index & + ", path: " & $self.path.mapIt( it.toHex ) & " )" + func toArray32*(bytes: openArray[byte]): array[32, byte] = result[0.. (self.manifest.numSlotBlocks - 1): + # pad blocks are 0 byte blocks + trace "Returning empty digest tree for pad block" + return success (self.emptyBlock, self.emptyDigestTree) + without blk =? await self.store.getBlock(self.manifest.treeCid, blkIdx), err: - error "Failed to get block CID for tree at index" + error "Failed to get block CID for tree at index", err = err.msg return failure(err) if blk.isEmpty: - success (DefaultEmptyBlock & self.blockPadBytes, self.emptyDigestTree) + success (self.emptyBlock, self.emptyDigestTree) else: without tree =? - Poseidon2Tree.digestTree(blk.data & self.blockPadBytes, self.cellSize.int), err: - error "Failed to create digest for block" + T.digestTree(blk.data, self.cellSize.int), err: + error "Failed to create digest for block", err = err.msg return failure(err) success (blk.data, tree) -proc getBlockHashes*( - self: SlotsBuilder, - slotIndex: Natural): Future[?!seq[Poseidon2Hash]] {.async.} = +proc getCellHashes*[T, H]( + self: SlotsBuilder[T, H], + slotIndex: Natural): Future[?!seq[H]] {.async.} = + ## Collect all the cells from a block and return + ## their hashes. + ## let treeCid = self.manifest.treeCid @@ -209,38 +182,44 @@ proc getBlockHashes*( logScope: treeCid = treeCid - blockCount = blockCount + origBlockCount = blockCount numberOfSlots = numberOfSlots - index = blkIdx slotIndex = slotIndex - let - hashes: seq[Poseidon2Hash] = collect(newSeq): - for blkIdx in self.strategy.getIndicies(slotIndex): - trace "Getting block CID for tree at index" + let hashes = collect(newSeq): + for i, blkIdx in self.strategy.getIndicies(slotIndex): + logScope: + blkIdx = blkIdx + pos = i - without (_, blockTree) =? (await self.buildBlockTree(blkIdx)) and - digest =? blockTree.root, err: - error "Failed to get block CID for block tree at index", err = err.msg - return failure(err) + trace "Getting block CID for tree at index" + without (_, tree) =? (await self.buildBlockTree(blkIdx, i)) and + digest =? tree.root, err: + error "Failed to get block CID for tree at index", err = err.msg + return failure(err) - digest + trace "Get block digest", digest = digest.toHex + digest success hashes -proc buildSlotTree*( - self: SlotsBuilder, - slotIndex: Natural): Future[?!Poseidon2Tree] {.async.} = - without blockHashes =? (await self.getBlockHashes(slotIndex)), err: +proc buildSlotTree*[T, H]( + self: SlotsBuilder[T, H], + slotIndex: Natural): Future[?!T] {.async.} = + ## Build the slot tree from the block digest hashes + ## and return the tree. + + without cellHashes =? (await self.getCellHashes(slotIndex)), err: error "Failed to select slot blocks", err = err.msg return failure(err) - Poseidon2Tree.init(blockHashes & self.slotsPadLeafs) + T.init(cellHashes) -proc buildSlot*( - self: SlotsBuilder, - slotIndex: Natural): Future[?!Poseidon2Hash] {.async.} = - ## Build a slot tree and store it in the block store. +proc buildSlot*[T, H]( + self: SlotsBuilder[T, H], + slotIndex: Natural): Future[?!H] {.async.} = + ## Build a slot tree and store the proofs in + ## the block store. ## logScope: @@ -272,12 +251,10 @@ proc buildSlot*( tree.root() -func buildVerifyTree*( - self: SlotsBuilder, - slotRoots: openArray[Poseidon2Hash]): ?!Poseidon2Tree = - Poseidon2Tree.init(@slotRoots & self.rootsPadLeafs) +func buildVerifyTree*[T, H](self: SlotsBuilder[T, H], slotRoots: openArray[H]): ?!T = + T.init(@slotRoots) -proc buildSlots*(self: SlotsBuilder): Future[?!void] {.async.} = +proc buildSlots*[T, H](self: SlotsBuilder[T, H]): Future[?!void] {.async.} = ## Build all slot trees and store them in the block store. ## @@ -300,14 +277,14 @@ proc buildSlots*(self: SlotsBuilder): Future[?!void] {.async.} = return failure(err) if verifyTree =? self.verifyTree and verifyRoot =? verifyTree.root: - if verifyRoot != root: # TODO: `!=` doesn't work for SecretBool + if not bool(verifyRoot == root): # TODO: `!=` doesn't work for SecretBool return failure "Existing slots root doesn't match reconstructed root." - self.verifyTree = some tree + self.verifiableTree = some tree success() -proc buildManifest*(self: SlotsBuilder): Future[?!Manifest] {.async.} = +proc buildManifest*[T, H](self: SlotsBuilder[T, H]): Future[?!Manifest] {.async.} = if err =? (await self.buildSlots()).errorOption: error "Failed to build slot roots", err = err.msg return failure(err) @@ -321,68 +298,88 @@ proc buildManifest*(self: SlotsBuilder): Future[?!Manifest] {.async.} = error "Failed to map slot roots to CIDs", err = err.msg return failure(err) - Manifest.new(self.manifest, rootProvingCid, rootCids) + Manifest.new( + self.manifest, + rootProvingCid, + rootCids, + self.cellSize, + self.strategy.strategyType) -proc new*( - T: type SlotsBuilder, +proc new*[T, H]( + _: type SlotsBuilder[T, H], store: BlockStore, manifest: Manifest, - strategy: ?IndexingStrategy = none IndexingStrategy, - cellSize = DefaultCellSize): ?!SlotsBuilder = + strategy = SteppedStrategy, + cellSize = DefaultCellSize): ?!SlotsBuilder[T, H] = if not manifest.protected: - return failure("Can only create SlotsBuilder using protected manifests.") + trace "Manifest is not protected." + return failure("Manifest is not protected.") + + logScope: + blockSize = manifest.blockSize + strategy = strategy + cellSize = cellSize if (manifest.blocksCount mod manifest.numSlots) != 0: + trace "Number of blocks must be divisable by number of slots." return failure("Number of blocks must be divisable by number of slots.") + let cellSize = if manifest.verifiable: manifest.cellSize else: cellSize if (manifest.blockSize mod cellSize) != 0.NBytes: + trace "Block size must be divisable by cell size." return failure("Block size must be divisable by cell size.") let - strategy = if strategy.isNone: - ? SteppedStrategy.init( - 0, manifest.blocksCount - 1, manifest.numSlots).catch - else: - strategy.get + numBlockCells = (manifest.blockSize div cellSize).int # number of cells per block + numSlotCells = manifest.numSlotBlocks * numBlockCells # number of uncorrected slot cells + pow2SlotCells = nextPowerOfTwo(numSlotCells) # pow2 cells per slot + numPadSlotBlocks = pow2SlotCells div numBlockCells # pow2 blocks per slot + numPadBlocksTotal = numPadSlotBlocks * manifest.numSlots # total number of pad blocks - # all trees have to be padded to power of two - numBlockCells = (manifest.blockSize div cellSize).int # number of cells per block - blockPadBytes = newSeq[byte](numBlockCells.nextPowerOfTwoPad * cellSize.int) # power of two padding for blocks - numSlotBlocks = (manifest.blocksCount div manifest.numSlots) - slotsPadLeafs = newSeqWith(numSlotBlocks.nextPowerOfTwoPad, Poseidon2Zero) # power of two padding for block roots - rootsPadLeafs = newSeqWith(manifest.numSlots.nextPowerOfTwoPad, Poseidon2Zero) - emptyDigestTree = ? Poseidon2Tree.digestTree(DefaultEmptyBlock & blockPadBytes, DefaultCellSize.int) + emptyBlock = newSeq[byte](manifest.blockSize.int) + emptyDigestTree = ? T.digestTree(emptyBlock, cellSize.int) - var self = SlotsBuilder( - store: store, - manifest: manifest, - strategy: strategy, - cellSize: cellSize, - blockPadBytes: blockPadBytes, - slotsPadLeafs: slotsPadLeafs, - rootsPadLeafs: rootsPadLeafs, - emptyDigestTree: emptyDigestTree) + strategy = ? strategy.init( + 0, + numPadBlocksTotal - 1, + manifest.numSlots).catch + + logScope: + numBlockCells = numBlockCells + numSlotCells = numSlotCells + pow2SlotCells = pow2SlotCells + numPadSlotBlocks = numPadSlotBlocks + numPadBlocksTotal = numPadBlocksTotal + strategy = strategy.strategyType + + trace "Creating slots builder" + + var + self = SlotsBuilder[T, H]( + store: store, + manifest: manifest, + strategy: strategy, + cellSize: cellSize, + emptyBlock: emptyBlock, + numSlotBlocks: numPadSlotBlocks, + emptyDigestTree: emptyDigestTree) if manifest.verifiable: - if manifest.slotRoots.len == 0 or manifest.slotRoots.len != manifest.numSlots: + if manifest.slotRoots.len == 0 or + manifest.slotRoots.len != manifest.numSlots: return failure "Manifest is verifiable but slot roots are missing or invalid." - let slotRoots = manifest.slotRoots.mapIt( (? it.fromSlotCid())) + let + slotRoots = manifest.slotRoots.mapIt( (? it.fromSlotCid() )) + tree = ? self.buildVerifyTree(slotRoots) + expectedRoot = ? manifest.verifyRoot.fromVerifyCid() + verifyRoot = ? tree.root - without tree =? self.buildVerifyTree(slotRoots), err: - error "Failed to build slot roots tree", err = err.msg - return failure(err) - - without expectedRoot =? manifest.verifyRoot.fromVerifyCid(), err: - error "Unable to convert manifest verifyRoot to hash", error = err.msg - return failure(err) - - if verifyRoot =? tree.root: - if verifyRoot != expectedRoot: - return failure "Existing slots root doesn't match reconstructed root." + if verifyRoot != expectedRoot: + return failure "Existing slots root doesn't match reconstructed root." self.slotRoots = slotRoots - self.verifyTree = some tree + self.verifiableTree = some tree success self diff --git a/codex/slots/sampler.nim b/codex/slots/sampler.nim index 5e0645b9..10ea2656 100644 --- a/codex/slots/sampler.nim +++ b/codex/slots/sampler.nim @@ -1,4 +1,9 @@ import ./sampler/sampler import ./sampler/utils +import ../merkletree + export sampler, utils + +type + Poseidon2Sampler* = DataSampler[Poseidon2Tree, Poseidon2Hash] diff --git a/codex/slots/sampler/sampler.nim b/codex/slots/sampler/sampler.nim index 802ebdf1..e04805db 100644 --- a/codex/slots/sampler/sampler.nim +++ b/codex/slots/sampler/sampler.nim @@ -8,182 +8,146 @@ ## those terms. import std/sugar -import std/sequtils +import pkg/chronicles import pkg/chronos import pkg/questionable import pkg/questionable/results -import pkg/constantine/math/arithmetic -import pkg/poseidon2 -import pkg/poseidon2/types -import pkg/poseidon2/io import pkg/stew/arrayops -import ../../logutils import ../../market import ../../blocktype as bt import ../../merkletree import ../../manifest import ../../stores +import ../converters import ../builder - +import ../types import ./utils logScope: topics = "codex datasampler" type - Cell* = seq[byte] - - Sample* = object - data*: Cell - slotProof*: Poseidon2Proof - cellProof*: Poseidon2Proof - slotBlockIdx*: Natural - blockCellIdx*: Natural - - ProofInput* = object - entropy*: Poseidon2Hash - verifyRoot*: Poseidon2Hash - verifyProof*: Poseidon2Proof - numSlots*: Natural - numCells*: Natural - slotIndex*: Natural - samples*: seq[Sample] - - DataSampler* = ref object of RootObj + DataSampler*[T, H] = ref object of RootObj index: Natural blockStore: BlockStore - # The following data is invariant over time for a given slot: - builder: SlotsBuilder + builder: SlotsBuilder[T, H] -proc new*( - T: type DataSampler, +func getCell*[T, H]( + self: DataSampler[T, H], + blkBytes: seq[byte], + blkCellIdx: Natural): seq[byte] = + + let + cellSize = self.builder.cellSize.uint64 + dataStart = cellSize * blkCellIdx.uint64 + dataEnd = dataStart + cellSize + + doAssert (dataEnd - dataStart) == cellSize, "Invalid cell size" + + toInputData[H](blkBytes[dataStart ..< dataEnd]) + +proc getSample*[T, H]( + self: DataSampler[T, H], + cellIdx: int, + slotTreeCid: Cid, + slotRoot: H): Future[?!Sample[H]] {.async.} = + + let + cellsPerBlock = self.builder.numBlockCells + blkCellIdx = cellIdx.toCellInBlk(cellsPerBlock) # block cell index + blkSlotIdx = cellIdx.toBlkInSlot(cellsPerBlock) # slot tree index + origBlockIdx = self.builder.slotIndicies(self.index)[blkSlotIdx] # convert to original dataset block index + + logScope: + cellIdx = cellIdx + blkSlotIdx = blkSlotIdx + blkCellIdx = blkCellIdx + origBlockIdx = origBlockIdx + + trace "Retrieving sample from block tree" + let + (_, proof) = (await self.blockStore.getCidAndProof( + slotTreeCid, blkSlotIdx.Natural)).valueOr: + return failure("Failed to get slot tree CID and proof") + + slotProof = proof.toVerifiableProof().valueOr: + return failure("Failed to get verifiable proof") + + (bytes, blkTree) = (await self.builder.buildBlockTree( + origBlockIdx, blkSlotIdx)).valueOr: + return failure("Failed to build block tree") + + cellData = self.getCell(bytes, blkCellIdx) + cellProof = blkTree.getProof(blkCellIdx).valueOr: + return failure("Failed to get proof from block tree") + + success Sample[H]( + cellData: cellData, + merklePaths: (cellProof.path & slotProof.path)) + +proc getProofInput*[T, H]( + self: DataSampler[T, H], + entropy: ProofChallenge, + nSamples: Natural): Future[?!ProofInput[H]] {.async.} = + ## Generate proofs as input to the proving circuit. + ## + + let + entropy = H.fromBytes( + array[31, byte].initCopyFrom(entropy[0..30])) # truncate to 31 bytes, otherwise it _might_ be greater than mod + + verifyTree = self.builder.verifyTree.toFailure.valueOr: + return failure("Failed to get verify tree") + + slotProof = verifyTree.getProof(self.index).valueOr: + return failure("Failed to get slot proof") + + datasetRoot = verifyTree.root().valueOr: + return failure("Failed to get dataset root") + + slotTreeCid = self.builder.manifest.slotRoots[self.index] + slotRoot = self.builder.slotRoots[self.index] + cellIdxs = entropy.cellIndices( + slotRoot, + self.builder.numSlotCells, + nSamples) + + logScope: + cells = cellIdxs + + trace "Collecting input for proof" + let samples = collect(newSeq): + for cellIdx in cellIdxs: + (await self.getSample(cellIdx, slotTreeCid, slotRoot)).valueOr: + return failure("Failed to get sample") + + success ProofInput[H]( + entropy: entropy, + datasetRoot: datasetRoot, + slotProof: slotProof.path, + nSlotsPerDataSet: self.builder.numSlots, + nCellsPerSlot: self.builder.numSlotCells, + slotRoot: slotRoot, + slotIndex: self.index, + samples: samples) + +proc new*[T, H]( + _: type DataSampler[T, H], index: Natural, blockStore: BlockStore, - builder: SlotsBuilder): ?!DataSampler = + builder: SlotsBuilder[T, H]): ?!DataSampler[T, H] = if index > builder.slotRoots.high: error "Slot index is out of range" return failure("Slot index is out of range") - success DataSampler( + if not builder.verifiable: + return failure("Cannot instantiate DataSampler for non-verifiable builder") + + success DataSampler[T, H]( index: index, blockStore: blockStore, builder: builder) - -proc getCell*(self: DataSampler, blkBytes: seq[byte], blkCellIdx: Natural): Cell = - let - cellSize = self.builder.cellSize.uint64 - dataStart = cellSize * blkCellIdx.uint64 - dataEnd = dataStart + cellSize - return blkBytes[dataStart ..< dataEnd] - -proc createProofSample(self: DataSampler, slotTreeCid: Cid, cellIdx: Natural): Future[?!Sample] {.async.} = - let - cellsPerBlock = self.builder.numBlockCells - blkCellIdx = cellIdx.toBlockCellIdx(cellsPerBlock) - slotBlkIdx = cellIdx.toBlockIdx(cellsPerBlock) - - logScope: - cellIdx = cellIdx - slotBlkIdx = slotBlkIdx - blkCellIdx = blkCellIdx - - without (cid, proof) =? await self.blockStore.getCidAndProof( - slotTreeCid, - slotBlkIdx.Natural), err: - error "Failed to get block from block store", err = err.msg - return failure(err) - - without slotProof =? proof.toVerifiableProof(), err: - error "Unable to convert slot proof to poseidon proof", error = err.msg - return failure(err) - - # If the cell index is greater than or equal to the UNPADDED number of slot cells, - # Then we're sampling inside a padded block. - # In this case, we use the pre-generated zero-data and pre-generated padding-proof for this cell index. - if cellIdx >= self.builder.numSlotCells: - trace "Sampling a padded block" - - without blockProof =? self.builder.emptyDigestTree.getProof(blkCellIdx), err: - error "Failed to get proof from empty block tree", err = err.msg - return failure(err) - - success(Sample( - data: newSeq[byte](self.builder.cellSize.int), - slotProof: slotProof, - cellProof: blockProof, - slotBlockIdx: slotBlkIdx.Natural, - blockCellIdx: blkCellIdx.Natural)) - - else: - trace "Sampling a dataset block" - # This converts our slotBlockIndex to a datasetBlockIndex using the - # indexing-strategy used by the builder. - # We need this to fetch the block data. We can't do it by slotTree + slotBlkIdx. - let datasetBlockIndex = self.builder.slotIndicies(self.index)[slotBlkIdx] - - without (bytes, blkTree) =? await self.builder.buildBlockTree(datasetBlockIndex), err: - error "Failed to build block tree", err = err.msg - return failure(err) - - without blockProof =? blkTree.getProof(blkCellIdx), err: - error "Failed to get proof from block tree", err = err.msg - return failure(err) - - success(Sample( - data: self.getCell(bytes, blkCellIdx), - slotProof: slotProof, - cellProof: blockProof, - slotBlockIdx: slotBlkIdx.Natural, - blockCellIdx: blkCellIdx.Natural)) - -proc getProofInput*( - self: DataSampler, - entropy: ProofChallenge, - nSamples: Natural): Future[?!ProofInput] {.async.} = - ## Generate proofs as input to the proving circuit. - ## - - let - entropy = Poseidon2Hash.fromBytes( - array[31, byte].initCopyFrom(entropy[0..30])) # truncate to 31 bytes, otherwise it _might_ be greater than mod - - without verifyTree =? self.builder.verifyTree and - verifyProof =? verifyTree.getProof(self.index) and - verifyRoot =? verifyTree.root(), err: - error "Failed to get slot proof from verify tree", err = err.msg - return failure(err) - - let slotTreeCid = self.builder.manifest.slotRoots[self.index] - - logScope: - index = self.index - samples = nSamples - slotTreeCid = slotTreeCid - - trace "Collecting input for proof" - - let cellIdxs = entropy.cellIndices( - self.builder.slotRoots[self.index], - self.builder.numSlotCellsPadded, - nSamples) - - trace "Found cell indices", cellIdxs - let samples = collect(newSeq): - for cellIdx in cellIdxs: - without sample =? (await self.createProofSample(slotTreeCid, cellIdx)), err: - error "Failed to create proof sample", error = err.msg - return failure(err) - sample - - success ProofInput( - entropy: entropy, - verifyRoot: verifyRoot, - verifyProof: verifyProof, - numSlots: self.builder.numSlots, - numCells: self.builder.numSlotCells, - slotIndex: self.index, - samples: samples) diff --git a/codex/slots/sampler/utils.nim b/codex/slots/sampler/utils.nim index 0379fb07..fa68e408 100644 --- a/codex/slots/sampler/utils.nim +++ b/codex/slots/sampler/utils.nim @@ -9,8 +9,9 @@ import std/sugar import std/bitops +import std/sequtils -import pkg/chronicles +import pkg/questionable/results import pkg/poseidon2 import pkg/poseidon2/io @@ -20,8 +21,11 @@ import pkg/constantine/math/io/io_fields import ../../merkletree +func toInputData*[H](data: seq[byte]): seq[byte] = + return toSeq(data.elements(H)).mapIt( @(it.toBytes) ).concat + func extractLowBits*[n: static int](elm: BigInt[n], k: int): uint64 = - assert( k > 0 and k <= 64 ) + doAssert( k > 0 and k <= 64 ) var r = 0'u64 for i in 0.. 0: + return failure("chunkSize must be greater than 0") + var index = 0 var leaves: seq[Poseidon2Hash] while index < bytes.len: let start = index let finish = min(index + chunkSize, bytes.len) - let digest = Sponge.digest(bytes.toOpenArray(start, finish - 1), rate = 2) + let digest = ? Poseidon2Hash.spongeDigest(bytes.toOpenArray(start, finish - 1), 2) leaves.add(digest) index += chunkSize return Poseidon2Tree.init(leaves) func digest*( _: type Poseidon2Tree, - bytes: openArray[byte], chunkSize: int): ?!Poseidon2Hash = + bytes: openArray[byte], + chunkSize: int): ?!Poseidon2Hash = ## Hashes chunks of data with a sponge of rate 2, and combines the ## resulting chunk hashes in a merkle root. ## @@ -41,7 +71,8 @@ func digest*( func digestMhash*( _: type Poseidon2Tree, - bytes: openArray[byte], chunkSize: int): ?!MultiHash = + bytes: openArray[byte], + chunkSize: int): ?!MultiHash = ## Hashes chunks of data with a sponge of rate 2 and ## returns the multihash of the root ## diff --git a/tests/circuits/fixtures/input.json b/tests/circuits/fixtures/input.json new file mode 100644 index 00000000..0de9bcfc --- /dev/null +++ b/tests/circuits/fixtures/input.json @@ -0,0 +1,527 @@ +{ + "dataSetRoot": "16074246370508166450132968585287196391860062495017081813239200574579640171677" +, "entropy": "1234567" +, "nCellsPerSlot": 512 +, "nSlotsPerDataSet": 11 +, "slotIndex": 3 +, "slotRoot": "20744935707483803411869804102043283881376973626291244537230284476834672019997" +, "slotProof": + [ "14279309641024220656349577745390262299143357053971618723978902485113885925133" + , "17350220251883387610715080716935498684002984280929482268590417788651882821293" + , "3614172556528990402229172918446087216573760062512459539027101853103043539066" + , "9593656216696187567506330076677122799107266567595923589824071605501987205034" + , "0" + , "0" + , "0" + , "0" + ] +, "cellData": + [ [ "211066599696340205996365563960462032209214145564176017965177408819390441927" + , "256399834032317991719099034134771774537377713676282398278615627599320306708" + , "40526956212941839024868120947422067322935297516255336725720469887577875470" + , "369406626072040375689238003388146123438765868500054546379159741776926336393" + , "333671948877941061129138970028865844558745266314244224980179694307884999701" + , "12844191661993811401197260475054004253686019503294245287625435635597431176" + , "103242505924551040184986153577926463300854479121445566984435820844932904541" + , "357267598134410031301285377503939045679462829143562392803919346036584141082" + , "162961530392479745020607594774288130869393650464001426863668385541077786641" + , "426633666684068667053061341108297711407154520752589357873877687002123230254" + , "131217200633679697678620573903653316618851166977399592012996945149000115543" + , "347806146382240882424437318028620747118202052207615339887983883245341422889" + , "373560143578047669373240014805175743607410583882636003337120578008437374619" + , "188643112726610812698950916978832639394489168469865816132268079928321994342" + , "261643693073361806247543578456646407192325702146752104760117340650316255422" + , "260425332276127964154119199351807753107358504026338378706506655351595199132" + , "374895089121103633563000003194929058314955925591115574099805066048344387554" + , "251251687633538360627384151887287228926166832908633974282160065378311171354" + , "72870348025150463132527129203816383011245664502016571773976961034605631401" + , "234969517550818492643515432666311755833657223377594670026839098818269671638" + , "250704662734070531273640113211555977086430125469327371276827055724111014200" + , "85287059658255939741261887818611116570376488685014337052369839580946343903" + , "148959658976765873884541474400081762855732313547557243964921157967555048302" + , "402116967301520959272239788104745348344918207829380669065019055837727389479" + , "440503687192139964066297025050080823601005280790824714962651368433530759519" + , "149064344353438643307231355233484617851197634669308957706338957575177745645" + , "249140840255377018814348914718469942883867200392561109698520706525194687651" + , "108796851176515124780842490199733462942992752881710253277179665118758071359" + , "168245155425564161902686596247453762364112240129335852645432169468767513906" + , "129930759770229612264501691941454447321585806001002088749773920103899362070" + , "26204732465033162738545933008873662562758704699080684615280202127289894343" + , "434343986187775289860542785529657444690073100436887472033336117760907652966" + , "361202432740487795596808128962740911600093857340619816047190021218849540225" + , "100616813001075101816809823591611435583084916492802624686700765550893945525" + , "262383566766515164611427346701355047932794351290691325516723194829671679460" + , "223966317895663049002893008659178463136086169222436544014405848127792334099" + , "416071800998333357259662686053338495720384342746822618737948080251761863079" + , "402582378631671531909245563300554883898015688826468151075059467077182712018" + , "271682867846395286938993638577506552857925968097084028550962231439839229096" + , "447239701195120035004067146414333903010840427278848992921567785105217019890" + , "275718367441702584521943480326858762208121719038684001399322597215141179102" + , "86424132534636958411139699704035287483951667275825975536719519441147485752" + , "149332313586183975744469174256094358432564607635406143904268565140988616920" + , "431284330776421588418608279008210842837281123158951642164956884286883748089" + , "328694410745471749523135644660455669483988686888634622076863114197617693825" + , "112671940998917362968156144648543607958275336559773039070338509488400224090" + , "40612851250697989627190554726382690498263128439797780029697069621854862060" + , "235047914228675997216342104196597257178021277585839376175077878186492271543" + , "169718735151210135199527910197065439221144015957220768545119706561079163228" + , "345850109040121443591415752965486014695272086901102608769402892906795715635" + , "107916794601837835951508003838161872232087225679609623116098837565956752373" + , "415195406060373162425374246348197423165008252112453298469572523506488563795" + , "18574536720926634955170276058049993354780447816096707123565164996905722992" + , "77316964375201096801231570737992491072607736322255206149311341101525354519" + , "198566211140075666401818378905444403588640345933666108724809349396921957675" + , "71926050707400318807625168942501384117254391471312636171205913503928815127" + , "303403754792341398295052586417779761162194818661412867094096550880325459639" + , "444796230931706624375881141460151785952798771079111017071124833045933389733" + , "430832727519144759265424205289522948157007336118070755365887670730658782114" + , "75431213985866235726407973044434444984663930761207296437571668004273515965" + , "9242635103653159191249730220870735855877366952081272723035956668095954838" + , "93770678769846326584848478152412123903909949977598807336203128684179492141" + , "438043261966019084676377174988310087831395864304423411701911688757793135582" + , "175357918416563657734972138036003712114814934655676571874083109097142591069" + , "301619681954194702458985259161884119574424456150215738560639417824784261940" + , "376627771252167062559065889174364784087884871999807562320457079200311413098" + , "77407" + ] + , [ "424838222746158988229624788694939151178615656210585621868910231014323837551" + , "113188486246955346418956949679485575685258850346101035778277727456423482970" + , "275449978206132565019222655023969430014622832597654643418394602485645803413" + , "407856757018138010439232009766252068440920591566039673660874626878413077905" + , "433369046493777496016384947949950877133856528218602671493669395706908819748" + , "258364166531180422149545015891786023981872586904946376136311798402581278793" + , "111997719221028147522803956659709775148434180015507797582340706052412284571" + , "370086597976426508280413876667101417393598181708677558733730556109327409076" + , "394139601979449259075910238117153992797849553309541269624848742084205563806" + , "224088276319080487199395482893988152025671468715318212801266537036640477323" + , "412710245119809501914481942088314642684754542082140451180970198371889738885" + , "353872602359946553306242341162348980635834907495492814598834657175405697176" + , "252575199506028475372678621140654219936768774012823764047177692104580452933" + , "259093153824033122452869556249315839899366493071746219770487886456301642099" + , "433829976798312333371154167497560676982294392632725612538640639617101218872" + , "69918581382122563555200898078544150952625715196904114153232367538572342772" + , "337937520623192257595352158476909569245988839238498098464935654555688460123" + , "264739973232292969253318643276671532055689422253184191167449284055983944338" + , "326116252818829775096345069850111970510714050346103409479803743342138260656" + , "236666502377413649728378889488706275212721356921124776708954261777813709815" + , "211625935799984260567718228446525455893664313064841539301444509150157287163" + , "60213206239417039999880027112341976360540689886703427811513517396638607512" + , "68310118105957780876770075529546844404225720757669797609686816545988561625" + , "423863085551351065136684030270731679105571943009795949621903966660399419936" + , "388914614294393005039878123500859325222684672184567792659076815268598434245" + , "449456790291560508709069826219925144971979653209755565240911568965768874382" + , "448810363770763694447869940916735951256986784286793489549428379909616059117" + , "93646909783664049092056237949587618925209622020026157405117796611689551192" + , "352210795298632954574896499649181574584074853828419384742874364724522457331" + , "37455517056393404525863484733101879886413925183061645520768912552476716150" + , "386617357584684336812125385078476270301738184058813703112840991226785114117" + , "309940292044597334261558429176136686101590729982259514472573656131579113438" + , "375815246167575100319857872432879650174355611853064771241069582477717074415" + , "332214507344122806007757734266883566559371568252728459951766124888176633706" + , "148990259460952914990881100852534318351247069504848477833147446514732789712" + , "328669527889838880414072022433859139004058211332184916573516704632073044118" + , "39278026039348543645873027549112998051601664395028652771103624299930924528" + , "147717660530589785119644237145092759103012624422229579698752386490700965238" + , "374018518701362017594752095877197725242352803195413267746619111489936980685" + , "19185486483883210730969367354195688373879769005461272861759636600984416877" + , "61866046506558157021682973167090213780467780519546382332208868591026703563" + , "186854966504766517012887726614015646154225796572138017810371160981778288347" + , "87813550250328892091332566928942844770632705056120813488729800874811845697" + , "207775163424060266085108794048371834145545842567796157378282772999548202308" + , "369987573847786237689249538753881486995686208870889713177975415012214427429" + , "240880979016395044518849230927615466120209140082149273390921042537474853143" + , "174902051454932913375934735427101804474275543418199101786687925733405159872" + , "342217255652950822372803598682842961053537267723988087801275319754065261308" + , "403207518067666945448161377960706451817747922771285796668778802535227939962" + , "407191459999036791052261270163259267557900498930952521056725210031161568230" + , "338216472523551728793268225845396218561132966393725938551091882807069657206" + , "118364222678576550870375112494142500603091119946985793934499723872824782886" + , "269721611028665323587192624288165848310917029418261851436925829954710472436" + , "227424498125745236125352117206136621428869662458452610254773560636280935711" + , "334380807433339401906359555583987757411855090694162252260781648609761248049" + , "42470806516174819075107446234449247453971524726021445768611797530804156161" + , "418994916402918322951830716592888390611524984156817012683478842068581638820" + , "363263142412048420546716019774090729399081311227606555141174736853886128407" + , "192292715923468025058557166341646729623133127372303791236447550026886802680" + , "450253878713722337767292128303865371116770532625906889925779639839924402495" + , "412596147086332805611392200560087191411541689130482740065137300252639590489" + , "264059866105067484811456793906835462997849523506366903974724979936196358724" + , "80922384260325792825608274004101667366364502159441286540209512108302572137" + , "69261112192907071823876894642934902051254647002357333297635511793652550535" + , "342810644068354896385837929029331085645928473943142618800192452300937098227" + , "228361826362950356801549793202622774869100858404479869989505302905936946659" + , "89244" + ] + , [ "359926778925154809567585559910064420821311221299937568759183366972901588855" + , "128688825421597555299113005875649976344934035310192572516197551102045945994" + , "225379354482053502673695304662016054746658452775856157938886424152134693969" + , "321872319934407904743844034025550739031781361848065513098922085967524180784" + , "250375637325405951645070615947051799520095842815922754899017741501395744611" + , "97676493052788080672307677749501730337551225267342432472194527468634722352" + , "140101187036396881926000630022834247990512766860086619783437252676747730662" + , "428833039353549335335605804240430918631639449874968791377641834408506136850" + , "418359203734539413977740838354554804415161215624809316660001820037711273005" + , "197411877418795659129213175603102238904459737200167987167255995825203749339" + , "221646252100316559257470803343058462853641953756978011126414924636869625612" + , "106393540293584181037890192557883541231531964825708650997196071036779482686" + , "121473330828208543539643554911190528237124877123871673169194452404939833883" + , "234055622144947293638512253368547046093971383516706577723613696162225606040" + , "68307451767502390304445005915787226559811433450721625085741437522802389574" + , "446891883436763112014492564462451523127134145501201571918449345324780633462" + , "83718652783543018019599555197511164121642363321504039439786358267060414949" + , "90267297500929836073049162292104311427365986272517761342871530827272320168" + , "398425606698859520268856768787424690872952910789754531465894080258173664751" + , "323570139379118444589557840594603212198136718240764273769210492735883659788" + , "318597103584099056378057647488068323974418467250708490151864712850204121402" + , "6299083430773359277240726214182464517380839990956316267425262319606033077" + , "27638206326925436960316131682014727983280820447721477666884742925275976240" + , "434344186848869917381375812528446841024676181532946456237439060027443649574" + , "64735754118644738348599761514174981324344130078598038275246522384474432918" + , "53068717269762105498508401788249005564862415051761175636612434108259085043" + , "35813044996911619267309099508360887777226716179396659295580849861836012116" + , "67751791392924142809580984450371772015056060429352159361446914484238646676" + , "68534949135677447506316576616742207938855454921330757052467057435206318183" + , "98510151949547604999069864337574320742530406479752176012935179772005228326" + , "342190252152505345443004241184891966319091967630257822491352072978326623645" + , "362701658859425316334005554473186516818256386066010799465369887406035738447" + , "266999116654850467726292928465517542818678046748008340458185725047959981772" + , "227089355966197874086821090531951502393729872265201602128464978982907992285" + , "240800343500959216904535208047288234867926058830277460630902462914796702354" + , "447956858573680756485556898469710354624642653441041335815079102198306530583" + , "89422712944117481549242421245588048728782658978853365197341587057196539094" + , "72610343179362050463955730204044877712105879926618304878262944723464870506" + , "8676698500519447254981838968537883138182811064381378248657969913325524054" + , "180453700216061196739413267121764366438386031039311941313645439527087166894" + , "63346784016053253727621352882430122335280702556586808389293772066791583857" + , "400031453850139978805133735852120986024101930860924735862305444817383365395" + , "230104622290558570218036071349472289358926019290368625724986905348610140188" + , "175689489221336091369196327293045073133701056385414159213338224521167050830" + , "73310331103509697419315970265031228794034932318600293733074730812482185479" + , "371383255255842707875498538452907102684511927320158672347778293876877893808" + , "165319345890230193939972313953881372394171342391835626454759321320435952720" + , "184753541001210613115361457830691571384268642766106935428207829332011259768" + , "378810733004878432563271790625801205570962762395854282745166380274493181314" + , "86321674336629444862383262780020871828941143514651008200999725989892879714" + , "332634533993388248915777870988529817692938793120418377552247997050250349749" + , "41742010257820712511267405684781534740157292266212120171929853763272599516" + , "224101330592139734390658213442359402546038743346294438455635537496290117560" + , "204363902046137087420878796391135987770418514008394389852388361468850216359" + , "296526036888463156867153847102999031430641220536577556044554029990778763710" + , "137568796227115931047082828377698464185467276723279763304686078869351280509" + , "147456720891843338735232645318045656082153590170441596326500204393398792771" + , "297291342309626392877004635010131510068476699687818509485687346071074942006" + , "20748013593486831233186810444485136836664689496258849465507672301203832324" + , "335431726883875036252568773858744219410188302724423071283643221351691013313" + , "50487384098835523033417523223562560857744547945136829388944024807752630716" + , "425952679139710019732649156938676226492714486376811103362264146658191708598" + , "439787938069461539508728805881194071103269524405653997968488488049426387373" + , "279863410796988495259178322026394289028023166886112128504434877538089779477" + , "398941099058270093463626870965433502581301413791423667994257456160377865247" + , "5759692644185723491187478313102540786562625675495805072053262277490225012" + , "115176" + ] + , [ "199440901482393381753657315848210955092644686957900737971520778206058989647" + , "339215657660349719251187938243949119362753238126598750470832739830379601048" + , "17957417011314891245400567671664723859427624136284133790923936126779445290" + , "294761585889095249928492042608516765584723814657871392207964321318076158536" + , "367304199921887970655898837985855454346911090426896946930048519042744277770" + , "173405546837606747721292792526074597519538685230485266741646923399938591491" + , "13202798104529529703580600642858924379886936325402696094624200032343206719" + , "28211272278315691894282764239983301742024079691520980592618486887749800025" + , "73792448247120972778500624350664849847034095641998271809779791788652649022" + , "386961947078838359430674078072441680475090687247027225632133013772954043342" + , "247859266401821616700765969075270662915024391205665146199401830650793676517" + , "243938047874995926342875119559105623088213951205962677439167259642163766960" + , "14909501249861872673329370269106359532506159818320693170564856401208688898" + , "200331653478898243177761429526240803993101536716611440775588088625522029071" + , "127891684617049394579738365914860024509007913559921966744972525605976847919" + , "202912167983786187592861727924433749852786012202809544200943965898118027816" + , "176370650316309755425558132466370508977563252342126855874617990006444464573" + , "179490319446297562731655155074196383539396893457024237113284509223965454107" + , "118703379899134287650980989454755985628620830085932176414465582081598659194" + , "102025594191113707886629488454876652037341696284939671367279050169222988689" + , "421132375430104331136732058548913808473025321492255503838896123601628815453" + , "328334791815856213206267892694535121226673194052951034497930366807851111845" + , "83012322813281668737061895967682970093636853452224812061110092135287899376" + , "329204708391107220275172926348002826875172581958734129645882445887919889321" + , "410748869385474485539728045765785256294532845070137964737879849265390625591" + , "197274807717335387012872999914232051341799797613667869923402386359379902675" + , "235713095185988155553500217595661312303861624791720350423698435045853678746" + , "150631584359141913552843813384841153102535679219851913018874172414598353488" + , "207783836843813911284913666774420250626019971129676431904589416081127483900" + , "15728034718954665549174230921445077500399069880555428849958014406422697976" + , "69799423545177501667748653663121504155622623013014583690766769624482972893" + , "265665371394145560256710553779588458846778960884758560117805511822299802326" + , "149195925869584039415331014261414953602245337159354350672030141190471260449" + , "162328395279114064180857718453973759378615891406692054752029241989300597156" + , "104643123291849369328362168243087456326274773811561124607060302871149280568" + , "320704123383295724141970902124509237736831907552238934395000039155541269937" + , "77914486216152383968400591704791756847610018757857965758408455442143531631" + , "238365259321088298905088250444146071545398991768186471689605160523129613763" + , "279409375422154593510552116666741774885392805494152031820287626934209978908" + , "195118776021452609708543280509123101181249086555819844047203390788132717252" + , "197977884437087886153482042453896079316138251415359773453987232734849953584" + , "168185043240980087638006965666857387510828226074470344841850764460631595331" + , "231157923359356077977363679818678437836536420121744865399935742538602805912" + , "177903771863742191900138437188329108771172098110036075491750018158192424072" + , "313552174443290416730632310997197097951229162137491595709733420111980331403" + , "273253450712049988786741336540196077743997302924525995219038781650977490211" + , "421908030281055821389377531613150504859996607596444776050212044919345332385" + , "180108184992593746898140529947178182204857361841304042854173884504394805936" + , "37075272799330399065679301151342697855905822543084867570322173216259074746" + , "364885615491975468180698816037289079302391684470748470356247509218051645743" + , "397482868106190800111749908137311511782294652288655656060379563261618687603" + , "192853269627895017416381451198403197013798085262867793715522216458130791820" + , "450480853450142791928572953497280890976328598410525218090104787739409705079" + , "40278654070502961330170439514434669768416784968274956021022493731441898222" + , "251277143131769020481025315216040765839561111684608785317366609258959942695" + , "95094468748825454459610961968601800404132682484160170977941285794444806916" + , "160586633865113902389134480029183924655750088163491531655080014223476604929" + , "211661229493873434581423107377168478957907088187044576032505407590783850232" + , "409651293631434750272174456674594508340952460788494035327119354167465019826" + , "233213211946836553080627522409887799285199986120567245752841080275284294566" + , "143182900674482411759481361336063079267405785923487419697568907351386146653" + , "430050085956999990041799366370428972519385994997821389120583306252090911051" + , "241257468571530133762629460194062384921386438426367660851087853915892684115" + , "106478922860328643074356032194102718325829441005019365153538120054339275205" + , "252933430690486626644000908036895289326526510137514385824014300035301154822" + , "242924628511152862437189415942615812459145003511499320946981326550434266392" + , "107566" + ] + , [ "10892488375325920610152294701785926476935321890453222549428070867493882259" + , "230776541958253414701326765204413805639078570664616139597900921490475143840" + , "162235550819840758141721889536295480113278275911087429090017541814695333320" + , "318634611531007856220026646570492630940047240387334221027051366284225674524" + , "347695480420330337439096561080975864031317110001559084872978387417651465445" + , "243301070227446762125488369714708670219289121859111929877695012393609726208" + , "312153141205681954392441579373652470334482191990502803524039048245142338874" + , "243769659456658813016931656268556893289414122189859136776671772112510762644" + , "235510946617019540983239794402008700611014664578713646813426116406215382253" + , "394638234040056271265534896894991100558052611842099314808878257003754175212" + , "112730195097163222179949980634599934392634120069300673310070655800491242211" + , "112545144551723145227061757291353149296490850338535641681237178578430772288" + , "399161925498018051746424503488168548076537557369822821644005390567188305750" + , "291823556779130095044851238536860577785281148245623338113991004798525195947" + , "443006765181360772964201996695219533539991249361936300010158089981376772939" + , "74018417655448362012716269253153545524112649147848318337218250865231619883" + , "361038295627629757439073080656763633087074408866229383288288831546300069767" + , "269655542872834422597617091710145830035767834433114567250727497135451412216" + , "58289072717559976781527037332370898163550414718655391280446986067842922181" + , "365399954331278626941053447122605263207706937018525782908294459265663426953" + , "83576501872896181822963149907518188809522341045703363171261203985876068484" + , "203403783686919490357886887779316222746544665267595235714814853282937072937" + , "226090172488558641139782927103632452136731207206963592401497570070700221117" + , "249813560776802008219945831355163226098682741553305882699128946352901227282" + , "236586835155013316983057471119105606475813711035522306026673814184519826069" + , "420611449257527132395951061920868895981487081726689195426673433565077012458" + , "414979562418422585161189066520079202660418565693776869373399931253524536378" + , "115851377630895049619958829726730418778383692593973233290077769966938018584" + , "248071158447148977966329235335508229928647083339924308951291087789494073866" + , "8254100651607835318906499132096926759832050649688561036854000785129084907" + , "91385483239630999205401307763890943367451672856478206313711901403403429289" + , "369346641925770698935046191632374874762045839747680407985443471202350286304" + , "236809023553698844817613980395588655518336237009075203833474757804664254158" + , "8367847400805682648908349286699722579431227561083498361702537964306290078" + , "599241730770400067632779114435635342549228985534229813617556932580328166" + , "347112528350917448294202348829052279076907614831011498643025223836596915573" + , "384244379244118003891043669466323943736726794634167201471569326059716944701" + , "118013777197672343498581960057939216208494837962825017767101107204031333144" + , "27234916267695376599463409893017377196853108034589808756909998459364893467" + , "443519198016088819704735929590164254445884637317806160485888215392818578737" + , "396780482611567392375183169345153676737175342167284140440545202776279411157" + , "420351155303051883480975795437743307852799862858964108014000673383502660760" + , "17379377743250873773932440622865094720355292220433235366224143179854831702" + , "299671454782683147939928632170233327590769402224392134648893444626929909373" + , "143062753141414050359792615867774312517100868919516205179025540179759009492" + , "79497692490953838158801094558761984613913564034406069659969793097043605498" + , "422748645389700647011491406944374966856916994331478229959954030359911549565" + , "101802829812014644970197499895811874607753186302439171072935333706660468030" + , "376428369998893026519415315112012919906032811618495880392785036762185101192" + , "193969030999254195249242252871597931610859408264053152789041067245597391073" + , "262277607928686742238285487190873200602833495734085188071246746209841324139" + , "154099884960502807271641574310268486840763221700920893692135020347157046386" + , "155875061164585018658671842995328931296342883770473498362059838106632382461" + , "248574435283666782825705601259695525637993294311364397935499480206725256362" + , "171325185063038052248966557755722232979612702743265316145563443527135798688" + , "19982746887818897250405980185937061235439217109294376948752373205830077881" + , "363719103724181291346130833008745948141602173373912337315631878022251200824" + , "174596812883797666354579747966720458118607233660798323559531788300018084931" + , "296611197821867585469311917529661698312828606304722408477045992233526328708" + , "115884038550627840768260751168697204665962542002024023842649383174518895165" + , "265597417366164841889730505737916646261040505851159477903649773521314216810" + , "59890857222664166616144499778264545115438678877460924559608721809777680238" + , "150275344313515259978222149421664752546204516114655782705875535407472455999" + , "119762211657733951640135703777940013374803447228667976561992857374027112851" + , "124750313254270944205036764607428674226393862430770588148329786501973600535" + , "223562415856611692667255745278751292230311455355383541649772348032933666931" + , "70851" + ] + ] +, "merklePaths": + [ [ "12330511756602656312909435206144412037562550923861053314147193494605624608532" + , "11626412651279744307149389430094868876817959605147995292836995419396445628874" + , "5992799448428980485292103987999399446160713735809250167288027256759377161164" + , "19665782623633007009046423286205585606029554995684266199914536255425604862856" + , "16487082247902739801276638140231448150060608377815926330854457096549924699346" + , "13757776896542890425183206586238760231244468647794123671233758868377423038254" + , "5689382212199289790852870716648320118282913659439556386010574440270030991956" + , "19397819444005071538172083272808000152189643623980212079104170950073560541073" + , "13602141253349313166682170066159161155345481788270338632198566205283140117430" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + ] + , [ "20475873274412005481064639175076772466269555202404881992985142339142752174247" + , "16346160910918020455716115524174351294558871582658635761456705803199256575588" + , "2853750013041818757293788273269978233503065226276819540991640203844566736443" + , "9192572535522846104757923561847416578417599996904208474460700268961782558170" + , "11041850361074018956732434352769265303294549935255362322653487210796196161858" + , "20835509643844784930831626622202658364476409300598072395494952478408974334325" + , "15426115581767819720710837762133134950520914636073261355445708100826108573907" + , "7565353224987902191368863653499353764559862741092477570130316358454603122676" + , "2622681935585012630617774892501744551457568716225188460692779556142778732663" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + ] + , [ "13099868869639061574800067722436547911616384523753701384396275064543709640456" + , "353757809120595355213201328586632712724405232919181040928026587840976194078" + , "17653300914565730132855106316678548541847283141888106932466281385199556950861" + , "15467462085462582082877261755656498905479817107855355753427299990712166382496" + , "8291733777946446853018893495264026584437749231931118771866890345692346711355" + , "15510790697317206014779022286261864844918915222875014882833700758879700055506" + , "5689382212199289790852870716648320118282913659439556386010574440270030991956" + , "19397819444005071538172083272808000152189643623980212079104170950073560541073" + , "13602141253349313166682170066159161155345481788270338632198566205283140117430" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + ] + , [ "123238869181525326412236116167249816873084559218151119452851092131080991962" + , "9610314342084317647296061595824603740114670828357751076517430572434680540425" + , "16802740554584732104294972716558962567961331277692246600846665155168171370476" + , "151083360419914122898584757765086723506432610661508069194962432698872036623" + , "10357032992337239725601662829902169825217513617307319193581711776597892496381" + , "10120699018002766520605012835043517238241846918467244955580419060582311503402" + , "21149604008153751948441881526949680605328007895979738537313721955134548786062" + , "5720106921878932614189421948890362637585879521377362100104826996201092964473" + , "2622681935585012630617774892501744551457568716225188460692779556142778732663" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + ] + , [ "19153513112714782931111012244694922243101677748840856395814929006033044311081" + , "21046138187228318287277629107063936540039891592394801899272249280765102572688" + , "18057980437430910028015917806534512217725128031222973066601095455076586015436" + , "5766677914654397407881589917461473873246279171605373166264025525757502238061" + , "12019967669236656188577515900815533059046454955207846938479617973037184411021" + , "14504305765289705714959523666100275156034056689367568164630385862257567596209" + , "7152002871325824138073253423783370852632926621899161541618248808716037342022" + , "9714587356194206699401761190093056901650105401919163689816999407566849779455" + , "13602141253349313166682170066159161155345481788270338632198566205283140117430" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + , "0" + ] + ] +} diff --git a/tests/codex/node/testcontracts.nim b/tests/codex/node/testcontracts.nim index 8eb235c9..373a4a24 100644 --- a/tests/codex/node/testcontracts.nim +++ b/tests/codex/node/testcontracts.nim @@ -31,6 +31,7 @@ import pkg/codex/erasure import pkg/codex/merkletree import pkg/codex/blocktype as bt import pkg/codex/utils/asynciter +import pkg/codex/indexingstrategy import pkg/codex/node {.all.} @@ -54,7 +55,7 @@ asyncchecksuite "Test Node - Host contracts": manifestCidStr: string manifestCid: Cid market: MockMarket - builder: SlotsBuilder + builder: Poseidon2Builder verifiable: Manifest verifiableBlock: bt.Block protected: Manifest @@ -84,7 +85,7 @@ asyncchecksuite "Test Node - Host contracts": (await localStore.putBlock(manifestBlock)).tryGet() protected = (await erasure.encode(manifest, 3, 2)).tryGet() - builder = SlotsBuilder.new(localStore, protected).tryGet() + builder = Poseidon2Builder.new(localStore, protected).tryGet() verifiable = (await builder.buildManifest()).tryGet() verifiableBlock = bt.Block.new( verifiable.encode().tryGet(), @@ -127,9 +128,12 @@ asyncchecksuite "Test Node - Host contracts": return success() (await onStore(request, 1.u256, onBlocks)).tryGet() - check fetchedBytes == 786432 + check fetchedBytes == 851968 - for index in builder.slotIndicies(1): + let indexer = verifiable.protectedStrategy.init( + 0, verifiable.blocksCount - 1, verifiable.numSlots) + + for index in indexer.getIndicies(1): let blk = (await localStore.getBlock(verifiable.treeCid, index)).tryGet expiryKey = (createBlockExpirationMetadataKey(blk.cid)).tryGet diff --git a/tests/codex/node/testnode.nim b/tests/codex/node/testnode.nim index 246b94f4..5f166281 100644 --- a/tests/codex/node/testnode.nim +++ b/tests/codex/node/testnode.nim @@ -126,7 +126,7 @@ asyncchecksuite "Test Node - Basic": codec = ManifestCodec).tryGet() protected = (await erasure.encode(manifest, 3, 2)).tryGet() - builder = SlotsBuilder.new(localStore, protected).tryGet() + builder = Poseidon2Builder.new(localStore, protected).tryGet() verifiable = (await builder.buildManifest()).tryGet() verifiableBlock = bt.Block.new( verifiable.encode().tryGet(), diff --git a/tests/codex/slots/backends/helpers.nim b/tests/codex/slots/backends/helpers.nim new file mode 100644 index 00000000..8d6395b2 --- /dev/null +++ b/tests/codex/slots/backends/helpers.nim @@ -0,0 +1,143 @@ + +import std/sequtils +import std/sugar +import std/strutils + +import pkg/poseidon2 +import pkg/constantine/math/arithmetic +import pkg/constantine/math/io/io_bigints +import pkg/constantine/math/io/io_fields + +import pkg/codex/merkletree +import pkg/codex/slots +import pkg/codex/slots/types +import pkg/codex/utils/json + +export types + +func fromCircomData*[H](cellData: seq[byte]): seq[H] = + var + pos = 0 + cellElms: seq[Bn254Fr] + while pos < cellData.len: + var + step = 32 + offset = min(pos + step, cellData.len) + data = cellData[pos.. 0): + let blk = bt.Block.new(chunk).tryGet() + discard await store.putBlock(blk) + blk + +proc createProtectedManifest*( + datasetBlocks: seq[bt.Block], + store: BlockStore, + numDatasetBlocks: int, + ecK: int, ecM: int, + blockSize: NBytes, + originalDatasetSize: int, + totalDatasetSize: int): + Future[tuple[manifest: Manifest, protected: Manifest]] {.async.} = + + let + cids = datasetBlocks.mapIt(it.cid) + datasetTree = CodexTree.init(cids[0.. " & $expected & ")": - let slotBlockIndex = toBlockIdx(input, numCells = 32) + let slotBlockIndex = toBlkInSlot(input, numCells = cellsPerBlock) check: slotBlockIndex == expected for (input, expected) in [(10, 10), (31, 31), (32, 0), (63, 31), (64, 0)]: test "Can get blockCellIndex from slotCellIndex (" & $input & " -> " & $expected & ")": - let blockCellIndex = toBlockCellIdx(input, numCells = 32) + let blockCellIndex = toCellInBlk(input, numCells = cellsPerBlock) check: blockCellIndex == expected diff --git a/tests/codex/slots/testsampler.nim b/tests/codex/slots/testsampler.nim index 3c3eb5f8..50a40c2c 100644 --- a/tests/codex/slots/testsampler.nim +++ b/tests/codex/slots/testsampler.nim @@ -1,155 +1,4 @@ -import std/sequtils -import std/sugar -import std/random -import std/strutils +import ./sampler/testsampler +import ./sampler/testutils -import pkg/questionable/results -import pkg/constantine/math/arithmetic -import pkg/constantine/math/io/io_fields -import pkg/poseidon2/io -import pkg/poseidon2 -import pkg/chronos -import pkg/nimcrypto -import pkg/codex/stores/cachestore -import pkg/codex/chunker -import pkg/codex/stores -import pkg/codex/blocktype as bt -import pkg/codex/contracts/requests -import pkg/codex/contracts -import pkg/codex/merkletree -import pkg/codex/stores/cachestore - -import pkg/codex/slots/sampler -import pkg/codex/slots/builder/builder - -import ../helpers -import ../examples -import ../merkletree/helpers -import ../../asynctest -import ./testsampler_expected -import ./provingtestenv - -asyncchecksuite "Test DataSampler": - var - env: ProvingTestEnvironment - dataSampler: DataSampler - blk: bt.Block - cell0Bytes: seq[byte] - cell1Bytes: seq[byte] - cell2Bytes: seq[byte] - - proc createDataSampler(): Future[void] {.async.} = - dataSampler = DataSampler.new( - datasetSlotIndex, - env.localStore, - SlotsBuilder.new(env.localStore, env.manifest).tryGet()).tryGet() - - setup: - randomize() - env = await createProvingTestEnvironment() - let bytes = newSeqWith(bytesPerBlock, rand(uint8)) - blk = bt.Block.new(bytes).tryGet() - cell0Bytes = bytes[0.. (protectedManifest.numSlotBlocks - 1): + emptyDigest + else: + SpongeMerkle.digest(datasetBlocks[idx].data, cellSize.int) - expectedHashes: seq[Poseidon2Hash] = collect(newSeq): - for blk in expectedBlock: - SpongeMerkle.digest(blk.data & blockPadBytes, cellSize.int) + cellHashes = (await builder.getCellHashes(0)).tryGet() - cellHashes = (await slotBuilder.getBlockHashes(i)).tryGet() - - check: - expectedHashes == cellHashes + check: + cellHashes.len == expectedHashes.len + cellHashes == expectedHashes test "Should build slot trees for all slots": let - steppedStrategy = SteppedStrategy.init(0, numTotalBlocks - 1, numSlots) - slotBuilder = SlotsBuilder.new( + steppedStrategy = Strategy.init( + 0, numPadBlocksTotal - 1, numSlots) + + builder = Poseidon2Builder.new( localStore, protectedManifest, cellSize = cellSize).tryGet() - for i in 0 ..< numSlots: + for i in 0.. (protectedManifest.numSlotBlocks - 1): + emptyDigest + else: + SpongeMerkle.digest(datasetBlocks[idx].data, cellSize.int) - expectedHashes: seq[Poseidon2Hash] = collect(newSeq): - for blk in expectedBlock: - SpongeMerkle.digest(blk.data & blockPadBytes, cellSize.int) - expectedRoot = Merkle.digest(expectedHashes & slotsPadLeafs) - - slotTree = (await slotBuilder.buildSlotTree(i)).tryGet() + expectedRoot = Merkle.digest(expectedHashes) + slotTree = (await builder.buildSlotTree(i)).tryGet() check: - expectedRoot == slotTree.root().tryGet() + slotTree.root().tryGet() == expectedRoot test "Should persist trees for all slots": let - slotBuilder = SlotsBuilder.new( + builder = Poseidon2Builder.new( localStore, protectedManifest, cellSize = cellSize).tryGet() - for i in 0 ..< numSlots: + for i in 0.. (protectedManifest.numSlotBlocks - 1): + emptyDigest + else: + SpongeMerkle.digest(datasetBlocks[idx].data, cellSize.int) - slotHashes: seq[Poseidon2Hash] = collect(newSeq): - for blk in expectedBlocks: - SpongeMerkle.digest(blk.data & blockPadBytes, cellSize.int) + Merkle.digest(slotHashes) - Merkle.digest(slotHashes & slotsPadLeafs) - - expectedRoot = Merkle.digest(slotsHashes & rootsPadLeafs) - rootHash = slotBuilder.buildVerifyTree(slotBuilder.slotRoots).tryGet().root.tryGet() + expectedRoot = Merkle.digest(slotsHashes) + rootHash = builder.buildVerifyTree(builder.slotRoots).tryGet().root.tryGet() check: expectedRoot == rootHash test "Should build correct verification root manifest": let - steppedStrategy = SteppedStrategy.init(0, numTotalBlocks - 1, numSlots) - slotBuilder = SlotsBuilder.new( + steppedStrategy = Strategy.init(0, numPadBlocksTotal - 1, numSlots) + builder = Poseidon2Builder.new( localStore, protectedManifest, cellSize = cellSize).tryGet() slotsHashes = collect(newSeq): - for i in 0 ..< numSlots: + for i in 0.. (protectedManifest.numSlotBlocks - 1): + emptyDigest + else: + SpongeMerkle.digest(datasetBlocks[idx].data, cellSize.int) - slotHashes: seq[Poseidon2Hash] = collect(newSeq): - for blk in expectedBlocks: - SpongeMerkle.digest(blk.data & blockPadBytes, cellSize.int) + Merkle.digest(slotHashes) - Merkle.digest(slotHashes & slotsPadLeafs) - - expectedRoot = Merkle.digest(slotsHashes & rootsPadLeafs) - manifest = (await slotBuilder.buildManifest()).tryGet() + expectedRoot = Merkle.digest(slotsHashes) + manifest = (await builder.buildManifest()).tryGet() mhash = manifest.verifyRoot.mhash.tryGet() mhashBytes = mhash.digestBytes rootHash = Poseidon2Hash.fromBytes(mhashBytes.toArray32).get @@ -334,69 +294,69 @@ suite "Slot builder": test "Should not build from verifiable manifest with 0 slots": var - slotBuilder = SlotsBuilder.new( + builder = Poseidon2Builder.new( localStore, protectedManifest, cellSize = cellSize).tryGet() - verifyManifest = (await slotBuilder.buildManifest()).tryGet() + verifyManifest = (await builder.buildManifest()).tryGet() verifyManifest.slotRoots = @[] - check SlotsBuilder.new( + check Poseidon2Builder.new( localStore, verifyManifest, cellSize = cellSize).isErr test "Should not build from verifiable manifest with incorrect number of slots": var - slotBuilder = SlotsBuilder.new( + builder = Poseidon2Builder.new( localStore, protectedManifest, cellSize = cellSize).tryGet() - verifyManifest = (await slotBuilder.buildManifest()).tryGet() + verifyManifest = (await builder.buildManifest()).tryGet() verifyManifest.slotRoots.del( verifyManifest.slotRoots.len - 1 ) - check SlotsBuilder.new( + check Poseidon2Builder.new( localStore, verifyManifest, cellSize = cellSize).isErr test "Should not build from verifiable manifest with invalid verify root": let - slotBuilder = SlotsBuilder.new( + builder = Poseidon2Builder.new( localStore, protectedManifest, cellSize = cellSize).tryGet() - verifyManifest = (await slotBuilder.buildManifest()).tryGet() + verifyManifest = (await builder.buildManifest()).tryGet() offset = verifyManifest.verifyRoot.data.buffer.len div 2 rng.shuffle( Rng.instance, verifyManifest.verifyRoot.data.buffer) - check SlotsBuilder.new( + check Poseidon2Builder.new( localStore, verifyManifest, cellSize = cellSize).isErr test "Should build from verifiable manifest": let - slotBuilder = SlotsBuilder.new( + builder = Poseidon2Builder.new( localStore, protectedManifest, cellSize = cellSize).tryGet() - verifyManifest = (await slotBuilder.buildManifest()).tryGet() + verifyManifest = (await builder.buildManifest()).tryGet() - verificationBuilder = SlotsBuilder.new( + verificationBuilder = Poseidon2Builder.new( localStore, verifyManifest, cellSize = cellSize).tryGet() check: - slotBuilder.slotRoots == verificationBuilder.slotRoots - slotBuilder.verifyRoot == verificationBuilder.verifyRoot + builder.slotRoots == verificationBuilder.slotRoots + builder.verifyRoot == verificationBuilder.verifyRoot diff --git a/tests/codex/testslots.nim b/tests/codex/testslots.nim index 6dfd298c..f37e0927 100644 --- a/tests/codex/testslots.nim +++ b/tests/codex/testslots.nim @@ -1,5 +1,4 @@ import ./slots/testslotbuilder -import ./slots/testutils import ./slots/testsampler import ./slots/testconverters