diff --git a/codex/slotbuilder/slotbuilder.nim b/codex/slotbuilder/slotbuilder.nim deleted file mode 100644 index 426c6a2b..00000000 --- a/codex/slotbuilder/slotbuilder.nim +++ /dev/null @@ -1,109 +0,0 @@ -import std/math -import std/sequtils -import pkg/libp2p -import pkg/chronos -import pkg/chronicles -import pkg/questionable/results -import ../merkletree -import ../stores -import ../manifest -import ../utils -import ../utils/digest - -const - # TODO: Unified with the CellSize specified in branch "data-sampler" - # Number of bytes in a cell. A cell is the smallest unit of data used - # in the proving circuit. - CellSize* = 2048 - -type - SlotBuilder* = object of RootObj - blockStore: BlockStore - manifest: Manifest - slotBlocks: int - -proc new*( - T: type SlotBuilder, - blockStore: BlockStore, - manifest: Manifest -): ?!SlotBuilder = - - if not manifest.protected: - return failure("Can only create SlotBuilder using protected manifests.") - - if (manifest.blocksCount mod manifest.ecK) != 0: - return failure("Number of blocks must be divisable by number of slots.") - - if (manifest.blockSize.int mod CellSize) != 0: - return failure("Block size must be divisable by cell size.") - - let slotBlocks = manifest.blocksCount div manifest.numberOfSlots - success SlotBuilder( - blockStore: blockStore, - manifest: manifest, - slotBlocks: slotBlocks) - -proc cellsPerBlock(self: SlotBuilder): int = - self.manifest.blockSize.int div CellSize - -proc selectSlotBlocks*( - self: SlotBuilder, - slotIndex: int): Future[?!seq[Poseidon2Hash]] {.async.} = - - let - treeCid = self.manifest.treeCid - blockCount = self.manifest.blocksCount - numberOfSlots = self.manifest.numberOfSlots - strategy = SteppedIndexingStrategy.new(0, blockCount - 1, numberOfSlots) - - logScope: - treeCid = treeCid - blockCount = blockCount - numberOfSlots = numberOfSlots - index = blockIndex - - var blocks = newSeq[Poseidon2Hash]() - for blockIndex in strategy.getIndicies(slotIndex): - without blk =? await self.blockStore.getBlock(treeCid, blockIndex), err: - error "Failed to get block CID for tree at index" - - return failure(err) - - without digestTree =? Poseidon2Tree.digest(blk.data, CellSize) and - blockDigest =? digestTree.root, err: - error "Failed to create digest for block" - - return failure(err) - - blocks.add(blockDigest) - # TODO: Remove this sleep. It's here to prevent us from locking up the thread. - await sleepAsync(10.millis) - - success blocks - -proc numPaddingCells*(self: SlotBuilder, slotBlocks: int): int = - let - numberOfCells = slotBlocks * self.cellsPerBlock - nextPowerOfTwo = nextPowerOfTwo(numberOfCells) - - return nextPowerOfTwo - numberOfCells - -proc buildSlotTree*(self: SlotBuilder, slotBlocks: seq[Cid], paddingCells: int): ?!Poseidon2Tree = - without emptyCid =? emptyCid(self.manifest.version, self.manifest.hcodec, self.manifest.codec), err: - error "Unable to initialize empty cid" - return failure(err) - - let paddingBlocks = divUp(paddingCells, self.cellsPerBlock) - let padding = newSeqWith(paddingBlocks, emptyCid) - - Poseidon2Tree.init(slotBlocks & padding) - -proc createSlots*(self: SlotBuilder, slotIndex: int): Future[?!Manifest] {.async.} = - without slotBlocks =? await self.selectSlotBlocks(slotIndex), err: - error "Failed to select slot blocks" - return failure(err) - - let paddingCells = self.numPaddingCells(slotBlocks.len) - - trace "Creating slot tree", slotIndex, nSlotBlocks = slotBlocks.len, paddingCells - return self.buildSlotTree(slotBlocks, paddingCells) diff --git a/codex/slots/slotbuilder.nim b/codex/slots/slotbuilder.nim new file mode 100644 index 00000000..343a6893 --- /dev/null +++ b/codex/slots/slotbuilder.nim @@ -0,0 +1,267 @@ +## Nim-Codex +## Copyright (c) 2023 Status Research & Development GmbH +## Licensed under either of +## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) +## * MIT license ([LICENSE-MIT](LICENSE-MIT)) +## at your option. +## This file may not be copied, modified, or distributed except according to +## those terms. + +{.push raises: [].} + +import std/math +import std/sequtils +import std/sugar + +import pkg/libp2p +import pkg/chronos +import pkg/chronicles +import pkg/questionable/results +import pkg/poseidon2 +import pkg/poseidon2/io + +import ../indexingstrategy +import ../merkletree +import ../stores +import ../manifest +import ../utils +import ../utils/digest + +const + # TODO: Unified with the CellSize specified in branch "data-sampler" + # Number of bytes in a cell. A cell is the smallest unit of data used + # in the proving circuit. + CellSize* = 2048 + +type + SlotBuilder* = object of RootObj + store: BlockStore + manifest: Manifest + strategy: IndexingStrategy + cellSize: int + blockPadBytes: seq[byte] + slotsPadLeafs: seq[Poseidon2Hash] + rootsPadLeafs: seq[Poseidon2Hash] + +func numBlockPadBytes*(self: SlotBuilder): Natural = + ## Number of padding bytes required for a pow2 + ## merkle tree for each block. + ## + + self.blockPadBytes.len + +func numSlotsPadLeafs*(self: SlotBuilder): Natural = + ## Number of padding field elements required for a pow2 + ## merkle tree for each slot. + ## + + self.slotsPadLeafs.len + +func numRootsPadLeafs*(self: SlotBuilder): Natural = + ## Number of padding field elements required for a pow2 + ## merkle tree for the slot roots. + ## + + self.rootsPadLeafs.len + +func numSlotBlocks*(self: SlotBuilder): Natural = + ## Number of blocks per slot. + ## + + self.manifest.blocksCount div self.manifest.numSlots + +func numBlockRoots*(self: SlotBuilder): Natural = + ## Number of cells per block. + ## + + self.manifest.blockSize.int div self.cellSize + +func toCellCid(cell: Poseidon2Hash): ?!Cid = + let + cellMhash = ? MultiHash.init(Pos2Bn128MrklCodec, cell.toBytes).mapFailure + cellCid = ? Cid.init(CIDv1, CodexSlotCell, cellMhash).mapFailure + + success cellCid + +func toSlotCid(root: Poseidon2Hash): ?!Cid = + let + mhash = ? MultiHash.init($multiCodec("identity"), root.toBytes).mapFailure + treeCid = ? Cid.init(CIDv1, SlotRootCodec, mhash).mapFailure + + success treeCid + +func toProvingCid(root: Poseidon2Hash): ?!Cid = + let + mhash = ? MultiHash.init($multiCodec("identity"), root.toBytes).mapFailure + treeCid = ? Cid.init(CIDv1, SlotProvingRootCodec, mhash).mapFailure + + success treeCid + +func mapToSlotCids(slotRoots: seq[Poseidon2Hash]): ?!seq[Cid] = + success slotRoots.mapIt( ? it.toSlotCid ) + +func toEncodableProof*( + proof: Poseidon2Proof): ?!CodexProof = + + let + encodableProof = CodexProof( + mcodec: multiCodec("identity"), # copy bytes as is + index: proof.index, + nleaves: proof.nleaves, + path: proof.path.mapIt( @(it.toBytes) )) + + success encodableProof + +func toVerifiableProof*( + proof: CodexProof): ?!Poseidon2Proof = + + let + verifiableProof = Poseidon2Proof( + index: proof.index, + nleaves: proof.nleaves, + path: proof.path.mapIt( + ? Poseidon2Hash.fromBytes(it.toArray32).toResult + )) + + success verifiableProof + +proc getCellHashes*( + self: SlotBuilder, + slotIndex: int): Future[?!seq[Poseidon2Hash]] {.async.} = + + let + treeCid = self.manifest.treeCid + blockCount = self.manifest.blocksCount + numberOfSlots = self.manifest.numSlots + + logScope: + treeCid = treeCid + blockCount = blockCount + numberOfSlots = numberOfSlots + index = blockIndex + slotIndex = slotIndex + + let + hashes: seq[Poseidon2Hash] = collect(newSeq): + for blockIndex in self.strategy.getIndicies(slotIndex): + trace "Getting block CID for tree at index" + + without blk =? (await self.store.getBlock(treeCid, blockIndex)), err: + error "Failed to get block CID for tree at index" + return failure(err) + + without digest =? Poseidon2Tree.digest(blk.data & self.blockPadBytes, self.cellSize), err: + error "Failed to create digest for block" + return failure(err) + + # TODO: Remove this sleep. It's here to prevent us from locking up the thread. + # await sleepAsync(10.millis) + + digest + + success hashes + +proc buildSlotTree*( + self: SlotBuilder, + slotIndex: int): Future[?!Poseidon2Tree] {.async.} = + without cellHashes =? (await self.getCellHashes(slotIndex)), err: + error "Failed to select slot blocks", err = err.msg + return failure(err) + + Poseidon2Tree.init(cellHashes & self.slotsPadLeafs) + +proc buildSlot*( + self: SlotBuilder, + slotIndex: int): Future[?!Poseidon2Hash] {.async.} = + ## Build a slot tree and store it in the block store. + ## + + without tree =? (await self.buildSlotTree(slotIndex)) and + treeCid =? tree.root.?toSlotCid, err: + error "Failed to build slot tree", err = err.msg + return failure(err) + + trace "Storing slot tree", treeCid, slotIndex, leaves = tree.leavesCount + for i, leaf in tree.leaves: + without cellCid =? leaf.toCellCid, err: + error "Failed to get CID for slot cell", err = err.msg + return failure(err) + + without proof =? tree.getProof(i) and + encodableProof =? proof.toEncodableProof, err: + error "Failed to get proof for slot tree", err = err.msg + return failure(err) + + if err =? (await self.store.putCidAndProof( + treeCid, i, cellCid, encodableProof)).errorOption: + error "Failed to store slot tree", err = err.msg + return failure(err) + + tree.root() + +proc buildSlots(self: SlotBuilder): Future[?!Manifest] {.async.} = + let + slotRoots: seq[Poseidon2Hash] = collect(newSeq): + for i in 0..