diff --git a/codex/blockexchange/engine/engine.nim b/codex/blockexchange/engine/engine.nim index 649edec6..c24a0104 100644 --- a/codex/blockexchange/engine/engine.nim +++ b/codex/blockexchange/engine/engine.nim @@ -450,7 +450,7 @@ proc wantListHandler*( trace "Sending presence to remote", items = presence.len await b.network.request.sendPresence(peer, presence) - trace "Scheduling a task for this peer, to look over their want-list", peer + trace "Scheduling a task to check want-list", peer if not b.scheduleTask(peerCtx): trace "Unable to schedule task for peer", peer diff --git a/codex/indexingstrategy.nim b/codex/indexingstrategy.nim index e4006ff1..aac9e802 100644 --- a/codex/indexingstrategy.nim +++ b/codex/indexingstrategy.nim @@ -103,5 +103,4 @@ proc new*( firstIndex: firstIndex, lastIndex: lastIndex, numberOfIterations: numberOfIterations, - step: divUp((lastIndex - firstIndex), numberOfIterations) - ) + step: divUp((lastIndex - firstIndex), numberOfIterations)) diff --git a/codex/node.nim b/codex/node.nim index affc4307..40e8ee26 100644 --- a/codex/node.nim +++ b/codex/node.nim @@ -102,20 +102,6 @@ proc storeManifest*( success blk -proc findPeer*( - self: CodexNodeRef, - peerId: PeerId): Future[?PeerRecord] {.async.} = - ## Find peer using the discovery service from the given CodexNode - ## - return await self.discovery.findPeer(peerId) - -proc connect*( - self: CodexNodeRef, - peerId: PeerId, - addrs: seq[MultiAddress] -): Future[void] = - self.switch.connect(peerId, addrs) - proc fetchManifest*( self: CodexNodeRef, cid: Cid): Future[?!Manifest] {.async.} = @@ -141,6 +127,20 @@ proc fetchManifest*( return manifest.success +proc findPeer*( + self: CodexNodeRef, + peerId: PeerId): Future[?PeerRecord] {.async.} = + ## Find peer using the discovery service from the given CodexNode + ## + return await self.discovery.findPeer(peerId) + +proc connect*( + self: CodexNodeRef, + peerId: PeerId, + addrs: seq[MultiAddress] +): Future[void] = + self.switch.connect(peerId, addrs) + proc updateExpiry*( self: CodexNodeRef, manifestCid: Cid, @@ -493,9 +493,8 @@ proc onStore( trace "Received a request to store a slot!" - without cid =? Cid.init(request.content.cid): + without cid =? Cid.init(request.content.cid).mapFailure, err: trace "Unable to parse Cid", cid - let err = newException(CodexError, "Unable to parse Cid") return failure(err) without manifest =? (await self.fetchManifest(cid)), err: @@ -521,15 +520,17 @@ proc onStore( if updateExpiryErr =? (await allFutureResult(ensureExpiryFutures)).errorOption: return failure(updateExpiryErr) - echo "blocksCb.isNil: ", blocksCb.isNil if not blocksCb.isNil and err =? (await blocksCb(blocks)).errorOption: trace "Unable to process blocks", err = err.msg return failure(err) return success() - if blksIter =? builder.slotIndicies(slotIdx) and - err =? (await self.fetchBatched(manifest.treeCid, blksIter, onBatch = updateExpiry)).errorOption: + if blksIter =? builder.slotIndiciesIter(slotIdx) and + err =? (await self.fetchBatched( + manifest.treeCid, + blksIter, + onBatch = updateExpiry)).errorOption: trace "Unable to fetch blocks", err = err.msg return failure(err) @@ -543,6 +544,47 @@ proc onStore( return success() +proc onProve( + self: CodexNodeRef, + slot: Slot, + challenge: ProofChallenge): Future[?!seq[byte]] {.async.} = + ## Generats a proof for a given slot and challenge + ## + + let + cidStr = slot.request.content.cid + slotIdx = slot.slotIndex.truncate(Natural) + + logScope: + cid = cidStr + slot = slotIdx + challenge = challenge + + trace "Received proof challenge" + + without cid =? Cid.init(cidStr).mapFailure, err: + error "Unable to parse Cid", cid, err = err.msg + return failure(err) + + without manifest =? await self.fetchManifest(cid), err: + error "Unable to fetch manifest for cid", err = err.msg + return failure(err) + + without builder =? SlotsBuilder.new(self.blockStore, manifest), err: + error "Unable to create slots builder", err = err.msg + return failure(err) + + without sampler =? DataSampler.new(slotIdx, self.blockStore, builder), err: + error "Unable to create data sampler", err = err.msg + return failure(err) + + without proofInput =? await sampler.getProofInput(challenge, nSamples = 3), err: + error "Unable to get proof input for slot", err = err.msg + return failure(err) + + # Todo: send proofInput to circuit. Get proof. (Profit, repeat.) + success(@[42'u8]) + proc onExpiryUpdate( self: CodexNodeRef, rootCid: string, @@ -561,13 +603,6 @@ proc onClear( # TODO: remove data from local storage discard -proc onProve( - self: CodexNodeRef, - slot: Slot, - challenge: ProofChallenge): Future[seq[byte]] {.async.} = - # TODO: generate proof - return @[42'u8] - proc start*(self: CodexNodeRef) {.async.} = if not self.engine.isNil: await self.engine.start() @@ -598,7 +633,7 @@ proc start*(self: CodexNodeRef) {.async.} = self.onClear(request, slotIndex) hostContracts.sales.onProve = - proc(slot: Slot, challenge: ProofChallenge): Future[seq[byte]] = + proc(slot: Slot, challenge: ProofChallenge): Future[?!seq[byte]] = # TODO: generate proof self.onProve(slot, challenge) diff --git a/codex/sales.nim b/codex/sales.nim index d7a265cb..7994db89 100644 --- a/codex/sales.nim +++ b/codex/sales.nim @@ -301,7 +301,7 @@ proc onStorageRequested(sales: Sales, expiry: UInt256) = logScope: - topics = " marketplace sales onStorageRequested" + topics = "marketplace sales onStorageRequested" requestId slots = ask.slots expiry diff --git a/codex/sales/salescontext.nim b/codex/sales/salescontext.nim index 5d0b8308..0c209db7 100644 --- a/codex/sales/salescontext.nim +++ b/codex/sales/salescontext.nim @@ -27,7 +27,7 @@ type OnStore* = proc(request: StorageRequest, slot: UInt256, blocksCb: BlocksCb): Future[?!void] {.gcsafe, upraises: [].} - OnProve* = proc(slot: Slot, challenge: ProofChallenge): Future[seq[byte]] {.gcsafe, upraises: [].} + OnProve* = proc(slot: Slot, challenge: ProofChallenge): Future[?!seq[byte]] {.gcsafe, upraises: [].} OnExpiryUpdate* = proc(rootCid: string, expiry: SecondsSince1970): Future[?!void] {.gcsafe, upraises: [].} OnClear* = proc(request: StorageRequest, slotIndex: UInt256) {.gcsafe, upraises: [].} diff --git a/codex/sales/states/initialproving.nim b/codex/sales/states/initialproving.nim index d44b058b..eeb1592f 100644 --- a/codex/sales/states/initialproving.nim +++ b/codex/sales/states/initialproving.nim @@ -1,9 +1,11 @@ import pkg/chronicles +import pkg/questionable/results import ../statemachine import ../salesagent import ./errorhandling import ./filling import ./cancelled +import ./errored import ./failed logScope: @@ -34,7 +36,10 @@ method run*(state: SaleInitialProving, machine: Machine): Future[?State] {.async let slot = Slot(request: request, slotIndex: data.slotIndex) challenge = await context.market.getChallenge(slot.id) - proof = await onProve(slot, challenge) + without proof =? (await onProve(slot, challenge)), err: + error "Failed to generate initial proof", error = err.msg + return some State(SaleErrored(error: err)) + debug "Finished proof calculation", requestId = $data.requestId return some State(SaleFilling(proof: proof)) diff --git a/codex/sales/states/proving.nim b/codex/sales/states/proving.nim index 367eaa9e..dffb8368 100644 --- a/codex/sales/states/proving.nim +++ b/codex/sales/states/proving.nim @@ -1,5 +1,6 @@ import std/options import pkg/chronicles +import pkg/questionable/results import ../../clock import ../statemachine import ../salesagent @@ -27,7 +28,10 @@ method prove*( currentPeriod: Period ) {.base, async.} = try: - let proof = await onProve(slot, challenge) + without proof =? (await onProve(slot, challenge)), err: + error "Failed to generate proof", error = err.msg + # In this state, there's nothing we can do except try again next time. + return debug "Submitting proof", currentPeriod = currentPeriod, slotId = $slot.id await market.submitProof(slot.id, proof) except CatchableError as e: diff --git a/codex/slots.nim b/codex/slots.nim index a34949f3..fb0827a4 100644 --- a/codex/slots.nim +++ b/codex/slots.nim @@ -1,3 +1,4 @@ import ./slots/builder +import ./slots/sampler -export builder +export builder, sampler diff --git a/codex/slots/builder.nim b/codex/slots/builder.nim index 9ee63f05..6cf9a371 100644 --- a/codex/slots/builder.nim +++ b/codex/slots/builder.nim @@ -1,327 +1,4 @@ -## Nim-Codex -## Copyright (c) 2023 Status Research & Development GmbH -## Licensed under either of -## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) -## * MIT license ([LICENSE-MIT](LICENSE-MIT)) -## at your option. -## This file may not be copied, modified, or distributed except according to -## those terms. - -{.push raises: [].} - -import std/math -import std/sequtils -import std/sugar - -import pkg/libp2p -import pkg/chronos -import pkg/chronicles -import pkg/questionable/results -import pkg/poseidon2 -import pkg/poseidon2/io -import pkg/constantine/math/arithmetic/finite_fields - -import ../indexingstrategy -import ../merkletree -import ../stores -import ../manifest -import ../utils -import ../utils/asynciter -import ../utils/digest -import ../utils/poseidon2digest - +import ./builder/builder import ./converters -export converters - -const - # TODO: Unified with the CellSize specified in branch "data-sampler" - # in the proving circuit. - CellSize* = 2048 - - DefaultEmptyBlock* = newSeq[byte](DefaultBlockSize.int) - DefaultEmptyCell* = newSeq[byte](DefaultCellSize.int) - -type - # TODO: should be a generic type that - # supports all merkle trees - SlotsBuilder* = ref object of RootObj - store: BlockStore - manifest: Manifest - strategy: IndexingStrategy - cellSize: int - blockEmptyDigest: Poseidon2Hash - blockPadBytes: seq[byte] - slotsPadLeafs: seq[Poseidon2Hash] - rootsPadLeafs: seq[Poseidon2Hash] - slotRoots: seq[Poseidon2Hash] - verifyRoot: ?Poseidon2Hash - -func slotRoots*(self: SlotsBuilder): seq[Poseidon2Hash] {.inline.} = - ## Returns the slot roots. - ## - - self.slotRoots - -func verifyRoot*(self: SlotsBuilder): ?Poseidon2Hash {.inline.} = - ## Returns the slots root (verification root). - ## - - self.verifyRoot - -func nextPowerOfTwoPad*(a: int): int = - ## Returns the difference between the original - ## value and the next power of two. - ## - - nextPowerOfTwo(a) - a - -func numBlockPadBytes*(self: SlotsBuilder): Natural = - ## Number of padding bytes required for a pow2 - ## merkle tree for each block. - ## - - self.blockPadBytes.len - -func numSlotsPadLeafs*(self: SlotsBuilder): Natural = - ## Number of padding field elements required for a pow2 - ## merkle tree for each slot. - ## - - self.slotsPadLeafs.len - -func numRootsPadLeafs*(self: SlotsBuilder): Natural = - ## Number of padding field elements required for a pow2 - ## merkle tree for the slot roots. - ## - - self.rootsPadLeafs.len - -func numSlots*(self: SlotsBuilder): Natural = - ## Number of slots. - ## - - self.manifest.numSlots - -func numSlotBlocks*(self: SlotsBuilder): Natural = - ## Number of blocks per slot. - ## - - self.manifest.blocksCount div self.manifest.numSlots - -func slotBytes*(self: SlotsBuilder): NBytes = - ## Number of bytes per slot. - ## - - (self.manifest.blockSize.int * self.numSlotBlocks).NBytes - -func numBlockCells*(self: SlotsBuilder): Natural = - ## Number of cells per block. - ## - - self.manifest.blockSize.int div self.cellSize - -func slotIndicies*(self: SlotsBuilder, slot: Natural): ?!Iter[int] = - ## Returns the slot indices. - ## TODO: should return an iterator - ## - - self.strategy.getIndicies(slot).catch - -proc getCellHashes*( - self: SlotsBuilder, - slotIndex: int): Future[?!seq[Poseidon2Hash]] {.async.} = - - let - treeCid = self.manifest.treeCid - blockCount = self.manifest.blocksCount - numberOfSlots = self.manifest.numSlots - - logScope: - treeCid = treeCid - blockCount = blockCount - numberOfSlots = numberOfSlots - index = blockIndex - slotIndex = slotIndex - - let - hashes: seq[Poseidon2Hash] = collect(newSeq): - for blockIndex in self.strategy.getIndicies(slotIndex): - trace "Getting block CID for tree at index" - - without blk =? (await self.store.getBlock(treeCid, blockIndex)), err: - error "Failed to get block CID for tree at index" - return failure(err) - - if blk.isEmpty: - self.blockEmptyDigest - else: - without digest =? Poseidon2Tree.digest(blk.data & self.blockPadBytes, self.cellSize), err: - error "Failed to create digest for block" - return failure(err) - - digest - - success hashes - -proc buildSlotTree*( - self: SlotsBuilder, - slotIndex: int): Future[?!Poseidon2Tree] {.async.} = - without cellHashes =? (await self.getCellHashes(slotIndex)), err: - error "Failed to select slot blocks", err = err.msg - return failure(err) - - Poseidon2Tree.init(cellHashes & self.slotsPadLeafs) - -proc buildSlot*( - self: SlotsBuilder, - slotIndex: Natural): Future[?!Poseidon2Hash] {.async.} = - ## Build a slot tree and store it in the block store. - ## - - logScope: - cid = self.manifest.treeCid - slotIndex = slotIndex - - trace "Building slot tree" - - without tree =? (await self.buildSlotTree(slotIndex)) and - treeCid =? tree.root.?toSlotCid, err: - error "Failed to build slot tree", err = err.msg - return failure(err) - - trace "Storing slot tree", treeCid, slotIndex, leaves = tree.leavesCount - for i, leaf in tree.leaves: - without cellCid =? leaf.toCellCid, err: - error "Failed to get CID for slot cell", err = err.msg - return failure(err) - - without proof =? tree.getProof(i) and - encodableProof =? proof.toEncodableProof, err: - error "Failed to get proof for slot tree", err = err.msg - return failure(err) - - if err =? (await self.store.putCidAndProof( - treeCid, i, cellCid, encodableProof)).errorOption: - error "Failed to store slot tree", err = err.msg - return failure(err) - - tree.root() - -func buildRootsTree*( - self: SlotsBuilder, - slotRoots: openArray[Poseidon2Hash]): ?!Poseidon2Tree = - Poseidon2Tree.init(@slotRoots & self.rootsPadLeafs) - -proc buildSlots*(self: SlotsBuilder): Future[?!void] {.async.} = - ## Build all slot trees and store them in the block store. - ## - - logScope: - cid = self.manifest.treeCid - blockCount = self.manifest.blocksCount - - trace "Building slots" - - if self.slotRoots.len == 0: - self.slotRoots = collect(newSeq): - for i in 0.. builder.slotRoots.high: + error "Slot index is out of range" + return failure("Slot index is out of range") + + success DataSampler( + index: index, + blockStore: blockStore, + builder: builder) + +proc getCell*(self: DataSampler, blkBytes: seq[byte], blkCellIdx: Natural): Cell = + let + cellSize = self.builder.cellSize.uint64 + dataStart = cellSize * blkCellIdx.uint64 + dataEnd = dataStart + cellSize + return blkBytes[dataStart ..< dataEnd] + +proc getProofInput*( + self: DataSampler, + entropy: ProofChallenge, + nSamples: Natural): Future[?!ProofInput] {.async.} = + ## Generate proofs as input to the proving circuit. + ## + + let + entropy = Poseidon2Hash.fromBytes( + array[31, byte].initCopyFrom(entropy[0..30])) # truncate to 31 bytes, otherwise it _might_ be greater than mod + + without verifyTree =? self.builder.verifyTree and + verifyProof =? verifyTree.getProof(self.index) and + verifyRoot =? verifyTree.root(), err: + error "Failed to get slot proof from verify tree", err = err.msg + return failure(err) + + let + slotTreeCid = self.builder.manifest.slotRoots[self.index] + cellsPerBlock = self.builder.numBlockCells + cellIdxs = entropy.cellIndices( + self.builder.slotRoots[self.index], + self.builder.numSlotCells, + nSamples) + + logScope: + index = self.index + samples = nSamples + cells = cellIdxs + slotTreeCid = slotTreeCid + + trace "Collecting input for proof" + let samples = collect(newSeq): + for cellIdx in cellIdxs: + let + blkCellIdx = cellIdx.toBlockCellIdx(cellsPerBlock) # block cell index + slotCellIdx = cellIdx.toBlockIdx(cellsPerBlock) # slot tree index + + logScope: + cellIdx = cellIdx + slotCellIdx = slotCellIdx + blkCellIdx = blkCellIdx + + without (cid, proof) =? await self.blockStore.getCidAndProof( + slotTreeCid, + slotCellIdx.Natural), err: + error "Failed to get block from block store", err = err.msg + return failure(err) + + without slotProof =? proof.toVerifiableProof(), err: + error "Unable to convert slot proof to poseidon proof", error = err.msg + return failure(err) + + # This converts our slotBlockIndex to a datasetBlockIndex using the + # indexing-strategy used by the builder. + # We need this to fetch the block data. We can't do it by slotTree + slotBlkIdx. + let datasetBlockIndex = self.builder.slotIndicies(self.index)[slotCellIdx] + + without (bytes, blkTree) =? await self.builder.buildBlockTree(datasetBlockIndex), err: + error "Failed to build block tree", err = err.msg + return failure(err) + + without blockProof =? blkTree.getProof(blkCellIdx), err: + error "Failed to get proof from block tree", err = err.msg + return failure(err) + + Sample( + data: self.getCell(bytes, blkCellIdx), + slotProof: slotProof, + cellProof: blockProof, + slotBlockIdx: slotCellIdx.Natural, + blockCellIdx: blkCellIdx.Natural) + + success ProofInput( + entropy: entropy, + verifyRoot: verifyRoot, + verifyProof: verifyProof, + numSlots: self.builder.numSlots, + numCells: self.builder.numSlotCells, + slotIndex: self.index, + samples: samples) diff --git a/codex/slots/sampler/utils.nim b/codex/slots/sampler/utils.nim new file mode 100644 index 00000000..c73f7b95 --- /dev/null +++ b/codex/slots/sampler/utils.nim @@ -0,0 +1,83 @@ +## Nim-Codex +## Copyright (c) 2024 Status Research & Development GmbH +## Licensed under either of +## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) +## * MIT license ([LICENSE-MIT](LICENSE-MIT)) +## at your option. +## This file may not be copied, modified, or distributed except according to +## those terms. + +import std/sugar +import std/bitops + +import pkg/poseidon2 +import pkg/poseidon2/io + +import pkg/constantine/math/arithmetic + +import pkg/constantine/math/io/io_fields + +import ../../merkletree + +func extractLowBits*[n: static int](elm: BigInt[n], k: int): uint64 = + assert( k > 0 and k <= 64 ) + var r = 0'u64 + for i in 0.. 0): + k += 1 + y = y shr 1 + return k + +func ceilingLog2*(x : int) : int = + if (x == 0): + return -1 + else: + return (floorLog2(x-1) + 1) + +func toBlockIdx*(cell: Natural, numCells: Natural): Natural = + let log2 = ceilingLog2(numCells) + doAssert( 1 shl log2 == numCells , "`numCells` is assumed to be a power of two" ) + + return cell div numCells + +func toBlockCellIdx*(cell: Natural, numCells: Natural): Natural = + let log2 = ceilingLog2(numCells) + doAssert( 1 shl log2 == numCells , "`numCells` is assumed to be a power of two" ) + + return cell mod numCells + +func cellIndex*( + entropy: Poseidon2Hash, + slotRoot: Poseidon2Hash, + numCells: Natural, counter: Natural): Natural = + let log2 = ceilingLog2(numCells) + doAssert( 1 shl log2 == numCells , "`numCells` is assumed to be a power of two" ) + + let hash = Sponge.digest( @[ slotRoot, entropy, counter.toF ], rate = 2 ) + + return int( extractLowBits(hash, log2) ) + +func cellIndices*( + entropy: Poseidon2Hash, + slotRoot: Poseidon2Hash, + numCells: Natural, nSamples: Natural): seq[Natural] = + + var indices: seq[Natural] + while (indices.len < nSamples): + let idx = cellIndex(entropy, slotRoot, numCells, indices.len + 1) + indices.add(idx.Natural) + indices + diff --git a/codex/stores/blockstore.nim b/codex/stores/blockstore.nim index fbe36152..d97d6948 100644 --- a/codex/stores/blockstore.nim +++ b/codex/stores/blockstore.nim @@ -89,7 +89,7 @@ method getCidAndProof*( ## Get a block proof from the blockstore ## - raiseAssert("putCidAndProof not implemented!") + raiseAssert("getCidAndProof not implemented!") method ensureExpiry*( self: BlockStore, diff --git a/codex/stores/networkstore.nim b/codex/stores/networkstore.nim index 24f4249d..59e4bb94 100644 --- a/codex/stores/networkstore.nim +++ b/codex/stores/networkstore.nim @@ -86,6 +86,15 @@ method putCidAndProof*( proof: CodexProof): Future[?!void] = self.localStore.putCidAndProof(treeCid, index, blockCid, proof) +method getCidAndProof*( + self: NetworkStore, + treeCid: Cid, + index: Natural): Future[?!(Cid, CodexProof)] = + ## Get a block proof from the blockstore + ## + + self.localStore.getCidAndProof(treeCid, index) + method ensureExpiry*( self: NetworkStore, cid: Cid, diff --git a/codex/stores/repostore.nim b/codex/stores/repostore.nim index 1d95b245..45faefbd 100644 --- a/codex/stores/repostore.nim +++ b/codex/stores/repostore.nim @@ -241,7 +241,7 @@ method ensureExpiry*( cid: Cid, expiry: SecondsSince1970 ): Future[?!void] {.async.} = - ## Ensure that block's assosicated expiry is at least given timestamp + ## Ensure that block's associated expiry is at least given timestamp ## If the current expiry is lower then it is updated to the given one, otherwise it is left intact ## @@ -262,8 +262,12 @@ method ensureExpiry*( error "Could not read datastore key", err = err.msg return failure(err) + logScope: + current = currentExpiry.toSecondsSince1970 + ensuring = expiry + if expiry <= currentExpiry.toSecondsSince1970: - trace "Current expiry is larger than or equal to the specified one, no action needed", current = currentExpiry.toSecondsSince1970, ensuring = expiry + trace "Expiry is larger than or equal to requested" return success() if err =? (await self.metaDs.put(expiryKey, expiry.toBytes)).errorOption: diff --git a/codex/utils/asynciter.nim b/codex/utils/asynciter.nim index dec605db..4c9a65ec 100644 --- a/codex/utils/asynciter.nim +++ b/codex/utils/asynciter.nim @@ -2,12 +2,11 @@ import std/sugar import pkg/questionable import pkg/chronos -import pkg/upraises type - Function*[T, U] = proc(fut: T): U {.upraises: [CatchableError], gcsafe, closure.} - IsFinished* = proc(): bool {.upraises: [], gcsafe, closure.} - GenNext*[T] = proc(): T {.upraises: [CatchableError], gcsafe, closure.} + Function*[T, U] = proc(fut: T): U {.raises: [CatchableError], gcsafe, closure.} + IsFinished* = proc(): bool {.raises: [], gcsafe, closure.} + GenNext*[T] = proc(): T {.raises: [CatchableError], gcsafe, closure.} Iter*[T] = ref object finished: bool next*: GenNext[T] @@ -36,7 +35,7 @@ proc map*[T, U](fut: Future[T], fn: Function[T, U]): Future[U] {.async.} = proc new*[T](_: type Iter, genNext: GenNext[T], isFinished: IsFinished, finishOnErr: bool = true): Iter[T] = var iter = Iter[T]() - proc next(): T {.upraises: [CatchableError].} = + proc next(): T {.raises: [CatchableError].} = if not iter.finished: var item: T try: diff --git a/tests/codex/examples.nim b/tests/codex/examples.nim index 28467c64..c313ad4a 100644 --- a/tests/codex/examples.nim +++ b/tests/codex/examples.nim @@ -7,6 +7,7 @@ import pkg/codex/rng import pkg/codex/stores import pkg/codex/blocktype as bt import pkg/codex/sales +import pkg/codex/merkletree import ../examples export examples @@ -71,3 +72,11 @@ proc example*(_: type Reservation): Reservation = size = uint16.example.u256, slotId = SlotId.example ) + +proc example*(_: type MerkleProof): MerkleProof = + MerkleProof.init(3, @[MultiHash.example]).tryget() + +proc example*(_: type Poseidon2Proof): Poseidon2Proof = + var example = MerkleProof[Poseidon2Hash, PoseidonKeysEnum]() + example.index = 123 + example diff --git a/tests/codex/node/testcontracts.nim b/tests/codex/node/testcontracts.nim index 460eef62..05e5a047 100644 --- a/tests/codex/node/testcontracts.nim +++ b/tests/codex/node/testcontracts.nim @@ -129,7 +129,7 @@ asyncchecksuite "Test Node - Host contracts": (await onStore(request, 1.u256, onBlocks)).tryGet() check fetchedBytes == 786432 - for index in !builder.slotIndicies(1): + for index in builder.slotIndicies(1): let blk = (await localStore.getBlock(verifiable.treeCid, index)).tryGet expiryKey = (createBlockExpirationMetadataKey(blk.cid)).tryGet diff --git a/tests/codex/sales/states/testinitialproving.nim b/tests/codex/sales/states/testinitialproving.nim index dcc4f406..e083b68b 100644 --- a/tests/codex/sales/states/testinitialproving.nim +++ b/tests/codex/sales/states/testinitialproving.nim @@ -6,6 +6,7 @@ import pkg/codex/sales/states/initialproving import pkg/codex/sales/states/cancelled import pkg/codex/sales/states/failed import pkg/codex/sales/states/filling +import pkg/codex/sales/states/errored import pkg/codex/sales/salesagent import pkg/codex/sales/salescontext import pkg/codex/market @@ -24,9 +25,9 @@ asyncchecksuite "sales state 'initialproving'": var receivedChallenge: ProofChallenge setup: - let onProve = proc (slot: Slot, challenge: ProofChallenge): Future[seq[byte]] {.async.} = + let onProve = proc (slot: Slot, challenge: ProofChallenge): Future[?!seq[byte]] {.async.} = receivedChallenge = challenge - return proof + return success(proof) let context = SalesContext( onProve: onProve.some, market: market @@ -56,3 +57,19 @@ asyncchecksuite "sales state 'initialproving'": let future = state.run(agent) check receivedChallenge == market.proofChallenge + + test "switches to errored state when onProve callback fails": + let onProveFailed: OnProve = proc(slot: Slot, challenge: ProofChallenge): Future[?!seq[byte]] {.async.} = + return failure("oh no!") + + let proofFailedContext = SalesContext( + onProve: onProveFailed.some, + market: market + ) + agent = newSalesAgent(proofFailedContext, + request.id, + slotIndex, + request.some) + + let next = await state.run(agent) + check !next of SaleErrored diff --git a/tests/codex/sales/states/testproving.nim b/tests/codex/sales/states/testproving.nim index f6473882..3658b06f 100644 --- a/tests/codex/sales/states/testproving.nim +++ b/tests/codex/sales/states/testproving.nim @@ -28,9 +28,9 @@ asyncchecksuite "sales state 'proving'": setup: clock = MockClock.new() market = MockMarket.new() - let onProve = proc (slot: Slot, challenge: ProofChallenge): Future[seq[byte]] {.async.} = + let onProve = proc (slot: Slot, challenge: ProofChallenge): Future[?!seq[byte]] {.async.} = receivedChallenge = challenge - return proof + return success(proof) let context = SalesContext(market: market, clock: clock, onProve: onProve.some) agent = newSalesAgent(context, request.id, diff --git a/tests/codex/sales/states/testsimulatedproving.nim b/tests/codex/sales/states/testsimulatedproving.nim index 359af769..164d4fdc 100644 --- a/tests/codex/sales/states/testsimulatedproving.nim +++ b/tests/codex/sales/states/testsimulatedproving.nim @@ -44,8 +44,8 @@ asyncchecksuite "sales state 'simulated-proving'": market.setProofRequired(slot.id, true) subscription = await market.subscribeProofSubmission(onProofSubmission) - let onProve = proc (slot: Slot, challenge: ProofChallenge): Future[seq[byte]] {.async.} = - return proof + let onProve = proc (slot: Slot, challenge: ProofChallenge): Future[?!seq[byte]] {.async.} = + return success(proof) let context = SalesContext(market: market, clock: clock, onProve: onProve.some) agent = newSalesAgent(context, request.id, diff --git a/tests/codex/sales/testsales.nim b/tests/codex/sales/testsales.nim index 6668b105..baa3fbc9 100644 --- a/tests/codex/sales/testsales.nim +++ b/tests/codex/sales/testsales.nim @@ -64,8 +64,8 @@ asyncchecksuite "Sales - start": return success() queue = sales.context.slotQueue - sales.onProve = proc(slot: Slot, challenge: ProofChallenge): Future[seq[byte]] {.async.} = - return proof + sales.onProve = proc(slot: Slot, challenge: ProofChallenge): Future[?!seq[byte]] {.async.} = + return success(proof) itemsProcessed = @[] request.expiry = (clock.now() + 42).u256 @@ -167,8 +167,8 @@ asyncchecksuite "Sales": return success() queue = sales.context.slotQueue - sales.onProve = proc(slot: Slot, challenge: ProofChallenge): Future[seq[byte]] {.async.} = - return proof + sales.onProve = proc(slot: Slot, challenge: ProofChallenge): Future[?!seq[byte]] {.async.} = + return success(proof) await sales.start() itemsProcessed = @[] @@ -369,7 +369,7 @@ asyncchecksuite "Sales": test "handles errors during state run": var saleFailed = false - sales.onProve = proc(slot: Slot, challenge: ProofChallenge): Future[seq[byte]] {.async.} = + sales.onProve = proc(slot: Slot, challenge: ProofChallenge): Future[?!seq[byte]] {.async.} = # raise exception so machine.onError is called raise newException(ValueError, "some error") @@ -394,9 +394,10 @@ asyncchecksuite "Sales": test "generates proof of storage": var provingRequest: StorageRequest var provingSlot: UInt256 - sales.onProve = proc(slot: Slot, challenge: ProofChallenge): Future[seq[byte]] {.async.} = + sales.onProve = proc(slot: Slot, challenge: ProofChallenge): Future[?!seq[byte]] {.async.} = provingRequest = slot.request provingSlot = slot.slotIndex + return success(exampleProof()) createAvailability() await market.requestStorage(request) check eventually provingRequest == request @@ -426,7 +427,7 @@ asyncchecksuite "Sales": test "calls onClear when storage becomes available again": # fail the proof intentionally to trigger `agent.finish(success=false)`, # which then calls the onClear callback - sales.onProve = proc(slot: Slot, challenge: ProofChallenge): Future[seq[byte]] {.async.} = + sales.onProve = proc(slot: Slot, challenge: ProofChallenge): Future[?!seq[byte]] {.async.} = raise newException(IOError, "proof failed") var clearedRequest: StorageRequest var clearedSlotIndex: UInt256 diff --git a/tests/codex/slots/provingtestenv.nim b/tests/codex/slots/provingtestenv.nim new file mode 100644 index 00000000..85fbab91 --- /dev/null +++ b/tests/codex/slots/provingtestenv.nim @@ -0,0 +1,172 @@ +import std/sequtils + +import pkg/questionable/results +import pkg/poseidon2/io +import pkg/poseidon2 +import pkg/chronos +import pkg/codex/stores/cachestore +import pkg/codex/chunker +import pkg/codex/stores +import pkg/codex/blocktype as bt +import pkg/codex/contracts/requests +import pkg/codex/contracts +import pkg/codex/merkletree +import pkg/codex/stores/cachestore +import pkg/codex/indexingstrategy + +import pkg/codex/slots/converters +import pkg/codex/slots/builder/builder +import pkg/codex/utils/poseidon2digest +import pkg/codex/utils/asynciter + +import ../helpers +import ../merkletree/helpers + +const + # The number of slot blocks and number of slots, combined with + # the bytes per block, make it so that there are exactly 256 cells + # in the dataset. + bytesPerBlock* = 64 * 1024 + cellsPerBlock* = bytesPerBlock div DefaultCellSize.int + numberOfSlotBlocks* = 4 + totalNumberOfSlots* = 2 + datasetSlotIndex* = 1 + cellsPerSlot* = (bytesPerBlock * numberOfSlotBlocks) div DefaultCellSize.int + totalNumCells = ((numberOfSlotBlocks * totalNumberOfSlots * bytesPerBlock) div DefaultCellSize.int) + +type + ProvingTestEnvironment* = ref object + # Invariant: + challenge*: Poseidon2Hash + # Variant: + localStore*: CacheStore + manifest*: Manifest + manifestBlock*: bt.Block + slot*: Slot + datasetBlocks*: seq[bt.Block] + slotTree*: Poseidon2Tree + slotRootCid*: Cid + slotRoots*: seq[Poseidon2Hash] + datasetToSlotTree*: Poseidon2Tree + datasetRootHash*: Poseidon2Hash + +proc createDatasetBlocks(self: ProvingTestEnvironment): Future[void] {.async.} = + var data: seq[byte] = @[] + + # This generates a number of blocks that have different data, such that + # Each cell in each block is unique, but nothing is random. + for i in 0 ..< totalNumCells: + data = data & (i.byte).repeat(DefaultCellSize.uint64) + + let chunker = MockChunker.new( + dataset = data, + chunkSize = bytesPerBlock) + + while true: + let chunk = await chunker.getBytes() + if chunk.len <= 0: + break + let b = bt.Block.new(chunk).tryGet() + self.datasetBlocks.add(b) + discard await self.localStore.putBlock(b) + +proc createSlotTree(self: ProvingTestEnvironment, dSlotIndex: uint64): Future[Poseidon2Tree] {.async.} = + let + slotSize = (bytesPerBlock * numberOfSlotBlocks).uint64 + blocksInSlot = slotSize div bytesPerBlock.uint64 + datasetBlockIndexingStrategy = SteppedIndexingStrategy.new(0, self.datasetBlocks.len - 1, totalNumberOfSlots) + datasetBlockIndices = toSeq(datasetBlockIndexingStrategy.getIndicies(dSlotIndex.int)) + + let + slotBlocks = datasetBlockIndices.mapIt(self.datasetBlocks[it]) + numBlockCells = bytesPerBlock.int div DefaultCellSize.int + blockPadBytes = newSeq[byte](numBlockCells.nextPowerOfTwoPad * DefaultCellSize.int) + slotBlockRoots = slotBlocks.mapIt(Poseidon2Tree.digest(it.data & blockPadBytes, DefaultCellSize.int).tryGet()) + tree = Poseidon2Tree.init(slotBlockRoots).tryGet() + treeCid = tree.root().tryGet().toSlotCid().tryGet() + + for i in 0 ..< numberOfSlotBlocks: + let + blkCid = slotBlockRoots[i].toCellCid().tryGet() + proof = tree.getProof(i).tryGet().toEncodableProof().tryGet() + + discard await self.localStore.putCidAndProof(treeCid, i, blkCid, proof) + + return tree + +proc createDatasetRootHashAndSlotTree(self: ProvingTestEnvironment): Future[void] {.async.} = + var slotTrees = newSeq[Poseidon2Tree]() + for i in 0 ..< totalNumberOfSlots: + slotTrees.add(await self.createSlotTree(i.uint64)) + self.slotTree = slotTrees[datasetSlotIndex] + self.slotRootCid = slotTrees[datasetSlotIndex].root().tryGet().toSlotCid().tryGet() + self.slotRoots = slotTrees.mapIt(it.root().tryGet()) + let rootsPadLeafs = newSeqWith(totalNumberOfSlots.nextPowerOfTwoPad, Poseidon2Zero) + self.datasetToSlotTree = Poseidon2Tree.init(self.slotRoots & rootsPadLeafs).tryGet() + self.datasetRootHash = self.datasetToSlotTree.root().tryGet() + +proc createManifest(self: ProvingTestEnvironment): Future[void] {.async.} = + let + cids = self.datasetBlocks.mapIt(it.cid) + tree = CodexTree.init(cids).tryGet() + treeCid = tree.rootCid(CIDv1, BlockCodec).tryGet() + + for i in 0 ..< self.datasetBlocks.len: + let + blk = self.datasetBlocks[i] + leafCid = blk.cid + proof = tree.getProof(i).tryGet() + discard await self.localStore.putBlock(blk) + discard await self.localStore.putCidAndProof(treeCid, i, leafCid, proof) + + # Basic manifest: + self.manifest = Manifest.new( + treeCid = treeCid, + blockSize = bytesPerBlock.NBytes, + datasetSize = (bytesPerBlock * numberOfSlotBlocks * totalNumberOfSlots).NBytes) + + # Protected manifest: + self.manifest = Manifest.new( + manifest = self.manifest, + treeCid = treeCid, + datasetSize = self.manifest.datasetSize, + ecK = totalNumberOfSlots, + ecM = 0 + ) + + # Verifiable manifest: + self.manifest = Manifest.new( + manifest = self.manifest, + verifyRoot = self.datasetRootHash.toVerifyCid().tryGet(), + slotRoots = self.slotRoots.mapIt(it.toSlotCid().tryGet()) + ).tryGet() + + self.manifestBlock = bt.Block.new(self.manifest.encode().tryGet(), codec = ManifestCodec).tryGet() + discard await self.localStore.putBlock(self.manifestBlock) + +proc createSlot(self: ProvingTestEnvironment): void = + self.slot = Slot( + request: StorageRequest( + ask: StorageAsk( + slots: totalNumberOfSlots.uint64, + slotSize: u256(bytesPerBlock * numberOfSlotBlocks) + ), + content: StorageContent( + cid: $self.manifestBlock.cid + ), + ), + slotIndex: u256(datasetSlotIndex) + ) + +proc createProvingTestEnvironment*(): Future[ProvingTestEnvironment] {.async.} = + var testEnv = ProvingTestEnvironment( + challenge: toF(12345) + ) + + testEnv.localStore = CacheStore.new() + await testEnv.createDatasetBlocks() + await testEnv.createDatasetRootHashAndSlotTree() + await testEnv.createManifest() + testEnv.createSlot() + + return testEnv diff --git a/tests/codex/slots/testconverters.nim b/tests/codex/slots/testconverters.nim new file mode 100644 index 00000000..d76302bb --- /dev/null +++ b/tests/codex/slots/testconverters.nim @@ -0,0 +1,47 @@ +import pkg/chronos +import pkg/asynctest +import pkg/poseidon2 +import pkg/poseidon2/io +import pkg/constantine/math/io/io_fields +import pkg/questionable/results +import pkg/codex/merkletree + +import pkg/codex/slots/converters +import ../examples +import ../merkletree/helpers + +let + hash: Poseidon2Hash = toF(12345) + +suite "Converters": + test "CellBlock cid": + let + cid = toCellCid(hash).tryGet() + value = fromCellCid(cid).tryGet() + + check: + hash.toDecimal() == value.toDecimal() + + test "Slot cid": + let + cid = toSlotCid(hash).tryGet() + value = fromSlotCid(cid).tryGet() + + check: + hash.toDecimal() == value.toDecimal() + + test "Verify cid": + let + cid = toVerifyCid(hash).tryGet() + value = fromVerifyCid(cid).tryGet() + + check: + hash.toDecimal() == value.toDecimal() + + test "Proof": + let + codexProof = toEncodableProof(Poseidon2Proof.example).tryGet() + poseidonProof = toVerifiableProof(codexProof).tryGet() + + check: + Poseidon2Proof.example == poseidonProof diff --git a/tests/codex/slots/testsampler.nim b/tests/codex/slots/testsampler.nim new file mode 100644 index 00000000..2ca2b40f --- /dev/null +++ b/tests/codex/slots/testsampler.nim @@ -0,0 +1,118 @@ +import std/sequtils +import std/sugar +import std/random +import std/strutils + +import pkg/questionable/results +import pkg/constantine/math/arithmetic +import pkg/constantine/math/io/io_fields +import pkg/poseidon2/io +import pkg/poseidon2 +import pkg/chronos +import pkg/asynctest +import pkg/codex/stores/cachestore +import pkg/codex/chunker +import pkg/codex/stores +import pkg/codex/blocktype as bt +import pkg/codex/contracts/requests +import pkg/codex/contracts +import pkg/codex/merkletree +import pkg/codex/stores/cachestore + +import pkg/codex/slots/sampler +import pkg/codex/slots/builder/builder + +import ../helpers +import ../examples +import ../merkletree/helpers +import testsampler_expected +import ./provingtestenv + +asyncchecksuite "Test DataSampler": + var + env: ProvingTestEnvironment + dataSampler: DataSampler + blk: bt.Block + cell0Bytes: seq[byte] + cell1Bytes: seq[byte] + cell2Bytes: seq[byte] + + proc createDataSampler(): Future[void] {.async.} = + dataSampler = DataSampler.new( + datasetSlotIndex, + env.localStore, + SlotsBuilder.new(env.localStore, env.manifest).tryGet()).tryGet() + + setup: + randomize() + env = await createProvingTestEnvironment() + let bytes = newSeqWith(bytesPerBlock, rand(uint8)) + blk = bt.Block.new(bytes).tryGet() + cell0Bytes = bytes[0.. " & $expected & ")": + let slotBlockIndex = toBlockIdx(input, numCells = 32) + + check: + slotBlockIndex == expected + + for (input, expected) in [(10, 10), (31, 31), (32, 0), (63, 31), (64, 0)]: + test "Can get blockCellIndex from slotCellIndex (" & $input & " -> " & $expected & ")": + let blockCellIndex = toBlockCellIdx(input, numCells = 32) + + check: + blockCellIndex == expected diff --git a/tests/codex/testmanifest.nim b/tests/codex/testmanifest.nim index 35feefb8..184bcdd0 100644 --- a/tests/codex/testmanifest.nim +++ b/tests/codex/testmanifest.nim @@ -39,11 +39,11 @@ checksuite "Manifest": slotLeavesCids = leaves.toSlotCids().tryGet tree = Poseidon2Tree.init(leaves).tryGet - slotsRootsCid = tree.root.tryGet.toSlotsRootsCid().tryGet + verifyCid = tree.root.tryGet.toVerifyCid().tryGet verifiableManifest = Manifest.new( manifest = protectedManifest, - verifyRoot = slotsRootsCid, + verifyRoot = verifyCid, slotRoots = slotLeavesCids ).tryGet() diff --git a/tests/codex/testslotbuilder.nim b/tests/codex/testslotbuilder.nim deleted file mode 100644 index 4402d7c2..00000000 --- a/tests/codex/testslotbuilder.nim +++ /dev/null @@ -1,3 +0,0 @@ -import ./slotbuilder/testslotbuilder - -{.warning[UnusedImport]: off.} diff --git a/tests/codex/testslots.nim b/tests/codex/testslots.nim new file mode 100644 index 00000000..6dfd298c --- /dev/null +++ b/tests/codex/testslots.nim @@ -0,0 +1,6 @@ +import ./slots/testslotbuilder +import ./slots/testutils +import ./slots/testsampler +import ./slots/testconverters + +{.warning[UnusedImport]: off.} diff --git a/tests/testCodex.nim b/tests/testCodex.nim index 85f6b26c..7fb34ff3 100644 --- a/tests/testCodex.nim +++ b/tests/testCodex.nim @@ -14,7 +14,7 @@ import ./codex/testsystemclock import ./codex/testvalidation import ./codex/testasyncstreamwrapper import ./codex/testmerkletree -import ./codex/testslotbuilder +import ./codex/testslots import ./codex/testindexingstrategy {.warning[UnusedImport]: off.}