From 8b12934fe242381df8a65ac6afd32c6e171dac4a Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Thu, 11 Jan 2024 10:45:23 -0600 Subject: [PATCH] Build slots (#668) Wiring in slots builder functionality into `requestStorage` --- codex/blocktype.nim | 1 - codex/codextypes.nim | 4 +- codex/erasure/erasure.nim | 43 ++-- codex/errors.nim | 3 +- codex/manifest/coders.nim | 14 +- codex/manifest/manifest.nim | 33 +-- codex/merkletree.nim | 5 + codex/merkletree/codex/codex.nim | 8 +- codex/node.nim | 225 ++++++++++++------ codex/purchasing/purchase.nim | 3 +- codex/rest/api.nim | 7 +- codex/slots.nim | 3 + codex/slots/{slotbuilder.nim => builder.nim} | 193 +++++++++++----- codex/utils/digest.nim | 53 +---- codex/utils/poseidon2digest.nim | 52 +++++ config.nims | 1 + tests/codex/merkletree/testcodextree.nim | 1 + tests/codex/merkletree/testmerkledigest.nim | 1 + tests/codex/slotbuilder/testslotbuilder.nim | 111 +++++++-- tests/codex/testchunking.nim | 2 +- tests/codex/testerasure.nim | 4 +- tests/codex/testmanifest.nim | 25 +- tests/codex/testnode.nim | 226 ++++++++++--------- tests/integration/testIntegration.nim | 19 +- vendor/nim-poseidon2 | 2 +- 25 files changed, 680 insertions(+), 359 deletions(-) create mode 100644 codex/slots.nim rename codex/slots/{slotbuilder.nim => builder.nim} (57%) create mode 100644 codex/utils/poseidon2digest.nim diff --git a/codex/blocktype.nim b/codex/blocktype.nim index afd64061..52bb5432 100644 --- a/codex/blocktype.nim +++ b/codex/blocktype.nim @@ -44,7 +44,6 @@ type else: cid*: Cid - proc `==`*(a, b: BlockAddress): bool = a.leaf == b.leaf and ( diff --git a/codex/codextypes.nim b/codex/codextypes.nim index b2d63225..6202fdcd 100644 --- a/codex/codextypes.nim +++ b/codex/codextypes.nim @@ -39,7 +39,7 @@ const BlockCodec* = multiCodec("codex-block") SlotRootCodec* = multiCodec("codex-slot-root") SlotProvingRootCodec* = multiCodec("codex-proving-root") - CodexSlotCell* = multiCodec("codex-slot-cell") + CodexSlotCellCodec* = multiCodec("codex-slot-cell") CodexHashesCodecs* = [ Sha256HashCodec, @@ -53,7 +53,7 @@ const BlockCodec, SlotRootCodec, SlotProvingRootCodec, - CodexSlotCell, + CodexSlotCellCodec, ] proc initEmptyCidTable(): ?!Table[(CidVersion, MultiCodec, MultiCodec), Cid] = diff --git a/codex/erasure/erasure.nim b/codex/erasure/erasure.nim index cd10b53a..b6977145 100644 --- a/codex/erasure/erasure.nim +++ b/codex/erasure/erasure.nim @@ -72,11 +72,11 @@ type store*: BlockStore EncodingParams = object - ecK: int - ecM: int - rounded: int - steps: int - blocksCount: int + ecK: Natural + ecM: Natural + rounded: Natural + steps: Natural + blocksCount: Natural func indexToPos(steps, idx, step: int): int {.inline.} = ## Convert an index to a position in the encoded @@ -112,7 +112,8 @@ proc getPendingBlocks( return await completedFut else: let (_, index) = await completedFut - raise newException(CatchableError, "Future for block id not found, tree cid: " & $manifest.treeCid & ", index: " & $index) + raise newException(CatchableError, + "Future for block id not found, tree cid: " & $manifest.treeCid & ", index: " & $index) Iter.new(genNext, isFinished) @@ -120,10 +121,10 @@ proc prepareEncodingData( self: Erasure, manifest: Manifest, params: EncodingParams, - step: int, + step: Natural, data: ref seq[seq[byte]], cids: ref seq[Cid], - emptyBlock: seq[byte]): Future[?!int] {.async.} = + emptyBlock: seq[byte]): Future[?!Natural] {.async.} = ## Prepare data for encoding ## @@ -157,16 +158,16 @@ proc prepareEncodingData( return failure(err) cids[idx] = emptyBlockCid - success(resolved) + success(resolved.Natural) proc prepareDecodingData( self: Erasure, encoded: Manifest, - step: int, + step: Natural, data: ref seq[seq[byte]], parityData: ref seq[seq[byte]], cids: ref seq[Cid], - emptyBlock: seq[byte]): Future[?!(int, int)] {.async.} = + emptyBlock: seq[byte]): Future[?!(Natural, Natural)] {.async.} = ## Prepare data for decoding ## `encoded` - the encoded manifest ## `step` - the current step @@ -222,14 +223,18 @@ proc prepareDecodingData( resolved.inc - return success (dataPieces, parityPieces) + return success (dataPieces.Natural, parityPieces.Natural) proc init*( _: type EncodingParams, manifest: Manifest, - ecK: int, ecM: int): ?!EncodingParams = + ecK: Natural, ecM: Natural): ?!EncodingParams = if ecK > manifest.blocksCount: - return failure("Unable to encode manifest, not enough blocks, ecK = " & $ecK & ", blocksCount = " & $manifest.blocksCount) + return failure( + "Unable to encode manifest, not enough blocks, ecK = " & + $ecK & + ", blocksCount = " & + $manifest.blocksCount) let rounded = roundUp(manifest.blocksCount, ecK) @@ -338,8 +343,8 @@ proc encodeData( proc encode*( self: Erasure, manifest: Manifest, - blocks: int, - parity: int): Future[?!Manifest] {.async.} = + blocks: Natural, + parity: Natural): Future[?!Manifest] {.async.} = ## Encode a manifest into one that is erasure protected. ## ## `manifest` - the original manifest to be encoded @@ -347,7 +352,7 @@ proc encode*( ## `parity` - the number of parity blocks to generate - M ## - without params =? EncodingParams.init(manifest, blocks, parity), err: + without params =? EncodingParams.init(manifest, blocks.int, parity.int), err: return failure(err) without encodedManifest =? await self.encodeData(manifest, params), err: @@ -373,7 +378,7 @@ proc decode*( var cids = seq[Cid].new() - recoveredIndices = newSeq[int]() + recoveredIndices = newSeq[Natural]() decoder = self.decoderProvider(encoded.blockSize.int, encoded.ecK, encoded.ecM) emptyBlock = newSeq[byte](encoded.blockSize.int) @@ -443,7 +448,7 @@ proc decode*( let idxIter = Iter .fromItems(recoveredIndices) - .filter((i: int) => i < tree.leavesCount) + .filter((i: Natural) => i < tree.leavesCount) if err =? (await self.store.putSomeProofs(tree, idxIter)).errorOption: return failure(err) diff --git a/codex/errors.nim b/codex/errors.nim index ff2789b9..9947d4b7 100644 --- a/codex/errors.nim +++ b/codex/errors.nim @@ -31,7 +31,8 @@ template mapFailure*[T, V, E]( template mapFailure*[T, V](exp: Result[T, V]): Result[T, ref CatchableError] = mapFailure(exp, CodexError) -template toResult*[T](exp: Option[T]): Result[T, ref CatchableError] = +# TODO: using a template here, causes bad codegen +func toFailure*[T](exp: Option[T]): Result[T, ref CatchableError] {.inline.} = if exp.isSome: success exp.get else: diff --git a/codex/manifest/coders.nim b/codex/manifest/coders.nim index 21069273..aef9d452 100644 --- a/codex/manifest/coders.nim +++ b/codex/manifest/coders.nim @@ -39,7 +39,7 @@ proc encode*(manifest: Manifest): ?!seq[byte] = # # ```protobuf # Message VerificationInfo { - # bytes verificationRoot = 1; # Decimal encoded field-element + # bytes verifyRoot = 1; # Decimal encoded field-element # repeated bytes slotRoots = 2; # Decimal encoded field-elements # } # Message ErasureInfo { @@ -78,7 +78,7 @@ proc encode*(manifest: Manifest): ?!seq[byte] = if manifest.verifiable: var verificationInfo = initProtoBuffer() - verificationInfo.write(1, manifest.verificationRoot.data.buffer) + verificationInfo.write(1, manifest.verifyRoot.data.buffer) for slotRoot in manifest.slotRoots: verificationInfo.write(2, slotRoot.data.buffer) erasureInfo.write(5, verificationInfo) @@ -109,7 +109,7 @@ proc decode*(_: type Manifest, data: openArray[byte]): ?!Manifest = blockSize: uint32 originalDatasetSize: uint32 ecK, ecM: uint32 - verificationRoot: seq[byte] + verifyRoot: seq[byte] slotRoots: seq[seq[byte]] # Decode `Header` message @@ -158,8 +158,8 @@ proc decode*(_: type Manifest, data: openArray[byte]): ?!Manifest = verifiable = pbVerificationInfo.buffer.len > 0 if verifiable: - if pbVerificationInfo.getField(1, verificationRoot).isErr: - return failure("Unable to decode `verificationRoot` from manifest!") + if pbVerificationInfo.getField(1, verifyRoot).isErr: + return failure("Unable to decode `verifyRoot` from manifest!") if pbVerificationInfo.getRequiredRepeatedField(2, slotRoots).isErr: return failure("Unable to decode `slotRoots` from manifest!") @@ -193,12 +193,12 @@ proc decode*(_: type Manifest, data: openArray[byte]): ?!Manifest = if verifiable: let - verificationRootCid = ? Cid.init(verificationRoot).mapFailure + verifyRootCid = ? Cid.init(verifyRoot).mapFailure slotRootCids = slotRoots.mapIt(? Cid.init(it).mapFailure) return Manifest.new( manifest = self, - verificationRoot = verificationRootCid, + verifyRoot = verifyRootCid, slotRoots = slotRootCids ) diff --git a/codex/manifest/manifest.nim b/codex/manifest/manifest.nim index 525d4f0e..a3dbd7cc 100644 --- a/codex/manifest/manifest.nim +++ b/codex/manifest/manifest.nim @@ -39,8 +39,8 @@ type originalDatasetSize: NBytes case verifiable {.serialize.}: bool # Verifiable datasets can be used to generate storage proofs of true: - verificationRoot: Cid - slotRoots: seq[Cid] + verifyRoot: Cid # Root of the top level merkle tree built from slot roots + slotRoots: seq[Cid] # Individual slot root built from the original dataset blocks else: discard else: @@ -92,8 +92,8 @@ proc blocksCount*(self: Manifest): int = proc verifiable*(self: Manifest): bool = self.verifiable -proc verificationRoot*(self: Manifest): Cid = - self.verificationRoot +proc verifyRoot*(self: Manifest): Cid = + self.verifyRoot proc slotRoots*(self: Manifest): seq[Cid] = self.slotRoots @@ -103,6 +103,7 @@ proc numSlots*(self: Manifest): int = 0 else: self.ecK + self.ecM + ############################################################ # Operations on block list ############################################################ @@ -159,7 +160,7 @@ proc `==`*(a, b: Manifest): bool = (a.originalDatasetSize == b.originalDatasetSize) and (a.verifiable == b.verifiable) and (if a.verifiable: - (a.verificationRoot == b.verificationRoot) and + (a.verifyRoot == b.verifyRoot) and (a.slotRoots == b.slotRoots) else: true) @@ -181,7 +182,7 @@ proc `$`*(self: Manifest): string = ", originalDatasetSize: " & $self.originalDatasetSize & ", verifiable: " & $self.verifiable & (if self.verifiable: - ", verificationRoot: " & $self.verificationRoot & + ", verifyRoot: " & $self.verifyRoot & ", slotRoots: " & $self.slotRoots else: "") @@ -287,15 +288,21 @@ proc new*( proc new*( T: type Manifest, manifest: Manifest, - verificationRoot: Cid, - slotRoots: seq[Cid]): ?!Manifest = + verifyRoot: Cid, + slotRoots: openArray[Cid]): ?!Manifest = ## Create a verifiable dataset from an ## protected one ## - if not manifest.protected: - return failure newException(CodexError, "Can create verifiable manifest only from protected manifest.") - success(Manifest( + if not manifest.protected: + return failure newException( + CodexError, "Can create verifiable manifest only from protected manifest.") + + if slotRoots.len != manifest.numSlots: + return failure newException( + CodexError, "Wrong number of slot roots.") + + success Manifest( treeCid: manifest.treeCid, datasetSize: manifest.datasetSize, version: manifest.version, @@ -308,5 +315,5 @@ proc new*( originalTreeCid: manifest.treeCid, originalDatasetSize: manifest.originalDatasetSize, verifiable: true, - verificationRoot: verificationRoot, - slotRoots: slotRoots)) + verifyRoot: verifyRoot, + slotRoots: @slotRoots) diff --git a/codex/merkletree.nim b/codex/merkletree.nim index d7f6b1d7..bf5e3630 100644 --- a/codex/merkletree.nim +++ b/codex/merkletree.nim @@ -3,3 +3,8 @@ import ./merkletree/codex import ./merkletree/poseidon2 export codex, poseidon2, merkletree + +type + SomeMerkleTree* = ByteTree | CodexTree | Poseidon2Tree + SomeMerkleProof* = ByteProof | CodexProof | Poseidon2Proof + SomeMerkleHash* = ByteHash | Poseidon2Hash diff --git a/codex/merkletree/codex/codex.nim b/codex/merkletree/codex/codex.nim index ea9790d0..49439eb3 100644 --- a/codex/merkletree/codex/codex.nim +++ b/codex/merkletree/codex/codex.nim @@ -23,6 +23,8 @@ import ../../rng import ../../errors import ../../blocktype +from ../../utils/digest import digestBytes + import ../merkletree export merkletree @@ -62,12 +64,6 @@ func digestSize*(self: (CodexTree or CodexProof)): int = self.mhash.size -func digestBytes*(mhash: MultiHash): seq[byte] = - ## Extract hash digestBytes - ## - - mhash.data.buffer[mhash.dpos..= (1, 4): --warning:"ObservableStores:off" diff --git a/tests/codex/merkletree/testcodextree.nim b/tests/codex/merkletree/testcodextree.nim index ace612f5..3312309c 100644 --- a/tests/codex/merkletree/testcodextree.nim +++ b/tests/codex/merkletree/testcodextree.nim @@ -9,6 +9,7 @@ import pkg/libp2p import pkg/codex/codextypes import pkg/codex/merkletree +import pkg/codex/utils/digest import ./helpers import ./generictreetests diff --git a/tests/codex/merkletree/testmerkledigest.nim b/tests/codex/merkletree/testmerkledigest.nim index 21945d2e..6ef40e4e 100644 --- a/tests/codex/merkletree/testmerkledigest.nim +++ b/tests/codex/merkletree/testmerkledigest.nim @@ -12,6 +12,7 @@ import pkg/questionable/results import pkg/codex/merkletree import pkg/codex/utils/digest +import pkg/codex/utils/poseidon2digest import ./helpers diff --git a/tests/codex/slotbuilder/testslotbuilder.nim b/tests/codex/slotbuilder/testslotbuilder.nim index ac75a41c..8cb21d77 100644 --- a/tests/codex/slotbuilder/testslotbuilder.nim +++ b/tests/codex/slotbuilder/testslotbuilder.nim @@ -11,6 +11,7 @@ import pkg/codex/rng import pkg/codex/stores import pkg/codex/chunker import pkg/codex/merkletree +import pkg/codex/manifest {.all.} import pkg/codex/utils import pkg/codex/utils/digest import pkg/datastore @@ -23,7 +24,10 @@ import ../examples import ../merkletree/helpers import pkg/codex/indexingstrategy {.all.} -import pkg/codex/slots/slotbuilder {.all.} +import pkg/codex/slots {.all.} + +privateAccess(SlotsBuilder) # enable access to private fields +privateAccess(Manifest) # enable access to private fields suite "Slot builder": let @@ -57,7 +61,7 @@ suite "Slot builder": manifest: Manifest protectedManifest: Manifest expectedEmptyCid: Cid - slotBuilder: SlotBuilder + slotBuilder: SlotsBuilder chunker: Chunker proc createBlocks(): Future[void] {.async.} = @@ -115,8 +119,6 @@ suite "Slot builder": protectedManifest.hcodec, protectedManifest.codec).tryGet() - privateAccess(SlotBuilder) # enable access to private fields - setup: let repoDs = SQLiteDatastore.new(Memory).tryGet() @@ -130,6 +132,9 @@ suite "Slot builder": teardown: await localStore.close() + # TODO: THIS IS A BUG IN asynctest, because it doesn't release the + # objects after the test is done, so we need to do it manually + # # Need to reset all objects because otherwise they get # captured by the test runner closures, not good! reset(datasetBlocks) @@ -148,8 +153,8 @@ suite "Slot builder": datasetSize = originalDatasetSize.NBytes) check: - SlotBuilder.new(localStore, unprotectedManifest, cellSize = cellSize) - .error.msg == "Can only create SlotBuilder using protected manifests." + SlotsBuilder.new(localStore, unprotectedManifest, cellSize = cellSize) + .error.msg == "Can only create SlotsBuilder using protected manifests." test "Number of blocks must be devisable by number of slots": let @@ -164,7 +169,7 @@ suite "Slot builder": ecM = ecM) check: - SlotBuilder.new(localStore, mismatchManifest, cellSize = cellSize) + SlotsBuilder.new(localStore, mismatchManifest, cellSize = cellSize) .error.msg == "Number of blocks must be divisable by number of slots." test "Block size must be divisable by cell size": @@ -180,11 +185,11 @@ suite "Slot builder": ecM = ecM) check: - SlotBuilder.new(localStore, mismatchManifest, cellSize = cellSize) + SlotsBuilder.new(localStore, mismatchManifest, cellSize = cellSize) .error.msg == "Block size must be divisable by cell size." test "Should build correct slot builder": - slotBuilder = SlotBuilder.new( + slotBuilder = SlotsBuilder.new( localStore, protectedManifest, cellSize = cellSize).tryGet() @@ -197,7 +202,7 @@ suite "Slot builder": test "Should build slot hashes for all slots": let steppedStrategy = SteppedIndexingStrategy.new(0, numTotalBlocks - 1, numSlots) - slotBuilder = SlotBuilder.new( + slotBuilder = SlotsBuilder.new( localStore, protectedManifest, cellSize = cellSize).tryGet() @@ -220,7 +225,7 @@ suite "Slot builder": test "Should build slot trees for all slots": let steppedStrategy = SteppedIndexingStrategy.new(0, numTotalBlocks - 1, numSlots) - slotBuilder = SlotBuilder.new( + slotBuilder = SlotsBuilder.new( localStore, protectedManifest, cellSize = cellSize).tryGet() @@ -243,7 +248,7 @@ suite "Slot builder": test "Should persist trees for all slots": let - slotBuilder = SlotBuilder.new( + slotBuilder = SlotsBuilder.new( localStore, protectedManifest, cellSize = cellSize).tryGet() @@ -268,12 +273,13 @@ suite "Slot builder": test "Should build correct verification root": let steppedStrategy = SteppedIndexingStrategy.new(0, numTotalBlocks - 1, numSlots) - slotBuilder = SlotBuilder.new( + slotBuilder = SlotsBuilder.new( localStore, protectedManifest, cellSize = cellSize).tryGet() - slotRoots = (await slotBuilder.buildSlots()).tryGet + (await slotBuilder.buildSlots()).tryGet + let slotsHashes = collect(newSeq): for i in 0 ..< numSlots: let @@ -288,7 +294,7 @@ suite "Slot builder": Merkle.digest(slotHashes & slotsPadLeafs) expectedRoot = Merkle.digest(slotsHashes & rootsPadLeafs) - rootHash = slotBuilder.buildRootsTree(slotRoots).tryGet().root.tryGet() + rootHash = slotBuilder.buildRootsTree(slotBuilder.slotRoots).tryGet().root.tryGet() check: expectedRoot == rootHash @@ -296,7 +302,7 @@ suite "Slot builder": test "Should build correct verification root manifest": let steppedStrategy = SteppedIndexingStrategy.new(0, numTotalBlocks - 1, numSlots) - slotBuilder = SlotBuilder.new( + slotBuilder = SlotsBuilder.new( localStore, protectedManifest, cellSize = cellSize).tryGet() @@ -316,9 +322,78 @@ suite "Slot builder": expectedRoot = Merkle.digest(slotsHashes & rootsPadLeafs) manifest = (await slotBuilder.buildManifest()).tryGet() - mhash = manifest.verificationRoot.mhash.tryGet() + mhash = manifest.verifyRoot.mhash.tryGet() mhashBytes = mhash.digestBytes - rootHash = Poseidon2Hash.fromBytes(mhashBytes.toArray32).toResult.tryGet() + rootHash = Poseidon2Hash.fromBytes(mhashBytes.toArray32).get check: expectedRoot == rootHash + + test "Should not build from verifiable manifest with 0 slots": + var + slotBuilder = SlotsBuilder.new( + localStore, + protectedManifest, + cellSize = cellSize).tryGet() + verifyManifest = (await slotBuilder.buildManifest()).tryGet() + + verifyManifest.slotRoots = @[] + check SlotsBuilder.new( + localStore, + verifyManifest, + cellSize = cellSize).isErr + + test "Should not build from verifiable manifest with incorrect number of slots": + var + slotBuilder = SlotsBuilder.new( + localStore, + protectedManifest, + cellSize = cellSize).tryGet() + + verifyManifest = (await slotBuilder.buildManifest()).tryGet() + + verifyManifest.slotRoots.del( + verifyManifest.slotRoots.len - 1 + ) + + check SlotsBuilder.new( + localStore, + verifyManifest, + cellSize = cellSize).isErr + + test "Should not build from verifiable manifest with slots root": + let + slotBuilder = SlotsBuilder.new( + localStore, + protectedManifest, + cellSize = cellSize).tryGet() + + verifyManifest = (await slotBuilder.buildManifest()).tryGet() + offset = verifyManifest.verifyRoot.data.buffer.len div 2 + + rng.shuffle( + Rng.instance, + verifyManifest.verifyRoot.data.buffer) + + check SlotsBuilder.new( + localStore, + verifyManifest, + cellSize = cellSize).isErr + + test "Should build from verifiable manifest": + let + slotBuilder = SlotsBuilder.new( + localStore, + protectedManifest, + cellSize = cellSize).tryGet() + + verifyManifest = (await slotBuilder.buildManifest()).tryGet() + + verificationBuilder = SlotsBuilder.new( + localStore, + verifyManifest, + cellSize = cellSize).tryGet() + + check: + slotBuilder.slotRoots == verificationBuilder.slotRoots + slotBuilder.verifyRoot == verificationBuilder.verifyRoot diff --git a/tests/codex/testchunking.nim b/tests/codex/testchunking.nim index 860e52b6..99eec1c1 100644 --- a/tests/codex/testchunking.nim +++ b/tests/codex/testchunking.nim @@ -61,7 +61,7 @@ asyncchecksuite "Chunking": test "should chunk file": let - (path, _, _) = instantiationInfo(-2, fullPaths = true) # get this file's name + path = currentSourcePath() file = open(path) fileChunker = FileChunker.new(file = file, chunkSize = 256'nb, pad = false) diff --git a/tests/codex/testerasure.nim b/tests/codex/testerasure.nim index b108c6f7..0c7e08a5 100644 --- a/tests/codex/testerasure.nim +++ b/tests/codex/testerasure.nim @@ -39,8 +39,8 @@ suite "Erasure encode/decode": let encoded = (await erasure.encode( manifest, - buffers, - parity)).tryGet() + buffers.Natural, + parity.Natural)).tryGet() check: encoded.blocksCount mod (buffers + parity) == 0 diff --git a/tests/codex/testmanifest.nim b/tests/codex/testmanifest.nim index 72878885..35feefb8 100644 --- a/tests/codex/testmanifest.nim +++ b/tests/codex/testmanifest.nim @@ -6,6 +6,10 @@ import pkg/asynctest import pkg/codex/chunker import pkg/codex/blocktype as bt import pkg/codex/manifest +import pkg/poseidon2 + +import pkg/codex/slots +import pkg/codex/merkletree import ./helpers import ./examples @@ -17,17 +21,30 @@ checksuite "Manifest": blockSize = 1.MiBs, datasetSize = 100.MiBs ) + protectedManifest = Manifest.new( manifest = manifest, treeCid = Cid.example, datasetSize = 200.MiBs, - eck = 10, - ecM = 10 + eck = 2, + ecM = 2 ) + + leaves = [ + 0.toF.Poseidon2Hash, + 1.toF.Poseidon2Hash, + 2.toF.Poseidon2Hash, + 3.toF.Poseidon2Hash] + + slotLeavesCids = leaves.toSlotCids().tryGet + + tree = Poseidon2Tree.init(leaves).tryGet + slotsRootsCid = tree.root.tryGet.toSlotsRootsCid().tryGet + verifiableManifest = Manifest.new( manifest = protectedManifest, - verificationRoot = Cid.example, - slotRoots = @[Cid.example, Cid.example] + verifyRoot = slotsRootsCid, + slotRoots = slotLeavesCids ).tryGet() proc encodeDecode(manifest: Manifest): Manifest = diff --git a/tests/codex/testnode.nim b/tests/codex/testnode.nim index 63b598c2..1ac99864 100644 --- a/tests/codex/testnode.nim +++ b/tests/codex/testnode.nim @@ -2,6 +2,8 @@ import std/os import std/options import std/math import std/times +import std/sequtils +import std/importutils import pkg/asynctest import pkg/chronos @@ -11,6 +13,8 @@ import pkg/datastore import pkg/questionable import pkg/questionable/results import pkg/stint +import pkg/poseidon2 +import pkg/poseidon2/io import pkg/nitro import pkg/codexdht/discv5/protocol as discv5 @@ -21,23 +25,61 @@ import pkg/codex/contracts import pkg/codex/systemclock import pkg/codex/blockexchange import pkg/codex/chunker -import pkg/codex/node +import pkg/codex/slots import pkg/codex/manifest import pkg/codex/discovery +import pkg/codex/erasure +import pkg/codex/merkletree import pkg/codex/blocktype as bt +import pkg/codex/node {.all.} + import ../examples import ./helpers import ./helpers/mockmarket import ./helpers/mockclock +privateAccess(CodexNodeRef) # enable access to private fields + proc toTimesDuration(d: chronos.Duration): times.Duration = - initDuration(seconds=d.seconds) + initDuration(seconds = d.seconds) + +proc drain( + stream: LPStream | Result[lpstream.LPStream, ref CatchableError]): + Future[seq[byte]] {.async.} = -asyncchecksuite "Test Node": let - (path, _, _) = instantiationInfo(-2, fullPaths = true) # get this file's name + stream = + when typeof(stream) is Result[lpstream.LPStream, ref CatchableError]: + stream.tryGet() + else: + stream + defer: + await stream.close() + + var data: seq[byte] + while not stream.atEof: + var + buf = newSeq[byte](DefaultBlockSize.int) + res = await stream.readOnce(addr buf[0], DefaultBlockSize.int) + check res <= DefaultBlockSize.int + buf.setLen(res) + data &= buf + + data + +proc pipeChunker(stream: BufferStream, chunker: Chunker) {.async.} = + try: + while ( + let chunk = await chunker.getBytes(); + chunk.len > 0): + await stream.pushData(chunk) + finally: + await stream.pushEof() + await stream.close() + +template setupAndTearDown() {.dirty.} = var file: File chunker: Chunker @@ -55,33 +97,13 @@ asyncchecksuite "Test Node": peerStore: PeerCtxStore pendingBlocks: PendingBlocksManager discovery: DiscoveryEngine + erasure: Erasure - proc fetch(T: type Manifest, chunker: Chunker): Future[Manifest] {.async.} = - # Collect blocks from Chunker into Manifest - await storeDataGetManifest(localStore, chunker) - - proc retrieve(cid: Cid): Future[seq[byte]] {.async.} = - # Retrieve an entire file contents by file Cid - let - oddChunkSize = math.trunc(DefaultBlockSize.float/1.359).int # Let's check that node.retrieve can correctly rechunk data - stream = (await node.retrieve(cid)).tryGet() - var - data: seq[byte] - - defer: await stream.close() - - while not stream.atEof: - var - buf = newSeq[byte](oddChunkSize) - res = await stream.readOnce(addr buf[0], oddChunkSize) - check res <= oddChunkSize - buf.setLen(res) - data &= buf - - return data + let + path = currentSourcePath().parentDir setup: - file = open(path.splitFile().dir /../ "fixtures" / "test.jpg") + file = open(path /../ "fixtures" / "test.jpg") chunker = FileChunker.new(file = file, chunkSize = DefaultBlockSize) switch = newStandardSwitch() wallet = WalletRef.new(EthPrivateKey.random()) @@ -90,7 +112,7 @@ asyncchecksuite "Test Node": clock = SystemClock.new() localStoreMetaDs = SQLiteDatastore.new(Memory).tryGet() localStoreRepoDs = SQLiteDatastore.new(Memory).tryGet() - localStore = RepoStore.new(localStoreRepoDs, localStoreMetaDs, clock=clock) + localStore = RepoStore.new(localStoreRepoDs, localStoreMetaDs, clock = clock) await localStore.start() blockDiscovery = Discovery.new( @@ -102,7 +124,8 @@ asyncchecksuite "Test Node": discovery = DiscoveryEngine.new(localStore, peerStore, network, blockDiscovery, pendingBlocks) engine = BlockExcEngine.new(localStore, wallet, network, discovery, peerStore, pendingBlocks) store = NetworkStore.new(engine, localStore) - node = CodexNodeRef.new(switch, store, engine, nil, blockDiscovery) # TODO: pass `Erasure` + erasure = Erasure.new(store, leoEncoderProvider, leoDecoderProvider) + node = CodexNodeRef.new(switch, store, engine, erasure, blockDiscovery) await node.start() @@ -110,9 +133,12 @@ asyncchecksuite "Test Node": close(file) await node.stop() +asyncchecksuite "Test Node - Basic": + setupAndTearDown() + test "Fetch Manifest": let - manifest = await Manifest.fetch(chunker) + manifest = await storeDataGetManifest(localStore, chunker) manifestBlock = bt.Block.new( manifest.encode().tryGet(), @@ -127,7 +153,7 @@ asyncchecksuite "Test Node": fetched == manifest test "Block Batching": - let manifest = await Manifest.fetch(chunker) + let manifest = await storeDataGetManifest(localStore, chunker) for batchSize in 1..12: (await node.fetchBatched( @@ -142,8 +168,9 @@ asyncchecksuite "Test Node": let stream = BufferStream.new() storeFut = node.store(stream) - oddChunkSize = math.trunc(DefaultBlockSize.float/3.14).NBytes # Let's check that node.store can correctly rechunk these odd chunks + oddChunkSize = math.trunc(DefaultBlockSize.float / 3.14).NBytes # Let's check that node.store can correctly rechunk these odd chunks oddChunker = FileChunker.new(file = file, chunkSize = oddChunkSize, pad = false) # TODO: doesn't work with pad=tue + var original: seq[byte] @@ -159,14 +186,9 @@ asyncchecksuite "Test Node": let manifestCid = (await storeFut).tryGet() - check: - (await localStore.hasBlock(manifestCid)).tryGet() - - let manifestBlock = (await localStore.getBlock(manifestCid)).tryGet() localManifest = Manifest.decode(manifestBlock).tryGet() - - let data = await retrieve(manifestCid) + data = await (await node.retrieve(manifestCid)).drain() check: data.len == localManifest.datasetSize.int @@ -186,80 +208,72 @@ asyncchecksuite "Test Node": await stream.readExactly(addr data[0], data.len) check string.fromBytes(data) == testString + test "Setup purchase request": + let + manifest = await storeDataGetManifest(localStore, chunker) + manifestBlock = bt.Block.new( + manifest.encode().tryGet(), + codec = ManifestCodec).tryGet() -asyncchecksuite "Test Node - host contracts": - let - (path, _, _) = instantiationInfo(-2, fullPaths = true) # get this file's name + protected = (await erasure.encode(manifest, 3, 2)).tryGet() + builder = SlotsBuilder.new(localStore, protected).tryGet() + verifiable = (await builder.buildManifest()).tryGet() + verifiableBlock = bt.Block.new( + verifiable.encode().tryGet(), + codec = ManifestCodec).tryGet() + + (await localStore.putBlock(manifestBlock)).tryGet() + + let + request = (await node.setupRequest( + cid = manifestBlock.cid, + nodes = 5, + tolerance = 2, + duration = 100.u256, + reward = 2.u256, + proofProbability = 3.u256, + expiry = 200.u256, + collateral = 200.u256)).tryGet + + check: + (await verifiableBlock.cid in localStore) == true + request.content.cid == $verifiableBlock.cid + request.content.merkleRoot == builder.verifyRoot.get.toBytes + +asyncchecksuite "Test Node - Host contracts": + setupAndTearDown() var - file: File - chunker: Chunker - switch: Switch - wallet: WalletRef - network: BlockExcNetwork - clock: MockClock - localStore: RepoStore - localStoreRepoDs: DataStore - localStoreMetaDs: DataStore - engine: BlockExcEngine - store: NetworkStore sales: Sales - node: CodexNodeRef - blockDiscovery: Discovery - peerStore: PeerCtxStore - pendingBlocks: PendingBlocksManager - discovery: DiscoveryEngine + purchasing: Purchasing manifest: Manifest - manifestCid: string - - proc fetch(T: type Manifest, chunker: Chunker): Future[Manifest] {.async.} = - # Collect blocks from Chunker into Manifest - await storeDataGetManifest(localStore, chunker) + manifestCidStr: string + manifestCid: Cid + market: MockMarket setup: - file = open(path.splitFile().dir /../ "fixtures" / "test.jpg") - chunker = FileChunker.new(file = file, chunkSize = DefaultBlockSize) - switch = newStandardSwitch() - wallet = WalletRef.new(EthPrivateKey.random()) - network = BlockExcNetwork.new(switch) - - clock = MockClock.new() - localStoreMetaDs = SQLiteDatastore.new(Memory).tryGet() - localStoreRepoDs = SQLiteDatastore.new(Memory).tryGet() - localStore = RepoStore.new(localStoreRepoDs, localStoreMetaDs, clock=clock) - await localStore.start() - - blockDiscovery = Discovery.new( - switch.peerInfo.privateKey, - announceAddrs = @[MultiAddress.init("/ip4/127.0.0.1/tcp/0") - .expect("Should return multiaddress")]) - peerStore = PeerCtxStore.new() - pendingBlocks = PendingBlocksManager.new() - discovery = DiscoveryEngine.new(localStore, peerStore, network, blockDiscovery, pendingBlocks) - engine = BlockExcEngine.new(localStore, wallet, network, discovery, peerStore, pendingBlocks) - store = NetworkStore.new(engine, localStore) - node = CodexNodeRef.new(switch, store, engine, nil, blockDiscovery) # TODO: pass `Erasure` - # Setup Host Contracts and dependencies - let market = MockMarket.new() + market = MockMarket.new() sales = Sales.new(market, clock, localStore) - let hostContracts = some HostInteractions.new(clock, sales) - node.contracts = (ClientInteractions.none, hostContracts, ValidatorInteractions.none) + + node.contracts = ( + none ClientInteractions, + some HostInteractions.new(clock, sales), + none ValidatorInteractions) await node.start() # Populate manifest in local store manifest = await storeDataGetManifest(localStore, chunker) - let manifestBlock = bt.Block.new( + let + manifestBlock = bt.Block.new( manifest.encode().tryGet(), - codec = ManifestCodec - ).tryGet() - manifestCid = $(manifestBlock.cid) - (await localStore.putBlock(manifestBlock)).tryGet() + codec = ManifestCodec).tryGet() - teardown: - close(file) - await node.stop() + manifestCid = manifestBlock.cid + manifestCidStr = $(manifestCid) + + (await localStore.putBlock(manifestBlock)).tryGet() test "onExpiryUpdate callback is set": check sales.onExpiryUpdate.isSome @@ -270,12 +284,13 @@ asyncchecksuite "Test Node - host contracts": expectedExpiry: SecondsSince1970 = clock.now + DefaultBlockTtl.seconds + 11123 expiryUpdateCallback = !sales.onExpiryUpdate - (await expiryUpdateCallback(manifestCid, expectedExpiry)).tryGet() + (await expiryUpdateCallback(manifestCidStr, expectedExpiry)).tryGet() for index in 0..