diff --git a/codex/blockexchange/engine/engine.nim b/codex/blockexchange/engine/engine.nim index 23683718..649edec6 100644 --- a/codex/blockexchange/engine/engine.nim +++ b/codex/blockexchange/engine/engine.nim @@ -361,7 +361,12 @@ proc blocksDeliveryHandler*( without proof =? bd.proof: error "Proof expected for a leaf block delivery" continue - if err =? (await b.localStore.putBlockCidAndProof(bd.address.treeCid, bd.address.index, bd.blk.cid, proof)).errorOption: + if err =? (await b.localStore.putCidAndProof( + bd.address.treeCid, + bd.address.index, + bd.blk.cid, + proof)).errorOption: + error "Unable to store proof and cid for a block" continue diff --git a/codex/codextypes.nim b/codex/codextypes.nim index ddf0ddff..b2d63225 100644 --- a/codex/codextypes.nim +++ b/codex/codextypes.nim @@ -39,6 +39,7 @@ const BlockCodec* = multiCodec("codex-block") SlotRootCodec* = multiCodec("codex-slot-root") SlotProvingRootCodec* = multiCodec("codex-proving-root") + CodexSlotCell* = multiCodec("codex-slot-cell") CodexHashesCodecs* = [ Sha256HashCodec, @@ -52,15 +53,15 @@ const BlockCodec, SlotRootCodec, SlotProvingRootCodec, + CodexSlotCell, ] - proc initEmptyCidTable(): ?!Table[(CidVersion, MultiCodec, MultiCodec), Cid] = ## Initialize padding blocks table ## ## TODO: Ideally this is done at compile time, but for now ## we do it at runtime because of an `importc` error that is - ## coming from somwhere in MultiHash that I can't track down. + ## coming from somewhere in MultiHash that I can't track down. ## let @@ -68,8 +69,6 @@ proc initEmptyCidTable(): ?!Table[(CidVersion, MultiCodec, MultiCodec), Cid] = PadHashes = { Sha256HashCodec: ? MultiHash.digest($Sha256HashCodec, emptyData).mapFailure, Sha512HashCodec: ? MultiHash.digest($Sha512HashCodec, emptyData).mapFailure, - Pos2Bn128SpngCodec: ? MultiHash.digest($Pos2Bn128SpngCodec, emptyData).mapFailure, - Pos2Bn128MrklCodec: ? MultiHash.digest($Pos2Bn128SpngCodec, emptyData).mapFailure, }.toTable var diff --git a/codex/erasure/erasure.nim b/codex/erasure/erasure.nim index 19013b4d..cd10b53a 100644 --- a/codex/erasure/erasure.nim +++ b/codex/erasure/erasure.nim @@ -25,6 +25,7 @@ import ../stores import ../blocktype as bt import ../utils import ../utils/asynciter +import ../indexingstrategy import pkg/stew/byteutils @@ -127,7 +128,12 @@ proc prepareEncodingData( ## let - indicies = toSeq(countup(step, params.rounded - 1, params.steps)) + strategy = SteppedIndexingStrategy.new( + firstIndex = 0, + lastIndex = params.rounded - 1, + numberOfIterations = params.steps + ) + indicies = strategy.getIndicies(step) pendingBlocksIter = self.getPendingBlocks(manifest, indicies.filterIt(it < manifest.blocksCount)) var resolved = 0 @@ -171,7 +177,12 @@ proc prepareDecodingData( ## let - indicies = toSeq(countup(step, encoded.blocksCount - 1, encoded.steps)) + strategy = SteppedIndexingStrategy.new( + firstIndex = 0, + lastIndex = encoded.blocksCount - 1, + numberOfIterations = encoded.steps + ) + indicies = strategy.getIndicies(step) pendingBlocksIter = self.getPendingBlocks(encoded, indicies) var @@ -213,7 +224,10 @@ proc prepareDecodingData( return success (dataPieces, parityPieces) -proc init(_: type EncodingParams, manifest: Manifest, ecK: int, ecM: int): ?!EncodingParams = +proc init*( + _: type EncodingParams, + manifest: Manifest, + ecK: int, ecM: int): ?!EncodingParams = if ecK > manifest.blocksCount: return failure("Unable to encode manifest, not enough blocks, ecK = " & $ecK & ", blocksCount = " & $manifest.blocksCount) diff --git a/codex/errors.nim b/codex/errors.nim index ebb07f04..ff2789b9 100644 --- a/codex/errors.nim +++ b/codex/errors.nim @@ -7,6 +7,8 @@ ## This file may not be copied, modified, or distributed except according to ## those terms. +import std/options + import pkg/stew/results import pkg/chronos import pkg/questionable/results @@ -29,6 +31,12 @@ template mapFailure*[T, V, E]( template mapFailure*[T, V](exp: Result[T, V]): Result[T, ref CatchableError] = mapFailure(exp, CodexError) +template toResult*[T](exp: Option[T]): Result[T, ref CatchableError] = + if exp.isSome: + success exp.get + else: + T.failure("Option is None") + proc allFutureResult*[T](fut: seq[Future[T]]): Future[?!void] {.async.} = try: await allFuturesThrowing(fut) diff --git a/codex/indexingstrategy.nim b/codex/indexingstrategy.nim new file mode 100644 index 00000000..b7774765 --- /dev/null +++ b/codex/indexingstrategy.nim @@ -0,0 +1,61 @@ +import std/sequtils +import ./utils + +# I'm choosing to use an assert here because: +# 1. These are a programmer errors and *should not* happen during application runtime. +# 2. Users don't have to deal with Result types. + +type + # Representing a strategy for grouping indices (of blocks usually) + # Given an interation-count as input, will produce a seq of + # selected indices. + IndexingStrategy* = ref object of RootObj + firstIndex*: int # Lowest index that can be returned + lastIndex*: int # Highest index that can be returned + numberOfIterations*: int # getIndices(iteration) will run from 0 ..< numberOfIterations + step*: int + + # Simplest approach: + # 0 => 0, 1, 2 + # 1 => 3, 4, 5 + # 2 => 6, 7, 8 + LinearIndexingStrategy* = ref object of IndexingStrategy + + # Stepped indexing: + # 0 => 0, 3, 6 + # 1 => 1, 4, 7 + # 2 => 2, 5, 8 + SteppedIndexingStrategy* = ref object of IndexingStrategy + +proc assertIteration(self: IndexingStrategy, iteration: int): void = + if iteration >= self.numberOfIterations: + raiseAssert("Indexing iteration can't be greater than or equal to numberOfIterations.") + +method getIndicies*(self: IndexingStrategy, iteration: int): seq[int] {.base.} = + raiseAssert("Not implemented") + +proc new*(T: type IndexingStrategy, firstIndex, lastIndex, numberOfIterations: int): T = + if firstIndex > lastIndex: + raiseAssert("firstIndex (" & $firstIndex & ") can't be greater than lastIndex (" & $lastIndex & ")") + if numberOfIterations <= 0: + raiseAssert("numberOfIteration (" & $numberOfIterations & ") must be greater than zero.") + + T( + firstIndex: firstIndex, + lastIndex: lastIndex, + numberOfIterations: numberOfIterations, + step: divUp((lastIndex - firstIndex), numberOfIterations) + ) + +method getIndicies*(self: LinearIndexingStrategy, iteration: int): seq[int] = + self.assertIteration(iteration) + + let + first = self.firstIndex + iteration * (self.step + 1) + last = min(first + self.step, self.lastIndex) + + toSeq(countup(first, last, 1)) + +method getIndicies*(self: SteppedIndexingStrategy, iteration: int): seq[int] = + self.assertIteration(iteration) + toSeq(countup(self.firstIndex + iteration, self.lastIndex, self.numberOfIterations)) diff --git a/codex/manifest/manifest.nim b/codex/manifest/manifest.nim index fcc2a704..525d4f0e 100644 --- a/codex/manifest/manifest.nim +++ b/codex/manifest/manifest.nim @@ -98,6 +98,11 @@ proc verificationRoot*(self: Manifest): Cid = proc slotRoots*(self: Manifest): seq[Cid] = self.slotRoots +proc numSlots*(self: Manifest): int = + if not self.protected: + 0 + else: + self.ecK + self.ecM ############################################################ # Operations on block list ############################################################ diff --git a/codex/merkletree/codex/codex.nim b/codex/merkletree/codex/codex.nim index 887bb1e1..ea9790d0 100644 --- a/codex/merkletree/codex/codex.nim +++ b/codex/merkletree/codex/codex.nim @@ -13,7 +13,6 @@ push: {.upraises: [].} import std/bitops import std/sequtils - import pkg/questionable import pkg/questionable/results import pkg/libp2p/[cid, multicodec, multihash] @@ -40,15 +39,15 @@ type ByteHash* = seq[byte] ByteTree* = MerkleTree[ByteHash, ByteTreeKey] - ByteTreeProof* = MerkleProof[ByteHash, ByteTreeKey] + ByteProof* = MerkleProof[ByteHash, ByteTreeKey] CodexTree* = ref object of ByteTree - mhash: MHash + mcodec*: MultiCodec - CodexProof* = ref object of ByteTreeProof - mhash: MHash + CodexProof* = ref object of ByteProof + mcodec*: MultiCodec -func getMhash*(mcodec: MultiCodec): ?!MHash = +func mhash*(mcodec: MultiCodec): ?!MHash = let mhash = CodeHashes.getOrDefault(mcodec) @@ -63,21 +62,15 @@ func digestSize*(self: (CodexTree or CodexProof)): int = self.mhash.size -func mcodec*(self: (CodexTree or CodexProof)): MultiCodec = - ## Multicodec - ## - - self.mhash.mcodec - -func bytes*(mhash: MultiHash): seq[byte] = - ## Extract hash bytes +func digestBytes*(mhash: MultiHash): seq[byte] = + ## Extract hash digestBytes ## mhash.data.buffer[mhash.dpos..