diff --git a/codex/blockexchange/engine/engine.nim b/codex/blockexchange/engine/engine.nim index 64aa624c..e42851c5 100644 --- a/codex/blockexchange/engine/engine.nim +++ b/codex/blockexchange/engine/engine.nim @@ -122,9 +122,9 @@ proc stop*(b: BlockExcEngine) {.async.} = proc sendWantHave( - b: BlockExcEngine, - address: BlockAddress, - selectedPeer: BlockExcPeerCtx, + b: BlockExcEngine, + address: BlockAddress, + selectedPeer: BlockExcPeerCtx, peers: seq[BlockExcPeerCtx]): Future[void] {.async.} = trace "Sending wantHave request to peers", address for p in peers: @@ -137,8 +137,8 @@ proc sendWantHave( wantType = WantType.WantHave) # we only want to know if the peer has the block proc sendWantBlock( - b: BlockExcEngine, - address: BlockAddress, + b: BlockExcEngine, + address: BlockAddress, blockPeer: BlockExcPeerCtx): Future[void] {.async.} = trace "Sending wantBlock request to", peer = blockPeer.id, address await b.network.request.sendWantList( @@ -189,14 +189,14 @@ proc requestBlock*( if peers.len == 0: b.discovery.queueFindBlocksReq(@[address.cidOrTreeCid]) - let maybePeer = + let maybePeer = if peers.len > 0: peers[hash(address) mod peers.len].some elif b.peers.len > 0: toSeq(b.peers)[hash(address) mod b.peers.len].some else: BlockExcPeerCtx.none - + if peer =? maybePeer: asyncSpawn b.monitorBlockHandle(blockFuture, address, peer.id) b.pendingBlocks.setInFlight(address) @@ -204,7 +204,7 @@ proc requestBlock*( codex_block_exchange_want_block_lists_sent.inc() await b.sendWantHave(address, peer, toSeq(b.peers)) codex_block_exchange_want_have_lists_sent.inc() - + return await blockFuture proc requestBlock*( @@ -320,7 +320,7 @@ proc validateBlockDelivery( if bd.address.leaf: without proof =? bd.proof: return failure("Missing proof") - + if proof.index != bd.address.index: return failure("Proof index " & $proof.index & " doesn't match leaf index " & $bd.address.index) @@ -539,7 +539,7 @@ proc taskHandler*(b: BlockExcEngine, task: BlockExcPeerCtx) {.gcsafe, async.} = trace "Handling lookup for entry", address = e.address if e.address.leaf: (await b.localStore.getBlockAndProof(e.address.treeCid, e.address.index)).map( - (blkAndProof: (Block, MerkleProof)) => + (blkAndProof: (Block, MerkleProof)) => BlockDelivery(address: e.address, blk: blkAndProof[0], proof: blkAndProof[1].some) ) else: diff --git a/codex/blockexchange/engine/pendingblocks.nim b/codex/blockexchange/engine/pendingblocks.nim index 328f3ead..c4763998 100644 --- a/codex/blockexchange/engine/pendingblocks.nim +++ b/codex/blockexchange/engine/pendingblocks.nim @@ -102,7 +102,7 @@ proc resolve*( retrievalDurationUs = (stopTime - startTime) div 1000 blockReq.handle.complete(bd.blk) - + codex_block_exchange_retrieval_time_us.set(retrievalDurationUs) trace "Block retrieval time", retrievalDurationUs, address = bd.address else: diff --git a/codex/blocktype.nim b/codex/blocktype.nim index 080e308a..a26a3157 100644 --- a/codex/blocktype.nim +++ b/codex/blocktype.nim @@ -96,7 +96,7 @@ func new*( codec = multiCodec("raw") ): ?!Block = ## creates a new block for both storage and network IO - ## + ## let hash = ? MultiHash.digest($mcodec, data).mapFailure @@ -132,7 +132,7 @@ func new*( proc emptyCid*(version: CidVersion, hcodec: MultiCodec, dcodec: MultiCodec): ?!Cid = ## Returns cid representing empty content, given cid version, hash codec and data codec - ## + ## const Sha256 = multiCodec("sha2-256") @@ -161,11 +161,11 @@ proc emptyBlock*(version: CidVersion, hcodec: MultiCodec): ?!Block = .flatMap((cid: Cid) => Block.new(cid = cid, data = @[])) proc emptyBlock*(cid: Cid): ?!Block = - cid.mhash.mapFailure.flatMap((mhash: MultiHash) => + cid.mhash.mapFailure.flatMap((mhash: MultiHash) => emptyBlock(cid.cidver, mhash.mcodec)) proc isEmpty*(cid: Cid): bool = - success(cid) == cid.mhash.mapFailure.flatMap((mhash: MultiHash) => + success(cid) == cid.mhash.mapFailure.flatMap((mhash: MultiHash) => emptyCid(cid.cidver, mhash.mcodec, cid.mcodec)) proc isEmpty*(blk: Block): bool = diff --git a/codex/erasure/erasure.nim b/codex/erasure/erasure.nim index 35109a99..1bab089a 100644 --- a/codex/erasure/erasure.nim +++ b/codex/erasure/erasure.nim @@ -102,7 +102,7 @@ proc getPendingBlocks( proc isFinished(): bool = pendingBlocks.len == 0 - proc genNext(): Future[(?!bt.Block, int)] {.async.} = + proc genNext(): Future[(?!bt.Block, int)] {.async.} = let completedFut = await one(pendingBlocks) if (let i = pendingBlocks.find(completedFut); i >= 0): pendingBlocks.del(i) @@ -112,7 +112,7 @@ proc getPendingBlocks( raise newException(CatchableError, "Future for block id not found, tree cid: " & $manifest.treeCid & ", index: " & $index) Iter.new(genNext, isFinished) - + proc prepareEncodingData( self: Erasure, manifest: Manifest, @@ -134,7 +134,7 @@ proc prepareEncodingData( without blk =? blkOrErr, err: warn "Failed retreiving a block", treeCid = manifest.treeCid, idx, msg = err.msg continue - + let pos = indexToPos(params.steps, idx, step) shallowCopy(data[pos], if blk.isEmpty: emptyBlock else: blk.data) cids[idx] = blk.cid @@ -168,7 +168,7 @@ proc prepareDecodingData( ## `emptyBlock` - the empty block to be used for padding ## - let + let indicies = toSeq(countup(step, encoded.blocksCount - 1, encoded.steps)) pendingBlocksIter = self.getPendingBlocks(encoded, indicies) diff --git a/codex/manifest/manifest.nim b/codex/manifest/manifest.nim index 53c37ce3..886f9ead 100644 --- a/codex/manifest/manifest.nim +++ b/codex/manifest/manifest.nim @@ -30,17 +30,17 @@ export types type Manifest* = ref object of RootObj - treeCid {.serialize.}: Cid # Root of the merkle tree - datasetSize {.serialize.}: NBytes # Total size of all blocks - blockSize {.serialize.}: NBytes # Size of each contained block (might not be needed if blocks are len-prefixed) - version: CidVersion # Cid version - hcodec: MultiCodec # Multihash codec - codec: MultiCodec # Data set codec - case protected {.serialize.}: bool # Protected datasets have erasure coded info + treeCid {.serialize.}: Cid # Root of the merkle tree + datasetSize {.serialize.}: NBytes # Total size of all blocks + blockSize {.serialize.}: NBytes # Size of each contained block (might not be needed if blocks are len-prefixed) + version: CidVersion # Cid version + hcodec: MultiCodec # Multihash codec + codec: MultiCodec # Data set codec + case protected {.serialize.}: bool # Protected datasets have erasure coded info of true: - ecK: int # Number of blocks to encode - ecM: int # Number of resulting parity blocks - originalTreeCid: Cid # The original root of the dataset being erasure coded + ecK: int # Number of blocks to encode + ecM: int # Number of resulting parity blocks + originalTreeCid: Cid # The original root of the dataset being erasure coded originalDatasetSize: NBytes else: discard diff --git a/codex/merkletree/merkletree.nim b/codex/merkletree/merkletree.nim index 7612f0d6..74ddba6d 100644 --- a/codex/merkletree/merkletree.nim +++ b/codex/merkletree/merkletree.nim @@ -46,7 +46,7 @@ type ########################################################### func computeTreeHeight(leavesCount: int): int = - if isPowerOfTwo(leavesCount): + if isPowerOfTwo(leavesCount): fastLog2(leavesCount) + 1 else: fastLog2(leavesCount) + 2 @@ -84,16 +84,16 @@ proc init*( proc addDataBlock*(self: var MerkleTreeBuilder, dataBlock: openArray[byte]): ?!void = ## Hashes the data block and adds the result of hashing to a buffer - ## + ## let oldLen = self.buffer.len self.buffer.setLen(oldLen + self.digestSize) digestFn(self.mcodec, self.buffer, oldLen, dataBlock) proc addLeaf*(self: var MerkleTreeBuilder, leaf: MultiHash): ?!void = if leaf.mcodec != self.mcodec or leaf.size != self.digestSize: - return failure("Expected mcodec to be " & $self.mcodec & " and digest size to be " & + return failure("Expected mcodec to be " & $self.mcodec & " and digest size to be " & $self.digestSize & " but was " & $leaf.mcodec & " and " & $leaf.size) - + let oldLen = self.buffer.len self.buffer.setLen(oldLen + self.digestSize) self.buffer[oldLen..= self.leavesCount: return failure("Index " & $index & " out of range [0.." & $(self.leavesCount - 1) & "]" ) - + success(self.nodeBufferToMultiHash(index)) proc getLeafCid*(self: MerkleTree, index: Natural, version = CIDv1, dataCodec = multiCodec("raw")): ?!Cid = @@ -216,7 +216,7 @@ proc height*(self: MerkleTree): Natural = proc getProof*(self: MerkleTree, index: Natural): ?!MerkleProof = ## Extracts proof from a tree for a given index - ## + ## ## Given a tree built from data blocks A, B and C ## H5 ## / \ @@ -230,7 +230,7 @@ proc getProof*(self: MerkleTree, index: Natural): ?!MerkleProof = ## - 0,[H1, H4] for data block A ## - 1,[H0, H4] for data block B ## - 2,[0x00, H3] for data block C - ## + ## if index >= self.leavesCount: return failure("Index " & $index & " out of range [0.." & $(self.leavesCount - 1) & "]" ) @@ -250,7 +250,7 @@ proc getProof*(self: MerkleTree, index: Natural): ?!MerkleProof = var dummyValue = if level.index == 0: zero else: one if siblingIndex < level.offset + level.width: - proofNodesBuffer[level.index * self.digestSize..<(level.index + 1) * self.digestSize] = + proofNodesBuffer[level.index * self.digestSize..<(level.index + 1) * self.digestSize] = self.nodesBuffer[siblingIndex * self.digestSize..<(siblingIndex + 1) * self.digestSize] else: proofNodesBuffer[level.index * self.digestSize..<(level.index + 1) * self.digestSize] = dummyValue @@ -281,9 +281,9 @@ proc init*( if totalNodes * digestSize == nodesBuffer.len: success( MerkleTree( - mcodec: mcodec, - digestSize: digestSize, - leavesCount: leavesCount, + mcodec: mcodec, + digestSize: digestSize, + leavesCount: leavesCount, nodesBuffer: nodesBuffer ) ) @@ -296,14 +296,14 @@ proc init*( ): ?!MerkleTree = without leaf =? leaves.?[0]: return failure("At least one leaf is required") - + var builder = ? MerkleTreeBuilder.init(mcodec = leaf.mcodec) for l in leaves: let res = builder.addLeaf(l) if res.isErr: return failure(res.error) - + builder.build() proc init*( @@ -311,7 +311,7 @@ proc init*( cids: openArray[Cid] ): ?!MerkleTree = var leaves = newSeq[MultiHash]() - + for cid in cids: let res = cid.mhash.mapFailure if res.isErr: @@ -345,7 +345,7 @@ proc verifyLeaf*(self: MerkleProof, leaf: MultiHash, treeRoot: MultiHash): ?!boo else: concatBuf[0..^1] = self.nodesBuffer[offset..<(offset + self.digestSize)] & digestBuf ? digestFn(self.mcodec, digestBuf, 0, concatBuf) - + let computedRoot = ? MultiHash.init(self.mcodec, digestBuf).mapFailure success(computedRoot == treeRoot) @@ -369,8 +369,8 @@ proc `$`*(self: MerkleProof): string = ", nodes: " & $self.nodes func `==`*(a, b: MerkleProof): bool = - (a.index == b.index) and - (a.mcodec == b.mcodec) and + (a.index == b.index) and + (a.mcodec == b.mcodec) and (a.digestSize == b.digestSize) and (a.nodesBuffer == b.nodesBuffer) @@ -385,11 +385,11 @@ proc init*( let mcodec = nodes[0].mcodec digestSize = nodes[0].size - + var nodesBuffer = newSeq[byte](nodes.len * digestSize) for nodeIndex, node in nodes: nodesBuffer[nodeIndex * digestSize..<(nodeIndex + 1) * digestSize] = node.data.buffer[node.dpos.. items[i]) @@ -81,7 +81,7 @@ proc fromRange*[U, V, S: Ordinal](_: type Iter, a: U, b: V, step: S = 1): Iter[U let u = i inc(i, step) u - + proc isFinished(): bool = (step > 0 and i > b) or (step < 0 and i < b)