From 44ad291b16508d187ed0daa1ccd7b51ddb6d782f Mon Sep 17 00:00:00 2001 From: Eric <5089238+emizzle@users.noreply.github.com> Date: Tue, 3 Feb 2026 09:43:14 +1100 Subject: [PATCH] refactor!: move merkletree to own repo (#1390) --- .gitmodules | 5 +- codex/blockexchange/engine/engine.nim | 5 +- codex/blockexchange/protobuf/message.nim | 9 +- codex/merkletree.nim | 4 +- codex/merkletree/{codex => }/coders.nim | 25 +- codex/merkletree/codex.nim | 4 - codex/merkletree/codex/codex.nim | 249 -------- codex/merkletree/merkletree.nim | 569 +++++++----------- codex/node.nim | 2 +- codex/stores/blockstore.nim | 14 +- codex/stores/cachestore.nim | 14 +- codex/stores/networkstore.nim | 8 +- codex/stores/repostore/operations.nim | 6 +- codex/stores/repostore/store.nim | 10 +- codex/stores/repostore/types.nim | 2 +- codex/stores/treehelper.nim | 6 +- .../blockexchange/discovery/testdiscovery.nim | 4 +- .../discovery/testdiscoveryengine.nim | 2 +- tests/codex/helpers/datasetutils.nim | 5 +- tests/codex/merkletree/generictreetests.nim | 118 ---- tests/codex/merkletree/helpers.nim | 4 +- tests/codex/merkletree/testcodexcoders.nim | 8 +- tests/codex/merkletree/testcodextree.nim | 39 +- tests/codex/slots/helpers.nim | 2 +- tests/codex/stores/commonstoretests.nim | 2 +- tests/codex/stores/testrepostore.nim | 2 +- vendor/nim-merkletree | 1 + 27 files changed, 317 insertions(+), 802 deletions(-) rename codex/merkletree/{codex => }/coders.nim (80%) delete mode 100644 codex/merkletree/codex.nim delete mode 100644 codex/merkletree/codex/codex.nim delete mode 100644 tests/codex/merkletree/generictreetests.nim create mode 160000 vendor/nim-merkletree diff --git a/.gitmodules b/.gitmodules index b8260d32..240f4b5c 100644 --- a/.gitmodules +++ b/.gitmodules @@ -206,4 +206,7 @@ path = vendor/nim-ngtcp2 url = https://github.com/vacp2p/nim-ngtcp2.git ignore = untracked - branch = main \ No newline at end of file + branch = main +[submodule "vendor/nim-merkletree"] + path = vendor/nim-merkletree + url = https://github.com/logos-storage/nim-merkletree diff --git a/codex/blockexchange/engine/engine.nim b/codex/blockexchange/engine/engine.nim index 003843f3..2da068d9 100644 --- a/codex/blockexchange/engine/engine.nim +++ b/codex/blockexchange/engine/engine.nim @@ -848,12 +848,13 @@ proc localLookup( ): Future[?!BlockDelivery] {.async: (raises: [CancelledError]).} = if address.leaf: (await self.localStore.getBlockAndProof(address.treeCid, address.index)).map( - (blkAndProof: (Block, CodexProof)) => + (blkAndProof: (Block, StorageMerkleProof)) => BlockDelivery(address: address, blk: blkAndProof[0], proof: blkAndProof[1].some) ) else: (await self.localStore.getBlock(address)).map( - (blk: Block) => BlockDelivery(address: address, blk: blk, proof: CodexProof.none) + (blk: Block) => + BlockDelivery(address: address, blk: blk, proof: StorageMerkleProof.none) ) iterator splitBatches[T](sequence: seq[T], batchSize: int): seq[T] = diff --git a/codex/blockexchange/protobuf/message.nim b/codex/blockexchange/protobuf/message.nim index 507831f7..645f9b4f 100644 --- a/codex/blockexchange/protobuf/message.nim +++ b/codex/blockexchange/protobuf/message.nim @@ -42,7 +42,7 @@ type BlockDelivery* = object blk*: Block address*: BlockAddress - proof*: ?CodexProof # Present only if `address.leaf` is true + proof*: ?StorageMerkleProof # Present only if `address.leaf` is true BlockPresenceType* = enum Have = 0 @@ -201,12 +201,13 @@ proc decode*(_: type BlockDelivery, pb: ProtoBuffer): ProtoResult[BlockDelivery] if value.address.leaf: var proofBuf = newSeq[byte]() if ?pb.getField(4, proofBuf): - let proof = ?CodexProof.decode(proofBuf).mapErr(x => ProtoError.IncorrectBlob) + let proof = + ?StorageMerkleProof.decode(proofBuf).mapErr(x => ProtoError.IncorrectBlob) value.proof = proof.some else: - value.proof = CodexProof.none + value.proof = StorageMerkleProof.none else: - value.proof = CodexProof.none + value.proof = StorageMerkleProof.none ok(value) diff --git a/codex/merkletree.nim b/codex/merkletree.nim index 2f4a4472..366af992 100644 --- a/codex/merkletree.nim +++ b/codex/merkletree.nim @@ -1,4 +1,4 @@ import ./merkletree/merkletree -import ./merkletree/codex +import ./merkletree/coders -export codex, merkletree +export merkletree, coders diff --git a/codex/merkletree/codex/coders.nim b/codex/merkletree/coders.nim similarity index 80% rename from codex/merkletree/codex/coders.nim rename to codex/merkletree/coders.nim index d81979c1..3210e13d 100644 --- a/codex/merkletree/codex/coders.nim +++ b/codex/merkletree/coders.nim @@ -15,15 +15,14 @@ import pkg/questionable/results import pkg/stew/byteutils import pkg/serde/json -import ../../units -import ../../errors - -import ./codex +import ../units +import ../errors +import ./merkletree const MaxMerkleTreeSize = 100.MiBs.uint const MaxMerkleProofSize = 1.MiBs.uint -proc encode*(self: CodexTree): seq[byte] = +proc encode*(self: StorageMerkleTree): seq[byte] = var pb = initProtoBuffer() pb.write(1, self.mcodec.uint64) pb.write(2, self.leavesCount.uint64) @@ -36,7 +35,7 @@ proc encode*(self: CodexTree): seq[byte] = pb.finish pb.buffer -proc decode*(_: type CodexTree, data: seq[byte]): ?!CodexTree = +proc decode*(_: type StorageMerkleTree, data: seq[byte]): ?!StorageMerkleTree = var pb = initProtoBuffer(data) var mcodecCode: uint64 var leavesCount: uint64 @@ -57,9 +56,9 @@ proc decode*(_: type CodexTree, data: seq[byte]): ?!CodexTree = discard ?initProtoBuffer(nodeBuff).getField(1, node).mapFailure nodes.add node - CodexTree.fromNodes(mcodec, nodes, leavesCount.int) + StorageMerkleTree.fromNodes(mcodec, nodes, leavesCount.int) -proc encode*(self: CodexProof): seq[byte] = +proc encode*(self: StorageMerkleProof): seq[byte] = var pb = initProtoBuffer() pb.write(1, self.mcodec.uint64) pb.write(2, self.index.uint64) @@ -74,7 +73,7 @@ proc encode*(self: CodexProof): seq[byte] = pb.finish pb.buffer -proc decode*(_: type CodexProof, data: seq[byte]): ?!CodexProof = +proc decode*(_: type StorageMerkleProof, data: seq[byte]): ?!StorageMerkleProof = var pb = initProtoBuffer(data) var mcodecCode: uint64 var index: uint64 @@ -99,9 +98,9 @@ proc decode*(_: type CodexProof, data: seq[byte]): ?!CodexProof = discard ?nodePb.getField(1, node).mapFailure nodes.add node - CodexProof.init(mcodec, index.int, nleaves.int, nodes) + StorageMerkleProof.init(mcodec, index.int, nleaves.int, nodes) -proc fromJson*(_: type CodexProof, json: JsonNode): ?!CodexProof = +proc fromJson*(_: type StorageMerkleProof, json: JsonNode): ?!StorageMerkleProof = expectJsonKind(Cid, JString, json) var bytes: seq[byte] try: @@ -109,7 +108,7 @@ proc fromJson*(_: type CodexProof, json: JsonNode): ?!CodexProof = except ValueError as err: return failure(err) - CodexProof.decode(bytes) + StorageMerkleProof.decode(bytes) -func `%`*(proof: CodexProof): JsonNode = +func `%`*(proof: StorageMerkleProof): JsonNode = %byteutils.toHex(proof.encode()) diff --git a/codex/merkletree/codex.nim b/codex/merkletree/codex.nim deleted file mode 100644 index 0e9a5874..00000000 --- a/codex/merkletree/codex.nim +++ /dev/null @@ -1,4 +0,0 @@ -import ./codex/codex -import ./codex/coders - -export codex, coders diff --git a/codex/merkletree/codex/codex.nim b/codex/merkletree/codex/codex.nim deleted file mode 100644 index 173befd0..00000000 --- a/codex/merkletree/codex/codex.nim +++ /dev/null @@ -1,249 +0,0 @@ -## Logos Storage -## Copyright (c) 2023 Status Research & Development GmbH -## Licensed under either of -## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) -## * MIT license ([LICENSE-MIT](LICENSE-MIT)) -## at your option. -## This file may not be copied, modified, or distributed except according to -## those terms. - -{.push raises: [].} - -import std/bitops -import std/[atomics, sequtils] - -import pkg/questionable -import pkg/questionable/results -import pkg/libp2p/[cid, multicodec, multihash] -import pkg/constantine/hashes -import pkg/taskpools -import pkg/chronos/threadsync -import ../../utils -import ../../rng -import ../../errors -import ../../codextypes - -from ../../utils/digest import digestBytes - -import ../merkletree - -export merkletree - -logScope: - topics = "codex merkletree" - -type - ByteTreeKey* {.pure.} = enum - KeyNone = 0x0.byte - KeyBottomLayer = 0x1.byte - KeyOdd = 0x2.byte - KeyOddAndBottomLayer = 0x3.byte - - ByteHash* = seq[byte] - ByteTree* = MerkleTree[ByteHash, ByteTreeKey] - ByteProof* = MerkleProof[ByteHash, ByteTreeKey] - - CodexTree* = ref object of ByteTree - mcodec*: MultiCodec - - CodexProof* = ref object of ByteProof - mcodec*: MultiCodec - -func getProof*(self: CodexTree, index: int): ?!CodexProof = - var proof = CodexProof(mcodec: self.mcodec) - - ?self.getProof(index, proof) - - success proof - -func verify*(self: CodexProof, leaf: MultiHash, root: MultiHash): ?!bool = - ## Verify hash - ## - - let - rootBytes = root.digestBytes - leafBytes = leaf.digestBytes - - if self.mcodec != root.mcodec or self.mcodec != leaf.mcodec: - return failure "Hash codec mismatch" - - if rootBytes.len != root.size and leafBytes.len != leaf.size: - return failure "Invalid hash length" - - self.verify(leafBytes, rootBytes) - -func verify*(self: CodexProof, leaf: Cid, root: Cid): ?!bool = - self.verify(?leaf.mhash.mapFailure, ?leaf.mhash.mapFailure) - -proc rootCid*(self: CodexTree, version = CIDv1, dataCodec = DatasetRootCodec): ?!Cid = - if (?self.root).len == 0: - return failure "Empty root" - - let mhash = ?MultiHash.init(self.mcodec, ?self.root).mapFailure - - Cid.init(version, DatasetRootCodec, mhash).mapFailure - -func getLeafCid*( - self: CodexTree, i: Natural, version = CIDv1, dataCodec = BlockCodec -): ?!Cid = - if i >= self.leavesCount: - return failure "Invalid leaf index " & $i - - let - leaf = self.leaves[i] - mhash = ?MultiHash.init($self.mcodec, leaf).mapFailure - - Cid.init(version, dataCodec, mhash).mapFailure - -proc `$`*(self: CodexTree): string = - let root = - if self.root.isOk: - byteutils.toHex(self.root.get) - else: - "none" - "CodexTree(" & " root: " & root & ", leavesCount: " & $self.leavesCount & ", levels: " & - $self.levels & ", mcodec: " & $self.mcodec & " )" - -proc `$`*(self: CodexProof): string = - "CodexProof(" & " nleaves: " & $self.nleaves & ", index: " & $self.index & ", path: " & - $self.path.mapIt(byteutils.toHex(it)) & ", mcodec: " & $self.mcodec & " )" - -func compress*(x, y: openArray[byte], key: ByteTreeKey, codec: MultiCodec): ?!ByteHash = - ## Compress two hashes - ## - let input = @x & @y & @[key.byte] - let digest = ?MultiHash.digest(codec, input).mapFailure - success digest.digestBytes - -func initTree(mcodec: MultiCodec, leaves: openArray[ByteHash]): ?!CodexTree = - if leaves.len == 0: - return failure "Empty leaves" - - let - compressor = proc(x, y: seq[byte], key: ByteTreeKey): ?!ByteHash {.noSideEffect.} = - compress(x, y, key, mcodec) - digestSize = ?mcodec.digestSize.mapFailure - Zero: ByteHash = newSeq[byte](digestSize) - - if digestSize != leaves[0].len: - return failure "Invalid hash length" - - var self = CodexTree(mcodec: mcodec) - ?self.prepare(compressor, Zero, leaves) - success self - -func init*( - _: type CodexTree, mcodec: MultiCodec = Sha256HashCodec, leaves: openArray[ByteHash] -): ?!CodexTree = - let tree = ?initTree(mcodec, leaves) - ?tree.compute() - success tree - -proc init*( - _: type CodexTree, - tp: Taskpool, - mcodec: MultiCodec = Sha256HashCodec, - leaves: seq[ByteHash], -): Future[?!CodexTree] {.async: (raises: [CancelledError]).} = - let tree = ?initTree(mcodec, leaves) - ?await tree.compute(tp) - success tree - -func init*(_: type CodexTree, leaves: openArray[MultiHash]): ?!CodexTree = - if leaves.len == 0: - return failure "Empty leaves" - - let - mcodec = leaves[0].mcodec - leaves = leaves.mapIt(it.digestBytes) - - CodexTree.init(mcodec, leaves) - -proc init*( - _: type CodexTree, tp: Taskpool, leaves: seq[MultiHash] -): Future[?!CodexTree] {.async: (raises: [CancelledError]).} = - if leaves.len == 0: - return failure "Empty leaves" - - let - mcodec = leaves[0].mcodec - leaves = leaves.mapIt(it.digestBytes) - - await CodexTree.init(tp, mcodec, leaves) - -func init*(_: type CodexTree, leaves: openArray[Cid]): ?!CodexTree = - if leaves.len == 0: - return failure "Empty leaves" - - let - mcodec = (?leaves[0].mhash.mapFailure).mcodec - leaves = leaves.mapIt((?it.mhash.mapFailure).digestBytes) - - CodexTree.init(mcodec, leaves) - -proc init*( - _: type CodexTree, tp: Taskpool, leaves: seq[Cid] -): Future[?!CodexTree] {.async: (raises: [CancelledError]).} = - if leaves.len == 0: - return failure("Empty leaves") - - let - mcodec = (?leaves[0].mhash.mapFailure).mcodec - leaves = leaves.mapIt((?it.mhash.mapFailure).digestBytes) - - await CodexTree.init(tp, mcodec, leaves) - -proc fromNodes*( - _: type CodexTree, - mcodec: MultiCodec = Sha256HashCodec, - nodes: openArray[ByteHash], - nleaves: int, -): ?!CodexTree = - if nodes.len == 0: - return failure "Empty nodes" - - let - digestSize = ?mcodec.digestSize.mapFailure - Zero = newSeq[byte](digestSize) - compressor = proc(x, y: seq[byte], key: ByteTreeKey): ?!ByteHash {.noSideEffect.} = - compress(x, y, key, mcodec) - - if digestSize != nodes[0].len: - return failure "Invalid hash length" - - var self = CodexTree(mcodec: mcodec) - ?self.fromNodes(compressor, Zero, nodes, nleaves) - - let - index = Rng.instance.rand(nleaves - 1) - proof = ?self.getProof(index) - - if not ?proof.verify(self.leaves[index], ?self.root): # sanity check - return failure "Unable to verify tree built from nodes" - - success self - -func init*( - _: type CodexProof, - mcodec: MultiCodec = Sha256HashCodec, - index: int, - nleaves: int, - nodes: openArray[ByteHash], -): ?!CodexProof = - if nodes.len == 0: - return failure "Empty nodes" - - let - digestSize = ?mcodec.digestSize.mapFailure - Zero = newSeq[byte](digestSize) - compressor = proc(x, y: seq[byte], key: ByteTreeKey): ?!seq[byte] {.noSideEffect.} = - compress(x, y, key, mcodec) - - success CodexProof( - compress: compressor, - zero: Zero, - mcodec: mcodec, - index: index, - nleaves: nleaves, - path: @nodes, - ) diff --git a/codex/merkletree/merkletree.nim b/codex/merkletree/merkletree.nim index 56491d84..0b5de28d 100644 --- a/codex/merkletree/merkletree.nim +++ b/codex/merkletree/merkletree.nim @@ -9,388 +9,247 @@ {.push raises: [].} -import std/[bitops, atomics, sequtils] -import stew/assign2 +import std/bitops +import std/[atomics, sequtils] +import pkg/questionable import pkg/questionable/results +import pkg/libp2p/[cid, multicodec, multihash] +import pkg/constantine/hashes import pkg/taskpools -import pkg/chronos import pkg/chronos/threadsync - +import pkg/merkletree +import ../utils +import ../rng import ../errors -import ../utils/sharedbuf +import ../codextypes -export sharedbuf +from ../utils/digest import digestBytes -template nodeData( - data: openArray[byte], offsets: openArray[int], nodeSize, i, j: int -): openArray[byte] = - ## Bytes of the j'th entry of the i'th level in the tree, starting with the - ## leaves (at level 0). - let start = (offsets[i] + j) * nodeSize - data.toOpenArray(start, start + nodeSize - 1) +export merkletree + +logScope: + topics = "codex merkletree" type - # TODO hash functions don't fail - removing the ?! from this function would - # significantly simplify the flow below - CompressFn*[H, K] = proc(x, y: H, key: K): ?!H {.noSideEffect, raises: [].} + ByteTreeKey* {.pure.} = enum + KeyNone = 0x0.byte + KeyBottomLayer = 0x1.byte + KeyOdd = 0x2.byte + KeyOddAndBottomLayer = 0x3.byte - CompressData[H, K] = object - fn: CompressFn[H, K] - nodeSize: int - zero: H + ByteHash* = seq[byte] + ByteTree* = MerkleTree[ByteHash, ByteTreeKey] + ByteProof* = MerkleProof[ByteHash, ByteTreeKey] - MerkleTreeObj*[H, K] = object of RootObj - store*: seq[byte] - ## Flattened merkle tree where hashes are assumed to be trivial bytes and - ## uniform in size. - ## - ## Each layer of the tree is stored serially starting with the leaves and - ## ending with the root. - ## - ## Beacuse the tree might not be balanced, `layerOffsets` contains the - ## index of the starting point of each level, for easy lookup. - layerOffsets*: seq[int] - ## Starting point of each level in the tree, starting from the leaves - - ## multiplied by the entry size, this is the offset in the payload where - ## the entries of that level start - ## - ## For example, a tree with 4 leaves will have [0, 4, 6] stored here. - ## - ## See nodesPerLevel function, from whic this sequence is derived - compress*: CompressData[H, K] + StorageMerkleTree* = ref object of ByteTree + mcodec*: MultiCodec - MerkleTree*[H, K] = ref MerkleTreeObj[H, K] + StorageMerkleProof* = ref object of ByteProof + mcodec*: MultiCodec - MerkleProof*[H, K] = ref object of RootObj - index*: int # linear index of the leaf, starting from 0 - path*: seq[H] # order: from the bottom to the top - nleaves*: int # number of leaves in the tree (=size of input) - compress*: CompressFn[H, K] # compress function - zero*: H # zero value - -func levels*[H, K](self: MerkleTree[H, K]): int = - return self.layerOffsets.len - -func depth*[H, K](self: MerkleTree[H, K]): int = - return self.levels() - 1 - -func nodesInLayer(offsets: openArray[int], layer: int): int = - if layer == offsets.high: - 1 - else: - offsets[layer + 1] - offsets[layer] - -func nodesInLayer(self: MerkleTree | MerkleTreeObj, layer: int): int = - self.layerOffsets.nodesInLayer(layer) - -func leavesCount*[H, K](self: MerkleTree[H, K]): int = - return self.nodesInLayer(0) - -func nodesPerLevel(nleaves: int): seq[int] = - ## Given a number of leaves, return a seq with the number of nodes at each - ## layer of the tree (from the bottom/leaves to the root) - ## - ## Ie For a tree of 4 leaves, return `[4, 2, 1]` - if nleaves <= 0: - return @[] - elif nleaves == 1: - return @[1, 1] # leaf and root - - var nodes: seq[int] = @[] - var m = nleaves - while true: - nodes.add(m) - if m == 1: - break - # Next layer size is ceil(m/2) - m = (m + 1) shr 1 - - nodes - -func layerOffsets(nleaves: int): seq[int] = - ## Given a number of leaves, return a seq of the starting offsets of each - ## layer in the node store that results from flattening the binary tree - ## - ## Ie For a tree of 4 leaves, return `[0, 4, 6]` - let nodes = nodesPerLevel(nleaves) - var tot = 0 - let offsets = nodes.mapIt: - let cur = tot - tot += it - cur - offsets - -template nodeData(self: MerkleTreeObj, i, j: int): openArray[byte] = - ## Bytes of the j'th node of the i'th level in the tree, starting with the - ## leaves (at level 0). - self.store.nodeData(self.layerOffsets, self.compress.nodeSize, i, j) - -func layer*[H, K]( - self: MerkleTree[H, K], layer: int -): seq[H] {.deprecated: "Expensive".} = - var nodes = newSeq[H](self.nodesInLayer(layer)) - for i, h in nodes.mpairs: - assign(h, self[].nodeData(layer, i)) - return nodes - -func leaves*[H, K](self: MerkleTree[H, K]): seq[H] {.deprecated: "Expensive".} = - self.layer(0) - -iterator layers*[H, K](self: MerkleTree[H, K]): seq[H] {.deprecated: "Expensive".} = - for i in 0 ..< self.layerOffsets.len: - yield self.layer(i) - -proc layers*[H, K](self: MerkleTree[H, K]): seq[seq[H]] {.deprecated: "Expensive".} = - for l in self.layers(): - result.add l - -iterator nodes*[H, K](self: MerkleTree[H, K]): H = - ## Iterate over the nodes of each layer starting with the leaves - var node: H - for i in 0 ..< self.layerOffsets.len: - let nodesInLayer = self.nodesInLayer(i) - for j in 0 ..< nodesInLayer: - assign(node, self[].nodeData(i, j)) - yield node - -func root*[H, K](self: MerkleTree[H, K]): ?!H = - mixin assign - if self.layerOffsets.len == 0: - return failure "invalid tree" - - var h: H - assign(h, self[].nodeData(self.layerOffsets.high(), 0)) - return success h - -func getProof*[H, K]( - self: MerkleTree[H, K], index: int, proof: MerkleProof[H, K] -): ?!void = - let depth = self.depth - let nleaves = self.leavesCount - - if not (index >= 0 and index < nleaves): - return failure "index out of bounds" - - var path: seq[H] = newSeq[H](depth) - var k = index - var m = nleaves - for i in 0 ..< depth: - let j = k xor 1 - - if (j < m): - assign(path[i], self[].nodeData(i, j)) - else: - path[i] = self.compress.zero - - k = k shr 1 - m = (m + 1) shr 1 - - proof.index = index - proof.path = path - proof.nleaves = nleaves - proof.compress = self.compress.fn - - success() - -func getProof*[H, K](self: MerkleTree[H, K], index: int): ?!MerkleProof[H, K] = - var proof = MerkleProof[H, K]() +func getProof*(self: StorageMerkleTree, index: int): ?!StorageMerkleProof = + var proof = StorageMerkleProof(mcodec: self.mcodec) ?self.getProof(index, proof) success proof -func reconstructRoot*[H, K](proof: MerkleProof[H, K], leaf: H): ?!H = - var - m = proof.nleaves - j = proof.index - h = leaf - bottomFlag = K.KeyBottomLayer +func verify*(self: StorageMerkleProof, leaf: MultiHash, root: MultiHash): ?!bool = + ## Verify hash + ## - for p in proof.path: - let oddIndex: bool = (bitand(j, 1) != 0) - if oddIndex: - # the index of the child is odd, so the node itself can't be odd (a bit counterintuitive, yeah :) - h = ?proof.compress(p, h, bottomFlag) + let + rootBytes = root.digestBytes + leafBytes = leaf.digestBytes + + if self.mcodec != root.mcodec or self.mcodec != leaf.mcodec: + return failure "Hash codec mismatch" + + if rootBytes.len != root.size and leafBytes.len != leaf.size: + return failure "Invalid hash length" + + self.verify(leafBytes, rootBytes) + +func verify*(self: StorageMerkleProof, leaf: Cid, root: Cid): ?!bool = + self.verify(?leaf.mhash.mapFailure, ?leaf.mhash.mapFailure) + +proc rootCid*( + self: StorageMerkleTree, version = CIDv1, dataCodec = DatasetRootCodec +): ?!Cid = + if (?self.root).len == 0: + return failure "Empty root" + + let mhash = ?MultiHash.init(self.mcodec, ?self.root).mapFailure + + Cid.init(version, DatasetRootCodec, mhash).mapFailure + +func getLeafCid*( + self: StorageMerkleTree, i: Natural, version = CIDv1, dataCodec = BlockCodec +): ?!Cid = + if i >= self.leavesCount: + return failure "Invalid leaf index " & $i + + let + leaf = self.leaves[i] + mhash = ?MultiHash.init($self.mcodec, leaf).mapFailure + + Cid.init(version, dataCodec, mhash).mapFailure + +proc `$`*(self: StorageMerkleTree): string = + let root = + if self.root.isOk: + byteutils.toHex(self.root.get) else: - if j == m - 1: - # single child => odd node - h = ?proof.compress(h, p, K(bottomFlag.ord + 2)) - else: - # even node - h = ?proof.compress(h, p, bottomFlag) - bottomFlag = K.KeyNone - j = j shr 1 - m = (m + 1) shr 1 + "none" + "StorageMerkleTree(" & " root: " & root & ", leavesCount: " & $self.leavesCount & + ", levels: " & $self.levels & ", mcodec: " & $self.mcodec & " )" - return success h +proc `$`*(self: StorageMerkleProof): string = + "StorageMerkleProof(" & " nleaves: " & $self.nleaves & ", index: " & $self.index & + ", path: " & $self.path.mapIt(byteutils.toHex(it)) & ", mcodec: " & $self.mcodec & + " )" -func verify*[H, K](proof: MerkleProof[H, K], leaf: H, root: H): ?!bool = - success bool(root == ?proof.reconstructRoot(leaf)) - -func fromNodes*[H, K]( - self: MerkleTree[H, K], - compressor: CompressFn, - zero: H, - nodes: openArray[H], - nleaves: int, -): ?!void = - mixin assign - - if nodes.len < 2: # At least leaf and root - return failure "Not enough nodes" - - if nleaves == 0: - return failure "No leaves" - - self.compress = CompressData[H, K](fn: compressor, nodeSize: nodes[0].len, zero: zero) - self.layerOffsets = layerOffsets(nleaves) - - if self.layerOffsets[^1] + 1 != nodes.len: - return failure "bad node count" - - self.store = newSeqUninit[byte](nodes.len * self.compress.nodeSize) - - for i in 0 ..< nodes.len: - assign( - self[].store.toOpenArray( - i * self.compress.nodeSize, (i + 1) * self.compress.nodeSize - 1 - ), - nodes[i], - ) - - success() - -func merkleTreeWorker[H, K]( - store: var openArray[byte], - offsets: openArray[int], - compress: CompressData[H, K], - layer: int, - isBottomLayer: static bool, -): ?!void = - ## Worker used to compute the merkle tree from the leaves that are assumed to - ## already be stored at the beginning of the `store`, as done by `prepare`. - - # Throughout, we use `assign` to convert from H to bytes and back, assuming - # this assignment can be done somewhat efficiently (ie memcpy) - because - # the code must work with multihash where len(H) is can differ, we cannot - # simply use a fixed-size array here. - mixin assign - - template nodeData(i, j: int): openArray[byte] = - # Pick out the bytes of node j in layer i - store.nodeData(offsets, compress.nodeSize, i, j) - - let m = offsets.nodesInLayer(layer) - - when not isBottomLayer: - if m == 1: - return success() - - let halfn: int = m div 2 - let n: int = 2 * halfn - let isOdd: bool = (n != m) - - # Because the compression function we work with works with H and not bytes, - # we need to extract H from the raw data - a little abstraction tax that - # ensures that properties like alignment of H are respected. - var a, b, tmp: H - - for i in 0 ..< halfn: - const key = when isBottomLayer: K.KeyBottomLayer else: K.KeyNone - - assign(a, nodeData(layer, i * 2)) - assign(b, nodeData(layer, i * 2 + 1)) - - tmp = ?compress.fn(a, b, key = key) - - assign(nodeData(layer + 1, i), tmp) - - if isOdd: - const key = when isBottomLayer: K.KeyOddAndBottomLayer else: K.KeyOdd - - assign(a, nodeData(layer, n)) - - tmp = ?compress.fn(a, compress.zero, key = key) - - assign(nodeData(layer + 1, halfn), tmp) - - merkleTreeWorker(store, offsets, compress, layer + 1, false) - -proc merkleTreeWorker[H, K]( - store: SharedBuf[byte], - offsets: SharedBuf[int], - compress: ptr CompressData[H, K], - signal: ThreadSignalPtr, -): bool = - defer: - discard signal.fireSync() - - let res = merkleTreeWorker( - store.toOpenArray(), offsets.toOpenArray(), compress[], 0, isBottomLayer = true - ) - - return res.isOk() - -func prepare*[H, K]( - self: MerkleTree[H, K], compressor: CompressFn, zero: H, leaves: openArray[H] -): ?!void = - ## Prepare the instance for computing the merkle tree of the given leaves using - ## the given compression function. After preparation, `compute` should be - ## called to perform the actual computation. `leaves` will be copied into the - ## tree so they can be freed after the call. +func compress*(x, y: openArray[byte], key: ByteTreeKey, codec: MultiCodec): ?!ByteHash = + ## Compress two hashes + ## + let input = @x & @y & @[key.byte] + let digest = ?MultiHash.digest(codec, input).mapFailure + success digest.digestBytes +func initTree(mcodec: MultiCodec, leaves: openArray[ByteHash]): ?!StorageMerkleTree = if leaves.len == 0: - return failure "No leaves" + return failure "Empty leaves" - self.compress = - CompressData[H, K](fn: compressor, nodeSize: leaves[0].len, zero: zero) - self.layerOffsets = layerOffsets(leaves.len) + let + compressor = proc(x, y: seq[byte], key: ByteTreeKey): ?!ByteHash {.noSideEffect.} = + compress(x, y, key, mcodec) + digestSize = ?mcodec.digestSize.mapFailure + Zero: ByteHash = newSeq[byte](digestSize) - self.store = newSeqUninit[byte]((self.layerOffsets[^1] + 1) * self.compress.nodeSize) + if digestSize != leaves[0].len: + return failure "Invalid hash length" - for j in 0 ..< leaves.len: - assign(self[].nodeData(0, j), leaves[j]) + var self = StorageMerkleTree(mcodec: mcodec) + ?self.prepare(compressor, Zero, leaves) + success self - return success() +func init*( + _: type StorageMerkleTree, + mcodec: MultiCodec = Sha256HashCodec, + leaves: openArray[ByteHash], +): ?!StorageMerkleTree = + let tree = ?initTree(mcodec, leaves) + ?tree.compute() + success tree -proc compute*[H, K](self: MerkleTree[H, K]): ?!void = - merkleTreeWorker( - self.store, self.layerOffsets, self.compress, 0, isBottomLayer = true +proc init*( + _: type StorageMerkleTree, + tp: Taskpool, + mcodec: MultiCodec = Sha256HashCodec, + leaves: seq[ByteHash], +): Future[?!StorageMerkleTree] {.async: (raises: [CancelledError]).} = + let tree = ?initTree(mcodec, leaves) + ?await tree.compute(tp) + success tree + +func init*( + _: type StorageMerkleTree, leaves: openArray[MultiHash] +): ?!StorageMerkleTree = + if leaves.len == 0: + return failure "Empty leaves" + + let + mcodec = leaves[0].mcodec + leaves = leaves.mapIt(it.digestBytes) + + StorageMerkleTree.init(mcodec, leaves) + +proc init*( + _: type StorageMerkleTree, tp: Taskpool, leaves: seq[MultiHash] +): Future[?!StorageMerkleTree] {.async: (raises: [CancelledError]).} = + if leaves.len == 0: + return failure "Empty leaves" + + let + mcodec = leaves[0].mcodec + leaves = leaves.mapIt(it.digestBytes) + + await StorageMerkleTree.init(tp, mcodec, leaves) + +func init*(_: type StorageMerkleTree, leaves: openArray[Cid]): ?!StorageMerkleTree = + if leaves.len == 0: + return failure "Empty leaves" + + let + mcodec = (?leaves[0].mhash.mapFailure).mcodec + leaves = leaves.mapIt((?it.mhash.mapFailure).digestBytes) + + StorageMerkleTree.init(mcodec, leaves) + +proc init*( + _: type StorageMerkleTree, tp: Taskpool, leaves: seq[Cid] +): Future[?!StorageMerkleTree] {.async: (raises: [CancelledError]).} = + if leaves.len == 0: + return failure("Empty leaves") + + let + mcodec = (?leaves[0].mhash.mapFailure).mcodec + leaves = leaves.mapIt((?it.mhash.mapFailure).digestBytes) + + await StorageMerkleTree.init(tp, mcodec, leaves) + +proc fromNodes*( + _: type StorageMerkleTree, + mcodec: MultiCodec = Sha256HashCodec, + nodes: openArray[ByteHash], + nleaves: int, +): ?!StorageMerkleTree = + if nodes.len == 0: + return failure "Empty nodes" + + let + digestSize = ?mcodec.digestSize.mapFailure + Zero = newSeq[byte](digestSize) + compressor = proc(x, y: seq[byte], key: ByteTreeKey): ?!ByteHash {.noSideEffect.} = + compress(x, y, key, mcodec) + + if digestSize != nodes[0].len: + return failure "Invalid hash length" + + var self = StorageMerkleTree(mcodec: mcodec) + ?self.fromNodes(compressor, Zero, nodes, nleaves) + + let + index = Rng.instance.rand(nleaves - 1) + proof = ?self.getProof(index) + + if not ?proof.verify(self.leaves[index], ?self.root): # sanity check + return failure "Unable to verify tree built from nodes" + + success self + +func init*( + _: type StorageMerkleProof, + mcodec: MultiCodec = Sha256HashCodec, + index: int, + nleaves: int, + nodes: openArray[ByteHash], +): ?!StorageMerkleProof = + if nodes.len == 0: + return failure "Empty nodes" + + let + digestSize = ?mcodec.digestSize.mapFailure + Zero = newSeq[byte](digestSize) + compressor = proc(x, y: seq[byte], key: ByteTreeKey): ?!seq[byte] {.noSideEffect.} = + compress(x, y, key, mcodec) + + success StorageMerkleProof( + compress: compressor, + zero: Zero, + mcodec: mcodec, + index: index, + nleaves: nleaves, + path: @nodes, ) - -proc compute*[H, K]( - self: MerkleTree[H, K], tp: Taskpool -): Future[?!void] {.async: (raises: []).} = - if tp.numThreads == 1: - # With a single thread, there's no point creating a separate task - return self.compute() - - # TODO this signal would benefit from reuse across computations - without signal =? ThreadSignalPtr.new(): - return failure("Unable to create thread signal") - - defer: - signal.close().expect("closing once works") - - let res = tp.spawn merkleTreeWorker( - SharedBuf.view(self.store), - SharedBuf.view(self.layerOffsets), - addr self.compress, - signal, - ) - - # To support cancellation, we'd have to ensure the task we posted to taskpools - # exits early - since we're not doing that, block cancellation attempts - try: - await noCancel signal.wait() - except AsyncError as exc: - # Since we initialized the signal, the OS or chronos is misbehaving. In any - # case, it would mean the task is still running which would cause a memory - # a memory violation if we let it run - panic instead - raiseAssert "Could not wait for signal, was it initialized? " & exc.msg - - if not res.sync(): - return failure("merkle tree task failed") - - return success() diff --git a/codex/node.nim b/codex/node.nim index 260a6313..1e2e0b80 100644 --- a/codex/node.nim +++ b/codex/node.nim @@ -445,7 +445,7 @@ proc store*( finally: await stream.close() - without tree =? (await CodexTree.init(self.taskPool, cids)), err: + without tree =? (await StorageMerkleTree.init(self.taskPool, cids)), err: return failure(err) without treeCid =? tree.rootCid(CIDv1, dataCodec), err: diff --git a/codex/stores/blockstore.nim b/codex/stores/blockstore.nim index 732d29e4..7edabc4d 100644 --- a/codex/stores/blockstore.nim +++ b/codex/stores/blockstore.nim @@ -80,7 +80,9 @@ method getBlocks*( method getBlockAndProof*( self: BlockStore, treeCid: Cid, index: Natural -): Future[?!(Block, CodexProof)] {.base, async: (raises: [CancelledError]), gcsafe.} = +): Future[?!(Block, StorageMerkleProof)] {. + base, async: (raises: [CancelledError]), gcsafe +.} = ## Get a block and associated inclusion proof by Cid of a merkle tree and an index of a leaf in a tree ## @@ -95,7 +97,11 @@ method putBlock*( raiseAssert("putBlock not implemented!") method putCidAndProof*( - self: BlockStore, treeCid: Cid, index: Natural, blockCid: Cid, proof: CodexProof + self: BlockStore, + treeCid: Cid, + index: Natural, + blockCid: Cid, + proof: StorageMerkleProof, ): Future[?!void] {.base, async: (raises: [CancelledError]), gcsafe.} = ## Put a block proof to the blockstore ## @@ -104,7 +110,9 @@ method putCidAndProof*( method getCidAndProof*( self: BlockStore, treeCid: Cid, index: Natural -): Future[?!(Cid, CodexProof)] {.base, async: (raises: [CancelledError]), gcsafe.} = +): Future[?!(Cid, StorageMerkleProof)] {. + base, async: (raises: [CancelledError]), gcsafe +.} = ## Get a block proof from the blockstore ## diff --git a/codex/stores/cachestore.nim b/codex/stores/cachestore.nim index cb5da7a3..98fa3deb 100644 --- a/codex/stores/cachestore.nim +++ b/codex/stores/cachestore.nim @@ -37,7 +37,7 @@ type currentSize*: NBytes size*: NBytes cache: LruCache[Cid, Block] - cidAndProofCache: LruCache[(Cid, Natural), (Cid, CodexProof)] + cidAndProofCache: LruCache[(Cid, Natural), (Cid, StorageMerkleProof)] InvalidBlockSize* = object of CodexError @@ -83,7 +83,7 @@ method getBlocks*( method getCidAndProof*( self: CacheStore, treeCid: Cid, index: Natural -): Future[?!(Cid, CodexProof)] {.async: (raises: [CancelledError]).} = +): Future[?!(Cid, StorageMerkleProof)] {.async: (raises: [CancelledError]).} = if cidAndProof =? self.cidAndProofCache.getOption((treeCid, index)): success(cidAndProof) else: @@ -103,7 +103,7 @@ method getBlock*( method getBlockAndProof*( self: CacheStore, treeCid: Cid, index: Natural -): Future[?!(Block, CodexProof)] {.async: (raises: [CancelledError]).} = +): Future[?!(Block, StorageMerkleProof)] {.async: (raises: [CancelledError]).} = without cidAndProof =? (await self.getCidAndProof(treeCid, index)), err: return failure(err) @@ -226,7 +226,11 @@ method putBlock*( return success() method putCidAndProof*( - self: CacheStore, treeCid: Cid, index: Natural, blockCid: Cid, proof: CodexProof + self: CacheStore, + treeCid: Cid, + index: Natural, + blockCid: Cid, + proof: StorageMerkleProof, ): Future[?!void] {.async: (raises: [CancelledError]).} = self.cidAndProofCache[(treeCid, index)] = (blockCid, proof) success() @@ -301,7 +305,7 @@ proc new*( currentSize = 0'nb size = int(cacheSize div chunkSize) cache = newLruCache[Cid, Block](size) - cidAndProofCache = newLruCache[(Cid, Natural), (Cid, CodexProof)](size) + cidAndProofCache = newLruCache[(Cid, Natural), (Cid, StorageMerkleProof)](size) store = CacheStore( cache: cache, cidAndProofCache: cidAndProofCache, diff --git a/codex/stores/networkstore.nim b/codex/stores/networkstore.nim index 200024f9..6aa76d8c 100644 --- a/codex/stores/networkstore.nim +++ b/codex/stores/networkstore.nim @@ -104,13 +104,17 @@ method putBlock*( return success() method putCidAndProof*( - self: NetworkStore, treeCid: Cid, index: Natural, blockCid: Cid, proof: CodexProof + self: NetworkStore, + treeCid: Cid, + index: Natural, + blockCid: Cid, + proof: StorageMerkleProof, ): Future[?!void] {.async: (raw: true, raises: [CancelledError]).} = self.localStore.putCidAndProof(treeCid, index, blockCid, proof) method getCidAndProof*( self: NetworkStore, treeCid: Cid, index: Natural -): Future[?!(Cid, CodexProof)] {.async: (raw: true, raises: [CancelledError]).} = +): Future[?!(Cid, StorageMerkleProof)] {.async: (raw: true, raises: [CancelledError]).} = ## Get a block proof from the blockstore ## diff --git a/codex/stores/repostore/operations.nim b/codex/stores/repostore/operations.nim index e27322ea..efd7f56f 100644 --- a/codex/stores/repostore/operations.nim +++ b/codex/stores/repostore/operations.nim @@ -33,7 +33,11 @@ declareGauge(codex_repostore_bytes_used, "codex repostore bytes used") declareGauge(codex_repostore_bytes_reserved, "codex repostore bytes reserved") proc putLeafMetadata*( - self: RepoStore, treeCid: Cid, index: Natural, blkCid: Cid, proof: CodexProof + self: RepoStore, + treeCid: Cid, + index: Natural, + blkCid: Cid, + proof: StorageMerkleProof, ): Future[?!StoreResultKind] {.async: (raises: [CancelledError]).} = without key =? createBlockCidAndProofMetadataKey(treeCid, index), err: return failure(err) diff --git a/codex/stores/repostore/store.nim b/codex/stores/repostore/store.nim index aab1a080..554bd62c 100644 --- a/codex/stores/repostore/store.nim +++ b/codex/stores/repostore/store.nim @@ -82,7 +82,7 @@ method getBlock*( method getBlockAndProof*( self: RepoStore, treeCid: Cid, index: Natural -): Future[?!(Block, CodexProof)] {.async: (raises: [CancelledError]).} = +): Future[?!(Block, StorageMerkleProof)] {.async: (raises: [CancelledError]).} = without leafMd =? await self.getLeafMetadata(treeCid, index), err: return failure(err) @@ -136,7 +136,11 @@ method ensureExpiry*( await self.ensureExpiry(leafMd.blkCid, expiry) method putCidAndProof*( - self: RepoStore, treeCid: Cid, index: Natural, blkCid: Cid, proof: CodexProof + self: RepoStore, + treeCid: Cid, + index: Natural, + blkCid: Cid, + proof: StorageMerkleProof, ): Future[?!void] {.async: (raises: [CancelledError]).} = ## Put a block to the blockstore ## @@ -163,7 +167,7 @@ method putCidAndProof*( method getCidAndProof*( self: RepoStore, treeCid: Cid, index: Natural -): Future[?!(Cid, CodexProof)] {.async: (raises: [CancelledError]).} = +): Future[?!(Cid, StorageMerkleProof)] {.async: (raises: [CancelledError]).} = without leafMd =? await self.getLeafMetadata(treeCid, index), err: return failure(err) diff --git a/codex/stores/repostore/types.nim b/codex/stores/repostore/types.nim index bd43dded..6936b92d 100644 --- a/codex/stores/repostore/types.nim +++ b/codex/stores/repostore/types.nim @@ -49,7 +49,7 @@ type LeafMetadata* {.serialize.} = object blkCid*: Cid - proof*: CodexProof + proof*: StorageMerkleProof BlockExpiration* {.serialize.} = object cid*: Cid diff --git a/codex/stores/treehelper.nim b/codex/stores/treehelper.nim index 9b0eb5d9..dc89a2ff 100644 --- a/codex/stores/treehelper.nim +++ b/codex/stores/treehelper.nim @@ -21,7 +21,7 @@ import ../utils/asynciter import ../merkletree proc putSomeProofs*( - store: BlockStore, tree: CodexTree, iter: Iter[int] + store: BlockStore, tree: StorageMerkleTree, iter: Iter[int] ): Future[?!void] {.async.} = without treeCid =? tree.rootCid, err: return failure(err) @@ -47,9 +47,9 @@ proc putSomeProofs*( success() proc putSomeProofs*( - store: BlockStore, tree: CodexTree, iter: Iter[Natural] + store: BlockStore, tree: StorageMerkleTree, iter: Iter[Natural] ): Future[?!void] = store.putSomeProofs(tree, iter.map((i: Natural) => i.ord)) -proc putAllProofs*(store: BlockStore, tree: CodexTree): Future[?!void] = +proc putAllProofs*(store: BlockStore, tree: StorageMerkleTree): Future[?!void] = store.putSomeProofs(tree, Iter[int].new(0 ..< tree.leavesCount)) diff --git a/tests/codex/blockexchange/discovery/testdiscovery.nim b/tests/codex/blockexchange/discovery/testdiscovery.nim index b828d57e..b8aec57b 100644 --- a/tests/codex/blockexchange/discovery/testdiscovery.nim +++ b/tests/codex/blockexchange/discovery/testdiscovery.nim @@ -25,7 +25,7 @@ asyncchecksuite "Block Advertising and Discovery": var blocks: seq[bt.Block] manifest: Manifest - tree: CodexTree + tree: StorageMerkleTree manifestBlock: bt.Block switch: Switch peerStore: PeerCtxStore @@ -158,7 +158,7 @@ asyncchecksuite "E2E - Multiple Nodes Discovery": blockexc: seq[NetworkStore] manifests: seq[Manifest] mBlocks: seq[bt.Block] - trees: seq[CodexTree] + trees: seq[StorageMerkleTree] setup: for _ in 0 ..< 4: diff --git a/tests/codex/blockexchange/discovery/testdiscoveryengine.nim b/tests/codex/blockexchange/discovery/testdiscoveryengine.nim index b7282a92..978fe690 100644 --- a/tests/codex/blockexchange/discovery/testdiscoveryengine.nim +++ b/tests/codex/blockexchange/discovery/testdiscoveryengine.nim @@ -27,7 +27,7 @@ asyncchecksuite "Test Discovery Engine": var blocks: seq[bt.Block] manifest: Manifest - tree: CodexTree + tree: StorageMerkleTree manifestBlock: bt.Block switch: Switch peerStore: PeerCtxStore diff --git a/tests/codex/helpers/datasetutils.nim b/tests/codex/helpers/datasetutils.nim index 56f26e34..d4aedc41 100644 --- a/tests/codex/helpers/datasetutils.nim +++ b/tests/codex/helpers/datasetutils.nim @@ -8,7 +8,8 @@ import pkg/codex/rng import ./randomchunker -type TestDataset* = tuple[blocks: seq[Block], tree: CodexTree, manifest: Manifest] +type TestDataset* = + tuple[blocks: seq[Block], tree: StorageMerkleTree, manifest: Manifest] proc makeRandomBlock*(size: NBytes): Block = let bytes = newSeqWith(size.int, rand(uint8)) @@ -34,7 +35,7 @@ proc makeDataset*(blocks: seq[Block]): ?!TestDataset = let datasetSize = blocks.mapIt(it.data.len).foldl(a + b) blockSize = blocks.mapIt(it.data.len).foldl(max(a, b)) - tree = ?CodexTree.init(blocks.mapIt(it.cid)) + tree = ?StorageMerkleTree.init(blocks.mapIt(it.cid)) treeCid = ?tree.rootCid manifest = Manifest.new( treeCid = treeCid, diff --git a/tests/codex/merkletree/generictreetests.nim b/tests/codex/merkletree/generictreetests.nim deleted file mode 100644 index e24cbad1..00000000 --- a/tests/codex/merkletree/generictreetests.nim +++ /dev/null @@ -1,118 +0,0 @@ -import pkg/unittest2 - -import pkg/codex/merkletree - -proc testGenericTree*[H, K, U]( - name: string, - data: openArray[H], - zero: H, - compress: proc(z, y: H, key: K): H, - makeTree: proc(data: seq[H]): U, -) = - let data = @data - - suite "Correctness tests - " & name: - test "Should build correct tree for single leaf": - let expectedRoot = compress(data[0], zero, K.KeyOddAndBottomLayer) - - let tree = makeTree(data[0 .. 0]) - check: - tree.root.tryGet == expectedRoot - - test "Should build correct tree for even bottom layer": - let expectedRoot = compress( - compress( - compress(data[0], data[1], K.KeyBottomLayer), - compress(data[2], data[3], K.KeyBottomLayer), - K.KeyNone, - ), - compress( - compress(data[4], data[5], K.KeyBottomLayer), - compress(data[6], data[7], K.KeyBottomLayer), - K.KeyNone, - ), - K.KeyNone, - ) - - let tree = makeTree(data[0 .. 7]) - - check: - tree.root.tryGet == expectedRoot - - test "Should build correct tree for odd bottom layer": - let expectedRoot = compress( - compress( - compress(data[0], data[1], K.KeyBottomLayer), - compress(data[2], data[3], K.KeyBottomLayer), - K.KeyNone, - ), - compress( - compress(data[4], data[5], K.KeyBottomLayer), - compress(data[6], zero, K.KeyOddAndBottomLayer), - K.KeyNone, - ), - K.KeyNone, - ) - - let tree = makeTree(data[0 .. 6]) - - check: - tree.root.tryGet == expectedRoot - - test "Should build correct tree for even bottom and odd upper layers": - let expectedRoot = compress( - compress( - compress( - compress(data[0], data[1], K.KeyBottomLayer), - compress(data[2], data[3], K.KeyBottomLayer), - K.KeyNone, - ), - compress( - compress(data[4], data[5], K.KeyBottomLayer), - compress(data[6], data[7], K.KeyBottomLayer), - K.KeyNone, - ), - K.KeyNone, - ), - compress( - compress(compress(data[8], data[9], K.KeyBottomLayer), zero, K.KeyOdd), - zero, - K.KeyOdd, - ), - K.KeyNone, - ) - - let tree = makeTree(data[0 .. 9]) - - check: - tree.root.tryGet == expectedRoot - - test "Should get and validate correct proofs": - let expectedRoot = compress( - compress( - compress( - compress(data[0], data[1], K.KeyBottomLayer), - compress(data[2], data[3], K.KeyBottomLayer), - K.KeyNone, - ), - compress( - compress(data[4], data[5], K.KeyBottomLayer), - compress(data[6], data[7], K.KeyBottomLayer), - K.KeyNone, - ), - K.KeyNone, - ), - compress( - compress(compress(data[8], data[9], K.KeyBottomLayer), zero, K.KeyOdd), - zero, - K.KeyOdd, - ), - K.KeyNone, - ) - - let tree = makeTree(data) - - for i in 0 ..< data.len: - let proof = tree.getProof(i).tryGet - check: - proof.verify(tree.leaves[i], expectedRoot).isOk diff --git a/tests/codex/merkletree/helpers.nim b/tests/codex/merkletree/helpers.nim index e322ac83..c2361b9d 100644 --- a/tests/codex/merkletree/helpers.nim +++ b/tests/codex/merkletree/helpers.nim @@ -3,9 +3,9 @@ import ../helpers export merkletree, helpers -proc `==`*(a, b: CodexTree): bool = +proc `==`*(a, b: StorageMerkleTree): bool = (a.mcodec == b.mcodec) and (a.leavesCount == b.leavesCount) and (a.levels == b.levels) -proc `==`*(a, b: CodexProof): bool = +proc `==`*(a, b: StorageMerkleProof): bool = (a.mcodec == b.mcodec) and (a.nleaves == b.nleaves) and (a.path == b.path) and (a.index == b.index) diff --git a/tests/codex/merkletree/testcodexcoders.nim b/tests/codex/merkletree/testcodexcoders.nim index 6da56844..eddafc9e 100644 --- a/tests/codex/merkletree/testcodexcoders.nim +++ b/tests/codex/merkletree/testcodexcoders.nim @@ -21,16 +21,16 @@ const data = [ suite "merkletree - coders": test "encoding and decoding a tree yields the same tree": let - tree = CodexTree.init(Sha256HashCodec, data).tryGet() + tree = StorageMerkleTree.init(Sha256HashCodec, data).tryGet() encodedBytes = tree.encode() - decodedTree = CodexTree.decode(encodedBytes).tryGet() + decodedTree = StorageMerkleTree.decode(encodedBytes).tryGet() check: tree == decodedTree test "encoding and decoding a proof yields the same proof": let - tree = CodexTree.init(Sha256HashCodec, data).tryGet() + tree = StorageMerkleTree.init(Sha256HashCodec, data).tryGet() proof = tree.getProof(4).tryGet() check: @@ -38,7 +38,7 @@ suite "merkletree - coders": let encodedBytes = proof.encode() - decodedProof = CodexProof.decode(encodedBytes).tryGet() + decodedProof = StorageMerkleProof.decode(encodedBytes).tryGet() check: proof == decodedProof diff --git a/tests/codex/merkletree/testcodextree.nim b/tests/codex/merkletree/testcodextree.nim index 16765dbb..15ec6ce9 100644 --- a/tests/codex/merkletree/testcodextree.nim +++ b/tests/codex/merkletree/testcodextree.nim @@ -12,7 +12,6 @@ import pkg/codex/utils/digest import pkg/taskpools import ./helpers -import ./generictreetests import ../../asynctest # TODO: Generalize to other hashes @@ -32,23 +31,23 @@ const ] sha256 = Sha256HashCodec -suite "Test CodexTree": +suite "Test StorageMerkleTree": test "Cannot init tree without any multihash leaves": check: - CodexTree.init(leaves = newSeq[MultiHash]()).isErr + StorageMerkleTree.init(leaves = newSeq[MultiHash]()).isErr test "Cannot init tree without any cid leaves": check: - CodexTree.init(leaves = newSeq[Cid]()).isErr + StorageMerkleTree.init(leaves = newSeq[Cid]()).isErr test "Cannot init tree without any byte leaves": check: - CodexTree.init(sha256, leaves = newSeq[ByteHash]()).isErr + StorageMerkleTree.init(sha256, leaves = newSeq[ByteHash]()).isErr test "Should build tree from multihash leaves": var expectedLeaves = data.mapIt(MultiHash.digest($sha256, it).tryGet()) - tree = CodexTree.init(leaves = expectedLeaves) + tree = StorageMerkleTree.init(leaves = expectedLeaves) check: tree.isOk @@ -62,7 +61,7 @@ suite "Test CodexTree": let expectedLeaves = data.mapIt(MultiHash.digest($sha256, it).tryGet()) - let tree = (await CodexTree.init(tp, leaves = expectedLeaves)) + let tree = (await StorageMerkleTree.init(tp, leaves = expectedLeaves)) check: tree.isOk tree.get().leaves == expectedLeaves.mapIt(it.digestBytes) @@ -73,7 +72,7 @@ suite "Test CodexTree": Cid.init(CidVersion.CIDv1, BlockCodec, MultiHash.digest($sha256, it).tryGet).tryGet ) - let tree = CodexTree.init(leaves = expectedLeaves) + let tree = StorageMerkleTree.init(leaves = expectedLeaves) check: tree.isOk @@ -89,7 +88,7 @@ suite "Test CodexTree": Cid.init(CidVersion.CIDv1, BlockCodec, MultiHash.digest($sha256, it).tryGet).tryGet ) - let tree = (await CodexTree.init(tp, leaves = expectedLeaves)) + let tree = (await StorageMerkleTree.init(tp, leaves = expectedLeaves)) check: tree.isOk @@ -106,8 +105,8 @@ suite "Test CodexTree": ) let - atree = (await CodexTree.init(tp, leaves = expectedLeaves)) - stree = CodexTree.init(leaves = expectedLeaves) + atree = (await StorageMerkleTree.init(tp, leaves = expectedLeaves)) + stree = StorageMerkleTree.init(leaves = expectedLeaves) check: toSeq(atree.get().nodes) == toSeq(stree.get().nodes) @@ -115,15 +114,15 @@ suite "Test CodexTree": # Single-leaf trees have their root separately computed let - atree1 = (await CodexTree.init(tp, leaves = expectedLeaves[0 .. 0])) - stree1 = CodexTree.init(leaves = expectedLeaves[0 .. 0]) + atree1 = (await StorageMerkleTree.init(tp, leaves = expectedLeaves[0 .. 0])) + stree1 = StorageMerkleTree.init(leaves = expectedLeaves[0 .. 0]) check: toSeq(atree.get().nodes) == toSeq(stree.get().nodes) atree.get().root == stree.get().root test "Should build from raw digestbytes (should not hash leaves)": - let tree = CodexTree.init(sha256, leaves = data).tryGet + let tree = StorageMerkleTree.init(sha256, leaves = data).tryGet check: tree.mcodec == sha256 @@ -134,7 +133,7 @@ suite "Test CodexTree": defer: tp.shutdown() - let tree = (await CodexTree.init(tp, sha256, leaves = @data)) + let tree = (await StorageMerkleTree.init(tp, sha256, leaves = @data)) check: tree.isOk @@ -143,8 +142,8 @@ suite "Test CodexTree": test "Should build from nodes": let - tree = CodexTree.init(sha256, leaves = data).tryGet - fromNodes = CodexTree.fromNodes( + tree = StorageMerkleTree.init(sha256, leaves = data).tryGet + fromNodes = StorageMerkleTree.fromNodes( nodes = toSeq(tree.nodes), nleaves = tree.leavesCount ).tryGet @@ -158,7 +157,5 @@ let compress = proc(x, y: seq[byte], key: ByteTreeKey): seq[byte] = compress(x, y, key, sha256).tryGet - makeTree = proc(data: seq[seq[byte]]): CodexTree = - CodexTree.init(sha256, leaves = data).tryGet - -testGenericTree("CodexTree", @data, zero, compress, makeTree) + makeTree = proc(data: seq[seq[byte]]): StorageMerkleTree = + StorageMerkleTree.init(sha256, leaves = data).tryGet diff --git a/tests/codex/slots/helpers.nim b/tests/codex/slots/helpers.nim index 52d9078b..17c4d401 100644 --- a/tests/codex/slots/helpers.nim +++ b/tests/codex/slots/helpers.nim @@ -46,7 +46,7 @@ proc makeManifest*( hcodec = Sha256HashCodec, dataCodec = BlockCodec, ): Future[?!Manifest] {.async.} = - without tree =? CodexTree.init(cids), err: + without tree =? StorageMerkleTree.init(cids), err: return failure(err) without treeCid =? tree.rootCid(CIDv1, dataCodec), err: diff --git a/tests/codex/stores/commonstoretests.nim b/tests/codex/stores/commonstoretests.nim index d3132773..03627f87 100644 --- a/tests/codex/stores/commonstoretests.nim +++ b/tests/codex/stores/commonstoretests.nim @@ -29,7 +29,7 @@ proc commonBlockStoreTests*( var newBlock, newBlock1, newBlock2, newBlock3: Block manifest: Manifest - tree: CodexTree + tree: StorageMerkleTree store: BlockStore setup: diff --git a/tests/codex/stores/testrepostore.nim b/tests/codex/stores/testrepostore.nim index 7eb9fd0d..b524c860 100644 --- a/tests/codex/stores/testrepostore.nim +++ b/tests/codex/stores/testrepostore.nim @@ -16,7 +16,7 @@ import pkg/codex/stores/repostore/operations import pkg/codex/blocktype as bt import pkg/codex/clock import pkg/codex/utils/safeasynciter -import pkg/codex/merkletree/codex +import pkg/codex/merkletree import ../../asynctest import ../helpers diff --git a/vendor/nim-merkletree b/vendor/nim-merkletree new file mode 160000 index 00000000..685c40cc --- /dev/null +++ b/vendor/nim-merkletree @@ -0,0 +1 @@ +Subproject commit 685c40cc5ec7d66112e8cf6be4eed59f619c51de