Rework merkle tree (#654)

* rework merkle tree support

* deps

* rename merkletree -> codexmerkletree

* treed and proof encoding/decoding

* small change to invoke proof verification

* rename merkletree to codexmerkletree

* style

* adding codex merkle and coders tests

* fixup imports

* remove new codecs for now

* bump deps

* adding trace statement

* properly serde of manifest block codecs

* use default hash codec

* add more trace logging to aid debugging

* misc

* remove double import

* revert un-needded change

* proof size changed

* bump poseidon2

* add from nodes test

* shorte file names

* remove upraises

* wip poseidon tree

* adjust file names

* misc

* shorten file names

* fix bad `elements` iter

* don't do asserts

* add fromNodes and converters

* root and getProof now return result

* add poseidon2 tree tests

* root now returns result

* misc

* had to make merkletree a ref, because nim blows up otherwise

* root returns a result

* root returns a result

* import poseidon tests

* bump

* merkle poseidon2 digest

* misc

* add merkle digest tests

* bump

* don't use checksuite

* Update tests/codex/merkletree/generictreetests.nim

Co-authored-by: markspanbroek <mark@spanbroek.net>
Signed-off-by: Dmitriy Ryajov <dryajov@gmail.com>

* Update codex/merkletree/merkletree.nim

Co-authored-by: markspanbroek <mark@spanbroek.net>
Signed-off-by: Dmitriy Ryajov <dryajov@gmail.com>

* Update codex/merkletree/merkletree.nim

Co-authored-by: markspanbroek <mark@spanbroek.net>
Signed-off-by: Dmitriy Ryajov <dryajov@gmail.com>

* Update tests/codex/merkletree/generictreetests.nim

Co-authored-by: markspanbroek <mark@spanbroek.net>
Signed-off-by: Dmitriy Ryajov <dryajov@gmail.com>

* missing return

* make toBool private (it's still needed otherwise comparison won't work)

* added `digestTree` that returns a tree and `digest` for root

* test against both poseidon trees - codex and poseidon2

* shorten merkle tree names

* don't compare trees - it's going to be too slow

* move comparison to mekrle helper

* remove merkle utils

---------

Signed-off-by: Dmitriy Ryajov <dryajov@gmail.com>
Co-authored-by: markspanbroek <mark@spanbroek.net>

codex/blockexchange/engine/engine.nim

@@ -328,11 +328,9 @@ proc validateBlockDelivery(
     without treeRoot =? bd.address.treeCid.mhash.mapFailure, err:
       return failure("Unable to get mhash from treeCid for block, nested err: " & err.msg)
 
-    without verifyOutcome =? proof.verifyLeaf(leaf, treeRoot), err:
+    if err =? proof.verify(leaf, treeRoot).errorOption:
       return failure("Unable to verify proof for block, nested err: " & err.msg)
 
-    if not verifyOutcome:
-      return failure("Provided inclusion proof is invalid")
   else: # not leaf
     if bd.address.cid != bd.blk.cid:
       return failure("Delivery cid " & $bd.address.cid & " doesn't match block cid " & $bd.blk.cid)
@@ -537,12 +535,12 @@ proc taskHandler*(b: BlockExcEngine, task: BlockExcPeerCtx) {.gcsafe, async.} =
       trace "Handling lookup for entry", address = e.address
       if e.address.leaf:
         (await b.localStore.getBlockAndProof(e.address.treeCid, e.address.index)).map(
-          (blkAndProof: (Block, MerkleProof)) =>
+          (blkAndProof: (Block, CodexProof)) =>
             BlockDelivery(address: e.address, blk: blkAndProof[0], proof: blkAndProof[1].some)
         )
       else:
         (await b.localStore.getBlock(e.address)).map(
-          (blk: Block) => BlockDelivery(address: e.address, blk: blk, proof: MerkleProof.none)
+          (blk: Block) => BlockDelivery(address: e.address, blk: blk, proof: CodexProof.none)
         )
 
     let

codex/blockexchange/engine/pendingblocks.nim

@@ -18,11 +18,9 @@ import pkg/chronicles
 import pkg/chronos
 import pkg/libp2p
 import pkg/metrics
-import pkg/questionable/results
 
 import ../protobuf/blockexc
 import ../../blocktype
-import ../../merkletree
 
 logScope:
   topics = "codex pendingblocks"

codex/blockexchange/peers/peercontext.nim

@@ -9,7 +9,6 @@
 
 import std/sequtils
 import std/tables
-import std/sugar
 import std/sets
 
 import pkg/chronicles

codex/blockexchange/protobuf/blockexc.nim

@@ -9,7 +9,6 @@
 
 import std/hashes
 import std/sequtils
-import pkg/libp2p
 import pkg/stew/endians2
 
 import message

codex/blockexchange/protobuf/message.nim

@@ -37,7 +37,7 @@ type
   BlockDelivery* = object
     blk*: Block
     address*: BlockAddress
-    proof*: ?MerkleProof    # Present only if `address.leaf` is true
+    proof*: ?CodexProof    # Present only if `address.leaf` is true
 
   BlockPresenceType* = enum
     Have = 0,
@@ -219,12 +219,12 @@ proc decode*(_: type BlockDelivery, pb: ProtoBuffer): ProtoResult[BlockDelivery]
   if value.address.leaf:
     var proofBuf = newSeq[byte]()
     if ? pb.getField(4, proofBuf):
-      let proof = ? MerkleProof.decode(proofBuf).mapErr(x => ProtoError.IncorrectBlob)
+      let proof = ? CodexProof.decode(proofBuf).mapErr(x => ProtoError.IncorrectBlob)
       value.proof = proof.some
     else:
-      value.proof = MerkleProof.none
+      value.proof = CodexProof.none
   else:
-    value.proof = MerkleProof.none
+    value.proof = CodexProof.none
 
   ok(value)
 

codex/blocktype.nim

@@ -34,6 +34,14 @@ const
   # should be divisible by 31 for PoR and by 64 for Leopard ECC
   DefaultBlockSize* = NBytes 31 * 64 * 33
 
+  # hashes
+  Sha256Hash* = multiCodec("sha2-256")
+
+  # CIDs
+  Raw = multiCodec("raw")
+  DagPB* = multiCodec("dag-pb")
+  DagJson* = multiCodec("dag-json")
+
 type
   Block* = ref object of RootObj
     cid*: Cid

codex/conf.nim

@@ -40,7 +40,12 @@ import ./units
 import ./utils
 
 export units
-export net, DefaultQuotaBytes, DefaultBlockTtl, DefaultBlockMaintenanceInterval, DefaultNumberOfBlocksToMaintainPerInterval
+export net
+export
+  DefaultQuotaBytes,
+  DefaultBlockTtl,
+  DefaultBlockMaintenanceInterval,
+  DefaultNumberOfBlocksToMaintainPerInterval
 
 const
   codex_enable_api_debug_peers* {.booldefine.} = false

codex/erasure/erasure.nim

@@ -97,7 +97,9 @@ proc getPendingBlocks(
   var
     # request blocks from the store
     pendingBlocks = indicies.map( (i: int) =>
-      self.store.getBlock(BlockAddress.init(manifest.treeCid, i)).map((r: ?!bt.Block) => (r, i)) # Get the data blocks (first K)
+      self.store.getBlock(
+        BlockAddress.init(manifest.treeCid, i)
+      ).map((r: ?!bt.Block) => (r, i)) # Get the data blocks (first K)
     )
 
   proc isFinished(): bool = pendingBlocks.len == 0
@@ -291,7 +293,7 @@ proc encodeData(
           return failure("Unable to store block!")
         idx.inc(params.steps)
 
-    without tree =? MerkleTree.init(cids[]), err:
+    without tree =? CodexTree.init(cids[]), err:
       return failure(err)
 
     without treeCid =? tree.rootCid, err:
@@ -308,6 +310,7 @@ proc encodeData(
       ecM = params.ecM
     )
 
+    trace "Encoded data successfully", treeCid, blocksCount = params.blocksCount
     return encodedManifest.success
   except CancelledError as exc:
     trace "Erasure coding encoding cancelled"
@@ -415,7 +418,7 @@ proc decode*(
   finally:
     decoder.release()
 
-  without tree =? MerkleTree.init(cids[0..<encoded.originalBlocksCount]), err:
+  without tree =? CodexTree.init(cids[0..<encoded.originalBlocksCount]), err:
     return failure(err)
 
   without treeCid =? tree.rootCid, err:

codex/manifest/coders.nim

@@ -54,7 +54,10 @@ proc encode*(_: DagPBCoder, manifest: Manifest): ?!seq[byte] =
   #     optional bytes treeCid = 1;       # cid (root) of the tree
   #     optional uint32 blockSize = 2;    # size of a single block
   #     optional uint64 datasetSize = 3;  # size of the dataset
-  #     optional ErasureInfo erasure = 4; # erasure coding info
+  #     optional codec: MultiCodec = 4;   # Dataset codec
+  #     optional hcodec: MultiCodec = 5   # Multihash codec
+  #     optional version: CidVersion = 6; # Cid version
+  #     optional ErasureInfo erasure = 7; # erasure coding info
   #   }
   # ```
   #
@@ -63,6 +66,9 @@ proc encode*(_: DagPBCoder, manifest: Manifest): ?!seq[byte] =
   header.write(1, manifest.treeCid.data.buffer)
   header.write(2, manifest.blockSize.uint32)
   header.write(3, manifest.datasetSize.uint32)
+  header.write(4, manifest.codec.uint32)
+  header.write(5, manifest.hcodec.uint32)
+  header.write(6, manifest.version.uint32)
   if manifest.protected:
     var erasureInfo = initProtoBuffer()
     erasureInfo.write(1, manifest.ecK.uint32)
@@ -78,7 +84,7 @@ proc encode*(_: DagPBCoder, manifest: Manifest): ?!seq[byte] =
       erasureInfo.write(5, verificationInfo)
 
     erasureInfo.finish()
-    header.write(4, erasureInfo)
+    header.write(7, erasureInfo)
 
   pbNode.write(1, header) # set the treeCid as the data field
   pbNode.finish()
@@ -97,6 +103,9 @@ proc decode*(_: DagPBCoder, data: openArray[byte]): ?!Manifest =
     treeCidBuf: seq[byte]
     originalTreeCid: seq[byte]
     datasetSize: uint32
+    codec: uint32
+    hcodec: uint32
+    version: uint32
     blockSize: uint32
     originalDatasetSize: uint32
     ecK, ecM: uint32
@@ -117,7 +126,16 @@ proc decode*(_: DagPBCoder, data: openArray[byte]): ?!Manifest =
   if pbHeader.getField(3, datasetSize).isErr:
     return failure("Unable to decode `datasetSize` from manifest!")
 
-  if pbHeader.getField(4, pbErasureInfo).isErr:
+  if pbHeader.getField(4, codec).isErr:
+    return failure("Unable to decode `codec` from manifest!")
+
+  if pbHeader.getField(5, hcodec).isErr:
+    return failure("Unable to decode `hcodec` from manifest!")
+
+  if pbHeader.getField(6, version).isErr:
+    return failure("Unable to decode `version` from manifest!")
+
+  if pbHeader.getField(7, pbErasureInfo).isErr:
     return failure("Unable to decode `erasureInfo` from manifest!")
 
   let protected = pbErasureInfo.buffer.len > 0
@@ -155,23 +173,21 @@ proc decode*(_: DagPBCoder, data: openArray[byte]): ?!Manifest =
         treeCid = treeCid,
         datasetSize = datasetSize.NBytes,
         blockSize = blockSize.NBytes,
-        version = treeCid.cidver,
+        version = CidVersion(version),
-        hcodec = (? treeCid.mhash.mapFailure).mcodec,
+        hcodec = hcodec.MultiCodec,
-        codec = treeCid.mcodec,
+        codec = codec.MultiCodec,
         ecK = ecK.int,
         ecM = ecM.int,
         originalTreeCid = ? Cid.init(originalTreeCid).mapFailure,
-        originalDatasetSize = originalDatasetSize.NBytes
+        originalDatasetSize = originalDatasetSize.NBytes)
-      )
       else:
         Manifest.new(
           treeCid = treeCid,
           datasetSize = datasetSize.NBytes,
           blockSize = blockSize.NBytes,
-          version = treeCid.cidver,
+          version = CidVersion(version),
-          hcodec = (? treeCid.mhash.mapFailure).mcodec,
+          hcodec = hcodec.MultiCodec,
-          codec = treeCid.mcodec
+          codec = codec.MultiCodec)
-        )
 
   ? self.verify()
 

codex/manifest/manifest.nim

@@ -33,9 +33,9 @@ type
     treeCid {.serialize.}: Cid              # Root of the merkle tree
     datasetSize {.serialize.}:  NBytes      # Total size of all blocks
     blockSize {.serialize.}: NBytes         # Size of each contained block (might not be needed if blocks are len-prefixed)
-    version: CidVersion                     # Cid version
-    hcodec: MultiCodec                      # Multihash codec
     codec: MultiCodec                       # Dataset codec
+    hcodec: MultiCodec                      # Multihash codec
+    version: CidVersion                     # Cid version
     case protected {.serialize.}: bool      # Protected datasets have erasure coded info
     of true:
       ecK: int                              # Number of blocks to encode
@@ -201,8 +201,7 @@ proc new*(
   version: CidVersion = CIDv1,
   hcodec = multiCodec("sha2-256"),
   codec = multiCodec("raw"),
-    protected = false,
+  protected = false): Manifest =
-): Manifest =
 
   T(
     treeCid: treeCid,
@@ -218,11 +217,11 @@ proc new*(
   manifest: Manifest,
   treeCid: Cid,
   datasetSize: NBytes,
-    ecK, ecM: int
+  ecK, ecM: int): Manifest =
-): Manifest =
   ## Create an erasure protected dataset from an
   ## unprotected one
   ##
+
   Manifest(
     treeCid: treeCid,
     datasetSize: datasetSize,
@@ -237,8 +236,7 @@ proc new*(
 
 proc new*(
   T: type Manifest,
-    manifest: Manifest
+  manifest: Manifest): Manifest =
-): Manifest =
   ## Create an unprotected dataset from an
   ## erasure protected one
   ##
@@ -254,10 +252,10 @@ proc new*(
 proc new*(
   T: type Manifest,
   data: openArray[byte],
-  decoder = ManifestContainers[$DagPBCodec]
+  decoder = ManifestContainers[$DagPBCodec]): ?!Manifest =
-): ?!Manifest =
   ## Create a manifest instance from given data
   ##
+
   Manifest.decode(data, decoder)
 
 proc new*(
@@ -271,8 +269,8 @@ proc new*(
   ecK: int,
   ecM: int,
   originalTreeCid: Cid,
-  originalDatasetSize: NBytes
+  originalDatasetSize: NBytes): Manifest =
-): Manifest =
+
   Manifest(
     treeCid: treeCid,
     datasetSize: datasetSize,
@@ -291,8 +289,7 @@ proc new*(
   T: type Manifest,
   manifest: Manifest,
   verificationRoot: Cid,
-    slotRoots: seq[Cid]
+  slotRoots: seq[Cid]): ?!Manifest =
-): ?!Manifest =
   ## Create a verifiable dataset from an
   ## protected one
   ##
@@ -313,5 +310,4 @@ proc new*(
     originalDatasetSize: manifest.originalDatasetSize,
     verifiable: true,
     verificationRoot: verificationRoot,
-    slotRoots: slotRoots
+    slotRoots: slotRoots))
-  ))

codex/manifest/types.nim

@@ -10,14 +10,12 @@
 # This module defines Manifest and all related types
 
 import std/tables
-import std/strutils
 import pkg/libp2p
 
 import ../units
 export units
 
 const
-  BlockCodec* = multiCodec("raw")
   DagPBCodec* = multiCodec("dag-pb")
 
 type

codex/merkletree.nim

@@ -1,4 +1,5 @@
 import ./merkletree/merkletree
-import ./merkletree/coders
+import ./merkletree/codex
+import ./merkletree/poseidon2
 
-export merkletree, coders
+export codex, poseidon2, merkletree

codex/merkletree/coders.nim

@@ -1,75 +0,0 @@
-## Nim-Codex
-## Copyright (c) 2023 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
-
-import pkg/libp2p
-import pkg/questionable
-import pkg/questionable/results
-
-import ./merkletree
-import ../units
-import ../errors
-
-const MaxMerkleTreeSize = 100.MiBs.uint
-const MaxMerkleProofSize = 1.MiBs.uint
-
-proc encode*(self: MerkleTree): seq[byte] =
-  var pb = initProtoBuffer(maxSize = MaxMerkleTreeSize)
-  pb.write(1, self.mcodec.uint64)
-  pb.write(2, self.digestSize.uint64)
-  pb.write(3, self.leavesCount.uint64)
-  pb.write(4, self.nodesBuffer)
-  pb.finish
-  pb.buffer
-
-proc decode*(_: type MerkleTree, data: seq[byte]): ?!MerkleTree =
-  var pb = initProtoBuffer(data, maxSize = MaxMerkleTreeSize)
-  var mcodecCode: uint64
-  var digestSize: uint64
-  var leavesCount: uint64
-  discard ? pb.getField(1, mcodecCode).mapFailure
-  discard ? pb.getField(2, digestSize).mapFailure
-  discard ? pb.getField(3, leavesCount).mapFailure
-
-  let mcodec = MultiCodec.codec(cast[int](mcodecCode))
-  if mcodec == InvalidMultiCodec:
-    return failure("Invalid MultiCodec code " & $cast[int](mcodec))
-
-  var nodesBuffer = newSeq[byte]()
-  discard ? pb.getField(4, nodesBuffer).mapFailure
-
-  let tree = ? MerkleTree.init(mcodec, digestSize, leavesCount, nodesBuffer)
-  success(tree)
-
-proc encode*(self: MerkleProof): seq[byte] =
-  var pb = initProtoBuffer(maxSize = MaxMerkleProofSize)
-  pb.write(1, self.mcodec.uint64)
-  pb.write(2, self.digestSize.uint64)
-  pb.write(3, self.index.uint64)
-  pb.write(4, self.nodesBuffer)
-  pb.finish
-  pb.buffer
-
-proc decode*(_: type MerkleProof, data: seq[byte]): ?!MerkleProof =
-  var pb = initProtoBuffer(data, maxSize = MaxMerkleProofSize)
-  var mcodecCode: uint64
-  var digestSize: uint64
-  var index: uint64
-  discard ? pb.getField(1, mcodecCode).mapFailure
-  discard ? pb.getField(2, digestSize).mapFailure
-  discard ? pb.getField(3, index).mapFailure
-
-  let mcodec = MultiCodec.codec(cast[int](mcodecCode))
-  if mcodec == InvalidMultiCodec:
-    return failure("Invalid MultiCodec code " & $cast[int](mcodec))
-
-  var nodesBuffer = newSeq[byte]()
-  discard ? pb.getField(4, nodesBuffer).mapFailure
-
-  let proof = ? MerkleProof.init(mcodec, digestSize, index, nodesBuffer)
-  success(proof)

codex/merkletree/codex.nim

@@ -0,0 +1,4 @@
+import ./codex/codex
+import ./codex/coders
+
+export codex, coders

codex/merkletree/codex/coders.nim

@@ -0,0 +1,102 @@
+## Nim-Codex
+## Copyright (c) 2023 Status Research & Development GmbH
+## Licensed under either of
+##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+## at your option.
+## This file may not be copied, modified, or distributed except according to
+## those terms.
+
+import pkg/upraises
+
+push: {.upraises: [].}
+
+import pkg/libp2p
+import pkg/questionable
+import pkg/questionable/results
+
+import ../../units
+import ../../errors
+
+import ./codex
+
+const MaxMerkleTreeSize = 100.MiBs.uint
+const MaxMerkleProofSize = 1.MiBs.uint
+
+proc encode*(self: CodexTree): seq[byte] =
+  var pb = initProtoBuffer(maxSize = MaxMerkleTreeSize)
+  pb.write(1, self.mcodec.uint64)
+  pb.write(2, self.leavesCount.uint64)
+  for node in self.nodes:
+    var nodesPb = initProtoBuffer(maxSize = MaxMerkleTreeSize)
+    nodesPb.write(1, node)
+    nodesPb.finish()
+    pb.write(3, nodesPb)
+
+  pb.finish
+  pb.buffer
+
+proc decode*(_: type CodexTree, data: seq[byte]): ?!CodexTree =
+  var pb = initProtoBuffer(data, maxSize = MaxMerkleTreeSize)
+  var mcodecCode: uint64
+  var leavesCount: uint64
+  discard ? pb.getField(1, mcodecCode).mapFailure
+  discard ? pb.getField(2, leavesCount).mapFailure
+
+  let mcodec = MultiCodec.codec(mcodecCode.int)
+  if mcodec == InvalidMultiCodec:
+    return failure("Invalid MultiCodec code " & $mcodecCode)
+
+  var
+    nodesBuff: seq[seq[byte]]
+    nodes: seq[ByteHash]
+
+  if ? pb.getRepeatedField(3, nodesBuff).mapFailure:
+    for nodeBuff in nodesBuff:
+      var node: ByteHash
+      discard ? initProtoBuffer(nodeBuff).getField(1, node).mapFailure
+      nodes.add node
+
+  CodexTree.fromNodes(mcodec, nodes, leavesCount.int)
+
+proc encode*(self: CodexProof): seq[byte] =
+  var pb = initProtoBuffer(maxSize = MaxMerkleProofSize)
+  pb.write(1, self.mcodec.uint64)
+  pb.write(2, self.index.uint64)
+  pb.write(3, self.nleaves.uint64)
+
+  for node in self.path:
+    var nodesPb = initProtoBuffer(maxSize = MaxMerkleTreeSize)
+    nodesPb.write(1, node)
+    nodesPb.finish()
+    pb.write(4, nodesPb)
+
+  pb.finish
+  pb.buffer
+
+proc decode*(_: type CodexProof, data: seq[byte]): ?!CodexProof =
+  var pb = initProtoBuffer(data, maxSize = MaxMerkleProofSize)
+  var mcodecCode: uint64
+  var index: uint64
+  var nleaves: uint64
+  discard ? pb.getField(1, mcodecCode).mapFailure
+
+  let mcodec = MultiCodec.codec(mcodecCode.int)
+  if mcodec == InvalidMultiCodec:
+    return failure("Invalid MultiCodec code " & $mcodecCode)
+
+  discard ? pb.getField(2, index).mapFailure
+  discard ? pb.getField(3, nleaves).mapFailure
+
+  var
+    nodesBuff: seq[seq[byte]]
+    nodes: seq[ByteHash]
+
+  if ? pb.getRepeatedField(4, nodesBuff).mapFailure:
+    for nodeBuff in nodesBuff:
+      var node: ByteHash
+      let nodePb = initProtoBuffer(nodeBuff)
+      discard ? nodePb.getField(1, node).mapFailure
+      nodes.add node
+
+  CodexProof.init(mcodec, index.int, nleaves.int, nodes)

codex/merkletree/codex/codex.nim

@@ -0,0 +1,270 @@
+## Nim-Codex
+## Copyright (c) 2023 Status Research & Development GmbH
+## Licensed under either of
+##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+## at your option.
+## This file may not be copied, modified, or distributed except according to
+## those terms.
+
+import pkg/upraises
+
+push: {.upraises: [].}
+
+import std/bitops
+import std/sequtils
+
+import pkg/questionable
+import pkg/questionable/results
+import pkg/libp2p/[cid, multicodec, multihash]
+import pkg/stew/byteutils
+
+import ../../utils
+import ../../rng
+import ../../errors
+import ../../blocktype
+
+import ../merkletree
+
+export merkletree
+
+logScope:
+  topics = "codex merkletree"
+
+const
+  DatasetRootCodec* = multiCodec("codex-root") # TODO: move to blocktype
+  BlockCodec* = multiCodec("raw") # TODO: fix multicodec to `codex-block` and move to blocktype
+
+type
+  ByteTreeKey* {.pure.} = enum
+    KeyNone               = 0x0.byte
+    KeyBottomLayer        = 0x1.byte
+    KeyOdd                = 0x2.byte
+    KeyOddAndBottomLayer  = 0x3.byte
+
+  ByteHash* = seq[byte]
+  ByteTree* = MerkleTree[ByteHash, ByteTreeKey]
+  ByteTreeProof* = MerkleProof[ByteHash, ByteTreeKey]
+
+  CodexTree* = ref object of ByteTree
+    mhash: MHash
+
+  CodexProof* = ref object of ByteTreeProof
+    mhash: MHash
+
+func getMhash*(mcodec: MultiCodec): ?!MHash =
+  let
+    mhash = CodeHashes.getOrDefault(mcodec)
+
+  if isNil(mhash.coder):
+    return failure "Invalid multihash codec"
+
+  success mhash
+
+func digestSize*(self: (CodexTree or CodexProof)): int =
+  ## Number of leaves
+  ##
+
+  self.mhash.size
+
+func mcodec*(self: (CodexTree or CodexProof)): MultiCodec =
+  ## Multicodec
+  ##
+
+  self.mhash.mcodec
+
+func bytes*(mhash: MultiHash): seq[byte] =
+  ## Extract hash bytes
+  ##
+
+  mhash.data.buffer[mhash.dpos..<mhash.dpos + mhash.size]
+
+func getProof*(self: CodexTree, index: int): ?!CodexProof =
+  var
+    proof = CodexProof(mhash: self.mhash)
+
+  ? self.getProof(index, proof)
+
+  success proof
+
+func verify*(self: CodexProof, leaf: MultiHash, root: MultiHash): ?!void =
+  ## Verify hash
+  ##
+
+  let
+    rootBytes = root.bytes
+    leafBytes = leaf.bytes
+
+  if self.mcodec != root.mcodec or
+    self.mcodec != leaf.mcodec:
+    return failure "Hash codec mismatch"
+
+  if rootBytes.len != root.size and
+    leafBytes.len != leaf.size:
+    return failure "Invalid hash length"
+
+  ? self.verify(leafBytes, rootBytes)
+
+  success()
+
+func verify*(self: CodexProof, leaf: Cid, root: Cid): ?!void =
+  self.verify(? leaf.mhash.mapFailure, ? leaf.mhash.mapFailure)
+
+proc rootCid*(
+  self: CodexTree,
+  version = CIDv1,
+  dataCodec = DatasetRootCodec): ?!Cid =
+
+  if (? self.root).len == 0:
+    return failure "Empty root"
+
+  let
+    mhash = ? MultiHash.init(self.mcodec, ? self.root).mapFailure
+
+  Cid.init(version, DatasetRootCodec, mhash).mapFailure
+
+func getLeafCid*(
+  self: CodexTree,
+  i: Natural,
+  version = CIDv1,
+  dataCodec = BlockCodec): ?!Cid =
+
+  if i >= self.leavesCount:
+    return failure "Invalid leaf index " & $i
+
+  let
+    leaf = self.leaves[i]
+    mhash = ? MultiHash.init($self.mcodec, leaf).mapFailure
+
+  Cid.init(version, dataCodec, mhash).mapFailure
+
+proc `$`*(self: CodexTree): string =
+  "CodexTree( mcodec: " &
+    $self.mcodec &
+    ", leavesCount: " &
+    $self.leavesCount & " )"
+
+proc `$`*(self: CodexProof): string =
+  "CodexProof( mcodec: " &
+    $self.mcodec & ", nleaves: " &
+    $self.nleaves & ", index: " &
+    $self.index & " )"
+
+func compress*(
+  x, y: openArray[byte],
+  key: ByteTreeKey,
+  mhash: MHash): ?!ByteHash =
+  ## Compress two hashes
+  ##
+
+  var digest = newSeq[byte](mhash.size)
+  mhash.coder(@x & @y & @[ key.byte ], digest)
+  success digest
+
+func init*(
+  _: type CodexTree,
+  mcodec: MultiCodec = multiCodec("sha2-256"),
+  leaves: openArray[ByteHash]): ?!CodexTree =
+
+  if leaves.len == 0:
+    return failure "Empty leaves"
+
+  let
+    mhash = ? mcodec.getMhash()
+    compressor = proc(x, y: seq[byte], key: ByteTreeKey): ?!ByteHash {.noSideEffect.} =
+      compress(x, y, key, mhash)
+    Zero: ByteHash = newSeq[byte](mhash.size)
+
+  if mhash.size != leaves[0].len:
+    return failure "Invalid hash length"
+
+  var
+    self = CodexTree(mhash: mhash, compress: compressor, zero: Zero)
+
+  self.layers = ? merkleTreeWorker(self, leaves, isBottomLayer = true)
+  success self
+
+func init*(
+  _: type CodexTree,
+  leaves: openArray[MultiHash]): ?!CodexTree =
+
+  if leaves.len == 0:
+    return failure "Empty leaves"
+
+  let
+    mcodec = leaves[0].mcodec
+    leaves = leaves.mapIt( it.bytes )
+
+  CodexTree.init(mcodec, leaves)
+
+func init*(
+  _: type CodexTree,
+  leaves: openArray[Cid]): ?!CodexTree =
+  if leaves.len == 0:
+    return failure "Empty leaves"
+
+  let
+    mcodec = (? leaves[0].mhash.mapFailure).mcodec
+    leaves = leaves.mapIt( (? it.mhash.mapFailure).bytes )
+
+  CodexTree.init(mcodec, leaves)
+
+proc fromNodes*(
+  _: type CodexTree,
+  mcodec: MultiCodec = multiCodec("sha2-256"),
+  nodes: openArray[ByteHash],
+  nleaves: int): ?!CodexTree =
+
+  if nodes.len == 0:
+    return failure "Empty nodes"
+
+  let
+    mhash = ? mcodec.getMhash()
+    Zero = newSeq[byte](mhash.size)
+    compressor = proc(x, y: seq[byte], key: ByteTreeKey): ?!ByteHash {.noSideEffect.} =
+      compress(x, y, key, mhash)
+
+  if mhash.size != nodes[0].len:
+    return failure "Invalid hash length"
+
+  var
+    self = CodexTree(compress: compressor, zero: Zero, mhash: mhash)
+    layer = nleaves
+    pos = 0
+
+  while pos < nodes.len:
+    self.layers.add( nodes[pos..<(pos + layer)] )
+    pos += layer
+    layer = divUp(layer, 2)
+
+  let
+    index = Rng.instance.rand(nleaves - 1)
+    proof = ? self.getProof(index)
+
+  ? proof.verify(self.leaves[index], ? self.root) # sanity check
+  success self
+
+func init*(
+  _: type CodexProof,
+  mcodec: MultiCodec = multiCodec("sha2-256"),
+  index: int,
+  nleaves: int,
+  nodes: openArray[ByteHash]): ?!CodexProof =
+
+  if nodes.len == 0:
+    return failure "Empty nodes"
+
+  let
+    mhash = ? mcodec.getMhash()
+    Zero = newSeq[byte](mhash.size)
+    compressor = proc(x, y: seq[byte], key: ByteTreeKey): ?!seq[byte] {.noSideEffect.} =
+      compress(x, y, key, mhash)
+
+
+  success CodexProof(
+    compress: compressor,
+    zero: Zero,
+    mhash: mhash,
+    index: index,
+    nleaves: nleaves,
+    path: @nodes)

codex/merkletree/merkletree.nim

@@ -7,412 +7,157 @@
 ## This file may not be copied, modified, or distributed except according to
 ## those terms.
 
-import std/math
+{.push raises: [].}
-import std/bitops
+
-import std/sequtils
+import std/bitops
-import std/sugar
-import std/algorithm
 
-import pkg/chronicles
-import pkg/questionable
 import pkg/questionable/results
-import pkg/nimcrypto/sha2
-import pkg/libp2p/[cid, multicodec, multihash, vbuffer]
-import pkg/stew/byteutils
 
 import ../errors
 
-logScope:
-  topics = "codex merkletree"
-
 type
-  MerkleTree* = object
+  PutFn*[H] = proc(i: Natural, x: H): ?!void {.noSideEffect, raises: [].}
-    mcodec: MultiCodec
+  GetFn*[H] = proc(i: Natural): ?!H {.noSideEffect, raises: [].}
-    digestSize: Natural
-    leavesCount: Natural
-    nodesBuffer*: seq[byte]
-  MerkleProof* = object
-    mcodec: MultiCodec
-    digestSize: Natural
-    index: Natural
-    nodesBuffer*: seq[byte]
-  MerkleTreeBuilder* = object
-    mcodec: MultiCodec
-    digestSize: Natural
-    buffer: seq[byte]
 
-###########################################################
+  StoreBackend*[H] = object
-# Helper functions
+    put: PutFn[H]
-###########################################################
+    get: GetFn[H]
 
-func computeTreeHeight(leavesCount: int): int =
+  CompressFn*[H, K] = proc (x, y: H, key: K): ?!H {.noSideEffect, raises: [].}
-  if isPowerOfTwo(leavesCount):
-    fastLog2(leavesCount) + 1
-  else:
-    fastLog2(leavesCount) + 2
 
-func computeLevels(leavesCount: int): seq[tuple[offset: int, width: int, index: int]] =
+  MerkleTree*[H, K] = ref object of RootObj
-  let height = computeTreeHeight(leavesCount)
+    layers*  : seq[seq[H]]
-  var levels = newSeq[tuple[offset: int, width: int, index: int]](height)
+    compress*: CompressFn[H, K]
+    zero*    : H
 
-  levels[0].offset = 0
+  MerkleProof*[H, K] = ref object of RootObj
-  levels[0].width = leavesCount
+    index*   : int                  # linear index of the leaf, starting from 0
-  levels[0].index = 0
+    path*    : seq[H]               # order: from the bottom to the top
-  for i in 1..<height:
+    nleaves* : int                  # number of leaves in the tree (=size of input)
-    levels[i].offset = levels[i - 1].offset + levels[i - 1].width
+    compress*: CompressFn[H, K]     # compress function
-    levels[i].width = (levels[i - 1].width + 1) div 2
+    zero*    : H                    # zero value
-    levels[i].index = i
+
-  levels
+func depth*[H, K](self: MerkleTree[H, K]): int =
+  return self.layers.len - 1
+
+func leavesCount*[H, K](self: MerkleTree[H, K]): int =
+  return self.layers[0].len
+
+func levels*[H, K](self: MerkleTree[H, K]): int =
+  return self.layers.len
+
+func leaves*[H, K](self: MerkleTree[H, K]): seq[H] =
+  return self.layers[0]
+
+iterator layers*[H, K](self: MerkleTree[H, K]): seq[H] =
+  for layer in self.layers:
+    yield layer
+
+iterator nodes*[H, K](self: MerkleTree[H, K]): H =
+  for layer in self.layers:
+    for node in layer:
+      yield node
+
+func root*[H, K](self: MerkleTree[H, K]): ?!H =
+  let last = self.layers[^1]
+  if last.len != 1:
+    return failure "invalid tree"
+
+  return success last[0]
+
+func getProof*[H, K](
+  self: MerkleTree[H, K],
+  index: int,
+  proof: MerkleProof[H, K]): ?!void =
+  let depth   = self.depth
+  let nleaves = self.leavesCount
+
+  if not (index >= 0 and index < nleaves):
+    return failure "index out of bounds"
+
+  var path : seq[H] = newSeq[H](depth)
+  var k = index
+  var m = nleaves
+  for i in 0..<depth:
+    let j = k xor 1
+    path[i] = if (j < m): self.layers[i][j] else: self.zero
+    k =  k      shr 1
+    m = (m + 1) shr 1
+
+  proof.index = index
+  proof.path  = path
+  proof.nleaves = nleaves
+  proof.compress = self.compress
 
-proc digestFn(mcodec: MultiCodec, dst: var openArray[byte], dstPos: int, data: openArray[byte]): ?!void =
-  var mhash = ? MultiHash.digest($mcodec, data).mapFailure
-  if (dstPos + mhash.size) > dst.len:
-    return failure("Not enough space in a destination buffer")
-  dst[dstPos..<dstPos + mhash.size] = mhash.data.buffer[mhash.dpos..<mhash.dpos + mhash.size]
   success()
 
-###########################################################
+func getProof*[H, K](self: MerkleTree[H, K], index: int): ?!MerkleProof[H, K] =
-# MerkleTreeBuilder
-###########################################################
-
-proc init*(
-  T: type MerkleTreeBuilder,
-  mcodec: MultiCodec = multiCodec("sha2-256")
-): ?!MerkleTreeBuilder =
-  let mhash = ? MultiHash.digest($mcodec, "".toBytes).mapFailure
-  success(MerkleTreeBuilder(mcodec: mcodec, digestSize: mhash.size, buffer: newSeq[byte]()))
-
-proc addDataBlock*(self: var MerkleTreeBuilder, dataBlock: openArray[byte]): ?!void =
-  ## Hashes the data block and adds the result of hashing to a buffer
-  ##
-  let oldLen = self.buffer.len
-  self.buffer.setLen(oldLen + self.digestSize)
-  digestFn(self.mcodec, self.buffer, oldLen, dataBlock)
-
-proc addLeaf*(self: var MerkleTreeBuilder, leaf: MultiHash): ?!void =
-  if leaf.mcodec != self.mcodec or leaf.size != self.digestSize:
-    return failure("Expected mcodec to be " & $self.mcodec & " and digest size to be " &
-      $self.digestSize & " but was " & $leaf.mcodec & " and " & $leaf.size)
-
-  let oldLen = self.buffer.len
-  self.buffer.setLen(oldLen + self.digestSize)
-  self.buffer[oldLen..<oldLen + self.digestSize] = leaf.data.buffer[leaf.dpos..<leaf.dpos + self.digestSize]
-  success()
-
-proc build*(self: MerkleTreeBuilder): ?!MerkleTree =
-  ## Builds a tree from previously added data blocks
-  ##
-  ## Tree built from data blocks A, B and C is
-  ##        H5=H(H3 & H4)
-  ##      /            \
-  ##    H3=H(H0 & H1)   H4=H(H2 & 0x00)
-  ##   /    \          /
-  ## H0=H(A) H1=H(B) H2=H(C)
-  ## |       |       |
-  ## A       B       C
-  ##
-  ## Memory layout is [H0, H1, H2, H3, H4, H5]
-  ##
-  let
-    mcodec = self.mcodec
-    digestSize = self.digestSize
-    leavesCount = self.buffer.len div self.digestSize
-
-  if leavesCount == 0:
-    return failure("At least one data block is required")
-
-  let levels = computeLevels(leavesCount)
-  let totalNodes = levels[^1].offset + 1
-
-  var tree = MerkleTree(mcodec: mcodec, digestSize: digestSize, leavesCount: leavesCount, nodesBuffer: newSeq[byte](totalNodes * digestSize))
-
-  # copy leaves
-  tree.nodesBuffer[0..<leavesCount * digestSize] = self.buffer[0..<leavesCount * digestSize]
-
-  # calculate intermediate nodes
-  var zero = newSeq[byte](digestSize)
-  var one = newSeq[byte](digestSize)
-  one[^1] = 0x01
-
   var
-    concatBuf = newSeq[byte](2 * digestSize)
+    proof = MerkleProof[H, K]()
-    prevLevel = levels[0]
-  for level in levels[1..^1]:
-    for i in 0..<level.width:
-      let parentIndex = level.offset + i
-      let leftChildIndex = prevLevel.offset + 2 * i
-      let rightChildIndex = leftChildIndex + 1
 
-      concatBuf[0..<digestSize] = tree.nodesBuffer[leftChildIndex * digestSize..<(leftChildIndex + 1) * digestSize]
+  ? self.getProof(index, proof)
 
-      var dummyValue = if prevLevel.index == 0: zero else: one
+  success proof
 
-      if rightChildIndex < prevLevel.offset + prevLevel.width:
+func reconstructRoot*[H, K](proof: MerkleProof[H, K], leaf: H): ?!H =
-        concatBuf[digestSize..^1] = tree.nodesBuffer[rightChildIndex * digestSize..<(rightChildIndex + 1) * digestSize]
+  var
+    m = proof.nleaves
+    j = proof.index
+    h = leaf
+    bottomFlag = K.KeyBottomLayer
+
+  for p in proof.path:
+    let oddIndex : bool = (bitand(j,1) != 0)
+    if oddIndex:
+      # the index of the child is odd, so the node itself can't be odd (a bit counterintuitive, yeah :)
+      h = ? proof.compress( p, h, bottomFlag )
     else:
-        concatBuf[digestSize..^1] = dummyValue
+      if j == m - 1:
-
+        # single child => odd node
-      ? digestFn(mcodec, tree.nodesBuffer, parentIndex * digestSize, concatBuf)
+        h = ? proof.compress( h, p, K(bottomFlag.ord + 2) )
-    prevLevel = level
-
-  return success(tree)
-
-###########################################################
-# MerkleTree
-###########################################################
-
-proc nodeBufferToMultiHash(self: (MerkleTree | MerkleProof), index: int): MultiHash =
-  var buf = newSeq[byte](self.digestSize)
-  let offset = index * self.digestSize
-  buf[0..^1] = self.nodesBuffer[offset..<(offset + self.digestSize)]
-
-  {.noSideEffect.}:
-    without mhash =? MultiHash.init($self.mcodec, buf).mapFailure, errx:
-      error "Error converting bytes to hash", msg = errx.msg
-  mhash
-
-proc len*(self: (MerkleTree | MerkleProof)): Natural =
-  self.nodesBuffer.len div self.digestSize
-
-proc nodes*(self: (MerkleTree | MerkleProof)): seq[MultiHash] {.noSideEffect.} =
-  toSeq(0..<self.len).map(i => self.nodeBufferToMultiHash(i))
-
-proc mcodec*(self: (MerkleTree | MerkleProof)): MultiCodec =
-  self.mcodec
-
-proc digestSize*(self: (MerkleTree | MerkleProof)): Natural =
-  self.digestSize
-
-proc root*(self: MerkleTree): MultiHash =
-  let rootIndex = self.len - 1
-  self.nodeBufferToMultiHash(rootIndex)
-
-proc rootCid*(self: MerkleTree, version = CIDv1, dataCodec = multiCodec("raw")): ?!Cid =
-  Cid.init(version, dataCodec, self.root).mapFailure
-
-iterator leaves*(self: MerkleTree): MultiHash =
-  for i in 0..<self.leavesCount:
-    yield self.nodeBufferToMultiHash(i)
-
-iterator leavesCids*(self: MerkleTree, version = CIDv1, dataCodec = multiCodec("raw")): ?!Cid =
-  for leaf in self.leaves:
-    yield Cid.init(version, dataCodec, leaf).mapFailure
-
-proc leavesCount*(self: MerkleTree): Natural =
-  self.leavesCount
-
-proc getLeaf*(self: MerkleTree, index: Natural): ?!MultiHash =
-  if index >= self.leavesCount:
-    return failure("Index " & $index & " out of range [0.." & $(self.leavesCount - 1) & "]" )
-
-  success(self.nodeBufferToMultiHash(index))
-
-proc getLeafCid*(self: MerkleTree, index: Natural, version = CIDv1, dataCodec = multiCodec("raw")): ?!Cid =
-  let leaf = ? self.getLeaf(index)
-  Cid.init(version, dataCodec, leaf).mapFailure
-
-proc height*(self: MerkleTree): Natural =
-  computeTreeHeight(self.leavesCount)
-
-proc getProof*(self: MerkleTree, index: Natural): ?!MerkleProof =
-  ## Extracts proof from a tree for a given index
-  ##
-  ## Given a tree built from data blocks A, B and C
-  ##         H5
-  ##      /     \
-  ##    H3       H4
-  ##   /  \     /
-  ## H0    H1 H2
-  ## |     |  |
-  ## A     B  C
-  ##
-  ## Proofs of inclusion (index and path) are
-  ## - 0,[H1, H4] for data block A
-  ## - 1,[H0, H4] for data block B
-  ## - 2,[0x00, H3] for data block C
-  ##
-  if index >= self.leavesCount:
-    return failure("Index " & $index & " out of range [0.." & $(self.leavesCount - 1) & "]" )
-
-  var zero = newSeq[byte](self.digestSize)
-  var one = newSeq[byte](self.digestSize)
-  one[^1] = 0x01
-
-  let levels = computeLevels(self.leavesCount)
-  var proofNodesBuffer = newSeq[byte]((levels.len - 1) * self.digestSize)
-  for level in levels[0..^2]:
-    let lr = index shr level.index
-    let siblingIndex = if lr mod 2 == 0:
-      level.offset + lr + 1
       else:
-      level.offset + lr - 1
+        # even node
+        h = ? proof.compress( h , p, bottomFlag )
+    bottomFlag = K.KeyNone
+    j =  j    shr 1
+    m = (m+1) shr 1
 
-    var dummyValue = if level.index == 0: zero else: one
+  return success h
 
-    if siblingIndex < level.offset + level.width:
+func verify*[H, K](proof: MerkleProof[H, K], leaf: H, root: H): ?!void =
-      proofNodesBuffer[level.index * self.digestSize..<(level.index + 1) * self.digestSize] =
+  return if bool(root == ? proof.reconstructRoot(leaf)):
-        self.nodesBuffer[siblingIndex * self.digestSize..<(siblingIndex + 1) * self.digestSize]
+      success()
     else:
-      proofNodesBuffer[level.index * self.digestSize..<(level.index + 1) * self.digestSize] = dummyValue
+      failure("invalid proof")
 
-  success(MerkleProof(mcodec: self.mcodec, digestSize: self.digestSize, index: index, nodesBuffer: proofNodesBuffer))
+func merkleTreeWorker*[H, K](
+  self: MerkleTree[H, K],
+  xs: openArray[H],
+  isBottomLayer: static bool): ?!seq[seq[H]] =
 
-proc `$`*(self: MerkleTree): string {.noSideEffect.} =
+  let a = low(xs)
-  "mcodec:" & $self.mcodec &
+  let b = high(xs)
-    ", digestSize: " & $self.digestSize &
+  let m = b - a + 1
-    ", leavesCount: " & $self.leavesCount &
-    ", nodes: " & $self.nodes
 
-proc `==`*(a, b: MerkleTree): bool =
+  when not isBottomLayer:
-  (a.mcodec == b.mcodec) and
+    if m == 1:
-  (a.digestSize == b.digestSize) and
+      return success @[ @xs ]
-  (a.leavesCount == b.leavesCount) and
-    (a.nodesBuffer == b.nodesBuffer)
 
-proc init*(
+  let halfn: int  = m div 2
-  T: type MerkleTree,
+  let n    : int  = 2 * halfn
-  mcodec: MultiCodec,
+  let isOdd: bool = (n != m)
-  digestSize: Natural,
+
-  leavesCount: Natural,
+  var ys: seq[H]
-  nodesBuffer: seq[byte]
+  if not isOdd:
-): ?!MerkleTree =
+    ys = newSeq[H](halfn)
-  let levels = computeLevels(leavesCount)
-  let totalNodes = levels[^1].offset + 1
-  if totalNodes * digestSize == nodesBuffer.len:
-    success(
-      MerkleTree(
-        mcodec: mcodec,
-        digestSize: digestSize,
-        leavesCount: leavesCount,
-        nodesBuffer: nodesBuffer
-      )
-    )
   else:
-    failure("Expected nodesBuffer len to be " & $(totalNodes * digestSize) & " but was " & $nodesBuffer.len)
+    ys = newSeq[H](halfn + 1)
 
-proc init*(
+  for i in 0..<halfn:
-  T: type MerkleTree,
+    const key = when isBottomLayer: K.KeyBottomLayer else: K.KeyNone
-  leaves: openArray[MultiHash]
+    ys[i] = ? self.compress( xs[a + 2 * i], xs[a + 2 * i + 1], key = key )
-): ?!MerkleTree =
+  if isOdd:
-  without leaf =? leaves.?[0]:
+    const key = when isBottomLayer: K.KeyOddAndBottomLayer else: K.KeyOdd
-    return failure("At least one leaf is required")
+    ys[halfn] = ? self.compress( xs[n], self.zero, key = key )
 
-  var builder = ? MerkleTreeBuilder.init(mcodec = leaf.mcodec)
+  success @[ @xs ] & ? self.merkleTreeWorker(ys, isBottomLayer = false)
-
-  for l in leaves:
-    let res = builder.addLeaf(l)
-    if res.isErr:
-      return failure(res.error)
-
-  builder.build()
-
-proc init*(
-  T: type MerkleTree,
-  cids: openArray[Cid]
-): ?!MerkleTree =
-  var leaves = newSeq[MultiHash]()
-
-  for cid in cids:
-    let res = cid.mhash.mapFailure
-    if res.isErr:
-      return failure(res.error)
-    else:
-      leaves.add(res.value)
-
-  MerkleTree.init(leaves)
-
-###########################################################
-# MerkleProof
-###########################################################
-
-proc verifyLeaf*(self: MerkleProof, leaf: MultiHash, treeRoot: MultiHash): ?!bool =
-  if leaf.mcodec != self.mcodec:
-    return failure("Leaf mcodec was " & $leaf.mcodec & ", but " & $self.mcodec & " expected")
-
-  if leaf.mcodec != self.mcodec:
-    return failure("Tree root mcodec was " & $treeRoot.mcodec & ", but " & $treeRoot.mcodec & " expected")
-
-  var digestBuf = newSeq[byte](self.digestSize)
-  digestBuf[0..^1] = leaf.data.buffer[leaf.dpos..<(leaf.dpos + self.digestSize)]
-
-  let proofLen = self.nodesBuffer.len div self.digestSize
-  var concatBuf = newSeq[byte](2 * self.digestSize)
-  for i in 0..<proofLen:
-    let offset = i * self.digestSize
-    let lr = self.index shr i
-    if lr mod 2 == 0:
-      concatBuf[0..^1] = digestBuf & self.nodesBuffer[offset..<(offset + self.digestSize)]
-    else:
-      concatBuf[0..^1] = self.nodesBuffer[offset..<(offset + self.digestSize)] & digestBuf
-    ? digestFn(self.mcodec, digestBuf, 0, concatBuf)
-
-  let computedRoot = ? MultiHash.init(self.mcodec, digestBuf).mapFailure
-
-  success(computedRoot == treeRoot)
-
-
-proc verifyDataBlock*(self: MerkleProof, dataBlock: openArray[byte], treeRoot: MultiHash): ?!bool =
-  var digestBuf = newSeq[byte](self.digestSize)
-  ? digestFn(self.mcodec, digestBuf, 0, dataBlock)
-
-  let leaf = ? MultiHash.init(self.mcodec, digestBuf).mapFailure
-
-  self.verifyLeaf(leaf, treeRoot)
-
-proc index*(self: MerkleProof): Natural =
-  self.index
-
-proc `$`*(self: MerkleProof): string =
-  "mcodec:" & $self.mcodec &
-    ", digestSize: " & $self.digestSize &
-    ", index: " & $self.index &
-    ", nodes: " & $self.nodes
-
-func `==`*(a, b: MerkleProof): bool =
-  (a.index == b.index) and
-    (a.mcodec == b.mcodec) and
-    (a.digestSize == b.digestSize) and
-    (a.nodesBuffer == b.nodesBuffer)
-
-proc init*(
-  T: type MerkleProof,
-  index: Natural,
-  nodes: seq[MultiHash]
-): ?!MerkleProof =
-  if nodes.len == 0:
-    return failure("At least one node is required")
-
-  let
-    mcodec = nodes[0].mcodec
-    digestSize = nodes[0].size
-
-  var nodesBuffer = newSeq[byte](nodes.len * digestSize)
-  for nodeIndex, node in nodes:
-    nodesBuffer[nodeIndex * digestSize..<(nodeIndex + 1) * digestSize] = node.data.buffer[node.dpos..<node.dpos + digestSize]
-
-  success(MerkleProof(mcodec: mcodec, digestSize: digestSize, index: index, nodesBuffer: nodesBuffer))
-
-func init*(
-  T: type MerkleProof,
-  mcodec: MultiCodec,
-  digestSize: Natural,
-  index: Natural,
-  nodesBuffer: seq[byte]
-): ?!MerkleProof =
-
-  if nodesBuffer.len mod digestSize != 0:
-    return failure("nodesBuffer len is not a multiple of digestSize")
-
-  let treeHeight = (nodesBuffer.len div digestSize) + 1
-  let maxLeavesCount = 1 shl treeHeight
-  if index < maxLeavesCount:
-    return success(
-      MerkleProof(
-        mcodec: mcodec,
-        digestSize: digestSize,
-        index: index,
-        nodesBuffer: nodesBuffer
-      )
-    )
-  else:
-    return failure("index higher than max leaves count")

codex/merkletree/poseidon2.nim

@@ -0,0 +1,104 @@
+## Nim-Codex
+## Copyright (c) 2023 Status Research & Development GmbH
+## Licensed under either of
+##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+## at your option.
+## This file may not be copied, modified, or distributed except according to
+## those terms.
+
+import std/sequtils
+
+import pkg/poseidon2
+import pkg/constantine/math/io/io_fields
+import pkg/constantine/platforms/abstractions
+import pkg/questionable/results
+
+import ../utils
+import ../rng
+
+import ./merkletree
+
+export merkletree, poseidon2
+
+const
+  KeyNoneF              = F.fromhex("0x0")
+  KeyBottomLayerF       = F.fromhex("0x1")
+  KeyOddF               = F.fromhex("0x2")
+  KeyOddAndBottomLayerF = F.fromhex("0x3")
+
+type
+  Poseidon2Hash* = F
+
+  PoseidonKeysEnum* = enum  # can't use non-ordinals as enum values
+    KeyNone
+    KeyBottomLayer
+    KeyOdd
+    KeyOddAndBottomLayer
+
+  Poseidon2Tree* = MerkleTree[Poseidon2Hash, PoseidonKeysEnum]
+  Poseidon2Proof* = MerkleProof[Poseidon2Hash, PoseidonKeysEnum]
+
+converter toKey*(key: PoseidonKeysEnum): Poseidon2Hash =
+  case key:
+  of KeyNone: KeyNoneF
+  of KeyBottomLayer: KeyBottomLayerF
+  of KeyOdd: KeyOddF
+  of KeyOddAndBottomLayer: KeyOddAndBottomLayerF
+
+func init*(
+  _: type Poseidon2Tree,
+  leaves: openArray[Poseidon2Hash]): ?!Poseidon2Tree =
+
+  if leaves.len == 0:
+    return failure "Empty leaves"
+
+  let
+    compressor = proc(
+      x, y: Poseidon2Hash,
+      key: PoseidonKeysEnum): ?!Poseidon2Hash {.noSideEffect.} =
+      success compress( x, y, key.toKey )
+
+  var
+    self = Poseidon2Tree(compress: compressor, zero: zero)
+
+  self.layers = ? merkleTreeWorker(self, leaves, isBottomLayer = true)
+  success self
+
+func init*(
+  _: type Poseidon2Tree,
+  leaves: openArray[array[31, byte]]): ?!Poseidon2Tree =
+  Poseidon2Tree.init(
+    leaves.mapIt( Poseidon2Hash.fromBytes(it) ))
+
+proc fromNodes*(
+  _: type Poseidon2Tree,
+  nodes: openArray[Poseidon2Hash],
+  nleaves: int): ?!Poseidon2Tree =
+
+  if nodes.len == 0:
+    return failure "Empty nodes"
+
+  let
+    compressor = proc(
+      x, y: Poseidon2Hash,
+      key: PoseidonKeysEnum): ?!Poseidon2Hash {.noSideEffect.} =
+      success compress( x, y, key.toKey )
+
+  var
+    self = Poseidon2Tree(compress: compressor, zero: zero)
+    layer = nleaves
+    pos = 0
+
+  while pos < nodes.len:
+    self.layers.add( nodes[pos..<(pos + layer)] )
+    pos += layer
+    layer = divUp(layer, 2)
+
+  let
+    index = Rng.instance.rand(nleaves - 1)
+    proof = ? self.getProof(index)
+
+  ? proof.verify(self.leaves[index], ? self.root) # sanity check
+
+  success self

codex/node.nim

@@ -243,7 +243,7 @@ proc store*(
   finally:
     await stream.close()
 
-  without tree =? MerkleTree.init(cids), err:
+  without tree =? CodexTree.init(cids), err:
     return failure(err)
 
   without treeCid =? tree.rootCid(CIDv1, dataCodec), err:

codex/sales/states/cancelled.nim

@@ -2,7 +2,6 @@ import pkg/chronicles
 import ../salesagent
 import ../statemachine
 import ./errorhandling
-import ./errored
 
 logScope:
   topics = "marketplace sales cancelled"

codex/stores.nim

@@ -6,4 +6,11 @@ import ./stores/maintenance
 import ./stores/keyutils
 import ./stores/treehelper
 
-export cachestore, blockstore, networkstore, repostore, maintenance, keyutils, treehelper
+export
+  cachestore,
+  blockstore,
+  networkstore,
+  repostore,
+  keyutils,
+  treehelper,
+  maintenance

codex/stores/blockstore.nim

@@ -49,7 +49,7 @@ method getBlock*(self: BlockStore, address: BlockAddress): Future[?!Block] {.bas
 
   raiseAssert("getBlock by addr not implemented!")
 
-method getBlockAndProof*(self: BlockStore, treeCid: Cid, index: Natural): Future[?!(Block, MerkleProof)] {.base.} =
+method getBlockAndProof*(self: BlockStore, treeCid: Cid, index: Natural): Future[?!(Block, CodexProof)] {.base.} =
   ## Get a block and associated inclusion proof by Cid of a merkle tree and an index of a leaf in a tree
   ##
 
@@ -70,7 +70,7 @@ method putBlockCidAndProof*(
   treeCid: Cid,
   index: Natural,
   blockCid: Cid,
-  proof: MerkleProof
+  proof: CodexProof
 ): Future[?!void] {.base.} =
   ## Put a block to the blockstore
   ##

codex/stores/cachestore.nim

@@ -39,7 +39,7 @@ type
     currentSize*: NBytes
     size*: NBytes
     cache: LruCache[Cid, Block]
-    cidAndProofCache: LruCache[(Cid, Natural), (Cid, MerkleProof)]
+    cidAndProofCache: LruCache[(Cid, Natural), (Cid, CodexProof)]
 
   InvalidBlockSize* = object of CodexError
 
@@ -65,7 +65,7 @@ method getBlock*(self: CacheStore, cid: Cid): Future[?!Block] {.async.} =
     trace "Error requesting block from cache", cid, error = exc.msg
     return failure exc
 
-proc getCidAndProof(self: CacheStore, treeCid: Cid, index: Natural): ?!(Cid, MerkleProof) =
+proc getCidAndProof(self: CacheStore, treeCid: Cid, index: Natural): ?!(Cid, CodexProof) =
   if cidAndProof =? self.cidAndProofCache.getOption((treeCid, index)):
     success(cidAndProof)
   else:
@@ -77,7 +77,7 @@ method getBlock*(self: CacheStore, treeCid: Cid, index: Natural): Future[?!Block
 
   await self.getBlock(cidAndProof[0])
 
-method getBlockAndProof*(self: CacheStore, treeCid: Cid, index: Natural): Future[?!(Block, MerkleProof)] {.async.} =
+method getBlockAndProof*(self: CacheStore, treeCid: Cid, index: Natural): Future[?!(Block, CodexProof)] {.async.} =
   without cidAndProof =? self.getCidAndProof(treeCid, index), err:
     return failure(err)
 
@@ -215,7 +215,7 @@ method putBlockCidAndProof*(
   treeCid: Cid,
   index: Natural,
   blockCid: Cid,
-  proof: MerkleProof
+  proof: CodexProof
 ): Future[?!void] {.async.} =
   self.cidAndProofCache[(treeCid, index)] = (blockCid, proof)
   success()
@@ -288,7 +288,7 @@ proc new*(
     currentSize = 0'nb
     size = int(cacheSize div chunkSize)
     cache = newLruCache[Cid, Block](size)
-    cidAndProofCache = newLruCache[(Cid, Natural), (Cid, MerkleProof)](size)
+    cidAndProofCache = newLruCache[(Cid, Natural), (Cid, CodexProof)](size)
     store = CacheStore(
       cache: cache,
       cidAndProofCache: cidAndProofCache,

codex/stores/keyutils.nim

@@ -47,4 +47,3 @@ proc createBlockExpirationMetadataQueryKey*(): ?!Key =
 
 proc createBlockCidAndProofMetadataKey*(treeCid: Cid, index: Natural): ?!Key =
   (BlockProofKey / $treeCid).flatMap((k: Key) => k / $index)
-  

codex/stores/networkstore.nim

@@ -11,8 +11,6 @@ import pkg/upraises
 
 push: {.upraises: [].}
 
-import std/sugar
-
 import pkg/chronicles
 import pkg/chronos
 import pkg/libp2p
@@ -87,7 +85,7 @@ method putBlockCidAndProof*(
   treeCid: Cid,
   index: Natural,
   blockCid: Cid,
-  proof: MerkleProof): Future[?!void] =
+  proof: CodexProof): Future[?!void] =
   self.localStore.putBlockCidAndProof(treeCid, index, blockCid, proof)
 
 method ensureExpiry*(

codex/stores/repostore.nim

@@ -77,7 +77,7 @@ func available*(self: RepoStore): uint =
 func available*(self: RepoStore, bytes: uint): bool =
   return bytes < self.available()
 
-proc encode(cidAndProof: (Cid, MerkleProof)): seq[byte] =
+proc encode(cidAndProof: (Cid, CodexProof)): seq[byte] =
   ## Encodes a tuple of cid and merkle proof in a following format:
   ## | 8-bytes | n-bytes | remaining bytes |
   ## |    n    |   cid   |      proof      |
@@ -93,14 +93,14 @@ proc encode(cidAndProof: (Cid, MerkleProof)): seq[byte] =
 
   @nBytes & cidBytes & proofBytes
 
-proc decode(_: type (Cid, MerkleProof), data: seq[byte]): ?!(Cid, MerkleProof) =
+proc decode(_: type (Cid, CodexProof), data: seq[byte]): ?!(Cid, CodexProof) =
   let
     n = uint64.fromBytesBE(data[0..<sizeof(uint64)]).int
     cid = ? Cid.init(data[sizeof(uint64)..<sizeof(uint64) + n]).mapFailure
-    proof = ? MerkleProof.decode(data[sizeof(uint64) + n..^1])
+    proof = ? CodexProof.decode(data[sizeof(uint64) + n..^1])
   success((cid, proof))
 
-proc decodeCid(_: type (Cid, MerkleProof), data: seq[byte]): ?!Cid =
+proc decodeCid(_: type (Cid, CodexProof), data: seq[byte]): ?!Cid =
   let
     n = uint64.fromBytesBE(data[0..<sizeof(uint64)]).int
     cid = ? Cid.init(data[sizeof(uint64)..<sizeof(uint64) + n]).mapFailure
@@ -111,7 +111,7 @@ method putBlockCidAndProof*(
   treeCid: Cid,
   index: Natural,
   blockCid: Cid,
-  proof: MerkleProof
+  proof: CodexProof
 ): Future[?!void] {.async.} =
   ## Put a block to the blockstore
   ##
@@ -119,6 +119,8 @@ method putBlockCidAndProof*(
   without key =? createBlockCidAndProofMetadataKey(treeCid, index), err:
     return failure(err)
 
+  trace "Storing block cid and proof with key", key
+
   let value = (blockCid, proof).encode()
 
   await self.metaDs.put(key, value)
@@ -127,7 +129,7 @@ proc getCidAndProof(
   self: RepoStore,
   treeCid: Cid,
   index: Natural
-): Future[?!(Cid, MerkleProof)] {.async.} =
+): Future[?!(Cid, CodexProof)] {.async.} =
   without key =? createBlockCidAndProofMetadataKey(treeCid, index), err:
     return failure(err)
 
@@ -137,23 +139,29 @@ proc getCidAndProof(
     else:
       return failure(err)
 
-  return (Cid, MerkleProof).decode(value)
+  without (cid, proof) =? (Cid, CodexProof).decode(value), err:
+    trace "Unable to decode cid and proof", err = err.msg
+    return failure(err)
+
+  trace "Got cid and proof for block", cid, proof = $proof
+  return success (cid, proof)
 
 proc getCid(
   self: RepoStore,
   treeCid: Cid,
-  index: Natural
+  index: Natural): Future[?!Cid] {.async.} =
-): Future[?!Cid] {.async.} =
   without key =? createBlockCidAndProofMetadataKey(treeCid, index), err:
     return failure(err)
 
   without value =? await self.metaDs.get(key), err:
     if err of DatastoreKeyNotFound:
+      trace "Cid not found", treeCid, index
       return failure(newException(BlockNotFoundError, err.msg))
     else:
+      trace "Error getting cid from datastore", err = err.msg, key
       return failure(err)
 
-  return (Cid, MerkleProof).decodeCid(value)
+  return (Cid, CodexProof).decodeCid(value)
 
 method getBlock*(self: RepoStore, cid: Cid): Future[?!Block] {.async.} =
   ## Get a block from the blockstore
@@ -181,7 +189,7 @@ method getBlock*(self: RepoStore, cid: Cid): Future[?!Block] {.async.} =
   return Block.new(cid, data, verify = true)
 
 
-method getBlockAndProof*(self: RepoStore, treeCid: Cid, index: Natural): Future[?!(Block, MerkleProof)] {.async.} =
+method getBlockAndProof*(self: RepoStore, treeCid: Cid, index: Natural): Future[?!(Block, CodexProof)] {.async.} =
   without cidAndProof =? await self.getCidAndProof(treeCid, index), err:
     return failure(err)
 
@@ -409,15 +417,17 @@ method delBlock*(self: RepoStore, treeCid: Cid, index: Natural): Future[?!void]
   without key =? createBlockCidAndProofMetadataKey(treeCid, index), err:
     return failure(err)
 
+  trace "Fetching proof", key
   without value =? await self.metaDs.get(key), err:
     if err of DatastoreKeyNotFound:
       return success()
     else:
       return failure(err)
 
-  without cid =? (Cid, MerkleProof).decodeCid(value), err:
+  without cid =? (Cid, CodexProof).decodeCid(value), err:
     return failure(err)
 
+  trace "Deleting block", cid
   if err =? (await self.delBlock(cid)).errorOption:
     return failure(err)
 

codex/stores/treehelper.nim

@@ -22,7 +22,7 @@ import ./blockstore
 import ../utils/asynciter
 import ../merkletree
 
-proc putSomeProofs*(store: BlockStore, tree: MerkleTree, iter: Iter[int]): Future[?!void] {.async.} =
+proc putSomeProofs*(store: BlockStore, tree: CodexTree, iter: Iter[int]): Future[?!void] {.async.} =
   without treeCid =? tree.rootCid, err:
     return failure(err)
 
@@ -43,10 +43,8 @@ proc putSomeProofs*(store: BlockStore, tree: MerkleTree, iter: Iter[int]): Futur
 
   success()
 
-proc putSomeProofs*(store: BlockStore, tree: MerkleTree, iter: Iter[Natural]): Future[?!void] =
+proc putSomeProofs*(store: BlockStore, tree: CodexTree, iter: Iter[Natural]): Future[?!void] =
   store.putSomeProofs(tree, iter.map((i: Natural) => i.ord))
 
-proc putAllProofs*(store: BlockStore, tree: MerkleTree): Future[?!void] =
+proc putAllProofs*(store: BlockStore, tree: CodexTree): Future[?!void] =
   store.putSomeProofs(tree, Iter.fromSlice(0..<tree.leavesCount))
-
-

codex/utils.nim

@@ -35,7 +35,6 @@ proc orElse*[A](a, b: Option[A]): Option[A] =
   else:
     b
 
-
 when not declared(parseDuration): # Odd code formatting to minimize diff v. mainLine
  const Whitespace = {' ', '\t', '\v', '\r', '\l', '\f'}
 

codex/utils/asynciter.nim

@@ -145,4 +145,3 @@ proc prefetch*[T](iter: Iter[T], n: Positive): Iter[T] =
     tryFetch(j)
 
   Iter.new(genNext, isFinished)
-

codex/utils/digest.nim

@@ -0,0 +1,40 @@
+## Nim-Codex
+## Copyright (c) 2023 Status Research & Development GmbH
+## Licensed under either of
+##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+## at your option.
+## This file may not be copied, modified, or distributed except according to
+## those terms.
+
+import pkg/poseidon2
+import pkg/poseidon2/io
+import pkg/questionable/results
+
+import ../merkletree
+
+func digestTree*(
+  _: type Poseidon2Tree,
+  bytes: openArray[byte], chunkSize: int): ?!Poseidon2Tree =
+  ## Hashes chunks of data with a sponge of rate 2, and combines the
+  ## resulting chunk hashes in a merkle root.
+  ##
+
+  var index = 0
+  var leaves: seq[Poseidon2Hash]
+  while index < bytes.len:
+    let start = index
+    let finish = min(index + chunkSize, bytes.len)
+    let digest = Sponge.digest(bytes.toOpenArray(start, finish - 1), rate = 2)
+    leaves.add(digest)
+    index += chunkSize
+  return Poseidon2Tree.init(leaves)
+
+func digest*(
+  _: type Poseidon2Tree,
+  bytes: openArray[byte], chunkSize: int): ?!Poseidon2Hash =
+  ## Hashes chunks of data with a sponge of rate 2, and combines the
+  ## resulting chunk hashes in a merkle root.
+  ##
+
+  (? Poseidon2Tree.digestTree(bytes, chunkSize)).root

codex/utils/timer.nim

@@ -10,9 +10,12 @@
 ## Timer
 ## Used to execute a callback in a loop
 
+import pkg/upraises
+
+push: {.upraises: [].}
+
 import pkg/chronos
 import pkg/chronicles
-import pkg/upraises
 
 type
   TimerCallback* = proc(): Future[void] {.gcsafe, upraises:[].}

tests/codex/blockexchange/discovery/testdiscovery.nim

@@ -26,7 +26,7 @@ asyncchecksuite "Block Advertising and Discovery":
   var
     blocks: seq[bt.Block]
     manifest: Manifest
-    tree: MerkleTree
+    tree: CodexTree
     manifestBlock: bt.Block
     switch: Switch
     peerStore: PeerCtxStore

tests/codex/helpers.nim

@@ -38,7 +38,7 @@ proc lenPrefix*(msg: openArray[byte]): seq[byte] =
 
   return buf
 
-proc makeManifestAndTree*(blocks: seq[Block]): ?!(Manifest, MerkleTree) =
+proc makeManifestAndTree*(blocks: seq[Block]): ?!(Manifest, CodexTree) =
 
   if blocks.len == 0:
     return failure("Blocks list was empty")
@@ -46,15 +46,12 @@ proc makeManifestAndTree*(blocks: seq[Block]): ?!(Manifest, MerkleTree) =
   let
     datasetSize = blocks.mapIt(it.data.len).foldl(a + b)
     blockSize = blocks.mapIt(it.data.len).foldl(max(a, b))
-    tree = ? MerkleTree.init(blocks.mapIt(it.cid))
+    tree = ? CodexTree.init(blocks.mapIt(it.cid))
     treeCid = ? tree.rootCid
     manifest = Manifest.new(
       treeCid = treeCid,
       blockSize = NBytes(blockSize),
-      datasetSize = NBytes(datasetSize),
+      datasetSize = NBytes(datasetSize))
-      version = CIDv1,
-      hcodec = tree.mcodec
-    )
 
   return success((manifest, tree))
 
@@ -88,13 +85,12 @@ proc storeDataGetManifest*(store: BlockStore, chunker: Chunker): Future[Manifest
     (await store.putBlock(blk)).tryGet()
 
   let
-    tree = MerkleTree.init(cids).tryGet()
+    tree = CodexTree.init(cids).tryGet()
     treeCid = tree.rootCid.tryGet()
     manifest = Manifest.new(
       treeCid = treeCid,
       blockSize = NBytes(chunker.chunkSize),
-      datasetSize = NBytes(chunker.offset),
+      datasetSize = NBytes(chunker.offset))
-    )
 
   for i in 0..<tree.leavesCount:
     let proof = tree.getProof(i).tryGet()

tests/codex/merkletree/generictreetests.nim

@@ -0,0 +1,130 @@
+import std/unittest
+import std/sequtils
+
+import pkg/codex/merkletree
+
+proc testGenericTree*[H, K, U](
+  name: string,
+  data: openArray[H],
+  zero: H,
+  compress: proc(z, y: H, key: K): H,
+  makeTree: proc(data: seq[H]): U) =
+
+  let
+    data = @data
+
+  suite "Correctness tests - " & name:
+
+    test "Should build correct tree for even bottom layer":
+      let
+        expectedRoot = compress(
+            compress(
+              compress(data[0], data[1], K.KeyBottomLayer),
+              compress(data[2], data[3], K.KeyBottomLayer),
+              K.KeyNone
+            ),
+            compress(
+              compress(data[4], data[5], K.KeyBottomLayer),
+              compress(data[6], data[7], K.KeyBottomLayer),
+              K.KeyNone
+            ),
+            K.KeyNone
+          )
+
+      let
+        tree = makeTree( data[0..7] )
+
+      check:
+        tree.root.tryGet == expectedRoot
+
+    test "Should build correct tree for odd bottom layer":
+      let
+        expectedRoot = compress(
+            compress(
+              compress(data[0], data[1], K.KeyBottomLayer),
+              compress(data[2], data[3], K.KeyBottomLayer),
+              K.KeyNone
+            ),
+            compress(
+              compress(data[4], data[5], K.KeyBottomLayer),
+              compress(data[6], zero, K.KeyOddAndBottomLayer),
+              K.KeyNone
+            ),
+            K.KeyNone
+          )
+
+      let
+        tree = makeTree( data[0..6] )
+
+      check:
+        tree.root.tryGet == expectedRoot
+
+    test "Should build correct tree for even bottom and odd upper layers":
+      let
+        expectedRoot = compress(
+          compress(
+            compress(
+              compress(data[0], data[1], K.KeyBottomLayer),
+              compress(data[2], data[3], K.KeyBottomLayer),
+              K.KeyNone
+            ),
+            compress(
+              compress(data[4], data[5], K.KeyBottomLayer),
+              compress(data[6], data[7], K.KeyBottomLayer),
+              K.KeyNone
+            ),
+            K.KeyNone
+          ),
+          compress(
+            compress(
+              compress(data[8], data[9], K.KeyBottomLayer),
+              zero,
+              K.KeyOdd
+            ),
+            zero,
+            K.KeyOdd
+          ),
+          K.KeyNone
+        )
+
+      let
+        tree = makeTree( data[0..9] )
+
+      check:
+        tree.root.tryGet == expectedRoot
+
+    test "Should get and validate correct proofs":
+      let
+        expectedRoot = compress(
+          compress(
+            compress(
+              compress(data[0], data[1], K.KeyBottomLayer),
+              compress(data[2], data[3], K.KeyBottomLayer),
+              K.KeyNone
+            ),
+            compress(
+              compress(data[4], data[5], K.KeyBottomLayer),
+              compress(data[6], data[7], K.KeyBottomLayer),
+              K.KeyNone
+            ),
+            K.KeyNone
+          ),
+          compress(
+            compress(
+              compress(data[8], data[9], K.KeyBottomLayer),
+              zero,
+              K.KeyOdd
+            ),
+            zero,
+            K.KeyOdd
+          ),
+          K.KeyNone
+        )
+
+      let
+        tree = makeTree( data )
+
+      for i in 0..<data.len:
+        let proof = tree.getProof(i).tryGet
+        check:
+          proof.verify(tree.leaves[i], expectedRoot).isOk

tests/codex/merkletree/helpers.nim

@@ -0,0 +1,31 @@
+import pkg/constantine/platforms/abstractions
+
+import pkg/codex/merkletree
+import ../helpers
+
+export merkletree, helpers
+
+converter toBool*(x: CtBool): bool =
+  bool(x)
+
+proc `==`*(a, b: Poseidon2Tree): bool =
+  (a.leavesCount == b.leavesCount) and
+  (a.levels == b.levels) and
+  (a.layers == b.layers)
+
+proc `==`*(a, b: Poseidon2Proof): bool =
+  (a.nleaves == b.nleaves) and
+  (a.index == b.index) and
+  (a.path.len == b.path.len) and
+  (a.path == b.path)
+
+proc `==`*(a, b: CodexTree): bool =
+  (a.mcodec == b.mcodec) and
+  (a.leavesCount == b.leavesCount) and
+  (a.levels == b.levels)
+
+proc `==`*(a, b: CodexProof): bool =
+  (a.mcodec == b.mcodec) and
+  (a.nleaves == b.nleaves) and
+  (a.path == b.path) and
+  (a.index == b.index)

tests/codex/merkletree/testcoders.nim

@@ -1,42 +0,0 @@
-import std/unittest
-
-import pkg/questionable/results
-import pkg/stew/byteutils
-
-import pkg/codex/merkletree
-import ../helpers
-
-checksuite "merkletree - coders":
-  const data =
-    [
-      "0123456789012345678901234567890123456789".toBytes,
-      "1234567890123456789012345678901234567890".toBytes,
-      "2345678901234567890123456789012345678901".toBytes,
-      "3456789012345678901234567890123456789012".toBytes,
-      "4567890123456789012345678901234567890123".toBytes,
-      "5678901234567890123456789012345678901234".toBytes,
-      "6789012345678901234567890123456789012345".toBytes,
-      "7890123456789012345678901234567890123456".toBytes,
-      "8901234567890123456789012345678901234567".toBytes,
-      "9012345678901234567890123456789012345678".toBytes,
-    ]
-
-  test "encoding and decoding a tree yields the same tree":
-    var builder = MerkleTreeBuilder.init(multiCodec("sha2-256")).tryGet()
-    builder.addDataBlock(data[0]).tryGet()
-    builder.addDataBlock(data[1]).tryGet()
-    builder.addDataBlock(data[2]).tryGet()
-    builder.addDataBlock(data[3]).tryGet()
-    builder.addDataBlock(data[4]).tryGet()
-    builder.addDataBlock(data[5]).tryGet()
-    builder.addDataBlock(data[6]).tryGet()
-    builder.addDataBlock(data[7]).tryGet()
-    builder.addDataBlock(data[8]).tryGet()
-    builder.addDataBlock(data[9]).tryGet()
-
-    let tree = builder.build().tryGet()
-    let encodedBytes = tree.encode()
-    let decodedTree = MerkleTree.decode(encodedBytes).tryGet()
-
-    check:
-      tree == decodedTree

tests/codex/merkletree/testcodexcoders.nim

@@ -0,0 +1,48 @@
+import std/unittest
+import std/sequtils
+
+import pkg/questionable/results
+import pkg/stew/byteutils
+
+import pkg/codex/merkletree
+import ./helpers
+
+const data =
+  [
+    "00000000000000000000000000000001".toBytes,
+    "00000000000000000000000000000002".toBytes,
+    "00000000000000000000000000000003".toBytes,
+    "00000000000000000000000000000004".toBytes,
+    "00000000000000000000000000000005".toBytes,
+    "00000000000000000000000000000006".toBytes,
+    "00000000000000000000000000000007".toBytes,
+    "00000000000000000000000000000008".toBytes,
+    "00000000000000000000000000000009".toBytes,
+    "00000000000000000000000000000010".toBytes,
+  ]
+
+checksuite "merkletree - coders":
+
+  test "encoding and decoding a tree yields the same tree":
+    let
+      tree = CodexTree.init(multiCodec("sha2-256"), data).tryGet()
+      encodedBytes = tree.encode()
+      decodedTree = CodexTree.decode(encodedBytes).tryGet()
+
+    check:
+      tree == decodedTree
+
+  test "encoding and decoding a proof yields the same proof":
+    let
+      tree = CodexTree.init(multiCodec("sha2-256"), data).tryGet()
+      proof = tree.getProof(4).tryGet()
+
+    check:
+      proof.verify(tree.leaves[4], tree.root.tryGet).isOk
+
+    let
+      encodedBytes = proof.encode()
+      decodedProof = CodexProof.decode(encodedBytes).tryGet()
+
+    check:
+      proof == decodedProof

tests/codex/merkletree/testcodextree.nim

@@ -0,0 +1,106 @@
+import std/unittest
+import std/sequtils
+import std/tables
+
+import pkg/questionable/results
+import pkg/stew/byteutils
+import pkg/nimcrypto/sha2
+import pkg/libp2p
+
+import pkg/codex/merkletree
+
+import ./helpers
+import ./generictreetests
+
+# TODO: Generalize to other hashes
+
+const
+  data =
+    [
+      "00000000000000000000000000000001".toBytes,
+      "00000000000000000000000000000002".toBytes,
+      "00000000000000000000000000000003".toBytes,
+      "00000000000000000000000000000004".toBytes,
+      "00000000000000000000000000000005".toBytes,
+      "00000000000000000000000000000006".toBytes,
+      "00000000000000000000000000000007".toBytes,
+      "00000000000000000000000000000008".toBytes,
+      "00000000000000000000000000000009".toBytes,
+      "00000000000000000000000000000010".toBytes,
+    ]
+  sha256 = multiCodec("sha2-256")
+
+suite "Test CodexTree":
+  test "Cannot init tree without any multihash leaves":
+    check:
+      CodexTree.init(leaves = newSeq[MultiHash]()).isErr
+
+  test "Cannot init tree without any cid leaves":
+    check:
+      CodexTree.init(leaves = newSeq[Cid]()).isErr
+
+  test "Cannot init tree without any byte leaves":
+    check:
+      CodexTree.init(sha256, leaves =  newSeq[ByteHash]()).isErr
+
+  test "Should build tree from multihash leaves":
+    var
+      expectedLeaves = data.mapIt(  MultiHash.digest($sha256, it).tryGet() )
+
+    var tree = CodexTree.init(leaves = expectedLeaves)
+    check:
+      tree.isOk
+      tree.get().leaves == expectedLeaves.mapIt( it.bytes )
+      tree.get().mcodec == sha256
+
+  test "Should build tree from cid leaves":
+    var
+      expectedLeaves = data.mapIt(
+        Cid.init(
+          CidVersion.CIDv1,
+          BlockCodec,
+          MultiHash.digest($sha256, it).tryGet
+      ).tryGet )
+
+    let
+      tree = CodexTree.init(leaves = expectedLeaves)
+
+    check:
+      tree.isOk
+      tree.get().leaves == expectedLeaves.mapIt( it.mhash.tryGet.bytes )
+      tree.get().mcodec == sha256
+
+  test "Should build from raw bytes (should not hash leaves)":
+    let
+      tree = CodexTree.init(sha256, leaves = data).tryGet
+
+    check:
+      tree.mcodec == sha256
+      tree.leaves == data
+
+  test "Should build from nodes":
+    let
+      tree = CodexTree.init(sha256, leaves = data).tryGet
+      fromNodes = CodexTree.fromNodes(
+        nodes = toSeq(tree.nodes),
+        nleaves = tree.leavesCount).tryGet
+
+    check:
+      tree.mcodec == sha256
+      tree == fromNodes
+
+let
+  mhash = sha256.getMhash().tryGet
+  zero: seq[byte] = newSeq[byte](mhash.size)
+  compress = proc(x, y: seq[byte], key: ByteTreeKey): seq[byte] =
+    compress(x, y, key, mhash).tryGet
+
+  makeTree = proc(data: seq[seq[byte]]): CodexTree =
+    CodexTree.init(sha256, leaves = data).tryGet
+
+testGenericTree(
+  "CodexTree",
+  @data,
+  zero,
+  compress,
+  makeTree)

tests/codex/merkletree/testmerkledigest.nim

@@ -0,0 +1,62 @@
+import std/unittest
+import std/sequtils
+import std/random
+
+import pkg/constantine/math/arithmetic
+
+import pkg/poseidon2
+import pkg/poseidon2/io
+import pkg/poseidon2/sponge
+
+import pkg/questionable/results
+
+import pkg/codex/merkletree
+import pkg/codex/utils/digest
+
+import ./helpers
+
+suite "Digest - MerkleTree":
+
+  const KB = 1024
+
+  test "Hashes chunks of data with sponge, and combines them in merkle root":
+    let bytes = newSeqWith(64*KB, rand(byte))
+    var leaves: seq[Poseidon2Hash]
+    for i in 0..<32:
+      let
+        chunk = bytes[(i*2*KB)..<((i+1)*2*KB)]
+        digest = Sponge.digest(chunk, rate = 2)
+      leaves.add(digest)
+
+    let
+      digest = Poseidon2Tree.digest(bytes, chunkSize = 2*KB).tryGet
+      spongeDigest = SpongeMerkle.digest(bytes, chunkSize = 2*KB)
+      codexPosTree = Poseidon2Tree.init(leaves).tryGet
+      rootDigest = codexPosTree.root.tryGet
+
+    check:
+      bool( digest == spongeDigest )
+      bool( digest == rootDigest )
+
+  test "Handles partial chunk at the end":
+
+    let bytes = newSeqWith(63*KB, rand(byte))
+    var leaves: seq[Poseidon2Hash]
+    for i in 0..<31:
+      let
+        chunk = bytes[(i*2*KB)..<((i+1)*2*KB)]
+        digest = Sponge.digest(chunk, rate = 2)
+      leaves.add(digest)
+
+    let partialChunk = bytes[(62*KB)..<(63*KB)]
+    leaves.add(Sponge.digest(partialChunk, rate = 2))
+
+    let
+      digest = Poseidon2Tree.digest(bytes, chunkSize = 2*KB).tryGet
+      spongeDigest = SpongeMerkle.digest(bytes, chunkSize = 2*KB)
+      codexPosTree = Poseidon2Tree.init(leaves).tryGet
+      rootDigest = codexPosTree.root.tryGet
+
+    check:
+      bool( digest == spongeDigest )
+      bool( digest == rootDigest )

tests/codex/merkletree/testmerkletree.nim

@@ -1,245 +0,0 @@
-import std/unittest
-import std/sequtils
-import std/tables
-
-import pkg/questionable/results
-import pkg/stew/byteutils
-import pkg/nimcrypto/sha2
-
-import pkg/codex/merkletree
-import ../helpers
-
-checksuite "merkletree":
-  const data =
-    [
-      "0123456789012345678901234567890123456789".toBytes,
-      "1234567890123456789012345678901234567890".toBytes,
-      "2345678901234567890123456789012345678901".toBytes,
-      "3456789012345678901234567890123456789012".toBytes,
-      "4567890123456789012345678901234567890123".toBytes,
-      "5678901234567890123456789012345678901234".toBytes,
-      "6789012345678901234567890123456789012345".toBytes,
-      "7890123456789012345678901234567890123456".toBytes,
-      "8901234567890123456789012345678901234567".toBytes,
-      "9012345678901234567890123456789012345678".toBytes,
-    ]
-
-  const sha256 = multiCodec("sha2-256")
-  const sha512 = multiCodec("sha2-512")
-
-  proc combine(a, b: MultiHash, codec: MultiCodec = sha256): MultiHash =
-    var buf = newSeq[byte](a.size + b.size)
-    copyMem(addr buf[0], unsafeAddr a.data.buffer[a.dpos], a.size)
-    copyMem(addr buf[a.size], unsafeAddr b.data.buffer[b.dpos], b.size)
-    return MultiHash.digest($codec, buf).tryGet()
-
-  var zeroHash: MultiHash
-  var oneHash: MultiHash
-
-  var expectedLeaves: array[data.len, MultiHash]
-  var builder: MerkleTreeBuilder
-
-  setup:
-    for i in 0..<data.len:
-      expectedLeaves[i] = MultiHash.digest($sha256, data[i]).tryGet()
-
-    builder = MerkleTreeBuilder.init(sha256).tryGet()
-    var zero: array[32, byte]
-    var one: array[32, byte]
-    one[^1] = 0x01
-    zeroHash = MultiHash.init($sha256, zero).tryGet()
-    oneHash = MultiHash.init($sha256, one).tryGet()
-
-  test "cannot init tree without any leafs":
-    let treeOrErr = MerkleTree.init(newSeq[MultiHash]())
-
-    check:
-      treeOrErr.isErr
-
-  test "tree with one leaf has expected structure":
-    builder.addDataBlock(data[0]).tryGet()
-
-    let tree = builder.build().tryGet()
-
-    check:
-      tree.leaves.toSeq == expectedLeaves[0..0]
-      tree.root == expectedLeaves[0]
-      tree.len == 1
-
-  test "tree with two leaves has expected structure":
-    builder.addDataBlock(data[0]).tryGet()
-    builder.addDataBlock(data[1]).tryGet()
-
-    let tree = builder.build().tryGet()
-
-    let expectedRoot = combine(expectedLeaves[0], expectedLeaves[1])
-
-    check:
-      tree.leaves.toSeq == expectedLeaves[0..1]
-      tree.len == 3
-      tree.root == expectedRoot
-
-  test "tree with three leaves has expected structure":
-    builder.addDataBlock(data[0]).tryGet()
-    builder.addDataBlock(data[1]).tryGet()
-    builder.addDataBlock(data[2]).tryGet()
-
-    let tree = builder.build().tryGet()
-
-    let
-      expectedRoot = combine(
-        combine(expectedLeaves[0], expectedLeaves[1]),
-        combine(expectedLeaves[2], zeroHash)
-      )
-
-    check:
-      tree.leaves.toSeq == expectedLeaves[0..2]
-      tree.len == 6
-      tree.root == expectedRoot
-
-  test "tree with nine leaves has expected structure":
-    builder.addDataBlock(data[0]).tryGet()
-    builder.addDataBlock(data[1]).tryGet()
-    builder.addDataBlock(data[2]).tryGet()
-    builder.addDataBlock(data[3]).tryGet()
-    builder.addDataBlock(data[4]).tryGet()
-    builder.addDataBlock(data[5]).tryGet()
-    builder.addDataBlock(data[6]).tryGet()
-    builder.addDataBlock(data[7]).tryGet()
-    builder.addDataBlock(data[8]).tryGet()
-
-    let tree = builder.build().tryGet()
-
-    let
-      expectedRoot = combine(
-        combine(
-          combine(
-            combine(expectedLeaves[0], expectedLeaves[1]),
-            combine(expectedLeaves[2], expectedLeaves[3]),
-          ),
-          combine(
-            combine(expectedLeaves[4], expectedLeaves[5]),
-            combine(expectedLeaves[6], expectedLeaves[7])
-          )
-        ),
-        combine(
-          combine(
-            combine(expectedLeaves[8], zeroHash),
-            oneHash
-          ),
-          oneHash
-        )
-      )
-
-    check:
-      tree.leaves.toSeq == expectedLeaves[0..8]
-      tree.len == 20
-      tree.root == expectedRoot
-
-  test "tree with two leaves provides expected and valid proofs":
-    builder.addDataBlock(data[0]).tryGet()
-    builder.addDataBlock(data[1]).tryGet()
-
-    let tree = builder.build().tryGet()
-
-    let expectedProofs = [
-      MerkleProof.init(0, @[expectedLeaves[1]]).tryGet(),
-      MerkleProof.init(1, @[expectedLeaves[0]]).tryGet(),
-    ]
-
-    check:
-      tree.getProof(0).tryGet() == expectedProofs[0]
-      tree.getProof(1).tryGet() == expectedProofs[1]
-
-    check:
-      tree.getProof(0).tryGet().verifyDataBlock(data[0], tree.root).tryGet()
-      tree.getProof(1).tryGet().verifyDataBlock(data[1], tree.root).tryGet()
-
-  test "tree with three leaves provides expected proofs":
-    builder.addDataBlock(data[0]).tryGet()
-    builder.addDataBlock(data[1]).tryGet()
-    builder.addDataBlock(data[2]).tryGet()
-
-    let tree = builder.build().tryGet()
-
-    let expectedProofs = [
-      MerkleProof.init(0, @[expectedLeaves[1], combine(expectedLeaves[2], zeroHash)]).tryGet(),
-      MerkleProof.init(1, @[expectedLeaves[0], combine(expectedLeaves[2], zeroHash)]).tryGet(),
-      MerkleProof.init(2, @[zeroHash, combine(expectedLeaves[0], expectedLeaves[1])]).tryGet(),
-    ]
-
-    check:
-      tree.getProof(0).tryGet() == expectedProofs[0]
-      tree.getProof(1).tryGet() == expectedProofs[1]
-      tree.getProof(2).tryGet() == expectedProofs[2]
-
-    check:
-      tree.getProof(0).tryGet().verifyDataBlock(data[0], tree.root).tryGet()
-      tree.getProof(1).tryGet().verifyDataBlock(data[1], tree.root).tryGet()
-      tree.getProof(2).tryGet().verifyDataBlock(data[2], tree.root).tryGet()
-
-  test "tree with nine leaves provides expected proofs":
-    builder.addDataBlock(data[0]).tryGet()
-    builder.addDataBlock(data[1]).tryGet()
-    builder.addDataBlock(data[2]).tryGet()
-    builder.addDataBlock(data[3]).tryGet()
-    builder.addDataBlock(data[4]).tryGet()
-    builder.addDataBlock(data[5]).tryGet()
-    builder.addDataBlock(data[6]).tryGet()
-    builder.addDataBlock(data[7]).tryGet()
-    builder.addDataBlock(data[8]).tryGet()
-
-    let tree = builder.build().tryGet()
-
-    let expectedProofs = {
-      4:
-        MerkleProof.init(4, @[
-          expectedLeaves[5],
-          combine(expectedLeaves[6], expectedLeaves[7]),
-          combine(
-              combine(expectedLeaves[0], expectedLeaves[1]),
-              combine(expectedLeaves[2], expectedLeaves[3]),
-          ),
-          combine(
-            combine(
-              combine(expectedLeaves[8], zeroHash),
-              oneHash
-            ),
-            oneHash
-          )
-        ]).tryGet(),
-      8:
-        MerkleProof.init(8, @[
-          zeroHash,
-          oneHash,
-          oneHash,
-          combine(
-            combine(
-              combine(expectedLeaves[0], expectedLeaves[1]),
-              combine(expectedLeaves[2], expectedLeaves[3]),
-            ),
-            combine(
-              combine(expectedLeaves[4], expectedLeaves[5]),
-              combine(expectedLeaves[6], expectedLeaves[7])
-            )
-          )
-        ]).tryGet(),
-    }.newTable
-
-    check:
-      tree.getProof(4).tryGet() == expectedProofs[4]
-      tree.getProof(8).tryGet() == expectedProofs[8]
-
-    check:
-      tree.getProof(4).tryGet().verifyDataBlock(data[4], tree.root).tryGet()
-      tree.getProof(8).tryGet().verifyDataBlock(data[8], tree.root).tryGet()
-
-  test "getProof fails for index out of bounds":
-    builder.addDataBlock(data[0]).tryGet()
-    builder.addDataBlock(data[1]).tryGet()
-    builder.addDataBlock(data[2]).tryGet()
-
-    let tree = builder.build().tryGet()
-
-    check:
-      isErr(tree.getProof(4))

tests/codex/merkletree/testposeidon2tree.nim

@@ -0,0 +1,88 @@
+import std/unittest
+import std/sequtils
+import std/sugar
+
+import pkg/poseidon2
+import pkg/poseidon2/io
+import pkg/questionable/results
+import pkg/results
+import pkg/stew/byteutils
+import pkg/stew/arrayops
+import constantine/math/arithmetic
+import constantine/math/io/io_bigints
+import pkg/constantine/math/io/io_fields
+import pkg/constantine/platforms/abstractions
+
+import pkg/codex/merkletree
+
+import ./generictreetests
+import ./helpers
+
+const
+  data =
+    [
+      "0000000000000000000000000000001".toBytes,
+      "0000000000000000000000000000002".toBytes,
+      "0000000000000000000000000000003".toBytes,
+      "0000000000000000000000000000004".toBytes,
+      "0000000000000000000000000000005".toBytes,
+      "0000000000000000000000000000006".toBytes,
+      "0000000000000000000000000000007".toBytes,
+      "0000000000000000000000000000008".toBytes,
+      "0000000000000000000000000000009".toBytes,
+      "0000000000000000000000000000010".toBytes,
+    ]
+
+suite "Test CodexTree":
+  var
+    expectedLeaves: seq[Poseidon2Hash]
+
+  setup:
+    expectedLeaves = toSeq( data.concat().elements(Poseidon2Hash) )
+
+  test "Should fail init tree from empty leaves":
+    check:
+      Poseidon2Tree.init( leaves = newSeq[Poseidon2Hash](0) ).isErr
+
+  test "Init tree from poseidon2 leaves":
+    let
+      tree = Poseidon2Tree.init( leaves = expectedLeaves ).tryGet
+
+    check:
+      tree.leaves == expectedLeaves
+
+  test "Init tree from byte leaves":
+    let
+      tree = Poseidon2Tree.init(
+        leaves = data.mapIt(
+          array[31, byte].initCopyFrom( it )
+        )).tryGet
+
+    check:
+      tree.leaves == expectedLeaves
+
+  test "Should build from nodes":
+    let
+      tree = Poseidon2Tree.init(leaves = expectedLeaves).tryGet
+      fromNodes = Poseidon2Tree.fromNodes(
+        nodes = toSeq(tree.nodes),
+        nleaves = tree.leavesCount).tryGet
+
+    check:
+      tree == fromNodes
+
+let
+  compressor = proc(
+    x, y: Poseidon2Hash,
+    key: PoseidonKeysEnum): Poseidon2Hash {.noSideEffect.} =
+    compress(x, y, key.toKey)
+
+  makeTree = proc(data: seq[Poseidon2Hash]): Poseidon2Tree =
+    Poseidon2Tree.init(leaves = data).tryGet
+
+testGenericTree(
+  "Poseidon2Tree",
+  toSeq( data.concat().elements(Poseidon2Hash) ),
+  zero,
+  compressor,
+  makeTree)

tests/codex/stores/commonstoretests.nim

@@ -30,7 +30,7 @@ proc commonBlockStoreTests*(name: string,
     var
       newBlock, newBlock1, newBlock2, newBlock3: Block
       manifest: Manifest
-      tree: MerkleTree
+      tree: CodexTree
       store: BlockStore
 
     setup:

tests/codex/testmerkletree.nim

@@ -1,4 +1,6 @@
-import ./merkletree/testmerkletree
+import ./merkletree/testcodextree
-import ./merkletree/testcoders
+import ./merkletree/testposeidon2tree
+import ./merkletree/testcodexcoders
+import ./merkletree/testmerkledigest
 
 {.warning[UnusedImport]: off.}

tests/integration/testIntegration.nim

@@ -51,7 +51,7 @@ twonodessuite "Integration tests", debug1 = false, debug2 = false:
     check:
       space.totalBlocks == 2.uint
       space.quotaMaxBytes == 8589934592.uint
-      space.quotaUsedBytes == 65518.uint
+      space.quotaUsedBytes == 65526.uint
       space.quotaReservedBytes == 12.uint
 
   test "node allows local file downloads":

vendor/nim-libp2p

@@ -1 +1 @@
-Subproject commit 440461b24b9e66542b34d26a0b908c17f6549d05
+Subproject commit 4f2259e1cef65085d092b2b713bb67f5aac55626

vendor/nim-poseidon2

@@ -1 +1 @@
-Subproject commit 9be7b0c134e64e3d57a38520a32af93a55a37c44
+Subproject commit 0cfecf7d780b8c3295d11251d64a49f3c7258fbf