diff --git a/codex/contentids_exts.nim b/codex/contentids_exts.nim new file mode 100644 index 00000000..9ef6fbb5 --- /dev/null +++ b/codex/contentids_exts.nim @@ -0,0 +1,8 @@ +const ContentIdsExts = [ + multiCodec("codex-root"), + multiCodec("codex-manifest"), + multiCodec("codex-block"), + multiCodec("codex-slot-root"), + multiCodec("codex-proving-root"), + multiCodec("codex-slot-cell"), +] diff --git a/codex/merkletree/codex/codex.nim b/codex/merkletree/codex/codex.nim index 0eec92e4..bf9f39d6 100644 --- a/codex/merkletree/codex/codex.nim +++ b/codex/merkletree/codex/codex.nim @@ -47,28 +47,6 @@ type CodexProof* = ref object of ByteProof mcodec*: MultiCodec -# CodeHashes is not exported from libp2p -# So we need to recreate it instead of -proc initMultiHashCodeTable(): Table[MultiCodec, MHash] {.compileTime.} = - for item in HashesList: - result[item.mcodec] = item - -const CodeHashes = initMultiHashCodeTable() - -func mhash*(mcodec: MultiCodec): ?!MHash = - let mhash = CodeHashes.getOrDefault(mcodec) - - if isNil(mhash.coder): - return failure "Invalid multihash codec" - - success mhash - -func digestSize*(self: (CodexTree or CodexProof)): int = - ## Number of leaves - ## - - self.mhash.size - func getProof*(self: CodexTree, index: int): ?!CodexProof = var proof = CodexProof(mcodec: self.mcodec) @@ -128,17 +106,12 @@ proc `$`*(self: CodexProof): string = "CodexProof(" & " nleaves: " & $self.nleaves & ", index: " & $self.index & ", path: " & $self.path.mapIt(byteutils.toHex(it)) & ", mcodec: " & $self.mcodec & " )" -func compress*(x, y: openArray[byte], key: ByteTreeKey, mhash: MHash): ?!ByteHash = +func compress*(x, y: openArray[byte], key: ByteTreeKey, codec: MultiCodec): ?!ByteHash = ## Compress two hashes ## - - # Using Constantine's SHA256 instead of mhash for optimal performance on 32-byte merkle node hashing - # See: https://github.com/codex-storage/nim-codex/issues/1162 - let input = @x & @y & @[key.byte] - var digest = hashes.sha256.hash(input) - - success @digest + let digest = ?MultiHash.digest(codec, input).mapFailure + success digest.digestBytes func init*( _: type CodexTree, mcodec: MultiCodec = Sha256HashCodec, leaves: openArray[ByteHash] @@ -147,12 +120,12 @@ func init*( return failure "Empty leaves" let - mhash = ?mcodec.mhash() compressor = proc(x, y: seq[byte], key: ByteTreeKey): ?!ByteHash {.noSideEffect.} = - compress(x, y, key, mhash) - Zero: ByteHash = newSeq[byte](mhash.size) + compress(x, y, key, mcodec) + digestSize = ?mcodec.digestSize.mapFailure + Zero: ByteHash = newSeq[byte](digestSize) - if mhash.size != leaves[0].len: + if digestSize != leaves[0].len: return failure "Invalid hash length" var self = CodexTree(mcodec: mcodec, compress: compressor, zero: Zero) @@ -190,12 +163,12 @@ proc fromNodes*( return failure "Empty nodes" let - mhash = ?mcodec.mhash() - Zero = newSeq[byte](mhash.size) + digestSize = ?mcodec.digestSize.mapFailure + Zero = newSeq[byte](digestSize) compressor = proc(x, y: seq[byte], key: ByteTreeKey): ?!ByteHash {.noSideEffect.} = - compress(x, y, key, mhash) + compress(x, y, key, mcodec) - if mhash.size != nodes[0].len: + if digestSize != nodes[0].len: return failure "Invalid hash length" var @@ -228,10 +201,10 @@ func init*( return failure "Empty nodes" let - mhash = ?mcodec.mhash() - Zero = newSeq[byte](mhash.size) + digestSize = ?mcodec.digestSize.mapFailure + Zero = newSeq[byte](digestSize) compressor = proc(x, y: seq[byte], key: ByteTreeKey): ?!seq[byte] {.noSideEffect.} = - compress(x, y, key, mhash) + compress(x, y, key, mcodec) success CodexProof( compress: compressor, diff --git a/codex/multicodec_exts.nim b/codex/multicodec_exts.nim new file mode 100644 index 00000000..b14fbbe6 --- /dev/null +++ b/codex/multicodec_exts.nim @@ -0,0 +1,11 @@ +const CodecExts = [ + ("poseidon2-alt_bn_128-sponge-r2", 0xCD10), # bn128 rate 2 sponge + ("poseidon2-alt_bn_128-merkle-2kb", 0xCD11), # bn128 2kb compress & merkleize + ("poseidon2-alt_bn_128-keyed-compress", 0xCD12), # bn128 keyed compress] + ("codex-manifest", 0xCD01), + ("codex-block", 0xCD02), + ("codex-root", 0xCD03), + ("codex-slot-root", 0xCD04), + ("codex-proving-root", 0xCD05), + ("codex-slot-cell", 0xCD06), +] diff --git a/codex/multihash_exts.nim b/codex/multihash_exts.nim new file mode 100644 index 00000000..e9477177 --- /dev/null +++ b/codex/multihash_exts.nim @@ -0,0 +1,40 @@ +import blscurve/bls_public_exports +import pkg/constantine/hashes +import poseidon2 + +proc sha2_256hash_constantine(data: openArray[byte], output: var openArray[byte]) = + # Using Constantine's SHA256 instead of mhash for optimal performance on 32-byte merkle node hashing + # See: https://github.com/codex-storage/nim-codex/issues/1162 + if len(output) > 0: + let digest = hashes.sha256.hash(data) + copyMem(addr output[0], addr digest[0], 32) + +proc poseidon2_sponge_rate2(data: openArray[byte], output: var openArray[byte]) = + if len(output) > 0: + var digest = poseidon2.Sponge.digest(data).toBytes() + copyMem(addr output[0], addr digest[0], uint(len(output))) + +proc poseidon2_merkle_2kb_sponge(data: openArray[byte], output: var openArray[byte]) = + if len(output) > 0: + var digest = poseidon2.SpongeMerkle.digest(data, 2048).toBytes() + copyMem(addr output[0], addr digest[0], uint(len(output))) + +const Sha2256MultiHash* = MHash( + mcodec: multiCodec("sha2-256"), + size: sha256.sizeDigest, + coder: sha2_256hash_constantine, +) +const HashExts = [ + # override sha2-256 hash function + Sha2256MultiHash, + MHash( + mcodec: multiCodec("poseidon2-alt_bn_128-sponge-r2"), + size: 32, + coder: poseidon2_sponge_rate2, + ), + MHash( + mcodec: multiCodec("poseidon2-alt_bn_128-merkle-2kb"), + size: 32, + coder: poseidon2_merkle_2kb_sponge, + ), +] diff --git a/config.nims b/config.nims index 8700f60e..b1bc4cbe 100644 --- a/config.nims +++ b/config.nims @@ -92,6 +92,12 @@ else: on --warningAsError: "ProveField:on" +--define: + "libp2p_multicodec_exts:../../../codex/multicodec_exts.nim" +--define: + "libp2p_multihash_exts:../../../codex/multihash_exts.nim" +--define: + "libp2p_contentids_exts:../../../codex/contentids_exts.nim" when (NimMajor, NimMinor) >= (1, 4): --warning: diff --git a/tests/codex/merkletree/testcodextree.nim b/tests/codex/merkletree/testcodextree.nim index 29390c16..91ccd7f5 100644 --- a/tests/codex/merkletree/testcodextree.nim +++ b/tests/codex/merkletree/testcodextree.nim @@ -82,10 +82,10 @@ suite "Test CodexTree": tree == fromNodes let - mhash = sha256.mhash().tryGet - zero: seq[byte] = newSeq[byte](mhash.size) + digestSize = sha256.digestSize.get + zero: seq[byte] = newSeq[byte](digestSize) compress = proc(x, y: seq[byte], key: ByteTreeKey): seq[byte] = - compress(x, y, key, mhash).tryGet + compress(x, y, key, sha256).tryGet makeTree = proc(data: seq[seq[byte]]): CodexTree = CodexTree.init(sha256, leaves = data).tryGet diff --git a/vendor/nim-libp2p b/vendor/nim-libp2p index 7eaf79fe..aa8ce46f 160000 --- a/vendor/nim-libp2p +++ b/vendor/nim-libp2p @@ -1 +1 @@ -Subproject commit 7eaf79fefe45b03a4281e3505d5fcc97b32df39c +Subproject commit aa8ce46f782240cb99a7222c474022b8cfd24e52