mirror of
https://github.com/codex-storage/nim-codex.git
synced 2025-02-13 05:17:58 +00:00
rework merkle tree support
This commit is contained in:
Dmitriy Ryajov
parent
c0f4512f62
commit
2736d78619
4
codex/merkletree/codexmerkletree.nim
Normal file
4
codex/merkletree/codexmerkletree.nim
Normal file
@ -0,0 +1,4 @@
|
||||
import ./codexmerkletree/codexmerkletree
|
||||
import ./codexmerkletree/coders
|
||||
|
||||
export codexmerkletree, coders
|
||||
110
codex/merkletree/codexmerkletree/coders.nim
Normal file
110
codex/merkletree/codexmerkletree/coders.nim
Normal file
@ -0,0 +1,110 @@
|
||||
## Nim-Codex
|
||||
## Copyright (c) 2023 Status Research & Development GmbH
|
||||
## Licensed under either of
|
||||
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
|
||||
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
|
||||
## at your option.
|
||||
## This file may not be copied, modified, or distributed except according to
|
||||
## those terms.
|
||||
|
||||
import pkg/upraises
|
||||
|
||||
push: {.upraises: [].}
|
||||
|
||||
import std/sequtils
|
||||
|
||||
import pkg/libp2p
|
||||
import pkg/questionable
|
||||
import pkg/questionable/results
|
||||
|
||||
import ../../units
|
||||
import ../../errors
|
||||
|
||||
import ./codexmerkletree
|
||||
|
||||
const MaxMerkleTreeSize = 100.MiBs.uint
|
||||
const MaxMerkleProofSize = 1.MiBs.uint
|
||||
|
||||
proc encode*(self: CodexMerkleTree): seq[byte] =
|
||||
var pb = initProtoBuffer(maxSize = MaxMerkleTreeSize)
|
||||
pb.write(1, self.mcodec.uint64)
|
||||
pb.write(2, self.digestSize.uint64)
|
||||
pb.write(3, self.leavesCount.uint64)
|
||||
var nodesPb = initProtoBuffer(maxSize = MaxMerkleTreeSize)
|
||||
for node in self.nodes:
|
||||
nodesPb.write(1, node)
|
||||
nodesPb.finish()
|
||||
pb.write(4, nodesPb)
|
||||
|
||||
pb.finish
|
||||
pb.buffer
|
||||
|
||||
proc decode*(_: type CodexMerkleTree, data: seq[byte]): ?!CodexMerkleTree =
|
||||
var pb = initProtoBuffer(data, maxSize = MaxMerkleTreeSize)
|
||||
var mcodecCode: uint64
|
||||
var digestSize: uint64
|
||||
var leavesCount: uint64
|
||||
discard ? pb.getField(1, mcodecCode).mapFailure
|
||||
discard ? pb.getField(2, digestSize).mapFailure
|
||||
discard ? pb.getField(3, leavesCount).mapFailure
|
||||
|
||||
let mcodec = MultiCodec.codec(mcodecCode.int)
|
||||
if mcodec == InvalidMultiCodec:
|
||||
return failure("Invalid MultiCodec code " & $mcodecCode)
|
||||
|
||||
var
|
||||
nodesBuff: seq[seq[byte]]
|
||||
nodes: seq[ByteHash]
|
||||
|
||||
if ? pb.getRepeatedField(4, nodesBuff).mapFailure:
|
||||
for nodeBuff in nodesBuff:
|
||||
var node: ByteHash
|
||||
let nodePb = initProtoBuffer(nodeBuff)
|
||||
discard ? nodePb.getField(1, node).mapFailure
|
||||
nodes.add node
|
||||
|
||||
let tree = ? CodexMerkleTree.fromNodes(mcodec, digestSize, leavesCount, nodesBuffer)
|
||||
success(tree)
|
||||
|
||||
proc encode*(self: CodexMerkleProof): seq[byte] =
|
||||
var pb = initProtoBuffer(maxSize = MaxMerkleProofSize)
|
||||
pb.write(1, self.mcodec.uint64)
|
||||
pb.write(2, self.digestSize.uint64)
|
||||
pb.write(3, self.index.uint64)
|
||||
var nodesPb = initProtoBuffer(maxSize = MaxMerkleTreeSize)
|
||||
for node in self.path:
|
||||
nodesPb.write(1, node)
|
||||
nodesPb.finish()
|
||||
pb.write(4, nodesPb)
|
||||
pb.finish
|
||||
pb.buffer
|
||||
|
||||
proc decode*(_: type CodexMerkleProof, data: seq[byte]): ?!CodexMerkleProof =
|
||||
var pb = initProtoBuffer(data, maxSize = MaxMerkleProofSize)
|
||||
var mcodecCode: uint64
|
||||
var digestSize: uint64
|
||||
var index: uint64
|
||||
discard ? pb.getField(1, mcodecCode).mapFailure
|
||||
|
||||
let mcodec = MultiCodec.codec(mcodecCode.int)
|
||||
if mcodec == InvalidMultiCodec:
|
||||
return failure("Invalid MultiCodec code " & $mcodecCode)
|
||||
|
||||
discard ? pb.getField(2, digestSize).mapFailure
|
||||
discard ? pb.getField(3, index).mapFailure
|
||||
|
||||
var
|
||||
nodesBuff: seq[seq[byte]]
|
||||
nodes: seq[ByteHash]
|
||||
|
||||
if ? pb.getRepeatedField(4, nodesBuff).mapFailure:
|
||||
for nodeBuff in nodesBuff:
|
||||
var node: ByteHash
|
||||
let nodePb = initProtoBuffer(nodeBuff)
|
||||
discard ? nodePb.getField(1, node).mapFailure
|
||||
nodes.add node
|
||||
|
||||
let
|
||||
proof = ? CodexMerkleProof.init(mcodec, index.int, nodes)
|
||||
|
||||
success(proof)
|
||||
245
codex/merkletree/codexmerkletree/codexmerkletree.nim
Normal file
245
codex/merkletree/codexmerkletree/codexmerkletree.nim
Normal file
@ -0,0 +1,245 @@
|
||||
## Nim-Codex
|
||||
## Copyright (c) 2023 Status Research & Development GmbH
|
||||
## Licensed under either of
|
||||
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
|
||||
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
|
||||
## at your option.
|
||||
## This file may not be copied, modified, or distributed except according to
|
||||
## those terms.
|
||||
|
||||
import pkg/upraises
|
||||
|
||||
push: {.upraises: [].}
|
||||
|
||||
import std/math
|
||||
import std/bitops
|
||||
import std/sequtils
|
||||
import std/sugar
|
||||
import std/algorithm
|
||||
import std/tables
|
||||
|
||||
import pkg/chronicles
|
||||
import pkg/questionable
|
||||
import pkg/questionable/results
|
||||
import pkg/libp2p/[cid, multicodec, multihash]
|
||||
import pkg/stew/byteutils
|
||||
|
||||
import ../../errors
|
||||
import ../../blocktype
|
||||
|
||||
import ../merkletree
|
||||
|
||||
export merkletree
|
||||
|
||||
logScope:
|
||||
topics = "codex merkletree"
|
||||
|
||||
type
|
||||
ByteTreeKey* {.pure.} = enum
|
||||
KeyNone = 0x0.byte
|
||||
KeyBottomLayer = 0x1.byte
|
||||
KeyOdd = 0x2.byte
|
||||
KeyOddAndBottomLayer = 0x3.byte
|
||||
|
||||
ByteHash* = seq[byte]
|
||||
ByteTree* = MerkleTree[ByteHash, ByteTreeKey]
|
||||
ByteTreeProof* = MerkleProof[ByteHash, ByteTreeKey]
|
||||
|
||||
CodexMerkleTree* = object of ByteTree
|
||||
mhash: MHash
|
||||
|
||||
CodexMerkleProof* = object of ByteTreeProof
|
||||
mhash: MHash
|
||||
|
||||
func getMhash*(mcodec: MultiCodec): ?!MHash =
|
||||
let
|
||||
mhash = CodeHashes.getOrDefault(mcodec)
|
||||
|
||||
if isNil(mhash.coder):
|
||||
return failure "Invalid multihash codec"
|
||||
|
||||
success mhash
|
||||
|
||||
func digestSize*(self: (CodexMerkleTree or CodexMerkleProof)): int =
|
||||
## Number of leaves
|
||||
##
|
||||
|
||||
self.mhash.size
|
||||
|
||||
func mcodec*(self: (CodexMerkleTree or CodexMerkleProof)): MultiCodec =
|
||||
## Multicodec
|
||||
##
|
||||
|
||||
self.mhash.mcodec
|
||||
|
||||
func bytes*(mhash: MultiHash): seq[byte] =
|
||||
## Extract hash bytes
|
||||
##
|
||||
|
||||
mhash.data.buffer[mhash.dpos..<mhash.dpos + mhash.size]
|
||||
|
||||
func getProof*(self: CodexMerkleTree, index: int): ?!CodexMerkleProof =
|
||||
var
|
||||
proof = CodexMerkleProof(mhash: self.mhash)
|
||||
|
||||
self.getProof(index, proof)
|
||||
|
||||
success proof
|
||||
|
||||
func verify*(self: CodexMerkleProof, root: MultiHash): ?!void =
|
||||
## Verify hash
|
||||
##
|
||||
|
||||
let
|
||||
bytes = root.bytes
|
||||
|
||||
if self.mcodec != root.mcodec:
|
||||
return failure "Hash codec mismatch"
|
||||
|
||||
if bytes.len != root.size:
|
||||
return failure "Invalid hash length"
|
||||
|
||||
? self.verify(bytes)
|
||||
|
||||
success()
|
||||
|
||||
proc rootCid*(
|
||||
self: CodexMerkleTree,
|
||||
version = CIDv1,
|
||||
dataCodec = DatasetRootCodec): ?!Cid =
|
||||
|
||||
if self.root.len == 0:
|
||||
return failure "Empty root"
|
||||
|
||||
Cid.init(
|
||||
version,
|
||||
dataCodec,
|
||||
? MultiHash.init(self.mcodec, self.root).mapFailure).mapFailure
|
||||
|
||||
func getLeafCid*(
|
||||
self: CodexMerkleTree,
|
||||
i: Natural,
|
||||
version = CIDv1,
|
||||
dataCodec = BlockCodec): ?!Cid =
|
||||
|
||||
if i >= self.leavesCount:
|
||||
return failure "Invalid leaf index " & $i
|
||||
|
||||
let
|
||||
leaf = self.leaves[i]
|
||||
|
||||
Cid.init(
|
||||
CidVersion.CIDv1,
|
||||
dataCodec,
|
||||
? MultiHash.init(self.mcodec, self.root).mapFailure).mapFailure
|
||||
|
||||
func compress*(
|
||||
x, y: openArray[byte],
|
||||
key: ByteTreeKey,
|
||||
mhash: MHash): ?!ByteHash =
|
||||
## Compress two hashes
|
||||
##
|
||||
|
||||
var digest = newSeq[byte](mhash.size)
|
||||
mhash.coder(@x & @y & @[ key.byte ], digest)
|
||||
success digest
|
||||
|
||||
func init*(
|
||||
_: type CodexMerkleTree,
|
||||
mcodec: MultiCodec,
|
||||
leaves: openArray[ByteHash]): ?!CodexMerkleTree =
|
||||
|
||||
if leaves.len == 0:
|
||||
return failure "Empty leaves"
|
||||
|
||||
let
|
||||
mhash = ? mcodec.getMhash()
|
||||
compressor = proc(x, y: seq[byte], key: ByteTreeKey): ?!ByteHash {.noSideEffect.} =
|
||||
compress(x, y, key, mhash)
|
||||
Zero: ByteHash = newSeq[byte](mhash.size)
|
||||
|
||||
if mhash.size != leaves[0].len:
|
||||
return failure "Invalid hash length"
|
||||
|
||||
var
|
||||
self = CodexMerkleTree(mhash: mhash, compress: compressor, zero: Zero)
|
||||
|
||||
self.layers = ? merkleTreeWorker(self, leaves, isBottomLayer = true)
|
||||
success self
|
||||
|
||||
func init*(
|
||||
_: type CodexMerkleTree,
|
||||
leaves: openArray[MultiHash]): ?!CodexMerkleTree =
|
||||
|
||||
if leaves.len == 0:
|
||||
return failure "Empty leaves"
|
||||
|
||||
let
|
||||
mcodec = leaves[0].mcodec
|
||||
leaves = leaves.mapIt( it.bytes )
|
||||
|
||||
CodexMerkleTree.init(mcodec, leaves)
|
||||
|
||||
func init*(
|
||||
_: type CodexMerkleTree,
|
||||
leaves: openArray[Cid]): ?!CodexMerkleTree =
|
||||
if leaves.len == 0:
|
||||
return failure "Empty leaves"
|
||||
|
||||
let
|
||||
mcodec = (? leaves[0].mhash.mapFailure).mcodec
|
||||
leaves = leaves.mapIt( (? it.mhash.mapFailure).bytes )
|
||||
|
||||
CodexMerkleTree.init(mcodec, leaves)
|
||||
|
||||
func fromNodes*(
|
||||
_: type CodexMerkleTree,
|
||||
mcodec: MultiCodec,
|
||||
nodes: openArray[seq[ByteHash]],
|
||||
nleaves: int): ?!CodexMerkleTree =
|
||||
|
||||
if nodes.len == 0:
|
||||
return failure "Empty nodes"
|
||||
|
||||
let
|
||||
mhash = ? mcodec.getMhash()
|
||||
Zero = newSeq[ByteHash](mhash.size)
|
||||
compressor = proc(x, y: openArray[byte], key: ByteTreeKey): ?!ByteHash {.noSideEffect.} =
|
||||
compress(x, y, key, mhash)
|
||||
|
||||
if mhash.size != nodes[0].len:
|
||||
return failure "Invalid hash length"
|
||||
|
||||
let
|
||||
self = CodexMerkleTree(compress: compressor, zero: Zero, mhash: mhash)
|
||||
|
||||
var
|
||||
layer = nleaves
|
||||
pos = 0
|
||||
|
||||
while layer > 0:
|
||||
self.layers.add( nodes[pos..<layer].toSeq() )
|
||||
pos += layer
|
||||
layer = layer shr 1
|
||||
|
||||
? self.proof(Rng.instance.rand(nleaves)).?verify(self.root) # sanity check
|
||||
|
||||
success self
|
||||
|
||||
func init*(
|
||||
_: type CodexMerkleProof,
|
||||
mcodec: MultiCodec,
|
||||
index: int,
|
||||
nodes: openArray[ByteHash]): ?!CodexMerkleProof =
|
||||
|
||||
if nodes.len == 0:
|
||||
return failure "Empty nodes"
|
||||
|
||||
let
|
||||
mhash = ? mcodec.getMhash()
|
||||
Zero: ByteHash = newSeq[byte](mhash.size)
|
||||
compressor = proc(x, y: seq[byte], key: ByteTreeKey): ?!seq[byte] {.noSideEffect.} =
|
||||
compress(x, y, key, mhash)
|
||||
self = CodexMerkleProof(compress: compressor, zero: Zero, mhash: mhash)
|
||||
|
||||
success self
|
||||
418
codex/merkletree/merkletreeold.nim
Normal file
418
codex/merkletree/merkletreeold.nim
Normal file
@ -0,0 +1,418 @@
|
||||
## Nim-Codex
|
||||
## Copyright (c) 2023 Status Research & Development GmbH
|
||||
## Licensed under either of
|
||||
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
|
||||
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
|
||||
## at your option.
|
||||
## This file may not be copied, modified, or distributed except according to
|
||||
## those terms.
|
||||
|
||||
import std/math
|
||||
import std/bitops
|
||||
import std/sequtils
|
||||
import std/sugar
|
||||
import std/algorithm
|
||||
|
||||
import pkg/chronicles
|
||||
import pkg/questionable
|
||||
import pkg/questionable/results
|
||||
import pkg/nimcrypto/sha2
|
||||
import pkg/libp2p/[cid, multicodec, multihash, vbuffer]
|
||||
import pkg/stew/byteutils
|
||||
|
||||
import ../errors
|
||||
|
||||
logScope:
|
||||
topics = "codex merkletree"
|
||||
|
||||
type
|
||||
MerkleTree* = object
|
||||
mcodec: MultiCodec
|
||||
digestSize: Natural
|
||||
leavesCount: Natural
|
||||
nodesBuffer*: seq[byte]
|
||||
MerkleProof* = object
|
||||
mcodec: MultiCodec
|
||||
digestSize: Natural
|
||||
index: Natural
|
||||
nodesBuffer*: seq[byte]
|
||||
MerkleTreeBuilder* = object
|
||||
mcodec: MultiCodec
|
||||
digestSize: Natural
|
||||
buffer: seq[byte]
|
||||
|
||||
###########################################################
|
||||
# Helper functions
|
||||
###########################################################
|
||||
|
||||
func computeTreeHeight(leavesCount: int): int =
|
||||
if isPowerOfTwo(leavesCount):
|
||||
fastLog2(leavesCount) + 1
|
||||
else:
|
||||
fastLog2(leavesCount) + 2
|
||||
|
||||
func computeLevels(leavesCount: int): seq[tuple[offset: int, width: int, index: int]] =
|
||||
let height = computeTreeHeight(leavesCount)
|
||||
var levels = newSeq[tuple[offset: int, width: int, index: int]](height)
|
||||
|
||||
levels[0].offset = 0
|
||||
levels[0].width = leavesCount
|
||||
levels[0].index = 0
|
||||
for i in 1..<height:
|
||||
levels[i].offset = levels[i - 1].offset + levels[i - 1].width
|
||||
levels[i].width = (levels[i - 1].width + 1) div 2
|
||||
levels[i].index = i
|
||||
levels
|
||||
|
||||
proc digestFn(mcodec: MultiCodec, dst: var openArray[byte], dstPos: int, data: openArray[byte]): ?!void =
|
||||
var mhash = ? MultiHash.digest($mcodec, data).mapFailure
|
||||
if (dstPos + mhash.size) > dst.len:
|
||||
return failure("Not enough space in a destination buffer")
|
||||
dst[dstPos..<dstPos + mhash.size] = mhash.data.buffer[mhash.dpos..<mhash.dpos + mhash.size]
|
||||
success()
|
||||
|
||||
###########################################################
|
||||
# MerkleTreeBuilder
|
||||
###########################################################
|
||||
|
||||
proc init*(
|
||||
T: type MerkleTreeBuilder,
|
||||
mcodec: MultiCodec = multiCodec("sha2-256")
|
||||
): ?!MerkleTreeBuilder =
|
||||
let mhash = ? MultiHash.digest($mcodec, "".toBytes).mapFailure
|
||||
success(MerkleTreeBuilder(mcodec: mcodec, digestSize: mhash.size, buffer: newSeq[byte]()))
|
||||
|
||||
proc addDataBlock*(self: var MerkleTreeBuilder, dataBlock: openArray[byte]): ?!void =
|
||||
## Hashes the data block and adds the result of hashing to a buffer
|
||||
##
|
||||
let oldLen = self.buffer.len
|
||||
self.buffer.setLen(oldLen + self.digestSize)
|
||||
digestFn(self.mcodec, self.buffer, oldLen, dataBlock)
|
||||
|
||||
proc addLeaf*(self: var MerkleTreeBuilder, leaf: MultiHash): ?!void =
|
||||
if leaf.mcodec != self.mcodec or leaf.size != self.digestSize:
|
||||
return failure("Expected mcodec to be " & $self.mcodec & " and digest size to be " &
|
||||
$self.digestSize & " but was " & $leaf.mcodec & " and " & $leaf.size)
|
||||
|
||||
let oldLen = self.buffer.len
|
||||
self.buffer.setLen(oldLen + self.digestSize)
|
||||
self.buffer[oldLen..<oldLen + self.digestSize] = leaf.data.buffer[leaf.dpos..<leaf.dpos + self.digestSize]
|
||||
success()
|
||||
|
||||
proc build*(self: MerkleTreeBuilder): ?!MerkleTree =
|
||||
## Builds a tree from previously added data blocks
|
||||
##
|
||||
## Tree built from data blocks A, B and C is
|
||||
## H5=H(H3 & H4)
|
||||
## / \
|
||||
## H3=H(H0 & H1) H4=H(H2 & 0x00)
|
||||
## / \ /
|
||||
## H0=H(A) H1=H(B) H2=H(C)
|
||||
## | | |
|
||||
## A B C
|
||||
##
|
||||
## Memory layout is [H0, H1, H2, H3, H4, H5]
|
||||
##
|
||||
let
|
||||
mcodec = self.mcodec
|
||||
digestSize = self.digestSize
|
||||
leavesCount = self.buffer.len div self.digestSize
|
||||
|
||||
if leavesCount == 0:
|
||||
return failure("At least one data block is required")
|
||||
|
||||
let levels = computeLevels(leavesCount)
|
||||
let totalNodes = levels[^1].offset + 1
|
||||
|
||||
var tree = MerkleTree(mcodec: mcodec, digestSize: digestSize, leavesCount: leavesCount, nodesBuffer: newSeq[byte](totalNodes * digestSize))
|
||||
|
||||
# copy leaves
|
||||
tree.nodesBuffer[0..<leavesCount * digestSize] = self.buffer[0..<leavesCount * digestSize]
|
||||
|
||||
# calculate intermediate nodes
|
||||
var zero = newSeq[byte](digestSize)
|
||||
var one = newSeq[byte](digestSize)
|
||||
one[^1] = 0x01
|
||||
|
||||
var
|
||||
concatBuf = newSeq[byte](2 * digestSize)
|
||||
prevLevel = levels[0]
|
||||
for level in levels[1..^1]:
|
||||
for i in 0..<level.width:
|
||||
let parentIndex = level.offset + i
|
||||
let leftChildIndex = prevLevel.offset + 2 * i
|
||||
let rightChildIndex = leftChildIndex + 1
|
||||
|
||||
concatBuf[0..<digestSize] = tree.nodesBuffer[leftChildIndex * digestSize..<(leftChildIndex + 1) * digestSize]
|
||||
|
||||
var dummyValue = if prevLevel.index == 0: zero else: one
|
||||
|
||||
if rightChildIndex < prevLevel.offset + prevLevel.width:
|
||||
concatBuf[digestSize..^1] = tree.nodesBuffer[rightChildIndex * digestSize..<(rightChildIndex + 1) * digestSize]
|
||||
else:
|
||||
concatBuf[digestSize..^1] = dummyValue
|
||||
|
||||
? digestFn(mcodec, tree.nodesBuffer, parentIndex * digestSize, concatBuf)
|
||||
prevLevel = level
|
||||
|
||||
return success(tree)
|
||||
|
||||
###########################################################
|
||||
# MerkleTree
|
||||
###########################################################
|
||||
|
||||
proc nodeBufferToMultiHash(self: (MerkleTree | MerkleProof), index: int): MultiHash =
|
||||
var buf = newSeq[byte](self.digestSize)
|
||||
let offset = index * self.digestSize
|
||||
buf[0..^1] = self.nodesBuffer[offset..<(offset + self.digestSize)]
|
||||
|
||||
{.noSideEffect.}:
|
||||
without mhash =? MultiHash.init($self.mcodec, buf).mapFailure, errx:
|
||||
error "Error converting bytes to hash", msg = errx.msg
|
||||
mhash
|
||||
|
||||
proc len*(self: (MerkleTree | MerkleProof)): Natural =
|
||||
self.nodesBuffer.len div self.digestSize
|
||||
|
||||
proc nodes*(self: (MerkleTree | MerkleProof)): seq[MultiHash] {.noSideEffect.} =
|
||||
toSeq(0..<self.len).map(i => self.nodeBufferToMultiHash(i))
|
||||
|
||||
proc mcodec*(self: (MerkleTree | MerkleProof)): MultiCodec =
|
||||
self.mcodec
|
||||
|
||||
proc digestSize*(self: (MerkleTree | MerkleProof)): Natural =
|
||||
self.digestSize
|
||||
|
||||
proc root*(self: MerkleTree): MultiHash =
|
||||
let rootIndex = self.len - 1
|
||||
self.nodeBufferToMultiHash(rootIndex)
|
||||
|
||||
proc rootCid*(self: MerkleTree, version = CIDv1, dataCodec = multiCodec("raw")): ?!Cid =
|
||||
Cid.init(version, dataCodec, self.root).mapFailure
|
||||
|
||||
iterator leaves*(self: MerkleTree): MultiHash =
|
||||
for i in 0..<self.leavesCount:
|
||||
yield self.nodeBufferToMultiHash(i)
|
||||
|
||||
iterator leavesCids*(self: MerkleTree, version = CIDv1, dataCodec = multiCodec("raw")): ?!Cid =
|
||||
for leaf in self.leaves:
|
||||
yield Cid.init(version, dataCodec, leaf).mapFailure
|
||||
|
||||
proc leavesCount*(self: MerkleTree): Natural =
|
||||
self.leavesCount
|
||||
|
||||
proc getLeaf*(self: MerkleTree, index: Natural): ?!MultiHash =
|
||||
if index >= self.leavesCount:
|
||||
return failure("Index " & $index & " out of range [0.." & $(self.leavesCount - 1) & "]" )
|
||||
|
||||
success(self.nodeBufferToMultiHash(index))
|
||||
|
||||
proc getLeafCid*(self: MerkleTree, index: Natural, version = CIDv1, dataCodec = multiCodec("raw")): ?!Cid =
|
||||
let leaf = ? self.getLeaf(index)
|
||||
Cid.init(version, dataCodec, leaf).mapFailure
|
||||
|
||||
proc height*(self: MerkleTree): Natural =
|
||||
computeTreeHeight(self.leavesCount)
|
||||
|
||||
proc getProof*(self: MerkleTree, index: Natural): ?!MerkleProof =
|
||||
## Extracts proof from a tree for a given index
|
||||
##
|
||||
## Given a tree built from data blocks A, B and C
|
||||
## H5
|
||||
## / \
|
||||
## H3 H4
|
||||
## / \ /
|
||||
## H0 H1 H2
|
||||
## | | |
|
||||
## A B C
|
||||
##
|
||||
## Proofs of inclusion (index and path) are
|
||||
## - 0,[H1, H4] for data block A
|
||||
## - 1,[H0, H4] for data block B
|
||||
## - 2,[0x00, H3] for data block C
|
||||
##
|
||||
if index >= self.leavesCount:
|
||||
return failure("Index " & $index & " out of range [0.." & $(self.leavesCount - 1) & "]" )
|
||||
|
||||
var zero = newSeq[byte](self.digestSize)
|
||||
var one = newSeq[byte](self.digestSize)
|
||||
one[^1] = 0x01
|
||||
|
||||
let levels = computeLevels(self.leavesCount)
|
||||
var proofNodesBuffer = newSeq[byte]((levels.len - 1) * self.digestSize)
|
||||
for level in levels[0..^2]:
|
||||
let lr = index shr level.index
|
||||
let siblingIndex = if lr mod 2 == 0:
|
||||
level.offset + lr + 1
|
||||
else:
|
||||
level.offset + lr - 1
|
||||
|
||||
var dummyValue = if level.index == 0: zero else: one
|
||||
|
||||
if siblingIndex < level.offset + level.width:
|
||||
proofNodesBuffer[level.index * self.digestSize..<(level.index + 1) * self.digestSize] =
|
||||
self.nodesBuffer[siblingIndex * self.digestSize..<(siblingIndex + 1) * self.digestSize]
|
||||
else:
|
||||
proofNodesBuffer[level.index * self.digestSize..<(level.index + 1) * self.digestSize] = dummyValue
|
||||
|
||||
success(MerkleProof(mcodec: self.mcodec, digestSize: self.digestSize, index: index, nodesBuffer: proofNodesBuffer))
|
||||
|
||||
proc `$`*(self: MerkleTree): string {.noSideEffect.} =
|
||||
"mcodec:" & $self.mcodec &
|
||||
", digestSize: " & $self.digestSize &
|
||||
", leavesCount: " & $self.leavesCount &
|
||||
", nodes: " & $self.nodes
|
||||
|
||||
proc `==`*(a, b: MerkleTree): bool =
|
||||
(a.mcodec == b.mcodec) and
|
||||
(a.digestSize == b.digestSize) and
|
||||
(a.leavesCount == b.leavesCount) and
|
||||
(a.nodesBuffer == b.nodesBuffer)
|
||||
|
||||
proc init*(
|
||||
T: type MerkleTree,
|
||||
mcodec: MultiCodec,
|
||||
digestSize: Natural,
|
||||
leavesCount: Natural,
|
||||
nodesBuffer: seq[byte]
|
||||
): ?!MerkleTree =
|
||||
let levels = computeLevels(leavesCount)
|
||||
let totalNodes = levels[^1].offset + 1
|
||||
if totalNodes * digestSize == nodesBuffer.len:
|
||||
success(
|
||||
MerkleTree(
|
||||
mcodec: mcodec,
|
||||
digestSize: digestSize,
|
||||
leavesCount: leavesCount,
|
||||
nodesBuffer: nodesBuffer
|
||||
)
|
||||
)
|
||||
else:
|
||||
failure("Expected nodesBuffer len to be " & $(totalNodes * digestSize) & " but was " & $nodesBuffer.len)
|
||||
|
||||
proc init*(
|
||||
T: type MerkleTree,
|
||||
leaves: openArray[MultiHash]
|
||||
): ?!MerkleTree =
|
||||
without leaf =? leaves.?[0]:
|
||||
return failure("At least one leaf is required")
|
||||
|
||||
var builder = ? MerkleTreeBuilder.init(mcodec = leaf.mcodec)
|
||||
|
||||
for l in leaves:
|
||||
let res = builder.addLeaf(l)
|
||||
if res.isErr:
|
||||
return failure(res.error)
|
||||
|
||||
builder.build()
|
||||
|
||||
proc init*(
|
||||
T: type MerkleTree,
|
||||
cids: openArray[Cid]
|
||||
): ?!MerkleTree =
|
||||
var leaves = newSeq[MultiHash]()
|
||||
|
||||
for cid in cids:
|
||||
let res = cid.mhash.mapFailure
|
||||
if res.isErr:
|
||||
return failure(res.error)
|
||||
else:
|
||||
leaves.add(res.value)
|
||||
|
||||
MerkleTree.init(leaves)
|
||||
|
||||
###########################################################
|
||||
# MerkleProof
|
||||
###########################################################
|
||||
|
||||
proc verifyLeaf*(self: MerkleProof, leaf: MultiHash, treeRoot: MultiHash): ?!bool =
|
||||
if leaf.mcodec != self.mcodec:
|
||||
return failure("Leaf mcodec was " & $leaf.mcodec & ", but " & $self.mcodec & " expected")
|
||||
|
||||
if leaf.mcodec != self.mcodec:
|
||||
return failure("Tree root mcodec was " & $treeRoot.mcodec & ", but " & $treeRoot.mcodec & " expected")
|
||||
|
||||
var digestBuf = newSeq[byte](self.digestSize)
|
||||
digestBuf[0..^1] = leaf.data.buffer[leaf.dpos..<(leaf.dpos + self.digestSize)]
|
||||
|
||||
let proofLen = self.nodesBuffer.len div self.digestSize
|
||||
var concatBuf = newSeq[byte](2 * self.digestSize)
|
||||
for i in 0..<proofLen:
|
||||
let offset = i * self.digestSize
|
||||
let lr = self.index shr i
|
||||
if lr mod 2 == 0:
|
||||
concatBuf[0..^1] = digestBuf & self.nodesBuffer[offset..<(offset + self.digestSize)]
|
||||
else:
|
||||
concatBuf[0..^1] = self.nodesBuffer[offset..<(offset + self.digestSize)] & digestBuf
|
||||
? digestFn(self.mcodec, digestBuf, 0, concatBuf)
|
||||
|
||||
let computedRoot = ? MultiHash.init(self.mcodec, digestBuf).mapFailure
|
||||
|
||||
success(computedRoot == treeRoot)
|
||||
|
||||
|
||||
proc verifyDataBlock*(self: MerkleProof, dataBlock: openArray[byte], treeRoot: MultiHash): ?!bool =
|
||||
var digestBuf = newSeq[byte](self.digestSize)
|
||||
? digestFn(self.mcodec, digestBuf, 0, dataBlock)
|
||||
|
||||
let leaf = ? MultiHash.init(self.mcodec, digestBuf).mapFailure
|
||||
|
||||
self.verifyLeaf(leaf, treeRoot)
|
||||
|
||||
proc index*(self: MerkleProof): Natural =
|
||||
self.index
|
||||
|
||||
proc `$`*(self: MerkleProof): string =
|
||||
"mcodec:" & $self.mcodec &
|
||||
", digestSize: " & $self.digestSize &
|
||||
", index: " & $self.index &
|
||||
", nodes: " & $self.nodes
|
||||
|
||||
func `==`*(a, b: MerkleProof): bool =
|
||||
(a.index == b.index) and
|
||||
(a.mcodec == b.mcodec) and
|
||||
(a.digestSize == b.digestSize) and
|
||||
(a.nodesBuffer == b.nodesBuffer)
|
||||
|
||||
proc init*(
|
||||
T: type MerkleProof,
|
||||
index: Natural,
|
||||
nodes: seq[MultiHash]
|
||||
): ?!MerkleProof =
|
||||
if nodes.len == 0:
|
||||
return failure("At least one node is required")
|
||||
|
||||
let
|
||||
mcodec = nodes[0].mcodec
|
||||
digestSize = nodes[0].size
|
||||
|
||||
var nodesBuffer = newSeq[byte](nodes.len * digestSize)
|
||||
for nodeIndex, node in nodes:
|
||||
nodesBuffer[nodeIndex * digestSize..<(nodeIndex + 1) * digestSize] = node.data.buffer[node.dpos..<node.dpos + digestSize]
|
||||
|
||||
success(MerkleProof(mcodec: mcodec, digestSize: digestSize, index: index, nodesBuffer: nodesBuffer))
|
||||
|
||||
func init*(
|
||||
T: type MerkleProof,
|
||||
mcodec: MultiCodec,
|
||||
digestSize: Natural,
|
||||
index: Natural,
|
||||
nodesBuffer: seq[byte]
|
||||
): ?!MerkleProof =
|
||||
|
||||
if nodesBuffer.len mod digestSize != 0:
|
||||
return failure("nodesBuffer len is not a multiple of digestSize")
|
||||
|
||||
let treeHeight = (nodesBuffer.len div digestSize) + 1
|
||||
let maxLeavesCount = 1 shl treeHeight
|
||||
if index < maxLeavesCount:
|
||||
return success(
|
||||
MerkleProof(
|
||||
mcodec: mcodec,
|
||||
digestSize: digestSize,
|
||||
index: index,
|
||||
nodesBuffer: nodesBuffer
|
||||
)
|
||||
)
|
||||
else:
|
||||
return failure("index higher than max leaves count")
|
||||
39
codex/merkletree/poseidon2merkletree.nim
Normal file
39
codex/merkletree/poseidon2merkletree.nim
Normal file
@ -0,0 +1,39 @@
|
||||
## Nim-Codex
|
||||
## Copyright (c) 2023 Status Research & Development GmbH
|
||||
## Licensed under either of
|
||||
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
|
||||
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
|
||||
## at your option.
|
||||
## This file may not be copied, modified, or distributed except according to
|
||||
## those terms.
|
||||
|
||||
import pkg/poseidon2
|
||||
import pkg/constantine/math/io/io_fields
|
||||
|
||||
import ./merkletree
|
||||
|
||||
export merkletree, poseidon2
|
||||
|
||||
type
|
||||
Poseidon2Hash* = F
|
||||
|
||||
PoseidonKeysEnum* {.pure.} = enum
|
||||
KeyNone = "0x0"
|
||||
KeyBottomLayer = "0x1"
|
||||
KeyOdd = "0x2"
|
||||
KeyOddAndBottomLayer = "0x3"
|
||||
|
||||
MerkleTreePoseidon2* = MerkleTree[Poseidon2Hash, PoseidonKeysEnum]
|
||||
MerkleProofPoseidon2* = MerkleProof[Poseidon2Hash, PoseidonKeysEnum]
|
||||
|
||||
converter toKey*(x: PoseidonKeysEnum): Poseidon2Hash =
|
||||
return Poseidon2Hash.fromHex($x)
|
||||
|
||||
func init*(_: type MerkleTreePoseidon2, leaves: seq[Poseidon2Hash]): MerkleTreePoseidon2 =
|
||||
let
|
||||
compress = proc(
|
||||
x, y: Poseidon2Hash,
|
||||
key: PoseidonKeysEnum): Poseidon2Hash {.noSideEffect.} =
|
||||
poseidon2.compress( x, y, key )
|
||||
|
||||
MerkleTreePoseidon2(compress: compress, leaves: leaves, zero: zero)
|
||||
90
tests/codex/merkletree/testcodexmerkletree.nim
Normal file
90
tests/codex/merkletree/testcodexmerkletree.nim
Normal file
@ -0,0 +1,90 @@
|
||||
import std/unittest
|
||||
import std/sequtils
|
||||
import std/tables
|
||||
|
||||
import pkg/questionable/results
|
||||
import pkg/stew/byteutils
|
||||
import pkg/nimcrypto/sha2
|
||||
|
||||
import pkg/codex/merkletree
|
||||
|
||||
import ../helpers
|
||||
import ./generictreetests
|
||||
|
||||
# TODO: Generalize to other hashes
|
||||
|
||||
const
|
||||
data =
|
||||
[
|
||||
"00000000000000000000000000000001".toBytes,
|
||||
"00000000000000000000000000000002".toBytes,
|
||||
"00000000000000000000000000000003".toBytes,
|
||||
"00000000000000000000000000000004".toBytes,
|
||||
"00000000000000000000000000000005".toBytes,
|
||||
"00000000000000000000000000000006".toBytes,
|
||||
"00000000000000000000000000000007".toBytes,
|
||||
"00000000000000000000000000000008".toBytes,
|
||||
"00000000000000000000000000000009".toBytes,
|
||||
"00000000000000000000000000000010".toBytes,
|
||||
]
|
||||
sha256 = multiCodec("sha2-256")
|
||||
|
||||
checksuite "merkletree":
|
||||
test "Cannot init tree without any multihash leaves":
|
||||
check:
|
||||
CodexMerkleTree.init(leaves = newSeq[MultiHash]()).isErr
|
||||
|
||||
test "Cannot init tree without any cid leaves":
|
||||
check:
|
||||
CodexMerkleTree.init(leaves = newSeq[Cid]()).isErr
|
||||
|
||||
test "Cannot init tree without any byte leaves":
|
||||
check:
|
||||
CodexMerkleTree.init(sha256, leaves = newSeq[ByteHash]()).isErr
|
||||
|
||||
test "Should build tree from multihash leaves":
|
||||
var
|
||||
expectedLeaves = data.mapIt( MultiHash.digest($sha256, it).tryGet() )
|
||||
|
||||
var tree = CodexMerkleTree.init(leaves = expectedLeaves)
|
||||
check:
|
||||
tree.isOk
|
||||
tree.get().leaves == expectedLeaves.mapIt( it.bytes )
|
||||
tree.get().mcodec == sha256
|
||||
|
||||
test "Should build tree from cid leaves":
|
||||
var
|
||||
expectedLeaves = data.mapIt( Cid.init(
|
||||
CidVersion.CIDv1, BlockCodec, MultiHash.digest($sha256, it).tryGet ).tryGet )
|
||||
|
||||
let
|
||||
tree = CodexMerkleTree.init(leaves = expectedLeaves)
|
||||
|
||||
check:
|
||||
tree.isOk
|
||||
tree.get().leaves == expectedLeaves.mapIt( it.mhash.tryGet.bytes )
|
||||
tree.get().mcodec == sha256
|
||||
|
||||
test "Should build from raw bytes (should not hash leaves)":
|
||||
let
|
||||
tree = CodexMerkleTree.init(sha256, leaves = data).tryGet
|
||||
|
||||
check:
|
||||
tree.mcodec == sha256
|
||||
tree.leaves == data
|
||||
|
||||
let
|
||||
mhash = sha256.getMhash().tryGet
|
||||
zero: seq[byte] = newSeq[byte](mhash.size)
|
||||
compress = proc(x, y: seq[byte], key: ByteTreeKey): seq[byte] =
|
||||
compress(x, y, key, mhash).tryGet
|
||||
|
||||
makeTree = proc(data: seq[seq[byte]]): CodexMerkleTree =
|
||||
CodexMerkleTree.init(sha256, leaves = data).tryGet
|
||||
|
||||
checkGenericTree(
|
||||
"CodexMerkleTree",
|
||||
@data,
|
||||
zero,
|
||||
compress,
|
||||
makeTree)
|
||||
Loading…
x
Reference in New Issue
Block a user