codex-storage
/
nim-codex
mirror of https://github.com/codex-storage/nim-codex.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

treed and proof encoding/decoding

This commit is contained in:
Dmitriy Ryajov 2023-12-13 21:28:13 -06:00
parent 1437df6847
commit 75976996ed
No known key found for this signature in database
GPG Key ID: DA8C680CE7C657A4
2 changed files with 65 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
codex/merkletree/codexmerkletree/coders.nim
View File

@ -28,13 +28,12 @@ const MaxMerkleProofSize = 1.MiBs.uint
proc encode*(self: CodexMerkleTree): seq[byte] = proc encode*(self: CodexMerkleTree): seq[byte] =
var pb = initProtoBuffer(maxSize = MaxMerkleTreeSize) var pb = initProtoBuffer(maxSize = MaxMerkleTreeSize)
pb.write(1, self.mcodec.uint64) pb.write(1, self.mcodec.uint64)
pb.write(2, self.digestSize.uint64) pb.write(2, self.leavesCount.uint64)
pb.write(3, self.leavesCount.uint64)
var nodesPb = initProtoBuffer(maxSize = MaxMerkleTreeSize)
for node in self.nodes: for node in self.nodes:
var nodesPb = initProtoBuffer(maxSize = MaxMerkleTreeSize)
nodesPb.write(1, node) nodesPb.write(1, node)
nodesPb.finish() nodesPb.finish()
pb.write(4, nodesPb) pb.write(3, nodesPb)
pb.finish pb.finish
pb.buffer pb.buffer
@ -42,11 +41,9 @@ proc encode*(self: CodexMerkleTree): seq[byte] =
proc decode*(_: type CodexMerkleTree, data: seq[byte]): ?!CodexMerkleTree = proc decode*(_: type CodexMerkleTree, data: seq[byte]): ?!CodexMerkleTree =
var pb = initProtoBuffer(data, maxSize = MaxMerkleTreeSize) var pb = initProtoBuffer(data, maxSize = MaxMerkleTreeSize)
var mcodecCode: uint64 var mcodecCode: uint64
var digestSize: uint64
var leavesCount: uint64 var leavesCount: uint64
discard ? pb.getField(1, mcodecCode).mapFailure discard ? pb.getField(1, mcodecCode).mapFailure
discard ? pb.getField(2, digestSize).mapFailure discard ? pb.getField(2, leavesCount).mapFailure
discard ? pb.getField(3, leavesCount).mapFailure
let mcodec = MultiCodec.codec(mcodecCode.int) let mcodec = MultiCodec.codec(mcodecCode.int)
if mcodec == InvalidMultiCodec: if mcodec == InvalidMultiCodec:
@ -56,42 +53,42 @@ proc decode*(_: type CodexMerkleTree, data: seq[byte]): ?!CodexMerkleTree =
nodesBuff: seq[seq[byte]] nodesBuff: seq[seq[byte]]
nodes: seq[ByteHash] nodes: seq[ByteHash]
if ? pb.getRepeatedField(4, nodesBuff).mapFailure: if ? pb.getRepeatedField(3, nodesBuff).mapFailure:
for nodeBuff in nodesBuff: for nodeBuff in nodesBuff:
var node: ByteHash var node: ByteHash
let nodePb = initProtoBuffer(nodeBuff) discard ? initProtoBuffer(nodeBuff).getField(1, node).mapFailure
discard ? nodePb.getField(1, node).mapFailure
nodes.add node nodes.add node
let tree = ? CodexMerkleTree.fromNodes(mcodec, digestSize, leavesCount, nodesBuffer) CodexMerkleTree.fromNodes(mcodec, nodes, leavesCount.int)
success(tree)
proc encode*(self: CodexMerkleProof): seq[byte] = proc encode*(self: CodexMerkleProof): seq[byte] =
var pb = initProtoBuffer(maxSize = MaxMerkleProofSize) var pb = initProtoBuffer(maxSize = MaxMerkleProofSize)
pb.write(1, self.mcodec.uint64) pb.write(1, self.mcodec.uint64)
pb.write(2, self.digestSize.uint64) pb.write(2, self.index.uint64)
pb.write(3, self.index.uint64) pb.write(3, self.nleaves.uint64)
var nodesPb = initProtoBuffer(maxSize = MaxMerkleTreeSize)
for node in self.path: for node in self.path:
var nodesPb = initProtoBuffer(maxSize = MaxMerkleTreeSize)
nodesPb.write(1, node) nodesPb.write(1, node)
nodesPb.finish() nodesPb.finish()
pb.write(4, nodesPb) pb.write(4, nodesPb)
pb.finish pb.finish
pb.buffer pb.buffer
proc decode*(_: type CodexMerkleProof, data: seq[byte]): ?!CodexMerkleProof = proc decode*(_: type CodexMerkleProof, data: seq[byte]): ?!CodexMerkleProof =
var pb = initProtoBuffer(data, maxSize = MaxMerkleProofSize) var pb = initProtoBuffer(data, maxSize = MaxMerkleProofSize)
var mcodecCode: uint64 var mcodecCode: uint64
var digestSize: uint64
var index: uint64 var index: uint64
var nleaves: uint64
discard ? pb.getField(1, mcodecCode).mapFailure discard ? pb.getField(1, mcodecCode).mapFailure
let mcodec = MultiCodec.codec(mcodecCode.int) let mcodec = MultiCodec.codec(mcodecCode.int)
if mcodec == InvalidMultiCodec: if mcodec == InvalidMultiCodec:
return failure("Invalid MultiCodec code " & $mcodecCode) return failure("Invalid MultiCodec code " & $mcodecCode)
discard ? pb.getField(2, digestSize).mapFailure discard ? pb.getField(2, index).mapFailure
discard ? pb.getField(3, index).mapFailure discard ? pb.getField(3, nleaves).mapFailure
var var
nodesBuff: seq[seq[byte]] nodesBuff: seq[seq[byte]]
@ -104,7 +101,4 @@ proc decode*(_: type CodexMerkleProof, data: seq[byte]): ?!CodexMerkleProof =
discard ? nodePb.getField(1, node).mapFailure discard ? nodePb.getField(1, node).mapFailure
nodes.add node nodes.add node
let CodexMerkleProof.init(mcodec, index.int, nleaves.int, nodes)
proof = ? CodexMerkleProof.init(mcodec, index.int, nodes)
success(proof)

65
codex/merkletree/codexmerkletree/codexmerkletree.nim
View File

@ -24,6 +24,8 @@ import pkg/questionable/results
import pkg/libp2p/[cid, multicodec, multihash] import pkg/libp2p/[cid, multicodec, multihash]
import pkg/stew/byteutils import pkg/stew/byteutils
import ../../utils
import ../../rng
import ../../errors import ../../errors
import ../../blocktype import ../../blocktype
@ -86,23 +88,29 @@ func getProof*(self: CodexMerkleTree, index: int): ?!CodexMerkleProof =
success proof success proof
func verify*(self: CodexMerkleProof, root: MultiHash): ?!void = func verify*(self: CodexMerkleProof, leaf: MultiHash, root: MultiHash): ?!void =
## Verify hash ## Verify hash
## ##
let let
bytes = root.bytes rootBytes = root.bytes
leafBytes = leaf.bytes
if self.mcodec != root.mcodec: if self.mcodec != root.mcodec or
self.mcodec != leaf.mcodec:
return failure "Hash codec mismatch" return failure "Hash codec mismatch"
if bytes.len != root.size: if rootBytes.len != root.size and
leafBytes.len != leaf.size:
return failure "Invalid hash length" return failure "Invalid hash length"
? self.verify(bytes) ? self.verify(leafBytes, rootBytes)
success() success()
func verify*(self: CodexMerkleProof, leaf: Cid, root: Cid): ?!void =
self.verify(? leaf.mhash.mapFailure, ? leaf.mhash.mapFailure)
proc rootCid*( proc rootCid*(
self: CodexMerkleTree, self: CodexMerkleTree,
version = CIDv1, version = CIDv1,
@ -133,6 +141,17 @@ func getLeafCid*(
dataCodec, dataCodec,
? MultiHash.init(self.mcodec, self.root).mapFailure).mapFailure ? MultiHash.init(self.mcodec, self.root).mapFailure).mapFailure
proc `==`*(a, b: CodexMerkleTree): bool =
(a.mcodec == b.mcodec) and
(a.leavesCount == b.leavesCount) and
(a.levels == b.levels)
proc `==`*(a, b: CodexMerkleProof): bool =
(a.mcodec == b.mcodec) and
(a.nleaves == b.nleaves) and
(a.path == b.path) and
(a.index == b.index)
func compress*( func compress*(
x, y: openArray[byte], x, y: openArray[byte],
key: ByteTreeKey, key: ByteTreeKey,
@ -192,10 +211,10 @@ func init*(
CodexMerkleTree.init(mcodec, leaves) CodexMerkleTree.init(mcodec, leaves)
func fromNodes*( proc fromNodes*(
_: type CodexMerkleTree, _: type CodexMerkleTree,
mcodec: MultiCodec, mcodec: MultiCodec,
nodes: openArray[seq[ByteHash]], nodes: openArray[ByteHash],
nleaves: int): ?!CodexMerkleTree = nleaves: int): ?!CodexMerkleTree =
if nodes.len == 0: if nodes.len == 0:
@ -203,33 +222,35 @@ func fromNodes*(
let let
mhash = ? mcodec.getMhash() mhash = ? mcodec.getMhash()
Zero = newSeq[ByteHash](mhash.size) Zero = newSeq[byte](mhash.size)
compressor = proc(x, y: openArray[byte], key: ByteTreeKey): ?!ByteHash {.noSideEffect.} = compressor = proc(x, y: seq[byte], key: ByteTreeKey): ?!ByteHash {.noSideEffect.} =
compress(x, y, key, mhash) compress(x, y, key, mhash)
if mhash.size != nodes[0].len: if mhash.size != nodes[0].len:
return failure "Invalid hash length" return failure "Invalid hash length"
let
self = CodexMerkleTree(compress: compressor, zero: Zero, mhash: mhash)
var var
self = CodexMerkleTree(compress: compressor, zero: Zero, mhash: mhash)
layer = nleaves layer = nleaves
pos = 0 pos = 0
while layer > 0: while pos < nodes.len:
self.layers.add( nodes[pos..<layer].toSeq() ) self.layers.add( nodes[pos..<(pos + layer)] )
pos += layer pos += layer
layer = layer shr 1 layer = divUp(layer, 2)
? self.proof(Rng.instance.rand(nleaves)).?verify(self.root) # sanity check let
index = Rng.instance.rand(nleaves - 1)
proof = ? self.getProof(index)
? proof.verify(self.leaves[index], self.root) # sanity check
success self success self
func init*( func init*(
_: type CodexMerkleProof, _: type CodexMerkleProof,
mcodec: MultiCodec, mcodec: MultiCodec,
index: int, index: int,
nleaves: int,
nodes: openArray[ByteHash]): ?!CodexMerkleProof = nodes: openArray[ByteHash]): ?!CodexMerkleProof =
if nodes.len == 0: if nodes.len == 0:
@ -237,9 +258,15 @@ func init*(
let let
mhash = ? mcodec.getMhash() mhash = ? mcodec.getMhash()
Zero: ByteHash = newSeq[byte](mhash.size) Zero = newSeq[byte](mhash.size)
compressor = proc(x, y: seq[byte], key: ByteTreeKey): ?!seq[byte] {.noSideEffect.} = compressor = proc(x, y: seq[byte], key: ByteTreeKey): ?!seq[byte] {.noSideEffect.} =
compress(x, y, key, mhash) compress(x, y, key, mhash)
self = CodexMerkleProof(compress: compressor, zero: Zero, mhash: mhash)
success self
success CodexMerkleProof(
compress: compressor,
zero: Zero,
mhash: mhash,
index: index,
nleaves: nleaves,
path: @nodes)