mirror of
https://github.com/logos-storage/logos-storage-nim.git
synced 2026-02-03 21:23:09 +00:00
refactor!: move merkletree to own repo (#1390)
This commit is contained in:
Eric
GitHub
parent
c6345fd6f7
commit
44ad291b16
5
.gitmodules
vendored
5
.gitmodules
vendored
@ -206,4 +206,7 @@
|
||||
path = vendor/nim-ngtcp2
|
||||
url = https://github.com/vacp2p/nim-ngtcp2.git
|
||||
ignore = untracked
|
||||
branch = main
|
||||
branch = main
|
||||
[submodule "vendor/nim-merkletree"]
|
||||
path = vendor/nim-merkletree
|
||||
url = https://github.com/logos-storage/nim-merkletree
|
||||
|
||||
@ -848,12 +848,13 @@ proc localLookup(
|
||||
): Future[?!BlockDelivery] {.async: (raises: [CancelledError]).} =
|
||||
if address.leaf:
|
||||
(await self.localStore.getBlockAndProof(address.treeCid, address.index)).map(
|
||||
(blkAndProof: (Block, CodexProof)) =>
|
||||
(blkAndProof: (Block, StorageMerkleProof)) =>
|
||||
BlockDelivery(address: address, blk: blkAndProof[0], proof: blkAndProof[1].some)
|
||||
)
|
||||
else:
|
||||
(await self.localStore.getBlock(address)).map(
|
||||
(blk: Block) => BlockDelivery(address: address, blk: blk, proof: CodexProof.none)
|
||||
(blk: Block) =>
|
||||
BlockDelivery(address: address, blk: blk, proof: StorageMerkleProof.none)
|
||||
)
|
||||
|
||||
iterator splitBatches[T](sequence: seq[T], batchSize: int): seq[T] =
|
||||
|
||||
@ -42,7 +42,7 @@ type
|
||||
BlockDelivery* = object
|
||||
blk*: Block
|
||||
address*: BlockAddress
|
||||
proof*: ?CodexProof # Present only if `address.leaf` is true
|
||||
proof*: ?StorageMerkleProof # Present only if `address.leaf` is true
|
||||
|
||||
BlockPresenceType* = enum
|
||||
Have = 0
|
||||
@ -201,12 +201,13 @@ proc decode*(_: type BlockDelivery, pb: ProtoBuffer): ProtoResult[BlockDelivery]
|
||||
if value.address.leaf:
|
||||
var proofBuf = newSeq[byte]()
|
||||
if ?pb.getField(4, proofBuf):
|
||||
let proof = ?CodexProof.decode(proofBuf).mapErr(x => ProtoError.IncorrectBlob)
|
||||
let proof =
|
||||
?StorageMerkleProof.decode(proofBuf).mapErr(x => ProtoError.IncorrectBlob)
|
||||
value.proof = proof.some
|
||||
else:
|
||||
value.proof = CodexProof.none
|
||||
value.proof = StorageMerkleProof.none
|
||||
else:
|
||||
value.proof = CodexProof.none
|
||||
value.proof = StorageMerkleProof.none
|
||||
|
||||
ok(value)
|
||||
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
import ./merkletree/merkletree
|
||||
import ./merkletree/codex
|
||||
import ./merkletree/coders
|
||||
|
||||
export codex, merkletree
|
||||
export merkletree, coders
|
||||
|
||||
@ -15,15 +15,14 @@ import pkg/questionable/results
|
||||
import pkg/stew/byteutils
|
||||
import pkg/serde/json
|
||||
|
||||
import ../../units
|
||||
import ../../errors
|
||||
|
||||
import ./codex
|
||||
import ../units
|
||||
import ../errors
|
||||
import ./merkletree
|
||||
|
||||
const MaxMerkleTreeSize = 100.MiBs.uint
|
||||
const MaxMerkleProofSize = 1.MiBs.uint
|
||||
|
||||
proc encode*(self: CodexTree): seq[byte] =
|
||||
proc encode*(self: StorageMerkleTree): seq[byte] =
|
||||
var pb = initProtoBuffer()
|
||||
pb.write(1, self.mcodec.uint64)
|
||||
pb.write(2, self.leavesCount.uint64)
|
||||
@ -36,7 +35,7 @@ proc encode*(self: CodexTree): seq[byte] =
|
||||
pb.finish
|
||||
pb.buffer
|
||||
|
||||
proc decode*(_: type CodexTree, data: seq[byte]): ?!CodexTree =
|
||||
proc decode*(_: type StorageMerkleTree, data: seq[byte]): ?!StorageMerkleTree =
|
||||
var pb = initProtoBuffer(data)
|
||||
var mcodecCode: uint64
|
||||
var leavesCount: uint64
|
||||
@ -57,9 +56,9 @@ proc decode*(_: type CodexTree, data: seq[byte]): ?!CodexTree =
|
||||
discard ?initProtoBuffer(nodeBuff).getField(1, node).mapFailure
|
||||
nodes.add node
|
||||
|
||||
CodexTree.fromNodes(mcodec, nodes, leavesCount.int)
|
||||
StorageMerkleTree.fromNodes(mcodec, nodes, leavesCount.int)
|
||||
|
||||
proc encode*(self: CodexProof): seq[byte] =
|
||||
proc encode*(self: StorageMerkleProof): seq[byte] =
|
||||
var pb = initProtoBuffer()
|
||||
pb.write(1, self.mcodec.uint64)
|
||||
pb.write(2, self.index.uint64)
|
||||
@ -74,7 +73,7 @@ proc encode*(self: CodexProof): seq[byte] =
|
||||
pb.finish
|
||||
pb.buffer
|
||||
|
||||
proc decode*(_: type CodexProof, data: seq[byte]): ?!CodexProof =
|
||||
proc decode*(_: type StorageMerkleProof, data: seq[byte]): ?!StorageMerkleProof =
|
||||
var pb = initProtoBuffer(data)
|
||||
var mcodecCode: uint64
|
||||
var index: uint64
|
||||
@ -99,9 +98,9 @@ proc decode*(_: type CodexProof, data: seq[byte]): ?!CodexProof =
|
||||
discard ?nodePb.getField(1, node).mapFailure
|
||||
nodes.add node
|
||||
|
||||
CodexProof.init(mcodec, index.int, nleaves.int, nodes)
|
||||
StorageMerkleProof.init(mcodec, index.int, nleaves.int, nodes)
|
||||
|
||||
proc fromJson*(_: type CodexProof, json: JsonNode): ?!CodexProof =
|
||||
proc fromJson*(_: type StorageMerkleProof, json: JsonNode): ?!StorageMerkleProof =
|
||||
expectJsonKind(Cid, JString, json)
|
||||
var bytes: seq[byte]
|
||||
try:
|
||||
@ -109,7 +108,7 @@ proc fromJson*(_: type CodexProof, json: JsonNode): ?!CodexProof =
|
||||
except ValueError as err:
|
||||
return failure(err)
|
||||
|
||||
CodexProof.decode(bytes)
|
||||
StorageMerkleProof.decode(bytes)
|
||||
|
||||
func `%`*(proof: CodexProof): JsonNode =
|
||||
func `%`*(proof: StorageMerkleProof): JsonNode =
|
||||
%byteutils.toHex(proof.encode())
|
||||
@ -1,4 +0,0 @@
|
||||
import ./codex/codex
|
||||
import ./codex/coders
|
||||
|
||||
export codex, coders
|
||||
@ -1,249 +0,0 @@
|
||||
## Logos Storage
|
||||
## Copyright (c) 2023 Status Research & Development GmbH
|
||||
## Licensed under either of
|
||||
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
|
||||
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
|
||||
## at your option.
|
||||
## This file may not be copied, modified, or distributed except according to
|
||||
## those terms.
|
||||
|
||||
{.push raises: [].}
|
||||
|
||||
import std/bitops
|
||||
import std/[atomics, sequtils]
|
||||
|
||||
import pkg/questionable
|
||||
import pkg/questionable/results
|
||||
import pkg/libp2p/[cid, multicodec, multihash]
|
||||
import pkg/constantine/hashes
|
||||
import pkg/taskpools
|
||||
import pkg/chronos/threadsync
|
||||
import ../../utils
|
||||
import ../../rng
|
||||
import ../../errors
|
||||
import ../../codextypes
|
||||
|
||||
from ../../utils/digest import digestBytes
|
||||
|
||||
import ../merkletree
|
||||
|
||||
export merkletree
|
||||
|
||||
logScope:
|
||||
topics = "codex merkletree"
|
||||
|
||||
type
|
||||
ByteTreeKey* {.pure.} = enum
|
||||
KeyNone = 0x0.byte
|
||||
KeyBottomLayer = 0x1.byte
|
||||
KeyOdd = 0x2.byte
|
||||
KeyOddAndBottomLayer = 0x3.byte
|
||||
|
||||
ByteHash* = seq[byte]
|
||||
ByteTree* = MerkleTree[ByteHash, ByteTreeKey]
|
||||
ByteProof* = MerkleProof[ByteHash, ByteTreeKey]
|
||||
|
||||
CodexTree* = ref object of ByteTree
|
||||
mcodec*: MultiCodec
|
||||
|
||||
CodexProof* = ref object of ByteProof
|
||||
mcodec*: MultiCodec
|
||||
|
||||
func getProof*(self: CodexTree, index: int): ?!CodexProof =
|
||||
var proof = CodexProof(mcodec: self.mcodec)
|
||||
|
||||
?self.getProof(index, proof)
|
||||
|
||||
success proof
|
||||
|
||||
func verify*(self: CodexProof, leaf: MultiHash, root: MultiHash): ?!bool =
|
||||
## Verify hash
|
||||
##
|
||||
|
||||
let
|
||||
rootBytes = root.digestBytes
|
||||
leafBytes = leaf.digestBytes
|
||||
|
||||
if self.mcodec != root.mcodec or self.mcodec != leaf.mcodec:
|
||||
return failure "Hash codec mismatch"
|
||||
|
||||
if rootBytes.len != root.size and leafBytes.len != leaf.size:
|
||||
return failure "Invalid hash length"
|
||||
|
||||
self.verify(leafBytes, rootBytes)
|
||||
|
||||
func verify*(self: CodexProof, leaf: Cid, root: Cid): ?!bool =
|
||||
self.verify(?leaf.mhash.mapFailure, ?leaf.mhash.mapFailure)
|
||||
|
||||
proc rootCid*(self: CodexTree, version = CIDv1, dataCodec = DatasetRootCodec): ?!Cid =
|
||||
if (?self.root).len == 0:
|
||||
return failure "Empty root"
|
||||
|
||||
let mhash = ?MultiHash.init(self.mcodec, ?self.root).mapFailure
|
||||
|
||||
Cid.init(version, DatasetRootCodec, mhash).mapFailure
|
||||
|
||||
func getLeafCid*(
|
||||
self: CodexTree, i: Natural, version = CIDv1, dataCodec = BlockCodec
|
||||
): ?!Cid =
|
||||
if i >= self.leavesCount:
|
||||
return failure "Invalid leaf index " & $i
|
||||
|
||||
let
|
||||
leaf = self.leaves[i]
|
||||
mhash = ?MultiHash.init($self.mcodec, leaf).mapFailure
|
||||
|
||||
Cid.init(version, dataCodec, mhash).mapFailure
|
||||
|
||||
proc `$`*(self: CodexTree): string =
|
||||
let root =
|
||||
if self.root.isOk:
|
||||
byteutils.toHex(self.root.get)
|
||||
else:
|
||||
"none"
|
||||
"CodexTree(" & " root: " & root & ", leavesCount: " & $self.leavesCount & ", levels: " &
|
||||
$self.levels & ", mcodec: " & $self.mcodec & " )"
|
||||
|
||||
proc `$`*(self: CodexProof): string =
|
||||
"CodexProof(" & " nleaves: " & $self.nleaves & ", index: " & $self.index & ", path: " &
|
||||
$self.path.mapIt(byteutils.toHex(it)) & ", mcodec: " & $self.mcodec & " )"
|
||||
|
||||
func compress*(x, y: openArray[byte], key: ByteTreeKey, codec: MultiCodec): ?!ByteHash =
|
||||
## Compress two hashes
|
||||
##
|
||||
let input = @x & @y & @[key.byte]
|
||||
let digest = ?MultiHash.digest(codec, input).mapFailure
|
||||
success digest.digestBytes
|
||||
|
||||
func initTree(mcodec: MultiCodec, leaves: openArray[ByteHash]): ?!CodexTree =
|
||||
if leaves.len == 0:
|
||||
return failure "Empty leaves"
|
||||
|
||||
let
|
||||
compressor = proc(x, y: seq[byte], key: ByteTreeKey): ?!ByteHash {.noSideEffect.} =
|
||||
compress(x, y, key, mcodec)
|
||||
digestSize = ?mcodec.digestSize.mapFailure
|
||||
Zero: ByteHash = newSeq[byte](digestSize)
|
||||
|
||||
if digestSize != leaves[0].len:
|
||||
return failure "Invalid hash length"
|
||||
|
||||
var self = CodexTree(mcodec: mcodec)
|
||||
?self.prepare(compressor, Zero, leaves)
|
||||
success self
|
||||
|
||||
func init*(
|
||||
_: type CodexTree, mcodec: MultiCodec = Sha256HashCodec, leaves: openArray[ByteHash]
|
||||
): ?!CodexTree =
|
||||
let tree = ?initTree(mcodec, leaves)
|
||||
?tree.compute()
|
||||
success tree
|
||||
|
||||
proc init*(
|
||||
_: type CodexTree,
|
||||
tp: Taskpool,
|
||||
mcodec: MultiCodec = Sha256HashCodec,
|
||||
leaves: seq[ByteHash],
|
||||
): Future[?!CodexTree] {.async: (raises: [CancelledError]).} =
|
||||
let tree = ?initTree(mcodec, leaves)
|
||||
?await tree.compute(tp)
|
||||
success tree
|
||||
|
||||
func init*(_: type CodexTree, leaves: openArray[MultiHash]): ?!CodexTree =
|
||||
if leaves.len == 0:
|
||||
return failure "Empty leaves"
|
||||
|
||||
let
|
||||
mcodec = leaves[0].mcodec
|
||||
leaves = leaves.mapIt(it.digestBytes)
|
||||
|
||||
CodexTree.init(mcodec, leaves)
|
||||
|
||||
proc init*(
|
||||
_: type CodexTree, tp: Taskpool, leaves: seq[MultiHash]
|
||||
): Future[?!CodexTree] {.async: (raises: [CancelledError]).} =
|
||||
if leaves.len == 0:
|
||||
return failure "Empty leaves"
|
||||
|
||||
let
|
||||
mcodec = leaves[0].mcodec
|
||||
leaves = leaves.mapIt(it.digestBytes)
|
||||
|
||||
await CodexTree.init(tp, mcodec, leaves)
|
||||
|
||||
func init*(_: type CodexTree, leaves: openArray[Cid]): ?!CodexTree =
|
||||
if leaves.len == 0:
|
||||
return failure "Empty leaves"
|
||||
|
||||
let
|
||||
mcodec = (?leaves[0].mhash.mapFailure).mcodec
|
||||
leaves = leaves.mapIt((?it.mhash.mapFailure).digestBytes)
|
||||
|
||||
CodexTree.init(mcodec, leaves)
|
||||
|
||||
proc init*(
|
||||
_: type CodexTree, tp: Taskpool, leaves: seq[Cid]
|
||||
): Future[?!CodexTree] {.async: (raises: [CancelledError]).} =
|
||||
if leaves.len == 0:
|
||||
return failure("Empty leaves")
|
||||
|
||||
let
|
||||
mcodec = (?leaves[0].mhash.mapFailure).mcodec
|
||||
leaves = leaves.mapIt((?it.mhash.mapFailure).digestBytes)
|
||||
|
||||
await CodexTree.init(tp, mcodec, leaves)
|
||||
|
||||
proc fromNodes*(
|
||||
_: type CodexTree,
|
||||
mcodec: MultiCodec = Sha256HashCodec,
|
||||
nodes: openArray[ByteHash],
|
||||
nleaves: int,
|
||||
): ?!CodexTree =
|
||||
if nodes.len == 0:
|
||||
return failure "Empty nodes"
|
||||
|
||||
let
|
||||
digestSize = ?mcodec.digestSize.mapFailure
|
||||
Zero = newSeq[byte](digestSize)
|
||||
compressor = proc(x, y: seq[byte], key: ByteTreeKey): ?!ByteHash {.noSideEffect.} =
|
||||
compress(x, y, key, mcodec)
|
||||
|
||||
if digestSize != nodes[0].len:
|
||||
return failure "Invalid hash length"
|
||||
|
||||
var self = CodexTree(mcodec: mcodec)
|
||||
?self.fromNodes(compressor, Zero, nodes, nleaves)
|
||||
|
||||
let
|
||||
index = Rng.instance.rand(nleaves - 1)
|
||||
proof = ?self.getProof(index)
|
||||
|
||||
if not ?proof.verify(self.leaves[index], ?self.root): # sanity check
|
||||
return failure "Unable to verify tree built from nodes"
|
||||
|
||||
success self
|
||||
|
||||
func init*(
|
||||
_: type CodexProof,
|
||||
mcodec: MultiCodec = Sha256HashCodec,
|
||||
index: int,
|
||||
nleaves: int,
|
||||
nodes: openArray[ByteHash],
|
||||
): ?!CodexProof =
|
||||
if nodes.len == 0:
|
||||
return failure "Empty nodes"
|
||||
|
||||
let
|
||||
digestSize = ?mcodec.digestSize.mapFailure
|
||||
Zero = newSeq[byte](digestSize)
|
||||
compressor = proc(x, y: seq[byte], key: ByteTreeKey): ?!seq[byte] {.noSideEffect.} =
|
||||
compress(x, y, key, mcodec)
|
||||
|
||||
success CodexProof(
|
||||
compress: compressor,
|
||||
zero: Zero,
|
||||
mcodec: mcodec,
|
||||
index: index,
|
||||
nleaves: nleaves,
|
||||
path: @nodes,
|
||||
)
|
||||
@ -9,388 +9,247 @@
|
||||
|
||||
{.push raises: [].}
|
||||
|
||||
import std/[bitops, atomics, sequtils]
|
||||
import stew/assign2
|
||||
import std/bitops
|
||||
import std/[atomics, sequtils]
|
||||
|
||||
import pkg/questionable
|
||||
import pkg/questionable/results
|
||||
import pkg/libp2p/[cid, multicodec, multihash]
|
||||
import pkg/constantine/hashes
|
||||
import pkg/taskpools
|
||||
import pkg/chronos
|
||||
import pkg/chronos/threadsync
|
||||
|
||||
import pkg/merkletree
|
||||
import ../utils
|
||||
import ../rng
|
||||
import ../errors
|
||||
import ../utils/sharedbuf
|
||||
import ../codextypes
|
||||
|
||||
export sharedbuf
|
||||
from ../utils/digest import digestBytes
|
||||
|
||||
template nodeData(
|
||||
data: openArray[byte], offsets: openArray[int], nodeSize, i, j: int
|
||||
): openArray[byte] =
|
||||
## Bytes of the j'th entry of the i'th level in the tree, starting with the
|
||||
## leaves (at level 0).
|
||||
let start = (offsets[i] + j) * nodeSize
|
||||
data.toOpenArray(start, start + nodeSize - 1)
|
||||
export merkletree
|
||||
|
||||
logScope:
|
||||
topics = "codex merkletree"
|
||||
|
||||
type
|
||||
# TODO hash functions don't fail - removing the ?! from this function would
|
||||
# significantly simplify the flow below
|
||||
CompressFn*[H, K] = proc(x, y: H, key: K): ?!H {.noSideEffect, raises: [].}
|
||||
ByteTreeKey* {.pure.} = enum
|
||||
KeyNone = 0x0.byte
|
||||
KeyBottomLayer = 0x1.byte
|
||||
KeyOdd = 0x2.byte
|
||||
KeyOddAndBottomLayer = 0x3.byte
|
||||
|
||||
CompressData[H, K] = object
|
||||
fn: CompressFn[H, K]
|
||||
nodeSize: int
|
||||
zero: H
|
||||
ByteHash* = seq[byte]
|
||||
ByteTree* = MerkleTree[ByteHash, ByteTreeKey]
|
||||
ByteProof* = MerkleProof[ByteHash, ByteTreeKey]
|
||||
|
||||
MerkleTreeObj*[H, K] = object of RootObj
|
||||
store*: seq[byte]
|
||||
## Flattened merkle tree where hashes are assumed to be trivial bytes and
|
||||
## uniform in size.
|
||||
##
|
||||
## Each layer of the tree is stored serially starting with the leaves and
|
||||
## ending with the root.
|
||||
##
|
||||
## Beacuse the tree might not be balanced, `layerOffsets` contains the
|
||||
## index of the starting point of each level, for easy lookup.
|
||||
layerOffsets*: seq[int]
|
||||
## Starting point of each level in the tree, starting from the leaves -
|
||||
## multiplied by the entry size, this is the offset in the payload where
|
||||
## the entries of that level start
|
||||
##
|
||||
## For example, a tree with 4 leaves will have [0, 4, 6] stored here.
|
||||
##
|
||||
## See nodesPerLevel function, from whic this sequence is derived
|
||||
compress*: CompressData[H, K]
|
||||
StorageMerkleTree* = ref object of ByteTree
|
||||
mcodec*: MultiCodec
|
||||
|
||||
MerkleTree*[H, K] = ref MerkleTreeObj[H, K]
|
||||
StorageMerkleProof* = ref object of ByteProof
|
||||
mcodec*: MultiCodec
|
||||
|
||||
MerkleProof*[H, K] = ref object of RootObj
|
||||
index*: int # linear index of the leaf, starting from 0
|
||||
path*: seq[H] # order: from the bottom to the top
|
||||
nleaves*: int # number of leaves in the tree (=size of input)
|
||||
compress*: CompressFn[H, K] # compress function
|
||||
zero*: H # zero value
|
||||
|
||||
func levels*[H, K](self: MerkleTree[H, K]): int =
|
||||
return self.layerOffsets.len
|
||||
|
||||
func depth*[H, K](self: MerkleTree[H, K]): int =
|
||||
return self.levels() - 1
|
||||
|
||||
func nodesInLayer(offsets: openArray[int], layer: int): int =
|
||||
if layer == offsets.high:
|
||||
1
|
||||
else:
|
||||
offsets[layer + 1] - offsets[layer]
|
||||
|
||||
func nodesInLayer(self: MerkleTree | MerkleTreeObj, layer: int): int =
|
||||
self.layerOffsets.nodesInLayer(layer)
|
||||
|
||||
func leavesCount*[H, K](self: MerkleTree[H, K]): int =
|
||||
return self.nodesInLayer(0)
|
||||
|
||||
func nodesPerLevel(nleaves: int): seq[int] =
|
||||
## Given a number of leaves, return a seq with the number of nodes at each
|
||||
## layer of the tree (from the bottom/leaves to the root)
|
||||
##
|
||||
## Ie For a tree of 4 leaves, return `[4, 2, 1]`
|
||||
if nleaves <= 0:
|
||||
return @[]
|
||||
elif nleaves == 1:
|
||||
return @[1, 1] # leaf and root
|
||||
|
||||
var nodes: seq[int] = @[]
|
||||
var m = nleaves
|
||||
while true:
|
||||
nodes.add(m)
|
||||
if m == 1:
|
||||
break
|
||||
# Next layer size is ceil(m/2)
|
||||
m = (m + 1) shr 1
|
||||
|
||||
nodes
|
||||
|
||||
func layerOffsets(nleaves: int): seq[int] =
|
||||
## Given a number of leaves, return a seq of the starting offsets of each
|
||||
## layer in the node store that results from flattening the binary tree
|
||||
##
|
||||
## Ie For a tree of 4 leaves, return `[0, 4, 6]`
|
||||
let nodes = nodesPerLevel(nleaves)
|
||||
var tot = 0
|
||||
let offsets = nodes.mapIt:
|
||||
let cur = tot
|
||||
tot += it
|
||||
cur
|
||||
offsets
|
||||
|
||||
template nodeData(self: MerkleTreeObj, i, j: int): openArray[byte] =
|
||||
## Bytes of the j'th node of the i'th level in the tree, starting with the
|
||||
## leaves (at level 0).
|
||||
self.store.nodeData(self.layerOffsets, self.compress.nodeSize, i, j)
|
||||
|
||||
func layer*[H, K](
|
||||
self: MerkleTree[H, K], layer: int
|
||||
): seq[H] {.deprecated: "Expensive".} =
|
||||
var nodes = newSeq[H](self.nodesInLayer(layer))
|
||||
for i, h in nodes.mpairs:
|
||||
assign(h, self[].nodeData(layer, i))
|
||||
return nodes
|
||||
|
||||
func leaves*[H, K](self: MerkleTree[H, K]): seq[H] {.deprecated: "Expensive".} =
|
||||
self.layer(0)
|
||||
|
||||
iterator layers*[H, K](self: MerkleTree[H, K]): seq[H] {.deprecated: "Expensive".} =
|
||||
for i in 0 ..< self.layerOffsets.len:
|
||||
yield self.layer(i)
|
||||
|
||||
proc layers*[H, K](self: MerkleTree[H, K]): seq[seq[H]] {.deprecated: "Expensive".} =
|
||||
for l in self.layers():
|
||||
result.add l
|
||||
|
||||
iterator nodes*[H, K](self: MerkleTree[H, K]): H =
|
||||
## Iterate over the nodes of each layer starting with the leaves
|
||||
var node: H
|
||||
for i in 0 ..< self.layerOffsets.len:
|
||||
let nodesInLayer = self.nodesInLayer(i)
|
||||
for j in 0 ..< nodesInLayer:
|
||||
assign(node, self[].nodeData(i, j))
|
||||
yield node
|
||||
|
||||
func root*[H, K](self: MerkleTree[H, K]): ?!H =
|
||||
mixin assign
|
||||
if self.layerOffsets.len == 0:
|
||||
return failure "invalid tree"
|
||||
|
||||
var h: H
|
||||
assign(h, self[].nodeData(self.layerOffsets.high(), 0))
|
||||
return success h
|
||||
|
||||
func getProof*[H, K](
|
||||
self: MerkleTree[H, K], index: int, proof: MerkleProof[H, K]
|
||||
): ?!void =
|
||||
let depth = self.depth
|
||||
let nleaves = self.leavesCount
|
||||
|
||||
if not (index >= 0 and index < nleaves):
|
||||
return failure "index out of bounds"
|
||||
|
||||
var path: seq[H] = newSeq[H](depth)
|
||||
var k = index
|
||||
var m = nleaves
|
||||
for i in 0 ..< depth:
|
||||
let j = k xor 1
|
||||
|
||||
if (j < m):
|
||||
assign(path[i], self[].nodeData(i, j))
|
||||
else:
|
||||
path[i] = self.compress.zero
|
||||
|
||||
k = k shr 1
|
||||
m = (m + 1) shr 1
|
||||
|
||||
proof.index = index
|
||||
proof.path = path
|
||||
proof.nleaves = nleaves
|
||||
proof.compress = self.compress.fn
|
||||
|
||||
success()
|
||||
|
||||
func getProof*[H, K](self: MerkleTree[H, K], index: int): ?!MerkleProof[H, K] =
|
||||
var proof = MerkleProof[H, K]()
|
||||
func getProof*(self: StorageMerkleTree, index: int): ?!StorageMerkleProof =
|
||||
var proof = StorageMerkleProof(mcodec: self.mcodec)
|
||||
|
||||
?self.getProof(index, proof)
|
||||
|
||||
success proof
|
||||
|
||||
func reconstructRoot*[H, K](proof: MerkleProof[H, K], leaf: H): ?!H =
|
||||
var
|
||||
m = proof.nleaves
|
||||
j = proof.index
|
||||
h = leaf
|
||||
bottomFlag = K.KeyBottomLayer
|
||||
func verify*(self: StorageMerkleProof, leaf: MultiHash, root: MultiHash): ?!bool =
|
||||
## Verify hash
|
||||
##
|
||||
|
||||
for p in proof.path:
|
||||
let oddIndex: bool = (bitand(j, 1) != 0)
|
||||
if oddIndex:
|
||||
# the index of the child is odd, so the node itself can't be odd (a bit counterintuitive, yeah :)
|
||||
h = ?proof.compress(p, h, bottomFlag)
|
||||
let
|
||||
rootBytes = root.digestBytes
|
||||
leafBytes = leaf.digestBytes
|
||||
|
||||
if self.mcodec != root.mcodec or self.mcodec != leaf.mcodec:
|
||||
return failure "Hash codec mismatch"
|
||||
|
||||
if rootBytes.len != root.size and leafBytes.len != leaf.size:
|
||||
return failure "Invalid hash length"
|
||||
|
||||
self.verify(leafBytes, rootBytes)
|
||||
|
||||
func verify*(self: StorageMerkleProof, leaf: Cid, root: Cid): ?!bool =
|
||||
self.verify(?leaf.mhash.mapFailure, ?leaf.mhash.mapFailure)
|
||||
|
||||
proc rootCid*(
|
||||
self: StorageMerkleTree, version = CIDv1, dataCodec = DatasetRootCodec
|
||||
): ?!Cid =
|
||||
if (?self.root).len == 0:
|
||||
return failure "Empty root"
|
||||
|
||||
let mhash = ?MultiHash.init(self.mcodec, ?self.root).mapFailure
|
||||
|
||||
Cid.init(version, DatasetRootCodec, mhash).mapFailure
|
||||
|
||||
func getLeafCid*(
|
||||
self: StorageMerkleTree, i: Natural, version = CIDv1, dataCodec = BlockCodec
|
||||
): ?!Cid =
|
||||
if i >= self.leavesCount:
|
||||
return failure "Invalid leaf index " & $i
|
||||
|
||||
let
|
||||
leaf = self.leaves[i]
|
||||
mhash = ?MultiHash.init($self.mcodec, leaf).mapFailure
|
||||
|
||||
Cid.init(version, dataCodec, mhash).mapFailure
|
||||
|
||||
proc `$`*(self: StorageMerkleTree): string =
|
||||
let root =
|
||||
if self.root.isOk:
|
||||
byteutils.toHex(self.root.get)
|
||||
else:
|
||||
if j == m - 1:
|
||||
# single child => odd node
|
||||
h = ?proof.compress(h, p, K(bottomFlag.ord + 2))
|
||||
else:
|
||||
# even node
|
||||
h = ?proof.compress(h, p, bottomFlag)
|
||||
bottomFlag = K.KeyNone
|
||||
j = j shr 1
|
||||
m = (m + 1) shr 1
|
||||
"none"
|
||||
"StorageMerkleTree(" & " root: " & root & ", leavesCount: " & $self.leavesCount &
|
||||
", levels: " & $self.levels & ", mcodec: " & $self.mcodec & " )"
|
||||
|
||||
return success h
|
||||
proc `$`*(self: StorageMerkleProof): string =
|
||||
"StorageMerkleProof(" & " nleaves: " & $self.nleaves & ", index: " & $self.index &
|
||||
", path: " & $self.path.mapIt(byteutils.toHex(it)) & ", mcodec: " & $self.mcodec &
|
||||
" )"
|
||||
|
||||
func verify*[H, K](proof: MerkleProof[H, K], leaf: H, root: H): ?!bool =
|
||||
success bool(root == ?proof.reconstructRoot(leaf))
|
||||
|
||||
func fromNodes*[H, K](
|
||||
self: MerkleTree[H, K],
|
||||
compressor: CompressFn,
|
||||
zero: H,
|
||||
nodes: openArray[H],
|
||||
nleaves: int,
|
||||
): ?!void =
|
||||
mixin assign
|
||||
|
||||
if nodes.len < 2: # At least leaf and root
|
||||
return failure "Not enough nodes"
|
||||
|
||||
if nleaves == 0:
|
||||
return failure "No leaves"
|
||||
|
||||
self.compress = CompressData[H, K](fn: compressor, nodeSize: nodes[0].len, zero: zero)
|
||||
self.layerOffsets = layerOffsets(nleaves)
|
||||
|
||||
if self.layerOffsets[^1] + 1 != nodes.len:
|
||||
return failure "bad node count"
|
||||
|
||||
self.store = newSeqUninit[byte](nodes.len * self.compress.nodeSize)
|
||||
|
||||
for i in 0 ..< nodes.len:
|
||||
assign(
|
||||
self[].store.toOpenArray(
|
||||
i * self.compress.nodeSize, (i + 1) * self.compress.nodeSize - 1
|
||||
),
|
||||
nodes[i],
|
||||
)
|
||||
|
||||
success()
|
||||
|
||||
func merkleTreeWorker[H, K](
|
||||
store: var openArray[byte],
|
||||
offsets: openArray[int],
|
||||
compress: CompressData[H, K],
|
||||
layer: int,
|
||||
isBottomLayer: static bool,
|
||||
): ?!void =
|
||||
## Worker used to compute the merkle tree from the leaves that are assumed to
|
||||
## already be stored at the beginning of the `store`, as done by `prepare`.
|
||||
|
||||
# Throughout, we use `assign` to convert from H to bytes and back, assuming
|
||||
# this assignment can be done somewhat efficiently (ie memcpy) - because
|
||||
# the code must work with multihash where len(H) is can differ, we cannot
|
||||
# simply use a fixed-size array here.
|
||||
mixin assign
|
||||
|
||||
template nodeData(i, j: int): openArray[byte] =
|
||||
# Pick out the bytes of node j in layer i
|
||||
store.nodeData(offsets, compress.nodeSize, i, j)
|
||||
|
||||
let m = offsets.nodesInLayer(layer)
|
||||
|
||||
when not isBottomLayer:
|
||||
if m == 1:
|
||||
return success()
|
||||
|
||||
let halfn: int = m div 2
|
||||
let n: int = 2 * halfn
|
||||
let isOdd: bool = (n != m)
|
||||
|
||||
# Because the compression function we work with works with H and not bytes,
|
||||
# we need to extract H from the raw data - a little abstraction tax that
|
||||
# ensures that properties like alignment of H are respected.
|
||||
var a, b, tmp: H
|
||||
|
||||
for i in 0 ..< halfn:
|
||||
const key = when isBottomLayer: K.KeyBottomLayer else: K.KeyNone
|
||||
|
||||
assign(a, nodeData(layer, i * 2))
|
||||
assign(b, nodeData(layer, i * 2 + 1))
|
||||
|
||||
tmp = ?compress.fn(a, b, key = key)
|
||||
|
||||
assign(nodeData(layer + 1, i), tmp)
|
||||
|
||||
if isOdd:
|
||||
const key = when isBottomLayer: K.KeyOddAndBottomLayer else: K.KeyOdd
|
||||
|
||||
assign(a, nodeData(layer, n))
|
||||
|
||||
tmp = ?compress.fn(a, compress.zero, key = key)
|
||||
|
||||
assign(nodeData(layer + 1, halfn), tmp)
|
||||
|
||||
merkleTreeWorker(store, offsets, compress, layer + 1, false)
|
||||
|
||||
proc merkleTreeWorker[H, K](
|
||||
store: SharedBuf[byte],
|
||||
offsets: SharedBuf[int],
|
||||
compress: ptr CompressData[H, K],
|
||||
signal: ThreadSignalPtr,
|
||||
): bool =
|
||||
defer:
|
||||
discard signal.fireSync()
|
||||
|
||||
let res = merkleTreeWorker(
|
||||
store.toOpenArray(), offsets.toOpenArray(), compress[], 0, isBottomLayer = true
|
||||
)
|
||||
|
||||
return res.isOk()
|
||||
|
||||
func prepare*[H, K](
|
||||
self: MerkleTree[H, K], compressor: CompressFn, zero: H, leaves: openArray[H]
|
||||
): ?!void =
|
||||
## Prepare the instance for computing the merkle tree of the given leaves using
|
||||
## the given compression function. After preparation, `compute` should be
|
||||
## called to perform the actual computation. `leaves` will be copied into the
|
||||
## tree so they can be freed after the call.
|
||||
func compress*(x, y: openArray[byte], key: ByteTreeKey, codec: MultiCodec): ?!ByteHash =
|
||||
## Compress two hashes
|
||||
##
|
||||
let input = @x & @y & @[key.byte]
|
||||
let digest = ?MultiHash.digest(codec, input).mapFailure
|
||||
success digest.digestBytes
|
||||
|
||||
func initTree(mcodec: MultiCodec, leaves: openArray[ByteHash]): ?!StorageMerkleTree =
|
||||
if leaves.len == 0:
|
||||
return failure "No leaves"
|
||||
return failure "Empty leaves"
|
||||
|
||||
self.compress =
|
||||
CompressData[H, K](fn: compressor, nodeSize: leaves[0].len, zero: zero)
|
||||
self.layerOffsets = layerOffsets(leaves.len)
|
||||
let
|
||||
compressor = proc(x, y: seq[byte], key: ByteTreeKey): ?!ByteHash {.noSideEffect.} =
|
||||
compress(x, y, key, mcodec)
|
||||
digestSize = ?mcodec.digestSize.mapFailure
|
||||
Zero: ByteHash = newSeq[byte](digestSize)
|
||||
|
||||
self.store = newSeqUninit[byte]((self.layerOffsets[^1] + 1) * self.compress.nodeSize)
|
||||
if digestSize != leaves[0].len:
|
||||
return failure "Invalid hash length"
|
||||
|
||||
for j in 0 ..< leaves.len:
|
||||
assign(self[].nodeData(0, j), leaves[j])
|
||||
var self = StorageMerkleTree(mcodec: mcodec)
|
||||
?self.prepare(compressor, Zero, leaves)
|
||||
success self
|
||||
|
||||
return success()
|
||||
func init*(
|
||||
_: type StorageMerkleTree,
|
||||
mcodec: MultiCodec = Sha256HashCodec,
|
||||
leaves: openArray[ByteHash],
|
||||
): ?!StorageMerkleTree =
|
||||
let tree = ?initTree(mcodec, leaves)
|
||||
?tree.compute()
|
||||
success tree
|
||||
|
||||
proc compute*[H, K](self: MerkleTree[H, K]): ?!void =
|
||||
merkleTreeWorker(
|
||||
self.store, self.layerOffsets, self.compress, 0, isBottomLayer = true
|
||||
proc init*(
|
||||
_: type StorageMerkleTree,
|
||||
tp: Taskpool,
|
||||
mcodec: MultiCodec = Sha256HashCodec,
|
||||
leaves: seq[ByteHash],
|
||||
): Future[?!StorageMerkleTree] {.async: (raises: [CancelledError]).} =
|
||||
let tree = ?initTree(mcodec, leaves)
|
||||
?await tree.compute(tp)
|
||||
success tree
|
||||
|
||||
func init*(
|
||||
_: type StorageMerkleTree, leaves: openArray[MultiHash]
|
||||
): ?!StorageMerkleTree =
|
||||
if leaves.len == 0:
|
||||
return failure "Empty leaves"
|
||||
|
||||
let
|
||||
mcodec = leaves[0].mcodec
|
||||
leaves = leaves.mapIt(it.digestBytes)
|
||||
|
||||
StorageMerkleTree.init(mcodec, leaves)
|
||||
|
||||
proc init*(
|
||||
_: type StorageMerkleTree, tp: Taskpool, leaves: seq[MultiHash]
|
||||
): Future[?!StorageMerkleTree] {.async: (raises: [CancelledError]).} =
|
||||
if leaves.len == 0:
|
||||
return failure "Empty leaves"
|
||||
|
||||
let
|
||||
mcodec = leaves[0].mcodec
|
||||
leaves = leaves.mapIt(it.digestBytes)
|
||||
|
||||
await StorageMerkleTree.init(tp, mcodec, leaves)
|
||||
|
||||
func init*(_: type StorageMerkleTree, leaves: openArray[Cid]): ?!StorageMerkleTree =
|
||||
if leaves.len == 0:
|
||||
return failure "Empty leaves"
|
||||
|
||||
let
|
||||
mcodec = (?leaves[0].mhash.mapFailure).mcodec
|
||||
leaves = leaves.mapIt((?it.mhash.mapFailure).digestBytes)
|
||||
|
||||
StorageMerkleTree.init(mcodec, leaves)
|
||||
|
||||
proc init*(
|
||||
_: type StorageMerkleTree, tp: Taskpool, leaves: seq[Cid]
|
||||
): Future[?!StorageMerkleTree] {.async: (raises: [CancelledError]).} =
|
||||
if leaves.len == 0:
|
||||
return failure("Empty leaves")
|
||||
|
||||
let
|
||||
mcodec = (?leaves[0].mhash.mapFailure).mcodec
|
||||
leaves = leaves.mapIt((?it.mhash.mapFailure).digestBytes)
|
||||
|
||||
await StorageMerkleTree.init(tp, mcodec, leaves)
|
||||
|
||||
proc fromNodes*(
|
||||
_: type StorageMerkleTree,
|
||||
mcodec: MultiCodec = Sha256HashCodec,
|
||||
nodes: openArray[ByteHash],
|
||||
nleaves: int,
|
||||
): ?!StorageMerkleTree =
|
||||
if nodes.len == 0:
|
||||
return failure "Empty nodes"
|
||||
|
||||
let
|
||||
digestSize = ?mcodec.digestSize.mapFailure
|
||||
Zero = newSeq[byte](digestSize)
|
||||
compressor = proc(x, y: seq[byte], key: ByteTreeKey): ?!ByteHash {.noSideEffect.} =
|
||||
compress(x, y, key, mcodec)
|
||||
|
||||
if digestSize != nodes[0].len:
|
||||
return failure "Invalid hash length"
|
||||
|
||||
var self = StorageMerkleTree(mcodec: mcodec)
|
||||
?self.fromNodes(compressor, Zero, nodes, nleaves)
|
||||
|
||||
let
|
||||
index = Rng.instance.rand(nleaves - 1)
|
||||
proof = ?self.getProof(index)
|
||||
|
||||
if not ?proof.verify(self.leaves[index], ?self.root): # sanity check
|
||||
return failure "Unable to verify tree built from nodes"
|
||||
|
||||
success self
|
||||
|
||||
func init*(
|
||||
_: type StorageMerkleProof,
|
||||
mcodec: MultiCodec = Sha256HashCodec,
|
||||
index: int,
|
||||
nleaves: int,
|
||||
nodes: openArray[ByteHash],
|
||||
): ?!StorageMerkleProof =
|
||||
if nodes.len == 0:
|
||||
return failure "Empty nodes"
|
||||
|
||||
let
|
||||
digestSize = ?mcodec.digestSize.mapFailure
|
||||
Zero = newSeq[byte](digestSize)
|
||||
compressor = proc(x, y: seq[byte], key: ByteTreeKey): ?!seq[byte] {.noSideEffect.} =
|
||||
compress(x, y, key, mcodec)
|
||||
|
||||
success StorageMerkleProof(
|
||||
compress: compressor,
|
||||
zero: Zero,
|
||||
mcodec: mcodec,
|
||||
index: index,
|
||||
nleaves: nleaves,
|
||||
path: @nodes,
|
||||
)
|
||||
|
||||
proc compute*[H, K](
|
||||
self: MerkleTree[H, K], tp: Taskpool
|
||||
): Future[?!void] {.async: (raises: []).} =
|
||||
if tp.numThreads == 1:
|
||||
# With a single thread, there's no point creating a separate task
|
||||
return self.compute()
|
||||
|
||||
# TODO this signal would benefit from reuse across computations
|
||||
without signal =? ThreadSignalPtr.new():
|
||||
return failure("Unable to create thread signal")
|
||||
|
||||
defer:
|
||||
signal.close().expect("closing once works")
|
||||
|
||||
let res = tp.spawn merkleTreeWorker(
|
||||
SharedBuf.view(self.store),
|
||||
SharedBuf.view(self.layerOffsets),
|
||||
addr self.compress,
|
||||
signal,
|
||||
)
|
||||
|
||||
# To support cancellation, we'd have to ensure the task we posted to taskpools
|
||||
# exits early - since we're not doing that, block cancellation attempts
|
||||
try:
|
||||
await noCancel signal.wait()
|
||||
except AsyncError as exc:
|
||||
# Since we initialized the signal, the OS or chronos is misbehaving. In any
|
||||
# case, it would mean the task is still running which would cause a memory
|
||||
# a memory violation if we let it run - panic instead
|
||||
raiseAssert "Could not wait for signal, was it initialized? " & exc.msg
|
||||
|
||||
if not res.sync():
|
||||
return failure("merkle tree task failed")
|
||||
|
||||
return success()
|
||||
|
||||
@ -445,7 +445,7 @@ proc store*(
|
||||
finally:
|
||||
await stream.close()
|
||||
|
||||
without tree =? (await CodexTree.init(self.taskPool, cids)), err:
|
||||
without tree =? (await StorageMerkleTree.init(self.taskPool, cids)), err:
|
||||
return failure(err)
|
||||
|
||||
without treeCid =? tree.rootCid(CIDv1, dataCodec), err:
|
||||
|
||||
@ -80,7 +80,9 @@ method getBlocks*(
|
||||
|
||||
method getBlockAndProof*(
|
||||
self: BlockStore, treeCid: Cid, index: Natural
|
||||
): Future[?!(Block, CodexProof)] {.base, async: (raises: [CancelledError]), gcsafe.} =
|
||||
): Future[?!(Block, StorageMerkleProof)] {.
|
||||
base, async: (raises: [CancelledError]), gcsafe
|
||||
.} =
|
||||
## Get a block and associated inclusion proof by Cid of a merkle tree and an index of a leaf in a tree
|
||||
##
|
||||
|
||||
@ -95,7 +97,11 @@ method putBlock*(
|
||||
raiseAssert("putBlock not implemented!")
|
||||
|
||||
method putCidAndProof*(
|
||||
self: BlockStore, treeCid: Cid, index: Natural, blockCid: Cid, proof: CodexProof
|
||||
self: BlockStore,
|
||||
treeCid: Cid,
|
||||
index: Natural,
|
||||
blockCid: Cid,
|
||||
proof: StorageMerkleProof,
|
||||
): Future[?!void] {.base, async: (raises: [CancelledError]), gcsafe.} =
|
||||
## Put a block proof to the blockstore
|
||||
##
|
||||
@ -104,7 +110,9 @@ method putCidAndProof*(
|
||||
|
||||
method getCidAndProof*(
|
||||
self: BlockStore, treeCid: Cid, index: Natural
|
||||
): Future[?!(Cid, CodexProof)] {.base, async: (raises: [CancelledError]), gcsafe.} =
|
||||
): Future[?!(Cid, StorageMerkleProof)] {.
|
||||
base, async: (raises: [CancelledError]), gcsafe
|
||||
.} =
|
||||
## Get a block proof from the blockstore
|
||||
##
|
||||
|
||||
|
||||
@ -37,7 +37,7 @@ type
|
||||
currentSize*: NBytes
|
||||
size*: NBytes
|
||||
cache: LruCache[Cid, Block]
|
||||
cidAndProofCache: LruCache[(Cid, Natural), (Cid, CodexProof)]
|
||||
cidAndProofCache: LruCache[(Cid, Natural), (Cid, StorageMerkleProof)]
|
||||
|
||||
InvalidBlockSize* = object of CodexError
|
||||
|
||||
@ -83,7 +83,7 @@ method getBlocks*(
|
||||
|
||||
method getCidAndProof*(
|
||||
self: CacheStore, treeCid: Cid, index: Natural
|
||||
): Future[?!(Cid, CodexProof)] {.async: (raises: [CancelledError]).} =
|
||||
): Future[?!(Cid, StorageMerkleProof)] {.async: (raises: [CancelledError]).} =
|
||||
if cidAndProof =? self.cidAndProofCache.getOption((treeCid, index)):
|
||||
success(cidAndProof)
|
||||
else:
|
||||
@ -103,7 +103,7 @@ method getBlock*(
|
||||
|
||||
method getBlockAndProof*(
|
||||
self: CacheStore, treeCid: Cid, index: Natural
|
||||
): Future[?!(Block, CodexProof)] {.async: (raises: [CancelledError]).} =
|
||||
): Future[?!(Block, StorageMerkleProof)] {.async: (raises: [CancelledError]).} =
|
||||
without cidAndProof =? (await self.getCidAndProof(treeCid, index)), err:
|
||||
return failure(err)
|
||||
|
||||
@ -226,7 +226,11 @@ method putBlock*(
|
||||
return success()
|
||||
|
||||
method putCidAndProof*(
|
||||
self: CacheStore, treeCid: Cid, index: Natural, blockCid: Cid, proof: CodexProof
|
||||
self: CacheStore,
|
||||
treeCid: Cid,
|
||||
index: Natural,
|
||||
blockCid: Cid,
|
||||
proof: StorageMerkleProof,
|
||||
): Future[?!void] {.async: (raises: [CancelledError]).} =
|
||||
self.cidAndProofCache[(treeCid, index)] = (blockCid, proof)
|
||||
success()
|
||||
@ -301,7 +305,7 @@ proc new*(
|
||||
currentSize = 0'nb
|
||||
size = int(cacheSize div chunkSize)
|
||||
cache = newLruCache[Cid, Block](size)
|
||||
cidAndProofCache = newLruCache[(Cid, Natural), (Cid, CodexProof)](size)
|
||||
cidAndProofCache = newLruCache[(Cid, Natural), (Cid, StorageMerkleProof)](size)
|
||||
store = CacheStore(
|
||||
cache: cache,
|
||||
cidAndProofCache: cidAndProofCache,
|
||||
|
||||
@ -104,13 +104,17 @@ method putBlock*(
|
||||
return success()
|
||||
|
||||
method putCidAndProof*(
|
||||
self: NetworkStore, treeCid: Cid, index: Natural, blockCid: Cid, proof: CodexProof
|
||||
self: NetworkStore,
|
||||
treeCid: Cid,
|
||||
index: Natural,
|
||||
blockCid: Cid,
|
||||
proof: StorageMerkleProof,
|
||||
): Future[?!void] {.async: (raw: true, raises: [CancelledError]).} =
|
||||
self.localStore.putCidAndProof(treeCid, index, blockCid, proof)
|
||||
|
||||
method getCidAndProof*(
|
||||
self: NetworkStore, treeCid: Cid, index: Natural
|
||||
): Future[?!(Cid, CodexProof)] {.async: (raw: true, raises: [CancelledError]).} =
|
||||
): Future[?!(Cid, StorageMerkleProof)] {.async: (raw: true, raises: [CancelledError]).} =
|
||||
## Get a block proof from the blockstore
|
||||
##
|
||||
|
||||
|
||||
@ -33,7 +33,11 @@ declareGauge(codex_repostore_bytes_used, "codex repostore bytes used")
|
||||
declareGauge(codex_repostore_bytes_reserved, "codex repostore bytes reserved")
|
||||
|
||||
proc putLeafMetadata*(
|
||||
self: RepoStore, treeCid: Cid, index: Natural, blkCid: Cid, proof: CodexProof
|
||||
self: RepoStore,
|
||||
treeCid: Cid,
|
||||
index: Natural,
|
||||
blkCid: Cid,
|
||||
proof: StorageMerkleProof,
|
||||
): Future[?!StoreResultKind] {.async: (raises: [CancelledError]).} =
|
||||
without key =? createBlockCidAndProofMetadataKey(treeCid, index), err:
|
||||
return failure(err)
|
||||
|
||||
@ -82,7 +82,7 @@ method getBlock*(
|
||||
|
||||
method getBlockAndProof*(
|
||||
self: RepoStore, treeCid: Cid, index: Natural
|
||||
): Future[?!(Block, CodexProof)] {.async: (raises: [CancelledError]).} =
|
||||
): Future[?!(Block, StorageMerkleProof)] {.async: (raises: [CancelledError]).} =
|
||||
without leafMd =? await self.getLeafMetadata(treeCid, index), err:
|
||||
return failure(err)
|
||||
|
||||
@ -136,7 +136,11 @@ method ensureExpiry*(
|
||||
await self.ensureExpiry(leafMd.blkCid, expiry)
|
||||
|
||||
method putCidAndProof*(
|
||||
self: RepoStore, treeCid: Cid, index: Natural, blkCid: Cid, proof: CodexProof
|
||||
self: RepoStore,
|
||||
treeCid: Cid,
|
||||
index: Natural,
|
||||
blkCid: Cid,
|
||||
proof: StorageMerkleProof,
|
||||
): Future[?!void] {.async: (raises: [CancelledError]).} =
|
||||
## Put a block to the blockstore
|
||||
##
|
||||
@ -163,7 +167,7 @@ method putCidAndProof*(
|
||||
|
||||
method getCidAndProof*(
|
||||
self: RepoStore, treeCid: Cid, index: Natural
|
||||
): Future[?!(Cid, CodexProof)] {.async: (raises: [CancelledError]).} =
|
||||
): Future[?!(Cid, StorageMerkleProof)] {.async: (raises: [CancelledError]).} =
|
||||
without leafMd =? await self.getLeafMetadata(treeCid, index), err:
|
||||
return failure(err)
|
||||
|
||||
|
||||
@ -49,7 +49,7 @@ type
|
||||
|
||||
LeafMetadata* {.serialize.} = object
|
||||
blkCid*: Cid
|
||||
proof*: CodexProof
|
||||
proof*: StorageMerkleProof
|
||||
|
||||
BlockExpiration* {.serialize.} = object
|
||||
cid*: Cid
|
||||
|
||||
@ -21,7 +21,7 @@ import ../utils/asynciter
|
||||
import ../merkletree
|
||||
|
||||
proc putSomeProofs*(
|
||||
store: BlockStore, tree: CodexTree, iter: Iter[int]
|
||||
store: BlockStore, tree: StorageMerkleTree, iter: Iter[int]
|
||||
): Future[?!void] {.async.} =
|
||||
without treeCid =? tree.rootCid, err:
|
||||
return failure(err)
|
||||
@ -47,9 +47,9 @@ proc putSomeProofs*(
|
||||
success()
|
||||
|
||||
proc putSomeProofs*(
|
||||
store: BlockStore, tree: CodexTree, iter: Iter[Natural]
|
||||
store: BlockStore, tree: StorageMerkleTree, iter: Iter[Natural]
|
||||
): Future[?!void] =
|
||||
store.putSomeProofs(tree, iter.map((i: Natural) => i.ord))
|
||||
|
||||
proc putAllProofs*(store: BlockStore, tree: CodexTree): Future[?!void] =
|
||||
proc putAllProofs*(store: BlockStore, tree: StorageMerkleTree): Future[?!void] =
|
||||
store.putSomeProofs(tree, Iter[int].new(0 ..< tree.leavesCount))
|
||||
|
||||
@ -25,7 +25,7 @@ asyncchecksuite "Block Advertising and Discovery":
|
||||
var
|
||||
blocks: seq[bt.Block]
|
||||
manifest: Manifest
|
||||
tree: CodexTree
|
||||
tree: StorageMerkleTree
|
||||
manifestBlock: bt.Block
|
||||
switch: Switch
|
||||
peerStore: PeerCtxStore
|
||||
@ -158,7 +158,7 @@ asyncchecksuite "E2E - Multiple Nodes Discovery":
|
||||
blockexc: seq[NetworkStore]
|
||||
manifests: seq[Manifest]
|
||||
mBlocks: seq[bt.Block]
|
||||
trees: seq[CodexTree]
|
||||
trees: seq[StorageMerkleTree]
|
||||
|
||||
setup:
|
||||
for _ in 0 ..< 4:
|
||||
|
||||
@ -27,7 +27,7 @@ asyncchecksuite "Test Discovery Engine":
|
||||
var
|
||||
blocks: seq[bt.Block]
|
||||
manifest: Manifest
|
||||
tree: CodexTree
|
||||
tree: StorageMerkleTree
|
||||
manifestBlock: bt.Block
|
||||
switch: Switch
|
||||
peerStore: PeerCtxStore
|
||||
|
||||
@ -8,7 +8,8 @@ import pkg/codex/rng
|
||||
|
||||
import ./randomchunker
|
||||
|
||||
type TestDataset* = tuple[blocks: seq[Block], tree: CodexTree, manifest: Manifest]
|
||||
type TestDataset* =
|
||||
tuple[blocks: seq[Block], tree: StorageMerkleTree, manifest: Manifest]
|
||||
|
||||
proc makeRandomBlock*(size: NBytes): Block =
|
||||
let bytes = newSeqWith(size.int, rand(uint8))
|
||||
@ -34,7 +35,7 @@ proc makeDataset*(blocks: seq[Block]): ?!TestDataset =
|
||||
let
|
||||
datasetSize = blocks.mapIt(it.data.len).foldl(a + b)
|
||||
blockSize = blocks.mapIt(it.data.len).foldl(max(a, b))
|
||||
tree = ?CodexTree.init(blocks.mapIt(it.cid))
|
||||
tree = ?StorageMerkleTree.init(blocks.mapIt(it.cid))
|
||||
treeCid = ?tree.rootCid
|
||||
manifest = Manifest.new(
|
||||
treeCid = treeCid,
|
||||
|
||||
@ -1,118 +0,0 @@
|
||||
import pkg/unittest2
|
||||
|
||||
import pkg/codex/merkletree
|
||||
|
||||
proc testGenericTree*[H, K, U](
|
||||
name: string,
|
||||
data: openArray[H],
|
||||
zero: H,
|
||||
compress: proc(z, y: H, key: K): H,
|
||||
makeTree: proc(data: seq[H]): U,
|
||||
) =
|
||||
let data = @data
|
||||
|
||||
suite "Correctness tests - " & name:
|
||||
test "Should build correct tree for single leaf":
|
||||
let expectedRoot = compress(data[0], zero, K.KeyOddAndBottomLayer)
|
||||
|
||||
let tree = makeTree(data[0 .. 0])
|
||||
check:
|
||||
tree.root.tryGet == expectedRoot
|
||||
|
||||
test "Should build correct tree for even bottom layer":
|
||||
let expectedRoot = compress(
|
||||
compress(
|
||||
compress(data[0], data[1], K.KeyBottomLayer),
|
||||
compress(data[2], data[3], K.KeyBottomLayer),
|
||||
K.KeyNone,
|
||||
),
|
||||
compress(
|
||||
compress(data[4], data[5], K.KeyBottomLayer),
|
||||
compress(data[6], data[7], K.KeyBottomLayer),
|
||||
K.KeyNone,
|
||||
),
|
||||
K.KeyNone,
|
||||
)
|
||||
|
||||
let tree = makeTree(data[0 .. 7])
|
||||
|
||||
check:
|
||||
tree.root.tryGet == expectedRoot
|
||||
|
||||
test "Should build correct tree for odd bottom layer":
|
||||
let expectedRoot = compress(
|
||||
compress(
|
||||
compress(data[0], data[1], K.KeyBottomLayer),
|
||||
compress(data[2], data[3], K.KeyBottomLayer),
|
||||
K.KeyNone,
|
||||
),
|
||||
compress(
|
||||
compress(data[4], data[5], K.KeyBottomLayer),
|
||||
compress(data[6], zero, K.KeyOddAndBottomLayer),
|
||||
K.KeyNone,
|
||||
),
|
||||
K.KeyNone,
|
||||
)
|
||||
|
||||
let tree = makeTree(data[0 .. 6])
|
||||
|
||||
check:
|
||||
tree.root.tryGet == expectedRoot
|
||||
|
||||
test "Should build correct tree for even bottom and odd upper layers":
|
||||
let expectedRoot = compress(
|
||||
compress(
|
||||
compress(
|
||||
compress(data[0], data[1], K.KeyBottomLayer),
|
||||
compress(data[2], data[3], K.KeyBottomLayer),
|
||||
K.KeyNone,
|
||||
),
|
||||
compress(
|
||||
compress(data[4], data[5], K.KeyBottomLayer),
|
||||
compress(data[6], data[7], K.KeyBottomLayer),
|
||||
K.KeyNone,
|
||||
),
|
||||
K.KeyNone,
|
||||
),
|
||||
compress(
|
||||
compress(compress(data[8], data[9], K.KeyBottomLayer), zero, K.KeyOdd),
|
||||
zero,
|
||||
K.KeyOdd,
|
||||
),
|
||||
K.KeyNone,
|
||||
)
|
||||
|
||||
let tree = makeTree(data[0 .. 9])
|
||||
|
||||
check:
|
||||
tree.root.tryGet == expectedRoot
|
||||
|
||||
test "Should get and validate correct proofs":
|
||||
let expectedRoot = compress(
|
||||
compress(
|
||||
compress(
|
||||
compress(data[0], data[1], K.KeyBottomLayer),
|
||||
compress(data[2], data[3], K.KeyBottomLayer),
|
||||
K.KeyNone,
|
||||
),
|
||||
compress(
|
||||
compress(data[4], data[5], K.KeyBottomLayer),
|
||||
compress(data[6], data[7], K.KeyBottomLayer),
|
||||
K.KeyNone,
|
||||
),
|
||||
K.KeyNone,
|
||||
),
|
||||
compress(
|
||||
compress(compress(data[8], data[9], K.KeyBottomLayer), zero, K.KeyOdd),
|
||||
zero,
|
||||
K.KeyOdd,
|
||||
),
|
||||
K.KeyNone,
|
||||
)
|
||||
|
||||
let tree = makeTree(data)
|
||||
|
||||
for i in 0 ..< data.len:
|
||||
let proof = tree.getProof(i).tryGet
|
||||
check:
|
||||
proof.verify(tree.leaves[i], expectedRoot).isOk
|
||||
@ -3,9 +3,9 @@ import ../helpers
|
||||
|
||||
export merkletree, helpers
|
||||
|
||||
proc `==`*(a, b: CodexTree): bool =
|
||||
proc `==`*(a, b: StorageMerkleTree): bool =
|
||||
(a.mcodec == b.mcodec) and (a.leavesCount == b.leavesCount) and (a.levels == b.levels)
|
||||
|
||||
proc `==`*(a, b: CodexProof): bool =
|
||||
proc `==`*(a, b: StorageMerkleProof): bool =
|
||||
(a.mcodec == b.mcodec) and (a.nleaves == b.nleaves) and (a.path == b.path) and
|
||||
(a.index == b.index)
|
||||
|
||||
@ -21,16 +21,16 @@ const data = [
|
||||
suite "merkletree - coders":
|
||||
test "encoding and decoding a tree yields the same tree":
|
||||
let
|
||||
tree = CodexTree.init(Sha256HashCodec, data).tryGet()
|
||||
tree = StorageMerkleTree.init(Sha256HashCodec, data).tryGet()
|
||||
encodedBytes = tree.encode()
|
||||
decodedTree = CodexTree.decode(encodedBytes).tryGet()
|
||||
decodedTree = StorageMerkleTree.decode(encodedBytes).tryGet()
|
||||
|
||||
check:
|
||||
tree == decodedTree
|
||||
|
||||
test "encoding and decoding a proof yields the same proof":
|
||||
let
|
||||
tree = CodexTree.init(Sha256HashCodec, data).tryGet()
|
||||
tree = StorageMerkleTree.init(Sha256HashCodec, data).tryGet()
|
||||
proof = tree.getProof(4).tryGet()
|
||||
|
||||
check:
|
||||
@ -38,7 +38,7 @@ suite "merkletree - coders":
|
||||
|
||||
let
|
||||
encodedBytes = proof.encode()
|
||||
decodedProof = CodexProof.decode(encodedBytes).tryGet()
|
||||
decodedProof = StorageMerkleProof.decode(encodedBytes).tryGet()
|
||||
|
||||
check:
|
||||
proof == decodedProof
|
||||
|
||||
@ -12,7 +12,6 @@ import pkg/codex/utils/digest
|
||||
import pkg/taskpools
|
||||
|
||||
import ./helpers
|
||||
import ./generictreetests
|
||||
import ../../asynctest
|
||||
|
||||
# TODO: Generalize to other hashes
|
||||
@ -32,23 +31,23 @@ const
|
||||
]
|
||||
sha256 = Sha256HashCodec
|
||||
|
||||
suite "Test CodexTree":
|
||||
suite "Test StorageMerkleTree":
|
||||
test "Cannot init tree without any multihash leaves":
|
||||
check:
|
||||
CodexTree.init(leaves = newSeq[MultiHash]()).isErr
|
||||
StorageMerkleTree.init(leaves = newSeq[MultiHash]()).isErr
|
||||
|
||||
test "Cannot init tree without any cid leaves":
|
||||
check:
|
||||
CodexTree.init(leaves = newSeq[Cid]()).isErr
|
||||
StorageMerkleTree.init(leaves = newSeq[Cid]()).isErr
|
||||
|
||||
test "Cannot init tree without any byte leaves":
|
||||
check:
|
||||
CodexTree.init(sha256, leaves = newSeq[ByteHash]()).isErr
|
||||
StorageMerkleTree.init(sha256, leaves = newSeq[ByteHash]()).isErr
|
||||
|
||||
test "Should build tree from multihash leaves":
|
||||
var
|
||||
expectedLeaves = data.mapIt(MultiHash.digest($sha256, it).tryGet())
|
||||
tree = CodexTree.init(leaves = expectedLeaves)
|
||||
tree = StorageMerkleTree.init(leaves = expectedLeaves)
|
||||
|
||||
check:
|
||||
tree.isOk
|
||||
@ -62,7 +61,7 @@ suite "Test CodexTree":
|
||||
|
||||
let expectedLeaves = data.mapIt(MultiHash.digest($sha256, it).tryGet())
|
||||
|
||||
let tree = (await CodexTree.init(tp, leaves = expectedLeaves))
|
||||
let tree = (await StorageMerkleTree.init(tp, leaves = expectedLeaves))
|
||||
check:
|
||||
tree.isOk
|
||||
tree.get().leaves == expectedLeaves.mapIt(it.digestBytes)
|
||||
@ -73,7 +72,7 @@ suite "Test CodexTree":
|
||||
Cid.init(CidVersion.CIDv1, BlockCodec, MultiHash.digest($sha256, it).tryGet).tryGet
|
||||
)
|
||||
|
||||
let tree = CodexTree.init(leaves = expectedLeaves)
|
||||
let tree = StorageMerkleTree.init(leaves = expectedLeaves)
|
||||
|
||||
check:
|
||||
tree.isOk
|
||||
@ -89,7 +88,7 @@ suite "Test CodexTree":
|
||||
Cid.init(CidVersion.CIDv1, BlockCodec, MultiHash.digest($sha256, it).tryGet).tryGet
|
||||
)
|
||||
|
||||
let tree = (await CodexTree.init(tp, leaves = expectedLeaves))
|
||||
let tree = (await StorageMerkleTree.init(tp, leaves = expectedLeaves))
|
||||
|
||||
check:
|
||||
tree.isOk
|
||||
@ -106,8 +105,8 @@ suite "Test CodexTree":
|
||||
)
|
||||
|
||||
let
|
||||
atree = (await CodexTree.init(tp, leaves = expectedLeaves))
|
||||
stree = CodexTree.init(leaves = expectedLeaves)
|
||||
atree = (await StorageMerkleTree.init(tp, leaves = expectedLeaves))
|
||||
stree = StorageMerkleTree.init(leaves = expectedLeaves)
|
||||
|
||||
check:
|
||||
toSeq(atree.get().nodes) == toSeq(stree.get().nodes)
|
||||
@ -115,15 +114,15 @@ suite "Test CodexTree":
|
||||
|
||||
# Single-leaf trees have their root separately computed
|
||||
let
|
||||
atree1 = (await CodexTree.init(tp, leaves = expectedLeaves[0 .. 0]))
|
||||
stree1 = CodexTree.init(leaves = expectedLeaves[0 .. 0])
|
||||
atree1 = (await StorageMerkleTree.init(tp, leaves = expectedLeaves[0 .. 0]))
|
||||
stree1 = StorageMerkleTree.init(leaves = expectedLeaves[0 .. 0])
|
||||
|
||||
check:
|
||||
toSeq(atree.get().nodes) == toSeq(stree.get().nodes)
|
||||
atree.get().root == stree.get().root
|
||||
|
||||
test "Should build from raw digestbytes (should not hash leaves)":
|
||||
let tree = CodexTree.init(sha256, leaves = data).tryGet
|
||||
let tree = StorageMerkleTree.init(sha256, leaves = data).tryGet
|
||||
|
||||
check:
|
||||
tree.mcodec == sha256
|
||||
@ -134,7 +133,7 @@ suite "Test CodexTree":
|
||||
defer:
|
||||
tp.shutdown()
|
||||
|
||||
let tree = (await CodexTree.init(tp, sha256, leaves = @data))
|
||||
let tree = (await StorageMerkleTree.init(tp, sha256, leaves = @data))
|
||||
|
||||
check:
|
||||
tree.isOk
|
||||
@ -143,8 +142,8 @@ suite "Test CodexTree":
|
||||
|
||||
test "Should build from nodes":
|
||||
let
|
||||
tree = CodexTree.init(sha256, leaves = data).tryGet
|
||||
fromNodes = CodexTree.fromNodes(
|
||||
tree = StorageMerkleTree.init(sha256, leaves = data).tryGet
|
||||
fromNodes = StorageMerkleTree.fromNodes(
|
||||
nodes = toSeq(tree.nodes), nleaves = tree.leavesCount
|
||||
).tryGet
|
||||
|
||||
@ -158,7 +157,5 @@ let
|
||||
compress = proc(x, y: seq[byte], key: ByteTreeKey): seq[byte] =
|
||||
compress(x, y, key, sha256).tryGet
|
||||
|
||||
makeTree = proc(data: seq[seq[byte]]): CodexTree =
|
||||
CodexTree.init(sha256, leaves = data).tryGet
|
||||
|
||||
testGenericTree("CodexTree", @data, zero, compress, makeTree)
|
||||
makeTree = proc(data: seq[seq[byte]]): StorageMerkleTree =
|
||||
StorageMerkleTree.init(sha256, leaves = data).tryGet
|
||||
|
||||
@ -46,7 +46,7 @@ proc makeManifest*(
|
||||
hcodec = Sha256HashCodec,
|
||||
dataCodec = BlockCodec,
|
||||
): Future[?!Manifest] {.async.} =
|
||||
without tree =? CodexTree.init(cids), err:
|
||||
without tree =? StorageMerkleTree.init(cids), err:
|
||||
return failure(err)
|
||||
|
||||
without treeCid =? tree.rootCid(CIDv1, dataCodec), err:
|
||||
|
||||
@ -29,7 +29,7 @@ proc commonBlockStoreTests*(
|
||||
var
|
||||
newBlock, newBlock1, newBlock2, newBlock3: Block
|
||||
manifest: Manifest
|
||||
tree: CodexTree
|
||||
tree: StorageMerkleTree
|
||||
store: BlockStore
|
||||
|
||||
setup:
|
||||
|
||||
@ -16,7 +16,7 @@ import pkg/codex/stores/repostore/operations
|
||||
import pkg/codex/blocktype as bt
|
||||
import pkg/codex/clock
|
||||
import pkg/codex/utils/safeasynciter
|
||||
import pkg/codex/merkletree/codex
|
||||
import pkg/codex/merkletree
|
||||
|
||||
import ../../asynctest
|
||||
import ../helpers
|
||||
|
||||
1
vendor/nim-merkletree
vendored
Submodule
1
vendor/nim-merkletree
vendored
Submodule
@ -0,0 +1 @@
|
||||
Subproject commit 685c40cc5ec7d66112e8cf6be4eed59f619c51de
|
||||
Loading…
x
Reference in New Issue
Block a user