mirror of
https://github.com/logos-storage/logos-storage-nim.git
synced 2026-03-02 10:23:07 +00:00
Rename codex merkletree types
- Rename CodexTree > StorageMerkleTree - Rename CodexProofs > StorageMerkleProof
This commit is contained in:
parent
d627b1738b
commit
204ee9e27f
@ -848,12 +848,12 @@ proc localLookup(
|
||||
): Future[?!BlockDelivery] {.async: (raises: [CancelledError]).} =
|
||||
if address.leaf:
|
||||
(await self.localStore.getBlockAndProof(address.treeCid, address.index)).map(
|
||||
(blkAndProof: (Block, CodexProof)) =>
|
||||
(blkAndProof: (Block, StorageMerkleProof)) =>
|
||||
BlockDelivery(address: address, blk: blkAndProof[0], proof: blkAndProof[1].some)
|
||||
)
|
||||
else:
|
||||
(await self.localStore.getBlock(address)).map(
|
||||
(blk: Block) => BlockDelivery(address: address, blk: blk, proof: CodexProof.none)
|
||||
(blk: Block) => BlockDelivery(address: address, blk: blk, proof: StorageMerkleProof.none)
|
||||
)
|
||||
|
||||
iterator splitBatches[T](sequence: seq[T], batchSize: int): seq[T] =
|
||||
|
||||
@ -42,7 +42,7 @@ type
|
||||
BlockDelivery* = object
|
||||
blk*: Block
|
||||
address*: BlockAddress
|
||||
proof*: ?CodexProof # Present only if `address.leaf` is true
|
||||
proof*: ?StorageMerkleProof # Present only if `address.leaf` is true
|
||||
|
||||
BlockPresenceType* = enum
|
||||
Have = 0
|
||||
@ -201,12 +201,12 @@ proc decode*(_: type BlockDelivery, pb: ProtoBuffer): ProtoResult[BlockDelivery]
|
||||
if value.address.leaf:
|
||||
var proofBuf = newSeq[byte]()
|
||||
if ?pb.getField(4, proofBuf):
|
||||
let proof = ?CodexProof.decode(proofBuf).mapErr(x => ProtoError.IncorrectBlob)
|
||||
let proof = ?StorageMerkleProof.decode(proofBuf).mapErr(x => ProtoError.IncorrectBlob)
|
||||
value.proof = proof.some
|
||||
else:
|
||||
value.proof = CodexProof.none
|
||||
value.proof = StorageMerkleProof.none
|
||||
else:
|
||||
value.proof = CodexProof.none
|
||||
value.proof = StorageMerkleProof.none
|
||||
|
||||
ok(value)
|
||||
|
||||
|
||||
@ -22,7 +22,7 @@ import ./merkletree
|
||||
const MaxMerkleTreeSize = 100.MiBs.uint
|
||||
const MaxMerkleProofSize = 1.MiBs.uint
|
||||
|
||||
proc encode*(self: CodexTree): seq[byte] =
|
||||
proc encode*(self: StorageMerkleTree): seq[byte] =
|
||||
var pb = initProtoBuffer()
|
||||
pb.write(1, self.mcodec.uint64)
|
||||
pb.write(2, self.leavesCount.uint64)
|
||||
@ -35,7 +35,7 @@ proc encode*(self: CodexTree): seq[byte] =
|
||||
pb.finish
|
||||
pb.buffer
|
||||
|
||||
proc decode*(_: type CodexTree, data: seq[byte]): ?!CodexTree =
|
||||
proc decode*(_: type StorageMerkleTree, data: seq[byte]): ?!StorageMerkleTree =
|
||||
var pb = initProtoBuffer(data)
|
||||
var mcodecCode: uint64
|
||||
var leavesCount: uint64
|
||||
@ -56,9 +56,9 @@ proc decode*(_: type CodexTree, data: seq[byte]): ?!CodexTree =
|
||||
discard ?initProtoBuffer(nodeBuff).getField(1, node).mapFailure
|
||||
nodes.add node
|
||||
|
||||
CodexTree.fromNodes(mcodec, nodes, leavesCount.int)
|
||||
StorageMerkleTree.fromNodes(mcodec, nodes, leavesCount.int)
|
||||
|
||||
proc encode*(self: CodexProof): seq[byte] =
|
||||
proc encode*(self: StorageMerkleProof): seq[byte] =
|
||||
var pb = initProtoBuffer()
|
||||
pb.write(1, self.mcodec.uint64)
|
||||
pb.write(2, self.index.uint64)
|
||||
@ -73,7 +73,7 @@ proc encode*(self: CodexProof): seq[byte] =
|
||||
pb.finish
|
||||
pb.buffer
|
||||
|
||||
proc decode*(_: type CodexProof, data: seq[byte]): ?!CodexProof =
|
||||
proc decode*(_: type StorageMerkleProof, data: seq[byte]): ?!StorageMerkleProof =
|
||||
var pb = initProtoBuffer(data)
|
||||
var mcodecCode: uint64
|
||||
var index: uint64
|
||||
@ -98,9 +98,9 @@ proc decode*(_: type CodexProof, data: seq[byte]): ?!CodexProof =
|
||||
discard ?nodePb.getField(1, node).mapFailure
|
||||
nodes.add node
|
||||
|
||||
CodexProof.init(mcodec, index.int, nleaves.int, nodes)
|
||||
StorageMerkleProof.init(mcodec, index.int, nleaves.int, nodes)
|
||||
|
||||
proc fromJson*(_: type CodexProof, json: JsonNode): ?!CodexProof =
|
||||
proc fromJson*(_: type StorageMerkleProof, json: JsonNode): ?!StorageMerkleProof =
|
||||
expectJsonKind(Cid, JString, json)
|
||||
var bytes: seq[byte]
|
||||
try:
|
||||
@ -108,7 +108,7 @@ proc fromJson*(_: type CodexProof, json: JsonNode): ?!CodexProof =
|
||||
except ValueError as err:
|
||||
return failure(err)
|
||||
|
||||
CodexProof.decode(bytes)
|
||||
StorageMerkleProof.decode(bytes)
|
||||
|
||||
func `%`*(proof: CodexProof): JsonNode =
|
||||
func `%`*(proof: StorageMerkleProof): JsonNode =
|
||||
%byteutils.toHex(proof.encode())
|
||||
|
||||
@ -43,20 +43,20 @@ type
|
||||
ByteTree* = MerkleTree[ByteHash, ByteTreeKey]
|
||||
ByteProof* = MerkleProof[ByteHash, ByteTreeKey]
|
||||
|
||||
CodexTree* = ref object of ByteTree
|
||||
StorageMerkleTree* = ref object of ByteTree
|
||||
mcodec*: MultiCodec
|
||||
|
||||
CodexProof* = ref object of ByteProof
|
||||
StorageMerkleProof* = ref object of ByteProof
|
||||
mcodec*: MultiCodec
|
||||
|
||||
func getProof*(self: CodexTree, index: int): ?!CodexProof =
|
||||
var proof = CodexProof(mcodec: self.mcodec)
|
||||
func getProof*(self: StorageMerkleTree, index: int): ?!StorageMerkleProof =
|
||||
var proof = StorageMerkleProof(mcodec: self.mcodec)
|
||||
|
||||
?self.getProof(index, proof)
|
||||
|
||||
success proof
|
||||
|
||||
func verify*(self: CodexProof, leaf: MultiHash, root: MultiHash): ?!bool =
|
||||
func verify*(self: StorageMerkleProof, leaf: MultiHash, root: MultiHash): ?!bool =
|
||||
## Verify hash
|
||||
##
|
||||
|
||||
@ -72,10 +72,10 @@ func verify*(self: CodexProof, leaf: MultiHash, root: MultiHash): ?!bool =
|
||||
|
||||
self.verify(leafBytes, rootBytes)
|
||||
|
||||
func verify*(self: CodexProof, leaf: Cid, root: Cid): ?!bool =
|
||||
func verify*(self: StorageMerkleProof, leaf: Cid, root: Cid): ?!bool =
|
||||
self.verify(?leaf.mhash.mapFailure, ?leaf.mhash.mapFailure)
|
||||
|
||||
proc rootCid*(self: CodexTree, version = CIDv1, dataCodec = DatasetRootCodec): ?!Cid =
|
||||
proc rootCid*(self: StorageMerkleTree, version = CIDv1, dataCodec = DatasetRootCodec): ?!Cid =
|
||||
if (?self.root).len == 0:
|
||||
return failure "Empty root"
|
||||
|
||||
@ -84,7 +84,7 @@ proc rootCid*(self: CodexTree, version = CIDv1, dataCodec = DatasetRootCodec): ?
|
||||
Cid.init(version, DatasetRootCodec, mhash).mapFailure
|
||||
|
||||
func getLeafCid*(
|
||||
self: CodexTree, i: Natural, version = CIDv1, dataCodec = BlockCodec
|
||||
self: StorageMerkleTree, i: Natural, version = CIDv1, dataCodec = BlockCodec
|
||||
): ?!Cid =
|
||||
if i >= self.leavesCount:
|
||||
return failure "Invalid leaf index " & $i
|
||||
@ -95,17 +95,17 @@ func getLeafCid*(
|
||||
|
||||
Cid.init(version, dataCodec, mhash).mapFailure
|
||||
|
||||
proc `$`*(self: CodexTree): string =
|
||||
proc `$`*(self: StorageMerkleTree): string =
|
||||
let root =
|
||||
if self.root.isOk:
|
||||
byteutils.toHex(self.root.get)
|
||||
else:
|
||||
"none"
|
||||
"CodexTree(" & " root: " & root & ", leavesCount: " & $self.leavesCount & ", levels: " &
|
||||
"StorageMerkleTree(" & " root: " & root & ", leavesCount: " & $self.leavesCount & ", levels: " &
|
||||
$self.levels & ", mcodec: " & $self.mcodec & " )"
|
||||
|
||||
proc `$`*(self: CodexProof): string =
|
||||
"CodexProof(" & " nleaves: " & $self.nleaves & ", index: " & $self.index & ", path: " &
|
||||
proc `$`*(self: StorageMerkleProof): string =
|
||||
"StorageMerkleProof(" & " nleaves: " & $self.nleaves & ", index: " & $self.index & ", path: " &
|
||||
$self.path.mapIt(byteutils.toHex(it)) & ", mcodec: " & $self.mcodec & " )"
|
||||
|
||||
func compress*(x, y: openArray[byte], key: ByteTreeKey, codec: MultiCodec): ?!ByteHash =
|
||||
@ -115,7 +115,7 @@ func compress*(x, y: openArray[byte], key: ByteTreeKey, codec: MultiCodec): ?!By
|
||||
let digest = ?MultiHash.digest(codec, input).mapFailure
|
||||
success digest.digestBytes
|
||||
|
||||
func initTree(mcodec: MultiCodec, leaves: openArray[ByteHash]): ?!CodexTree =
|
||||
func initTree(mcodec: MultiCodec, leaves: openArray[ByteHash]): ?!StorageMerkleTree =
|
||||
if leaves.len == 0:
|
||||
return failure "Empty leaves"
|
||||
|
||||
@ -128,28 +128,28 @@ func initTree(mcodec: MultiCodec, leaves: openArray[ByteHash]): ?!CodexTree =
|
||||
if digestSize != leaves[0].len:
|
||||
return failure "Invalid hash length"
|
||||
|
||||
var self = CodexTree(mcodec: mcodec)
|
||||
var self = StorageMerkleTree(mcodec: mcodec)
|
||||
?self.prepare(compressor, Zero, leaves)
|
||||
success self
|
||||
|
||||
func init*(
|
||||
_: type CodexTree, mcodec: MultiCodec = Sha256HashCodec, leaves: openArray[ByteHash]
|
||||
): ?!CodexTree =
|
||||
_: type StorageMerkleTree, mcodec: MultiCodec = Sha256HashCodec, leaves: openArray[ByteHash]
|
||||
): ?!StorageMerkleTree =
|
||||
let tree = ?initTree(mcodec, leaves)
|
||||
?tree.compute()
|
||||
success tree
|
||||
|
||||
proc init*(
|
||||
_: type CodexTree,
|
||||
_: type StorageMerkleTree,
|
||||
tp: Taskpool,
|
||||
mcodec: MultiCodec = Sha256HashCodec,
|
||||
leaves: seq[ByteHash],
|
||||
): Future[?!CodexTree] {.async: (raises: [CancelledError]).} =
|
||||
): Future[?!StorageMerkleTree] {.async: (raises: [CancelledError]).} =
|
||||
let tree = ?initTree(mcodec, leaves)
|
||||
?await tree.compute(tp)
|
||||
success tree
|
||||
|
||||
func init*(_: type CodexTree, leaves: openArray[MultiHash]): ?!CodexTree =
|
||||
func init*(_: type StorageMerkleTree, leaves: openArray[MultiHash]): ?!StorageMerkleTree =
|
||||
if leaves.len == 0:
|
||||
return failure "Empty leaves"
|
||||
|
||||
@ -157,11 +157,11 @@ func init*(_: type CodexTree, leaves: openArray[MultiHash]): ?!CodexTree =
|
||||
mcodec = leaves[0].mcodec
|
||||
leaves = leaves.mapIt(it.digestBytes)
|
||||
|
||||
CodexTree.init(mcodec, leaves)
|
||||
StorageMerkleTree.init(mcodec, leaves)
|
||||
|
||||
proc init*(
|
||||
_: type CodexTree, tp: Taskpool, leaves: seq[MultiHash]
|
||||
): Future[?!CodexTree] {.async: (raises: [CancelledError]).} =
|
||||
_: type StorageMerkleTree, tp: Taskpool, leaves: seq[MultiHash]
|
||||
): Future[?!StorageMerkleTree] {.async: (raises: [CancelledError]).} =
|
||||
if leaves.len == 0:
|
||||
return failure "Empty leaves"
|
||||
|
||||
@ -169,9 +169,9 @@ proc init*(
|
||||
mcodec = leaves[0].mcodec
|
||||
leaves = leaves.mapIt(it.digestBytes)
|
||||
|
||||
await CodexTree.init(tp, mcodec, leaves)
|
||||
await StorageMerkleTree.init(tp, mcodec, leaves)
|
||||
|
||||
func init*(_: type CodexTree, leaves: openArray[Cid]): ?!CodexTree =
|
||||
func init*(_: type StorageMerkleTree, leaves: openArray[Cid]): ?!StorageMerkleTree =
|
||||
if leaves.len == 0:
|
||||
return failure "Empty leaves"
|
||||
|
||||
@ -179,11 +179,11 @@ func init*(_: type CodexTree, leaves: openArray[Cid]): ?!CodexTree =
|
||||
mcodec = (?leaves[0].mhash.mapFailure).mcodec
|
||||
leaves = leaves.mapIt((?it.mhash.mapFailure).digestBytes)
|
||||
|
||||
CodexTree.init(mcodec, leaves)
|
||||
StorageMerkleTree.init(mcodec, leaves)
|
||||
|
||||
proc init*(
|
||||
_: type CodexTree, tp: Taskpool, leaves: seq[Cid]
|
||||
): Future[?!CodexTree] {.async: (raises: [CancelledError]).} =
|
||||
_: type StorageMerkleTree, tp: Taskpool, leaves: seq[Cid]
|
||||
): Future[?!StorageMerkleTree] {.async: (raises: [CancelledError]).} =
|
||||
if leaves.len == 0:
|
||||
return failure("Empty leaves")
|
||||
|
||||
@ -191,14 +191,14 @@ proc init*(
|
||||
mcodec = (?leaves[0].mhash.mapFailure).mcodec
|
||||
leaves = leaves.mapIt((?it.mhash.mapFailure).digestBytes)
|
||||
|
||||
await CodexTree.init(tp, mcodec, leaves)
|
||||
await StorageMerkleTree.init(tp, mcodec, leaves)
|
||||
|
||||
proc fromNodes*(
|
||||
_: type CodexTree,
|
||||
_: type StorageMerkleTree,
|
||||
mcodec: MultiCodec = Sha256HashCodec,
|
||||
nodes: openArray[ByteHash],
|
||||
nleaves: int,
|
||||
): ?!CodexTree =
|
||||
): ?!StorageMerkleTree =
|
||||
if nodes.len == 0:
|
||||
return failure "Empty nodes"
|
||||
|
||||
@ -211,7 +211,7 @@ proc fromNodes*(
|
||||
if digestSize != nodes[0].len:
|
||||
return failure "Invalid hash length"
|
||||
|
||||
var self = CodexTree(mcodec: mcodec)
|
||||
var self = StorageMerkleTree(mcodec: mcodec)
|
||||
?self.fromNodes(compressor, Zero, nodes, nleaves)
|
||||
|
||||
let
|
||||
@ -224,12 +224,12 @@ proc fromNodes*(
|
||||
success self
|
||||
|
||||
func init*(
|
||||
_: type CodexProof,
|
||||
_: type StorageMerkleProof,
|
||||
mcodec: MultiCodec = Sha256HashCodec,
|
||||
index: int,
|
||||
nleaves: int,
|
||||
nodes: openArray[ByteHash],
|
||||
): ?!CodexProof =
|
||||
): ?!StorageMerkleProof =
|
||||
if nodes.len == 0:
|
||||
return failure "Empty nodes"
|
||||
|
||||
@ -239,7 +239,7 @@ func init*(
|
||||
compressor = proc(x, y: seq[byte], key: ByteTreeKey): ?!seq[byte] {.noSideEffect.} =
|
||||
compress(x, y, key, mcodec)
|
||||
|
||||
success CodexProof(
|
||||
success StorageMerkleProof(
|
||||
compress: compressor,
|
||||
zero: Zero,
|
||||
mcodec: mcodec,
|
||||
|
||||
@ -445,7 +445,7 @@ proc store*(
|
||||
finally:
|
||||
await stream.close()
|
||||
|
||||
without tree =? (await CodexTree.init(self.taskPool, cids)), err:
|
||||
without tree =? (await StorageMerkleTree.init(self.taskPool, cids)), err:
|
||||
return failure(err)
|
||||
|
||||
without treeCid =? tree.rootCid(CIDv1, dataCodec), err:
|
||||
|
||||
@ -80,7 +80,7 @@ method getBlocks*(
|
||||
|
||||
method getBlockAndProof*(
|
||||
self: BlockStore, treeCid: Cid, index: Natural
|
||||
): Future[?!(Block, CodexProof)] {.base, async: (raises: [CancelledError]), gcsafe.} =
|
||||
): Future[?!(Block, StorageMerkleProof)] {.base, async: (raises: [CancelledError]), gcsafe.} =
|
||||
## Get a block and associated inclusion proof by Cid of a merkle tree and an index of a leaf in a tree
|
||||
##
|
||||
|
||||
@ -95,7 +95,7 @@ method putBlock*(
|
||||
raiseAssert("putBlock not implemented!")
|
||||
|
||||
method putCidAndProof*(
|
||||
self: BlockStore, treeCid: Cid, index: Natural, blockCid: Cid, proof: CodexProof
|
||||
self: BlockStore, treeCid: Cid, index: Natural, blockCid: Cid, proof: StorageMerkleProof
|
||||
): Future[?!void] {.base, async: (raises: [CancelledError]), gcsafe.} =
|
||||
## Put a block proof to the blockstore
|
||||
##
|
||||
@ -104,7 +104,7 @@ method putCidAndProof*(
|
||||
|
||||
method getCidAndProof*(
|
||||
self: BlockStore, treeCid: Cid, index: Natural
|
||||
): Future[?!(Cid, CodexProof)] {.base, async: (raises: [CancelledError]), gcsafe.} =
|
||||
): Future[?!(Cid, StorageMerkleProof)] {.base, async: (raises: [CancelledError]), gcsafe.} =
|
||||
## Get a block proof from the blockstore
|
||||
##
|
||||
|
||||
|
||||
@ -37,7 +37,7 @@ type
|
||||
currentSize*: NBytes
|
||||
size*: NBytes
|
||||
cache: LruCache[Cid, Block]
|
||||
cidAndProofCache: LruCache[(Cid, Natural), (Cid, CodexProof)]
|
||||
cidAndProofCache: LruCache[(Cid, Natural), (Cid, StorageMerkleProof)]
|
||||
|
||||
InvalidBlockSize* = object of CodexError
|
||||
|
||||
@ -83,7 +83,7 @@ method getBlocks*(
|
||||
|
||||
method getCidAndProof*(
|
||||
self: CacheStore, treeCid: Cid, index: Natural
|
||||
): Future[?!(Cid, CodexProof)] {.async: (raises: [CancelledError]).} =
|
||||
): Future[?!(Cid, StorageMerkleProof)] {.async: (raises: [CancelledError]).} =
|
||||
if cidAndProof =? self.cidAndProofCache.getOption((treeCid, index)):
|
||||
success(cidAndProof)
|
||||
else:
|
||||
@ -103,7 +103,7 @@ method getBlock*(
|
||||
|
||||
method getBlockAndProof*(
|
||||
self: CacheStore, treeCid: Cid, index: Natural
|
||||
): Future[?!(Block, CodexProof)] {.async: (raises: [CancelledError]).} =
|
||||
): Future[?!(Block, StorageMerkleProof)] {.async: (raises: [CancelledError]).} =
|
||||
without cidAndProof =? (await self.getCidAndProof(treeCid, index)), err:
|
||||
return failure(err)
|
||||
|
||||
@ -226,7 +226,7 @@ method putBlock*(
|
||||
return success()
|
||||
|
||||
method putCidAndProof*(
|
||||
self: CacheStore, treeCid: Cid, index: Natural, blockCid: Cid, proof: CodexProof
|
||||
self: CacheStore, treeCid: Cid, index: Natural, blockCid: Cid, proof: StorageMerkleProof
|
||||
): Future[?!void] {.async: (raises: [CancelledError]).} =
|
||||
self.cidAndProofCache[(treeCid, index)] = (blockCid, proof)
|
||||
success()
|
||||
@ -301,7 +301,7 @@ proc new*(
|
||||
currentSize = 0'nb
|
||||
size = int(cacheSize div chunkSize)
|
||||
cache = newLruCache[Cid, Block](size)
|
||||
cidAndProofCache = newLruCache[(Cid, Natural), (Cid, CodexProof)](size)
|
||||
cidAndProofCache = newLruCache[(Cid, Natural), (Cid, StorageMerkleProof)](size)
|
||||
store = CacheStore(
|
||||
cache: cache,
|
||||
cidAndProofCache: cidAndProofCache,
|
||||
|
||||
@ -104,13 +104,13 @@ method putBlock*(
|
||||
return success()
|
||||
|
||||
method putCidAndProof*(
|
||||
self: NetworkStore, treeCid: Cid, index: Natural, blockCid: Cid, proof: CodexProof
|
||||
self: NetworkStore, treeCid: Cid, index: Natural, blockCid: Cid, proof: StorageMerkleProof
|
||||
): Future[?!void] {.async: (raw: true, raises: [CancelledError]).} =
|
||||
self.localStore.putCidAndProof(treeCid, index, blockCid, proof)
|
||||
|
||||
method getCidAndProof*(
|
||||
self: NetworkStore, treeCid: Cid, index: Natural
|
||||
): Future[?!(Cid, CodexProof)] {.async: (raw: true, raises: [CancelledError]).} =
|
||||
): Future[?!(Cid, StorageMerkleProof)] {.async: (raw: true, raises: [CancelledError]).} =
|
||||
## Get a block proof from the blockstore
|
||||
##
|
||||
|
||||
|
||||
@ -33,7 +33,7 @@ declareGauge(codex_repostore_bytes_used, "codex repostore bytes used")
|
||||
declareGauge(codex_repostore_bytes_reserved, "codex repostore bytes reserved")
|
||||
|
||||
proc putLeafMetadata*(
|
||||
self: RepoStore, treeCid: Cid, index: Natural, blkCid: Cid, proof: CodexProof
|
||||
self: RepoStore, treeCid: Cid, index: Natural, blkCid: Cid, proof: StorageMerkleProof
|
||||
): Future[?!StoreResultKind] {.async: (raises: [CancelledError]).} =
|
||||
without key =? createBlockCidAndProofMetadataKey(treeCid, index), err:
|
||||
return failure(err)
|
||||
|
||||
@ -82,7 +82,7 @@ method getBlock*(
|
||||
|
||||
method getBlockAndProof*(
|
||||
self: RepoStore, treeCid: Cid, index: Natural
|
||||
): Future[?!(Block, CodexProof)] {.async: (raises: [CancelledError]).} =
|
||||
): Future[?!(Block, StorageMerkleProof)] {.async: (raises: [CancelledError]).} =
|
||||
without leafMd =? await self.getLeafMetadata(treeCid, index), err:
|
||||
return failure(err)
|
||||
|
||||
@ -136,7 +136,7 @@ method ensureExpiry*(
|
||||
await self.ensureExpiry(leafMd.blkCid, expiry)
|
||||
|
||||
method putCidAndProof*(
|
||||
self: RepoStore, treeCid: Cid, index: Natural, blkCid: Cid, proof: CodexProof
|
||||
self: RepoStore, treeCid: Cid, index: Natural, blkCid: Cid, proof: StorageMerkleProof
|
||||
): Future[?!void] {.async: (raises: [CancelledError]).} =
|
||||
## Put a block to the blockstore
|
||||
##
|
||||
@ -163,7 +163,7 @@ method putCidAndProof*(
|
||||
|
||||
method getCidAndProof*(
|
||||
self: RepoStore, treeCid: Cid, index: Natural
|
||||
): Future[?!(Cid, CodexProof)] {.async: (raises: [CancelledError]).} =
|
||||
): Future[?!(Cid, StorageMerkleProof)] {.async: (raises: [CancelledError]).} =
|
||||
without leafMd =? await self.getLeafMetadata(treeCid, index), err:
|
||||
return failure(err)
|
||||
|
||||
|
||||
@ -49,7 +49,7 @@ type
|
||||
|
||||
LeafMetadata* {.serialize.} = object
|
||||
blkCid*: Cid
|
||||
proof*: CodexProof
|
||||
proof*: StorageMerkleProof
|
||||
|
||||
BlockExpiration* {.serialize.} = object
|
||||
cid*: Cid
|
||||
|
||||
@ -21,7 +21,7 @@ import ../utils/asynciter
|
||||
import ../merkletree
|
||||
|
||||
proc putSomeProofs*(
|
||||
store: BlockStore, tree: CodexTree, iter: Iter[int]
|
||||
store: BlockStore, tree: StorageMerkleTree, iter: Iter[int]
|
||||
): Future[?!void] {.async.} =
|
||||
without treeCid =? tree.rootCid, err:
|
||||
return failure(err)
|
||||
@ -47,9 +47,9 @@ proc putSomeProofs*(
|
||||
success()
|
||||
|
||||
proc putSomeProofs*(
|
||||
store: BlockStore, tree: CodexTree, iter: Iter[Natural]
|
||||
store: BlockStore, tree: StorageMerkleTree, iter: Iter[Natural]
|
||||
): Future[?!void] =
|
||||
store.putSomeProofs(tree, iter.map((i: Natural) => i.ord))
|
||||
|
||||
proc putAllProofs*(store: BlockStore, tree: CodexTree): Future[?!void] =
|
||||
proc putAllProofs*(store: BlockStore, tree: StorageMerkleTree): Future[?!void] =
|
||||
store.putSomeProofs(tree, Iter[int].new(0 ..< tree.leavesCount))
|
||||
|
||||
@ -25,7 +25,7 @@ asyncchecksuite "Block Advertising and Discovery":
|
||||
var
|
||||
blocks: seq[bt.Block]
|
||||
manifest: Manifest
|
||||
tree: CodexTree
|
||||
tree: StorageMerkleTree
|
||||
manifestBlock: bt.Block
|
||||
switch: Switch
|
||||
peerStore: PeerCtxStore
|
||||
@ -158,7 +158,7 @@ asyncchecksuite "E2E - Multiple Nodes Discovery":
|
||||
blockexc: seq[NetworkStore]
|
||||
manifests: seq[Manifest]
|
||||
mBlocks: seq[bt.Block]
|
||||
trees: seq[CodexTree]
|
||||
trees: seq[StorageMerkleTree]
|
||||
|
||||
setup:
|
||||
for _ in 0 ..< 4:
|
||||
|
||||
@ -27,7 +27,7 @@ asyncchecksuite "Test Discovery Engine":
|
||||
var
|
||||
blocks: seq[bt.Block]
|
||||
manifest: Manifest
|
||||
tree: CodexTree
|
||||
tree: StorageMerkleTree
|
||||
manifestBlock: bt.Block
|
||||
switch: Switch
|
||||
peerStore: PeerCtxStore
|
||||
|
||||
@ -8,7 +8,7 @@ import pkg/codex/rng
|
||||
|
||||
import ./randomchunker
|
||||
|
||||
type TestDataset* = tuple[blocks: seq[Block], tree: CodexTree, manifest: Manifest]
|
||||
type TestDataset* = tuple[blocks: seq[Block], tree: StorageMerkleTree, manifest: Manifest]
|
||||
|
||||
proc makeRandomBlock*(size: NBytes): Block =
|
||||
let bytes = newSeqWith(size.int, rand(uint8))
|
||||
@ -34,7 +34,7 @@ proc makeDataset*(blocks: seq[Block]): ?!TestDataset =
|
||||
let
|
||||
datasetSize = blocks.mapIt(it.data.len).foldl(a + b)
|
||||
blockSize = blocks.mapIt(it.data.len).foldl(max(a, b))
|
||||
tree = ?CodexTree.init(blocks.mapIt(it.cid))
|
||||
tree = ?StorageMerkleTree.init(blocks.mapIt(it.cid))
|
||||
treeCid = ?tree.rootCid
|
||||
manifest = Manifest.new(
|
||||
treeCid = treeCid,
|
||||
|
||||
@ -3,9 +3,9 @@ import ../helpers
|
||||
|
||||
export merkletree, helpers
|
||||
|
||||
proc `==`*(a, b: CodexTree): bool =
|
||||
proc `==`*(a, b: StorageMerkleTree): bool =
|
||||
(a.mcodec == b.mcodec) and (a.leavesCount == b.leavesCount) and (a.levels == b.levels)
|
||||
|
||||
proc `==`*(a, b: CodexProof): bool =
|
||||
proc `==`*(a, b: StorageMerkleProof): bool =
|
||||
(a.mcodec == b.mcodec) and (a.nleaves == b.nleaves) and (a.path == b.path) and
|
||||
(a.index == b.index)
|
||||
|
||||
@ -21,16 +21,16 @@ const data = [
|
||||
suite "merkletree - coders":
|
||||
test "encoding and decoding a tree yields the same tree":
|
||||
let
|
||||
tree = CodexTree.init(Sha256HashCodec, data).tryGet()
|
||||
tree = StorageMerkleTree.init(Sha256HashCodec, data).tryGet()
|
||||
encodedBytes = tree.encode()
|
||||
decodedTree = CodexTree.decode(encodedBytes).tryGet()
|
||||
decodedTree = StorageMerkleTree.decode(encodedBytes).tryGet()
|
||||
|
||||
check:
|
||||
tree == decodedTree
|
||||
|
||||
test "encoding and decoding a proof yields the same proof":
|
||||
let
|
||||
tree = CodexTree.init(Sha256HashCodec, data).tryGet()
|
||||
tree = StorageMerkleTree.init(Sha256HashCodec, data).tryGet()
|
||||
proof = tree.getProof(4).tryGet()
|
||||
|
||||
check:
|
||||
@ -38,7 +38,7 @@ suite "merkletree - coders":
|
||||
|
||||
let
|
||||
encodedBytes = proof.encode()
|
||||
decodedProof = CodexProof.decode(encodedBytes).tryGet()
|
||||
decodedProof = StorageMerkleProof.decode(encodedBytes).tryGet()
|
||||
|
||||
check:
|
||||
proof == decodedProof
|
||||
|
||||
@ -32,23 +32,23 @@ const
|
||||
]
|
||||
sha256 = Sha256HashCodec
|
||||
|
||||
suite "Test CodexTree":
|
||||
suite "Test StorageMerkleTree":
|
||||
test "Cannot init tree without any multihash leaves":
|
||||
check:
|
||||
CodexTree.init(leaves = newSeq[MultiHash]()).isErr
|
||||
StorageMerkleTree.init(leaves = newSeq[MultiHash]()).isErr
|
||||
|
||||
test "Cannot init tree without any cid leaves":
|
||||
check:
|
||||
CodexTree.init(leaves = newSeq[Cid]()).isErr
|
||||
StorageMerkleTree.init(leaves = newSeq[Cid]()).isErr
|
||||
|
||||
test "Cannot init tree without any byte leaves":
|
||||
check:
|
||||
CodexTree.init(sha256, leaves = newSeq[ByteHash]()).isErr
|
||||
StorageMerkleTree.init(sha256, leaves = newSeq[ByteHash]()).isErr
|
||||
|
||||
test "Should build tree from multihash leaves":
|
||||
var
|
||||
expectedLeaves = data.mapIt(MultiHash.digest($sha256, it).tryGet())
|
||||
tree = CodexTree.init(leaves = expectedLeaves)
|
||||
tree = StorageMerkleTree.init(leaves = expectedLeaves)
|
||||
|
||||
check:
|
||||
tree.isOk
|
||||
@ -62,7 +62,7 @@ suite "Test CodexTree":
|
||||
|
||||
let expectedLeaves = data.mapIt(MultiHash.digest($sha256, it).tryGet())
|
||||
|
||||
let tree = (await CodexTree.init(tp, leaves = expectedLeaves))
|
||||
let tree = (await StorageMerkleTree.init(tp, leaves = expectedLeaves))
|
||||
check:
|
||||
tree.isOk
|
||||
tree.get().leaves == expectedLeaves.mapIt(it.digestBytes)
|
||||
@ -73,7 +73,7 @@ suite "Test CodexTree":
|
||||
Cid.init(CidVersion.CIDv1, BlockCodec, MultiHash.digest($sha256, it).tryGet).tryGet
|
||||
)
|
||||
|
||||
let tree = CodexTree.init(leaves = expectedLeaves)
|
||||
let tree = StorageMerkleTree.init(leaves = expectedLeaves)
|
||||
|
||||
check:
|
||||
tree.isOk
|
||||
@ -89,7 +89,7 @@ suite "Test CodexTree":
|
||||
Cid.init(CidVersion.CIDv1, BlockCodec, MultiHash.digest($sha256, it).tryGet).tryGet
|
||||
)
|
||||
|
||||
let tree = (await CodexTree.init(tp, leaves = expectedLeaves))
|
||||
let tree = (await StorageMerkleTree.init(tp, leaves = expectedLeaves))
|
||||
|
||||
check:
|
||||
tree.isOk
|
||||
@ -106,8 +106,8 @@ suite "Test CodexTree":
|
||||
)
|
||||
|
||||
let
|
||||
atree = (await CodexTree.init(tp, leaves = expectedLeaves))
|
||||
stree = CodexTree.init(leaves = expectedLeaves)
|
||||
atree = (await StorageMerkleTree.init(tp, leaves = expectedLeaves))
|
||||
stree = StorageMerkleTree.init(leaves = expectedLeaves)
|
||||
|
||||
check:
|
||||
toSeq(atree.get().nodes) == toSeq(stree.get().nodes)
|
||||
@ -115,15 +115,15 @@ suite "Test CodexTree":
|
||||
|
||||
# Single-leaf trees have their root separately computed
|
||||
let
|
||||
atree1 = (await CodexTree.init(tp, leaves = expectedLeaves[0 .. 0]))
|
||||
stree1 = CodexTree.init(leaves = expectedLeaves[0 .. 0])
|
||||
atree1 = (await StorageMerkleTree.init(tp, leaves = expectedLeaves[0 .. 0]))
|
||||
stree1 = StorageMerkleTree.init(leaves = expectedLeaves[0 .. 0])
|
||||
|
||||
check:
|
||||
toSeq(atree.get().nodes) == toSeq(stree.get().nodes)
|
||||
atree.get().root == stree.get().root
|
||||
|
||||
test "Should build from raw digestbytes (should not hash leaves)":
|
||||
let tree = CodexTree.init(sha256, leaves = data).tryGet
|
||||
let tree = StorageMerkleTree.init(sha256, leaves = data).tryGet
|
||||
|
||||
check:
|
||||
tree.mcodec == sha256
|
||||
@ -134,7 +134,7 @@ suite "Test CodexTree":
|
||||
defer:
|
||||
tp.shutdown()
|
||||
|
||||
let tree = (await CodexTree.init(tp, sha256, leaves = @data))
|
||||
let tree = (await StorageMerkleTree.init(tp, sha256, leaves = @data))
|
||||
|
||||
check:
|
||||
tree.isOk
|
||||
@ -143,8 +143,8 @@ suite "Test CodexTree":
|
||||
|
||||
test "Should build from nodes":
|
||||
let
|
||||
tree = CodexTree.init(sha256, leaves = data).tryGet
|
||||
fromNodes = CodexTree.fromNodes(
|
||||
tree = StorageMerkleTree.init(sha256, leaves = data).tryGet
|
||||
fromNodes = StorageMerkleTree.fromNodes(
|
||||
nodes = toSeq(tree.nodes), nleaves = tree.leavesCount
|
||||
).tryGet
|
||||
|
||||
@ -158,7 +158,7 @@ let
|
||||
compress = proc(x, y: seq[byte], key: ByteTreeKey): seq[byte] =
|
||||
compress(x, y, key, sha256).tryGet
|
||||
|
||||
makeTree = proc(data: seq[seq[byte]]): CodexTree =
|
||||
CodexTree.init(sha256, leaves = data).tryGet
|
||||
makeTree = proc(data: seq[seq[byte]]): StorageMerkleTree =
|
||||
StorageMerkleTree.init(sha256, leaves = data).tryGet
|
||||
|
||||
testGenericTree("CodexTree", @data, zero, compress, makeTree)
|
||||
testGenericTree("StorageMerkleTree", @data, zero, compress, makeTree)
|
||||
|
||||
@ -46,7 +46,7 @@ proc makeManifest*(
|
||||
hcodec = Sha256HashCodec,
|
||||
dataCodec = BlockCodec,
|
||||
): Future[?!Manifest] {.async.} =
|
||||
without tree =? CodexTree.init(cids), err:
|
||||
without tree =? StorageMerkleTree.init(cids), err:
|
||||
return failure(err)
|
||||
|
||||
without treeCid =? tree.rootCid(CIDv1, dataCodec), err:
|
||||
|
||||
@ -29,7 +29,7 @@ proc commonBlockStoreTests*(
|
||||
var
|
||||
newBlock, newBlock1, newBlock2, newBlock3: Block
|
||||
manifest: Manifest
|
||||
tree: CodexTree
|
||||
tree: StorageMerkleTree
|
||||
store: BlockStore
|
||||
|
||||
setup:
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user