From e8d2622bc2d1f4961d59987c24db9f51bfb9cb34 Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Wed, 12 Jan 2022 18:38:41 -0600 Subject: [PATCH] move manifest container to it's own file --- dagger/blocksmanifest.nim | 158 ++++++++++++++++++++++++++++++++++ dagger/manifest.nim | 143 +----------------------------- dagger/node.nim | 2 +- tests/dagger/testmanifest.nim | 2 +- tests/dagger/testnode.nim | 2 +- 5 files changed, 165 insertions(+), 142 deletions(-) create mode 100644 dagger/blocksmanifest.nim diff --git a/dagger/blocksmanifest.nim b/dagger/blocksmanifest.nim new file mode 100644 index 00000000..1969fdf0 --- /dev/null +++ b/dagger/blocksmanifest.nim @@ -0,0 +1,158 @@ +## Nim-Dagger +## Copyright (c) 2021 Status Research & Development GmbH +## Licensed under either of +## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) +## * MIT license ([LICENSE-MIT](LICENSE-MIT)) +## at your option. +## This file may not be copied, modified, or distributed except according to +## those terms. + +{.push raises: [Defect].} + +import std/tables + +import pkg/libp2p +import pkg/questionable +import pkg/questionable/results +import pkg/chronicles +import pkg/chronos + +import ./manifest +import ./errors + +export manifest + +const + ManifestCodec* = multiCodec("dag-pb") + +var + emptyDigests {.threadvar.}: array[CIDv0..CIDv1, Table[MultiCodec, MultiHash]] + once {.threadvar.}: bool + +template EmptyDigests: untyped = + if not once: + emptyDigests = [ + CIDv0: { + multiCodec("sha2-256"): Cid + .init("bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku") + .get() + .mhash + .get() + }.toTable, + CIDv1: { + multiCodec("sha2-256"): Cid.init("QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n") + .get() + .mhash + .get() + }.toTable, + ] + + once = true + emptyDigests + +type + BlocksManifest* = object + manifest: Manifest + version*: CidVersion + hcodec*: MultiCodec + codec*: MultiCodec + +proc len*(b: BlocksManifest): int = b.manifest.blocks.len + +iterator items*(b: BlocksManifest): Cid = + for b in b.manifest.blocks: + yield b + +template hashBytes(mh: MultiHash): seq[byte] = + ## get the hash bytes of a multihash object + ## + + mh.data.buffer[mh.dpos..(mh.dpos + mh.size - 1)] + +proc cid*(b: var BlocksManifest): ?!Cid = + ## Generate a root hash using the treehash algorithm + ## + + if htree =? b.manifest.cid: + return htree.success + + var + stack: seq[MultiHash] + + for cid in b.manifest.blocks: + stack.add(? cid.mhash.mapFailure) + + while stack.len > 1: + let + (b1, b2) = (stack.pop(), stack.pop()) + mh = ? MultiHash.digest( + $b.hcodec, + (b1.hashBytes() & b2.hashBytes())) + .mapFailure + stack.add(mh) + + if stack.len == 1: + let cid = ? Cid.init( + b.version, + b.codec, + (? EmptyDigests[b.version][b.hcodec].catch)) + .mapFailure + + b.manifest.cid = cid.some + return cid.success + +proc put*(b: var BlocksManifest, cid: Cid) = + b.manifest.cid = Cid.none + trace "Adding cid to manifest", cid + b.manifest.blocks.add(cid) + +proc contains*(b: BlocksManifest, cid: Cid): bool = + cid in b.manifest.blocks + +proc encode*(b: var BlocksManifest): ?!seq[byte] = + if b.manifest.cid.isNone: + b.manifest.cid = (? b.cid).some + + b.manifest.encode() + +proc init*( + T: type BlocksManifest, + blocks: openArray[Cid] = [], + version = CIDv1, + hcodec = multiCodec("sha2-256"), + codec = multiCodec("raw")): ?!T = + ## Create a manifest using array of `Cid`s + ## + + if hcodec notin EmptyDigests[version]: + return failure("Unsupported manifest hash codec!") + + T( + manifest: Manifest(blocks: @blocks), + version: version, + codec: codec, + hcodec: hcodec, + ).success + +proc init*( + T: type BlocksManifest, + data: openArray[byte]): ?!T = + ## Create manifest from a raw data blob + ## (in dag-pb for for now) + ## + + let + manifest = ? Manifest.decode(data) + cid = !manifest.cid + mhash = ? cid.mhash.mapFailure + + var blockManifest = ? BlocksManifest.init( + manifest.blocks, + cid.version, + mhash.mcodec, + cid.mcodec) + + if cid != ? blockManifest.cid: + return failure("Decoded content hash doesn't match!") + + blockManifest.success diff --git a/dagger/manifest.nim b/dagger/manifest.nim index 8aa4b4c1..494f2e68 100644 --- a/dagger/manifest.nim +++ b/dagger/manifest.nim @@ -7,125 +7,32 @@ ## This file may not be copied, modified, or distributed except according to ## those terms. -{.push raises: [Defect].} - -import std/tables - -import pkg/libp2p import pkg/libp2p/protobuf/minprotobuf +import pkg/libp2p import pkg/questionable import pkg/questionable/results -import pkg/chronicles -import pkg/chronos -import ./blocktype import ./errors -const - ManifestCodec* = multiCodec("dag-pb") - -var - emptyDigests {.threadvar.}: array[CIDv0..CIDv1, Table[MultiCodec, MultiHash]] - once {.threadvar.}: bool - -template EmptyDigests: untyped = - if not once: - emptyDigests = [ - CIDv0: { - multiCodec("sha2-256"): Cid - .init("bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku") - .get() - .mhash - .get() - }.toTable, - CIDv1: { - multiCodec("sha2-256"): Cid.init("QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n") - .get() - .mhash - .get() - }.toTable, - ] - - once = true - emptyDigests - type Manifest* = object cid*: ?Cid blocks*: seq[Cid] - BlocksManifest* = object - manifest: Manifest - version*: CidVersion - hcodec*: MultiCodec - codec*: MultiCodec - -proc len*(b: BlocksManifest): int = b.manifest.blocks.len - -iterator items*(b: BlocksManifest): Cid = - for b in b.manifest.blocks: - yield b - -template hashBytes(mh: MultiHash): seq[byte] = - ## get the hash bytes of a multihash object - ## - - mh.data.buffer[mh.dpos..(mh.dpos + mh.size - 1)] - -proc cid*(b: var BlocksManifest): ?!Cid = - ## Generate a root hash using the treehash algorithm - ## - - if htree =? b.manifest.cid: - return htree.success - - var - stack: seq[MultiHash] - - for cid in b.manifest.blocks: - stack.add(? cid.mhash.mapFailure) - - while stack.len > 1: - let - (b1, b2) = (stack.pop(), stack.pop()) - mh = ? MultiHash.digest( - $b.hcodec, - (b1.hashBytes() & b2.hashBytes())) - .mapFailure - stack.add(mh) - - if stack.len == 1: - let cid = ? Cid.init( - b.version, - b.codec, - (? EmptyDigests[b.version][b.hcodec].catch)) - .mapFailure - - b.manifest.cid = cid.some - return cid.success - -proc put*(b: var BlocksManifest, cid: Cid) = - b.manifest.cid = Cid.none - trace "Adding cid to manifest", cid - b.manifest.blocks.add(cid) - -proc contains*(b: BlocksManifest, cid: Cid): bool = - cid in b.manifest.blocks - -proc encode*(b: var BlocksManifest): ?!seq[byte] = +proc encode*(b: var Manifest): ?!seq[byte] = ## Encode the manifest into a ``ManifestCodec`` ## multicodec container (Dag-pb) for now ## var pbNode = initProtoBuffer() - for c in b.manifest.blocks: + for c in b.blocks: var pbLink = initProtoBuffer() pbLink.write(1, c.data.buffer) # write Cid links pbLink.finish() pbNode.write(2, pbLink) - let cid = ? b.cid + let cid = !b.cid pbNode.write(1, cid.data.buffer) # set the treeHash Cid as the data field pbNode.finish() @@ -156,45 +63,3 @@ proc decode*(_: type Manifest, data: openArray[byte]): ?!Manifest = blocks.add(? Cid.init(blockBuf).mapFailure) Manifest(cid: cid.some, blocks: blocks).success - -proc init*( - T: type BlocksManifest, - blocks: openArray[Cid] = [], - version = CIDv1, - hcodec = multiCodec("sha2-256"), - codec = multiCodec("raw")): ?!T = - ## Create a manifest using array of `Cid`s - ## - - if hcodec notin EmptyDigests[version]: - return failure("Unsupported manifest hash codec!") - - T( - manifest: Manifest(blocks: @blocks), - version: version, - codec: codec, - hcodec: hcodec, - ).success - -proc init*( - T: type BlocksManifest, - data: openArray[byte]): ?!T = - ## Create manifest from a raw data blob - ## (in dag-pb for for now) - ## - - let - manifest = ? Manifest.decode(data) - cid = !manifest.cid - mhash = ? cid.mhash.mapFailure - - var blockManifest = ? BlocksManifest.init( - manifest.blocks, - cid.version, - mhash.mcodec, - cid.mcodec) - - if cid != ? blockManifest.cid: - return failure("Decoded content hash doesn't match!") - - blockManifest.success diff --git a/dagger/node.nim b/dagger/node.nim index fe1b8010..6d1e84d2 100644 --- a/dagger/node.nim +++ b/dagger/node.nim @@ -21,7 +21,7 @@ import pkg/libp2p/signed_envelope import ./chunker import ./blocktype as bt -import ./manifest +import ./blocksmanifest import ./stores/blockstore import ./blockexchange diff --git a/tests/dagger/testmanifest.nim b/tests/dagger/testmanifest.nim index e3fa5b46..fcf9dd8d 100644 --- a/tests/dagger/testmanifest.nim +++ b/tests/dagger/testmanifest.nim @@ -9,7 +9,7 @@ import pkg/stew/byteutils import pkg/dagger/chunker import pkg/dagger/blocktype as bt -import pkg/dagger/manifest +import pkg/dagger/blocksmanifest import ./helpers diff --git a/tests/dagger/testnode.nim b/tests/dagger/testnode.nim index 3a666a74..70ceb280 100644 --- a/tests/dagger/testnode.nim +++ b/tests/dagger/testnode.nim @@ -12,7 +12,7 @@ import pkg/dagger/stores import pkg/dagger/blockexchange import pkg/dagger/chunker import pkg/dagger/node -import pkg/dagger/manifest +import pkg/dagger/blocksmanifest import pkg/dagger/blocktype as bt import ./helpers