diff --git a/dagger/manifest/coders.nim b/dagger/manifest/coders.nim index 1a3413b8..1be35c85 100644 --- a/dagger/manifest/coders.nim +++ b/dagger/manifest/coders.nim @@ -17,7 +17,7 @@ import pkg/questionable/results import pkg/chronicles import pkg/chronos -import ./types +import ./manifest import ../errors const @@ -27,6 +27,12 @@ type ManifestCoderType*[codec: static MultiCodec] = object DagPBCoder* = ManifestCoderType[multiCodec("dag-pb")] +const + # TODO: move somewhere better? + ManifestContainers* = { + $DagPBCodec: DagPBCoder() + }.toTable + func encode*(_: DagPBCoder, manifest: Manifest): ?!seq[byte] = ## Encode the manifest into a ``ManifestCodec`` ## multicodec container (Dag-pb) for now @@ -40,8 +46,25 @@ func encode*(_: DagPBCoder, manifest: Manifest): ?!seq[byte] = pbLink.finish() pbNode.write(2, pbLink) + # NOTE: The `Data` field in the the `dag-pb` + # contains the following protobuf `Message` + # + # ```protobuf + # Message Header { + # optional bytes rootHash = 1; # the root (tree) hash + # optional uint32 blockSize = 2; # size of a single block + # optional uint32 blocksLen = 3; # total amount of blocks + # } + # ``` + # + let cid = !manifest.rootHash - pbNode.write(1, cid.data.buffer) # set the rootHash Cid as the data field + var header = initProtoBuffer() + header.write(1, cid.data.buffer) + header.write(2, manifest.blockSize.uint32) + header.write(3, manifest.blocks.len.uint32) + + pbNode.write(1, header.buffer) # set the rootHash Cid as the data field pbNode.finish() return pbNode.buffer.success @@ -52,13 +75,27 @@ func decode*(_: DagPBCoder, data: openArray[byte]): ?!Manifest = var pbNode = initProtoBuffer(data) - cidBuf: seq[byte] + pbHeader: ProtoBuffer + rootHash: seq[byte] + blockSize: uint32 + blocksLen: uint32 blocks: seq[Cid] - if pbNode.getField(1, cidBuf).isErr: - return failure("Unable to decode Cid from manifest!") + # Decode `Header` message + if pbNode.getField(1, pbHeader).isErr: + return failure("Unable to decode `Header` from dag-pb manifest!") - let cid = ? Cid.init(cidBuf).mapFailure + # Decode `Header` contents + if pbHeader.getField(1, rootHash).isErr: + return failure("Unable to decode `rootHash` from manifest!") + + if pbHeader.getField(2, blockSize).isErr: + return failure("Unable to decode `blockSize` from manifest!") + + if pbHeader.getField(3, blocksLen).isErr: + return failure("Unable to decode `blocksLen` from manifest!") + + let rootHashCid = ? Cid.init(rootHash).mapFailure var linksBuf: seq[seq[byte]] if pbNode.getRepeatedField(2, linksBuf).isOk: for pbLinkBuf in linksBuf: @@ -70,4 +107,31 @@ func decode*(_: DagPBCoder, data: openArray[byte]): ?!Manifest = if pbLink.getField(1, blockBuf).isOk: blocks.add(? Cid.init(blockBuf).mapFailure) - Manifest(rootHash: cid.some, blocks: blocks).success + if blocksLen.int != blocks.len: + return failure("Total blocks and length of blocks in header don't match!") + + Manifest( + rootHash: rootHashCid.some, + blockSize: blockSize.int, + blocks: blocks, + hcodec: (? rootHashCid.mhash.mapFailure).mcodec, + codec: rootHashCid.mcodec, + version: rootHashCid.cidver).success + +proc encode*(self: var Manifest, encoder = ManifestContainers[$DagPBCodec]): ?!seq[byte] = + ## Encode a manifest using `encoder` + ## + + if self.rootHash.isNone: + ? self.makeRoot() + + encoder.encode(self) + +func decode*( + _: type Manifest, + data: openArray[byte], + decoder = ManifestContainers[$DagPBCodec]): ?!Manifest = + ## Decode a manifest using `decoder` + ## + + decoder.decode(data) diff --git a/dagger/manifest/manifest.nim b/dagger/manifest/manifest.nim index 8e6df348..f4268d36 100644 --- a/dagger/manifest/manifest.nim +++ b/dagger/manifest/manifest.nim @@ -16,17 +16,42 @@ import pkg/questionable import pkg/questionable/results import pkg/chronicles -import ./types -import ./coders import ../errors import ../blocktype -export coders, Manifest +template EmptyDigests: untyped = + var + emptyDigests {.global, threadvar.}: + array[CIDv0..CIDv1, Table[MultiCodec, MultiHash]] -const - ManifestContainers* = { - $DagPBCodec: DagPBCoder() - }.toTable + once: + emptyDigests = [ + CIDv0: { + multiCodec("sha2-256"): Cid + .init("bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku") + .get() + .mhash + .get() + }.toTable, + CIDv1: { + multiCodec("sha2-256"): Cid + .init("QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n") + .get() + .mhash + .get() + }.toTable, + ] + + emptyDigests + +type + Manifest* = object of RootObj + rootHash*: ?Cid # root (tree) hash of the contained data set + blockSize*: int # size of each contained block (might not be needed if blocks are len-prefixed) + blocks*: seq[Cid] # block Cid + version*: CidVersion # Cid version + hcodec*: MultiCodec # Multihash codec + codec*: MultiCodec # Data set codec func len*(self: Manifest): int = self.blocks.len @@ -66,7 +91,7 @@ template hashBytes(mh: MultiHash): seq[byte] = mh.data.buffer[mh.dpos..(mh.dpos + mh.size - 1)] -proc makeRoot(self: var Manifest): ?!void = +proc makeRoot*(self: var Manifest): ?!void = ## Create a tree hash root of the contained ## block hashes ## @@ -106,24 +131,6 @@ proc cid*(self: var Manifest): ?!Cid = (!self.rootHash).success -proc encode*(self: var Manifest, encoder = ManifestContainers[$DagPBCodec]): ?!seq[byte] = - ## Encode a manifest using `encoder` - ## - - if self.rootHash.isNone: - ? self.makeRoot() - - encoder.encode(self) - -func decode*( - _: type Manifest, - data: openArray[byte], - decoder = ManifestContainers[$DagPBCodec]): ?!Manifest = - ## Decode a manifest using `decoder` - ## - - decoder.decode(data) - proc init*( T: type Manifest, blocks: openArray[Cid] = [], @@ -144,3 +151,8 @@ proc init*( hcodec: hcodec, blockSize: blockSize ).success + +proc init*( + T: type Manifest, + data: openArray[byte]): ?!T = + Manifest.decode(data) diff --git a/dagger/manifest/types.nim b/dagger/manifest/types.nim deleted file mode 100644 index 233e3fa5..00000000 --- a/dagger/manifest/types.nim +++ /dev/null @@ -1,47 +0,0 @@ -## Nim-Dagger -## Copyright (c) 2022 Status Research & Development GmbH -## Licensed under either of -## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) -## * MIT license ([LICENSE-MIT](LICENSE-MIT)) -## at your option. -## This file may not be copied, modified, or distributed except according to -## those terms. - -{.push raises: [Defect].} - -import pkg/libp2p -import pkg/questionable - -template EmptyDigests*: untyped = - var - emptyDigests {.global, threadvar.}: - array[CIDv0..CIDv1, Table[MultiCodec, MultiHash]] - - once: - emptyDigests = [ - CIDv0: { - multiCodec("sha2-256"): Cid - .init("bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku") - .get() - .mhash - .get() - }.toTable, - CIDv1: { - multiCodec("sha2-256"): Cid - .init("QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n") - .get() - .mhash - .get() - }.toTable, - ] - - emptyDigests - -type - Manifest* = object of RootObj - rootHash*: ?Cid - blockSize*: int - blocks*: seq[Cid] - version*: CidVersion - hcodec*: MultiCodec - codec*: MultiCodec