mirror of
https://github.com/logos-storage/logos-storage-nim.git
synced 2026-01-08 16:33:11 +00:00
fix decoding/encoding
This commit is contained in:
Dmitriy Ryajov
parent
42d222bbc2
commit
572748e7d7
@ -17,7 +17,7 @@ import pkg/questionable/results
|
||||
import pkg/chronicles
|
||||
import pkg/chronos
|
||||
|
||||
import ./types
|
||||
import ./manifest
|
||||
import ../errors
|
||||
|
||||
const
|
||||
@ -27,6 +27,12 @@ type
|
||||
ManifestCoderType*[codec: static MultiCodec] = object
|
||||
DagPBCoder* = ManifestCoderType[multiCodec("dag-pb")]
|
||||
|
||||
const
|
||||
# TODO: move somewhere better?
|
||||
ManifestContainers* = {
|
||||
$DagPBCodec: DagPBCoder()
|
||||
}.toTable
|
||||
|
||||
func encode*(_: DagPBCoder, manifest: Manifest): ?!seq[byte] =
|
||||
## Encode the manifest into a ``ManifestCodec``
|
||||
## multicodec container (Dag-pb) for now
|
||||
@ -40,8 +46,25 @@ func encode*(_: DagPBCoder, manifest: Manifest): ?!seq[byte] =
|
||||
pbLink.finish()
|
||||
pbNode.write(2, pbLink)
|
||||
|
||||
# NOTE: The `Data` field in the the `dag-pb`
|
||||
# contains the following protobuf `Message`
|
||||
#
|
||||
# ```protobuf
|
||||
# Message Header {
|
||||
# optional bytes rootHash = 1; # the root (tree) hash
|
||||
# optional uint32 blockSize = 2; # size of a single block
|
||||
# optional uint32 blocksLen = 3; # total amount of blocks
|
||||
# }
|
||||
# ```
|
||||
#
|
||||
|
||||
let cid = !manifest.rootHash
|
||||
pbNode.write(1, cid.data.buffer) # set the rootHash Cid as the data field
|
||||
var header = initProtoBuffer()
|
||||
header.write(1, cid.data.buffer)
|
||||
header.write(2, manifest.blockSize.uint32)
|
||||
header.write(3, manifest.blocks.len.uint32)
|
||||
|
||||
pbNode.write(1, header.buffer) # set the rootHash Cid as the data field
|
||||
pbNode.finish()
|
||||
|
||||
return pbNode.buffer.success
|
||||
@ -52,13 +75,27 @@ func decode*(_: DagPBCoder, data: openArray[byte]): ?!Manifest =
|
||||
|
||||
var
|
||||
pbNode = initProtoBuffer(data)
|
||||
cidBuf: seq[byte]
|
||||
pbHeader: ProtoBuffer
|
||||
rootHash: seq[byte]
|
||||
blockSize: uint32
|
||||
blocksLen: uint32
|
||||
blocks: seq[Cid]
|
||||
|
||||
if pbNode.getField(1, cidBuf).isErr:
|
||||
return failure("Unable to decode Cid from manifest!")
|
||||
# Decode `Header` message
|
||||
if pbNode.getField(1, pbHeader).isErr:
|
||||
return failure("Unable to decode `Header` from dag-pb manifest!")
|
||||
|
||||
let cid = ? Cid.init(cidBuf).mapFailure
|
||||
# Decode `Header` contents
|
||||
if pbHeader.getField(1, rootHash).isErr:
|
||||
return failure("Unable to decode `rootHash` from manifest!")
|
||||
|
||||
if pbHeader.getField(2, blockSize).isErr:
|
||||
return failure("Unable to decode `blockSize` from manifest!")
|
||||
|
||||
if pbHeader.getField(3, blocksLen).isErr:
|
||||
return failure("Unable to decode `blocksLen` from manifest!")
|
||||
|
||||
let rootHashCid = ? Cid.init(rootHash).mapFailure
|
||||
var linksBuf: seq[seq[byte]]
|
||||
if pbNode.getRepeatedField(2, linksBuf).isOk:
|
||||
for pbLinkBuf in linksBuf:
|
||||
@ -70,4 +107,31 @@ func decode*(_: DagPBCoder, data: openArray[byte]): ?!Manifest =
|
||||
if pbLink.getField(1, blockBuf).isOk:
|
||||
blocks.add(? Cid.init(blockBuf).mapFailure)
|
||||
|
||||
Manifest(rootHash: cid.some, blocks: blocks).success
|
||||
if blocksLen.int != blocks.len:
|
||||
return failure("Total blocks and length of blocks in header don't match!")
|
||||
|
||||
Manifest(
|
||||
rootHash: rootHashCid.some,
|
||||
blockSize: blockSize.int,
|
||||
blocks: blocks,
|
||||
hcodec: (? rootHashCid.mhash.mapFailure).mcodec,
|
||||
codec: rootHashCid.mcodec,
|
||||
version: rootHashCid.cidver).success
|
||||
|
||||
proc encode*(self: var Manifest, encoder = ManifestContainers[$DagPBCodec]): ?!seq[byte] =
|
||||
## Encode a manifest using `encoder`
|
||||
##
|
||||
|
||||
if self.rootHash.isNone:
|
||||
? self.makeRoot()
|
||||
|
||||
encoder.encode(self)
|
||||
|
||||
func decode*(
|
||||
_: type Manifest,
|
||||
data: openArray[byte],
|
||||
decoder = ManifestContainers[$DagPBCodec]): ?!Manifest =
|
||||
## Decode a manifest using `decoder`
|
||||
##
|
||||
|
||||
decoder.decode(data)
|
||||
|
||||
@ -16,17 +16,42 @@ import pkg/questionable
|
||||
import pkg/questionable/results
|
||||
import pkg/chronicles
|
||||
|
||||
import ./types
|
||||
import ./coders
|
||||
import ../errors
|
||||
import ../blocktype
|
||||
|
||||
export coders, Manifest
|
||||
template EmptyDigests: untyped =
|
||||
var
|
||||
emptyDigests {.global, threadvar.}:
|
||||
array[CIDv0..CIDv1, Table[MultiCodec, MultiHash]]
|
||||
|
||||
const
|
||||
ManifestContainers* = {
|
||||
$DagPBCodec: DagPBCoder()
|
||||
}.toTable
|
||||
once:
|
||||
emptyDigests = [
|
||||
CIDv0: {
|
||||
multiCodec("sha2-256"): Cid
|
||||
.init("bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku")
|
||||
.get()
|
||||
.mhash
|
||||
.get()
|
||||
}.toTable,
|
||||
CIDv1: {
|
||||
multiCodec("sha2-256"): Cid
|
||||
.init("QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n")
|
||||
.get()
|
||||
.mhash
|
||||
.get()
|
||||
}.toTable,
|
||||
]
|
||||
|
||||
emptyDigests
|
||||
|
||||
type
|
||||
Manifest* = object of RootObj
|
||||
rootHash*: ?Cid # root (tree) hash of the contained data set
|
||||
blockSize*: int # size of each contained block (might not be needed if blocks are len-prefixed)
|
||||
blocks*: seq[Cid] # block Cid
|
||||
version*: CidVersion # Cid version
|
||||
hcodec*: MultiCodec # Multihash codec
|
||||
codec*: MultiCodec # Data set codec
|
||||
|
||||
func len*(self: Manifest): int =
|
||||
self.blocks.len
|
||||
@ -66,7 +91,7 @@ template hashBytes(mh: MultiHash): seq[byte] =
|
||||
|
||||
mh.data.buffer[mh.dpos..(mh.dpos + mh.size - 1)]
|
||||
|
||||
proc makeRoot(self: var Manifest): ?!void =
|
||||
proc makeRoot*(self: var Manifest): ?!void =
|
||||
## Create a tree hash root of the contained
|
||||
## block hashes
|
||||
##
|
||||
@ -106,24 +131,6 @@ proc cid*(self: var Manifest): ?!Cid =
|
||||
|
||||
(!self.rootHash).success
|
||||
|
||||
proc encode*(self: var Manifest, encoder = ManifestContainers[$DagPBCodec]): ?!seq[byte] =
|
||||
## Encode a manifest using `encoder`
|
||||
##
|
||||
|
||||
if self.rootHash.isNone:
|
||||
? self.makeRoot()
|
||||
|
||||
encoder.encode(self)
|
||||
|
||||
func decode*(
|
||||
_: type Manifest,
|
||||
data: openArray[byte],
|
||||
decoder = ManifestContainers[$DagPBCodec]): ?!Manifest =
|
||||
## Decode a manifest using `decoder`
|
||||
##
|
||||
|
||||
decoder.decode(data)
|
||||
|
||||
proc init*(
|
||||
T: type Manifest,
|
||||
blocks: openArray[Cid] = [],
|
||||
@ -144,3 +151,8 @@ proc init*(
|
||||
hcodec: hcodec,
|
||||
blockSize: blockSize
|
||||
).success
|
||||
|
||||
proc init*(
|
||||
T: type Manifest,
|
||||
data: openArray[byte]): ?!T =
|
||||
Manifest.decode(data)
|
||||
|
||||
@ -1,47 +0,0 @@
|
||||
## Nim-Dagger
|
||||
## Copyright (c) 2022 Status Research & Development GmbH
|
||||
## Licensed under either of
|
||||
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
|
||||
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
|
||||
## at your option.
|
||||
## This file may not be copied, modified, or distributed except according to
|
||||
## those terms.
|
||||
|
||||
{.push raises: [Defect].}
|
||||
|
||||
import pkg/libp2p
|
||||
import pkg/questionable
|
||||
|
||||
template EmptyDigests*: untyped =
|
||||
var
|
||||
emptyDigests {.global, threadvar.}:
|
||||
array[CIDv0..CIDv1, Table[MultiCodec, MultiHash]]
|
||||
|
||||
once:
|
||||
emptyDigests = [
|
||||
CIDv0: {
|
||||
multiCodec("sha2-256"): Cid
|
||||
.init("bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku")
|
||||
.get()
|
||||
.mhash
|
||||
.get()
|
||||
}.toTable,
|
||||
CIDv1: {
|
||||
multiCodec("sha2-256"): Cid
|
||||
.init("QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n")
|
||||
.get()
|
||||
.mhash
|
||||
.get()
|
||||
}.toTable,
|
||||
]
|
||||
|
||||
emptyDigests
|
||||
|
||||
type
|
||||
Manifest* = object of RootObj
|
||||
rootHash*: ?Cid
|
||||
blockSize*: int
|
||||
blocks*: seq[Cid]
|
||||
version*: CidVersion
|
||||
hcodec*: MultiCodec
|
||||
codec*: MultiCodec
|
||||
Loading…
x
Reference in New Issue
Block a user