From 24ade730614caacd1f53be9bbe0caa5c3068ab57 Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Thu, 9 Nov 2023 12:47:41 +0100 Subject: [PATCH] manifest: make protection recursive We can protect in multiple dimensions, which require the manifest to describe all dimensions, either directly, or by making the structure recursive. Here we make it recursive. Signed-off-by: Csaba Kiraly --- codex/manifest/coders.nim | 26 ++++++++++---------------- codex/manifest/manifest.nim | 35 +++++++++++++---------------------- 2 files changed, 23 insertions(+), 38 deletions(-) diff --git a/codex/manifest/coders.nim b/codex/manifest/coders.nim index db504617..ee876265 100644 --- a/codex/manifest/coders.nim +++ b/codex/manifest/coders.nim @@ -26,7 +26,7 @@ import ../errors import ../blocktype import ./types -proc encode*(_: DagPBCoder, manifest: Manifest): ?!seq[byte] = +proc encode*(coder: DagPBCoder, manifest: Manifest): ?!seq[byte] = ## Encode the manifest into a ``ManifestCodec`` ## multicodec container (Dag-pb) for now ## @@ -41,8 +41,7 @@ proc encode*(_: DagPBCoder, manifest: Manifest): ?!seq[byte] = # Message ErasureInfo { # optional uint32 ecK = 1; # number of encoded blocks # optional uint32 ecM = 2; # number of parity blocks - # optional bytes originalTreeCid = 3; # cid of the original dataset - # optional uint32 originalDatasetSize = 4; # size of the original dataset + # optional bytes originalManifest = 3; # manifest of the original dataset # } # Message Header { # optional bytes treeCid = 1; # cid (root) of the tree @@ -61,8 +60,7 @@ proc encode*(_: DagPBCoder, manifest: Manifest): ?!seq[byte] = var erasureInfo = initProtoBuffer() erasureInfo.write(1, manifest.ecK.uint32) erasureInfo.write(2, manifest.ecM.uint32) - erasureInfo.write(3, manifest.originalTreeCid.data.buffer) - erasureInfo.write(4, manifest.originalDatasetSize.uint32) + erasureInfo.write(3, ? coder.encode(manifest.originalManifest)) # TODO: fix check erasureInfo.finish() header.write(4, erasureInfo) @@ -72,7 +70,7 @@ proc encode*(_: DagPBCoder, manifest: Manifest): ?!seq[byte] = return pbNode.buffer.success -proc decode*(_: DagPBCoder, data: openArray[byte]): ?!Manifest = +proc decode*(coder: DagPBCoder, data: openArray[byte]): ?!Manifest = ## Decode a manifest from a data blob ## @@ -81,10 +79,9 @@ proc decode*(_: DagPBCoder, data: openArray[byte]): ?!Manifest = pbHeader: ProtoBuffer pbErasureInfo: ProtoBuffer treeCidBuf: seq[byte] - originalTreeCid: seq[byte] + originalManifest: Manifest datasetSize: uint32 blockSize: uint32 - originalDatasetSize: uint32 ecK, ecM: uint32 # Decode `Header` message @@ -112,12 +109,10 @@ proc decode*(_: DagPBCoder, data: openArray[byte]): ?!Manifest = if pbErasureInfo.getField(2, ecM).isErr: return failure("Unable to decode `M` from manifest!") - if pbErasureInfo.getField(3, originalTreeCid).isErr: - return failure("Unable to decode `originalTreeCid` from manifest!") - - if pbErasureInfo.getField(4, originalDatasetSize).isErr: - return failure("Unable to decode `originalDatasetSize` from manifest!") - + var buffer = newSeq[byte]() + if pbErasureInfo.getField(3, buffer).isErr: + return failure("Unable to decode `originalManifest` from manifest!") + originalManifest = coder.decode(buffer).get # TODO: fix check let treeCid = ? Cid.init(treeCidBuf).mapFailure @@ -133,8 +128,7 @@ proc decode*(_: DagPBCoder, data: openArray[byte]): ?!Manifest = codec = treeCid.mcodec, ecK = ecK.int, ecM = ecM.int, - originalTreeCid = ? Cid.init(originalTreeCid).mapFailure, - originalDatasetSize = originalDatasetSize.NBytes + originalManifest originalManifest ) else: Manifest.new( diff --git a/codex/manifest/manifest.nim b/codex/manifest/manifest.nim index 886f9ead..db7c341e 100644 --- a/codex/manifest/manifest.nim +++ b/codex/manifest/manifest.nim @@ -40,8 +40,7 @@ type of true: ecK: int # Number of blocks to encode ecM: int # Number of resulting parity blocks - originalTreeCid: Cid # The original root of the dataset being erasure coded - originalDatasetSize: NBytes + originalManifest: Manifest # The original Manifest being erasure coded else: discard @@ -73,11 +72,15 @@ proc ecK*(self: Manifest): int = proc ecM*(self: Manifest): int = self.ecM +proc originalManifest*(self: Manifest): Manifest = + self.originalManifest + proc originalTreeCid*(self: Manifest): Cid = - self.originalTreeCid + self.originalManifest.treeCid proc originalBlocksCount*(self: Manifest): int = - divUp(self.originalDatasetSize.int, self.blockSize.int) + divUp(self.originalManifest.datasetSize.int, self.blockSize.int) + proc originalDatasetSize*(self: Manifest): NBytes = self.originalDatasetSize @@ -141,8 +144,7 @@ proc `==`*(a, b: Manifest): bool = (if a.protected: (a.ecK == b.ecK) and (a.ecM == b.ecM) and - (a.originalTreeCid == b.originalTreeCid) and - (a.originalDatasetSize == b.originalDatasetSize) + (a.originalManifest == b.originalManifest) else: true) @@ -157,8 +159,7 @@ proc `$`*(self: Manifest): string = (if self.protected: ", ecK: " & $self.ecK & ", ecM: " & $self.ecM & - ", originalTreeCid: " & $self.originalTreeCid & - ", originalDatasetSize: " & $self.originalDatasetSize + ", originalManifest: " & $self.originalManifest else: "") @@ -205,8 +206,7 @@ proc new*( blockSize: manifest.blockSize, protected: true, ecK: ecK, ecM: ecM, - originalTreeCid: manifest.treeCid, - originalDatasetSize: manifest.datasetSize) + originalManifest: manifest) proc new*( T: type Manifest, @@ -215,14 +215,7 @@ proc new*( ## Create an unprotected dataset from an ## erasure protected one ## - Manifest( - treeCid: manifest.originalTreeCid, - datasetSize: manifest.originalDatasetSize, - version: manifest.version, - codec: manifest.codec, - hcodec: manifest.hcodec, - blockSize: manifest.blockSize, - protected: false) + manifest.originalManifest proc new*( T: type Manifest, @@ -243,8 +236,7 @@ proc new*( codec: MultiCodec, ecK: int, ecM: int, - originalTreeCid: Cid, - originalDatasetSize: NBytes + originalManifest: Manifest ): Manifest = Manifest( treeCid: treeCid, @@ -256,6 +248,5 @@ proc new*( protected: true, ecK: ecK, ecM: ecM, - originalTreeCid: originalTreeCid, - originalDatasetSize: originalDatasetSize + originalManifest: originalManifest )