codex-storage
/
nim-codex
mirror of https://github.com/codex-storage/nim-codex.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

manifest: make protection recursive 

We can protect in multiple dimensions, which require
the manifest to describe all dimensions, either directly,
or by making the structure recursive.
Here we make it recursive.

Signed-off-by: Csaba Kiraly <csaba.kiraly@gmail.com>
This commit is contained in:
Csaba Kiraly 2023-11-09 12:47:41 +01:00
parent 40d7714a67
commit 24ade73061
No known key found for this signature in database
GPG Key ID: 0FE274EE8C95166E
2 changed files with 23 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
codex/manifest/coders.nim
View File

@ -26,7 +26,7 @@ import ../errors
import ../blocktype
import ./types
proc encode*(_: DagPBCoder, manifest: Manifest): ?!seq[byte] =
proc encode*(coder: DagPBCoder, manifest: Manifest): ?!seq[byte] =
## Encode the manifest into a ``ManifestCodec``
## multicodec container (Dag-pb) for now
##
@ -41,8 +41,7 @@ proc encode*(_: DagPBCoder, manifest: Manifest): ?!seq[byte] =
# Message ErasureInfo {
# optional uint32 ecK = 1; # number of encoded blocks
# optional uint32 ecM = 2; # number of parity blocks
# optional bytes originalTreeCid = 3; # cid of the original dataset
# optional uint32 originalDatasetSize = 4; # size of the original dataset
# optional bytes originalManifest = 3; # manifest of the original dataset
# }
# Message Header {
# optional bytes treeCid = 1; # cid (root) of the tree
@ -61,8 +60,7 @@ proc encode*(_: DagPBCoder, manifest: Manifest): ?!seq[byte] =
var erasureInfo = initProtoBuffer()
erasureInfo.write(1, manifest.ecK.uint32)
erasureInfo.write(2, manifest.ecM.uint32)
erasureInfo.write(3, manifest.originalTreeCid.data.buffer)
erasureInfo.write(4, manifest.originalDatasetSize.uint32)
erasureInfo.write(3, ? coder.encode(manifest.originalManifest)) # TODO: fix check
erasureInfo.finish()
header.write(4, erasureInfo)
@ -72,7 +70,7 @@ proc encode*(_: DagPBCoder, manifest: Manifest): ?!seq[byte] =
return pbNode.buffer.success
proc decode*(_: DagPBCoder, data: openArray[byte]): ?!Manifest =
proc decode*(coder: DagPBCoder, data: openArray[byte]): ?!Manifest =
## Decode a manifest from a data blob
##
@ -81,10 +79,9 @@ proc decode*(_: DagPBCoder, data: openArray[byte]): ?!Manifest =
pbHeader: ProtoBuffer
pbErasureInfo: ProtoBuffer
treeCidBuf: seq[byte]
originalTreeCid: seq[byte]
originalManifest: Manifest
datasetSize: uint32
blockSize: uint32
originalDatasetSize: uint32
ecK, ecM: uint32
# Decode `Header` message
@ -112,12 +109,10 @@ proc decode*(_: DagPBCoder, data: openArray[byte]): ?!Manifest =
if pbErasureInfo.getField(2, ecM).isErr:
return failure("Unable to decode `M` from manifest!")
if pbErasureInfo.getField(3, originalTreeCid).isErr:
return failure("Unable to decode `originalTreeCid` from manifest!")
if pbErasureInfo.getField(4, originalDatasetSize).isErr:
return failure("Unable to decode `originalDatasetSize` from manifest!")
var buffer = newSeq[byte]()
if pbErasureInfo.getField(3, buffer).isErr:
return failure("Unable to decode `originalManifest` from manifest!")
originalManifest = coder.decode(buffer).get # TODO: fix check
let
treeCid = ? Cid.init(treeCidBuf).mapFailure
@ -133,8 +128,7 @@ proc decode*(_: DagPBCoder, data: openArray[byte]): ?!Manifest =
codec = treeCid.mcodec,
ecK = ecK.int,
ecM = ecM.int,
originalTreeCid = ? Cid.init(originalTreeCid).mapFailure,
originalDatasetSize = originalDatasetSize.NBytes
originalManifest originalManifest
)
else:
Manifest.new(

35
codex/manifest/manifest.nim
View File

@ -40,8 +40,7 @@ type
of true:
ecK: int # Number of blocks to encode
ecM: int # Number of resulting parity blocks
originalTreeCid: Cid # The original root of the dataset being erasure coded
originalDatasetSize: NBytes
originalManifest: Manifest # The original Manifest being erasure coded
else:
discard
@ -73,11 +72,15 @@ proc ecK*(self: Manifest): int =
proc ecM*(self: Manifest): int =
self.ecM
proc originalManifest*(self: Manifest): Manifest =
self.originalManifest
proc originalTreeCid*(self: Manifest): Cid =
self.originalTreeCid
self.originalManifest.treeCid
proc originalBlocksCount*(self: Manifest): int =
divUp(self.originalDatasetSize.int, self.blockSize.int)
divUp(self.originalManifest.datasetSize.int, self.blockSize.int)
proc originalDatasetSize*(self: Manifest): NBytes =
self.originalDatasetSize
@ -141,8 +144,7 @@ proc `==`*(a, b: Manifest): bool =
(if a.protected:
(a.ecK == b.ecK) and
(a.ecM == b.ecM) and
(a.originalTreeCid == b.originalTreeCid) and
(a.originalDatasetSize == b.originalDatasetSize)
(a.originalManifest == b.originalManifest)
else:
true)
@ -157,8 +159,7 @@ proc `$`*(self: Manifest): string =
(if self.protected:
", ecK: " & $self.ecK &
", ecM: " & $self.ecM &
", originalTreeCid: " & $self.originalTreeCid &
", originalDatasetSize: " & $self.originalDatasetSize
", originalManifest: " & $self.originalManifest
else:
"")
@ -205,8 +206,7 @@ proc new*(
blockSize: manifest.blockSize,
protected: true,
ecK: ecK, ecM: ecM,
originalTreeCid: manifest.treeCid,
originalDatasetSize: manifest.datasetSize)
originalManifest: manifest)
proc new*(
T: type Manifest,
@ -215,14 +215,7 @@ proc new*(
## Create an unprotected dataset from an
## erasure protected one
##
Manifest(
treeCid: manifest.originalTreeCid,
datasetSize: manifest.originalDatasetSize,
version: manifest.version,
codec: manifest.codec,
hcodec: manifest.hcodec,
blockSize: manifest.blockSize,
protected: false)
manifest.originalManifest
proc new*(
T: type Manifest,
@ -243,8 +236,7 @@ proc new*(
codec: MultiCodec,
ecK: int,
ecM: int,
originalTreeCid: Cid,
originalDatasetSize: NBytes
originalManifest: Manifest
): Manifest =
Manifest(
treeCid: treeCid,
@ -256,6 +248,5 @@ proc new*(
protected: true,
ecK: ecK,
ecM: ecM,
originalTreeCid: originalTreeCid,
originalDatasetSize: originalDatasetSize
originalManifest: originalManifest
)