2022-05-19 19:56:03 +00:00
|
|
|
## Nim-Codex
|
2022-03-14 16:06:36 +00:00
|
|
|
## Copyright (c) 2022 Status Research & Development GmbH
|
|
|
|
## Licensed under either of
|
|
|
|
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
|
|
|
|
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
|
|
|
|
## at your option.
|
|
|
|
## This file may not be copied, modified, or distributed except according to
|
|
|
|
## those terms.
|
|
|
|
|
2022-08-24 12:15:59 +00:00
|
|
|
# This module defines all operations on Manifest
|
|
|
|
|
2022-03-18 22:17:51 +00:00
|
|
|
import pkg/upraises
|
|
|
|
|
|
|
|
push: {.upraises: [].}
|
2022-03-14 16:06:36 +00:00
|
|
|
|
|
|
|
import pkg/libp2p/protobuf/minprotobuf
|
|
|
|
import pkg/libp2p
|
|
|
|
import pkg/questionable
|
|
|
|
import pkg/questionable/results
|
|
|
|
import pkg/chronicles
|
|
|
|
|
|
|
|
import ../errors
|
2022-08-24 12:15:59 +00:00
|
|
|
import ../utils
|
2023-11-09 08:47:09 +00:00
|
|
|
import ../utils/json
|
2023-07-06 23:23:27 +00:00
|
|
|
import ../units
|
2022-03-15 18:47:31 +00:00
|
|
|
import ../blocktype
|
2022-04-06 00:34:29 +00:00
|
|
|
import ./types
|
2022-03-14 16:06:36 +00:00
|
|
|
|
2023-07-06 23:23:27 +00:00
|
|
|
export types
|
|
|
|
|
2023-07-19 14:06:59 +00:00
|
|
|
type
|
|
|
|
Manifest* = ref object of RootObj
|
2023-11-14 17:52:27 +00:00
|
|
|
treeCid {.serialize.}: Cid # Root of the merkle tree
|
|
|
|
datasetSize {.serialize.}: NBytes # Total size of all blocks
|
|
|
|
blockSize {.serialize.}: NBytes # Size of each contained block (might not be needed if blocks are len-prefixed)
|
|
|
|
version: CidVersion # Cid version
|
|
|
|
hcodec: MultiCodec # Multihash codec
|
|
|
|
codec: MultiCodec # Data set codec
|
|
|
|
case protected {.serialize.}: bool # Protected datasets have erasure coded info
|
2023-07-19 14:06:59 +00:00
|
|
|
of true:
|
2023-11-14 17:52:27 +00:00
|
|
|
ecK: int # Number of blocks to encode
|
|
|
|
ecM: int # Number of resulting parity blocks
|
2023-11-14 17:28:24 +00:00
|
|
|
interleave: int # How far apart are blocks of an erasure code according to original index
|
2023-11-09 11:47:41 +00:00
|
|
|
originalManifest: Manifest # The original Manifest being erasure coded
|
2023-07-19 14:06:59 +00:00
|
|
|
else:
|
|
|
|
discard
|
|
|
|
|
|
|
|
############################################################
|
|
|
|
# Accessors
|
|
|
|
############################################################
|
|
|
|
|
|
|
|
proc blockSize*(self: Manifest): NBytes =
|
|
|
|
self.blockSize
|
|
|
|
|
2023-11-14 12:02:17 +00:00
|
|
|
proc datasetSize*(self: Manifest): NBytes =
|
|
|
|
self.datasetSize
|
2023-07-19 14:06:59 +00:00
|
|
|
|
|
|
|
proc version*(self: Manifest): CidVersion =
|
|
|
|
self.version
|
|
|
|
|
|
|
|
proc hcodec*(self: Manifest): MultiCodec =
|
|
|
|
self.hcodec
|
|
|
|
|
|
|
|
proc codec*(self: Manifest): MultiCodec =
|
|
|
|
self.codec
|
|
|
|
|
|
|
|
proc protected*(self: Manifest): bool =
|
|
|
|
self.protected
|
|
|
|
|
|
|
|
proc ecK*(self: Manifest): int =
|
|
|
|
self.ecK
|
|
|
|
|
|
|
|
proc ecM*(self: Manifest): int =
|
|
|
|
self.ecM
|
|
|
|
|
2023-11-14 17:28:24 +00:00
|
|
|
proc interleave*(self: Manifest): int =
|
|
|
|
self.interleave
|
|
|
|
|
2023-11-09 11:47:41 +00:00
|
|
|
proc originalManifest*(self: Manifest): Manifest =
|
|
|
|
self.originalManifest
|
|
|
|
|
2023-11-14 12:02:17 +00:00
|
|
|
proc originalTreeCid*(self: Manifest): Cid =
|
2023-11-09 11:47:41 +00:00
|
|
|
self.originalManifest.treeCid
|
2023-07-19 14:06:59 +00:00
|
|
|
|
2023-11-14 12:02:17 +00:00
|
|
|
proc originalBlocksCount*(self: Manifest): int =
|
2023-11-09 11:47:41 +00:00
|
|
|
divUp(self.originalManifest.datasetSize.int, self.blockSize.int)
|
|
|
|
|
2023-11-16 10:52:02 +00:00
|
|
|
proc unprotectedBlocksCount*(self: Manifest): int =
|
|
|
|
var mfest = self
|
|
|
|
while mfest.protected:
|
|
|
|
mfest = mfest.originalManifest
|
|
|
|
divUp(mfest.datasetSize.int, self.blockSize.int)
|
2023-07-19 14:06:59 +00:00
|
|
|
|
2023-11-14 12:02:17 +00:00
|
|
|
proc originalDatasetSize*(self: Manifest): NBytes =
|
|
|
|
self.originalDatasetSize
|
2022-03-14 16:06:36 +00:00
|
|
|
|
2023-11-14 12:02:17 +00:00
|
|
|
proc treeCid*(self: Manifest): Cid =
|
|
|
|
self.treeCid
|
2022-03-14 16:06:36 +00:00
|
|
|
|
2023-11-14 12:02:17 +00:00
|
|
|
proc blocksCount*(self: Manifest): int =
|
|
|
|
divUp(self.datasetSize.int, self.blockSize.int)
|
2022-03-14 16:06:36 +00:00
|
|
|
|
2023-11-14 12:02:17 +00:00
|
|
|
############################################################
|
|
|
|
# Operations on block list
|
|
|
|
############################################################
|
2022-03-14 16:06:36 +00:00
|
|
|
|
2022-12-03 00:00:55 +00:00
|
|
|
func isManifest*(cid: Cid): ?!bool =
|
2023-08-01 23:47:57 +00:00
|
|
|
let res = ?cid.contentType().mapFailure(CodexError)
|
|
|
|
($(res) in ManifestContainers).success
|
2022-12-03 00:00:55 +00:00
|
|
|
|
|
|
|
func isManifest*(mc: MultiCodec): ?!bool =
|
|
|
|
($mc in ManifestContainers).success
|
|
|
|
|
2022-08-24 12:15:59 +00:00
|
|
|
############################################################
|
|
|
|
# Various sizes and verification
|
|
|
|
############################################################
|
|
|
|
|
2023-07-06 23:23:27 +00:00
|
|
|
func bytes*(self: Manifest, pad = true): NBytes =
|
2022-08-24 12:15:59 +00:00
|
|
|
## Compute how many bytes corresponding StoreStream(Manifest, pad) will return
|
|
|
|
if pad or self.protected:
|
2023-11-14 12:02:17 +00:00
|
|
|
self.blocksCount.NBytes * self.blockSize
|
2022-08-24 12:15:59 +00:00
|
|
|
else:
|
2023-11-14 12:02:17 +00:00
|
|
|
self.datasetSize
|
2022-08-24 12:15:59 +00:00
|
|
|
|
|
|
|
func rounded*(self: Manifest): int =
|
|
|
|
## Number of data blocks in *protected* manifest including padding at the end
|
2023-11-14 12:02:17 +00:00
|
|
|
roundUp(self.originalBlocksCount, self.ecK)
|
2022-08-24 12:15:59 +00:00
|
|
|
|
|
|
|
func steps*(self: Manifest): int =
|
|
|
|
## Number of EC groups in *protected* manifest
|
2023-11-14 17:28:24 +00:00
|
|
|
divUp(self.originalBlocksCount, self.ecK * self.interleave)
|
2022-08-24 12:15:59 +00:00
|
|
|
|
|
|
|
func verify*(self: Manifest): ?!void =
|
|
|
|
## Check manifest correctness
|
|
|
|
##
|
|
|
|
|
2023-11-14 12:02:17 +00:00
|
|
|
if self.protected and (self.blocksCount != self.steps * (self.ecK + self.ecM)):
|
|
|
|
return failure newException(CodexError, "Broken manifest: wrong originalBlocksCount")
|
2022-08-24 12:15:59 +00:00
|
|
|
|
|
|
|
return success()
|
|
|
|
|
2023-11-14 12:02:17 +00:00
|
|
|
proc cid*(self: Manifest): ?!Cid {.deprecated: "use treeCid instead".} =
|
|
|
|
self.treeCid.success
|
|
|
|
|
|
|
|
proc `==`*(a, b: Manifest): bool =
|
|
|
|
(a.treeCid == b.treeCid) and
|
|
|
|
(a.datasetSize == b.datasetSize) and
|
|
|
|
(a.blockSize == b.blockSize) and
|
|
|
|
(a.version == b.version) and
|
|
|
|
(a.hcodec == b.hcodec) and
|
|
|
|
(a.codec == b.codec) and
|
|
|
|
(a.protected == b.protected) and
|
|
|
|
(if a.protected:
|
|
|
|
(a.ecK == b.ecK) and
|
|
|
|
(a.ecM == b.ecM) and
|
2023-11-14 17:28:24 +00:00
|
|
|
(a.interleave == b.interleave) and
|
2023-11-09 11:47:41 +00:00
|
|
|
(a.originalManifest == b.originalManifest)
|
2023-11-14 12:02:17 +00:00
|
|
|
else:
|
|
|
|
true)
|
|
|
|
|
|
|
|
proc `$`*(self: Manifest): string =
|
|
|
|
"treeCid: " & $self.treeCid &
|
|
|
|
", datasetSize: " & $self.datasetSize &
|
|
|
|
", blockSize: " & $self.blockSize &
|
|
|
|
", version: " & $self.version &
|
|
|
|
", hcodec: " & $self.hcodec &
|
|
|
|
", codec: " & $self.codec &
|
|
|
|
", protected: " & $self.protected &
|
|
|
|
(if self.protected:
|
|
|
|
", ecK: " & $self.ecK &
|
|
|
|
", ecM: " & $self.ecM &
|
2023-11-14 17:28:24 +00:00
|
|
|
", interleave: " & $self.interleave &
|
2023-11-09 11:47:41 +00:00
|
|
|
", originalManifest: " & $self.originalManifest
|
2023-11-14 12:02:17 +00:00
|
|
|
else:
|
|
|
|
"")
|
2022-08-24 12:15:59 +00:00
|
|
|
|
|
|
|
############################################################
|
|
|
|
# Constructors
|
|
|
|
############################################################
|
|
|
|
|
2022-03-17 13:56:46 +00:00
|
|
|
proc new*(
|
2023-06-22 15:11:18 +00:00
|
|
|
T: type Manifest,
|
2023-11-14 12:02:17 +00:00
|
|
|
treeCid: Cid,
|
|
|
|
blockSize: NBytes,
|
|
|
|
datasetSize: NBytes,
|
|
|
|
version: CidVersion = CIDv1,
|
2023-06-22 15:11:18 +00:00
|
|
|
hcodec = multiCodec("sha2-256"),
|
|
|
|
codec = multiCodec("raw"),
|
2023-11-14 12:02:17 +00:00
|
|
|
protected = false,
|
|
|
|
): Manifest =
|
2022-03-14 16:06:36 +00:00
|
|
|
|
|
|
|
T(
|
2023-11-14 12:02:17 +00:00
|
|
|
treeCid: treeCid,
|
|
|
|
blockSize: blockSize,
|
|
|
|
datasetSize: datasetSize,
|
2022-03-14 16:06:36 +00:00
|
|
|
version: version,
|
|
|
|
codec: codec,
|
|
|
|
hcodec: hcodec,
|
2023-11-14 12:02:17 +00:00
|
|
|
protected: protected)
|
2022-04-05 00:46:13 +00:00
|
|
|
|
|
|
|
proc new*(
|
2023-06-22 15:11:18 +00:00
|
|
|
T: type Manifest,
|
|
|
|
manifest: Manifest,
|
2023-11-14 12:02:17 +00:00
|
|
|
treeCid: Cid,
|
|
|
|
datasetSize: NBytes,
|
2023-11-14 17:28:24 +00:00
|
|
|
ecK, ecM: int,
|
|
|
|
interleave: int
|
2023-11-14 12:02:17 +00:00
|
|
|
): Manifest =
|
2022-04-05 00:46:13 +00:00
|
|
|
## Create an erasure protected dataset from an
|
2023-11-14 12:02:17 +00:00
|
|
|
## unprotected one
|
2022-04-05 00:46:13 +00:00
|
|
|
##
|
2023-11-14 12:02:17 +00:00
|
|
|
Manifest(
|
|
|
|
treeCid: treeCid,
|
|
|
|
datasetSize: datasetSize,
|
|
|
|
version: manifest.version,
|
|
|
|
codec: manifest.codec,
|
|
|
|
hcodec: manifest.hcodec,
|
|
|
|
blockSize: manifest.blockSize,
|
|
|
|
protected: true,
|
|
|
|
ecK: ecK, ecM: ecM,
|
2023-11-14 17:28:24 +00:00
|
|
|
interleave: interleave,
|
2023-11-09 11:47:41 +00:00
|
|
|
originalManifest: manifest)
|
2022-04-05 00:46:13 +00:00
|
|
|
|
2023-11-14 12:02:17 +00:00
|
|
|
proc new*(
|
|
|
|
T: type Manifest,
|
|
|
|
manifest: Manifest
|
|
|
|
): Manifest =
|
|
|
|
## Create an unprotected dataset from an
|
|
|
|
## erasure protected one
|
|
|
|
##
|
2023-11-09 11:47:41 +00:00
|
|
|
manifest.originalManifest
|
2022-03-15 18:47:31 +00:00
|
|
|
|
2022-03-17 13:56:46 +00:00
|
|
|
proc new*(
|
2022-03-15 18:47:31 +00:00
|
|
|
T: type Manifest,
|
2022-04-06 00:34:29 +00:00
|
|
|
data: openArray[byte],
|
2023-06-22 15:11:18 +00:00
|
|
|
decoder = ManifestContainers[$DagPBCodec]
|
|
|
|
): ?!Manifest =
|
|
|
|
## Create a manifest instance from given data
|
2023-07-19 14:06:59 +00:00
|
|
|
##
|
2022-04-06 00:34:29 +00:00
|
|
|
Manifest.decode(data, decoder)
|
2023-07-19 14:06:59 +00:00
|
|
|
|
|
|
|
proc new*(
|
|
|
|
T: type Manifest,
|
2023-11-14 12:02:17 +00:00
|
|
|
treeCid: Cid,
|
|
|
|
datasetSize: NBytes,
|
2023-07-19 14:06:59 +00:00
|
|
|
blockSize: NBytes,
|
|
|
|
version: CidVersion,
|
|
|
|
hcodec: MultiCodec,
|
|
|
|
codec: MultiCodec,
|
|
|
|
ecK: int,
|
|
|
|
ecM: int,
|
2023-11-14 17:28:24 +00:00
|
|
|
interleave: int,
|
2023-11-09 11:47:41 +00:00
|
|
|
originalManifest: Manifest
|
2023-07-19 14:06:59 +00:00
|
|
|
): Manifest =
|
|
|
|
Manifest(
|
2023-11-14 12:02:17 +00:00
|
|
|
treeCid: treeCid,
|
|
|
|
datasetSize: datasetSize,
|
2023-07-19 14:06:59 +00:00
|
|
|
blockSize: blockSize,
|
|
|
|
version: version,
|
|
|
|
hcodec: hcodec,
|
|
|
|
codec: codec,
|
|
|
|
protected: true,
|
|
|
|
ecK: ecK,
|
|
|
|
ecM: ecM,
|
2023-11-14 17:28:24 +00:00
|
|
|
interleave: interleave,
|
2023-11-09 11:47:41 +00:00
|
|
|
originalManifest: originalManifest
|
2023-07-19 14:06:59 +00:00
|
|
|
)
|