nim-codex/codex/manifest/manifest.nim

## Nim-Codex
## Copyright (c) 2022 Status Research & Development GmbH
## Licensed under either of
##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
## at your option.
## This file may not be copied, modified, or distributed except according to
## those terms.

# This module defines all operations on Manifest

import pkg/upraises

push: {.upraises: [].}

import pkg/libp2p/protobuf/minprotobuf
import pkg/libp2p/[cid, multihash, multicodec]
import pkg/questionable/results

import ../errors
import ../utils
import ../utils/json
import ../units
import ../blocktype
import ../indexingstrategy
import ../logutils


# TODO: Manifest should be reworked to more concrete types,
# perhaps using inheritance
type
  Manifest* = ref object of RootObj
    treeCid {.serialize.}: Cid              # Root of the merkle tree
    datasetSize {.serialize.}: NBytes       # Total size of all blocks
    blockSize {.serialize.}: NBytes         # Size of each contained block (might not be needed if blocks are len-prefixed)
    codec: MultiCodec                       # Dataset codec
    hcodec: MultiCodec                      # Multihash codec
    version: CidVersion                     # Cid version
    filename {.serialize.}: ?string          # The filename of the content uploaded (optional)
    mimetype {.serialize.}: ?string          # The mimetype of the content uploaded (optional)
    uploadedAt {.serialize.}: ?int64          # The UTC creation timestamp in seconds
    case protected {.serialize.}: bool      # Protected datasets have erasure coded info
    of true:
      ecK: int                              # Number of blocks to encode
      ecM: int                              # Number of resulting parity blocks
      originalTreeCid: Cid                  # The original root of the dataset being erasure coded
      originalDatasetSize: NBytes
      protectedStrategy: StrategyType       # Indexing strategy used to build the slot roots
      case verifiable {.serialize.}: bool   # Verifiable datasets can be used to generate storage proofs
      of true:
        verifyRoot: Cid                     # Root of the top level merkle tree built from slot roots
        slotRoots: seq[Cid]                 # Individual slot root built from the original dataset blocks
        cellSize: NBytes                    # Size of each slot cell
        verifiableStrategy: StrategyType    # Indexing strategy used to build the slot roots
      else:
        discard
    else:
      discard

############################################################
# Accessors
############################################################

func blockSize*(self: Manifest): NBytes =
  self.blockSize

func datasetSize*(self: Manifest): NBytes =
  self.datasetSize

func version*(self: Manifest): CidVersion =
  self.version

func hcodec*(self: Manifest): MultiCodec =
  self.hcodec

func codec*(self: Manifest): MultiCodec =
  self.codec

func protected*(self: Manifest): bool =
  self.protected

func ecK*(self: Manifest): int =
  self.ecK

func ecM*(self: Manifest): int =
  self.ecM

func originalTreeCid*(self: Manifest): Cid =
  self.originalTreeCid

func originalBlocksCount*(self: Manifest): int =
  divUp(self.originalDatasetSize.int, self.blockSize.int)

func originalDatasetSize*(self: Manifest): NBytes =
  self.originalDatasetSize

func treeCid*(self: Manifest): Cid =
  self.treeCid

func blocksCount*(self: Manifest): int =
  divUp(self.datasetSize.int, self.blockSize.int)

func verifiable*(self: Manifest): bool =
  bool (self.protected and self.verifiable)

func verifyRoot*(self: Manifest): Cid =
  self.verifyRoot

func slotRoots*(self: Manifest): seq[Cid] =
  self.slotRoots

func numSlots*(self: Manifest): int =
  self.ecK + self.ecM

func cellSize*(self: Manifest): NBytes =
  self.cellSize

func protectedStrategy*(self: Manifest): StrategyType =
  self.protectedStrategy

func verifiableStrategy*(self: Manifest): StrategyType =
  self.verifiableStrategy

func numSlotBlocks*(self: Manifest): int =
  divUp(self.blocksCount, self.numSlots)

func filename*(self: Manifest): ?string =
  self.filename

func mimetype*(self: Manifest): ?string =
  self.mimetype

func uploadedAt*(self: Manifest): ?int64 =
  self.uploadedAt
############################################################
# Operations on block list
############################################################

func isManifest*(cid: Cid): ?!bool =
  success (ManifestCodec == ? cid.contentType().mapFailure(CodexError))

func isManifest*(mc: MultiCodec): ?!bool =
  success mc == ManifestCodec

############################################################
# Various sizes and verification
############################################################

func rounded*(self: Manifest): int =
  ## Number of data blocks in *protected* manifest including padding at the end
  roundUp(self.originalBlocksCount, self.ecK)

func steps*(self: Manifest): int =
  ## Number of EC groups in *protected* manifest
  divUp(self.rounded, self.ecK)

func verify*(self: Manifest): ?!void =
  ## Check manifest correctness
  ##

  if self.protected and (self.blocksCount != self.steps * (self.ecK + self.ecM)):
    return failure newException(CodexError, "Broken manifest: wrong originalBlocksCount")

  return success()

func cid*(self: Manifest): ?!Cid {.deprecated: "use treeCid instead".} =
  self.treeCid.success

func `==`*(a, b: Manifest): bool =
  (a.treeCid == b.treeCid) and
  (a.datasetSize == b.datasetSize) and
  (a.blockSize == b.blockSize) and
  (a.version == b.version) and
  (a.hcodec == b.hcodec) and
  (a.codec == b.codec) and
  (a.protected == b.protected) and
  (a.filename == b.filename) and
  (a.mimetype == b.mimetype) and
  (a.uploadedAt == b.uploadedAt) and
    (if a.protected:
      (a.ecK == b.ecK) and
      (a.ecM == b.ecM) and
      (a.originalTreeCid == b.originalTreeCid) and
      (a.originalDatasetSize == b.originalDatasetSize) and
      (a.protectedStrategy == b.protectedStrategy) and
      (a.verifiable == b.verifiable) and
        (if a.verifiable:
          (a.verifyRoot == b.verifyRoot) and
          (a.slotRoots == b.slotRoots) and
          (a.cellSize == b.cellSize) and
          (a.verifiableStrategy == b.verifiableStrategy)
        else:
          true)
    else:
      true)

func `$`*(self: Manifest): string =
  result = "treeCid: " & $self.treeCid &
    ", datasetSize: " & $self.datasetSize &
    ", blockSize: " & $self.blockSize &
    ", version: " & $self.version &
    ", hcodec: " & $self.hcodec &
    ", codec: " & $self.codec &
    ", protected: " & $self.protected

  if self.filename.isSome:
    result &= ", filename: " & $self.filename

  if self.mimetype.isSome:
    result &= ", mimetype: " & $self.mimetype

  if self.uploadedAt.isSome:
    result &= ", uploadedAt: " & $self.uploadedAt

  result &= (if self.protected:
    ", ecK: " & $self.ecK &
    ", ecM: " & $self.ecM &
    ", originalTreeCid: " & $self.originalTreeCid &
    ", originalDatasetSize: " & $self.originalDatasetSize &
    ", verifiable: " & $self.verifiable &
    (if self.verifiable:
      ", verifyRoot: " & $self.verifyRoot &
      ", slotRoots: " & $self.slotRoots
    else:
      "")
  else:
    "")

  return result

############################################################
# Constructors
############################################################

func new*(
  T: type Manifest,
  treeCid: Cid,
  blockSize: NBytes,
  datasetSize: NBytes,
  version: CidVersion = CIDv1,
  hcodec = Sha256HashCodec,
  codec = BlockCodec,
  protected = false,
  filename: ?string = string.none,
  mimetype: ?string = string.none,
  uploadedAt: ?int64 = int64.none): Manifest =

  T(
    treeCid: treeCid,
    blockSize: blockSize,
    datasetSize: datasetSize,
    version: version,
    codec: codec,
    hcodec: hcodec,
    protected: protected,
    filename: filename,
    mimetype: mimetype,
    uploadedAt: uploadedAt)

func new*(
  T: type Manifest,
  manifest: Manifest,
  treeCid: Cid,
  datasetSize: NBytes,
  ecK, ecM: int,
  strategy = SteppedStrategy): Manifest =
  ## Create an erasure protected dataset from an
  ## unprotected one
  ##

  Manifest(
    treeCid: treeCid,
    datasetSize: datasetSize,
    version: manifest.version,
    codec: manifest.codec,
    hcodec: manifest.hcodec,
    blockSize: manifest.blockSize,
    protected: true,
    ecK: ecK, ecM: ecM,
    originalTreeCid: manifest.treeCid,
    originalDatasetSize: manifest.datasetSize,
    protectedStrategy: strategy,
    filename: manifest.filename,
    mimetype: manifest.mimetype,
    uploadedAt: manifest.uploadedAt
    )

func new*(
  T: type Manifest,
  manifest: Manifest): Manifest =
  ## Create an unprotected dataset from an
  ## erasure protected one
  ##

  Manifest(
    treeCid: manifest.originalTreeCid,
    datasetSize: manifest.originalDatasetSize,
    version: manifest.version,
    codec: manifest.codec,
    hcodec: manifest.hcodec,
    blockSize: manifest.blockSize,
    protected: false,
    filename: manifest.filename,
    mimetype: manifest.mimetype,
    uploadedAt: manifest.uploadedAt)

func new*(
  T: type Manifest,
  treeCid: Cid,
  datasetSize: NBytes,
  blockSize: NBytes,
  version: CidVersion,
  hcodec: MultiCodec,
  codec: MultiCodec,
  ecK: int,
  ecM: int,
  originalTreeCid: Cid,
  originalDatasetSize: NBytes,
  strategy = SteppedStrategy,
  filename: ?string = string.none,
  mimetype: ?string = string.none,
  uploadedAt: ?int64 = int64.none): Manifest =

  Manifest(
    treeCid: treeCid,
    datasetSize: datasetSize,
    blockSize: blockSize,
    version: version,
    hcodec: hcodec,
    codec: codec,
    protected: true,
    ecK: ecK,
    ecM: ecM,
    originalTreeCid: originalTreeCid,
    originalDatasetSize: originalDatasetSize,
    protectedStrategy: strategy,
    filename: filename, 
    mimetype: mimetype,
    uploadedAt: uploadedAt)

func new*(
  T: type Manifest,
  manifest: Manifest,
  verifyRoot: Cid,
  slotRoots: openArray[Cid],
  cellSize = DefaultCellSize,
  strategy = LinearStrategy): ?!Manifest =
  ## Create a verifiable dataset from an
  ## protected one
  ##

  if not manifest.protected:
    return failure newException(
      CodexError, "Can create verifiable manifest only from protected manifest.")

  if slotRoots.len != manifest.numSlots:
    return failure newException(
      CodexError, "Wrong number of slot roots.")

  success Manifest(
    treeCid: manifest.treeCid,
    datasetSize: manifest.datasetSize,
    version: manifest.version,
    codec: manifest.codec,
    hcodec: manifest.hcodec,
    blockSize: manifest.blockSize,
    protected: true,
    ecK: manifest.ecK,
    ecM: manifest.ecM,
    originalTreeCid: manifest.originalTreeCid,
    originalDatasetSize: manifest.originalDatasetSize,
    protectedStrategy: manifest.protectedStrategy,
    verifiable: true,
    verifyRoot: verifyRoot,
    slotRoots: @slotRoots,
    cellSize: cellSize,
    verifiableStrategy: strategy,
    filename: manifest.filename,
    mimetype: manifest.mimetype,
    uploadedAt: manifest.uploadedAt
    )

func new*(
  T: type Manifest,
  data: openArray[byte]): ?!Manifest =
  ## Create a manifest instance from given data
  ##

  Manifest.decode(data)