Verifiable manifests (#642)

* Adds test for encoding/decoding protected manifest

* Setting up verifiable manifest

* mysterious mysteries

* Successful encoding test for verifiable manifests

* extracts toF out of users of manifest code

* Update codex/manifest/coders.nim

Co-authored-by: Dmitriy Ryajov <dryajov@gmail.com>
Signed-off-by: Ben Bierens <39762930+benbierens@users.noreply.github.com>

* Review comments by Dmitriy

* Adds missing verifiable print to $ method.

* Replace poseidon2 F type with int as temporary stand-in for verification hashes

* Replaces verification hash placeholder with CID

---------

Signed-off-by: Ben Bierens <39762930+benbierens@users.noreply.github.com>
Co-authored-by: Dmitriy Ryajov <dryajov@gmail.com>

codex/manifest/coders.nim

@@ -14,6 +14,7 @@ import pkg/upraises
 push: {.upraises: [].}
 
 import std/tables
+import std/sequtils
 
 import pkg/libp2p
 import pkg/questionable
@@ -38,11 +39,16 @@ proc encode*(_: DagPBCoder, manifest: Manifest): ?!seq[byte] =
   # contains the following protobuf `Message`
   #
   # ```protobuf
+  #   Message VerificationInfo {
+  #     bytes verificationRoot = 1;            # Decimal encoded field-element
+  #     repeated bytes slotRoots = 2;     # Decimal encoded field-elements
+  #   }
   #   Message ErasureInfo {
-  #     optional uint32 ecK = 1;                  # number of encoded blocks
+  #     optional uint32 ecK = 1;                            # number of encoded blocks
-  #     optional uint32 ecM = 2;                  # number of parity blocks
+  #     optional uint32 ecM = 2;                            # number of parity blocks
-  #     optional bytes originalTreeCid = 3;       # cid of the original dataset
+  #     optional bytes originalTreeCid = 3;                 # cid of the original dataset
-  #     optional uint32 originalDatasetSize = 4;  # size of the original dataset
+  #     optional uint32 originalDatasetSize = 4;            # size of the original dataset
+  #     optional VerificationInformation verification = 5;  # verification information
   #   }
   #   Message Header {
   #     optional bytes treeCid = 1;       # cid (root) of the tree
@@ -63,8 +69,15 @@ proc encode*(_: DagPBCoder, manifest: Manifest): ?!seq[byte] =
     erasureInfo.write(2, manifest.ecM.uint32)
     erasureInfo.write(3, manifest.originalTreeCid.data.buffer)
     erasureInfo.write(4, manifest.originalDatasetSize.uint32)
-    erasureInfo.finish()
 
+    if manifest.verifiable:
+      var verificationInfo = initProtoBuffer()
+      verificationInfo.write(1, manifest.verificationRoot.data.buffer)
+      for slotRoot in manifest.slotRoots:
+        verificationInfo.write(2, slotRoot.data.buffer)
+      erasureInfo.write(5, verificationInfo)
+
+    erasureInfo.finish()
     header.write(4, erasureInfo)
 
   pbNode.write(1, header) # set the treeCid as the data field
@@ -80,12 +93,15 @@ proc decode*(_: DagPBCoder, data: openArray[byte]): ?!Manifest =
     pbNode = initProtoBuffer(data)
     pbHeader: ProtoBuffer
     pbErasureInfo: ProtoBuffer
+    pbVerificationInfo: ProtoBuffer
     treeCidBuf: seq[byte]
     originalTreeCid: seq[byte]
     datasetSize: uint32
     blockSize: uint32
     originalDatasetSize: uint32
     ecK, ecM: uint32
+    verificationRoot: seq[byte]
+    slotRoots: seq[seq[byte]]
 
   # Decode `Header` message
   if pbNode.getField(1, pbHeader).isErr:
@@ -105,6 +121,7 @@ proc decode*(_: DagPBCoder, data: openArray[byte]): ?!Manifest =
     return failure("Unable to decode `erasureInfo` from manifest!")
 
   let protected = pbErasureInfo.buffer.len > 0
+  var verifiable = false
   if protected:
     if pbErasureInfo.getField(1, ecK).isErr:
       return failure("Unable to decode `K` from manifest!")
@@ -118,8 +135,18 @@ proc decode*(_: DagPBCoder, data: openArray[byte]): ?!Manifest =
     if pbErasureInfo.getField(4, originalDatasetSize).isErr:
       return failure("Unable to decode `originalDatasetSize` from manifest!")
 
+    if pbErasureInfo.getField(5, pbVerificationInfo).isErr:
+      return failure("Unable to decode `verificationInfo` from manifest!")
 
-  let 
+    verifiable = pbVerificationInfo.buffer.len > 0
+    if verifiable:
+      if pbVerificationInfo.getField(1, verificationRoot).isErr:
+        return failure("Unable to decode `verificationRoot` from manifest!")
+
+      if pbVerificationInfo.getRequiredRepeatedField(2, slotRoots).isErr:
+        return failure("Unable to decode `slotRoots` from manifest!")
+
+  let
     treeCid = ? Cid.init(treeCidBuf).mapFailure
 
   let
@@ -147,6 +174,18 @@ proc decode*(_: DagPBCoder, data: openArray[byte]): ?!Manifest =
         )
 
   ? self.verify()
+
+  if verifiable:
+    let
+      verificationRootCid = ? Cid.init(verificationRoot).mapFailure
+      slotRootCids = slotRoots.mapIt(? Cid.init(it).mapFailure)
+
+    return Manifest.new(
+      manifest = self,
+      verificationRoot = verificationRootCid,
+      slotRoots = slotRootCids
+    )
+
   self.success
 
 proc encode*(

codex/manifest/manifest.nim

@@ -42,6 +42,12 @@ type
       ecM: int                              # Number of resulting parity blocks
       originalTreeCid: Cid                  # The original root of the dataset being erasure coded
       originalDatasetSize: NBytes
+      case verifiable {.serialize.}: bool   # Verifiable datasets can be used to generate storage proofs
+      of true:
+        verificationRoot: Cid
+        slotRoots: seq[Cid]
+      else:
+        discard
     else:
       discard
 
@@ -88,6 +94,15 @@ proc treeCid*(self: Manifest): Cid =
 proc blocksCount*(self: Manifest): int =
   divUp(self.datasetSize.int, self.blockSize.int)
 
+proc verifiable*(self: Manifest): bool =
+  self.verifiable
+
+proc verificationRoot*(self: Manifest): Cid =
+  self.verificationRoot
+
+proc slotRoots*(self: Manifest): seq[Cid] =
+  self.slotRoots
+
 ############################################################
 # Operations on block list
 ############################################################
@@ -142,7 +157,13 @@ proc `==`*(a, b: Manifest): bool =
       (a.ecK == b.ecK) and
       (a.ecM == b.ecM) and
       (a.originalTreeCid == b.originalTreeCid) and
-      (a.originalDatasetSize == b.originalDatasetSize)
+      (a.originalDatasetSize == b.originalDatasetSize) and
+      (a.verifiable == b.verifiable) and
+        (if a.verifiable:
+          (a.verificationRoot == b.verificationRoot) and
+          (a.slotRoots == b.slotRoots)
+        else:
+          true)
     else:
       true)
 
@@ -158,7 +179,13 @@ proc `$`*(self: Manifest): string =
       ", ecK: " & $self.ecK &
       ", ecM: " & $self.ecM &
       ", originalTreeCid: " & $self.originalTreeCid &
-      ", originalDatasetSize: " & $self.originalDatasetSize
+      ", originalDatasetSize: " & $self.originalDatasetSize &
+      ", verifiable: " & $self.verifiable &
+      (if self.verifiable:
+        ", verificationRoot: " & $self.verificationRoot &
+        ", slotRoots: " & $self.slotRoots
+      else:
+        "")
     else:
       "")
 
@@ -259,3 +286,32 @@ proc new*(
     originalTreeCid: originalTreeCid,
     originalDatasetSize: originalDatasetSize
   )
+
+proc new*(
+    T: type Manifest,
+    manifest: Manifest,
+    verificationRoot: Cid,
+    slotRoots: seq[Cid]
+): ?!Manifest =
+  ## Create a verifiable dataset from an
+  ## protected one
+  ##
+  if not manifest.protected:
+    return failure newException(CodexError, "Can create verifiable manifest only from protected manifest.")
+
+  success(Manifest(
+    treeCid: manifest.treeCid,
+    datasetSize: manifest.datasetSize,
+    version: manifest.version,
+    codec: manifest.codec,
+    hcodec: manifest.hcodec,
+    blockSize: manifest.blockSize,
+    protected: true,
+    ecK: manifest.ecK,
+    ecM: manifest.ecM,
+    originalTreeCid: manifest.treeCid,
+    originalDatasetSize: manifest.originalDatasetSize,
+    verifiable: true,
+    verificationRoot: verificationRoot,
+    slotRoots: slotRoots
+  ))

codex/manifest/types.nim

@@ -10,6 +10,7 @@
 # This module defines Manifest and all related types
 
 import std/tables
+import std/strutils
 import pkg/libp2p
 
 import ../units

config.nims

@@ -115,6 +115,7 @@ switch("define", "chronicles_sinks=textlines[dynamic],json[dynamic],textlines[dy
 
 # Workaround for assembler incompatibility between constantine and secp256k1
 switch("define", "use_asm_syntax_intel=false")
+switch("define", "ctt_asm=false")
 
 # begin Nimble config (version 1)
 when system.fileExists("nimble.paths"):

tests/codex/testmanifest.nim

@@ -3,8 +3,6 @@ import std/sequtils
 import pkg/chronos
 import pkg/questionable/results
 import pkg/asynctest
-import pkg/stew/byteutils
-
 import pkg/codex/chunker
 import pkg/codex/blocktype as bt
 import pkg/codex/manifest
@@ -13,17 +11,37 @@ import ./helpers
 import ./examples
 
 checksuite "Manifest":
-  test "Should encode/decode to/from manifest":
+  let
-    var
+    manifest = Manifest.new(
-      manifest = Manifest.new(
+      treeCid = Cid.example,
-        treeCid = Cid.example,
+      blockSize = 1.MiBs,
-        blockSize = 1.MiBs,
+      datasetSize = 100.MiBs
-        datasetSize = 100.MiBs)
+    )
+    protectedManifest = Manifest.new(
+      manifest = manifest,
+      treeCid = Cid.example,
+      datasetSize = 200.MiBs,
+      eck = 10,
+      ecM = 10
+    )
+    verifiableManifest = Manifest.new(
+      manifest = protectedManifest,
+      verificationRoot = Cid.example,
+      slotRoots = @[Cid.example, Cid.example]
+    ).tryGet()
 
-    let
+  proc encodeDecode(manifest: Manifest): Manifest =
-      e = manifest.encode().tryGet()
+    let e = manifest.encode().tryGet()
-      decoded = Manifest.decode(e).tryGet()
+    Manifest.decode(e).tryGet()
 
+  test "Should encode/decode to/from base manifest":
     check:
-      decoded == manifest
+      encodeDecode(manifest) == manifest
 
+  test "Should encode/decode to/from protected manifest":
+    check:
+      encodeDecode(protectedManifest) == protectedManifest
+
+  test "Should encode/decode to/from verifiable manifest":
+    check:
+      encodeDecode(verifiableManifest) == verifiableManifest

vendor/constantine

@@ -1 +1 @@
-Subproject commit 5f7ba18f2ed351260015397c9eae079a6decaee1
+Subproject commit 8367d7d19cdbba874aab961b70d272e742184c37

vendor/nim-poseidon2

@@ -1 +1 @@
-Subproject commit af6737492971dd05b2a6b03404e490f865266f15
+Subproject commit 9be7b0c134e64e3d57a38520a32af93a55a37c44