From 0c3d1dd563aa4e695b6e34e683357f06185a3069 Mon Sep 17 00:00:00 2001 From: Ben Bierens <39762930+benbierens@users.noreply.github.com> Date: Tue, 12 Dec 2023 09:11:54 +0100 Subject: [PATCH] Verifiable manifests (#642) * Adds test for encoding/decoding protected manifest * Setting up verifiable manifest * mysterious mysteries * Successful encoding test for verifiable manifests * extracts toF out of users of manifest code * Update codex/manifest/coders.nim Co-authored-by: Dmitriy Ryajov Signed-off-by: Ben Bierens <39762930+benbierens@users.noreply.github.com> * Review comments by Dmitriy * Adds missing verifiable print to $ method. * Replace poseidon2 F type with int as temporary stand-in for verification hashes * Replaces verification hash placeholder with CID --------- Signed-off-by: Ben Bierens <39762930+benbierens@users.noreply.github.com> Co-authored-by: Dmitriy Ryajov --- codex/manifest/coders.nim | 51 ++++++++++++++++++++++++++---- codex/manifest/manifest.nim | 60 ++++++++++++++++++++++++++++++++++-- codex/manifest/types.nim | 1 + config.nims | 1 + tests/codex/testmanifest.nim | 42 +++++++++++++++++-------- vendor/constantine | 2 +- vendor/nim-poseidon2 | 2 +- 7 files changed, 137 insertions(+), 22 deletions(-) diff --git a/codex/manifest/coders.nim b/codex/manifest/coders.nim index db504617..7bf173f9 100644 --- a/codex/manifest/coders.nim +++ b/codex/manifest/coders.nim @@ -14,6 +14,7 @@ import pkg/upraises push: {.upraises: [].} import std/tables +import std/sequtils import pkg/libp2p import pkg/questionable @@ -38,11 +39,16 @@ proc encode*(_: DagPBCoder, manifest: Manifest): ?!seq[byte] = # contains the following protobuf `Message` # # ```protobuf + # Message VerificationInfo { + # bytes verificationRoot = 1; # Decimal encoded field-element + # repeated bytes slotRoots = 2; # Decimal encoded field-elements + # } # Message ErasureInfo { - # optional uint32 ecK = 1; # number of encoded blocks - # optional uint32 ecM = 2; # number of parity blocks - # optional bytes originalTreeCid = 3; # cid of the original dataset - # optional uint32 originalDatasetSize = 4; # size of the original dataset + # optional uint32 ecK = 1; # number of encoded blocks + # optional uint32 ecM = 2; # number of parity blocks + # optional bytes originalTreeCid = 3; # cid of the original dataset + # optional uint32 originalDatasetSize = 4; # size of the original dataset + # optional VerificationInformation verification = 5; # verification information # } # Message Header { # optional bytes treeCid = 1; # cid (root) of the tree @@ -63,8 +69,15 @@ proc encode*(_: DagPBCoder, manifest: Manifest): ?!seq[byte] = erasureInfo.write(2, manifest.ecM.uint32) erasureInfo.write(3, manifest.originalTreeCid.data.buffer) erasureInfo.write(4, manifest.originalDatasetSize.uint32) - erasureInfo.finish() + if manifest.verifiable: + var verificationInfo = initProtoBuffer() + verificationInfo.write(1, manifest.verificationRoot.data.buffer) + for slotRoot in manifest.slotRoots: + verificationInfo.write(2, slotRoot.data.buffer) + erasureInfo.write(5, verificationInfo) + + erasureInfo.finish() header.write(4, erasureInfo) pbNode.write(1, header) # set the treeCid as the data field @@ -80,12 +93,15 @@ proc decode*(_: DagPBCoder, data: openArray[byte]): ?!Manifest = pbNode = initProtoBuffer(data) pbHeader: ProtoBuffer pbErasureInfo: ProtoBuffer + pbVerificationInfo: ProtoBuffer treeCidBuf: seq[byte] originalTreeCid: seq[byte] datasetSize: uint32 blockSize: uint32 originalDatasetSize: uint32 ecK, ecM: uint32 + verificationRoot: seq[byte] + slotRoots: seq[seq[byte]] # Decode `Header` message if pbNode.getField(1, pbHeader).isErr: @@ -105,6 +121,7 @@ proc decode*(_: DagPBCoder, data: openArray[byte]): ?!Manifest = return failure("Unable to decode `erasureInfo` from manifest!") let protected = pbErasureInfo.buffer.len > 0 + var verifiable = false if protected: if pbErasureInfo.getField(1, ecK).isErr: return failure("Unable to decode `K` from manifest!") @@ -118,8 +135,18 @@ proc decode*(_: DagPBCoder, data: openArray[byte]): ?!Manifest = if pbErasureInfo.getField(4, originalDatasetSize).isErr: return failure("Unable to decode `originalDatasetSize` from manifest!") + if pbErasureInfo.getField(5, pbVerificationInfo).isErr: + return failure("Unable to decode `verificationInfo` from manifest!") - let + verifiable = pbVerificationInfo.buffer.len > 0 + if verifiable: + if pbVerificationInfo.getField(1, verificationRoot).isErr: + return failure("Unable to decode `verificationRoot` from manifest!") + + if pbVerificationInfo.getRequiredRepeatedField(2, slotRoots).isErr: + return failure("Unable to decode `slotRoots` from manifest!") + + let treeCid = ? Cid.init(treeCidBuf).mapFailure let @@ -147,6 +174,18 @@ proc decode*(_: DagPBCoder, data: openArray[byte]): ?!Manifest = ) ? self.verify() + + if verifiable: + let + verificationRootCid = ? Cid.init(verificationRoot).mapFailure + slotRootCids = slotRoots.mapIt(? Cid.init(it).mapFailure) + + return Manifest.new( + manifest = self, + verificationRoot = verificationRootCid, + slotRoots = slotRootCids + ) + self.success proc encode*( diff --git a/codex/manifest/manifest.nim b/codex/manifest/manifest.nim index 886f9ead..8f8df55c 100644 --- a/codex/manifest/manifest.nim +++ b/codex/manifest/manifest.nim @@ -42,6 +42,12 @@ type ecM: int # Number of resulting parity blocks originalTreeCid: Cid # The original root of the dataset being erasure coded originalDatasetSize: NBytes + case verifiable {.serialize.}: bool # Verifiable datasets can be used to generate storage proofs + of true: + verificationRoot: Cid + slotRoots: seq[Cid] + else: + discard else: discard @@ -88,6 +94,15 @@ proc treeCid*(self: Manifest): Cid = proc blocksCount*(self: Manifest): int = divUp(self.datasetSize.int, self.blockSize.int) +proc verifiable*(self: Manifest): bool = + self.verifiable + +proc verificationRoot*(self: Manifest): Cid = + self.verificationRoot + +proc slotRoots*(self: Manifest): seq[Cid] = + self.slotRoots + ############################################################ # Operations on block list ############################################################ @@ -142,7 +157,13 @@ proc `==`*(a, b: Manifest): bool = (a.ecK == b.ecK) and (a.ecM == b.ecM) and (a.originalTreeCid == b.originalTreeCid) and - (a.originalDatasetSize == b.originalDatasetSize) + (a.originalDatasetSize == b.originalDatasetSize) and + (a.verifiable == b.verifiable) and + (if a.verifiable: + (a.verificationRoot == b.verificationRoot) and + (a.slotRoots == b.slotRoots) + else: + true) else: true) @@ -158,7 +179,13 @@ proc `$`*(self: Manifest): string = ", ecK: " & $self.ecK & ", ecM: " & $self.ecM & ", originalTreeCid: " & $self.originalTreeCid & - ", originalDatasetSize: " & $self.originalDatasetSize + ", originalDatasetSize: " & $self.originalDatasetSize & + ", verifiable: " & $self.verifiable & + (if self.verifiable: + ", verificationRoot: " & $self.verificationRoot & + ", slotRoots: " & $self.slotRoots + else: + "") else: "") @@ -259,3 +286,32 @@ proc new*( originalTreeCid: originalTreeCid, originalDatasetSize: originalDatasetSize ) + +proc new*( + T: type Manifest, + manifest: Manifest, + verificationRoot: Cid, + slotRoots: seq[Cid] +): ?!Manifest = + ## Create a verifiable dataset from an + ## protected one + ## + if not manifest.protected: + return failure newException(CodexError, "Can create verifiable manifest only from protected manifest.") + + success(Manifest( + treeCid: manifest.treeCid, + datasetSize: manifest.datasetSize, + version: manifest.version, + codec: manifest.codec, + hcodec: manifest.hcodec, + blockSize: manifest.blockSize, + protected: true, + ecK: manifest.ecK, + ecM: manifest.ecM, + originalTreeCid: manifest.treeCid, + originalDatasetSize: manifest.originalDatasetSize, + verifiable: true, + verificationRoot: verificationRoot, + slotRoots: slotRoots + )) diff --git a/codex/manifest/types.nim b/codex/manifest/types.nim index 710b6c28..613de2d7 100644 --- a/codex/manifest/types.nim +++ b/codex/manifest/types.nim @@ -10,6 +10,7 @@ # This module defines Manifest and all related types import std/tables +import std/strutils import pkg/libp2p import ../units diff --git a/config.nims b/config.nims index 031cdaa7..2f96ddfe 100644 --- a/config.nims +++ b/config.nims @@ -115,6 +115,7 @@ switch("define", "chronicles_sinks=textlines[dynamic],json[dynamic],textlines[dy # Workaround for assembler incompatibility between constantine and secp256k1 switch("define", "use_asm_syntax_intel=false") +switch("define", "ctt_asm=false") # begin Nimble config (version 1) when system.fileExists("nimble.paths"): diff --git a/tests/codex/testmanifest.nim b/tests/codex/testmanifest.nim index e2e4e152..72878885 100644 --- a/tests/codex/testmanifest.nim +++ b/tests/codex/testmanifest.nim @@ -3,8 +3,6 @@ import std/sequtils import pkg/chronos import pkg/questionable/results import pkg/asynctest -import pkg/stew/byteutils - import pkg/codex/chunker import pkg/codex/blocktype as bt import pkg/codex/manifest @@ -13,17 +11,37 @@ import ./helpers import ./examples checksuite "Manifest": - test "Should encode/decode to/from manifest": - var - manifest = Manifest.new( - treeCid = Cid.example, - blockSize = 1.MiBs, - datasetSize = 100.MiBs) + let + manifest = Manifest.new( + treeCid = Cid.example, + blockSize = 1.MiBs, + datasetSize = 100.MiBs + ) + protectedManifest = Manifest.new( + manifest = manifest, + treeCid = Cid.example, + datasetSize = 200.MiBs, + eck = 10, + ecM = 10 + ) + verifiableManifest = Manifest.new( + manifest = protectedManifest, + verificationRoot = Cid.example, + slotRoots = @[Cid.example, Cid.example] + ).tryGet() - let - e = manifest.encode().tryGet() - decoded = Manifest.decode(e).tryGet() + proc encodeDecode(manifest: Manifest): Manifest = + let e = manifest.encode().tryGet() + Manifest.decode(e).tryGet() + test "Should encode/decode to/from base manifest": check: - decoded == manifest + encodeDecode(manifest) == manifest + test "Should encode/decode to/from protected manifest": + check: + encodeDecode(protectedManifest) == protectedManifest + + test "Should encode/decode to/from verifiable manifest": + check: + encodeDecode(verifiableManifest) == verifiableManifest diff --git a/vendor/constantine b/vendor/constantine index 5f7ba18f..8367d7d1 160000 --- a/vendor/constantine +++ b/vendor/constantine @@ -1 +1 @@ -Subproject commit 5f7ba18f2ed351260015397c9eae079a6decaee1 +Subproject commit 8367d7d19cdbba874aab961b70d272e742184c37 diff --git a/vendor/nim-poseidon2 b/vendor/nim-poseidon2 index af673749..9be7b0c1 160000 --- a/vendor/nim-poseidon2 +++ b/vendor/nim-poseidon2 @@ -1 +1 @@ -Subproject commit af6737492971dd05b2a6b03404e490f865266f15 +Subproject commit 9be7b0c134e64e3d57a38520a32af93a55a37c44