Verifiable manifests (#642)
* Adds test for encoding/decoding protected manifest * Setting up verifiable manifest * mysterious mysteries * Successful encoding test for verifiable manifests * extracts toF out of users of manifest code * Update codex/manifest/coders.nim Co-authored-by: Dmitriy Ryajov <dryajov@gmail.com> Signed-off-by: Ben Bierens <39762930+benbierens@users.noreply.github.com> * Review comments by Dmitriy * Adds missing verifiable print to $ method. * Replace poseidon2 F type with int as temporary stand-in for verification hashes * Replaces verification hash placeholder with CID --------- Signed-off-by: Ben Bierens <39762930+benbierens@users.noreply.github.com> Co-authored-by: Dmitriy Ryajov <dryajov@gmail.com>
This commit is contained in:
Ben Bierens
GitHub
parent
a9f8090bb4
commit
0c3d1dd563
|
|
@ -14,6 +14,7 @@ import pkg/upraises
|
|||
push: {.upraises: [].}
|
||||
|
||||
import std/tables
|
||||
import std/sequtils
|
||||
|
||||
import pkg/libp2p
|
||||
import pkg/questionable
|
||||
|
|
@ -38,11 +39,16 @@ proc encode*(_: DagPBCoder, manifest: Manifest): ?!seq[byte] =
|
|||
# contains the following protobuf `Message`
|
||||
#
|
||||
# ```protobuf
|
||||
# Message VerificationInfo {
|
||||
# bytes verificationRoot = 1; # Decimal encoded field-element
|
||||
# repeated bytes slotRoots = 2; # Decimal encoded field-elements
|
||||
# }
|
||||
# Message ErasureInfo {
|
||||
# optional uint32 ecK = 1; # number of encoded blocks
|
||||
# optional uint32 ecM = 2; # number of parity blocks
|
||||
# optional bytes originalTreeCid = 3; # cid of the original dataset
|
||||
# optional uint32 originalDatasetSize = 4; # size of the original dataset
|
||||
# optional VerificationInformation verification = 5; # verification information
|
||||
# }
|
||||
# Message Header {
|
||||
# optional bytes treeCid = 1; # cid (root) of the tree
|
||||
|
|
@ -63,8 +69,15 @@ proc encode*(_: DagPBCoder, manifest: Manifest): ?!seq[byte] =
|
|||
erasureInfo.write(2, manifest.ecM.uint32)
|
||||
erasureInfo.write(3, manifest.originalTreeCid.data.buffer)
|
||||
erasureInfo.write(4, manifest.originalDatasetSize.uint32)
|
||||
erasureInfo.finish()
|
||||
|
||||
if manifest.verifiable:
|
||||
var verificationInfo = initProtoBuffer()
|
||||
verificationInfo.write(1, manifest.verificationRoot.data.buffer)
|
||||
for slotRoot in manifest.slotRoots:
|
||||
verificationInfo.write(2, slotRoot.data.buffer)
|
||||
erasureInfo.write(5, verificationInfo)
|
||||
|
||||
erasureInfo.finish()
|
||||
header.write(4, erasureInfo)
|
||||
|
||||
pbNode.write(1, header) # set the treeCid as the data field
|
||||
|
|
@ -80,12 +93,15 @@ proc decode*(_: DagPBCoder, data: openArray[byte]): ?!Manifest =
|
|||
pbNode = initProtoBuffer(data)
|
||||
pbHeader: ProtoBuffer
|
||||
pbErasureInfo: ProtoBuffer
|
||||
pbVerificationInfo: ProtoBuffer
|
||||
treeCidBuf: seq[byte]
|
||||
originalTreeCid: seq[byte]
|
||||
datasetSize: uint32
|
||||
blockSize: uint32
|
||||
originalDatasetSize: uint32
|
||||
ecK, ecM: uint32
|
||||
verificationRoot: seq[byte]
|
||||
slotRoots: seq[seq[byte]]
|
||||
|
||||
# Decode `Header` message
|
||||
if pbNode.getField(1, pbHeader).isErr:
|
||||
|
|
@ -105,6 +121,7 @@ proc decode*(_: DagPBCoder, data: openArray[byte]): ?!Manifest =
|
|||
return failure("Unable to decode `erasureInfo` from manifest!")
|
||||
|
||||
let protected = pbErasureInfo.buffer.len > 0
|
||||
var verifiable = false
|
||||
if protected:
|
||||
if pbErasureInfo.getField(1, ecK).isErr:
|
||||
return failure("Unable to decode `K` from manifest!")
|
||||
|
|
@ -118,6 +135,16 @@ proc decode*(_: DagPBCoder, data: openArray[byte]): ?!Manifest =
|
|||
if pbErasureInfo.getField(4, originalDatasetSize).isErr:
|
||||
return failure("Unable to decode `originalDatasetSize` from manifest!")
|
||||
|
||||
if pbErasureInfo.getField(5, pbVerificationInfo).isErr:
|
||||
return failure("Unable to decode `verificationInfo` from manifest!")
|
||||
|
||||
verifiable = pbVerificationInfo.buffer.len > 0
|
||||
if verifiable:
|
||||
if pbVerificationInfo.getField(1, verificationRoot).isErr:
|
||||
return failure("Unable to decode `verificationRoot` from manifest!")
|
||||
|
||||
if pbVerificationInfo.getRequiredRepeatedField(2, slotRoots).isErr:
|
||||
return failure("Unable to decode `slotRoots` from manifest!")
|
||||
|
||||
let
|
||||
treeCid = ? Cid.init(treeCidBuf).mapFailure
|
||||
|
|
@ -147,6 +174,18 @@ proc decode*(_: DagPBCoder, data: openArray[byte]): ?!Manifest =
|
|||
)
|
||||
|
||||
? self.verify()
|
||||
|
||||
if verifiable:
|
||||
let
|
||||
verificationRootCid = ? Cid.init(verificationRoot).mapFailure
|
||||
slotRootCids = slotRoots.mapIt(? Cid.init(it).mapFailure)
|
||||
|
||||
return Manifest.new(
|
||||
manifest = self,
|
||||
verificationRoot = verificationRootCid,
|
||||
slotRoots = slotRootCids
|
||||
)
|
||||
|
||||
self.success
|
||||
|
||||
proc encode*(
|
||||
|
|
|
|||
|
|
@ -42,6 +42,12 @@ type
|
|||
ecM: int # Number of resulting parity blocks
|
||||
originalTreeCid: Cid # The original root of the dataset being erasure coded
|
||||
originalDatasetSize: NBytes
|
||||
case verifiable {.serialize.}: bool # Verifiable datasets can be used to generate storage proofs
|
||||
of true:
|
||||
verificationRoot: Cid
|
||||
slotRoots: seq[Cid]
|
||||
else:
|
||||
discard
|
||||
else:
|
||||
discard
|
||||
|
||||
|
|
@ -88,6 +94,15 @@ proc treeCid*(self: Manifest): Cid =
|
|||
proc blocksCount*(self: Manifest): int =
|
||||
divUp(self.datasetSize.int, self.blockSize.int)
|
||||
|
||||
proc verifiable*(self: Manifest): bool =
|
||||
self.verifiable
|
||||
|
||||
proc verificationRoot*(self: Manifest): Cid =
|
||||
self.verificationRoot
|
||||
|
||||
proc slotRoots*(self: Manifest): seq[Cid] =
|
||||
self.slotRoots
|
||||
|
||||
############################################################
|
||||
# Operations on block list
|
||||
############################################################
|
||||
|
|
@ -142,7 +157,13 @@ proc `==`*(a, b: Manifest): bool =
|
|||
(a.ecK == b.ecK) and
|
||||
(a.ecM == b.ecM) and
|
||||
(a.originalTreeCid == b.originalTreeCid) and
|
||||
(a.originalDatasetSize == b.originalDatasetSize)
|
||||
(a.originalDatasetSize == b.originalDatasetSize) and
|
||||
(a.verifiable == b.verifiable) and
|
||||
(if a.verifiable:
|
||||
(a.verificationRoot == b.verificationRoot) and
|
||||
(a.slotRoots == b.slotRoots)
|
||||
else:
|
||||
true)
|
||||
else:
|
||||
true)
|
||||
|
||||
|
|
@ -158,7 +179,13 @@ proc `$`*(self: Manifest): string =
|
|||
", ecK: " & $self.ecK &
|
||||
", ecM: " & $self.ecM &
|
||||
", originalTreeCid: " & $self.originalTreeCid &
|
||||
", originalDatasetSize: " & $self.originalDatasetSize
|
||||
", originalDatasetSize: " & $self.originalDatasetSize &
|
||||
", verifiable: " & $self.verifiable &
|
||||
(if self.verifiable:
|
||||
", verificationRoot: " & $self.verificationRoot &
|
||||
", slotRoots: " & $self.slotRoots
|
||||
else:
|
||||
"")
|
||||
else:
|
||||
"")
|
||||
|
||||
|
|
@ -259,3 +286,32 @@ proc new*(
|
|||
originalTreeCid: originalTreeCid,
|
||||
originalDatasetSize: originalDatasetSize
|
||||
)
|
||||
|
||||
proc new*(
|
||||
T: type Manifest,
|
||||
manifest: Manifest,
|
||||
verificationRoot: Cid,
|
||||
slotRoots: seq[Cid]
|
||||
): ?!Manifest =
|
||||
## Create a verifiable dataset from an
|
||||
## protected one
|
||||
##
|
||||
if not manifest.protected:
|
||||
return failure newException(CodexError, "Can create verifiable manifest only from protected manifest.")
|
||||
|
||||
success(Manifest(
|
||||
treeCid: manifest.treeCid,
|
||||
datasetSize: manifest.datasetSize,
|
||||
version: manifest.version,
|
||||
codec: manifest.codec,
|
||||
hcodec: manifest.hcodec,
|
||||
blockSize: manifest.blockSize,
|
||||
protected: true,
|
||||
ecK: manifest.ecK,
|
||||
ecM: manifest.ecM,
|
||||
originalTreeCid: manifest.treeCid,
|
||||
originalDatasetSize: manifest.originalDatasetSize,
|
||||
verifiable: true,
|
||||
verificationRoot: verificationRoot,
|
||||
slotRoots: slotRoots
|
||||
))
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@
|
|||
# This module defines Manifest and all related types
|
||||
|
||||
import std/tables
|
||||
import std/strutils
|
||||
import pkg/libp2p
|
||||
|
||||
import ../units
|
||||
|
|
|
|||
|
|
@ -115,6 +115,7 @@ switch("define", "chronicles_sinks=textlines[dynamic],json[dynamic],textlines[dy
|
|||
|
||||
# Workaround for assembler incompatibility between constantine and secp256k1
|
||||
switch("define", "use_asm_syntax_intel=false")
|
||||
switch("define", "ctt_asm=false")
|
||||
|
||||
# begin Nimble config (version 1)
|
||||
when system.fileExists("nimble.paths"):
|
||||
|
|
|
|||
|
|
@ -3,8 +3,6 @@ import std/sequtils
|
|||
import pkg/chronos
|
||||
import pkg/questionable/results
|
||||
import pkg/asynctest
|
||||
import pkg/stew/byteutils
|
||||
|
||||
import pkg/codex/chunker
|
||||
import pkg/codex/blocktype as bt
|
||||
import pkg/codex/manifest
|
||||
|
|
@ -13,17 +11,37 @@ import ./helpers
|
|||
import ./examples
|
||||
|
||||
checksuite "Manifest":
|
||||
test "Should encode/decode to/from manifest":
|
||||
var
|
||||
let
|
||||
manifest = Manifest.new(
|
||||
treeCid = Cid.example,
|
||||
blockSize = 1.MiBs,
|
||||
datasetSize = 100.MiBs)
|
||||
datasetSize = 100.MiBs
|
||||
)
|
||||
protectedManifest = Manifest.new(
|
||||
manifest = manifest,
|
||||
treeCid = Cid.example,
|
||||
datasetSize = 200.MiBs,
|
||||
eck = 10,
|
||||
ecM = 10
|
||||
)
|
||||
verifiableManifest = Manifest.new(
|
||||
manifest = protectedManifest,
|
||||
verificationRoot = Cid.example,
|
||||
slotRoots = @[Cid.example, Cid.example]
|
||||
).tryGet()
|
||||
|
||||
let
|
||||
e = manifest.encode().tryGet()
|
||||
decoded = Manifest.decode(e).tryGet()
|
||||
proc encodeDecode(manifest: Manifest): Manifest =
|
||||
let e = manifest.encode().tryGet()
|
||||
Manifest.decode(e).tryGet()
|
||||
|
||||
test "Should encode/decode to/from base manifest":
|
||||
check:
|
||||
decoded == manifest
|
||||
encodeDecode(manifest) == manifest
|
||||
|
||||
test "Should encode/decode to/from protected manifest":
|
||||
check:
|
||||
encodeDecode(protectedManifest) == protectedManifest
|
||||
|
||||
test "Should encode/decode to/from verifiable manifest":
|
||||
check:
|
||||
encodeDecode(verifiableManifest) == verifiableManifest
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
Subproject commit 5f7ba18f2ed351260015397c9eae079a6decaee1
|
||||
Subproject commit 8367d7d19cdbba874aab961b70d272e742184c37
|
||||
|
|
@ -1 +1 @@
|
|||
Subproject commit af6737492971dd05b2a6b03404e490f865266f15
|
||||
Subproject commit 9be7b0c134e64e3d57a38520a32af93a55a37c44
|
||||
Loading…
Reference in New Issue