status-im
/
nim-codex
mirror of https://github.com/status-im/nim-codex.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Verifiable manifests (#642) 

* Adds test for encoding/decoding protected manifest

* Setting up verifiable manifest

* mysterious mysteries

* Successful encoding test for verifiable manifests

* extracts toF out of users of manifest code

* Update codex/manifest/coders.nim

Co-authored-by: Dmitriy Ryajov <dryajov@gmail.com>
Signed-off-by: Ben Bierens <39762930+benbierens@users.noreply.github.com>

* Review comments by Dmitriy

* Adds missing verifiable print to $ method.

* Replace poseidon2 F type with int as temporary stand-in for verification hashes

* Replaces verification hash placeholder with CID

---------

Signed-off-by: Ben Bierens <39762930+benbierens@users.noreply.github.com>
Co-authored-by: Dmitriy Ryajov <dryajov@gmail.com>
This commit is contained in:
Ben Bierens 2023-12-12 09:11:54 +01:00 committed by GitHub
parent a9f8090bb4
commit 0c3d1dd563
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 137 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
codex/manifest/coders.nim
View File

@ -14,6 +14,7 @@ import pkg/upraises
push: {.upraises: [].}
import std/tables
import std/sequtils
import pkg/libp2p
import pkg/questionable
@ -38,11 +39,16 @@ proc encode*(_: DagPBCoder, manifest: Manifest): ?!seq[byte] =
# contains the following protobuf `Message`
#
# ```protobuf
# Message VerificationInfo {
# bytes verificationRoot = 1; # Decimal encoded field-element
# repeated bytes slotRoots = 2; # Decimal encoded field-elements
# }
# Message ErasureInfo {
# optional uint32 ecK = 1; # number of encoded blocks
# optional uint32 ecM = 2; # number of parity blocks
# optional bytes originalTreeCid = 3; # cid of the original dataset
# optional uint32 originalDatasetSize = 4; # size of the original dataset
# optional uint32 ecK = 1; # number of encoded blocks
# optional uint32 ecM = 2; # number of parity blocks
# optional bytes originalTreeCid = 3; # cid of the original dataset
# optional uint32 originalDatasetSize = 4; # size of the original dataset
# optional VerificationInformation verification = 5; # verification information
# }
# Message Header {
# optional bytes treeCid = 1; # cid (root) of the tree
@ -63,8 +69,15 @@ proc encode*(_: DagPBCoder, manifest: Manifest): ?!seq[byte] =
erasureInfo.write(2, manifest.ecM.uint32)
erasureInfo.write(3, manifest.originalTreeCid.data.buffer)
erasureInfo.write(4, manifest.originalDatasetSize.uint32)
erasureInfo.finish()
if manifest.verifiable:
var verificationInfo = initProtoBuffer()
verificationInfo.write(1, manifest.verificationRoot.data.buffer)
for slotRoot in manifest.slotRoots:
verificationInfo.write(2, slotRoot.data.buffer)
erasureInfo.write(5, verificationInfo)
erasureInfo.finish()
header.write(4, erasureInfo)
pbNode.write(1, header) # set the treeCid as the data field
@ -80,12 +93,15 @@ proc decode*(_: DagPBCoder, data: openArray[byte]): ?!Manifest =
pbNode = initProtoBuffer(data)
pbHeader: ProtoBuffer
pbErasureInfo: ProtoBuffer
pbVerificationInfo: ProtoBuffer
treeCidBuf: seq[byte]
originalTreeCid: seq[byte]
datasetSize: uint32
blockSize: uint32
originalDatasetSize: uint32
ecK, ecM: uint32
verificationRoot: seq[byte]
slotRoots: seq[seq[byte]]
# Decode `Header` message
if pbNode.getField(1, pbHeader).isErr:
@ -105,6 +121,7 @@ proc decode*(_: DagPBCoder, data: openArray[byte]): ?!Manifest =
return failure("Unable to decode `erasureInfo` from manifest!")
let protected = pbErasureInfo.buffer.len > 0
var verifiable = false
if protected:
if pbErasureInfo.getField(1, ecK).isErr:
return failure("Unable to decode `K` from manifest!")
@ -118,8 +135,18 @@ proc decode*(_: DagPBCoder, data: openArray[byte]): ?!Manifest =
if pbErasureInfo.getField(4, originalDatasetSize).isErr:
return failure("Unable to decode `originalDatasetSize` from manifest!")
if pbErasureInfo.getField(5, pbVerificationInfo).isErr:
return failure("Unable to decode `verificationInfo` from manifest!")
let
verifiable = pbVerificationInfo.buffer.len > 0
if verifiable:
if pbVerificationInfo.getField(1, verificationRoot).isErr:
return failure("Unable to decode `verificationRoot` from manifest!")
if pbVerificationInfo.getRequiredRepeatedField(2, slotRoots).isErr:
return failure("Unable to decode `slotRoots` from manifest!")
let
treeCid = ? Cid.init(treeCidBuf).mapFailure
let
@ -147,6 +174,18 @@ proc decode*(_: DagPBCoder, data: openArray[byte]): ?!Manifest =
)
? self.verify()
if verifiable:
let
verificationRootCid = ? Cid.init(verificationRoot).mapFailure
slotRootCids = slotRoots.mapIt(? Cid.init(it).mapFailure)
return Manifest.new(
manifest = self,
verificationRoot = verificationRootCid,
slotRoots = slotRootCids
)
self.success
proc encode*(

60
codex/manifest/manifest.nim
View File

@ -42,6 +42,12 @@ type
ecM: int # Number of resulting parity blocks
originalTreeCid: Cid # The original root of the dataset being erasure coded
originalDatasetSize: NBytes
case verifiable {.serialize.}: bool # Verifiable datasets can be used to generate storage proofs
of true:
verificationRoot: Cid
slotRoots: seq[Cid]
else:
discard
else:
discard
@ -88,6 +94,15 @@ proc treeCid*(self: Manifest): Cid =
proc blocksCount*(self: Manifest): int =
divUp(self.datasetSize.int, self.blockSize.int)
proc verifiable*(self: Manifest): bool =
self.verifiable
proc verificationRoot*(self: Manifest): Cid =
self.verificationRoot
proc slotRoots*(self: Manifest): seq[Cid] =
self.slotRoots
############################################################
# Operations on block list
############################################################
@ -142,7 +157,13 @@ proc `==`*(a, b: Manifest): bool =
(a.ecK == b.ecK) and
(a.ecM == b.ecM) and
(a.originalTreeCid == b.originalTreeCid) and
(a.originalDatasetSize == b.originalDatasetSize)
(a.originalDatasetSize == b.originalDatasetSize) and
(a.verifiable == b.verifiable) and
(if a.verifiable:
(a.verificationRoot == b.verificationRoot) and
(a.slotRoots == b.slotRoots)
else:
true)
else:
true)
@ -158,7 +179,13 @@ proc `$`*(self: Manifest): string =
", ecK: " & $self.ecK &
", ecM: " & $self.ecM &
", originalTreeCid: " & $self.originalTreeCid &
", originalDatasetSize: " & $self.originalDatasetSize
", originalDatasetSize: " & $self.originalDatasetSize &
", verifiable: " & $self.verifiable &
(if self.verifiable:
", verificationRoot: " & $self.verificationRoot &
", slotRoots: " & $self.slotRoots
else:
"")
else:
"")
@ -259,3 +286,32 @@ proc new*(
originalTreeCid: originalTreeCid,
originalDatasetSize: originalDatasetSize
)
proc new*(
T: type Manifest,
manifest: Manifest,
verificationRoot: Cid,
slotRoots: seq[Cid]
): ?!Manifest =
## Create a verifiable dataset from an
## protected one
##
if not manifest.protected:
return failure newException(CodexError, "Can create verifiable manifest only from protected manifest.")
success(Manifest(
treeCid: manifest.treeCid,
datasetSize: manifest.datasetSize,
version: manifest.version,
codec: manifest.codec,
hcodec: manifest.hcodec,
blockSize: manifest.blockSize,
protected: true,
ecK: manifest.ecK,
ecM: manifest.ecM,
originalTreeCid: manifest.treeCid,
originalDatasetSize: manifest.originalDatasetSize,
verifiable: true,
verificationRoot: verificationRoot,
slotRoots: slotRoots
))

1
codex/manifest/types.nim
View File

@ -10,6 +10,7 @@
# This module defines Manifest and all related types
import std/tables
import std/strutils
import pkg/libp2p
import ../units

1
config.nims
View File

@ -115,6 +115,7 @@ switch("define", "chronicles_sinks=textlines[dynamic],json[dynamic],textlines[dy
# Workaround for assembler incompatibility between constantine and secp256k1
switch("define", "use_asm_syntax_intel=false")
switch("define", "ctt_asm=false")
# begin Nimble config (version 1)
when system.fileExists("nimble.paths"):

42
tests/codex/testmanifest.nim
View File

@ -3,8 +3,6 @@ import std/sequtils
import pkg/chronos
import pkg/questionable/results
import pkg/asynctest
import pkg/stew/byteutils
import pkg/codex/chunker
import pkg/codex/blocktype as bt
import pkg/codex/manifest
@ -13,17 +11,37 @@ import ./helpers
import ./examples
checksuite "Manifest":
test "Should encode/decode to/from manifest":
var
manifest = Manifest.new(
treeCid = Cid.example,
blockSize = 1.MiBs,
datasetSize = 100.MiBs)
let
manifest = Manifest.new(
treeCid = Cid.example,
blockSize = 1.MiBs,
datasetSize = 100.MiBs
)
protectedManifest = Manifest.new(
manifest = manifest,
treeCid = Cid.example,
datasetSize = 200.MiBs,
eck = 10,
ecM = 10
)
verifiableManifest = Manifest.new(
manifest = protectedManifest,
verificationRoot = Cid.example,
slotRoots = @[Cid.example, Cid.example]
).tryGet()
let
e = manifest.encode().tryGet()
decoded = Manifest.decode(e).tryGet()
proc encodeDecode(manifest: Manifest): Manifest =
let e = manifest.encode().tryGet()
Manifest.decode(e).tryGet()
test "Should encode/decode to/from base manifest":
check:
decoded == manifest
encodeDecode(manifest) == manifest
test "Should encode/decode to/from protected manifest":
check:
encodeDecode(protectedManifest) == protectedManifest
test "Should encode/decode to/from verifiable manifest":
check:
encodeDecode(verifiableManifest) == verifiableManifest

2
vendor/constantine vendored

@ -1 +1 @@
Subproject commit 5f7ba18f2ed351260015397c9eae079a6decaee1
Subproject commit 8367d7d19cdbba874aab961b70d272e742184c37

2
vendor/nim-poseidon2 vendored

@ -1 +1 @@
Subproject commit af6737492971dd05b2a6b03404e490f865266f15
Subproject commit 9be7b0c134e64e3d57a38520a32af93a55a37c44