status-im
/
nim-codex
mirror of https://github.com/status-im/nim-codex.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

cleanup manifest and wire in strategy (#696)

This commit is contained in:
Dmitriy Ryajov 2024-02-07 14:54:57 -06:00 committed by GitHub
parent 1e957da109
commit afec86b3cf
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 136 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
codex/erasure/backend.nim
View File

@ -14,15 +14,15 @@ push: {.upraises: [].}
import ../stores import ../stores
type type
Backend* = ref object of RootObj ErasureBackend* = ref object of RootObj
blockSize*: int # block size in bytes blockSize*: int # block size in bytes
buffers*: int # number of original pieces buffers*: int # number of original pieces
parity*: int # number of redundancy pieces parity*: int # number of redundancy pieces
EncoderBackend* = ref object of Backend EncoderBackend* = ref object of ErasureBackend
DecoderBackend* = ref object of Backend DecoderBackend* = ref object of ErasureBackend
method release*(self: Backend) {.base.} = method release*(self: ErasureBackend) {.base.} =
## release the backend ## release the backend
## ##
raiseAssert("not implemented!") raiseAssert("not implemented!")

31
codex/erasure/erasure.nim
View File

@ -77,6 +77,7 @@ type
rounded: Natural rounded: Natural
steps: Natural steps: Natural
blocksCount: Natural blocksCount: Natural
strategy: StrategyType
func indexToPos(steps, idx, step: int): int {.inline.} = func indexToPos(steps, idx, step: int): int {.inline.} =
## Convert an index to a position in the encoded ## Convert an index to a position in the encoded
@ -130,7 +131,7 @@ proc prepareEncodingData(
## ##
let let
strategy = SteppedStrategy.init( strategy = params.strategy.init(
firstIndex = 0, firstIndex = 0,
lastIndex = params.rounded - 1, lastIndex = params.rounded - 1,
iterations = params.steps iterations = params.steps
@ -179,7 +180,7 @@ proc prepareDecodingData(
## ##
let let
strategy = SteppedStrategy.init( strategy = encoded.protectedStrategy.init(
firstIndex = 0, firstIndex = 0,
lastIndex = encoded.blocksCount - 1, lastIndex = encoded.blocksCount - 1,
iterations = encoded.steps iterations = encoded.steps
@ -229,7 +230,8 @@ proc prepareDecodingData(
proc init*( proc init*(
_: type EncodingParams, _: type EncodingParams,
manifest: Manifest, manifest: Manifest,
ecK: Natural, ecM: Natural): ?!EncodingParams = ecK: Natural, ecM: Natural,
strategy: StrategyType): ?!EncodingParams =
if ecK > manifest.blocksCount: if ecK > manifest.blocksCount:
return failure( return failure(
"Unable to encode manifest, not enough blocks, ecK = " & "Unable to encode manifest, not enough blocks, ecK = " &
@ -242,13 +244,14 @@ proc init*(
steps = divUp(manifest.blocksCount, ecK) steps = divUp(manifest.blocksCount, ecK)
blocksCount = rounded + (steps * ecM) blocksCount = rounded + (steps * ecM)
EncodingParams( success EncodingParams(
ecK: ecK, ecK: ecK,
ecM: ecM, ecM: ecM,
rounded: rounded, rounded: rounded,
steps: steps, steps: steps,
blocksCount: blocksCount blocksCount: blocksCount,
).success strategy: strategy
)
proc encodeData( proc encodeData(
self: Erasure, self: Erasure,
@ -327,11 +330,12 @@ proc encodeData(
treeCid = treeCid, treeCid = treeCid,
datasetSize = (manifest.blockSize.int * params.blocksCount).NBytes, datasetSize = (manifest.blockSize.int * params.blocksCount).NBytes,
ecK = params.ecK, ecK = params.ecK,
ecM = params.ecM ecM = params.ecM,
strategy = params.strategy
) )
trace "Encoded data successfully", treeCid, blocksCount = params.blocksCount trace "Encoded data successfully", treeCid, blocksCount = params.blocksCount
return encodedManifest.success success encodedManifest
except CancelledError as exc: except CancelledError as exc:
trace "Erasure coding encoding cancelled" trace "Erasure coding encoding cancelled"
raise exc # cancellation needs to be propagated raise exc # cancellation needs to be propagated
@ -345,7 +349,8 @@ proc encode*(
self: Erasure, self: Erasure,
manifest: Manifest, manifest: Manifest,
blocks: Natural, blocks: Natural,
parity: Natural): Future[?!Manifest] {.async.} = parity: Natural,
strategy = SteppedStrategy): Future[?!Manifest] {.async.} =
## Encode a manifest into one that is erasure protected. ## Encode a manifest into one that is erasure protected.
## ##
## `manifest` - the original manifest to be encoded ## `manifest` - the original manifest to be encoded
@ -353,7 +358,7 @@ proc encode*(
## `parity` - the number of parity blocks to generate - M ## `parity` - the number of parity blocks to generate - M
## ##
without params =? EncodingParams.init(manifest, blocks.int, parity.int), err: without params =? EncodingParams.init(manifest, blocks.int, parity.int, strategy), err:
return failure(err) return failure(err)
without encodedManifest =? await self.encodeData(manifest, params), err: without encodedManifest =? await self.encodeData(manifest, params), err:
@ -363,8 +368,7 @@ proc encode*(
proc decode*( proc decode*(
self: Erasure, self: Erasure,
encoded: Manifest encoded: Manifest): Future[?!Manifest] {.async.} =
): Future[?!Manifest] {.async.} =
## Decode a protected manifest into it's original ## Decode a protected manifest into it's original
## manifest ## manifest
## ##
@ -468,8 +472,7 @@ proc new*(
T: type Erasure, T: type Erasure,
store: BlockStore, store: BlockStore,
encoderProvider: EncoderProvider, encoderProvider: EncoderProvider,
decoderProvider: DecoderProvider decoderProvider: DecoderProvider): Erasure =
): Erasure =
## Create a new Erasure instance for encoding and decoding manifests ## Create a new Erasure instance for encoding and decoding manifests
Erasure( Erasure(

27
codex/manifest/coders.nim
View File

@ -25,6 +25,7 @@ import ./manifest
import ../errors import ../errors
import ../blocktype import ../blocktype
import ../logutils import ../logutils
import ../indexingstrategy
proc encode*(manifest: Manifest): ?!seq[byte] = proc encode*(manifest: Manifest): ?!seq[byte] =
## Encode the manifest into a ``ManifestCodec`` ## Encode the manifest into a ``ManifestCodec``
@ -75,13 +76,16 @@ proc encode*(manifest: Manifest): ?!seq[byte] =
erasureInfo.write(2, manifest.ecM.uint32) erasureInfo.write(2, manifest.ecM.uint32)
erasureInfo.write(3, manifest.originalTreeCid.data.buffer) erasureInfo.write(3, manifest.originalTreeCid.data.buffer)
erasureInfo.write(4, manifest.originalDatasetSize.uint32) erasureInfo.write(4, manifest.originalDatasetSize.uint32)
erasureInfo.write(5, manifest.protectedStrategy.uint32)
if manifest.verifiable: if manifest.verifiable:
var verificationInfo = initProtoBuffer() var verificationInfo = initProtoBuffer()
verificationInfo.write(1, manifest.verifyRoot.data.buffer) verificationInfo.write(1, manifest.verifyRoot.data.buffer)
for slotRoot in manifest.slotRoots: for slotRoot in manifest.slotRoots:
verificationInfo.write(2, slotRoot.data.buffer) verificationInfo.write(2, slotRoot.data.buffer)
erasureInfo.write(5, verificationInfo) verificationInfo.write(3, manifest.cellSize.uint32)
verificationInfo.write(4, manifest.verifiableStrategy.uint32)
erasureInfo.write(6, verificationInfo)
erasureInfo.finish() erasureInfo.finish()
header.write(7, erasureInfo) header.write(7, erasureInfo)
@ -109,8 +113,11 @@ proc decode*(_: type Manifest, data: openArray[byte]): ?!Manifest =
blockSize: uint32 blockSize: uint32
originalDatasetSize: uint32 originalDatasetSize: uint32
ecK, ecM: uint32 ecK, ecM: uint32
protectedStrategy: uint32
verifyRoot: seq[byte] verifyRoot: seq[byte]
slotRoots: seq[seq[byte]] slotRoots: seq[seq[byte]]
cellSize: uint32
verifiableStrategy: uint32
# Decode `Header` message # Decode `Header` message
if pbNode.getField(1, pbHeader).isErr: if pbNode.getField(1, pbHeader).isErr:
@ -153,7 +160,10 @@ proc decode*(_: type Manifest, data: openArray[byte]): ?!Manifest =
if pbErasureInfo.getField(4, originalDatasetSize).isErr: if pbErasureInfo.getField(4, originalDatasetSize).isErr:
return failure("Unable to decode `originalDatasetSize` from manifest!") return failure("Unable to decode `originalDatasetSize` from manifest!")
if pbErasureInfo.getField(5, pbVerificationInfo).isErr: if pbErasureInfo.getField(5, protectedStrategy).isErr:
return failure("Unable to decode `protectedStrategy` from manifest!")
if pbErasureInfo.getField(6, pbVerificationInfo).isErr:
return failure("Unable to decode `verificationInfo` from manifest!") return failure("Unable to decode `verificationInfo` from manifest!")
verifiable = pbVerificationInfo.buffer.len > 0 verifiable = pbVerificationInfo.buffer.len > 0
@ -164,6 +174,12 @@ proc decode*(_: type Manifest, data: openArray[byte]): ?!Manifest =
if pbVerificationInfo.getRequiredRepeatedField(2, slotRoots).isErr: if pbVerificationInfo.getRequiredRepeatedField(2, slotRoots).isErr:
return failure("Unable to decode `slotRoots` from manifest!") return failure("Unable to decode `slotRoots` from manifest!")
if pbVerificationInfo.getField(3, cellSize).isErr:
return failure("Unable to decode `cellSize` from manifest!")
if pbVerificationInfo.getField(4, verifiableStrategy).isErr:
return failure("Unable to decode `verifiableStrategy` from manifest!")
let let
treeCid = ? Cid.init(treeCidBuf).mapFailure treeCid = ? Cid.init(treeCidBuf).mapFailure
@ -179,7 +195,8 @@ proc decode*(_: type Manifest, data: openArray[byte]): ?!Manifest =
ecK = ecK.int, ecK = ecK.int,
ecM = ecM.int, ecM = ecM.int,
originalTreeCid = ? Cid.init(originalTreeCid).mapFailure, originalTreeCid = ? Cid.init(originalTreeCid).mapFailure,
originalDatasetSize = originalDatasetSize.NBytes) originalDatasetSize = originalDatasetSize.NBytes,
strategy = StrategyType(protectedStrategy))
else: else:
Manifest.new( Manifest.new(
treeCid = treeCid, treeCid = treeCid,
@ -199,7 +216,9 @@ proc decode*(_: type Manifest, data: openArray[byte]): ?!Manifest =
return Manifest.new( return Manifest.new(
manifest = self, manifest = self,
verifyRoot = verifyRootCid, verifyRoot = verifyRootCid,
slotRoots = slotRootCids slotRoots = slotRootCids,
cellSize = cellSize.NBytes,
strategy = StrategyType(verifiableStrategy)
) )
self.success self.success

116
codex/manifest/manifest.nim
View File

@ -22,8 +22,12 @@ import ../utils
import ../utils/json import ../utils/json
import ../units import ../units
import ../blocktype import ../blocktype
import ../indexingstrategy
import ../logutils import ../logutils
# TODO: Manifest should be reworked to more concrete types,
# perhaps using inheritance
type type
Manifest* = ref object of RootObj Manifest* = ref object of RootObj
treeCid {.serialize.}: Cid # Root of the merkle tree treeCid {.serialize.}: Cid # Root of the merkle tree
@ -38,10 +42,13 @@ type
ecM: int # Number of resulting parity blocks ecM: int # Number of resulting parity blocks
originalTreeCid: Cid # The original root of the dataset being erasure coded originalTreeCid: Cid # The original root of the dataset being erasure coded
originalDatasetSize: NBytes originalDatasetSize: NBytes
protectedStrategy: StrategyType # Indexing strategy used to build the slot roots
case verifiable {.serialize.}: bool # Verifiable datasets can be used to generate storage proofs case verifiable {.serialize.}: bool # Verifiable datasets can be used to generate storage proofs
of true: of true:
verifyRoot: Cid # Root of the top level merkle tree built from slot roots verifyRoot: Cid # Root of the top level merkle tree built from slot roots
slotRoots: seq[Cid] # Individual slot root built from the original dataset blocks slotRoots: seq[Cid] # Individual slot root built from the original dataset blocks
cellSize: NBytes # Size of each slot cell
verifiableStrategy: StrategyType # Indexing strategy used to build the slot roots
else: else:
discard discard
else: else:
@ -51,60 +58,69 @@ type
# Accessors # Accessors
############################################################ ############################################################
proc blockSize*(self: Manifest): NBytes = func blockSize*(self: Manifest): NBytes =
self.blockSize self.blockSize
proc datasetSize*(self: Manifest): NBytes = func datasetSize*(self: Manifest): NBytes =
self.datasetSize self.datasetSize
proc version*(self: Manifest): CidVersion = func version*(self: Manifest): CidVersion =
self.version self.version
proc hcodec*(self: Manifest): MultiCodec = func hcodec*(self: Manifest): MultiCodec =
self.hcodec self.hcodec
proc codec*(self: Manifest): MultiCodec = func codec*(self: Manifest): MultiCodec =
self.codec self.codec
proc protected*(self: Manifest): bool = func protected*(self: Manifest): bool =
self.protected self.protected
proc ecK*(self: Manifest): int = func ecK*(self: Manifest): int =
self.ecK self.ecK
proc ecM*(self: Manifest): int = func ecM*(self: Manifest): int =
self.ecM self.ecM
proc originalTreeCid*(self: Manifest): Cid = func originalTreeCid*(self: Manifest): Cid =
self.originalTreeCid self.originalTreeCid
proc originalBlocksCount*(self: Manifest): int = func originalBlocksCount*(self: Manifest): int =
divUp(self.originalDatasetSize.int, self.blockSize.int) divUp(self.originalDatasetSize.int, self.blockSize.int)
proc originalDatasetSize*(self: Manifest): NBytes = func originalDatasetSize*(self: Manifest): NBytes =
self.originalDatasetSize self.originalDatasetSize
proc treeCid*(self: Manifest): Cid = func treeCid*(self: Manifest): Cid =
self.treeCid self.treeCid
proc blocksCount*(self: Manifest): int = func blocksCount*(self: Manifest): int =
divUp(self.datasetSize.int, self.blockSize.int) divUp(self.datasetSize.int, self.blockSize.int)
proc verifiable*(self: Manifest): bool = func verifiable*(self: Manifest): bool =
self.verifiable bool (self.protected and self.verifiable)
proc verifyRoot*(self: Manifest): Cid = func verifyRoot*(self: Manifest): Cid =
self.verifyRoot self.verifyRoot
proc slotRoots*(self: Manifest): seq[Cid] = func slotRoots*(self: Manifest): seq[Cid] =
self.slotRoots self.slotRoots
proc numSlots*(self: Manifest): int = func numSlots*(self: Manifest): int =
if not self.protected:
0
else:
self.ecK + self.ecM self.ecK + self.ecM
func cellSize*(self: Manifest): NBytes =
self.cellSize
func protectedStrategy*(self: Manifest): StrategyType =
self.protectedStrategy
func verifiableStrategy*(self: Manifest): StrategyType =
self.verifiableStrategy
func numSlotBlocks*(self: Manifest): int =
divUp(self.blocksCount, self.numSlots)
############################################################ ############################################################
# Operations on block list # Operations on block list
############################################################ ############################################################
@ -143,10 +159,10 @@ func verify*(self: Manifest): ?!void =
return success() return success()
proc cid*(self: Manifest): ?!Cid {.deprecated: "use treeCid instead".} = func cid*(self: Manifest): ?!Cid {.deprecated: "use treeCid instead".} =
self.treeCid.success self.treeCid.success
proc `==`*(a, b: Manifest): bool = func `==`*(a, b: Manifest): bool =
(a.treeCid == b.treeCid) and (a.treeCid == b.treeCid) and
(a.datasetSize == b.datasetSize) and (a.datasetSize == b.datasetSize) and
(a.blockSize == b.blockSize) and (a.blockSize == b.blockSize) and
@ -159,16 +175,19 @@ proc `==`*(a, b: Manifest): bool =
(a.ecM == b.ecM) and (a.ecM == b.ecM) and
(a.originalTreeCid == b.originalTreeCid) and (a.originalTreeCid == b.originalTreeCid) and
(a.originalDatasetSize == b.originalDatasetSize) and (a.originalDatasetSize == b.originalDatasetSize) and
(a.protectedStrategy == b.protectedStrategy) and
(a.verifiable == b.verifiable) and (a.verifiable == b.verifiable) and
(if a.verifiable: (if a.verifiable:
(a.verifyRoot == b.verifyRoot) and (a.verifyRoot == b.verifyRoot) and
(a.slotRoots == b.slotRoots) (a.slotRoots == b.slotRoots) and
(a.cellSize == b.cellSize) and
(a.verifiableStrategy == b.verifiableStrategy)
else: else:
true) true)
else: else:
true) true)
proc `$`*(self: Manifest): string = func `$`*(self: Manifest): string =
"treeCid: " & $self.treeCid & "treeCid: " & $self.treeCid &
", datasetSize: " & $self.datasetSize & ", datasetSize: " & $self.datasetSize &
", blockSize: " & $self.blockSize & ", blockSize: " & $self.blockSize &
@ -194,7 +213,7 @@ proc `$`*(self: Manifest): string =
# Constructors # Constructors
############################################################ ############################################################
proc new*( func new*(
T: type Manifest, T: type Manifest,
treeCid: Cid, treeCid: Cid,
blockSize: NBytes, blockSize: NBytes,
@ -213,12 +232,13 @@ proc new*(
hcodec: hcodec, hcodec: hcodec,
protected: protected) protected: protected)
proc new*( func new*(
T: type Manifest, T: type Manifest,
manifest: Manifest, manifest: Manifest,
treeCid: Cid, treeCid: Cid,
datasetSize: NBytes, datasetSize: NBytes,
ecK, ecM: int): Manifest = ecK, ecM: int,
strategy: StrategyType): Manifest =
## Create an erasure protected dataset from an ## Create an erasure protected dataset from an
## unprotected one ## unprotected one
## ##
@ -233,9 +253,10 @@ proc new*(
protected: true, protected: true,
ecK: ecK, ecM: ecM, ecK: ecK, ecM: ecM,
originalTreeCid: manifest.treeCid, originalTreeCid: manifest.treeCid,
originalDatasetSize: manifest.datasetSize) originalDatasetSize: manifest.datasetSize,
protectedStrategy: strategy)
proc new*( func new*(
T: type Manifest, T: type Manifest,
manifest: Manifest): Manifest = manifest: Manifest): Manifest =
## Create an unprotected dataset from an ## Create an unprotected dataset from an
@ -251,15 +272,7 @@ proc new*(
blockSize: manifest.blockSize, blockSize: manifest.blockSize,
protected: false) protected: false)
proc new*( func new*(
T: type Manifest,
data: openArray[byte]): ?!Manifest =
## Create a manifest instance from given data
##
Manifest.decode(data)
proc new*(
T: type Manifest, T: type Manifest,
treeCid: Cid, treeCid: Cid,
datasetSize: NBytes, datasetSize: NBytes,
@ -270,7 +283,8 @@ proc new*(
ecK: int, ecK: int,
ecM: int, ecM: int,
originalTreeCid: Cid, originalTreeCid: Cid,
originalDatasetSize: NBytes): Manifest = originalDatasetSize: NBytes,
strategy: StrategyType): Manifest =
Manifest( Manifest(
treeCid: treeCid, treeCid: treeCid,
@ -283,14 +297,16 @@ proc new*(
ecK: ecK, ecK: ecK,
ecM: ecM, ecM: ecM,
originalTreeCid: originalTreeCid, originalTreeCid: originalTreeCid,
originalDatasetSize: originalDatasetSize originalDatasetSize: originalDatasetSize,
) protectedStrategy: strategy)
proc new*( func new*(
T: type Manifest, T: type Manifest,
manifest: Manifest, manifest: Manifest,
verifyRoot: Cid, verifyRoot: Cid,
slotRoots: openArray[Cid]): ?!Manifest = slotRoots: openArray[Cid],
cellSize = DefaultCellSize,
strategy = SteppedStrategy): ?!Manifest =
## Create a verifiable dataset from an ## Create a verifiable dataset from an
## protected one ## protected one
## ##
@ -317,4 +333,14 @@ proc new*(
originalDatasetSize: manifest.originalDatasetSize, originalDatasetSize: manifest.originalDatasetSize,
verifiable: true, verifiable: true,
verifyRoot: verifyRoot, verifyRoot: verifyRoot,
slotRoots: @slotRoots) slotRoots: @slotRoots,
cellSize: cellSize,
verifiableStrategy: strategy)
func new*(
T: type Manifest,
data: openArray[byte]): ?!Manifest =
## Create a manifest instance from given data
##
Manifest.decode(data)

3
tests/codex/slots/provingtestenv.nim
View File

@ -142,7 +142,8 @@ proc createManifest(self: ProvingTestEnvironment): Future[void] {.async.} =
treeCid = treeCid, treeCid = treeCid,
datasetSize = self.manifest.datasetSize, datasetSize = self.manifest.datasetSize,
ecK = totalNumberOfSlots, ecK = totalNumberOfSlots,
ecM = 0 ecM = 0,
strategy = StrategyType.SteppedStrategy
) )
# Verifiable manifest: # Verifiable manifest:

9
tests/codex/slots/testslotbuilder.nim
View File

@ -100,7 +100,8 @@ suite "Slot builder":
treeCid = protectedTreeCid, treeCid = protectedTreeCid,
datasetSize = totalDatasetSize.NBytes, datasetSize = totalDatasetSize.NBytes,
ecK = ecK, ecK = ecK,
ecM = ecM) ecM = ecM,
strategy = StrategyType.SteppedStrategy)
let let
manifestBlock = bt.Block.new( manifestBlock = bt.Block.new(
@ -166,7 +167,8 @@ suite "Slot builder":
treeCid = Cid.example, treeCid = Cid.example,
datasetSize = totalDatasetSize.NBytes, datasetSize = totalDatasetSize.NBytes,
ecK = ecK - 1, ecK = ecK - 1,
ecM = ecM) ecM = ecM,
strategy = StrategyType.SteppedStrategy)
check: check:
SlotsBuilder.new(localStore, mismatchManifest, cellSize = cellSize) SlotsBuilder.new(localStore, mismatchManifest, cellSize = cellSize)
@ -182,7 +184,8 @@ suite "Slot builder":
treeCid = Cid.example, treeCid = Cid.example,
datasetSize = (totalDatasetSize - 1).NBytes, datasetSize = (totalDatasetSize - 1).NBytes,
ecK = ecK, ecK = ecK,
ecM = ecM) ecM = ecM,
strategy = StrategyType.SteppedStrategy)
check: check:
SlotsBuilder.new(localStore, mismatchManifest, cellSize = cellSize) SlotsBuilder.new(localStore, mismatchManifest, cellSize = cellSize)

1
tests/codex/testerasure.nim
View File

@ -11,6 +11,7 @@ import pkg/codex/stores
import pkg/codex/blocktype as bt import pkg/codex/blocktype as bt
import pkg/codex/rng import pkg/codex/rng
import pkg/codex/utils import pkg/codex/utils
import pkg/codex/indexingstrategy
import ../asynctest import ../asynctest
import ./helpers import ./helpers

4
tests/codex/testmanifest.nim
View File

@ -9,6 +9,7 @@ import pkg/poseidon2
import pkg/codex/slots import pkg/codex/slots
import pkg/codex/merkletree import pkg/codex/merkletree
import pkg/codex/indexingstrategy
import ../asynctest import ../asynctest
import ./helpers import ./helpers
@ -27,7 +28,8 @@ checksuite "Manifest":
treeCid = Cid.example, treeCid = Cid.example,
datasetSize = 200.MiBs, datasetSize = 200.MiBs,
eck = 2, eck = 2,
ecM = 2 ecM = 2,
strategy = SteppedStrategy
) )
leaves = [ leaves = [