mirror of
https://github.com/logos-storage/logos-storage-nim.git
synced 2026-05-02 09:33:23 +00:00
2D slot encoding
Part of https://github.com/logos-storage/logos-storage-nim/issues/896 Signed-off-by: Chrysostomos Nanakos <chris@include.gr>
This commit is contained in:
Chrysostomos Nanakos
parent
7eb2fb12cc
commit
617d2982a3
@ -305,9 +305,11 @@ proc new*(
|
||||
store = NetworkStore.new(engine, repoStore)
|
||||
prover =
|
||||
if config.prover:
|
||||
let backend =
|
||||
config.initializeBackend().expect("Unable to create prover backend.")
|
||||
some Prover.new(store, backend, config.numProofSamples)
|
||||
let
|
||||
backend =
|
||||
config.initializeBackend().expect("Unable to create prover backend.")
|
||||
erasure = Erasure.new(store, leoEncoderProvider, leoDecoderProvider, taskpool)
|
||||
some Prover.new(store, erasure, backend, config.numProofSamples)
|
||||
else:
|
||||
none Prover
|
||||
|
||||
|
||||
@ -12,7 +12,7 @@ import pkg/upraises
|
||||
push:
|
||||
{.upraises: [].}
|
||||
|
||||
import std/[sugar, atomics, sequtils]
|
||||
import std/[sugar, atomics, sequtils, math, algorithm]
|
||||
|
||||
import pkg/chronos
|
||||
import pkg/chronos/threadsync
|
||||
@ -29,6 +29,7 @@ import ../clock
|
||||
import ../blocktype as bt
|
||||
import ../utils
|
||||
import ../utils/asynciter
|
||||
import ../utils/encoding2d
|
||||
import ../indexingstrategy
|
||||
import ../errors
|
||||
import ../utils/arrayutils
|
||||
@ -85,6 +86,13 @@ type
|
||||
blocksCount: Natural
|
||||
strategy: StrategyType
|
||||
|
||||
EncodingParams2D = object
|
||||
ecK: Natural
|
||||
ecM: Natural
|
||||
numSlotBlocks: Natural
|
||||
protectedNumSlotBlocks: Natural
|
||||
strategy: StrategyType
|
||||
|
||||
ErasureError* = object of CodexError
|
||||
InsufficientBlocksError* = object of ErasureError
|
||||
# Minimum size, in bytes, that the dataset must have had
|
||||
@ -120,8 +128,86 @@ func indexToPos(steps, idx, step: int): int {.inline.} =
|
||||
|
||||
(idx - step) div steps
|
||||
|
||||
proc init*(
|
||||
_: type EncodingParams,
|
||||
manifest: Manifest,
|
||||
ecK: Natural,
|
||||
ecM: Natural,
|
||||
strategy: StrategyType,
|
||||
): ?!EncodingParams =
|
||||
if ecK > manifest.blocksCount:
|
||||
let exc = (ref InsufficientBlocksError)(
|
||||
msg:
|
||||
"Unable to encode manifest, not enough blocks, ecK = " & $ecK &
|
||||
", blocksCount = " & $manifest.blocksCount,
|
||||
minSize: ecK.NBytes * manifest.blockSize,
|
||||
)
|
||||
return failure(exc)
|
||||
|
||||
let
|
||||
rounded = roundUp(manifest.blocksCount, ecK)
|
||||
steps = divUp(rounded, ecK)
|
||||
blocksCount = rounded + (steps * ecM)
|
||||
|
||||
success EncodingParams(
|
||||
ecK: ecK,
|
||||
ecM: ecM,
|
||||
rounded: rounded,
|
||||
steps: steps,
|
||||
blocksCount: blocksCount,
|
||||
strategy: strategy,
|
||||
)
|
||||
|
||||
proc init(
|
||||
_: type EncodingParams2D, manifest: Manifest, strategy: StrategyType
|
||||
): ?!EncodingParams2D =
|
||||
if not manifest.protected:
|
||||
return failure("2D encoding requires a protected manifest")
|
||||
|
||||
let
|
||||
numSlotBlocks = manifest.numSlotBlocks
|
||||
(ecK, ecM) = ?encoding2d.calculate2DErasureParams(numSlotBlocks.uint64)
|
||||
|
||||
success EncodingParams2D(
|
||||
ecK: ecK,
|
||||
ecM: ecM,
|
||||
numSlotBlocks: (ecK + ecM) * (ecK + ecM),
|
||||
protectedNumSlotBlocks: numSlotBlocks.Natural,
|
||||
strategy: strategy,
|
||||
)
|
||||
|
||||
proc calculate2DIndices*(
|
||||
self: EncodingParams2D, manifest: Manifest, slotIndex: int
|
||||
): ?!Iter[int] =
|
||||
without slotIndicesIter =?
|
||||
self.strategy
|
||||
.init(0, manifest.blocksCount - 1, manifest.numSlots)
|
||||
.getIndices(slotIndex).catch, err:
|
||||
return failure(err)
|
||||
|
||||
let nextAvailableIndex =
|
||||
if slotIndex == 0:
|
||||
manifest.blocksCount
|
||||
else:
|
||||
let additionalIndicesPerSlot = self.numSlotBlocks - self.protectedNumSlotBlocks
|
||||
manifest.blocksCount + (slotIndex * additionalIndicesPerSlot)
|
||||
|
||||
let
|
||||
additionalIndices = self.numSlotBlocks - self.protectedNumSlotBlocks
|
||||
combinedIter = Iter[int].new(
|
||||
iterator (): int {.gcsafe.} =
|
||||
for idx in slotIndicesIter:
|
||||
yield idx
|
||||
|
||||
for i in 0 ..< additionalIndices:
|
||||
yield nextAvailableIndex + i
|
||||
|
||||
)
|
||||
|
||||
success combinedIter
|
||||
|
||||
proc getPendingBlocks(
|
||||
self: Erasure, manifest: Manifest, indices: seq[int]
|
||||
self: Erasure, manifest: Manifest, indicesIter: Iter[int]
|
||||
): AsyncIter[(?!bt.Block, int)] =
|
||||
## Get pending blocks iterator
|
||||
##
|
||||
@ -133,7 +219,7 @@ proc getPendingBlocks(
|
||||
## avoids closure capture issues
|
||||
return (await fut, i)
|
||||
|
||||
for blockIndex in indices:
|
||||
for blockIndex in indicesIter:
|
||||
# request blocks from the store
|
||||
let fut = self.store.getBlock(BlockAddress.init(manifest.treeCid, blockIndex))
|
||||
pendingBlocks.add(attachIndex(fut, blockIndex))
|
||||
@ -168,13 +254,22 @@ proc prepareEncodingData(
|
||||
## Prepare data for encoding
|
||||
##
|
||||
|
||||
let strategy = params.strategy.init(
|
||||
firstIndex = 0, lastIndex = params.rounded - 1, iterations = params.steps
|
||||
)
|
||||
|
||||
without indicesIter =? strategy.getIndices(step).catch, err:
|
||||
return failure(err)
|
||||
|
||||
let
|
||||
strategy = params.strategy.init(
|
||||
firstIndex = 0, lastIndex = params.rounded - 1, iterations = params.steps
|
||||
indices = Iter[int].new(
|
||||
iterator (): int {.gcsafe.} =
|
||||
for idx in indicesIter:
|
||||
if idx < manifest.blocksCount:
|
||||
yield idx
|
||||
|
||||
)
|
||||
indices = toSeq(strategy.getIndices(step))
|
||||
pendingBlocksIter =
|
||||
self.getPendingBlocks(manifest, indices.filterIt(it < manifest.blocksCount))
|
||||
pendingBlocksIter = self.getPendingBlocks(manifest, indices)
|
||||
|
||||
var resolved = 0
|
||||
for fut in pendingBlocksIter:
|
||||
@ -189,14 +284,19 @@ proc prepareEncodingData(
|
||||
|
||||
resolved.inc()
|
||||
|
||||
for idx in indices.filterIt(it >= manifest.blocksCount):
|
||||
let pos = indexToPos(params.steps, idx, step)
|
||||
trace "Padding with empty block", idx
|
||||
shallowCopy(data[pos], emptyBlock)
|
||||
without emptyBlockCid =? emptyCid(manifest.version, manifest.hcodec, manifest.codec),
|
||||
err:
|
||||
return failure(err)
|
||||
cids[idx] = emptyBlockCid
|
||||
without paddingIndicesIter =? strategy.getIndices(step).catch, err:
|
||||
return failure(err)
|
||||
|
||||
without emptyBlockCid =? emptyCid(manifest.version, manifest.hcodec, manifest.codec),
|
||||
err:
|
||||
return failure(err)
|
||||
|
||||
for idx in paddingIndicesIter:
|
||||
if idx >= manifest.blocksCount:
|
||||
let pos = indexToPos(params.steps, idx, step)
|
||||
trace "Padding with empty block", idx
|
||||
shallowCopy(data[pos], emptyBlock)
|
||||
cids[idx] = emptyBlockCid
|
||||
|
||||
success(resolved.Natural)
|
||||
|
||||
@ -222,7 +322,7 @@ proc prepareDecodingData(
|
||||
strategy = encoded.protectedStrategy.init(
|
||||
firstIndex = 0, lastIndex = encoded.blocksCount - 1, iterations = encoded.steps
|
||||
)
|
||||
indices = toSeq(strategy.getIndices(step))
|
||||
indices = strategy.getIndices(step)
|
||||
pendingBlocksIter = self.getPendingBlocks(encoded, indices)
|
||||
|
||||
var
|
||||
@ -265,36 +365,6 @@ proc prepareDecodingData(
|
||||
|
||||
return success (dataPieces.Natural, parityPieces.Natural)
|
||||
|
||||
proc init*(
|
||||
_: type EncodingParams,
|
||||
manifest: Manifest,
|
||||
ecK: Natural,
|
||||
ecM: Natural,
|
||||
strategy: StrategyType,
|
||||
): ?!EncodingParams =
|
||||
if ecK > manifest.blocksCount:
|
||||
let exc = (ref InsufficientBlocksError)(
|
||||
msg:
|
||||
"Unable to encode manifest, not enough blocks, ecK = " & $ecK &
|
||||
", blocksCount = " & $manifest.blocksCount,
|
||||
minSize: ecK.NBytes * manifest.blockSize,
|
||||
)
|
||||
return failure(exc)
|
||||
|
||||
let
|
||||
rounded = roundUp(manifest.blocksCount, ecK)
|
||||
steps = divUp(rounded, ecK)
|
||||
blocksCount = rounded + (steps * ecM)
|
||||
|
||||
success EncodingParams(
|
||||
ecK: ecK,
|
||||
ecM: ecM,
|
||||
rounded: rounded,
|
||||
steps: steps,
|
||||
blocksCount: blocksCount,
|
||||
strategy: strategy,
|
||||
)
|
||||
|
||||
proc leopardEncodeTask(tp: Taskpool, task: ptr EncodeTask) {.gcsafe.} =
|
||||
# Task suitable for running in taskpools - look, no GC!
|
||||
let encoder =
|
||||
@ -477,6 +547,364 @@ proc encode*(
|
||||
|
||||
return success encodedManifest
|
||||
|
||||
proc prepare2DRowData(
|
||||
self: Erasure,
|
||||
manifest: Manifest,
|
||||
rowIndices: seq[int],
|
||||
rowIndex: int,
|
||||
params: EncodingParams2D,
|
||||
rowData: ref seq[seq[byte]],
|
||||
cids: ref seq[Cid],
|
||||
emptyBlock: seq[byte],
|
||||
slotIndices: seq[int],
|
||||
): Future[?!Natural] {.async.} =
|
||||
var resolved = 0
|
||||
|
||||
let
|
||||
rowIndicesIter = Iter[int].new(
|
||||
iterator (): int {.gcsafe.} =
|
||||
for idx in rowIndices:
|
||||
if idx < manifest.blocksCount:
|
||||
yield idx
|
||||
|
||||
)
|
||||
pendingBlocksIter = self.getPendingBlocks(manifest, rowIndicesIter)
|
||||
|
||||
for fut in pendingBlocksIter:
|
||||
let (blkOrErr, idx) = await fut
|
||||
without blk =? blkOrErr, err:
|
||||
warn "Failed retrieving a block for 2D encoding row",
|
||||
treeCid = manifest.treeCid, idx, rowIndex, msg = err.msg
|
||||
return failure(err)
|
||||
|
||||
let colPos = rowIndices.find(idx)
|
||||
if colPos >= 0:
|
||||
shallowCopy(rowData[colPos], if blk.isEmpty: emptyBlock else: blk.data)
|
||||
|
||||
let slotPos = slotIndices.find(idx)
|
||||
if slotPos >= 0 and slotPos < cids[].len:
|
||||
cids[slotPos] = blk.cid
|
||||
resolved.inc()
|
||||
|
||||
for col in resolved ..< params.ecK:
|
||||
shallowCopy(rowData[col], emptyBlock)
|
||||
|
||||
success(resolved.Natural)
|
||||
|
||||
proc encode2DRows(
|
||||
self: Erasure,
|
||||
manifest: Manifest,
|
||||
params: EncodingParams2D,
|
||||
slotIndex: int,
|
||||
cids: ref seq[Cid],
|
||||
emptyBlock: seq[byte],
|
||||
): Future[?!void] {.async.} =
|
||||
without allIndicesIter =? params.calculate2DIndices(manifest, slotIndex), err:
|
||||
return failure("Failed to calculate 2D indices: " & err.msg)
|
||||
|
||||
var
|
||||
slotIndices: seq[int] = @[]
|
||||
additionalIndices: seq[int] = @[]
|
||||
indexCount = 0
|
||||
|
||||
for idx in allIndicesIter:
|
||||
if indexCount < params.protectedNumSlotBlocks:
|
||||
slotIndices.add(idx)
|
||||
else:
|
||||
additionalIndices.add(idx)
|
||||
indexCount.inc()
|
||||
|
||||
without emptyBlockCid =? emptyCid(manifest.version, manifest.hcodec, manifest.codec),
|
||||
err:
|
||||
return failure(err)
|
||||
|
||||
let
|
||||
dataPositionsNeeded = params.ecK * params.ecK
|
||||
paddingBlockCount = max(0, dataPositionsNeeded - params.protectedNumSlotBlocks)
|
||||
|
||||
for i in 0 ..< paddingBlockCount:
|
||||
if i < additionalIndices.len:
|
||||
let cidPos = slotIndices.len + i
|
||||
if cidPos < cids[].len:
|
||||
cids[cidPos] = emptyBlockCid
|
||||
|
||||
var nextParityPos = slotIndices.len + paddingBlockCount
|
||||
|
||||
for rowIdx in 0 ..< params.ecK:
|
||||
var
|
||||
rowData = seq[seq[byte]].new()
|
||||
rowParity = createDoubleArray(params.ecM, manifest.blockSize.int)
|
||||
defer:
|
||||
freeDoubleArray(rowParity, params.ecM)
|
||||
|
||||
rowData[].setLen(params.ecK)
|
||||
|
||||
without slotIndicesIter =?
|
||||
params.strategy
|
||||
.init(0, manifest.blocksCount - 1, manifest.numSlots)
|
||||
.getIndices(slotIndex).catch, err:
|
||||
trace "Unable to get slot indices for row", rowIdx, error = err.msg
|
||||
return failure(err)
|
||||
|
||||
var
|
||||
rowIndices: seq[int] = @[]
|
||||
currentIdx = 0
|
||||
|
||||
for slotIdx in slotIndicesIter:
|
||||
let dataIdx = currentIdx
|
||||
if dataIdx >= rowIdx * params.ecK and dataIdx < (rowIdx + 1) * params.ecK:
|
||||
rowIndices.add(slotIdx)
|
||||
currentIdx.inc
|
||||
if currentIdx >= params.protectedNumSlotBlocks:
|
||||
break
|
||||
|
||||
await sleepAsync(10.millis)
|
||||
|
||||
without resolved =? (
|
||||
await self.prepare2DRowData(
|
||||
manifest, rowIndices, rowIdx, params, rowData, cids, emptyBlock, slotIndices
|
||||
)
|
||||
), err:
|
||||
trace "Unable to prepare row data for 2D encoding", rowIdx, error = err.msg
|
||||
return failure(err)
|
||||
|
||||
if err =? (
|
||||
await self.asyncEncode(
|
||||
manifest.blockSize.int, params.ecK, params.ecM, rowData, rowParity
|
||||
)
|
||||
).errorOption:
|
||||
return failure(err)
|
||||
|
||||
for parityCol in 0 ..< params.ecM:
|
||||
var innerPtr: ptr UncheckedArray[byte] = rowParity[][parityCol]
|
||||
|
||||
without blk =? bt.Block.new(innerPtr.toOpenArray(0, manifest.blockSize.int - 1)),
|
||||
error:
|
||||
trace "Unable to create row parity block", rowIdx, parityCol, err = error.msg
|
||||
return failure(error)
|
||||
|
||||
if nextParityPos < cids[].len:
|
||||
cids[nextParityPos] = blk.cid
|
||||
nextParityPos.inc()
|
||||
|
||||
if error =? (await self.store.putBlock(blk)).errorOption:
|
||||
warn "Unable to store row parity block!", cid = blk.cid, msg = error.msg
|
||||
return failure("Unable to store row parity block!")
|
||||
|
||||
success()
|
||||
|
||||
proc prepare2DColumnData(
|
||||
self: Erasure,
|
||||
manifest: Manifest,
|
||||
colIndex: int,
|
||||
params: EncodingParams2D,
|
||||
colData: ref seq[seq[byte]],
|
||||
cids: ref seq[Cid],
|
||||
emptyBlock: seq[byte],
|
||||
columnIndices: seq[int],
|
||||
slotIndices: seq[int],
|
||||
): Future[?!Natural] {.async.} =
|
||||
var resolved = 0
|
||||
|
||||
if colIndex < params.ecK:
|
||||
let columnIndicesIter = Iter[int].new(
|
||||
iterator (): int {.gcsafe.} =
|
||||
for idx in columnIndices:
|
||||
if idx < manifest.blocksCount:
|
||||
yield idx
|
||||
|
||||
)
|
||||
|
||||
let pendingBlocksIter = self.getPendingBlocks(manifest, columnIndicesIter)
|
||||
var colPosition = 0
|
||||
|
||||
for fut in pendingBlocksIter:
|
||||
let (blkOrErr, idx) = await fut
|
||||
without blk =? blkOrErr, err:
|
||||
return failure("Failed retrieving data block for column encoding: " & err.msg)
|
||||
|
||||
if colPosition < params.ecK:
|
||||
shallowCopy(colData[colPosition], if blk.isEmpty: emptyBlock else: blk.data)
|
||||
resolved.inc()
|
||||
colPosition.inc()
|
||||
|
||||
for row in colPosition ..< params.ecK:
|
||||
shallowCopy(colData[row], emptyBlock)
|
||||
else:
|
||||
for row in 0 ..< params.ecK:
|
||||
let
|
||||
parityCol = colIndex - params.ecK
|
||||
dataPositionsNeeded = params.ecK * params.ecK
|
||||
paddingCount = max(0, dataPositionsNeeded - params.protectedNumSlotBlocks)
|
||||
cidPos = slotIndices.len + paddingCount + (row * params.ecM + parityCol)
|
||||
|
||||
if cidPos >= 0 and cidPos < cids[].len and not cids[cidPos].isEmpty:
|
||||
without blk =? await self.store.getBlock(BlockAddress.init(cids[cidPos])), err:
|
||||
return failure(
|
||||
"Failed retrieving row parity block for column encoding: " & err.msg
|
||||
)
|
||||
|
||||
shallowCopy(colData[row], if blk.isEmpty: emptyBlock else: blk.data)
|
||||
resolved.inc()
|
||||
else:
|
||||
return failure("Row parity block CID not found at position " & $cidPos)
|
||||
|
||||
success(resolved.Natural)
|
||||
|
||||
proc encode2DColumns(
|
||||
self: Erasure,
|
||||
manifest: Manifest,
|
||||
params: EncodingParams2D,
|
||||
slotIndex: int,
|
||||
cids: ref seq[Cid],
|
||||
emptyBlock: seq[byte],
|
||||
): Future[?!void] {.async.} =
|
||||
without allIndicesIter =? params.calculate2DIndices(manifest, slotIndex), err:
|
||||
return failure("Failed to calculate 2D indices: " & err.msg)
|
||||
|
||||
var
|
||||
slotIndices: seq[int] = @[]
|
||||
additionalIndices: seq[int] = @[]
|
||||
indexCount = 0
|
||||
|
||||
for idx in allIndicesIter:
|
||||
if indexCount < params.protectedNumSlotBlocks:
|
||||
slotIndices.add(idx)
|
||||
else:
|
||||
additionalIndices.add(idx)
|
||||
indexCount.inc()
|
||||
|
||||
let
|
||||
dataPositionsNeeded = params.ecK * params.ecK
|
||||
paddingCount = max(0, dataPositionsNeeded - params.protectedNumSlotBlocks)
|
||||
rowParityCount = params.ecK * params.ecM
|
||||
var nextColParityPos = slotIndices.len + paddingCount + rowParityCount
|
||||
|
||||
for colIdx in 0 ..< (params.ecK + params.ecM):
|
||||
var
|
||||
colData = seq[seq[byte]].new()
|
||||
colParity = createDoubleArray(params.ecM, manifest.blockSize.int)
|
||||
|
||||
defer:
|
||||
freeDoubleArray(colParity, params.ecM)
|
||||
|
||||
colData[].setLen(params.ecK)
|
||||
|
||||
without slotIndicesIter =?
|
||||
params.strategy
|
||||
.init(0, manifest.blocksCount - 1, manifest.numSlots)
|
||||
.getIndices(slotIndex).catch, err:
|
||||
trace "Unable to get slot indices for column", colIdx, error = err.msg
|
||||
return failure(err)
|
||||
|
||||
var
|
||||
columnIndices: seq[int] = @[]
|
||||
currentIdx = 0
|
||||
for slotIdx in slotIndicesIter:
|
||||
let
|
||||
row = currentIdx div params.ecK
|
||||
col = currentIdx mod params.ecK
|
||||
if col == colIdx and row < params.ecK:
|
||||
columnIndices.add(slotIdx)
|
||||
currentIdx.inc
|
||||
if currentIdx >= params.protectedNumSlotBlocks:
|
||||
break
|
||||
|
||||
await sleepAsync(10.millis)
|
||||
|
||||
without resolved =? (
|
||||
await self.prepare2DColumnData(
|
||||
manifest, colIdx, params, colData, cids, emptyBlock, columnIndices, slotIndices
|
||||
)
|
||||
), err:
|
||||
trace "Unable to prepare column data for 2D encoding", colIdx, error = err.msg
|
||||
return failure(err)
|
||||
|
||||
if err =? (
|
||||
await self.asyncEncode(
|
||||
manifest.blockSize.int, params.ecK, params.ecM, colData, colParity
|
||||
)
|
||||
).errorOption:
|
||||
return failure(err)
|
||||
|
||||
for parityRow in 0 ..< params.ecM:
|
||||
var innerPtr: ptr UncheckedArray[byte] = colParity[][parityRow]
|
||||
|
||||
without blk =? bt.Block.new(innerPtr.toOpenArray(0, manifest.blockSize.int - 1)),
|
||||
error:
|
||||
trace "Unable to create column parity block", colIdx, parityRow, err = error.msg
|
||||
return failure(error)
|
||||
|
||||
if nextColParityPos < cids[].len:
|
||||
cids[nextColParityPos] = blk.cid
|
||||
nextColParityPos.inc()
|
||||
|
||||
if error =? (await self.store.putBlock(blk)).errorOption:
|
||||
warn "Unable to store column parity block!", cid = blk.cid, msg = error.msg
|
||||
return failure("Unable to store column parity block!")
|
||||
|
||||
success()
|
||||
|
||||
proc encode2DSlot*(
|
||||
self: Erasure, manifest: Manifest, strategy: StrategyType, slotIndex: int
|
||||
): Future[?!Cid] {.async.} =
|
||||
if not manifest.protected:
|
||||
return failure("2D encoding requires a protected manifest")
|
||||
|
||||
without params =? EncodingParams2D.init(manifest, strategy), err:
|
||||
return failure(err)
|
||||
|
||||
logScope:
|
||||
slotIndex = slotIndex
|
||||
ecK = params.ecK
|
||||
ecM = params.ecM
|
||||
|
||||
trace "Starting 2D erasure encoding for slot"
|
||||
|
||||
var
|
||||
cids = seq[Cid].new()
|
||||
emptyBlock = newSeq[byte](manifest.blockSize.int)
|
||||
|
||||
cids[].setLen(params.numSlotBlocks)
|
||||
|
||||
try:
|
||||
if err =?
|
||||
(await self.encode2DRows(manifest, params, slotIndex, cids, emptyBlock)).errorOption:
|
||||
trace "Row encoding failed", error = err.msg
|
||||
return failure(err)
|
||||
|
||||
trace "Row encoding completed for slot"
|
||||
|
||||
if err =? (
|
||||
await self.encode2DColumns(manifest, params, slotIndex, cids, emptyBlock)
|
||||
).errorOption:
|
||||
trace "Column encoding failed", error = err.msg
|
||||
return failure(err)
|
||||
|
||||
trace "Column encoding completed for slot"
|
||||
|
||||
without tree =? CodexTree.init(cids[]), err:
|
||||
return failure(err)
|
||||
|
||||
without treeCid =? tree.rootCid, err:
|
||||
return failure(err)
|
||||
|
||||
if err =? (await self.store.putAllProofs(tree)).errorOption:
|
||||
return failure(err)
|
||||
|
||||
trace "2D erasure encoding completed successfully for slot",
|
||||
treeCid,
|
||||
totalBlocks = params.numSlotBlocks,
|
||||
protectedNumSlotBlocks = params.protectedNumSlotBlocks
|
||||
|
||||
success treeCid
|
||||
except CancelledError as exc:
|
||||
trace "2D erasure coding encoding cancelled for slot"
|
||||
raise exc
|
||||
except CatchableError as exc:
|
||||
trace "2D erasure coding encoding error for slot", exc = exc.msg
|
||||
return failure(exc)
|
||||
|
||||
proc leopardDecodeTask(tp: Taskpool, task: ptr DecodeTask) {.gcsafe.} =
|
||||
# Task suitable for running in taskpools - look, no GC!
|
||||
let decoder =
|
||||
|
||||
@ -90,6 +90,8 @@ proc encode*(manifest: Manifest): ?!seq[byte] =
|
||||
verificationInfo.write(2, slotRoot.data.buffer)
|
||||
verificationInfo.write(3, manifest.cellSize.uint32)
|
||||
verificationInfo.write(4, manifest.verifiableStrategy.uint32)
|
||||
for slotEncodedTreeCid in manifest.slotEncodedTreeCids:
|
||||
verificationInfo.write(5, slotEncodedTreeCid.data.buffer)
|
||||
erasureInfo.write(6, verificationInfo)
|
||||
|
||||
erasureInfo.finish()
|
||||
@ -127,6 +129,7 @@ proc decode*(_: type Manifest, data: openArray[byte]): ?!Manifest =
|
||||
protectedStrategy: uint32
|
||||
verifyRoot: seq[byte]
|
||||
slotRoots: seq[seq[byte]]
|
||||
slotEncodedTreeCids: seq[seq[byte]]
|
||||
cellSize: uint32
|
||||
verifiableStrategy: uint32
|
||||
filename: string
|
||||
@ -199,6 +202,9 @@ proc decode*(_: type Manifest, data: openArray[byte]): ?!Manifest =
|
||||
if pbVerificationInfo.getField(4, verifiableStrategy).isErr:
|
||||
return failure("Unable to decode `verifiableStrategy` from manifest!")
|
||||
|
||||
if pbVerificationInfo.getRepeatedField(5, slotEncodedTreeCids).isErr:
|
||||
slotEncodedTreeCids = @[]
|
||||
|
||||
let treeCid = ?Cid.init(treeCidBuf).mapFailure
|
||||
|
||||
var filenameOption = if filename.len == 0: string.none else: filename.some
|
||||
@ -239,11 +245,17 @@ proc decode*(_: type Manifest, data: openArray[byte]): ?!Manifest =
|
||||
let
|
||||
verifyRootCid = ?Cid.init(verifyRoot).mapFailure
|
||||
slotRootCids = slotRoots.mapIt(?Cid.init(it).mapFailure)
|
||||
slotEncodedTreeCidsSeq =
|
||||
if slotEncodedTreeCids.len > 0:
|
||||
slotEncodedTreeCids.mapIt(?Cid.init(it).mapFailure)
|
||||
else:
|
||||
newSeq[Cid](slotRootCids.len)
|
||||
|
||||
return Manifest.new(
|
||||
manifest = self,
|
||||
verifyRoot = verifyRootCid,
|
||||
slotRoots = slotRootCids,
|
||||
slotEncodedTreeCids = slotEncodedTreeCidsSeq,
|
||||
cellSize = cellSize.NBytes,
|
||||
strategy = StrategyType(verifiableStrategy),
|
||||
)
|
||||
|
||||
@ -52,6 +52,7 @@ type Manifest* = ref object of RootObj
|
||||
slotRoots: seq[Cid] # Individual slot root built from the original dataset blocks
|
||||
cellSize: NBytes # Size of each slot cell
|
||||
verifiableStrategy: StrategyType # Indexing strategy used to build the slot roots
|
||||
slotEncodedTreeCids: seq[Cid]
|
||||
else:
|
||||
discard
|
||||
else:
|
||||
@ -109,6 +110,9 @@ func verifyRoot*(self: Manifest): Cid =
|
||||
func slotRoots*(self: Manifest): seq[Cid] =
|
||||
self.slotRoots
|
||||
|
||||
func slotEncodedTreeCids*(self: Manifest): seq[Cid] =
|
||||
self.slotEncodedTreeCids
|
||||
|
||||
func numSlots*(self: Manifest): int =
|
||||
self.ecK + self.ecM
|
||||
|
||||
@ -324,6 +328,7 @@ func new*(
|
||||
manifest: Manifest,
|
||||
verifyRoot: Cid,
|
||||
slotRoots: openArray[Cid],
|
||||
slotEncodedTreeCids: openArray[Cid],
|
||||
cellSize = DefaultCellSize,
|
||||
strategy = LinearStrategy,
|
||||
): ?!Manifest =
|
||||
@ -339,6 +344,9 @@ func new*(
|
||||
if slotRoots.len != manifest.numSlots:
|
||||
return failure newException(CodexError, "Wrong number of slot roots.")
|
||||
|
||||
if slotEncodedTreeCids.len != manifest.numSlots:
|
||||
return failure newException(CodexError, "Wrong number of 2D tree CIDs.")
|
||||
|
||||
success Manifest(
|
||||
treeCid: manifest.treeCid,
|
||||
datasetSize: manifest.datasetSize,
|
||||
@ -355,6 +363,7 @@ func new*(
|
||||
verifiable: true,
|
||||
verifyRoot: verifyRoot,
|
||||
slotRoots: @slotRoots,
|
||||
slotEncodedTreeCids: @slotEncodedTreeCids,
|
||||
cellSize: cellSize,
|
||||
verifiableStrategy: strategy,
|
||||
filename: manifest.filename,
|
||||
|
||||
@ -540,7 +540,9 @@ proc setupRequest(
|
||||
trace "Unable to erasure code dataset"
|
||||
return failure(error)
|
||||
|
||||
without builder =? Poseidon2Builder.new(self.networkStore.localStore, encoded), err:
|
||||
without builder =? Poseidon2Builder.new(
|
||||
self.networkStore.localStore, encoded, erasure
|
||||
), err:
|
||||
trace "Unable to create slot builder"
|
||||
return failure(err)
|
||||
|
||||
@ -643,8 +645,14 @@ proc onStore(
|
||||
trace "Unable to fetch manifest for cid", cid, err = err.msg
|
||||
return failure(err)
|
||||
|
||||
let erasure = Erasure.new(
|
||||
self.networkStore, leoEncoderProvider, leoDecoderProvider, self.taskpool
|
||||
)
|
||||
|
||||
without builder =?
|
||||
Poseidon2Builder.new(self.networkStore, manifest, manifest.verifiableStrategy), err:
|
||||
Poseidon2Builder.new(
|
||||
self.networkStore, manifest, erasure, manifest.verifiableStrategy
|
||||
), err:
|
||||
trace "Unable to create slots builder", err = err.msg
|
||||
return failure(err)
|
||||
|
||||
@ -678,9 +686,6 @@ proc onStore(
|
||||
if isRepairing:
|
||||
trace "start repairing slot", slotIdx
|
||||
try:
|
||||
let erasure = Erasure.new(
|
||||
self.networkStore, leoEncoderProvider, leoDecoderProvider, self.taskpool
|
||||
)
|
||||
if err =? (await erasure.repair(manifest)).errorOption:
|
||||
error "Unable to erasure decode repairing manifest",
|
||||
cid = manifest.treeCid, exc = err.msg
|
||||
|
||||
@ -12,6 +12,7 @@
|
||||
import std/math
|
||||
import std/sequtils
|
||||
import std/sugar
|
||||
import std/[random, sets]
|
||||
|
||||
import pkg/libp2p
|
||||
import pkg/chronos
|
||||
@ -21,8 +22,10 @@ import pkg/constantine/math/io/io_fields
|
||||
|
||||
import ../../logutils
|
||||
import ../../utils
|
||||
import ../../utils/encoding2d
|
||||
import ../../stores
|
||||
import ../../manifest
|
||||
import ../../erasure
|
||||
import ../../merkletree
|
||||
import ../../utils/asynciter
|
||||
import ../../indexingstrategy
|
||||
@ -37,7 +40,8 @@ logScope:
|
||||
type SlotsBuilder*[T, H] = ref object of RootObj
|
||||
store: BlockStore
|
||||
manifest: Manifest # current manifest
|
||||
strategy: IndexingStrategy # indexing strategy
|
||||
erasure: Erasure
|
||||
strategy: StrategyType # indexing strategy
|
||||
cellSize: NBytes # cell size
|
||||
numSlotBlocks: Natural
|
||||
# number of blocks per slot (should yield a power of two number of cells)
|
||||
@ -45,6 +49,8 @@ type SlotsBuilder*[T, H] = ref object of RootObj
|
||||
emptyBlock: seq[byte] # empty block
|
||||
verifiableTree: ?T # verification tree (dataset tree)
|
||||
emptyDigestTree: T # empty digest tree for empty blocks
|
||||
numSlotBlocksEncoded: Natural # number of blocks per slot after slot encoding (2D)
|
||||
slotEncodedTreeCids: seq[Cid] # Encoded tree CIDs
|
||||
|
||||
func verifiable*[T, H](self: SlotsBuilder[T, H]): bool {.inline.} =
|
||||
## Returns true if the slots are verifiable.
|
||||
@ -113,27 +119,38 @@ func numSlotCells*[T, H](self: SlotsBuilder[T, H]): Natural =
|
||||
|
||||
self.numBlockCells * self.numSlotBlocks
|
||||
|
||||
func slotIndicesIter*[T, H](self: SlotsBuilder[T, H], slot: Natural): ?!Iter[int] =
|
||||
## Returns the slot indices.
|
||||
##
|
||||
|
||||
self.strategy.getIndices(slot).catch
|
||||
|
||||
func slotIndices*[T, H](self: SlotsBuilder[T, H], slot: Natural): seq[int] =
|
||||
## Returns the slot indices.
|
||||
##
|
||||
|
||||
if iter =? self.strategy.getIndices(slot).catch:
|
||||
return toSeq(iter)
|
||||
|
||||
func manifest*[T, H](self: SlotsBuilder[T, H]): Manifest =
|
||||
## Returns the manifest.
|
||||
##
|
||||
|
||||
self.manifest
|
||||
|
||||
func numSlotBlocksEncoded*[T, H](self: SlotsBuilder[T, H]): Natural =
|
||||
## Number of blocks per slot for encoded (2D) slots.
|
||||
##
|
||||
|
||||
self.numSlotBlocksEncoded
|
||||
|
||||
func numSlotCellsEncoded*[T, H](self: SlotsBuilder[T, H]): Natural =
|
||||
## Number of cells per slot for encoded (2D) slots.
|
||||
##
|
||||
|
||||
(self.numSlotBlocksEncoded * self.numBlockCells).Natural
|
||||
|
||||
func numSlotBlocksPadded*[T, H](self: SlotsBuilder[T, H]): Natural =
|
||||
## Returns the padded number of blocks (power of two) for tree construction
|
||||
##
|
||||
|
||||
nextPowerOfTwo(self.numSlotBlocksEncoded)
|
||||
|
||||
func isEmptyBlockIndex*[T, H](self: SlotsBuilder[T, H], blkIdx: Natural): bool =
|
||||
## Returns true if this block index should be filled with empty blocks
|
||||
##
|
||||
|
||||
blkIdx >= self.numSlotBlocksEncoded
|
||||
|
||||
proc buildBlockTree*[T, H](
|
||||
self: SlotsBuilder[T, H], blkIdx: Natural, slotPos: Natural
|
||||
self: SlotsBuilder[T, H], treeCid: Cid, blkIdx: Natural
|
||||
): Future[?!(seq[byte], T)] {.async: (raises: [CancelledError]).} =
|
||||
## Build the block digest tree and return a tuple with the
|
||||
## block data and the tree.
|
||||
@ -141,18 +158,15 @@ proc buildBlockTree*[T, H](
|
||||
|
||||
logScope:
|
||||
blkIdx = blkIdx
|
||||
slotPos = slotPos
|
||||
numSlotBlocks = self.manifest.numSlotBlocks
|
||||
cellSize = self.cellSize
|
||||
|
||||
trace "Building block tree"
|
||||
|
||||
if slotPos > (self.manifest.numSlotBlocks - 1):
|
||||
# pad blocks are 0 byte blocks
|
||||
trace "Returning empty digest tree for pad block"
|
||||
if self.isEmptyBlockIndex(blkIdx):
|
||||
return success (self.emptyBlock, self.emptyDigestTree)
|
||||
|
||||
without blk =? await self.store.getBlock(self.manifest.treeCid, blkIdx), err:
|
||||
without blk =? await self.store.getBlock(treeCid, blkIdx), err:
|
||||
error "Failed to get block CID for tree at index", err = err.msg
|
||||
return failure(err)
|
||||
|
||||
@ -167,11 +181,10 @@ proc buildBlockTree*[T, H](
|
||||
|
||||
proc getCellHashes*[T, H](
|
||||
self: SlotsBuilder[T, H], slotIndex: Natural
|
||||
): Future[?!seq[H]] {.async: (raises: [CancelledError, IndexingError]).} =
|
||||
## Collect all the cells from a block and return
|
||||
): Future[?!seq[H]] {.async: (raises: [CancelledError]).} =
|
||||
## Collect all the cells from a 2D encoded slot and return
|
||||
## their hashes.
|
||||
##
|
||||
|
||||
let
|
||||
treeCid = self.manifest.treeCid
|
||||
blockCount = self.manifest.blocksCount
|
||||
@ -183,21 +196,31 @@ proc getCellHashes*[T, H](
|
||||
numberOfSlots = numberOfSlots
|
||||
slotIndex = slotIndex
|
||||
|
||||
trace "Starting 2D encoding for slot to get cell hashes"
|
||||
|
||||
without encoded2DTreeCid =?
|
||||
?(await self.erasure.encode2DSlot(self.manifest, self.strategy, slotIndex)).catch,
|
||||
err:
|
||||
error "Failed to 2D encode slot for cell hashes", err = err.msg
|
||||
return failure(err)
|
||||
|
||||
trace "2D encoding completed, collecting cell hashes from encoded blocks"
|
||||
|
||||
let hashes = collect(newSeq):
|
||||
for i, blkIdx in self.strategy.getIndices(slotIndex):
|
||||
for blkIdx in 0 ..< self.numSlotBlocksPadded:
|
||||
logScope:
|
||||
blkIdx = blkIdx
|
||||
pos = i
|
||||
|
||||
trace "Getting block CID for tree at index"
|
||||
without (_, tree) =? (await self.buildBlockTree(blkIdx, i)) and digest =? tree.root,
|
||||
err:
|
||||
trace "Getting 2D encoded block for cell hash"
|
||||
without (_, tree) =? (await self.buildBlockTree(encoded2DTreeCid, blkIdx)) and
|
||||
digest =? tree.root, err:
|
||||
error "Failed to get block CID for tree at index", err = err.msg
|
||||
return failure(err)
|
||||
|
||||
trace "Get block digest", digest = digest.toHex
|
||||
digest
|
||||
|
||||
self.slotEncodedTreeCids[slotIndex] = encoded2DTreeCid
|
||||
success hashes
|
||||
|
||||
proc buildSlotTree*[T, H](
|
||||
@ -303,13 +326,15 @@ proc buildManifest*[T, H](
|
||||
return failure(err)
|
||||
|
||||
Manifest.new(
|
||||
self.manifest, rootProvingCid, rootCids, self.cellSize, self.strategy.strategyType
|
||||
self.manifest, rootProvingCid, rootCids, self.slotEncodedTreeCids, self.cellSize,
|
||||
self.strategy,
|
||||
)
|
||||
|
||||
proc new*[T, H](
|
||||
_: type SlotsBuilder[T, H],
|
||||
store: BlockStore,
|
||||
manifest: Manifest,
|
||||
erasure: Erasure,
|
||||
strategy = LinearStrategy,
|
||||
cellSize = DefaultCellSize,
|
||||
): ?!SlotsBuilder[T, H] =
|
||||
@ -317,6 +342,9 @@ proc new*[T, H](
|
||||
trace "Manifest is not protected."
|
||||
return failure("Manifest is not protected.")
|
||||
|
||||
if strategy != LinearStrategy:
|
||||
return failure("strategy is not linear.")
|
||||
|
||||
logScope:
|
||||
blockSize = manifest.blockSize
|
||||
strategy = strategy
|
||||
@ -333,56 +361,32 @@ proc new*[T, H](
|
||||
trace msg
|
||||
return failure(msg)
|
||||
|
||||
without numSlotBlocksEncoded =? encoding2d.calculate2DSlotBlocks(manifest), err:
|
||||
return failure(err)
|
||||
|
||||
let
|
||||
numSlotBlocks = manifest.numSlotBlocks
|
||||
numBlockCells = (manifest.blockSize div cellSize).int # number of cells per block
|
||||
numSlotCells = manifest.numSlotBlocks * numBlockCells
|
||||
# number of uncorrected slot cells
|
||||
pow2SlotCells = nextPowerOfTwo(numSlotCells) # pow2 cells per slot
|
||||
numPadSlotBlocks = (pow2SlotCells div numBlockCells) - numSlotBlocks
|
||||
# pow2 blocks per slot
|
||||
|
||||
numSlotBlocksTotal =
|
||||
# pad blocks per slot
|
||||
if numPadSlotBlocks > 0:
|
||||
numPadSlotBlocks + numSlotBlocks
|
||||
else:
|
||||
numSlotBlocks
|
||||
|
||||
numBlocksTotal = numSlotBlocksTotal * manifest.numSlots # number of blocks per slot
|
||||
|
||||
emptyBlock = newSeq[byte](manifest.blockSize.int)
|
||||
emptyDigestTree = ?T.digestTree(emptyBlock, cellSize.int)
|
||||
|
||||
strategy =
|
||||
?strategy.init(
|
||||
0,
|
||||
manifest.blocksCount - 1,
|
||||
manifest.numSlots,
|
||||
manifest.numSlots,
|
||||
numPadSlotBlocks,
|
||||
).catch
|
||||
|
||||
logScope:
|
||||
numSlotBlocks = numSlotBlocks
|
||||
numBlockCells = numBlockCells
|
||||
numSlotCells = numSlotCells
|
||||
pow2SlotCells = pow2SlotCells
|
||||
numPadSlotBlocks = numPadSlotBlocks
|
||||
numBlocksTotal = numBlocksTotal
|
||||
numSlotBlocksTotal = numSlotBlocksTotal
|
||||
strategy = strategy.strategyType
|
||||
numSlotBlocksEncoded = numSlotBlocksEncoded
|
||||
strategy = strategy
|
||||
|
||||
trace "Creating slots builder"
|
||||
|
||||
var self = SlotsBuilder[T, H](
|
||||
store: store,
|
||||
manifest: manifest,
|
||||
erasure: erasure,
|
||||
strategy: strategy,
|
||||
cellSize: cellSize,
|
||||
emptyBlock: emptyBlock,
|
||||
numSlotBlocks: numSlotBlocksTotal,
|
||||
numSlotBlocks: numSlotBlocks,
|
||||
numSlotBlocksEncoded: numSlotBlocksEncoded,
|
||||
emptyDigestTree: emptyDigestTree,
|
||||
slotEncodedTreeCids: newSeq[Cid](manifest.numSlots),
|
||||
)
|
||||
|
||||
if manifest.verifiable:
|
||||
@ -400,5 +404,6 @@ proc new*[T, H](
|
||||
|
||||
self.slotRoots = slotRoots
|
||||
self.verifiableTree = some tree
|
||||
self.slotEncodedTreeCids = manifest.slotEncodedTreeCids
|
||||
|
||||
success self
|
||||
|
||||
@ -17,6 +17,7 @@ import pkg/questionable/results
|
||||
import pkg/libp2p/cid
|
||||
|
||||
import ../../manifest
|
||||
import ../../erasure
|
||||
import ../../merkletree
|
||||
import ../../stores
|
||||
import ../../market
|
||||
@ -46,6 +47,7 @@ type
|
||||
Prover* = ref object of RootObj
|
||||
backend: AnyBackend
|
||||
store: BlockStore
|
||||
erasure: Erasure
|
||||
nSamples: int
|
||||
|
||||
proc prove*(
|
||||
@ -61,7 +63,7 @@ proc prove*(
|
||||
|
||||
trace "Received proof challenge"
|
||||
|
||||
without builder =? AnyBuilder.new(self.store, manifest), err:
|
||||
without builder =? AnyBuilder.new(self.store, manifest, self.erasure), err:
|
||||
error "Unable to create slots builder", err = err.msg
|
||||
return failure(err)
|
||||
|
||||
@ -88,6 +90,10 @@ proc verify*(
|
||||
self.backend.verify(proof, inputs)
|
||||
|
||||
proc new*(
|
||||
_: type Prover, store: BlockStore, backend: AnyBackend, nSamples: int
|
||||
_: type Prover,
|
||||
store: BlockStore,
|
||||
erasure: Erasure,
|
||||
backend: AnyBackend,
|
||||
nSamples: int,
|
||||
): Prover =
|
||||
Prover(store: store, backend: backend, nSamples: nSamples)
|
||||
Prover(store: store, backend: backend, nSamples: nSamples, erasure: erasure)
|
||||
|
||||
@ -7,7 +7,7 @@
|
||||
## This file may not be copied, modified, or distributed except according to
|
||||
## those terms.
|
||||
|
||||
import std/sugar
|
||||
import std/[sugar, math]
|
||||
|
||||
import pkg/chronos
|
||||
import pkg/questionable
|
||||
@ -53,14 +53,11 @@ proc getSample*[T, H](
|
||||
cellsPerBlock = self.builder.numBlockCells
|
||||
blkCellIdx = cellIdx.toCellInBlk(cellsPerBlock) # block cell index
|
||||
blkSlotIdx = cellIdx.toBlkInSlot(cellsPerBlock) # slot tree index
|
||||
origBlockIdx = self.builder.slotIndices(self.index)[blkSlotIdx]
|
||||
# convert to original dataset block index
|
||||
|
||||
logScope:
|
||||
cellIdx = cellIdx
|
||||
blkSlotIdx = blkSlotIdx
|
||||
blkCellIdx = blkCellIdx
|
||||
origBlockIdx = origBlockIdx
|
||||
|
||||
trace "Retrieving sample from block tree"
|
||||
let
|
||||
@ -70,7 +67,11 @@ proc getSample*[T, H](
|
||||
slotProof = proof.toVerifiableProof().valueOr:
|
||||
return failure("Failed to get verifiable proof")
|
||||
|
||||
(bytes, blkTree) = (await self.builder.buildBlockTree(origBlockIdx, blkSlotIdx)).valueOr:
|
||||
slotEncodedTreeCid = self.builder.manifest.slotEncodedTreeCids[self.index]
|
||||
|
||||
(bytes, blkTree) = (
|
||||
await self.builder.buildBlockTree(slotEncodedTreeCid, blkSlotIdx)
|
||||
).valueOr:
|
||||
return failure("Failed to build block tree")
|
||||
|
||||
cellData = self.getCell(bytes, blkCellIdx)
|
||||
@ -100,23 +101,30 @@ proc getProofInput*[T, H](
|
||||
|
||||
slotTreeCid = self.builder.manifest.slotRoots[self.index]
|
||||
slotRoot = self.builder.slotRoots[self.index]
|
||||
cellIdxs = entropy.cellIndices(slotRoot, self.builder.numSlotCells, nSamples)
|
||||
numSlotCellsEncoded = self.builder.numSlotCellsEncoded
|
||||
nCellsPerSlotEncoded = nextPowerOfTwo(numSlotCellsEncoded)
|
||||
cellIdxs = entropy.cellIndices(slotRoot, nCellsPerSlotEncoded, nSamples)
|
||||
|
||||
logScope:
|
||||
cells = cellIdxs
|
||||
|
||||
trace "Collecting input for proof"
|
||||
let samples = collect(newSeq):
|
||||
for cellIdx in cellIdxs:
|
||||
(await self.getSample(cellIdx, slotTreeCid, slotRoot)).valueOr:
|
||||
return failure("Failed to get sample")
|
||||
let sampleFutures = cellIdxs.mapIt(self.getSample(it, slotTreeCid, slotRoot))
|
||||
|
||||
var samples: seq[Sample[H]] = @[]
|
||||
for i in 0 ..< sampleFutures.len:
|
||||
let res = await sampleFutures[i]
|
||||
if res.isErr:
|
||||
return
|
||||
failure("Failed to get sample for cell " & $cellIdxs[i] & ": " & res.error.msg)
|
||||
samples.add(res.get())
|
||||
|
||||
success ProofInputs[H](
|
||||
entropy: entropy,
|
||||
datasetRoot: datasetRoot,
|
||||
slotProof: slotProof.path,
|
||||
nSlotsPerDataSet: self.builder.numSlots,
|
||||
nCellsPerSlot: self.builder.numSlotCells,
|
||||
nCellsPerSlot: nCellsPerSlotEncoded,
|
||||
slotRoot: slotRoot,
|
||||
slotIndex: self.index,
|
||||
samples: samples,
|
||||
|
||||
42
codex/utils/encoding2d.nim
Normal file
42
codex/utils/encoding2d.nim
Normal file
@ -0,0 +1,42 @@
|
||||
## Nim-Codex
|
||||
## Copyright (c) 2025 Status Research & Development GmbH
|
||||
## Licensed under either of
|
||||
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
|
||||
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
|
||||
## at your option.
|
||||
## This file may not be copied, modified, or distributed except according to
|
||||
## those terms.
|
||||
import pkg/questionable/results
|
||||
|
||||
import std/math
|
||||
|
||||
import ../manifest
|
||||
|
||||
const maxBlocks = 2147483648.uint64 # 2^31 - maximum for 2D grid
|
||||
|
||||
func calculate2DErasureParams*(numBlocks: uint64): ?!(int, int) =
|
||||
## Calculate K and M for 2D erasure coding for ~2x output with GF(2^16) constraint
|
||||
if numBlocks > maxBlocks:
|
||||
return failure(
|
||||
"Dataset too large: " & $numBlocks & " blocks exceeds maximum of " & $maxBlocks &
|
||||
" blocks for 2D erasure coding"
|
||||
)
|
||||
|
||||
let
|
||||
totalDim = min(round(sqrt(2.0 * numBlocks.float)).int, 65536)
|
||||
minK = ceil(sqrt(numBlocks.float)).int
|
||||
k = max(minK, round(totalDim.float / sqrt(2.0)).int)
|
||||
m = max(1, min(totalDim - k, 65536 - k))
|
||||
|
||||
success (k, m)
|
||||
|
||||
func calculate2DSlotBlocks*(manifest: Manifest): ?!Natural =
|
||||
## Calculate number of blocks per slot for 2D encoding
|
||||
if not manifest.protected:
|
||||
return failure("2D encoding requires a protected manifest")
|
||||
|
||||
let numSlotBlocks = manifest.numSlotBlocks
|
||||
without (ecK, ecM) =? calculate2DErasureParams(numSlotBlocks.uint64), err:
|
||||
return failure(err)
|
||||
|
||||
success ((ecK + ecM) * (ecK + ecM)).Natural
|
||||
@ -78,6 +78,7 @@ template setupAndTearDown*() {.dirty.} =
|
||||
pendingBlocks: PendingBlocksManager
|
||||
discovery: DiscoveryEngine
|
||||
advertiser: Advertiser
|
||||
taskpool: Taskpool
|
||||
|
||||
let
|
||||
path = currentSourcePath().parentDir
|
||||
@ -85,6 +86,7 @@ template setupAndTearDown*() {.dirty.} =
|
||||
metaTmp = TempLevelDb.new()
|
||||
|
||||
setup:
|
||||
taskpool = Taskpool.new()
|
||||
file = open(path /../ "" /../ "fixtures" / "test.jpg")
|
||||
chunker = FileChunker.new(file = file, chunkSize = DefaultBlockSize)
|
||||
switch = newStandardSwitch()
|
||||
@ -119,7 +121,7 @@ template setupAndTearDown*() {.dirty.} =
|
||||
engine = engine,
|
||||
prover = Prover.none,
|
||||
discovery = blockDiscovery,
|
||||
taskpool = Taskpool.new(),
|
||||
taskpool = taskpool,
|
||||
)
|
||||
|
||||
teardown:
|
||||
@ -127,3 +129,5 @@ template setupAndTearDown*() {.dirty.} =
|
||||
await node.stop()
|
||||
await metaTmp.destroyDb()
|
||||
await repoTmp.destroyDb()
|
||||
if not taskpool.isNil:
|
||||
taskpool.shutdown()
|
||||
|
||||
@ -75,14 +75,14 @@ asyncchecksuite "Test Node - Host contracts":
|
||||
let
|
||||
manifestBlock =
|
||||
bt.Block.new(manifest.encode().tryGet(), codec = ManifestCodec).tryGet()
|
||||
erasure = Erasure.new(store, leoEncoderProvider, leoDecoderProvider, Taskpool.new)
|
||||
erasure = Erasure.new(store, leoEncoderProvider, leoDecoderProvider, taskpool)
|
||||
|
||||
manifestCid = manifestBlock.cid
|
||||
|
||||
(await localStore.putBlock(manifestBlock)).tryGet()
|
||||
|
||||
protected = (await erasure.encode(manifest, 3, 2)).tryGet()
|
||||
builder = Poseidon2Builder.new(localStore, protected).tryGet()
|
||||
builder = Poseidon2Builder.new(localStore, protected, erasure).tryGet()
|
||||
verifiable = (await builder.buildManifest()).tryGet()
|
||||
verifiableBlock =
|
||||
bt.Block.new(verifiable.encode().tryGet(), codec = ManifestCodec).tryGet()
|
||||
|
||||
@ -180,7 +180,7 @@ asyncchecksuite "Test Node - Basic":
|
||||
manifestBlock =
|
||||
bt.Block.new(manifest.encode().tryGet(), codec = ManifestCodec).tryGet()
|
||||
protected = (await erasure.encode(manifest, 3, 2)).tryGet()
|
||||
builder = Poseidon2Builder.new(localStore, protected).tryGet()
|
||||
builder = Poseidon2Builder.new(localStore, protected, erasure).tryGet()
|
||||
verifiable = (await builder.buildManifest()).tryGet()
|
||||
verifiableBlock =
|
||||
bt.Block.new(verifiable.encode().tryGet(), codec = ManifestCodec).tryGet()
|
||||
|
||||
@ -100,7 +100,7 @@ asyncchecksuite "Test Node - Slot Repair":
|
||||
(await localStore.putBlock(manifestBlock)).tryGet()
|
||||
|
||||
protected = (await erasure.encode(manifest, ecK, ecM)).tryGet()
|
||||
builder = Poseidon2Builder.new(localStore, protected).tryGet()
|
||||
builder = Poseidon2Builder.new(localStore, protected, erasure).tryGet()
|
||||
verifiable = (await builder.buildManifest()).tryGet()
|
||||
verifiableBlock =
|
||||
bt.Block.new(verifiable.encode().tryGet(), codec = ManifestCodec).tryGet()
|
||||
@ -179,7 +179,7 @@ asyncchecksuite "Test Node - Slot Repair":
|
||||
(await localStore.putBlock(manifestBlock)).tryGet()
|
||||
|
||||
protected = (await erasure.encode(manifest, ecK, ecM)).tryGet()
|
||||
builder = Poseidon2Builder.new(localStore, protected).tryGet()
|
||||
builder = Poseidon2Builder.new(localStore, protected, erasure).tryGet()
|
||||
verifiable = (await builder.buildManifest()).tryGet()
|
||||
verifiableBlock =
|
||||
bt.Block.new(verifiable.encode().tryGet(), codec = ManifestCodec).tryGet()
|
||||
|
||||
@ -3,6 +3,7 @@ import std/options
|
||||
import ../../../asynctest
|
||||
|
||||
import pkg/chronos
|
||||
import pkg/taskpools
|
||||
import pkg/poseidon2
|
||||
import pkg/serde/json
|
||||
|
||||
@ -12,6 +13,7 @@ import pkg/codex/merkletree
|
||||
import pkg/codex/codextypes
|
||||
import pkg/codex/manifest
|
||||
import pkg/codex/stores
|
||||
import pkg/codex/erasure
|
||||
|
||||
import ./helpers
|
||||
import ../helpers
|
||||
@ -77,6 +79,8 @@ suite "Test Circom Compat Backend":
|
||||
challenge: array[32, byte]
|
||||
builder: Poseidon2Builder
|
||||
sampler: Poseidon2Sampler
|
||||
taskpool: Taskpool
|
||||
erasure: Erasure
|
||||
|
||||
setup:
|
||||
let
|
||||
@ -84,12 +88,14 @@ suite "Test Circom Compat Backend":
|
||||
metaDs = metaTmp.newDb()
|
||||
|
||||
store = RepoStore.new(repoDs, metaDs)
|
||||
taskpool = Taskpool.new()
|
||||
erasure = Erasure.new(store, leoEncoderProvider, leoDecoderProvider, taskpool)
|
||||
|
||||
(manifest, protected, verifiable) = await createVerifiableManifest(
|
||||
store, numDatasetBlocks, ecK, ecM, blockSize, cellSize
|
||||
store, numDatasetBlocks, ecK, ecM, blockSize, cellSize, erasure
|
||||
)
|
||||
|
||||
builder = Poseidon2Builder.new(store, verifiable).tryGet
|
||||
builder = Poseidon2Builder.new(store, verifiable, erasure).tryGet
|
||||
sampler = Poseidon2Sampler.new(slotId, store, builder).tryGet
|
||||
|
||||
circom = CircomCompat.init(r1cs, wasm, zkey)
|
||||
@ -102,6 +108,11 @@ suite "Test Circom Compat Backend":
|
||||
await repoTmp.destroyDb()
|
||||
await metaTmp.destroyDb()
|
||||
|
||||
if not taskpool.isNil:
|
||||
taskpool.shutdown()
|
||||
|
||||
reset(taskpool)
|
||||
|
||||
test "Should verify with correct input":
|
||||
var proof = circom.prove(proofInputs).tryGet
|
||||
|
||||
|
||||
@ -1,11 +1,13 @@
|
||||
import std/sugar
|
||||
|
||||
import pkg/chronos
|
||||
import pkg/taskpools
|
||||
import pkg/libp2p/cid
|
||||
|
||||
import pkg/codex/codextypes
|
||||
import pkg/codex/stores
|
||||
import pkg/codex/merkletree
|
||||
import pkg/codex/erasure
|
||||
import pkg/codex/manifest
|
||||
import pkg/codex/blocktype as bt
|
||||
import pkg/codex/chunker
|
||||
@ -145,6 +147,7 @@ proc createVerifiableManifest*(
|
||||
ecM: int,
|
||||
blockSize: NBytes,
|
||||
cellSize: NBytes,
|
||||
erasure: Erasure,
|
||||
): Future[tuple[manifest: Manifest, protected: Manifest, verifiable: Manifest]] {.
|
||||
async
|
||||
.} =
|
||||
@ -165,7 +168,10 @@ proc createVerifiableManifest*(
|
||||
totalDatasetSize,
|
||||
)
|
||||
|
||||
builder = Poseidon2Builder.new(store, protectedManifest, cellSize = cellSize).tryGet
|
||||
let
|
||||
builder = Poseidon2Builder.new(
|
||||
store, protectedManifest, erasure, cellSize = cellSize
|
||||
).tryGet
|
||||
verifiableManifest = (await builder.buildManifest()).tryGet
|
||||
|
||||
# build the slots and manifest
|
||||
|
||||
@ -1,11 +1,12 @@
|
||||
import std/sequtils
|
||||
import std/options
|
||||
import std/[sequtils, options, math]
|
||||
|
||||
import ../../../asynctest
|
||||
|
||||
import pkg/taskpools
|
||||
import pkg/questionable/results
|
||||
|
||||
import pkg/codex/stores
|
||||
import pkg/codex/erasure
|
||||
import pkg/codex/merkletree
|
||||
import pkg/codex/utils/json
|
||||
import pkg/codex/codextypes
|
||||
@ -87,6 +88,8 @@ suite "Test Sampler":
|
||||
manifest: Manifest
|
||||
protected: Manifest
|
||||
verifiable: Manifest
|
||||
taskpool: Taskpool
|
||||
erasure: Erasure
|
||||
|
||||
setup:
|
||||
let
|
||||
@ -94,19 +97,26 @@ suite "Test Sampler":
|
||||
metaDs = metaTmp.newDb()
|
||||
|
||||
store = RepoStore.new(repoDs, metaDs)
|
||||
taskpool = Taskpool.new()
|
||||
erasure = Erasure.new(store, leoEncoderProvider, leoDecoderProvider, taskpool)
|
||||
|
||||
(manifest, protected, verifiable) = await createVerifiableManifest(
|
||||
store, datasetBlocks, ecK, ecM, blockSize, cellSize
|
||||
store, datasetBlocks, ecK, ecM, blockSize, cellSize, erasure
|
||||
)
|
||||
|
||||
# create sampler
|
||||
builder = Poseidon2Builder.new(store, verifiable).tryGet
|
||||
builder = Poseidon2Builder.new(store, verifiable, erasure).tryGet
|
||||
|
||||
teardown:
|
||||
await store.close()
|
||||
await repoTmp.destroyDb()
|
||||
await metaTmp.destroyDb()
|
||||
|
||||
if not taskpool.isNil:
|
||||
taskpool.shutdown()
|
||||
|
||||
reset(taskpool)
|
||||
|
||||
test "Should fail instantiating for invalid slot index":
|
||||
let sampler = Poseidon2Sampler.new(builder.slotRoots.len, store, builder)
|
||||
|
||||
@ -114,7 +124,7 @@ suite "Test Sampler":
|
||||
|
||||
test "Should fail instantiating for non verifiable builder":
|
||||
let
|
||||
nonVerifiableBuilder = Poseidon2Builder.new(store, protected).tryGet
|
||||
nonVerifiableBuilder = Poseidon2Builder.new(store, protected, erasure).tryGet
|
||||
sampler = Poseidon2Sampler.new(slotIndex, store, nonVerifiableBuilder)
|
||||
|
||||
check sampler.isErr
|
||||
@ -129,22 +139,24 @@ suite "Test Sampler":
|
||||
slotTreeCid = verifiable.slotRoots[slotIndex]
|
||||
# get slot tree cid to retrieve proof from storage
|
||||
slotRoot = builder.slotRoots[slotIndex] # get slot root hash
|
||||
cellIdxs = entropy.cellIndices(slotRoot, builder.numSlotCells, nSamples)
|
||||
|
||||
nSlotCellsEncoded = builder.numSlotCellsEncoded
|
||||
pow2SlotCellsEncoded = nextPowerOfTwo(nSlotCellsEncoded)
|
||||
cellIdxs = entropy.cellIndices(slotRoot, pow2SlotCellsEncoded, nSamples)
|
||||
nBlockCells = builder.numBlockCells
|
||||
nSlotCells = builder.numSlotCells
|
||||
|
||||
for i, cellIdx in cellIdxs:
|
||||
let
|
||||
sample = (await sampler.getSample(cellIdx, slotTreeCid, slotRoot)).tryGet
|
||||
|
||||
cellProof = Poseidon2Proof.init(
|
||||
cellIdx.toCellInBlk(nBlockCells), nSlotCells, sample.merklePaths[0 ..< 5]
|
||||
cellIdx.toCellInBlk(nBlockCells),
|
||||
nSlotCellsEncoded,
|
||||
sample.merklePaths[0 ..< 5],
|
||||
).tryGet
|
||||
|
||||
slotProof = Poseidon2Proof.init(
|
||||
cellIdx.toBlkInSlot(nBlockCells),
|
||||
nSlotCells,
|
||||
nSlotCellsEncoded,
|
||||
sample.merklePaths[5 ..< sample.merklePaths.len],
|
||||
).tryGet
|
||||
|
||||
|
||||
@ -1,12 +1,14 @@
|
||||
import ../../asynctest
|
||||
|
||||
import pkg/chronos
|
||||
import pkg/taskpools
|
||||
import pkg/libp2p/cid
|
||||
|
||||
import pkg/codex/merkletree
|
||||
import pkg/codex/chunker
|
||||
import pkg/codex/blocktype as bt
|
||||
import pkg/codex/slots
|
||||
import pkg/codex/erasure
|
||||
import pkg/codex/stores
|
||||
import pkg/codex/conf
|
||||
import pkg/confutils/defs
|
||||
@ -29,6 +31,8 @@ suite "Test Prover":
|
||||
var
|
||||
store: BlockStore
|
||||
prover: Prover
|
||||
taskpool: Taskpool
|
||||
erasure: Erasure
|
||||
|
||||
setup:
|
||||
let
|
||||
@ -47,20 +51,30 @@ suite "Test Prover":
|
||||
backend = config.initializeBackend().tryGet()
|
||||
|
||||
store = RepoStore.new(repoDs, metaDs)
|
||||
prover = Prover.new(store, backend, config.numProofSamples)
|
||||
taskpool = Taskpool.new()
|
||||
erasure = Erasure.new(store, leoEncoderProvider, leoDecoderProvider, taskpool)
|
||||
prover = Prover.new(store, erasure, backend, config.numProofSamples)
|
||||
|
||||
teardown:
|
||||
await repoTmp.destroyDb()
|
||||
await metaTmp.destroyDb()
|
||||
|
||||
if not taskpool.isNil:
|
||||
taskpool.shutdown()
|
||||
|
||||
reset(store)
|
||||
reset(prover)
|
||||
reset(taskpool)
|
||||
|
||||
test "Should sample and prove a slot":
|
||||
let (_, _, verifiable) = await createVerifiableManifest(
|
||||
store,
|
||||
8, # number of blocks in the original dataset (before EC)
|
||||
5, # ecK
|
||||
3, # ecM
|
||||
5, # number of blocks in the original dataset (before EC)
|
||||
2, # ecK
|
||||
1, # ecM
|
||||
blockSize,
|
||||
cellSize,
|
||||
erasure,
|
||||
)
|
||||
|
||||
let (inputs, proof) = (await prover.prove(1, verifiable, challenge)).tryGet
|
||||
@ -80,6 +94,7 @@ suite "Test Prover":
|
||||
1, # ecM
|
||||
blockSize,
|
||||
cellSize,
|
||||
erasure,
|
||||
)
|
||||
|
||||
let (inputs, proof) = (await prover.prove(1, verifiable, challenge)).tryGet
|
||||
|
||||
@ -4,6 +4,7 @@ import std/importutils
|
||||
import ../../asynctest
|
||||
|
||||
import pkg/chronos
|
||||
import pkg/taskpools
|
||||
import pkg/questionable/results
|
||||
import pkg/codex/blocktype as bt
|
||||
import pkg/codex/rng
|
||||
@ -12,6 +13,8 @@ import pkg/codex/chunker
|
||||
import pkg/codex/merkletree
|
||||
import pkg/codex/manifest {.all.}
|
||||
import pkg/codex/utils
|
||||
import pkg/codex/utils/encoding2d
|
||||
import pkg/codex/erasure
|
||||
import pkg/codex/utils/digest
|
||||
import pkg/poseidon2
|
||||
import pkg/poseidon2/io
|
||||
@ -44,22 +47,6 @@ suite "Slot builder":
|
||||
originalDatasetSize = numDatasetBlocks * blockSize.int
|
||||
totalDatasetSize = numTotalBlocks * blockSize.int
|
||||
|
||||
numSlotBlocks = numTotalBlocks div numSlots
|
||||
numBlockCells = (blockSize div cellSize).int # number of cells per block
|
||||
numSlotCells = numSlotBlocks * numBlockCells # number of uncorrected slot cells
|
||||
pow2SlotCells = nextPowerOfTwo(numSlotCells) # pow2 cells per slot
|
||||
numPadSlotBlocks = (pow2SlotCells div numBlockCells) - numSlotBlocks
|
||||
# pow2 blocks per slot
|
||||
|
||||
numSlotBlocksTotal =
|
||||
# pad blocks per slot
|
||||
if numPadSlotBlocks > 0:
|
||||
numPadSlotBlocks + numSlotBlocks
|
||||
else:
|
||||
numSlotBlocks
|
||||
|
||||
numBlocksTotal = numSlotBlocksTotal * numSlots
|
||||
|
||||
# empty digest
|
||||
emptyDigest = SpongeMerkle.digest(newSeq[byte](blockSize.int), cellSize.int)
|
||||
repoTmp = TempLevelDb.new()
|
||||
@ -72,6 +59,8 @@ suite "Slot builder":
|
||||
protectedManifest: Manifest
|
||||
builder: Poseidon2Builder
|
||||
chunker: Chunker
|
||||
taskpool: Taskpool
|
||||
erasure: Erasure
|
||||
|
||||
setup:
|
||||
let
|
||||
@ -79,6 +68,8 @@ suite "Slot builder":
|
||||
metaDs = metaTmp.newDb()
|
||||
|
||||
localStore = RepoStore.new(repoDs, metaDs)
|
||||
taskpool = Taskpool.new()
|
||||
erasure = Erasure.new(localStore, leoEncoderProvider, leoDecoderProvider, taskpool)
|
||||
chunker =
|
||||
RandomChunker.new(Rng.instance(), size = totalDatasetSize, chunkSize = blockSize)
|
||||
datasetBlocks = await chunker.createBlocks(localStore)
|
||||
@ -93,6 +84,9 @@ suite "Slot builder":
|
||||
await repoTmp.destroyDb()
|
||||
await metaTmp.destroyDb()
|
||||
|
||||
if not taskpool.isNil:
|
||||
taskpool.shutdown()
|
||||
|
||||
# TODO: THIS IS A BUG IN asynctest, because it doesn't release the
|
||||
# objects after the test is done, so we need to do it manually
|
||||
#
|
||||
@ -104,6 +98,7 @@ suite "Slot builder":
|
||||
reset(protectedManifest)
|
||||
reset(builder)
|
||||
reset(chunker)
|
||||
reset(taskpool)
|
||||
|
||||
test "Can only create builder with protected manifest":
|
||||
let unprotectedManifest = Manifest.new(
|
||||
@ -113,8 +108,9 @@ suite "Slot builder":
|
||||
)
|
||||
|
||||
check:
|
||||
Poseidon2Builder.new(localStore, unprotectedManifest, cellSize = cellSize).error.msg ==
|
||||
"Manifest is not protected."
|
||||
Poseidon2Builder.new(
|
||||
localStore, unprotectedManifest, erasure, cellSize = cellSize
|
||||
).error.msg == "Manifest is not protected."
|
||||
|
||||
test "Number of blocks must be devisable by number of slots":
|
||||
let mismatchManifest = Manifest.new(
|
||||
@ -131,7 +127,7 @@ suite "Slot builder":
|
||||
)
|
||||
|
||||
check:
|
||||
Poseidon2Builder.new(localStore, mismatchManifest, cellSize = cellSize).error.msg ==
|
||||
Poseidon2Builder.new(localStore, mismatchManifest, erasure, cellSize = cellSize).error.msg ==
|
||||
"Number of blocks must be divisible by number of slots."
|
||||
|
||||
test "Block size must be divisable by cell size":
|
||||
@ -149,39 +145,56 @@ suite "Slot builder":
|
||||
)
|
||||
|
||||
check:
|
||||
Poseidon2Builder.new(localStore, mismatchManifest, cellSize = cellSize).error.msg ==
|
||||
Poseidon2Builder.new(localStore, mismatchManifest, erasure, cellSize = cellSize).error.msg ==
|
||||
"Block size must be divisible by cell size."
|
||||
|
||||
test "Should build correct slot builder":
|
||||
builder =
|
||||
Poseidon2Builder.new(localStore, protectedManifest, cellSize = cellSize).tryGet()
|
||||
builder = Poseidon2Builder
|
||||
.new(localStore, protectedManifest, erasure, cellSize = cellSize)
|
||||
.tryGet()
|
||||
|
||||
let
|
||||
numBlockCells = (blockSize div cellSize).int
|
||||
numSlotBlocks = protectedManifest.numSlotBlocks
|
||||
numSlotBlocksEncoded =
|
||||
encoding2d.calculate2DSlotBlocks(protectedManifest).tryGet()
|
||||
numSlotCells = numSlotBlocks * numBlockCells
|
||||
numSlotCellsEncoded = numSlotBlocksEncoded * numBlockCells
|
||||
numBlocksTotal = numSlotBlocks * numSlots
|
||||
|
||||
check:
|
||||
builder.cellSize == cellSize
|
||||
builder.numSlots == numSlots
|
||||
builder.numBlockCells == numBlockCells
|
||||
builder.numSlotBlocks == numSlotBlocksTotal
|
||||
builder.numSlotCells == pow2SlotCells
|
||||
builder.numSlotBlocks == numSlotBlocks
|
||||
builder.numSlotBlocksEncoded == numSlotBlocksEncoded
|
||||
builder.numSlotCells == numSlotCells
|
||||
builder.numSlotCellsEncoded == numSlotCellsEncoded
|
||||
builder.numBlocks == numBlocksTotal
|
||||
|
||||
test "Should build slot hashes for all slots":
|
||||
let
|
||||
linearStrategy = Strategy.init(
|
||||
0, protectedManifest.blocksCount - 1, numSlots, numSlots, numPadSlotBlocks
|
||||
)
|
||||
|
||||
builder = Poseidon2Builder
|
||||
.new(localStore, protectedManifest, cellSize = cellSize)
|
||||
.tryGet()
|
||||
let builder = Poseidon2Builder
|
||||
.new(localStore, protectedManifest, erasure, cellSize = cellSize)
|
||||
.tryGet()
|
||||
|
||||
for i in 0 ..< numSlots:
|
||||
let
|
||||
encoded2DTreeCid =
|
||||
(await erasure.encode2DSlot(protectedManifest, Strategy, i)).tryGet()
|
||||
numSlotBlocksEncoded =
|
||||
encoding2d.calculate2DSlotBlocks(protectedManifest).tryGet()
|
||||
powNumSlotBlocksEncoded = nextPowerOfTwo(numSlotBlocksEncoded)
|
||||
|
||||
expectedHashes = collect(newSeq):
|
||||
for j, idx in linearStrategy.getIndices(i):
|
||||
if j > (protectedManifest.numSlotBlocks - 1):
|
||||
for idx in 0 ..< powNumSlotBlocksEncoded:
|
||||
if idx >= numSlotBlocksEncoded:
|
||||
emptyDigest
|
||||
else:
|
||||
SpongeMerkle.digest(datasetBlocks[idx].data, cellSize.int)
|
||||
let blk = (await localStore.getBlock(encoded2DTreeCid, idx)).tryGet()
|
||||
if blk.isEmpty:
|
||||
emptyDigest
|
||||
else:
|
||||
SpongeMerkle.digest(blk.data, cellSize.int)
|
||||
|
||||
cellHashes = (await builder.getCellHashes(i)).tryGet()
|
||||
|
||||
@ -190,23 +203,27 @@ suite "Slot builder":
|
||||
cellHashes == expectedHashes
|
||||
|
||||
test "Should build slot trees for all slots":
|
||||
let
|
||||
linearStrategy = Strategy.init(
|
||||
0, protectedManifest.blocksCount - 1, numSlots, numSlots, numPadSlotBlocks
|
||||
)
|
||||
|
||||
builder = Poseidon2Builder
|
||||
.new(localStore, protectedManifest, cellSize = cellSize)
|
||||
.tryGet()
|
||||
let builder = Poseidon2Builder
|
||||
.new(localStore, protectedManifest, erasure, cellSize = cellSize)
|
||||
.tryGet()
|
||||
|
||||
for i in 0 ..< numSlots:
|
||||
let
|
||||
encoded2DTreeCid =
|
||||
(await erasure.encode2DSlot(protectedManifest, Strategy, i)).tryGet()
|
||||
numSlotBlocksEncoded =
|
||||
encoding2d.calculate2DSlotBlocks(protectedManifest).tryGet()
|
||||
powNumSlotBlocksEncoded = nextPowerOfTwo(numSlotBlocksEncoded)
|
||||
expectedHashes = collect(newSeq):
|
||||
for j, idx in linearStrategy.getIndices(i):
|
||||
if j > (protectedManifest.numSlotBlocks - 1):
|
||||
for idx in 0 ..< powNumSlotBlocksEncoded:
|
||||
if idx >= numSlotBlocksEncoded:
|
||||
emptyDigest
|
||||
else:
|
||||
SpongeMerkle.digest(datasetBlocks[idx].data, cellSize.int)
|
||||
let blk = (await localStore.getBlock(encoded2DTreeCid, idx)).tryGet()
|
||||
if blk.isEmpty:
|
||||
emptyDigest
|
||||
else:
|
||||
SpongeMerkle.digest(blk.data, cellSize.int)
|
||||
|
||||
expectedRoot = Merkle.digest(expectedHashes)
|
||||
slotTree = (await builder.buildSlotTree(i)).tryGet()
|
||||
@ -215,16 +232,18 @@ suite "Slot builder":
|
||||
slotTree.root().tryGet() == expectedRoot
|
||||
|
||||
test "Should persist trees for all slots":
|
||||
let builder =
|
||||
Poseidon2Builder.new(localStore, protectedManifest, cellSize = cellSize).tryGet()
|
||||
let builder = Poseidon2Builder
|
||||
.new(localStore, protectedManifest, erasure, cellSize = cellSize)
|
||||
.tryGet()
|
||||
|
||||
for i in 0 ..< numSlots:
|
||||
let
|
||||
slotTree = (await builder.buildSlotTree(i)).tryGet()
|
||||
slotRoot = (await builder.buildSlot(i)).tryGet()
|
||||
slotCid = slotRoot.toSlotCid().tryGet()
|
||||
pow2NumSlotBlocksEncoded = nextPowerOfTwo(builder.numSlotBlocksEncoded)
|
||||
|
||||
for cellIndex in 0 ..< numPadSlotBlocks:
|
||||
for cellIndex in 0 ..< pow2NumSlotBlocksEncoded:
|
||||
let
|
||||
(cellCid, proof) =
|
||||
(await localStore.getCidAndProof(slotCid, cellIndex)).tryGet()
|
||||
@ -237,24 +256,30 @@ suite "Slot builder":
|
||||
verifiableProof.nleaves == posProof.nleaves
|
||||
|
||||
test "Should build correct verification root":
|
||||
let
|
||||
linearStrategy = Strategy.init(
|
||||
0, protectedManifest.blocksCount - 1, numSlots, numSlots, numPadSlotBlocks
|
||||
)
|
||||
builder = Poseidon2Builder
|
||||
.new(localStore, protectedManifest, cellSize = cellSize)
|
||||
.tryGet()
|
||||
let builder = Poseidon2Builder
|
||||
.new(localStore, protectedManifest, erasure, cellSize = cellSize)
|
||||
.tryGet()
|
||||
|
||||
(await builder.buildSlots()).tryGet
|
||||
let
|
||||
slotsHashes = collect(newSeq):
|
||||
for i in 0 ..< numSlots:
|
||||
let slotHashes = collect(newSeq):
|
||||
for j, idx in linearStrategy.getIndices(i):
|
||||
if j > (protectedManifest.numSlotBlocks - 1):
|
||||
emptyDigest
|
||||
else:
|
||||
SpongeMerkle.digest(datasetBlocks[idx].data, cellSize.int)
|
||||
let
|
||||
encoded2DTreeCid =
|
||||
(await erasure.encode2DSlot(protectedManifest, Strategy, i)).tryGet()
|
||||
numSlotBlocksEncoded =
|
||||
encoding2d.calculate2DSlotBlocks(protectedManifest).tryGet()
|
||||
powNumSlotBlocksEncoded = nextPowerOfTwo(numSlotBlocksEncoded)
|
||||
slotHashes = collect(newSeq):
|
||||
for idx in 0 ..< powNumSlotBlocksEncoded:
|
||||
if idx >= numSlotBlocksEncoded:
|
||||
emptyDigest
|
||||
else:
|
||||
let blk = (await localStore.getBlock(encoded2DTreeCid, idx)).tryGet()
|
||||
if blk.isEmpty:
|
||||
emptyDigest
|
||||
else:
|
||||
SpongeMerkle.digest(blk.data, cellSize.int)
|
||||
|
||||
Merkle.digest(slotHashes)
|
||||
|
||||
@ -266,21 +291,28 @@ suite "Slot builder":
|
||||
|
||||
test "Should build correct verification root manifest":
|
||||
let
|
||||
linearStrategy = Strategy.init(
|
||||
0, protectedManifest.blocksCount - 1, numSlots, numSlots, numPadSlotBlocks
|
||||
)
|
||||
builder = Poseidon2Builder
|
||||
.new(localStore, protectedManifest, cellSize = cellSize)
|
||||
.new(localStore, protectedManifest, erasure, cellSize = cellSize)
|
||||
.tryGet()
|
||||
|
||||
slotsHashes = collect(newSeq):
|
||||
for i in 0 ..< numSlots:
|
||||
let slotHashes = collect(newSeq):
|
||||
for j, idx in linearStrategy.getIndices(i):
|
||||
if j > (protectedManifest.numSlotBlocks - 1):
|
||||
emptyDigest
|
||||
else:
|
||||
SpongeMerkle.digest(datasetBlocks[idx].data, cellSize.int)
|
||||
let
|
||||
encoded2DTreeCid =
|
||||
(await erasure.encode2DSlot(protectedManifest, Strategy, i)).tryGet()
|
||||
numSlotBlocksEncoded =
|
||||
encoding2d.calculate2DSlotBlocks(protectedManifest).tryGet()
|
||||
powNumSlotBlocksEncoded = nextPowerOfTwo(numSlotBlocksEncoded)
|
||||
slotHashes = collect(newSeq):
|
||||
for idx in 0 ..< powNumSlotBlocksEncoded:
|
||||
if idx >= numSlotBlocksEncoded:
|
||||
emptyDigest
|
||||
else:
|
||||
let blk = (await localStore.getBlock(encoded2DTreeCid, idx)).tryGet()
|
||||
if blk.isEmpty:
|
||||
emptyDigest
|
||||
else:
|
||||
SpongeMerkle.digest(blk.data, cellSize.int)
|
||||
|
||||
Merkle.digest(slotHashes)
|
||||
|
||||
@ -296,45 +328,47 @@ suite "Slot builder":
|
||||
test "Should not build from verifiable manifest with 0 slots":
|
||||
var
|
||||
builder = Poseidon2Builder
|
||||
.new(localStore, protectedManifest, cellSize = cellSize)
|
||||
.new(localStore, protectedManifest, erasure, cellSize = cellSize)
|
||||
.tryGet()
|
||||
verifyManifest = (await builder.buildManifest()).tryGet()
|
||||
|
||||
verifyManifest.slotRoots = @[]
|
||||
check Poseidon2Builder.new(localStore, verifyManifest, cellSize = cellSize).isErr
|
||||
check Poseidon2Builder.new(localStore, verifyManifest, erasure, cellSize = cellSize).isErr
|
||||
|
||||
test "Should not build from verifiable manifest with incorrect number of slots":
|
||||
var
|
||||
builder = Poseidon2Builder
|
||||
.new(localStore, protectedManifest, cellSize = cellSize)
|
||||
.new(localStore, protectedManifest, erasure, cellSize = cellSize)
|
||||
.tryGet()
|
||||
|
||||
verifyManifest = (await builder.buildManifest()).tryGet()
|
||||
|
||||
verifyManifest.slotRoots.del(verifyManifest.slotRoots.len - 1)
|
||||
|
||||
check Poseidon2Builder.new(localStore, verifyManifest, cellSize = cellSize).isErr
|
||||
check Poseidon2Builder.new(localStore, verifyManifest, erasure, cellSize = cellSize).isErr
|
||||
|
||||
test "Should not build from verifiable manifest with invalid verify root":
|
||||
let builder =
|
||||
Poseidon2Builder.new(localStore, protectedManifest, cellSize = cellSize).tryGet()
|
||||
let builder = Poseidon2Builder
|
||||
.new(localStore, protectedManifest, erasure, cellSize = cellSize)
|
||||
.tryGet()
|
||||
|
||||
var verifyManifest = (await builder.buildManifest()).tryGet()
|
||||
|
||||
rng.shuffle(Rng.instance, verifyManifest.verifyRoot.data.buffer)
|
||||
|
||||
check Poseidon2Builder.new(localStore, verifyManifest, cellSize = cellSize).isErr
|
||||
check Poseidon2Builder.new(localStore, verifyManifest, erasure, cellSize = cellSize).isErr
|
||||
|
||||
test "Should build from verifiable manifest":
|
||||
let
|
||||
builder = Poseidon2Builder
|
||||
.new(localStore, protectedManifest, cellSize = cellSize)
|
||||
.new(localStore, protectedManifest, erasure, cellSize = cellSize)
|
||||
.tryGet()
|
||||
|
||||
verifyManifest = (await builder.buildManifest()).tryGet()
|
||||
|
||||
verificationBuilder =
|
||||
Poseidon2Builder.new(localStore, verifyManifest, cellSize = cellSize).tryGet()
|
||||
verificationBuilder = Poseidon2Builder
|
||||
.new(localStore, verifyManifest, erasure, cellSize = cellSize)
|
||||
.tryGet()
|
||||
|
||||
check:
|
||||
builder.slotRoots == verificationBuilder.slotRoots
|
||||
|
||||
@ -274,8 +274,12 @@ suite "Erasure encode/decode":
|
||||
slotCids = collect(newSeq):
|
||||
for i in 0 ..< encoded.numSlots:
|
||||
Cid.example
|
||||
slotEncodedTreeCids = collect(newSeq):
|
||||
for i in 0 ..< encoded.numSlots:
|
||||
Cid.example
|
||||
|
||||
verifiable = Manifest.new(encoded, Cid.example, slotCids).tryGet()
|
||||
verifiable =
|
||||
Manifest.new(encoded, Cid.example, slotCids, slotEncodedTreeCids).tryGet()
|
||||
|
||||
decoded = (await erasure.decode(verifiable)).tryGet()
|
||||
|
||||
|
||||
@ -32,13 +32,17 @@ suite "Manifest":
|
||||
]
|
||||
|
||||
slotLeavesCids = leaves.toSlotCids().tryGet
|
||||
slotEncodedTreeCids = [Cid.example, Cid.example, Cid.example, Cid.example]
|
||||
|
||||
tree = Poseidon2Tree.init(leaves).tryGet
|
||||
verifyCid = tree.root.tryGet.toVerifyCid().tryGet
|
||||
|
||||
verifiableManifest = Manifest
|
||||
.new(
|
||||
manifest = protectedManifest, verifyRoot = verifyCid, slotRoots = slotLeavesCids
|
||||
manifest = protectedManifest,
|
||||
verifyRoot = verifyCid,
|
||||
slotRoots = slotLeavesCids,
|
||||
slotEncodedTreeCids = slotEncodedTreeCids,
|
||||
)
|
||||
.tryGet()
|
||||
|
||||
@ -91,6 +95,7 @@ suite "Manifest - Attribute Inheritance":
|
||||
manifest = makeProtectedManifest(SteppedStrategy),
|
||||
verifyRoot = Cid.example,
|
||||
slotRoots = @[Cid.example, Cid.example],
|
||||
slotEncodedTreeCids = @[Cid.example, Cid.example],
|
||||
)
|
||||
.tryGet()
|
||||
|
||||
@ -101,6 +106,7 @@ suite "Manifest - Attribute Inheritance":
|
||||
manifest = makeProtectedManifest(LinearStrategy),
|
||||
verifyRoot = Cid.example,
|
||||
slotRoots = @[Cid.example, Cid.example],
|
||||
slotEncodedTreeCids = @[Cid.example, Cid.example],
|
||||
)
|
||||
.tryGet()
|
||||
|
||||
@ -112,6 +118,7 @@ suite "Manifest - Attribute Inheritance":
|
||||
manifest = makeProtectedManifest(SteppedStrategy),
|
||||
verifyRoot = Cid.example,
|
||||
slotRoots = @[Cid.example, Cid.example],
|
||||
slotEncodedTreeCids = @[Cid.example, Cid.example],
|
||||
)
|
||||
.tryGet()
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user