mirror of
https://github.com/codex-storage/nim-codex.git
synced 2025-02-02 08:05:15 +00:00
reworking slotbuilder
This commit is contained in:
Dmitriy Ryajov
parent
98f60411a6
commit
9c9730d23a
@ -1,109 +0,0 @@
|
||||
import std/math
|
||||
import std/sequtils
|
||||
import pkg/libp2p
|
||||
import pkg/chronos
|
||||
import pkg/chronicles
|
||||
import pkg/questionable/results
|
||||
import ../merkletree
|
||||
import ../stores
|
||||
import ../manifest
|
||||
import ../utils
|
||||
import ../utils/digest
|
||||
|
||||
const
|
||||
# TODO: Unified with the CellSize specified in branch "data-sampler"
|
||||
# Number of bytes in a cell. A cell is the smallest unit of data used
|
||||
# in the proving circuit.
|
||||
CellSize* = 2048
|
||||
|
||||
type
|
||||
SlotBuilder* = object of RootObj
|
||||
blockStore: BlockStore
|
||||
manifest: Manifest
|
||||
slotBlocks: int
|
||||
|
||||
proc new*(
|
||||
T: type SlotBuilder,
|
||||
blockStore: BlockStore,
|
||||
manifest: Manifest
|
||||
): ?!SlotBuilder =
|
||||
|
||||
if not manifest.protected:
|
||||
return failure("Can only create SlotBuilder using protected manifests.")
|
||||
|
||||
if (manifest.blocksCount mod manifest.ecK) != 0:
|
||||
return failure("Number of blocks must be divisable by number of slots.")
|
||||
|
||||
if (manifest.blockSize.int mod CellSize) != 0:
|
||||
return failure("Block size must be divisable by cell size.")
|
||||
|
||||
let slotBlocks = manifest.blocksCount div manifest.numberOfSlots
|
||||
success SlotBuilder(
|
||||
blockStore: blockStore,
|
||||
manifest: manifest,
|
||||
slotBlocks: slotBlocks)
|
||||
|
||||
proc cellsPerBlock(self: SlotBuilder): int =
|
||||
self.manifest.blockSize.int div CellSize
|
||||
|
||||
proc selectSlotBlocks*(
|
||||
self: SlotBuilder,
|
||||
slotIndex: int): Future[?!seq[Poseidon2Hash]] {.async.} =
|
||||
|
||||
let
|
||||
treeCid = self.manifest.treeCid
|
||||
blockCount = self.manifest.blocksCount
|
||||
numberOfSlots = self.manifest.numberOfSlots
|
||||
strategy = SteppedIndexingStrategy.new(0, blockCount - 1, numberOfSlots)
|
||||
|
||||
logScope:
|
||||
treeCid = treeCid
|
||||
blockCount = blockCount
|
||||
numberOfSlots = numberOfSlots
|
||||
index = blockIndex
|
||||
|
||||
var blocks = newSeq[Poseidon2Hash]()
|
||||
for blockIndex in strategy.getIndicies(slotIndex):
|
||||
without blk =? await self.blockStore.getBlock(treeCid, blockIndex), err:
|
||||
error "Failed to get block CID for tree at index"
|
||||
|
||||
return failure(err)
|
||||
|
||||
without digestTree =? Poseidon2Tree.digest(blk.data, CellSize) and
|
||||
blockDigest =? digestTree.root, err:
|
||||
error "Failed to create digest for block"
|
||||
|
||||
return failure(err)
|
||||
|
||||
blocks.add(blockDigest)
|
||||
# TODO: Remove this sleep. It's here to prevent us from locking up the thread.
|
||||
await sleepAsync(10.millis)
|
||||
|
||||
success blocks
|
||||
|
||||
proc numPaddingCells*(self: SlotBuilder, slotBlocks: int): int =
|
||||
let
|
||||
numberOfCells = slotBlocks * self.cellsPerBlock
|
||||
nextPowerOfTwo = nextPowerOfTwo(numberOfCells)
|
||||
|
||||
return nextPowerOfTwo - numberOfCells
|
||||
|
||||
proc buildSlotTree*(self: SlotBuilder, slotBlocks: seq[Cid], paddingCells: int): ?!Poseidon2Tree =
|
||||
without emptyCid =? emptyCid(self.manifest.version, self.manifest.hcodec, self.manifest.codec), err:
|
||||
error "Unable to initialize empty cid"
|
||||
return failure(err)
|
||||
|
||||
let paddingBlocks = divUp(paddingCells, self.cellsPerBlock)
|
||||
let padding = newSeqWith(paddingBlocks, emptyCid)
|
||||
|
||||
Poseidon2Tree.init(slotBlocks & padding)
|
||||
|
||||
proc createSlots*(self: SlotBuilder, slotIndex: int): Future[?!Manifest] {.async.} =
|
||||
without slotBlocks =? await self.selectSlotBlocks(slotIndex), err:
|
||||
error "Failed to select slot blocks"
|
||||
return failure(err)
|
||||
|
||||
let paddingCells = self.numPaddingCells(slotBlocks.len)
|
||||
|
||||
trace "Creating slot tree", slotIndex, nSlotBlocks = slotBlocks.len, paddingCells
|
||||
return self.buildSlotTree(slotBlocks, paddingCells)
|
||||
267
codex/slots/slotbuilder.nim
Normal file
267
codex/slots/slotbuilder.nim
Normal file
@ -0,0 +1,267 @@
|
||||
## Nim-Codex
|
||||
## Copyright (c) 2023 Status Research & Development GmbH
|
||||
## Licensed under either of
|
||||
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
|
||||
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
|
||||
## at your option.
|
||||
## This file may not be copied, modified, or distributed except according to
|
||||
## those terms.
|
||||
|
||||
{.push raises: [].}
|
||||
|
||||
import std/math
|
||||
import std/sequtils
|
||||
import std/sugar
|
||||
|
||||
import pkg/libp2p
|
||||
import pkg/chronos
|
||||
import pkg/chronicles
|
||||
import pkg/questionable/results
|
||||
import pkg/poseidon2
|
||||
import pkg/poseidon2/io
|
||||
|
||||
import ../indexingstrategy
|
||||
import ../merkletree
|
||||
import ../stores
|
||||
import ../manifest
|
||||
import ../utils
|
||||
import ../utils/digest
|
||||
|
||||
const
|
||||
# TODO: Unified with the CellSize specified in branch "data-sampler"
|
||||
# Number of bytes in a cell. A cell is the smallest unit of data used
|
||||
# in the proving circuit.
|
||||
CellSize* = 2048
|
||||
|
||||
type
|
||||
SlotBuilder* = object of RootObj
|
||||
store: BlockStore
|
||||
manifest: Manifest
|
||||
strategy: IndexingStrategy
|
||||
cellSize: int
|
||||
blockPadBytes: seq[byte]
|
||||
slotsPadLeafs: seq[Poseidon2Hash]
|
||||
rootsPadLeafs: seq[Poseidon2Hash]
|
||||
|
||||
func numBlockPadBytes*(self: SlotBuilder): Natural =
|
||||
## Number of padding bytes required for a pow2
|
||||
## merkle tree for each block.
|
||||
##
|
||||
|
||||
self.blockPadBytes.len
|
||||
|
||||
func numSlotsPadLeafs*(self: SlotBuilder): Natural =
|
||||
## Number of padding field elements required for a pow2
|
||||
## merkle tree for each slot.
|
||||
##
|
||||
|
||||
self.slotsPadLeafs.len
|
||||
|
||||
func numRootsPadLeafs*(self: SlotBuilder): Natural =
|
||||
## Number of padding field elements required for a pow2
|
||||
## merkle tree for the slot roots.
|
||||
##
|
||||
|
||||
self.rootsPadLeafs.len
|
||||
|
||||
func numSlotBlocks*(self: SlotBuilder): Natural =
|
||||
## Number of blocks per slot.
|
||||
##
|
||||
|
||||
self.manifest.blocksCount div self.manifest.numSlots
|
||||
|
||||
func numBlockRoots*(self: SlotBuilder): Natural =
|
||||
## Number of cells per block.
|
||||
##
|
||||
|
||||
self.manifest.blockSize.int div self.cellSize
|
||||
|
||||
func toCellCid(cell: Poseidon2Hash): ?!Cid =
|
||||
let
|
||||
cellMhash = ? MultiHash.init(Pos2Bn128MrklCodec, cell.toBytes).mapFailure
|
||||
cellCid = ? Cid.init(CIDv1, CodexSlotCell, cellMhash).mapFailure
|
||||
|
||||
success cellCid
|
||||
|
||||
func toSlotCid(root: Poseidon2Hash): ?!Cid =
|
||||
let
|
||||
mhash = ? MultiHash.init($multiCodec("identity"), root.toBytes).mapFailure
|
||||
treeCid = ? Cid.init(CIDv1, SlotRootCodec, mhash).mapFailure
|
||||
|
||||
success treeCid
|
||||
|
||||
func toProvingCid(root: Poseidon2Hash): ?!Cid =
|
||||
let
|
||||
mhash = ? MultiHash.init($multiCodec("identity"), root.toBytes).mapFailure
|
||||
treeCid = ? Cid.init(CIDv1, SlotProvingRootCodec, mhash).mapFailure
|
||||
|
||||
success treeCid
|
||||
|
||||
func mapToSlotCids(slotRoots: seq[Poseidon2Hash]): ?!seq[Cid] =
|
||||
success slotRoots.mapIt( ? it.toSlotCid )
|
||||
|
||||
func toEncodableProof*(
|
||||
proof: Poseidon2Proof): ?!CodexProof =
|
||||
|
||||
let
|
||||
encodableProof = CodexProof(
|
||||
mcodec: multiCodec("identity"), # copy bytes as is
|
||||
index: proof.index,
|
||||
nleaves: proof.nleaves,
|
||||
path: proof.path.mapIt( @(it.toBytes) ))
|
||||
|
||||
success encodableProof
|
||||
|
||||
func toVerifiableProof*(
|
||||
proof: CodexProof): ?!Poseidon2Proof =
|
||||
|
||||
let
|
||||
verifiableProof = Poseidon2Proof(
|
||||
index: proof.index,
|
||||
nleaves: proof.nleaves,
|
||||
path: proof.path.mapIt(
|
||||
? Poseidon2Hash.fromBytes(it.toArray32).toResult
|
||||
))
|
||||
|
||||
success verifiableProof
|
||||
|
||||
proc getCellHashes*(
|
||||
self: SlotBuilder,
|
||||
slotIndex: int): Future[?!seq[Poseidon2Hash]] {.async.} =
|
||||
|
||||
let
|
||||
treeCid = self.manifest.treeCid
|
||||
blockCount = self.manifest.blocksCount
|
||||
numberOfSlots = self.manifest.numSlots
|
||||
|
||||
logScope:
|
||||
treeCid = treeCid
|
||||
blockCount = blockCount
|
||||
numberOfSlots = numberOfSlots
|
||||
index = blockIndex
|
||||
slotIndex = slotIndex
|
||||
|
||||
let
|
||||
hashes: seq[Poseidon2Hash] = collect(newSeq):
|
||||
for blockIndex in self.strategy.getIndicies(slotIndex):
|
||||
trace "Getting block CID for tree at index"
|
||||
|
||||
without blk =? (await self.store.getBlock(treeCid, blockIndex)), err:
|
||||
error "Failed to get block CID for tree at index"
|
||||
return failure(err)
|
||||
|
||||
without digest =? Poseidon2Tree.digest(blk.data & self.blockPadBytes, self.cellSize), err:
|
||||
error "Failed to create digest for block"
|
||||
return failure(err)
|
||||
|
||||
# TODO: Remove this sleep. It's here to prevent us from locking up the thread.
|
||||
# await sleepAsync(10.millis)
|
||||
|
||||
digest
|
||||
|
||||
success hashes
|
||||
|
||||
proc buildSlotTree*(
|
||||
self: SlotBuilder,
|
||||
slotIndex: int): Future[?!Poseidon2Tree] {.async.} =
|
||||
without cellHashes =? (await self.getCellHashes(slotIndex)), err:
|
||||
error "Failed to select slot blocks", err = err.msg
|
||||
return failure(err)
|
||||
|
||||
Poseidon2Tree.init(cellHashes & self.slotsPadLeafs)
|
||||
|
||||
proc buildSlot*(
|
||||
self: SlotBuilder,
|
||||
slotIndex: int): Future[?!Poseidon2Hash] {.async.} =
|
||||
## Build a slot tree and store it in the block store.
|
||||
##
|
||||
|
||||
without tree =? (await self.buildSlotTree(slotIndex)) and
|
||||
treeCid =? tree.root.?toSlotCid, err:
|
||||
error "Failed to build slot tree", err = err.msg
|
||||
return failure(err)
|
||||
|
||||
trace "Storing slot tree", treeCid, slotIndex, leaves = tree.leavesCount
|
||||
for i, leaf in tree.leaves:
|
||||
without cellCid =? leaf.toCellCid, err:
|
||||
error "Failed to get CID for slot cell", err = err.msg
|
||||
return failure(err)
|
||||
|
||||
without proof =? tree.getProof(i) and
|
||||
encodableProof =? proof.toEncodableProof, err:
|
||||
error "Failed to get proof for slot tree", err = err.msg
|
||||
return failure(err)
|
||||
|
||||
if err =? (await self.store.putCidAndProof(
|
||||
treeCid, i, cellCid, encodableProof)).errorOption:
|
||||
error "Failed to store slot tree", err = err.msg
|
||||
return failure(err)
|
||||
|
||||
tree.root()
|
||||
|
||||
proc buildSlots(self: SlotBuilder): Future[?!Manifest] {.async.} =
|
||||
let
|
||||
slotRoots: seq[Poseidon2Hash] = collect(newSeq):
|
||||
for i in 0..<self.manifest.numSlots:
|
||||
without root =? (await self.buildSlot(i)), err:
|
||||
error "Failed to build slot", err = err.msg, index = i
|
||||
return failure(err)
|
||||
root
|
||||
|
||||
without provingRootCid =? Poseidon2Tree.init(slotRoots & self.rootsPadLeafs).?root.?toProvingCid, err:
|
||||
error "Failed to build proving tree", err = err.msg
|
||||
return failure(err)
|
||||
|
||||
without rootCids =? slotRoots.mapToSlotCids(), err:
|
||||
error "Failed to map slot roots to CIDs", err = err.msg
|
||||
return failure(err)
|
||||
|
||||
Manifest.new(self.manifest, provingRootCid, rootCids)
|
||||
|
||||
func nextPowerOfTwoPad*(a: int): int =
|
||||
## Returns the next power of two of `a` and `b` and the difference between
|
||||
## the original value and the next power of two.
|
||||
##
|
||||
|
||||
nextPowerOfTwo(a) - a
|
||||
|
||||
proc new*(
|
||||
T: type SlotBuilder,
|
||||
store: BlockStore,
|
||||
manifest: Manifest,
|
||||
strategy: IndexingStrategy = nil,
|
||||
cellSize = CellSize): ?!SlotBuilder =
|
||||
|
||||
if not manifest.protected:
|
||||
return failure("Can only create SlotBuilder using protected manifests.")
|
||||
|
||||
if (manifest.blocksCount mod manifest.numSlots) != 0:
|
||||
return failure("Number of blocks must be divisable by number of slots.")
|
||||
|
||||
if (manifest.blockSize.int mod cellSize) != 0:
|
||||
return failure("Block size must be divisable by cell size.")
|
||||
|
||||
let
|
||||
strategy = if strategy == nil:
|
||||
SteppedIndexingStrategy.new(
|
||||
0, manifest.blocksCount - 1, manifest.numSlots)
|
||||
else:
|
||||
strategy
|
||||
|
||||
# all trees have to be padded to power of two
|
||||
numBlockCells = manifest.blockSize.int div cellSize # number of cells per block
|
||||
blockPadBytes
|
||||
= newSeq[byte](numBlockCells.nextPowerOfTwoPad * cellSize) # power of two padding for blocks
|
||||
slotsPadLeafs
|
||||
= newSeqWith((manifest.blocksCount div manifest.numSlots).nextPowerOfTwoPad, Poseidon2Zero) # power of two padding for block roots
|
||||
rootsPadLeafs
|
||||
= newSeqWith(manifest.numSlots.nextPowerOfTwoPad, Poseidon2Zero)
|
||||
|
||||
success SlotBuilder(
|
||||
store: store,
|
||||
manifest: manifest,
|
||||
strategy: strategy,
|
||||
cellSize: cellSize,
|
||||
blockPadBytes: blockPadBytes,
|
||||
slotsPadLeafs: slotsPadLeafs,
|
||||
rootsPadLeafs: rootsPadLeafs)
|
||||
@ -1,5 +1,8 @@
|
||||
import std/sequtils
|
||||
import std/math
|
||||
import std/importutils
|
||||
import std/sugar
|
||||
|
||||
import pkg/chronos
|
||||
import pkg/asynctest
|
||||
import pkg/questionable/results
|
||||
@ -9,29 +12,53 @@ import pkg/codex/stores
|
||||
import pkg/codex/chunker
|
||||
import pkg/codex/merkletree
|
||||
import pkg/codex/utils
|
||||
import pkg/codex/utils/digest
|
||||
import pkg/datastore
|
||||
import pkg/poseidon2
|
||||
import pkg/poseidon2/io
|
||||
import constantine/math/io/io_fields
|
||||
|
||||
import ../helpers
|
||||
import ../examples
|
||||
import ../merkletree/helpers
|
||||
|
||||
import pkg/codex/manifest/indexingstrategy
|
||||
import pkg/codex/slotbuilder/slotbuilder
|
||||
import pkg/codex/indexingstrategy {.all.}
|
||||
import pkg/codex/slots/slotbuilder {.all.}
|
||||
|
||||
asyncchecksuite "Slot builder":
|
||||
suite "Slot builder":
|
||||
let
|
||||
blockSize = 64 * 1024
|
||||
numberOfCellsPerBlock = blockSize div CellSize
|
||||
numberOfSlotBlocks = 6
|
||||
numberOfSlots = 5
|
||||
numberOfDatasetBlocks = numberOfSlotBlocks * numberOfSlots
|
||||
datasetSize = numberOfDatasetBlocks * blockSize
|
||||
chunker = RandomChunker.new(Rng.instance(), size = datasetSize, chunkSize = blockSize)
|
||||
blockSize = 1024
|
||||
cellSize = 64
|
||||
ecK = 3
|
||||
ecM = 2
|
||||
|
||||
numSlots = ecK + ecM
|
||||
numDatasetBlocks = 100
|
||||
numBlockCells = blockSize div cellSize
|
||||
|
||||
numTotalBlocks = calcEcBlocksCount(numDatasetBlocks, ecK, ecM) # total number of blocks in the dataset after
|
||||
# EC (should will match number of slots)
|
||||
originalDatasetSize = numDatasetBlocks * blockSize # size of the dataset before EC
|
||||
totalDatasetSize = numTotalBlocks * blockSize # size of the dataset after EC
|
||||
numTotalSlotBlocks = nextPowerOfTwo(numTotalBlocks div numSlots)
|
||||
|
||||
blockPadBytes =
|
||||
newSeq[byte](numBlockCells.nextPowerOfTwoPad * cellSize) # power of two padding for blocks
|
||||
|
||||
slotsPadLeafs =
|
||||
newSeqWith((numTotalBlocks div numSlots).nextPowerOfTwoPad, Poseidon2Zero) # power of two padding for block roots
|
||||
|
||||
rootsPadLeafs =
|
||||
newSeqWith(numSlots.nextPowerOfTwoPad, Poseidon2Zero)
|
||||
|
||||
var
|
||||
datasetBlocks: seq[bt.Block]
|
||||
localStore = CacheStore.new()
|
||||
localStore: BlockStore
|
||||
manifest: Manifest
|
||||
protectedManifest: Manifest
|
||||
expectedEmptyCid: Cid
|
||||
slotBuilder: SlotBuilder
|
||||
chunker: Chunker
|
||||
|
||||
proc createBlocks(): Future[void] {.async.} =
|
||||
while true:
|
||||
@ -45,132 +72,225 @@ asyncchecksuite "Slot builder":
|
||||
proc createProtectedManifest(): Future[void] {.async.} =
|
||||
let
|
||||
cids = datasetBlocks.mapIt(it.cid)
|
||||
tree = Poseidon2Tree.init(cids).tryGet()
|
||||
treeCid = tree.rootCid().tryGet()
|
||||
datasetTree = CodexTree.init(cids[0..<numDatasetBlocks]).tryGet()
|
||||
datasetTreeCid = datasetTree.rootCid().tryGet()
|
||||
|
||||
protectedTree = CodexTree.init(cids).tryGet()
|
||||
protectedTreeCid = protectedTree.rootCid().tryGet()
|
||||
|
||||
for index, cid in cids[0..<numDatasetBlocks]:
|
||||
let proof = datasetTree.getProof(index).tryget()
|
||||
(await localStore.putCidAndProof(datasetTreeCid, index, cid, proof)).tryGet
|
||||
|
||||
for index, cid in cids:
|
||||
let proof = tree.getProof(index).tryget()
|
||||
discard await localStore.putBlockCidAndProof(treeCid, index, cid, proof)
|
||||
let proof = protectedTree.getProof(index).tryget()
|
||||
(await localStore.putCidAndProof(protectedTreeCid, index, cid, proof)).tryGet
|
||||
|
||||
manifest = Manifest.new(
|
||||
treeCid = datasetTreeCid,
|
||||
blockSize = blockSize.NBytes,
|
||||
datasetSize = originalDatasetSize.NBytes)
|
||||
|
||||
protectedManifest = Manifest.new(
|
||||
manifest = Manifest.new(
|
||||
treeCid = treeCid,
|
||||
blockSize = blockSize.NBytes,
|
||||
datasetSize = datasetSize.NBytes),
|
||||
treeCid = treeCid,
|
||||
datasetSize = datasetSize.NBytes,
|
||||
ecK = numberOfSlots,
|
||||
ecM = 0
|
||||
)
|
||||
manifest = manifest,
|
||||
treeCid = protectedTreeCid,
|
||||
datasetSize = totalDatasetSize.NBytes,
|
||||
ecK = ecK,
|
||||
ecM = ecM)
|
||||
|
||||
let manifestBlock = bt.Block.new(protectedManifest.encode().tryGet(), codec = DagPBCodec).tryGet()
|
||||
discard await localStore.putBlock(manifestBlock)
|
||||
expectedEmptyCid = emptyCid(protectedManifest.version, protectedManifest.hcodec, protectedManifest.codec).tryget()
|
||||
let
|
||||
manifestBlock = bt.Block.new(
|
||||
manifest.encode().tryGet(),
|
||||
codec = ManifestCodec).tryGet()
|
||||
|
||||
protectedManifestBlock = bt.Block.new(
|
||||
protectedManifest.encode().tryGet(),
|
||||
codec = ManifestCodec).tryGet()
|
||||
|
||||
(await localStore.putBlock(manifestBlock)).tryGet()
|
||||
(await localStore.putBlock(protectedManifestBlock)).tryGet()
|
||||
|
||||
expectedEmptyCid = emptyCid(
|
||||
protectedManifest.version,
|
||||
protectedManifest.hcodec,
|
||||
protectedManifest.codec).tryGet()
|
||||
|
||||
privateAccess(SlotBuilder) # enable access to private fields
|
||||
|
||||
setup:
|
||||
let
|
||||
repoDs = SQLiteDatastore.new(Memory).tryGet()
|
||||
metaDs = SQLiteDatastore.new(Memory).tryGet()
|
||||
localStore = RepoStore.new(repoDs, metaDs)
|
||||
|
||||
chunker = RandomChunker.new(Rng.instance(), size = totalDatasetSize, chunkSize = blockSize)
|
||||
await createBlocks()
|
||||
await createProtectedManifest()
|
||||
slotBuilder = SlotBuilder.new(localStore, protectedManifest).tryGet()
|
||||
|
||||
teardown:
|
||||
await localStore.close()
|
||||
|
||||
# Need to reset all objects because otherwise they get
|
||||
# captured by the test runner closures, not good!
|
||||
reset(datasetBlocks)
|
||||
reset(localStore)
|
||||
reset(manifest)
|
||||
reset(protectedManifest)
|
||||
reset(expectedEmptyCid)
|
||||
reset(slotBuilder)
|
||||
reset(chunker)
|
||||
|
||||
test "Can only create slotBuilder with protected manifest":
|
||||
let unprotectedManifest = Manifest.new(
|
||||
treeCid = Cid.example,
|
||||
blockSize = blockSize.NBytes,
|
||||
datasetSize = datasetSize.NBytes)
|
||||
|
||||
check:
|
||||
SlotBuilder.new(localStore, unprotectedManifest).isErr
|
||||
|
||||
test "Number of blocks must be devisable by number of slots":
|
||||
let mismatchManifest = Manifest.new(
|
||||
manifest = Manifest.new(
|
||||
let
|
||||
unprotectedManifest = Manifest.new(
|
||||
treeCid = Cid.example,
|
||||
blockSize = blockSize.NBytes,
|
||||
datasetSize = datasetSize.NBytes),
|
||||
treeCid = Cid.example,
|
||||
datasetSize = datasetSize.NBytes,
|
||||
ecK = numberOfSlots - 1,
|
||||
ecM = 0
|
||||
)
|
||||
datasetSize = originalDatasetSize.NBytes)
|
||||
|
||||
check:
|
||||
SlotBuilder.new(localStore, mismatchManifest).isErr
|
||||
SlotBuilder.new(localStore, unprotectedManifest, cellSize = cellSize)
|
||||
.error.msg == "Can only create SlotBuilder using protected manifests."
|
||||
|
||||
test "Number of blocks must be devisable by number of slots":
|
||||
let
|
||||
mismatchManifest = Manifest.new(
|
||||
manifest = Manifest.new(
|
||||
treeCid = Cid.example,
|
||||
blockSize = blockSize.NBytes,
|
||||
datasetSize = originalDatasetSize.NBytes),
|
||||
treeCid = Cid.example,
|
||||
datasetSize = totalDatasetSize.NBytes,
|
||||
ecK = ecK - 1,
|
||||
ecM = ecM)
|
||||
|
||||
check:
|
||||
SlotBuilder.new(localStore, mismatchManifest, cellSize = cellSize)
|
||||
.error.msg == "Number of blocks must be divisable by number of slots."
|
||||
|
||||
test "Block size must be divisable by cell size":
|
||||
let mismatchManifest = Manifest.new(
|
||||
manifest = Manifest.new(
|
||||
treeCid = Cid.example,
|
||||
blockSize = (blockSize - 1).NBytes,
|
||||
datasetSize = (datasetSize - numberOfDatasetBlocks).NBytes),
|
||||
treeCid = Cid.example,
|
||||
datasetSize = (datasetSize - numberOfDatasetBlocks).NBytes,
|
||||
ecK = numberOfSlots,
|
||||
ecM = 0
|
||||
)
|
||||
|
||||
check:
|
||||
SlotBuilder.new(localStore, mismatchManifest).isErr
|
||||
|
||||
for nSlotBlocks in [1, 12, 123, 1234, 12345]:
|
||||
test "Can calculate the number of padding cells (" & $nSlotBlocks & ")":
|
||||
let
|
||||
nPadCells = slotBuilder.calculateNumberOfPaddingCells(nSlotBlocks)
|
||||
totalSlotBytes = nSlotBlocks * blockSize
|
||||
totalSlotCells = totalSlotBytes div CellSize
|
||||
expectedPadCells = nextPowerOfTwo(totalSlotCells) - totalSlotCells
|
||||
check:
|
||||
expectedPadCells == nPadCells
|
||||
|
||||
for i in 0 ..< numberOfSlots:
|
||||
test "Can select slot block CIDs (index: " & $i & ")":
|
||||
let
|
||||
steppedStrategy = SteppedIndexingStrategy.new(0, numberOfDatasetBlocks - 1, numberOfSlots)
|
||||
expectedDatasetBlockIndicies = steppedStrategy.getIndicies(i)
|
||||
expectedBlockCids = expectedDatasetBlockIndicies.mapIt(datasetBlocks[it].cid)
|
||||
|
||||
slotBlockCids = (await slotBuilder.selectSlotBlocks(i)).tryGet()
|
||||
|
||||
check:
|
||||
expectedBlockCids == slotBlockCids
|
||||
|
||||
test "Can create slot tree (index: " & $i & ")":
|
||||
let
|
||||
expectedSlotBlockCids = (await slotBuilder.selectSlotBlocks(i)).tryGet()
|
||||
expectedNumPadBlocks = divUp(slotBuilder.calculateNumberOfPaddingCells(expectedSlotBlockCids.len), numberOfCellsPerBlock)
|
||||
|
||||
slotTree = (await slotBuilder.createSlotTree(i)).tryGet()
|
||||
|
||||
check:
|
||||
# Tree size
|
||||
slotTree.leavesCount == expectedSlotBlockCids.len + expectedNumPadBlocks
|
||||
|
||||
for i in 0 ..< numberOfSlotBlocks:
|
||||
check:
|
||||
# Each slot block
|
||||
slotTree.getLeafCid(i).tryget() == expectedSlotBlockCids[i]
|
||||
|
||||
for i in 0 ..< expectedNumPadBlocks:
|
||||
check:
|
||||
# Each pad block
|
||||
slotTree.getLeafCid(numberOfSlotBlocks + i).tryget() == expectedEmptyCid
|
||||
|
||||
test "Can create slot tree":
|
||||
let
|
||||
slotBlockCids = datasetBlocks[0 ..< numberOfSlotBlocks].mapIt(it.cid)
|
||||
numPadCells = numberOfCellsPerBlock div 2 # We expect 1 pad block.
|
||||
|
||||
slotTree = slotBuilder.buildSlotTree(slotBlockCids, numPadCells).tryGet()
|
||||
mismatchManifest = Manifest.new(
|
||||
manifest = Manifest.new(
|
||||
treeCid = Cid.example,
|
||||
blockSize = (blockSize + 1).NBytes,
|
||||
datasetSize = (originalDatasetSize - 1).NBytes),
|
||||
treeCid = Cid.example,
|
||||
datasetSize = (totalDatasetSize - 1).NBytes,
|
||||
ecK = ecK,
|
||||
ecM = ecM)
|
||||
|
||||
check:
|
||||
# Tree size
|
||||
slotTree.leavesCount == slotBlockCids.len + 1
|
||||
SlotBuilder.new(localStore, mismatchManifest, cellSize = cellSize)
|
||||
.error.msg == "Block size must be divisable by cell size."
|
||||
|
||||
test "Should build correct slot builder":
|
||||
slotBuilder = SlotBuilder.new(
|
||||
localStore,
|
||||
protectedManifest,
|
||||
cellSize = cellSize).tryGet()
|
||||
|
||||
check:
|
||||
slotBuilder.numBlockPadBytes == blockPadBytes.len
|
||||
slotBuilder.numSlotsPadLeafs == slotsPadLeafs.len
|
||||
slotBuilder.numRootsPadLeafs == rootsPadLeafs.len
|
||||
|
||||
test "Should build slot hashes for all slots":
|
||||
let
|
||||
steppedStrategy = SteppedIndexingStrategy.new(0, numTotalBlocks - 1, numSlots)
|
||||
slotBuilder = SlotBuilder.new(
|
||||
localStore,
|
||||
protectedManifest,
|
||||
cellSize = cellSize).tryGet()
|
||||
|
||||
for i in 0 ..< numSlots:
|
||||
let
|
||||
expectedBlock = steppedStrategy
|
||||
.getIndicies(i)
|
||||
.mapIt( datasetBlocks[it] )
|
||||
|
||||
expectedHashes: seq[Poseidon2Hash] = collect(newSeq):
|
||||
for blk in expectedBlock:
|
||||
SpongeMerkle.digest(blk.data & blockPadBytes, cellSize)
|
||||
|
||||
cellHashes = (await slotBuilder.getCellHashes(i)).tryGet()
|
||||
|
||||
for i in 0 ..< numberOfSlotBlocks:
|
||||
check:
|
||||
# Each slot block
|
||||
slotTree.getLeafCid(i).tryget() == slotBlockCids[i]
|
||||
expectedHashes == cellHashes
|
||||
|
||||
test "Should build slot trees for all slots":
|
||||
let
|
||||
steppedStrategy = SteppedIndexingStrategy.new(0, numTotalBlocks - 1, numSlots)
|
||||
slotBuilder = SlotBuilder.new(
|
||||
localStore,
|
||||
protectedManifest,
|
||||
cellSize = cellSize).tryGet()
|
||||
|
||||
for i in 0 ..< numSlots:
|
||||
let
|
||||
expectedBlock = steppedStrategy
|
||||
.getIndicies(i)
|
||||
.mapIt( datasetBlocks[it] )
|
||||
|
||||
expectedHashes: seq[Poseidon2Hash] = collect(newSeq):
|
||||
for blk in expectedBlock:
|
||||
SpongeMerkle.digest(blk.data & blockPadBytes, cellSize)
|
||||
expectedRoot = Merkle.digest(expectedHashes & slotsPadLeafs)
|
||||
|
||||
slotTree = (await slotBuilder.buildSlotTree(i)).tryGet()
|
||||
|
||||
check:
|
||||
expectedRoot == slotTree.root().tryGet()
|
||||
|
||||
test "Should persist trees for all slots":
|
||||
let
|
||||
slotBuilder = SlotBuilder.new(
|
||||
localStore,
|
||||
protectedManifest,
|
||||
cellSize = cellSize).tryGet()
|
||||
|
||||
for i in 0 ..< numSlots:
|
||||
let
|
||||
slotTree = (await slotBuilder.buildSlotTree(i)).tryGet()
|
||||
slotRoot = (await slotBuilder.buildSlot(i)).tryGet()
|
||||
slotCid = slotRoot.toSlotCid().tryGet()
|
||||
|
||||
for cellIndex in 0..<numTotalSlotBlocks:
|
||||
let
|
||||
(cellCid, proof) = (await localStore.getCidAndProof(slotCid, cellIndex)).tryGet()
|
||||
verifiableProof = proof.toVerifiableProof().tryGet()
|
||||
posProof = slotTree.getProof(cellIndex).tryGet
|
||||
|
||||
check:
|
||||
verifiableProof.index == posProof.index
|
||||
verifiableProof.nleaves == posProof.nleaves
|
||||
verifiableProof.path == posProof.path
|
||||
|
||||
test "Should build correct verification root":
|
||||
let
|
||||
steppedStrategy = SteppedIndexingStrategy.new(0, numTotalBlocks - 1, numSlots)
|
||||
slotBuilder = SlotBuilder.new(
|
||||
localStore,
|
||||
protectedManifest,
|
||||
cellSize = cellSize).tryGet()
|
||||
|
||||
slotsHashes = collect(newSeq):
|
||||
for i in 0 ..< numSlots:
|
||||
let
|
||||
expectedBlocks = steppedStrategy
|
||||
.getIndicies(i)
|
||||
.mapIt( datasetBlocks[it] )
|
||||
|
||||
slotHashes: seq[Poseidon2Hash] = collect(newSeq):
|
||||
for blk in expectedBlocks:
|
||||
SpongeMerkle.digest(blk.data & blockPadBytes, cellSize)
|
||||
|
||||
Merkle.digest(slotHashes & slotsPadLeafs)
|
||||
|
||||
expectedRoot = Merkle.digest(slotsHashes & rootsPadLeafs)
|
||||
manifest = (await slotBuilder.buildSlots()).tryGet()
|
||||
mhash = manifest.verificationRoot.mhash.tryGet()
|
||||
mhashBytes = mhash.digestBytes
|
||||
rootHash = Poseidon2Hash.fromBytes(mhashBytes.toArray32).toResult.tryGet()
|
||||
|
||||
check:
|
||||
# 1 pad block
|
||||
slotTree.getLeafCid(numberOfSlotBlocks).tryget() == expectedEmptyCid
|
||||
|
||||
expectedRoot == rootHash
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user