nim-codex/tests/codex/slots/testslotbuilder.nim

400 lines
12 KiB
Nim

import std/sequtils
import std/math
import std/importutils
import std/sugar
import pkg/chronos
import pkg/asynctest
import pkg/questionable/results
import pkg/codex/blocktype as bt
import pkg/codex/rng
import pkg/codex/stores
import pkg/codex/chunker
import pkg/codex/merkletree
import pkg/codex/manifest {.all.}
import pkg/codex/utils
import pkg/codex/utils/digest
import pkg/datastore
import pkg/poseidon2
import pkg/poseidon2/io
import constantine/math/io/io_fields
import ../helpers
import ../examples
import ../merkletree/helpers
import pkg/codex/indexingstrategy {.all.}
import pkg/codex/slots {.all.}
privateAccess(SlotsBuilder) # enable access to private fields
privateAccess(Manifest) # enable access to private fields
suite "Slot builder":
let
blockSize = NBytes 1024
cellSize = NBytes 64
ecK = 3
ecM = 2
numSlots = ecK + ecM
numDatasetBlocks = 100
numBlockCells = (blockSize div cellSize).int
numTotalBlocks = calcEcBlocksCount(numDatasetBlocks, ecK, ecM) # total number of blocks in the dataset after
# EC (should will match number of slots)
originalDatasetSize = numDatasetBlocks * blockSize.int # size of the dataset before EC
totalDatasetSize = numTotalBlocks * blockSize.int # size of the dataset after EC
numTotalSlotBlocks = nextPowerOfTwo(numTotalBlocks div numSlots)
blockPadBytes =
newSeq[byte](numBlockCells.nextPowerOfTwoPad * cellSize.int) # power of two padding for blocks
slotsPadLeafs =
newSeqWith((numTotalBlocks div numSlots).nextPowerOfTwoPad, Poseidon2Zero) # power of two padding for block roots
rootsPadLeafs =
newSeqWith(numSlots.nextPowerOfTwoPad, Poseidon2Zero)
var
datasetBlocks: seq[bt.Block]
localStore: BlockStore
manifest: Manifest
protectedManifest: Manifest
expectedEmptyCid: Cid
slotBuilder: SlotsBuilder
chunker: Chunker
proc createBlocks(): Future[void] {.async.} =
while true:
let chunk = await chunker.getBytes()
if chunk.len <= 0:
break
let blk = bt.Block.new(chunk).tryGet()
datasetBlocks.add(blk)
discard await localStore.putBlock(blk)
proc createProtectedManifest(): Future[void] {.async.} =
let
cids = datasetBlocks.mapIt(it.cid)
datasetTree = CodexTree.init(cids[0..<numDatasetBlocks]).tryGet()
datasetTreeCid = datasetTree.rootCid().tryGet()
protectedTree = CodexTree.init(cids).tryGet()
protectedTreeCid = protectedTree.rootCid().tryGet()
for index, cid in cids[0..<numDatasetBlocks]:
let proof = datasetTree.getProof(index).tryget()
(await localStore.putCidAndProof(datasetTreeCid, index, cid, proof)).tryGet
for index, cid in cids:
let proof = protectedTree.getProof(index).tryget()
(await localStore.putCidAndProof(protectedTreeCid, index, cid, proof)).tryGet
manifest = Manifest.new(
treeCid = datasetTreeCid,
blockSize = blockSize.NBytes,
datasetSize = originalDatasetSize.NBytes)
protectedManifest = Manifest.new(
manifest = manifest,
treeCid = protectedTreeCid,
datasetSize = totalDatasetSize.NBytes,
ecK = ecK,
ecM = ecM)
let
manifestBlock = bt.Block.new(
manifest.encode().tryGet(),
codec = ManifestCodec).tryGet()
protectedManifestBlock = bt.Block.new(
protectedManifest.encode().tryGet(),
codec = ManifestCodec).tryGet()
(await localStore.putBlock(manifestBlock)).tryGet()
(await localStore.putBlock(protectedManifestBlock)).tryGet()
expectedEmptyCid = emptyCid(
protectedManifest.version,
protectedManifest.hcodec,
protectedManifest.codec).tryGet()
setup:
let
repoDs = SQLiteDatastore.new(Memory).tryGet()
metaDs = SQLiteDatastore.new(Memory).tryGet()
localStore = RepoStore.new(repoDs, metaDs)
chunker = RandomChunker.new(Rng.instance(), size = totalDatasetSize, chunkSize = blockSize)
await createBlocks()
await createProtectedManifest()
teardown:
await localStore.close()
# TODO: THIS IS A BUG IN asynctest, because it doesn't release the
# objects after the test is done, so we need to do it manually
#
# Need to reset all objects because otherwise they get
# captured by the test runner closures, not good!
reset(datasetBlocks)
reset(localStore)
reset(manifest)
reset(protectedManifest)
reset(expectedEmptyCid)
reset(slotBuilder)
reset(chunker)
test "Can only create slotBuilder with protected manifest":
let
unprotectedManifest = Manifest.new(
treeCid = Cid.example,
blockSize = blockSize.NBytes,
datasetSize = originalDatasetSize.NBytes)
check:
SlotsBuilder.new(localStore, unprotectedManifest, cellSize = cellSize)
.error.msg == "Can only create SlotsBuilder using protected manifests."
test "Number of blocks must be devisable by number of slots":
let
mismatchManifest = Manifest.new(
manifest = Manifest.new(
treeCid = Cid.example,
blockSize = blockSize.NBytes,
datasetSize = originalDatasetSize.NBytes),
treeCid = Cid.example,
datasetSize = totalDatasetSize.NBytes,
ecK = ecK - 1,
ecM = ecM)
check:
SlotsBuilder.new(localStore, mismatchManifest, cellSize = cellSize)
.error.msg == "Number of blocks must be divisable by number of slots."
test "Block size must be divisable by cell size":
let
mismatchManifest = Manifest.new(
manifest = Manifest.new(
treeCid = Cid.example,
blockSize = (blockSize + 1).NBytes,
datasetSize = (originalDatasetSize - 1).NBytes),
treeCid = Cid.example,
datasetSize = (totalDatasetSize - 1).NBytes,
ecK = ecK,
ecM = ecM)
check:
SlotsBuilder.new(localStore, mismatchManifest, cellSize = cellSize)
.error.msg == "Block size must be divisable by cell size."
test "Should build correct slot builder":
slotBuilder = SlotsBuilder.new(
localStore,
protectedManifest,
cellSize = cellSize).tryGet()
check:
slotBuilder.numBlockPadBytes == blockPadBytes.len
slotBuilder.numSlotsPadLeafs == slotsPadLeafs.len
slotBuilder.numRootsPadLeafs == rootsPadLeafs.len
test "Should build slot hashes for all slots":
let
steppedStrategy = SteppedIndexingStrategy.new(0, numTotalBlocks - 1, numSlots)
slotBuilder = SlotsBuilder.new(
localStore,
protectedManifest,
cellSize = cellSize).tryGet()
for i in 0 ..< numSlots:
let
expectedBlock = steppedStrategy
.getIndicies(i)
.mapIt( datasetBlocks[it] )
expectedHashes: seq[Poseidon2Hash] = collect(newSeq):
for blk in expectedBlock:
SpongeMerkle.digest(blk.data & blockPadBytes, cellSize.int)
cellHashes = (await slotBuilder.getCellHashes(i)).tryGet()
check:
expectedHashes == cellHashes
test "Should build slot trees for all slots":
let
steppedStrategy = SteppedIndexingStrategy.new(0, numTotalBlocks - 1, numSlots)
slotBuilder = SlotsBuilder.new(
localStore,
protectedManifest,
cellSize = cellSize).tryGet()
for i in 0 ..< numSlots:
let
expectedBlock = steppedStrategy
.getIndicies(i)
.mapIt( datasetBlocks[it] )
expectedHashes: seq[Poseidon2Hash] = collect(newSeq):
for blk in expectedBlock:
SpongeMerkle.digest(blk.data & blockPadBytes, cellSize.int)
expectedRoot = Merkle.digest(expectedHashes & slotsPadLeafs)
slotTree = (await slotBuilder.buildSlotTree(i)).tryGet()
check:
expectedRoot == slotTree.root().tryGet()
test "Should persist trees for all slots":
let
slotBuilder = SlotsBuilder.new(
localStore,
protectedManifest,
cellSize = cellSize).tryGet()
for i in 0 ..< numSlots:
let
slotTree = (await slotBuilder.buildSlotTree(i)).tryGet()
slotRoot = (await slotBuilder.buildSlot(i)).tryGet()
slotCid = slotRoot.toSlotCid().tryGet()
for cellIndex in 0..<numTotalSlotBlocks:
let
(cellCid, proof) = (await localStore.getCidAndProof(slotCid, cellIndex)).tryGet()
verifiableProof = proof.toVerifiableProof().tryGet()
posProof = slotTree.getProof(cellIndex).tryGet
check:
verifiableProof.index == posProof.index
verifiableProof.nleaves == posProof.nleaves
verifiableProof.path == posProof.path
test "Should build correct verification root":
let
steppedStrategy = SteppedIndexingStrategy.new(0, numTotalBlocks - 1, numSlots)
slotBuilder = SlotsBuilder.new(
localStore,
protectedManifest,
cellSize = cellSize).tryGet()
(await slotBuilder.buildSlots()).tryGet
let
slotsHashes = collect(newSeq):
for i in 0 ..< numSlots:
let
expectedBlocks = steppedStrategy
.getIndicies(i)
.mapIt( datasetBlocks[it] )
slotHashes: seq[Poseidon2Hash] = collect(newSeq):
for blk in expectedBlocks:
SpongeMerkle.digest(blk.data & blockPadBytes, cellSize.int)
Merkle.digest(slotHashes & slotsPadLeafs)
expectedRoot = Merkle.digest(slotsHashes & rootsPadLeafs)
rootHash = slotBuilder.buildVerifyTree(slotBuilder.slotRoots).tryGet().root.tryGet()
check:
expectedRoot == rootHash
test "Should build correct verification root manifest":
let
steppedStrategy = SteppedIndexingStrategy.new(0, numTotalBlocks - 1, numSlots)
slotBuilder = SlotsBuilder.new(
localStore,
protectedManifest,
cellSize = cellSize).tryGet()
slotsHashes = collect(newSeq):
for i in 0 ..< numSlots:
let
expectedBlocks = steppedStrategy
.getIndicies(i)
.mapIt( datasetBlocks[it] )
slotHashes: seq[Poseidon2Hash] = collect(newSeq):
for blk in expectedBlocks:
SpongeMerkle.digest(blk.data & blockPadBytes, cellSize.int)
Merkle.digest(slotHashes & slotsPadLeafs)
expectedRoot = Merkle.digest(slotsHashes & rootsPadLeafs)
manifest = (await slotBuilder.buildManifest()).tryGet()
mhash = manifest.verifyRoot.mhash.tryGet()
mhashBytes = mhash.digestBytes
rootHash = Poseidon2Hash.fromBytes(mhashBytes.toArray32).get
check:
expectedRoot == rootHash
test "Should not build from verifiable manifest with 0 slots":
var
slotBuilder = SlotsBuilder.new(
localStore,
protectedManifest,
cellSize = cellSize).tryGet()
verifyManifest = (await slotBuilder.buildManifest()).tryGet()
verifyManifest.slotRoots = @[]
check SlotsBuilder.new(
localStore,
verifyManifest,
cellSize = cellSize).isErr
test "Should not build from verifiable manifest with incorrect number of slots":
var
slotBuilder = SlotsBuilder.new(
localStore,
protectedManifest,
cellSize = cellSize).tryGet()
verifyManifest = (await slotBuilder.buildManifest()).tryGet()
verifyManifest.slotRoots.del(
verifyManifest.slotRoots.len - 1
)
check SlotsBuilder.new(
localStore,
verifyManifest,
cellSize = cellSize).isErr
test "Should not build from verifiable manifest with invalid verify root":
let
slotBuilder = SlotsBuilder.new(
localStore,
protectedManifest,
cellSize = cellSize).tryGet()
verifyManifest = (await slotBuilder.buildManifest()).tryGet()
offset = verifyManifest.verifyRoot.data.buffer.len div 2
rng.shuffle(
Rng.instance,
verifyManifest.verifyRoot.data.buffer)
check SlotsBuilder.new(
localStore,
verifyManifest,
cellSize = cellSize).isErr
test "Should build from verifiable manifest":
let
slotBuilder = SlotsBuilder.new(
localStore,
protectedManifest,
cellSize = cellSize).tryGet()
verifyManifest = (await slotBuilder.buildManifest()).tryGet()
verificationBuilder = SlotsBuilder.new(
localStore,
verifyManifest,
cellSize = cellSize).tryGet()
check:
slotBuilder.slotRoots == verificationBuilder.slotRoots
slotBuilder.verifyRoot == verificationBuilder.verifyRoot