Restores tests for sampler

This commit is contained in:
benbierens 2024-01-16 15:20:48 +01:00 committed by Dmitriy Ryajov
parent 4e19384a86
commit 9c1ea719d8
7 changed files with 154 additions and 328 deletions

View File

@ -1,4 +1,4 @@
import ./builder/builder
import ./builder/converters
import ./converters
export builder, converters

View File

@ -348,16 +348,7 @@ proc new*(
if manifest.slotRoots.len == 0 or manifest.slotRoots.len != manifest.numSlots:
return failure "Manifest is verifiable but slot roots are missing or invalid."
let
# slotRoot = ? Poseidon2Hash.fromBytes(
# ( ? manifest.verifyRoot.mhash.mapFailure ).digestBytes.toArray32
# ).toFailure
slotRoots = manifest.slotRoots.mapIt(it.fromSlotCid().toFailure)
# ? Poseidon2Hash.fromBytes(
# ( ? it.mhash.mapFailure ).digestBytes.toArray32
# ).toFailure
# )
let slotRoots = manifest.slotRoots.mapIt( (? it.fromSlotCid()))
without tree =? self.buildVerifyTree(slotRoots), err:
error "Failed to build slot roots tree", err = err.msg

View File

@ -39,6 +39,8 @@ type
data*: Cell
slotProof*: Poseidon2Proof
cellProof*: Poseidon2Proof
slotBlockIdx*: Natural
blockCellIdx*: Natural
ProofInput* = object
entropy*: Poseidon2Hash
@ -96,6 +98,7 @@ proc getProofInput*(
let
slotTreeCid = self.builder.manifest.slotRoots[self.index]
numCellsPerBlock = (self.builder.manifest.blockSize div self.builder.cellSize).Natural
cellIdxs = entropy.cellIndices(
self.builder.slotRoots[self.index],
self.builder.numSlotCells,
@ -111,21 +114,30 @@ proc getProofInput*(
let samples = collect(newSeq):
for cellIdx in cellIdxs:
let
blockIdx = cellIdx.toBlockIdx(self.builder.numSlotCells)
blkCellIdx = cellIdx.toBlockCellIdx(self.builder.numBlockCells)
blockIdx = cellIdx.toBlockIdx(numCellsPerBlock)
blkCellIdx = cellIdx.toBlockCellIdx(numCellsPerBlock)
logScope:
cellIdx = cellIdx
blockIdx = blockIdx
blkCellIdx = blkCellIdx
without (cid, slotProof) =? await self.blockStore.getCidAndProof(
without (cid, proof) =? await self.blockStore.getCidAndProof(
slotTreeCid,
blockIdx.Natural), err:
error "Failed to get block from block store", err = err.msg
return failure(err)
without (bytes, blkTree) =? await self.builder.buildBlockTree(blockIdx), err:
without slotProof =? proof.toVerifiableProof(), err:
error "Unable to convert slot proof to poseidon proof", error = err.msg
return failure(err)
# This converts our slotBlockIndex to a datasetBlockIndex using the
# indexing-strategy used by the builder.
# We need this to fetch the block data. We can't do it by slotTree + slotBlkIdx.
let datasetBlockIndex = self.builder.slotIndicies(self.index)[blockIdx]
without (bytes, blkTree) =? await self.builder.buildBlockTree(datasetBlockIndex), err:
error "Failed to build block tree", err = err.msg
return failure(err)
@ -135,7 +147,13 @@ proc getProofInput*(
let cellData = self.getCell(bytes, blkCellIdx)
Sample(data: cellData, slotProof: slotProof, cellProof: blockProof)
Sample(
data: cellData,
slotProof: slotProof,
cellProof: blockProof,
slotBlockIdx: blockIdx.Natural,
blockCellIdx: blkCellIdx.Natural
)
success ProofInput(
entropy: entropy,

View File

@ -80,3 +80,4 @@ func cellIndices*(
let idx = cellIndex(entropy, slotRoot, numCells, indices.len + 1)
indices.add(idx.Natural)
indices

View File

@ -78,7 +78,9 @@ proc createSlotTree(self: ProvingTestEnvironment, dSlotIndex: uint64): Future[Po
let
slotBlocks = datasetBlockIndices.mapIt(self.datasetBlocks[it])
slotBlockRoots = slotBlocks.mapIt(Poseidon2Tree.digest(it.data, DefaultCellSize.int).tryGet())
numBlockCells = bytesPerBlock.int div DefaultCellSize.int
blockPadBytes = newSeq[byte](numBlockCells.nextPowerOfTwoPad * DefaultCellSize.int)
slotBlockRoots = slotBlocks.mapIt(Poseidon2Tree.digest(it.data & blockPadBytes, DefaultCellSize.int).tryGet())
tree = Poseidon2Tree.init(slotBlockRoots).tryGet()
treeCid = tree.root().tryGet().toSlotCid().tryGet()
@ -145,6 +147,7 @@ proc createSlot(self: ProvingTestEnvironment): void =
self.slot = Slot(
request: StorageRequest(
ask: StorageAsk(
slots: totalNumberOfSlots.uint64,
slotSize: u256(bytesPerBlock * numberOfSlotBlocks)
),
content: StorageContent(

View File

@ -1,329 +1,117 @@
# import std/sequtils
# import std/sugar
# import std/random
import std/sequtils
import std/sugar
import std/random
import std/strutils
# import pkg/questionable/results
# import pkg/constantine/math/arithmetic
# import pkg/constantine/math/io/io_fields
# import pkg/poseidon2/types
# import pkg/poseidon2/io
# import pkg/poseidon2
# import pkg/chronos
# import pkg/asynctest
# import pkg/codex/stores/cachestore
# import pkg/codex/chunker
# import pkg/codex/stores
# import pkg/codex/blocktype as bt
# import pkg/codex/contracts/requests
# import pkg/codex/contracts
# import pkg/codex/merkletree
# import pkg/codex/stores/cachestore
import pkg/questionable/results
import pkg/constantine/math/arithmetic
import pkg/constantine/math/io/io_fields
import pkg/poseidon2/io
import pkg/poseidon2
import pkg/chronos
import pkg/asynctest
import pkg/codex/stores/cachestore
import pkg/codex/chunker
import pkg/codex/stores
import pkg/codex/blocktype as bt
import pkg/codex/contracts/requests
import pkg/codex/contracts
import pkg/codex/merkletree
import pkg/codex/stores/cachestore
# import pkg/codex/slots
# import pkg/codex/proof/misc
# import pkg/codex/proof/types
import pkg/codex/slots/sampler
import pkg/codex/slots/builder/builder
# import ../helpers
# import ../examples
# import testdatasampler_expected
import ../helpers
import ../examples
import ../merkletree/helpers
import testsampler_expected
import ./provingtestenv
# let
# bytesPerBlock = 64 * 1024
# challenge: FieldElement = toF(12345)
# datasetRootHash: FieldElement = toF(6789)
asyncchecksuite "Test DataSampler":
var
env: ProvingTestEnvironment
dataSampler: DataSampler
blk: bt.Block
cell0Bytes: seq[byte]
cell1Bytes: seq[byte]
cell2Bytes: seq[byte]
# asyncchecksuite "Test proof datasampler - components":
# let
# numberOfSlotBlocks = 16
# slot = Slot(
# request: StorageRequest(
# ask: StorageAsk(
# slots: 10,
# slotSize: u256(bytesPerBlock * numberOfSlotBlocks),
# ),
# content: StorageContent(
# cid: $Cid.example
# )
# ),
# slotIndex: u256(3)
# )
proc createDataSampler(): Future[void] {.async.} =
dataSampler = DataSampler.new(
datasetSlotIndex,
env.localStore,
SlotsBuilder.new(env.localStore, env.manifest).tryGet()).tryGet()
# test "Number of cells is a power of two":
# # This is to check that the data used for testing is sane.
# proc isPow2(value: int): bool =
# let log2 = ceilingLog2(value)
# return (1 shl log2) == value
setup:
env = await createProvingTestEnvironment()
let bytes = newSeqWith(bytesPerBlock, rand(uint8))
blk = bt.Block.new(bytes).tryGet()
cell0Bytes = bytes[0..<DefaultCellSize.uint64]
cell1Bytes = bytes[DefaultCellSize.uint64..<(DefaultCellSize.uint64*2)]
cell2Bytes = bytes[(DefaultCellSize.uint64*2)..<(DefaultCellSize.uint64*3)]
# let numberOfCells = getNumberOfCellsInSlot(slot).int
await createDataSampler()
# check:
# isPow2(numberOfCells)
teardown:
reset(env)
reset(dataSampler)
# test "Extract low bits":
# proc extract(value: uint64, nBits: int): uint64 =
# let big = toF(value).toBig()
# return extractLowBits(big, nBits)
test "Can get cell from block":
let
sample0 = dataSampler.getCell(blk.data, 0)
sample1 = dataSampler.getCell(blk.data, 1)
sample2 = dataSampler.getCell(blk.data, 2)
# check:
# extract(0x88, 4) == 0x8.uint64
# extract(0x88, 7) == 0x8.uint64
# extract(0x9A, 5) == 0x1A.uint64
# extract(0x9A, 7) == 0x1A.uint64
# extract(0x1248, 10) == 0x248.uint64
# extract(0x1248, 12) == 0x248.uint64
# extract(0x1248306A560C9AC0.uint64, 10) == 0x2C0.uint64
# extract(0x1248306A560C9AC0.uint64, 12) == 0xAC0.uint64
# extract(0x1248306A560C9AC0.uint64, 50) == 0x306A560C9AC0.uint64
# extract(0x1248306A560C9AC0.uint64, 52) == 0x8306A560C9AC0.uint64
check:
sample0 == cell0Bytes
sample1 == cell1Bytes
sample2 == cell2Bytes
# test "Should calculate total number of cells in Slot":
# let
# slotSizeInBytes = (slot.request.ask.slotSize).truncate(uint64)
# expectedNumberOfCells = slotSizeInBytes div CellSize
test "Can gather proof input":
let
nSamples = 3
challengeBytes = env.challenge.toBytes()
input = (await dataSampler.getProofInput(challengeBytes, nSamples)).tryget()
# check:
# expectedNumberOfCells == 512
# expectedNumberOfCells == getNumberOfCellsInSlot(slot)
proc equal(a: Poseidon2Hash, b: Poseidon2Hash): bool =
a.toDecimal() == b.toDecimal()
# asyncchecksuite "Test proof datasampler - main":
# let
# # The number of slot blocks and number of slots, combined with
# # the bytes per block, make it so that there are exactly 256 cells
# # in the dataset.
# numberOfSlotBlocks = 4
# totalNumberOfSlots = 2
# datasetSlotIndex = 1
# localStore = CacheStore.new()
# datasetToSlotProof = MerkleProof.example
proc toStr(proof: Poseidon2Proof): string =
let a = proof.path.mapIt(toHex(it))
join(a)
# var
# manifest: Manifest
# manifestBlock: bt.Block
# slot: Slot
# datasetBlocks: seq[bt.Block]
# slotPoseidonTree: MerkleTree
# dataSampler: DataSampler
let
expectedBlockSlotProofs = getExpectedBlockSlotProofs()
expectedCellBlockProofs = getExpectedCellBlockProofs()
expectedCellData = getExpectedCellData()
expectedProof = env.datasetToSlotTree.getProof(datasetSlotIndex).tryGet()
# proc createDatasetBlocks(): Future[void] {.async.} =
# let numberOfCellsNeeded = (numberOfSlotBlocks * totalNumberOfSlots * bytesPerBlock).uint64 div CellSize
# var data: seq[byte] = @[]
check:
equal(input.verifyRoot, env.datasetRootHash)
equal(input.entropy, env.challenge)
input.numCells == ((bytesPerBlock * numberOfSlotBlocks) div DefaultCellSize.int).Natural
input.numSlots == totalNumberOfSlots.Natural
input.slotIndex == env.slot.slotIndex.truncate(Natural)
input.verifyProof == expectedProof
# # This generates a number of blocks that have different data, such that
# # Each cell in each block is unique, but nothing is random.
# for i in 0 ..< numberOfCellsNeeded:
# data = data & (i.byte).repeat(CellSize)
# block-slot proofs
input.samples[0].slotBlockIdx == 2
input.samples[1].slotBlockIdx == 2
input.samples[2].slotBlockIdx == 0
toStr(input.samples[0].slotProof) == expectedBlockSlotProofs[0]
toStr(input.samples[1].slotProof) == expectedBlockSlotProofs[1]
toStr(input.samples[2].slotProof) == expectedBlockSlotProofs[2]
# let chunker = MockChunker.new(
# dataset = data,
# chunkSize = bytesPerBlock)
# cell-block proofs
input.samples[0].blockCellIdx == 26
input.samples[1].blockCellIdx == 29
input.samples[2].blockCellIdx == 29
toStr(input.samples[0].cellProof) == expectedCellBlockProofs[0]
toStr(input.samples[1].cellProof) == expectedCellBlockProofs[1]
toStr(input.samples[2].cellProof) == expectedCellBlockProofs[2]
# while true:
# let chunk = await chunker.getBytes()
# if chunk.len <= 0:
# break
# let b = bt.Block.new(chunk).tryGet()
# datasetBlocks.add(b)
# discard await localStore.putBlock(b)
# proc createManifest(): Future[void] {.async.} =
# let
# cids = datasetBlocks.mapIt(it.cid)
# tree = MerkleTree.init(cids).tryGet()
# treeCid = tree.rootCid().tryGet()
# for index, cid in cids:
# let proof = tree.getProof(index).tryget()
# discard await localStore.putBlockCidAndProof(treeCid, index, cid, proof)
# manifest = Manifest.new(
# treeCid = treeCid,
# blockSize = bytesPerBlock.NBytes,
# datasetSize = (bytesPerBlock * numberOfSlotBlocks * totalNumberOfSlots).NBytes)
# manifestBlock = bt.Block.new(manifest.encode().tryGet(), codec = DagPBCodec).tryGet()
# proc createSlot(): void =
# slot = Slot(
# request: StorageRequest(
# ask: StorageAsk(
# slotSize: u256(bytesPerBlock * numberOfSlotBlocks)
# ),
# content: StorageContent(
# cid: $manifestBlock.cid
# ),
# ),
# slotIndex: u256(datasetSlotIndex)
# )
# proc createSlotPoseidonTree(): void =
# let
# slotSize = slot.request.ask.slotSize.truncate(uint64)
# blocksInSlot = slotSize div bytesPerBlock.uint64
# datasetSlotIndex = slot.slotIndex.truncate(uint64)
# datasetBlockIndexFirst = datasetSlotIndex * blocksInSlot
# datasetBlockIndexLast = datasetBlockIndexFirst + numberOfSlotBlocks.uint64
# slotBlocks = datasetBlocks[datasetBlockIndexFirst ..< datasetBlockIndexLast]
# slotBlockCids = slotBlocks.mapIt(it.cid)
# slotPoseidonTree = MerkleTree.init(slotBlockCids).tryGet()
# proc createDataSampler(): Future[void] {.async.} =
# dataSampler = (await DataSampler.new(
# slot,
# localStore,
# datasetRootHash,
# slotPoseidonTree,
# datasetToSlotProof
# )).tryGet()
# setup:
# await createDatasetBlocks()
# await createManifest()
# createSlot()
# discard await localStore.putBlock(manifestBlock)
# createSlotPoseidonTree()
# await createDataSampler()
# test "Number of cells is a power of two":
# # This is to check that the data used for testing is sane.
# proc isPow2(value: int): bool =
# let log2 = ceilingLog2(value)
# return (1 shl log2) == value
# let numberOfCells = getNumberOfCellsInSlot(slot).int
# check:
# isPow2(numberOfCells)
# let knownIndices = @[74.uint64, 41.uint64, 51.uint64]
# test "Can find single slot-cell index":
# proc slotCellIndex(i: int): uint64 =
# let counter: FieldElement = toF(i)
# return dataSampler.findSlotCellIndex(challenge, counter)
# proc getExpectedIndex(i: int): uint64 =
# let
# numberOfCellsInSlot = (bytesPerBlock * numberOfSlotBlocks) div CellSize.int
# slotRootHash = toF(1234) # TODO - replace with slotPoseidonTree.root when it is a poseidon tree.
# hash = Sponge.digest(@[slotRootHash, challenge, toF(i)], rate = 2)
# return extractLowBits(hash.toBig(), ceilingLog2(numberOfCellsInSlot))
# check:
# slotCellIndex(1) == getExpectedIndex(1)
# slotCellIndex(1) == knownIndices[0]
# slotCellIndex(2) == getExpectedIndex(2)
# slotCellIndex(2) == knownIndices[1]
# slotCellIndex(3) == getExpectedIndex(3)
# slotCellIndex(3) == knownIndices[2]
# test "Can find sequence of slot-cell indices":
# proc slotCellIndices(n: int): seq[uint64] =
# dataSampler.findSlotCellIndices(challenge, n)
# proc getExpectedIndices(n: int): seq[uint64] =
# return collect(newSeq, (for i in 1..n: dataSampler.findSlotCellIndex(challenge, toF(i))))
# check:
# slotCellIndices(3) == getExpectedIndices(3)
# slotCellIndices(3) == knownIndices
# let
# bytes = newSeqWith(bytesPerBlock, rand(uint8))
# blk = bt.Block.new(bytes).tryGet()
# cell0Bytes = bytes[0..<CellSize]
# cell1Bytes = bytes[CellSize..<(CellSize*2)]
# cell2Bytes = bytes[(CellSize*2)..<(CellSize*3)]
# test "Can get cell from block":
# let
# sample0 = dataSampler.getCellFromBlock(blk, 0)
# sample1 = dataSampler.getCellFromBlock(blk, 1)
# sample2 = dataSampler.getCellFromBlock(blk, 2)
# check:
# sample0 == cell0Bytes
# sample1 == cell1Bytes
# sample2 == cell2Bytes
# test "Can convert block into cells":
# let cells = dataSampler.getBlockCells(blk)
# check:
# cells.len == (bytesPerBlock div CellSize.int)
# cells[0] == cell0Bytes
# cells[1] == cell1Bytes
# cells[2] == cell2Bytes
# test "Can create mini tree for block cells":
# let miniTree = dataSampler.getBlockCellMiniTree(blk).tryGet()
# let
# cell0Proof = miniTree.getProof(0).tryGet()
# cell1Proof = miniTree.getProof(1).tryGet()
# cell2Proof = miniTree.getProof(2).tryGet()
# check:
# cell0Proof.verifyDataBlock(cell0Bytes, miniTree.root).tryGet()
# cell1Proof.verifyDataBlock(cell1Bytes, miniTree.root).tryGet()
# cell2Proof.verifyDataBlock(cell2Bytes, miniTree.root).tryGet()
# test "Can gather proof input":
# # This is the main entry point for this module, and what it's all about.
# let
# nSamples = 3
# input = (await dataSampler.getProofInput(challenge, nSamples)).tryget()
# proc equal(a: FieldElement, b: FieldElement): bool =
# a.toDecimal() == b.toDecimal()
# proc toStr(proof: MerkleProof): string =
# toHex(proof.nodesBuffer)
# let
# expectedMerkleProofs = getExpectedSlotToBlockProofs()
# expectedCellData = getExpectedCellData()
# check:
# # datasetRoot*: FieldElement
# equal(input.datasetRoot, datasetRootHash)
# # entropy*: FieldElement
# equal(input.entropy, challenge)
# # numberOfCellsInSlot*: uint64
# input.numberOfCellsInSlot == (bytesPerBlock * numberOfSlotBlocks).uint64 div CellSize
# # numberOfSlots*: uint64
# input.numberOfSlots == slot.request.ask.slots
# # datasetSlotIndex*: uint64
# input.datasetSlotIndex == slot.slotIndex.truncate(uint64)
# # slotRoot*: FieldElement
# equal(input.slotRoot, toF(1234)) # TODO - when slotPoseidonTree is a poseidon tree, its root should be a FieldElement.
# # datasetToSlotProof*: MerkleProof
# input.datasetToSlotProof == datasetToSlotProof
# # proofSamples*: seq[ProofSample]
# toStr(input.proofSamples[0].merkleProof) == expectedMerkleProofs[0]
# toStr(input.proofSamples[1].merkleProof) == expectedMerkleProofs[1]
# toStr(input.proofSamples[2].merkleProof) == expectedMerkleProofs[2]
# # cell data
# toHex(input.proofSamples[0].cellData) == expectedCellData[0]
# toHex(input.proofSamples[1].cellData) == expectedCellData[1]
# toHex(input.proofSamples[2].cellData) == expectedCellData[2]
# # input.slotToBlockProofs.mapIt(toStr(it)) == expectedSlotToBlockProofs
# # input.blockToCellProofs.mapIt(toStr(it)) == expectedBlockToCellProofs
# # toHex(input.sampleData) == expectedSampleData
# for (input, expected) in [(10, 0), (31, 0), (32, 1), (63, 1), (64, 2)]:
# test "Can get slotBlockIndex from slotCellIndex (" & $input & " -> " & $expected & ")":
# let
# slotCellIndex = input.uint64
# slotBlockIndex = dataSampler.getSlotBlockIndexForSlotCellIndex(slotCellIndex)
# check:
# slotBlockIndex == expected.uint64
# for (input, expected) in [(10, 10), (31, 31), (32, 0), (63, 31), (64, 0)]:
# test "Can get blockCellIndex from slotCellIndex (" & $input & " -> " & $expected & ")":
# let
# slotCellIndex = input.uint64
# blockCellIndex = dataSampler.getBlockCellIndexForSlotCellIndex(slotCellIndex)
# check:
# blockCellIndex == expected.uint64
# # cell data
toHex(input.samples[0].data) == expectedCellData[0]
toHex(input.samples[1].data) == expectedCellData[1]
toHex(input.samples[2].data) == expectedCellData[2]

View File

@ -0,0 +1,25 @@
# Snapshot of expected values for testsampler.
import std/strutils
import pkg/codex/codextypes
proc getExpectedCellBlockProofs*(): seq[string] =
@[
"0x189890bedf2a40f2757554c5f089811e07601543a576e2d40d68a1bd295adbee0x059f227fe687a7abd9c3d9878c0b812aa7829c85d30c23154b8824a907909b060x2c685fc951f1684fd3979f7e3a558fa6aeed3f960ef0a9e2ba51cc62cff7de0e0x10ebae3fb12d502044216e40319726ed36308b9ae4ab3fb0a36c77e5614c6fbd0x1decd3fb7ff458261149731115657cecd7eb2fe4a6cf84f3c6761aa8b0dd6b9a",
"0x2567cd93b3fe058b31908656c05d3a09fd33cc0d7df3c7c005d991a8cda60ba80x16e4788105082295706c49c604216518f16ca9dd106012c7c98e6ee138893f6e0x1c258af996aecf9bba249130182ccfe44a9090bc58fe59063a06db67efb7b5240x10ebae3fb12d502044216e40319726ed36308b9ae4ab3fb0a36c77e5614c6fbd0x1decd3fb7ff458261149731115657cecd7eb2fe4a6cf84f3c6761aa8b0dd6b9a",
"0x0568735989a51526104eddbcf386b8aaef186a2d31afce0c2671c8ce8dd8cd1a0x20d06082668338924981a9e0e4f18e7ec6e2b7912e7fb74c1b6dc921b824def60x2fd45662152ae87192971a0e9b7d50de48d7bc8ab22e5711680173a302120bf00x0f528a58c839889e4bb9195e2bcbc2addb7022e47c8fb11bbdeba0a0e9c6f4cb0x0edf43ec0f277500371537a4d566f3f352d0c49bfa9d4659e07d776ffe119437"
]
proc getExpectedBlockSlotProofs*(): seq[string] =
@[
"0x0684458ea77eca59be05e368bb26b7ca318b8e836100c415e60136876c01ba170x2a66917fa49371e835376fcece0d854c77008ac1195740963b1ac4491ee1aaf1",
"0x0684458ea77eca59be05e368bb26b7ca318b8e836100c415e60136876c01ba170x2a66917fa49371e835376fcece0d854c77008ac1195740963b1ac4491ee1aaf1",
"0x03883ad2637a4c68f29bc0910400259291d9c3d730de7e3925adbf26c80b7f440x2d6a888f50b14b0c686f64c4bd0d8389cd555cdf0e3d6f387682c4722ac2a674"
]
proc getExpectedCellData*(): seq[string] =
@[
"BA".repeat(DefaultCellSize.int),
"BD".repeat(DefaultCellSize.int),
"3D".repeat(DefaultCellSize.int)
]