mirror of
https://github.com/logos-storage/logos-storage-nim.git
synced 2026-01-23 07:43:10 +00:00
move tests around
This commit is contained in:
Dmitriy Ryajov
parent
a169e7fe24
commit
f7861df6c7
@ -129,7 +129,7 @@ asyncchecksuite "Test Node - Host contracts":
|
||||
(await onStore(request, 1.u256, onBlocks)).tryGet()
|
||||
check fetchedBytes == 786432
|
||||
|
||||
for index in !builder.slotIndicies(1):
|
||||
for index in builder.slotIndicies(1):
|
||||
let
|
||||
blk = (await localStore.getBlock(verifiable.treeCid, index)).tryGet
|
||||
expiryKey = (createBlockExpirationMetadataKey(blk.cid)).tryGet
|
||||
|
||||
@ -1,329 +0,0 @@
|
||||
import std/sequtils
|
||||
import std/sugar
|
||||
import std/random
|
||||
|
||||
import pkg/questionable/results
|
||||
import pkg/constantine/math/arithmetic
|
||||
import pkg/constantine/math/io/io_fields
|
||||
import pkg/poseidon2/types
|
||||
import pkg/poseidon2/io
|
||||
import pkg/poseidon2
|
||||
import pkg/chronos
|
||||
import pkg/asynctest
|
||||
import pkg/codex/stores/cachestore
|
||||
import pkg/codex/chunker
|
||||
import pkg/codex/stores
|
||||
import pkg/codex/blocktype as bt
|
||||
import pkg/codex/contracts/requests
|
||||
import pkg/codex/contracts
|
||||
import pkg/codex/merkletree
|
||||
import pkg/codex/stores/cachestore
|
||||
|
||||
import pkg/codex/proof/datasampler
|
||||
import pkg/codex/proof/misc
|
||||
import pkg/codex/proof/types
|
||||
|
||||
import ../helpers
|
||||
import ../examples
|
||||
import testdatasampler_expected
|
||||
|
||||
let
|
||||
bytesPerBlock = 64 * 1024
|
||||
challenge: FieldElement = toF(12345)
|
||||
datasetRootHash: FieldElement = toF(6789)
|
||||
|
||||
asyncchecksuite "Test proof datasampler - components":
|
||||
let
|
||||
numberOfSlotBlocks = 16
|
||||
slot = Slot(
|
||||
request: StorageRequest(
|
||||
ask: StorageAsk(
|
||||
slots: 10,
|
||||
slotSize: u256(bytesPerBlock * numberOfSlotBlocks),
|
||||
),
|
||||
content: StorageContent(
|
||||
cid: $Cid.example
|
||||
)
|
||||
),
|
||||
slotIndex: u256(3)
|
||||
)
|
||||
|
||||
test "Number of cells is a power of two":
|
||||
# This is to check that the data used for testing is sane.
|
||||
proc isPow2(value: int): bool =
|
||||
let log2 = ceilingLog2(value)
|
||||
return (1 shl log2) == value
|
||||
|
||||
let numberOfCells = getNumberOfCellsInSlot(slot).int
|
||||
|
||||
check:
|
||||
isPow2(numberOfCells)
|
||||
|
||||
test "Extract low bits":
|
||||
proc extract(value: uint64, nBits: int): uint64 =
|
||||
let big = toF(value).toBig()
|
||||
return extractLowBits(big, nBits)
|
||||
|
||||
check:
|
||||
extract(0x88, 4) == 0x8.uint64
|
||||
extract(0x88, 7) == 0x8.uint64
|
||||
extract(0x9A, 5) == 0x1A.uint64
|
||||
extract(0x9A, 7) == 0x1A.uint64
|
||||
extract(0x1248, 10) == 0x248.uint64
|
||||
extract(0x1248, 12) == 0x248.uint64
|
||||
extract(0x1248306A560C9AC0.uint64, 10) == 0x2C0.uint64
|
||||
extract(0x1248306A560C9AC0.uint64, 12) == 0xAC0.uint64
|
||||
extract(0x1248306A560C9AC0.uint64, 50) == 0x306A560C9AC0.uint64
|
||||
extract(0x1248306A560C9AC0.uint64, 52) == 0x8306A560C9AC0.uint64
|
||||
|
||||
test "Should calculate total number of cells in Slot":
|
||||
let
|
||||
slotSizeInBytes = (slot.request.ask.slotSize).truncate(uint64)
|
||||
expectedNumberOfCells = slotSizeInBytes div CellSize
|
||||
|
||||
check:
|
||||
expectedNumberOfCells == 512
|
||||
expectedNumberOfCells == getNumberOfCellsInSlot(slot)
|
||||
|
||||
asyncchecksuite "Test proof datasampler - main":
|
||||
let
|
||||
# The number of slot blocks and number of slots, combined with
|
||||
# the bytes per block, make it so that there are exactly 256 cells
|
||||
# in the dataset.
|
||||
numberOfSlotBlocks = 4
|
||||
totalNumberOfSlots = 2
|
||||
datasetSlotIndex = 1
|
||||
localStore = CacheStore.new()
|
||||
datasetToSlotProof = MerkleProof.example
|
||||
|
||||
var
|
||||
manifest: Manifest
|
||||
manifestBlock: bt.Block
|
||||
slot: Slot
|
||||
datasetBlocks: seq[bt.Block]
|
||||
slotPoseidonTree: MerkleTree
|
||||
dataSampler: DataSampler
|
||||
|
||||
proc createDatasetBlocks(): Future[void] {.async.} =
|
||||
let numberOfCellsNeeded = (numberOfSlotBlocks * totalNumberOfSlots * bytesPerBlock).uint64 div CellSize
|
||||
var data: seq[byte] = @[]
|
||||
|
||||
# This generates a number of blocks that have different data, such that
|
||||
# Each cell in each block is unique, but nothing is random.
|
||||
for i in 0 ..< numberOfCellsNeeded:
|
||||
data = data & (i.byte).repeat(CellSize)
|
||||
|
||||
let chunker = MockChunker.new(
|
||||
dataset = data,
|
||||
chunkSize = bytesPerBlock)
|
||||
|
||||
while true:
|
||||
let chunk = await chunker.getBytes()
|
||||
if chunk.len <= 0:
|
||||
break
|
||||
let b = bt.Block.new(chunk).tryGet()
|
||||
datasetBlocks.add(b)
|
||||
discard await localStore.putBlock(b)
|
||||
|
||||
proc createManifest(): Future[void] {.async.} =
|
||||
let
|
||||
cids = datasetBlocks.mapIt(it.cid)
|
||||
tree = MerkleTree.init(cids).tryGet()
|
||||
treeCid = tree.rootCid().tryGet()
|
||||
|
||||
for index, cid in cids:
|
||||
let proof = tree.getProof(index).tryget()
|
||||
discard await localStore.putBlockCidAndProof(treeCid, index, cid, proof)
|
||||
|
||||
manifest = Manifest.new(
|
||||
treeCid = treeCid,
|
||||
blockSize = bytesPerBlock.NBytes,
|
||||
datasetSize = (bytesPerBlock * numberOfSlotBlocks * totalNumberOfSlots).NBytes)
|
||||
manifestBlock = bt.Block.new(manifest.encode().tryGet(), codec = DagPBCodec).tryGet()
|
||||
|
||||
proc createSlot(): void =
|
||||
slot = Slot(
|
||||
request: StorageRequest(
|
||||
ask: StorageAsk(
|
||||
slotSize: u256(bytesPerBlock * numberOfSlotBlocks)
|
||||
),
|
||||
content: StorageContent(
|
||||
cid: $manifestBlock.cid
|
||||
),
|
||||
),
|
||||
slotIndex: u256(datasetSlotIndex)
|
||||
)
|
||||
|
||||
proc createSlotPoseidonTree(): void =
|
||||
let
|
||||
slotSize = slot.request.ask.slotSize.truncate(uint64)
|
||||
blocksInSlot = slotSize div bytesPerBlock.uint64
|
||||
datasetSlotIndex = slot.slotIndex.truncate(uint64)
|
||||
datasetBlockIndexFirst = datasetSlotIndex * blocksInSlot
|
||||
datasetBlockIndexLast = datasetBlockIndexFirst + numberOfSlotBlocks.uint64
|
||||
slotBlocks = datasetBlocks[datasetBlockIndexFirst ..< datasetBlockIndexLast]
|
||||
slotBlockCids = slotBlocks.mapIt(it.cid)
|
||||
slotPoseidonTree = MerkleTree.init(slotBlockCids).tryGet()
|
||||
|
||||
proc createDataSampler(): Future[void] {.async.} =
|
||||
dataSampler = (await DataSampler.new(
|
||||
slot,
|
||||
localStore,
|
||||
datasetRootHash,
|
||||
slotPoseidonTree,
|
||||
datasetToSlotProof
|
||||
)).tryGet()
|
||||
|
||||
setup:
|
||||
await createDatasetBlocks()
|
||||
await createManifest()
|
||||
createSlot()
|
||||
discard await localStore.putBlock(manifestBlock)
|
||||
createSlotPoseidonTree()
|
||||
await createDataSampler()
|
||||
|
||||
test "Number of cells is a power of two":
|
||||
# This is to check that the data used for testing is sane.
|
||||
proc isPow2(value: int): bool =
|
||||
let log2 = ceilingLog2(value)
|
||||
return (1 shl log2) == value
|
||||
|
||||
let numberOfCells = getNumberOfCellsInSlot(slot).int
|
||||
|
||||
check:
|
||||
isPow2(numberOfCells)
|
||||
|
||||
let knownIndices = @[74.uint64, 41.uint64, 51.uint64]
|
||||
|
||||
test "Can find single slot-cell index":
|
||||
proc slotCellIndex(i: int): uint64 =
|
||||
let counter: FieldElement = toF(i)
|
||||
return dataSampler.findSlotCellIndex(challenge, counter)
|
||||
|
||||
proc getExpectedIndex(i: int): uint64 =
|
||||
let
|
||||
numberOfCellsInSlot = (bytesPerBlock * numberOfSlotBlocks) div CellSize.int
|
||||
slotRootHash = toF(1234) # TODO - replace with slotPoseidonTree.root when it is a poseidon tree.
|
||||
hash = Sponge.digest(@[slotRootHash, challenge, toF(i)], rate = 2)
|
||||
return extractLowBits(hash.toBig(), ceilingLog2(numberOfCellsInSlot))
|
||||
|
||||
check:
|
||||
slotCellIndex(1) == getExpectedIndex(1)
|
||||
slotCellIndex(1) == knownIndices[0]
|
||||
slotCellIndex(2) == getExpectedIndex(2)
|
||||
slotCellIndex(2) == knownIndices[1]
|
||||
slotCellIndex(3) == getExpectedIndex(3)
|
||||
slotCellIndex(3) == knownIndices[2]
|
||||
|
||||
test "Can find sequence of slot-cell indices":
|
||||
proc slotCellIndices(n: int): seq[uint64] =
|
||||
dataSampler.findSlotCellIndices(challenge, n)
|
||||
|
||||
proc getExpectedIndices(n: int): seq[uint64] =
|
||||
return collect(newSeq, (for i in 1..n: dataSampler.findSlotCellIndex(challenge, toF(i))))
|
||||
|
||||
check:
|
||||
slotCellIndices(3) == getExpectedIndices(3)
|
||||
slotCellIndices(3) == knownIndices
|
||||
|
||||
let
|
||||
bytes = newSeqWith(bytesPerBlock, rand(uint8))
|
||||
blk = bt.Block.new(bytes).tryGet()
|
||||
cell0Bytes = bytes[0..<CellSize]
|
||||
cell1Bytes = bytes[CellSize..<(CellSize*2)]
|
||||
cell2Bytes = bytes[(CellSize*2)..<(CellSize*3)]
|
||||
|
||||
test "Can get cell from block":
|
||||
let
|
||||
sample0 = dataSampler.getCellFromBlock(blk, 0)
|
||||
sample1 = dataSampler.getCellFromBlock(blk, 1)
|
||||
sample2 = dataSampler.getCellFromBlock(blk, 2)
|
||||
|
||||
check:
|
||||
sample0 == cell0Bytes
|
||||
sample1 == cell1Bytes
|
||||
sample2 == cell2Bytes
|
||||
|
||||
test "Can convert block into cells":
|
||||
let cells = dataSampler.getBlockCells(blk)
|
||||
|
||||
check:
|
||||
cells.len == (bytesPerBlock div CellSize.int)
|
||||
cells[0] == cell0Bytes
|
||||
cells[1] == cell1Bytes
|
||||
cells[2] == cell2Bytes
|
||||
|
||||
test "Can create mini tree for block cells":
|
||||
let miniTree = dataSampler.getBlockCellMiniTree(blk).tryGet()
|
||||
|
||||
let
|
||||
cell0Proof = miniTree.getProof(0).tryGet()
|
||||
cell1Proof = miniTree.getProof(1).tryGet()
|
||||
cell2Proof = miniTree.getProof(2).tryGet()
|
||||
|
||||
check:
|
||||
cell0Proof.verifyDataBlock(cell0Bytes, miniTree.root).tryGet()
|
||||
cell1Proof.verifyDataBlock(cell1Bytes, miniTree.root).tryGet()
|
||||
cell2Proof.verifyDataBlock(cell2Bytes, miniTree.root).tryGet()
|
||||
|
||||
test "Can gather proof input":
|
||||
# This is the main entry point for this module, and what it's all about.
|
||||
let
|
||||
nSamples = 3
|
||||
input = (await dataSampler.getProofInput(challenge, nSamples)).tryget()
|
||||
|
||||
proc equal(a: FieldElement, b: FieldElement): bool =
|
||||
a.toDecimal() == b.toDecimal()
|
||||
|
||||
proc toStr(proof: MerkleProof): string =
|
||||
toHex(proof.nodesBuffer)
|
||||
|
||||
let
|
||||
expectedMerkleProofs = getExpectedSlotToBlockProofs()
|
||||
expectedCellData = getExpectedCellData()
|
||||
|
||||
check:
|
||||
# datasetRoot*: FieldElement
|
||||
equal(input.datasetRoot, datasetRootHash)
|
||||
# entropy*: FieldElement
|
||||
equal(input.entropy, challenge)
|
||||
# numberOfCellsInSlot*: uint64
|
||||
input.numberOfCellsInSlot == (bytesPerBlock * numberOfSlotBlocks).uint64 div CellSize
|
||||
# numberOfSlots*: uint64
|
||||
input.numberOfSlots == slot.request.ask.slots
|
||||
# datasetSlotIndex*: uint64
|
||||
input.datasetSlotIndex == slot.slotIndex.truncate(uint64)
|
||||
# slotRoot*: FieldElement
|
||||
equal(input.slotRoot, toF(1234)) # TODO - when slotPoseidonTree is a poseidon tree, its root should be a FieldElement.
|
||||
# datasetToSlotProof*: MerkleProof
|
||||
input.datasetToSlotProof == datasetToSlotProof
|
||||
# proofSamples*: seq[ProofSample]
|
||||
toStr(input.proofSamples[0].merkleProof) == expectedMerkleProofs[0]
|
||||
toStr(input.proofSamples[1].merkleProof) == expectedMerkleProofs[1]
|
||||
toStr(input.proofSamples[2].merkleProof) == expectedMerkleProofs[2]
|
||||
# cell data
|
||||
toHex(input.proofSamples[0].cellData) == expectedCellData[0]
|
||||
toHex(input.proofSamples[1].cellData) == expectedCellData[1]
|
||||
toHex(input.proofSamples[2].cellData) == expectedCellData[2]
|
||||
|
||||
# input.slotToBlockProofs.mapIt(toStr(it)) == expectedSlotToBlockProofs
|
||||
# input.blockToCellProofs.mapIt(toStr(it)) == expectedBlockToCellProofs
|
||||
# toHex(input.sampleData) == expectedSampleData
|
||||
|
||||
for (input, expected) in [(10, 0), (31, 0), (32, 1), (63, 1), (64, 2)]:
|
||||
test "Can get slotBlockIndex from slotCellIndex (" & $input & " -> " & $expected & ")":
|
||||
let
|
||||
slotCellIndex = input.uint64
|
||||
slotBlockIndex = dataSampler.getSlotBlockIndexForSlotCellIndex(slotCellIndex)
|
||||
|
||||
check:
|
||||
slotBlockIndex == expected.uint64
|
||||
|
||||
for (input, expected) in [(10, 10), (31, 31), (32, 0), (63, 31), (64, 0)]:
|
||||
test "Can get blockCellIndex from slotCellIndex (" & $input & " -> " & $expected & ")":
|
||||
let
|
||||
slotCellIndex = input.uint64
|
||||
blockCellIndex = dataSampler.getBlockCellIndexForSlotCellIndex(slotCellIndex)
|
||||
|
||||
check:
|
||||
blockCellIndex == expected.uint64
|
||||
@ -1,23 +0,0 @@
|
||||
import pkg/chronos
|
||||
import pkg/asynctest
|
||||
import pkg/codex/proof/indexing
|
||||
import pkg/codex/contracts/requests
|
||||
import ../helpers
|
||||
|
||||
let
|
||||
bytesPerBlock = 64 * 1024
|
||||
numberOfSlotBlocks = 16
|
||||
blockSize = bytesPerBlock.uint64
|
||||
slot = Slot(
|
||||
request: StorageRequest(
|
||||
ask: StorageAsk(
|
||||
slots: 10,
|
||||
slotSize: u256(bytesPerBlock * numberOfSlotBlocks)
|
||||
),
|
||||
content: StorageContent(),
|
||||
),
|
||||
slotIndex: u256(3)
|
||||
)
|
||||
|
||||
checksuite "Test indexing":
|
||||
|
||||
@ -1,177 +0,0 @@
|
||||
import std/sequtils
|
||||
|
||||
import pkg/chronos
|
||||
import pkg/asynctest
|
||||
import pkg/codex/rng
|
||||
import pkg/codex/stores/cachestore
|
||||
import pkg/codex/chunker
|
||||
import pkg/codex/stores
|
||||
import pkg/codex/blocktype as bt
|
||||
import pkg/codex/contracts/requests
|
||||
import pkg/codex/contracts
|
||||
import pkg/codex/merkletree
|
||||
|
||||
import pkg/codex/proof/slotblocks
|
||||
|
||||
import ../helpers
|
||||
import ../examples
|
||||
|
||||
let
|
||||
bytesPerBlock = 64 * 1024
|
||||
numberOfSlotBlocks = 4
|
||||
datasetSlotIndex = 3
|
||||
|
||||
asyncchecksuite "Test slotblocks - manifest":
|
||||
let
|
||||
localStore = CacheStore.new()
|
||||
manifest = Manifest.new(
|
||||
treeCid = Cid.example,
|
||||
blockSize = 1.MiBs,
|
||||
datasetSize = 100.MiBs)
|
||||
|
||||
var
|
||||
manifestBlock = bt.Block.new(manifest.encode().tryGet(), codec = DagPBCodec).tryGet()
|
||||
slot = Slot(
|
||||
request: StorageRequest(
|
||||
ask: StorageAsk(
|
||||
slotSize: u256(bytesPerBlock * numberOfSlotBlocks)
|
||||
),
|
||||
content: StorageContent(
|
||||
cid: $manifestBlock.cid
|
||||
),
|
||||
),
|
||||
slotIndex: u256(datasetSlotIndex)
|
||||
)
|
||||
|
||||
setup:
|
||||
discard await localStore.putBlock(manifestBlock)
|
||||
|
||||
proc getManifest(store: BlockStore): Future[?!Manifest] {.async.} =
|
||||
without slotBlocks =? await SlotBlocks.new(slot, store), err:
|
||||
return failure(err)
|
||||
return success(slotBlocks.manifest)
|
||||
|
||||
test "Can get manifest for slot":
|
||||
let m = (await getManifest(localStore)).tryGet()
|
||||
|
||||
check:
|
||||
m.treeCid == manifest.treeCid
|
||||
|
||||
test "Can fail to get manifest for invalid cid":
|
||||
slot.request.content.cid = "invalid"
|
||||
let m = (await getManifest(localStore))
|
||||
|
||||
check:
|
||||
m.isErr
|
||||
|
||||
test "Can fail to get manifest when manifest block not found":
|
||||
let
|
||||
emptyStore = CacheStore.new()
|
||||
m = (await getManifest(emptyStore))
|
||||
|
||||
check:
|
||||
m.isErr
|
||||
|
||||
test "Can fail to get manifest when manifest fails to decode":
|
||||
manifestBlock.data = @[]
|
||||
|
||||
let m = (await getManifest(localStore))
|
||||
|
||||
check:
|
||||
m.isErr
|
||||
|
||||
|
||||
asyncchecksuite "Test slotblocks - slot blocks by index":
|
||||
let
|
||||
totalNumberOfSlots = 4
|
||||
localStore = CacheStore.new()
|
||||
chunker = RandomChunker.new(rng.Rng.instance(),
|
||||
size = bytesPerBlock * numberOfSlotBlocks * totalNumberOfSlots,
|
||||
chunkSize = bytesPerBlock)
|
||||
|
||||
var
|
||||
manifest: Manifest
|
||||
manifestBlock: bt.Block
|
||||
slot: Slot
|
||||
datasetBlocks: seq[bt.Block]
|
||||
slotBlocks: SlotBlocks
|
||||
|
||||
proc createDatasetBlocks(): Future[void] {.async.} =
|
||||
while true:
|
||||
let chunk = await chunker.getBytes()
|
||||
if chunk.len <= 0:
|
||||
break
|
||||
let b = bt.Block.new(chunk).tryGet()
|
||||
datasetBlocks.add(b)
|
||||
discard await localStore.putBlock(b)
|
||||
|
||||
proc createManifest(): Future[void] {.async.} =
|
||||
let
|
||||
cids = datasetBlocks.mapIt(it.cid)
|
||||
tree = MerkleTree.init(cids).tryGet()
|
||||
treeCid = tree.rootCid().tryGet()
|
||||
|
||||
for index, cid in cids:
|
||||
let proof = tree.getProof(index).tryget()
|
||||
discard await localStore.putBlockCidAndProof(treeCid, index, cid, proof)
|
||||
|
||||
manifest = Manifest.new(
|
||||
treeCid = treeCid,
|
||||
blockSize = bytesPerBlock.NBytes,
|
||||
datasetSize = (bytesPerBlock * numberOfSlotBlocks * totalNumberOfSlots).NBytes)
|
||||
manifestBlock = bt.Block.new(manifest.encode().tryGet(), codec = DagPBCodec).tryGet()
|
||||
|
||||
proc createSlot(): void =
|
||||
slot = Slot(
|
||||
request: StorageRequest(
|
||||
ask: StorageAsk(
|
||||
slotSize: u256(bytesPerBlock * numberOfSlotBlocks)
|
||||
),
|
||||
content: StorageContent(
|
||||
cid: $manifestBlock.cid
|
||||
),
|
||||
),
|
||||
slotIndex: u256(datasetSlotIndex)
|
||||
)
|
||||
|
||||
proc createSlotBlocks(): Future[void] {.async.} =
|
||||
slotBlocks = (await SlotBlocks.new(slot, localStore)).tryGet()
|
||||
|
||||
setup:
|
||||
await createDatasetBlocks()
|
||||
await createManifest()
|
||||
createSlot()
|
||||
discard await localStore.putBlock(manifestBlock)
|
||||
await createSlotBlocks()
|
||||
|
||||
for input in 0 ..< numberOfSlotBlocks:
|
||||
test "Can get datasetBlockIndex from slotBlockIndex (" & $input & ")":
|
||||
let
|
||||
slotBlockIndex = input.uint64
|
||||
datasetBlockIndex = slotBlocks.getDatasetBlockIndexForSlotBlockIndex(slotBlockIndex)
|
||||
datasetSlotIndex = slot.slotIndex.truncate(uint64)
|
||||
expectedIndex = (numberOfSlotBlocks.uint64 * datasetSlotIndex) + slotBlockIndex
|
||||
|
||||
check:
|
||||
datasetBlockIndex == expectedIndex
|
||||
|
||||
for input in [0, 1, numberOfSlotBlocks-1]:
|
||||
test "Can get slot block by index (" & $input & ")":
|
||||
let
|
||||
slotBlockIndex = input.uint64
|
||||
slotBlock = (await slotBlocks.getSlotBlock(slotBlockIndex)).tryget()
|
||||
expectedDatasetBlockIndex = slotBlocks.getDatasetBlockIndexForSlotBlockIndex(slotBlockIndex)
|
||||
expectedBlock = datasetBlocks[expectedDatasetBlockIndex]
|
||||
|
||||
check:
|
||||
slotBlock.cid == expectedBlock.cid
|
||||
slotBlock.data == expectedBlock.data
|
||||
|
||||
test "Can fail to get block when index is out of range":
|
||||
let
|
||||
b1 = await slotBlocks.getSlotBlock(numberOfSlotBlocks.uint64)
|
||||
b2 = await slotBlocks.getSlotBlock((numberOfSlotBlocks + 1).uint64)
|
||||
|
||||
check:
|
||||
b1.isErr
|
||||
b2.isErr
|
||||
329
tests/codex/slots/proof/testdatasampler.nim
Normal file
329
tests/codex/slots/proof/testdatasampler.nim
Normal file
@ -0,0 +1,329 @@
|
||||
# import std/sequtils
|
||||
# import std/sugar
|
||||
# import std/random
|
||||
|
||||
# import pkg/questionable/results
|
||||
# import pkg/constantine/math/arithmetic
|
||||
# import pkg/constantine/math/io/io_fields
|
||||
# import pkg/poseidon2/types
|
||||
# import pkg/poseidon2/io
|
||||
# import pkg/poseidon2
|
||||
# import pkg/chronos
|
||||
# import pkg/asynctest
|
||||
# import pkg/codex/stores/cachestore
|
||||
# import pkg/codex/chunker
|
||||
# import pkg/codex/stores
|
||||
# import pkg/codex/blocktype as bt
|
||||
# import pkg/codex/contracts/requests
|
||||
# import pkg/codex/contracts
|
||||
# import pkg/codex/merkletree
|
||||
# import pkg/codex/stores/cachestore
|
||||
|
||||
# import pkg/codex/slots
|
||||
# import pkg/codex/proof/misc
|
||||
# import pkg/codex/proof/types
|
||||
|
||||
# import ../helpers
|
||||
# import ../examples
|
||||
# import testdatasampler_expected
|
||||
|
||||
# let
|
||||
# bytesPerBlock = 64 * 1024
|
||||
# challenge: FieldElement = toF(12345)
|
||||
# datasetRootHash: FieldElement = toF(6789)
|
||||
|
||||
# asyncchecksuite "Test proof datasampler - components":
|
||||
# let
|
||||
# numberOfSlotBlocks = 16
|
||||
# slot = Slot(
|
||||
# request: StorageRequest(
|
||||
# ask: StorageAsk(
|
||||
# slots: 10,
|
||||
# slotSize: u256(bytesPerBlock * numberOfSlotBlocks),
|
||||
# ),
|
||||
# content: StorageContent(
|
||||
# cid: $Cid.example
|
||||
# )
|
||||
# ),
|
||||
# slotIndex: u256(3)
|
||||
# )
|
||||
|
||||
# test "Number of cells is a power of two":
|
||||
# # This is to check that the data used for testing is sane.
|
||||
# proc isPow2(value: int): bool =
|
||||
# let log2 = ceilingLog2(value)
|
||||
# return (1 shl log2) == value
|
||||
|
||||
# let numberOfCells = getNumberOfCellsInSlot(slot).int
|
||||
|
||||
# check:
|
||||
# isPow2(numberOfCells)
|
||||
|
||||
# test "Extract low bits":
|
||||
# proc extract(value: uint64, nBits: int): uint64 =
|
||||
# let big = toF(value).toBig()
|
||||
# return extractLowBits(big, nBits)
|
||||
|
||||
# check:
|
||||
# extract(0x88, 4) == 0x8.uint64
|
||||
# extract(0x88, 7) == 0x8.uint64
|
||||
# extract(0x9A, 5) == 0x1A.uint64
|
||||
# extract(0x9A, 7) == 0x1A.uint64
|
||||
# extract(0x1248, 10) == 0x248.uint64
|
||||
# extract(0x1248, 12) == 0x248.uint64
|
||||
# extract(0x1248306A560C9AC0.uint64, 10) == 0x2C0.uint64
|
||||
# extract(0x1248306A560C9AC0.uint64, 12) == 0xAC0.uint64
|
||||
# extract(0x1248306A560C9AC0.uint64, 50) == 0x306A560C9AC0.uint64
|
||||
# extract(0x1248306A560C9AC0.uint64, 52) == 0x8306A560C9AC0.uint64
|
||||
|
||||
# test "Should calculate total number of cells in Slot":
|
||||
# let
|
||||
# slotSizeInBytes = (slot.request.ask.slotSize).truncate(uint64)
|
||||
# expectedNumberOfCells = slotSizeInBytes div CellSize
|
||||
|
||||
# check:
|
||||
# expectedNumberOfCells == 512
|
||||
# expectedNumberOfCells == getNumberOfCellsInSlot(slot)
|
||||
|
||||
# asyncchecksuite "Test proof datasampler - main":
|
||||
# let
|
||||
# # The number of slot blocks and number of slots, combined with
|
||||
# # the bytes per block, make it so that there are exactly 256 cells
|
||||
# # in the dataset.
|
||||
# numberOfSlotBlocks = 4
|
||||
# totalNumberOfSlots = 2
|
||||
# datasetSlotIndex = 1
|
||||
# localStore = CacheStore.new()
|
||||
# datasetToSlotProof = MerkleProof.example
|
||||
|
||||
# var
|
||||
# manifest: Manifest
|
||||
# manifestBlock: bt.Block
|
||||
# slot: Slot
|
||||
# datasetBlocks: seq[bt.Block]
|
||||
# slotPoseidonTree: MerkleTree
|
||||
# dataSampler: DataSampler
|
||||
|
||||
# proc createDatasetBlocks(): Future[void] {.async.} =
|
||||
# let numberOfCellsNeeded = (numberOfSlotBlocks * totalNumberOfSlots * bytesPerBlock).uint64 div CellSize
|
||||
# var data: seq[byte] = @[]
|
||||
|
||||
# # This generates a number of blocks that have different data, such that
|
||||
# # Each cell in each block is unique, but nothing is random.
|
||||
# for i in 0 ..< numberOfCellsNeeded:
|
||||
# data = data & (i.byte).repeat(CellSize)
|
||||
|
||||
# let chunker = MockChunker.new(
|
||||
# dataset = data,
|
||||
# chunkSize = bytesPerBlock)
|
||||
|
||||
# while true:
|
||||
# let chunk = await chunker.getBytes()
|
||||
# if chunk.len <= 0:
|
||||
# break
|
||||
# let b = bt.Block.new(chunk).tryGet()
|
||||
# datasetBlocks.add(b)
|
||||
# discard await localStore.putBlock(b)
|
||||
|
||||
# proc createManifest(): Future[void] {.async.} =
|
||||
# let
|
||||
# cids = datasetBlocks.mapIt(it.cid)
|
||||
# tree = MerkleTree.init(cids).tryGet()
|
||||
# treeCid = tree.rootCid().tryGet()
|
||||
|
||||
# for index, cid in cids:
|
||||
# let proof = tree.getProof(index).tryget()
|
||||
# discard await localStore.putBlockCidAndProof(treeCid, index, cid, proof)
|
||||
|
||||
# manifest = Manifest.new(
|
||||
# treeCid = treeCid,
|
||||
# blockSize = bytesPerBlock.NBytes,
|
||||
# datasetSize = (bytesPerBlock * numberOfSlotBlocks * totalNumberOfSlots).NBytes)
|
||||
# manifestBlock = bt.Block.new(manifest.encode().tryGet(), codec = DagPBCodec).tryGet()
|
||||
|
||||
# proc createSlot(): void =
|
||||
# slot = Slot(
|
||||
# request: StorageRequest(
|
||||
# ask: StorageAsk(
|
||||
# slotSize: u256(bytesPerBlock * numberOfSlotBlocks)
|
||||
# ),
|
||||
# content: StorageContent(
|
||||
# cid: $manifestBlock.cid
|
||||
# ),
|
||||
# ),
|
||||
# slotIndex: u256(datasetSlotIndex)
|
||||
# )
|
||||
|
||||
# proc createSlotPoseidonTree(): void =
|
||||
# let
|
||||
# slotSize = slot.request.ask.slotSize.truncate(uint64)
|
||||
# blocksInSlot = slotSize div bytesPerBlock.uint64
|
||||
# datasetSlotIndex = slot.slotIndex.truncate(uint64)
|
||||
# datasetBlockIndexFirst = datasetSlotIndex * blocksInSlot
|
||||
# datasetBlockIndexLast = datasetBlockIndexFirst + numberOfSlotBlocks.uint64
|
||||
# slotBlocks = datasetBlocks[datasetBlockIndexFirst ..< datasetBlockIndexLast]
|
||||
# slotBlockCids = slotBlocks.mapIt(it.cid)
|
||||
# slotPoseidonTree = MerkleTree.init(slotBlockCids).tryGet()
|
||||
|
||||
# proc createDataSampler(): Future[void] {.async.} =
|
||||
# dataSampler = (await DataSampler.new(
|
||||
# slot,
|
||||
# localStore,
|
||||
# datasetRootHash,
|
||||
# slotPoseidonTree,
|
||||
# datasetToSlotProof
|
||||
# )).tryGet()
|
||||
|
||||
# setup:
|
||||
# await createDatasetBlocks()
|
||||
# await createManifest()
|
||||
# createSlot()
|
||||
# discard await localStore.putBlock(manifestBlock)
|
||||
# createSlotPoseidonTree()
|
||||
# await createDataSampler()
|
||||
|
||||
# test "Number of cells is a power of two":
|
||||
# # This is to check that the data used for testing is sane.
|
||||
# proc isPow2(value: int): bool =
|
||||
# let log2 = ceilingLog2(value)
|
||||
# return (1 shl log2) == value
|
||||
|
||||
# let numberOfCells = getNumberOfCellsInSlot(slot).int
|
||||
|
||||
# check:
|
||||
# isPow2(numberOfCells)
|
||||
|
||||
# let knownIndices = @[74.uint64, 41.uint64, 51.uint64]
|
||||
|
||||
# test "Can find single slot-cell index":
|
||||
# proc slotCellIndex(i: int): uint64 =
|
||||
# let counter: FieldElement = toF(i)
|
||||
# return dataSampler.findSlotCellIndex(challenge, counter)
|
||||
|
||||
# proc getExpectedIndex(i: int): uint64 =
|
||||
# let
|
||||
# numberOfCellsInSlot = (bytesPerBlock * numberOfSlotBlocks) div CellSize.int
|
||||
# slotRootHash = toF(1234) # TODO - replace with slotPoseidonTree.root when it is a poseidon tree.
|
||||
# hash = Sponge.digest(@[slotRootHash, challenge, toF(i)], rate = 2)
|
||||
# return extractLowBits(hash.toBig(), ceilingLog2(numberOfCellsInSlot))
|
||||
|
||||
# check:
|
||||
# slotCellIndex(1) == getExpectedIndex(1)
|
||||
# slotCellIndex(1) == knownIndices[0]
|
||||
# slotCellIndex(2) == getExpectedIndex(2)
|
||||
# slotCellIndex(2) == knownIndices[1]
|
||||
# slotCellIndex(3) == getExpectedIndex(3)
|
||||
# slotCellIndex(3) == knownIndices[2]
|
||||
|
||||
# test "Can find sequence of slot-cell indices":
|
||||
# proc slotCellIndices(n: int): seq[uint64] =
|
||||
# dataSampler.findSlotCellIndices(challenge, n)
|
||||
|
||||
# proc getExpectedIndices(n: int): seq[uint64] =
|
||||
# return collect(newSeq, (for i in 1..n: dataSampler.findSlotCellIndex(challenge, toF(i))))
|
||||
|
||||
# check:
|
||||
# slotCellIndices(3) == getExpectedIndices(3)
|
||||
# slotCellIndices(3) == knownIndices
|
||||
|
||||
# let
|
||||
# bytes = newSeqWith(bytesPerBlock, rand(uint8))
|
||||
# blk = bt.Block.new(bytes).tryGet()
|
||||
# cell0Bytes = bytes[0..<CellSize]
|
||||
# cell1Bytes = bytes[CellSize..<(CellSize*2)]
|
||||
# cell2Bytes = bytes[(CellSize*2)..<(CellSize*3)]
|
||||
|
||||
# test "Can get cell from block":
|
||||
# let
|
||||
# sample0 = dataSampler.getCellFromBlock(blk, 0)
|
||||
# sample1 = dataSampler.getCellFromBlock(blk, 1)
|
||||
# sample2 = dataSampler.getCellFromBlock(blk, 2)
|
||||
|
||||
# check:
|
||||
# sample0 == cell0Bytes
|
||||
# sample1 == cell1Bytes
|
||||
# sample2 == cell2Bytes
|
||||
|
||||
# test "Can convert block into cells":
|
||||
# let cells = dataSampler.getBlockCells(blk)
|
||||
|
||||
# check:
|
||||
# cells.len == (bytesPerBlock div CellSize.int)
|
||||
# cells[0] == cell0Bytes
|
||||
# cells[1] == cell1Bytes
|
||||
# cells[2] == cell2Bytes
|
||||
|
||||
# test "Can create mini tree for block cells":
|
||||
# let miniTree = dataSampler.getBlockCellMiniTree(blk).tryGet()
|
||||
|
||||
# let
|
||||
# cell0Proof = miniTree.getProof(0).tryGet()
|
||||
# cell1Proof = miniTree.getProof(1).tryGet()
|
||||
# cell2Proof = miniTree.getProof(2).tryGet()
|
||||
|
||||
# check:
|
||||
# cell0Proof.verifyDataBlock(cell0Bytes, miniTree.root).tryGet()
|
||||
# cell1Proof.verifyDataBlock(cell1Bytes, miniTree.root).tryGet()
|
||||
# cell2Proof.verifyDataBlock(cell2Bytes, miniTree.root).tryGet()
|
||||
|
||||
# test "Can gather proof input":
|
||||
# # This is the main entry point for this module, and what it's all about.
|
||||
# let
|
||||
# nSamples = 3
|
||||
# input = (await dataSampler.getProofInput(challenge, nSamples)).tryget()
|
||||
|
||||
# proc equal(a: FieldElement, b: FieldElement): bool =
|
||||
# a.toDecimal() == b.toDecimal()
|
||||
|
||||
# proc toStr(proof: MerkleProof): string =
|
||||
# toHex(proof.nodesBuffer)
|
||||
|
||||
# let
|
||||
# expectedMerkleProofs = getExpectedSlotToBlockProofs()
|
||||
# expectedCellData = getExpectedCellData()
|
||||
|
||||
# check:
|
||||
# # datasetRoot*: FieldElement
|
||||
# equal(input.datasetRoot, datasetRootHash)
|
||||
# # entropy*: FieldElement
|
||||
# equal(input.entropy, challenge)
|
||||
# # numberOfCellsInSlot*: uint64
|
||||
# input.numberOfCellsInSlot == (bytesPerBlock * numberOfSlotBlocks).uint64 div CellSize
|
||||
# # numberOfSlots*: uint64
|
||||
# input.numberOfSlots == slot.request.ask.slots
|
||||
# # datasetSlotIndex*: uint64
|
||||
# input.datasetSlotIndex == slot.slotIndex.truncate(uint64)
|
||||
# # slotRoot*: FieldElement
|
||||
# equal(input.slotRoot, toF(1234)) # TODO - when slotPoseidonTree is a poseidon tree, its root should be a FieldElement.
|
||||
# # datasetToSlotProof*: MerkleProof
|
||||
# input.datasetToSlotProof == datasetToSlotProof
|
||||
# # proofSamples*: seq[ProofSample]
|
||||
# toStr(input.proofSamples[0].merkleProof) == expectedMerkleProofs[0]
|
||||
# toStr(input.proofSamples[1].merkleProof) == expectedMerkleProofs[1]
|
||||
# toStr(input.proofSamples[2].merkleProof) == expectedMerkleProofs[2]
|
||||
# # cell data
|
||||
# toHex(input.proofSamples[0].cellData) == expectedCellData[0]
|
||||
# toHex(input.proofSamples[1].cellData) == expectedCellData[1]
|
||||
# toHex(input.proofSamples[2].cellData) == expectedCellData[2]
|
||||
|
||||
# # input.slotToBlockProofs.mapIt(toStr(it)) == expectedSlotToBlockProofs
|
||||
# # input.blockToCellProofs.mapIt(toStr(it)) == expectedBlockToCellProofs
|
||||
# # toHex(input.sampleData) == expectedSampleData
|
||||
|
||||
# for (input, expected) in [(10, 0), (31, 0), (32, 1), (63, 1), (64, 2)]:
|
||||
# test "Can get slotBlockIndex from slotCellIndex (" & $input & " -> " & $expected & ")":
|
||||
# let
|
||||
# slotCellIndex = input.uint64
|
||||
# slotBlockIndex = dataSampler.getSlotBlockIndexForSlotCellIndex(slotCellIndex)
|
||||
|
||||
# check:
|
||||
# slotBlockIndex == expected.uint64
|
||||
|
||||
# for (input, expected) in [(10, 10), (31, 31), (32, 0), (63, 31), (64, 0)]:
|
||||
# test "Can get blockCellIndex from slotCellIndex (" & $input & " -> " & $expected & ")":
|
||||
# let
|
||||
# slotCellIndex = input.uint64
|
||||
# blockCellIndex = dataSampler.getBlockCellIndexForSlotCellIndex(slotCellIndex)
|
||||
|
||||
# check:
|
||||
# blockCellIndex == expected.uint64
|
||||
@ -31,23 +31,23 @@ privateAccess(Manifest) # enable access to private fields
|
||||
|
||||
suite "Slot builder":
|
||||
let
|
||||
blockSize = 1024
|
||||
cellSize = 64
|
||||
blockSize = NBytes 1024
|
||||
cellSize = NBytes 64
|
||||
ecK = 3
|
||||
ecM = 2
|
||||
|
||||
numSlots = ecK + ecM
|
||||
numDatasetBlocks = 100
|
||||
numBlockCells = blockSize div cellSize
|
||||
numBlockCells = (blockSize div cellSize).int
|
||||
|
||||
numTotalBlocks = calcEcBlocksCount(numDatasetBlocks, ecK, ecM) # total number of blocks in the dataset after
|
||||
# EC (should will match number of slots)
|
||||
originalDatasetSize = numDatasetBlocks * blockSize # size of the dataset before EC
|
||||
totalDatasetSize = numTotalBlocks * blockSize # size of the dataset after EC
|
||||
originalDatasetSize = numDatasetBlocks * blockSize.int # size of the dataset before EC
|
||||
totalDatasetSize = numTotalBlocks * blockSize.int # size of the dataset after EC
|
||||
numTotalSlotBlocks = nextPowerOfTwo(numTotalBlocks div numSlots)
|
||||
|
||||
blockPadBytes =
|
||||
newSeq[byte](numBlockCells.nextPowerOfTwoPad * cellSize) # power of two padding for blocks
|
||||
newSeq[byte](numBlockCells.nextPowerOfTwoPad * cellSize.int) # power of two padding for blocks
|
||||
|
||||
slotsPadLeafs =
|
||||
newSeqWith((numTotalBlocks div numSlots).nextPowerOfTwoPad, Poseidon2Zero) # power of two padding for block roots
|
||||
@ -215,7 +215,7 @@ suite "Slot builder":
|
||||
|
||||
expectedHashes: seq[Poseidon2Hash] = collect(newSeq):
|
||||
for blk in expectedBlock:
|
||||
SpongeMerkle.digest(blk.data & blockPadBytes, cellSize)
|
||||
SpongeMerkle.digest(blk.data & blockPadBytes, cellSize.int)
|
||||
|
||||
cellHashes = (await slotBuilder.getCellHashes(i)).tryGet()
|
||||
|
||||
@ -238,7 +238,7 @@ suite "Slot builder":
|
||||
|
||||
expectedHashes: seq[Poseidon2Hash] = collect(newSeq):
|
||||
for blk in expectedBlock:
|
||||
SpongeMerkle.digest(blk.data & blockPadBytes, cellSize)
|
||||
SpongeMerkle.digest(blk.data & blockPadBytes, cellSize.int)
|
||||
expectedRoot = Merkle.digest(expectedHashes & slotsPadLeafs)
|
||||
|
||||
slotTree = (await slotBuilder.buildSlotTree(i)).tryGet()
|
||||
@ -289,12 +289,12 @@ suite "Slot builder":
|
||||
|
||||
slotHashes: seq[Poseidon2Hash] = collect(newSeq):
|
||||
for blk in expectedBlocks:
|
||||
SpongeMerkle.digest(blk.data & blockPadBytes, cellSize)
|
||||
SpongeMerkle.digest(blk.data & blockPadBytes, cellSize.int)
|
||||
|
||||
Merkle.digest(slotHashes & slotsPadLeafs)
|
||||
|
||||
expectedRoot = Merkle.digest(slotsHashes & rootsPadLeafs)
|
||||
rootHash = slotBuilder.buildRootsTree(slotBuilder.slotRoots).tryGet().root.tryGet()
|
||||
rootHash = slotBuilder.buildVerifyTree(slotBuilder.slotRoots).tryGet().root.tryGet()
|
||||
|
||||
check:
|
||||
expectedRoot == rootHash
|
||||
@ -316,7 +316,7 @@ suite "Slot builder":
|
||||
|
||||
slotHashes: seq[Poseidon2Hash] = collect(newSeq):
|
||||
for blk in expectedBlocks:
|
||||
SpongeMerkle.digest(blk.data & blockPadBytes, cellSize)
|
||||
SpongeMerkle.digest(blk.data & blockPadBytes, cellSize.int)
|
||||
|
||||
Merkle.digest(slotHashes & slotsPadLeafs)
|
||||
|
||||
@ -1,5 +0,0 @@
|
||||
import ./proof/testdatasampler
|
||||
import ./proof/testslotblocks
|
||||
import ./proof/testindexing
|
||||
|
||||
{.warning[UnusedImport]: off.}
|
||||
@ -1,3 +0,0 @@
|
||||
import ./slotbuilder/testslotbuilder
|
||||
|
||||
{.warning[UnusedImport]: off.}
|
||||
4
tests/codex/testslots.nim
Normal file
4
tests/codex/testslots.nim
Normal file
@ -0,0 +1,4 @@
|
||||
import ./slots/testslotbuilder
|
||||
import ./slots/proof/testdatasampler
|
||||
|
||||
{.warning[UnusedImport]: off.}
|
||||
@ -6,7 +6,6 @@ import ./codex/testmanifest
|
||||
import ./codex/testnode
|
||||
import ./codex/teststorestream
|
||||
import ./codex/testpurchasing
|
||||
import ./codex/testproof
|
||||
import ./codex/testsales
|
||||
import ./codex/testerasure
|
||||
import ./codex/testutils
|
||||
@ -15,7 +14,7 @@ import ./codex/testsystemclock
|
||||
import ./codex/testvalidation
|
||||
import ./codex/testasyncstreamwrapper
|
||||
import ./codex/testmerkletree
|
||||
import ./codex/testslotbuilder
|
||||
import ./codex/testslots
|
||||
import ./codex/testindexingstrategy
|
||||
|
||||
{.warning[UnusedImport]: off.}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user