Restores tests for proof sampler utils
This commit is contained in:
benbierens
Dmitriy Ryajov
parent
d80ddd41df
commit
4e19384a86
|
|
@ -30,8 +30,7 @@ import ../../utils
|
|||
import ../../utils/asynciter
|
||||
import ../../utils/digest
|
||||
import ../../utils/poseidon2digest
|
||||
|
||||
import ./converters
|
||||
import ../converters
|
||||
|
||||
export converters
|
||||
|
||||
|
|
@ -297,7 +296,7 @@ proc buildManifest*(self: SlotsBuilder): Future[?!Manifest] {.async.} =
|
|||
error "Failed to map slot roots to CIDs", err = err.msg
|
||||
return failure(err)
|
||||
|
||||
without rootProvingCidRes =? self.verifyRoot.?toSlotsRootsCid() and
|
||||
without rootProvingCidRes =? self.verifyRoot.?toVerifyCid() and
|
||||
rootProvingCid =? rootProvingCidRes, err: # TODO: why doesn't `.?` unpack the result?
|
||||
error "Failed to map slot roots to CIDs", err = err.msg
|
||||
return failure(err)
|
||||
|
|
@ -350,15 +349,15 @@ proc new*(
|
|||
return failure "Manifest is verifiable but slot roots are missing or invalid."
|
||||
|
||||
let
|
||||
slotRoot = ? Poseidon2Hash.fromBytes(
|
||||
( ? manifest.verifyRoot.mhash.mapFailure ).digestBytes.toArray32
|
||||
).toFailure
|
||||
# slotRoot = ? Poseidon2Hash.fromBytes(
|
||||
# ( ? manifest.verifyRoot.mhash.mapFailure ).digestBytes.toArray32
|
||||
# ).toFailure
|
||||
|
||||
slotRoots = manifest.slotRoots.mapIt(
|
||||
? Poseidon2Hash.fromBytes(
|
||||
( ? it.mhash.mapFailure ).digestBytes.toArray32
|
||||
).toFailure
|
||||
)
|
||||
slotRoots = manifest.slotRoots.mapIt(it.fromSlotCid().toFailure)
|
||||
# ? Poseidon2Hash.fromBytes(
|
||||
# ( ? it.mhash.mapFailure ).digestBytes.toArray32
|
||||
# ).toFailure
|
||||
# )
|
||||
|
||||
without tree =? self.buildVerifyTree(slotRoots), err:
|
||||
error "Failed to build slot roots tree", err = err.msg
|
||||
|
|
|
|||
|
|
@ -1,69 +0,0 @@
|
|||
## Nim-Codex
|
||||
## Copyright (c) 2024 Status Research & Development GmbH
|
||||
## Licensed under either of
|
||||
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
|
||||
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
|
||||
## at your option.
|
||||
## This file may not be copied, modified, or distributed except according to
|
||||
## those terms.
|
||||
|
||||
import std/sequtils
|
||||
|
||||
import pkg/libp2p
|
||||
import pkg/questionable
|
||||
import pkg/questionable/results
|
||||
import pkg/poseidon2
|
||||
import pkg/poseidon2/io
|
||||
|
||||
import ../../codextypes
|
||||
import ../../merkletree
|
||||
import ../../errors
|
||||
|
||||
func toCellCid*(cell: Poseidon2Hash): ?!Cid =
|
||||
let
|
||||
cellMhash = ? MultiHash.init(Pos2Bn128MrklCodec, cell.toBytes).mapFailure
|
||||
cellCid = ? Cid.init(CIDv1, CodexSlotCellCodec, cellMhash).mapFailure
|
||||
|
||||
success cellCid
|
||||
|
||||
func toSlotCid*(root: Poseidon2Hash): ?!Cid =
|
||||
let
|
||||
mhash = ? MultiHash.init($multiCodec("identity"), root.toBytes).mapFailure
|
||||
treeCid = ? Cid.init(CIDv1, SlotRootCodec, mhash).mapFailure
|
||||
|
||||
success treeCid
|
||||
|
||||
func toSlotCids*(slotRoots: openArray[Poseidon2Hash]): ?!seq[Cid] =
|
||||
success slotRoots.mapIt( ? it.toSlotCid )
|
||||
|
||||
func toSlotsRootsCid*(root: Poseidon2Hash): ?!Cid =
|
||||
let
|
||||
mhash = ? MultiHash.init($multiCodec("identity"), root.toBytes).mapFailure
|
||||
treeCid = ? Cid.init(CIDv1, SlotProvingRootCodec, mhash).mapFailure
|
||||
|
||||
success treeCid
|
||||
|
||||
func toEncodableProof*(
|
||||
proof: Poseidon2Proof): ?!CodexProof =
|
||||
|
||||
let
|
||||
encodableProof = CodexProof(
|
||||
mcodec: multiCodec("identity"), # copy bytes as is
|
||||
index: proof.index,
|
||||
nleaves: proof.nleaves,
|
||||
path: proof.path.mapIt( @( it.toBytes ) ))
|
||||
|
||||
success encodableProof
|
||||
|
||||
func toVerifiableProof*(
|
||||
proof: CodexProof): ?!Poseidon2Proof =
|
||||
|
||||
let
|
||||
verifiableProof = Poseidon2Proof(
|
||||
index: proof.index,
|
||||
nleaves: proof.nleaves,
|
||||
path: proof.path.mapIt(
|
||||
? Poseidon2Hash.fromBytes(it.toArray32).toFailure
|
||||
))
|
||||
|
||||
success verifiableProof
|
||||
|
|
@ -0,0 +1,89 @@
|
|||
## Nim-Codex
|
||||
## Copyright (c) 2024 Status Research & Development GmbH
|
||||
## Licensed under either of
|
||||
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
|
||||
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
|
||||
## at your option.
|
||||
## This file may not be copied, modified, or distributed except according to
|
||||
## those terms.
|
||||
|
||||
import std/sequtils
|
||||
|
||||
import pkg/libp2p
|
||||
import pkg/stew/arrayops
|
||||
import pkg/questionable
|
||||
import pkg/questionable/results
|
||||
import pkg/poseidon2
|
||||
import pkg/poseidon2/io
|
||||
|
||||
import ../codextypes
|
||||
import ../merkletree
|
||||
import ../errors
|
||||
import ../utils/digest
|
||||
|
||||
func toCid(hash: Poseidon2Hash, mcodec: MultiCodec, cidCodec: MultiCodec): ?!Cid =
|
||||
let
|
||||
mhash = ? MultiHash.init(mcodec, hash.toBytes).mapFailure
|
||||
treeCid = ? Cid.init(CIDv1, cidCodec, mhash).mapFailure
|
||||
success treeCid
|
||||
|
||||
proc toPoseidon2Hash(cid: Cid, mcodec: MultiCodec, cidCodec: MultiCodec): ?!Poseidon2Hash =
|
||||
if cid.cidver != CIDv1:
|
||||
return failure("Unexpected CID version")
|
||||
|
||||
if cid.mcodec != cidCodec:
|
||||
return failure("Cid is not of expected codec. Was: " & $cid.mcodec & " but expected: " & $cidCodec)
|
||||
|
||||
let
|
||||
mhash = ? cid.mhash.mapFailure
|
||||
bytes: array[32, byte] = array[32, byte].initCopyFrom(mhash.digestBytes())
|
||||
hash = Poseidon2Hash.fromBytes(bytes)
|
||||
if not hash.isSome():
|
||||
return failure("Unable to convert Cid to Poseidon2Hash")
|
||||
return success(hash.get())
|
||||
|
||||
func toCellCid*(hash: Poseidon2Hash): ?!Cid =
|
||||
toCid(hash, Pos2Bn128MrklCodec, CodexSlotCellCodec)
|
||||
|
||||
func fromCellCid*(cid: Cid): ?!Poseidon2Hash =
|
||||
toPoseidon2Hash(cid, Pos2Bn128MrklCodec, CodexSlotCellCodec)
|
||||
|
||||
func toSlotCid*(hash: Poseidon2Hash): ?!Cid =
|
||||
toCid(hash, multiCodec("identity"), SlotRootCodec)
|
||||
|
||||
func toSlotCids*(slotRoots: openArray[Poseidon2Hash]): ?!seq[Cid] =
|
||||
success slotRoots.mapIt( ? it.toSlotCid )
|
||||
|
||||
func fromSlotCid*(cid: Cid): ?!Poseidon2Hash =
|
||||
toPoseidon2Hash(cid, multiCodec("identity"), SlotRootCodec)
|
||||
|
||||
func toVerifyCid*(hash: Poseidon2Hash): ?!Cid =
|
||||
toCid(hash, multiCodec("identity"), SlotProvingRootCodec)
|
||||
|
||||
func fromVerifyCid*(cid: Cid): ?!Poseidon2Hash =
|
||||
toPoseidon2Hash(cid, multiCodec("identity"), SlotProvingRootCodec)
|
||||
|
||||
func toEncodableProof*(
|
||||
proof: Poseidon2Proof): ?!CodexProof =
|
||||
|
||||
let
|
||||
encodableProof = CodexProof(
|
||||
mcodec: multiCodec("identity"),
|
||||
index: proof.index,
|
||||
nleaves: proof.nleaves,
|
||||
path: proof.path.mapIt( @( it.toBytes ) ))
|
||||
|
||||
success encodableProof
|
||||
|
||||
func toVerifiableProof*(
|
||||
proof: CodexProof): ?!Poseidon2Proof =
|
||||
|
||||
let
|
||||
verifiableProof = Poseidon2Proof(
|
||||
index: proof.index,
|
||||
nleaves: proof.nleaves,
|
||||
path: proof.path.mapIt(
|
||||
? Poseidon2Hash.fromBytes(it.toArray32).toFailure
|
||||
))
|
||||
|
||||
success verifiableProof
|
||||
|
|
@ -15,6 +15,8 @@ import pkg/poseidon2/io
|
|||
|
||||
import pkg/constantine/math/arithmetic
|
||||
|
||||
import pkg/constantine/math/io/io_fields
|
||||
|
||||
import ../../merkletree
|
||||
|
||||
func extractLowBits*[n: static int](elm: BigInt[n], k: int): uint64 =
|
||||
|
|
@ -64,8 +66,7 @@ func cellIndex*(
|
|||
let log2 = ceilingLog2(numCells)
|
||||
doAssert( 1 shl log2 == numCells , "`numCells` is assumed to be a power of two" )
|
||||
|
||||
let
|
||||
hash = Sponge.digest( @[ entropy, slotRoot, counter.toF ], rate = 2 )
|
||||
let hash = Sponge.digest( @[ slotRoot, entropy, counter.toF ], rate = 2 )
|
||||
|
||||
return int( extractLowBits(hash, log2) )
|
||||
|
||||
|
|
@ -74,7 +75,8 @@ func cellIndices*(
|
|||
slotRoot: Poseidon2Hash,
|
||||
numCells: Natural, nSamples: Natural): seq[Natural] =
|
||||
|
||||
var indices: seq[int]
|
||||
var indices: seq[Natural]
|
||||
while (indices.len < nSamples):
|
||||
let idx = entropy.cellIndex(slotRoot, numCells, indices.len + 1)
|
||||
indices.add(idx)
|
||||
let idx = cellIndex(entropy, slotRoot, numCells, indices.len + 1)
|
||||
indices.add(idx.Natural)
|
||||
indices
|
||||
|
|
|
|||
|
|
@ -75,3 +75,8 @@ proc example*(_: type Reservation): Reservation =
|
|||
|
||||
proc example*(_: type MerkleProof): MerkleProof =
|
||||
MerkleProof.init(3, @[MultiHash.example]).tryget()
|
||||
|
||||
proc example*(_: type Poseidon2Proof): Poseidon2Proof =
|
||||
var example = MerkleProof[Poseidon2Hash, PoseidonKeysEnum]()
|
||||
example.index = 123
|
||||
example
|
||||
|
|
|
|||
|
|
@ -0,0 +1,168 @@
|
|||
import std/sequtils
|
||||
|
||||
import pkg/questionable/results
|
||||
import pkg/poseidon2/io
|
||||
import pkg/poseidon2
|
||||
import pkg/chronos
|
||||
import pkg/codex/stores/cachestore
|
||||
import pkg/codex/chunker
|
||||
import pkg/codex/stores
|
||||
import pkg/codex/blocktype as bt
|
||||
import pkg/codex/contracts/requests
|
||||
import pkg/codex/contracts
|
||||
import pkg/codex/merkletree
|
||||
import pkg/codex/stores/cachestore
|
||||
import pkg/codex/indexingstrategy
|
||||
|
||||
import pkg/codex/slots/converters
|
||||
import pkg/codex/slots/builder/builder
|
||||
import pkg/codex/utils/poseidon2digest
|
||||
import pkg/codex/utils/asynciter
|
||||
|
||||
import ../helpers
|
||||
import ../merkletree/helpers
|
||||
|
||||
type
|
||||
ProvingTestEnvironment* = ref object
|
||||
# Invariant:
|
||||
challenge*: Poseidon2Hash
|
||||
# Variant:
|
||||
localStore*: CacheStore
|
||||
manifest*: Manifest
|
||||
manifestBlock*: bt.Block
|
||||
slot*: Slot
|
||||
datasetBlocks*: seq[bt.Block]
|
||||
slotTree*: Poseidon2Tree
|
||||
slotRootCid*: Cid
|
||||
slotRoots*: seq[Poseidon2Hash]
|
||||
datasetToSlotTree*: Poseidon2Tree
|
||||
datasetRootHash*: Poseidon2Hash
|
||||
|
||||
const
|
||||
# The number of slot blocks and number of slots, combined with
|
||||
# the bytes per block, make it so that there are exactly 256 cells
|
||||
# in the dataset.
|
||||
bytesPerBlock* = 64 * 1024
|
||||
numberOfSlotBlocks* = 4
|
||||
totalNumberOfSlots* = 2
|
||||
datasetSlotIndex* = 1
|
||||
cellsPerSlot* = (bytesPerBlock * numberOfSlotBlocks) div DefaultCellSize.int
|
||||
|
||||
proc createDatasetBlocks(self: ProvingTestEnvironment): Future[void] {.async.} =
|
||||
let numberOfCellsNeeded = (numberOfSlotBlocks * totalNumberOfSlots * bytesPerBlock).uint64 div DefaultCellSize.uint64
|
||||
var data: seq[byte] = @[]
|
||||
|
||||
# This generates a number of blocks that have different data, such that
|
||||
# Each cell in each block is unique, but nothing is random.
|
||||
for i in 0 ..< numberOfCellsNeeded:
|
||||
data = data & (i.byte).repeat(DefaultCellSize.uint64)
|
||||
|
||||
let chunker = MockChunker.new(
|
||||
dataset = data,
|
||||
chunkSize = bytesPerBlock)
|
||||
|
||||
while true:
|
||||
let chunk = await chunker.getBytes()
|
||||
if chunk.len <= 0:
|
||||
break
|
||||
let b = bt.Block.new(chunk).tryGet()
|
||||
self.datasetBlocks.add(b)
|
||||
discard await self.localStore.putBlock(b)
|
||||
|
||||
proc createSlotTree(self: ProvingTestEnvironment, dSlotIndex: uint64): Future[Poseidon2Tree] {.async.} =
|
||||
let
|
||||
slotSize = (bytesPerBlock * numberOfSlotBlocks).uint64
|
||||
blocksInSlot = slotSize div bytesPerBlock.uint64
|
||||
datasetBlockIndexingStrategy = SteppedIndexingStrategy.new(0, self.datasetBlocks.len - 1, totalNumberOfSlots)
|
||||
datasetBlockIndices = toSeq(datasetBlockIndexingStrategy.getIndicies(dSlotIndex.int))
|
||||
|
||||
let
|
||||
slotBlocks = datasetBlockIndices.mapIt(self.datasetBlocks[it])
|
||||
slotBlockRoots = slotBlocks.mapIt(Poseidon2Tree.digest(it.data, DefaultCellSize.int).tryGet())
|
||||
tree = Poseidon2Tree.init(slotBlockRoots).tryGet()
|
||||
treeCid = tree.root().tryGet().toSlotCid().tryGet()
|
||||
|
||||
for i in 0 ..< numberOfSlotBlocks:
|
||||
let
|
||||
blkCid = slotBlockRoots[i].toCellCid().tryGet()
|
||||
proof = tree.getProof(i).tryGet().toEncodableProof().tryGet()
|
||||
|
||||
discard await self.localStore.putCidAndProof(treeCid, i, blkCid, proof)
|
||||
|
||||
return tree
|
||||
|
||||
proc createDatasetRootHashAndSlotTree(self: ProvingTestEnvironment): Future[void] {.async.} =
|
||||
var slotTrees = newSeq[Poseidon2Tree]()
|
||||
for i in 0 ..< totalNumberOfSlots:
|
||||
slotTrees.add(await self.createSlotTree(i.uint64))
|
||||
self.slotTree = slotTrees[datasetSlotIndex]
|
||||
self.slotRootCid = slotTrees[datasetSlotIndex].root().tryGet().toSlotCid().tryGet()
|
||||
self.slotRoots = slotTrees.mapIt(it.root().tryGet())
|
||||
let rootsPadLeafs = newSeqWith(totalNumberOfSlots.nextPowerOfTwoPad, Poseidon2Zero)
|
||||
self.datasetToSlotTree = Poseidon2Tree.init(self.slotRoots & rootsPadLeafs).tryGet()
|
||||
self.datasetRootHash = self.datasetToSlotTree.root().tryGet()
|
||||
|
||||
proc createManifest(self: ProvingTestEnvironment): Future[void] {.async.} =
|
||||
let
|
||||
cids = self.datasetBlocks.mapIt(it.cid)
|
||||
tree = CodexTree.init(cids).tryGet()
|
||||
treeCid = tree.rootCid(CIDv1, BlockCodec).tryGet()
|
||||
|
||||
for i in 0 ..< self.datasetBlocks.len:
|
||||
let
|
||||
blk = self.datasetBlocks[i]
|
||||
leafCid = blk.cid
|
||||
proof = tree.getProof(i).tryGet()
|
||||
discard await self.localStore.putBlock(blk)
|
||||
discard await self.localStore.putCidAndProof(treeCid, i, leafCid, proof)
|
||||
|
||||
# Basic manifest:
|
||||
self.manifest = Manifest.new(
|
||||
treeCid = treeCid,
|
||||
blockSize = bytesPerBlock.NBytes,
|
||||
datasetSize = (bytesPerBlock * numberOfSlotBlocks * totalNumberOfSlots).NBytes)
|
||||
|
||||
# Protected manifest:
|
||||
self.manifest = Manifest.new(
|
||||
manifest = self.manifest,
|
||||
treeCid = treeCid,
|
||||
datasetSize = self.manifest.datasetSize,
|
||||
ecK = totalNumberOfSlots,
|
||||
ecM = 0
|
||||
)
|
||||
|
||||
# Verifiable manifest:
|
||||
self.manifest = Manifest.new(
|
||||
manifest = self.manifest,
|
||||
verifyRoot = self.datasetRootHash.toVerifyCid().tryGet(),
|
||||
slotRoots = self.slotRoots.mapIt(it.toSlotCid().tryGet())
|
||||
).tryGet()
|
||||
|
||||
self.manifestBlock = bt.Block.new(self.manifest.encode().tryGet(), codec = ManifestCodec).tryGet()
|
||||
discard await self.localStore.putBlock(self.manifestBlock)
|
||||
|
||||
proc createSlot(self: ProvingTestEnvironment): void =
|
||||
self.slot = Slot(
|
||||
request: StorageRequest(
|
||||
ask: StorageAsk(
|
||||
slotSize: u256(bytesPerBlock * numberOfSlotBlocks)
|
||||
),
|
||||
content: StorageContent(
|
||||
cid: $self.manifestBlock.cid
|
||||
),
|
||||
),
|
||||
slotIndex: u256(datasetSlotIndex)
|
||||
)
|
||||
|
||||
proc createProvingTestEnvironment*(): Future[ProvingTestEnvironment] {.async.} =
|
||||
var testEnv = ProvingTestEnvironment(
|
||||
challenge: toF(12345)
|
||||
)
|
||||
|
||||
testEnv.localStore = CacheStore.new()
|
||||
await testEnv.createDatasetBlocks()
|
||||
await testEnv.createDatasetRootHashAndSlotTree()
|
||||
await testEnv.createManifest()
|
||||
testEnv.createSlot()
|
||||
|
||||
return testEnv
|
||||
|
|
@ -0,0 +1,47 @@
|
|||
import pkg/chronos
|
||||
import pkg/asynctest
|
||||
import pkg/poseidon2
|
||||
import pkg/poseidon2/io
|
||||
import pkg/constantine/math/io/io_fields
|
||||
import pkg/questionable/results
|
||||
import pkg/codex/merkletree
|
||||
|
||||
import pkg/codex/slots/converters
|
||||
import ../examples
|
||||
import ../merkletree/helpers
|
||||
|
||||
let
|
||||
hash: Poseidon2Hash = toF(12345)
|
||||
|
||||
suite "Converters":
|
||||
test "CellBlock cid":
|
||||
let
|
||||
cid = toCellCid(hash).tryGet()
|
||||
value = fromCellCid(cid).tryGet()
|
||||
|
||||
check:
|
||||
hash.toDecimal() == value.toDecimal()
|
||||
|
||||
test "Slot cid":
|
||||
let
|
||||
cid = toSlotCid(hash).tryGet()
|
||||
value = fromSlotCid(cid).tryGet()
|
||||
|
||||
check:
|
||||
hash.toDecimal() == value.toDecimal()
|
||||
|
||||
test "Verify cid":
|
||||
let
|
||||
cid = toVerifyCid(hash).tryGet()
|
||||
value = fromVerifyCid(cid).tryGet()
|
||||
|
||||
check:
|
||||
hash.toDecimal() == value.toDecimal()
|
||||
|
||||
test "Proof":
|
||||
let
|
||||
codexProof = toEncodableProof(Poseidon2Proof.example).tryGet()
|
||||
poseidonProof = toVerifiableProof(codexProof).tryGet()
|
||||
|
||||
check:
|
||||
Poseidon2Proof.example == poseidonProof
|
||||
|
|
@ -0,0 +1,329 @@
|
|||
# import std/sequtils
|
||||
# import std/sugar
|
||||
# import std/random
|
||||
|
||||
# import pkg/questionable/results
|
||||
# import pkg/constantine/math/arithmetic
|
||||
# import pkg/constantine/math/io/io_fields
|
||||
# import pkg/poseidon2/types
|
||||
# import pkg/poseidon2/io
|
||||
# import pkg/poseidon2
|
||||
# import pkg/chronos
|
||||
# import pkg/asynctest
|
||||
# import pkg/codex/stores/cachestore
|
||||
# import pkg/codex/chunker
|
||||
# import pkg/codex/stores
|
||||
# import pkg/codex/blocktype as bt
|
||||
# import pkg/codex/contracts/requests
|
||||
# import pkg/codex/contracts
|
||||
# import pkg/codex/merkletree
|
||||
# import pkg/codex/stores/cachestore
|
||||
|
||||
# import pkg/codex/slots
|
||||
# import pkg/codex/proof/misc
|
||||
# import pkg/codex/proof/types
|
||||
|
||||
# import ../helpers
|
||||
# import ../examples
|
||||
# import testdatasampler_expected
|
||||
|
||||
# let
|
||||
# bytesPerBlock = 64 * 1024
|
||||
# challenge: FieldElement = toF(12345)
|
||||
# datasetRootHash: FieldElement = toF(6789)
|
||||
|
||||
# asyncchecksuite "Test proof datasampler - components":
|
||||
# let
|
||||
# numberOfSlotBlocks = 16
|
||||
# slot = Slot(
|
||||
# request: StorageRequest(
|
||||
# ask: StorageAsk(
|
||||
# slots: 10,
|
||||
# slotSize: u256(bytesPerBlock * numberOfSlotBlocks),
|
||||
# ),
|
||||
# content: StorageContent(
|
||||
# cid: $Cid.example
|
||||
# )
|
||||
# ),
|
||||
# slotIndex: u256(3)
|
||||
# )
|
||||
|
||||
# test "Number of cells is a power of two":
|
||||
# # This is to check that the data used for testing is sane.
|
||||
# proc isPow2(value: int): bool =
|
||||
# let log2 = ceilingLog2(value)
|
||||
# return (1 shl log2) == value
|
||||
|
||||
# let numberOfCells = getNumberOfCellsInSlot(slot).int
|
||||
|
||||
# check:
|
||||
# isPow2(numberOfCells)
|
||||
|
||||
# test "Extract low bits":
|
||||
# proc extract(value: uint64, nBits: int): uint64 =
|
||||
# let big = toF(value).toBig()
|
||||
# return extractLowBits(big, nBits)
|
||||
|
||||
# check:
|
||||
# extract(0x88, 4) == 0x8.uint64
|
||||
# extract(0x88, 7) == 0x8.uint64
|
||||
# extract(0x9A, 5) == 0x1A.uint64
|
||||
# extract(0x9A, 7) == 0x1A.uint64
|
||||
# extract(0x1248, 10) == 0x248.uint64
|
||||
# extract(0x1248, 12) == 0x248.uint64
|
||||
# extract(0x1248306A560C9AC0.uint64, 10) == 0x2C0.uint64
|
||||
# extract(0x1248306A560C9AC0.uint64, 12) == 0xAC0.uint64
|
||||
# extract(0x1248306A560C9AC0.uint64, 50) == 0x306A560C9AC0.uint64
|
||||
# extract(0x1248306A560C9AC0.uint64, 52) == 0x8306A560C9AC0.uint64
|
||||
|
||||
# test "Should calculate total number of cells in Slot":
|
||||
# let
|
||||
# slotSizeInBytes = (slot.request.ask.slotSize).truncate(uint64)
|
||||
# expectedNumberOfCells = slotSizeInBytes div CellSize
|
||||
|
||||
# check:
|
||||
# expectedNumberOfCells == 512
|
||||
# expectedNumberOfCells == getNumberOfCellsInSlot(slot)
|
||||
|
||||
# asyncchecksuite "Test proof datasampler - main":
|
||||
# let
|
||||
# # The number of slot blocks and number of slots, combined with
|
||||
# # the bytes per block, make it so that there are exactly 256 cells
|
||||
# # in the dataset.
|
||||
# numberOfSlotBlocks = 4
|
||||
# totalNumberOfSlots = 2
|
||||
# datasetSlotIndex = 1
|
||||
# localStore = CacheStore.new()
|
||||
# datasetToSlotProof = MerkleProof.example
|
||||
|
||||
# var
|
||||
# manifest: Manifest
|
||||
# manifestBlock: bt.Block
|
||||
# slot: Slot
|
||||
# datasetBlocks: seq[bt.Block]
|
||||
# slotPoseidonTree: MerkleTree
|
||||
# dataSampler: DataSampler
|
||||
|
||||
# proc createDatasetBlocks(): Future[void] {.async.} =
|
||||
# let numberOfCellsNeeded = (numberOfSlotBlocks * totalNumberOfSlots * bytesPerBlock).uint64 div CellSize
|
||||
# var data: seq[byte] = @[]
|
||||
|
||||
# # This generates a number of blocks that have different data, such that
|
||||
# # Each cell in each block is unique, but nothing is random.
|
||||
# for i in 0 ..< numberOfCellsNeeded:
|
||||
# data = data & (i.byte).repeat(CellSize)
|
||||
|
||||
# let chunker = MockChunker.new(
|
||||
# dataset = data,
|
||||
# chunkSize = bytesPerBlock)
|
||||
|
||||
# while true:
|
||||
# let chunk = await chunker.getBytes()
|
||||
# if chunk.len <= 0:
|
||||
# break
|
||||
# let b = bt.Block.new(chunk).tryGet()
|
||||
# datasetBlocks.add(b)
|
||||
# discard await localStore.putBlock(b)
|
||||
|
||||
# proc createManifest(): Future[void] {.async.} =
|
||||
# let
|
||||
# cids = datasetBlocks.mapIt(it.cid)
|
||||
# tree = MerkleTree.init(cids).tryGet()
|
||||
# treeCid = tree.rootCid().tryGet()
|
||||
|
||||
# for index, cid in cids:
|
||||
# let proof = tree.getProof(index).tryget()
|
||||
# discard await localStore.putBlockCidAndProof(treeCid, index, cid, proof)
|
||||
|
||||
# manifest = Manifest.new(
|
||||
# treeCid = treeCid,
|
||||
# blockSize = bytesPerBlock.NBytes,
|
||||
# datasetSize = (bytesPerBlock * numberOfSlotBlocks * totalNumberOfSlots).NBytes)
|
||||
# manifestBlock = bt.Block.new(manifest.encode().tryGet(), codec = DagPBCodec).tryGet()
|
||||
|
||||
# proc createSlot(): void =
|
||||
# slot = Slot(
|
||||
# request: StorageRequest(
|
||||
# ask: StorageAsk(
|
||||
# slotSize: u256(bytesPerBlock * numberOfSlotBlocks)
|
||||
# ),
|
||||
# content: StorageContent(
|
||||
# cid: $manifestBlock.cid
|
||||
# ),
|
||||
# ),
|
||||
# slotIndex: u256(datasetSlotIndex)
|
||||
# )
|
||||
|
||||
# proc createSlotPoseidonTree(): void =
|
||||
# let
|
||||
# slotSize = slot.request.ask.slotSize.truncate(uint64)
|
||||
# blocksInSlot = slotSize div bytesPerBlock.uint64
|
||||
# datasetSlotIndex = slot.slotIndex.truncate(uint64)
|
||||
# datasetBlockIndexFirst = datasetSlotIndex * blocksInSlot
|
||||
# datasetBlockIndexLast = datasetBlockIndexFirst + numberOfSlotBlocks.uint64
|
||||
# slotBlocks = datasetBlocks[datasetBlockIndexFirst ..< datasetBlockIndexLast]
|
||||
# slotBlockCids = slotBlocks.mapIt(it.cid)
|
||||
# slotPoseidonTree = MerkleTree.init(slotBlockCids).tryGet()
|
||||
|
||||
# proc createDataSampler(): Future[void] {.async.} =
|
||||
# dataSampler = (await DataSampler.new(
|
||||
# slot,
|
||||
# localStore,
|
||||
# datasetRootHash,
|
||||
# slotPoseidonTree,
|
||||
# datasetToSlotProof
|
||||
# )).tryGet()
|
||||
|
||||
# setup:
|
||||
# await createDatasetBlocks()
|
||||
# await createManifest()
|
||||
# createSlot()
|
||||
# discard await localStore.putBlock(manifestBlock)
|
||||
# createSlotPoseidonTree()
|
||||
# await createDataSampler()
|
||||
|
||||
# test "Number of cells is a power of two":
|
||||
# # This is to check that the data used for testing is sane.
|
||||
# proc isPow2(value: int): bool =
|
||||
# let log2 = ceilingLog2(value)
|
||||
# return (1 shl log2) == value
|
||||
|
||||
# let numberOfCells = getNumberOfCellsInSlot(slot).int
|
||||
|
||||
# check:
|
||||
# isPow2(numberOfCells)
|
||||
|
||||
# let knownIndices = @[74.uint64, 41.uint64, 51.uint64]
|
||||
|
||||
# test "Can find single slot-cell index":
|
||||
# proc slotCellIndex(i: int): uint64 =
|
||||
# let counter: FieldElement = toF(i)
|
||||
# return dataSampler.findSlotCellIndex(challenge, counter)
|
||||
|
||||
# proc getExpectedIndex(i: int): uint64 =
|
||||
# let
|
||||
# numberOfCellsInSlot = (bytesPerBlock * numberOfSlotBlocks) div CellSize.int
|
||||
# slotRootHash = toF(1234) # TODO - replace with slotPoseidonTree.root when it is a poseidon tree.
|
||||
# hash = Sponge.digest(@[slotRootHash, challenge, toF(i)], rate = 2)
|
||||
# return extractLowBits(hash.toBig(), ceilingLog2(numberOfCellsInSlot))
|
||||
|
||||
# check:
|
||||
# slotCellIndex(1) == getExpectedIndex(1)
|
||||
# slotCellIndex(1) == knownIndices[0]
|
||||
# slotCellIndex(2) == getExpectedIndex(2)
|
||||
# slotCellIndex(2) == knownIndices[1]
|
||||
# slotCellIndex(3) == getExpectedIndex(3)
|
||||
# slotCellIndex(3) == knownIndices[2]
|
||||
|
||||
# test "Can find sequence of slot-cell indices":
|
||||
# proc slotCellIndices(n: int): seq[uint64] =
|
||||
# dataSampler.findSlotCellIndices(challenge, n)
|
||||
|
||||
# proc getExpectedIndices(n: int): seq[uint64] =
|
||||
# return collect(newSeq, (for i in 1..n: dataSampler.findSlotCellIndex(challenge, toF(i))))
|
||||
|
||||
# check:
|
||||
# slotCellIndices(3) == getExpectedIndices(3)
|
||||
# slotCellIndices(3) == knownIndices
|
||||
|
||||
# let
|
||||
# bytes = newSeqWith(bytesPerBlock, rand(uint8))
|
||||
# blk = bt.Block.new(bytes).tryGet()
|
||||
# cell0Bytes = bytes[0..<CellSize]
|
||||
# cell1Bytes = bytes[CellSize..<(CellSize*2)]
|
||||
# cell2Bytes = bytes[(CellSize*2)..<(CellSize*3)]
|
||||
|
||||
# test "Can get cell from block":
|
||||
# let
|
||||
# sample0 = dataSampler.getCellFromBlock(blk, 0)
|
||||
# sample1 = dataSampler.getCellFromBlock(blk, 1)
|
||||
# sample2 = dataSampler.getCellFromBlock(blk, 2)
|
||||
|
||||
# check:
|
||||
# sample0 == cell0Bytes
|
||||
# sample1 == cell1Bytes
|
||||
# sample2 == cell2Bytes
|
||||
|
||||
# test "Can convert block into cells":
|
||||
# let cells = dataSampler.getBlockCells(blk)
|
||||
|
||||
# check:
|
||||
# cells.len == (bytesPerBlock div CellSize.int)
|
||||
# cells[0] == cell0Bytes
|
||||
# cells[1] == cell1Bytes
|
||||
# cells[2] == cell2Bytes
|
||||
|
||||
# test "Can create mini tree for block cells":
|
||||
# let miniTree = dataSampler.getBlockCellMiniTree(blk).tryGet()
|
||||
|
||||
# let
|
||||
# cell0Proof = miniTree.getProof(0).tryGet()
|
||||
# cell1Proof = miniTree.getProof(1).tryGet()
|
||||
# cell2Proof = miniTree.getProof(2).tryGet()
|
||||
|
||||
# check:
|
||||
# cell0Proof.verifyDataBlock(cell0Bytes, miniTree.root).tryGet()
|
||||
# cell1Proof.verifyDataBlock(cell1Bytes, miniTree.root).tryGet()
|
||||
# cell2Proof.verifyDataBlock(cell2Bytes, miniTree.root).tryGet()
|
||||
|
||||
# test "Can gather proof input":
|
||||
# # This is the main entry point for this module, and what it's all about.
|
||||
# let
|
||||
# nSamples = 3
|
||||
# input = (await dataSampler.getProofInput(challenge, nSamples)).tryget()
|
||||
|
||||
# proc equal(a: FieldElement, b: FieldElement): bool =
|
||||
# a.toDecimal() == b.toDecimal()
|
||||
|
||||
# proc toStr(proof: MerkleProof): string =
|
||||
# toHex(proof.nodesBuffer)
|
||||
|
||||
# let
|
||||
# expectedMerkleProofs = getExpectedSlotToBlockProofs()
|
||||
# expectedCellData = getExpectedCellData()
|
||||
|
||||
# check:
|
||||
# # datasetRoot*: FieldElement
|
||||
# equal(input.datasetRoot, datasetRootHash)
|
||||
# # entropy*: FieldElement
|
||||
# equal(input.entropy, challenge)
|
||||
# # numberOfCellsInSlot*: uint64
|
||||
# input.numberOfCellsInSlot == (bytesPerBlock * numberOfSlotBlocks).uint64 div CellSize
|
||||
# # numberOfSlots*: uint64
|
||||
# input.numberOfSlots == slot.request.ask.slots
|
||||
# # datasetSlotIndex*: uint64
|
||||
# input.datasetSlotIndex == slot.slotIndex.truncate(uint64)
|
||||
# # slotRoot*: FieldElement
|
||||
# equal(input.slotRoot, toF(1234)) # TODO - when slotPoseidonTree is a poseidon tree, its root should be a FieldElement.
|
||||
# # datasetToSlotProof*: MerkleProof
|
||||
# input.datasetToSlotProof == datasetToSlotProof
|
||||
# # proofSamples*: seq[ProofSample]
|
||||
# toStr(input.proofSamples[0].merkleProof) == expectedMerkleProofs[0]
|
||||
# toStr(input.proofSamples[1].merkleProof) == expectedMerkleProofs[1]
|
||||
# toStr(input.proofSamples[2].merkleProof) == expectedMerkleProofs[2]
|
||||
# # cell data
|
||||
# toHex(input.proofSamples[0].cellData) == expectedCellData[0]
|
||||
# toHex(input.proofSamples[1].cellData) == expectedCellData[1]
|
||||
# toHex(input.proofSamples[2].cellData) == expectedCellData[2]
|
||||
|
||||
# # input.slotToBlockProofs.mapIt(toStr(it)) == expectedSlotToBlockProofs
|
||||
# # input.blockToCellProofs.mapIt(toStr(it)) == expectedBlockToCellProofs
|
||||
# # toHex(input.sampleData) == expectedSampleData
|
||||
|
||||
# for (input, expected) in [(10, 0), (31, 0), (32, 1), (63, 1), (64, 2)]:
|
||||
# test "Can get slotBlockIndex from slotCellIndex (" & $input & " -> " & $expected & ")":
|
||||
# let
|
||||
# slotCellIndex = input.uint64
|
||||
# slotBlockIndex = dataSampler.getSlotBlockIndexForSlotCellIndex(slotCellIndex)
|
||||
|
||||
# check:
|
||||
# slotBlockIndex == expected.uint64
|
||||
|
||||
# for (input, expected) in [(10, 10), (31, 31), (32, 0), (63, 31), (64, 0)]:
|
||||
# test "Can get blockCellIndex from slotCellIndex (" & $input & " -> " & $expected & ")":
|
||||
# let
|
||||
# slotCellIndex = input.uint64
|
||||
# blockCellIndex = dataSampler.getBlockCellIndexForSlotCellIndex(slotCellIndex)
|
||||
|
||||
# check:
|
||||
# blockCellIndex == expected.uint64
|
||||
|
|
@ -0,0 +1,104 @@
|
|||
import std/sequtils
|
||||
import std/sugar
|
||||
import std/random
|
||||
import std/strutils
|
||||
|
||||
import pkg/questionable/results
|
||||
import pkg/constantine/math/arithmetic
|
||||
import pkg/constantine/math/io/io_fields
|
||||
import pkg/poseidon2/io
|
||||
import pkg/poseidon2
|
||||
import pkg/chronos
|
||||
import pkg/asynctest
|
||||
import pkg/codex/stores/cachestore
|
||||
import pkg/codex/chunker
|
||||
import pkg/codex/stores
|
||||
import pkg/codex/blocktype as bt
|
||||
import pkg/codex/contracts/requests
|
||||
import pkg/codex/contracts
|
||||
import pkg/codex/merkletree
|
||||
import pkg/codex/stores/cachestore
|
||||
|
||||
import pkg/codex/slots/sampler/utils
|
||||
|
||||
import ../helpers
|
||||
import ../examples
|
||||
import ../merkletree/helpers
|
||||
import ./provingtestenv
|
||||
|
||||
asyncchecksuite "Test proof sampler utils":
|
||||
let knownIndices: seq[Natural] = @[90, 93, 29]
|
||||
|
||||
var
|
||||
env: ProvingTestEnvironment
|
||||
slotRoot: Poseidon2Hash
|
||||
numCells: Natural
|
||||
|
||||
setup:
|
||||
env = await createProvingTestEnvironment()
|
||||
slotRoot = env.slotRoots[datasetSlotIndex]
|
||||
numCells = cellsPerSlot
|
||||
|
||||
teardown:
|
||||
reset(env)
|
||||
|
||||
test "Extract low bits":
|
||||
proc extract(value: uint64, nBits: int): uint64 =
|
||||
let big = toF(value).toBig()
|
||||
return extractLowBits(big, nBits)
|
||||
|
||||
check:
|
||||
extract(0x88, 4) == 0x8.uint64
|
||||
extract(0x88, 7) == 0x8.uint64
|
||||
extract(0x9A, 5) == 0x1A.uint64
|
||||
extract(0x9A, 7) == 0x1A.uint64
|
||||
extract(0x1248, 10) == 0x248.uint64
|
||||
extract(0x1248, 12) == 0x248.uint64
|
||||
extract(0x1248306A560C9AC0.uint64, 10) == 0x2C0.uint64
|
||||
extract(0x1248306A560C9AC0.uint64, 12) == 0xAC0.uint64
|
||||
extract(0x1248306A560C9AC0.uint64, 50) == 0x306A560C9AC0.uint64
|
||||
extract(0x1248306A560C9AC0.uint64, 52) == 0x8306A560C9AC0.uint64
|
||||
|
||||
test "Can find single slot-cell index":
|
||||
proc slotCellIndex(i: Natural): Natural =
|
||||
return cellIndex(env.challenge, slotRoot, numCells, i)
|
||||
|
||||
proc getExpectedIndex(i: int): Natural =
|
||||
let
|
||||
numberOfCellsInSlot = (bytesPerBlock * numberOfSlotBlocks) div DefaultCellSize.uint64.int
|
||||
hash = Sponge.digest(@[slotRoot, env.challenge, toF(i)], rate = 2)
|
||||
|
||||
return int(extractLowBits(hash.toBig(), ceilingLog2(numberOfCellsInSlot)))
|
||||
|
||||
check:
|
||||
slotCellIndex(1) == getExpectedIndex(1)
|
||||
slotCellIndex(1) == knownIndices[0]
|
||||
slotCellIndex(2) == getExpectedIndex(2)
|
||||
slotCellIndex(2) == knownIndices[1]
|
||||
slotCellIndex(3) == getExpectedIndex(3)
|
||||
slotCellIndex(3) == knownIndices[2]
|
||||
|
||||
test "Can find sequence of slot-cell indices":
|
||||
proc slotCellIndices(n: int): seq[Natural] =
|
||||
cellIndices(env.challenge, slotRoot, numCells, n)
|
||||
|
||||
proc getExpectedIndices(n: int): seq[Natural] =
|
||||
return collect(newSeq, (for i in 1..n: cellIndex(env.challenge, slotRoot, numCells, i)))
|
||||
|
||||
check:
|
||||
slotCellIndices(3) == getExpectedIndices(3)
|
||||
slotCellIndices(3) == knownIndices
|
||||
|
||||
for (input, expected) in [(10, 0), (31, 0), (32, 1), (63, 1), (64, 2)]:
|
||||
test "Can get slotBlockIndex from slotCellIndex (" & $input & " -> " & $expected & ")":
|
||||
let slotBlockIndex = toBlockIdx(input, numCells = 32)
|
||||
|
||||
check:
|
||||
slotBlockIndex == expected
|
||||
|
||||
for (input, expected) in [(10, 10), (31, 31), (32, 0), (63, 31), (64, 0)]:
|
||||
test "Can get blockCellIndex from slotCellIndex (" & $input & " -> " & $expected & ")":
|
||||
let blockCellIndex = toBlockCellIdx(input, numCells = 32)
|
||||
|
||||
check:
|
||||
blockCellIndex == expected
|
||||
|
|
@ -1,4 +1,7 @@
|
|||
import ./slots/testslotbuilder
|
||||
import ./slots/proof/testdatasampler
|
||||
import ./slots/testutils
|
||||
import ./slots/testsampler
|
||||
import ./slots/testconverters
|
||||
|
||||
{.warning[UnusedImport]: off.}
|
||||
|
|
|
|||
Loading…
Reference in New Issue