From 4e19384a86dc62dc0bcc9d16cc43629db4e2653a Mon Sep 17 00:00:00 2001 From: benbierens Date: Tue, 16 Jan 2024 13:36:34 +0100 Subject: [PATCH] Restores tests for proof sampler utils --- codex/slots/builder/builder.nim | 21 +- codex/slots/builder/converters.nim | 69 ------ codex/slots/converters.nim | 89 ++++++++ codex/slots/sampler/utils.nim | 12 +- tests/codex/examples.nim | 5 + tests/codex/slots/provingtestenv.nim | 168 ++++++++++++++ tests/codex/slots/testconverters.nim | 47 ++++ tests/codex/slots/testsampler.nim | 329 +++++++++++++++++++++++++++ tests/codex/slots/testutils.nim | 104 +++++++++ tests/codex/testslots.nim | 3 + 10 files changed, 762 insertions(+), 85 deletions(-) delete mode 100644 codex/slots/builder/converters.nim create mode 100644 codex/slots/converters.nim create mode 100644 tests/codex/slots/provingtestenv.nim create mode 100644 tests/codex/slots/testconverters.nim create mode 100644 tests/codex/slots/testsampler.nim create mode 100644 tests/codex/slots/testutils.nim diff --git a/codex/slots/builder/builder.nim b/codex/slots/builder/builder.nim index 6b979b99..a4baa637 100644 --- a/codex/slots/builder/builder.nim +++ b/codex/slots/builder/builder.nim @@ -30,8 +30,7 @@ import ../../utils import ../../utils/asynciter import ../../utils/digest import ../../utils/poseidon2digest - -import ./converters +import ../converters export converters @@ -297,7 +296,7 @@ proc buildManifest*(self: SlotsBuilder): Future[?!Manifest] {.async.} = error "Failed to map slot roots to CIDs", err = err.msg return failure(err) - without rootProvingCidRes =? self.verifyRoot.?toSlotsRootsCid() and + without rootProvingCidRes =? self.verifyRoot.?toVerifyCid() and rootProvingCid =? rootProvingCidRes, err: # TODO: why doesn't `.?` unpack the result? error "Failed to map slot roots to CIDs", err = err.msg return failure(err) @@ -350,15 +349,15 @@ proc new*( return failure "Manifest is verifiable but slot roots are missing or invalid." let - slotRoot = ? Poseidon2Hash.fromBytes( - ( ? manifest.verifyRoot.mhash.mapFailure ).digestBytes.toArray32 - ).toFailure + # slotRoot = ? Poseidon2Hash.fromBytes( + # ( ? manifest.verifyRoot.mhash.mapFailure ).digestBytes.toArray32 + # ).toFailure - slotRoots = manifest.slotRoots.mapIt( - ? Poseidon2Hash.fromBytes( - ( ? it.mhash.mapFailure ).digestBytes.toArray32 - ).toFailure - ) + slotRoots = manifest.slotRoots.mapIt(it.fromSlotCid().toFailure) + # ? Poseidon2Hash.fromBytes( + # ( ? it.mhash.mapFailure ).digestBytes.toArray32 + # ).toFailure + # ) without tree =? self.buildVerifyTree(slotRoots), err: error "Failed to build slot roots tree", err = err.msg diff --git a/codex/slots/builder/converters.nim b/codex/slots/builder/converters.nim deleted file mode 100644 index 6b46d3d9..00000000 --- a/codex/slots/builder/converters.nim +++ /dev/null @@ -1,69 +0,0 @@ -## Nim-Codex -## Copyright (c) 2024 Status Research & Development GmbH -## Licensed under either of -## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) -## * MIT license ([LICENSE-MIT](LICENSE-MIT)) -## at your option. -## This file may not be copied, modified, or distributed except according to -## those terms. - -import std/sequtils - -import pkg/libp2p -import pkg/questionable -import pkg/questionable/results -import pkg/poseidon2 -import pkg/poseidon2/io - -import ../../codextypes -import ../../merkletree -import ../../errors - -func toCellCid*(cell: Poseidon2Hash): ?!Cid = - let - cellMhash = ? MultiHash.init(Pos2Bn128MrklCodec, cell.toBytes).mapFailure - cellCid = ? Cid.init(CIDv1, CodexSlotCellCodec, cellMhash).mapFailure - - success cellCid - -func toSlotCid*(root: Poseidon2Hash): ?!Cid = - let - mhash = ? MultiHash.init($multiCodec("identity"), root.toBytes).mapFailure - treeCid = ? Cid.init(CIDv1, SlotRootCodec, mhash).mapFailure - - success treeCid - -func toSlotCids*(slotRoots: openArray[Poseidon2Hash]): ?!seq[Cid] = - success slotRoots.mapIt( ? it.toSlotCid ) - -func toSlotsRootsCid*(root: Poseidon2Hash): ?!Cid = - let - mhash = ? MultiHash.init($multiCodec("identity"), root.toBytes).mapFailure - treeCid = ? Cid.init(CIDv1, SlotProvingRootCodec, mhash).mapFailure - - success treeCid - -func toEncodableProof*( - proof: Poseidon2Proof): ?!CodexProof = - - let - encodableProof = CodexProof( - mcodec: multiCodec("identity"), # copy bytes as is - index: proof.index, - nleaves: proof.nleaves, - path: proof.path.mapIt( @( it.toBytes ) )) - - success encodableProof - -func toVerifiableProof*( - proof: CodexProof): ?!Poseidon2Proof = - - let - verifiableProof = Poseidon2Proof( - index: proof.index, - nleaves: proof.nleaves, - path: proof.path.mapIt( - ? Poseidon2Hash.fromBytes(it.toArray32).toFailure - )) - - success verifiableProof diff --git a/codex/slots/converters.nim b/codex/slots/converters.nim new file mode 100644 index 00000000..81cbc9ce --- /dev/null +++ b/codex/slots/converters.nim @@ -0,0 +1,89 @@ +## Nim-Codex +## Copyright (c) 2024 Status Research & Development GmbH +## Licensed under either of +## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) +## * MIT license ([LICENSE-MIT](LICENSE-MIT)) +## at your option. +## This file may not be copied, modified, or distributed except according to +## those terms. + +import std/sequtils + +import pkg/libp2p +import pkg/stew/arrayops +import pkg/questionable +import pkg/questionable/results +import pkg/poseidon2 +import pkg/poseidon2/io + +import ../codextypes +import ../merkletree +import ../errors +import ../utils/digest + +func toCid(hash: Poseidon2Hash, mcodec: MultiCodec, cidCodec: MultiCodec): ?!Cid = + let + mhash = ? MultiHash.init(mcodec, hash.toBytes).mapFailure + treeCid = ? Cid.init(CIDv1, cidCodec, mhash).mapFailure + success treeCid + +proc toPoseidon2Hash(cid: Cid, mcodec: MultiCodec, cidCodec: MultiCodec): ?!Poseidon2Hash = + if cid.cidver != CIDv1: + return failure("Unexpected CID version") + + if cid.mcodec != cidCodec: + return failure("Cid is not of expected codec. Was: " & $cid.mcodec & " but expected: " & $cidCodec) + + let + mhash = ? cid.mhash.mapFailure + bytes: array[32, byte] = array[32, byte].initCopyFrom(mhash.digestBytes()) + hash = Poseidon2Hash.fromBytes(bytes) + if not hash.isSome(): + return failure("Unable to convert Cid to Poseidon2Hash") + return success(hash.get()) + +func toCellCid*(hash: Poseidon2Hash): ?!Cid = + toCid(hash, Pos2Bn128MrklCodec, CodexSlotCellCodec) + +func fromCellCid*(cid: Cid): ?!Poseidon2Hash = + toPoseidon2Hash(cid, Pos2Bn128MrklCodec, CodexSlotCellCodec) + +func toSlotCid*(hash: Poseidon2Hash): ?!Cid = + toCid(hash, multiCodec("identity"), SlotRootCodec) + +func toSlotCids*(slotRoots: openArray[Poseidon2Hash]): ?!seq[Cid] = + success slotRoots.mapIt( ? it.toSlotCid ) + +func fromSlotCid*(cid: Cid): ?!Poseidon2Hash = + toPoseidon2Hash(cid, multiCodec("identity"), SlotRootCodec) + +func toVerifyCid*(hash: Poseidon2Hash): ?!Cid = + toCid(hash, multiCodec("identity"), SlotProvingRootCodec) + +func fromVerifyCid*(cid: Cid): ?!Poseidon2Hash = + toPoseidon2Hash(cid, multiCodec("identity"), SlotProvingRootCodec) + +func toEncodableProof*( + proof: Poseidon2Proof): ?!CodexProof = + + let + encodableProof = CodexProof( + mcodec: multiCodec("identity"), + index: proof.index, + nleaves: proof.nleaves, + path: proof.path.mapIt( @( it.toBytes ) )) + + success encodableProof + +func toVerifiableProof*( + proof: CodexProof): ?!Poseidon2Proof = + + let + verifiableProof = Poseidon2Proof( + index: proof.index, + nleaves: proof.nleaves, + path: proof.path.mapIt( + ? Poseidon2Hash.fromBytes(it.toArray32).toFailure + )) + + success verifiableProof diff --git a/codex/slots/sampler/utils.nim b/codex/slots/sampler/utils.nim index 484ae7f6..c23721ac 100644 --- a/codex/slots/sampler/utils.nim +++ b/codex/slots/sampler/utils.nim @@ -15,6 +15,8 @@ import pkg/poseidon2/io import pkg/constantine/math/arithmetic +import pkg/constantine/math/io/io_fields + import ../../merkletree func extractLowBits*[n: static int](elm: BigInt[n], k: int): uint64 = @@ -64,8 +66,7 @@ func cellIndex*( let log2 = ceilingLog2(numCells) doAssert( 1 shl log2 == numCells , "`numCells` is assumed to be a power of two" ) - let - hash = Sponge.digest( @[ entropy, slotRoot, counter.toF ], rate = 2 ) + let hash = Sponge.digest( @[ slotRoot, entropy, counter.toF ], rate = 2 ) return int( extractLowBits(hash, log2) ) @@ -74,7 +75,8 @@ func cellIndices*( slotRoot: Poseidon2Hash, numCells: Natural, nSamples: Natural): seq[Natural] = - var indices: seq[int] + var indices: seq[Natural] while (indices.len < nSamples): - let idx = entropy.cellIndex(slotRoot, numCells, indices.len + 1) - indices.add(idx) + let idx = cellIndex(entropy, slotRoot, numCells, indices.len + 1) + indices.add(idx.Natural) + indices diff --git a/tests/codex/examples.nim b/tests/codex/examples.nim index 1bf8a473..c313ad4a 100644 --- a/tests/codex/examples.nim +++ b/tests/codex/examples.nim @@ -75,3 +75,8 @@ proc example*(_: type Reservation): Reservation = proc example*(_: type MerkleProof): MerkleProof = MerkleProof.init(3, @[MultiHash.example]).tryget() + +proc example*(_: type Poseidon2Proof): Poseidon2Proof = + var example = MerkleProof[Poseidon2Hash, PoseidonKeysEnum]() + example.index = 123 + example diff --git a/tests/codex/slots/provingtestenv.nim b/tests/codex/slots/provingtestenv.nim new file mode 100644 index 00000000..e6041f80 --- /dev/null +++ b/tests/codex/slots/provingtestenv.nim @@ -0,0 +1,168 @@ +import std/sequtils + +import pkg/questionable/results +import pkg/poseidon2/io +import pkg/poseidon2 +import pkg/chronos +import pkg/codex/stores/cachestore +import pkg/codex/chunker +import pkg/codex/stores +import pkg/codex/blocktype as bt +import pkg/codex/contracts/requests +import pkg/codex/contracts +import pkg/codex/merkletree +import pkg/codex/stores/cachestore +import pkg/codex/indexingstrategy + +import pkg/codex/slots/converters +import pkg/codex/slots/builder/builder +import pkg/codex/utils/poseidon2digest +import pkg/codex/utils/asynciter + +import ../helpers +import ../merkletree/helpers + +type + ProvingTestEnvironment* = ref object + # Invariant: + challenge*: Poseidon2Hash + # Variant: + localStore*: CacheStore + manifest*: Manifest + manifestBlock*: bt.Block + slot*: Slot + datasetBlocks*: seq[bt.Block] + slotTree*: Poseidon2Tree + slotRootCid*: Cid + slotRoots*: seq[Poseidon2Hash] + datasetToSlotTree*: Poseidon2Tree + datasetRootHash*: Poseidon2Hash + +const + # The number of slot blocks and number of slots, combined with + # the bytes per block, make it so that there are exactly 256 cells + # in the dataset. + bytesPerBlock* = 64 * 1024 + numberOfSlotBlocks* = 4 + totalNumberOfSlots* = 2 + datasetSlotIndex* = 1 + cellsPerSlot* = (bytesPerBlock * numberOfSlotBlocks) div DefaultCellSize.int + +proc createDatasetBlocks(self: ProvingTestEnvironment): Future[void] {.async.} = + let numberOfCellsNeeded = (numberOfSlotBlocks * totalNumberOfSlots * bytesPerBlock).uint64 div DefaultCellSize.uint64 + var data: seq[byte] = @[] + + # This generates a number of blocks that have different data, such that + # Each cell in each block is unique, but nothing is random. + for i in 0 ..< numberOfCellsNeeded: + data = data & (i.byte).repeat(DefaultCellSize.uint64) + + let chunker = MockChunker.new( + dataset = data, + chunkSize = bytesPerBlock) + + while true: + let chunk = await chunker.getBytes() + if chunk.len <= 0: + break + let b = bt.Block.new(chunk).tryGet() + self.datasetBlocks.add(b) + discard await self.localStore.putBlock(b) + +proc createSlotTree(self: ProvingTestEnvironment, dSlotIndex: uint64): Future[Poseidon2Tree] {.async.} = + let + slotSize = (bytesPerBlock * numberOfSlotBlocks).uint64 + blocksInSlot = slotSize div bytesPerBlock.uint64 + datasetBlockIndexingStrategy = SteppedIndexingStrategy.new(0, self.datasetBlocks.len - 1, totalNumberOfSlots) + datasetBlockIndices = toSeq(datasetBlockIndexingStrategy.getIndicies(dSlotIndex.int)) + + let + slotBlocks = datasetBlockIndices.mapIt(self.datasetBlocks[it]) + slotBlockRoots = slotBlocks.mapIt(Poseidon2Tree.digest(it.data, DefaultCellSize.int).tryGet()) + tree = Poseidon2Tree.init(slotBlockRoots).tryGet() + treeCid = tree.root().tryGet().toSlotCid().tryGet() + + for i in 0 ..< numberOfSlotBlocks: + let + blkCid = slotBlockRoots[i].toCellCid().tryGet() + proof = tree.getProof(i).tryGet().toEncodableProof().tryGet() + + discard await self.localStore.putCidAndProof(treeCid, i, blkCid, proof) + + return tree + +proc createDatasetRootHashAndSlotTree(self: ProvingTestEnvironment): Future[void] {.async.} = + var slotTrees = newSeq[Poseidon2Tree]() + for i in 0 ..< totalNumberOfSlots: + slotTrees.add(await self.createSlotTree(i.uint64)) + self.slotTree = slotTrees[datasetSlotIndex] + self.slotRootCid = slotTrees[datasetSlotIndex].root().tryGet().toSlotCid().tryGet() + self.slotRoots = slotTrees.mapIt(it.root().tryGet()) + let rootsPadLeafs = newSeqWith(totalNumberOfSlots.nextPowerOfTwoPad, Poseidon2Zero) + self.datasetToSlotTree = Poseidon2Tree.init(self.slotRoots & rootsPadLeafs).tryGet() + self.datasetRootHash = self.datasetToSlotTree.root().tryGet() + +proc createManifest(self: ProvingTestEnvironment): Future[void] {.async.} = + let + cids = self.datasetBlocks.mapIt(it.cid) + tree = CodexTree.init(cids).tryGet() + treeCid = tree.rootCid(CIDv1, BlockCodec).tryGet() + + for i in 0 ..< self.datasetBlocks.len: + let + blk = self.datasetBlocks[i] + leafCid = blk.cid + proof = tree.getProof(i).tryGet() + discard await self.localStore.putBlock(blk) + discard await self.localStore.putCidAndProof(treeCid, i, leafCid, proof) + + # Basic manifest: + self.manifest = Manifest.new( + treeCid = treeCid, + blockSize = bytesPerBlock.NBytes, + datasetSize = (bytesPerBlock * numberOfSlotBlocks * totalNumberOfSlots).NBytes) + + # Protected manifest: + self.manifest = Manifest.new( + manifest = self.manifest, + treeCid = treeCid, + datasetSize = self.manifest.datasetSize, + ecK = totalNumberOfSlots, + ecM = 0 + ) + + # Verifiable manifest: + self.manifest = Manifest.new( + manifest = self.manifest, + verifyRoot = self.datasetRootHash.toVerifyCid().tryGet(), + slotRoots = self.slotRoots.mapIt(it.toSlotCid().tryGet()) + ).tryGet() + + self.manifestBlock = bt.Block.new(self.manifest.encode().tryGet(), codec = ManifestCodec).tryGet() + discard await self.localStore.putBlock(self.manifestBlock) + +proc createSlot(self: ProvingTestEnvironment): void = + self.slot = Slot( + request: StorageRequest( + ask: StorageAsk( + slotSize: u256(bytesPerBlock * numberOfSlotBlocks) + ), + content: StorageContent( + cid: $self.manifestBlock.cid + ), + ), + slotIndex: u256(datasetSlotIndex) + ) + +proc createProvingTestEnvironment*(): Future[ProvingTestEnvironment] {.async.} = + var testEnv = ProvingTestEnvironment( + challenge: toF(12345) + ) + + testEnv.localStore = CacheStore.new() + await testEnv.createDatasetBlocks() + await testEnv.createDatasetRootHashAndSlotTree() + await testEnv.createManifest() + testEnv.createSlot() + + return testEnv diff --git a/tests/codex/slots/testconverters.nim b/tests/codex/slots/testconverters.nim new file mode 100644 index 00000000..d76302bb --- /dev/null +++ b/tests/codex/slots/testconverters.nim @@ -0,0 +1,47 @@ +import pkg/chronos +import pkg/asynctest +import pkg/poseidon2 +import pkg/poseidon2/io +import pkg/constantine/math/io/io_fields +import pkg/questionable/results +import pkg/codex/merkletree + +import pkg/codex/slots/converters +import ../examples +import ../merkletree/helpers + +let + hash: Poseidon2Hash = toF(12345) + +suite "Converters": + test "CellBlock cid": + let + cid = toCellCid(hash).tryGet() + value = fromCellCid(cid).tryGet() + + check: + hash.toDecimal() == value.toDecimal() + + test "Slot cid": + let + cid = toSlotCid(hash).tryGet() + value = fromSlotCid(cid).tryGet() + + check: + hash.toDecimal() == value.toDecimal() + + test "Verify cid": + let + cid = toVerifyCid(hash).tryGet() + value = fromVerifyCid(cid).tryGet() + + check: + hash.toDecimal() == value.toDecimal() + + test "Proof": + let + codexProof = toEncodableProof(Poseidon2Proof.example).tryGet() + poseidonProof = toVerifiableProof(codexProof).tryGet() + + check: + Poseidon2Proof.example == poseidonProof diff --git a/tests/codex/slots/testsampler.nim b/tests/codex/slots/testsampler.nim new file mode 100644 index 00000000..525b2a13 --- /dev/null +++ b/tests/codex/slots/testsampler.nim @@ -0,0 +1,329 @@ +# import std/sequtils +# import std/sugar +# import std/random + +# import pkg/questionable/results +# import pkg/constantine/math/arithmetic +# import pkg/constantine/math/io/io_fields +# import pkg/poseidon2/types +# import pkg/poseidon2/io +# import pkg/poseidon2 +# import pkg/chronos +# import pkg/asynctest +# import pkg/codex/stores/cachestore +# import pkg/codex/chunker +# import pkg/codex/stores +# import pkg/codex/blocktype as bt +# import pkg/codex/contracts/requests +# import pkg/codex/contracts +# import pkg/codex/merkletree +# import pkg/codex/stores/cachestore + +# import pkg/codex/slots +# import pkg/codex/proof/misc +# import pkg/codex/proof/types + +# import ../helpers +# import ../examples +# import testdatasampler_expected + +# let +# bytesPerBlock = 64 * 1024 +# challenge: FieldElement = toF(12345) +# datasetRootHash: FieldElement = toF(6789) + +# asyncchecksuite "Test proof datasampler - components": +# let +# numberOfSlotBlocks = 16 +# slot = Slot( +# request: StorageRequest( +# ask: StorageAsk( +# slots: 10, +# slotSize: u256(bytesPerBlock * numberOfSlotBlocks), +# ), +# content: StorageContent( +# cid: $Cid.example +# ) +# ), +# slotIndex: u256(3) +# ) + +# test "Number of cells is a power of two": +# # This is to check that the data used for testing is sane. +# proc isPow2(value: int): bool = +# let log2 = ceilingLog2(value) +# return (1 shl log2) == value + +# let numberOfCells = getNumberOfCellsInSlot(slot).int + +# check: +# isPow2(numberOfCells) + +# test "Extract low bits": +# proc extract(value: uint64, nBits: int): uint64 = +# let big = toF(value).toBig() +# return extractLowBits(big, nBits) + +# check: +# extract(0x88, 4) == 0x8.uint64 +# extract(0x88, 7) == 0x8.uint64 +# extract(0x9A, 5) == 0x1A.uint64 +# extract(0x9A, 7) == 0x1A.uint64 +# extract(0x1248, 10) == 0x248.uint64 +# extract(0x1248, 12) == 0x248.uint64 +# extract(0x1248306A560C9AC0.uint64, 10) == 0x2C0.uint64 +# extract(0x1248306A560C9AC0.uint64, 12) == 0xAC0.uint64 +# extract(0x1248306A560C9AC0.uint64, 50) == 0x306A560C9AC0.uint64 +# extract(0x1248306A560C9AC0.uint64, 52) == 0x8306A560C9AC0.uint64 + +# test "Should calculate total number of cells in Slot": +# let +# slotSizeInBytes = (slot.request.ask.slotSize).truncate(uint64) +# expectedNumberOfCells = slotSizeInBytes div CellSize + +# check: +# expectedNumberOfCells == 512 +# expectedNumberOfCells == getNumberOfCellsInSlot(slot) + +# asyncchecksuite "Test proof datasampler - main": +# let +# # The number of slot blocks and number of slots, combined with +# # the bytes per block, make it so that there are exactly 256 cells +# # in the dataset. +# numberOfSlotBlocks = 4 +# totalNumberOfSlots = 2 +# datasetSlotIndex = 1 +# localStore = CacheStore.new() +# datasetToSlotProof = MerkleProof.example + +# var +# manifest: Manifest +# manifestBlock: bt.Block +# slot: Slot +# datasetBlocks: seq[bt.Block] +# slotPoseidonTree: MerkleTree +# dataSampler: DataSampler + +# proc createDatasetBlocks(): Future[void] {.async.} = +# let numberOfCellsNeeded = (numberOfSlotBlocks * totalNumberOfSlots * bytesPerBlock).uint64 div CellSize +# var data: seq[byte] = @[] + +# # This generates a number of blocks that have different data, such that +# # Each cell in each block is unique, but nothing is random. +# for i in 0 ..< numberOfCellsNeeded: +# data = data & (i.byte).repeat(CellSize) + +# let chunker = MockChunker.new( +# dataset = data, +# chunkSize = bytesPerBlock) + +# while true: +# let chunk = await chunker.getBytes() +# if chunk.len <= 0: +# break +# let b = bt.Block.new(chunk).tryGet() +# datasetBlocks.add(b) +# discard await localStore.putBlock(b) + +# proc createManifest(): Future[void] {.async.} = +# let +# cids = datasetBlocks.mapIt(it.cid) +# tree = MerkleTree.init(cids).tryGet() +# treeCid = tree.rootCid().tryGet() + +# for index, cid in cids: +# let proof = tree.getProof(index).tryget() +# discard await localStore.putBlockCidAndProof(treeCid, index, cid, proof) + +# manifest = Manifest.new( +# treeCid = treeCid, +# blockSize = bytesPerBlock.NBytes, +# datasetSize = (bytesPerBlock * numberOfSlotBlocks * totalNumberOfSlots).NBytes) +# manifestBlock = bt.Block.new(manifest.encode().tryGet(), codec = DagPBCodec).tryGet() + +# proc createSlot(): void = +# slot = Slot( +# request: StorageRequest( +# ask: StorageAsk( +# slotSize: u256(bytesPerBlock * numberOfSlotBlocks) +# ), +# content: StorageContent( +# cid: $manifestBlock.cid +# ), +# ), +# slotIndex: u256(datasetSlotIndex) +# ) + +# proc createSlotPoseidonTree(): void = +# let +# slotSize = slot.request.ask.slotSize.truncate(uint64) +# blocksInSlot = slotSize div bytesPerBlock.uint64 +# datasetSlotIndex = slot.slotIndex.truncate(uint64) +# datasetBlockIndexFirst = datasetSlotIndex * blocksInSlot +# datasetBlockIndexLast = datasetBlockIndexFirst + numberOfSlotBlocks.uint64 +# slotBlocks = datasetBlocks[datasetBlockIndexFirst ..< datasetBlockIndexLast] +# slotBlockCids = slotBlocks.mapIt(it.cid) +# slotPoseidonTree = MerkleTree.init(slotBlockCids).tryGet() + +# proc createDataSampler(): Future[void] {.async.} = +# dataSampler = (await DataSampler.new( +# slot, +# localStore, +# datasetRootHash, +# slotPoseidonTree, +# datasetToSlotProof +# )).tryGet() + +# setup: +# await createDatasetBlocks() +# await createManifest() +# createSlot() +# discard await localStore.putBlock(manifestBlock) +# createSlotPoseidonTree() +# await createDataSampler() + +# test "Number of cells is a power of two": +# # This is to check that the data used for testing is sane. +# proc isPow2(value: int): bool = +# let log2 = ceilingLog2(value) +# return (1 shl log2) == value + +# let numberOfCells = getNumberOfCellsInSlot(slot).int + +# check: +# isPow2(numberOfCells) + +# let knownIndices = @[74.uint64, 41.uint64, 51.uint64] + +# test "Can find single slot-cell index": +# proc slotCellIndex(i: int): uint64 = +# let counter: FieldElement = toF(i) +# return dataSampler.findSlotCellIndex(challenge, counter) + +# proc getExpectedIndex(i: int): uint64 = +# let +# numberOfCellsInSlot = (bytesPerBlock * numberOfSlotBlocks) div CellSize.int +# slotRootHash = toF(1234) # TODO - replace with slotPoseidonTree.root when it is a poseidon tree. +# hash = Sponge.digest(@[slotRootHash, challenge, toF(i)], rate = 2) +# return extractLowBits(hash.toBig(), ceilingLog2(numberOfCellsInSlot)) + +# check: +# slotCellIndex(1) == getExpectedIndex(1) +# slotCellIndex(1) == knownIndices[0] +# slotCellIndex(2) == getExpectedIndex(2) +# slotCellIndex(2) == knownIndices[1] +# slotCellIndex(3) == getExpectedIndex(3) +# slotCellIndex(3) == knownIndices[2] + +# test "Can find sequence of slot-cell indices": +# proc slotCellIndices(n: int): seq[uint64] = +# dataSampler.findSlotCellIndices(challenge, n) + +# proc getExpectedIndices(n: int): seq[uint64] = +# return collect(newSeq, (for i in 1..n: dataSampler.findSlotCellIndex(challenge, toF(i)))) + +# check: +# slotCellIndices(3) == getExpectedIndices(3) +# slotCellIndices(3) == knownIndices + +# let +# bytes = newSeqWith(bytesPerBlock, rand(uint8)) +# blk = bt.Block.new(bytes).tryGet() +# cell0Bytes = bytes[0.. " & $expected & ")": +# let +# slotCellIndex = input.uint64 +# slotBlockIndex = dataSampler.getSlotBlockIndexForSlotCellIndex(slotCellIndex) + +# check: +# slotBlockIndex == expected.uint64 + +# for (input, expected) in [(10, 10), (31, 31), (32, 0), (63, 31), (64, 0)]: +# test "Can get blockCellIndex from slotCellIndex (" & $input & " -> " & $expected & ")": +# let +# slotCellIndex = input.uint64 +# blockCellIndex = dataSampler.getBlockCellIndexForSlotCellIndex(slotCellIndex) + +# check: +# blockCellIndex == expected.uint64 diff --git a/tests/codex/slots/testutils.nim b/tests/codex/slots/testutils.nim new file mode 100644 index 00000000..04e38b40 --- /dev/null +++ b/tests/codex/slots/testutils.nim @@ -0,0 +1,104 @@ +import std/sequtils +import std/sugar +import std/random +import std/strutils + +import pkg/questionable/results +import pkg/constantine/math/arithmetic +import pkg/constantine/math/io/io_fields +import pkg/poseidon2/io +import pkg/poseidon2 +import pkg/chronos +import pkg/asynctest +import pkg/codex/stores/cachestore +import pkg/codex/chunker +import pkg/codex/stores +import pkg/codex/blocktype as bt +import pkg/codex/contracts/requests +import pkg/codex/contracts +import pkg/codex/merkletree +import pkg/codex/stores/cachestore + +import pkg/codex/slots/sampler/utils + +import ../helpers +import ../examples +import ../merkletree/helpers +import ./provingtestenv + +asyncchecksuite "Test proof sampler utils": + let knownIndices: seq[Natural] = @[90, 93, 29] + + var + env: ProvingTestEnvironment + slotRoot: Poseidon2Hash + numCells: Natural + + setup: + env = await createProvingTestEnvironment() + slotRoot = env.slotRoots[datasetSlotIndex] + numCells = cellsPerSlot + + teardown: + reset(env) + + test "Extract low bits": + proc extract(value: uint64, nBits: int): uint64 = + let big = toF(value).toBig() + return extractLowBits(big, nBits) + + check: + extract(0x88, 4) == 0x8.uint64 + extract(0x88, 7) == 0x8.uint64 + extract(0x9A, 5) == 0x1A.uint64 + extract(0x9A, 7) == 0x1A.uint64 + extract(0x1248, 10) == 0x248.uint64 + extract(0x1248, 12) == 0x248.uint64 + extract(0x1248306A560C9AC0.uint64, 10) == 0x2C0.uint64 + extract(0x1248306A560C9AC0.uint64, 12) == 0xAC0.uint64 + extract(0x1248306A560C9AC0.uint64, 50) == 0x306A560C9AC0.uint64 + extract(0x1248306A560C9AC0.uint64, 52) == 0x8306A560C9AC0.uint64 + + test "Can find single slot-cell index": + proc slotCellIndex(i: Natural): Natural = + return cellIndex(env.challenge, slotRoot, numCells, i) + + proc getExpectedIndex(i: int): Natural = + let + numberOfCellsInSlot = (bytesPerBlock * numberOfSlotBlocks) div DefaultCellSize.uint64.int + hash = Sponge.digest(@[slotRoot, env.challenge, toF(i)], rate = 2) + + return int(extractLowBits(hash.toBig(), ceilingLog2(numberOfCellsInSlot))) + + check: + slotCellIndex(1) == getExpectedIndex(1) + slotCellIndex(1) == knownIndices[0] + slotCellIndex(2) == getExpectedIndex(2) + slotCellIndex(2) == knownIndices[1] + slotCellIndex(3) == getExpectedIndex(3) + slotCellIndex(3) == knownIndices[2] + + test "Can find sequence of slot-cell indices": + proc slotCellIndices(n: int): seq[Natural] = + cellIndices(env.challenge, slotRoot, numCells, n) + + proc getExpectedIndices(n: int): seq[Natural] = + return collect(newSeq, (for i in 1..n: cellIndex(env.challenge, slotRoot, numCells, i))) + + check: + slotCellIndices(3) == getExpectedIndices(3) + slotCellIndices(3) == knownIndices + + for (input, expected) in [(10, 0), (31, 0), (32, 1), (63, 1), (64, 2)]: + test "Can get slotBlockIndex from slotCellIndex (" & $input & " -> " & $expected & ")": + let slotBlockIndex = toBlockIdx(input, numCells = 32) + + check: + slotBlockIndex == expected + + for (input, expected) in [(10, 10), (31, 31), (32, 0), (63, 31), (64, 0)]: + test "Can get blockCellIndex from slotCellIndex (" & $input & " -> " & $expected & ")": + let blockCellIndex = toBlockCellIdx(input, numCells = 32) + + check: + blockCellIndex == expected diff --git a/tests/codex/testslots.nim b/tests/codex/testslots.nim index 471fd5c6..ffa9847d 100644 --- a/tests/codex/testslots.nim +++ b/tests/codex/testslots.nim @@ -1,4 +1,7 @@ import ./slots/testslotbuilder import ./slots/proof/testdatasampler +import ./slots/testutils +import ./slots/testsampler +import ./slots/testconverters {.warning[UnusedImport]: off.}