diff --git a/codex/slots/builder.nim b/codex/slots/builder.nim index 64f5774d..6cf9a371 100644 --- a/codex/slots/builder.nim +++ b/codex/slots/builder.nim @@ -1,4 +1,4 @@ import ./builder/builder -import ./builder/converters +import ./converters export builder, converters diff --git a/codex/slots/builder/builder.nim b/codex/slots/builder/builder.nim index a4baa637..ce0de057 100644 --- a/codex/slots/builder/builder.nim +++ b/codex/slots/builder/builder.nim @@ -348,16 +348,7 @@ proc new*( if manifest.slotRoots.len == 0 or manifest.slotRoots.len != manifest.numSlots: return failure "Manifest is verifiable but slot roots are missing or invalid." - let - # slotRoot = ? Poseidon2Hash.fromBytes( - # ( ? manifest.verifyRoot.mhash.mapFailure ).digestBytes.toArray32 - # ).toFailure - - slotRoots = manifest.slotRoots.mapIt(it.fromSlotCid().toFailure) - # ? Poseidon2Hash.fromBytes( - # ( ? it.mhash.mapFailure ).digestBytes.toArray32 - # ).toFailure - # ) + let slotRoots = manifest.slotRoots.mapIt( (? it.fromSlotCid())) without tree =? self.buildVerifyTree(slotRoots), err: error "Failed to build slot roots tree", err = err.msg diff --git a/codex/slots/sampler/sampler.nim b/codex/slots/sampler/sampler.nim index d626733f..bb0ed13b 100644 --- a/codex/slots/sampler/sampler.nim +++ b/codex/slots/sampler/sampler.nim @@ -39,6 +39,8 @@ type data*: Cell slotProof*: Poseidon2Proof cellProof*: Poseidon2Proof + slotBlockIdx*: Natural + blockCellIdx*: Natural ProofInput* = object entropy*: Poseidon2Hash @@ -96,6 +98,7 @@ proc getProofInput*( let slotTreeCid = self.builder.manifest.slotRoots[self.index] + numCellsPerBlock = (self.builder.manifest.blockSize div self.builder.cellSize).Natural cellIdxs = entropy.cellIndices( self.builder.slotRoots[self.index], self.builder.numSlotCells, @@ -111,21 +114,30 @@ proc getProofInput*( let samples = collect(newSeq): for cellIdx in cellIdxs: let - blockIdx = cellIdx.toBlockIdx(self.builder.numSlotCells) - blkCellIdx = cellIdx.toBlockCellIdx(self.builder.numBlockCells) + blockIdx = cellIdx.toBlockIdx(numCellsPerBlock) + blkCellIdx = cellIdx.toBlockCellIdx(numCellsPerBlock) logScope: cellIdx = cellIdx blockIdx = blockIdx blkCellIdx = blkCellIdx - without (cid, slotProof) =? await self.blockStore.getCidAndProof( + without (cid, proof) =? await self.blockStore.getCidAndProof( slotTreeCid, blockIdx.Natural), err: error "Failed to get block from block store", err = err.msg return failure(err) - without (bytes, blkTree) =? await self.builder.buildBlockTree(blockIdx), err: + without slotProof =? proof.toVerifiableProof(), err: + error "Unable to convert slot proof to poseidon proof", error = err.msg + return failure(err) + + # This converts our slotBlockIndex to a datasetBlockIndex using the + # indexing-strategy used by the builder. + # We need this to fetch the block data. We can't do it by slotTree + slotBlkIdx. + let datasetBlockIndex = self.builder.slotIndicies(self.index)[blockIdx] + + without (bytes, blkTree) =? await self.builder.buildBlockTree(datasetBlockIndex), err: error "Failed to build block tree", err = err.msg return failure(err) @@ -135,7 +147,13 @@ proc getProofInput*( let cellData = self.getCell(bytes, blkCellIdx) - Sample(data: cellData, slotProof: slotProof, cellProof: blockProof) + Sample( + data: cellData, + slotProof: slotProof, + cellProof: blockProof, + slotBlockIdx: blockIdx.Natural, + blockCellIdx: blkCellIdx.Natural + ) success ProofInput( entropy: entropy, diff --git a/codex/slots/sampler/utils.nim b/codex/slots/sampler/utils.nim index c23721ac..0d494fd6 100644 --- a/codex/slots/sampler/utils.nim +++ b/codex/slots/sampler/utils.nim @@ -80,3 +80,4 @@ func cellIndices*( let idx = cellIndex(entropy, slotRoot, numCells, indices.len + 1) indices.add(idx.Natural) indices + diff --git a/tests/codex/slots/provingtestenv.nim b/tests/codex/slots/provingtestenv.nim index e6041f80..95faab67 100644 --- a/tests/codex/slots/provingtestenv.nim +++ b/tests/codex/slots/provingtestenv.nim @@ -78,7 +78,9 @@ proc createSlotTree(self: ProvingTestEnvironment, dSlotIndex: uint64): Future[Po let slotBlocks = datasetBlockIndices.mapIt(self.datasetBlocks[it]) - slotBlockRoots = slotBlocks.mapIt(Poseidon2Tree.digest(it.data, DefaultCellSize.int).tryGet()) + numBlockCells = bytesPerBlock.int div DefaultCellSize.int + blockPadBytes = newSeq[byte](numBlockCells.nextPowerOfTwoPad * DefaultCellSize.int) + slotBlockRoots = slotBlocks.mapIt(Poseidon2Tree.digest(it.data & blockPadBytes, DefaultCellSize.int).tryGet()) tree = Poseidon2Tree.init(slotBlockRoots).tryGet() treeCid = tree.root().tryGet().toSlotCid().tryGet() @@ -145,6 +147,7 @@ proc createSlot(self: ProvingTestEnvironment): void = self.slot = Slot( request: StorageRequest( ask: StorageAsk( + slots: totalNumberOfSlots.uint64, slotSize: u256(bytesPerBlock * numberOfSlotBlocks) ), content: StorageContent( diff --git a/tests/codex/slots/testsampler.nim b/tests/codex/slots/testsampler.nim index 525b2a13..afb79e5e 100644 --- a/tests/codex/slots/testsampler.nim +++ b/tests/codex/slots/testsampler.nim @@ -1,329 +1,117 @@ -# import std/sequtils -# import std/sugar -# import std/random +import std/sequtils +import std/sugar +import std/random +import std/strutils -# import pkg/questionable/results -# import pkg/constantine/math/arithmetic -# import pkg/constantine/math/io/io_fields -# import pkg/poseidon2/types -# import pkg/poseidon2/io -# import pkg/poseidon2 -# import pkg/chronos -# import pkg/asynctest -# import pkg/codex/stores/cachestore -# import pkg/codex/chunker -# import pkg/codex/stores -# import pkg/codex/blocktype as bt -# import pkg/codex/contracts/requests -# import pkg/codex/contracts -# import pkg/codex/merkletree -# import pkg/codex/stores/cachestore +import pkg/questionable/results +import pkg/constantine/math/arithmetic +import pkg/constantine/math/io/io_fields +import pkg/poseidon2/io +import pkg/poseidon2 +import pkg/chronos +import pkg/asynctest +import pkg/codex/stores/cachestore +import pkg/codex/chunker +import pkg/codex/stores +import pkg/codex/blocktype as bt +import pkg/codex/contracts/requests +import pkg/codex/contracts +import pkg/codex/merkletree +import pkg/codex/stores/cachestore -# import pkg/codex/slots -# import pkg/codex/proof/misc -# import pkg/codex/proof/types +import pkg/codex/slots/sampler +import pkg/codex/slots/builder/builder -# import ../helpers -# import ../examples -# import testdatasampler_expected +import ../helpers +import ../examples +import ../merkletree/helpers +import testsampler_expected +import ./provingtestenv -# let -# bytesPerBlock = 64 * 1024 -# challenge: FieldElement = toF(12345) -# datasetRootHash: FieldElement = toF(6789) +asyncchecksuite "Test DataSampler": + var + env: ProvingTestEnvironment + dataSampler: DataSampler + blk: bt.Block + cell0Bytes: seq[byte] + cell1Bytes: seq[byte] + cell2Bytes: seq[byte] -# asyncchecksuite "Test proof datasampler - components": -# let -# numberOfSlotBlocks = 16 -# slot = Slot( -# request: StorageRequest( -# ask: StorageAsk( -# slots: 10, -# slotSize: u256(bytesPerBlock * numberOfSlotBlocks), -# ), -# content: StorageContent( -# cid: $Cid.example -# ) -# ), -# slotIndex: u256(3) -# ) + proc createDataSampler(): Future[void] {.async.} = + dataSampler = DataSampler.new( + datasetSlotIndex, + env.localStore, + SlotsBuilder.new(env.localStore, env.manifest).tryGet()).tryGet() -# test "Number of cells is a power of two": -# # This is to check that the data used for testing is sane. -# proc isPow2(value: int): bool = -# let log2 = ceilingLog2(value) -# return (1 shl log2) == value + setup: + env = await createProvingTestEnvironment() + let bytes = newSeqWith(bytesPerBlock, rand(uint8)) + blk = bt.Block.new(bytes).tryGet() + cell0Bytes = bytes[0.. " & $expected & ")": -# let -# slotCellIndex = input.uint64 -# slotBlockIndex = dataSampler.getSlotBlockIndexForSlotCellIndex(slotCellIndex) - -# check: -# slotBlockIndex == expected.uint64 - -# for (input, expected) in [(10, 10), (31, 31), (32, 0), (63, 31), (64, 0)]: -# test "Can get blockCellIndex from slotCellIndex (" & $input & " -> " & $expected & ")": -# let -# slotCellIndex = input.uint64 -# blockCellIndex = dataSampler.getBlockCellIndexForSlotCellIndex(slotCellIndex) - -# check: -# blockCellIndex == expected.uint64 + # # cell data + toHex(input.samples[0].data) == expectedCellData[0] + toHex(input.samples[1].data) == expectedCellData[1] + toHex(input.samples[2].data) == expectedCellData[2] diff --git a/tests/codex/slots/testsampler_expected.nim b/tests/codex/slots/testsampler_expected.nim new file mode 100644 index 00000000..7d041f86 --- /dev/null +++ b/tests/codex/slots/testsampler_expected.nim @@ -0,0 +1,25 @@ +# Snapshot of expected values for testsampler. + +import std/strutils +import pkg/codex/codextypes + +proc getExpectedCellBlockProofs*(): seq[string] = + @[ + "0x189890bedf2a40f2757554c5f089811e07601543a576e2d40d68a1bd295adbee0x059f227fe687a7abd9c3d9878c0b812aa7829c85d30c23154b8824a907909b060x2c685fc951f1684fd3979f7e3a558fa6aeed3f960ef0a9e2ba51cc62cff7de0e0x10ebae3fb12d502044216e40319726ed36308b9ae4ab3fb0a36c77e5614c6fbd0x1decd3fb7ff458261149731115657cecd7eb2fe4a6cf84f3c6761aa8b0dd6b9a", + "0x2567cd93b3fe058b31908656c05d3a09fd33cc0d7df3c7c005d991a8cda60ba80x16e4788105082295706c49c604216518f16ca9dd106012c7c98e6ee138893f6e0x1c258af996aecf9bba249130182ccfe44a9090bc58fe59063a06db67efb7b5240x10ebae3fb12d502044216e40319726ed36308b9ae4ab3fb0a36c77e5614c6fbd0x1decd3fb7ff458261149731115657cecd7eb2fe4a6cf84f3c6761aa8b0dd6b9a", + "0x0568735989a51526104eddbcf386b8aaef186a2d31afce0c2671c8ce8dd8cd1a0x20d06082668338924981a9e0e4f18e7ec6e2b7912e7fb74c1b6dc921b824def60x2fd45662152ae87192971a0e9b7d50de48d7bc8ab22e5711680173a302120bf00x0f528a58c839889e4bb9195e2bcbc2addb7022e47c8fb11bbdeba0a0e9c6f4cb0x0edf43ec0f277500371537a4d566f3f352d0c49bfa9d4659e07d776ffe119437" + ] + +proc getExpectedBlockSlotProofs*(): seq[string] = + @[ + "0x0684458ea77eca59be05e368bb26b7ca318b8e836100c415e60136876c01ba170x2a66917fa49371e835376fcece0d854c77008ac1195740963b1ac4491ee1aaf1", + "0x0684458ea77eca59be05e368bb26b7ca318b8e836100c415e60136876c01ba170x2a66917fa49371e835376fcece0d854c77008ac1195740963b1ac4491ee1aaf1", + "0x03883ad2637a4c68f29bc0910400259291d9c3d730de7e3925adbf26c80b7f440x2d6a888f50b14b0c686f64c4bd0d8389cd555cdf0e3d6f387682c4722ac2a674" + ] + +proc getExpectedCellData*(): seq[string] = + @[ + "BA".repeat(DefaultCellSize.int), + "BD".repeat(DefaultCellSize.int), + "3D".repeat(DefaultCellSize.int) + ]