From 0cc4563eba6e6046d3cc28e1fbeb2f7a87ae3326 Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 24 Nov 2023 14:53:50 +0100 Subject: [PATCH] splits up indexing --- codex/proof/datasampler.nim | 91 ++++++++++----------------- codex/proof/indexing.nim | 24 +++++++ codex/proof/slotblocks.nim | 14 +---- codex/proof/types.nim | 17 +++++ tests/codex/examples.nim | 4 ++ tests/codex/proof/testdatasampler.nim | 87 ++++++++++--------------- tests/codex/proof/testindexing.nim | 52 +++++++++++++++ tests/codex/proof/testslotblocks.nim | 15 +---- tests/codex/testproof.nim | 1 + 9 files changed, 170 insertions(+), 135 deletions(-) create mode 100644 codex/proof/indexing.nim create mode 100644 codex/proof/types.nim create mode 100644 tests/codex/proof/testindexing.nim diff --git a/codex/proof/datasampler.nim b/codex/proof/datasampler.nim index 40c6e5ed..3134b1f6 100644 --- a/codex/proof/datasampler.nim +++ b/codex/proof/datasampler.nim @@ -17,20 +17,8 @@ import pkg/poseidon2 import misc import slotblocks - -const - # Size of a cell. - # A cell is a sample of storage-data selected for proving. - CellSize* = 2048.uint64 - -type - DSFieldElement* = F - DSCellIndex* = uint64 - DSCell* = seq[byte] - ProofInput* = ref object - blockInclProofs*: seq[MerkleProof] - cellInclProofs*: seq[MerkleProof] - sampleData*: seq[byte] +import indexing +import types func extractLowBits*[n: static int](A: BigInt[n], k: int): uint64 = assert(k > 0 and k <= 64) @@ -45,7 +33,7 @@ func extractLowBits*[n: static int](A: BigInt[n], k: int): uint64 = r = bitor(r, 1'u64 shl i) return r -proc getCellIndex(fe: DSFieldElement, numberOfCells: int): uint64 = +proc convertToSlotCellIndex(fe: DSFieldElement, numberOfCells: int): uint64 = let log2 = ceilingLog2(numberOfCells) assert((1 shl log2) == numberOfCells , "expected `numberOfCells` to be a power of two.") @@ -54,48 +42,32 @@ proc getCellIndex(fe: DSFieldElement, numberOfCells: int): uint64 = proc getNumberOfCellsInSlot*(slot: Slot): uint64 = (slot.request.ask.slotSize.truncate(uint64) div CellSize) -proc findCellIndex*( +proc findSlotCellIndex*( slotRootHash: DSFieldElement, challenge: DSFieldElement, counter: DSFieldElement, - numberOfCells: uint64): DSCellIndex = - # Computes the cell index for a single sample. + numberOfCells: uint64): DSSlotCellIndex = + # Computes the slot-cell index for a single sample. let input = @[slotRootHash, challenge, counter] hash = Sponge.digest(input, rate = 2) - index = getCellIndex(hash, numberOfCells.int) + index = convertToSlotCellIndex(hash, numberOfCells.int) return index -func findCellIndices*( +func findSlotCellIndices*( slot: Slot, slotRootHash: DSFieldElement, challenge: DSFieldElement, - nSamples: int): seq[DSCellIndex] = - # Computes nSamples cell indices. + nSamples: int): seq[DSSlotCellIndex] = + # Computes nSamples slot-cell indices. let numberOfCells = getNumberOfCellsInSlot(slot) - return collect(newSeq, (for i in 1..nSamples: findCellIndex(slotRootHash, challenge, toF(i), numberOfCells))) + return collect(newSeq, (for i in 1..nSamples: findSlotCellIndex(slotRootHash, challenge, toF(i), numberOfCells))) -proc getSlotBlockIndex*(cellIndex: DSCellIndex, blockSize: uint64): uint64 = - let numberOfCellsPerBlock = blockSize div CellSize - return cellIndex div numberOfCellsPerBlock - -proc getDatasetBlockIndex*(slot: Slot, slotBlockIndex: uint64, blockSize: uint64): uint64 = +proc getCellFromBlock*(blk: bt.Block, slotCellIndex: DSSlotCellIndex, blockSize: uint64): DSCell = let - slotIndex = slot.slotIndex.truncate(uint64) - slotSize = slot.request.ask.slotSize.truncate(uint64) - blocksInSlot = slotSize div blockSize - - return (blocksInSlot * slotIndex) + slotBlockIndex - -proc getCellIndexInBlock*(cellIndex: DSCellIndex, blockSize: uint64): uint64 = - let numberOfCellsPerBlock = blockSize div CellSize - return cellIndex mod numberOfCellsPerBlock - -proc getCellFromBlock*(blk: bt.Block, cellIndex: DSCellIndex, blockSize: uint64): DSCell = - let - inBlockCellIndex = getCellIndexInBlock(cellIndex, blockSize) - dataStart = (CellSize * inBlockCellIndex) + blockCellIndex = getBlockCellIndexForSlotCellIndex(slotCellIndex, blockSize) + dataStart = (CellSize * blockCellIndex) dataEnd = dataStart + CellSize return blk.data[dataStart ..< dataEnd] @@ -124,13 +96,14 @@ proc getProofInput*( slot: Slot, blockStore: BlockStore, slotRootHash: DSFieldElement, - dataSetPoseidonTree: MerkleTree, + slotPoseidonTree: MerkleTree, + datasetToSlotProof: MerkleProof, challenge: DSFieldElement, nSamples: int ): Future[?!ProofInput] {.async.} = var - blockProofs: seq[MerkleProof] - cellProofs: seq[MerkleProof] + slotToBlockProofs: seq[MerkleProof] + blockToCellProofs: seq[MerkleProof] sampleData: seq[byte] without manifest =? await getManifestForSlot(slot, blockStore), err: @@ -139,10 +112,13 @@ proc getProofInput*( let blockSize = manifest.blockSize.uint64 - cellIndices = findCellIndices(slot, slotRootHash, challenge, nSamples) + slotCellIndices = findSlotCellIndices(slot, slotRootHash, challenge, nSamples) + + for slotCellIndex in slotCellIndices: + let + slotBlockIndex = getSlotBlockIndexForSlotCellIndex(slotCellIndex, blockSize) + datasetBlockIndex = getDatasetBlockIndexForSlotBlockIndex(slot, slotBlockIndex, blockSize) - for cellIndex in cellIndices: - let slotBlockIndex = getSlotBlockIndex(cellIndex, blockSize) without blk =? await getSlotBlock(slot, blockStore, manifest, slotBlockIndex), err: error "Failed to get slot block" return failure(err) @@ -151,22 +127,23 @@ proc getProofInput*( error "Failed to calculate minitree for block" return failure(err) - # without blockProof =? dataSetPoseidonTree.getProof(???block index in dataset!), err: - # error "Failed to get dataset inclusion proof" - # return failure(err) - # blockProofs.add(blockProof) + without blockProof =? slotPoseidonTree.getProof(datasetBlockIndex), err: + error "Failed to get dataset inclusion proof" + return failure(err) + slotToBlockProofs.add(blockProof) - without cellProof =? miniTree.getProof(cellIndex), err: + without cellProof =? miniTree.getProof(slotCellIndex), err: error "Failed to get cell inclusion proof" return failure(err) - cellProofs.add(cellProof) + blockToCellProofs.add(cellProof) - let cell = getCellFromBlock(blk, cellIndex, blockSize) + let cell = getCellFromBlock(blk, slotCellIndex, blockSize) sampleData = sampleData & cell trace "Successfully collected proof input data" success(ProofInput( - blockInclProofs: blockProofs, - cellInclProofs: cellProofs, + datasetToSlotProof: datasetToSlotProof, + slotToBlockProofs: slotToBlockProofs, + blockToCellProofs: blockToCellProofs, sampleData: sampleData )) diff --git a/codex/proof/indexing.nim b/codex/proof/indexing.nim new file mode 100644 index 00000000..4925b03b --- /dev/null +++ b/codex/proof/indexing.nim @@ -0,0 +1,24 @@ +import pkg/chronicles +import ../contracts/requests +import types + +# Index naming convention: +# "Index" => The index of an ElementType within a ContainerType. +# Some examples: +# SlotBlockIndex => The index of a Block within a Slot. +# DatasetBlockIndex => The index of a Block within a Dataset. + +proc getSlotBlockIndexForSlotCellIndex*(cellIndex: DSSlotCellIndex, blockSize: uint64): uint64 = + let numberOfCellsPerBlock = blockSize div CellSize + return cellIndex div numberOfCellsPerBlock + +proc getBlockCellIndexForSlotCellIndex*(cellIndex: DSSlotCellIndex, blockSize: uint64): uint64 = + let numberOfCellsPerBlock = blockSize div CellSize + return cellIndex mod numberOfCellsPerBlock + +proc getDatasetBlockIndexForSlotBlockIndex*(slot: Slot, blockSize: uint64, slotBlockIndex: uint64): uint64 = + let + slotSize = slot.request.ask.slotSize.truncate(uint64) + blocksInSlot = slotSize div blockSize + datasetSlotIndex = slot.slotIndex.truncate(uint64) + return (datasetSlotIndex * blocksInSlot) + slotBlockIndex diff --git a/codex/proof/slotblocks.nim b/codex/proof/slotblocks.nim index 398c3fa2..a9ea8341 100644 --- a/codex/proof/slotblocks.nim +++ b/codex/proof/slotblocks.nim @@ -10,6 +10,7 @@ import pkg/questionable/results import ../contracts/requests import ../stores/blockstore import ../manifest +import indexing proc getManifestForSlot*(slot: Slot, blockstore: BlockStore): Future[?!Manifest] {.async.} = without manifestBlockCid =? Cid.init(slot.request.content.cid).mapFailure, err: @@ -26,21 +27,10 @@ proc getManifestForSlot*(slot: Slot, blockstore: BlockStore): Future[?!Manifest] return success(manifest) -proc getIndexForSlotBlock*(slot: Slot, blockSize: NBytes, slotBlockIndex: uint64): uint64 = - let - slotSize = slot.request.ask.slotSize.truncate(uint64) - blocksInSlot = slotSize div blockSize.uint64 - slotIndex = slot.slotIndex.truncate(uint64) - - datasetIndex = (slotIndex * blocksInSlot) + slotBlockIndex - - trace "Converted slotBlockIndex to datasetIndex", slotBlockIndex, datasetIndex - return datasetIndex - proc getSlotBlock*(slot: Slot, blockstore: BlockStore, manifest: Manifest, slotBlockIndex: uint64): Future[?!Block] {.async.} = let blocksInManifest = (manifest.datasetSize div manifest.blockSize).uint64 - datasetIndex = getIndexForSlotBlock(slot, manifest.blockSize, slotBlockIndex) + datasetIndex = getDatasetBlockIndexForSlotBlockIndex(slot, manifest.blockSize.uint64, slotBlockIndex) if datasetIndex >= blocksInManifest: return failure("Found slotBlockIndex that is out-of-range: " & $datasetIndex) diff --git a/codex/proof/types.nim b/codex/proof/types.nim new file mode 100644 index 00000000..f91c7b38 --- /dev/null +++ b/codex/proof/types.nim @@ -0,0 +1,17 @@ +import pkg/poseidon2/types +import ../merkletree + +const + # Size of a cell. + # A cell is a sample of storage-data selected for proving. + CellSize* = 2048.uint64 + +type + DSFieldElement* = F + DSSlotCellIndex* = uint64 + DSCell* = seq[byte] + ProofInput* = ref object + datasetToSlotProof*: MerkleProof + slotToBlockProofs*: seq[MerkleProof] + blockToCellProofs*: seq[MerkleProof] + sampleData*: seq[byte] diff --git a/tests/codex/examples.nim b/tests/codex/examples.nim index 28467c64..1bf8a473 100644 --- a/tests/codex/examples.nim +++ b/tests/codex/examples.nim @@ -7,6 +7,7 @@ import pkg/codex/rng import pkg/codex/stores import pkg/codex/blocktype as bt import pkg/codex/sales +import pkg/codex/merkletree import ../examples export examples @@ -71,3 +72,6 @@ proc example*(_: type Reservation): Reservation = size = uint16.example.u256, slotId = SlotId.example ) + +proc example*(_: type MerkleProof): MerkleProof = + MerkleProof.init(3, @[MultiHash.example]).tryget() diff --git a/tests/codex/proof/testdatasampler.nim b/tests/codex/proof/testdatasampler.nim index f9eaf6be..2ccbe4bc 100644 --- a/tests/codex/proof/testdatasampler.nim +++ b/tests/codex/proof/testdatasampler.nim @@ -28,6 +28,8 @@ import pkg/codex/stores/cachestore import pkg/codex/proof/datasampler import pkg/codex/proof/misc +import pkg/codex/proof/types +#import pkg/codex/proof/indexing import ../helpers import ../examples @@ -116,69 +118,36 @@ asyncchecksuite "Test proof datasampler": let knownIndices = @[178.uint64, 277.uint64, 366.uint64] - test "Can find single cell index": + test "Can find single slot-cell index": let numberOfCells = getNumberOfCellsInSlot(slot) - proc cellIndex(i: int): DSCellIndex = + proc slotCellIndex(i: int): DSSlotCellIndex = let counter: DSFieldElement = toF(i) - return findCellIndex(slotRootHash, challenge, counter, numberOfCells) + return findSlotCellIndex(slotRootHash, challenge, counter, numberOfCells) - proc getExpectedIndex(i: int): DSCellIndex = + proc getExpectedIndex(i: int): DSSlotCellIndex = let hash = Sponge.digest(@[slotRootHash, challenge, toF(i)], rate = 2) return extractLowBits(hash.toBig(), ceilingLog2(numberOfCells.int)) check: - cellIndex(1) == getExpectedIndex(1) - cellIndex(1) == knownIndices[0] - cellIndex(2) == getExpectedIndex(2) - cellIndex(2) == knownIndices[1] - cellIndex(3) == getExpectedIndex(3) - cellIndex(3) == knownIndices[2] + slotCellIndex(1) == getExpectedIndex(1) + slotCellIndex(1) == knownIndices[0] + slotCellIndex(2) == getExpectedIndex(2) + slotCellIndex(2) == knownIndices[1] + slotCellIndex(3) == getExpectedIndex(3) + slotCellIndex(3) == knownIndices[2] - test "Can find sequence of cell indices": - proc cellIndices(n: int): seq[DSCellIndex] = - findCellIndices(slot, slotRootHash, challenge, n) + test "Can find sequence of slot-cell indices": + proc slotCellIndices(n: int): seq[DSSlotCellIndex] = + findSlotCellIndices(slot, slotRootHash, challenge, n) let numberOfCells = getNumberOfCellsInSlot(slot) - proc getExpectedIndices(n: int): seq[DSCellIndex] = - return collect(newSeq, (for i in 1..n: findCellIndex(slotRootHash, challenge, toF(i), numberOfCells))) + proc getExpectedIndices(n: int): seq[DSSlotCellIndex] = + return collect(newSeq, (for i in 1..n: findSlotCellIndex(slotRootHash, challenge, toF(i), numberOfCells))) check: - cellIndices(3) == getExpectedIndices(3) - cellIndices(3) == knownIndices - - for (input, expected) in [(10, 0), (31, 0), (32, 1), (63, 1), (64, 2)]: - test "Can get slotBlockIndex from cell index (" & $input & " -> " & $expected & ")": - let - cellIndex = input.uint64 - blockSize = (64 * 1024).uint64 - - slotBlockIndex = getSlotBlockIndex(cellIndex, blockSize) - - check: - slotBlockIndex == expected.uint64 - - for input in 0 ..< numberOfSlotBlocks: - test "Can get datasetBlockIndex from slotBlockIndex (" & $input & ")": - let - slotBlockIndex = input.uint64 - datasetBlockIndex = getDatasetBlockIndex(slot, slotBlockIndex, bytesPerBlock.uint64) - slotIndex = slot.slotIndex.truncate(uint64) - expectedIndex = (numberOfSlotBlocks.uint64 * slotIndex) + slotBlockIndex - - check: - datasetBlockIndex == expectedIndex - - for (input, expected) in [(10, 10), (31, 31), (32, 0), (63, 31), (64, 0)]: - test "Can get cellIndexInBlock from cell index (" & $input & " -> " & $expected & ")": - let - cellIndex = input.uint64 - blockSize = (64 * 1024).uint64 - - cellIndexInBlock = getCellIndexInBlock(cellIndex, blockSize) - - check: - cellIndexInBlock == expected.uint64 + slotCellIndices(3) == getExpectedIndices(3) + slotCellIndices(3) == knownIndices test "Can get cell from block": let @@ -234,9 +203,19 @@ asyncchecksuite "Test proof datasampler": # This is the main entry point for this module, and what it's all about. let localStore = CacheStore.new() - dataSetPoseidonTree = MerkleTree.init(@[Cid.example]).tryget() - a = (await getProofInput(slot, localStore, slotRootHash, dataSetPoseidonTree, challenge, 3)).tryget() + datasetToSlotProof = MerkleProof.example + slotPoseidonTree = MerkleTree.init(@[Cid.example]).tryget() + nSamples = 3 - echo "a.blockInclProofs: " & $a.blockInclProofs.len - echo "a.cellInclProofs: " & $a.cellInclProofs.len + a = (await getProofInput( + slot, + localStore, + slotRootHash, + slotPoseidonTree, + datasetToSlotProof, + challenge, + nSamples)).tryget() + + echo "a.slotToBlockProofs: " & $a.slotToBlockProofs.len + echo "a.blockToCellProofs: " & $a.blockToCellProofs.len echo "a.sampleData: " & $a.sampleData.len diff --git a/tests/codex/proof/testindexing.nim b/tests/codex/proof/testindexing.nim new file mode 100644 index 00000000..725f6d64 --- /dev/null +++ b/tests/codex/proof/testindexing.nim @@ -0,0 +1,52 @@ +import pkg/chronos +import pkg/asynctest +import pkg/codex/proof/indexing +import pkg/codex/contracts/requests +import ../helpers + +let + bytesPerBlock = 64 * 1024 + numberOfSlotBlocks = 16 + blockSize = bytesPerBlock.uint64 + slot = Slot( + request: StorageRequest( + ask: StorageAsk( + slots: 10, + slotSize: u256(bytesPerBlock * numberOfSlotBlocks) + ), + content: StorageContent(), + ), + slotIndex: u256(3) + ) + +checksuite "Test indexing": + for (input, expected) in [(10, 0), (31, 0), (32, 1), (63, 1), (64, 2)]: + test "Can get slotBlockIndex from slotCellIndex (" & $input & " -> " & $expected & ")": + let + slotCellIndex = input.uint64 + + slotBlockIndex = getSlotBlockIndexForSlotCellIndex(slotCellIndex, blockSize) + + check: + slotBlockIndex == expected.uint64 + + for input in 0 ..< numberOfSlotBlocks: + test "Can get datasetBlockIndex from slotBlockIndex (" & $input & ")": + let + slotBlockIndex = input.uint64 + datasetBlockIndex = getDatasetBlockIndexForSlotBlockIndex(slot, blockSize, slotBlockIndex) + datasetSlotIndex = slot.slotIndex.truncate(uint64) + expectedIndex = (numberOfSlotBlocks.uint64 * datasetSlotIndex) + slotBlockIndex + + check: + datasetBlockIndex == expectedIndex + + for (input, expected) in [(10, 10), (31, 31), (32, 0), (63, 31), (64, 0)]: + test "Can get blockCellIndex from slotCellIndex (" & $input & " -> " & $expected & ")": + let + slotCellIndex = input.uint64 + + blockCellIndex = getBlockCellIndexForSlotCellIndex(slotCellIndex, blockSize) + + check: + blockCellIndex == expected.uint64 diff --git a/tests/codex/proof/testslotblocks.nim b/tests/codex/proof/testslotblocks.nim index d042224b..e9eb8c60 100644 --- a/tests/codex/proof/testslotblocks.nim +++ b/tests/codex/proof/testslotblocks.nim @@ -12,6 +12,7 @@ import pkg/codex/contracts import pkg/codex/merkletree import pkg/codex/proof/slotblocks +import pkg/codex/proof/indexing import ../helpers import ../examples @@ -135,22 +136,12 @@ asyncchecksuite "Test slotblocks - slot blocks by index": createSlot() discard await localStore.putBlock(manifestBlock) - for (input, expected) in [(0, 12), (1, 13), (10, 22)]: - test "Can get index for slot block (" & $input & " -> " & $expected & ")": - let - index = getIndexForSlotBlock(slot, bytesPerBlock.NBytes, input.uint64) - expectedIndex = (slotIndex * numberOfSlotBlocks + input).uint64 - - check: - index == expected.uint64 - index == expectedIndex - for input in [0, 1, numberOfSlotBlocks-1]: test "Can get slot block by index (" & $input & ")": let slotBlock = (await getSlotBlock(slot, localStore, input.uint64)).tryget() - expectedIndex = getIndexForSlotBlock(slot, bytesPerBlock.NBytes, input.uint64) - expectedBlock = datasetBlocks[expectedIndex] + expectedDatasetBlockIndex = getDatasetBlockIndexForSlotBlockIndex(slot, bytesPerBlock.uint64, input.uint64) + expectedBlock = datasetBlocks[expectedDatasetBlockIndex] check: slotBlock.cid == expectedBlock.cid diff --git a/tests/codex/testproof.nim b/tests/codex/testproof.nim index 3b562fea..a04992e8 100644 --- a/tests/codex/testproof.nim +++ b/tests/codex/testproof.nim @@ -1,4 +1,5 @@ import ./proof/testdatasampler import ./proof/testslotblocks +import ./proof/testindexing {.warning[UnusedImport]: off.}