splits up indexing

codex/proof/datasampler.nim

@@ -17,20 +17,8 @@ import pkg/poseidon2
 
 import misc
 import slotblocks
-
-const
-  # Size of a cell.
-  # A cell is a sample of storage-data selected for proving.
-  CellSize* = 2048.uint64
-
-type
-  DSFieldElement* = F
-  DSCellIndex* = uint64
-  DSCell* = seq[byte]
-  ProofInput* = ref object
-    blockInclProofs*: seq[MerkleProof]
-    cellInclProofs*: seq[MerkleProof]
-    sampleData*: seq[byte]
+import indexing
+import types
 
 func extractLowBits*[n: static int](A: BigInt[n], k: int): uint64 =
   assert(k > 0 and k <= 64)
@@ -45,7 +33,7 @@ func extractLowBits*[n: static int](A: BigInt[n], k: int): uint64 =
       r = bitor(r, 1'u64 shl i)
   return r
 
-proc getCellIndex(fe: DSFieldElement, numberOfCells: int): uint64 =
+proc convertToSlotCellIndex(fe: DSFieldElement, numberOfCells: int): uint64 =
   let log2 = ceilingLog2(numberOfCells)
   assert((1 shl log2) == numberOfCells , "expected `numberOfCells` to be a power of two.")
 
@@ -54,48 +42,32 @@ proc getCellIndex(fe: DSFieldElement, numberOfCells: int): uint64 =
 proc getNumberOfCellsInSlot*(slot: Slot): uint64 =
   (slot.request.ask.slotSize.truncate(uint64) div CellSize)
 
-proc findCellIndex*(
+proc findSlotCellIndex*(
   slotRootHash: DSFieldElement,
   challenge: DSFieldElement,
   counter: DSFieldElement,
-  numberOfCells: uint64): DSCellIndex =
-  # Computes the cell index for a single sample.
+  numberOfCells: uint64): DSSlotCellIndex =
+  # Computes the slot-cell index for a single sample.
   let
     input = @[slotRootHash, challenge, counter]
     hash = Sponge.digest(input, rate = 2)
-    index = getCellIndex(hash, numberOfCells.int)
+    index = convertToSlotCellIndex(hash, numberOfCells.int)
 
   return index
 
-func findCellIndices*(
+func findSlotCellIndices*(
   slot: Slot,
   slotRootHash: DSFieldElement,
   challenge: DSFieldElement,
-  nSamples: int): seq[DSCellIndex] =
-  # Computes nSamples cell indices.
+  nSamples: int): seq[DSSlotCellIndex] =
+  # Computes nSamples slot-cell indices.
   let numberOfCells = getNumberOfCellsInSlot(slot)
-  return collect(newSeq, (for i in 1..nSamples: findCellIndex(slotRootHash, challenge, toF(i), numberOfCells)))
+  return collect(newSeq, (for i in 1..nSamples: findSlotCellIndex(slotRootHash, challenge, toF(i), numberOfCells)))
 
-proc getSlotBlockIndex*(cellIndex: DSCellIndex, blockSize: uint64): uint64 =
-  let numberOfCellsPerBlock = blockSize div CellSize
-  return cellIndex div numberOfCellsPerBlock
-
-proc getDatasetBlockIndex*(slot: Slot, slotBlockIndex: uint64, blockSize: uint64): uint64 =
+proc getCellFromBlock*(blk: bt.Block, slotCellIndex: DSSlotCellIndex, blockSize: uint64): DSCell =
   let
-    slotIndex = slot.slotIndex.truncate(uint64)
-    slotSize = slot.request.ask.slotSize.truncate(uint64)
-    blocksInSlot = slotSize div blockSize
-
-  return (blocksInSlot * slotIndex) + slotBlockIndex
-
-proc getCellIndexInBlock*(cellIndex: DSCellIndex, blockSize: uint64): uint64 =
-  let numberOfCellsPerBlock = blockSize div CellSize
-  return cellIndex mod numberOfCellsPerBlock
-
-proc getCellFromBlock*(blk: bt.Block, cellIndex: DSCellIndex, blockSize: uint64): DSCell =
-  let
-    inBlockCellIndex = getCellIndexInBlock(cellIndex, blockSize)
-    dataStart = (CellSize * inBlockCellIndex)
+    blockCellIndex = getBlockCellIndexForSlotCellIndex(slotCellIndex, blockSize)
+    dataStart = (CellSize * blockCellIndex)
     dataEnd = dataStart + CellSize
 
   return blk.data[dataStart ..< dataEnd]
@@ -124,13 +96,14 @@ proc getProofInput*(
   slot: Slot,
   blockStore: BlockStore,
   slotRootHash: DSFieldElement,
-  dataSetPoseidonTree: MerkleTree,
+  slotPoseidonTree: MerkleTree,
+  datasetToSlotProof: MerkleProof,
   challenge: DSFieldElement,
   nSamples: int
 ): Future[?!ProofInput] {.async.} =
   var
-    blockProofs: seq[MerkleProof]
-    cellProofs: seq[MerkleProof]
+    slotToBlockProofs: seq[MerkleProof]
+    blockToCellProofs: seq[MerkleProof]
     sampleData: seq[byte]
 
   without manifest =? await getManifestForSlot(slot, blockStore), err:
@@ -139,10 +112,13 @@ proc getProofInput*(
 
   let
     blockSize = manifest.blockSize.uint64
-    cellIndices = findCellIndices(slot, slotRootHash, challenge, nSamples)
+    slotCellIndices = findSlotCellIndices(slot, slotRootHash, challenge, nSamples)
+
+  for slotCellIndex in slotCellIndices:
+    let
+      slotBlockIndex = getSlotBlockIndexForSlotCellIndex(slotCellIndex, blockSize)
+      datasetBlockIndex = getDatasetBlockIndexForSlotBlockIndex(slot, slotBlockIndex, blockSize)
 
-  for cellIndex in cellIndices:
-    let slotBlockIndex = getSlotBlockIndex(cellIndex, blockSize)
     without blk =? await getSlotBlock(slot, blockStore, manifest, slotBlockIndex), err:
       error "Failed to get slot block"
       return failure(err)
@@ -151,22 +127,23 @@ proc getProofInput*(
       error "Failed to calculate minitree for block"
       return failure(err)
 
-    # without blockProof =? dataSetPoseidonTree.getProof(???block index in dataset!), err:
-    #   error "Failed to get dataset inclusion proof"
-    #   return failure(err)
-    # blockProofs.add(blockProof)
+    without blockProof =? slotPoseidonTree.getProof(datasetBlockIndex), err:
+      error "Failed to get dataset inclusion proof"
+      return failure(err)
+    slotToBlockProofs.add(blockProof)
 
-    without cellProof =? miniTree.getProof(cellIndex), err:
+    without cellProof =? miniTree.getProof(slotCellIndex), err:
       error "Failed to get cell inclusion proof"
       return failure(err)
-    cellProofs.add(cellProof)
+    blockToCellProofs.add(cellProof)
 
-    let cell = getCellFromBlock(blk, cellIndex, blockSize)
+    let cell = getCellFromBlock(blk, slotCellIndex, blockSize)
     sampleData = sampleData & cell
 
   trace "Successfully collected proof input data"
   success(ProofInput(
-    blockInclProofs: blockProofs,
-    cellInclProofs: cellProofs,
+    datasetToSlotProof: datasetToSlotProof,
+    slotToBlockProofs: slotToBlockProofs,
+    blockToCellProofs: blockToCellProofs,
     sampleData: sampleData
   ))

codex/proof/indexing.nim

@@ -0,0 +1,24 @@
+import pkg/chronicles
+import ../contracts/requests
+import types
+
+# Index naming convention:
+# "<ContainerType><ElementType>Index" => The index of an ElementType within a ContainerType.
+# Some examples:
+# SlotBlockIndex => The index of a Block within a Slot.
+# DatasetBlockIndex => The index of a Block within a Dataset.
+
+proc getSlotBlockIndexForSlotCellIndex*(cellIndex: DSSlotCellIndex, blockSize: uint64): uint64 =
+  let numberOfCellsPerBlock = blockSize div CellSize
+  return cellIndex div numberOfCellsPerBlock
+
+proc getBlockCellIndexForSlotCellIndex*(cellIndex: DSSlotCellIndex, blockSize: uint64): uint64 =
+  let numberOfCellsPerBlock = blockSize div CellSize
+  return cellIndex mod numberOfCellsPerBlock
+
+proc getDatasetBlockIndexForSlotBlockIndex*(slot: Slot, blockSize: uint64, slotBlockIndex: uint64): uint64 =
+  let
+    slotSize = slot.request.ask.slotSize.truncate(uint64)
+    blocksInSlot = slotSize div blockSize
+    datasetSlotIndex = slot.slotIndex.truncate(uint64)
+  return (datasetSlotIndex * blocksInSlot) + slotBlockIndex

codex/proof/slotblocks.nim

@@ -10,6 +10,7 @@ import pkg/questionable/results
 import ../contracts/requests
 import ../stores/blockstore
 import ../manifest
+import indexing
 
 proc getManifestForSlot*(slot: Slot, blockstore: BlockStore): Future[?!Manifest] {.async.} =
   without manifestBlockCid =? Cid.init(slot.request.content.cid).mapFailure, err:
@@ -26,21 +27,10 @@ proc getManifestForSlot*(slot: Slot, blockstore: BlockStore): Future[?!Manifest]
 
   return success(manifest)
 
-proc getIndexForSlotBlock*(slot: Slot, blockSize: NBytes, slotBlockIndex: uint64): uint64 =
-  let
-    slotSize = slot.request.ask.slotSize.truncate(uint64)
-    blocksInSlot = slotSize div blockSize.uint64
-    slotIndex = slot.slotIndex.truncate(uint64)
-
-    datasetIndex = (slotIndex * blocksInSlot) + slotBlockIndex
-
-  trace "Converted slotBlockIndex to datasetIndex", slotBlockIndex, datasetIndex
-  return datasetIndex
-
 proc getSlotBlock*(slot: Slot, blockstore: BlockStore, manifest: Manifest, slotBlockIndex: uint64): Future[?!Block] {.async.} =
   let
     blocksInManifest = (manifest.datasetSize div manifest.blockSize).uint64
-    datasetIndex = getIndexForSlotBlock(slot, manifest.blockSize, slotBlockIndex)
+    datasetIndex = getDatasetBlockIndexForSlotBlockIndex(slot, manifest.blockSize.uint64, slotBlockIndex)
 
   if datasetIndex >= blocksInManifest:
     return failure("Found slotBlockIndex that is out-of-range: " & $datasetIndex)

codex/proof/types.nim

@@ -0,0 +1,17 @@
+import pkg/poseidon2/types
+import ../merkletree
+
+const
+  # Size of a cell.
+  # A cell is a sample of storage-data selected for proving.
+  CellSize* = 2048.uint64
+
+type
+  DSFieldElement* = F
+  DSSlotCellIndex* = uint64
+  DSCell* = seq[byte]
+  ProofInput* = ref object
+    datasetToSlotProof*: MerkleProof
+    slotToBlockProofs*: seq[MerkleProof]
+    blockToCellProofs*: seq[MerkleProof]
+    sampleData*: seq[byte]

tests/codex/examples.nim

@@ -7,6 +7,7 @@ import pkg/codex/rng
 import pkg/codex/stores
 import pkg/codex/blocktype as bt
 import pkg/codex/sales
+import pkg/codex/merkletree
 import ../examples
 
 export examples
@@ -71,3 +72,6 @@ proc example*(_: type Reservation): Reservation =
     size = uint16.example.u256,
     slotId = SlotId.example
   )
+
+proc example*(_: type MerkleProof): MerkleProof =
+  MerkleProof.init(3, @[MultiHash.example]).tryget()

tests/codex/proof/testdatasampler.nim

@@ -28,6 +28,8 @@ import pkg/codex/stores/cachestore
 
 import pkg/codex/proof/datasampler
 import pkg/codex/proof/misc
+import pkg/codex/proof/types
+#import pkg/codex/proof/indexing
 
 import ../helpers
 import ../examples
@@ -116,69 +118,36 @@ asyncchecksuite "Test proof datasampler":
 
   let knownIndices = @[178.uint64, 277.uint64, 366.uint64]
 
-  test "Can find single cell index":
+  test "Can find single slot-cell index":
     let numberOfCells = getNumberOfCellsInSlot(slot)
 
-    proc cellIndex(i: int): DSCellIndex =
+    proc slotCellIndex(i: int): DSSlotCellIndex =
       let counter: DSFieldElement = toF(i)
-      return findCellIndex(slotRootHash, challenge, counter, numberOfCells)
+      return findSlotCellIndex(slotRootHash, challenge, counter, numberOfCells)
 
-    proc getExpectedIndex(i: int): DSCellIndex =
+    proc getExpectedIndex(i: int): DSSlotCellIndex =
       let hash = Sponge.digest(@[slotRootHash, challenge, toF(i)], rate = 2)
       return extractLowBits(hash.toBig(), ceilingLog2(numberOfCells.int))
 
     check:
-      cellIndex(1) == getExpectedIndex(1)
-      cellIndex(1) == knownIndices[0]
-      cellIndex(2) == getExpectedIndex(2)
-      cellIndex(2) == knownIndices[1]
-      cellIndex(3) == getExpectedIndex(3)
-      cellIndex(3) == knownIndices[2]
+      slotCellIndex(1) == getExpectedIndex(1)
+      slotCellIndex(1) == knownIndices[0]
+      slotCellIndex(2) == getExpectedIndex(2)
+      slotCellIndex(2) == knownIndices[1]
+      slotCellIndex(3) == getExpectedIndex(3)
+      slotCellIndex(3) == knownIndices[2]
 
-  test "Can find sequence of cell indices":
-    proc cellIndices(n: int): seq[DSCellIndex]  =
-      findCellIndices(slot, slotRootHash, challenge, n)
+  test "Can find sequence of slot-cell indices":
+    proc slotCellIndices(n: int): seq[DSSlotCellIndex]  =
+      findSlotCellIndices(slot, slotRootHash, challenge, n)
 
     let numberOfCells = getNumberOfCellsInSlot(slot)
-    proc getExpectedIndices(n: int): seq[DSCellIndex]  =
-      return collect(newSeq, (for i in 1..n: findCellIndex(slotRootHash, challenge, toF(i), numberOfCells)))
+    proc getExpectedIndices(n: int): seq[DSSlotCellIndex]  =
+      return collect(newSeq, (for i in 1..n: findSlotCellIndex(slotRootHash, challenge, toF(i), numberOfCells)))
 
     check:
-      cellIndices(3) == getExpectedIndices(3)
-      cellIndices(3) == knownIndices
-
-  for (input, expected) in [(10, 0), (31, 0), (32, 1), (63, 1), (64, 2)]:
-    test "Can get slotBlockIndex from cell index (" & $input & " -> " & $expected & ")":
-      let
-        cellIndex = input.uint64
-        blockSize = (64 * 1024).uint64
-
-        slotBlockIndex = getSlotBlockIndex(cellIndex, blockSize)
-
-      check:
-        slotBlockIndex == expected.uint64
-
-  for input in 0 ..< numberOfSlotBlocks:
-    test "Can get datasetBlockIndex from slotBlockIndex (" & $input & ")":
-      let
-        slotBlockIndex = input.uint64
-        datasetBlockIndex = getDatasetBlockIndex(slot, slotBlockIndex, bytesPerBlock.uint64)
-        slotIndex = slot.slotIndex.truncate(uint64)
-        expectedIndex = (numberOfSlotBlocks.uint64 * slotIndex) + slotBlockIndex
-
-      check:
-        datasetBlockIndex == expectedIndex
-
-  for (input, expected) in [(10, 10), (31, 31), (32, 0), (63, 31), (64, 0)]:
-    test "Can get cellIndexInBlock from cell index (" & $input & " -> " & $expected & ")":
-      let
-        cellIndex = input.uint64
-        blockSize = (64 * 1024).uint64
-
-        cellIndexInBlock = getCellIndexInBlock(cellIndex, blockSize)
-
-      check:
-        cellIndexInBlock == expected.uint64
+      slotCellIndices(3) == getExpectedIndices(3)
+      slotCellIndices(3) == knownIndices
 
   test "Can get cell from block":
     let
@@ -234,9 +203,19 @@ asyncchecksuite "Test proof datasampler":
     # This is the main entry point for this module, and what it's all about.
     let
       localStore = CacheStore.new()
-      dataSetPoseidonTree = MerkleTree.init(@[Cid.example]).tryget()
-      a = (await getProofInput(slot, localStore, slotRootHash, dataSetPoseidonTree, challenge, 3)).tryget()
+      datasetToSlotProof = MerkleProof.example
+      slotPoseidonTree = MerkleTree.init(@[Cid.example]).tryget()
+      nSamples = 3
 
-    echo "a.blockInclProofs: " & $a.blockInclProofs.len
-    echo "a.cellInclProofs: " & $a.cellInclProofs.len
+      a = (await getProofInput(
+        slot,
+        localStore,
+        slotRootHash,
+        slotPoseidonTree,
+        datasetToSlotProof,
+        challenge,
+        nSamples)).tryget()
+
+    echo "a.slotToBlockProofs: " & $a.slotToBlockProofs.len
+    echo "a.blockToCellProofs: " & $a.blockToCellProofs.len
     echo "a.sampleData: " & $a.sampleData.len

tests/codex/proof/testindexing.nim

@@ -0,0 +1,52 @@
+import pkg/chronos
+import pkg/asynctest
+import pkg/codex/proof/indexing
+import pkg/codex/contracts/requests
+import ../helpers
+
+let
+  bytesPerBlock = 64 * 1024
+  numberOfSlotBlocks = 16
+  blockSize = bytesPerBlock.uint64
+  slot = Slot(
+    request: StorageRequest(
+      ask: StorageAsk(
+        slots: 10,
+        slotSize: u256(bytesPerBlock * numberOfSlotBlocks)
+      ),
+      content: StorageContent(),
+    ),
+    slotIndex: u256(3)
+  )
+
+checksuite "Test indexing":
+  for (input, expected) in [(10, 0), (31, 0), (32, 1), (63, 1), (64, 2)]:
+    test "Can get slotBlockIndex from slotCellIndex (" & $input & " -> " & $expected & ")":
+      let
+        slotCellIndex = input.uint64
+
+        slotBlockIndex = getSlotBlockIndexForSlotCellIndex(slotCellIndex, blockSize)
+
+      check:
+        slotBlockIndex == expected.uint64
+
+  for input in 0 ..< numberOfSlotBlocks:
+    test "Can get datasetBlockIndex from slotBlockIndex (" & $input & ")":
+      let
+        slotBlockIndex = input.uint64
+        datasetBlockIndex = getDatasetBlockIndexForSlotBlockIndex(slot, blockSize, slotBlockIndex)
+        datasetSlotIndex = slot.slotIndex.truncate(uint64)
+        expectedIndex = (numberOfSlotBlocks.uint64 * datasetSlotIndex) + slotBlockIndex
+
+      check:
+        datasetBlockIndex == expectedIndex
+
+  for (input, expected) in [(10, 10), (31, 31), (32, 0), (63, 31), (64, 0)]:
+    test "Can get blockCellIndex from slotCellIndex (" & $input & " -> " & $expected & ")":
+      let
+        slotCellIndex = input.uint64
+
+        blockCellIndex = getBlockCellIndexForSlotCellIndex(slotCellIndex, blockSize)
+
+      check:
+        blockCellIndex == expected.uint64

tests/codex/proof/testslotblocks.nim

@@ -12,6 +12,7 @@ import pkg/codex/contracts
 import pkg/codex/merkletree
 
 import pkg/codex/proof/slotblocks
+import pkg/codex/proof/indexing
 
 import ../helpers
 import ../examples
@@ -135,22 +136,12 @@ asyncchecksuite "Test slotblocks - slot blocks by index":
     createSlot()
     discard await localStore.putBlock(manifestBlock)
 
-  for (input, expected) in [(0, 12), (1, 13), (10, 22)]:
-    test "Can get index for slot block (" & $input & " -> " & $expected & ")":
-      let
-        index = getIndexForSlotBlock(slot, bytesPerBlock.NBytes, input.uint64)
-        expectedIndex = (slotIndex * numberOfSlotBlocks + input).uint64
-
-      check:
-        index == expected.uint64
-        index == expectedIndex
-
   for input in [0, 1, numberOfSlotBlocks-1]:
     test "Can get slot block by index (" & $input & ")":
       let
         slotBlock = (await getSlotBlock(slot, localStore, input.uint64)).tryget()
-        expectedIndex = getIndexForSlotBlock(slot, bytesPerBlock.NBytes, input.uint64)
-        expectedBlock = datasetBlocks[expectedIndex]
+        expectedDatasetBlockIndex = getDatasetBlockIndexForSlotBlockIndex(slot, bytesPerBlock.uint64, input.uint64)
+        expectedBlock = datasetBlocks[expectedDatasetBlockIndex]
 
       check:
         slotBlock.cid == expectedBlock.cid

tests/codex/testproof.nim

@@ -1,4 +1,5 @@
 import ./proof/testdatasampler
 import ./proof/testslotblocks
+import ./proof/testindexing
 
 {.warning[UnusedImport]: off.}