almost there

codex/proof/datasampler.nim

@@ -20,6 +20,9 @@ import slotblocks
 import indexing
 import types
 
+logScope:
+  topics = "codex datasampler"
+
 func extractLowBits*[n: static int](A: BigInt[n], k: int): uint64 =
   assert(k > 0 and k <= 64)
   var r: uint64 = 0
@@ -114,10 +117,12 @@ proc getProofInput*(
     blockSize = manifest.blockSize.uint64
     slotCellIndices = findSlotCellIndices(slot, slotRootHash, challenge, nSamples)
 
+  trace "Collecing input for proof", selectedSlotCellIndices = $slotCellIndices
   for slotCellIndex in slotCellIndices:
     let
       slotBlockIndex = getSlotBlockIndexForSlotCellIndex(slotCellIndex, blockSize)
-      datasetBlockIndex = getDatasetBlockIndexForSlotBlockIndex(slot, slotBlockIndex, blockSize)
+      datasetBlockIndex = getDatasetBlockIndexForSlotBlockIndex(slot, blockSize, slotBlockIndex)
+      blockCellIndex = getBlockCellIndexForSlotCellIndex(slotCellIndex, blockSize)
 
     without blk =? await getSlotBlock(slot, blockStore, manifest, slotBlockIndex), err:
       error "Failed to get slot block"
@@ -127,20 +132,20 @@ proc getProofInput*(
       error "Failed to calculate minitree for block"
       return failure(err)
 
-    without blockProof =? slotPoseidonTree.getProof(datasetBlockIndex), err:
-      error "Failed to get dataset inclusion proof"
+    without blockProof =? slotPoseidonTree.getProof(slotBlockIndex), err:
+      error "Failed to get slot-to-block inclusion proof"
       return failure(err)
     slotToBlockProofs.add(blockProof)
 
-    without cellProof =? miniTree.getProof(slotCellIndex), err:
-      error "Failed to get cell inclusion proof"
+    without cellProof =? miniTree.getProof(blockCellIndex), err:
+      error "Failed to get block-to-cell inclusion proof"
       return failure(err)
     blockToCellProofs.add(cellProof)
 
     let cell = getCellFromBlock(blk, slotCellIndex, blockSize)
     sampleData = sampleData & cell
 
-  trace "Successfully collected proof input data"
+  trace "Successfully collected proof input"
   success(ProofInput(
     datasetToSlotProof: datasetToSlotProof,
     slotToBlockProofs: slotToBlockProofs,

codex/proof/indexing.nim

@@ -8,13 +8,13 @@ import types
 # SlotBlockIndex => The index of a Block within a Slot.
 # DatasetBlockIndex => The index of a Block within a Dataset.
 
-proc getSlotBlockIndexForSlotCellIndex*(cellIndex: DSSlotCellIndex, blockSize: uint64): uint64 =
+proc getSlotBlockIndexForSlotCellIndex*(slotCellIndex: DSSlotCellIndex, blockSize: uint64): uint64 =
   let numberOfCellsPerBlock = blockSize div CellSize
-  return cellIndex div numberOfCellsPerBlock
+  return slotCellIndex div numberOfCellsPerBlock
 
-proc getBlockCellIndexForSlotCellIndex*(cellIndex: DSSlotCellIndex, blockSize: uint64): uint64 =
+proc getBlockCellIndexForSlotCellIndex*(slotCellIndex: DSSlotCellIndex, blockSize: uint64): uint64 =
   let numberOfCellsPerBlock = blockSize div CellSize
-  return cellIndex mod numberOfCellsPerBlock
+  return slotCellIndex mod numberOfCellsPerBlock
 
 proc getDatasetBlockIndexForSlotBlockIndex*(slot: Slot, blockSize: uint64, slotBlockIndex: uint64): uint64 =
   let

tests/codex/proof/testdatasampler.nim

@@ -1,26 +1,17 @@
-import std/os
-import std/strutils
 import std/sequtils
 import std/sugar
 import std/random
 
-import pkg/questionable
 import pkg/questionable/results
 import pkg/constantine/math/arithmetic
 import pkg/poseidon2/types
 import pkg/poseidon2
 import pkg/chronos
 import pkg/asynctest
-import pkg/stew/byteutils
-import pkg/stew/endians2
-import pkg/datastore
-import pkg/codex/rng
 import pkg/codex/stores/cachestore
 import pkg/codex/chunker
 import pkg/codex/stores
 import pkg/codex/blocktype as bt
-import pkg/codex/clock
-import pkg/codex/utils/asynciter
 import pkg/codex/contracts/requests
 import pkg/codex/contracts
 import pkg/codex/merkletree
@@ -29,55 +20,32 @@ import pkg/codex/stores/cachestore
 import pkg/codex/proof/datasampler
 import pkg/codex/proof/misc
 import pkg/codex/proof/types
-#import pkg/codex/proof/indexing
+import pkg/codex/proof/indexing
 
 import ../helpers
 import ../examples
+import testdatasampler_expected
 
 let
   bytesPerBlock = 64 * 1024
-  numberOfSlotBlocks = 16
   challenge: DSFieldElement = toF(12345)
   slotRootHash: DSFieldElement = toF(6789)
-  slot = Slot(
-    request: StorageRequest(
-      client: Address.example,
-      ask: StorageAsk(
-        slots: 10,
-        slotSize: u256(bytesPerBlock * numberOfSlotBlocks),
-        duration: UInt256.example,
-        proofProbability: UInt256.example,
-        reward: UInt256.example,
-        collateral: UInt256.example,
-        maxSlotLoss: 123.uint64
+
+asyncchecksuite "Test proof datasampler - components":
+  let
+    numberOfSlotBlocks = 16
+    slot = Slot(
+      request: StorageRequest(
+        ask: StorageAsk(
+          slots: 10,
+          slotSize: u256(bytesPerBlock * numberOfSlotBlocks),
+        ),
+        content: StorageContent(
+          cid: $Cid.example
+        )
       ),
-      content: StorageContent(
-        cid: "cidstringtodo",
-        erasure: StorageErasure(),
-        por: StoragePoR()
-      ),
-      expiry: UInt256.example,
-      nonce: Nonce.example
-    ),
-    slotIndex: u256(3)
-  )
-
-asyncchecksuite "Test proof datasampler":
-  let chunker = RandomChunker.new(rng.Rng.instance(),
-    size = bytesPerBlock * numberOfSlotBlocks,
-    chunkSize = bytesPerBlock)
-
-  var slotBlocks: seq[bt.Block]
-
-  proc createSlotBlocks(): Future[void] {.async.} =
-    while true:
-      let chunk = await chunker.getBytes()
-      if chunk.len <= 0:
-        break
-      slotBlocks.add(bt.Block.new(chunk).tryGet())
-
-  setup:
-    await createSlotBlocks()
+      slotIndex: u256(3)
+    )
 
   test "Number of cells is a power of two":
     # This is to check that the data used for testing is sane.
@@ -199,15 +167,89 @@ asyncchecksuite "Test proof datasampler":
       cell1Proof.verifyDataBlock(cell1Bytes, miniTree.root).tryGet()
       cell2Proof.verifyDataBlock(cell2Bytes, miniTree.root).tryGet()
 
+asyncchecksuite "Test proof datasampler - main":
+  let
+    numberOfSlotBlocks = 4 # 16
+    totalNumberOfSlots = 2 #4
+    datasetSlotIndex = 1
+    localStore = CacheStore.new()
+
+  var
+    manifest: Manifest
+    manifestBlock: bt.Block
+    slot: Slot
+    datasetBlocks: seq[bt.Block]
+
+  proc createDatasetBlocks(): Future[void] {.async.} =
+    let numberOfCellsNeeded = (numberOfSlotBlocks * totalNumberOfSlots * bytesPerBlock).uint64 div CellSize
+    var data: seq[byte] = @[]
+
+    # This generates a number of blocks that have different data, such that
+    # Each cell in each block is unique, but nothing is random.
+    for i in 0 ..< numberOfCellsNeeded:
+      data = data & (i.byte).repeat(CellSize)
+
+    let chunker = MockChunker.new(
+      dataset = data,
+      chunkSize = bytesPerBlock)
+
+    while true:
+      let chunk = await chunker.getBytes()
+      if chunk.len <= 0:
+        break
+      let b = bt.Block.new(chunk).tryGet()
+      datasetBlocks.add(b)
+      discard await localStore.putBlock(b)
+
+  proc createManifest(): Future[void] {.async.} =
+    let
+      cids = datasetBlocks.mapIt(it.cid)
+      tree = MerkleTree.init(cids).tryGet()
+      treeCid = tree.rootCid().tryGet()
+
+    for index, cid in cids:
+      let proof = tree.getProof(index).tryget()
+      discard await localStore.putBlockCidAndProof(treeCid, index, cid, proof)
+
+    manifest = Manifest.new(
+      treeCid = treeCid,
+      blockSize = bytesPerBlock.NBytes,
+      datasetSize = (bytesPerBlock * numberOfSlotBlocks * totalNumberOfSlots).NBytes)
+    manifestBlock = bt.Block.new(manifest.encode().tryGet(), codec = DagPBCodec).tryGet()
+
+  proc createSlot(): void =
+    slot = Slot(
+      request: StorageRequest(
+        ask: StorageAsk(
+          slotSize: u256(bytesPerBlock * numberOfSlotBlocks)
+        ),
+        content: StorageContent(
+          cid: $manifestBlock.cid
+        ),
+      ),
+      slotIndex: u256(datasetSlotIndex)
+    )
+
+  setup:
+    await createDatasetBlocks()
+    await createManifest()
+    createSlot()
+    discard await localStore.putBlock(manifestBlock)
+
   test "Can gather proof input":
     # This is the main entry point for this module, and what it's all about.
     let
-      localStore = CacheStore.new()
       datasetToSlotProof = MerkleProof.example
-      slotPoseidonTree = MerkleTree.init(@[Cid.example]).tryget()
+      datasetBlockIndexFirst = getDatasetBlockIndexForSlotBlockIndex(slot, bytesPerBlock.uint64, 0.uint64)
+      datasetBlockIndexLast = datasetBlockIndexFirst + numberOfSlotBlocks.uint64
+      slotBlocks = datasetBlocks[datasetBlockIndexFirst ..< datasetBlockIndexLast]
+      slotBlockCids = slotBlocks.mapIt(it.cid)
+      slotPoseidonTree = MerkleTree.init(slotBlockCids).tryGet()
       nSamples = 3
 
-      a = (await getProofInput(
+    discard await localStore.putBlock(manifestBlock)
+
+    let a = (await getProofInput(
         slot,
         localStore,
         slotRootHash,
@@ -216,6 +258,16 @@ asyncchecksuite "Test proof datasampler":
         challenge,
         nSamples)).tryget()
 
-    echo "a.slotToBlockProofs: " & $a.slotToBlockProofs.len
-    echo "a.blockToCellProofs: " & $a.blockToCellProofs.len
-    echo "a.sampleData: " & $a.sampleData.len
+    proc toStr(proof: MerkleProof): string =
+      toHex(proof.nodesBuffer)
+
+    let
+      expectedSlotToBlockProofs = getExpectedSlotToBlockProofs()
+      expectedBlockToCellProofs = getExpectedBlockToCellProofs()
+      expectedSampleData = getExpectedSampleData()
+
+    check:
+      a.datasetToSlotProof == datasetToSlotProof
+      a.slotToBlockProofs.mapIt(toStr(it)) == expectedSlotToBlockProofs
+      a.blockToCellProofs.mapIt(toStr(it)) == expectedBlockToCellProofs
+      toHex(a.sampleData) == expectedSampleData

tests/codex/proof/testdatasampler_expected.nim

@@ -0,0 +1,23 @@
+# Snapshot of expected values for testdatasampler.
+
+import std/strutils
+import pkg/codex/proof/types
+
+proc getExpectedSlotToBlockProofs*(): seq[string] =
+  @[
+    "6ABAF89654125BBA9B99CA2635D605167D27D89641CE7E4ECF4F08E898C669B8DB784CEC822621E2DCB502F0135D48FEBACB228379FAC5ED295ED9F7A369107C",
+    "AD85DD1D1A218A253DAF27CC7874E63C162838C7F8D676F0689212D7C3A5E2EADB784CEC822621E2DCB502F0135D48FEBACB228379FAC5ED295ED9F7A369107C",
+    "BB8629AA6FF8B5818A2A3C260CB3F028E5E61DC9819DD415B168194736E23229B77FD63AEB6A517F5AE514C8A9CCBAFED5500E595572018ECC9C4205F06140E8"
+  ]
+
+proc getExpectedBlockToCellProofs*(): seq[string] =
+  @[
+    "43AAA4A0B89548E8694C5C266BEBCC7BD0758F25005E606EDF559536D2227A7261CD0E8636D0CA8B899A070ABA5D9F815C6678877FAF71B984F5CF15300357E1C4C4A35D47EB0BECF6FB31E6C33CDB8A04D7EA0B29C2021678361F27CFBC96B355B36B13CC749463F14A7A0452A2F765E5951547B16CFB53D3824888226EE5D7FDB433239D7F55029C2C97C635A6B1214D1257C7D85C1C649758BA1AF8700E24",
+    "F5D40D9722D943D936AF8EFDE070C5E5D8BD0EC57A89F7BF0AA864DCD8BC77D9D8BC10DDDEB897A82318994242DCF2E07BC631B063DD6F9D4D13133FDED2127DC5840256978CA55F93AF0D6C730660813CF6A29C9B0C0153C8E01D152196A03E94AF1258C023CC638A060314259FA97B5CB743CBCE7AA483839BEA37DB71DF97C4F0DD83C0F38D791C75A1406EC9CFC212E7B302508CDD5780728B55CD5F132E",
+    "BB4E5BB9F2E0E104F04E2F49D8D39A9259C1419252A0B1150D9F0FE077A2A2258F594C0DC1A29099A3E3AAEB21C0BF00823383FC06F8A09C25462FC2DA11D51EAD968AA8200FBEA0A62FA1F83A46B3EA5D24859FF028351BEB15EA4D94F987D0CCD48366D7038EF9B0BD35FE287E111DB85A19F0E82E93CB9D40423776164D72332992C2849E7EF3AC7685A8DB37F3C44EFF9366BFE3F5FD80AF2B047C62E4A0"
+  ]
+
+proc getExpectedSampleData*(): string =
+  "B2".repeat(CellSize) &
+  "59".repeat(CellSize) &
+  "EE".repeat(CellSize)

tests/codex/proof/testslotblocks.nim

@@ -20,7 +20,7 @@ import ../examples
 let
   bytesPerBlock = 64 * 1024
   numberOfSlotBlocks = 4
-  slotIndex = 3
+  datasetSlotIndex = 3
 
 asyncchecksuite "Test slotblocks - manifest":
   let
@@ -41,7 +41,7 @@ asyncchecksuite "Test slotblocks - manifest":
           cid: $manifestBlock.cid
         ),
       ),
-      slotIndex: u256(slotIndex)
+      slotIndex: u256(datasetSlotIndex)
     )
 
   setup:
@@ -116,7 +116,6 @@ asyncchecksuite "Test slotblocks - slot blocks by index":
       datasetSize = (bytesPerBlock * numberOfSlotBlocks * totalNumberOfSlots).NBytes)
     manifestBlock = bt.Block.new(manifest.encode().tryGet(), codec = DagPBCodec).tryGet()
 
-
   proc createSlot(): void =
     slot = Slot(
       request: StorageRequest(
@@ -127,7 +126,7 @@ asyncchecksuite "Test slotblocks - slot blocks by index":
           cid: $manifestBlock.cid
         ),
       ),
-      slotIndex: u256(slotIndex)
+      slotIndex: u256(datasetSlotIndex)
     )
 
   setup: