From 26f238409c989e4fb689254e4d94ee07e19902c9 Mon Sep 17 00:00:00 2001 From: benbierens Date: Sun, 26 Nov 2023 17:05:11 +0100 Subject: [PATCH] almost there --- codex/proof/datasampler.nim | 17 +- codex/proof/indexing.nim | 8 +- tests/codex/proof/testdatasampler.nim | 162 ++++++++++++------ .../codex/proof/testdatasampler_expected.nim | 23 +++ tests/codex/proof/testslotblocks.nim | 7 +- 5 files changed, 148 insertions(+), 69 deletions(-) create mode 100644 tests/codex/proof/testdatasampler_expected.nim diff --git a/codex/proof/datasampler.nim b/codex/proof/datasampler.nim index 3134b1f6..d30aa7c6 100644 --- a/codex/proof/datasampler.nim +++ b/codex/proof/datasampler.nim @@ -20,6 +20,9 @@ import slotblocks import indexing import types +logScope: + topics = "codex datasampler" + func extractLowBits*[n: static int](A: BigInt[n], k: int): uint64 = assert(k > 0 and k <= 64) var r: uint64 = 0 @@ -114,10 +117,12 @@ proc getProofInput*( blockSize = manifest.blockSize.uint64 slotCellIndices = findSlotCellIndices(slot, slotRootHash, challenge, nSamples) + trace "Collecing input for proof", selectedSlotCellIndices = $slotCellIndices for slotCellIndex in slotCellIndices: let slotBlockIndex = getSlotBlockIndexForSlotCellIndex(slotCellIndex, blockSize) - datasetBlockIndex = getDatasetBlockIndexForSlotBlockIndex(slot, slotBlockIndex, blockSize) + datasetBlockIndex = getDatasetBlockIndexForSlotBlockIndex(slot, blockSize, slotBlockIndex) + blockCellIndex = getBlockCellIndexForSlotCellIndex(slotCellIndex, blockSize) without blk =? await getSlotBlock(slot, blockStore, manifest, slotBlockIndex), err: error "Failed to get slot block" @@ -127,20 +132,20 @@ proc getProofInput*( error "Failed to calculate minitree for block" return failure(err) - without blockProof =? slotPoseidonTree.getProof(datasetBlockIndex), err: - error "Failed to get dataset inclusion proof" + without blockProof =? slotPoseidonTree.getProof(slotBlockIndex), err: + error "Failed to get slot-to-block inclusion proof" return failure(err) slotToBlockProofs.add(blockProof) - without cellProof =? miniTree.getProof(slotCellIndex), err: - error "Failed to get cell inclusion proof" + without cellProof =? miniTree.getProof(blockCellIndex), err: + error "Failed to get block-to-cell inclusion proof" return failure(err) blockToCellProofs.add(cellProof) let cell = getCellFromBlock(blk, slotCellIndex, blockSize) sampleData = sampleData & cell - trace "Successfully collected proof input data" + trace "Successfully collected proof input" success(ProofInput( datasetToSlotProof: datasetToSlotProof, slotToBlockProofs: slotToBlockProofs, diff --git a/codex/proof/indexing.nim b/codex/proof/indexing.nim index 4925b03b..c673fd8a 100644 --- a/codex/proof/indexing.nim +++ b/codex/proof/indexing.nim @@ -8,13 +8,13 @@ import types # SlotBlockIndex => The index of a Block within a Slot. # DatasetBlockIndex => The index of a Block within a Dataset. -proc getSlotBlockIndexForSlotCellIndex*(cellIndex: DSSlotCellIndex, blockSize: uint64): uint64 = +proc getSlotBlockIndexForSlotCellIndex*(slotCellIndex: DSSlotCellIndex, blockSize: uint64): uint64 = let numberOfCellsPerBlock = blockSize div CellSize - return cellIndex div numberOfCellsPerBlock + return slotCellIndex div numberOfCellsPerBlock -proc getBlockCellIndexForSlotCellIndex*(cellIndex: DSSlotCellIndex, blockSize: uint64): uint64 = +proc getBlockCellIndexForSlotCellIndex*(slotCellIndex: DSSlotCellIndex, blockSize: uint64): uint64 = let numberOfCellsPerBlock = blockSize div CellSize - return cellIndex mod numberOfCellsPerBlock + return slotCellIndex mod numberOfCellsPerBlock proc getDatasetBlockIndexForSlotBlockIndex*(slot: Slot, blockSize: uint64, slotBlockIndex: uint64): uint64 = let diff --git a/tests/codex/proof/testdatasampler.nim b/tests/codex/proof/testdatasampler.nim index 2ccbe4bc..440de0ff 100644 --- a/tests/codex/proof/testdatasampler.nim +++ b/tests/codex/proof/testdatasampler.nim @@ -1,26 +1,17 @@ -import std/os -import std/strutils import std/sequtils import std/sugar import std/random -import pkg/questionable import pkg/questionable/results import pkg/constantine/math/arithmetic import pkg/poseidon2/types import pkg/poseidon2 import pkg/chronos import pkg/asynctest -import pkg/stew/byteutils -import pkg/stew/endians2 -import pkg/datastore -import pkg/codex/rng import pkg/codex/stores/cachestore import pkg/codex/chunker import pkg/codex/stores import pkg/codex/blocktype as bt -import pkg/codex/clock -import pkg/codex/utils/asynciter import pkg/codex/contracts/requests import pkg/codex/contracts import pkg/codex/merkletree @@ -29,55 +20,32 @@ import pkg/codex/stores/cachestore import pkg/codex/proof/datasampler import pkg/codex/proof/misc import pkg/codex/proof/types -#import pkg/codex/proof/indexing +import pkg/codex/proof/indexing import ../helpers import ../examples +import testdatasampler_expected let bytesPerBlock = 64 * 1024 - numberOfSlotBlocks = 16 challenge: DSFieldElement = toF(12345) slotRootHash: DSFieldElement = toF(6789) - slot = Slot( - request: StorageRequest( - client: Address.example, - ask: StorageAsk( - slots: 10, - slotSize: u256(bytesPerBlock * numberOfSlotBlocks), - duration: UInt256.example, - proofProbability: UInt256.example, - reward: UInt256.example, - collateral: UInt256.example, - maxSlotLoss: 123.uint64 + +asyncchecksuite "Test proof datasampler - components": + let + numberOfSlotBlocks = 16 + slot = Slot( + request: StorageRequest( + ask: StorageAsk( + slots: 10, + slotSize: u256(bytesPerBlock * numberOfSlotBlocks), + ), + content: StorageContent( + cid: $Cid.example + ) ), - content: StorageContent( - cid: "cidstringtodo", - erasure: StorageErasure(), - por: StoragePoR() - ), - expiry: UInt256.example, - nonce: Nonce.example - ), - slotIndex: u256(3) - ) - -asyncchecksuite "Test proof datasampler": - let chunker = RandomChunker.new(rng.Rng.instance(), - size = bytesPerBlock * numberOfSlotBlocks, - chunkSize = bytesPerBlock) - - var slotBlocks: seq[bt.Block] - - proc createSlotBlocks(): Future[void] {.async.} = - while true: - let chunk = await chunker.getBytes() - if chunk.len <= 0: - break - slotBlocks.add(bt.Block.new(chunk).tryGet()) - - setup: - await createSlotBlocks() + slotIndex: u256(3) + ) test "Number of cells is a power of two": # This is to check that the data used for testing is sane. @@ -199,15 +167,89 @@ asyncchecksuite "Test proof datasampler": cell1Proof.verifyDataBlock(cell1Bytes, miniTree.root).tryGet() cell2Proof.verifyDataBlock(cell2Bytes, miniTree.root).tryGet() +asyncchecksuite "Test proof datasampler - main": + let + numberOfSlotBlocks = 4 # 16 + totalNumberOfSlots = 2 #4 + datasetSlotIndex = 1 + localStore = CacheStore.new() + + var + manifest: Manifest + manifestBlock: bt.Block + slot: Slot + datasetBlocks: seq[bt.Block] + + proc createDatasetBlocks(): Future[void] {.async.} = + let numberOfCellsNeeded = (numberOfSlotBlocks * totalNumberOfSlots * bytesPerBlock).uint64 div CellSize + var data: seq[byte] = @[] + + # This generates a number of blocks that have different data, such that + # Each cell in each block is unique, but nothing is random. + for i in 0 ..< numberOfCellsNeeded: + data = data & (i.byte).repeat(CellSize) + + let chunker = MockChunker.new( + dataset = data, + chunkSize = bytesPerBlock) + + while true: + let chunk = await chunker.getBytes() + if chunk.len <= 0: + break + let b = bt.Block.new(chunk).tryGet() + datasetBlocks.add(b) + discard await localStore.putBlock(b) + + proc createManifest(): Future[void] {.async.} = + let + cids = datasetBlocks.mapIt(it.cid) + tree = MerkleTree.init(cids).tryGet() + treeCid = tree.rootCid().tryGet() + + for index, cid in cids: + let proof = tree.getProof(index).tryget() + discard await localStore.putBlockCidAndProof(treeCid, index, cid, proof) + + manifest = Manifest.new( + treeCid = treeCid, + blockSize = bytesPerBlock.NBytes, + datasetSize = (bytesPerBlock * numberOfSlotBlocks * totalNumberOfSlots).NBytes) + manifestBlock = bt.Block.new(manifest.encode().tryGet(), codec = DagPBCodec).tryGet() + + proc createSlot(): void = + slot = Slot( + request: StorageRequest( + ask: StorageAsk( + slotSize: u256(bytesPerBlock * numberOfSlotBlocks) + ), + content: StorageContent( + cid: $manifestBlock.cid + ), + ), + slotIndex: u256(datasetSlotIndex) + ) + + setup: + await createDatasetBlocks() + await createManifest() + createSlot() + discard await localStore.putBlock(manifestBlock) + test "Can gather proof input": # This is the main entry point for this module, and what it's all about. let - localStore = CacheStore.new() datasetToSlotProof = MerkleProof.example - slotPoseidonTree = MerkleTree.init(@[Cid.example]).tryget() + datasetBlockIndexFirst = getDatasetBlockIndexForSlotBlockIndex(slot, bytesPerBlock.uint64, 0.uint64) + datasetBlockIndexLast = datasetBlockIndexFirst + numberOfSlotBlocks.uint64 + slotBlocks = datasetBlocks[datasetBlockIndexFirst ..< datasetBlockIndexLast] + slotBlockCids = slotBlocks.mapIt(it.cid) + slotPoseidonTree = MerkleTree.init(slotBlockCids).tryGet() nSamples = 3 - a = (await getProofInput( + discard await localStore.putBlock(manifestBlock) + + let a = (await getProofInput( slot, localStore, slotRootHash, @@ -216,6 +258,16 @@ asyncchecksuite "Test proof datasampler": challenge, nSamples)).tryget() - echo "a.slotToBlockProofs: " & $a.slotToBlockProofs.len - echo "a.blockToCellProofs: " & $a.blockToCellProofs.len - echo "a.sampleData: " & $a.sampleData.len + proc toStr(proof: MerkleProof): string = + toHex(proof.nodesBuffer) + + let + expectedSlotToBlockProofs = getExpectedSlotToBlockProofs() + expectedBlockToCellProofs = getExpectedBlockToCellProofs() + expectedSampleData = getExpectedSampleData() + + check: + a.datasetToSlotProof == datasetToSlotProof + a.slotToBlockProofs.mapIt(toStr(it)) == expectedSlotToBlockProofs + a.blockToCellProofs.mapIt(toStr(it)) == expectedBlockToCellProofs + toHex(a.sampleData) == expectedSampleData diff --git a/tests/codex/proof/testdatasampler_expected.nim b/tests/codex/proof/testdatasampler_expected.nim new file mode 100644 index 00000000..71000600 --- /dev/null +++ b/tests/codex/proof/testdatasampler_expected.nim @@ -0,0 +1,23 @@ +# Snapshot of expected values for testdatasampler. + +import std/strutils +import pkg/codex/proof/types + +proc getExpectedSlotToBlockProofs*(): seq[string] = + @[ + "6ABAF89654125BBA9B99CA2635D605167D27D89641CE7E4ECF4F08E898C669B8DB784CEC822621E2DCB502F0135D48FEBACB228379FAC5ED295ED9F7A369107C", + "AD85DD1D1A218A253DAF27CC7874E63C162838C7F8D676F0689212D7C3A5E2EADB784CEC822621E2DCB502F0135D48FEBACB228379FAC5ED295ED9F7A369107C", + "BB8629AA6FF8B5818A2A3C260CB3F028E5E61DC9819DD415B168194736E23229B77FD63AEB6A517F5AE514C8A9CCBAFED5500E595572018ECC9C4205F06140E8" + ] + +proc getExpectedBlockToCellProofs*(): seq[string] = + @[ + "43AAA4A0B89548E8694C5C266BEBCC7BD0758F25005E606EDF559536D2227A7261CD0E8636D0CA8B899A070ABA5D9F815C6678877FAF71B984F5CF15300357E1C4C4A35D47EB0BECF6FB31E6C33CDB8A04D7EA0B29C2021678361F27CFBC96B355B36B13CC749463F14A7A0452A2F765E5951547B16CFB53D3824888226EE5D7FDB433239D7F55029C2C97C635A6B1214D1257C7D85C1C649758BA1AF8700E24", + "F5D40D9722D943D936AF8EFDE070C5E5D8BD0EC57A89F7BF0AA864DCD8BC77D9D8BC10DDDEB897A82318994242DCF2E07BC631B063DD6F9D4D13133FDED2127DC5840256978CA55F93AF0D6C730660813CF6A29C9B0C0153C8E01D152196A03E94AF1258C023CC638A060314259FA97B5CB743CBCE7AA483839BEA37DB71DF97C4F0DD83C0F38D791C75A1406EC9CFC212E7B302508CDD5780728B55CD5F132E", + "BB4E5BB9F2E0E104F04E2F49D8D39A9259C1419252A0B1150D9F0FE077A2A2258F594C0DC1A29099A3E3AAEB21C0BF00823383FC06F8A09C25462FC2DA11D51EAD968AA8200FBEA0A62FA1F83A46B3EA5D24859FF028351BEB15EA4D94F987D0CCD48366D7038EF9B0BD35FE287E111DB85A19F0E82E93CB9D40423776164D72332992C2849E7EF3AC7685A8DB37F3C44EFF9366BFE3F5FD80AF2B047C62E4A0" + ] + +proc getExpectedSampleData*(): string = + "B2".repeat(CellSize) & + "59".repeat(CellSize) & + "EE".repeat(CellSize) diff --git a/tests/codex/proof/testslotblocks.nim b/tests/codex/proof/testslotblocks.nim index e9eb8c60..289444bd 100644 --- a/tests/codex/proof/testslotblocks.nim +++ b/tests/codex/proof/testslotblocks.nim @@ -20,7 +20,7 @@ import ../examples let bytesPerBlock = 64 * 1024 numberOfSlotBlocks = 4 - slotIndex = 3 + datasetSlotIndex = 3 asyncchecksuite "Test slotblocks - manifest": let @@ -41,7 +41,7 @@ asyncchecksuite "Test slotblocks - manifest": cid: $manifestBlock.cid ), ), - slotIndex: u256(slotIndex) + slotIndex: u256(datasetSlotIndex) ) setup: @@ -116,7 +116,6 @@ asyncchecksuite "Test slotblocks - slot blocks by index": datasetSize = (bytesPerBlock * numberOfSlotBlocks * totalNumberOfSlots).NBytes) manifestBlock = bt.Block.new(manifest.encode().tryGet(), codec = DagPBCodec).tryGet() - proc createSlot(): void = slot = Slot( request: StorageRequest( @@ -127,7 +126,7 @@ asyncchecksuite "Test slotblocks - slot blocks by index": cid: $manifestBlock.cid ), ), - slotIndex: u256(slotIndex) + slotIndex: u256(datasetSlotIndex) ) setup: