From 0497114e44f7c2b3441bae2448fccd81d8e4f878 Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Sun, 18 Feb 2024 20:19:59 -0600 Subject: [PATCH] Use correct blocks count in builder (#709) --- codex/node.nim | 11 ++++-- codex/slots/builder/builder.nim | 40 ++++++++++++------- tests/codex/node/testcontracts.nim | 4 +- tests/codex/slots/testslotbuilder.nim | 57 +++++++++++++++------------ 4 files changed, 66 insertions(+), 46 deletions(-) diff --git a/codex/node.nim b/codex/node.nim index 90c46669..68896bb1 100644 --- a/codex/node.nim +++ b/codex/node.nim @@ -531,10 +531,13 @@ proc onStore( return success() without indexer =? manifest.protectedStrategy.init( - 0, manifest.blocksCount() - 1, manifest.numSlots).catch and - blksIter =? indexer.getIndicies(slotIdx).catch, err: - trace "Unable to create indexing strategy from protected manifest", err = err.msg - return failure(err) + 0, manifest.numSlotBlocks() - 1, manifest.numSlots).catch, err: + trace "Unable to create indexing strategy from protected manifest", err = err.msg + return failure(err) + + without blksIter =? indexer.getIndicies(slotIdx).catch, err: + trace "Unable to get indicies from strategy", err = err.msg + return failure(err) if err =? (await self.fetchBatched( manifest.treeCid, diff --git a/codex/slots/builder/builder.nim b/codex/slots/builder/builder.nim index d8e513ed..674fd28e 100644 --- a/codex/slots/builder/builder.nim +++ b/codex/slots/builder/builder.nim @@ -331,27 +331,37 @@ proc new*[T, H]( return failure("Block size must be divisable by cell size.") let - numBlockCells = (manifest.blockSize div cellSize).int # number of cells per block - numSlotCells = manifest.numSlotBlocks * numBlockCells # number of uncorrected slot cells - pow2SlotCells = nextPowerOfTwo(numSlotCells) # pow2 cells per slot - numPadSlotBlocks = pow2SlotCells div numBlockCells # pow2 blocks per slot - numPadBlocksTotal = numPadSlotBlocks * manifest.numSlots # total number of pad blocks + numSlotBlocks = manifest.numSlotBlocks + numBlockCells = (manifest.blockSize div cellSize).int # number of cells per block + numSlotCells = manifest.numSlotBlocks * numBlockCells # number of uncorrected slot cells + pow2SlotCells = nextPowerOfTwo(numSlotCells) # pow2 cells per slot + numPadSlotBlocks = (pow2SlotCells div numBlockCells) - numSlotBlocks # pow2 blocks per slot - emptyBlock = newSeq[byte](manifest.blockSize.int) - emptyDigestTree = ? T.digestTree(emptyBlock, cellSize.int) + numSlotBlocksTotal = # pad blocks per slot + if numPadSlotBlocks > 0: + numPadSlotBlocks + numSlotBlocks + else: + numSlotBlocks + + numBlocksTotal = numSlotBlocksTotal * manifest.numSlots # number of blocks per slot + + emptyBlock = newSeq[byte](manifest.blockSize.int) + emptyDigestTree = ? T.digestTree(emptyBlock, cellSize.int) strategy = ? strategy.init( 0, - numPadBlocksTotal - 1, + numBlocksTotal - 1, manifest.numSlots).catch logScope: - numBlockCells = numBlockCells - numSlotCells = numSlotCells - pow2SlotCells = pow2SlotCells - numPadSlotBlocks = numPadSlotBlocks - numPadBlocksTotal = numPadBlocksTotal - strategy = strategy.strategyType + numSlotBlocks = numSlotBlocks + numBlockCells = numBlockCells + numSlotCells = numSlotCells + pow2SlotCells = pow2SlotCells + numPadSlotBlocks = numPadSlotBlocks + numBlocksTotal = numBlocksTotal + numSlotBlocksTotal = numSlotBlocksTotal + strategy = strategy.strategyType trace "Creating slots builder" @@ -362,7 +372,7 @@ proc new*[T, H]( strategy: strategy, cellSize: cellSize, emptyBlock: emptyBlock, - numSlotBlocks: numPadSlotBlocks, + numSlotBlocks: numSlotBlocksTotal, emptyDigestTree: emptyDigestTree) if manifest.verifiable: diff --git a/tests/codex/node/testcontracts.nim b/tests/codex/node/testcontracts.nim index f7997560..c140ee6f 100644 --- a/tests/codex/node/testcontracts.nim +++ b/tests/codex/node/testcontracts.nim @@ -128,10 +128,10 @@ asyncchecksuite "Test Node - Host contracts": return success() (await onStore(request, 1.u256, onBlocks)).tryGet() - check fetchedBytes == 851968 + check fetchedBytes == 262144 let indexer = verifiable.protectedStrategy.init( - 0, verifiable.blocksCount - 1, verifiable.numSlots) + 0, verifiable.numSlotBlocks() - 1, verifiable.numSlots) for index in indexer.getIndicies(1): let diff --git a/tests/codex/slots/testslotbuilder.nim b/tests/codex/slots/testslotbuilder.nim index 08bb7f24..4b38ec1a 100644 --- a/tests/codex/slots/testslotbuilder.nim +++ b/tests/codex/slots/testslotbuilder.nim @@ -43,18 +43,25 @@ suite "Slot builder": ecM = 2 numSlots = ecK + ecM - numDatasetBlocks = 50 + numDatasetBlocks = 8 numTotalBlocks = calcEcBlocksCount(numDatasetBlocks, ecK, ecM) # total number of blocks in the dataset after # EC (should will match number of slots) originalDatasetSize = numDatasetBlocks * blockSize.int totalDatasetSize = numTotalBlocks * blockSize.int - numBlockCells = (blockSize div cellSize).int # number of cells per block - numSlotBlocks = numTotalBlocks div numSlots # number of blocks per slot - numSlotCells = numSlotBlocks * numBlockCells # number of uncorrected slot cells - pow2SlotCells = nextPowerOfTwo(numSlotCells) # pow2 cells per slot - numPadSlotBlocks = pow2SlotCells div numBlockCells # pow2 blocks per slot - numPadBlocksTotal = numPadSlotBlocks * numSlots # total number of pad blocks + numSlotBlocks = numTotalBlocks div numSlots + numBlockCells = (blockSize div cellSize).int # number of cells per block + numSlotCells = numSlotBlocks * numBlockCells # number of uncorrected slot cells + pow2SlotCells = nextPowerOfTwo(numSlotCells) # pow2 cells per slot + numPadSlotBlocks = (pow2SlotCells div numBlockCells) - numSlotBlocks # pow2 blocks per slot + + numSlotBlocksTotal = # pad blocks per slot + if numPadSlotBlocks > 0: + numPadSlotBlocks + numSlotBlocks + else: + numSlotBlocks + + numBlocksTotal = numSlotBlocksTotal * numSlots # empty digest emptyDigest = SpongeMerkle.digest(newSeq[byte](blockSize.int), cellSize.int) @@ -157,39 +164,39 @@ suite "Slot builder": builder.cellSize == cellSize builder.numSlots == numSlots builder.numBlockCells == numBlockCells - builder.numSlotBlocks == numPadSlotBlocks + builder.numSlotBlocks == numSlotBlocksTotal builder.numSlotCells == pow2SlotCells - builder.numBlocks == numPadBlocksTotal + builder.numBlocks == numBlocksTotal test "Should build slot hashes for all slots": let steppedStrategy = Strategy.init( - 0, numPadBlocksTotal - 1, numSlots) + 0, numBlocksTotal - 1, numSlots) builder = Poseidon2Builder.new( localStore, protectedManifest, cellSize = cellSize).tryGet() - # for i in 0.. (protectedManifest.numSlotBlocks - 1): - emptyDigest - else: - SpongeMerkle.digest(datasetBlocks[idx].data, cellSize.int) + for i in 0.. (protectedManifest.numSlotBlocks - 1): + emptyDigest + else: + SpongeMerkle.digest(datasetBlocks[idx].data, cellSize.int) - cellHashes = (await builder.getCellHashes(0)).tryGet() + cellHashes = (await builder.getCellHashes(i)).tryGet() - check: - cellHashes.len == expectedHashes.len - cellHashes == expectedHashes + check: + cellHashes.len == expectedHashes.len + cellHashes == expectedHashes test "Should build slot trees for all slots": let steppedStrategy = Strategy.init( - 0, numPadBlocksTotal - 1, numSlots) + 0, numBlocksTotal - 1, numSlots) builder = Poseidon2Builder.new( localStore, @@ -237,7 +244,7 @@ suite "Slot builder": test "Should build correct verification root": let - steppedStrategy = Strategy.init(0, numPadBlocksTotal - 1, numSlots) + steppedStrategy = Strategy.init(0, numBlocksTotal - 1, numSlots) builder = Poseidon2Builder.new( localStore, protectedManifest, @@ -265,7 +272,7 @@ suite "Slot builder": test "Should build correct verification root manifest": let - steppedStrategy = Strategy.init(0, numPadBlocksTotal - 1, numSlots) + steppedStrategy = Strategy.init(0, numBlocksTotal - 1, numSlots) builder = Poseidon2Builder.new( localStore, protectedManifest,