status-im
/
nim-codex
mirror of https://github.com/status-im/nim-codex.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Use correct blocks count in builder (#709)

This commit is contained in:
Dmitriy Ryajov 2024-02-18 20:19:59 -06:00 committed by GitHub
parent 3fce267352
commit 0497114e44
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 66 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
codex/node.nim
View File

@ -531,10 +531,13 @@ proc onStore(
return success() return success()
without indexer =? manifest.protectedStrategy.init( without indexer =? manifest.protectedStrategy.init(
0, manifest.blocksCount() - 1, manifest.numSlots).catch and 0, manifest.numSlotBlocks() - 1, manifest.numSlots).catch, err:
blksIter =? indexer.getIndicies(slotIdx).catch, err: trace "Unable to create indexing strategy from protected manifest", err = err.msg
trace "Unable to create indexing strategy from protected manifest", err = err.msg return failure(err)
return failure(err)
without blksIter =? indexer.getIndicies(slotIdx).catch, err:
trace "Unable to get indicies from strategy", err = err.msg
return failure(err)
if err =? (await self.fetchBatched( if err =? (await self.fetchBatched(
manifest.treeCid, manifest.treeCid,

40
codex/slots/builder/builder.nim
View File

@ -331,27 +331,37 @@ proc new*[T, H](
return failure("Block size must be divisable by cell size.") return failure("Block size must be divisable by cell size.")
let let
numBlockCells = (manifest.blockSize div cellSize).int # number of cells per block numSlotBlocks = manifest.numSlotBlocks
numSlotCells = manifest.numSlotBlocks * numBlockCells # number of uncorrected slot cells numBlockCells = (manifest.blockSize div cellSize).int # number of cells per block
pow2SlotCells = nextPowerOfTwo(numSlotCells) # pow2 cells per slot numSlotCells = manifest.numSlotBlocks * numBlockCells # number of uncorrected slot cells
numPadSlotBlocks = pow2SlotCells div numBlockCells # pow2 blocks per slot pow2SlotCells = nextPowerOfTwo(numSlotCells) # pow2 cells per slot
numPadBlocksTotal = numPadSlotBlocks * manifest.numSlots # total number of pad blocks numPadSlotBlocks = (pow2SlotCells div numBlockCells) - numSlotBlocks # pow2 blocks per slot
emptyBlock = newSeq[byte](manifest.blockSize.int) numSlotBlocksTotal = # pad blocks per slot
emptyDigestTree = ? T.digestTree(emptyBlock, cellSize.int) if numPadSlotBlocks > 0:
numPadSlotBlocks + numSlotBlocks
else:
numSlotBlocks
numBlocksTotal = numSlotBlocksTotal * manifest.numSlots # number of blocks per slot
emptyBlock = newSeq[byte](manifest.blockSize.int)
emptyDigestTree = ? T.digestTree(emptyBlock, cellSize.int)
strategy = ? strategy.init( strategy = ? strategy.init(
0, 0,
numPadBlocksTotal - 1, numBlocksTotal - 1,
manifest.numSlots).catch manifest.numSlots).catch
logScope: logScope:
numBlockCells = numBlockCells numSlotBlocks = numSlotBlocks
numSlotCells = numSlotCells numBlockCells = numBlockCells
pow2SlotCells = pow2SlotCells numSlotCells = numSlotCells
numPadSlotBlocks = numPadSlotBlocks pow2SlotCells = pow2SlotCells
numPadBlocksTotal = numPadBlocksTotal numPadSlotBlocks = numPadSlotBlocks
strategy = strategy.strategyType numBlocksTotal = numBlocksTotal
numSlotBlocksTotal = numSlotBlocksTotal
strategy = strategy.strategyType
trace "Creating slots builder" trace "Creating slots builder"
@ -362,7 +372,7 @@ proc new*[T, H](
strategy: strategy, strategy: strategy,
cellSize: cellSize, cellSize: cellSize,
emptyBlock: emptyBlock, emptyBlock: emptyBlock,
numSlotBlocks: numPadSlotBlocks, numSlotBlocks: numSlotBlocksTotal,
emptyDigestTree: emptyDigestTree) emptyDigestTree: emptyDigestTree)
if manifest.verifiable: if manifest.verifiable:

4
tests/codex/node/testcontracts.nim
View File

@ -128,10 +128,10 @@ asyncchecksuite "Test Node - Host contracts":
return success() return success()
(await onStore(request, 1.u256, onBlocks)).tryGet() (await onStore(request, 1.u256, onBlocks)).tryGet()
check fetchedBytes == 851968 check fetchedBytes == 262144
let indexer = verifiable.protectedStrategy.init( let indexer = verifiable.protectedStrategy.init(
0, verifiable.blocksCount - 1, verifiable.numSlots) 0, verifiable.numSlotBlocks() - 1, verifiable.numSlots)
for index in indexer.getIndicies(1): for index in indexer.getIndicies(1):
let let

57
tests/codex/slots/testslotbuilder.nim
View File

@ -43,18 +43,25 @@ suite "Slot builder":
ecM = 2 ecM = 2
numSlots = ecK + ecM numSlots = ecK + ecM
numDatasetBlocks = 50 numDatasetBlocks = 8
numTotalBlocks = calcEcBlocksCount(numDatasetBlocks, ecK, ecM) # total number of blocks in the dataset after numTotalBlocks = calcEcBlocksCount(numDatasetBlocks, ecK, ecM) # total number of blocks in the dataset after
# EC (should will match number of slots) # EC (should will match number of slots)
originalDatasetSize = numDatasetBlocks * blockSize.int originalDatasetSize = numDatasetBlocks * blockSize.int
totalDatasetSize = numTotalBlocks * blockSize.int totalDatasetSize = numTotalBlocks * blockSize.int
numBlockCells = (blockSize div cellSize).int # number of cells per block numSlotBlocks = numTotalBlocks div numSlots
numSlotBlocks = numTotalBlocks div numSlots # number of blocks per slot numBlockCells = (blockSize div cellSize).int # number of cells per block
numSlotCells = numSlotBlocks * numBlockCells # number of uncorrected slot cells numSlotCells = numSlotBlocks * numBlockCells # number of uncorrected slot cells
pow2SlotCells = nextPowerOfTwo(numSlotCells) # pow2 cells per slot pow2SlotCells = nextPowerOfTwo(numSlotCells) # pow2 cells per slot
numPadSlotBlocks = pow2SlotCells div numBlockCells # pow2 blocks per slot numPadSlotBlocks = (pow2SlotCells div numBlockCells) - numSlotBlocks # pow2 blocks per slot
numPadBlocksTotal = numPadSlotBlocks * numSlots # total number of pad blocks
numSlotBlocksTotal = # pad blocks per slot
if numPadSlotBlocks > 0:
numPadSlotBlocks + numSlotBlocks
else:
numSlotBlocks
numBlocksTotal = numSlotBlocksTotal * numSlots
# empty digest # empty digest
emptyDigest = SpongeMerkle.digest(newSeq[byte](blockSize.int), cellSize.int) emptyDigest = SpongeMerkle.digest(newSeq[byte](blockSize.int), cellSize.int)
@ -157,39 +164,39 @@ suite "Slot builder":
builder.cellSize == cellSize builder.cellSize == cellSize
builder.numSlots == numSlots builder.numSlots == numSlots
builder.numBlockCells == numBlockCells builder.numBlockCells == numBlockCells
builder.numSlotBlocks == numPadSlotBlocks builder.numSlotBlocks == numSlotBlocksTotal
builder.numSlotCells == pow2SlotCells builder.numSlotCells == pow2SlotCells
builder.numBlocks == numPadBlocksTotal builder.numBlocks == numBlocksTotal
test "Should build slot hashes for all slots": test "Should build slot hashes for all slots":
let let
steppedStrategy = Strategy.init( steppedStrategy = Strategy.init(
0, numPadBlocksTotal - 1, numSlots) 0, numBlocksTotal - 1, numSlots)
builder = Poseidon2Builder.new( builder = Poseidon2Builder.new(
localStore, localStore,
protectedManifest, protectedManifest,
cellSize = cellSize).tryGet() cellSize = cellSize).tryGet()
# for i in 0..<numSlots: for i in 0..<numSlots:
let let
expectedHashes = collect(newSeq): expectedHashes = collect(newSeq):
for j, idx in steppedStrategy.getIndicies(0): for j, idx in steppedStrategy.getIndicies(i):
if j > (protectedManifest.numSlotBlocks - 1): if j > (protectedManifest.numSlotBlocks - 1):
emptyDigest emptyDigest
else: else:
SpongeMerkle.digest(datasetBlocks[idx].data, cellSize.int) SpongeMerkle.digest(datasetBlocks[idx].data, cellSize.int)
cellHashes = (await builder.getCellHashes(0)).tryGet() cellHashes = (await builder.getCellHashes(i)).tryGet()
check: check:
cellHashes.len == expectedHashes.len cellHashes.len == expectedHashes.len
cellHashes == expectedHashes cellHashes == expectedHashes
test "Should build slot trees for all slots": test "Should build slot trees for all slots":
let let
steppedStrategy = Strategy.init( steppedStrategy = Strategy.init(
0, numPadBlocksTotal - 1, numSlots) 0, numBlocksTotal - 1, numSlots)
builder = Poseidon2Builder.new( builder = Poseidon2Builder.new(
localStore, localStore,
@ -237,7 +244,7 @@ suite "Slot builder":
test "Should build correct verification root": test "Should build correct verification root":
let let
steppedStrategy = Strategy.init(0, numPadBlocksTotal - 1, numSlots) steppedStrategy = Strategy.init(0, numBlocksTotal - 1, numSlots)
builder = Poseidon2Builder.new( builder = Poseidon2Builder.new(
localStore, localStore,
protectedManifest, protectedManifest,
@ -265,7 +272,7 @@ suite "Slot builder":
test "Should build correct verification root manifest": test "Should build correct verification root manifest":
let let
steppedStrategy = Strategy.init(0, numPadBlocksTotal - 1, numSlots) steppedStrategy = Strategy.init(0, numBlocksTotal - 1, numSlots)
builder = Poseidon2Builder.new( builder = Poseidon2Builder.new(
localStore, localStore,
protectedManifest, protectedManifest,