diff --git a/hive_integration/nodocker/engine/node.nim b/hive_integration/nodocker/engine/node.nim index 30b22b29d..4c1c0dfe1 100644 --- a/hive_integration/nodocker/engine/node.nim +++ b/hive_integration/nodocker/engine/node.nim @@ -108,11 +108,11 @@ proc setBlock*(c: ChainRef; blk: EthBlock): Result[void, string] = return err("Could not persist header") try: - c.db.persistTransactions(header.number, blk.transactions) - c.db.persistReceipts(vmState.receipts) + c.db.persistTransactions(header.number, header.txRoot, blk.transactions) + c.db.persistReceipts(header.receiptsRoot, vmState.receipts) if blk.withdrawals.isSome: - c.db.persistWithdrawals(blk.withdrawals.get) + c.db.persistWithdrawals(header.withdrawalsRoot, blk.withdrawals.get) except CatchableError as exc: return err(exc.msg) diff --git a/nimbus/core/chain/forked_chain.nim b/nimbus/core/chain/forked_chain.nim index ee700fb63..b9c279dc6 100644 --- a/nimbus/core/chain/forked_chain.nim +++ b/nimbus/core/chain/forked_chain.nim @@ -169,16 +169,21 @@ proc replaySegment(c: ForkedChainRef, target: Hash256) = proc writeBaggage(c: ForkedChainRef, target: Hash256) = # Write baggage from base+1 to target block + template header(): BlockHeader = + blk.blk.header + shouldNotKeyError: var prevHash = target while prevHash != c.baseHash: let blk = c.blocks[prevHash] - c.db.persistTransactions(blk.blk.header.number, blk.blk.transactions) - c.db.persistReceipts(blk.receipts) + c.db.persistTransactions(header.number, header.txRoot, blk.blk.transactions) + c.db.persistReceipts(header.receiptsRoot, blk.receipts) discard c.db.persistUncles(blk.blk.uncles) if blk.blk.withdrawals.isSome: - c.db.persistWithdrawals(blk.blk.withdrawals.get) - prevHash = blk.blk.header.parentHash + c.db.persistWithdrawals( + header.withdrawalsRoot.expect("WithdrawalsRoot should be verified before"), + blk.blk.withdrawals.get) + prevHash = header.parentHash func updateBase(c: ForkedChainRef, newBaseHash: Hash256, diff --git a/nimbus/core/chain/persist_blocks.nim b/nimbus/core/chain/persist_blocks.nim index a2dba0e37..beb814f61 100644 --- a/nimbus/core/chain/persist_blocks.nim +++ b/nimbus/core/chain/persist_blocks.nim @@ -170,13 +170,15 @@ proc persistBlocksImpl( return err("Could not persist header") if NoPersistTransactions notin flags: - c.db.persistTransactions(header.number, blk.transactions) + c.db.persistTransactions(header.number, header.txRoot, blk.transactions) if NoPersistReceipts notin flags: - c.db.persistReceipts(vmState.receipts) + c.db.persistReceipts(header.receiptsRoot, vmState.receipts) if NoPersistWithdrawals notin flags and blk.withdrawals.isSome: - c.db.persistWithdrawals(blk.withdrawals.get) + c.db.persistWithdrawals( + header.withdrawalsRoot.expect("WithdrawalsRoot should be verified before"), + blk.withdrawals.get) # update currentBlock *after* we persist it # so the rpc return consistent result diff --git a/nimbus/db/core_db/base/base_desc.nim b/nimbus/db/core_db/base/base_desc.nim index 3f655a080..98f8470b1 100644 --- a/nimbus/db/core_db/base/base_desc.nim +++ b/nimbus/db/core_db/base/base_desc.nim @@ -61,9 +61,6 @@ type CoreDbColType* = enum CtGeneric = 2 # columns smaller than 2 are not provided - CtReceipts - CtTxs - CtWithdrawals CoreDbCaptFlags* {.pure.} = enum PersistPut diff --git a/nimbus/db/core_db/core_apps.nim b/nimbus/db/core_db/core_apps.nim index 08099b9ce..066ff570a 100644 --- a/nimbus/db/core_db/core_apps.nim +++ b/nimbus/db/core_db/core_apps.nim @@ -114,30 +114,23 @@ iterator findNewAncestors( iterator getBlockTransactionData*( db: CoreDbRef; - transactionRoot: Hash256; + txRoot: Hash256; ): Blob = const info = "getBlockTransactionData()" block body: - if transactionRoot == EMPTY_ROOT_HASH: + if txRoot == EMPTY_ROOT_HASH: break body - let - transactionDb = db.ctx.getColumn CtTxs - state = transactionDb.state(updateOk=true).valueOr: - raiseAssert info & ": " & $$error - if state != transactionRoot: - warn logTxt info, transactionRoot, state, error="state mismatch" - break body - var transactionIdx = 0'u64 - while true: - let transactionKey = rlp.encode(transactionIdx) - let data = transactionDb.fetch(transactionKey).valueOr: - if error.error != MptNotFound: - warn logTxt info, transactionRoot, - transactionKey, action="fetch()", error=($$error) - break body - yield data - inc transactionIdx + let kvt = db.newKvt() + for idx in 0'u16..header, stateRoot->blockNum discard kvt.del(genericHashKey(blockHash).toOpenArray) -proc getTransaction*( +proc getTransactionByIndex*( db: CoreDbRef; txRoot: Hash256; - txIndex: uint64; + txIndex: uint16; res: var Transaction; ): bool = const info = "getTransaction()" - let - clearOk = txRoot == EMPTY_ROOT_HASH - mpt = db.ctx.getColumn(CtTxs, clearData=clearOk) - if not clearOk: - let state = mpt.state(updateOk=true).valueOr: - raiseAssert info & ": " & $$error - if state != txRoot: - warn logTxt info, txRoot, state, error="state mismatch" - return false - let - txData = mpt.fetch(rlp.encode(txIndex)).valueOr: - if error.error != MptNotFound: - warn logTxt info, txIndex, error=($$error) - return false + + let kvt = db.newKvt() + let key = hashIndexKey(txRoot, txIndex) + let txData = kvt.getOrEmpty(key).valueOr: + warn logTxt "getTransaction()", + txRoot, key, action="getOrEmpty()", error=($$error) + return false + if txData.len == 0: + return false + try: res = rlp.decode(txData, Transaction) - except RlpError as e: - warn logTxt info, txRoot, action="rlp.decode()", name=($e.name), msg=e.msg + except RlpError as exc: + warn logTxt info, + txRoot, action="rlp.decode()", error=exc.msg return false true @@ -614,24 +588,19 @@ proc getTransactionCount*( ): int = const info = "getTransactionCount()" - let - clearOk = txRoot == EMPTY_ROOT_HASH - mpt = db.ctx.getColumn(CtTxs, clearData=clearOk) - if not clearOk: - let state = mpt.state(updateOk=true).valueOr: - raiseAssert info & ": " & $$error - if state != txRoot: - warn logTxt info, txRoot, state, error="state mismatch" - return 0 - var txCount = 0 + + let kvt = db.newKvt() + var txCount = 0'u16 while true: - let hasPath = mpt.hasPath(rlp.encode(txCount.uint)).valueOr: - warn logTxt info, txCount, action="hasPath()", error=($$error) + let key = hashIndexKey(txRoot, txCount) + let yes = kvt.hasKey(key).valueOr: + warn logTxt info, + txRoot, key, action="hasKey()", error=($$error) return 0 - if hasPath: + if yes: inc txCount else: - return txCount + return txCount.int doAssert(false, "unreachable") @@ -667,15 +636,17 @@ proc getUncles*( proc persistWithdrawals*( db: CoreDbRef; + withdrawalsRoot: Hash256; withdrawals: openArray[Withdrawal]; ) = const info = "persistWithdrawals()" if withdrawals.len == 0: return - let mpt = db.ctx.getColumn(CtWithdrawals, clearData=true) + let kvt = db.newKvt() for idx, wd in withdrawals: - mpt.merge(rlp.encode(idx.uint), rlp.encode(wd)).isOkOr: - warn logTxt info, idx, error=($$error) + let key = hashIndexKey(withdrawalsRoot, idx.uint16) + kvt.put(key, rlp.encode(wd)).isOkOr: + warn logTxt info, idx, action="put()", error=($$error) return proc getWithdrawals*( @@ -683,8 +654,8 @@ proc getWithdrawals*( withdrawalsRoot: Hash256; ): seq[Withdrawal] {.gcsafe, raises: [RlpError].} = - for encodedWd in db.getWithdrawalsData(withdrawalsRoot): - result.add(rlp.decode(encodedWd, Withdrawal)) + for wd in db.getWithdrawals(withdrawalsRoot): + result.add(wd) proc getTransactions*( db: CoreDbRef; @@ -837,14 +808,17 @@ proc setHead*( proc persistReceipts*( db: CoreDbRef; + receiptsRoot: Hash256; receipts: openArray[Receipt]; ) = const info = "persistReceipts()" if receipts.len == 0: return - let mpt = db.ctx.getColumn(CtReceipts, clearData=true) + + let kvt = db.newKvt() for idx, rec in receipts: - mpt.merge(rlp.encode(idx.uint), rlp.encode(rec)).isOkOr: + let key = hashIndexKey(receiptsRoot, idx.uint16) + kvt.put(key, rlp.encode(rec)).isOkOr: warn logTxt info, idx, action="merge()", error=($$error) proc getReceipts*( diff --git a/nimbus/db/storage_types.nim b/nimbus/db/storage_types.nim index 0c562d742..c0e808bcf 100644 --- a/nimbus/db/storage_types.nim +++ b/nimbus/db/storage_types.nim @@ -33,6 +33,8 @@ type data*: array[33, byte] dataEndPos*: uint8 # the last populated position in the data + HashIndexKey* = array[34, byte] + func genericHashKey*(h: Hash256): DbKey {.inline.} = result.data[0] = byte ord(genericHash) result.data[1 .. 32] = h.data @@ -102,6 +104,11 @@ func skeletonBodyKey*(h: Hash256): DbKey {.inline.} = result.data[1 .. 32] = h.data result.dataEndPos = uint8 32 +func hashIndexKey*(hash: Hash256, index: uint16): HashIndexKey = + result[0..31] = hash.data + result[32] = byte(index and 0xFF) + result[33] = byte((index shl 8) and 0xFF) + template toOpenArray*(k: DbKey): openArray[byte] = k.data.toOpenArray(0, int(k.dataEndPos)) diff --git a/nimbus/graphql/ethapi.nim b/nimbus/graphql/ethapi.nim index aaf88a844..533fe3120 100644 --- a/nimbus/graphql/ethapi.nim +++ b/nimbus/graphql/ethapi.nim @@ -315,7 +315,7 @@ proc getWithdrawals(ctx: GraphqlContextRef, header: common.BlockHeader): RespRes proc getTxAt(ctx: GraphqlContextRef, header: common.BlockHeader, index: uint64): RespResult = try: var tx: Transaction - if getTransaction(ctx.chainDB, header.txRoot, index, tx): + if getTransactionByIndex(ctx.chainDB, header.txRoot, index.uint16, tx): let txn = txNode(ctx, tx, index, header.number, header.baseFeePerGas) var i = 0'u64 diff --git a/nimbus/rpc/p2p.nim b/nimbus/rpc/p2p.nim index 26a9d6fb4..2aa9cbc9c 100644 --- a/nimbus/rpc/p2p.nim +++ b/nimbus/rpc/p2p.nim @@ -11,8 +11,12 @@ import std/[sequtils, times, tables, typetraits], - json_rpc/rpcserver, stint, stew/byteutils, - json_serialization, web3/conversions, json_serialization/stew/results, + json_rpc/rpcserver, + stint, + stew/byteutils, + json_serialization, + web3/conversions, + json_serialization/stew/results, eth/common/eth_types_json_serialization, eth/[keys, rlp, p2p], ".."/[transaction, evm/state, constants], @@ -414,7 +418,7 @@ proc setupEthRpc*( let header = chainDB.getBlockHeader(txDetails.blockNumber) var tx: Transaction - if chainDB.getTransaction(header.txRoot, txDetails.index, tx): + if chainDB.getTransactionByIndex(header.txRoot, uint16(txDetails.index), tx): result = populateTransactionObject(tx, Opt.some(header), Opt.some(txDetails.index)) server.rpc("eth_getTransactionByBlockHashAndIndex") do(data: Web3Hash, quantity: Web3Quantity) -> TransactionObject: @@ -429,7 +433,7 @@ proc setupEthRpc*( return nil var tx: Transaction - if chainDB.getTransaction(header.txRoot, index, tx): + if chainDB.getTransactionByIndex(header.txRoot, uint16(index), tx): result = populateTransactionObject(tx, Opt.some(header), Opt.some(index)) else: result = nil @@ -444,7 +448,7 @@ proc setupEthRpc*( index = uint64(quantity) var tx: Transaction - if chainDB.getTransaction(header.txRoot, index, tx): + if chainDB.getTransactionByIndex(header.txRoot, uint16(index), tx): result = populateTransactionObject(tx, Opt.some(header), Opt.some(index)) else: result = nil @@ -461,7 +465,7 @@ proc setupEthRpc*( let header = chainDB.getBlockHeader(txDetails.blockNumber) var tx: Transaction - if not chainDB.getTransaction(header.txRoot, txDetails.index, tx): + if not chainDB.getTransactionByIndex(header.txRoot, uint16(txDetails.index), tx): return nil var diff --git a/premix/prestate.nim b/premix/prestate.nim index 5b70e6d77..fede7a7cd 100644 --- a/premix/prestate.nim +++ b/premix/prestate.nim @@ -24,7 +24,7 @@ proc generatePrestate*(nimbus, geth: JsonNode, blockNumber: BlockNumber, parent: kvt = chainDB.newKvt() discard chainDB.setHead(parent, true) - chainDB.persistTransactions(blockNumber, blk.transactions) + chainDB.persistTransactions(blockNumber, header.txRoot, blk.transactions) discard chainDB.persistUncles(blk.uncles) kvt.put(genericHashKey(headerHash).toOpenArray, rlp.encode(header)).isOkOr: diff --git a/tests/test_rpc.nim b/tests/test_rpc.nim index 3787c8c88..e3e09a416 100644 --- a/tests/test_rpc.nim +++ b/tests/test_rpc.nim @@ -87,8 +87,8 @@ proc persistFixtureBlock(chainDB: CoreDbRef) = # Manually inserting header to avoid any parent checks discard chainDB.newKvt.put(genericHashKey(header.blockHash).toOpenArray, rlp.encode(header)) chainDB.addBlockNumberToHashLookup(header) - chainDB.persistTransactions(header.number, getBlockBody4514995().transactions) - chainDB.persistReceipts(getReceipts4514995()) + chainDB.persistTransactions(header.number, header.txRoot, getBlockBody4514995().transactions) + chainDB.persistReceipts(header.receiptsRoot, getReceipts4514995()) proc setupEnv(com: CommonRef, signer, ks2: EthAddress, ctx: EthContext): TestEnv = var @@ -153,9 +153,9 @@ proc setupEnv(com: CommonRef, signer, ks2: EthAddress, ctx: EthContext): TestEnv signedTx1 = signTransaction(unsignedTx1, acc.privateKey, com.chainId, eip155) signedTx2 = signTransaction(unsignedTx2, acc.privateKey, com.chainId, eip155) txs = [signedTx1, signedTx2] - com.db.persistTransactions(blockNumber, txs) - let txRoot = com.db.ctx.getColumn(CtTxs).state(updateOk=true).valueOr(EMPTY_ROOT_HASH) + let txRoot = calcTxRoot(txs) + com.db.persistTransactions(blockNumber, txRoot, txs) vmState.receipts = newSeq[Receipt](txs.len) vmState.cumulativeGasUsed = 0 diff --git a/tests/test_txpool2.nim b/tests/test_txpool2.nim index 78ee4c2c0..66efabae1 100644 --- a/tests/test_txpool2.nim +++ b/tests/test_txpool2.nim @@ -337,6 +337,77 @@ proc runTxHeadDelta(noisy = true) = balance = sdb.getBalance(recipient) check balance == expected +proc runGetBlockBodyTest() = + var + env = initEnv(Cancun) + blockTime = EthTime.now() + parentHeader: BlockHeader + currentHeader: BlockHeader + + suite "Test get parent transactions after persistBlock": + test "TxPool create first block": + let + tx1 = env.makeTx(recipient, 1.u256) + tx2 = env.makeTx(recipient, 2.u256) + + check env.xp.addLocal(PooledTransaction(tx: tx1), true).isOk + check env.xp.addLocal(PooledTransaction(tx: tx2), true).isOk + + env.com.pos.prevRandao = prevRandao + env.com.pos.feeRecipient = feeRecipient + env.com.pos.timestamp = blockTime + + let r = env.xp.assembleBlock() + if r.isErr: + check false + return + + let blk = r.get.blk + check env.chain.persistBlocks([blk]).isOk + parentHeader = blk.header + check env.xp.smartHead(parentHeader) + check blk.transactions.len == 2 + + test "TxPool create second block": + let + tx1 = env.makeTx(recipient, 3.u256) + tx2 = env.makeTx(recipient, 4.u256) + tx3 = env.makeTx(recipient, 5.u256) + + check env.xp.addLocal(PooledTransaction(tx: tx1), true).isOk + check env.xp.addLocal(PooledTransaction(tx: tx2), true).isOk + check env.xp.addLocal(PooledTransaction(tx: tx3), true).isOk + + env.com.pos.prevRandao = prevRandao + env.com.pos.feeRecipient = feeRecipient + env.com.pos.timestamp = blockTime + 1 + + let r = env.xp.assembleBlock() + if r.isErr: + check false + return + + let blk = r.get.blk + check env.chain.persistBlocks([blk]).isOk + currentHeader = blk.header + check env.xp.smartHead(currentHeader) + check blk.transactions.len == 3 + + test "Get current block body": + var body: BlockBody + check env.com.db.getBlockBody(currentHeader, body) + check body.transactions.len == 3 + check env.com.db.getReceipts(currentHeader.receiptsRoot).len == 3 + check env.com.db.getTransactionCount(currentHeader.txRoot) == 3 + + test "Get parent block body": + # Make sure parent baggage doesn't swept away by aristo + var body: BlockBody + check env.com.db.getBlockBody(parentHeader, body) + check body.transactions.len == 2 + check env.com.db.getReceipts(parentHeader.receiptsRoot).len == 2 + check env.com.db.getTransactionCount(parentHeader.txRoot) == 2 + proc txPool2Main*() = const noisy = defined(debug) @@ -346,6 +417,7 @@ proc txPool2Main*() = runTxPoolPosTest() runTxPoolBlobhashTest() noisy.runTxHeadDelta + runGetBlockBodyTest() when isMainModule: txPool2Main()