From 70a1f768f79d96b97e6f1001f13f7b23771bf943 Mon Sep 17 00:00:00 2001 From: andri lim Date: Thu, 7 Nov 2024 10:43:25 +0700 Subject: [PATCH] Engine API: Route more wiring from CoreDb to ForkedChain (#2844) --- nimbus/beacon/api_handler/api_forkchoice.nim | 2 +- nimbus/beacon/api_handler/api_getbodies.nim | 34 +---- nimbus/beacon/api_handler/api_utils.nim | 11 +- nimbus/beacon/beacon_engine.nim | 23 ++-- nimbus/beacon/web3_eth_conv.nim | 37 ++--- nimbus/core/chain/chain_desc.nim | 8 +- nimbus/core/chain/persist_blocks.nim | 24 ---- nimbus/db/core_db/core_apps.nim | 135 +------------------ tests/test_rpc.nim | 12 +- 9 files changed, 40 insertions(+), 246 deletions(-) diff --git a/nimbus/beacon/api_handler/api_forkchoice.nim b/nimbus/beacon/api_handler/api_forkchoice.nim index b48085c17..d88ad4c95 100644 --- a/nimbus/beacon/api_handler/api_forkchoice.nim +++ b/nimbus/beacon/api_handler/api_forkchoice.nim @@ -177,7 +177,7 @@ proc forkchoiceUpdated*(ben: BeaconEngineRef, db.safeHeaderHash(safeBlockHash) chain.forkChoice(blockHash, finalizedBlockHash).isOkOr: - return invalidFCU(error, com, header) + return invalidFCU(error, chain, header) # If payload generation was requested, create a new block to be potentially # sealed by the beacon client. The payload will be requested later, and we diff --git a/nimbus/beacon/api_handler/api_getbodies.nim b/nimbus/beacon/api_handler/api_getbodies.nim index d43985fbf..ee0a9b4b3 100644 --- a/nimbus/beacon/api_handler/api_getbodies.nim +++ b/nimbus/beacon/api_handler/api_getbodies.nim @@ -13,7 +13,6 @@ import ../web3_eth_conv, ../beacon_engine, web3/execution_types, - ../../db/core_db, ./api_utils {.push gcsafe, raises:[CatchableError].} @@ -21,32 +20,6 @@ import const maxBodyRequest = 32 -proc getPayloadBodyByHeader(db: CoreDbRef, - header: Header, - output: var seq[Opt[ExecutionPayloadBodyV1]]) {.raises:[].} = - - let body = db.getBlockBody(header).valueOr: - output.add Opt.none(ExecutionPayloadBodyV1) - return - - let txs = w3Txs body.transactions - var wds: seq[WithdrawalV1] - if body.withdrawals.isSome: - for w in body.withdrawals.get: - wds.add w3Withdrawal(w) - - output.add( - Opt.some(ExecutionPayloadBodyV1( - transactions: txs, - # pre Shanghai block return null withdrawals - # post Shanghai block return at least empty slice - withdrawals: if header.withdrawalsRoot.isSome: - Opt.some(wds) - else: - Opt.none(seq[WithdrawalV1]) - )) - ) - func toPayloadBody(blk: Block): ExecutionPayloadBodyV1 {.raises:[].} = var wds: seq[WithdrawalV1] if blk.withdrawals.isSome: @@ -87,9 +60,6 @@ proc getPayloadBodiesByRange*(ben: BeaconEngineRef, if count > maxBodyRequest: raise tooLargeRequest("request exceeds max allowed " & $maxBodyRequest) - let - db = ben.com.db - var last = start+count-1 @@ -102,10 +72,10 @@ proc getPayloadBodiesByRange*(ben: BeaconEngineRef, # get bodies from database for bn in start..ben.chain.baseNumber: - let header = db.getBlockHeader(bn).valueOr: + let blk = ben.chain.blockByNumber(bn).valueOr: result.add Opt.none(ExecutionPayloadBodyV1) continue - db.getPayloadBodyByHeader(header, result) + result.add Opt.some(blk.toPayloadBody) if last > ben.chain.baseNumber: let blocks = ben.chain.blockFromBaseTo(last) diff --git a/nimbus/beacon/api_handler/api_utils.nim b/nimbus/beacon/api_handler/api_utils.nim index ba69235de..5c1d2cef6 100644 --- a/nimbus/beacon/api_handler/api_utils.nim +++ b/nimbus/beacon/api_handler/api_utils.nim @@ -9,6 +9,7 @@ import std/[typetraits, strutils], + web3/execution_types, json_rpc/errors, nimcrypto/sha2, stew/endians2, @@ -17,7 +18,7 @@ import ../../db/core_db, ../../utils/utils, ../../common/common, - web3/execution_types, + ../../core/chain, ../web3_eth_conv {.push gcsafe, raises:[].} @@ -185,12 +186,12 @@ proc latestValidHash*(db: CoreDbRef, default(Hash32) proc invalidFCU*(validationError: string, - com: CommonRef, - header: common.Header): ForkchoiceUpdatedResponse = - let parent = com.db.getBlockHeader(header.parentHash).valueOr: + chain: ForkedChainRef, + header: Header): ForkchoiceUpdatedResponse = + let parent = chain.headerByHash(header.parentHash).valueOr: return invalidFCU(validationError) let blockHash = - latestValidHash(com.db, parent, com.ttd.get(high(UInt256))) + latestValidHash(chain.db, parent, chain.com.ttd.get(high(UInt256))) invalidFCU(validationError, blockHash) diff --git a/nimbus/beacon/beacon_engine.nim b/nimbus/beacon/beacon_engine.nim index c8168041b..a26128149 100644 --- a/nimbus/beacon/beacon_engine.nim +++ b/nimbus/beacon/beacon_engine.nim @@ -9,15 +9,14 @@ import std/[sequtils, tables], - ./web3_eth_conv, - ./payload_conv, + eth/common/[hashes, headers], chronicles, web3/execution_types, + ./web3_eth_conv, + ./payload_conv, ./payload_queue, ./api_handler/api_utils, - ../db/core_db, - ../core/[tx_pool, casper, chain], - eth/common/[hashes, headers] + ../core/[tx_pool, casper, chain] export chain, @@ -178,8 +177,8 @@ proc generateExecutionBundle*(ben: BeaconEngineRef, if bundle.blobsBundle.isSome: template blobData: untyped = bundle.blobsBundle.get blobsBundle = Opt.some BlobsBundleV1( - commitments: blobData.commitments.mapIt it.Web3KZGCommitment, - proofs: blobData.proofs.mapIt it.Web3KZGProof, + commitments: blobData.commitments, + proofs: blobData.proofs, blobs: blobData.blobs.mapIt it.Web3Blob) ok ExecutionBundle( @@ -196,13 +195,13 @@ func setInvalidAncestor*(ben: BeaconEngineRef, header: Header, blockHash: Hash32 # bad ancestor. If yes, it constructs the payload failure response to return. proc checkInvalidAncestor*(ben: BeaconEngineRef, check, head: Hash32): Opt[PayloadStatusV1] = - proc latestValidHash(db: CoreDbRef, invalid: auto): Hash32 = - let parent = db.getBlockHeader(invalid.parentHash).valueOr: - return invalid.parentHash + proc latestValidHash(chain: ForkedChainRef, invalid: auto): Hash32 = + let parent = chain.headerByHash(invalid.parentHash).valueOr: + return invalid.parentHash if parent.difficulty != 0.u256: return default(Hash32) invalid.parentHash - + # If the hash to check is unknown, return valid ben.invalidTipsets.withValue(check, invalid) do: # If the bad hash was hit too many times, evict it and try to reprocess in @@ -244,7 +243,7 @@ proc checkInvalidAncestor*(ben: BeaconEngineRef, ben.invalidTipsets[head] = invalid[] # If the last valid hash is the terminal pow block, return 0x0 for latest valid hash - let lastValid = latestValidHash(ben.com.db, invalid) + let lastValid = latestValidHash(ben.chain, invalid) return Opt.some invalidStatus(lastValid, "links to previously rejected block") do: return Opt.none(PayloadStatusV1) diff --git a/nimbus/beacon/web3_eth_conv.nim b/nimbus/beacon/web3_eth_conv.nim index 35ba204e5..d4947707d 100644 --- a/nimbus/beacon/web3_eth_conv.nim +++ b/nimbus/beacon/web3_eth_conv.nim @@ -26,11 +26,8 @@ export type Web3Quantity* = web3types.Quantity Web3ExtraData* = web3types.DynamicBytes[0, 32] - Web3BlockNumber* = Quantity Web3Tx* = engine_api_types.TypedTransaction Web3Blob* = engine_api_types.Blob - Web3KZGProof* = engine_api_types.KzgProof - Web3KZGCommitment* = engine_api_types.KzgCommitment {.push gcsafe, raises:[].} @@ -57,17 +54,17 @@ func u64*(x: Opt[Web3Quantity]): Opt[uint64] = if x.isNone: Opt.none(uint64) else: Opt.some(uint64 x.get) -func u256*(x: Web3BlockNumber): UInt256 = +func u256*(x: Web3Quantity): UInt256 = u256(x.uint64) -func u256*(x: common.FixedBytes[32]): UInt256 = +func u256*(x: FixedBytes[32]): UInt256 = UInt256.fromBytesBE(x.data) -func ethTime*(x: Web3Quantity): common.EthTime = - common.EthTime(x) +func ethTime*(x: Web3Quantity): EthTime = + EthTime(x) -func ethGasInt*(x: Web3Quantity): common.GasInt = - common.GasInt x +func ethGasInt*(x: Web3Quantity): GasInt = + GasInt x func ethBlob*(x: Web3ExtraData): seq[byte] = distinctBase x @@ -79,14 +76,14 @@ func ethWithdrawal*(x: WithdrawalV1): common.Withdrawal = result.amount = x.amount.uint64 func ethWithdrawals*(list: openArray[WithdrawalV1]): - seq[common.Withdrawal] = - result = newSeqOfCap[common.Withdrawal](list.len) + seq[Withdrawal] = + result = newSeqOfCap[Withdrawal](list.len) for x in list: result.add ethWithdrawal(x) func ethWithdrawals*(x: Opt[seq[WithdrawalV1]]): - Opt[seq[common.Withdrawal]] = - if x.isNone: Opt.none(seq[common.Withdrawal]) + Opt[seq[Withdrawal]] = + if x.isNone: Opt.none(seq[Withdrawal]) else: Opt.some(ethWithdrawals x.get) func ethTx*(x: Web3Tx): common.Transaction {.gcsafe, raises:[RlpError].} = @@ -105,10 +102,10 @@ func ethTxs*(list: openArray[Web3Tx]): func w3Qty*(x: UInt256): Web3Quantity = Web3Quantity x.truncate(uint64) -func w3Qty*(x: common.EthTime): Web3Quantity = +func w3Qty*(x: EthTime): Web3Quantity = Web3Quantity x.uint64 -func w3Qty*(x: common.EthTime, y: int): Web3Quantity = +func w3Qty*(x: EthTime, y: int): Web3Quantity = Web3Quantity(x + y.EthTime) func w3Qty*(x: Web3Quantity, y: int): Web3Quantity = @@ -130,16 +127,6 @@ func w3Qty*(x: uint64): Web3Quantity = func w3Qty*(x: int64): Web3Quantity = Web3Quantity(x) -func w3BlockNumber*(x: Opt[uint64]): Opt[Web3BlockNumber] = - if x.isNone: Opt.none(Web3BlockNumber) - else: Opt.some(Web3BlockNumber x.get) - -func w3BlockNumber*(x: uint64): Web3BlockNumber = - Web3BlockNumber(x) - -func w3BlockNumber*(x: UInt256): Web3BlockNumber = - Web3BlockNumber x.truncate(uint64) - func w3ExtraData*(x: seq[byte]): Web3ExtraData = Web3ExtraData x diff --git a/nimbus/core/chain/chain_desc.nim b/nimbus/core/chain/chain_desc.nim index 1eb87a2a0..388b8454e 100644 --- a/nimbus/core/chain/chain_desc.nim +++ b/nimbus/core/chain/chain_desc.nim @@ -61,7 +61,7 @@ proc newChain*(com: CommonRef): ChainRef = com: com, extraValidation: extraValidation, ) - + # ------------------------------------------------------------------------------ # Public `Chain` getters # ------------------------------------------------------------------------------ @@ -85,12 +85,6 @@ func verifyFrom*(c: ChainRef): BlockNumber = ## Getter c.verifyFrom -proc currentBlock*(c: ChainRef): Result[Header, string] = - ## currentBlock retrieves the current head block of the canonical chain. - ## Ideally the block should be retrieved from the blockchain's internal cache. - ## but now it's enough to retrieve it from database - c.db.getCanonicalHead() - # ------------------------------------------------------------------------------ # Public `Chain` setters # ------------------------------------------------------------------------------ diff --git a/nimbus/core/chain/persist_blocks.nim b/nimbus/core/chain/persist_blocks.nim index 5a58c60d3..4d69511d6 100644 --- a/nimbus/core/chain/persist_blocks.nim +++ b/nimbus/core/chain/persist_blocks.nim @@ -198,30 +198,6 @@ proc persistBlocksImpl( ok((blks, txs, gas)) -# ------------------------------------------------------------------------------ -# Public `ChainDB` methods -# ------------------------------------------------------------------------------ - -proc insertBlockWithoutSetHead*(c: ChainRef, blk: Block): Result[void, string] = - discard ?c.persistBlocksImpl([blk], {NoPersistHeader, NoPersistReceipts}) - c.db.persistHeader(blk.header.blockHash, blk.header, c.com.startOfHistory) - -proc setCanonical*(c: ChainRef, header: Header): Result[void, string] = - if header.parentHash == default(Hash32): - return c.db.setHead(header) - - var body = ?c.db.getBlockBody(header) - discard - ?c.persistBlocksImpl( - [Block.init(header, move(body))], {NoPersistHeader, NoPersistTransactions} - ) - - c.db.setHead(header) - -proc setCanonical*(c: ChainRef, blockHash: Hash32): Result[void, string] = - let header = ?c.db.getBlockHeader(blockHash) - setCanonical(c, header) - proc persistBlocks*( c: ChainRef, blocks: openArray[Block], flags: PersistBlockFlags = {} ): Result[PersistStats, string] = diff --git a/nimbus/db/core_db/core_apps.nim b/nimbus/db/core_db/core_apps.nim index f63d66236..e4e3d8794 100644 --- a/nimbus/db/core_db/core_apps.nim +++ b/nimbus/db/core_db/core_apps.nim @@ -68,33 +68,6 @@ template wrapRlpException(info: static[string]; code: untyped) = except RlpError as e: return err(info & ": " & e.msg) -# ------------------------------------------------------------------------------ -# Private iterators -# ------------------------------------------------------------------------------ - -proc findNewAncestors( - db: CoreDbRef; - header: Header; - ): Result[seq[Header], string] = - ## Returns the chain leading up from the given header until the first - ## ancestor it has in common with our canonical chain. - var - h = header - res = newSeq[Header]() - while true: - let orig = ?db.getBlockHeader(h.number) - if orig.rlpHash == h.rlpHash: - break - - res.add(h) - - if h.parentHash == GENESIS_PARENT_HASH: - break - else: - h = ?db.getBlockHeader(h.parentHash) - - ok(res) - # ------------------------------------------------------------------------------ # Public iterators # ------------------------------------------------------------------------------ @@ -174,108 +147,6 @@ iterator getReceipts*( break body yield rlp.decode(data, Receipt) -# ------------------------------------------------------------------------------ -# Private helpers -# ------------------------------------------------------------------------------ - -proc removeTransactionFromCanonicalChain( - db: CoreDbRef; - transactionHash: Hash32; - ) = - ## Removes the transaction specified by the given hash from the canonical - ## chain. - db.ctx.getKvt.del(transactionHashToBlockKey(transactionHash).toOpenArray).isOkOr: - warn "removeTransactionFromCanonicalChain", - transactionHash, error=($$error) - -proc setAsCanonicalChainHead( - db: CoreDbRef; - headerHash: Hash32; - header: Header; - ): Result[void, string] = - ## Sets the header as the canonical chain HEAD. - # TODO This code handles reorgs - this should be moved elsewhere because we'll - # be handling reorgs mainly in-memory - if header.number == 0 or - db.getCanonicalHeaderHash().valueOr(default(Hash32)) != header.parentHash: - var newCanonicalHeaders = ?db.findNewAncestors(header) - reverse(newCanonicalHeaders) - for h in newCanonicalHeaders: - let - oldHash = ?db.getBlockHash(h.number) - oldHeader = ?db.getBlockHeader(oldHash) - for txHash in db.getBlockTransactionHashes(oldHeader): - db.removeTransactionFromCanonicalChain(txHash) - # TODO re-add txn to internal pending pool (only if local sender) - - for h in newCanonicalHeaders: - # TODO don't recompute block hash - db.addBlockNumberToHashLookup(h.number, h.blockHash) - - let canonicalHeadHash = canonicalHeadHashKey() - db.ctx.getKvt.put(canonicalHeadHash.toOpenArray, rlp.encode(headerHash)).isOkOr: - return err($$error) - ok() - -proc markCanonicalChain( - db: CoreDbRef; - header: Header; - headerHash: Hash32; - ): Result[void, string] = - ## mark this chain as canonical by adding block number to hash lookup - ## down to forking point - const - info = "markCanonicalChain()" - var - currHash = headerHash - currHeader = header - - # mark current header as canonical - let - kvt = db.ctx.getKvt() - key = blockNumberToHashKey(currHeader.number) - kvt.put(key.toOpenArray, rlp.encode(currHash)).isOkOr: - return err($$error) - - # it is a genesis block, done - if currHeader.parentHash == default(Hash32): - return ok() - - # mark ancestor blocks as canonical too - currHash = currHeader.parentHash - currHeader = ?db.getBlockHeader(currHeader.parentHash) - - template rlpDecodeOrZero(data: openArray[byte]): Hash32 = - try: - rlp.decode(data, Hash32) - except RlpError as exc: - warn info, key, error=exc.msg - default(Hash32) - - while currHash != default(Hash32): - let key = blockNumberToHashKey(currHeader.number) - let data = kvt.getOrEmpty(key.toOpenArray).valueOr: - return err($$error) - if data.len == 0: - # not marked, mark it - kvt.put(key.toOpenArray, rlp.encode(currHash)).isOkOr: - return err($$error) - elif rlpDecodeOrZero(data) != currHash: - # replace prev chain - kvt.put(key.toOpenArray, rlp.encode(currHash)).isOkOr: - return err($$error) - else: - # forking point, done - break - - if currHeader.parentHash == default(Hash32): - break - - currHash = currHeader.parentHash - currHeader = ?db.getBlockHeader(currHeader.parentHash) - - ok() - # ------------------------------------------------------------------------------ # Public functions # ------------------------------------------------------------------------------ @@ -632,9 +503,6 @@ proc setHead*( db: CoreDbRef; blockHash: Hash32; ): Result[void, string] = - let header = ?db.getBlockHeader(blockHash) - ?db.markCanonicalChain(header, blockHash) - let canonicalHeadHash = canonicalHeadHashKey() db.ctx.getKvt.put(canonicalHeadHash.toOpenArray, rlp.encode(blockHash)).isOkOr: return err($$error) @@ -650,7 +518,6 @@ proc setHead*( if writeHeader: kvt.put(genericHashKey(headerHash).toOpenArray, rlp.encode(header)).isOkOr: return err($$error) - ?db.markCanonicalChain(header, headerHash) let canonicalHeadHash = canonicalHeadHashKey() kvt.put(canonicalHeadHash.toOpenArray, rlp.encode(headerHash)).isOkOr: return err($$error) @@ -751,7 +618,7 @@ proc persistHeader*( if score <= canonScore: return ok() - db.setAsCanonicalChainHead(blockHash, header) + db.setHead(blockHash) proc persistHeader*( db: CoreDbRef; diff --git a/tests/test_rpc.nim b/tests/test_rpc.nim index 9313d4bc3..4cadef1ea 100644 --- a/tests/test_rpc.nim +++ b/tests/test_rpc.nim @@ -433,14 +433,14 @@ proc rpcMain*() = test "eth_getTransactionByHash": let res = await client.eth_getTransactionByHash(env.txHash) check res.isNil.not - check res.blockNumber.get() == w3BlockNumber(1'u64) + check res.blockNumber.get() == w3Qty(1'u64) let res2 = await client.eth_getTransactionByHash(env.blockHash) check res2.isNil test "eth_getTransactionByBlockHashAndIndex": let res = await client.eth_getTransactionByBlockHashAndIndex(env.blockHash, w3Qty(0'u64)) check res.isNil.not - check res.blockNumber.get() == w3BlockNumber(1'u64) + check res.blockNumber.get() == w3Qty(1'u64) let res2 = await client.eth_getTransactionByBlockHashAndIndex(env.blockHash, w3Qty(3'u64)) check res2.isNil @@ -451,7 +451,7 @@ proc rpcMain*() = test "eth_getTransactionByBlockNumberAndIndex": let res = await client.eth_getTransactionByBlockNumberAndIndex("latest", w3Qty(1'u64)) check res.isNil.not - check res.blockNumber.get() == w3BlockNumber(1'u64) + check res.blockNumber.get() == w3Qty(1'u64) let res2 = await client.eth_getTransactionByBlockNumberAndIndex("latest", w3Qty(3'u64)) check res2.isNil @@ -470,7 +470,7 @@ proc rpcMain*() = # test "eth_getTransactionReceipt": # let res = await client.eth_getTransactionReceipt(env.txHash) # check res.isNil.not - # check res.blockNumber == w3BlockNumber(1'u64) + # check res.blockNumber == w3Qty(1'u64) # let res2 = await client.eth_getTransactionReceipt(env.blockHash) # check res2.isNil @@ -478,7 +478,7 @@ proc rpcMain*() = test "eth_getUncleByBlockHashAndIndex": let res = await client.eth_getUncleByBlockHashAndIndex(env.blockHash, w3Qty(0'u64)) check res.isNil.not - check res.number == w3BlockNumber(1'u64) + check res.number == w3Qty(1'u64) let res2 = await client.eth_getUncleByBlockHashAndIndex(env.blockHash, w3Qty(1'u64)) check res2.isNil @@ -489,7 +489,7 @@ proc rpcMain*() = test "eth_getUncleByBlockNumberAndIndex": let res = await client.eth_getUncleByBlockNumberAndIndex("latest", w3Qty(0'u64)) check res.isNil.not - check res.number == w3BlockNumber(1'u64) + check res.number == w3Qty(1'u64) let res2 = await client.eth_getUncleByBlockNumberAndIndex("latest", w3Qty(1'u64)) check res2.isNil