Engine API: Route more wiring from CoreDb to ForkedChain (#2844)

nimbus/beacon/api_handler/api_forkchoice.nim

@@ -177,7 +177,7 @@ proc forkchoiceUpdated*(ben: BeaconEngineRef,
     db.safeHeaderHash(safeBlockHash)
 
   chain.forkChoice(blockHash, finalizedBlockHash).isOkOr:
-    return invalidFCU(error, com, header)
+    return invalidFCU(error, chain, header)
 
   # If payload generation was requested, create a new block to be potentially
   # sealed by the beacon client. The payload will be requested later, and we

nimbus/beacon/api_handler/api_getbodies.nim

@@ -13,7 +13,6 @@ import
   ../web3_eth_conv,
   ../beacon_engine,
   web3/execution_types,
-  ../../db/core_db,
   ./api_utils
 
 {.push gcsafe, raises:[CatchableError].}
@@ -21,32 +20,6 @@ import
 const
   maxBodyRequest = 32
 
-proc getPayloadBodyByHeader(db: CoreDbRef,
-        header: Header,
-        output: var seq[Opt[ExecutionPayloadBodyV1]]) {.raises:[].} =
-
-  let body = db.getBlockBody(header).valueOr:
-    output.add Opt.none(ExecutionPayloadBodyV1)
-    return
-
-  let txs = w3Txs body.transactions
-  var wds: seq[WithdrawalV1]
-  if body.withdrawals.isSome:
-    for w in body.withdrawals.get:
-      wds.add w3Withdrawal(w)
-
-  output.add(
-    Opt.some(ExecutionPayloadBodyV1(
-      transactions: txs,
-      # pre Shanghai block return null withdrawals
-      # post Shanghai block return at least empty slice
-      withdrawals: if header.withdrawalsRoot.isSome:
-                     Opt.some(wds)
-                   else:
-                     Opt.none(seq[WithdrawalV1])
-    ))
-  )
-
 func toPayloadBody(blk: Block): ExecutionPayloadBodyV1 {.raises:[].}  =
   var wds: seq[WithdrawalV1]
   if blk.withdrawals.isSome:
@@ -87,9 +60,6 @@ proc getPayloadBodiesByRange*(ben: BeaconEngineRef,
   if count > maxBodyRequest:
     raise tooLargeRequest("request exceeds max allowed " & $maxBodyRequest)
 
-  let
-    db = ben.com.db
-
   var
     last = start+count-1
 
@@ -102,10 +72,10 @@ proc getPayloadBodiesByRange*(ben: BeaconEngineRef,
 
   # get bodies from database
   for bn in start..ben.chain.baseNumber:
-    let header = db.getBlockHeader(bn).valueOr:
+    let blk = ben.chain.blockByNumber(bn).valueOr:
       result.add Opt.none(ExecutionPayloadBodyV1)
       continue
-    db.getPayloadBodyByHeader(header, result)
+    result.add Opt.some(blk.toPayloadBody)
 
   if last > ben.chain.baseNumber:
     let blocks = ben.chain.blockFromBaseTo(last)

nimbus/beacon/api_handler/api_utils.nim

@@ -9,6 +9,7 @@
 
 import
   std/[typetraits, strutils],
+  web3/execution_types,
   json_rpc/errors,
   nimcrypto/sha2,
   stew/endians2,
@@ -17,7 +18,7 @@ import
   ../../db/core_db,
   ../../utils/utils,
   ../../common/common,
-  web3/execution_types,
+  ../../core/chain,
   ../web3_eth_conv
 
 {.push gcsafe, raises:[].}
@@ -185,12 +186,12 @@ proc latestValidHash*(db: CoreDbRef,
     default(Hash32)
 
 proc invalidFCU*(validationError: string,
-                 com: CommonRef,
+                 chain: ForkedChainRef,
-                 header: common.Header): ForkchoiceUpdatedResponse =
+                 header: Header): ForkchoiceUpdatedResponse =
-  let parent = com.db.getBlockHeader(header.parentHash).valueOr:
+  let parent = chain.headerByHash(header.parentHash).valueOr:
     return invalidFCU(validationError)
 
   let blockHash =
-    latestValidHash(com.db, parent, com.ttd.get(high(UInt256)))
+    latestValidHash(chain.db, parent, chain.com.ttd.get(high(UInt256)))
 
   invalidFCU(validationError, blockHash)

nimbus/beacon/beacon_engine.nim

@@ -9,15 +9,14 @@
 
 import
   std/[sequtils, tables],
-  ./web3_eth_conv,
+  eth/common/[hashes, headers],
-  ./payload_conv,
   chronicles,
   web3/execution_types,
+  ./web3_eth_conv,
+  ./payload_conv,
   ./payload_queue,
   ./api_handler/api_utils,
-  ../db/core_db,
+  ../core/[tx_pool, casper, chain]
-  ../core/[tx_pool, casper, chain],
-  eth/common/[hashes, headers]
 
 export
   chain,
@@ -178,8 +177,8 @@ proc generateExecutionBundle*(ben: BeaconEngineRef,
     if bundle.blobsBundle.isSome:
       template blobData: untyped = bundle.blobsBundle.get
       blobsBundle = Opt.some BlobsBundleV1(
-        commitments: blobData.commitments.mapIt it.Web3KZGCommitment,
+        commitments: blobData.commitments,
-        proofs: blobData.proofs.mapIt it.Web3KZGProof,
+        proofs: blobData.proofs,
         blobs: blobData.blobs.mapIt it.Web3Blob)
 
     ok ExecutionBundle(
@@ -196,13 +195,13 @@ func setInvalidAncestor*(ben: BeaconEngineRef, header: Header, blockHash: Hash32
 # bad ancestor. If yes, it constructs the payload failure response to return.
 proc checkInvalidAncestor*(ben: BeaconEngineRef,
                            check, head: Hash32): Opt[PayloadStatusV1] =
-  proc latestValidHash(db: CoreDbRef, invalid: auto): Hash32 =
+  proc latestValidHash(chain: ForkedChainRef, invalid: auto): Hash32 =
-    let parent = db.getBlockHeader(invalid.parentHash).valueOr:
+    let parent = chain.headerByHash(invalid.parentHash).valueOr:
-      return invalid.parentHash      
+      return invalid.parentHash
     if parent.difficulty != 0.u256:
       return default(Hash32)
     invalid.parentHash
-  
+
   # If the hash to check is unknown, return valid
   ben.invalidTipsets.withValue(check, invalid) do:
     # If the bad hash was hit too many times, evict it and try to reprocess in
@@ -244,7 +243,7 @@ proc checkInvalidAncestor*(ben: BeaconEngineRef,
       ben.invalidTipsets[head] = invalid[]
 
     # If the last valid hash is the terminal pow block, return 0x0 for latest valid hash
-    let lastValid = latestValidHash(ben.com.db, invalid)
+    let lastValid = latestValidHash(ben.chain, invalid)
     return Opt.some invalidStatus(lastValid, "links to previously rejected block")
   do:
     return Opt.none(PayloadStatusV1)

nimbus/beacon/web3_eth_conv.nim

@@ -26,11 +26,8 @@ export
 type
   Web3Quantity*      = web3types.Quantity
   Web3ExtraData*     = web3types.DynamicBytes[0, 32]
-  Web3BlockNumber*   = Quantity
   Web3Tx*            = engine_api_types.TypedTransaction
   Web3Blob*          = engine_api_types.Blob
-  Web3KZGProof*      = engine_api_types.KzgProof
-  Web3KZGCommitment* = engine_api_types.KzgCommitment
 
 {.push gcsafe, raises:[].}
 
@@ -57,17 +54,17 @@ func u64*(x: Opt[Web3Quantity]): Opt[uint64] =
   if x.isNone: Opt.none(uint64)
   else: Opt.some(uint64 x.get)
 
-func u256*(x: Web3BlockNumber): UInt256 =
+func u256*(x: Web3Quantity): UInt256 =
   u256(x.uint64)
 
-func u256*(x: common.FixedBytes[32]): UInt256 =
+func u256*(x: FixedBytes[32]): UInt256 =
   UInt256.fromBytesBE(x.data)
 
-func ethTime*(x: Web3Quantity): common.EthTime =
+func ethTime*(x: Web3Quantity): EthTime =
-  common.EthTime(x)
+  EthTime(x)
 
-func ethGasInt*(x: Web3Quantity): common.GasInt =
+func ethGasInt*(x: Web3Quantity): GasInt =
-  common.GasInt x
+  GasInt x
 
 func ethBlob*(x: Web3ExtraData): seq[byte] =
   distinctBase x
@@ -79,14 +76,14 @@ func ethWithdrawal*(x: WithdrawalV1): common.Withdrawal =
   result.amount = x.amount.uint64
 
 func ethWithdrawals*(list: openArray[WithdrawalV1]):
-                       seq[common.Withdrawal] =
+                       seq[Withdrawal] =
-  result = newSeqOfCap[common.Withdrawal](list.len)
+  result = newSeqOfCap[Withdrawal](list.len)
   for x in list:
     result.add ethWithdrawal(x)
 
 func ethWithdrawals*(x: Opt[seq[WithdrawalV1]]):
-                       Opt[seq[common.Withdrawal]] =
+                       Opt[seq[Withdrawal]] =
-  if x.isNone: Opt.none(seq[common.Withdrawal])
+  if x.isNone: Opt.none(seq[Withdrawal])
   else: Opt.some(ethWithdrawals x.get)
 
 func ethTx*(x: Web3Tx): common.Transaction {.gcsafe, raises:[RlpError].} =
@@ -105,10 +102,10 @@ func ethTxs*(list: openArray[Web3Tx]):
 func w3Qty*(x: UInt256): Web3Quantity =
   Web3Quantity x.truncate(uint64)
 
-func w3Qty*(x: common.EthTime): Web3Quantity =
+func w3Qty*(x: EthTime): Web3Quantity =
   Web3Quantity x.uint64
 
-func w3Qty*(x: common.EthTime, y: int): Web3Quantity =
+func w3Qty*(x: EthTime, y: int): Web3Quantity =
   Web3Quantity(x + y.EthTime)
 
 func w3Qty*(x: Web3Quantity, y: int): Web3Quantity =
@@ -130,16 +127,6 @@ func w3Qty*(x: uint64): Web3Quantity =
 func w3Qty*(x: int64): Web3Quantity =
   Web3Quantity(x)
 
-func w3BlockNumber*(x: Opt[uint64]): Opt[Web3BlockNumber] =
-  if x.isNone: Opt.none(Web3BlockNumber)
-  else: Opt.some(Web3BlockNumber x.get)
-
-func w3BlockNumber*(x: uint64): Web3BlockNumber =
-  Web3BlockNumber(x)
-
-func w3BlockNumber*(x: UInt256): Web3BlockNumber =
-  Web3BlockNumber x.truncate(uint64)
-
 func w3ExtraData*(x: seq[byte]): Web3ExtraData =
   Web3ExtraData x
 

nimbus/core/chain/chain_desc.nim

@@ -61,7 +61,7 @@ proc newChain*(com: CommonRef): ChainRef =
     com: com,
     extraValidation: extraValidation,
   )
-  
+
 # ------------------------------------------------------------------------------
 # Public `Chain` getters
 # ------------------------------------------------------------------------------
@@ -85,12 +85,6 @@ func verifyFrom*(c: ChainRef): BlockNumber =
   ## Getter
   c.verifyFrom
 
-proc currentBlock*(c: ChainRef): Result[Header, string] =
-  ## currentBlock retrieves the current head block of the canonical chain.
-  ## Ideally the block should be retrieved from the blockchain's internal cache.
-  ## but now it's enough to retrieve it from database
-  c.db.getCanonicalHead()
-
 # ------------------------------------------------------------------------------
 # Public `Chain` setters
 # ------------------------------------------------------------------------------

nimbus/core/chain/persist_blocks.nim

@@ -198,30 +198,6 @@ proc persistBlocksImpl(
 
   ok((blks, txs, gas))
 
-# ------------------------------------------------------------------------------
-# Public `ChainDB` methods
-# ------------------------------------------------------------------------------
-
-proc insertBlockWithoutSetHead*(c: ChainRef, blk: Block): Result[void, string] =
-  discard ?c.persistBlocksImpl([blk], {NoPersistHeader, NoPersistReceipts})
-  c.db.persistHeader(blk.header.blockHash, blk.header, c.com.startOfHistory)
-  
-proc setCanonical*(c: ChainRef, header: Header): Result[void, string] =
-  if header.parentHash == default(Hash32):
-    return c.db.setHead(header)
-      
-  var body = ?c.db.getBlockBody(header)    
-  discard
-    ?c.persistBlocksImpl(
-      [Block.init(header, move(body))], {NoPersistHeader, NoPersistTransactions}
-    )
-
-  c.db.setHead(header)
-    
-proc setCanonical*(c: ChainRef, blockHash: Hash32): Result[void, string] =
-  let header = ?c.db.getBlockHeader(blockHash)
-  setCanonical(c, header)
-
 proc persistBlocks*(
     c: ChainRef, blocks: openArray[Block], flags: PersistBlockFlags = {}
 ): Result[PersistStats, string] =

nimbus/db/core_db/core_apps.nim

@@ -68,33 +68,6 @@ template wrapRlpException(info: static[string]; code: untyped) =
   except RlpError as e:
     return err(info & ": " & e.msg)
 
-# ------------------------------------------------------------------------------
-# Private iterators
-# ------------------------------------------------------------------------------
-
-proc findNewAncestors(
-    db: CoreDbRef;
-    header: Header;
-      ): Result[seq[Header], string] =
-  ## Returns the chain leading up from the given header until the first
-  ## ancestor it has in common with our canonical chain.
-  var
-    h = header
-    res = newSeq[Header]()
-  while true:
-    let orig = ?db.getBlockHeader(h.number)
-    if orig.rlpHash == h.rlpHash:
-      break
-
-    res.add(h)
-
-    if h.parentHash == GENESIS_PARENT_HASH:
-      break
-    else:
-      h = ?db.getBlockHeader(h.parentHash)
-
-  ok(res)
-
 # ------------------------------------------------------------------------------
 # Public iterators
 # ------------------------------------------------------------------------------
@@ -174,108 +147,6 @@ iterator getReceipts*(
         break body
       yield rlp.decode(data, Receipt)
 
-# ------------------------------------------------------------------------------
-# Private helpers
-# ------------------------------------------------------------------------------
-
-proc removeTransactionFromCanonicalChain(
-    db: CoreDbRef;
-    transactionHash: Hash32;
-      ) =
-  ## Removes the transaction specified by the given hash from the canonical
-  ## chain.
-  db.ctx.getKvt.del(transactionHashToBlockKey(transactionHash).toOpenArray).isOkOr:
-    warn "removeTransactionFromCanonicalChain",
-      transactionHash, error=($$error)
-
-proc setAsCanonicalChainHead(
-    db: CoreDbRef;
-    headerHash: Hash32;
-    header: Header;
-      ): Result[void, string] =
-  ## Sets the header as the canonical chain HEAD.
-  # TODO This code handles reorgs - this should be moved elsewhere because we'll
-  #      be handling reorgs mainly in-memory
-  if header.number == 0 or
-      db.getCanonicalHeaderHash().valueOr(default(Hash32)) != header.parentHash:
-    var newCanonicalHeaders = ?db.findNewAncestors(header)
-    reverse(newCanonicalHeaders)
-    for h in newCanonicalHeaders:
-      let
-        oldHash = ?db.getBlockHash(h.number)
-        oldHeader = ?db.getBlockHeader(oldHash)
-      for txHash in db.getBlockTransactionHashes(oldHeader):
-        db.removeTransactionFromCanonicalChain(txHash)
-        # TODO re-add txn to internal pending pool (only if local sender)
-
-    for h in newCanonicalHeaders:
-      # TODO don't recompute block hash
-      db.addBlockNumberToHashLookup(h.number, h.blockHash)
-
-  let canonicalHeadHash = canonicalHeadHashKey()
-  db.ctx.getKvt.put(canonicalHeadHash.toOpenArray, rlp.encode(headerHash)).isOkOr:
-    return err($$error)
-  ok()
-
-proc markCanonicalChain(
-    db: CoreDbRef;
-    header: Header;
-    headerHash: Hash32;
-      ): Result[void, string] =
-  ## mark this chain as canonical by adding block number to hash lookup
-  ## down to forking point
-  const
-    info = "markCanonicalChain()"
-  var
-    currHash = headerHash
-    currHeader = header
-
-  # mark current header as canonical
-  let
-    kvt = db.ctx.getKvt()
-    key = blockNumberToHashKey(currHeader.number)
-  kvt.put(key.toOpenArray, rlp.encode(currHash)).isOkOr:
-    return err($$error)
-
-  # it is a genesis block, done
-  if currHeader.parentHash == default(Hash32):
-    return ok()
-
-  # mark ancestor blocks as canonical too
-  currHash = currHeader.parentHash
-  currHeader = ?db.getBlockHeader(currHeader.parentHash)
-
-  template rlpDecodeOrZero(data: openArray[byte]): Hash32 =
-    try:
-      rlp.decode(data, Hash32)
-    except RlpError as exc:
-      warn info, key, error=exc.msg
-      default(Hash32)
-
-  while currHash != default(Hash32):
-    let key = blockNumberToHashKey(currHeader.number)
-    let data = kvt.getOrEmpty(key.toOpenArray).valueOr:
-      return err($$error)
-    if data.len == 0:
-      # not marked, mark it
-      kvt.put(key.toOpenArray, rlp.encode(currHash)).isOkOr:
-        return err($$error)
-    elif rlpDecodeOrZero(data) != currHash:
-      # replace prev chain
-      kvt.put(key.toOpenArray, rlp.encode(currHash)).isOkOr:
-        return err($$error)
-    else:
-      # forking point, done
-      break
-
-    if currHeader.parentHash == default(Hash32):
-      break
-
-    currHash = currHeader.parentHash
-    currHeader = ?db.getBlockHeader(currHeader.parentHash)
-
-  ok()
-
 # ------------------------------------------------------------------------------
 # Public functions
 # ------------------------------------------------------------------------------
@@ -632,9 +503,6 @@ proc setHead*(
     db: CoreDbRef;
     blockHash: Hash32;
       ): Result[void, string] =
-  let header = ?db.getBlockHeader(blockHash)
-  ?db.markCanonicalChain(header, blockHash)
-
   let canonicalHeadHash = canonicalHeadHashKey()
   db.ctx.getKvt.put(canonicalHeadHash.toOpenArray, rlp.encode(blockHash)).isOkOr:
     return err($$error)
@@ -650,7 +518,6 @@ proc setHead*(
   if writeHeader:
     kvt.put(genericHashKey(headerHash).toOpenArray, rlp.encode(header)).isOkOr:
       return err($$error)
-  ?db.markCanonicalChain(header, headerHash)
   let canonicalHeadHash = canonicalHeadHashKey()
   kvt.put(canonicalHeadHash.toOpenArray, rlp.encode(headerHash)).isOkOr:
     return err($$error)
@@ -751,7 +618,7 @@ proc persistHeader*(
     if score <= canonScore:
       return ok()
 
-  db.setAsCanonicalChainHead(blockHash, header)
+  db.setHead(blockHash)
 
 proc persistHeader*(
     db: CoreDbRef;

tests/test_rpc.nim

@@ -433,14 +433,14 @@ proc rpcMain*() =
     test "eth_getTransactionByHash":
       let res = await client.eth_getTransactionByHash(env.txHash)
       check res.isNil.not
-      check res.blockNumber.get() == w3BlockNumber(1'u64)
+      check res.blockNumber.get() == w3Qty(1'u64)
       let res2 = await client.eth_getTransactionByHash(env.blockHash)
       check res2.isNil
 
     test "eth_getTransactionByBlockHashAndIndex":
       let res = await client.eth_getTransactionByBlockHashAndIndex(env.blockHash, w3Qty(0'u64))
       check res.isNil.not
-      check res.blockNumber.get() == w3BlockNumber(1'u64)
+      check res.blockNumber.get() == w3Qty(1'u64)
 
       let res2 = await client.eth_getTransactionByBlockHashAndIndex(env.blockHash, w3Qty(3'u64))
       check res2.isNil
@@ -451,7 +451,7 @@ proc rpcMain*() =
     test "eth_getTransactionByBlockNumberAndIndex":
       let res = await client.eth_getTransactionByBlockNumberAndIndex("latest", w3Qty(1'u64))
       check res.isNil.not
-      check res.blockNumber.get() == w3BlockNumber(1'u64)
+      check res.blockNumber.get() == w3Qty(1'u64)
 
       let res2 = await client.eth_getTransactionByBlockNumberAndIndex("latest", w3Qty(3'u64))
       check res2.isNil
@@ -470,7 +470,7 @@ proc rpcMain*() =
     # test "eth_getTransactionReceipt":
     #   let res = await client.eth_getTransactionReceipt(env.txHash)
     #   check res.isNil.not
-    #   check res.blockNumber == w3BlockNumber(1'u64)
+    #   check res.blockNumber == w3Qty(1'u64)
 
     #   let res2 = await client.eth_getTransactionReceipt(env.blockHash)
     #   check res2.isNil
@@ -478,7 +478,7 @@ proc rpcMain*() =
     test "eth_getUncleByBlockHashAndIndex":
       let res = await client.eth_getUncleByBlockHashAndIndex(env.blockHash, w3Qty(0'u64))
       check res.isNil.not
-      check res.number == w3BlockNumber(1'u64)
+      check res.number == w3Qty(1'u64)
 
       let res2 = await client.eth_getUncleByBlockHashAndIndex(env.blockHash, w3Qty(1'u64))
       check res2.isNil
@@ -489,7 +489,7 @@ proc rpcMain*() =
     test "eth_getUncleByBlockNumberAndIndex":
       let res = await client.eth_getUncleByBlockNumberAndIndex("latest", w3Qty(0'u64))
       check res.isNil.not
-      check res.number == w3BlockNumber(1'u64)
+      check res.number == w3Qty(1'u64)
 
       let res2 = await client.eth_getUncleByBlockNumberAndIndex("latest", w3Qty(1'u64))
       check res2.isNil