Implementation of Nimbus eth_getProof RPC Endpoint. (#1960)

* Initial implementation of eth_getProof endpoint.

* Implemented generation of account and storage proofs.

* Minor fixes and additional tests.

* Refactor getBranch code into a separate file.

* Improve usage of test data.

* Fix copyright year.

* Return zero hash for codeHash and storageHash if account doesn't exist.

* Update copyright notice and moved trie key hashing to inside getBranch proc.

nimbus/db/distinct_tries.nim

@@ -1,5 +1,5 @@
 # Nimbus
-# Copyright (c) 2023 Status Research & Development GmbH
+# Copyright (c) 2023-2024 Status Research & Development GmbH
 # Licensed under either of
 #  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
 #    http://www.apache.org/licenses/LICENSE-2.0)

nimbus/db/state_db.nim

@@ -1,5 +1,5 @@
 # Nimbus
-# Copyright (c) 2018-2023 Status Research & Development GmbH
+# Copyright (c) 2018-2024 Status Research & Development GmbH
 # Licensed under either of
 #  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
 #  * MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT)
@@ -29,6 +29,8 @@ export
   isDeadAccount,
   isEmptyAccount,
   newAccountStateDB,
-  rootHash
+  rootHash,
+  getAccountProof,
+  getStorageProof
 
 # End

nimbus/db/state_db/base.nim

@@ -1,5 +1,5 @@
 # Nimbus
-# Copyright (c) 2018-2023 Status Research & Development GmbH
+# Copyright (c) 2018-2024 Status Research & Development GmbH
 # Licensed under either of
 #  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
 #  * MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT)
@@ -8,10 +8,10 @@
 import
   std/[sets, strformat],
   chronicles,
-  eth/[common, rlp],
+  eth/[common, rlp, trie/trie_defs],
   ../../constants,
   ../../utils/utils,
-  ".."/[core_db, distinct_tries, storage_types]
+  ".."/[core_db, distinct_tries, storage_types, trie_get_branch]
 
 logScope:
   topics = "state_db"
@@ -55,6 +55,10 @@ type
     when aleth_compat:
       cleared: HashSet[EthAddress]
 
+  MptNodeRlpBytes* = seq[byte]
+  AccountProof* = seq[MptNodeRlpBytes]
+  SlotProof* = seq[MptNodeRlpBytes]
+
 proc pruneTrie*(db: AccountStateDB): bool =
   db.trie.isPruning
 
@@ -249,6 +253,27 @@ proc isDeadAccount*(db: AccountStateDB, address: EthAddress): bool =
   else:
     result = true
 
+proc removeEmptyRlpNode(branch: var seq[MptNodeRlpBytes]) =
+  if branch.len() == 1 and branch[0] == emptyRlp:
+    branch.del(0)
+
+proc getAccountProof*(db: AccountStateDB, address: EthAddress): AccountProof =
+  var branch = db.trie.phk().getBranch(address)
+  removeEmptyRlpNode(branch)
+  branch
+
+proc getStorageProof*(db: AccountStateDB, address: EthAddress, slots: seq[UInt256]): seq[SlotProof] =
+  var account = db.getAccount(address)
+  var storageTrie = db.getStorageTrie(account)
+
+  var slotProofs = newSeqOfCap[SlotProof](slots.len())
+  for slot in slots:
+    var branch = storageTrie.phk().getBranch(createTrieKeyFromSlot(slot))
+    removeEmptyRlpNode(branch)
+    slotProofs.add(branch)
+
+  slotProofs
+
 # Note: `state_db.getCommittedStorage()` is nowhere used.
 #
 #proc getCommittedStorage*(db: AccountStateDB, address: EthAddress, slot: UInt256): UInt256 =

nimbus/db/state_db/read_only.nim

@@ -1,5 +1,5 @@
 # Nimbus
-# Copyright (c) 2018-2023 Status Research & Development GmbH
+# Copyright (c) 2018-2024 Status Research & Development GmbH
 # Licensed under either of
 #  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)
 #    or http://www.apache.org/licenses/LICENSE-2.0)
@@ -28,6 +28,8 @@ proc hasCodeOrNonce*(db: ReadOnlyStateDB, address: EthAddress): bool {.borrow.}
 proc accountExists*(db: ReadOnlyStateDB, address: EthAddress): bool {.borrow.}
 proc isDeadAccount*(db: ReadOnlyStateDB, address: EthAddress): bool {.borrow.}
 proc isEmptyAccount*(db: ReadOnlyStateDB, address: EthAddress): bool {.borrow.}
+proc getAccountProof*(db: ReadOnlyStateDB, address: EthAddress): AccountProof {.borrow.}
+proc getStorageProof*(db: ReadOnlyStateDB, address: EthAddress, slots: seq[UInt256]): seq[SlotProof] {.borrow.}
 #proc getCommittedStorage*(db: ReadOnlyStateDB, address: EthAddress, slot: UInt256): UInt256 {.borrow.}
 
 # End

nimbus/db/trie_get_branch.nim

@@ -0,0 +1,97 @@
+# Nimbus
+# Copyright (c) 2023-2024 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
+#    http://www.apache.org/licenses/LICENSE-2.0)
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT) or
+#    http://opensource.org/licenses/MIT)
+# at your option. This file may not be copied, modified, or distributed except
+# according to those terms.
+
+# This implementation of getBranch on the CoreDbPhkRef type is a temporary solution
+# which can be removed once we get an equivient proc defined on the CoreDbPhkRef type
+# in the db layer.
+
+{.push raises: [].}
+
+import
+  eth/[rlp, trie/nibbles],
+  "."/[core_db]
+
+type
+  TrieNodeKey = object
+    hash: KeccakHash
+    usedBytes: uint8
+
+template len(key: TrieNodeKey): int =
+  key.usedBytes.int
+
+template asDbKey(k: TrieNodeKey): untyped =
+  doAssert k.usedBytes == 32
+  k.hash.data
+
+template extensionNodeKey(r: Rlp): auto =
+  hexPrefixDecode r.listElem(0).toBytes
+
+proc getLocalBytes(x: TrieNodeKey): seq[byte] =
+  ## This proc should be used on nodes using the optimization
+  ## of short values within the key.
+  doAssert x.usedBytes < 32
+  x.hash.data[0..<x.usedBytes]
+
+proc dbGet(db: CoreDbRef, data: openArray[byte]): seq[byte]
+  {.gcsafe, raises: [].} =
+  db.kvt.get(data)
+
+template keyToLocalBytes(db: CoreDbRef, k: TrieNodeKey): seq[byte] =
+  if k.len < 32: k.getLocalBytes
+  else: dbGet(db, k.asDbKey)
+
+proc expectHash(r: Rlp): seq[byte] {.raises: [RlpError].} =
+  result = r.toBytes
+  if result.len != 32:
+    raise newException(RlpTypeMismatch,
+      "RLP expected to be a Keccak hash value, but has an incorrect length")
+
+template getNode(db: CoreDbRef, elem: Rlp): untyped =
+  if elem.isList: @(elem.rawData)
+  else: dbGet(db, elem.expectHash)
+
+proc getBranchAux(
+    db: CoreDbRef, node: openArray[byte],
+    fullPath: NibblesSeq,
+    pathIndex: int,
+    output: var seq[seq[byte]]) {.raises: [RlpError].} =
+  var nodeRlp = rlpFromBytes node
+  if not nodeRlp.hasData or nodeRlp.isEmpty: return
+
+  let path = fullPath.slice(pathIndex)
+  case nodeRlp.listLen
+  of 2:
+    let (isLeaf, k) = nodeRlp.extensionNodeKey
+    let sharedNibbles = sharedPrefixLen(path, k)
+    if sharedNibbles == k.len:
+      let value = nodeRlp.listElem(1)
+      if not isLeaf:
+        let nextLookup = getNode(db, value)
+        output.add nextLookup
+        getBranchAux(db, nextLookup, fullPath, pathIndex + sharedNibbles, output)
+  of 17:
+    if path.len != 0:
+      var branch = nodeRlp.listElem(path[0].int)
+      if not branch.isEmpty:
+        let nextLookup = getNode(db, branch)
+        output.add nextLookup
+        getBranchAux(db, nextLookup, fullPath, pathIndex + 1, output)
+  else:
+    raise newException(RlpError, "node has an unexpected number of children")
+
+proc getBranch*(
+    self: CoreDbPhkRef;
+    key: openArray[byte]): seq[seq[byte]] {.raises: [RlpError].} =
+  let keyHash = keccakHash(key).data
+  result = @[]
+  var node = keyToLocalBytes(self.parent(), TrieNodeKey(
+        hash: self.rootHash(), usedBytes: self.rootHash().data.len().uint8))
+  result.add node
+  getBranchAux(self.parent(), node, initNibbleRange(keyHash), 0, result)

nimbus/rpc/p2p.nim

@@ -1,5 +1,5 @@
 # Nimbus
-# Copyright (c) 2018-2023 Status Research & Development GmbH
+# Copyright (c) 2018-2024 Status Research & Development GmbH
 # Licensed under either of
 #  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
 #  * MIT license ([LICENSE-MIT](LICENSE-MIT))
@@ -503,6 +503,49 @@ proc setupEthRpc*(
       )
       return logs
 
+  server.rpc("eth_getProof") do(data: Web3Address, slots: seq[UInt256], quantityTag: BlockTag) -> ProofResponse:
+    ## Returns information about an account and storage slots (if the account is a contract
+    ## and the slots are requested) along with account and storage proofs which prove the
+    ## existence of the values in the state.
+    ## See spec here: https://eips.ethereum.org/EIPS/eip-1186
+    ##
+    ## data: address of the account.
+    ## slots: integers of the positions in the storage to return with storage proofs.
+    ## quantityTag: integer block number, or the string "latest", "earliest" or "pending", see the default block parameter.
+    ## Returns: the proof response containing the account, account proof and storage proof
+
+    let
+      accDB = stateDBFromTag(quantityTag)
+      address = data.ethAddr
+      acc = accDB.getAccount(address)
+      accExists = accDB.accountExists(address)
+      accountProof = accDB.getAccountProof(address)
+      slotProofs = accDB.getStorageProof(address, slots)
+
+    var storage = newSeqOfCap[StorageProof](slots.len)
+
+    for i, slotKey in slots:
+      let (slotValue, _) = accDB.getStorage(address, u256(slotKey))
+      storage.add(StorageProof(
+          key: u256(slotKey),
+          value: slotValue,
+          proof: seq[RlpEncodedBytes](slotProofs[i])))
+
+    if accExists:
+      ProofResponse(
+            address: w3Addr(address),
+            accountProof: seq[RlpEncodedBytes](accountProof),
+            balance: acc.balance,
+            nonce: w3Qty(acc.nonce),
+            codeHash: w3Hash(acc.codeHash),
+            storageHash: w3Hash(acc.storageRoot),
+            storageProof: storage)
+    else:
+      ProofResponse(
+            address: w3Addr(address),
+            accountProof: seq[RlpEncodedBytes](accountProof),
+            storageProof: storage)
+
 #[
   server.rpc("eth_newFilter") do(filterOptions: FilterOptions) -> int:
     ## Creates a filter object, based on filter options, to notify when the state changes (logs).

tests/test_rpc.nim

@@ -1,18 +1,18 @@
 # Nimbus
-# Copyright (c) 2018-2023 Status Research & Development GmbH
+# Copyright (c) 2018-2024 Status Research & Development GmbH
 # Licensed under either of
 #  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
 #  * MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT)
 # at your option. This file may not be copied, modified, or distributed except according to those terms.
 
 import
-  std/[json, tables, os, typetraits, times],
+  std/[json, os, typetraits, times, sequtils],
   asynctest, web3/eth_api,
-  nimcrypto/[hash], stew/byteutils,
+  stew/byteutils,
   json_rpc/[rpcserver, rpcclient],
-  eth/[rlp, keys, p2p/private/p2p_types],
+  nimcrypto/[keccak, hash],
-  ../nimbus/[constants, transaction, config,
+  eth/[rlp, keys, trie/hexary_proof_verification],
-             vm_state, vm_types, version],
+  ../nimbus/[constants, transaction, config, vm_state, vm_types, version],
   ../nimbus/db/[ledger, storage_types],
   ../nimbus/sync/protocol,
   ../nimbus/core/[tx_pool, chain, executor, executor/executor_helpers, pow/difficulty],
@@ -47,6 +47,41 @@ func w3Addr(x: string): Web3Address =
 func w3Hash(x: string): Web3Hash =
   Web3Hash hexToByteArray[32](x)
 
+func zeroHash(): Web3Hash =
+  w3Hash("0x0000000000000000000000000000000000000000000000000000000000000000")
+
+func emptyCodeHash(): Web3Hash =
+  w3Hash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470")
+
+func emptyStorageHash(): Web3Hash =
+  w3Hash("0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421")
+
+proc verifyAccountProof(trustedStateRoot: Web3Hash, res: ProofResponse): MptProofVerificationResult =
+  let
+    key = toSeq(keccakHash(res.address.ethAddr).data)
+    value = rlp.encode(Account(
+        nonce: res.nonce.uint64,
+        balance: res.balance,
+        storageRoot: fromHex(Hash256, res.storageHash.toHex()),
+        codeHash: fromHex(Hash256, res.codeHash.toHex())))
+
+  verifyMptProof(
+    seq[seq[byte]](res.accountProof),
+    fromHex(KeccakHash, trustedStateRoot.toHex()),
+    key,
+    value)
+
+proc verifySlotProof(trustedStateRoot: Web3Hash, slot: StorageProof): MptProofVerificationResult =
+  let
+    key = toSeq(keccakHash(toBytesBE(slot.key)).data)
+    value = rlp.encode(slot.value)
+
+  verifyMptProof(
+    seq[seq[byte]](slot.proof),
+    fromHex(KeccakHash, trustedStateRoot.toHex()),
+    key,
+    value)
+
 proc persistFixtureBlock(chainDB: CoreDbRef) =
   let header = getBlockHeader4514995()
   # Manually inserting header to avoid any parent checks
@@ -80,6 +115,23 @@ proc setupEnv(com: CommonRef, signer, ks2: EthAddress, ctx: EthContext): TestEnv
   vmState.stateDB.setCode(ks2, code)
   vmState.stateDB.addBalance(signer, 9_000_000_000.u256)
 
+
+  # Test data created for eth_getProof tests
+  let regularAcc = ethAddr("0x0000000000000000000000000000000000000001")
+  vmState.stateDB.addBalance(regularAcc, 2_000_000_000.u256)
+  vmState.stateDB.setNonce(regularAcc, 1.uint64)
+
+  let contractAccWithStorage = ethAddr("0x0000000000000000000000000000000000000002")
+  vmState.stateDB.addBalance(contractAccWithStorage, 1_000_000_000.u256)
+  vmState.stateDB.setNonce(contractAccWithStorage, 2.uint64)
+  vmState.stateDB.setCode(contractAccWithStorage, code)
+  vmState.stateDB.setStorage(contractAccWithStorage, u256(0), u256(1234))
+  vmState.stateDB.setStorage(contractAccWithStorage, u256(1), u256(2345))
+
+  let contractAccNoStorage = ethAddr("0x0000000000000000000000000000000000000003")
+  vmState.stateDB.setCode(contractAccNoStorage, code)
+
+
   let
     unsignedTx1 = Transaction(
       txType  : TxLegacy,
@@ -526,6 +578,158 @@ proc rpcMain*() =
       check:
         len(logs) == 2
 
+    test "eth_getProof - Non existent account and storage slots":
+      let blockData = await client.eth_getBlockByNumber("latest", true)
+
+      block:
+        # account doesn't exist
+        let
+          address = w3Addr("0x0000000000000000000000000000000000000004")
+          proofResponse = await client.eth_getProof(address, @[], blockId(1'u64))
+          storageProof = proofResponse.storageProof
+
+        check:
+          proofResponse.address == address
+          verifyAccountProof(blockData.stateRoot, proofResponse).isMissing()
+          proofResponse.balance == 0.u256
+          proofResponse.codeHash == zeroHash()
+          proofResponse.nonce == w3Qty(0.uint64)
+          proofResponse.storageHash == zeroHash()
+          storageProof.len() == 0
+
+      block:
+        # account exists but requested slots don't exist
+        let
+          address = w3Addr("0x0000000000000000000000000000000000000001")
+          slot1Key = 0.u256
+          slot2Key = 1.u256
+          proofResponse = await client.eth_getProof(address, @[slot1Key, slot2Key], blockId(1'u64))
+          storageProof = proofResponse.storageProof
+
+        check:
+          proofResponse.address == address
+          verifyAccountProof(blockData.stateRoot, proofResponse).isValid()
+          proofResponse.balance == 2_000_000_000.u256
+          proofResponse.codeHash == emptyCodeHash()
+          proofResponse.nonce == w3Qty(1.uint64)
+          proofResponse.storageHash == emptyStorageHash()
+          storageProof.len() == 2
+          storageProof[0].key == slot1Key
+          storageProof[0].proof.len() == 0
+          storageProof[0].value == 0.u256
+          storageProof[1].key == slot2Key
+          storageProof[1].proof.len() == 0
+          storageProof[1].value == 0.u256
+
+      block:
+        # contract account with no storage slots
+        let
+          address = w3Addr("0x0000000000000000000000000000000000000003")
+          slot1Key = 0.u256 # Doesn't exist
+          proofResponse = await client.eth_getProof(address, @[slot1Key], blockId(1'u64))
+          storageProof = proofResponse.storageProof
+
+        check:
+          proofResponse.address == address
+          verifyAccountProof(blockData.stateRoot, proofResponse).isValid()
+          proofResponse.balance == 0.u256
+          proofResponse.codeHash == w3Hash("0x09044b55d7aba83cb8ac3d2c9c8d8bcadbfc33f06f1be65e8cc1e4ddab5f3074")
+          proofResponse.nonce == w3Qty(0.uint64)
+          proofResponse.storageHash == emptyStorageHash()
+          storageProof.len() == 1
+          storageProof[0].key == slot1Key
+          storageProof[0].proof.len() == 0
+          storageProof[0].value == 0.u256
+
+    test "eth_getProof - Existing accounts and storage slots":
+      let blockData = await client.eth_getBlockByNumber("latest", true)
+
+      block:
+        # contract account with storage slots
+        let
+          address = w3Addr("0x0000000000000000000000000000000000000002")
+          slot1Key = 0.u256
+          slot2Key = 1.u256
+          slot3Key = 2.u256 # Doesn't exist
+          proofResponse = await client.eth_getProof(address, @[slot1Key, slot2Key, slot3Key], blockId(1'u64))
+          storageProof = proofResponse.storageProof
+
+        check:
+          proofResponse.address == address
+          verifyAccountProof(blockData.stateRoot, proofResponse).isValid()
+          proofResponse.balance == 1_000_000_000.u256
+          proofResponse.codeHash == w3Hash("0x09044b55d7aba83cb8ac3d2c9c8d8bcadbfc33f06f1be65e8cc1e4ddab5f3074")
+          proofResponse.nonce == w3Qty(2.uint64)
+          proofResponse.storageHash == w3Hash("0x2ed06ec37dad4cd8c8fc1a1172d633a8973987fa6995b14a7c0a50c0e8d1a9c3")
+          storageProof.len() == 3
+          storageProof[0].key == slot1Key
+          storageProof[0].proof.len() > 0
+          storageProof[0].value == 1234.u256
+          storageProof[1].key == slot2Key
+          storageProof[1].proof.len() > 0
+          storageProof[1].value == 2345.u256
+          storageProof[2].key == slot3Key
+          storageProof[2].proof.len() > 0
+          storageProof[2].value == 0.u256
+          verifySlotProof(proofResponse.storageHash, storageProof[0]).isValid()
+          verifySlotProof(proofResponse.storageHash, storageProof[1]).isValid()
+          verifySlotProof(proofResponse.storageHash, storageProof[2]).isMissing()
+
+      block:
+        # externally owned account
+        let
+          address = w3Addr("0x0000000000000000000000000000000000000001")
+          proofResponse = await client.eth_getProof(address, @[], blockId(1'u64))
+          storageProof = proofResponse.storageProof
+
+        check:
+          proofResponse.address == address
+          verifyAccountProof(blockData.stateRoot, proofResponse).isValid()
+          proofResponse.balance == 2_000_000_000.u256
+          proofResponse.codeHash == emptyCodeHash()
+          proofResponse.nonce == w3Qty(1.uint64)
+          proofResponse.storageHash == emptyStorageHash()
+          storageProof.len() == 0
+
+    test "eth_getProof - Multiple blocks":
+      let blockData = await client.eth_getBlockByNumber("latest", true)
+
+      block:
+        # block 0 - account doesn't exist yet
+        let
+          address = w3Addr("0x0000000000000000000000000000000000000002")
+          slot1Key = 100.u256
+          proofResponse = await client.eth_getProof(address, @[slot1Key], blockId(0'u64))
+          storageProof = proofResponse.storageProof
+
+        check:
+          proofResponse.address == address
+          verifyAccountProof(blockData.stateRoot, proofResponse).kind == InvalidProof
+          proofResponse.balance == 0.u256
+          proofResponse.codeHash == zeroHash()
+          proofResponse.nonce == w3Qty(0.uint64)
+          proofResponse.storageHash == zeroHash()
+          storageProof.len() == 1
+          verifySlotProof(proofResponse.storageHash, storageProof[0]).kind == InvalidProof
+
+      block:
+        # block 1 - account has balance, code and storage
+        let
+          address = w3Addr("0x0000000000000000000000000000000000000002")
+          slot2Key = 1.u256
+          proofResponse = await client.eth_getProof(address, @[slot2Key], blockId(1'u64))
+          storageProof = proofResponse.storageProof
+
+        check:
+          proofResponse.address == address
+          verifyAccountProof(blockData.stateRoot, proofResponse).isValid()
+          proofResponse.balance == 1_000_000_000.u256
+          proofResponse.codeHash == w3Hash("0x09044b55d7aba83cb8ac3d2c9c8d8bcadbfc33f06f1be65e8cc1e4ddab5f3074")
+          proofResponse.nonce == w3Qty(2.uint64)
+          proofResponse.storageHash == w3Hash("0x2ed06ec37dad4cd8c8fc1a1172d633a8973987fa6995b14a7c0a50c0e8d1a9c3")
+          storageProof.len() == 1
+          verifySlotProof(proofResponse.storageHash, storageProof[0]).isValid()
+
     rpcServer.stop()
     rpcServer.close()