From 79c6bdc214d0f8167c900bc2d2161961dc4c5d54 Mon Sep 17 00:00:00 2001 From: web3-developer <51288821+web3-developer@users.noreply.github.com> Date: Tue, 9 Jan 2024 10:05:52 +0800 Subject: [PATCH] Implementation of Nimbus eth_getProof RPC Endpoint. (#1960) * Initial implementation of eth_getProof endpoint. * Implemented generation of account and storage proofs. * Minor fixes and additional tests. * Refactor getBranch code into a separate file. * Improve usage of test data. * Fix copyright year. * Return zero hash for codeHash and storageHash if account doesn't exist. * Update copyright notice and moved trie key hashing to inside getBranch proc. --- nimbus/db/distinct_tries.nim | 2 +- nimbus/db/state_db.nim | 6 +- nimbus/db/state_db/base.nim | 31 ++++- nimbus/db/state_db/read_only.nim | 4 +- nimbus/db/trie_get_branch.nim | 97 ++++++++++++++ nimbus/rpc/p2p.nim | 45 ++++++- tests/test_rpc.nim | 216 ++++++++++++++++++++++++++++++- 7 files changed, 387 insertions(+), 14 deletions(-) create mode 100644 nimbus/db/trie_get_branch.nim diff --git a/nimbus/db/distinct_tries.nim b/nimbus/db/distinct_tries.nim index 342b678fe..015f0d807 100644 --- a/nimbus/db/distinct_tries.nim +++ b/nimbus/db/distinct_tries.nim @@ -1,5 +1,5 @@ # Nimbus -# Copyright (c) 2023 Status Research & Development GmbH +# Copyright (c) 2023-2024 Status Research & Development GmbH # Licensed under either of # * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or # http://www.apache.org/licenses/LICENSE-2.0) diff --git a/nimbus/db/state_db.nim b/nimbus/db/state_db.nim index 391f9b682..750a2d419 100644 --- a/nimbus/db/state_db.nim +++ b/nimbus/db/state_db.nim @@ -1,5 +1,5 @@ # Nimbus -# Copyright (c) 2018-2023 Status Research & Development GmbH +# Copyright (c) 2018-2024 Status Research & Development GmbH # Licensed under either of # * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0) # * MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT) @@ -29,6 +29,8 @@ export isDeadAccount, isEmptyAccount, newAccountStateDB, - rootHash + rootHash, + getAccountProof, + getStorageProof # End diff --git a/nimbus/db/state_db/base.nim b/nimbus/db/state_db/base.nim index 90819cc25..67464fc7a 100644 --- a/nimbus/db/state_db/base.nim +++ b/nimbus/db/state_db/base.nim @@ -1,5 +1,5 @@ # Nimbus -# Copyright (c) 2018-2023 Status Research & Development GmbH +# Copyright (c) 2018-2024 Status Research & Development GmbH # Licensed under either of # * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0) # * MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT) @@ -8,10 +8,10 @@ import std/[sets, strformat], chronicles, - eth/[common, rlp], + eth/[common, rlp, trie/trie_defs], ../../constants, ../../utils/utils, - ".."/[core_db, distinct_tries, storage_types] + ".."/[core_db, distinct_tries, storage_types, trie_get_branch] logScope: topics = "state_db" @@ -55,6 +55,10 @@ type when aleth_compat: cleared: HashSet[EthAddress] + MptNodeRlpBytes* = seq[byte] + AccountProof* = seq[MptNodeRlpBytes] + SlotProof* = seq[MptNodeRlpBytes] + proc pruneTrie*(db: AccountStateDB): bool = db.trie.isPruning @@ -249,6 +253,27 @@ proc isDeadAccount*(db: AccountStateDB, address: EthAddress): bool = else: result = true +proc removeEmptyRlpNode(branch: var seq[MptNodeRlpBytes]) = + if branch.len() == 1 and branch[0] == emptyRlp: + branch.del(0) + +proc getAccountProof*(db: AccountStateDB, address: EthAddress): AccountProof = + var branch = db.trie.phk().getBranch(address) + removeEmptyRlpNode(branch) + branch + +proc getStorageProof*(db: AccountStateDB, address: EthAddress, slots: seq[UInt256]): seq[SlotProof] = + var account = db.getAccount(address) + var storageTrie = db.getStorageTrie(account) + + var slotProofs = newSeqOfCap[SlotProof](slots.len()) + for slot in slots: + var branch = storageTrie.phk().getBranch(createTrieKeyFromSlot(slot)) + removeEmptyRlpNode(branch) + slotProofs.add(branch) + + slotProofs + # Note: `state_db.getCommittedStorage()` is nowhere used. # #proc getCommittedStorage*(db: AccountStateDB, address: EthAddress, slot: UInt256): UInt256 = diff --git a/nimbus/db/state_db/read_only.nim b/nimbus/db/state_db/read_only.nim index 6a9eed13f..9f53b1213 100644 --- a/nimbus/db/state_db/read_only.nim +++ b/nimbus/db/state_db/read_only.nim @@ -1,5 +1,5 @@ # Nimbus -# Copyright (c) 2018-2023 Status Research & Development GmbH +# Copyright (c) 2018-2024 Status Research & Development GmbH # Licensed under either of # * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) # or http://www.apache.org/licenses/LICENSE-2.0) @@ -28,6 +28,8 @@ proc hasCodeOrNonce*(db: ReadOnlyStateDB, address: EthAddress): bool {.borrow.} proc accountExists*(db: ReadOnlyStateDB, address: EthAddress): bool {.borrow.} proc isDeadAccount*(db: ReadOnlyStateDB, address: EthAddress): bool {.borrow.} proc isEmptyAccount*(db: ReadOnlyStateDB, address: EthAddress): bool {.borrow.} +proc getAccountProof*(db: ReadOnlyStateDB, address: EthAddress): AccountProof {.borrow.} +proc getStorageProof*(db: ReadOnlyStateDB, address: EthAddress, slots: seq[UInt256]): seq[SlotProof] {.borrow.} #proc getCommittedStorage*(db: ReadOnlyStateDB, address: EthAddress, slot: UInt256): UInt256 {.borrow.} # End diff --git a/nimbus/db/trie_get_branch.nim b/nimbus/db/trie_get_branch.nim new file mode 100644 index 000000000..3753f4546 --- /dev/null +++ b/nimbus/db/trie_get_branch.nim @@ -0,0 +1,97 @@ +# Nimbus +# Copyright (c) 2023-2024 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or +# http://www.apache.org/licenses/LICENSE-2.0) +# * MIT license ([LICENSE-MIT](LICENSE-MIT) or +# http://opensource.org/licenses/MIT) +# at your option. This file may not be copied, modified, or distributed except +# according to those terms. + +# This implementation of getBranch on the CoreDbPhkRef type is a temporary solution +# which can be removed once we get an equivient proc defined on the CoreDbPhkRef type +# in the db layer. + +{.push raises: [].} + +import + eth/[rlp, trie/nibbles], + "."/[core_db] + +type + TrieNodeKey = object + hash: KeccakHash + usedBytes: uint8 + +template len(key: TrieNodeKey): int = + key.usedBytes.int + +template asDbKey(k: TrieNodeKey): untyped = + doAssert k.usedBytes == 32 + k.hash.data + +template extensionNodeKey(r: Rlp): auto = + hexPrefixDecode r.listElem(0).toBytes + +proc getLocalBytes(x: TrieNodeKey): seq[byte] = + ## This proc should be used on nodes using the optimization + ## of short values within the key. + doAssert x.usedBytes < 32 + x.hash.data[0.. ProofResponse: + ## Returns information about an account and storage slots (if the account is a contract + ## and the slots are requested) along with account and storage proofs which prove the + ## existence of the values in the state. + ## See spec here: https://eips.ethereum.org/EIPS/eip-1186 + ## + ## data: address of the account. + ## slots: integers of the positions in the storage to return with storage proofs. + ## quantityTag: integer block number, or the string "latest", "earliest" or "pending", see the default block parameter. + ## Returns: the proof response containing the account, account proof and storage proof + + let + accDB = stateDBFromTag(quantityTag) + address = data.ethAddr + acc = accDB.getAccount(address) + accExists = accDB.accountExists(address) + accountProof = accDB.getAccountProof(address) + slotProofs = accDB.getStorageProof(address, slots) + + var storage = newSeqOfCap[StorageProof](slots.len) + + for i, slotKey in slots: + let (slotValue, _) = accDB.getStorage(address, u256(slotKey)) + storage.add(StorageProof( + key: u256(slotKey), + value: slotValue, + proof: seq[RlpEncodedBytes](slotProofs[i]))) + + if accExists: + ProofResponse( + address: w3Addr(address), + accountProof: seq[RlpEncodedBytes](accountProof), + balance: acc.balance, + nonce: w3Qty(acc.nonce), + codeHash: w3Hash(acc.codeHash), + storageHash: w3Hash(acc.storageRoot), + storageProof: storage) + else: + ProofResponse( + address: w3Addr(address), + accountProof: seq[RlpEncodedBytes](accountProof), + storageProof: storage) + #[ server.rpc("eth_newFilter") do(filterOptions: FilterOptions) -> int: ## Creates a filter object, based on filter options, to notify when the state changes (logs). diff --git a/tests/test_rpc.nim b/tests/test_rpc.nim index 278cb752d..29e466723 100644 --- a/tests/test_rpc.nim +++ b/tests/test_rpc.nim @@ -1,18 +1,18 @@ # Nimbus -# Copyright (c) 2018-2023 Status Research & Development GmbH +# Copyright (c) 2018-2024 Status Research & Development GmbH # Licensed under either of # * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0) # * MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT) # at your option. This file may not be copied, modified, or distributed except according to those terms. import - std/[json, tables, os, typetraits, times], + std/[json, os, typetraits, times, sequtils], asynctest, web3/eth_api, - nimcrypto/[hash], stew/byteutils, + stew/byteutils, json_rpc/[rpcserver, rpcclient], - eth/[rlp, keys, p2p/private/p2p_types], - ../nimbus/[constants, transaction, config, - vm_state, vm_types, version], + nimcrypto/[keccak, hash], + eth/[rlp, keys, trie/hexary_proof_verification], + ../nimbus/[constants, transaction, config, vm_state, vm_types, version], ../nimbus/db/[ledger, storage_types], ../nimbus/sync/protocol, ../nimbus/core/[tx_pool, chain, executor, executor/executor_helpers, pow/difficulty], @@ -47,6 +47,41 @@ func w3Addr(x: string): Web3Address = func w3Hash(x: string): Web3Hash = Web3Hash hexToByteArray[32](x) +func zeroHash(): Web3Hash = + w3Hash("0x0000000000000000000000000000000000000000000000000000000000000000") + +func emptyCodeHash(): Web3Hash = + w3Hash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470") + +func emptyStorageHash(): Web3Hash = + w3Hash("0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421") + +proc verifyAccountProof(trustedStateRoot: Web3Hash, res: ProofResponse): MptProofVerificationResult = + let + key = toSeq(keccakHash(res.address.ethAddr).data) + value = rlp.encode(Account( + nonce: res.nonce.uint64, + balance: res.balance, + storageRoot: fromHex(Hash256, res.storageHash.toHex()), + codeHash: fromHex(Hash256, res.codeHash.toHex()))) + + verifyMptProof( + seq[seq[byte]](res.accountProof), + fromHex(KeccakHash, trustedStateRoot.toHex()), + key, + value) + +proc verifySlotProof(trustedStateRoot: Web3Hash, slot: StorageProof): MptProofVerificationResult = + let + key = toSeq(keccakHash(toBytesBE(slot.key)).data) + value = rlp.encode(slot.value) + + verifyMptProof( + seq[seq[byte]](slot.proof), + fromHex(KeccakHash, trustedStateRoot.toHex()), + key, + value) + proc persistFixtureBlock(chainDB: CoreDbRef) = let header = getBlockHeader4514995() # Manually inserting header to avoid any parent checks @@ -80,6 +115,23 @@ proc setupEnv(com: CommonRef, signer, ks2: EthAddress, ctx: EthContext): TestEnv vmState.stateDB.setCode(ks2, code) vmState.stateDB.addBalance(signer, 9_000_000_000.u256) + + # Test data created for eth_getProof tests + let regularAcc = ethAddr("0x0000000000000000000000000000000000000001") + vmState.stateDB.addBalance(regularAcc, 2_000_000_000.u256) + vmState.stateDB.setNonce(regularAcc, 1.uint64) + + let contractAccWithStorage = ethAddr("0x0000000000000000000000000000000000000002") + vmState.stateDB.addBalance(contractAccWithStorage, 1_000_000_000.u256) + vmState.stateDB.setNonce(contractAccWithStorage, 2.uint64) + vmState.stateDB.setCode(contractAccWithStorage, code) + vmState.stateDB.setStorage(contractAccWithStorage, u256(0), u256(1234)) + vmState.stateDB.setStorage(contractAccWithStorage, u256(1), u256(2345)) + + let contractAccNoStorage = ethAddr("0x0000000000000000000000000000000000000003") + vmState.stateDB.setCode(contractAccNoStorage, code) + + let unsignedTx1 = Transaction( txType : TxLegacy, @@ -526,6 +578,158 @@ proc rpcMain*() = check: len(logs) == 2 + test "eth_getProof - Non existent account and storage slots": + let blockData = await client.eth_getBlockByNumber("latest", true) + + block: + # account doesn't exist + let + address = w3Addr("0x0000000000000000000000000000000000000004") + proofResponse = await client.eth_getProof(address, @[], blockId(1'u64)) + storageProof = proofResponse.storageProof + + check: + proofResponse.address == address + verifyAccountProof(blockData.stateRoot, proofResponse).isMissing() + proofResponse.balance == 0.u256 + proofResponse.codeHash == zeroHash() + proofResponse.nonce == w3Qty(0.uint64) + proofResponse.storageHash == zeroHash() + storageProof.len() == 0 + + block: + # account exists but requested slots don't exist + let + address = w3Addr("0x0000000000000000000000000000000000000001") + slot1Key = 0.u256 + slot2Key = 1.u256 + proofResponse = await client.eth_getProof(address, @[slot1Key, slot2Key], blockId(1'u64)) + storageProof = proofResponse.storageProof + + check: + proofResponse.address == address + verifyAccountProof(blockData.stateRoot, proofResponse).isValid() + proofResponse.balance == 2_000_000_000.u256 + proofResponse.codeHash == emptyCodeHash() + proofResponse.nonce == w3Qty(1.uint64) + proofResponse.storageHash == emptyStorageHash() + storageProof.len() == 2 + storageProof[0].key == slot1Key + storageProof[0].proof.len() == 0 + storageProof[0].value == 0.u256 + storageProof[1].key == slot2Key + storageProof[1].proof.len() == 0 + storageProof[1].value == 0.u256 + + block: + # contract account with no storage slots + let + address = w3Addr("0x0000000000000000000000000000000000000003") + slot1Key = 0.u256 # Doesn't exist + proofResponse = await client.eth_getProof(address, @[slot1Key], blockId(1'u64)) + storageProof = proofResponse.storageProof + + check: + proofResponse.address == address + verifyAccountProof(blockData.stateRoot, proofResponse).isValid() + proofResponse.balance == 0.u256 + proofResponse.codeHash == w3Hash("0x09044b55d7aba83cb8ac3d2c9c8d8bcadbfc33f06f1be65e8cc1e4ddab5f3074") + proofResponse.nonce == w3Qty(0.uint64) + proofResponse.storageHash == emptyStorageHash() + storageProof.len() == 1 + storageProof[0].key == slot1Key + storageProof[0].proof.len() == 0 + storageProof[0].value == 0.u256 + + test "eth_getProof - Existing accounts and storage slots": + let blockData = await client.eth_getBlockByNumber("latest", true) + + block: + # contract account with storage slots + let + address = w3Addr("0x0000000000000000000000000000000000000002") + slot1Key = 0.u256 + slot2Key = 1.u256 + slot3Key = 2.u256 # Doesn't exist + proofResponse = await client.eth_getProof(address, @[slot1Key, slot2Key, slot3Key], blockId(1'u64)) + storageProof = proofResponse.storageProof + + check: + proofResponse.address == address + verifyAccountProof(blockData.stateRoot, proofResponse).isValid() + proofResponse.balance == 1_000_000_000.u256 + proofResponse.codeHash == w3Hash("0x09044b55d7aba83cb8ac3d2c9c8d8bcadbfc33f06f1be65e8cc1e4ddab5f3074") + proofResponse.nonce == w3Qty(2.uint64) + proofResponse.storageHash == w3Hash("0x2ed06ec37dad4cd8c8fc1a1172d633a8973987fa6995b14a7c0a50c0e8d1a9c3") + storageProof.len() == 3 + storageProof[0].key == slot1Key + storageProof[0].proof.len() > 0 + storageProof[0].value == 1234.u256 + storageProof[1].key == slot2Key + storageProof[1].proof.len() > 0 + storageProof[1].value == 2345.u256 + storageProof[2].key == slot3Key + storageProof[2].proof.len() > 0 + storageProof[2].value == 0.u256 + verifySlotProof(proofResponse.storageHash, storageProof[0]).isValid() + verifySlotProof(proofResponse.storageHash, storageProof[1]).isValid() + verifySlotProof(proofResponse.storageHash, storageProof[2]).isMissing() + + block: + # externally owned account + let + address = w3Addr("0x0000000000000000000000000000000000000001") + proofResponse = await client.eth_getProof(address, @[], blockId(1'u64)) + storageProof = proofResponse.storageProof + + check: + proofResponse.address == address + verifyAccountProof(blockData.stateRoot, proofResponse).isValid() + proofResponse.balance == 2_000_000_000.u256 + proofResponse.codeHash == emptyCodeHash() + proofResponse.nonce == w3Qty(1.uint64) + proofResponse.storageHash == emptyStorageHash() + storageProof.len() == 0 + + test "eth_getProof - Multiple blocks": + let blockData = await client.eth_getBlockByNumber("latest", true) + + block: + # block 0 - account doesn't exist yet + let + address = w3Addr("0x0000000000000000000000000000000000000002") + slot1Key = 100.u256 + proofResponse = await client.eth_getProof(address, @[slot1Key], blockId(0'u64)) + storageProof = proofResponse.storageProof + + check: + proofResponse.address == address + verifyAccountProof(blockData.stateRoot, proofResponse).kind == InvalidProof + proofResponse.balance == 0.u256 + proofResponse.codeHash == zeroHash() + proofResponse.nonce == w3Qty(0.uint64) + proofResponse.storageHash == zeroHash() + storageProof.len() == 1 + verifySlotProof(proofResponse.storageHash, storageProof[0]).kind == InvalidProof + + block: + # block 1 - account has balance, code and storage + let + address = w3Addr("0x0000000000000000000000000000000000000002") + slot2Key = 1.u256 + proofResponse = await client.eth_getProof(address, @[slot2Key], blockId(1'u64)) + storageProof = proofResponse.storageProof + + check: + proofResponse.address == address + verifyAccountProof(blockData.stateRoot, proofResponse).isValid() + proofResponse.balance == 1_000_000_000.u256 + proofResponse.codeHash == w3Hash("0x09044b55d7aba83cb8ac3d2c9c8d8bcadbfc33f06f1be65e8cc1e4ddab5f3074") + proofResponse.nonce == w3Qty(2.uint64) + proofResponse.storageHash == w3Hash("0x2ed06ec37dad4cd8c8fc1a1172d633a8973987fa6995b14a7c0a50c0e8d1a9c3") + storageProof.len() == 1 + verifySlotProof(proofResponse.storageHash, storageProof[0]).isValid() + rpcServer.stop() rpcServer.close()