diff --git a/nimbus/rpc/TODO-P2P.md b/nimbus/rpc/TODO-P2P.md index 285185288..1b2965dfe 100644 --- a/nimbus/rpc/TODO-P2P.md +++ b/nimbus/rpc/TODO-P2P.md @@ -1,5 +1,5 @@ After change to the *Aristo* single state database, the proof logic in -*p2p.nim* and *experimental.nim* is not supported anymore. +*p2p.nim* is not supported anymore. The proof logic in question refers to functions *state_db.getAccountProof()* and *state_db.getStorageProof()* which used now unsupported internal access diff --git a/nimbus/rpc/experimental.nim b/nimbus/rpc/experimental.nim index 27926ca77..55e57ceb6 100644 --- a/nimbus/rpc/experimental.nim +++ b/nimbus/rpc/experimental.nim @@ -9,99 +9,9 @@ {.push raises: [].} -import - std/[typetraits], - json_rpc/rpcserver, - web3/conversions, - eth/p2p, - stint, - ../core/executor/process_block, - ../[transaction, constants], - ../beacon/web3_eth_conv, - ../stateless/multi_keys, - ../evm/[state, types], - ../common/common, - ../utils/utils, - ../db/ledger, - ./rpc_types, - ./rpc_utils, - ./filters, - ./p2p - -type - BlockHeader = eth_types.BlockHeader - -proc getMultiKeys*( - com: CommonRef, - blockHeader: BlockHeader, - statePostExecution: bool): MultiKeysRef - {.raises: [BlockNotFound, ValueError].} = - - let - chainDB = com.db - blk = chainDB.getEthBlock(blockHeader.number) - # Initializing the VM will throw a Defect if the state doesn't exist. - # Once we enable pruning we will need to check if the block state has been pruned - # before trying to initialize the VM as we do here. - vmState = BaseVMState.new(blk.header, com).valueOr: - raise newException(ValueError, "Cannot create vm state") - - vmState.collectWitnessData = true # Enable saving witness data - vmState.com.hardForkTransition(blockHeader) - - let dbTx = vmState.com.db.ctx.newTransaction() - defer: dbTx.dispose() - - # Execute the block of transactions and collect the keys of the touched account state - processBlock(vmState, blk).expect("success") - - let mkeys = vmState.stateDB.makeMultiKeys() - - dbTx.rollback() - - mkeys - -proc getBlockProofs*( - accDB: LedgerRef, - mkeys: MultiKeysRef): seq[ProofResponse] = - - var blockProofs = newSeq[ProofResponse]() - - for keyData in mkeys.keys: - let address = keyData.address - var slots = newSeq[UInt256]() - - if not keyData.storageKeys.isNil and accDB.accountExists(address): - for slotData in keyData.storageKeys.keys: - slots.add(fromBytesBE(UInt256, slotData.storageSlot)) - - blockProofs.add(getProof(accDB, address, slots)) - - return blockProofs +import json_rpc/rpcserver, ../constants, ../common/common proc setupExpRpc*(com: CommonRef, server: RpcServer) = + # Currently no experimental endpoints - let chainDB = com.db - - proc getStateDB(header: BlockHeader): LedgerRef = - ## Retrieves the account db from canonical head - # we don't use accounst_cache here because it's only read operations - LedgerRef.init(chainDB, header.stateRoot) - - server.rpc("exp_getProofsByBlockNumber") do(quantityTag: BlockTag, statePostExecution: bool) -> seq[ProofResponse]: - ## Returns the block proofs for a block by block number or tag. - ## - ## quantityTag: integer of a block number, or the string "earliest", "latest" or "pending", as in the default block parameter. - ## statePostExecution: bool which indicates whether to return the proofs based on the state before or after executing the block. - ## Returns seq[ProofResponse] - - let - blockHeader = chainDB.headerFromTag(quantityTag) - mkeys = getMultiKeys(com, blockHeader, statePostExecution) - - let accDB = if statePostExecution: - getStateDB(blockHeader) - else: - getStateDB(chainDB.getBlockHeader(blockHeader.parentHash)) - - return getBlockProofs(accDB, mkeys) + discard diff --git a/tests/all_tests.nim b/tests/all_tests.nim index c7e93a788..7b01b67a2 100644 --- a/tests/all_tests.nim +++ b/tests/all_tests.nim @@ -40,6 +40,5 @@ cliBuilder: ./test_eip4844, ./test_beacon/test_skeleton, #./test_getproof_json, -- fails - #./test_rpc_experimental_json, -- fails ./test_aristo, ./test_coredb diff --git a/tests/rpc/experimental_rpc_client.nim b/tests/rpc/experimental_rpc_client.nim deleted file mode 100644 index 5ec2bcc44..000000000 --- a/tests/rpc/experimental_rpc_client.nim +++ /dev/null @@ -1,20 +0,0 @@ -# Nimbus -# Copyright (c) 2024 Status Research & Development GmbH -# Licensed and distributed under either of -# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). -# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). -# at your option. This file may not be copied, modified, or distributed except according to those terms. - -import - json_rpc/rpcclient, - json_rpc/errors, - web3/eth_api - -export - rpcclient, - errors, - eth_api_types, - conversions - -createRpcSigsFromNim(RpcClient): - proc exp_getProofsByBlockNumber(blockId: BlockIdentifier, statePostExecution: bool): seq[ProofResponse] diff --git a/tests/test_rpc_experimental_json.nim b/tests/test_rpc_experimental_json.nim deleted file mode 100644 index aa1973480..000000000 --- a/tests/test_rpc_experimental_json.nim +++ /dev/null @@ -1,244 +0,0 @@ -# Nimbus -# Copyright (c) 2024 Status Research & Development GmbH -# Licensed under either of -# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0) -# * MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT) -# at your option. This file may not be copied, modified, or distributed except according to those terms. - -import - std/[json, os], - asynctest, - json_rpc/[rpcclient, rpcserver], - stew/byteutils, - ../nimbus/core/chain, - ../nimbus/common/common, - ../nimbus/rpc, - ../nimbus/db/[ledger, core_db], - ./rpc/experimental_rpc_client - -type - Hash256 = eth_types.Hash256 - -func ethAddr*(x: Address): EthAddress = - EthAddress x - -template toHash256(hash: untyped): Hash256 = - fromHex(Hash256, hash.toHex()) - -proc importBlockData(node: JsonNode): (CommonRef, Hash256, Hash256, UInt256) {. raises: [Exception].} = - var - blockNumber = UInt256.fromHex(node["blockNumber"].getStr()) - memoryDB = newCoreDbRef DefaultDbMemory - config = chainConfigForNetwork(MainNet) - com = CommonRef.new(memoryDB, config) - state = node["state"] - - for k, v in state: - let key = hexToSeqByte(k) - let value = hexToSeqByte(v.getStr()) - memoryDB.kvt.put(key, value) - - let - parentNumber = blockNumber - 1 - parent = com.db.getBlockHeader(parentNumber) - blk = com.db.getEthBlock(blockNumber) - chain = newChain(com) - - # it's ok if setHead fails here because of missing ancestors - discard com.db.setHead(parent, true) - let validationResult = chain.persistBlocks([blk]) - doAssert validationResult.isOk() - - return (com, parent.stateRoot, blk.header.stateRoot, blockNumber) - -proc checkAndValidateProofs( - db: CoreDbRef, - parentStateRoot: KeccakHash, - expectedStateRoot: KeccakHash, - proofs: seq[ProofResponse]) = - - let - stateDB = LedgerRef.init(db, parentStateRoot) - - check: - proofs.len() > 0 - - for proof in proofs: - let - address = proof.address.ethAddr() - balance = proof.balance - nonce = proof.nonce.uint64 - codeHash = proof.codeHash.toHash256() - storageHash = proof.storageHash.toHash256() - slotProofs = proof.storageProof - - # TODO: Fix this test. Update the code by checking if codeHash has changed - # and calling eth_getCode to set the updated code in the stateDB - # if code.len() > 0: - # stateDB.setCode(address, code) - - stateDB.setBalance(address, balance) - stateDB.setNonce(address, nonce) - - for slotProof in slotProofs: - stateDB.setStorage(address, slotProof.key, slotProof.value) - - # the account doesn't exist due to a self destruct - if codeHash == ZERO_HASH256 and storageHash == ZERO_HASH256: - stateDB.deleteAccount(address) - - stateDB.persist() - - check stateDB.getBalance(address) == balance - check stateDB.getNonce(address) == nonce - - if codeHash == ZERO_HASH256 or codeHash == EMPTY_CODE_HASH: - check stateDB.getCode(address).len() == 0 - check stateDB.getCodeHash(address) == EMPTY_CODE_HASH - else: - check stateDB.getCodeHash(address) == codeHash - - if storageHash == ZERO_HASH256 or storageHash == EMPTY_ROOT_HASH: - check stateDB.getStorageRoot(address) == EMPTY_ROOT_HASH - else: - check stateDB.getStorageRoot(address) == storageHash - - check stateDB.rootHash == expectedStateRoot - -proc importBlockDataFromFile(file: string): (CommonRef, Hash256, Hash256, UInt256) {. raises: [].} = - try: - let - fileJson = json.parseFile("tests" / "fixtures" / "PersistBlockTests" / file) - return importBlockData(fileJson) - except Exception as ex: - doAssert false, ex.msg - -proc rpcExperimentalJsonMain*() = - - suite "rpc experimental json tests": - - let importFiles = [ - "block97.json", - "block98.json", - "block46147.json", - "block46400.json", - "block46402.json", - "block47205.json", - "block47216.json", - "block48712.json", - "block48915.json", - "block49018.json", - "block49439.json", - "block49891.json", - "block50111.json", - "block78458.json", - "block81383.json", - "block81666.json", - "block85858.json", - "block146675.json", - "block116524.json", - "block196647.json", - "block226147.json", - "block226522.json", - "block231501.json", - "block243826.json", - "block248032.json", - "block299804.json", - "block420301.json", - "block512335.json", - "block652148.json", - "block668910.json", - "block1017395.json", - "block1149150.json", - "block1155095.json", - "block1317742.json", - "block1352922.json", - "block1368834.json", - "block1417555.json", - "block1431916.json", - "block1487668.json", - "block1920000.json", - "block1927662.json", - "block2463413.json", - "block2675000.json", - "block2675002.json", - "block4370000.json" - ] - - let - RPC_HOST = "127.0.0.1" - RPC_PORT = 0 # let the OS choose a port - - var - rpcServer = newRpcHttpServerWithParams(initTAddress(RPC_HOST, RPC_PORT)).valueOr: - echo "Failed to create RPC server: ", error - quit(QuitFailure) - client = newRpcHttpClient() - - rpcServer.start() - waitFor client.connect(RPC_HOST, rpcServer.localAddress[0].port, secure = false) - - - test "exp_getProofsByBlockNumber - latest block pre-execution state": - for file in importFiles: - let (com, parentStateRoot, _, _) = importBlockDataFromFile(file) - - setupExpRpc(com, rpcServer) - - let proofs = await client.exp_getProofsByBlockNumber("latest", false) - - checkAndValidateProofs(com.db, parentStateRoot, parentStateRoot, proofs) - - test "exp_getProofsByBlockNumber - latest block post-execution state": - for file in importFiles: - let (com, parentStateRoot, stateRoot, _) = importBlockDataFromFile(file) - - setupExpRpc(com, rpcServer) - - let proofs = await client.exp_getProofsByBlockNumber("latest", true) - - checkAndValidateProofs(com.db, parentStateRoot, stateRoot, proofs) - - test "exp_getProofsByBlockNumber - block by number pre-execution state": - for file in importFiles: - let - (com, parentStateRoot, _, blockNumber) = importBlockDataFromFile(file) - blockNum = blockId(blockNumber.truncate(uint64)) - - setupExpRpc(com, rpcServer) - - let proofs = await client.exp_getProofsByBlockNumber(blockNum, false) - - checkAndValidateProofs(com.db, parentStateRoot, parentStateRoot, proofs) - - test "exp_getProofsByBlockNumber - block by number post-execution state": - for file in importFiles: - let - (com, parentStateRoot, stateRoot, blockNumber) = importBlockDataFromFile(file) - blockNum = blockId(blockNumber.truncate(uint64)) - - setupExpRpc(com, rpcServer) - - let proofs = await client.exp_getProofsByBlockNumber(blockNum, true) - - checkAndValidateProofs(com.db, parentStateRoot, stateRoot, proofs) - - test "exp_getProofsByBlockNumber - block by number that doesn't exist": - for file in importFiles: - let - (com, _, _, blockNumber) = importBlockDataFromFile(file) - blockNum = blockId(blockNumber.truncate(uint64) + 1) # doesn't exist - - setupExpRpc(com, rpcServer) - - expect JsonRpcError: - discard await client.exp_getProofsByBlockNumber(blockNum, false) - - expect JsonRpcError: - discard await client.exp_getProofsByBlockNumber(blockNum, true) - - waitFor rpcServer.stop() - waitFor rpcServer.closeWait() - -when isMainModule: - rpcExperimentalJsonMain() diff --git a/tests/test_rpc_getproofs_track_state_changes.nim b/tests/test_rpc_getproofs_track_state_changes.nim deleted file mode 100644 index e6dda6730..000000000 --- a/tests/test_rpc_getproofs_track_state_changes.nim +++ /dev/null @@ -1,147 +0,0 @@ -# Nimbus -# Copyright (c) 2024 Status Research & Development GmbH -# Licensed under either of -# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0) -# * MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT) -# at your option. This file may not be copied, modified, or distributed except according to those terms. - -# This test is intended to be run manually against a running instance of Nimbus-Eth1 -# so it should not be added to the test runner or to the CI test suite. -# -# It uses the exp_getProofsByBlockNumber endpoint to get the list of state updates -# for each block, it then applies these updates against a local test state and -# then checks the state root against the expected state root which is pulled from -# the block header. The local test state is persisted to disk so that the data can -# be re-used between separate test runs. The default database directory is the -# current directory but this can be changed by setting the DATABASE_PATH const below. -# -# To run the test: -# 1. Sync Nimbus up to or past the block number/s that you wish to test against. -# You can use the premix persist tool to do this if Nimbus is not able to sync. -# 2. Start Nimbus with the http RPC-API enabled with the 'eth' and 'exp' namespaces -# turned on using this command: build/nimbus --rpc --rpc-api=eth,exp -# 3. Start the test. - -import - unittest2, - web3/eth_api, - json_rpc/rpcclient, - stew/byteutils, - ../nimbus/core/chain, - ../nimbus/common/common, - ../nimbus/rpc, - ../nimbus/db/opts, - ../nimbus/db/core_db, - ../nimbus/db/core_db/persistent, - ../nimbus/db/ledger, - ./rpc/experimental_rpc_client - -const - RPC_HOST = "127.0.0.1" - RPC_PORT = Port(8545) - DATABASE_PATH = "." - START_BLOCK = 330_000 - END_BLOCK = 1_000_000 - -type - Hash256 = eth_types.Hash256 - -template ethAddr*(x: Address): EthAddress = - EthAddress x - -template toHash256(hash: untyped): Hash256 = - fromHex(Hash256, hash.toHex()) - -proc updateStateUsingProofsAndCheckStateRoot( - stateDB: LedgerRef, - expectedStateRoot: Hash256, - proofs: seq[ProofResponse]) = - - check: - proofs.len() > 0 - - for proof in proofs: - let - address = proof.address.ethAddr() - balance = proof.balance - nonce = proof.nonce.uint64 - codeHash = proof.codeHash.toHash256() - storageHash = proof.storageHash.toHash256() - slotProofs = proof.storageProof - - if (balance == 0 and nonce == 0 and codeHash == ZERO_HASH256 and storageHash == ZERO_HASH256): - # Account doesn't exist: - # The account was deleted due to a self destruct and the data no longer exists in the state. - # The RPC API correctly returns zeroed values in this scenario which is the same behavior - # implemented by geth. - stateDB.setCode(address, @[]) - stateDB.clearStorage(address) - stateDB.deleteAccount(address) - elif (balance == 0 and nonce == 0 and codeHash == EMPTY_CODE_HASH and storageHash == EMPTY_ROOT_HASH): - # Account exists but is empty: - # The account was deleted due to a self destruct or the storage was cleared/set to zero - # and the bytecode is empty. - # The RPC API correctly returns codeHash == EMPTY_CODE_HASH and storageHash == EMPTY_ROOT_HASH - # in this scenario which is the same behavior implemented by geth. - stateDB.setCode(address, @[]) - stateDB.clearStorage(address) - stateDB.setBalance(address, 0.u256) - stateDB.setNonce(address, 0) - else: - # Account exists and is not empty: - stateDB.setBalance(address, balance) - stateDB.setNonce(address, nonce) - for slotProof in slotProofs: - stateDB.setStorage(address, slotProof.key, slotProof.value) - - check stateDB.getBalance(address) == balance - check stateDB.getNonce(address) == nonce - - if codeHash == ZERO_HASH256 or codeHash == EMPTY_CODE_HASH: - check stateDB.getCode(address).len() == 0 - check stateDB.getCodeHash(address) == EMPTY_CODE_HASH - else: - check stateDB.getCodeHash(address) == codeHash - - if storageHash == ZERO_HASH256 or storageHash == EMPTY_ROOT_HASH: - check stateDB.getStorageRoot(address) == EMPTY_ROOT_HASH - else: - check stateDB.getStorageRoot(address) == storageHash - - check stateDB.rootHash == expectedStateRoot - -proc rpcGetProofsTrackStateChangesMain*() = - - suite "rpc getProofs track state changes tests": - - let client = newRpcHttpClient() - waitFor client.connect(RPC_HOST, RPC_PORT, secure = false) - - test "Test tracking the changes introduced in every block": - - let com = CommonRef.new(newCoreDbRef( - DefaultDbPersistent, DATABASE_PATH, DbOptions.init())) - - let - blockHeader = waitFor client.eth_getBlockByNumber(blockId(START_BLOCK), false) - stateDB = LedgerRef.init(com.db, blockHeader.stateRoot.toHash256()) - - for i in START_BLOCK..END_BLOCK: - let - blockNum = blockId(i.uint64) - blockHeader: BlockObject = waitFor client.eth_getBlockByNumber(blockNum, false) - proofs = waitFor client.exp_getProofsByBlockNumber(blockNum, true) - - updateStateUsingProofsAndCheckStateRoot( - stateDB, - blockHeader.stateRoot.toHash256(), - proofs) - - if i mod 1000 == 0: - echo "Block number: ", i - echo "Expected block stateRoot: ", blockHeader.stateRoot - echo "Actual block stateRoot: ", stateDB.rootHash - doAssert blockHeader.stateRoot.toHash256() == stateDB.rootHash - -when isMainModule: - rpcGetProofsTrackStateChangesMain() diff --git a/tests/test_tools_build.nim b/tests/test_tools_build.nim index fcaf014e7..3df9a6478 100644 --- a/tests/test_tools_build.nim +++ b/tests/test_tools_build.nim @@ -9,7 +9,7 @@ # when try to test buildability of these tools. # They never run in the CI so it is ok to combine them -{. warning[UnusedImport]:off .} +{.warning[UnusedImport]: off.} import #../premix/premix, # -- currently disabled (no tracer at the moment) @@ -29,5 +29,4 @@ import ../tools/t8n/t8n_test, ../tools/t8n/t8n_debug, ../tools/evmstate/evmstate, - ../tools/evmstate/evmstate_test, - ./test_rpc_getproofs_track_state_changes + ../tools/evmstate/evmstate_test