From a6960c3d0a5aca7e5fd320d2a2553a6d332ac5fe Mon Sep 17 00:00:00 2001 From: andri lim Date: Mon, 17 Jun 2024 19:19:12 +0700 Subject: [PATCH] Enable test_accounts_cache (#2373) The module name is a misnomer, because AccountsCache have been replaced by LedgerRef. But the test still applicable. Instead of replaying unsupported goerli blocks, we generate our own transactions and block. --- tests/all_tests.nim | 4 +- tests/customgenesis/cancun123.json | 38 +++ tests/test_accounts_cache.nim | 390 --------------------------- tests/test_ledger.nim | 408 +++++++++++++++++++++++++++++ 4 files changed, 448 insertions(+), 392 deletions(-) create mode 100644 tests/customgenesis/cancun123.json delete mode 100644 tests/test_accounts_cache.nim create mode 100644 tests/test_ledger.nim diff --git a/tests/all_tests.nim b/tests/all_tests.nim index 530d7ebb8..9362a2991 100644 --- a/tests/all_tests.nim +++ b/tests/all_tests.nim @@ -11,7 +11,7 @@ import ./all_tests_macro cliBuilder: import ./test_code_stream, - #./test_accounts_cache, -- does not compile + ./test_ledger, ./test_jwt_auth, ./test_gas_meter, ./test_memory, @@ -34,7 +34,7 @@ cliBuilder: ./test_transaction_json, # TODO: some of test_blockchain_json's test cases failing # see issue #2260 - ./test_blockchain_json, + ./test_blockchain_json, ./test_forkid, ./test_multi_keys, ./test_misc, diff --git a/tests/customgenesis/cancun123.json b/tests/customgenesis/cancun123.json new file mode 100644 index 000000000..662441780 --- /dev/null +++ b/tests/customgenesis/cancun123.json @@ -0,0 +1,38 @@ +{ + "config": { + "chainId": 123, + "homesteadBlock": 0, + "eip150Block": 0, + "eip155Block": 0, + "eip158Block": 0, + "byzantiumBlock": 0, + "constantinopleBlock": 0, + "petersburgBlock": 0, + "istanbulBlock": 0, + "muirGlacierBlock": 0, + "berlinBlock": 0, + "londonBlock": 0, + "terminalTotalDifficulty": 0, + "terminalTotalDifficultyPassed": true, + "shanghaiTime": 0, + "cancunTime": 0 + }, + "genesis": { + "nonce": "0x42", + "timestamp": "0x0", + "extraData": "", + "gasLimit": "0x1C9C380", + "difficulty": "0x0", + "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "coinbase": "0x0000000000000000000000000000000000000000", + "alloc": { + "0x73cf19657412508833f618a15e8251306b3e6ee5": { + "balance": "0x6d6172697573766477000000" + } + }, + "number": "0x0", + "gasUsed": "0x0", + "parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "baseFeePerGas": "0x7" + } +} diff --git a/tests/test_accounts_cache.nim b/tests/test_accounts_cache.nim deleted file mode 100644 index d484021af..000000000 --- a/tests/test_accounts_cache.nim +++ /dev/null @@ -1,390 +0,0 @@ -# Nimbus -# Copyright (c) 2018-2024 Status Research & Development GmbH -# Licensed under either of -# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or -# http://www.apache.org/licenses/LICENSE-2.0) -# * MIT license ([LICENSE-MIT](LICENSE-MIT) or -# http://opensource.org/licenses/MIT) -# at your option. This file may not be copied, modified, or distributed except -# according to those terms. - -import - std/[os, strformat, strutils, tables], - chronicles, - stew/byteutils, - ../nimbus/db/ledger, - ../nimbus/common/common, - ../nimbus/core/chain, - ../nimbus/transaction, - ../nimbus/constants, - ../nimbus/evm/state, - ../nimbus/evm/types, - ./replay/undump_blocks, - unittest2 - -type - CaptureSpecs = tuple - network: NetworkId - file: string - numBlocks: int - numTxs: int - -const - baseDir = [".", "tests", ".." / "tests", $DirSep] # path containg repo - repoDir = ["replay", "status", "test_clique"] # alternative repo paths - - goerliCapture: CaptureSpecs = ( - network: GoerliNet, - file: "goerli68161.txt.gz", - numBlocks: 5500, # unconditionally load blocks - numTxs: 10) # txs following (not in block chain) - -when false: - goerliCapture1: CaptureSpecs = ( - GoerliNet, goerliCapture.file, 5500, 10000) - - mainCapture: CaptureSpecs = ( - MainNet, "mainnet843841.txt.gz", 50000, 3000) - -var - xdb: CoreDbRef - txs: seq[Transaction] - txi: seq[int] # selected index into txs[] (crashable sender addresses) - -# ------------------------------------------------------------------------------ -# Helpers -# ------------------------------------------------------------------------------ - -proc findFilePath(file: string): string = - result = "?unknown?" / file - for dir in baseDir: - for repo in repoDir: - let path = dir / repo / file - if path.fileExists: - return path - -proc pp*(a: EthAddress): string = - a.toHex[32 .. 39].toLowerAscii - -proc pp*(tx: Transaction): string = - # "(" & tx.ecRecover.value.pp & "," & $tx.nonce & ")" - "(" & tx.getSender.pp & "," & $tx.nonce & ")" - -proc pp*(h: KeccakHash): string = - h.data.toHex[52 .. 63].toLowerAscii - -proc pp*(tx: Transaction; vmState: BaseVMState): string = - let address = tx.getSender - "(" & address.pp & - "," & $tx.nonce & - ";" & $vmState.readOnlyStateDB.getNonce(address) & - "," & $vmState.readOnlyStateDB.getBalance(address) & - ")" - -when false: - proc setTraceLevel = - discard - when defined(chronicles_runtime_filtering) and loggingEnabled: - setLogLevel(LogLevel.TRACE) - - proc setErrorLevel = - discard - when defined(chronicles_runtime_filtering) and loggingEnabled: - setLogLevel(LogLevel.ERROR) - -# ------------------------------------------------------------------------------ -# Private functions -# ------------------------------------------------------------------------------ - -proc blockChainForTesting*(network: NetworkId): CommonRef = - result = CommonRef.new( - newCoreDbRef LegacyDbMemory, - networkId = network, - params = network.networkParams) - initializeEmptyDb(result) - -proc importBlocks(com: CommonRef; h: seq[BlockHeader]; b: seq[BlockBody]) = - if com.newChain.persistBlocks(h,b).isErr: - raiseAssert "persistBlocks() failed at block #" & $h[0].blockNumber - -proc getVmState(com: CommonRef; number: BlockNumber): BaseVMState = - BaseVMState.new(com.db.getBlockHeader(number), com) - -# ------------------------------------------------------------------------------ -# Crash test function, finding out about how the transaction framework works .. -# ------------------------------------------------------------------------------ - -proc modBalance(ac: LedgerRef, address: EthAddress) = - ## This function is crucial for profucing the crash. If must - ## modify the balance so that the database gets written. - # ac.blindBalanceSetter(address) - ac.addBalance(address, 1.u256) - - -proc runTrial2ok(vmState: BaseVMState; inx: int) = - ## Run two blocks, the first one with *rollback*. - let eAddr = txs[inx].getSender - - block: - let accTx = vmState.stateDB.beginSavepoint - vmState.stateDB.modBalance(eAddr) - vmState.stateDB.rollback(accTx) - - block: - let accTx = vmState.stateDB.beginSavepoint - vmState.stateDB.modBalance(eAddr) - vmState.stateDB.commit(accTx) - - vmState.stateDB.persist() - - -proc runTrial3(vmState: BaseVMState; inx: int; rollback: bool) = - ## Run three blocks, the second one optionally with *rollback*. - let eAddr = txs[inx].getSender - - block: - let accTx = vmState.stateDB.beginSavepoint - vmState.stateDB.modBalance(eAddr) - vmState.stateDB.commit(accTx) - vmState.stateDB.persist() - - block: - let accTx = vmState.stateDB.beginSavepoint - vmState.stateDB.modBalance(eAddr) - - if rollback: - vmState.stateDB.rollback(accTx) - break - - vmState.stateDB.commit(accTx) - vmState.stateDB.persist() - - block: - let accTx = vmState.stateDB.beginSavepoint - vmState.stateDB.modBalance(eAddr) - vmState.stateDB.commit(accTx) - vmState.stateDB.persist() - - -proc runTrial3crash(vmState: BaseVMState; inx: int; noisy = false) = - ## Run three blocks with extra db frames and *rollback*. - let eAddr = txs[inx].getSender - - block: - let dbTx = xdb.beginTransaction() - - block: - let accTx = vmState.stateDB.beginSavepoint - vmState.stateDB.modBalance(eAddr) - vmState.stateDB.commit(accTx) - vmState.stateDB.persist() - - block: - let accTx = vmState.stateDB.beginSavepoint - vmState.stateDB.modBalance(eAddr) - vmState.stateDB.rollback(accTx) - - # The following statement will cause a crash at the next `persist()` call. - dbTx.rollback() - - # In order to survive without an exception in the next `persist()` call, the - # following function could be added to `db/ledger`: - # - # proc clobberRootHash*(ac: LedgerRef; root: KeccakHash; prune = true) = - # ac.trie = initAccountsTrie(ac.db, rootHash, prune) - # - # Then, beginning this very function `runTrial3crash()` with - # - # let stateRoot = vmState.stateDB.rootHash - # - # the survival statement would be to re-assign the state-root via - # - # vmState.stateDB.clobberRootHash(stateRoot) - # - # Also mind this comment from Andri: - # - # [..] but as a reminder, only reinit the ac.trie is not enough, you - # should consider the accounts in the cache too. if there is any accounts - # in the cache they must in sync with the new rootHash. - # - block: - let dbTx = xdb.beginTransaction() - - block: - let accTx = vmState.stateDB.beginSavepoint - vmState.stateDB.modBalance(eAddr) - vmState.stateDB.commit(accTx) - - try: - vmState.stateDB.persist() - except AssertionDefect as e: - if noisy: - let msg = e.msg.rsplit($DirSep,1)[^1] - echo &"*** runVmExec({eAddr.pp}): {e.name}: {msg}" - dbTx.dispose() - raise e - - vmState.stateDB.persist() - - dbTx.commit() - - -proc runTrial4(vmState: BaseVMState; inx: int; rollback: bool) = - ## Like `runTrial3()` but with four blocks and extra db transaction frames. - let eAddr = txs[inx].getSender - - block: - let dbTx = xdb.beginTransaction() - - block: - let accTx = vmState.stateDB.beginSavepoint - vmState.stateDB.modBalance(eAddr) - vmState.stateDB.commit(accTx) - vmState.stateDB.persist() - - block: - let accTx = vmState.stateDB.beginSavepoint - vmState.stateDB.modBalance(eAddr) - vmState.stateDB.commit(accTx) - vmState.stateDB.persist() - - block: - let accTx = vmState.stateDB.beginSavepoint - vmState.stateDB.modBalance(eAddr) - - if rollback: - vmState.stateDB.rollback(accTx) - break - - vmState.stateDB.commit(accTx) - vmState.stateDB.persist() - - # There must be no dbTx.rollback() here unless `vmState.stateDB` is - # discarded and/or re-initialised. - dbTx.commit() - - block: - let dbTx = xdb.beginTransaction() - - block: - let accTx = vmState.stateDB.beginSavepoint - vmState.stateDB.modBalance(eAddr) - vmState.stateDB.commit(accTx) - vmState.stateDB.persist() - - dbTx.commit() - -# ------------------------------------------------------------------------------ -# Test Runner -# ------------------------------------------------------------------------------ - -proc runner(noisy = true; capture = goerliCapture) = - let - loadBlocks = capture.numBlocks.u256 - loadTxs = capture.numTxs - fileInfo = capture.file.splitFile.name.split(".")[0] - filePath = capture.file.findFilePath - com = capture.network.blockChainForTesting - - txs.reset - xdb = com.db - - suite &"StateDB nesting scenarios": - var topNumber: BlockNumber - - test &"Import from {fileInfo}": - # Import minimum amount of blocks, then collect transactions - for chain in filePath.undumpBlocks: - let leadBlkNum = chain[0].header.number - topNumber = chain[^1].header.number - - if loadTxs <= txs.len: - break - - # Verify Genesis - if leadBlkNum == 0.u256: - doAssert chain[0][0] == xdb.getBlockHeader(0.u256) - continue - - # Import block chain blocks - if leadBlkNum < loadBlocks: - com.importBlocks(chain) - continue - - # Import transactions - for inx in 0 ..< chain.len: - let blkTxs = chain[inx].transactions - - # Continue importing up until first non-trivial block - if txs.len == 0 and blkTxs.len == 0: - com.importBlocks([chain[inx]]) - continue - - # Load transactions - txs.add blkTxs - - - test &"Collect unique sender addresses from {txs.len} txs," & - &" head=#{xdb.getCanonicalHead.number}, top=#{topNumber}": - var seen: Table[EthAddress,bool] - for n,tx in txs: - let a = tx.getSender - if not seen.hasKey(a): - seen[a] = true - txi.add n - - test &"Run {txi.len} two-step trials with rollback": - let dbTx = xdb.beginTransaction() - defer: dbTx.dispose() - for n in txi: - let vmState = com.getVmState(xdb.getCanonicalHead.number) - vmState.runTrial2ok(n) - - test &"Run {txi.len} three-step trials with rollback": - let dbTx = xdb.beginTransaction() - defer: dbTx.dispose() - for n in txi: - let vmState = com.getVmState(xdb.getCanonicalHead.number) - vmState.runTrial3(n, rollback = true) - - test &"Run {txi.len} three-step trials with extra db frame rollback" & - " throwing Exceptions": - let dbTx = xdb.beginTransaction() - defer: dbTx.dispose() - for n in txi: - let vmState = com.getVmState(xdb.getCanonicalHead.number) - expect AssertionDefect: - vmState.runTrial3crash(n, noisy) - - test &"Run {txi.len} tree-step trials without rollback": - let dbTx = xdb.beginTransaction() - defer: dbTx.dispose() - for n in txi: - let vmState = com.getVmState(xdb.getCanonicalHead.number) - vmState.runTrial3(n, rollback = false) - - test &"Run {txi.len} four-step trials with rollback and db frames": - let dbTx = xdb.beginTransaction() - defer: dbTx.dispose() - for n in txi: - let vmState = com.getVmState(xdb.getCanonicalHead.number) - vmState.runTrial4(n, rollback = true) - -# ------------------------------------------------------------------------------ -# Main function(s) -# ------------------------------------------------------------------------------ - -proc accountsCacheMain*(noisy = defined(debug)) = - noisy.runner - -when isMainModule: - var noisy = defined(debug) - #noisy = true - - setErrorLevel() - noisy.runner # mainCapture - # noisy.runner goerliCapture2 - -# ------------------------------------------------------------------------------ -# End -# ------------------------------------------------------------------------------ diff --git a/tests/test_ledger.nim b/tests/test_ledger.nim new file mode 100644 index 000000000..97e600bb2 --- /dev/null +++ b/tests/test_ledger.nim @@ -0,0 +1,408 @@ +# Nimbus +# Copyright (c) 2018-2024 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or +# http://www.apache.org/licenses/LICENSE-2.0) +# * MIT license ([LICENSE-MIT](LICENSE-MIT) or +# http://opensource.org/licenses/MIT) +# at your option. This file may not be copied, modified, or distributed except +# according to those terms. + +import + std/[strformat, strutils], + eth/keys, + stew/byteutils, + stew/endians2, + ../nimbus/config, + ../nimbus/db/ledger, + ../nimbus/common/common, + ../nimbus/core/chain, + ../nimbus/core/tx_pool, + ../nimbus/core/casper, + ../nimbus/transaction, + ../nimbus/constants, + unittest2 + +const + genesisFile = "tests/customgenesis/cancun123.json" + hexPrivKey = "af1a9be9f1a54421cac82943820a0fe0f601bb5f4f6d0bccc81c613f0ce6ae22" + senderAddr = hexToByteArray[20]("73cf19657412508833f618a15e8251306b3e6ee5") + +type + TestEnv = object + com: CommonRef + xdb: CoreDbRef + txs: seq[Transaction] + txi: seq[int] # selected index into txs[] (crashable sender addresses) + vaultKey: PrivateKey + nonce : uint64 + chainId : ChainId + xp : TxPoolRef + chain : ChainRef + +# ------------------------------------------------------------------------------ +# Helpers +# ------------------------------------------------------------------------------ + +proc pp*(a: EthAddress): string = + a.toHex[32 .. 39].toLowerAscii + +proc pp*(tx: Transaction): string = + # "(" & tx.ecRecover.value.pp & "," & $tx.nonce & ")" + "(" & tx.getSender.pp & "," & $tx.nonce & ")" + +proc pp*(h: KeccakHash): string = + h.data.toHex[52 .. 63].toLowerAscii + +proc pp*(tx: Transaction; ledger: LedgerRef): string = + let address = tx.getSender + "(" & address.pp & + "," & $tx.nonce & + ";" & $ledger.getNonce(address) & + "," & $ledger.getBalance(address) & + ")" + +when isMainModule: + import chronicles + + proc setTraceLevel = + discard + when defined(chronicles_runtime_filtering) and loggingEnabled: + setLogLevel(LogLevel.TRACE) + + proc setErrorLevel = + discard + when defined(chronicles_runtime_filtering) and loggingEnabled: + setLogLevel(LogLevel.ERROR) + +# ------------------------------------------------------------------------------ +# Private functions +# ------------------------------------------------------------------------------ +proc privKey(keyHex: string): PrivateKey = + let kRes = PrivateKey.fromHex(keyHex) + if kRes.isErr: + echo kRes.error + quit(QuitFailure) + + kRes.get() + +proc initEnv(): TestEnv = + let + conf = makeConfig(@[ + "--custom-network:" & genesisFile + ]) + + let + com = CommonRef.new( + newCoreDbRef DefaultDbMemory, + conf.networkId, + conf.networkParams + ) + com.initializeEmptyDb() + + TestEnv( + com : com, + xdb : com.db, + vaultKey: privKey(hexPrivKey), + nonce : 0'u64, + chainId : conf.networkParams.config.chainId, + xp : TxPoolRef.new(com), + chain : newChain(com), + ) + +func makeTx( + env: var TestEnv, + recipient: EthAddress, + amount: UInt256, + payload: openArray[byte] = []): Transaction = + const + gasLimit = 75000.GasInt + gasPrice = 30.gwei + + let tx = Transaction( + txType : TxLegacy, + chainId : env.chainId, + nonce : AccountNonce(env.nonce), + gasPrice: gasPrice, + gasLimit: gasLimit, + to : Opt.some(recipient), + value : amount, + payload : @payload + ) + + inc env.nonce + signTransaction(tx, env.vaultKey, env.chainId, eip155 = true) + +func initAddr(z: int): EthAddress = + const L = sizeof(result) + result[L-sizeof(uint32)..^1] = toBytesBE(z.uint32) + +proc importBlocks(env: TestEnv; blk: EthBlock) = + let res = env.chain.persistBlocks([blk]) + if res.isErr: + debugEcho res.error + raiseAssert "persistBlocks() failed at block #" & $blk.header.number + +proc getLedger(com: CommonRef; header: BlockHeader): LedgerRef = + LedgerRef.init(com.db, header.stateRoot) + +func getRecipient(tx: Transaction): EthAddress = + tx.to.expect("transaction have no recipient") + +# ------------------------------------------------------------------------------ +# Crash test function, finding out about how the transaction framework works .. +# ------------------------------------------------------------------------------ + +proc modBalance(ac: LedgerRef, address: EthAddress) = + ## This function is crucial for profucing the crash. If must + ## modify the balance so that the database gets written. + # ac.blindBalanceSetter(address) + ac.addBalance(address, 1.u256) + + +proc runTrial2ok(env: TestEnv, ledger: LedgerRef; inx: int) = + ## Run two blocks, the first one with *rollback*. + let eAddr = env.txs[inx].getRecipient + + block: + let accTx = ledger.beginSavepoint + ledger.modBalance(eAddr) + ledger.rollback(accTx) + + block: + let accTx = ledger.beginSavepoint + ledger.modBalance(eAddr) + ledger.commit(accTx) + + ledger.persist() + + +proc runTrial3(env: TestEnv, ledger: LedgerRef; inx: int; rollback: bool) = + ## Run three blocks, the second one optionally with *rollback*. + let eAddr = env.txs[inx].getRecipient + + block: + let accTx = ledger.beginSavepoint + ledger.modBalance(eAddr) + ledger.commit(accTx) + ledger.persist() + + block: + let accTx = ledger.beginSavepoint + ledger.modBalance(eAddr) + + if rollback: + ledger.rollback(accTx) + break + + ledger.commit(accTx) + ledger.persist() + + block: + let accTx = ledger.beginSavepoint + ledger.modBalance(eAddr) + ledger.commit(accTx) + ledger.persist() + + +proc runTrial3Survive(env: TestEnv, ledger: LedgerRef; inx: int; noisy = false) = + ## Run three blocks with extra db frames and *rollback*. + let eAddr = env.txs[inx].getRecipient + + block: + let dbTx = env.xdb.newTransaction() + + block: + let accTx = ledger.beginSavepoint + ledger.modBalance(eAddr) + ledger.commit(accTx) + ledger.persist() + + block: + let accTx = ledger.beginSavepoint + ledger.modBalance(eAddr) + ledger.rollback(accTx) + + dbTx.rollback() + + block: + let dbTx = env.xdb.newTransaction() + + block: + let accTx = ledger.beginSavepoint + ledger.modBalance(eAddr) + ledger.commit(accTx) + + ledger.persist() + + ledger.persist() + + dbTx.commit() + + +proc runTrial4(env: TestEnv, ledger: LedgerRef; inx: int; rollback: bool) = + ## Like `runTrial3()` but with four blocks and extra db transaction frames. + let eAddr = env.txs[inx].getRecipient + + block: + let dbTx = env.xdb.newTransaction() + + block: + let accTx = ledger.beginSavepoint + ledger.modBalance(eAddr) + ledger.commit(accTx) + ledger.persist() + + block: + let accTx = ledger.beginSavepoint + ledger.modBalance(eAddr) + ledger.commit(accTx) + ledger.persist() + + block: + let accTx = ledger.beginSavepoint + ledger.modBalance(eAddr) + + if rollback: + ledger.rollback(accTx) + break + + ledger.commit(accTx) + ledger.persist() + + # There must be no dbTx.rollback() here unless `ledger` is + # discarded and/or re-initialised. + dbTx.commit() + + block: + let dbTx = env.xdb.newTransaction() + + block: + let accTx = ledger.beginSavepoint + ledger.modBalance(eAddr) + ledger.commit(accTx) + ledger.persist() + + dbTx.commit() + +# ------------------------------------------------------------------------------ +# Test Runner +# ------------------------------------------------------------------------------ + +const + NumTransactions = 17 + NumBlocks = 13 + feeRecipient = initAddr(401) + prevRandao = EMPTY_UNCLE_HASH # it can be any valid hash + +proc runner(noisy = true) = + suite "StateDB nesting scenarios": + var env = initEnv() + + test "Create transactions and blocks": + var + recipientSeed = 501 + blockTime = EthTime.now() + + for _ in 0..