nimbus-eth1/tests/test_generalstate_json.nim

259 lines
9.2 KiB
Nim
Raw Normal View History

# Nimbus
# Copyright (c) 2018 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT)
# at your option. This file may not be copied, modified, or distributed except according to those terms.
import
std/[strutils, tables, json, times, os, sets, options],
./test_helpers, ./test_allowed_to_fail,
../nimbus/p2p/executor, test_config,
2019-11-13 14:49:39 +00:00
../nimbus/transaction,
2022-07-28 19:34:38 +00:00
../nimbus/[vm_state, vm_types, utils, chain_config],
../nimbus/db/[db_chain, accounts_cache],
../nimbus/forks,
chronicles,
eth/[rlp, common],
eth/trie/[db, trie_defs],
unittest2,
stew/[results, byteutils]
2019-03-18 01:55:02 +00:00
type
Tester = object
name: string
header: BlockHeader
pre: JsonNode
tx: Transaction
expectedHash: string
expectedLogs: string
fork: Fork
debugMode: bool
trace: bool
2019-03-21 09:01:26 +00:00
index: int
2019-02-28 08:22:07 +00:00
2019-03-18 14:18:04 +00:00
proc toBytes(x: string): seq[byte] =
result = newSeq[byte](x.len)
for i in 0..<x.len: result[i] = x[i].byte
Redesign of BaseVMState descriptor (#923) * Redesign of BaseVMState descriptor why: BaseVMState provides an environment for executing transactions. The current descriptor also provides data that cannot generally be known within the execution environment, e.g. the total gasUsed which is available not before after all transactions have finished. Also, the BaseVMState constructor has been replaced by a constructor that does not need pre-initialised input of the account database. also: Previous constructor and some fields are provided with a deprecated annotation (producing a lot of noise.) * Replace legacy directives in production sources * Replace legacy directives in unit test sources * fix CI (missing premix update) * Remove legacy directives * chase CI problem * rebased * Re-introduce 'AccountsCache' constructor optimisation for 'BaseVmState' re-initialisation why: Constructing a new 'AccountsCache' descriptor can be avoided sometimes when the current state root is properly positioned already. Such a feature existed already as the update function 'initStateDB()' for the 'BaseChanDB' where the accounts cache was linked into this desctiptor. The function 'initStateDB()' was removed and re-implemented into the 'BaseVmState' constructor without optimisation. The old version was of restricted use as a wrong accounts cache state would unconditionally throw an exception rather than conceptually ask for a remedy. The optimised 'BaseVmState' re-initialisation has been implemented for the 'persistBlocks()' function. also: moved some test helpers to 'test/replay' folder * Remove unused & undocumented fields from Chain descriptor why: Reduces attack surface in general & improves reading the code.
2022-01-18 16:19:32 +00:00
method getAncestorHash*(vmState: BaseVMState; blockNumber: BlockNumber): Hash256 {.gcsafe.} =
2019-03-18 14:18:04 +00:00
if blockNumber >= vmState.blockNumber:
return
elif blockNumber < 0:
return
elif blockNumber < vmState.blockNumber - 256:
return
else:
return keccakHash(toBytes($blockNumber))
proc dumpAccount(stateDB: ReadOnlyStateDB, address: EthAddress, name: string): JsonNode =
2019-03-18 01:55:02 +00:00
result = %{
"name": %name,
"address": %($address),
"nonce": %toHex(stateDB.getNonce(address)),
"balance": %stateDB.getBalance(address).toHex(),
"codehash": %($stateDB.getCodeHash(address)),
"storageRoot": %($stateDB.getStorageRoot(address))
2019-03-18 01:55:02 +00:00
}
2019-03-21 09:01:26 +00:00
proc dumpDebugData(tester: Tester, vmState: BaseVMState, sender: EthAddress, gasUsed: GasInt, success: bool) =
let recipient = tester.tx.getRecipient(sender)
2019-03-18 01:55:02 +00:00
let miner = tester.header.coinbase
var accounts = newJObject()
2019-03-18 01:55:02 +00:00
accounts[$miner] = dumpAccount(vmState.readOnlyStateDB, miner, "miner")
accounts[$sender] = dumpAccount(vmState.readOnlyStateDB, sender, "sender")
accounts[$recipient] = dumpAccount(vmState.readOnlyStateDB, recipient, "recipient")
let accountList = [sender, miner, recipient]
var i = 0
for ac, _ in tester.pre:
let account = ethAddressFromHex(ac)
if account notin accountList:
accounts[$account] = dumpAccount(vmState.readOnlyStateDB, account, "pre" & $i)
inc i
let tracingResult = if tester.trace: vmState.getTracingResult() else: %[]
2019-03-18 01:55:02 +00:00
let debugData = %{
"gasUsed": %gasUsed,
"structLogs": tracingResult,
2019-03-18 01:55:02 +00:00
"accounts": accounts
}
2019-03-21 09:01:26 +00:00
let status = if success: "_success" else: "_failed"
writeFile("debug_" & tester.name & "_" & $tester.index & status & ".json", debugData.pretty())
2019-03-18 01:55:02 +00:00
2022-07-28 19:34:38 +00:00
# using only one networkParams will reduce execution
# time ~90% instead of create it for every test
let chainParams = networkParams(MainNet)
2019-03-18 01:55:02 +00:00
proc testFixtureIndexes(tester: Tester, testStatusIMPL: var TestStatus) =
Redesign of BaseVMState descriptor (#923) * Redesign of BaseVMState descriptor why: BaseVMState provides an environment for executing transactions. The current descriptor also provides data that cannot generally be known within the execution environment, e.g. the total gasUsed which is available not before after all transactions have finished. Also, the BaseVMState constructor has been replaced by a constructor that does not need pre-initialised input of the account database. also: Previous constructor and some fields are provided with a deprecated annotation (producing a lot of noise.) * Replace legacy directives in production sources * Replace legacy directives in unit test sources * fix CI (missing premix update) * Remove legacy directives * chase CI problem * rebased * Re-introduce 'AccountsCache' constructor optimisation for 'BaseVmState' re-initialisation why: Constructing a new 'AccountsCache' descriptor can be avoided sometimes when the current state root is properly positioned already. Such a feature existed already as the update function 'initStateDB()' for the 'BaseChanDB' where the accounts cache was linked into this desctiptor. The function 'initStateDB()' was removed and re-implemented into the 'BaseVmState' constructor without optimisation. The old version was of restricted use as a wrong accounts cache state would unconditionally throw an exception rather than conceptually ask for a remedy. The optimised 'BaseVmState' re-initialisation has been implemented for the 'persistBlocks()' function. also: moved some test helpers to 'test/replay' folder * Remove unused & undocumented fields from Chain descriptor why: Reduces attack surface in general & improves reading the code.
2022-01-18 16:19:32 +00:00
let
2022-07-28 19:34:38 +00:00
chainDB = newBaseChainDB(newMemoryDB(), getConfiguration().pruning, params = chainParams)
parent = BlockHeader(stateRoot: emptyRlpHash)
# can't be assigned to
# tester.header.parentHash = parent.blockHash
chainDB.setScore(parent.blockHash, 0.u256)
if tester.fork >= FkParis:
chainDB.config.terminalTotalDifficulty = some(0.u256)
let vmState = BaseVMState.new(
parent = parent,
Redesign of BaseVMState descriptor (#923) * Redesign of BaseVMState descriptor why: BaseVMState provides an environment for executing transactions. The current descriptor also provides data that cannot generally be known within the execution environment, e.g. the total gasUsed which is available not before after all transactions have finished. Also, the BaseVMState constructor has been replaced by a constructor that does not need pre-initialised input of the account database. also: Previous constructor and some fields are provided with a deprecated annotation (producing a lot of noise.) * Replace legacy directives in production sources * Replace legacy directives in unit test sources * fix CI (missing premix update) * Remove legacy directives * chase CI problem * rebased * Re-introduce 'AccountsCache' constructor optimisation for 'BaseVmState' re-initialisation why: Constructing a new 'AccountsCache' descriptor can be avoided sometimes when the current state root is properly positioned already. Such a feature existed already as the update function 'initStateDB()' for the 'BaseChanDB' where the accounts cache was linked into this desctiptor. The function 'initStateDB()' was removed and re-implemented into the 'BaseVmState' constructor without optimisation. The old version was of restricted use as a wrong accounts cache state would unconditionally throw an exception rather than conceptually ask for a remedy. The optimised 'BaseVmState' re-initialisation has been implemented for the 'persistBlocks()' function. also: moved some test helpers to 'test/replay' folder * Remove unused & undocumented fields from Chain descriptor why: Reduces attack surface in general & improves reading the code.
2022-01-18 16:19:32 +00:00
header = tester.header,
chainDB = chainDB,
tracerFlags = (if tester.trace: {TracerFlags.EnableTracing} else: {}),
pruneTrie = chainDB.pruneTrie)
2019-11-28 10:02:11 +00:00
2019-03-21 09:01:26 +00:00
var gasUsed: GasInt
let sender = tester.tx.getSender()
vmState.mutateStateDB:
2019-03-18 01:55:02 +00:00
setupStateDB(tester.pre, db)
# this is an important step when using accounts_cache
# it will affect the account storage's location
# during the next call to `getComittedStorage`
db.persist()
2019-11-19 06:12:13 +00:00
defer:
let obtainedHash = "0x" & `$`(vmState.readOnlyStateDB.rootHash).toLowerAscii
2019-03-18 01:55:02 +00:00
check obtainedHash == tester.expectedHash
2019-02-28 08:22:07 +00:00
let logEntries = vmState.getAndClearLogEntries()
let actualLogsHash = hashLogEntries(logEntries)
2019-03-18 01:55:02 +00:00
let expectedLogsHash = toLowerAscii(tester.expectedLogs)
2019-02-28 08:22:07 +00:00
check(expectedLogsHash == actualLogsHash)
2019-03-21 09:01:26 +00:00
if tester.debugMode:
let success = expectedLogsHash == actualLogsHash and obtainedHash == tester.expectedHash
tester.dumpDebugData(vmState, sender, gasUsed, success)
let rc = vmState.processTransaction(
tester.tx, sender, tester.header, tester.fork)
2022-04-08 04:54:11 +00:00
if rc.isOk:
gasUsed = rc.value
2018-09-18 00:35:41 +00:00
2020-01-10 11:18:36 +00:00
# This is necessary due to the manner in which the state tests are
# generated. State tests are generated from the BlockChainTest tests
# in which these transactions are included in the larger context of a
# block and thus, the mechanisms which would touch/create/clear the
# coinbase account based on the mining reward are present during test
# generation, but not part of the execution, thus we must artificially
# create the account in VMs prior to the state clearing rules,
# as well as conditionally cleaning up the coinbase account when left
# empty in VMs after the state clearing rules came into effect.
let miner = tester.header.coinbase
if miner in vmState.selfDestructs:
2020-01-10 11:18:36 +00:00
vmState.mutateStateDB:
db.addBalance(miner, 0.u256)
if tester.fork >= FkSpurious:
if db.isEmptyAccount(miner):
db.deleteAccount(miner)
# this is an important step when using accounts_cache
# it will affect the account storage's location
# during the next call to `getComittedStorage`
# and the result of rootHash
db.persist()
2019-03-18 03:05:24 +00:00
proc testFixture(fixtures: JsonNode, testStatusIMPL: var TestStatus,
trace = false, debugMode = false, supportedForks: set[Fork] = supportedForks) =
2019-03-18 01:55:02 +00:00
var tester: Tester
var fixture: JsonNode
for label, child in fixtures:
fixture = child
2019-03-18 01:55:02 +00:00
tester.name = label
break
let fenv = fixture["env"]
2019-03-18 01:55:02 +00:00
tester.header = BlockHeader(
coinbase : fenv["currentCoinbase"].getStr.ethAddressFromHex,
difficulty : fromHex(UInt256, fenv{"currentDifficulty"}.getStr),
blockNumber: fenv{"currentNumber"}.getHexadecimalInt.u256,
gasLimit : fenv{"currentGasLimit"}.getHexadecimalInt.GasInt,
timestamp : fenv{"currentTimestamp"}.getHexadecimalInt.int64.fromUnix,
stateRoot : emptyRlpHash
)
if "currentBaseFee" in fenv:
tester.header.baseFee = fromHex(UInt256, fenv{"currentBaseFee"}.getStr)
if "currentRandom" in fenv:
tester.header.prevRandao.data = hexToByteArray[32](fenv{"currentRandom"}.getStr)
2019-03-21 09:01:26 +00:00
let specifyIndex = getConfiguration().index
tester.trace = trace
2019-03-18 01:55:02 +00:00
tester.debugMode = debugMode
let ftrans = fixture["transaction"]
2019-03-18 03:05:24 +00:00
var testedInFork = false
var numIndex = -1
for fork in supportedForks:
2019-03-21 14:35:25 +00:00
if fixture["post"].hasKey(forkNames[fork]):
numIndex = fixture["post"][forkNames[fork]].len
for expectation in fixture["post"][forkNames[fork]]:
2019-03-21 09:01:26 +00:00
inc tester.index
2019-03-21 14:35:25 +00:00
if specifyIndex > 0 and tester.index != specifyIndex:
2019-03-21 09:01:26 +00:00
continue
testedInFork = true
2019-03-18 01:55:02 +00:00
tester.expectedHash = expectation["hash"].getStr
tester.expectedLogs = expectation["logs"].getStr
let
indexes = expectation["indexes"]
dataIndex = indexes["data"].getInt
gasIndex = indexes["gas"].getInt
valueIndex = indexes["value"].getInt
2019-03-18 01:55:02 +00:00
tester.tx = ftrans.getFixtureTransaction(dataIndex, gasIndex, valueIndex)
tester.pre = fixture["pre"]
tester.fork = fork
testFixtureIndexes(tester, testStatusIMPL)
2019-03-18 03:05:24 +00:00
if not testedInFork:
echo "test subject '", tester.name, "' not tested in any forks/subtests"
if specifyIndex <= 0 or specifyIndex > numIndex:
echo "Maximum subtest available: ", numIndex
else:
echo "available forks in this test:"
for fork in test_helpers.supportedForks:
if fixture["post"].hasKey(forkNames[fork]):
echo fork
2019-03-18 03:05:24 +00:00
2019-09-21 05:45:23 +00:00
proc generalStateJsonMain*(debugMode = false) =
const
legacyFolder = "eth_tests" / "LegacyTests" / "Constantinople" / "GeneralStateTests"
newFolder = "eth_tests" / "GeneralStateTests"
2021-01-14 14:33:18 +00:00
let config = getConfiguration()
if config.testSubject == "" or not debugMode:
2019-03-18 01:55:02 +00:00
# run all test fixtures
2021-01-14 14:33:18 +00:00
if config.legacy:
suite "generalstate json tests":
jsonTest(legacyFolder , "GeneralStateTests", testFixture, skipGSTTests)
else:
suite "new generalstate json tests":
jsonTest(newFolder, "newGeneralStateTests", testFixture, skipNewGSTTests)
2019-03-18 01:55:02 +00:00
else:
# execute single test in debug mode
2019-03-18 03:05:24 +00:00
if config.testSubject.len == 0:
echo "missing test subject"
quit(QuitFailure)
let folder = if config.legacy: legacyFolder else: newFolder
let path = "tests" / "fixtures" / folder
2019-03-18 03:05:24 +00:00
let n = json.parseFile(path / config.testSubject)
2019-03-18 01:55:02 +00:00
var testStatusIMPL: TestStatus
2019-03-18 03:05:24 +00:00
var forks: set[Fork] = {}
forks.incl config.fork
testFixture(n, testStatusIMPL, config.trace, true, forks)
2019-03-18 03:05:24 +00:00
when isMainModule:
var message: string
## Processing command line arguments
if processArguments(message) != Success:
echo message
quit(QuitFailure)
else:
if len(message) > 0:
echo message
quit(QuitSuccess)
2019-09-21 05:45:23 +00:00
generalStateJsonMain(true)