nimbus-eth1/tests/test_helpers.nim

302 lines
10 KiB
Nim
Raw Normal View History

# Nimbus
# Copyright (c) 2018 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT)
# at your option. This file may not be copied, modified, or distributed except according to those terms.
2018-01-17 11:24:09 +00:00
import
2019-11-13 14:49:39 +00:00
os, macros, json, strformat, strutils, parseutils, os, tables,
2020-07-21 06:15:06 +00:00
stew/byteutils, net, eth/[common, keys, rlp, p2p], unittest2,
testutils/markdown_reports,
../nimbus/[constants, config, transaction, utils, errors, forks],
../nimbus/db/accounts_cache,
../nimbus/context
2018-01-17 11:24:09 +00:00
func revmap(x: Table[Fork, string]): Table[string, Fork] =
result = initTable[string, Fork]()
for k, v in x:
result[v] = k
const
# from https://ethereum-tests.readthedocs.io/en/latest/test_types/state_tests.html
forkNames* = {
FkFrontier: "Frontier",
FkHomestead: "Homestead",
FkTangerine: "EIP150",
FkSpurious: "EIP158",
FkByzantium: "Byzantium",
2020-03-04 09:32:30 +00:00
FkConstantinople: "Constantinople",
FkPetersburg: "ConstantinopleFix",
FkIstanbul: "Istanbul",
FkBerlin: "Berlin",
FkLondon: "London",
FkParis: "Merge"
}.toTable
supportedForks* = {
FkFrontier,
FkHomestead,
FkTangerine,
FkSpurious,
FkByzantium,
FkConstantinople,
2020-03-04 09:32:30 +00:00
FkPetersburg,
FkIstanbul,
FkBerlin,
FkLondon,
FkParis}
nameToFork* = revmap(forkNames)
func skipNothing*(folder: string, name: string): bool = false
2019-03-15 15:34:05 +00:00
proc lacksSupportedForks*(fixtures: JsonNode): bool =
# XXX: Until Nimbus supports all forks, some of the GeneralStateTests won't work.
2019-03-15 15:34:05 +00:00
var fixture: JsonNode
for label, child in fixtures:
fixture = child
break
# not all fixtures make a distinction between forks, so default to accepting
# them all, until we find the ones that specify forks in their "post" section
result = false
2019-03-15 15:34:05 +00:00
if fixture.kind == JObject and fixture.hasKey("transaction") and fixture.hasKey("post"):
result = true
for fork in supportedForks:
2019-03-15 15:34:05 +00:00
if fixture["post"].hasKey(forkNames[fork]):
result = false
break
var status = initOrderedTable[string, OrderedTable[string, Status]]()
proc jsonTestImpl*(inputFolder, outputName: string, handler, skipTest: NimNode): NimNode {.compileTime.} =
let
testStatusIMPL = ident("testStatusIMPL")
testName = ident("testName")
# workaround for strformat in quote do: https://github.com/nim-lang/Nim/issues/8220
2020-04-15 11:09:49 +00:00
symbol {.used.} = newIdentNode"symbol"
final {.used.} = newIdentNode"final"
name {.used.} = newIdentNode"name"
formatted {.used.} = newStrLitNode"{symbol[final]} {name:<64}{$final}{'\n'}"
2018-02-13 17:18:08 +00:00
result = quote:
var filenames: seq[string] = @[]
for filename in walkDirRec("tests" / "fixtures" / `inputFolder`):
if not filename.endsWith(".json"):
continue
var (folder, name) = filename.splitPath()
let last = folder.splitPath().tail
if not status.hasKey(last):
status[last] = initOrderedTable[string, Status]()
status[last][name] = Status.Skip
if `skipTest`(last, name):
continue
filenames.add(filename)
doAssert(filenames.len > 0)
for fname in filenames:
test fname:
{.gcsafe.}:
let
filename = `testName` # the first argument passed to the `test` template
(folder, name) = filename.splitPath()
last = folder.splitPath().tail
# we set this here because exceptions might be raised in the handler:
status[last][name] = Status.Fail
2022-04-08 04:54:11 +00:00
let fixtures = parseJson(readFile(filename))
if fixtures.lacksSupportedForks:
status[last][name] = Status.Skip
skip()
else:
2020-07-21 06:15:06 +00:00
# when not paralleliseTests:
# echo filename
`handler`(fixtures, `testStatusIMPL`)
if `testStatusIMPL` == OK:
status[last][name] = Status.OK
elif `testStatusIMPL` == SKIPPED:
status[last][name] = Status.Skip
suiteTeardown:
status.sort do (a: (string, OrderedTable[string, Status]),
b: (string, OrderedTable[string, Status])) -> int: cmp(a[0], b[0])
generateReport(`outputName`, status)
status.clear()
2018-01-17 11:24:09 +00:00
macro jsonTest*(inputFolder, outputName: static[string], handler: untyped, skipTest: untyped = skipNothing): untyped =
result = jsonTestImpl(inputFolder, outputName, handler, skipTest)
macro jsonTest*(inputFolder: static[string], handler: untyped, skipTest: untyped = skipNothing): untyped =
result = jsonTestImpl(inputFolder, inputFolder, handler, skipTest)
func ethAddressFromHex*(s: string): EthAddress = hexToByteArray(s, result)
2018-05-30 16:11:15 +00:00
func safeHexToSeqByte*(hexStr: string): seq[byte] =
if hexStr == "":
@[]
else:
hexStr.hexToSeqByte
func getHexadecimalInt*(j: JsonNode): int64 =
# parseutils.parseHex works with int which will overflow in 32 bit
2022-04-08 04:54:11 +00:00
var data: StUint[64]
data = fromHex(StUint[64], j.getStr)
result = cast[int64](data)
Redesign of BaseVMState descriptor (#923) * Redesign of BaseVMState descriptor why: BaseVMState provides an environment for executing transactions. The current descriptor also provides data that cannot generally be known within the execution environment, e.g. the total gasUsed which is available not before after all transactions have finished. Also, the BaseVMState constructor has been replaced by a constructor that does not need pre-initialised input of the account database. also: Previous constructor and some fields are provided with a deprecated annotation (producing a lot of noise.) * Replace legacy directives in production sources * Replace legacy directives in unit test sources * fix CI (missing premix update) * Remove legacy directives * chase CI problem * rebased * Re-introduce 'AccountsCache' constructor optimisation for 'BaseVmState' re-initialisation why: Constructing a new 'AccountsCache' descriptor can be avoided sometimes when the current state root is properly positioned already. Such a feature existed already as the update function 'initStateDB()' for the 'BaseChanDB' where the accounts cache was linked into this desctiptor. The function 'initStateDB()' was removed and re-implemented into the 'BaseVmState' constructor without optimisation. The old version was of restricted use as a wrong accounts cache state would unconditionally throw an exception rather than conceptually ask for a remedy. The optimised 'BaseVmState' re-initialisation has been implemented for the 'persistBlocks()' function. also: moved some test helpers to 'test/replay' folder * Remove unused & undocumented fields from Chain descriptor why: Reduces attack surface in general & improves reading the code.
2022-01-18 16:19:32 +00:00
proc setupStateDB*(wantedState: JsonNode, stateDB: AccountsCache) =
for ac, accountData in wantedState:
let account = ethAddressFromHex(ac)
2018-02-13 17:18:08 +00:00
for slot, value in accountData{"storage"}:
stateDB.setStorage(account, fromHex(UInt256, slot), fromHex(UInt256, value.getStr))
let nonce = accountData{"nonce"}.getHexadecimalInt.AccountNonce
let code = accountData{"code"}.getStr.safeHexToSeqByte
let balance = UInt256.fromHex accountData{"balance"}.getStr
2018-01-17 11:24:09 +00:00
2018-02-13 17:18:08 +00:00
stateDB.setNonce(account, nonce)
stateDB.setCode(account, code)
stateDB.setBalance(account, balance)
2018-12-31 03:28:12 +00:00
proc verifyStateDB*(wantedState: JsonNode, stateDB: ReadOnlyStateDB) =
for ac, accountData in wantedState:
let account = ethAddressFromHex(ac)
for slot, value in accountData{"storage"}:
let
slotId = UInt256.fromHex slot
wantedValue = UInt256.fromHex value.getStr
2020-05-30 03:14:59 +00:00
let actualValue = stateDB.getStorage(account, slotId)
#if not found:
# raise newException(ValidationError, "account not found: " & ac)
2019-09-23 15:50:13 +00:00
if actualValue != wantedValue:
2019-09-25 13:05:33 +00:00
raise newException(ValidationError, &"{ac} storageDiff: [{slot}] {actualValue.toHex} != {wantedValue.toHex}")
let
wantedCode = hexToSeqByte(accountData{"code"}.getStr)
wantedBalance = UInt256.fromHex accountData{"balance"}.getStr
2019-09-23 15:50:13 +00:00
wantedNonce = accountData{"nonce"}.getHexadecimalInt.AccountNonce
actualCode = stateDB.getCode(account)
actualBalance = stateDB.getBalance(account)
actualNonce = stateDB.getNonce(account)
2019-09-23 15:50:13 +00:00
if wantedCode != actualCode:
2021-01-13 11:17:38 +00:00
raise newException(ValidationError, &"{ac} codeDiff {wantedCode.toHex} != {actualCode.toHex}")
2019-09-23 15:50:13 +00:00
if wantedBalance != actualBalance:
2019-09-25 13:05:33 +00:00
raise newException(ValidationError, &"{ac} balanceDiff {wantedBalance.toHex} != {actualBalance.toHex}")
2019-09-23 15:50:13 +00:00
if wantedNonce != actualNonce:
2019-09-25 13:05:33 +00:00
raise newException(ValidationError, &"{ac} nonceDiff {wantedNonce.toHex} != {actualNonce.toHex}")
proc parseAccessList(n: JsonNode): AccessList =
if n.kind == JNull:
return
for x in n:
var ap = AccessPair(
address: parseAddress(x["address"].getStr)
)
let sks = x["storageKeys"]
for sk in sks:
ap.storageKeys.add hexToByteArray[32](sk.getStr())
result.add ap
proc getFixtureTransaction*(j: JsonNode, dataIndex, gasIndex, valueIndex: int): Transaction =
let dynamicFeeTx = "gasPrice" notin j
let nonce = j["nonce"].getHexadecimalInt.AccountNonce
let gasLimit = j["gasLimit"][gasIndex].getHexadecimalInt
var toAddr: Option[EthAddress]
# Fixture transactions with `"to": ""` are contract creations.
#
# Fixture transactions with `"to": "0x..."` or `"to": "..."` where `...` are
# 40 hex digits are call/transfer transactions. Even if the digits are all
# zeros, because the all-zeros address is a legitimate account.
#
# There are no other formats. The number of digits if present is always 40,
# "0x" prefix is used in some but not all fixtures, and upper case hex digits
# occur in a few.
let rawTo = j["to"].getStr
if rawTo != "":
toAddr = some(rawTo.parseAddress)
let hexStr = j["value"][valueIndex].getStr
# stTransactionTest/ValueOverflow.json
# prevent parsing exception and subtitute it with max uint256
let value = if ':' in hexStr: high(UInt256) else: fromHex(UInt256, hexStr)
let payload = j["data"][dataIndex].getStr.safeHexToSeqByte
var secretKey = j["secretKey"].getStr
removePrefix(secretKey, "0x")
let privateKey = PrivateKey.fromHex(secretKey).tryGet()
if dynamicFeeTx:
let accList = j["accessLists"][dataIndex]
var tx = Transaction(
txType: TxEip1559,
nonce: nonce,
maxFee: j["maxFeePerGas"].getHexadecimalInt,
maxPriorityFee: j["maxPriorityFeePerGas"].getHexadecimalInt,
gasLimit: gasLimit,
to: toAddr,
value: value,
payload: payload,
accessList: parseAccessList(accList),
chainId: ChainId(1)
)
return signTransaction(tx, privateKey, ChainId(1), false)
let gasPrice = j["gasPrice"].getHexadecimalInt
if j.hasKey("accessLists"):
let accList = j["accessLists"][dataIndex]
var tx = Transaction(
txType: TxEip2930,
nonce: nonce,
gasPrice: gasPrice,
gasLimit: gasLimit,
to: toAddr,
value: value,
payload: payload,
accessList: parseAccessList(accList),
chainId: ChainId(1)
)
signTransaction(tx, privateKey, ChainId(1), false)
else:
var tx = Transaction(
txType: TxLegacy,
nonce: nonce,
gasPrice: gasPrice,
gasLimit: gasLimit,
to: toAddr,
value: value,
payload: payload
)
signTransaction(tx, privateKey, ChainId(1), false)
2019-03-18 01:55:02 +00:00
proc hashLogEntries*(logs: seq[Log]): string =
toLowerAscii("0x" & $keccakHash(rlp.encode(logs)))
2019-04-24 20:45:29 +00:00
proc setupEthNode*(
conf: NimbusConf, ctx: EthContext,
capabilities: varargs[ProtocolInfo, `protocolInfo`]): EthereumNode =
let keypair = ctx.getNetKeys(conf.netKey, conf.dataDir.string).tryGet()
let srvAddress = Address(
ip: conf.listenAddress, tcpPort: conf.tcpPort, udpPort: conf.udpPort)
var node = newEthereumNode(
keypair, srvAddress,
conf.networkId,
nil, conf.agentString,
addAllCapabilities = false,
bindUdpPort = conf.udpPort, bindTcpPort = conf.tcpPort)
2019-04-24 20:45:29 +00:00
for capability in capabilities:
node.addCapability capability
node
proc makeTestConfig*(): NimbusConf =
# commandLineParams() will not works inside all_tests
makeConfig(@[])