2018-04-06 14:52:10 +00:00
|
|
|
# Nimbus
|
|
|
|
# Copyright (c) 2018 Status Research & Development GmbH
|
|
|
|
# Licensed under either of
|
|
|
|
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
|
|
|
|
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT)
|
|
|
|
# at your option. This file may not be copied, modified, or distributed except according to those terms.
|
|
|
|
|
2018-01-16 17:05:20 +00:00
|
|
|
import
|
2020-04-15 11:09:49 +00:00
|
|
|
chronicles, strformat, macros, options, times,
|
2020-07-21 06:15:06 +00:00
|
|
|
sets, eth/[common, keys],
|
2021-06-01 11:04:47 +00:00
|
|
|
../constants, ../errors, ../forks,
|
|
|
|
./interpreter/[opcode_values, gas_meter, gas_costs],
|
2021-04-01 11:53:22 +00:00
|
|
|
./code_stream, ./memory, ./message, ./stack, ./types, ./state,
|
|
|
|
../db/[accounts_cache, db_chain],
|
|
|
|
../utils/header, ./precompiles,
|
|
|
|
./transaction_tracer, ../utils
|
2020-01-17 14:54:28 +00:00
|
|
|
|
2020-07-21 08:12:59 +00:00
|
|
|
when defined(chronicles_log_level):
|
|
|
|
import stew/byteutils
|
|
|
|
|
2020-01-17 14:54:28 +00:00
|
|
|
when defined(evmc_enabled):
|
2020-07-21 06:15:06 +00:00
|
|
|
import evmc/evmc, evmc_helpers, evmc_api, stew/ranges/ptr_arith
|
2018-05-24 10:01:59 +00:00
|
|
|
|
2018-08-23 03:38:00 +00:00
|
|
|
logScope:
|
|
|
|
topics = "vm computation"
|
|
|
|
|
2020-01-16 10:23:51 +00:00
|
|
|
const
|
2020-01-16 16:22:43 +00:00
|
|
|
evmc_enabled* = defined(evmc_enabled)
|
2020-01-16 10:23:51 +00:00
|
|
|
|
|
|
|
template getCoinbase*(c: Computation): EthAddress =
|
|
|
|
when evmc_enabled:
|
2020-01-22 07:47:27 +00:00
|
|
|
c.host.getTxContext().block_coinbase
|
2020-01-16 10:23:51 +00:00
|
|
|
else:
|
|
|
|
c.vmState.coinbase
|
|
|
|
|
|
|
|
template getTimestamp*(c: Computation): int64 =
|
|
|
|
when evmc_enabled:
|
|
|
|
c.host.getTxContext().block_timestamp
|
|
|
|
else:
|
|
|
|
c.vmState.timestamp.toUnix
|
|
|
|
|
|
|
|
template getBlockNumber*(c: Computation): Uint256 =
|
|
|
|
when evmc_enabled:
|
|
|
|
c.host.getTxContext().block_number.u256
|
|
|
|
else:
|
|
|
|
c.vmState.blockNumber.blockNumberToVmWord
|
|
|
|
|
|
|
|
template getDifficulty*(c: Computation): DifficultyInt =
|
|
|
|
when evmc_enabled:
|
|
|
|
Uint256.fromEvmc c.host.getTxContext().block_difficulty
|
|
|
|
else:
|
|
|
|
c.vmState.difficulty
|
|
|
|
|
|
|
|
template getGasLimit*(c: Computation): GasInt =
|
|
|
|
when evmc_enabled:
|
|
|
|
c.host.getTxContext().block_gas_limit.GasInt
|
|
|
|
else:
|
|
|
|
c.vmState.gasLimit
|
|
|
|
|
2021-06-27 13:18:17 +00:00
|
|
|
template getBaseFee*(c: Computation): Uint256 =
|
|
|
|
when evmc_enabled:
|
|
|
|
Uint256.fromEvmc c.host.getTxContext().block_base_fee
|
|
|
|
else:
|
|
|
|
c.vmState.baseFee
|
|
|
|
|
2020-01-16 10:23:51 +00:00
|
|
|
template getChainId*(c: Computation): uint =
|
|
|
|
when evmc_enabled:
|
|
|
|
Uint256.fromEvmc(c.host.getTxContext().chain_id).truncate(uint)
|
|
|
|
else:
|
2021-02-13 09:32:48 +00:00
|
|
|
c.vmState.chaindb.config.chainId.uint
|
2020-01-16 10:23:51 +00:00
|
|
|
|
|
|
|
template getOrigin*(c: Computation): EthAddress =
|
|
|
|
when evmc_enabled:
|
2020-01-22 07:47:27 +00:00
|
|
|
c.host.getTxContext().tx_origin
|
2020-01-16 10:23:51 +00:00
|
|
|
else:
|
|
|
|
c.vmState.txOrigin
|
|
|
|
|
|
|
|
template getGasPrice*(c: Computation): GasInt =
|
|
|
|
when evmc_enabled:
|
|
|
|
Uint256.fromEvmc(c.host.getTxContext().tx_gas_price).truncate(GasInt)
|
|
|
|
else:
|
|
|
|
c.vmState.txGasPrice
|
|
|
|
|
|
|
|
template getBlockHash*(c: Computation, blockNumber: Uint256): Hash256 =
|
2020-01-16 14:34:16 +00:00
|
|
|
when evmc_enabled:
|
|
|
|
c.host.getBlockHash(blockNumber)
|
|
|
|
else:
|
2020-01-16 10:23:51 +00:00
|
|
|
c.vmState.getAncestorHash(blockNumber.vmWordToBlockNumber)
|
|
|
|
|
2020-01-16 14:43:29 +00:00
|
|
|
template accountExists*(c: Computation, address: EthAddress): bool =
|
|
|
|
when evmc_enabled:
|
|
|
|
c.host.accountExists(address)
|
|
|
|
else:
|
2020-01-17 11:48:14 +00:00
|
|
|
if c.fork >= FkSpurious:
|
|
|
|
not c.vmState.readOnlyStateDB.isDeadAccount(address)
|
|
|
|
else:
|
|
|
|
c.vmState.readOnlyStateDB.accountExists(address)
|
2020-01-16 14:43:29 +00:00
|
|
|
|
2020-01-16 14:56:59 +00:00
|
|
|
template getStorage*(c: Computation, slot: Uint256): Uint256 =
|
|
|
|
when evmc_enabled:
|
|
|
|
c.host.getStorage(c.msg.contractAddress, slot)
|
|
|
|
else:
|
2020-05-30 03:14:59 +00:00
|
|
|
c.vmState.readOnlyStateDB.getStorage(c.msg.contractAddress, slot)
|
2020-01-16 14:56:59 +00:00
|
|
|
|
2020-01-16 15:07:04 +00:00
|
|
|
template getBalance*(c: Computation, address: EthAddress): Uint256 =
|
|
|
|
when evmc_enabled:
|
|
|
|
c.host.getBalance(address)
|
|
|
|
else:
|
|
|
|
c.vmState.readOnlyStateDB.getBalance(address)
|
|
|
|
|
2020-01-16 15:15:20 +00:00
|
|
|
template getCodeSize*(c: Computation, address: EthAddress): uint =
|
|
|
|
when evmc_enabled:
|
|
|
|
c.host.getCodeSize(address)
|
|
|
|
else:
|
2020-05-30 03:14:59 +00:00
|
|
|
uint(c.vmState.readOnlyStateDB.getCodeSize(address))
|
2020-01-16 15:48:22 +00:00
|
|
|
|
|
|
|
template getCodeHash*(c: Computation, address: EthAddress): Hash256 =
|
|
|
|
when evmc_enabled:
|
|
|
|
c.host.getCodeHash(address)
|
|
|
|
else:
|
|
|
|
let db = c.vmState.readOnlyStateDB
|
|
|
|
if not db.accountExists(address) or db.isEmptyAccount(address):
|
|
|
|
default(Hash256)
|
|
|
|
else:
|
|
|
|
db.getCodeHash(address)
|
2020-01-16 15:15:20 +00:00
|
|
|
|
2020-01-16 16:12:46 +00:00
|
|
|
template selfDestruct*(c: Computation, address: EthAddress) =
|
|
|
|
when evmc_enabled:
|
|
|
|
c.host.selfDestruct(c.msg.contractAddress, address)
|
|
|
|
else:
|
2020-01-22 04:48:02 +00:00
|
|
|
c.execSelfDestruct(address)
|
2020-01-16 16:12:46 +00:00
|
|
|
|
2020-04-20 18:12:44 +00:00
|
|
|
template getCode*(c: Computation, address: EthAddress): seq[byte] =
|
2020-01-16 16:12:46 +00:00
|
|
|
when evmc_enabled:
|
2020-04-20 18:12:44 +00:00
|
|
|
c.host.copyCode(address)
|
2020-01-16 16:12:46 +00:00
|
|
|
else:
|
|
|
|
c.vmState.readOnlyStateDB.getCode(address)
|
|
|
|
|
Transaction: EVMC fix, `CREATE2` salt is a 256-bit blob not a number
This changes fixes a bug in `CREATE2` ops when used with EVMC.
Because it changes the salt type, it affects non-EVMC code as well.
The salt was passed through EVMC with the wrong byte order, although this went
unnoticed as the Nimbus host flipped the byte order before using it.
This was found when running Nimbus with third-party EVM,
["evmone"](https://github.com/ethereum/evmone).
There are different ways to remedy this.
If treated as a number, Nimbus EVM would byte-flip the value when calling EVMC,
then Nimbus host would flip the received value. Finally, it would be flipped a
third time when generating the address in `generateSafeAddress`. The first two
flips can be eliminated by negotiation (like other numbers), but there would
always be one flip.
As a bit pattern, Nimbus EVM would flip the same way it does when dealing with
hashes on the stack (e.g. with `getBlockHash`). Nimbus host wouldn't flip at
all - and when using third-party EVMs there would be no flips in Nimbus.
Because this value is not for arithmetic, any bit pattern is valid, and there
shouldn't be any flips when using a third-party EVM, the bit-pattern
interpretation is favoured. The only flip is done in Nimbus EVM (and might be
eliminated in an optimised version).
As suggested, we'll define a new "opaque 256 bits" type to hold this value.
(Similar to `Hash256`, but the salt isn't necessarily a hash.)
Signed-off-by: Jamie Lokier <jamie@shareable.org>
2021-06-09 22:32:42 +00:00
|
|
|
proc generateContractAddress(c: Computation, salt: ContractSalt): EthAddress =
|
2020-01-20 12:55:29 +00:00
|
|
|
if c.msg.kind == evmcCreate:
|
|
|
|
let creationNonce = c.vmState.readOnlyStateDb().getNonce(c.msg.sender)
|
|
|
|
result = generateAddress(c.msg.sender, creationNonce)
|
|
|
|
else:
|
2020-01-31 01:08:44 +00:00
|
|
|
result = generateSafeAddress(c.msg.sender, salt, c.msg.data)
|
2020-01-20 12:55:29 +00:00
|
|
|
|
2020-07-28 16:48:45 +00:00
|
|
|
import stew/byteutils
|
|
|
|
|
Transaction: EVMC fix, `CREATE2` salt is a 256-bit blob not a number
This changes fixes a bug in `CREATE2` ops when used with EVMC.
Because it changes the salt type, it affects non-EVMC code as well.
The salt was passed through EVMC with the wrong byte order, although this went
unnoticed as the Nimbus host flipped the byte order before using it.
This was found when running Nimbus with third-party EVM,
["evmone"](https://github.com/ethereum/evmone).
There are different ways to remedy this.
If treated as a number, Nimbus EVM would byte-flip the value when calling EVMC,
then Nimbus host would flip the received value. Finally, it would be flipped a
third time when generating the address in `generateSafeAddress`. The first two
flips can be eliminated by negotiation (like other numbers), but there would
always be one flip.
As a bit pattern, Nimbus EVM would flip the same way it does when dealing with
hashes on the stack (e.g. with `getBlockHash`). Nimbus host wouldn't flip at
all - and when using third-party EVMs there would be no flips in Nimbus.
Because this value is not for arithmetic, any bit pattern is valid, and there
shouldn't be any flips when using a third-party EVM, the bit-pattern
interpretation is favoured. The only flip is done in Nimbus EVM (and might be
eliminated in an optimised version).
As suggested, we'll define a new "opaque 256 bits" type to hold this value.
(Similar to `Hash256`, but the salt isn't necessarily a hash.)
Signed-off-by: Jamie Lokier <jamie@shareable.org>
2021-06-09 22:32:42 +00:00
|
|
|
proc newComputation*(vmState: BaseVMState, message: Message,
|
|
|
|
salt: ContractSalt = ZERO_CONTRACTSALT): Computation =
|
Refactor interpreter dispatch (#65)
* move forks constants, rename errors
* Move vm/utils to vm/interpreter/utils
* initial opcodes refactoring
* Add refactored Comparison & Bitwise Logic Operations
* Add sha3 and address, simplify macro, support pop 0
* balance, origin, caller, callValue
* fix gas copy opcodes gas costs, add callDataLoad/Size/Copy, CodeSize/Copy and gas price opcode
* Update with 30s, 40s, 50s opcodes + impl of balance + stack improvement
* add push, dup, swap, log, create and call operations
* finish opcode implementation
* Add the new dispatching logic
* Pass the opcode test
* Make test_vm_json compile
* halt execution without exceptions for Return, Revert, selfdestruct (fix #62)
* Properly catch and recover from EVM exceptions (stack underflow ...)
* Fix byte op
* Fix jump regressions
* Update for latest devel, don't import old dispatch code as quasiBoolean macro is broken by latest devel
* Fix sha3 regression on empty memory slice and until end of range slice
* Fix padding / range error on expXY_success (gas computation left)
* update logging procs
* Add tracing - expXY_success is not a regression, sload stub was accidentally passing the test
* Reuse the same stub as OO implementation
* Delete previous opcode implementation
* Delete object oriented fork code
* Delete exceptions that were used as control flows
* delete base.nim :fire:, yet another OO remnants
* Delete opcode table
* Enable omputed gotos and compile-time gas fees
* Revert const gasCosts -> generates SIGSEGV
* inline push, swap and dup opcodes
* loggers are now template again, why does this pass new tests?
* Trigger CI rebuild after rocksdb fix https://github.com/status-im/nim-rocksdb/pull/5
* Address review comment on "push" + VMTests in debug mode (not release)
* Address review comment: don't tag fork by default, make opcode impl grepable
* Static compilation fixes after rebasing
* fix the initialization of the VM database
* add a missing import
* Deactivate balance and sload test following #59
* Reactivate stack check (deactivated in #59, necessary to pass tests)
* Merge remaining opcodes implementation from #59
* Merge callDataLoad and codeCopy fixes, todo simplify see #67
2018-07-06 07:52:31 +00:00
|
|
|
new result
|
2018-01-16 17:05:20 +00:00
|
|
|
result.vmState = vmState
|
|
|
|
result.msg = message
|
|
|
|
result.memory = Memory()
|
|
|
|
result.stack = newStack()
|
2020-11-25 12:09:16 +00:00
|
|
|
result.returnStack = @[]
|
2018-07-18 12:18:17 +00:00
|
|
|
result.gasMeter.init(message.gas)
|
2020-01-09 06:25:53 +00:00
|
|
|
result.touchedAccounts = initHashSet[EthAddress]()
|
2021-05-11 09:05:58 +00:00
|
|
|
result.selfDestructs = initHashSet[EthAddress]()
|
2020-01-16 10:23:51 +00:00
|
|
|
|
2020-01-20 12:55:29 +00:00
|
|
|
if result.msg.isCreate():
|
|
|
|
result.msg.contractAddress = result.generateContractAddress(salt)
|
2020-01-20 14:02:06 +00:00
|
|
|
result.code = newCodeStream(message.data)
|
|
|
|
message.data = @[]
|
|
|
|
else:
|
2020-04-20 18:12:44 +00:00
|
|
|
result.code = newCodeStream(vmState.readOnlyStateDb.getCode(message.codeAddress))
|
2020-01-20 12:55:29 +00:00
|
|
|
|
2020-01-16 10:23:51 +00:00
|
|
|
when evmc_enabled:
|
|
|
|
result.host.init(
|
|
|
|
nim_host_get_interface(),
|
|
|
|
cast[evmc_host_context](result)
|
|
|
|
)
|
|
|
|
|
2021-05-18 13:43:38 +00:00
|
|
|
proc newComputation*(vmState: BaseVMState, message: Message, code: seq[byte]): Computation =
|
|
|
|
new result
|
|
|
|
result.vmState = vmState
|
|
|
|
result.msg = message
|
|
|
|
result.memory = Memory()
|
|
|
|
result.stack = newStack()
|
|
|
|
result.returnStack = @[]
|
|
|
|
result.gasMeter.init(message.gas)
|
|
|
|
result.touchedAccounts = initHashSet[EthAddress]()
|
|
|
|
result.selfDestructs = initHashSet[EthAddress]()
|
|
|
|
result.code = newCodeStream(code)
|
|
|
|
|
|
|
|
when evmc_enabled:
|
|
|
|
result.host.init(
|
|
|
|
nim_host_get_interface(),
|
|
|
|
cast[evmc_host_context](result)
|
|
|
|
)
|
|
|
|
|
2020-01-16 07:01:59 +00:00
|
|
|
template gasCosts*(c: Computation): untyped =
|
|
|
|
c.vmState.gasCosts
|
|
|
|
|
|
|
|
template fork*(c: Computation): untyped =
|
|
|
|
c.vmState.fork
|
|
|
|
|
2020-01-10 01:26:17 +00:00
|
|
|
proc isOriginComputation*(c: Computation): bool =
|
2018-01-16 17:05:20 +00:00
|
|
|
# Is this computation the computation initiated by a transaction
|
2020-01-16 06:36:58 +00:00
|
|
|
c.msg.sender == c.vmState.txOrigin
|
2018-01-16 17:05:20 +00:00
|
|
|
|
2020-01-10 01:26:17 +00:00
|
|
|
template isSuccess*(c: Computation): bool =
|
2018-01-16 17:05:20 +00:00
|
|
|
c.error.isNil
|
|
|
|
|
2020-01-10 01:26:17 +00:00
|
|
|
template isError*(c: Computation): bool =
|
2018-01-16 17:05:20 +00:00
|
|
|
not c.isSuccess
|
|
|
|
|
2020-01-10 01:26:17 +00:00
|
|
|
func shouldBurnGas*(c: Computation): bool =
|
2018-01-16 17:05:20 +00:00
|
|
|
c.isError and c.error.burnsGas
|
|
|
|
|
2021-05-11 09:05:58 +00:00
|
|
|
proc isSelfDestructed*(c: Computation, address: EthAddress): bool =
|
|
|
|
result = address in c.selfDestructs
|
2019-03-15 11:16:47 +00:00
|
|
|
|
2020-01-10 01:26:17 +00:00
|
|
|
proc snapshot*(c: Computation) =
|
2021-10-28 09:42:39 +00:00
|
|
|
c.savePoint = c.vmState.stateDB.beginSavePoint()
|
2019-02-17 00:30:02 +00:00
|
|
|
|
2020-01-10 01:26:17 +00:00
|
|
|
proc commit*(c: Computation) =
|
2021-10-28 09:42:39 +00:00
|
|
|
c.vmState.stateDB.commit(c.savePoint)
|
2018-01-16 17:05:20 +00:00
|
|
|
|
2020-01-10 01:26:17 +00:00
|
|
|
proc dispose*(c: Computation) {.inline.} =
|
2021-10-28 09:42:39 +00:00
|
|
|
c.vmState.stateDB.safeDispose(c.savePoint)
|
EVM: Small patch that reduces EVM stack usage to almost nothing
There's been a lot of talk about the Nimbus EVM "stack problem". I think we
assumed changing it would require big changes to the interpreter code, touching
a lot of functions.
It turned out to be a low hanging fruit.
This patch solves the stack problem, but hardly touches anything. The change
in EVM stack memory is from 13 MB worst case to just 48 kB, a 250x reduction.
I've been doing work on the database/storage/trie code. While looking at the
API between the EVM and the database/storage/trie, this stack patch stood out
and made itself obvious. As it's tiny, rather than more talk, here it is.
Note: This patch is intentionally small, non-invasive, and hopefully easy to
understand, so that it doesn't conflict with other work done on the EVM, and
can easily be grafted into any other EVM structure.
Motivation
==========
- We run out of space and crash on some targets, unless the stack limit is
raised above its default. Surprise segmentation faults are unhelpful.
- Some CI targets have been disabled for months due to this.
- Because usage borders on the system limits, when working on
database/storage/trie/sync code (called from the EVM), segmentation faults
occur and are misleading. They cause lost time due to thinking there's a
crash bug in the code being worked on, when there's nothing wrong with it.
- Sometimes unrelated, trivial code changes elsewhere trigger CI test failures.
It looks like abrupt termination. A simple, recent patch was crashing in
`make test` even though it was a trivial refactor. Turns out it pushed the
stack over the edge.
- A large stack has to be scanned by the Nim garbage collector sometimes.
Larger stack means slower GC and memory allocation.
- The structure of this small patch suggests how to weave async into the EVM
with almost no changes to the EVM, and no async transformation overhead.
- The patch seemed obvious when working on the API between EVM and storage.
Measurements before
===================
All these tests were run on Ubuntu 20.04 server, x86-64. This is one of the
targets that has been disabled for a while in CI in EVMC mode due to crashing,
and excessive stack usage is the cause.
Testing commit 0c34a8e3 `2021-04-08 17:46:00 +0200 CI: use MSYS2 on Windows`.
$ rm -f build/all_tests && make ENABLE_EVMC=1 test
$ ulimit -S -s 16384 # Requires larger stack than default to avoid crash.
$ ./build/all_tests 9 | tee tlog
[Suite] persist block json tests
...
Stack range 38496 depthHigh 3
...
Stack range 13140272 depthHigh 1024
[OK] tests/fixtures/PersistBlockTests/block1431916.json
These tests use 13.14 MB of stack to run, and so crash with the default stack
limit on Ubuntu Server 20.04 (8MB). Exactly 12832 bytes per EVM call stack
frame. It's interesting to see some stack frames take a bit more.
$ rm -f build/all_tests && make ENABLE_EVMC=1 test
$ ulimit -S -s 16384 # Requires larger stack than default.
$ ./build/all_tests 7 | tee tlog
[Suite] new generalstate json tests
...
Stack range 15488 depthHigh 2
...
Stack range 3539312 depthHigh 457
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest639.json
...
Stack range 3756144 depthHigh 485
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest458.json
...
Stack range 7929968 depthHigh 1024
[OK] tests/fixtures/eth_tests/GeneralStateTests/stCreate2/Create2OnDepth1024.json
These tests use 7.92MB of stack to run. About 7264 bytes per EVM call stack
frame. It _only just_ avoids crashing with the default Ubuntu Server stack
limit of 8 MB. However, it still crashes on Windows x86-64, which is why the
CI target is currently disabled.
On Linux where this passes, this is so borderline that it affects work and
testing of storage and sync code, because that's called from the EVM. Which
was a motivation for dealing with the stack instead of letting this linger.
Also, this stack greatly exceeds the default thread stack size.
$ rm -f build/all_tests && make ENABLE_EVMC=0 test
$ ulimit -S -s 16384 # Requires larger stack than default to avoid crash.
$ ./build/all_tests 9 | tee tlog
[Suite] persist block json tests
...
Stack range 33216 depthHigh 3
...
Stack range 11338032 depthHigh 1024
[OK] tests/fixtures/PersistBlockTests/block1431916.json
These tests use 11.33 MB stack to run, and so crash with a default stack limit
of 8MB. Exactly 11072 bytes per EVM call stack frame. It's interesting to see
some stack frames take a bit more.
$ rm -f build/all_tests && make ENABLE_EVMC=0 test
$ ulimit -S -s 16384 # Requires larger stack than default.
$ ./build/all_tests 7 | tee tlog
[Suite] new generalstate json tests
...
Stack range 10224 depthHigh 2
...
Stack range 2471760 depthHigh 457
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest639.json
...
Stack range 2623184 depthHigh 485
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest458.json
...
Stack range 5537824 depthHigh 1024
[OK] tests/fixtures/eth_tests/GeneralStateTests/stCreate2/Create2OnDepth1024.json
These tests use 5.54 MB of stack to run, and avoid crashing on with a default
stack limit of 8 MB. About 5408 bytes per EVM call stack frame.
However, this is uncomfortably close to the limit, as the stack frame size is
sensitive to changes in the code.
Also, this stack greatly exceeds the default thread stack size.
Measurements after
==================
(This patch doesn't address EVMC mode, which is not our default. EVMC stack
usage remains about the same. EVMC mode is addressed in another tiny patch.)
$ rm -f build/all_tests && make ENABLE_EVMC=0 test
$ ulimit -S -s 80 # Because we can! 80k stack.
$ ./build/all_tests 9 | tee tlog
[Suite] persist block json tests
...
Stack range 496 depthHigh 3
...
Stack range 49504 depthHigh 1024
[OK] tests/fixtures/PersistBlockTests/block1431916.json
$ rm -f build/all_tests && make ENABLE_EVMC=0 test
$ ulimit -S -s 72 # Because we can! 72k stack.
$ ./build/all_tests 7 | tee tlog
[Suite] new generalstate json tests
...
Stack range 448 depthHigh 2
...
Stack range 22288 depthHigh 457
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest639.json
...
Stack range 23632 depthHigh 485
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest458.json
...
Stack range 49504 depthHigh 1024
[OK] tests/fixtures/eth_tests/GeneralStateTests/stCreate2/Create2OnDepth1024.json
For both tests, a satisfying *48 bytes* per EVM call stack frame, and EVM takes
not much more than 48 kB. With other overheads, both tests run in 80 kB stack
total at maximum EVM depth.
We must add some headroom on this for database activity called from the EVM,
and different compile targets. But it means the EVM itself is no longer a
stack burden.
This is much smaller than the default thread stack size on Linux (2MB), with
plenty of margin. It's even smaller than Linux from a long time ago (128kB),
and some small embedded C targets. (Just fyi, though, some JVM environments
allocated just 32 kB to thread stacks.)
This size is also well suited to running EVMs in threads, if that's useful.
Subtle exception handling and `dispose`
=======================================
It is important that each `snapshot` has a corresponding `dispose` in the event
of an exception being raised. This code does do that, but in a subtle way.
The pair of functions `execCallOrCreate` and `execCallOrCreateAux` are
equivalent to the following code, where you can see `dispose` more clearly:
proc execCallOrCreate*(c: Computation) =
defer: c.dispose()
if c.beforeExec():
return
c.executeOpcodes()
while not c.continuation.isNil:
c.child.execCallOrCreate()
c.child = nil
(c.continuation)()
c.executeOpcodes()
c.afterExec()
That works fine, but only reduces the stack used to 300-700 kB instead of 48 kB.
To get lower we split the above into separate `execCallOrCreate` and
`execCallOrCreateAux`. Only the outermost has `defer`, and instead of handling
one level, it walks the entire `c.parent` chain calling `dispose` if needed.
The inner one avoids `defer`, which greatly reduces the size of its stackframe.
`c` is a `var` parameter, at each level of recursion. So the outermost proc
sees the temporary changes made by all inner calls. This is why `c` is updated
and the `c.parent` chain is maintained at each step.
Signed-off-by: Jamie Lokier <jamie@shareable.org>
2021-04-12 17:06:31 +00:00
|
|
|
c.savePoint = nil
|
2019-02-18 11:45:18 +00:00
|
|
|
|
2020-01-10 01:26:17 +00:00
|
|
|
proc rollback*(c: Computation) =
|
2021-10-28 09:42:39 +00:00
|
|
|
c.vmState.stateDB.rollback(c.savePoint)
|
2019-04-01 01:54:02 +00:00
|
|
|
|
2020-01-10 01:26:17 +00:00
|
|
|
proc setError*(c: Computation, msg: string, burnsGas = false) {.inline.} =
|
|
|
|
c.error = Error(info: msg, burnsGas: burnsGas)
|
2019-04-01 01:54:02 +00:00
|
|
|
|
EVM: `writeContract` fixes, never return contract code as `RETURNDATA`
This fixes #867 "EIP-170 related consensus error at Goerli block 5080941", and
equivalent on other networks.
This combines a change on the EVM-caller side with an EVM-side change from
@jangko 6548ff98 "fixes CREATE/CREATE2's `returndata` bug", making the caller
EVM ignore any data except from `REVERT`.
Either change works by itself. The reason for both is to ensure we definitely
comply with ambiguous EVMC expectations from either side of that boundary, and
it makes the internal API clearer.
As well as fixing a specific consensus issue, there are some other EVM logic
changes too: Refactored `writeContract`, how `RETURNDATA` is handled inside the
EVM, and changed behaviour with quirks before EIP-2 (Homestead).
The fix allows sync to pass block 5080941 on Goerli, and probably equivalent on
other networks. Here's a trace at batch 5080897..5081088:
```
TRC 2021-10-01 21:18:12.883+01:00 Persisting blocks file=persist_blocks.nim:43 fromBlock=5080897 toBlock=5081088
...
DBG 2021-10-01 21:18:13.270+01:00 Contract code size exceeds EIP170 topics="vm computation" file=computation.nim:236 limit=24577 actual=31411
DBG 2021-10-01 21:18:13.271+01:00 gasUsed neq cumulativeGasUsed file=process_block.nim:68 block=5080941/0A3537BC5BDFC637349E1C77D9648F2F65E2BF973ABF7956618F854B769DF626 gasUsed=3129669 cumulativeGasUsed=3132615
TRC 2021-10-01 21:18:13.271+01:00 peer disconnected file=blockchain_sync.nim:407 peer=<IP:PORT>
```
Although it says "Contract code size" and "gasUsed", this bug is more general
than either contract size or gas. It's due to incorrect behaviour of EVM
instructions `RETURNDATA` and `RETURNDATASIZE`.
Sometimes when `writeContract` decides to reject writing the contract for any
of several reasons (for example just insufficient gas), the unwritten contract
code was being used as the "return data", and given to the caller. If the
caller used `RETURNDATA` or `RETURNDATASIZE` ops, those incorrectly reported
the contract code that didn't get written.
EIP-211 (https://eips.ethereum.org/EIPS/eip-211) describes `RETURNDATA`:
> "`CREATE` and `CREATE2` are considered to return the empty buffer in the
> success case and the failure data in the failure case".
The language is ambiguous. In fact "failure case" means when the contract uses
`REVERT` to finish. It doesn't mean other failures like out of gas, EIP-170
limit, EIP-3541, etc.
To be thorough, and to ensure we always do the right thing with real EVMC when
that's finalised, this patch fixes the `RETURNDATA` issue in two places, either
of which make Goerli block 5080941 pass.
`writeContract` has been refactored to be caller, and so has where it's called.
It sets an error in the usual way if contract writing is rejected -- that's
anticipating EVMC, where we'll use different error codes later.
Overall four behaviour changes:
1. On the callee side, it doesn't set `c.outputData` except for `REVERT`.
2. On the caller side, it doesn't read `child.outputData` except for `REVERT`.
3. There was a bug in processing before Homestead fork (EIP-2). We did not
match the spec or other implementations; now we do. When there's
insufficient gas, before Homestead it's treated as success but with an empty
contract.
https://github.com/ethereum/pyethereum/blob/d117c8f3fd93359fc641fd850fa799436f7c43b5/ethereum/processblock.py#L304
https://github.com/ethereum/go-ethereum/blob/401354976bb4/core/vm/instructions.go#L586
4. The Byzantium check has been removed, as it's unnecessary.
Signed-off-by: Jamie Lokier <jamie@shareable.org>
2021-12-02 19:44:51 +00:00
|
|
|
proc writeContract*(c: Computation) =
|
|
|
|
template withExtra(tracer: untyped, args: varargs[untyped]) =
|
|
|
|
tracer args, newContract=($c.msg.contractAddress),
|
|
|
|
blockNumber=c.vmState.blockNumber, blockHash=($c.vmState.blockHash)
|
|
|
|
|
|
|
|
# In each check below, they are guarded by `len > 0`. This includes writing
|
|
|
|
# out the code, because the account already has zero-length code to handle
|
|
|
|
# nested calls (this is specified). May as well simplify other branches.
|
|
|
|
let (len, fork) = (c.output.len, c.fork)
|
|
|
|
if len == 0:
|
|
|
|
return
|
|
|
|
|
|
|
|
# EIP-3541 constraint (https://eips.ethereum.org/EIPS/eip-3541).
|
|
|
|
if fork >= FkLondon and c.output[0] == 0xEF.byte:
|
|
|
|
withExtra trace, "New contract code starts with 0xEF byte, not allowed by EIP-3541"
|
|
|
|
# TODO: Return `EVMC_CONTRACT_VALIDATION_FAILURE` (like Silkworm).
|
|
|
|
c.setError("EVMC_CONTRACT_VALIDATION_FAILURE", true)
|
|
|
|
return
|
|
|
|
|
|
|
|
# EIP-170 constraint (https://eips.ethereum.org/EIPS/eip-3541).
|
|
|
|
if fork >= FkSpurious and len > EIP170_MAX_CODE_SIZE:
|
|
|
|
withExtra trace, "New contract code exceeds EIP-170 limit",
|
|
|
|
codeSize=len, maxSize=EIP170_MAX_CODE_SIZE
|
|
|
|
# TODO: Return `EVMC_OUT_OF_GAS` (like Silkworm).
|
|
|
|
c.setError("EVMC_OUT_OF_GAS", true)
|
|
|
|
return
|
2021-06-28 10:14:08 +00:00
|
|
|
|
2021-10-19 11:52:01 +00:00
|
|
|
# Charge gas and write the code even if the code address is self-destructed.
|
|
|
|
# Non-empty code in a newly created, self-destructed account is possible if
|
|
|
|
# the init code calls `DELEGATECALL` or `CALLCODE` to other code which uses
|
|
|
|
# `SELFDESTRUCT`. This shows on Mainnet blocks 6001128..6001204, where the
|
|
|
|
# gas difference matters. The new code can be called later in the
|
|
|
|
# transaction too, before self-destruction wipes the account at the end.
|
2019-03-19 15:19:08 +00:00
|
|
|
|
EVM: `writeContract` fixes, never return contract code as `RETURNDATA`
This fixes #867 "EIP-170 related consensus error at Goerli block 5080941", and
equivalent on other networks.
This combines a change on the EVM-caller side with an EVM-side change from
@jangko 6548ff98 "fixes CREATE/CREATE2's `returndata` bug", making the caller
EVM ignore any data except from `REVERT`.
Either change works by itself. The reason for both is to ensure we definitely
comply with ambiguous EVMC expectations from either side of that boundary, and
it makes the internal API clearer.
As well as fixing a specific consensus issue, there are some other EVM logic
changes too: Refactored `writeContract`, how `RETURNDATA` is handled inside the
EVM, and changed behaviour with quirks before EIP-2 (Homestead).
The fix allows sync to pass block 5080941 on Goerli, and probably equivalent on
other networks. Here's a trace at batch 5080897..5081088:
```
TRC 2021-10-01 21:18:12.883+01:00 Persisting blocks file=persist_blocks.nim:43 fromBlock=5080897 toBlock=5081088
...
DBG 2021-10-01 21:18:13.270+01:00 Contract code size exceeds EIP170 topics="vm computation" file=computation.nim:236 limit=24577 actual=31411
DBG 2021-10-01 21:18:13.271+01:00 gasUsed neq cumulativeGasUsed file=process_block.nim:68 block=5080941/0A3537BC5BDFC637349E1C77D9648F2F65E2BF973ABF7956618F854B769DF626 gasUsed=3129669 cumulativeGasUsed=3132615
TRC 2021-10-01 21:18:13.271+01:00 peer disconnected file=blockchain_sync.nim:407 peer=<IP:PORT>
```
Although it says "Contract code size" and "gasUsed", this bug is more general
than either contract size or gas. It's due to incorrect behaviour of EVM
instructions `RETURNDATA` and `RETURNDATASIZE`.
Sometimes when `writeContract` decides to reject writing the contract for any
of several reasons (for example just insufficient gas), the unwritten contract
code was being used as the "return data", and given to the caller. If the
caller used `RETURNDATA` or `RETURNDATASIZE` ops, those incorrectly reported
the contract code that didn't get written.
EIP-211 (https://eips.ethereum.org/EIPS/eip-211) describes `RETURNDATA`:
> "`CREATE` and `CREATE2` are considered to return the empty buffer in the
> success case and the failure data in the failure case".
The language is ambiguous. In fact "failure case" means when the contract uses
`REVERT` to finish. It doesn't mean other failures like out of gas, EIP-170
limit, EIP-3541, etc.
To be thorough, and to ensure we always do the right thing with real EVMC when
that's finalised, this patch fixes the `RETURNDATA` issue in two places, either
of which make Goerli block 5080941 pass.
`writeContract` has been refactored to be caller, and so has where it's called.
It sets an error in the usual way if contract writing is rejected -- that's
anticipating EVMC, where we'll use different error codes later.
Overall four behaviour changes:
1. On the callee side, it doesn't set `c.outputData` except for `REVERT`.
2. On the caller side, it doesn't read `child.outputData` except for `REVERT`.
3. There was a bug in processing before Homestead fork (EIP-2). We did not
match the spec or other implementations; now we do. When there's
insufficient gas, before Homestead it's treated as success but with an empty
contract.
https://github.com/ethereum/pyethereum/blob/d117c8f3fd93359fc641fd850fa799436f7c43b5/ethereum/processblock.py#L304
https://github.com/ethereum/go-ethereum/blob/401354976bb4/core/vm/instructions.go#L586
4. The Byzantium check has been removed, as it's unnecessary.
Signed-off-by: Jamie Lokier <jamie@shareable.org>
2021-12-02 19:44:51 +00:00
|
|
|
let gasParams = GasParams(kind: Create, cr_memLength: len)
|
2020-01-10 01:26:17 +00:00
|
|
|
let codeCost = c.gasCosts[Create].c_handler(0.u256, gasParams).gasCost
|
EVM: `writeContract` fixes, never return contract code as `RETURNDATA`
This fixes #867 "EIP-170 related consensus error at Goerli block 5080941", and
equivalent on other networks.
This combines a change on the EVM-caller side with an EVM-side change from
@jangko 6548ff98 "fixes CREATE/CREATE2's `returndata` bug", making the caller
EVM ignore any data except from `REVERT`.
Either change works by itself. The reason for both is to ensure we definitely
comply with ambiguous EVMC expectations from either side of that boundary, and
it makes the internal API clearer.
As well as fixing a specific consensus issue, there are some other EVM logic
changes too: Refactored `writeContract`, how `RETURNDATA` is handled inside the
EVM, and changed behaviour with quirks before EIP-2 (Homestead).
The fix allows sync to pass block 5080941 on Goerli, and probably equivalent on
other networks. Here's a trace at batch 5080897..5081088:
```
TRC 2021-10-01 21:18:12.883+01:00 Persisting blocks file=persist_blocks.nim:43 fromBlock=5080897 toBlock=5081088
...
DBG 2021-10-01 21:18:13.270+01:00 Contract code size exceeds EIP170 topics="vm computation" file=computation.nim:236 limit=24577 actual=31411
DBG 2021-10-01 21:18:13.271+01:00 gasUsed neq cumulativeGasUsed file=process_block.nim:68 block=5080941/0A3537BC5BDFC637349E1C77D9648F2F65E2BF973ABF7956618F854B769DF626 gasUsed=3129669 cumulativeGasUsed=3132615
TRC 2021-10-01 21:18:13.271+01:00 peer disconnected file=blockchain_sync.nim:407 peer=<IP:PORT>
```
Although it says "Contract code size" and "gasUsed", this bug is more general
than either contract size or gas. It's due to incorrect behaviour of EVM
instructions `RETURNDATA` and `RETURNDATASIZE`.
Sometimes when `writeContract` decides to reject writing the contract for any
of several reasons (for example just insufficient gas), the unwritten contract
code was being used as the "return data", and given to the caller. If the
caller used `RETURNDATA` or `RETURNDATASIZE` ops, those incorrectly reported
the contract code that didn't get written.
EIP-211 (https://eips.ethereum.org/EIPS/eip-211) describes `RETURNDATA`:
> "`CREATE` and `CREATE2` are considered to return the empty buffer in the
> success case and the failure data in the failure case".
The language is ambiguous. In fact "failure case" means when the contract uses
`REVERT` to finish. It doesn't mean other failures like out of gas, EIP-170
limit, EIP-3541, etc.
To be thorough, and to ensure we always do the right thing with real EVMC when
that's finalised, this patch fixes the `RETURNDATA` issue in two places, either
of which make Goerli block 5080941 pass.
`writeContract` has been refactored to be caller, and so has where it's called.
It sets an error in the usual way if contract writing is rejected -- that's
anticipating EVMC, where we'll use different error codes later.
Overall four behaviour changes:
1. On the callee side, it doesn't set `c.outputData` except for `REVERT`.
2. On the caller side, it doesn't read `child.outputData` except for `REVERT`.
3. There was a bug in processing before Homestead fork (EIP-2). We did not
match the spec or other implementations; now we do. When there's
insufficient gas, before Homestead it's treated as success but with an empty
contract.
https://github.com/ethereum/pyethereum/blob/d117c8f3fd93359fc641fd850fa799436f7c43b5/ethereum/processblock.py#L304
https://github.com/ethereum/go-ethereum/blob/401354976bb4/core/vm/instructions.go#L586
4. The Byzantium check has been removed, as it's unnecessary.
Signed-off-by: Jamie Lokier <jamie@shareable.org>
2021-12-02 19:44:51 +00:00
|
|
|
if codeCost <= c.gasMeter.gasRemaining:
|
|
|
|
c.gasMeter.consumeGas(codeCost, reason = "Write new contract code")
|
2020-01-10 01:26:17 +00:00
|
|
|
c.vmState.mutateStateDb:
|
EVM: `writeContract` fixes, never return contract code as `RETURNDATA`
This fixes #867 "EIP-170 related consensus error at Goerli block 5080941", and
equivalent on other networks.
This combines a change on the EVM-caller side with an EVM-side change from
@jangko 6548ff98 "fixes CREATE/CREATE2's `returndata` bug", making the caller
EVM ignore any data except from `REVERT`.
Either change works by itself. The reason for both is to ensure we definitely
comply with ambiguous EVMC expectations from either side of that boundary, and
it makes the internal API clearer.
As well as fixing a specific consensus issue, there are some other EVM logic
changes too: Refactored `writeContract`, how `RETURNDATA` is handled inside the
EVM, and changed behaviour with quirks before EIP-2 (Homestead).
The fix allows sync to pass block 5080941 on Goerli, and probably equivalent on
other networks. Here's a trace at batch 5080897..5081088:
```
TRC 2021-10-01 21:18:12.883+01:00 Persisting blocks file=persist_blocks.nim:43 fromBlock=5080897 toBlock=5081088
...
DBG 2021-10-01 21:18:13.270+01:00 Contract code size exceeds EIP170 topics="vm computation" file=computation.nim:236 limit=24577 actual=31411
DBG 2021-10-01 21:18:13.271+01:00 gasUsed neq cumulativeGasUsed file=process_block.nim:68 block=5080941/0A3537BC5BDFC637349E1C77D9648F2F65E2BF973ABF7956618F854B769DF626 gasUsed=3129669 cumulativeGasUsed=3132615
TRC 2021-10-01 21:18:13.271+01:00 peer disconnected file=blockchain_sync.nim:407 peer=<IP:PORT>
```
Although it says "Contract code size" and "gasUsed", this bug is more general
than either contract size or gas. It's due to incorrect behaviour of EVM
instructions `RETURNDATA` and `RETURNDATASIZE`.
Sometimes when `writeContract` decides to reject writing the contract for any
of several reasons (for example just insufficient gas), the unwritten contract
code was being used as the "return data", and given to the caller. If the
caller used `RETURNDATA` or `RETURNDATASIZE` ops, those incorrectly reported
the contract code that didn't get written.
EIP-211 (https://eips.ethereum.org/EIPS/eip-211) describes `RETURNDATA`:
> "`CREATE` and `CREATE2` are considered to return the empty buffer in the
> success case and the failure data in the failure case".
The language is ambiguous. In fact "failure case" means when the contract uses
`REVERT` to finish. It doesn't mean other failures like out of gas, EIP-170
limit, EIP-3541, etc.
To be thorough, and to ensure we always do the right thing with real EVMC when
that's finalised, this patch fixes the `RETURNDATA` issue in two places, either
of which make Goerli block 5080941 pass.
`writeContract` has been refactored to be caller, and so has where it's called.
It sets an error in the usual way if contract writing is rejected -- that's
anticipating EVMC, where we'll use different error codes later.
Overall four behaviour changes:
1. On the callee side, it doesn't set `c.outputData` except for `REVERT`.
2. On the caller side, it doesn't read `child.outputData` except for `REVERT`.
3. There was a bug in processing before Homestead fork (EIP-2). We did not
match the spec or other implementations; now we do. When there's
insufficient gas, before Homestead it's treated as success but with an empty
contract.
https://github.com/ethereum/pyethereum/blob/d117c8f3fd93359fc641fd850fa799436f7c43b5/ethereum/processblock.py#L304
https://github.com/ethereum/go-ethereum/blob/401354976bb4/core/vm/instructions.go#L586
4. The Byzantium check has been removed, as it's unnecessary.
Signed-off-by: Jamie Lokier <jamie@shareable.org>
2021-12-02 19:44:51 +00:00
|
|
|
db.setCode(c.msg.contractAddress, c.output)
|
|
|
|
withExtra trace, "Writing new contract code"
|
|
|
|
return
|
|
|
|
|
|
|
|
if fork >= FkHomestead:
|
|
|
|
# EIP-2 (https://eips.ethereum.org/EIPS/eip-2).
|
|
|
|
# TODO: Return `EVMC_OUT_OF_GAS` (like Silkworm).
|
|
|
|
c.setError("EVMC_OUT_OF_GAS", true)
|
2019-03-19 15:19:08 +00:00
|
|
|
else:
|
EVM: `writeContract` fixes, never return contract code as `RETURNDATA`
This fixes #867 "EIP-170 related consensus error at Goerli block 5080941", and
equivalent on other networks.
This combines a change on the EVM-caller side with an EVM-side change from
@jangko 6548ff98 "fixes CREATE/CREATE2's `returndata` bug", making the caller
EVM ignore any data except from `REVERT`.
Either change works by itself. The reason for both is to ensure we definitely
comply with ambiguous EVMC expectations from either side of that boundary, and
it makes the internal API clearer.
As well as fixing a specific consensus issue, there are some other EVM logic
changes too: Refactored `writeContract`, how `RETURNDATA` is handled inside the
EVM, and changed behaviour with quirks before EIP-2 (Homestead).
The fix allows sync to pass block 5080941 on Goerli, and probably equivalent on
other networks. Here's a trace at batch 5080897..5081088:
```
TRC 2021-10-01 21:18:12.883+01:00 Persisting blocks file=persist_blocks.nim:43 fromBlock=5080897 toBlock=5081088
...
DBG 2021-10-01 21:18:13.270+01:00 Contract code size exceeds EIP170 topics="vm computation" file=computation.nim:236 limit=24577 actual=31411
DBG 2021-10-01 21:18:13.271+01:00 gasUsed neq cumulativeGasUsed file=process_block.nim:68 block=5080941/0A3537BC5BDFC637349E1C77D9648F2F65E2BF973ABF7956618F854B769DF626 gasUsed=3129669 cumulativeGasUsed=3132615
TRC 2021-10-01 21:18:13.271+01:00 peer disconnected file=blockchain_sync.nim:407 peer=<IP:PORT>
```
Although it says "Contract code size" and "gasUsed", this bug is more general
than either contract size or gas. It's due to incorrect behaviour of EVM
instructions `RETURNDATA` and `RETURNDATASIZE`.
Sometimes when `writeContract` decides to reject writing the contract for any
of several reasons (for example just insufficient gas), the unwritten contract
code was being used as the "return data", and given to the caller. If the
caller used `RETURNDATA` or `RETURNDATASIZE` ops, those incorrectly reported
the contract code that didn't get written.
EIP-211 (https://eips.ethereum.org/EIPS/eip-211) describes `RETURNDATA`:
> "`CREATE` and `CREATE2` are considered to return the empty buffer in the
> success case and the failure data in the failure case".
The language is ambiguous. In fact "failure case" means when the contract uses
`REVERT` to finish. It doesn't mean other failures like out of gas, EIP-170
limit, EIP-3541, etc.
To be thorough, and to ensure we always do the right thing with real EVMC when
that's finalised, this patch fixes the `RETURNDATA` issue in two places, either
of which make Goerli block 5080941 pass.
`writeContract` has been refactored to be caller, and so has where it's called.
It sets an error in the usual way if contract writing is rejected -- that's
anticipating EVMC, where we'll use different error codes later.
Overall four behaviour changes:
1. On the callee side, it doesn't set `c.outputData` except for `REVERT`.
2. On the caller side, it doesn't read `child.outputData` except for `REVERT`.
3. There was a bug in processing before Homestead fork (EIP-2). We did not
match the spec or other implementations; now we do. When there's
insufficient gas, before Homestead it's treated as success but with an empty
contract.
https://github.com/ethereum/pyethereum/blob/d117c8f3fd93359fc641fd850fa799436f7c43b5/ethereum/processblock.py#L304
https://github.com/ethereum/go-ethereum/blob/401354976bb4/core/vm/instructions.go#L586
4. The Byzantium check has been removed, as it's unnecessary.
Signed-off-by: Jamie Lokier <jamie@shareable.org>
2021-12-02 19:44:51 +00:00
|
|
|
# Before EIP-2, when out of gas for code storage, the account ends up with
|
|
|
|
# zero-length code and no error. No gas is charged. Code cited in EIP-2:
|
|
|
|
# https://github.com/ethereum/pyethereum/blob/d117c8f3fd93/ethereum/processblock.py#L304
|
|
|
|
# https://github.com/ethereum/go-ethereum/blob/401354976bb4/core/vm/instructions.go#L586
|
|
|
|
# The account already has zero-length code to handle nested calls.
|
|
|
|
withExtra trace, "New contract given empty code by pre-Homestead rules"
|
2019-03-19 15:19:08 +00:00
|
|
|
|
2020-01-09 07:52:19 +00:00
|
|
|
proc initAddress(x: int): EthAddress {.compileTime.} = result[19] = x.byte
|
2020-02-04 11:18:30 +00:00
|
|
|
const ripemdAddr = initAddress(3)
|
2020-01-10 01:26:17 +00:00
|
|
|
proc executeOpcodes*(c: Computation) {.gcsafe.}
|
2019-03-19 16:30:35 +00:00
|
|
|
|
EVM: Small patch that reduces EVM stack usage to almost nothing
There's been a lot of talk about the Nimbus EVM "stack problem". I think we
assumed changing it would require big changes to the interpreter code, touching
a lot of functions.
It turned out to be a low hanging fruit.
This patch solves the stack problem, but hardly touches anything. The change
in EVM stack memory is from 13 MB worst case to just 48 kB, a 250x reduction.
I've been doing work on the database/storage/trie code. While looking at the
API between the EVM and the database/storage/trie, this stack patch stood out
and made itself obvious. As it's tiny, rather than more talk, here it is.
Note: This patch is intentionally small, non-invasive, and hopefully easy to
understand, so that it doesn't conflict with other work done on the EVM, and
can easily be grafted into any other EVM structure.
Motivation
==========
- We run out of space and crash on some targets, unless the stack limit is
raised above its default. Surprise segmentation faults are unhelpful.
- Some CI targets have been disabled for months due to this.
- Because usage borders on the system limits, when working on
database/storage/trie/sync code (called from the EVM), segmentation faults
occur and are misleading. They cause lost time due to thinking there's a
crash bug in the code being worked on, when there's nothing wrong with it.
- Sometimes unrelated, trivial code changes elsewhere trigger CI test failures.
It looks like abrupt termination. A simple, recent patch was crashing in
`make test` even though it was a trivial refactor. Turns out it pushed the
stack over the edge.
- A large stack has to be scanned by the Nim garbage collector sometimes.
Larger stack means slower GC and memory allocation.
- The structure of this small patch suggests how to weave async into the EVM
with almost no changes to the EVM, and no async transformation overhead.
- The patch seemed obvious when working on the API between EVM and storage.
Measurements before
===================
All these tests were run on Ubuntu 20.04 server, x86-64. This is one of the
targets that has been disabled for a while in CI in EVMC mode due to crashing,
and excessive stack usage is the cause.
Testing commit 0c34a8e3 `2021-04-08 17:46:00 +0200 CI: use MSYS2 on Windows`.
$ rm -f build/all_tests && make ENABLE_EVMC=1 test
$ ulimit -S -s 16384 # Requires larger stack than default to avoid crash.
$ ./build/all_tests 9 | tee tlog
[Suite] persist block json tests
...
Stack range 38496 depthHigh 3
...
Stack range 13140272 depthHigh 1024
[OK] tests/fixtures/PersistBlockTests/block1431916.json
These tests use 13.14 MB of stack to run, and so crash with the default stack
limit on Ubuntu Server 20.04 (8MB). Exactly 12832 bytes per EVM call stack
frame. It's interesting to see some stack frames take a bit more.
$ rm -f build/all_tests && make ENABLE_EVMC=1 test
$ ulimit -S -s 16384 # Requires larger stack than default.
$ ./build/all_tests 7 | tee tlog
[Suite] new generalstate json tests
...
Stack range 15488 depthHigh 2
...
Stack range 3539312 depthHigh 457
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest639.json
...
Stack range 3756144 depthHigh 485
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest458.json
...
Stack range 7929968 depthHigh 1024
[OK] tests/fixtures/eth_tests/GeneralStateTests/stCreate2/Create2OnDepth1024.json
These tests use 7.92MB of stack to run. About 7264 bytes per EVM call stack
frame. It _only just_ avoids crashing with the default Ubuntu Server stack
limit of 8 MB. However, it still crashes on Windows x86-64, which is why the
CI target is currently disabled.
On Linux where this passes, this is so borderline that it affects work and
testing of storage and sync code, because that's called from the EVM. Which
was a motivation for dealing with the stack instead of letting this linger.
Also, this stack greatly exceeds the default thread stack size.
$ rm -f build/all_tests && make ENABLE_EVMC=0 test
$ ulimit -S -s 16384 # Requires larger stack than default to avoid crash.
$ ./build/all_tests 9 | tee tlog
[Suite] persist block json tests
...
Stack range 33216 depthHigh 3
...
Stack range 11338032 depthHigh 1024
[OK] tests/fixtures/PersistBlockTests/block1431916.json
These tests use 11.33 MB stack to run, and so crash with a default stack limit
of 8MB. Exactly 11072 bytes per EVM call stack frame. It's interesting to see
some stack frames take a bit more.
$ rm -f build/all_tests && make ENABLE_EVMC=0 test
$ ulimit -S -s 16384 # Requires larger stack than default.
$ ./build/all_tests 7 | tee tlog
[Suite] new generalstate json tests
...
Stack range 10224 depthHigh 2
...
Stack range 2471760 depthHigh 457
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest639.json
...
Stack range 2623184 depthHigh 485
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest458.json
...
Stack range 5537824 depthHigh 1024
[OK] tests/fixtures/eth_tests/GeneralStateTests/stCreate2/Create2OnDepth1024.json
These tests use 5.54 MB of stack to run, and avoid crashing on with a default
stack limit of 8 MB. About 5408 bytes per EVM call stack frame.
However, this is uncomfortably close to the limit, as the stack frame size is
sensitive to changes in the code.
Also, this stack greatly exceeds the default thread stack size.
Measurements after
==================
(This patch doesn't address EVMC mode, which is not our default. EVMC stack
usage remains about the same. EVMC mode is addressed in another tiny patch.)
$ rm -f build/all_tests && make ENABLE_EVMC=0 test
$ ulimit -S -s 80 # Because we can! 80k stack.
$ ./build/all_tests 9 | tee tlog
[Suite] persist block json tests
...
Stack range 496 depthHigh 3
...
Stack range 49504 depthHigh 1024
[OK] tests/fixtures/PersistBlockTests/block1431916.json
$ rm -f build/all_tests && make ENABLE_EVMC=0 test
$ ulimit -S -s 72 # Because we can! 72k stack.
$ ./build/all_tests 7 | tee tlog
[Suite] new generalstate json tests
...
Stack range 448 depthHigh 2
...
Stack range 22288 depthHigh 457
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest639.json
...
Stack range 23632 depthHigh 485
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest458.json
...
Stack range 49504 depthHigh 1024
[OK] tests/fixtures/eth_tests/GeneralStateTests/stCreate2/Create2OnDepth1024.json
For both tests, a satisfying *48 bytes* per EVM call stack frame, and EVM takes
not much more than 48 kB. With other overheads, both tests run in 80 kB stack
total at maximum EVM depth.
We must add some headroom on this for database activity called from the EVM,
and different compile targets. But it means the EVM itself is no longer a
stack burden.
This is much smaller than the default thread stack size on Linux (2MB), with
plenty of margin. It's even smaller than Linux from a long time ago (128kB),
and some small embedded C targets. (Just fyi, though, some JVM environments
allocated just 32 kB to thread stacks.)
This size is also well suited to running EVMs in threads, if that's useful.
Subtle exception handling and `dispose`
=======================================
It is important that each `snapshot` has a corresponding `dispose` in the event
of an exception being raised. This code does do that, but in a subtle way.
The pair of functions `execCallOrCreate` and `execCallOrCreateAux` are
equivalent to the following code, where you can see `dispose` more clearly:
proc execCallOrCreate*(c: Computation) =
defer: c.dispose()
if c.beforeExec():
return
c.executeOpcodes()
while not c.continuation.isNil:
c.child.execCallOrCreate()
c.child = nil
(c.continuation)()
c.executeOpcodes()
c.afterExec()
That works fine, but only reduces the stack used to 300-700 kB instead of 48 kB.
To get lower we split the above into separate `execCallOrCreate` and
`execCallOrCreateAux`. Only the outermost has `defer`, and instead of handling
one level, it walks the entire `c.parent` chain calling `dispose` if needed.
The inner one avoids `defer`, which greatly reduces the size of its stackframe.
`c` is a `var` parameter, at each level of recursion. So the outermost proc
sees the temporary changes made by all inner calls. This is why `c` is updated
and the `c.parent` chain is maintained at each step.
Signed-off-by: Jamie Lokier <jamie@shareable.org>
2021-04-12 17:06:31 +00:00
|
|
|
proc beforeExecCall(c: Computation) =
|
2020-02-03 05:37:33 +00:00
|
|
|
c.snapshot()
|
|
|
|
if c.msg.kind == evmcCall:
|
|
|
|
c.vmState.mutateStateDb:
|
|
|
|
db.subBalance(c.msg.sender, c.msg.value)
|
|
|
|
db.addBalance(c.msg.contractAddress, c.msg.value)
|
|
|
|
|
EVM: Small patch that reduces EVM stack usage to almost nothing
There's been a lot of talk about the Nimbus EVM "stack problem". I think we
assumed changing it would require big changes to the interpreter code, touching
a lot of functions.
It turned out to be a low hanging fruit.
This patch solves the stack problem, but hardly touches anything. The change
in EVM stack memory is from 13 MB worst case to just 48 kB, a 250x reduction.
I've been doing work on the database/storage/trie code. While looking at the
API between the EVM and the database/storage/trie, this stack patch stood out
and made itself obvious. As it's tiny, rather than more talk, here it is.
Note: This patch is intentionally small, non-invasive, and hopefully easy to
understand, so that it doesn't conflict with other work done on the EVM, and
can easily be grafted into any other EVM structure.
Motivation
==========
- We run out of space and crash on some targets, unless the stack limit is
raised above its default. Surprise segmentation faults are unhelpful.
- Some CI targets have been disabled for months due to this.
- Because usage borders on the system limits, when working on
database/storage/trie/sync code (called from the EVM), segmentation faults
occur and are misleading. They cause lost time due to thinking there's a
crash bug in the code being worked on, when there's nothing wrong with it.
- Sometimes unrelated, trivial code changes elsewhere trigger CI test failures.
It looks like abrupt termination. A simple, recent patch was crashing in
`make test` even though it was a trivial refactor. Turns out it pushed the
stack over the edge.
- A large stack has to be scanned by the Nim garbage collector sometimes.
Larger stack means slower GC and memory allocation.
- The structure of this small patch suggests how to weave async into the EVM
with almost no changes to the EVM, and no async transformation overhead.
- The patch seemed obvious when working on the API between EVM and storage.
Measurements before
===================
All these tests were run on Ubuntu 20.04 server, x86-64. This is one of the
targets that has been disabled for a while in CI in EVMC mode due to crashing,
and excessive stack usage is the cause.
Testing commit 0c34a8e3 `2021-04-08 17:46:00 +0200 CI: use MSYS2 on Windows`.
$ rm -f build/all_tests && make ENABLE_EVMC=1 test
$ ulimit -S -s 16384 # Requires larger stack than default to avoid crash.
$ ./build/all_tests 9 | tee tlog
[Suite] persist block json tests
...
Stack range 38496 depthHigh 3
...
Stack range 13140272 depthHigh 1024
[OK] tests/fixtures/PersistBlockTests/block1431916.json
These tests use 13.14 MB of stack to run, and so crash with the default stack
limit on Ubuntu Server 20.04 (8MB). Exactly 12832 bytes per EVM call stack
frame. It's interesting to see some stack frames take a bit more.
$ rm -f build/all_tests && make ENABLE_EVMC=1 test
$ ulimit -S -s 16384 # Requires larger stack than default.
$ ./build/all_tests 7 | tee tlog
[Suite] new generalstate json tests
...
Stack range 15488 depthHigh 2
...
Stack range 3539312 depthHigh 457
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest639.json
...
Stack range 3756144 depthHigh 485
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest458.json
...
Stack range 7929968 depthHigh 1024
[OK] tests/fixtures/eth_tests/GeneralStateTests/stCreate2/Create2OnDepth1024.json
These tests use 7.92MB of stack to run. About 7264 bytes per EVM call stack
frame. It _only just_ avoids crashing with the default Ubuntu Server stack
limit of 8 MB. However, it still crashes on Windows x86-64, which is why the
CI target is currently disabled.
On Linux where this passes, this is so borderline that it affects work and
testing of storage and sync code, because that's called from the EVM. Which
was a motivation for dealing with the stack instead of letting this linger.
Also, this stack greatly exceeds the default thread stack size.
$ rm -f build/all_tests && make ENABLE_EVMC=0 test
$ ulimit -S -s 16384 # Requires larger stack than default to avoid crash.
$ ./build/all_tests 9 | tee tlog
[Suite] persist block json tests
...
Stack range 33216 depthHigh 3
...
Stack range 11338032 depthHigh 1024
[OK] tests/fixtures/PersistBlockTests/block1431916.json
These tests use 11.33 MB stack to run, and so crash with a default stack limit
of 8MB. Exactly 11072 bytes per EVM call stack frame. It's interesting to see
some stack frames take a bit more.
$ rm -f build/all_tests && make ENABLE_EVMC=0 test
$ ulimit -S -s 16384 # Requires larger stack than default.
$ ./build/all_tests 7 | tee tlog
[Suite] new generalstate json tests
...
Stack range 10224 depthHigh 2
...
Stack range 2471760 depthHigh 457
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest639.json
...
Stack range 2623184 depthHigh 485
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest458.json
...
Stack range 5537824 depthHigh 1024
[OK] tests/fixtures/eth_tests/GeneralStateTests/stCreate2/Create2OnDepth1024.json
These tests use 5.54 MB of stack to run, and avoid crashing on with a default
stack limit of 8 MB. About 5408 bytes per EVM call stack frame.
However, this is uncomfortably close to the limit, as the stack frame size is
sensitive to changes in the code.
Also, this stack greatly exceeds the default thread stack size.
Measurements after
==================
(This patch doesn't address EVMC mode, which is not our default. EVMC stack
usage remains about the same. EVMC mode is addressed in another tiny patch.)
$ rm -f build/all_tests && make ENABLE_EVMC=0 test
$ ulimit -S -s 80 # Because we can! 80k stack.
$ ./build/all_tests 9 | tee tlog
[Suite] persist block json tests
...
Stack range 496 depthHigh 3
...
Stack range 49504 depthHigh 1024
[OK] tests/fixtures/PersistBlockTests/block1431916.json
$ rm -f build/all_tests && make ENABLE_EVMC=0 test
$ ulimit -S -s 72 # Because we can! 72k stack.
$ ./build/all_tests 7 | tee tlog
[Suite] new generalstate json tests
...
Stack range 448 depthHigh 2
...
Stack range 22288 depthHigh 457
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest639.json
...
Stack range 23632 depthHigh 485
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest458.json
...
Stack range 49504 depthHigh 1024
[OK] tests/fixtures/eth_tests/GeneralStateTests/stCreate2/Create2OnDepth1024.json
For both tests, a satisfying *48 bytes* per EVM call stack frame, and EVM takes
not much more than 48 kB. With other overheads, both tests run in 80 kB stack
total at maximum EVM depth.
We must add some headroom on this for database activity called from the EVM,
and different compile targets. But it means the EVM itself is no longer a
stack burden.
This is much smaller than the default thread stack size on Linux (2MB), with
plenty of margin. It's even smaller than Linux from a long time ago (128kB),
and some small embedded C targets. (Just fyi, though, some JVM environments
allocated just 32 kB to thread stacks.)
This size is also well suited to running EVMs in threads, if that's useful.
Subtle exception handling and `dispose`
=======================================
It is important that each `snapshot` has a corresponding `dispose` in the event
of an exception being raised. This code does do that, but in a subtle way.
The pair of functions `execCallOrCreate` and `execCallOrCreateAux` are
equivalent to the following code, where you can see `dispose` more clearly:
proc execCallOrCreate*(c: Computation) =
defer: c.dispose()
if c.beforeExec():
return
c.executeOpcodes()
while not c.continuation.isNil:
c.child.execCallOrCreate()
c.child = nil
(c.continuation)()
c.executeOpcodes()
c.afterExec()
That works fine, but only reduces the stack used to 300-700 kB instead of 48 kB.
To get lower we split the above into separate `execCallOrCreate` and
`execCallOrCreateAux`. Only the outermost has `defer`, and instead of handling
one level, it walks the entire `c.parent` chain calling `dispose` if needed.
The inner one avoids `defer`, which greatly reduces the size of its stackframe.
`c` is a `var` parameter, at each level of recursion. So the outermost proc
sees the temporary changes made by all inner calls. This is why `c` is updated
and the `c.parent` chain is maintained at each step.
Signed-off-by: Jamie Lokier <jamie@shareable.org>
2021-04-12 17:06:31 +00:00
|
|
|
proc afterExecCall(c: Computation) =
|
2020-02-04 11:18:30 +00:00
|
|
|
## Collect all of the accounts that *may* need to be deleted based on EIP161
|
|
|
|
## https://github.com/ethereum/EIPs/blob/master/EIPS/eip-161.md
|
|
|
|
## also see: https://github.com/ethereum/EIPs/issues/716
|
|
|
|
|
2021-01-13 01:08:56 +00:00
|
|
|
if c.isError or c.fork >= FKByzantium:
|
2020-02-04 11:00:23 +00:00
|
|
|
if c.msg.contractAddress == ripemdAddr:
|
2020-02-04 11:18:30 +00:00
|
|
|
# Special case to account for geth+parity bug
|
2020-02-04 11:00:23 +00:00
|
|
|
c.vmState.touchedAccounts.incl c.msg.contractAddress
|
2020-02-04 11:18:30 +00:00
|
|
|
|
2020-02-03 05:37:33 +00:00
|
|
|
if c.isSuccess:
|
|
|
|
c.commit()
|
2020-02-04 11:00:23 +00:00
|
|
|
c.touchedAccounts.incl c.msg.contractAddress
|
2020-02-03 05:37:33 +00:00
|
|
|
else:
|
|
|
|
c.rollback()
|
|
|
|
|
EVM: Small patch that reduces EVM stack usage to almost nothing
There's been a lot of talk about the Nimbus EVM "stack problem". I think we
assumed changing it would require big changes to the interpreter code, touching
a lot of functions.
It turned out to be a low hanging fruit.
This patch solves the stack problem, but hardly touches anything. The change
in EVM stack memory is from 13 MB worst case to just 48 kB, a 250x reduction.
I've been doing work on the database/storage/trie code. While looking at the
API between the EVM and the database/storage/trie, this stack patch stood out
and made itself obvious. As it's tiny, rather than more talk, here it is.
Note: This patch is intentionally small, non-invasive, and hopefully easy to
understand, so that it doesn't conflict with other work done on the EVM, and
can easily be grafted into any other EVM structure.
Motivation
==========
- We run out of space and crash on some targets, unless the stack limit is
raised above its default. Surprise segmentation faults are unhelpful.
- Some CI targets have been disabled for months due to this.
- Because usage borders on the system limits, when working on
database/storage/trie/sync code (called from the EVM), segmentation faults
occur and are misleading. They cause lost time due to thinking there's a
crash bug in the code being worked on, when there's nothing wrong with it.
- Sometimes unrelated, trivial code changes elsewhere trigger CI test failures.
It looks like abrupt termination. A simple, recent patch was crashing in
`make test` even though it was a trivial refactor. Turns out it pushed the
stack over the edge.
- A large stack has to be scanned by the Nim garbage collector sometimes.
Larger stack means slower GC and memory allocation.
- The structure of this small patch suggests how to weave async into the EVM
with almost no changes to the EVM, and no async transformation overhead.
- The patch seemed obvious when working on the API between EVM and storage.
Measurements before
===================
All these tests were run on Ubuntu 20.04 server, x86-64. This is one of the
targets that has been disabled for a while in CI in EVMC mode due to crashing,
and excessive stack usage is the cause.
Testing commit 0c34a8e3 `2021-04-08 17:46:00 +0200 CI: use MSYS2 on Windows`.
$ rm -f build/all_tests && make ENABLE_EVMC=1 test
$ ulimit -S -s 16384 # Requires larger stack than default to avoid crash.
$ ./build/all_tests 9 | tee tlog
[Suite] persist block json tests
...
Stack range 38496 depthHigh 3
...
Stack range 13140272 depthHigh 1024
[OK] tests/fixtures/PersistBlockTests/block1431916.json
These tests use 13.14 MB of stack to run, and so crash with the default stack
limit on Ubuntu Server 20.04 (8MB). Exactly 12832 bytes per EVM call stack
frame. It's interesting to see some stack frames take a bit more.
$ rm -f build/all_tests && make ENABLE_EVMC=1 test
$ ulimit -S -s 16384 # Requires larger stack than default.
$ ./build/all_tests 7 | tee tlog
[Suite] new generalstate json tests
...
Stack range 15488 depthHigh 2
...
Stack range 3539312 depthHigh 457
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest639.json
...
Stack range 3756144 depthHigh 485
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest458.json
...
Stack range 7929968 depthHigh 1024
[OK] tests/fixtures/eth_tests/GeneralStateTests/stCreate2/Create2OnDepth1024.json
These tests use 7.92MB of stack to run. About 7264 bytes per EVM call stack
frame. It _only just_ avoids crashing with the default Ubuntu Server stack
limit of 8 MB. However, it still crashes on Windows x86-64, which is why the
CI target is currently disabled.
On Linux where this passes, this is so borderline that it affects work and
testing of storage and sync code, because that's called from the EVM. Which
was a motivation for dealing with the stack instead of letting this linger.
Also, this stack greatly exceeds the default thread stack size.
$ rm -f build/all_tests && make ENABLE_EVMC=0 test
$ ulimit -S -s 16384 # Requires larger stack than default to avoid crash.
$ ./build/all_tests 9 | tee tlog
[Suite] persist block json tests
...
Stack range 33216 depthHigh 3
...
Stack range 11338032 depthHigh 1024
[OK] tests/fixtures/PersistBlockTests/block1431916.json
These tests use 11.33 MB stack to run, and so crash with a default stack limit
of 8MB. Exactly 11072 bytes per EVM call stack frame. It's interesting to see
some stack frames take a bit more.
$ rm -f build/all_tests && make ENABLE_EVMC=0 test
$ ulimit -S -s 16384 # Requires larger stack than default.
$ ./build/all_tests 7 | tee tlog
[Suite] new generalstate json tests
...
Stack range 10224 depthHigh 2
...
Stack range 2471760 depthHigh 457
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest639.json
...
Stack range 2623184 depthHigh 485
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest458.json
...
Stack range 5537824 depthHigh 1024
[OK] tests/fixtures/eth_tests/GeneralStateTests/stCreate2/Create2OnDepth1024.json
These tests use 5.54 MB of stack to run, and avoid crashing on with a default
stack limit of 8 MB. About 5408 bytes per EVM call stack frame.
However, this is uncomfortably close to the limit, as the stack frame size is
sensitive to changes in the code.
Also, this stack greatly exceeds the default thread stack size.
Measurements after
==================
(This patch doesn't address EVMC mode, which is not our default. EVMC stack
usage remains about the same. EVMC mode is addressed in another tiny patch.)
$ rm -f build/all_tests && make ENABLE_EVMC=0 test
$ ulimit -S -s 80 # Because we can! 80k stack.
$ ./build/all_tests 9 | tee tlog
[Suite] persist block json tests
...
Stack range 496 depthHigh 3
...
Stack range 49504 depthHigh 1024
[OK] tests/fixtures/PersistBlockTests/block1431916.json
$ rm -f build/all_tests && make ENABLE_EVMC=0 test
$ ulimit -S -s 72 # Because we can! 72k stack.
$ ./build/all_tests 7 | tee tlog
[Suite] new generalstate json tests
...
Stack range 448 depthHigh 2
...
Stack range 22288 depthHigh 457
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest639.json
...
Stack range 23632 depthHigh 485
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest458.json
...
Stack range 49504 depthHigh 1024
[OK] tests/fixtures/eth_tests/GeneralStateTests/stCreate2/Create2OnDepth1024.json
For both tests, a satisfying *48 bytes* per EVM call stack frame, and EVM takes
not much more than 48 kB. With other overheads, both tests run in 80 kB stack
total at maximum EVM depth.
We must add some headroom on this for database activity called from the EVM,
and different compile targets. But it means the EVM itself is no longer a
stack burden.
This is much smaller than the default thread stack size on Linux (2MB), with
plenty of margin. It's even smaller than Linux from a long time ago (128kB),
and some small embedded C targets. (Just fyi, though, some JVM environments
allocated just 32 kB to thread stacks.)
This size is also well suited to running EVMs in threads, if that's useful.
Subtle exception handling and `dispose`
=======================================
It is important that each `snapshot` has a corresponding `dispose` in the event
of an exception being raised. This code does do that, but in a subtle way.
The pair of functions `execCallOrCreate` and `execCallOrCreateAux` are
equivalent to the following code, where you can see `dispose` more clearly:
proc execCallOrCreate*(c: Computation) =
defer: c.dispose()
if c.beforeExec():
return
c.executeOpcodes()
while not c.continuation.isNil:
c.child.execCallOrCreate()
c.child = nil
(c.continuation)()
c.executeOpcodes()
c.afterExec()
That works fine, but only reduces the stack used to 300-700 kB instead of 48 kB.
To get lower we split the above into separate `execCallOrCreate` and
`execCallOrCreateAux`. Only the outermost has `defer`, and instead of handling
one level, it walks the entire `c.parent` chain calling `dispose` if needed.
The inner one avoids `defer`, which greatly reduces the size of its stackframe.
`c` is a `var` parameter, at each level of recursion. So the outermost proc
sees the temporary changes made by all inner calls. This is why `c` is updated
and the `c.parent` chain is maintained at each step.
Signed-off-by: Jamie Lokier <jamie@shareable.org>
2021-04-12 17:06:31 +00:00
|
|
|
proc beforeExecCreate(c: Computation): bool =
|
2020-01-31 01:08:44 +00:00
|
|
|
c.vmState.mutateStateDB:
|
|
|
|
db.incNonce(c.msg.sender)
|
|
|
|
|
2020-12-11 10:51:17 +00:00
|
|
|
# We add this to the access list _before_ taking a snapshot.
|
|
|
|
# Even if the creation fails, the access-list change should not be rolled back
|
|
|
|
# EIP2929
|
|
|
|
if c.fork >= FkBerlin:
|
|
|
|
db.accessList(c.msg.contractAddress)
|
|
|
|
|
2020-01-31 01:08:44 +00:00
|
|
|
c.snapshot()
|
|
|
|
|
|
|
|
if c.vmState.readOnlyStateDb().hasCodeOrNonce(c.msg.contractAddress):
|
|
|
|
c.setError("Address collision when creating contract address={c.msg.contractAddress.toHex}", true)
|
|
|
|
c.rollback()
|
EVM: Small patch that reduces EVM stack usage to almost nothing
There's been a lot of talk about the Nimbus EVM "stack problem". I think we
assumed changing it would require big changes to the interpreter code, touching
a lot of functions.
It turned out to be a low hanging fruit.
This patch solves the stack problem, but hardly touches anything. The change
in EVM stack memory is from 13 MB worst case to just 48 kB, a 250x reduction.
I've been doing work on the database/storage/trie code. While looking at the
API between the EVM and the database/storage/trie, this stack patch stood out
and made itself obvious. As it's tiny, rather than more talk, here it is.
Note: This patch is intentionally small, non-invasive, and hopefully easy to
understand, so that it doesn't conflict with other work done on the EVM, and
can easily be grafted into any other EVM structure.
Motivation
==========
- We run out of space and crash on some targets, unless the stack limit is
raised above its default. Surprise segmentation faults are unhelpful.
- Some CI targets have been disabled for months due to this.
- Because usage borders on the system limits, when working on
database/storage/trie/sync code (called from the EVM), segmentation faults
occur and are misleading. They cause lost time due to thinking there's a
crash bug in the code being worked on, when there's nothing wrong with it.
- Sometimes unrelated, trivial code changes elsewhere trigger CI test failures.
It looks like abrupt termination. A simple, recent patch was crashing in
`make test` even though it was a trivial refactor. Turns out it pushed the
stack over the edge.
- A large stack has to be scanned by the Nim garbage collector sometimes.
Larger stack means slower GC and memory allocation.
- The structure of this small patch suggests how to weave async into the EVM
with almost no changes to the EVM, and no async transformation overhead.
- The patch seemed obvious when working on the API between EVM and storage.
Measurements before
===================
All these tests were run on Ubuntu 20.04 server, x86-64. This is one of the
targets that has been disabled for a while in CI in EVMC mode due to crashing,
and excessive stack usage is the cause.
Testing commit 0c34a8e3 `2021-04-08 17:46:00 +0200 CI: use MSYS2 on Windows`.
$ rm -f build/all_tests && make ENABLE_EVMC=1 test
$ ulimit -S -s 16384 # Requires larger stack than default to avoid crash.
$ ./build/all_tests 9 | tee tlog
[Suite] persist block json tests
...
Stack range 38496 depthHigh 3
...
Stack range 13140272 depthHigh 1024
[OK] tests/fixtures/PersistBlockTests/block1431916.json
These tests use 13.14 MB of stack to run, and so crash with the default stack
limit on Ubuntu Server 20.04 (8MB). Exactly 12832 bytes per EVM call stack
frame. It's interesting to see some stack frames take a bit more.
$ rm -f build/all_tests && make ENABLE_EVMC=1 test
$ ulimit -S -s 16384 # Requires larger stack than default.
$ ./build/all_tests 7 | tee tlog
[Suite] new generalstate json tests
...
Stack range 15488 depthHigh 2
...
Stack range 3539312 depthHigh 457
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest639.json
...
Stack range 3756144 depthHigh 485
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest458.json
...
Stack range 7929968 depthHigh 1024
[OK] tests/fixtures/eth_tests/GeneralStateTests/stCreate2/Create2OnDepth1024.json
These tests use 7.92MB of stack to run. About 7264 bytes per EVM call stack
frame. It _only just_ avoids crashing with the default Ubuntu Server stack
limit of 8 MB. However, it still crashes on Windows x86-64, which is why the
CI target is currently disabled.
On Linux where this passes, this is so borderline that it affects work and
testing of storage and sync code, because that's called from the EVM. Which
was a motivation for dealing with the stack instead of letting this linger.
Also, this stack greatly exceeds the default thread stack size.
$ rm -f build/all_tests && make ENABLE_EVMC=0 test
$ ulimit -S -s 16384 # Requires larger stack than default to avoid crash.
$ ./build/all_tests 9 | tee tlog
[Suite] persist block json tests
...
Stack range 33216 depthHigh 3
...
Stack range 11338032 depthHigh 1024
[OK] tests/fixtures/PersistBlockTests/block1431916.json
These tests use 11.33 MB stack to run, and so crash with a default stack limit
of 8MB. Exactly 11072 bytes per EVM call stack frame. It's interesting to see
some stack frames take a bit more.
$ rm -f build/all_tests && make ENABLE_EVMC=0 test
$ ulimit -S -s 16384 # Requires larger stack than default.
$ ./build/all_tests 7 | tee tlog
[Suite] new generalstate json tests
...
Stack range 10224 depthHigh 2
...
Stack range 2471760 depthHigh 457
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest639.json
...
Stack range 2623184 depthHigh 485
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest458.json
...
Stack range 5537824 depthHigh 1024
[OK] tests/fixtures/eth_tests/GeneralStateTests/stCreate2/Create2OnDepth1024.json
These tests use 5.54 MB of stack to run, and avoid crashing on with a default
stack limit of 8 MB. About 5408 bytes per EVM call stack frame.
However, this is uncomfortably close to the limit, as the stack frame size is
sensitive to changes in the code.
Also, this stack greatly exceeds the default thread stack size.
Measurements after
==================
(This patch doesn't address EVMC mode, which is not our default. EVMC stack
usage remains about the same. EVMC mode is addressed in another tiny patch.)
$ rm -f build/all_tests && make ENABLE_EVMC=0 test
$ ulimit -S -s 80 # Because we can! 80k stack.
$ ./build/all_tests 9 | tee tlog
[Suite] persist block json tests
...
Stack range 496 depthHigh 3
...
Stack range 49504 depthHigh 1024
[OK] tests/fixtures/PersistBlockTests/block1431916.json
$ rm -f build/all_tests && make ENABLE_EVMC=0 test
$ ulimit -S -s 72 # Because we can! 72k stack.
$ ./build/all_tests 7 | tee tlog
[Suite] new generalstate json tests
...
Stack range 448 depthHigh 2
...
Stack range 22288 depthHigh 457
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest639.json
...
Stack range 23632 depthHigh 485
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest458.json
...
Stack range 49504 depthHigh 1024
[OK] tests/fixtures/eth_tests/GeneralStateTests/stCreate2/Create2OnDepth1024.json
For both tests, a satisfying *48 bytes* per EVM call stack frame, and EVM takes
not much more than 48 kB. With other overheads, both tests run in 80 kB stack
total at maximum EVM depth.
We must add some headroom on this for database activity called from the EVM,
and different compile targets. But it means the EVM itself is no longer a
stack burden.
This is much smaller than the default thread stack size on Linux (2MB), with
plenty of margin. It's even smaller than Linux from a long time ago (128kB),
and some small embedded C targets. (Just fyi, though, some JVM environments
allocated just 32 kB to thread stacks.)
This size is also well suited to running EVMs in threads, if that's useful.
Subtle exception handling and `dispose`
=======================================
It is important that each `snapshot` has a corresponding `dispose` in the event
of an exception being raised. This code does do that, but in a subtle way.
The pair of functions `execCallOrCreate` and `execCallOrCreateAux` are
equivalent to the following code, where you can see `dispose` more clearly:
proc execCallOrCreate*(c: Computation) =
defer: c.dispose()
if c.beforeExec():
return
c.executeOpcodes()
while not c.continuation.isNil:
c.child.execCallOrCreate()
c.child = nil
(c.continuation)()
c.executeOpcodes()
c.afterExec()
That works fine, but only reduces the stack used to 300-700 kB instead of 48 kB.
To get lower we split the above into separate `execCallOrCreate` and
`execCallOrCreateAux`. Only the outermost has `defer`, and instead of handling
one level, it walks the entire `c.parent` chain calling `dispose` if needed.
The inner one avoids `defer`, which greatly reduces the size of its stackframe.
`c` is a `var` parameter, at each level of recursion. So the outermost proc
sees the temporary changes made by all inner calls. This is why `c` is updated
and the `c.parent` chain is maintained at each step.
Signed-off-by: Jamie Lokier <jamie@shareable.org>
2021-04-12 17:06:31 +00:00
|
|
|
return true
|
2020-01-31 01:08:44 +00:00
|
|
|
|
|
|
|
c.vmState.mutateStateDb:
|
|
|
|
db.subBalance(c.msg.sender, c.msg.value)
|
|
|
|
db.addBalance(c.msg.contractAddress, c.msg.value)
|
|
|
|
db.clearStorage(c.msg.contractAddress)
|
|
|
|
if c.fork >= FkSpurious:
|
|
|
|
# EIP161 nonce incrementation
|
|
|
|
db.incNonce(c.msg.contractAddress)
|
|
|
|
|
EVM: Small patch that reduces EVM stack usage to almost nothing
There's been a lot of talk about the Nimbus EVM "stack problem". I think we
assumed changing it would require big changes to the interpreter code, touching
a lot of functions.
It turned out to be a low hanging fruit.
This patch solves the stack problem, but hardly touches anything. The change
in EVM stack memory is from 13 MB worst case to just 48 kB, a 250x reduction.
I've been doing work on the database/storage/trie code. While looking at the
API between the EVM and the database/storage/trie, this stack patch stood out
and made itself obvious. As it's tiny, rather than more talk, here it is.
Note: This patch is intentionally small, non-invasive, and hopefully easy to
understand, so that it doesn't conflict with other work done on the EVM, and
can easily be grafted into any other EVM structure.
Motivation
==========
- We run out of space and crash on some targets, unless the stack limit is
raised above its default. Surprise segmentation faults are unhelpful.
- Some CI targets have been disabled for months due to this.
- Because usage borders on the system limits, when working on
database/storage/trie/sync code (called from the EVM), segmentation faults
occur and are misleading. They cause lost time due to thinking there's a
crash bug in the code being worked on, when there's nothing wrong with it.
- Sometimes unrelated, trivial code changes elsewhere trigger CI test failures.
It looks like abrupt termination. A simple, recent patch was crashing in
`make test` even though it was a trivial refactor. Turns out it pushed the
stack over the edge.
- A large stack has to be scanned by the Nim garbage collector sometimes.
Larger stack means slower GC and memory allocation.
- The structure of this small patch suggests how to weave async into the EVM
with almost no changes to the EVM, and no async transformation overhead.
- The patch seemed obvious when working on the API between EVM and storage.
Measurements before
===================
All these tests were run on Ubuntu 20.04 server, x86-64. This is one of the
targets that has been disabled for a while in CI in EVMC mode due to crashing,
and excessive stack usage is the cause.
Testing commit 0c34a8e3 `2021-04-08 17:46:00 +0200 CI: use MSYS2 on Windows`.
$ rm -f build/all_tests && make ENABLE_EVMC=1 test
$ ulimit -S -s 16384 # Requires larger stack than default to avoid crash.
$ ./build/all_tests 9 | tee tlog
[Suite] persist block json tests
...
Stack range 38496 depthHigh 3
...
Stack range 13140272 depthHigh 1024
[OK] tests/fixtures/PersistBlockTests/block1431916.json
These tests use 13.14 MB of stack to run, and so crash with the default stack
limit on Ubuntu Server 20.04 (8MB). Exactly 12832 bytes per EVM call stack
frame. It's interesting to see some stack frames take a bit more.
$ rm -f build/all_tests && make ENABLE_EVMC=1 test
$ ulimit -S -s 16384 # Requires larger stack than default.
$ ./build/all_tests 7 | tee tlog
[Suite] new generalstate json tests
...
Stack range 15488 depthHigh 2
...
Stack range 3539312 depthHigh 457
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest639.json
...
Stack range 3756144 depthHigh 485
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest458.json
...
Stack range 7929968 depthHigh 1024
[OK] tests/fixtures/eth_tests/GeneralStateTests/stCreate2/Create2OnDepth1024.json
These tests use 7.92MB of stack to run. About 7264 bytes per EVM call stack
frame. It _only just_ avoids crashing with the default Ubuntu Server stack
limit of 8 MB. However, it still crashes on Windows x86-64, which is why the
CI target is currently disabled.
On Linux where this passes, this is so borderline that it affects work and
testing of storage and sync code, because that's called from the EVM. Which
was a motivation for dealing with the stack instead of letting this linger.
Also, this stack greatly exceeds the default thread stack size.
$ rm -f build/all_tests && make ENABLE_EVMC=0 test
$ ulimit -S -s 16384 # Requires larger stack than default to avoid crash.
$ ./build/all_tests 9 | tee tlog
[Suite] persist block json tests
...
Stack range 33216 depthHigh 3
...
Stack range 11338032 depthHigh 1024
[OK] tests/fixtures/PersistBlockTests/block1431916.json
These tests use 11.33 MB stack to run, and so crash with a default stack limit
of 8MB. Exactly 11072 bytes per EVM call stack frame. It's interesting to see
some stack frames take a bit more.
$ rm -f build/all_tests && make ENABLE_EVMC=0 test
$ ulimit -S -s 16384 # Requires larger stack than default.
$ ./build/all_tests 7 | tee tlog
[Suite] new generalstate json tests
...
Stack range 10224 depthHigh 2
...
Stack range 2471760 depthHigh 457
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest639.json
...
Stack range 2623184 depthHigh 485
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest458.json
...
Stack range 5537824 depthHigh 1024
[OK] tests/fixtures/eth_tests/GeneralStateTests/stCreate2/Create2OnDepth1024.json
These tests use 5.54 MB of stack to run, and avoid crashing on with a default
stack limit of 8 MB. About 5408 bytes per EVM call stack frame.
However, this is uncomfortably close to the limit, as the stack frame size is
sensitive to changes in the code.
Also, this stack greatly exceeds the default thread stack size.
Measurements after
==================
(This patch doesn't address EVMC mode, which is not our default. EVMC stack
usage remains about the same. EVMC mode is addressed in another tiny patch.)
$ rm -f build/all_tests && make ENABLE_EVMC=0 test
$ ulimit -S -s 80 # Because we can! 80k stack.
$ ./build/all_tests 9 | tee tlog
[Suite] persist block json tests
...
Stack range 496 depthHigh 3
...
Stack range 49504 depthHigh 1024
[OK] tests/fixtures/PersistBlockTests/block1431916.json
$ rm -f build/all_tests && make ENABLE_EVMC=0 test
$ ulimit -S -s 72 # Because we can! 72k stack.
$ ./build/all_tests 7 | tee tlog
[Suite] new generalstate json tests
...
Stack range 448 depthHigh 2
...
Stack range 22288 depthHigh 457
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest639.json
...
Stack range 23632 depthHigh 485
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest458.json
...
Stack range 49504 depthHigh 1024
[OK] tests/fixtures/eth_tests/GeneralStateTests/stCreate2/Create2OnDepth1024.json
For both tests, a satisfying *48 bytes* per EVM call stack frame, and EVM takes
not much more than 48 kB. With other overheads, both tests run in 80 kB stack
total at maximum EVM depth.
We must add some headroom on this for database activity called from the EVM,
and different compile targets. But it means the EVM itself is no longer a
stack burden.
This is much smaller than the default thread stack size on Linux (2MB), with
plenty of margin. It's even smaller than Linux from a long time ago (128kB),
and some small embedded C targets. (Just fyi, though, some JVM environments
allocated just 32 kB to thread stacks.)
This size is also well suited to running EVMs in threads, if that's useful.
Subtle exception handling and `dispose`
=======================================
It is important that each `snapshot` has a corresponding `dispose` in the event
of an exception being raised. This code does do that, but in a subtle way.
The pair of functions `execCallOrCreate` and `execCallOrCreateAux` are
equivalent to the following code, where you can see `dispose` more clearly:
proc execCallOrCreate*(c: Computation) =
defer: c.dispose()
if c.beforeExec():
return
c.executeOpcodes()
while not c.continuation.isNil:
c.child.execCallOrCreate()
c.child = nil
(c.continuation)()
c.executeOpcodes()
c.afterExec()
That works fine, but only reduces the stack used to 300-700 kB instead of 48 kB.
To get lower we split the above into separate `execCallOrCreate` and
`execCallOrCreateAux`. Only the outermost has `defer`, and instead of handling
one level, it walks the entire `c.parent` chain calling `dispose` if needed.
The inner one avoids `defer`, which greatly reduces the size of its stackframe.
`c` is a `var` parameter, at each level of recursion. So the outermost proc
sees the temporary changes made by all inner calls. This is why `c` is updated
and the `c.parent` chain is maintained at each step.
Signed-off-by: Jamie Lokier <jamie@shareable.org>
2021-04-12 17:06:31 +00:00
|
|
|
return false
|
2020-01-31 01:08:44 +00:00
|
|
|
|
EVM: Small patch that reduces EVM stack usage to almost nothing
There's been a lot of talk about the Nimbus EVM "stack problem". I think we
assumed changing it would require big changes to the interpreter code, touching
a lot of functions.
It turned out to be a low hanging fruit.
This patch solves the stack problem, but hardly touches anything. The change
in EVM stack memory is from 13 MB worst case to just 48 kB, a 250x reduction.
I've been doing work on the database/storage/trie code. While looking at the
API between the EVM and the database/storage/trie, this stack patch stood out
and made itself obvious. As it's tiny, rather than more talk, here it is.
Note: This patch is intentionally small, non-invasive, and hopefully easy to
understand, so that it doesn't conflict with other work done on the EVM, and
can easily be grafted into any other EVM structure.
Motivation
==========
- We run out of space and crash on some targets, unless the stack limit is
raised above its default. Surprise segmentation faults are unhelpful.
- Some CI targets have been disabled for months due to this.
- Because usage borders on the system limits, when working on
database/storage/trie/sync code (called from the EVM), segmentation faults
occur and are misleading. They cause lost time due to thinking there's a
crash bug in the code being worked on, when there's nothing wrong with it.
- Sometimes unrelated, trivial code changes elsewhere trigger CI test failures.
It looks like abrupt termination. A simple, recent patch was crashing in
`make test` even though it was a trivial refactor. Turns out it pushed the
stack over the edge.
- A large stack has to be scanned by the Nim garbage collector sometimes.
Larger stack means slower GC and memory allocation.
- The structure of this small patch suggests how to weave async into the EVM
with almost no changes to the EVM, and no async transformation overhead.
- The patch seemed obvious when working on the API between EVM and storage.
Measurements before
===================
All these tests were run on Ubuntu 20.04 server, x86-64. This is one of the
targets that has been disabled for a while in CI in EVMC mode due to crashing,
and excessive stack usage is the cause.
Testing commit 0c34a8e3 `2021-04-08 17:46:00 +0200 CI: use MSYS2 on Windows`.
$ rm -f build/all_tests && make ENABLE_EVMC=1 test
$ ulimit -S -s 16384 # Requires larger stack than default to avoid crash.
$ ./build/all_tests 9 | tee tlog
[Suite] persist block json tests
...
Stack range 38496 depthHigh 3
...
Stack range 13140272 depthHigh 1024
[OK] tests/fixtures/PersistBlockTests/block1431916.json
These tests use 13.14 MB of stack to run, and so crash with the default stack
limit on Ubuntu Server 20.04 (8MB). Exactly 12832 bytes per EVM call stack
frame. It's interesting to see some stack frames take a bit more.
$ rm -f build/all_tests && make ENABLE_EVMC=1 test
$ ulimit -S -s 16384 # Requires larger stack than default.
$ ./build/all_tests 7 | tee tlog
[Suite] new generalstate json tests
...
Stack range 15488 depthHigh 2
...
Stack range 3539312 depthHigh 457
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest639.json
...
Stack range 3756144 depthHigh 485
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest458.json
...
Stack range 7929968 depthHigh 1024
[OK] tests/fixtures/eth_tests/GeneralStateTests/stCreate2/Create2OnDepth1024.json
These tests use 7.92MB of stack to run. About 7264 bytes per EVM call stack
frame. It _only just_ avoids crashing with the default Ubuntu Server stack
limit of 8 MB. However, it still crashes on Windows x86-64, which is why the
CI target is currently disabled.
On Linux where this passes, this is so borderline that it affects work and
testing of storage and sync code, because that's called from the EVM. Which
was a motivation for dealing with the stack instead of letting this linger.
Also, this stack greatly exceeds the default thread stack size.
$ rm -f build/all_tests && make ENABLE_EVMC=0 test
$ ulimit -S -s 16384 # Requires larger stack than default to avoid crash.
$ ./build/all_tests 9 | tee tlog
[Suite] persist block json tests
...
Stack range 33216 depthHigh 3
...
Stack range 11338032 depthHigh 1024
[OK] tests/fixtures/PersistBlockTests/block1431916.json
These tests use 11.33 MB stack to run, and so crash with a default stack limit
of 8MB. Exactly 11072 bytes per EVM call stack frame. It's interesting to see
some stack frames take a bit more.
$ rm -f build/all_tests && make ENABLE_EVMC=0 test
$ ulimit -S -s 16384 # Requires larger stack than default.
$ ./build/all_tests 7 | tee tlog
[Suite] new generalstate json tests
...
Stack range 10224 depthHigh 2
...
Stack range 2471760 depthHigh 457
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest639.json
...
Stack range 2623184 depthHigh 485
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest458.json
...
Stack range 5537824 depthHigh 1024
[OK] tests/fixtures/eth_tests/GeneralStateTests/stCreate2/Create2OnDepth1024.json
These tests use 5.54 MB of stack to run, and avoid crashing on with a default
stack limit of 8 MB. About 5408 bytes per EVM call stack frame.
However, this is uncomfortably close to the limit, as the stack frame size is
sensitive to changes in the code.
Also, this stack greatly exceeds the default thread stack size.
Measurements after
==================
(This patch doesn't address EVMC mode, which is not our default. EVMC stack
usage remains about the same. EVMC mode is addressed in another tiny patch.)
$ rm -f build/all_tests && make ENABLE_EVMC=0 test
$ ulimit -S -s 80 # Because we can! 80k stack.
$ ./build/all_tests 9 | tee tlog
[Suite] persist block json tests
...
Stack range 496 depthHigh 3
...
Stack range 49504 depthHigh 1024
[OK] tests/fixtures/PersistBlockTests/block1431916.json
$ rm -f build/all_tests && make ENABLE_EVMC=0 test
$ ulimit -S -s 72 # Because we can! 72k stack.
$ ./build/all_tests 7 | tee tlog
[Suite] new generalstate json tests
...
Stack range 448 depthHigh 2
...
Stack range 22288 depthHigh 457
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest639.json
...
Stack range 23632 depthHigh 485
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest458.json
...
Stack range 49504 depthHigh 1024
[OK] tests/fixtures/eth_tests/GeneralStateTests/stCreate2/Create2OnDepth1024.json
For both tests, a satisfying *48 bytes* per EVM call stack frame, and EVM takes
not much more than 48 kB. With other overheads, both tests run in 80 kB stack
total at maximum EVM depth.
We must add some headroom on this for database activity called from the EVM,
and different compile targets. But it means the EVM itself is no longer a
stack burden.
This is much smaller than the default thread stack size on Linux (2MB), with
plenty of margin. It's even smaller than Linux from a long time ago (128kB),
and some small embedded C targets. (Just fyi, though, some JVM environments
allocated just 32 kB to thread stacks.)
This size is also well suited to running EVMs in threads, if that's useful.
Subtle exception handling and `dispose`
=======================================
It is important that each `snapshot` has a corresponding `dispose` in the event
of an exception being raised. This code does do that, but in a subtle way.
The pair of functions `execCallOrCreate` and `execCallOrCreateAux` are
equivalent to the following code, where you can see `dispose` more clearly:
proc execCallOrCreate*(c: Computation) =
defer: c.dispose()
if c.beforeExec():
return
c.executeOpcodes()
while not c.continuation.isNil:
c.child.execCallOrCreate()
c.child = nil
(c.continuation)()
c.executeOpcodes()
c.afterExec()
That works fine, but only reduces the stack used to 300-700 kB instead of 48 kB.
To get lower we split the above into separate `execCallOrCreate` and
`execCallOrCreateAux`. Only the outermost has `defer`, and instead of handling
one level, it walks the entire `c.parent` chain calling `dispose` if needed.
The inner one avoids `defer`, which greatly reduces the size of its stackframe.
`c` is a `var` parameter, at each level of recursion. So the outermost proc
sees the temporary changes made by all inner calls. This is why `c` is updated
and the `c.parent` chain is maintained at each step.
Signed-off-by: Jamie Lokier <jamie@shareable.org>
2021-04-12 17:06:31 +00:00
|
|
|
proc afterExecCreate(c: Computation) =
|
2020-01-31 01:08:44 +00:00
|
|
|
if c.isSuccess:
|
EVM: `writeContract` fixes, never return contract code as `RETURNDATA`
This fixes #867 "EIP-170 related consensus error at Goerli block 5080941", and
equivalent on other networks.
This combines a change on the EVM-caller side with an EVM-side change from
@jangko 6548ff98 "fixes CREATE/CREATE2's `returndata` bug", making the caller
EVM ignore any data except from `REVERT`.
Either change works by itself. The reason for both is to ensure we definitely
comply with ambiguous EVMC expectations from either side of that boundary, and
it makes the internal API clearer.
As well as fixing a specific consensus issue, there are some other EVM logic
changes too: Refactored `writeContract`, how `RETURNDATA` is handled inside the
EVM, and changed behaviour with quirks before EIP-2 (Homestead).
The fix allows sync to pass block 5080941 on Goerli, and probably equivalent on
other networks. Here's a trace at batch 5080897..5081088:
```
TRC 2021-10-01 21:18:12.883+01:00 Persisting blocks file=persist_blocks.nim:43 fromBlock=5080897 toBlock=5081088
...
DBG 2021-10-01 21:18:13.270+01:00 Contract code size exceeds EIP170 topics="vm computation" file=computation.nim:236 limit=24577 actual=31411
DBG 2021-10-01 21:18:13.271+01:00 gasUsed neq cumulativeGasUsed file=process_block.nim:68 block=5080941/0A3537BC5BDFC637349E1C77D9648F2F65E2BF973ABF7956618F854B769DF626 gasUsed=3129669 cumulativeGasUsed=3132615
TRC 2021-10-01 21:18:13.271+01:00 peer disconnected file=blockchain_sync.nim:407 peer=<IP:PORT>
```
Although it says "Contract code size" and "gasUsed", this bug is more general
than either contract size or gas. It's due to incorrect behaviour of EVM
instructions `RETURNDATA` and `RETURNDATASIZE`.
Sometimes when `writeContract` decides to reject writing the contract for any
of several reasons (for example just insufficient gas), the unwritten contract
code was being used as the "return data", and given to the caller. If the
caller used `RETURNDATA` or `RETURNDATASIZE` ops, those incorrectly reported
the contract code that didn't get written.
EIP-211 (https://eips.ethereum.org/EIPS/eip-211) describes `RETURNDATA`:
> "`CREATE` and `CREATE2` are considered to return the empty buffer in the
> success case and the failure data in the failure case".
The language is ambiguous. In fact "failure case" means when the contract uses
`REVERT` to finish. It doesn't mean other failures like out of gas, EIP-170
limit, EIP-3541, etc.
To be thorough, and to ensure we always do the right thing with real EVMC when
that's finalised, this patch fixes the `RETURNDATA` issue in two places, either
of which make Goerli block 5080941 pass.
`writeContract` has been refactored to be caller, and so has where it's called.
It sets an error in the usual way if contract writing is rejected -- that's
anticipating EVMC, where we'll use different error codes later.
Overall four behaviour changes:
1. On the callee side, it doesn't set `c.outputData` except for `REVERT`.
2. On the caller side, it doesn't read `child.outputData` except for `REVERT`.
3. There was a bug in processing before Homestead fork (EIP-2). We did not
match the spec or other implementations; now we do. When there's
insufficient gas, before Homestead it's treated as success but with an empty
contract.
https://github.com/ethereum/pyethereum/blob/d117c8f3fd93359fc641fd850fa799436f7c43b5/ethereum/processblock.py#L304
https://github.com/ethereum/go-ethereum/blob/401354976bb4/core/vm/instructions.go#L586
4. The Byzantium check has been removed, as it's unnecessary.
Signed-off-by: Jamie Lokier <jamie@shareable.org>
2021-12-02 19:44:51 +00:00
|
|
|
# This can change `c.isSuccess`.
|
|
|
|
c.writeContract()
|
|
|
|
# Contract code should never be returned to the caller. Only data from
|
|
|
|
# `REVERT` is returned after a create. Clearing in this branch covers the
|
|
|
|
# right cases, particularly important with EVMC where it must be cleared.
|
|
|
|
if c.output.len > 0:
|
|
|
|
c.output = @[]
|
2020-01-31 01:08:44 +00:00
|
|
|
|
|
|
|
if c.isSuccess:
|
|
|
|
c.commit()
|
|
|
|
else:
|
|
|
|
c.rollback()
|
|
|
|
|
EVMC: Small stacks when using EVMC, closes #575 (segfaults)
This patch reduces stack space used with EVM in ENABLE_EVMC=1 mode, from 13 MB
worst case to 550 kB, a 24x reduction.
This completes fixing the "stack problem" and closes #575 (`EVM: Different
segmentation faults when running the test suite with EVMC`).
It also closes #256 (`recursive EVM call trigger unrecoverable stack overflow`).
After this patch, it is possible to re-enable the CI targets which had to be
disabled due to #575.
This change is also a required precursor for switching over to "nearly EVMC" as
the clean and focused Nimbus-internal API between EVM and sync/database
processes, and is also key to the use of Chronos `async` in those processes
when calling the EVM.
(The motivation is the internal interface has to be substantially changed
_anyway_ for the parallel sync and database processes, and EVMC turns out to be
well-designed and well-suited for this. It provides good separation between
modules, and suits our needs better than our other current interface. Might as
well use a good one designed by someone else. EVMC is 98% done in Nimbus
thanks to great work done before by @jangko, and we can use Nimbus-specific
extensions where we need flexibility, including for performance. Being aligned
with the ecosystem is a useful bonus feature.)
All tests below were run on Ubuntu 20.04 LTS server, x86-64. This matches one
of the targets that has been disabled for a while in CI in EVMC mode due to
stack overflow crashing the tests, so it's a good choice.
Measurements before
===================
Testing commit `e76e0144 2021-04-22 11:29:42 +0700 add submodules: graphql and
toml-serialization`.
$ rm -f build/all_tests && make ENABLE_EVMC=1 test
$ ulimit -S -s 16384 # Requires larger stack than default to avoid crash.
$ ./build/all_tests 9 | tee tlog
[Suite] persist block json tests
...
Stack range 38416 depthHigh 3
...
Stack range 13074720 depthHigh 1024
[OK] tests/fixtures/PersistBlockTests/block1431916.json
These tests use 13.07 MB of stack to run, and so crash with the default stack
limit on Ubuntu Server 20.04 (8MB). Exactly 12768 bytes per EVM call stack
frame.
$ rm -f build/all_tests && make ENABLE_EVMC=1 test
$ ulimit -S -s 16384 # Requires larger stack than default.
$ ./build/all_tests 7 | tee tlog
[Suite] new generalstate json tests
...
Stack range 14384 depthHigh 2
...
Stack range 3495456 depthHigh 457
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest639.json
...
Stack range 3709600 depthHigh 485
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest458.json
...
Stack range 7831600 depthHigh 1024
[OK] tests/fixtures/eth_tests/GeneralStateTests/stCreate2/Create2OnDepth1024.json
These tests use 7.83MB of stack to run. About 7648 bytes per EVM call stack
frame. It _only just_ avoids crashing with the default Ubuntu Server stack
limit of 8 MB. However, it still crashes on Windows x86-64, which is why the
Windows CI EVMC target is currently disabled.
On Linux where this passes, this is so borderline that it affects work and
testing of the complex storage code, because that's called from the EVM.
Also, this greatly exceeds the default thread stack size.
Measurements after
==================
$ rm -f build/all_tests && make ENABLE_EVMC=1 test
$ ulimit -S -s 600 # Because we can! 600k stack.
$ ./build/all_tests 9 | tee tlog
[Suite] persist block json tests
...
Stack range 1936 depthHigh 3
...
Stack range 556272 depthHigh 1022
Stack range 556512 depthHigh 1023
Stack range 556816 depthHigh 1023
Stack range 557056 depthHigh 1024
Stack range 557360 depthHigh 1024
[OK] tests/fixtures/PersistBlockTests/block1431916.json
$ rm -f build/all_tests && make ENABLE_EVMC=1 test
$ ulimit -S -s 600 # Because we can! 600k stack.
$ ./build/all_tests 7 | tee tlog
[Suite] new generalstate json tests
...
Stack range 1392 depthHigh 2
...
Stack range 248912 depthHigh 457
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest639.json
...
Stack range 264144 depthHigh 485
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest458.json
...
Stack range 557360 depthHigh 1024
[OK] tests/fixtures/eth_tests/GeneralStateTests/stStaticCall/static_CallRecursiveBombPreCall.json
For both tests, a satisfying *544 bytes* per EVM call stack frame, and EVM
takes less than 600 kB total. With other overheads, both tests run in 600 kB
stack total at maximum EVM depth.
We must add some headroom on this for database activity called from the EVM,
and different compile targets. But it means the EVM itself is no longer a
stack burden.
This is much smaller than the default thread stack size on Linux (2MB), with
plenty of margin. (Just fyi, it isn't smaller than a _small_ thread stack on
Linux from a long time ago (128kB), and some small embedded C targets.)
This size is well suited to running EVMs in threads.
Further reduction
=================
This patch solves the stack problem. Windows and Linux 64-bit EVMC CI targets
can be re-enabled, and there is no longer a problem with stack usage.
We can reduce further to ~340 bytes per frame and 350 kB total, while still
complying with EVMC. But as this involves changing how errors are handled to
comply fully with EVMC, and removing `dispose` calls, it's not worth doing now
while there are other EVMC changes in progress that will have the same effect.
A Nimbus-specific extension will allow us to avoid recursion with EVMC anyway,
bringing bytes per frame to zero. We need the extension anyway, to support
Chronos `async` which parallel transaction processing is built around.
Interop with non-Nimbus over EVMC won't let us avoid recursion, but then we
can't control the stack frame size either. To prevent stack overflow in
interop I anticipate using (this method in Aleth)
[https://github.com/ethereum/aleth/blob/6e96ce34e3f131e2d42f3cb00741b54e05ab029d/libethereum/ExtVM.cpp#L61].
Smoke test other versions of GCC and Clang/LLVM
===============================================
As all builds including Windows use GCC or Apple's Clang/LLVM, this is just to
verify we're in the right ballpark on all targets. I've only checked `x86_64`
though, not 32-bit, and not ARM.
It's interesting to see GCC 10 uses less stack. This is because it optimises
`struct` returns better, sometimes skipping an intermediate copy. Here it
benefits the EVMC API, but I found GCC 10 also improves the larger stack usage
of the rest of `nimbus-eth1` as well.
Apple clang 12.0.0 (clang-1200.0.26.2) on MacOS 10.15:
- 544 bytes per EVM call stack frame
GCC 10.3.0 (Ubuntu 10.3.0-1ubuntu1) on Ubuntu 21.04:
- 464 bytes per EVM call stack frame
GCC 10.2.0 (Ubuntu 10.2.0-5ubuntu1~20.04) on Ubuntu 20.04 LTS:
- 464 bytes per EVM call stack frame
GCC 11.0.1 20210417 (experimental; Ubuntu 11-20210417-1ubuntu1) on Ubuntu 21.04:
- 8 bytes per EVM call stack frame
GCC 9.3.0 (Ubuntu 9.3.0-17ubuntu1~20.04) on Ubuntu 20.04 LTS:
- 544 bytes per EVM call stack frame
GCC 8.4.0 (Ubuntu 8.4.0-3ubuntu2) on Ubuntu 20.04 LTS:
- 544 bytes per EVM call stack frame
GCC 7.5.0 (Ubuntu 7.5.0-6ubuntu2) on Ubuntu 20.04 LTS:
- 544 bytes per EVM call stack frame
GCC 9.2.1 20191008 (Ubuntu 9.2.1-9ubuntu2) on Ubuntu 19.10:
- 528 bytes per EVM call stack frame
Signed-off-by: Jamie Lokier <jamie@shareable.org>
2021-04-26 15:01:00 +00:00
|
|
|
proc beforeExec(c: Computation): bool {.noinline.} =
|
EVM: Small patch that reduces EVM stack usage to almost nothing
There's been a lot of talk about the Nimbus EVM "stack problem". I think we
assumed changing it would require big changes to the interpreter code, touching
a lot of functions.
It turned out to be a low hanging fruit.
This patch solves the stack problem, but hardly touches anything. The change
in EVM stack memory is from 13 MB worst case to just 48 kB, a 250x reduction.
I've been doing work on the database/storage/trie code. While looking at the
API between the EVM and the database/storage/trie, this stack patch stood out
and made itself obvious. As it's tiny, rather than more talk, here it is.
Note: This patch is intentionally small, non-invasive, and hopefully easy to
understand, so that it doesn't conflict with other work done on the EVM, and
can easily be grafted into any other EVM structure.
Motivation
==========
- We run out of space and crash on some targets, unless the stack limit is
raised above its default. Surprise segmentation faults are unhelpful.
- Some CI targets have been disabled for months due to this.
- Because usage borders on the system limits, when working on
database/storage/trie/sync code (called from the EVM), segmentation faults
occur and are misleading. They cause lost time due to thinking there's a
crash bug in the code being worked on, when there's nothing wrong with it.
- Sometimes unrelated, trivial code changes elsewhere trigger CI test failures.
It looks like abrupt termination. A simple, recent patch was crashing in
`make test` even though it was a trivial refactor. Turns out it pushed the
stack over the edge.
- A large stack has to be scanned by the Nim garbage collector sometimes.
Larger stack means slower GC and memory allocation.
- The structure of this small patch suggests how to weave async into the EVM
with almost no changes to the EVM, and no async transformation overhead.
- The patch seemed obvious when working on the API between EVM and storage.
Measurements before
===================
All these tests were run on Ubuntu 20.04 server, x86-64. This is one of the
targets that has been disabled for a while in CI in EVMC mode due to crashing,
and excessive stack usage is the cause.
Testing commit 0c34a8e3 `2021-04-08 17:46:00 +0200 CI: use MSYS2 on Windows`.
$ rm -f build/all_tests && make ENABLE_EVMC=1 test
$ ulimit -S -s 16384 # Requires larger stack than default to avoid crash.
$ ./build/all_tests 9 | tee tlog
[Suite] persist block json tests
...
Stack range 38496 depthHigh 3
...
Stack range 13140272 depthHigh 1024
[OK] tests/fixtures/PersistBlockTests/block1431916.json
These tests use 13.14 MB of stack to run, and so crash with the default stack
limit on Ubuntu Server 20.04 (8MB). Exactly 12832 bytes per EVM call stack
frame. It's interesting to see some stack frames take a bit more.
$ rm -f build/all_tests && make ENABLE_EVMC=1 test
$ ulimit -S -s 16384 # Requires larger stack than default.
$ ./build/all_tests 7 | tee tlog
[Suite] new generalstate json tests
...
Stack range 15488 depthHigh 2
...
Stack range 3539312 depthHigh 457
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest639.json
...
Stack range 3756144 depthHigh 485
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest458.json
...
Stack range 7929968 depthHigh 1024
[OK] tests/fixtures/eth_tests/GeneralStateTests/stCreate2/Create2OnDepth1024.json
These tests use 7.92MB of stack to run. About 7264 bytes per EVM call stack
frame. It _only just_ avoids crashing with the default Ubuntu Server stack
limit of 8 MB. However, it still crashes on Windows x86-64, which is why the
CI target is currently disabled.
On Linux where this passes, this is so borderline that it affects work and
testing of storage and sync code, because that's called from the EVM. Which
was a motivation for dealing with the stack instead of letting this linger.
Also, this stack greatly exceeds the default thread stack size.
$ rm -f build/all_tests && make ENABLE_EVMC=0 test
$ ulimit -S -s 16384 # Requires larger stack than default to avoid crash.
$ ./build/all_tests 9 | tee tlog
[Suite] persist block json tests
...
Stack range 33216 depthHigh 3
...
Stack range 11338032 depthHigh 1024
[OK] tests/fixtures/PersistBlockTests/block1431916.json
These tests use 11.33 MB stack to run, and so crash with a default stack limit
of 8MB. Exactly 11072 bytes per EVM call stack frame. It's interesting to see
some stack frames take a bit more.
$ rm -f build/all_tests && make ENABLE_EVMC=0 test
$ ulimit -S -s 16384 # Requires larger stack than default.
$ ./build/all_tests 7 | tee tlog
[Suite] new generalstate json tests
...
Stack range 10224 depthHigh 2
...
Stack range 2471760 depthHigh 457
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest639.json
...
Stack range 2623184 depthHigh 485
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest458.json
...
Stack range 5537824 depthHigh 1024
[OK] tests/fixtures/eth_tests/GeneralStateTests/stCreate2/Create2OnDepth1024.json
These tests use 5.54 MB of stack to run, and avoid crashing on with a default
stack limit of 8 MB. About 5408 bytes per EVM call stack frame.
However, this is uncomfortably close to the limit, as the stack frame size is
sensitive to changes in the code.
Also, this stack greatly exceeds the default thread stack size.
Measurements after
==================
(This patch doesn't address EVMC mode, which is not our default. EVMC stack
usage remains about the same. EVMC mode is addressed in another tiny patch.)
$ rm -f build/all_tests && make ENABLE_EVMC=0 test
$ ulimit -S -s 80 # Because we can! 80k stack.
$ ./build/all_tests 9 | tee tlog
[Suite] persist block json tests
...
Stack range 496 depthHigh 3
...
Stack range 49504 depthHigh 1024
[OK] tests/fixtures/PersistBlockTests/block1431916.json
$ rm -f build/all_tests && make ENABLE_EVMC=0 test
$ ulimit -S -s 72 # Because we can! 72k stack.
$ ./build/all_tests 7 | tee tlog
[Suite] new generalstate json tests
...
Stack range 448 depthHigh 2
...
Stack range 22288 depthHigh 457
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest639.json
...
Stack range 23632 depthHigh 485
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest458.json
...
Stack range 49504 depthHigh 1024
[OK] tests/fixtures/eth_tests/GeneralStateTests/stCreate2/Create2OnDepth1024.json
For both tests, a satisfying *48 bytes* per EVM call stack frame, and EVM takes
not much more than 48 kB. With other overheads, both tests run in 80 kB stack
total at maximum EVM depth.
We must add some headroom on this for database activity called from the EVM,
and different compile targets. But it means the EVM itself is no longer a
stack burden.
This is much smaller than the default thread stack size on Linux (2MB), with
plenty of margin. It's even smaller than Linux from a long time ago (128kB),
and some small embedded C targets. (Just fyi, though, some JVM environments
allocated just 32 kB to thread stacks.)
This size is also well suited to running EVMs in threads, if that's useful.
Subtle exception handling and `dispose`
=======================================
It is important that each `snapshot` has a corresponding `dispose` in the event
of an exception being raised. This code does do that, but in a subtle way.
The pair of functions `execCallOrCreate` and `execCallOrCreateAux` are
equivalent to the following code, where you can see `dispose` more clearly:
proc execCallOrCreate*(c: Computation) =
defer: c.dispose()
if c.beforeExec():
return
c.executeOpcodes()
while not c.continuation.isNil:
c.child.execCallOrCreate()
c.child = nil
(c.continuation)()
c.executeOpcodes()
c.afterExec()
That works fine, but only reduces the stack used to 300-700 kB instead of 48 kB.
To get lower we split the above into separate `execCallOrCreate` and
`execCallOrCreateAux`. Only the outermost has `defer`, and instead of handling
one level, it walks the entire `c.parent` chain calling `dispose` if needed.
The inner one avoids `defer`, which greatly reduces the size of its stackframe.
`c` is a `var` parameter, at each level of recursion. So the outermost proc
sees the temporary changes made by all inner calls. This is why `c` is updated
and the `c.parent` chain is maintained at each step.
Signed-off-by: Jamie Lokier <jamie@shareable.org>
2021-04-12 17:06:31 +00:00
|
|
|
if not c.msg.isCreate:
|
|
|
|
c.beforeExecCall()
|
|
|
|
false
|
|
|
|
else:
|
|
|
|
c.beforeExecCreate()
|
|
|
|
|
EVMC: Small stacks when using EVMC, closes #575 (segfaults)
This patch reduces stack space used with EVM in ENABLE_EVMC=1 mode, from 13 MB
worst case to 550 kB, a 24x reduction.
This completes fixing the "stack problem" and closes #575 (`EVM: Different
segmentation faults when running the test suite with EVMC`).
It also closes #256 (`recursive EVM call trigger unrecoverable stack overflow`).
After this patch, it is possible to re-enable the CI targets which had to be
disabled due to #575.
This change is also a required precursor for switching over to "nearly EVMC" as
the clean and focused Nimbus-internal API between EVM and sync/database
processes, and is also key to the use of Chronos `async` in those processes
when calling the EVM.
(The motivation is the internal interface has to be substantially changed
_anyway_ for the parallel sync and database processes, and EVMC turns out to be
well-designed and well-suited for this. It provides good separation between
modules, and suits our needs better than our other current interface. Might as
well use a good one designed by someone else. EVMC is 98% done in Nimbus
thanks to great work done before by @jangko, and we can use Nimbus-specific
extensions where we need flexibility, including for performance. Being aligned
with the ecosystem is a useful bonus feature.)
All tests below were run on Ubuntu 20.04 LTS server, x86-64. This matches one
of the targets that has been disabled for a while in CI in EVMC mode due to
stack overflow crashing the tests, so it's a good choice.
Measurements before
===================
Testing commit `e76e0144 2021-04-22 11:29:42 +0700 add submodules: graphql and
toml-serialization`.
$ rm -f build/all_tests && make ENABLE_EVMC=1 test
$ ulimit -S -s 16384 # Requires larger stack than default to avoid crash.
$ ./build/all_tests 9 | tee tlog
[Suite] persist block json tests
...
Stack range 38416 depthHigh 3
...
Stack range 13074720 depthHigh 1024
[OK] tests/fixtures/PersistBlockTests/block1431916.json
These tests use 13.07 MB of stack to run, and so crash with the default stack
limit on Ubuntu Server 20.04 (8MB). Exactly 12768 bytes per EVM call stack
frame.
$ rm -f build/all_tests && make ENABLE_EVMC=1 test
$ ulimit -S -s 16384 # Requires larger stack than default.
$ ./build/all_tests 7 | tee tlog
[Suite] new generalstate json tests
...
Stack range 14384 depthHigh 2
...
Stack range 3495456 depthHigh 457
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest639.json
...
Stack range 3709600 depthHigh 485
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest458.json
...
Stack range 7831600 depthHigh 1024
[OK] tests/fixtures/eth_tests/GeneralStateTests/stCreate2/Create2OnDepth1024.json
These tests use 7.83MB of stack to run. About 7648 bytes per EVM call stack
frame. It _only just_ avoids crashing with the default Ubuntu Server stack
limit of 8 MB. However, it still crashes on Windows x86-64, which is why the
Windows CI EVMC target is currently disabled.
On Linux where this passes, this is so borderline that it affects work and
testing of the complex storage code, because that's called from the EVM.
Also, this greatly exceeds the default thread stack size.
Measurements after
==================
$ rm -f build/all_tests && make ENABLE_EVMC=1 test
$ ulimit -S -s 600 # Because we can! 600k stack.
$ ./build/all_tests 9 | tee tlog
[Suite] persist block json tests
...
Stack range 1936 depthHigh 3
...
Stack range 556272 depthHigh 1022
Stack range 556512 depthHigh 1023
Stack range 556816 depthHigh 1023
Stack range 557056 depthHigh 1024
Stack range 557360 depthHigh 1024
[OK] tests/fixtures/PersistBlockTests/block1431916.json
$ rm -f build/all_tests && make ENABLE_EVMC=1 test
$ ulimit -S -s 600 # Because we can! 600k stack.
$ ./build/all_tests 7 | tee tlog
[Suite] new generalstate json tests
...
Stack range 1392 depthHigh 2
...
Stack range 248912 depthHigh 457
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest639.json
...
Stack range 264144 depthHigh 485
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest458.json
...
Stack range 557360 depthHigh 1024
[OK] tests/fixtures/eth_tests/GeneralStateTests/stStaticCall/static_CallRecursiveBombPreCall.json
For both tests, a satisfying *544 bytes* per EVM call stack frame, and EVM
takes less than 600 kB total. With other overheads, both tests run in 600 kB
stack total at maximum EVM depth.
We must add some headroom on this for database activity called from the EVM,
and different compile targets. But it means the EVM itself is no longer a
stack burden.
This is much smaller than the default thread stack size on Linux (2MB), with
plenty of margin. (Just fyi, it isn't smaller than a _small_ thread stack on
Linux from a long time ago (128kB), and some small embedded C targets.)
This size is well suited to running EVMs in threads.
Further reduction
=================
This patch solves the stack problem. Windows and Linux 64-bit EVMC CI targets
can be re-enabled, and there is no longer a problem with stack usage.
We can reduce further to ~340 bytes per frame and 350 kB total, while still
complying with EVMC. But as this involves changing how errors are handled to
comply fully with EVMC, and removing `dispose` calls, it's not worth doing now
while there are other EVMC changes in progress that will have the same effect.
A Nimbus-specific extension will allow us to avoid recursion with EVMC anyway,
bringing bytes per frame to zero. We need the extension anyway, to support
Chronos `async` which parallel transaction processing is built around.
Interop with non-Nimbus over EVMC won't let us avoid recursion, but then we
can't control the stack frame size either. To prevent stack overflow in
interop I anticipate using (this method in Aleth)
[https://github.com/ethereum/aleth/blob/6e96ce34e3f131e2d42f3cb00741b54e05ab029d/libethereum/ExtVM.cpp#L61].
Smoke test other versions of GCC and Clang/LLVM
===============================================
As all builds including Windows use GCC or Apple's Clang/LLVM, this is just to
verify we're in the right ballpark on all targets. I've only checked `x86_64`
though, not 32-bit, and not ARM.
It's interesting to see GCC 10 uses less stack. This is because it optimises
`struct` returns better, sometimes skipping an intermediate copy. Here it
benefits the EVMC API, but I found GCC 10 also improves the larger stack usage
of the rest of `nimbus-eth1` as well.
Apple clang 12.0.0 (clang-1200.0.26.2) on MacOS 10.15:
- 544 bytes per EVM call stack frame
GCC 10.3.0 (Ubuntu 10.3.0-1ubuntu1) on Ubuntu 21.04:
- 464 bytes per EVM call stack frame
GCC 10.2.0 (Ubuntu 10.2.0-5ubuntu1~20.04) on Ubuntu 20.04 LTS:
- 464 bytes per EVM call stack frame
GCC 11.0.1 20210417 (experimental; Ubuntu 11-20210417-1ubuntu1) on Ubuntu 21.04:
- 8 bytes per EVM call stack frame
GCC 9.3.0 (Ubuntu 9.3.0-17ubuntu1~20.04) on Ubuntu 20.04 LTS:
- 544 bytes per EVM call stack frame
GCC 8.4.0 (Ubuntu 8.4.0-3ubuntu2) on Ubuntu 20.04 LTS:
- 544 bytes per EVM call stack frame
GCC 7.5.0 (Ubuntu 7.5.0-6ubuntu2) on Ubuntu 20.04 LTS:
- 544 bytes per EVM call stack frame
GCC 9.2.1 20191008 (Ubuntu 9.2.1-9ubuntu2) on Ubuntu 19.10:
- 528 bytes per EVM call stack frame
Signed-off-by: Jamie Lokier <jamie@shareable.org>
2021-04-26 15:01:00 +00:00
|
|
|
proc afterExec(c: Computation) {.noinline.} =
|
EVM: Small patch that reduces EVM stack usage to almost nothing
There's been a lot of talk about the Nimbus EVM "stack problem". I think we
assumed changing it would require big changes to the interpreter code, touching
a lot of functions.
It turned out to be a low hanging fruit.
This patch solves the stack problem, but hardly touches anything. The change
in EVM stack memory is from 13 MB worst case to just 48 kB, a 250x reduction.
I've been doing work on the database/storage/trie code. While looking at the
API between the EVM and the database/storage/trie, this stack patch stood out
and made itself obvious. As it's tiny, rather than more talk, here it is.
Note: This patch is intentionally small, non-invasive, and hopefully easy to
understand, so that it doesn't conflict with other work done on the EVM, and
can easily be grafted into any other EVM structure.
Motivation
==========
- We run out of space and crash on some targets, unless the stack limit is
raised above its default. Surprise segmentation faults are unhelpful.
- Some CI targets have been disabled for months due to this.
- Because usage borders on the system limits, when working on
database/storage/trie/sync code (called from the EVM), segmentation faults
occur and are misleading. They cause lost time due to thinking there's a
crash bug in the code being worked on, when there's nothing wrong with it.
- Sometimes unrelated, trivial code changes elsewhere trigger CI test failures.
It looks like abrupt termination. A simple, recent patch was crashing in
`make test` even though it was a trivial refactor. Turns out it pushed the
stack over the edge.
- A large stack has to be scanned by the Nim garbage collector sometimes.
Larger stack means slower GC and memory allocation.
- The structure of this small patch suggests how to weave async into the EVM
with almost no changes to the EVM, and no async transformation overhead.
- The patch seemed obvious when working on the API between EVM and storage.
Measurements before
===================
All these tests were run on Ubuntu 20.04 server, x86-64. This is one of the
targets that has been disabled for a while in CI in EVMC mode due to crashing,
and excessive stack usage is the cause.
Testing commit 0c34a8e3 `2021-04-08 17:46:00 +0200 CI: use MSYS2 on Windows`.
$ rm -f build/all_tests && make ENABLE_EVMC=1 test
$ ulimit -S -s 16384 # Requires larger stack than default to avoid crash.
$ ./build/all_tests 9 | tee tlog
[Suite] persist block json tests
...
Stack range 38496 depthHigh 3
...
Stack range 13140272 depthHigh 1024
[OK] tests/fixtures/PersistBlockTests/block1431916.json
These tests use 13.14 MB of stack to run, and so crash with the default stack
limit on Ubuntu Server 20.04 (8MB). Exactly 12832 bytes per EVM call stack
frame. It's interesting to see some stack frames take a bit more.
$ rm -f build/all_tests && make ENABLE_EVMC=1 test
$ ulimit -S -s 16384 # Requires larger stack than default.
$ ./build/all_tests 7 | tee tlog
[Suite] new generalstate json tests
...
Stack range 15488 depthHigh 2
...
Stack range 3539312 depthHigh 457
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest639.json
...
Stack range 3756144 depthHigh 485
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest458.json
...
Stack range 7929968 depthHigh 1024
[OK] tests/fixtures/eth_tests/GeneralStateTests/stCreate2/Create2OnDepth1024.json
These tests use 7.92MB of stack to run. About 7264 bytes per EVM call stack
frame. It _only just_ avoids crashing with the default Ubuntu Server stack
limit of 8 MB. However, it still crashes on Windows x86-64, which is why the
CI target is currently disabled.
On Linux where this passes, this is so borderline that it affects work and
testing of storage and sync code, because that's called from the EVM. Which
was a motivation for dealing with the stack instead of letting this linger.
Also, this stack greatly exceeds the default thread stack size.
$ rm -f build/all_tests && make ENABLE_EVMC=0 test
$ ulimit -S -s 16384 # Requires larger stack than default to avoid crash.
$ ./build/all_tests 9 | tee tlog
[Suite] persist block json tests
...
Stack range 33216 depthHigh 3
...
Stack range 11338032 depthHigh 1024
[OK] tests/fixtures/PersistBlockTests/block1431916.json
These tests use 11.33 MB stack to run, and so crash with a default stack limit
of 8MB. Exactly 11072 bytes per EVM call stack frame. It's interesting to see
some stack frames take a bit more.
$ rm -f build/all_tests && make ENABLE_EVMC=0 test
$ ulimit -S -s 16384 # Requires larger stack than default.
$ ./build/all_tests 7 | tee tlog
[Suite] new generalstate json tests
...
Stack range 10224 depthHigh 2
...
Stack range 2471760 depthHigh 457
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest639.json
...
Stack range 2623184 depthHigh 485
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest458.json
...
Stack range 5537824 depthHigh 1024
[OK] tests/fixtures/eth_tests/GeneralStateTests/stCreate2/Create2OnDepth1024.json
These tests use 5.54 MB of stack to run, and avoid crashing on with a default
stack limit of 8 MB. About 5408 bytes per EVM call stack frame.
However, this is uncomfortably close to the limit, as the stack frame size is
sensitive to changes in the code.
Also, this stack greatly exceeds the default thread stack size.
Measurements after
==================
(This patch doesn't address EVMC mode, which is not our default. EVMC stack
usage remains about the same. EVMC mode is addressed in another tiny patch.)
$ rm -f build/all_tests && make ENABLE_EVMC=0 test
$ ulimit -S -s 80 # Because we can! 80k stack.
$ ./build/all_tests 9 | tee tlog
[Suite] persist block json tests
...
Stack range 496 depthHigh 3
...
Stack range 49504 depthHigh 1024
[OK] tests/fixtures/PersistBlockTests/block1431916.json
$ rm -f build/all_tests && make ENABLE_EVMC=0 test
$ ulimit -S -s 72 # Because we can! 72k stack.
$ ./build/all_tests 7 | tee tlog
[Suite] new generalstate json tests
...
Stack range 448 depthHigh 2
...
Stack range 22288 depthHigh 457
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest639.json
...
Stack range 23632 depthHigh 485
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest458.json
...
Stack range 49504 depthHigh 1024
[OK] tests/fixtures/eth_tests/GeneralStateTests/stCreate2/Create2OnDepth1024.json
For both tests, a satisfying *48 bytes* per EVM call stack frame, and EVM takes
not much more than 48 kB. With other overheads, both tests run in 80 kB stack
total at maximum EVM depth.
We must add some headroom on this for database activity called from the EVM,
and different compile targets. But it means the EVM itself is no longer a
stack burden.
This is much smaller than the default thread stack size on Linux (2MB), with
plenty of margin. It's even smaller than Linux from a long time ago (128kB),
and some small embedded C targets. (Just fyi, though, some JVM environments
allocated just 32 kB to thread stacks.)
This size is also well suited to running EVMs in threads, if that's useful.
Subtle exception handling and `dispose`
=======================================
It is important that each `snapshot` has a corresponding `dispose` in the event
of an exception being raised. This code does do that, but in a subtle way.
The pair of functions `execCallOrCreate` and `execCallOrCreateAux` are
equivalent to the following code, where you can see `dispose` more clearly:
proc execCallOrCreate*(c: Computation) =
defer: c.dispose()
if c.beforeExec():
return
c.executeOpcodes()
while not c.continuation.isNil:
c.child.execCallOrCreate()
c.child = nil
(c.continuation)()
c.executeOpcodes()
c.afterExec()
That works fine, but only reduces the stack used to 300-700 kB instead of 48 kB.
To get lower we split the above into separate `execCallOrCreate` and
`execCallOrCreateAux`. Only the outermost has `defer`, and instead of handling
one level, it walks the entire `c.parent` chain calling `dispose` if needed.
The inner one avoids `defer`, which greatly reduces the size of its stackframe.
`c` is a `var` parameter, at each level of recursion. So the outermost proc
sees the temporary changes made by all inner calls. This is why `c` is updated
and the `c.parent` chain is maintained at each step.
Signed-off-by: Jamie Lokier <jamie@shareable.org>
2021-04-12 17:06:31 +00:00
|
|
|
if not c.msg.isCreate:
|
|
|
|
c.afterExecCall()
|
|
|
|
else:
|
|
|
|
c.afterExecCreate()
|
|
|
|
|
EVMC: Small stacks when using EVMC, closes #575 (segfaults)
This patch reduces stack space used with EVM in ENABLE_EVMC=1 mode, from 13 MB
worst case to 550 kB, a 24x reduction.
This completes fixing the "stack problem" and closes #575 (`EVM: Different
segmentation faults when running the test suite with EVMC`).
It also closes #256 (`recursive EVM call trigger unrecoverable stack overflow`).
After this patch, it is possible to re-enable the CI targets which had to be
disabled due to #575.
This change is also a required precursor for switching over to "nearly EVMC" as
the clean and focused Nimbus-internal API between EVM and sync/database
processes, and is also key to the use of Chronos `async` in those processes
when calling the EVM.
(The motivation is the internal interface has to be substantially changed
_anyway_ for the parallel sync and database processes, and EVMC turns out to be
well-designed and well-suited for this. It provides good separation between
modules, and suits our needs better than our other current interface. Might as
well use a good one designed by someone else. EVMC is 98% done in Nimbus
thanks to great work done before by @jangko, and we can use Nimbus-specific
extensions where we need flexibility, including for performance. Being aligned
with the ecosystem is a useful bonus feature.)
All tests below were run on Ubuntu 20.04 LTS server, x86-64. This matches one
of the targets that has been disabled for a while in CI in EVMC mode due to
stack overflow crashing the tests, so it's a good choice.
Measurements before
===================
Testing commit `e76e0144 2021-04-22 11:29:42 +0700 add submodules: graphql and
toml-serialization`.
$ rm -f build/all_tests && make ENABLE_EVMC=1 test
$ ulimit -S -s 16384 # Requires larger stack than default to avoid crash.
$ ./build/all_tests 9 | tee tlog
[Suite] persist block json tests
...
Stack range 38416 depthHigh 3
...
Stack range 13074720 depthHigh 1024
[OK] tests/fixtures/PersistBlockTests/block1431916.json
These tests use 13.07 MB of stack to run, and so crash with the default stack
limit on Ubuntu Server 20.04 (8MB). Exactly 12768 bytes per EVM call stack
frame.
$ rm -f build/all_tests && make ENABLE_EVMC=1 test
$ ulimit -S -s 16384 # Requires larger stack than default.
$ ./build/all_tests 7 | tee tlog
[Suite] new generalstate json tests
...
Stack range 14384 depthHigh 2
...
Stack range 3495456 depthHigh 457
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest639.json
...
Stack range 3709600 depthHigh 485
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest458.json
...
Stack range 7831600 depthHigh 1024
[OK] tests/fixtures/eth_tests/GeneralStateTests/stCreate2/Create2OnDepth1024.json
These tests use 7.83MB of stack to run. About 7648 bytes per EVM call stack
frame. It _only just_ avoids crashing with the default Ubuntu Server stack
limit of 8 MB. However, it still crashes on Windows x86-64, which is why the
Windows CI EVMC target is currently disabled.
On Linux where this passes, this is so borderline that it affects work and
testing of the complex storage code, because that's called from the EVM.
Also, this greatly exceeds the default thread stack size.
Measurements after
==================
$ rm -f build/all_tests && make ENABLE_EVMC=1 test
$ ulimit -S -s 600 # Because we can! 600k stack.
$ ./build/all_tests 9 | tee tlog
[Suite] persist block json tests
...
Stack range 1936 depthHigh 3
...
Stack range 556272 depthHigh 1022
Stack range 556512 depthHigh 1023
Stack range 556816 depthHigh 1023
Stack range 557056 depthHigh 1024
Stack range 557360 depthHigh 1024
[OK] tests/fixtures/PersistBlockTests/block1431916.json
$ rm -f build/all_tests && make ENABLE_EVMC=1 test
$ ulimit -S -s 600 # Because we can! 600k stack.
$ ./build/all_tests 7 | tee tlog
[Suite] new generalstate json tests
...
Stack range 1392 depthHigh 2
...
Stack range 248912 depthHigh 457
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest639.json
...
Stack range 264144 depthHigh 485
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest458.json
...
Stack range 557360 depthHigh 1024
[OK] tests/fixtures/eth_tests/GeneralStateTests/stStaticCall/static_CallRecursiveBombPreCall.json
For both tests, a satisfying *544 bytes* per EVM call stack frame, and EVM
takes less than 600 kB total. With other overheads, both tests run in 600 kB
stack total at maximum EVM depth.
We must add some headroom on this for database activity called from the EVM,
and different compile targets. But it means the EVM itself is no longer a
stack burden.
This is much smaller than the default thread stack size on Linux (2MB), with
plenty of margin. (Just fyi, it isn't smaller than a _small_ thread stack on
Linux from a long time ago (128kB), and some small embedded C targets.)
This size is well suited to running EVMs in threads.
Further reduction
=================
This patch solves the stack problem. Windows and Linux 64-bit EVMC CI targets
can be re-enabled, and there is no longer a problem with stack usage.
We can reduce further to ~340 bytes per frame and 350 kB total, while still
complying with EVMC. But as this involves changing how errors are handled to
comply fully with EVMC, and removing `dispose` calls, it's not worth doing now
while there are other EVMC changes in progress that will have the same effect.
A Nimbus-specific extension will allow us to avoid recursion with EVMC anyway,
bringing bytes per frame to zero. We need the extension anyway, to support
Chronos `async` which parallel transaction processing is built around.
Interop with non-Nimbus over EVMC won't let us avoid recursion, but then we
can't control the stack frame size either. To prevent stack overflow in
interop I anticipate using (this method in Aleth)
[https://github.com/ethereum/aleth/blob/6e96ce34e3f131e2d42f3cb00741b54e05ab029d/libethereum/ExtVM.cpp#L61].
Smoke test other versions of GCC and Clang/LLVM
===============================================
As all builds including Windows use GCC or Apple's Clang/LLVM, this is just to
verify we're in the right ballpark on all targets. I've only checked `x86_64`
though, not 32-bit, and not ARM.
It's interesting to see GCC 10 uses less stack. This is because it optimises
`struct` returns better, sometimes skipping an intermediate copy. Here it
benefits the EVMC API, but I found GCC 10 also improves the larger stack usage
of the rest of `nimbus-eth1` as well.
Apple clang 12.0.0 (clang-1200.0.26.2) on MacOS 10.15:
- 544 bytes per EVM call stack frame
GCC 10.3.0 (Ubuntu 10.3.0-1ubuntu1) on Ubuntu 21.04:
- 464 bytes per EVM call stack frame
GCC 10.2.0 (Ubuntu 10.2.0-5ubuntu1~20.04) on Ubuntu 20.04 LTS:
- 464 bytes per EVM call stack frame
GCC 11.0.1 20210417 (experimental; Ubuntu 11-20210417-1ubuntu1) on Ubuntu 21.04:
- 8 bytes per EVM call stack frame
GCC 9.3.0 (Ubuntu 9.3.0-17ubuntu1~20.04) on Ubuntu 20.04 LTS:
- 544 bytes per EVM call stack frame
GCC 8.4.0 (Ubuntu 8.4.0-3ubuntu2) on Ubuntu 20.04 LTS:
- 544 bytes per EVM call stack frame
GCC 7.5.0 (Ubuntu 7.5.0-6ubuntu2) on Ubuntu 20.04 LTS:
- 544 bytes per EVM call stack frame
GCC 9.2.1 20191008 (Ubuntu 9.2.1-9ubuntu2) on Ubuntu 19.10:
- 528 bytes per EVM call stack frame
Signed-off-by: Jamie Lokier <jamie@shareable.org>
2021-04-26 15:01:00 +00:00
|
|
|
template chainTo*(c: Computation, toChild: typeof(c.child), after: untyped) =
|
EVM: Small patch that reduces EVM stack usage to almost nothing
There's been a lot of talk about the Nimbus EVM "stack problem". I think we
assumed changing it would require big changes to the interpreter code, touching
a lot of functions.
It turned out to be a low hanging fruit.
This patch solves the stack problem, but hardly touches anything. The change
in EVM stack memory is from 13 MB worst case to just 48 kB, a 250x reduction.
I've been doing work on the database/storage/trie code. While looking at the
API between the EVM and the database/storage/trie, this stack patch stood out
and made itself obvious. As it's tiny, rather than more talk, here it is.
Note: This patch is intentionally small, non-invasive, and hopefully easy to
understand, so that it doesn't conflict with other work done on the EVM, and
can easily be grafted into any other EVM structure.
Motivation
==========
- We run out of space and crash on some targets, unless the stack limit is
raised above its default. Surprise segmentation faults are unhelpful.
- Some CI targets have been disabled for months due to this.
- Because usage borders on the system limits, when working on
database/storage/trie/sync code (called from the EVM), segmentation faults
occur and are misleading. They cause lost time due to thinking there's a
crash bug in the code being worked on, when there's nothing wrong with it.
- Sometimes unrelated, trivial code changes elsewhere trigger CI test failures.
It looks like abrupt termination. A simple, recent patch was crashing in
`make test` even though it was a trivial refactor. Turns out it pushed the
stack over the edge.
- A large stack has to be scanned by the Nim garbage collector sometimes.
Larger stack means slower GC and memory allocation.
- The structure of this small patch suggests how to weave async into the EVM
with almost no changes to the EVM, and no async transformation overhead.
- The patch seemed obvious when working on the API between EVM and storage.
Measurements before
===================
All these tests were run on Ubuntu 20.04 server, x86-64. This is one of the
targets that has been disabled for a while in CI in EVMC mode due to crashing,
and excessive stack usage is the cause.
Testing commit 0c34a8e3 `2021-04-08 17:46:00 +0200 CI: use MSYS2 on Windows`.
$ rm -f build/all_tests && make ENABLE_EVMC=1 test
$ ulimit -S -s 16384 # Requires larger stack than default to avoid crash.
$ ./build/all_tests 9 | tee tlog
[Suite] persist block json tests
...
Stack range 38496 depthHigh 3
...
Stack range 13140272 depthHigh 1024
[OK] tests/fixtures/PersistBlockTests/block1431916.json
These tests use 13.14 MB of stack to run, and so crash with the default stack
limit on Ubuntu Server 20.04 (8MB). Exactly 12832 bytes per EVM call stack
frame. It's interesting to see some stack frames take a bit more.
$ rm -f build/all_tests && make ENABLE_EVMC=1 test
$ ulimit -S -s 16384 # Requires larger stack than default.
$ ./build/all_tests 7 | tee tlog
[Suite] new generalstate json tests
...
Stack range 15488 depthHigh 2
...
Stack range 3539312 depthHigh 457
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest639.json
...
Stack range 3756144 depthHigh 485
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest458.json
...
Stack range 7929968 depthHigh 1024
[OK] tests/fixtures/eth_tests/GeneralStateTests/stCreate2/Create2OnDepth1024.json
These tests use 7.92MB of stack to run. About 7264 bytes per EVM call stack
frame. It _only just_ avoids crashing with the default Ubuntu Server stack
limit of 8 MB. However, it still crashes on Windows x86-64, which is why the
CI target is currently disabled.
On Linux where this passes, this is so borderline that it affects work and
testing of storage and sync code, because that's called from the EVM. Which
was a motivation for dealing with the stack instead of letting this linger.
Also, this stack greatly exceeds the default thread stack size.
$ rm -f build/all_tests && make ENABLE_EVMC=0 test
$ ulimit -S -s 16384 # Requires larger stack than default to avoid crash.
$ ./build/all_tests 9 | tee tlog
[Suite] persist block json tests
...
Stack range 33216 depthHigh 3
...
Stack range 11338032 depthHigh 1024
[OK] tests/fixtures/PersistBlockTests/block1431916.json
These tests use 11.33 MB stack to run, and so crash with a default stack limit
of 8MB. Exactly 11072 bytes per EVM call stack frame. It's interesting to see
some stack frames take a bit more.
$ rm -f build/all_tests && make ENABLE_EVMC=0 test
$ ulimit -S -s 16384 # Requires larger stack than default.
$ ./build/all_tests 7 | tee tlog
[Suite] new generalstate json tests
...
Stack range 10224 depthHigh 2
...
Stack range 2471760 depthHigh 457
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest639.json
...
Stack range 2623184 depthHigh 485
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest458.json
...
Stack range 5537824 depthHigh 1024
[OK] tests/fixtures/eth_tests/GeneralStateTests/stCreate2/Create2OnDepth1024.json
These tests use 5.54 MB of stack to run, and avoid crashing on with a default
stack limit of 8 MB. About 5408 bytes per EVM call stack frame.
However, this is uncomfortably close to the limit, as the stack frame size is
sensitive to changes in the code.
Also, this stack greatly exceeds the default thread stack size.
Measurements after
==================
(This patch doesn't address EVMC mode, which is not our default. EVMC stack
usage remains about the same. EVMC mode is addressed in another tiny patch.)
$ rm -f build/all_tests && make ENABLE_EVMC=0 test
$ ulimit -S -s 80 # Because we can! 80k stack.
$ ./build/all_tests 9 | tee tlog
[Suite] persist block json tests
...
Stack range 496 depthHigh 3
...
Stack range 49504 depthHigh 1024
[OK] tests/fixtures/PersistBlockTests/block1431916.json
$ rm -f build/all_tests && make ENABLE_EVMC=0 test
$ ulimit -S -s 72 # Because we can! 72k stack.
$ ./build/all_tests 7 | tee tlog
[Suite] new generalstate json tests
...
Stack range 448 depthHigh 2
...
Stack range 22288 depthHigh 457
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest639.json
...
Stack range 23632 depthHigh 485
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest458.json
...
Stack range 49504 depthHigh 1024
[OK] tests/fixtures/eth_tests/GeneralStateTests/stCreate2/Create2OnDepth1024.json
For both tests, a satisfying *48 bytes* per EVM call stack frame, and EVM takes
not much more than 48 kB. With other overheads, both tests run in 80 kB stack
total at maximum EVM depth.
We must add some headroom on this for database activity called from the EVM,
and different compile targets. But it means the EVM itself is no longer a
stack burden.
This is much smaller than the default thread stack size on Linux (2MB), with
plenty of margin. It's even smaller than Linux from a long time ago (128kB),
and some small embedded C targets. (Just fyi, though, some JVM environments
allocated just 32 kB to thread stacks.)
This size is also well suited to running EVMs in threads, if that's useful.
Subtle exception handling and `dispose`
=======================================
It is important that each `snapshot` has a corresponding `dispose` in the event
of an exception being raised. This code does do that, but in a subtle way.
The pair of functions `execCallOrCreate` and `execCallOrCreateAux` are
equivalent to the following code, where you can see `dispose` more clearly:
proc execCallOrCreate*(c: Computation) =
defer: c.dispose()
if c.beforeExec():
return
c.executeOpcodes()
while not c.continuation.isNil:
c.child.execCallOrCreate()
c.child = nil
(c.continuation)()
c.executeOpcodes()
c.afterExec()
That works fine, but only reduces the stack used to 300-700 kB instead of 48 kB.
To get lower we split the above into separate `execCallOrCreate` and
`execCallOrCreateAux`. Only the outermost has `defer`, and instead of handling
one level, it walks the entire `c.parent` chain calling `dispose` if needed.
The inner one avoids `defer`, which greatly reduces the size of its stackframe.
`c` is a `var` parameter, at each level of recursion. So the outermost proc
sees the temporary changes made by all inner calls. This is why `c` is updated
and the `c.parent` chain is maintained at each step.
Signed-off-by: Jamie Lokier <jamie@shareable.org>
2021-04-12 17:06:31 +00:00
|
|
|
c.child = toChild
|
|
|
|
c.continuation = proc() =
|
2021-05-11 14:51:31 +00:00
|
|
|
c.continuation = nil
|
EVM: Small patch that reduces EVM stack usage to almost nothing
There's been a lot of talk about the Nimbus EVM "stack problem". I think we
assumed changing it would require big changes to the interpreter code, touching
a lot of functions.
It turned out to be a low hanging fruit.
This patch solves the stack problem, but hardly touches anything. The change
in EVM stack memory is from 13 MB worst case to just 48 kB, a 250x reduction.
I've been doing work on the database/storage/trie code. While looking at the
API between the EVM and the database/storage/trie, this stack patch stood out
and made itself obvious. As it's tiny, rather than more talk, here it is.
Note: This patch is intentionally small, non-invasive, and hopefully easy to
understand, so that it doesn't conflict with other work done on the EVM, and
can easily be grafted into any other EVM structure.
Motivation
==========
- We run out of space and crash on some targets, unless the stack limit is
raised above its default. Surprise segmentation faults are unhelpful.
- Some CI targets have been disabled for months due to this.
- Because usage borders on the system limits, when working on
database/storage/trie/sync code (called from the EVM), segmentation faults
occur and are misleading. They cause lost time due to thinking there's a
crash bug in the code being worked on, when there's nothing wrong with it.
- Sometimes unrelated, trivial code changes elsewhere trigger CI test failures.
It looks like abrupt termination. A simple, recent patch was crashing in
`make test` even though it was a trivial refactor. Turns out it pushed the
stack over the edge.
- A large stack has to be scanned by the Nim garbage collector sometimes.
Larger stack means slower GC and memory allocation.
- The structure of this small patch suggests how to weave async into the EVM
with almost no changes to the EVM, and no async transformation overhead.
- The patch seemed obvious when working on the API between EVM and storage.
Measurements before
===================
All these tests were run on Ubuntu 20.04 server, x86-64. This is one of the
targets that has been disabled for a while in CI in EVMC mode due to crashing,
and excessive stack usage is the cause.
Testing commit 0c34a8e3 `2021-04-08 17:46:00 +0200 CI: use MSYS2 on Windows`.
$ rm -f build/all_tests && make ENABLE_EVMC=1 test
$ ulimit -S -s 16384 # Requires larger stack than default to avoid crash.
$ ./build/all_tests 9 | tee tlog
[Suite] persist block json tests
...
Stack range 38496 depthHigh 3
...
Stack range 13140272 depthHigh 1024
[OK] tests/fixtures/PersistBlockTests/block1431916.json
These tests use 13.14 MB of stack to run, and so crash with the default stack
limit on Ubuntu Server 20.04 (8MB). Exactly 12832 bytes per EVM call stack
frame. It's interesting to see some stack frames take a bit more.
$ rm -f build/all_tests && make ENABLE_EVMC=1 test
$ ulimit -S -s 16384 # Requires larger stack than default.
$ ./build/all_tests 7 | tee tlog
[Suite] new generalstate json tests
...
Stack range 15488 depthHigh 2
...
Stack range 3539312 depthHigh 457
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest639.json
...
Stack range 3756144 depthHigh 485
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest458.json
...
Stack range 7929968 depthHigh 1024
[OK] tests/fixtures/eth_tests/GeneralStateTests/stCreate2/Create2OnDepth1024.json
These tests use 7.92MB of stack to run. About 7264 bytes per EVM call stack
frame. It _only just_ avoids crashing with the default Ubuntu Server stack
limit of 8 MB. However, it still crashes on Windows x86-64, which is why the
CI target is currently disabled.
On Linux where this passes, this is so borderline that it affects work and
testing of storage and sync code, because that's called from the EVM. Which
was a motivation for dealing with the stack instead of letting this linger.
Also, this stack greatly exceeds the default thread stack size.
$ rm -f build/all_tests && make ENABLE_EVMC=0 test
$ ulimit -S -s 16384 # Requires larger stack than default to avoid crash.
$ ./build/all_tests 9 | tee tlog
[Suite] persist block json tests
...
Stack range 33216 depthHigh 3
...
Stack range 11338032 depthHigh 1024
[OK] tests/fixtures/PersistBlockTests/block1431916.json
These tests use 11.33 MB stack to run, and so crash with a default stack limit
of 8MB. Exactly 11072 bytes per EVM call stack frame. It's interesting to see
some stack frames take a bit more.
$ rm -f build/all_tests && make ENABLE_EVMC=0 test
$ ulimit -S -s 16384 # Requires larger stack than default.
$ ./build/all_tests 7 | tee tlog
[Suite] new generalstate json tests
...
Stack range 10224 depthHigh 2
...
Stack range 2471760 depthHigh 457
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest639.json
...
Stack range 2623184 depthHigh 485
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest458.json
...
Stack range 5537824 depthHigh 1024
[OK] tests/fixtures/eth_tests/GeneralStateTests/stCreate2/Create2OnDepth1024.json
These tests use 5.54 MB of stack to run, and avoid crashing on with a default
stack limit of 8 MB. About 5408 bytes per EVM call stack frame.
However, this is uncomfortably close to the limit, as the stack frame size is
sensitive to changes in the code.
Also, this stack greatly exceeds the default thread stack size.
Measurements after
==================
(This patch doesn't address EVMC mode, which is not our default. EVMC stack
usage remains about the same. EVMC mode is addressed in another tiny patch.)
$ rm -f build/all_tests && make ENABLE_EVMC=0 test
$ ulimit -S -s 80 # Because we can! 80k stack.
$ ./build/all_tests 9 | tee tlog
[Suite] persist block json tests
...
Stack range 496 depthHigh 3
...
Stack range 49504 depthHigh 1024
[OK] tests/fixtures/PersistBlockTests/block1431916.json
$ rm -f build/all_tests && make ENABLE_EVMC=0 test
$ ulimit -S -s 72 # Because we can! 72k stack.
$ ./build/all_tests 7 | tee tlog
[Suite] new generalstate json tests
...
Stack range 448 depthHigh 2
...
Stack range 22288 depthHigh 457
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest639.json
...
Stack range 23632 depthHigh 485
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest458.json
...
Stack range 49504 depthHigh 1024
[OK] tests/fixtures/eth_tests/GeneralStateTests/stCreate2/Create2OnDepth1024.json
For both tests, a satisfying *48 bytes* per EVM call stack frame, and EVM takes
not much more than 48 kB. With other overheads, both tests run in 80 kB stack
total at maximum EVM depth.
We must add some headroom on this for database activity called from the EVM,
and different compile targets. But it means the EVM itself is no longer a
stack burden.
This is much smaller than the default thread stack size on Linux (2MB), with
plenty of margin. It's even smaller than Linux from a long time ago (128kB),
and some small embedded C targets. (Just fyi, though, some JVM environments
allocated just 32 kB to thread stacks.)
This size is also well suited to running EVMs in threads, if that's useful.
Subtle exception handling and `dispose`
=======================================
It is important that each `snapshot` has a corresponding `dispose` in the event
of an exception being raised. This code does do that, but in a subtle way.
The pair of functions `execCallOrCreate` and `execCallOrCreateAux` are
equivalent to the following code, where you can see `dispose` more clearly:
proc execCallOrCreate*(c: Computation) =
defer: c.dispose()
if c.beforeExec():
return
c.executeOpcodes()
while not c.continuation.isNil:
c.child.execCallOrCreate()
c.child = nil
(c.continuation)()
c.executeOpcodes()
c.afterExec()
That works fine, but only reduces the stack used to 300-700 kB instead of 48 kB.
To get lower we split the above into separate `execCallOrCreate` and
`execCallOrCreateAux`. Only the outermost has `defer`, and instead of handling
one level, it walks the entire `c.parent` chain calling `dispose` if needed.
The inner one avoids `defer`, which greatly reduces the size of its stackframe.
`c` is a `var` parameter, at each level of recursion. So the outermost proc
sees the temporary changes made by all inner calls. This is why `c` is updated
and the `c.parent` chain is maintained at each step.
Signed-off-by: Jamie Lokier <jamie@shareable.org>
2021-04-12 17:06:31 +00:00
|
|
|
after
|
|
|
|
|
EVMC: Small stacks when using EVMC, closes #575 (segfaults)
This patch reduces stack space used with EVM in ENABLE_EVMC=1 mode, from 13 MB
worst case to 550 kB, a 24x reduction.
This completes fixing the "stack problem" and closes #575 (`EVM: Different
segmentation faults when running the test suite with EVMC`).
It also closes #256 (`recursive EVM call trigger unrecoverable stack overflow`).
After this patch, it is possible to re-enable the CI targets which had to be
disabled due to #575.
This change is also a required precursor for switching over to "nearly EVMC" as
the clean and focused Nimbus-internal API between EVM and sync/database
processes, and is also key to the use of Chronos `async` in those processes
when calling the EVM.
(The motivation is the internal interface has to be substantially changed
_anyway_ for the parallel sync and database processes, and EVMC turns out to be
well-designed and well-suited for this. It provides good separation between
modules, and suits our needs better than our other current interface. Might as
well use a good one designed by someone else. EVMC is 98% done in Nimbus
thanks to great work done before by @jangko, and we can use Nimbus-specific
extensions where we need flexibility, including for performance. Being aligned
with the ecosystem is a useful bonus feature.)
All tests below were run on Ubuntu 20.04 LTS server, x86-64. This matches one
of the targets that has been disabled for a while in CI in EVMC mode due to
stack overflow crashing the tests, so it's a good choice.
Measurements before
===================
Testing commit `e76e0144 2021-04-22 11:29:42 +0700 add submodules: graphql and
toml-serialization`.
$ rm -f build/all_tests && make ENABLE_EVMC=1 test
$ ulimit -S -s 16384 # Requires larger stack than default to avoid crash.
$ ./build/all_tests 9 | tee tlog
[Suite] persist block json tests
...
Stack range 38416 depthHigh 3
...
Stack range 13074720 depthHigh 1024
[OK] tests/fixtures/PersistBlockTests/block1431916.json
These tests use 13.07 MB of stack to run, and so crash with the default stack
limit on Ubuntu Server 20.04 (8MB). Exactly 12768 bytes per EVM call stack
frame.
$ rm -f build/all_tests && make ENABLE_EVMC=1 test
$ ulimit -S -s 16384 # Requires larger stack than default.
$ ./build/all_tests 7 | tee tlog
[Suite] new generalstate json tests
...
Stack range 14384 depthHigh 2
...
Stack range 3495456 depthHigh 457
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest639.json
...
Stack range 3709600 depthHigh 485
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest458.json
...
Stack range 7831600 depthHigh 1024
[OK] tests/fixtures/eth_tests/GeneralStateTests/stCreate2/Create2OnDepth1024.json
These tests use 7.83MB of stack to run. About 7648 bytes per EVM call stack
frame. It _only just_ avoids crashing with the default Ubuntu Server stack
limit of 8 MB. However, it still crashes on Windows x86-64, which is why the
Windows CI EVMC target is currently disabled.
On Linux where this passes, this is so borderline that it affects work and
testing of the complex storage code, because that's called from the EVM.
Also, this greatly exceeds the default thread stack size.
Measurements after
==================
$ rm -f build/all_tests && make ENABLE_EVMC=1 test
$ ulimit -S -s 600 # Because we can! 600k stack.
$ ./build/all_tests 9 | tee tlog
[Suite] persist block json tests
...
Stack range 1936 depthHigh 3
...
Stack range 556272 depthHigh 1022
Stack range 556512 depthHigh 1023
Stack range 556816 depthHigh 1023
Stack range 557056 depthHigh 1024
Stack range 557360 depthHigh 1024
[OK] tests/fixtures/PersistBlockTests/block1431916.json
$ rm -f build/all_tests && make ENABLE_EVMC=1 test
$ ulimit -S -s 600 # Because we can! 600k stack.
$ ./build/all_tests 7 | tee tlog
[Suite] new generalstate json tests
...
Stack range 1392 depthHigh 2
...
Stack range 248912 depthHigh 457
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest639.json
...
Stack range 264144 depthHigh 485
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest458.json
...
Stack range 557360 depthHigh 1024
[OK] tests/fixtures/eth_tests/GeneralStateTests/stStaticCall/static_CallRecursiveBombPreCall.json
For both tests, a satisfying *544 bytes* per EVM call stack frame, and EVM
takes less than 600 kB total. With other overheads, both tests run in 600 kB
stack total at maximum EVM depth.
We must add some headroom on this for database activity called from the EVM,
and different compile targets. But it means the EVM itself is no longer a
stack burden.
This is much smaller than the default thread stack size on Linux (2MB), with
plenty of margin. (Just fyi, it isn't smaller than a _small_ thread stack on
Linux from a long time ago (128kB), and some small embedded C targets.)
This size is well suited to running EVMs in threads.
Further reduction
=================
This patch solves the stack problem. Windows and Linux 64-bit EVMC CI targets
can be re-enabled, and there is no longer a problem with stack usage.
We can reduce further to ~340 bytes per frame and 350 kB total, while still
complying with EVMC. But as this involves changing how errors are handled to
comply fully with EVMC, and removing `dispose` calls, it's not worth doing now
while there are other EVMC changes in progress that will have the same effect.
A Nimbus-specific extension will allow us to avoid recursion with EVMC anyway,
bringing bytes per frame to zero. We need the extension anyway, to support
Chronos `async` which parallel transaction processing is built around.
Interop with non-Nimbus over EVMC won't let us avoid recursion, but then we
can't control the stack frame size either. To prevent stack overflow in
interop I anticipate using (this method in Aleth)
[https://github.com/ethereum/aleth/blob/6e96ce34e3f131e2d42f3cb00741b54e05ab029d/libethereum/ExtVM.cpp#L61].
Smoke test other versions of GCC and Clang/LLVM
===============================================
As all builds including Windows use GCC or Apple's Clang/LLVM, this is just to
verify we're in the right ballpark on all targets. I've only checked `x86_64`
though, not 32-bit, and not ARM.
It's interesting to see GCC 10 uses less stack. This is because it optimises
`struct` returns better, sometimes skipping an intermediate copy. Here it
benefits the EVMC API, but I found GCC 10 also improves the larger stack usage
of the rest of `nimbus-eth1` as well.
Apple clang 12.0.0 (clang-1200.0.26.2) on MacOS 10.15:
- 544 bytes per EVM call stack frame
GCC 10.3.0 (Ubuntu 10.3.0-1ubuntu1) on Ubuntu 21.04:
- 464 bytes per EVM call stack frame
GCC 10.2.0 (Ubuntu 10.2.0-5ubuntu1~20.04) on Ubuntu 20.04 LTS:
- 464 bytes per EVM call stack frame
GCC 11.0.1 20210417 (experimental; Ubuntu 11-20210417-1ubuntu1) on Ubuntu 21.04:
- 8 bytes per EVM call stack frame
GCC 9.3.0 (Ubuntu 9.3.0-17ubuntu1~20.04) on Ubuntu 20.04 LTS:
- 544 bytes per EVM call stack frame
GCC 8.4.0 (Ubuntu 8.4.0-3ubuntu2) on Ubuntu 20.04 LTS:
- 544 bytes per EVM call stack frame
GCC 7.5.0 (Ubuntu 7.5.0-6ubuntu2) on Ubuntu 20.04 LTS:
- 544 bytes per EVM call stack frame
GCC 9.2.1 20191008 (Ubuntu 9.2.1-9ubuntu2) on Ubuntu 19.10:
- 528 bytes per EVM call stack frame
Signed-off-by: Jamie Lokier <jamie@shareable.org>
2021-04-26 15:01:00 +00:00
|
|
|
when vm_use_recursion:
|
|
|
|
# Recursion with tiny stack frame per level.
|
|
|
|
proc execCallOrCreate*(c: Computation) =
|
|
|
|
defer: c.dispose()
|
|
|
|
if c.beforeExec():
|
|
|
|
return
|
|
|
|
c.executeOpcodes()
|
|
|
|
while not c.continuation.isNil:
|
|
|
|
when evmc_enabled:
|
|
|
|
c.res = c.host.call(c.child[])
|
|
|
|
else:
|
|
|
|
execCallOrCreate(c.child)
|
|
|
|
c.child = nil
|
|
|
|
c.executeOpcodes()
|
|
|
|
c.afterExec()
|
2021-04-20 14:53:25 +00:00
|
|
|
|
EVMC: Small stacks when using EVMC, closes #575 (segfaults)
This patch reduces stack space used with EVM in ENABLE_EVMC=1 mode, from 13 MB
worst case to 550 kB, a 24x reduction.
This completes fixing the "stack problem" and closes #575 (`EVM: Different
segmentation faults when running the test suite with EVMC`).
It also closes #256 (`recursive EVM call trigger unrecoverable stack overflow`).
After this patch, it is possible to re-enable the CI targets which had to be
disabled due to #575.
This change is also a required precursor for switching over to "nearly EVMC" as
the clean and focused Nimbus-internal API between EVM and sync/database
processes, and is also key to the use of Chronos `async` in those processes
when calling the EVM.
(The motivation is the internal interface has to be substantially changed
_anyway_ for the parallel sync and database processes, and EVMC turns out to be
well-designed and well-suited for this. It provides good separation between
modules, and suits our needs better than our other current interface. Might as
well use a good one designed by someone else. EVMC is 98% done in Nimbus
thanks to great work done before by @jangko, and we can use Nimbus-specific
extensions where we need flexibility, including for performance. Being aligned
with the ecosystem is a useful bonus feature.)
All tests below were run on Ubuntu 20.04 LTS server, x86-64. This matches one
of the targets that has been disabled for a while in CI in EVMC mode due to
stack overflow crashing the tests, so it's a good choice.
Measurements before
===================
Testing commit `e76e0144 2021-04-22 11:29:42 +0700 add submodules: graphql and
toml-serialization`.
$ rm -f build/all_tests && make ENABLE_EVMC=1 test
$ ulimit -S -s 16384 # Requires larger stack than default to avoid crash.
$ ./build/all_tests 9 | tee tlog
[Suite] persist block json tests
...
Stack range 38416 depthHigh 3
...
Stack range 13074720 depthHigh 1024
[OK] tests/fixtures/PersistBlockTests/block1431916.json
These tests use 13.07 MB of stack to run, and so crash with the default stack
limit on Ubuntu Server 20.04 (8MB). Exactly 12768 bytes per EVM call stack
frame.
$ rm -f build/all_tests && make ENABLE_EVMC=1 test
$ ulimit -S -s 16384 # Requires larger stack than default.
$ ./build/all_tests 7 | tee tlog
[Suite] new generalstate json tests
...
Stack range 14384 depthHigh 2
...
Stack range 3495456 depthHigh 457
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest639.json
...
Stack range 3709600 depthHigh 485
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest458.json
...
Stack range 7831600 depthHigh 1024
[OK] tests/fixtures/eth_tests/GeneralStateTests/stCreate2/Create2OnDepth1024.json
These tests use 7.83MB of stack to run. About 7648 bytes per EVM call stack
frame. It _only just_ avoids crashing with the default Ubuntu Server stack
limit of 8 MB. However, it still crashes on Windows x86-64, which is why the
Windows CI EVMC target is currently disabled.
On Linux where this passes, this is so borderline that it affects work and
testing of the complex storage code, because that's called from the EVM.
Also, this greatly exceeds the default thread stack size.
Measurements after
==================
$ rm -f build/all_tests && make ENABLE_EVMC=1 test
$ ulimit -S -s 600 # Because we can! 600k stack.
$ ./build/all_tests 9 | tee tlog
[Suite] persist block json tests
...
Stack range 1936 depthHigh 3
...
Stack range 556272 depthHigh 1022
Stack range 556512 depthHigh 1023
Stack range 556816 depthHigh 1023
Stack range 557056 depthHigh 1024
Stack range 557360 depthHigh 1024
[OK] tests/fixtures/PersistBlockTests/block1431916.json
$ rm -f build/all_tests && make ENABLE_EVMC=1 test
$ ulimit -S -s 600 # Because we can! 600k stack.
$ ./build/all_tests 7 | tee tlog
[Suite] new generalstate json tests
...
Stack range 1392 depthHigh 2
...
Stack range 248912 depthHigh 457
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest639.json
...
Stack range 264144 depthHigh 485
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest458.json
...
Stack range 557360 depthHigh 1024
[OK] tests/fixtures/eth_tests/GeneralStateTests/stStaticCall/static_CallRecursiveBombPreCall.json
For both tests, a satisfying *544 bytes* per EVM call stack frame, and EVM
takes less than 600 kB total. With other overheads, both tests run in 600 kB
stack total at maximum EVM depth.
We must add some headroom on this for database activity called from the EVM,
and different compile targets. But it means the EVM itself is no longer a
stack burden.
This is much smaller than the default thread stack size on Linux (2MB), with
plenty of margin. (Just fyi, it isn't smaller than a _small_ thread stack on
Linux from a long time ago (128kB), and some small embedded C targets.)
This size is well suited to running EVMs in threads.
Further reduction
=================
This patch solves the stack problem. Windows and Linux 64-bit EVMC CI targets
can be re-enabled, and there is no longer a problem with stack usage.
We can reduce further to ~340 bytes per frame and 350 kB total, while still
complying with EVMC. But as this involves changing how errors are handled to
comply fully with EVMC, and removing `dispose` calls, it's not worth doing now
while there are other EVMC changes in progress that will have the same effect.
A Nimbus-specific extension will allow us to avoid recursion with EVMC anyway,
bringing bytes per frame to zero. We need the extension anyway, to support
Chronos `async` which parallel transaction processing is built around.
Interop with non-Nimbus over EVMC won't let us avoid recursion, but then we
can't control the stack frame size either. To prevent stack overflow in
interop I anticipate using (this method in Aleth)
[https://github.com/ethereum/aleth/blob/6e96ce34e3f131e2d42f3cb00741b54e05ab029d/libethereum/ExtVM.cpp#L61].
Smoke test other versions of GCC and Clang/LLVM
===============================================
As all builds including Windows use GCC or Apple's Clang/LLVM, this is just to
verify we're in the right ballpark on all targets. I've only checked `x86_64`
though, not 32-bit, and not ARM.
It's interesting to see GCC 10 uses less stack. This is because it optimises
`struct` returns better, sometimes skipping an intermediate copy. Here it
benefits the EVMC API, but I found GCC 10 also improves the larger stack usage
of the rest of `nimbus-eth1` as well.
Apple clang 12.0.0 (clang-1200.0.26.2) on MacOS 10.15:
- 544 bytes per EVM call stack frame
GCC 10.3.0 (Ubuntu 10.3.0-1ubuntu1) on Ubuntu 21.04:
- 464 bytes per EVM call stack frame
GCC 10.2.0 (Ubuntu 10.2.0-5ubuntu1~20.04) on Ubuntu 20.04 LTS:
- 464 bytes per EVM call stack frame
GCC 11.0.1 20210417 (experimental; Ubuntu 11-20210417-1ubuntu1) on Ubuntu 21.04:
- 8 bytes per EVM call stack frame
GCC 9.3.0 (Ubuntu 9.3.0-17ubuntu1~20.04) on Ubuntu 20.04 LTS:
- 544 bytes per EVM call stack frame
GCC 8.4.0 (Ubuntu 8.4.0-3ubuntu2) on Ubuntu 20.04 LTS:
- 544 bytes per EVM call stack frame
GCC 7.5.0 (Ubuntu 7.5.0-6ubuntu2) on Ubuntu 20.04 LTS:
- 544 bytes per EVM call stack frame
GCC 9.2.1 20191008 (Ubuntu 9.2.1-9ubuntu2) on Ubuntu 19.10:
- 528 bytes per EVM call stack frame
Signed-off-by: Jamie Lokier <jamie@shareable.org>
2021-04-26 15:01:00 +00:00
|
|
|
else:
|
2021-04-20 14:53:25 +00:00
|
|
|
# No actual recursion, but simulate recursion including before/after/dispose.
|
EVMC: Small stacks when using EVMC, closes #575 (segfaults)
This patch reduces stack space used with EVM in ENABLE_EVMC=1 mode, from 13 MB
worst case to 550 kB, a 24x reduction.
This completes fixing the "stack problem" and closes #575 (`EVM: Different
segmentation faults when running the test suite with EVMC`).
It also closes #256 (`recursive EVM call trigger unrecoverable stack overflow`).
After this patch, it is possible to re-enable the CI targets which had to be
disabled due to #575.
This change is also a required precursor for switching over to "nearly EVMC" as
the clean and focused Nimbus-internal API between EVM and sync/database
processes, and is also key to the use of Chronos `async` in those processes
when calling the EVM.
(The motivation is the internal interface has to be substantially changed
_anyway_ for the parallel sync and database processes, and EVMC turns out to be
well-designed and well-suited for this. It provides good separation between
modules, and suits our needs better than our other current interface. Might as
well use a good one designed by someone else. EVMC is 98% done in Nimbus
thanks to great work done before by @jangko, and we can use Nimbus-specific
extensions where we need flexibility, including for performance. Being aligned
with the ecosystem is a useful bonus feature.)
All tests below were run on Ubuntu 20.04 LTS server, x86-64. This matches one
of the targets that has been disabled for a while in CI in EVMC mode due to
stack overflow crashing the tests, so it's a good choice.
Measurements before
===================
Testing commit `e76e0144 2021-04-22 11:29:42 +0700 add submodules: graphql and
toml-serialization`.
$ rm -f build/all_tests && make ENABLE_EVMC=1 test
$ ulimit -S -s 16384 # Requires larger stack than default to avoid crash.
$ ./build/all_tests 9 | tee tlog
[Suite] persist block json tests
...
Stack range 38416 depthHigh 3
...
Stack range 13074720 depthHigh 1024
[OK] tests/fixtures/PersistBlockTests/block1431916.json
These tests use 13.07 MB of stack to run, and so crash with the default stack
limit on Ubuntu Server 20.04 (8MB). Exactly 12768 bytes per EVM call stack
frame.
$ rm -f build/all_tests && make ENABLE_EVMC=1 test
$ ulimit -S -s 16384 # Requires larger stack than default.
$ ./build/all_tests 7 | tee tlog
[Suite] new generalstate json tests
...
Stack range 14384 depthHigh 2
...
Stack range 3495456 depthHigh 457
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest639.json
...
Stack range 3709600 depthHigh 485
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest458.json
...
Stack range 7831600 depthHigh 1024
[OK] tests/fixtures/eth_tests/GeneralStateTests/stCreate2/Create2OnDepth1024.json
These tests use 7.83MB of stack to run. About 7648 bytes per EVM call stack
frame. It _only just_ avoids crashing with the default Ubuntu Server stack
limit of 8 MB. However, it still crashes on Windows x86-64, which is why the
Windows CI EVMC target is currently disabled.
On Linux where this passes, this is so borderline that it affects work and
testing of the complex storage code, because that's called from the EVM.
Also, this greatly exceeds the default thread stack size.
Measurements after
==================
$ rm -f build/all_tests && make ENABLE_EVMC=1 test
$ ulimit -S -s 600 # Because we can! 600k stack.
$ ./build/all_tests 9 | tee tlog
[Suite] persist block json tests
...
Stack range 1936 depthHigh 3
...
Stack range 556272 depthHigh 1022
Stack range 556512 depthHigh 1023
Stack range 556816 depthHigh 1023
Stack range 557056 depthHigh 1024
Stack range 557360 depthHigh 1024
[OK] tests/fixtures/PersistBlockTests/block1431916.json
$ rm -f build/all_tests && make ENABLE_EVMC=1 test
$ ulimit -S -s 600 # Because we can! 600k stack.
$ ./build/all_tests 7 | tee tlog
[Suite] new generalstate json tests
...
Stack range 1392 depthHigh 2
...
Stack range 248912 depthHigh 457
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest639.json
...
Stack range 264144 depthHigh 485
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest458.json
...
Stack range 557360 depthHigh 1024
[OK] tests/fixtures/eth_tests/GeneralStateTests/stStaticCall/static_CallRecursiveBombPreCall.json
For both tests, a satisfying *544 bytes* per EVM call stack frame, and EVM
takes less than 600 kB total. With other overheads, both tests run in 600 kB
stack total at maximum EVM depth.
We must add some headroom on this for database activity called from the EVM,
and different compile targets. But it means the EVM itself is no longer a
stack burden.
This is much smaller than the default thread stack size on Linux (2MB), with
plenty of margin. (Just fyi, it isn't smaller than a _small_ thread stack on
Linux from a long time ago (128kB), and some small embedded C targets.)
This size is well suited to running EVMs in threads.
Further reduction
=================
This patch solves the stack problem. Windows and Linux 64-bit EVMC CI targets
can be re-enabled, and there is no longer a problem with stack usage.
We can reduce further to ~340 bytes per frame and 350 kB total, while still
complying with EVMC. But as this involves changing how errors are handled to
comply fully with EVMC, and removing `dispose` calls, it's not worth doing now
while there are other EVMC changes in progress that will have the same effect.
A Nimbus-specific extension will allow us to avoid recursion with EVMC anyway,
bringing bytes per frame to zero. We need the extension anyway, to support
Chronos `async` which parallel transaction processing is built around.
Interop with non-Nimbus over EVMC won't let us avoid recursion, but then we
can't control the stack frame size either. To prevent stack overflow in
interop I anticipate using (this method in Aleth)
[https://github.com/ethereum/aleth/blob/6e96ce34e3f131e2d42f3cb00741b54e05ab029d/libethereum/ExtVM.cpp#L61].
Smoke test other versions of GCC and Clang/LLVM
===============================================
As all builds including Windows use GCC or Apple's Clang/LLVM, this is just to
verify we're in the right ballpark on all targets. I've only checked `x86_64`
though, not 32-bit, and not ARM.
It's interesting to see GCC 10 uses less stack. This is because it optimises
`struct` returns better, sometimes skipping an intermediate copy. Here it
benefits the EVMC API, but I found GCC 10 also improves the larger stack usage
of the rest of `nimbus-eth1` as well.
Apple clang 12.0.0 (clang-1200.0.26.2) on MacOS 10.15:
- 544 bytes per EVM call stack frame
GCC 10.3.0 (Ubuntu 10.3.0-1ubuntu1) on Ubuntu 21.04:
- 464 bytes per EVM call stack frame
GCC 10.2.0 (Ubuntu 10.2.0-5ubuntu1~20.04) on Ubuntu 20.04 LTS:
- 464 bytes per EVM call stack frame
GCC 11.0.1 20210417 (experimental; Ubuntu 11-20210417-1ubuntu1) on Ubuntu 21.04:
- 8 bytes per EVM call stack frame
GCC 9.3.0 (Ubuntu 9.3.0-17ubuntu1~20.04) on Ubuntu 20.04 LTS:
- 544 bytes per EVM call stack frame
GCC 8.4.0 (Ubuntu 8.4.0-3ubuntu2) on Ubuntu 20.04 LTS:
- 544 bytes per EVM call stack frame
GCC 7.5.0 (Ubuntu 7.5.0-6ubuntu2) on Ubuntu 20.04 LTS:
- 544 bytes per EVM call stack frame
GCC 9.2.1 20191008 (Ubuntu 9.2.1-9ubuntu2) on Ubuntu 19.10:
- 528 bytes per EVM call stack frame
Signed-off-by: Jamie Lokier <jamie@shareable.org>
2021-04-26 15:01:00 +00:00
|
|
|
proc execCallOrCreate*(cParam: Computation) =
|
|
|
|
var (c, before) = (cParam, true)
|
|
|
|
defer:
|
|
|
|
while not c.isNil:
|
|
|
|
c.dispose()
|
|
|
|
c = c.parent
|
2021-04-20 14:53:25 +00:00
|
|
|
while true:
|
EVMC: Small stacks when using EVMC, closes #575 (segfaults)
This patch reduces stack space used with EVM in ENABLE_EVMC=1 mode, from 13 MB
worst case to 550 kB, a 24x reduction.
This completes fixing the "stack problem" and closes #575 (`EVM: Different
segmentation faults when running the test suite with EVMC`).
It also closes #256 (`recursive EVM call trigger unrecoverable stack overflow`).
After this patch, it is possible to re-enable the CI targets which had to be
disabled due to #575.
This change is also a required precursor for switching over to "nearly EVMC" as
the clean and focused Nimbus-internal API between EVM and sync/database
processes, and is also key to the use of Chronos `async` in those processes
when calling the EVM.
(The motivation is the internal interface has to be substantially changed
_anyway_ for the parallel sync and database processes, and EVMC turns out to be
well-designed and well-suited for this. It provides good separation between
modules, and suits our needs better than our other current interface. Might as
well use a good one designed by someone else. EVMC is 98% done in Nimbus
thanks to great work done before by @jangko, and we can use Nimbus-specific
extensions where we need flexibility, including for performance. Being aligned
with the ecosystem is a useful bonus feature.)
All tests below were run on Ubuntu 20.04 LTS server, x86-64. This matches one
of the targets that has been disabled for a while in CI in EVMC mode due to
stack overflow crashing the tests, so it's a good choice.
Measurements before
===================
Testing commit `e76e0144 2021-04-22 11:29:42 +0700 add submodules: graphql and
toml-serialization`.
$ rm -f build/all_tests && make ENABLE_EVMC=1 test
$ ulimit -S -s 16384 # Requires larger stack than default to avoid crash.
$ ./build/all_tests 9 | tee tlog
[Suite] persist block json tests
...
Stack range 38416 depthHigh 3
...
Stack range 13074720 depthHigh 1024
[OK] tests/fixtures/PersistBlockTests/block1431916.json
These tests use 13.07 MB of stack to run, and so crash with the default stack
limit on Ubuntu Server 20.04 (8MB). Exactly 12768 bytes per EVM call stack
frame.
$ rm -f build/all_tests && make ENABLE_EVMC=1 test
$ ulimit -S -s 16384 # Requires larger stack than default.
$ ./build/all_tests 7 | tee tlog
[Suite] new generalstate json tests
...
Stack range 14384 depthHigh 2
...
Stack range 3495456 depthHigh 457
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest639.json
...
Stack range 3709600 depthHigh 485
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest458.json
...
Stack range 7831600 depthHigh 1024
[OK] tests/fixtures/eth_tests/GeneralStateTests/stCreate2/Create2OnDepth1024.json
These tests use 7.83MB of stack to run. About 7648 bytes per EVM call stack
frame. It _only just_ avoids crashing with the default Ubuntu Server stack
limit of 8 MB. However, it still crashes on Windows x86-64, which is why the
Windows CI EVMC target is currently disabled.
On Linux where this passes, this is so borderline that it affects work and
testing of the complex storage code, because that's called from the EVM.
Also, this greatly exceeds the default thread stack size.
Measurements after
==================
$ rm -f build/all_tests && make ENABLE_EVMC=1 test
$ ulimit -S -s 600 # Because we can! 600k stack.
$ ./build/all_tests 9 | tee tlog
[Suite] persist block json tests
...
Stack range 1936 depthHigh 3
...
Stack range 556272 depthHigh 1022
Stack range 556512 depthHigh 1023
Stack range 556816 depthHigh 1023
Stack range 557056 depthHigh 1024
Stack range 557360 depthHigh 1024
[OK] tests/fixtures/PersistBlockTests/block1431916.json
$ rm -f build/all_tests && make ENABLE_EVMC=1 test
$ ulimit -S -s 600 # Because we can! 600k stack.
$ ./build/all_tests 7 | tee tlog
[Suite] new generalstate json tests
...
Stack range 1392 depthHigh 2
...
Stack range 248912 depthHigh 457
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest639.json
...
Stack range 264144 depthHigh 485
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest458.json
...
Stack range 557360 depthHigh 1024
[OK] tests/fixtures/eth_tests/GeneralStateTests/stStaticCall/static_CallRecursiveBombPreCall.json
For both tests, a satisfying *544 bytes* per EVM call stack frame, and EVM
takes less than 600 kB total. With other overheads, both tests run in 600 kB
stack total at maximum EVM depth.
We must add some headroom on this for database activity called from the EVM,
and different compile targets. But it means the EVM itself is no longer a
stack burden.
This is much smaller than the default thread stack size on Linux (2MB), with
plenty of margin. (Just fyi, it isn't smaller than a _small_ thread stack on
Linux from a long time ago (128kB), and some small embedded C targets.)
This size is well suited to running EVMs in threads.
Further reduction
=================
This patch solves the stack problem. Windows and Linux 64-bit EVMC CI targets
can be re-enabled, and there is no longer a problem with stack usage.
We can reduce further to ~340 bytes per frame and 350 kB total, while still
complying with EVMC. But as this involves changing how errors are handled to
comply fully with EVMC, and removing `dispose` calls, it's not worth doing now
while there are other EVMC changes in progress that will have the same effect.
A Nimbus-specific extension will allow us to avoid recursion with EVMC anyway,
bringing bytes per frame to zero. We need the extension anyway, to support
Chronos `async` which parallel transaction processing is built around.
Interop with non-Nimbus over EVMC won't let us avoid recursion, but then we
can't control the stack frame size either. To prevent stack overflow in
interop I anticipate using (this method in Aleth)
[https://github.com/ethereum/aleth/blob/6e96ce34e3f131e2d42f3cb00741b54e05ab029d/libethereum/ExtVM.cpp#L61].
Smoke test other versions of GCC and Clang/LLVM
===============================================
As all builds including Windows use GCC or Apple's Clang/LLVM, this is just to
verify we're in the right ballpark on all targets. I've only checked `x86_64`
though, not 32-bit, and not ARM.
It's interesting to see GCC 10 uses less stack. This is because it optimises
`struct` returns better, sometimes skipping an intermediate copy. Here it
benefits the EVMC API, but I found GCC 10 also improves the larger stack usage
of the rest of `nimbus-eth1` as well.
Apple clang 12.0.0 (clang-1200.0.26.2) on MacOS 10.15:
- 544 bytes per EVM call stack frame
GCC 10.3.0 (Ubuntu 10.3.0-1ubuntu1) on Ubuntu 21.04:
- 464 bytes per EVM call stack frame
GCC 10.2.0 (Ubuntu 10.2.0-5ubuntu1~20.04) on Ubuntu 20.04 LTS:
- 464 bytes per EVM call stack frame
GCC 11.0.1 20210417 (experimental; Ubuntu 11-20210417-1ubuntu1) on Ubuntu 21.04:
- 8 bytes per EVM call stack frame
GCC 9.3.0 (Ubuntu 9.3.0-17ubuntu1~20.04) on Ubuntu 20.04 LTS:
- 544 bytes per EVM call stack frame
GCC 8.4.0 (Ubuntu 8.4.0-3ubuntu2) on Ubuntu 20.04 LTS:
- 544 bytes per EVM call stack frame
GCC 7.5.0 (Ubuntu 7.5.0-6ubuntu2) on Ubuntu 20.04 LTS:
- 544 bytes per EVM call stack frame
GCC 9.2.1 20191008 (Ubuntu 9.2.1-9ubuntu2) on Ubuntu 19.10:
- 528 bytes per EVM call stack frame
Signed-off-by: Jamie Lokier <jamie@shareable.org>
2021-04-26 15:01:00 +00:00
|
|
|
while true:
|
|
|
|
if before and c.beforeExec():
|
|
|
|
break
|
|
|
|
c.executeOpcodes()
|
|
|
|
if c.continuation.isNil:
|
|
|
|
c.afterExec()
|
|
|
|
break
|
|
|
|
(before, c.child, c, c.parent) = (true, nil.Computation, c.child, c)
|
|
|
|
if c.parent.isNil:
|
2021-04-20 14:53:25 +00:00
|
|
|
break
|
EVMC: Small stacks when using EVMC, closes #575 (segfaults)
This patch reduces stack space used with EVM in ENABLE_EVMC=1 mode, from 13 MB
worst case to 550 kB, a 24x reduction.
This completes fixing the "stack problem" and closes #575 (`EVM: Different
segmentation faults when running the test suite with EVMC`).
It also closes #256 (`recursive EVM call trigger unrecoverable stack overflow`).
After this patch, it is possible to re-enable the CI targets which had to be
disabled due to #575.
This change is also a required precursor for switching over to "nearly EVMC" as
the clean and focused Nimbus-internal API between EVM and sync/database
processes, and is also key to the use of Chronos `async` in those processes
when calling the EVM.
(The motivation is the internal interface has to be substantially changed
_anyway_ for the parallel sync and database processes, and EVMC turns out to be
well-designed and well-suited for this. It provides good separation between
modules, and suits our needs better than our other current interface. Might as
well use a good one designed by someone else. EVMC is 98% done in Nimbus
thanks to great work done before by @jangko, and we can use Nimbus-specific
extensions where we need flexibility, including for performance. Being aligned
with the ecosystem is a useful bonus feature.)
All tests below were run on Ubuntu 20.04 LTS server, x86-64. This matches one
of the targets that has been disabled for a while in CI in EVMC mode due to
stack overflow crashing the tests, so it's a good choice.
Measurements before
===================
Testing commit `e76e0144 2021-04-22 11:29:42 +0700 add submodules: graphql and
toml-serialization`.
$ rm -f build/all_tests && make ENABLE_EVMC=1 test
$ ulimit -S -s 16384 # Requires larger stack than default to avoid crash.
$ ./build/all_tests 9 | tee tlog
[Suite] persist block json tests
...
Stack range 38416 depthHigh 3
...
Stack range 13074720 depthHigh 1024
[OK] tests/fixtures/PersistBlockTests/block1431916.json
These tests use 13.07 MB of stack to run, and so crash with the default stack
limit on Ubuntu Server 20.04 (8MB). Exactly 12768 bytes per EVM call stack
frame.
$ rm -f build/all_tests && make ENABLE_EVMC=1 test
$ ulimit -S -s 16384 # Requires larger stack than default.
$ ./build/all_tests 7 | tee tlog
[Suite] new generalstate json tests
...
Stack range 14384 depthHigh 2
...
Stack range 3495456 depthHigh 457
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest639.json
...
Stack range 3709600 depthHigh 485
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest458.json
...
Stack range 7831600 depthHigh 1024
[OK] tests/fixtures/eth_tests/GeneralStateTests/stCreate2/Create2OnDepth1024.json
These tests use 7.83MB of stack to run. About 7648 bytes per EVM call stack
frame. It _only just_ avoids crashing with the default Ubuntu Server stack
limit of 8 MB. However, it still crashes on Windows x86-64, which is why the
Windows CI EVMC target is currently disabled.
On Linux where this passes, this is so borderline that it affects work and
testing of the complex storage code, because that's called from the EVM.
Also, this greatly exceeds the default thread stack size.
Measurements after
==================
$ rm -f build/all_tests && make ENABLE_EVMC=1 test
$ ulimit -S -s 600 # Because we can! 600k stack.
$ ./build/all_tests 9 | tee tlog
[Suite] persist block json tests
...
Stack range 1936 depthHigh 3
...
Stack range 556272 depthHigh 1022
Stack range 556512 depthHigh 1023
Stack range 556816 depthHigh 1023
Stack range 557056 depthHigh 1024
Stack range 557360 depthHigh 1024
[OK] tests/fixtures/PersistBlockTests/block1431916.json
$ rm -f build/all_tests && make ENABLE_EVMC=1 test
$ ulimit -S -s 600 # Because we can! 600k stack.
$ ./build/all_tests 7 | tee tlog
[Suite] new generalstate json tests
...
Stack range 1392 depthHigh 2
...
Stack range 248912 depthHigh 457
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest639.json
...
Stack range 264144 depthHigh 485
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest458.json
...
Stack range 557360 depthHigh 1024
[OK] tests/fixtures/eth_tests/GeneralStateTests/stStaticCall/static_CallRecursiveBombPreCall.json
For both tests, a satisfying *544 bytes* per EVM call stack frame, and EVM
takes less than 600 kB total. With other overheads, both tests run in 600 kB
stack total at maximum EVM depth.
We must add some headroom on this for database activity called from the EVM,
and different compile targets. But it means the EVM itself is no longer a
stack burden.
This is much smaller than the default thread stack size on Linux (2MB), with
plenty of margin. (Just fyi, it isn't smaller than a _small_ thread stack on
Linux from a long time ago (128kB), and some small embedded C targets.)
This size is well suited to running EVMs in threads.
Further reduction
=================
This patch solves the stack problem. Windows and Linux 64-bit EVMC CI targets
can be re-enabled, and there is no longer a problem with stack usage.
We can reduce further to ~340 bytes per frame and 350 kB total, while still
complying with EVMC. But as this involves changing how errors are handled to
comply fully with EVMC, and removing `dispose` calls, it's not worth doing now
while there are other EVMC changes in progress that will have the same effect.
A Nimbus-specific extension will allow us to avoid recursion with EVMC anyway,
bringing bytes per frame to zero. We need the extension anyway, to support
Chronos `async` which parallel transaction processing is built around.
Interop with non-Nimbus over EVMC won't let us avoid recursion, but then we
can't control the stack frame size either. To prevent stack overflow in
interop I anticipate using (this method in Aleth)
[https://github.com/ethereum/aleth/blob/6e96ce34e3f131e2d42f3cb00741b54e05ab029d/libethereum/ExtVM.cpp#L61].
Smoke test other versions of GCC and Clang/LLVM
===============================================
As all builds including Windows use GCC or Apple's Clang/LLVM, this is just to
verify we're in the right ballpark on all targets. I've only checked `x86_64`
though, not 32-bit, and not ARM.
It's interesting to see GCC 10 uses less stack. This is because it optimises
`struct` returns better, sometimes skipping an intermediate copy. Here it
benefits the EVMC API, but I found GCC 10 also improves the larger stack usage
of the rest of `nimbus-eth1` as well.
Apple clang 12.0.0 (clang-1200.0.26.2) on MacOS 10.15:
- 544 bytes per EVM call stack frame
GCC 10.3.0 (Ubuntu 10.3.0-1ubuntu1) on Ubuntu 21.04:
- 464 bytes per EVM call stack frame
GCC 10.2.0 (Ubuntu 10.2.0-5ubuntu1~20.04) on Ubuntu 20.04 LTS:
- 464 bytes per EVM call stack frame
GCC 11.0.1 20210417 (experimental; Ubuntu 11-20210417-1ubuntu1) on Ubuntu 21.04:
- 8 bytes per EVM call stack frame
GCC 9.3.0 (Ubuntu 9.3.0-17ubuntu1~20.04) on Ubuntu 20.04 LTS:
- 544 bytes per EVM call stack frame
GCC 8.4.0 (Ubuntu 8.4.0-3ubuntu2) on Ubuntu 20.04 LTS:
- 544 bytes per EVM call stack frame
GCC 7.5.0 (Ubuntu 7.5.0-6ubuntu2) on Ubuntu 20.04 LTS:
- 544 bytes per EVM call stack frame
GCC 9.2.1 20191008 (Ubuntu 9.2.1-9ubuntu2) on Ubuntu 19.10:
- 528 bytes per EVM call stack frame
Signed-off-by: Jamie Lokier <jamie@shareable.org>
2021-04-26 15:01:00 +00:00
|
|
|
c.dispose()
|
|
|
|
(before, c.parent, c) = (false, nil.Computation, c.parent)
|
EVM: Small patch that reduces EVM stack usage to almost nothing
There's been a lot of talk about the Nimbus EVM "stack problem". I think we
assumed changing it would require big changes to the interpreter code, touching
a lot of functions.
It turned out to be a low hanging fruit.
This patch solves the stack problem, but hardly touches anything. The change
in EVM stack memory is from 13 MB worst case to just 48 kB, a 250x reduction.
I've been doing work on the database/storage/trie code. While looking at the
API between the EVM and the database/storage/trie, this stack patch stood out
and made itself obvious. As it's tiny, rather than more talk, here it is.
Note: This patch is intentionally small, non-invasive, and hopefully easy to
understand, so that it doesn't conflict with other work done on the EVM, and
can easily be grafted into any other EVM structure.
Motivation
==========
- We run out of space and crash on some targets, unless the stack limit is
raised above its default. Surprise segmentation faults are unhelpful.
- Some CI targets have been disabled for months due to this.
- Because usage borders on the system limits, when working on
database/storage/trie/sync code (called from the EVM), segmentation faults
occur and are misleading. They cause lost time due to thinking there's a
crash bug in the code being worked on, when there's nothing wrong with it.
- Sometimes unrelated, trivial code changes elsewhere trigger CI test failures.
It looks like abrupt termination. A simple, recent patch was crashing in
`make test` even though it was a trivial refactor. Turns out it pushed the
stack over the edge.
- A large stack has to be scanned by the Nim garbage collector sometimes.
Larger stack means slower GC and memory allocation.
- The structure of this small patch suggests how to weave async into the EVM
with almost no changes to the EVM, and no async transformation overhead.
- The patch seemed obvious when working on the API between EVM and storage.
Measurements before
===================
All these tests were run on Ubuntu 20.04 server, x86-64. This is one of the
targets that has been disabled for a while in CI in EVMC mode due to crashing,
and excessive stack usage is the cause.
Testing commit 0c34a8e3 `2021-04-08 17:46:00 +0200 CI: use MSYS2 on Windows`.
$ rm -f build/all_tests && make ENABLE_EVMC=1 test
$ ulimit -S -s 16384 # Requires larger stack than default to avoid crash.
$ ./build/all_tests 9 | tee tlog
[Suite] persist block json tests
...
Stack range 38496 depthHigh 3
...
Stack range 13140272 depthHigh 1024
[OK] tests/fixtures/PersistBlockTests/block1431916.json
These tests use 13.14 MB of stack to run, and so crash with the default stack
limit on Ubuntu Server 20.04 (8MB). Exactly 12832 bytes per EVM call stack
frame. It's interesting to see some stack frames take a bit more.
$ rm -f build/all_tests && make ENABLE_EVMC=1 test
$ ulimit -S -s 16384 # Requires larger stack than default.
$ ./build/all_tests 7 | tee tlog
[Suite] new generalstate json tests
...
Stack range 15488 depthHigh 2
...
Stack range 3539312 depthHigh 457
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest639.json
...
Stack range 3756144 depthHigh 485
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest458.json
...
Stack range 7929968 depthHigh 1024
[OK] tests/fixtures/eth_tests/GeneralStateTests/stCreate2/Create2OnDepth1024.json
These tests use 7.92MB of stack to run. About 7264 bytes per EVM call stack
frame. It _only just_ avoids crashing with the default Ubuntu Server stack
limit of 8 MB. However, it still crashes on Windows x86-64, which is why the
CI target is currently disabled.
On Linux where this passes, this is so borderline that it affects work and
testing of storage and sync code, because that's called from the EVM. Which
was a motivation for dealing with the stack instead of letting this linger.
Also, this stack greatly exceeds the default thread stack size.
$ rm -f build/all_tests && make ENABLE_EVMC=0 test
$ ulimit -S -s 16384 # Requires larger stack than default to avoid crash.
$ ./build/all_tests 9 | tee tlog
[Suite] persist block json tests
...
Stack range 33216 depthHigh 3
...
Stack range 11338032 depthHigh 1024
[OK] tests/fixtures/PersistBlockTests/block1431916.json
These tests use 11.33 MB stack to run, and so crash with a default stack limit
of 8MB. Exactly 11072 bytes per EVM call stack frame. It's interesting to see
some stack frames take a bit more.
$ rm -f build/all_tests && make ENABLE_EVMC=0 test
$ ulimit -S -s 16384 # Requires larger stack than default.
$ ./build/all_tests 7 | tee tlog
[Suite] new generalstate json tests
...
Stack range 10224 depthHigh 2
...
Stack range 2471760 depthHigh 457
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest639.json
...
Stack range 2623184 depthHigh 485
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest458.json
...
Stack range 5537824 depthHigh 1024
[OK] tests/fixtures/eth_tests/GeneralStateTests/stCreate2/Create2OnDepth1024.json
These tests use 5.54 MB of stack to run, and avoid crashing on with a default
stack limit of 8 MB. About 5408 bytes per EVM call stack frame.
However, this is uncomfortably close to the limit, as the stack frame size is
sensitive to changes in the code.
Also, this stack greatly exceeds the default thread stack size.
Measurements after
==================
(This patch doesn't address EVMC mode, which is not our default. EVMC stack
usage remains about the same. EVMC mode is addressed in another tiny patch.)
$ rm -f build/all_tests && make ENABLE_EVMC=0 test
$ ulimit -S -s 80 # Because we can! 80k stack.
$ ./build/all_tests 9 | tee tlog
[Suite] persist block json tests
...
Stack range 496 depthHigh 3
...
Stack range 49504 depthHigh 1024
[OK] tests/fixtures/PersistBlockTests/block1431916.json
$ rm -f build/all_tests && make ENABLE_EVMC=0 test
$ ulimit -S -s 72 # Because we can! 72k stack.
$ ./build/all_tests 7 | tee tlog
[Suite] new generalstate json tests
...
Stack range 448 depthHigh 2
...
Stack range 22288 depthHigh 457
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest639.json
...
Stack range 23632 depthHigh 485
[OK] tests/fixtures/eth_tests/GeneralStateTests/stRandom2/randomStatetest458.json
...
Stack range 49504 depthHigh 1024
[OK] tests/fixtures/eth_tests/GeneralStateTests/stCreate2/Create2OnDepth1024.json
For both tests, a satisfying *48 bytes* per EVM call stack frame, and EVM takes
not much more than 48 kB. With other overheads, both tests run in 80 kB stack
total at maximum EVM depth.
We must add some headroom on this for database activity called from the EVM,
and different compile targets. But it means the EVM itself is no longer a
stack burden.
This is much smaller than the default thread stack size on Linux (2MB), with
plenty of margin. It's even smaller than Linux from a long time ago (128kB),
and some small embedded C targets. (Just fyi, though, some JVM environments
allocated just 32 kB to thread stacks.)
This size is also well suited to running EVMs in threads, if that's useful.
Subtle exception handling and `dispose`
=======================================
It is important that each `snapshot` has a corresponding `dispose` in the event
of an exception being raised. This code does do that, but in a subtle way.
The pair of functions `execCallOrCreate` and `execCallOrCreateAux` are
equivalent to the following code, where you can see `dispose` more clearly:
proc execCallOrCreate*(c: Computation) =
defer: c.dispose()
if c.beforeExec():
return
c.executeOpcodes()
while not c.continuation.isNil:
c.child.execCallOrCreate()
c.child = nil
(c.continuation)()
c.executeOpcodes()
c.afterExec()
That works fine, but only reduces the stack used to 300-700 kB instead of 48 kB.
To get lower we split the above into separate `execCallOrCreate` and
`execCallOrCreateAux`. Only the outermost has `defer`, and instead of handling
one level, it walks the entire `c.parent` chain calling `dispose` if needed.
The inner one avoids `defer`, which greatly reduces the size of its stackframe.
`c` is a `var` parameter, at each level of recursion. So the outermost proc
sees the temporary changes made by all inner calls. This is why `c` is updated
and the `c.parent` chain is maintained at each step.
Signed-off-by: Jamie Lokier <jamie@shareable.org>
2021-04-12 17:06:31 +00:00
|
|
|
|
2020-01-31 01:08:44 +00:00
|
|
|
proc merge*(c, child: Computation) =
|
|
|
|
c.logEntries.add child.logEntries
|
|
|
|
c.gasMeter.refundGas(child.gasMeter.gasRefunded)
|
2021-05-11 09:05:58 +00:00
|
|
|
c.selfDestructs.incl child.selfDestructs
|
2020-01-31 01:08:44 +00:00
|
|
|
c.touchedAccounts.incl child.touchedAccounts
|
|
|
|
|
2020-01-22 04:48:02 +00:00
|
|
|
proc execSelfDestruct*(c: Computation, beneficiary: EthAddress) =
|
2020-01-17 11:58:03 +00:00
|
|
|
c.vmState.mutateStateDB:
|
|
|
|
let
|
|
|
|
localBalance = c.getBalance(c.msg.contractAddress)
|
|
|
|
beneficiaryBalance = c.getBalance(beneficiary)
|
|
|
|
|
|
|
|
# Transfer to beneficiary
|
|
|
|
db.setBalance(beneficiary, localBalance + beneficiaryBalance)
|
|
|
|
|
|
|
|
# Zero the balance of the address being deleted.
|
|
|
|
# This must come after sending to beneficiary in case the
|
|
|
|
# contract named itself as the beneficiary.
|
|
|
|
db.setBalance(c.msg.contractAddress, 0.u256)
|
|
|
|
|
|
|
|
trace "SELFDESTRUCT",
|
|
|
|
contractAddress = c.msg.contractAddress.toHex,
|
|
|
|
localBalance = localBalance.toString,
|
|
|
|
beneficiary = beneficiary.toHex
|
|
|
|
|
2020-01-09 06:25:53 +00:00
|
|
|
c.touchedAccounts.incl beneficiary
|
2020-01-17 11:58:03 +00:00
|
|
|
# Register the account to be deleted
|
2021-05-11 09:05:58 +00:00
|
|
|
c.selfDestructs.incl(c.msg.contractAddress)
|
2018-01-16 17:05:20 +00:00
|
|
|
|
2020-01-10 01:26:17 +00:00
|
|
|
proc addLogEntry*(c: Computation, log: Log) {.inline.} =
|
2019-02-27 14:04:42 +00:00
|
|
|
c.logEntries.add(log)
|
2018-01-16 17:05:20 +00:00
|
|
|
|
2020-01-10 01:26:17 +00:00
|
|
|
proc getGasRefund*(c: Computation): GasInt =
|
2019-12-19 09:31:06 +00:00
|
|
|
if c.isSuccess:
|
|
|
|
result = c.gasMeter.gasRefunded
|
2018-01-16 17:05:20 +00:00
|
|
|
|
2020-01-10 11:18:36 +00:00
|
|
|
proc refundSelfDestruct*(c: Computation) =
|
|
|
|
let cost = gasFees[c.fork][RefundSelfDestruct]
|
2021-05-11 09:05:58 +00:00
|
|
|
c.gasMeter.refundGas(cost * c.selfDestructs.len)
|
2020-01-10 11:18:36 +00:00
|
|
|
|
2020-06-06 03:05:11 +00:00
|
|
|
proc tracingEnabled*(c: Computation): bool {.inline.} =
|
|
|
|
EnableTracing in c.vmState.tracer.flags
|
2018-12-03 10:54:19 +00:00
|
|
|
|
2020-06-06 03:05:11 +00:00
|
|
|
proc traceOpCodeStarted*(c: Computation, op: Op): int {.inline.} =
|
2018-12-11 09:23:15 +00:00
|
|
|
c.vmState.tracer.traceOpCodeStarted(c, op)
|
2018-12-03 10:54:19 +00:00
|
|
|
|
2020-06-06 03:05:11 +00:00
|
|
|
proc traceOpCodeEnded*(c: Computation, op: Op, lastIndex: int) {.inline.} =
|
2019-02-21 08:17:43 +00:00
|
|
|
c.vmState.tracer.traceOpCodeEnded(c, op, lastIndex)
|
2018-12-03 16:22:08 +00:00
|
|
|
|
2020-06-06 03:05:11 +00:00
|
|
|
proc traceError*(c: Computation) {.inline.} =
|
2018-12-03 16:22:08 +00:00
|
|
|
c.vmState.tracer.traceError(c)
|
2019-02-25 13:02:16 +00:00
|
|
|
|
2020-06-06 03:05:11 +00:00
|
|
|
proc prepareTracer*(c: Computation) {.inline.} =
|
2019-02-25 13:02:16 +00:00
|
|
|
c.vmState.tracer.prepare(c.msg.depth)
|
2019-03-19 16:30:35 +00:00
|
|
|
|
|
|
|
include interpreter_dispatch
|
2020-01-17 14:54:28 +00:00
|
|
|
|
|
|
|
when defined(evmc_enabled):
|
|
|
|
include evmc_host
|