diff --git a/Makefile b/Makefile index fb17f2003..4b269b4ad 100644 --- a/Makefile +++ b/Makefile @@ -175,6 +175,10 @@ test: | build deps $(ENV_SCRIPT) nim test_rocksdb $(NIM_PARAMS) nimbus.nims $(ENV_SCRIPT) nim test $(NIM_PARAMS) nimbus.nims +# builds and runs an EVM-related subset of the nimbus test suite +test-evm: | build deps + $(ENV_SCRIPT) nim test_evm $(NIM_PARAMS) nimbus.nims + # Primitive reproducibility test. # # On some platforms, with some GCC versions, it may not be possible to get a diff --git a/nimbus.nimble b/nimbus.nimble index 299e504f9..650007f4f 100644 --- a/nimbus.nimble +++ b/nimbus.nimble @@ -66,6 +66,9 @@ task test, "Run tests": task test_rocksdb, "Run rocksdb tests": test "tests/db", "test_kvstore_rocksdb", "-d:chronicles_log_level=ERROR -d:unittest2DisableParamFiltering" +task test_evm, "Run EVM tests": + test "tests", "evm_tests", "-d:chronicles_log_level=ERROR -d:unittest2DisableParamFiltering" + ## Fluffy tasks task fluffy, "Build fluffy": diff --git a/nimbus/transaction/call_common.nim b/nimbus/transaction/call_common.nim index 5f394104e..94d7d1c27 100644 --- a/nimbus/transaction/call_common.nim +++ b/nimbus/transaction/call_common.nim @@ -8,6 +8,7 @@ import eth/common/eth_types, stint, options, stew/ranges/ptr_arith, + chronos, ".."/[vm_types, vm_state, vm_computation, vm_state_transactions], ".."/[vm_internals, vm_precompiles, vm_gas_costs], ".."/[db/accounts_cache, forks], @@ -160,6 +161,7 @@ proc setupHost(call: CallParams): TransactionHost = let cMsg = hostToComputationMessage(host.msg) host.computation = newComputation(vmState, cMsg, code) + shallowCopy(host.code, code) else: @@ -197,10 +199,12 @@ when defined(evmc_enabled): {.gcsafe.}: callResult.release(callResult) -proc runComputation*(call: CallParams): CallResult = - let host = setupHost(call) - let c = host.computation +# FIXME-awkwardFactoring: the factoring out of the pre and +# post parts feels awkward to me, but for now I'd really like +# not to have too much duplicated code between sync and async. +# --Adam +proc prepareToRunComputation(host: TransactionHost, call: CallParams) = # Must come after `setupHost` for correct fork. if not call.noAccessList: initialAccessListEIP2929(call) @@ -210,11 +214,9 @@ proc runComputation*(call: CallParams): CallResult = host.vmState.mutateStateDB: db.subBalance(call.sender, call.gasLimit.u256 * call.gasPrice.u256) - when defined(evmc_enabled): - doExecEvmc(host, call) - else: - execComputation(host.computation) - +proc calculateAndPossiblyRefundGas(host: TransactionHost, call: CallParams): GasInt = + let c = host.computation + # EIP-3529: Reduction in refunds let MaxRefundQuotient = if host.vmState.fork >= FkLondon: 5.GasInt @@ -222,19 +224,23 @@ proc runComputation*(call: CallParams): CallResult = 2.GasInt # Calculated gas used, taking into account refund rules. - var gasRemaining: GasInt = 0 if call.noRefund: - gasRemaining = c.gasMeter.gasRemaining + result = c.gasMeter.gasRemaining elif not c.shouldBurnGas: let maxRefund = (call.gasLimit - c.gasMeter.gasRemaining) div MaxRefundQuotient let refund = min(c.getGasRefund(), maxRefund) c.gasMeter.returnGas(refund) - gasRemaining = c.gasMeter.gasRemaining + result = c.gasMeter.gasRemaining # Refund for unused gas. - if gasRemaining > 0 and not call.noGasCharge: + if result > 0 and not call.noGasCharge: host.vmState.mutateStateDB: - db.addBalance(call.sender, gasRemaining.u256 * call.gasPrice.u256) + db.addBalance(call.sender, result.u256 * call.gasPrice.u256) + +proc finishRunningComputation(host: TransactionHost, call: CallParams): CallResult = + let c = host.computation + + let gasRemaining = calculateAndPossiblyRefundGas(host, call) result.isError = c.isError result.gasUsed = call.gasLimit - gasRemaining @@ -244,3 +250,27 @@ proc runComputation*(call: CallParams): CallResult = shallowCopy(result.logEntries, c.logEntries) result.stack = c.stack result.memory = c.memory + +proc runComputation*(call: CallParams): CallResult = + let host = setupHost(call) + prepareToRunComputation(host, call) + + when defined(evmc_enabled): + doExecEvmc(host, call) + else: + execComputation(host.computation) + + finishRunningComputation(host, call) + +# FIXME-duplicatedForAsync +proc asyncRunComputation*(call: CallParams): Future[CallResult] {.async.} = + let host = setupHost(call) + prepareToRunComputation(host, call) + + # FIXME-asyncAndEvmc: I'm not sure what to do with EVMC at the moment. + # when defined(evmc_enabled): + # doExecEvmc(host, call) + # else: + await asyncExecComputation(host.computation) + + return finishRunningComputation(host, call) diff --git a/nimbus/transaction/call_evm.nim b/nimbus/transaction/call_evm.nim index 77398cb05..3e57f3095 100644 --- a/nimbus/transaction/call_evm.nim +++ b/nimbus/transaction/call_evm.nim @@ -9,6 +9,7 @@ import std/[options, times], chronicles, + chronos, eth/[common/eth_types_rlp, trie/db], stint, ".."/[vm_types, vm_state, vm_gas_costs, forks, constants], @@ -177,8 +178,10 @@ proc rpcEstimateGas*(cd: RpcCallData, header: BlockHeader, chainDB: BaseChainDB, hi -proc txCallEvm*(tx: Transaction, sender: EthAddress, vmState: BaseVMState, fork: Fork): GasInt = - var call = CallParams( +proc callParamsForTx(tx: Transaction, sender: EthAddress, vmState: BaseVMState, fork: Fork): CallParams = + # Is there a nice idiom for this kind of thing? Should I + # just be writing this as a bunch of assignment statements? + result = CallParams( vmState: vmState, forkOverride: some(fork), gasPrice: tx.gasPrice, @@ -190,11 +193,10 @@ proc txCallEvm*(tx: Transaction, sender: EthAddress, vmState: BaseVMState, fork: input: tx.payload ) if tx.txType > TxLegacy: - shallowCopy(call.accessList, tx.accessList) - return runComputation(call).gasUsed + shallowCopy(result.accessList, tx.accessList) -proc testCallEvm*(tx: Transaction, sender: EthAddress, vmState: BaseVMState, fork: Fork): CallResult = - var call = CallParams( +proc callParamsForTest(tx: Transaction, sender: EthAddress, vmState: BaseVMState, fork: Fork): CallParams = + result = CallParams( vmState: vmState, forkOverride: some(fork), gasPrice: tx.gasPrice, @@ -209,5 +211,17 @@ proc testCallEvm*(tx: Transaction, sender: EthAddress, vmState: BaseVMState, for noRefund: true, # Don't apply gas refund/burn rule. ) if tx.txType > TxLegacy: - shallowCopy(call.accessList, tx.accessList) + shallowCopy(result.accessList, tx.accessList) + +proc txCallEvm*(tx: Transaction, sender: EthAddress, vmState: BaseVMState, fork: Fork): GasInt = + let call = callParamsForTx(tx, sender, vmState, fork) + return runComputation(call).gasUsed + +proc testCallEvm*(tx: Transaction, sender: EthAddress, vmState: BaseVMState, fork: Fork): CallResult = + let call = callParamsForTest(tx, sender, vmState, fork) runComputation(call) + +# FIXME-duplicatedForAsync +proc asyncTestCallEvm*(tx: Transaction, sender: EthAddress, vmState: BaseVMState, fork: Fork): Future[CallResult] {.async.} = + let call = callParamsForTest(tx, sender, vmState, fork) + return await asyncRunComputation(call) diff --git a/nimbus/vm2/async_operations.nim b/nimbus/vm2/async_operations.nim new file mode 100644 index 000000000..c9b0c6ff3 --- /dev/null +++ b/nimbus/vm2/async_operations.nim @@ -0,0 +1,104 @@ +# Nimbus +# Copyright (c) 2022 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or +# http://www.apache.org/licenses/LICENSE-2.0) +# * MIT license ([LICENSE-MIT](LICENSE-MIT) or +# http://opensource.org/licenses/MIT) +# at your option. This file may not be copied, modified, or distributed except +# according to those terms. + +import + chronos, + stint, + json_rpc/rpcclient, + web3, + ./computation, + ./state, + ./types, + ../db/accounts_cache + + + +# Used in synchronous mode. +proc noLazyDataSource*(): LazyDataSource = + LazyDataSource( + ifNecessaryGetStorage: (proc(c: Computation, slot: UInt256): Future[void] {.async.} = + discard + ) + ) + +# Will be used in asynchronous on-demand-data-fetching mode, once +# that is implemented. +proc realLazyDataSource*(client: RpcClient): LazyDataSource = + LazyDataSource( + ifNecessaryGetStorage: (proc(c: Computation, slot: UInt256): Future[void] {.async.} = + # TODO: find some way to check whether we already have it. + # This is WRONG, but good enough for now, considering this + # code is unused except in a few tests. I'm working on + # doing this properly. + if not c.getStorage(slot).isZero: return + + # FIXME-onDemandStorageNotImplementedYet + # (I sketched in this code, but haven't actually tried running it yet.) + echo("Attempting to for-real fetch slot " & $(slot)) + # ethAddressStr("0xfff33a3bd36abdbd412707b8e310d6011454a7ae") + # check hexDataStr(0.u256).string == res.string + let ethAddress = c.msg.contractAddress + let address: Address = Address(ethAddress) + let quantity: int = slot.truncate(int) # this is probably wrong; what's the right way to convert this? + let blockId: BlockIdentifier = blockId(c.vmState.parent.blockNumber.truncate(uint64)) # ditto + let res = await client.eth_getStorageAt(address, quantity, blockId) + echo("Fetched slot " & $(slot) & ", result is " & $(res)) + let v = res # will res be the actual value, or do I need to convert or something? + + # Before implementing this, see the note from Zahary here: + # https://github.com/status-im/nimbus-eth1/pull/1260#discussion_r999669139 + # + # c.vmState.mutateStateDB: + # db.setStorage(c.msg.contractAddress, slot, UInt256.fromBytesBE(v)) + ) + ) + +# Used for unit testing. Contains some prepopulated data. +proc fakeLazyDataSource*(fakePairs: seq[tuple[key, val: array[32, byte]]]): LazyDataSource = + LazyDataSource( + ifNecessaryGetStorage: (proc(c: Computation, slot: UInt256): Future[void] {.async.} = + # See the comment above. + if not c.getStorage(slot).isZero: return + + # FIXME-writeAutomatedTestsToShowThatItCanRunConcurrently + + # For now, until I've implemented some more automated way to + # capture and verify the fact that this can run concurrently, + # this is useful just to see in the console that the echo + # statements from multiple Computations can run at the same + # time and be interleaved. + # echo("Attempting to fake-fetch slot " & $(slot)) + # await sleepAsync(2.seconds) + + let slotBytes = toBytesBE(slot) + # The linear search is obviously slow, but doesn't matter + # for tests with only a few initialStorage entries. Fix + # this if we ever want to write tests with more. + for (k, v) in fakePairs: + if slotBytes == k: + c.vmState.mutateStateDB: + db.setStorage(c.msg.contractAddress, slot, UInt256.fromBytesBE(v)) + break + + # echo("Finished fake-fetch of slot " & $(slot)) + ) + ) + + + + +# Gotta find the place where we're creating a Computation without setting +# its asyncFactory in the first place, but this is fine for now. +proc asyncFactory*(c: Computation): AsyncOperationFactory = + # Does Nim have an "ifNil" macro/template? + if isNil(c.vmState.asyncFactory): + AsyncOperationFactory(lazyDataSource: noLazyDataSource()) # AARDVARK - can I make a singleton? + else: + c.vmState.asyncFactory diff --git a/nimbus/vm2/computation.nim b/nimbus/vm2/computation.nim index e235ad44a..32d25ca12 100644 --- a/nimbus/vm2/computation.nim +++ b/nimbus/vm2/computation.nim @@ -21,6 +21,7 @@ import ./transaction_tracer, ./types, chronicles, + chronos, eth/[common, keys], options, sets @@ -304,6 +305,13 @@ template chainTo*(c: Computation, toChild: typeof(c.child), after: untyped) = c.continuation = nil after +# Register an async operation to be performed before the continuation is called. +template asyncChainTo*(c: Computation, asyncOperation: Future[void], after: untyped) = + c.pendingAsyncOperation = asyncOperation + c.continuation = proc() = + c.continuation = nil + after + proc merge*(c, child: Computation) = c.logEntries.add child.logEntries c.gasMeter.refundGas(child.gasMeter.gasRefunded) diff --git a/nimbus/vm2/interpreter/op_dispatcher.nim b/nimbus/vm2/interpreter/op_dispatcher.nim index 5c7bb063e..6fbba38e6 100644 --- a/nimbus/vm2/interpreter/op_dispatcher.nim +++ b/nimbus/vm2/interpreter/op_dispatcher.nim @@ -98,7 +98,7 @@ proc toCaseStmt(forkArg, opArg, k: NimNode): NimNode = # Wrap innner case/switch into outer case/switch let branchStmt = block: case op - of Create, Create2, Call, CallCode, DelegateCall, StaticCall: + of Create, Create2, Call, CallCode, DelegateCall, StaticCall, Sload: quote do: `forkCaseSubExpr` if not `k`.cpt.continuation.isNil: @@ -135,7 +135,7 @@ template genLowMemDispatcher*(fork: Fork; op: Op; k: Vm2Ctx) = handleOtherDirective(fork, op, k) case c.instr - of Create, Create2, Call, CallCode, DelegateCall, StaticCall: + of Create, Create2, Call, CallCode, DelegateCall, StaticCall, Sload: if not k.cpt.continuation.isNil: break of Return, Revert, SelfDestruct: diff --git a/nimbus/vm2/interpreter/op_handlers/oph_memory.nim b/nimbus/vm2/interpreter/op_handlers/oph_memory.nim index 5e674ca93..a0d9f3026 100644 --- a/nimbus/vm2/interpreter/op_handlers/oph_memory.nim +++ b/nimbus/vm2/interpreter/op_handlers/oph_memory.nim @@ -14,6 +14,7 @@ import ../../../errors, + ../../async_operations, ../../code_stream, ../../computation, ../../memory, @@ -163,10 +164,11 @@ const sloadOp: Vm2OpFn = proc (k: var Vm2Ctx) = ## 0x54, Load word from storage. - let (slot) = k.cpt.stack.popInt(1) - k.cpt.stack.push: - k.cpt.getStorage(slot) - + let cpt = k.cpt # so it can safely be captured by the asyncChainTo closure below + let (slot) = cpt.stack.popInt(1) + cpt.asyncChainTo(asyncFactory(cpt).lazyDataSource.ifNecessaryGetStorage(cpt, slot)): + cpt.stack.push: + cpt.getStorage(slot) sloadEIP2929Op: Vm2OpFn = proc (k: var Vm2Ctx) = ## 0x54, EIP2929: Load word from storage for Berlin and later diff --git a/nimbus/vm2/interpreter_dispatch.nim b/nimbus/vm2/interpreter_dispatch.nim index 70bb7dff6..1f969fa3e 100644 --- a/nimbus/vm2/interpreter_dispatch.nim +++ b/nimbus/vm2/interpreter_dispatch.nim @@ -25,6 +25,7 @@ import ./state, ./types, chronicles, + chronos, eth/[common, keys], macros, options, @@ -45,12 +46,20 @@ const # Private functions # ------------------------------------------------------------------------------ -proc selectVM(c: Computation, fork: Fork) {.gcsafe.} = +proc selectVM(c: Computation, fork: Fork, shouldPrepareTracer: bool) {.gcsafe.} = ## Op code execution handler main loop. var desc: Vm2Ctx desc.cpt = c - if c.tracingEnabled: + # It's important not to re-prepare the tracer after + # an async operation, only after a call/create. + # + # That is, tracingEnabled is checked in many places, and + # indicates something like, "Do we want tracing to be + # enabled?", whereas shouldPrepareTracer is more like, + # "Are we at a spot right now where we want to re-initialize + # the tracer?" + if c.tracingEnabled and shouldPrepareTracer: c.prepareTracer() while true: @@ -195,7 +204,7 @@ proc afterExec(c: Computation) = # Public functions # ------------------------------------------------------------------------------ -proc executeOpcodes*(c: Computation) = +proc executeOpcodes*(c: Computation, shouldPrepareTracer: bool = true) = let fork = c.fork block: @@ -205,7 +214,7 @@ proc executeOpcodes*(c: Computation) = try: if not c.continuation.isNil: (c.continuation)() - c.selectVM(fork) + c.selectVM(fork, shouldPrepareTracer) except CatchableError as e: c.setError( &"Opcode Dispatch Error msg={e.msg}, depth={c.msg.depth}", true) @@ -222,17 +231,25 @@ when vm_use_recursion: return c.executeOpcodes() while not c.continuation.isNil: - when evmc_enabled: - c.res = c.host.call(c.child[]) + # If there's a continuation, then it's because there's either + # a child (i.e. call or create) or a pendingAsyncOperation. + if not c.pendingAsyncOperation.isNil: + let p = c.pendingAsyncOperation + c.pendingAsyncOperation = nil + doAssert(p.finished(), "In synchronous mode, every async operation should be an already-resolved Future.") + c.executeOpcodes(false) else: - execCallOrCreate(c.child) - c.child = nil - c.executeOpcodes() + when evmc_enabled: + c.res = c.host.call(c.child[]) + else: + execCallOrCreate(c.child) + c.child = nil + c.executeOpcodes() c.afterExec() else: proc execCallOrCreate*(cParam: Computation) = - var (c, before) = (cParam, true) + var (c, before, shouldPrepareTracer) = (cParam, true, true) defer: while not c.isNil: c.dispose() @@ -243,15 +260,60 @@ else: while true: if before and c.beforeExec(): break - c.executeOpcodes() + c.executeOpcodes(shouldPrepareTracer) if c.continuation.isNil: c.afterExec() break - (before, c.child, c, c.parent) = (true, nil.Computation, c.child, c) + if not c.pendingAsyncOperation.isNil: + before = false + shouldPrepareTracer = false + let p = c.pendingAsyncOperation + c.pendingAsyncOperation = nil + doAssert(p.finished(), "In synchronous mode, every async operation should be an already-resolved Future.") + else: + (before, shouldPrepareTracer, c.child, c, c.parent) = (true, true, nil.Computation, c.child, c) if c.parent.isNil: break c.dispose() - (before, c.parent, c) = (false, nil.Computation, c.parent) + (before, shouldPrepareTracer, c.parent, c) = (false, true, nil.Computation, c.parent) + +# FIXME-duplicatedForAsync +# +# In the long run I'd like to make some clever macro/template to +# eliminate the duplication between the synchronous and +# asynchronous versions. But for now let's stick with this for +# simplicity. +# +# Also, I've based this on the recursive one (above), which I think +# is okay because the "async" pragma is going to rewrite this whole +# thing to use callbacks anyway. But maybe I'm wrong? It isn't hard +# to write the async version of the iterative one, but this one is +# a bit shorter and feels cleaner, so if it works just as well I'd +# rather use this one. --Adam +proc asyncExecCallOrCreate*(c: Computation): Future[void] {.async.} = + defer: c.dispose() + if c.beforeExec(): + return + c.executeOpcodes() + while not c.continuation.isNil: + # If there's a continuation, then it's because there's either + # a child (i.e. call or create) or a pendingAsyncOperation. + if not c.pendingAsyncOperation.isNil: + let p = c.pendingAsyncOperation + c.pendingAsyncOperation = nil + await p + c.executeOpcodes(false) + else: + when evmc_enabled: + # FIXME-asyncAndEvmc + # Note that this is NOT async. I'm not sure how/whether I + # can do EVMC asynchronously. + c.res = c.host.call(c.child[]) + else: + await asyncExecCallOrCreate(c.child) + c.child = nil + c.executeOpcodes() + c.afterExec() # ------------------------------------------------------------------------------ # End diff --git a/nimbus/vm2/state_transactions.nim b/nimbus/vm2/state_transactions.nim index bebeef492..73ecd7dcc 100644 --- a/nimbus/vm2/state_transactions.nim +++ b/nimbus/vm2/state_transactions.nim @@ -21,6 +21,7 @@ import ./state, ./types, chronicles, + chronos, eth/common, eth/common/eth_types, options, @@ -46,13 +47,17 @@ proc refundGas*(c: Computation, tx: Transaction, sender: EthAddress) = db.addBalance(sender, c.gasMeter.gasRemaining.u256 * tx.gasPrice.u256) -proc execComputation*(c: Computation) = +# FIXME-awkwardFactoring: the factoring out of the pre and +# post parts feels awkward to me, but for now I'd really like +# not to have too much duplicated code between sync and async. +# --Adam + +proc preExecComputation(c: Computation) = if not c.msg.isCreate: c.vmState.mutateStateDB: db.incNonce(c.msg.sender) - c.execCallOrCreate() - +proc postExecComputation(c: Computation) = if c.isSuccess: if c.fork < FkLondon: # EIP-3529: Reduction in refunds @@ -62,3 +67,14 @@ proc execComputation*(c: Computation) = c.vmState.touchedAccounts.incl c.touchedAccounts c.vmState.status = c.isSuccess + +proc execComputation*(c: Computation) = + c.preExecComputation() + c.execCallOrCreate() + c.postExecComputation() + +# FIXME-duplicatedForAsync +proc asyncExecComputation*(c: Computation): Future[void] {.async.} = + c.preExecComputation() + await c.asyncExecCallOrCreate() + c.postExecComputation() diff --git a/nimbus/vm2/types.nim b/nimbus/vm2/types.nim index c2be949b2..9c98ed8df 100644 --- a/nimbus/vm2/types.nim +++ b/nimbus/vm2/types.nim @@ -11,6 +11,8 @@ import tables, eth/common, options, json, sets, + chronos, stint, + json_rpc/rpcclient, ./stack, ./memory, ./code_stream, ../forks, ./interpreter/[gas_costs, op_codes], # TODO - will be hidden at a lower layer @@ -55,6 +57,7 @@ type gasCosts* : GasCosts fork* : Fork minerAddress* : EthAddress + asyncFactory* : AsyncOperationFactory TracerFlags* {.pure.} = enum EnableTracing @@ -96,6 +99,7 @@ type res*: nimbus_result else: parent*, child*: Computation + pendingAsyncOperation*: Future[void] continuation*: proc() {.gcsafe.} Error* = ref object @@ -127,3 +131,9 @@ type value*: UInt256 data*: seq[byte] flags*: MsgFlags + + LazyDataSource* = ref object of RootObj + ifNecessaryGetStorage*: proc(c: Computation, slot: UInt256): Future[void] {.gcsafe.} + + AsyncOperationFactory* = ref object of RootObj + lazyDataSource*: LazyDataSource diff --git a/nimbus/vm_state_transactions.nim b/nimbus/vm_state_transactions.nim index b1bdfbea6..67334e9b7 100644 --- a/nimbus/vm_state_transactions.nim +++ b/nimbus/vm_state_transactions.nim @@ -12,6 +12,7 @@ import vm2/state_transactions as vmx export + vmx.asyncExecComputation, vmx.execComputation # End diff --git a/tests/evm_tests.nim b/tests/evm_tests.nim new file mode 100644 index 000000000..a872b95ac --- /dev/null +++ b/tests/evm_tests.nim @@ -0,0 +1,27 @@ +# Nimbus +# Copyright (c) 2022 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0) +# * MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT) +# at your option. This file may not be copied, modified, or distributed except according to those terms. + +import ../test_macro + +{. warning[UnusedImport]:off .} + +# This file is just meant to gather a bunch of EVM tests in one +# place. I want to be able to gradually add to this test suite. +# --Adam + +# FIXME-asyncAndEvmc +# The test_op_memory_lazy test fails under EVMC. +when not defined(evmc_enabled): + cliBuilder: + import ./test_op_arith, + ./test_op_bit, + ./test_op_env, + ./test_op_memory, + ./test_op_misc, + ./test_op_custom, + ./test_tracer_json, + ./test_op_memory_lazy diff --git a/tests/macro_assembler.nim b/tests/macro_assembler.nim index 033a0d797..49c6eb0f3 100644 --- a/tests/macro_assembler.nim +++ b/tests/macro_assembler.nim @@ -6,11 +6,16 @@ import import options, eth/trie/[db, hexary], ../nimbus/db/[db_chain, accounts_cache], + ../nimbus/vm2/[async_operations, types], ../nimbus/vm_internals, ../nimbus/forks, - ../nimbus/transaction/call_evm, + ../nimbus/transaction/[call_common, call_evm], ../nimbus/[transaction, chain_config, genesis, vm_types, vm_state], ../nimbus/utils/difficulty +# Need to exclude ServerCommand because it contains something +# called Stop that interferes with the EVM operation named Stop. +import chronos except ServerCommand + export byteutils {.experimental: "dynamicBindSym".} @@ -28,8 +33,11 @@ type Assembler* = object title*: string + chainDBIdentName*: string + vmStateIdentName*: string stack*: seq[VMWord] memory*: seq[VMWord] + initialStorage*: seq[Storage] storage*: seq[Storage] code*: seq[byte] logs*: seq[Log] @@ -40,6 +48,10 @@ type output*: seq[byte] fork*: Fork + ConcurrencyTest* = object + title*: string + assemblers*: seq[Assembler] + const idToOpcode = CacheTable"NimbusMacroAssembler" @@ -81,6 +93,16 @@ proc parseStorage(list: NimNode): seq[Storage] = for val in list: result.add validateStorage(val) +proc parseStringLiteral(node: NimNode): string = + let strNode = node[0] + strNode.expectKind(nnkStrLit) + strNode.strVal + +proc parseIdent(node: NimNode): string = + let identNode = node[0] + identNode.expectKind(nnkIdent) + identNode.strVal + proc parseSuccess(list: NimNode): bool = list.expectKind nnkStmtList list[0].expectKind(nnkIdent) @@ -180,21 +202,114 @@ proc parseGasUsed(gas: NimNode): GasInt = gas[0].expectKind(nnkIntLit) result = gas[0].intVal -proc generateVMProxy(boa: Assembler): NimNode = +proc parseAssembler(list: NimNode): Assembler = + result.success = true + result.fork = FkFrontier + result.gasUsed = -1 + list.expectKind nnkStmtList + for callSection in list: + callSection.expectKind(nnkCall) + let label = callSection[0].strVal + let body = callSection[1] + case label.normalize + of "title": result.title = parseStringLiteral(body) + of "vmstate": result.vmStateIdentName = parseIdent(body) + of "chaindb": result.chainDBIdentName = parseIdent(body) + of "code" : result.code = parseCode(body) + of "memory": result.memory = parseVMWords(body) + of "stack" : result.stack = parseVMWords(body) + of "storage": result.storage = parseStorage(body) + of "initialstorage": result.initialStorage = parseStorage(body) + of "logs": result.logs = parseLogs(body) + of "success": result.success = parseSuccess(body) + of "data": result.data = parseData(body) + of "output": result.output = parseData(body) + of "fork": result.fork = parseFork(body) + of "gasused": result.gasUsed = parseGasUsed(body) + else: error("unknown section '" & label & "'", callSection[0]) + +proc parseAssemblers(list: NimNode): seq[Assembler] = + result = @[] + list.expectKind nnkStmtList + for callSection in list: + # Should we do something with the label? Or is the + # assembler's "title" section good enough? + # let label = callSection[0].strVal + let body = callSection[1] + result.add parseAssembler(body) + +proc parseConcurrencyTest(list: NimNode): ConcurrencyTest = + list.expectKind nnkStmtList + for callSection in list: + callSection.expectKind(nnkCall) + let label = callSection[0].strVal + let body = callSection[1] + case label.normalize + of "title": result.title = parseStringLiteral(body) + of "assemblers": result.assemblers = parseAssemblers(body) + else: error("unknown section '" & label & "'", callSection[0]) + +type VMProxy = tuple[sym: NimNode, pr: NimNode] + +proc generateVMProxy(boa: Assembler, shouldBeAsync: bool): VMProxy = let - vmProxy = genSym(nskProc, "vmProxy") - chainDB = ident("chainDB") - vmState = ident("vmState") - title = boa.title + vmProxySym = genSym(nskProc, "asyncVMProxy") + chainDB = ident(if boa.chainDBIdentName == "": "chainDB" else: boa.chainDBIdentName) + vmState = ident(if boa.vmStateIdentName == "": "vmState" else: boa.vmStateIdentName) body = newLitFixed(boa) + returnType = if shouldBeAsync: + quote do: Future[bool] + else: + quote do: bool + runVMProcName = ident(if shouldBeAsync: "asyncRunVM" else: "runVM") + vmProxyProc = quote do: + proc `vmProxySym`(): `returnType` = + let boa = `body` + let asyncFactory = + AsyncOperationFactory( + lazyDataSource: + if len(boa.initialStorage) == 0: + noLazyDataSource() + else: + fakeLazyDataSource(boa.initialStorage)) + `runVMProcName`(`vmState`, `chainDB`, boa, asyncFactory) + (vmProxySym, vmProxyProc) + +proc generateAssemblerTest(boa: Assembler): NimNode = + let + (vmProxySym, vmProxyProc) = generateVMProxy(boa, false) + title: string = boa.title result = quote do: test `title`: - proc `vmProxy`(): bool = - let boa = `body` - runVM(`vmState`, `chainDB`, boa) + `vmProxyProc` {.gcsafe.}: - check `vmProxy`() + check `vmProxySym`() + + when defined(macro_assembler_debug): + echo result.toStrLit.strVal + +type + AsyncVMProxyTestProc* = proc(): Future[bool] + +proc generateConcurrencyTest(t: ConcurrencyTest): NimNode = + let + vmProxies: seq[VMProxy] = t.assemblers.map(proc(boa: Assembler): VMProxy = generateVMProxy(boa, true)) + vmProxyProcs: seq[NimNode] = vmProxies.map(proc(x: VMProxy): NimNode = x.pr) + vmProxySyms: seq[NimNode] = vmProxies.map(proc(x: VMProxy): NimNode = x.sym) + title: string = t.title + + let runVMProxy = quote do: + {.gcsafe.}: + let procs: seq[AsyncVMProxyTestProc] = @(`vmProxySyms`) + let futures: seq[Future[bool]] = procs.map(proc(s: AsyncVMProxyTestProc): Future[bool] = s()) + waitFor(allFutures(futures)) + + # Is there a way to use "quote" (or something like it) to splice + # in a statement list? + let stmtList = newStmtList(vmProxyProcs) + stmtList.add(runVMProxy) + result = newCall("test", newStrLitNode(title), stmtList) when defined(macro_assembler_debug): echo result.toStrLit.strVal @@ -220,26 +335,9 @@ proc initDatabase*(networkId = MainNet): (BaseVMState, BaseChainDB) = (vmState, db) -proc runVM*(vmState: BaseVMState, chainDB: BaseChainDB, boa: Assembler): bool = - const codeAddress = hexToByteArray[20]("460121576cc7df020759730751f92bd62fd78dd6") - let privateKey = PrivateKey.fromHex("7a28b5ba57c53603b0b07b56bba752f7784bf506fa95edc395f5cf6c7514fe9d")[] - - vmState.mutateStateDB: - db.setCode(codeAddress, boa.code) - db.setBalance(codeAddress, 1_000_000.u256) - - let unsignedTx = Transaction( - txType: TxLegacy, - nonce: 0, - gasPrice: 1.GasInt, - gasLimit: 500_000_000.GasInt, - to: codeAddress.some, - value: 500.u256, - payload: boa.data - ) - let tx = signTransaction(unsignedTx, privateKey, chainDB.config.chainId, false) - let asmResult = testCallEvm(tx, tx.getSender, vmState, boa.fork) +const codeAddress = hexToByteArray[20]("460121576cc7df020759730751f92bd62fd78dd6") +proc verifyAsmResult(vmState: BaseVMState, chainDB: BaseChainDB, boa: Assembler, asmResult: CallResult): bool = if not asmResult.isError: if boa.success == false: error "different success value", expected=boa.success, actual=true @@ -332,30 +430,43 @@ proc runVM*(vmState: BaseVMState, chainDB: BaseChainDB, boa: Assembler): bool = result = true +proc createSignedTx(boaData: Blob, chainId: ChainId): Transaction = + let privateKey = PrivateKey.fromHex("7a28b5ba57c53603b0b07b56bba752f7784bf506fa95edc395f5cf6c7514fe9d")[] + let unsignedTx = Transaction( + txType: TxLegacy, + nonce: 0, + gasPrice: 1.GasInt, + gasLimit: 500_000_000.GasInt, + to: codeAddress.some, + value: 500.u256, + payload: boaData + ) + signTransaction(unsignedTx, privateKey, chainId, false) + +proc runVM*(vmState: BaseVMState, chainDB: BaseChainDB, boa: Assembler, asyncFactory: AsyncOperationFactory): bool = + vmState.asyncFactory = asyncFactory + vmState.mutateStateDB: + db.setCode(codeAddress, boa.code) + db.setBalance(codeAddress, 1_000_000.u256) + let tx = createSignedTx(boa.data, chainDB.config.chainId) + let asmResult = testCallEvm(tx, tx.getSender, vmState, boa.fork) + verifyAsmResult(vmState, chainDB, boa, asmResult) + +# FIXME-duplicatedForAsync +proc asyncRunVM*(vmState: BaseVMState, chainDB: BaseChainDB, boa: Assembler, asyncFactory: AsyncOperationFactory): Future[bool] {.async.} = + vmState.asyncFactory = asyncFactory + vmState.mutateStateDB: + db.setCode(codeAddress, boa.code) + db.setBalance(codeAddress, 1_000_000.u256) + let tx = createSignedTx(boa.data, chainDB.config.chainId) + let asmResult = await asyncTestCallEvm(tx, tx.getSender, vmState, boa.fork) + return verifyAsmResult(vmState, chainDB, boa, asmResult) + macro assembler*(list: untyped): untyped = - var boa = Assembler(success: true, fork: FkFrontier, gasUsed: -1) - list.expectKind nnkStmtList - for callSection in list: - callSection.expectKind(nnkCall) - let label = callSection[0].strVal - let body = callSection[1] - case label.normalize - of "title": - let title = body[0] - title.expectKind(nnkStrLit) - boa.title = title.strVal - of "code" : boa.code = parseCode(body) - of "memory": boa.memory = parseVMWords(body) - of "stack" : boa.stack = parseVMWords(body) - of "storage": boa.storage = parseStorage(body) - of "logs": boa.logs = parseLogs(body) - of "success": boa.success = parseSuccess(body) - of "data": boa.data = parseData(body) - of "output": boa.output = parseData(body) - of "fork": boa.fork = parseFork(body) - of "gasused": boa.gasUsed = parseGasUsed(body) - else: error("unknown section '" & label & "'", callSection[0]) - result = boa.generateVMProxy() + result = parseAssembler(list).generateAssemblerTest() + +macro concurrentAssemblers*(list: untyped): untyped = + result = parseConcurrencyTest(list).generateConcurrencyTest() macro evmByteCode*(list: untyped): untyped = list.expectKind nnkStmtList diff --git a/tests/test_op_memory_lazy.nim b/tests/test_op_memory_lazy.nim new file mode 100644 index 000000000..60989897a --- /dev/null +++ b/tests/test_op_memory_lazy.nim @@ -0,0 +1,74 @@ +import macro_assembler, unittest2, macros, strutils + +proc opMemoryLazyMain*() = + suite "Lazy Loading With Memory Opcodes": + let (vmState, chainDB) = initDatabase() + + assembler: # SLOAD OP with (fake) lazy data fetching + title: "LAZY_SLOAD_1" + initialStorage: + "0xAA": "0x42" + code: + PUSH1 "0xAA" + SLOAD + PUSH1 "0x01" + ADD + PUSH1 "0xAA" + SSTORE + PUSH1 "0xAA" + SLOAD + storage: + "0xAA": "0x43" + stack: + "0x0000000000000000000000000000000000000000000000000000000000000043" + + let (vmState1, chainDB1) = initDatabase() + let (vmState2, chainDB2) = initDatabase() + concurrentAssemblers: + title: "Concurrent Assemblers" + assemblers: + asm1: + title: "asm1" + vmState: vmState1 + chainDB: chainDB1 + initialStorage: + "0xBB": "0x42" + "0xCC": "0x20" + code: + PUSH1 "0xBB" + SLOAD + PUSH1 "0xCC" + SLOAD + ADD + PUSH1 "0xBB" + SSTORE + PUSH1 "0xBB" + SLOAD + storage: + "0xBB": "0x62" + "0xCC": "0x20" + stack: "0x0000000000000000000000000000000000000000000000000000000000000062" + asm2: + title: "asm2" + vmState: vmState2 + chainDB: chainDB2 + initialStorage: + "0xDD": "0x30" + "0xEE": "0x20" + code: + PUSH1 "0xDD" + SLOAD + PUSH1 "0xEE" + SLOAD + ADD + PUSH1 "0xEE" + SSTORE + PUSH1 "0xEE" + SLOAD + storage: + "0xDD": "0x30" + "0xEE": "0x50" + stack: "0x0000000000000000000000000000000000000000000000000000000000000050" + +when isMainModule: + opMemoryLazyMain()