Added basic async capabilities for vm2. (#1260)

* Added basic async capabilities for vm2.

This is a whole new Git branch, not the same one as last time
(https://github.com/status-im/nimbus-eth1/pull/1250) - there wasn't
much worth salvaging. Main differences:

I didn't do the "each opcode has to specify an async handler" junk
that I put in last time. Instead, in oph_memory.nim you can see
sloadOp calling asyncChainTo and passing in an async operation.
That async operation is then run by the execCallOrCreate (or
asyncExecCallOrCreate) code in interpreter_dispatch.nim.

In the test code, the (previously existing) macro called "assembler"
now allows you to add a section called "initialStorage", specifying
fake data to be used by the EVM computation run by that test. (In
the long run we'll obviously want to write tests that for-real use
the JSON-RPC API to asynchronously fetch data; for now, this was
just an expedient way to write a basic unit test that exercises the
async-EVM code pathway.)

There's also a new macro called "concurrentAssemblers" that allows
you to write a test that runs multiple assemblers concurrently (and
then waits for them all to finish). There's one example test using
this, in test_op_memory_lazy.nim, though you can't actually see it
doing so unless you uncomment some echo statements in
async_operations.nim (in which case you can see the two concurrently
running EVM computations each printing out what they're doing, and
you'll see that they interleave).

A question: is it possible to make EVMC work asynchronously? (For
now, this code compiles and "make test" passes even if ENABLE_EVMC
is turned on, but it doesn't actually work asynchronously, it just
falls back on doing the usual synchronous EVMC thing. See
FIXME-asyncAndEvmc.)

* Moved the AsyncOperationFactory to the BaseVMState object.

* Made the AsyncOperationFactory into a table of fn pointers.

Also ditched the plain-data Vm2AsyncOperation type; it wasn't
really serving much purpose. Instead, the pendingAsyncOperation
field directly contains the Future.

* Removed the hasStorage idea.

It's not the right solution to the "how do we know whether we
still need to fetch the storage value or not?" problem. I
haven't implemented the right solution yet, but at least
we're better off not putting in a wrong one.

* Added/modified/removed some comments.

(Based on feedback on the PR.)

* Removed the waitFor from execCallOrCreate.

There was some back-and-forth in the PR regarding whether nested
waitFor calls are acceptable:

https://github.com/status-im/nimbus-eth1/pull/1260#discussion_r998587449

The eventual decision was to just change the waitFor to a doAssert
(since we probably won't want this extra functionality when running
synchronously anyway) to make sure that the Future is already
finished.

Makefile

@@ -175,6 +175,10 @@ test: | build deps
 	$(ENV_SCRIPT) nim test_rocksdb $(NIM_PARAMS) nimbus.nims
 	$(ENV_SCRIPT) nim test $(NIM_PARAMS) nimbus.nims
 
+# builds and runs an EVM-related subset of the nimbus test suite
+test-evm: | build deps
+	$(ENV_SCRIPT) nim test_evm $(NIM_PARAMS) nimbus.nims
+
 # Primitive reproducibility test.
 #
 # On some platforms, with some GCC versions, it may not be possible to get a

nimbus.nimble

@@ -66,6 +66,9 @@ task test, "Run tests":
 task test_rocksdb, "Run rocksdb tests":
   test "tests/db", "test_kvstore_rocksdb", "-d:chronicles_log_level=ERROR -d:unittest2DisableParamFiltering"
 
+task test_evm, "Run EVM tests":
+  test "tests", "evm_tests", "-d:chronicles_log_level=ERROR -d:unittest2DisableParamFiltering"
+
 ## Fluffy tasks
 
 task fluffy, "Build fluffy":

nimbus/transaction/call_common.nim

@@ -8,6 +8,7 @@
 
 import
   eth/common/eth_types, stint, options, stew/ranges/ptr_arith,
+  chronos,
   ".."/[vm_types, vm_state, vm_computation, vm_state_transactions],
   ".."/[vm_internals, vm_precompiles, vm_gas_costs],
   ".."/[db/accounts_cache, forks],
@@ -160,6 +161,7 @@ proc setupHost(call: CallParams): TransactionHost =
 
     let cMsg = hostToComputationMessage(host.msg)
     host.computation = newComputation(vmState, cMsg, code)
+
     shallowCopy(host.code, code)
 
   else:
@@ -197,10 +199,12 @@ when defined(evmc_enabled):
       {.gcsafe.}:
         callResult.release(callResult)
 
-proc runComputation*(call: CallParams): CallResult =
+# FIXME-awkwardFactoring: the factoring out of the pre and
-  let host = setupHost(call)
+# post parts feels awkward to me, but for now I'd really like
-  let c = host.computation
+# not to have too much duplicated code between sync and async.
+# --Adam
 
+proc prepareToRunComputation(host: TransactionHost, call: CallParams) =
   # Must come after `setupHost` for correct fork.
   if not call.noAccessList:
     initialAccessListEIP2929(call)
@@ -210,10 +214,8 @@ proc runComputation*(call: CallParams): CallResult =
     host.vmState.mutateStateDB:
       db.subBalance(call.sender, call.gasLimit.u256 * call.gasPrice.u256)
 
-  when defined(evmc_enabled):
+proc calculateAndPossiblyRefundGas(host: TransactionHost, call: CallParams): GasInt =
-    doExecEvmc(host, call)
+  let c = host.computation
-  else:
-    execComputation(host.computation)
   
   # EIP-3529: Reduction in refunds
   let MaxRefundQuotient = if host.vmState.fork >= FkLondon:
@@ -222,19 +224,23 @@ proc runComputation*(call: CallParams): CallResult =
                             2.GasInt
 
   # Calculated gas used, taking into account refund rules.
-  var gasRemaining: GasInt = 0
   if call.noRefund:
-    gasRemaining = c.gasMeter.gasRemaining
+    result = c.gasMeter.gasRemaining
   elif not c.shouldBurnGas:
     let maxRefund = (call.gasLimit - c.gasMeter.gasRemaining) div MaxRefundQuotient
     let refund = min(c.getGasRefund(), maxRefund)
     c.gasMeter.returnGas(refund)
-    gasRemaining = c.gasMeter.gasRemaining
+    result = c.gasMeter.gasRemaining
 
   # Refund for unused gas.
-  if gasRemaining > 0 and not call.noGasCharge:
+  if result > 0 and not call.noGasCharge:
     host.vmState.mutateStateDB:
-      db.addBalance(call.sender, gasRemaining.u256 * call.gasPrice.u256)
+      db.addBalance(call.sender, result.u256 * call.gasPrice.u256)
+
+proc finishRunningComputation(host: TransactionHost, call: CallParams): CallResult =
+  let c = host.computation
+  
+  let gasRemaining = calculateAndPossiblyRefundGas(host, call)
 
   result.isError = c.isError
   result.gasUsed = call.gasLimit - gasRemaining
@@ -244,3 +250,27 @@ proc runComputation*(call: CallParams): CallResult =
   shallowCopy(result.logEntries, c.logEntries)
   result.stack = c.stack
   result.memory = c.memory
+
+proc runComputation*(call: CallParams): CallResult =
+  let host = setupHost(call)
+  prepareToRunComputation(host, call)
+
+  when defined(evmc_enabled):
+    doExecEvmc(host, call)
+  else:
+    execComputation(host.computation)
+
+  finishRunningComputation(host, call)
+
+# FIXME-duplicatedForAsync
+proc asyncRunComputation*(call: CallParams): Future[CallResult] {.async.} =
+  let host = setupHost(call)
+  prepareToRunComputation(host, call)
+
+  # FIXME-asyncAndEvmc: I'm not sure what to do with EVMC at the moment.
+  # when defined(evmc_enabled):
+  #   doExecEvmc(host, call)
+  # else:
+  await asyncExecComputation(host.computation)
+
+  return finishRunningComputation(host, call)

nimbus/transaction/call_evm.nim

@@ -9,6 +9,7 @@
 import
   std/[options, times],
   chronicles,
+  chronos,
   eth/[common/eth_types_rlp, trie/db],
   stint,
   ".."/[vm_types, vm_state, vm_gas_costs, forks, constants],
@@ -177,8 +178,10 @@ proc rpcEstimateGas*(cd: RpcCallData, header: BlockHeader, chainDB: BaseChainDB,
 
   hi
 
-proc txCallEvm*(tx: Transaction, sender: EthAddress, vmState: BaseVMState, fork: Fork): GasInt =
+proc callParamsForTx(tx: Transaction, sender: EthAddress, vmState: BaseVMState, fork: Fork): CallParams =
-  var call = CallParams(
+  # Is there a nice idiom for this kind of thing? Should I
+  # just be writing this as a bunch of assignment statements?
+  result = CallParams(
     vmState:      vmState,
     forkOverride: some(fork),
     gasPrice:     tx.gasPrice,
@@ -190,11 +193,10 @@ proc txCallEvm*(tx: Transaction, sender: EthAddress, vmState: BaseVMState, fork:
     input:        tx.payload
   )
   if tx.txType > TxLegacy:
-    shallowCopy(call.accessList, tx.accessList)
+    shallowCopy(result.accessList, tx.accessList)
-  return runComputation(call).gasUsed
 
-proc testCallEvm*(tx: Transaction, sender: EthAddress, vmState: BaseVMState, fork: Fork): CallResult =
+proc callParamsForTest(tx: Transaction, sender: EthAddress, vmState: BaseVMState, fork: Fork): CallParams =
-  var call = CallParams(
+  result = CallParams(
     vmState:      vmState,
     forkOverride: some(fork),
     gasPrice:     tx.gasPrice,
@@ -209,5 +211,17 @@ proc testCallEvm*(tx: Transaction, sender: EthAddress, vmState: BaseVMState, for
     noRefund:     true, # Don't apply gas refund/burn rule.
   )
   if tx.txType > TxLegacy:
-    shallowCopy(call.accessList, tx.accessList)
+    shallowCopy(result.accessList, tx.accessList)
+
+proc txCallEvm*(tx: Transaction, sender: EthAddress, vmState: BaseVMState, fork: Fork): GasInt =
+  let call = callParamsForTx(tx, sender, vmState, fork)
+  return runComputation(call).gasUsed
+
+proc testCallEvm*(tx: Transaction, sender: EthAddress, vmState: BaseVMState, fork: Fork): CallResult =
+  let call = callParamsForTest(tx, sender, vmState, fork)
   runComputation(call)
+
+# FIXME-duplicatedForAsync
+proc asyncTestCallEvm*(tx: Transaction, sender: EthAddress, vmState: BaseVMState, fork: Fork): Future[CallResult] {.async.} =
+  let call = callParamsForTest(tx, sender, vmState, fork)
+  return await asyncRunComputation(call)

nimbus/vm2/async_operations.nim

@@ -0,0 +1,104 @@
+# Nimbus
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
+#    http://www.apache.org/licenses/LICENSE-2.0)
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT) or
+#    http://opensource.org/licenses/MIT)
+# at your option. This file may not be copied, modified, or distributed except
+# according to those terms.
+
+import
+  chronos,
+  stint,
+  json_rpc/rpcclient,
+  web3,
+  ./computation,
+  ./state,
+  ./types,
+  ../db/accounts_cache
+
+
+
+# Used in synchronous mode.
+proc noLazyDataSource*(): LazyDataSource =
+  LazyDataSource(
+    ifNecessaryGetStorage: (proc(c: Computation, slot: UInt256): Future[void] {.async.} =
+      discard
+    )
+  )
+
+# Will be used in asynchronous on-demand-data-fetching mode, once
+# that is implemented.
+proc realLazyDataSource*(client: RpcClient): LazyDataSource =
+  LazyDataSource(
+    ifNecessaryGetStorage: (proc(c: Computation, slot: UInt256): Future[void] {.async.} =
+      # TODO: find some way to check whether we already have it.
+      # This is WRONG, but good enough for now, considering this
+      # code is unused except in a few tests. I'm working on
+      # doing this properly.
+      if not c.getStorage(slot).isZero: return
+
+      # FIXME-onDemandStorageNotImplementedYet
+      # (I sketched in this code, but haven't actually tried running it yet.)
+      echo("Attempting to for-real fetch slot " & $(slot))
+      # ethAddressStr("0xfff33a3bd36abdbd412707b8e310d6011454a7ae")
+      # check hexDataStr(0.u256).string == res.string
+      let ethAddress = c.msg.contractAddress
+      let address: Address = Address(ethAddress)
+      let quantity: int = slot.truncate(int)  # this is probably wrong; what's the right way to convert this?
+      let blockId: BlockIdentifier = blockId(c.vmState.parent.blockNumber.truncate(uint64)) # ditto
+      let res = await client.eth_getStorageAt(address, quantity, blockId)
+      echo("Fetched slot " & $(slot) & ", result is " & $(res))
+      let v = res  # will res be the actual value, or do I need to convert or something?
+      
+      # Before implementing this, see the note from Zahary here:
+      # https://github.com/status-im/nimbus-eth1/pull/1260#discussion_r999669139
+      # 
+      # c.vmState.mutateStateDB:
+      #   db.setStorage(c.msg.contractAddress, slot, UInt256.fromBytesBE(v))
+    )
+  )
+
+# Used for unit testing. Contains some prepopulated data.
+proc fakeLazyDataSource*(fakePairs: seq[tuple[key, val: array[32, byte]]]): LazyDataSource =
+  LazyDataSource(
+    ifNecessaryGetStorage: (proc(c: Computation, slot: UInt256): Future[void] {.async.} =
+      # See the comment above.
+      if not c.getStorage(slot).isZero: return
+
+      # FIXME-writeAutomatedTestsToShowThatItCanRunConcurrently
+
+      # For now, until I've implemented some more automated way to
+      # capture and verify the fact that this can run concurrently,
+      # this is useful just to see in the console that the echo
+      # statements from multiple Computations can run at the same
+      # time and be interleaved.
+      # echo("Attempting to fake-fetch slot " & $(slot))
+      # await sleepAsync(2.seconds)  
+
+      let slotBytes = toBytesBE(slot)
+      # The linear search is obviously slow, but doesn't matter
+      # for tests with only a few initialStorage entries. Fix
+      # this if we ever want to write tests with more.
+      for (k, v) in fakePairs:
+        if slotBytes == k:
+          c.vmState.mutateStateDB:
+            db.setStorage(c.msg.contractAddress, slot, UInt256.fromBytesBE(v))
+          break
+
+      # echo("Finished fake-fetch of slot " & $(slot))
+    )
+  )
+
+
+
+
+# Gotta find the place where we're creating a Computation without setting
+# its asyncFactory in the first place, but this is fine for now.
+proc asyncFactory*(c: Computation): AsyncOperationFactory =
+  # Does Nim have an "ifNil" macro/template?
+  if isNil(c.vmState.asyncFactory):
+    AsyncOperationFactory(lazyDataSource: noLazyDataSource()) # AARDVARK - can I make a singleton?
+  else:
+    c.vmState.asyncFactory

nimbus/vm2/computation.nim

@@ -21,6 +21,7 @@ import
   ./transaction_tracer,
   ./types,
   chronicles,
+  chronos,
   eth/[common, keys],
   options,
   sets
@@ -304,6 +305,13 @@ template chainTo*(c: Computation, toChild: typeof(c.child), after: untyped) =
     c.continuation = nil
     after
 
+# Register an async operation to be performed before the continuation is called.
+template asyncChainTo*(c: Computation, asyncOperation: Future[void], after: untyped) =
+  c.pendingAsyncOperation = asyncOperation
+  c.continuation = proc() =
+    c.continuation = nil
+    after
+
 proc merge*(c, child: Computation) =
   c.logEntries.add child.logEntries
   c.gasMeter.refundGas(child.gasMeter.gasRefunded)

nimbus/vm2/interpreter/op_dispatcher.nim

@@ -98,7 +98,7 @@ proc toCaseStmt(forkArg, opArg, k: NimNode): NimNode =
     # Wrap innner case/switch into outer case/switch
     let branchStmt = block:
       case op
-      of Create, Create2, Call, CallCode, DelegateCall, StaticCall:
+      of Create, Create2, Call, CallCode, DelegateCall, StaticCall, Sload:
         quote do:
           `forkCaseSubExpr`
           if not `k`.cpt.continuation.isNil:
@@ -135,7 +135,7 @@ template genLowMemDispatcher*(fork: Fork; op: Op; k: Vm2Ctx) =
     handleOtherDirective(fork, op, k)
 
   case c.instr
-  of Create, Create2, Call, CallCode, DelegateCall, StaticCall:
+  of Create, Create2, Call, CallCode, DelegateCall, StaticCall, Sload:
     if not k.cpt.continuation.isNil:
       break
   of Return, Revert, SelfDestruct:

nimbus/vm2/interpreter/op_handlers/oph_memory.nim

@@ -14,6 +14,7 @@
 
 import
   ../../../errors,
+  ../../async_operations,
   ../../code_stream,
   ../../computation,
   ../../memory,
@@ -163,10 +164,11 @@ const
 
   sloadOp: Vm2OpFn = proc (k: var Vm2Ctx) =
     ## 0x54, Load word from storage.
-    let (slot) = k.cpt.stack.popInt(1)
+    let cpt = k.cpt  # so it can safely be captured by the asyncChainTo closure below
-    k.cpt.stack.push:
+    let (slot) = cpt.stack.popInt(1)
-      k.cpt.getStorage(slot)
+    cpt.asyncChainTo(asyncFactory(cpt).lazyDataSource.ifNecessaryGetStorage(cpt, slot)):
-
+      cpt.stack.push:
+        cpt.getStorage(slot)
 
   sloadEIP2929Op: Vm2OpFn = proc (k: var Vm2Ctx) =
     ## 0x54, EIP2929: Load word from storage for Berlin and later

nimbus/vm2/interpreter_dispatch.nim

@@ -25,6 +25,7 @@ import
   ./state,
   ./types,
   chronicles,
+  chronos,
   eth/[common, keys],
   macros,
   options,
@@ -45,12 +46,20 @@ const
 # Private functions
 # ------------------------------------------------------------------------------
 
-proc selectVM(c: Computation, fork: Fork) {.gcsafe.} =
+proc selectVM(c: Computation, fork: Fork, shouldPrepareTracer: bool) {.gcsafe.} =
   ## Op code execution handler main loop.
   var desc: Vm2Ctx
   desc.cpt = c
 
-  if c.tracingEnabled:
+  # It's important not to re-prepare the tracer after
+  # an async operation, only after a call/create.
+  #
+  # That is, tracingEnabled is checked in many places, and
+  # indicates something like, "Do we want tracing to be
+  # enabled?", whereas shouldPrepareTracer is more like,
+  # "Are we at a spot right now where we want to re-initialize
+  # the tracer?"
+  if c.tracingEnabled and shouldPrepareTracer:
     c.prepareTracer()
 
   while true:
@@ -195,7 +204,7 @@ proc afterExec(c: Computation) =
 # Public functions
 # ------------------------------------------------------------------------------
 
-proc executeOpcodes*(c: Computation) =
+proc executeOpcodes*(c: Computation, shouldPrepareTracer: bool = true) =
   let fork = c.fork
 
   block:
@@ -205,7 +214,7 @@ proc executeOpcodes*(c: Computation) =
     try:
       if not c.continuation.isNil:
         (c.continuation)()
-      c.selectVM(fork)
+      c.selectVM(fork, shouldPrepareTracer)
     except CatchableError as e:
       c.setError(
         &"Opcode Dispatch Error msg={e.msg}, depth={c.msg.depth}", true)
@@ -222,6 +231,14 @@ when vm_use_recursion:
       return
     c.executeOpcodes()
     while not c.continuation.isNil:
+      # If there's a continuation, then it's because there's either
+      # a child (i.e. call or create) or a pendingAsyncOperation.
+      if not c.pendingAsyncOperation.isNil:
+        let p = c.pendingAsyncOperation
+        c.pendingAsyncOperation = nil
+        doAssert(p.finished(), "In synchronous mode, every async operation should be an already-resolved Future.")
+        c.executeOpcodes(false)
+      else:
         when evmc_enabled:
           c.res = c.host.call(c.child[])
         else:
@@ -232,7 +249,7 @@ when vm_use_recursion:
 
 else:
   proc execCallOrCreate*(cParam: Computation) =
-    var (c, before) = (cParam, true)
+    var (c, before, shouldPrepareTracer) = (cParam, true, true)
     defer:
       while not c.isNil:
         c.dispose()
@@ -243,15 +260,60 @@ else:
       while true:
         if before and c.beforeExec():
           break
-        c.executeOpcodes()
+        c.executeOpcodes(shouldPrepareTracer)
         if c.continuation.isNil:
           c.afterExec()
           break
-        (before, c.child, c, c.parent) = (true, nil.Computation, c.child, c)
+        if not c.pendingAsyncOperation.isNil:
+          before = false
+          shouldPrepareTracer = false
+          let p = c.pendingAsyncOperation
+          c.pendingAsyncOperation = nil
+          doAssert(p.finished(), "In synchronous mode, every async operation should be an already-resolved Future.")
+        else:
+          (before, shouldPrepareTracer, c.child, c, c.parent) = (true, true, nil.Computation, c.child, c)
       if c.parent.isNil:
         break
       c.dispose()
-      (before, c.parent, c) = (false, nil.Computation, c.parent)
+      (before, shouldPrepareTracer, c.parent, c) = (false, true, nil.Computation, c.parent)
+
+# FIXME-duplicatedForAsync
+#
+# In the long run I'd like to make some clever macro/template to
+# eliminate the duplication between the synchronous and
+# asynchronous versions. But for now let's stick with this for
+# simplicity. 
+#
+# Also, I've based this on the recursive one (above), which I think
+# is okay because the "async" pragma is going to rewrite this whole
+# thing to use callbacks anyway. But maybe I'm wrong? It isn't hard
+# to write the async version of the iterative one, but this one is
+# a bit shorter and feels cleaner, so if it works just as well I'd
+# rather use this one. --Adam
+proc asyncExecCallOrCreate*(c: Computation): Future[void] {.async.} =
+  defer: c.dispose()
+  if c.beforeExec():
+    return
+  c.executeOpcodes()
+  while not c.continuation.isNil:
+    # If there's a continuation, then it's because there's either
+    # a child (i.e. call or create) or a pendingAsyncOperation.
+    if not c.pendingAsyncOperation.isNil:
+      let p = c.pendingAsyncOperation
+      c.pendingAsyncOperation = nil
+      await p
+      c.executeOpcodes(false)
+    else:
+      when evmc_enabled:
+        # FIXME-asyncAndEvmc
+        # Note that this is NOT async. I'm not sure how/whether I
+        # can do EVMC asynchronously.
+        c.res = c.host.call(c.child[])
+      else:
+        await asyncExecCallOrCreate(c.child)
+      c.child = nil
+      c.executeOpcodes()
+  c.afterExec()
 
 # ------------------------------------------------------------------------------
 # End

nimbus/vm2/state_transactions.nim

@@ -21,6 +21,7 @@ import
   ./state,
   ./types,
   chronicles,
+  chronos,
   eth/common,
   eth/common/eth_types,
   options,
@@ -46,13 +47,17 @@ proc refundGas*(c: Computation, tx: Transaction, sender: EthAddress) =
     db.addBalance(sender, c.gasMeter.gasRemaining.u256 * tx.gasPrice.u256)
 
 
-proc execComputation*(c: Computation) =
+# FIXME-awkwardFactoring: the factoring out of the pre and
+# post parts feels awkward to me, but for now I'd really like
+# not to have too much duplicated code between sync and async.
+# --Adam
+
+proc preExecComputation(c: Computation) =
   if not c.msg.isCreate:
     c.vmState.mutateStateDB:
       db.incNonce(c.msg.sender)
 
-  c.execCallOrCreate()
+proc postExecComputation(c: Computation) =
-
   if c.isSuccess:
     if c.fork < FkLondon:
       # EIP-3529: Reduction in refunds
@@ -62,3 +67,14 @@ proc execComputation*(c: Computation) =
     c.vmState.touchedAccounts.incl c.touchedAccounts
 
   c.vmState.status = c.isSuccess
+
+proc execComputation*(c: Computation) =
+  c.preExecComputation()
+  c.execCallOrCreate()
+  c.postExecComputation()
+
+# FIXME-duplicatedForAsync
+proc asyncExecComputation*(c: Computation): Future[void] {.async.} =
+  c.preExecComputation()
+  await c.asyncExecCallOrCreate()
+  c.postExecComputation()

nimbus/vm2/types.nim

@@ -11,6 +11,8 @@
 import
   tables, eth/common,
   options, json, sets,
+  chronos, stint,
+  json_rpc/rpcclient,
   ./stack,  ./memory, ./code_stream, ../forks,
   ./interpreter/[gas_costs, op_codes],
   # TODO - will be hidden at a lower layer
@@ -55,6 +57,7 @@ type
     gasCosts*      : GasCosts
     fork*          : Fork
     minerAddress*  : EthAddress
+    asyncFactory*  : AsyncOperationFactory
 
   TracerFlags* {.pure.} = enum
     EnableTracing
@@ -96,6 +99,7 @@ type
       res*:                 nimbus_result
     else:
       parent*, child*:      Computation
+    pendingAsyncOperation*: Future[void]
     continuation*:          proc() {.gcsafe.}
 
   Error* = ref object
@@ -127,3 +131,9 @@ type
     value*:            UInt256
     data*:             seq[byte]
     flags*:            MsgFlags
+
+  LazyDataSource* = ref object of RootObj
+    ifNecessaryGetStorage*: proc(c: Computation, slot: UInt256): Future[void] {.gcsafe.}
+  
+  AsyncOperationFactory* = ref object of RootObj
+    lazyDataSource*: LazyDataSource

nimbus/vm_state_transactions.nim

@@ -12,6 +12,7 @@ import
   vm2/state_transactions as vmx
 
 export
+  vmx.asyncExecComputation,
   vmx.execComputation
 
 # End

tests/evm_tests.nim

@@ -0,0 +1,27 @@
+# Nimbus
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT)
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+import ../test_macro
+
+{. warning[UnusedImport]:off .}
+
+# This file is just meant to gather a bunch of EVM tests in one
+# place. I want to be able to gradually add to this test suite.
+# --Adam
+
+# FIXME-asyncAndEvmc
+# The test_op_memory_lazy test fails under EVMC.
+when not defined(evmc_enabled):
+  cliBuilder:
+    import  ./test_op_arith,
+            ./test_op_bit,
+            ./test_op_env,
+            ./test_op_memory,
+            ./test_op_misc,
+            ./test_op_custom,
+            ./test_tracer_json,
+            ./test_op_memory_lazy

tests/macro_assembler.nim

@@ -6,11 +6,16 @@ import
 import
   options, eth/trie/[db, hexary],
   ../nimbus/db/[db_chain, accounts_cache],
+  ../nimbus/vm2/[async_operations, types],
   ../nimbus/vm_internals, ../nimbus/forks,
-  ../nimbus/transaction/call_evm,
+  ../nimbus/transaction/[call_common, call_evm],
   ../nimbus/[transaction, chain_config, genesis, vm_types, vm_state],
   ../nimbus/utils/difficulty
 
+# Need to exclude ServerCommand because it contains something
+# called Stop that interferes with the EVM operation named Stop.
+import chronos except ServerCommand
+
 export byteutils
 {.experimental: "dynamicBindSym".}
 
@@ -28,8 +33,11 @@ type
 
   Assembler* = object
     title*: string
+    chainDBIdentName*: string
+    vmStateIdentName*: string
     stack*: seq[VMWord]
     memory*: seq[VMWord]
+    initialStorage*: seq[Storage]
     storage*: seq[Storage]
     code*: seq[byte]
     logs*: seq[Log]
@@ -40,6 +48,10 @@ type
     output*: seq[byte]
     fork*: Fork
 
+  ConcurrencyTest* = object
+    title*: string
+    assemblers*: seq[Assembler]
+
 const
   idToOpcode = CacheTable"NimbusMacroAssembler"
 
@@ -81,6 +93,16 @@ proc parseStorage(list: NimNode): seq[Storage] =
   for val in list:
     result.add validateStorage(val)
 
+proc parseStringLiteral(node: NimNode): string =
+  let strNode = node[0]
+  strNode.expectKind(nnkStrLit)
+  strNode.strVal
+
+proc parseIdent(node: NimNode): string =
+  let identNode = node[0]
+  identNode.expectKind(nnkIdent)
+  identNode.strVal
+
 proc parseSuccess(list: NimNode): bool =
   list.expectKind nnkStmtList
   list[0].expectKind(nnkIdent)
@@ -180,21 +202,114 @@ proc parseGasUsed(gas: NimNode): GasInt =
   gas[0].expectKind(nnkIntLit)
   result = gas[0].intVal
 
-proc generateVMProxy(boa: Assembler): NimNode =
+proc parseAssembler(list: NimNode): Assembler =
+  result.success = true
+  result.fork = FkFrontier
+  result.gasUsed = -1
+  list.expectKind nnkStmtList
+  for callSection in list:
+    callSection.expectKind(nnkCall)
+    let label = callSection[0].strVal
+    let body  = callSection[1]
+    case label.normalize
+    of "title": result.title = parseStringLiteral(body)
+    of "vmstate": result.vmStateIdentName = parseIdent(body)
+    of "chaindb": result.chainDBIdentName = parseIdent(body)
+    of "code"  : result.code = parseCode(body)
+    of "memory": result.memory = parseVMWords(body)
+    of "stack" : result.stack = parseVMWords(body)
+    of "storage": result.storage = parseStorage(body)
+    of "initialstorage": result.initialStorage = parseStorage(body)
+    of "logs": result.logs = parseLogs(body)
+    of "success": result.success = parseSuccess(body)
+    of "data": result.data = parseData(body)
+    of "output": result.output = parseData(body)
+    of "fork": result.fork = parseFork(body)
+    of "gasused": result.gasUsed = parseGasUsed(body)
+    else: error("unknown section '" & label & "'", callSection[0])
+
+proc parseAssemblers(list: NimNode): seq[Assembler] =
+  result = @[]
+  list.expectKind nnkStmtList
+  for callSection in list:
+    # Should we do something with the label? Or is the
+    # assembler's "title" section good enough?
+    # let label = callSection[0].strVal
+    let body  = callSection[1]
+    result.add parseAssembler(body)
+
+proc parseConcurrencyTest(list: NimNode): ConcurrencyTest =
+  list.expectKind nnkStmtList
+  for callSection in list:
+    callSection.expectKind(nnkCall)
+    let label = callSection[0].strVal
+    let body  = callSection[1]
+    case label.normalize
+    of "title": result.title = parseStringLiteral(body)
+    of "assemblers": result.assemblers = parseAssemblers(body)
+    else: error("unknown section '" & label & "'", callSection[0])
+
+type VMProxy = tuple[sym: NimNode, pr: NimNode]
+
+proc generateVMProxy(boa: Assembler, shouldBeAsync: bool): VMProxy =
   let
-    vmProxy = genSym(nskProc, "vmProxy")
+    vmProxySym = genSym(nskProc, "asyncVMProxy")
-    chainDB = ident("chainDB")
+    chainDB = ident(if boa.chainDBIdentName == "": "chainDB" else: boa.chainDBIdentName)
-    vmState = ident("vmState")
+    vmState = ident(if boa.vmStateIdentName == "": "vmState" else: boa.vmStateIdentName)
-    title = boa.title
     body = newLitFixed(boa)
+    returnType = if shouldBeAsync:
+                   quote do: Future[bool]
+                 else:
+                   quote do: bool
+    runVMProcName = ident(if shouldBeAsync: "asyncRunVM" else: "runVM")
+    vmProxyProc = quote do:
+      proc `vmProxySym`(): `returnType` =
+        let boa = `body`
+        let asyncFactory =
+          AsyncOperationFactory(
+            lazyDataSource:
+              if len(boa.initialStorage) == 0:
+                noLazyDataSource()
+              else:
+                fakeLazyDataSource(boa.initialStorage))
+        `runVMProcName`(`vmState`, `chainDB`, boa, asyncFactory)
+  (vmProxySym, vmProxyProc)
+
+proc generateAssemblerTest(boa: Assembler): NimNode =
+  let
+    (vmProxySym, vmProxyProc) = generateVMProxy(boa, false)
+    title: string = boa.title
 
   result = quote do:
     test `title`:
-      proc `vmProxy`(): bool =
+      `vmProxyProc`
-        let boa = `body`
-        runVM(`vmState`, `chainDB`, boa)
       {.gcsafe.}:
-        check `vmProxy`()
+        check `vmProxySym`()
+
+  when defined(macro_assembler_debug):
+    echo result.toStrLit.strVal
+
+type
+  AsyncVMProxyTestProc* = proc(): Future[bool]
+
+proc generateConcurrencyTest(t: ConcurrencyTest): NimNode =
+  let
+    vmProxies: seq[VMProxy] = t.assemblers.map(proc(boa: Assembler): VMProxy = generateVMProxy(boa, true))
+    vmProxyProcs: seq[NimNode] = vmProxies.map(proc(x: VMProxy): NimNode = x.pr)
+    vmProxySyms: seq[NimNode] = vmProxies.map(proc(x: VMProxy): NimNode = x.sym)
+    title: string = t.title
+
+  let runVMProxy = quote do:
+    {.gcsafe.}:
+      let procs: seq[AsyncVMProxyTestProc] = @(`vmProxySyms`)
+      let futures: seq[Future[bool]] = procs.map(proc(s: AsyncVMProxyTestProc): Future[bool] = s())
+      waitFor(allFutures(futures))
+
+  # Is there a way to use "quote" (or something like it) to splice
+  # in a statement list?
+  let stmtList = newStmtList(vmProxyProcs)
+  stmtList.add(runVMProxy)
+  result = newCall("test", newStrLitNode(title), stmtList)
 
   when defined(macro_assembler_debug):
     echo result.toStrLit.strVal
@@ -220,26 +335,9 @@ proc initDatabase*(networkId = MainNet): (BaseVMState, BaseChainDB) =
 
   (vmState, db)
 
-proc runVM*(vmState: BaseVMState, chainDB: BaseChainDB, boa: Assembler): bool =
+const codeAddress = hexToByteArray[20]("460121576cc7df020759730751f92bd62fd78dd6")
-  const codeAddress = hexToByteArray[20]("460121576cc7df020759730751f92bd62fd78dd6")
-  let privateKey = PrivateKey.fromHex("7a28b5ba57c53603b0b07b56bba752f7784bf506fa95edc395f5cf6c7514fe9d")[]
-
-  vmState.mutateStateDB:
-    db.setCode(codeAddress, boa.code)
-    db.setBalance(codeAddress, 1_000_000.u256)
-
-  let unsignedTx = Transaction(
-    txType: TxLegacy,
-    nonce: 0,
-    gasPrice: 1.GasInt,
-    gasLimit: 500_000_000.GasInt,
-    to: codeAddress.some,
-    value: 500.u256,
-    payload: boa.data
-  )
-  let tx = signTransaction(unsignedTx, privateKey, chainDB.config.chainId, false)
-  let asmResult = testCallEvm(tx, tx.getSender, vmState, boa.fork)
 
+proc verifyAsmResult(vmState: BaseVMState, chainDB: BaseChainDB, boa: Assembler, asmResult: CallResult): bool =
   if not asmResult.isError:
     if boa.success == false:
       error "different success value", expected=boa.success, actual=true
@@ -332,30 +430,43 @@ proc runVM*(vmState: BaseVMState, chainDB: BaseChainDB, boa: Assembler): bool =
 
   result = true
 
+proc createSignedTx(boaData: Blob, chainId: ChainId): Transaction =
+  let privateKey = PrivateKey.fromHex("7a28b5ba57c53603b0b07b56bba752f7784bf506fa95edc395f5cf6c7514fe9d")[]
+  let unsignedTx = Transaction(
+    txType: TxLegacy,
+    nonce: 0,
+    gasPrice: 1.GasInt,
+    gasLimit: 500_000_000.GasInt,
+    to: codeAddress.some,
+    value: 500.u256,
+    payload: boaData
+  )
+  signTransaction(unsignedTx, privateKey, chainId, false)
+
+proc runVM*(vmState: BaseVMState, chainDB: BaseChainDB, boa: Assembler, asyncFactory: AsyncOperationFactory): bool =
+  vmState.asyncFactory = asyncFactory
+  vmState.mutateStateDB:
+    db.setCode(codeAddress, boa.code)
+    db.setBalance(codeAddress, 1_000_000.u256)
+  let tx = createSignedTx(boa.data, chainDB.config.chainId)
+  let asmResult = testCallEvm(tx, tx.getSender, vmState, boa.fork)
+  verifyAsmResult(vmState, chainDB, boa, asmResult)
+
+# FIXME-duplicatedForAsync
+proc asyncRunVM*(vmState: BaseVMState, chainDB: BaseChainDB, boa: Assembler, asyncFactory: AsyncOperationFactory): Future[bool] {.async.} =
+  vmState.asyncFactory = asyncFactory
+  vmState.mutateStateDB:
+    db.setCode(codeAddress, boa.code)
+    db.setBalance(codeAddress, 1_000_000.u256)
+  let tx = createSignedTx(boa.data, chainDB.config.chainId)
+  let asmResult = await asyncTestCallEvm(tx, tx.getSender, vmState, boa.fork)
+  return verifyAsmResult(vmState, chainDB, boa, asmResult)
+
 macro assembler*(list: untyped): untyped =
-  var boa = Assembler(success: true, fork: FkFrontier, gasUsed: -1)
+  result = parseAssembler(list).generateAssemblerTest()
-  list.expectKind nnkStmtList
+
-  for callSection in list:
+macro concurrentAssemblers*(list: untyped): untyped =
-    callSection.expectKind(nnkCall)
+  result = parseConcurrencyTest(list).generateConcurrencyTest()
-    let label = callSection[0].strVal
-    let body  = callSection[1]
-    case label.normalize
-    of "title":
-      let title = body[0]
-      title.expectKind(nnkStrLit)
-      boa.title = title.strVal
-    of "code"  : boa.code = parseCode(body)
-    of "memory": boa.memory = parseVMWords(body)
-    of "stack" : boa.stack = parseVMWords(body)
-    of "storage": boa.storage = parseStorage(body)
-    of "logs": boa.logs = parseLogs(body)
-    of "success": boa.success = parseSuccess(body)
-    of "data": boa.data = parseData(body)
-    of "output": boa.output = parseData(body)
-    of "fork": boa.fork = parseFork(body)
-    of "gasused": boa.gasUsed = parseGasUsed(body)
-    else: error("unknown section '" & label & "'", callSection[0])
-  result = boa.generateVMProxy()
 
 macro evmByteCode*(list: untyped): untyped =
   list.expectKind nnkStmtList

tests/test_op_memory_lazy.nim

@@ -0,0 +1,74 @@
+import macro_assembler, unittest2, macros, strutils
+
+proc opMemoryLazyMain*() =
+  suite "Lazy Loading With Memory Opcodes":
+    let (vmState, chainDB) = initDatabase()
+
+    assembler: # SLOAD OP with (fake) lazy data fetching
+      title: "LAZY_SLOAD_1"
+      initialStorage:
+        "0xAA": "0x42"
+      code:
+        PUSH1 "0xAA"
+        SLOAD
+        PUSH1 "0x01"
+        ADD
+        PUSH1 "0xAA"
+        SSTORE
+        PUSH1 "0xAA"
+        SLOAD
+      storage:
+        "0xAA": "0x43"
+      stack:
+        "0x0000000000000000000000000000000000000000000000000000000000000043"
+
+    let (vmState1, chainDB1) = initDatabase()
+    let (vmState2, chainDB2) = initDatabase()
+    concurrentAssemblers:
+      title: "Concurrent Assemblers"
+      assemblers:
+        asm1:
+          title: "asm1"
+          vmState: vmState1
+          chainDB: chainDB1
+          initialStorage:
+            "0xBB": "0x42"
+            "0xCC": "0x20"
+          code:
+            PUSH1 "0xBB"
+            SLOAD
+            PUSH1 "0xCC"
+            SLOAD
+            ADD
+            PUSH1 "0xBB"
+            SSTORE
+            PUSH1 "0xBB"
+            SLOAD
+          storage:
+            "0xBB": "0x62"
+            "0xCC": "0x20"
+          stack: "0x0000000000000000000000000000000000000000000000000000000000000062"
+        asm2:
+          title: "asm2"
+          vmState: vmState2
+          chainDB: chainDB2
+          initialStorage:
+            "0xDD": "0x30"
+            "0xEE": "0x20"
+          code:
+            PUSH1 "0xDD"
+            SLOAD
+            PUSH1 "0xEE"
+            SLOAD
+            ADD
+            PUSH1 "0xEE"
+            SSTORE
+            PUSH1 "0xEE"
+            SLOAD
+          storage:
+            "0xDD": "0x30"
+            "0xEE": "0x50"
+          stack: "0x0000000000000000000000000000000000000000000000000000000000000050"
+
+when isMainModule:
+  opMemoryLazyMain()