implement transaction tracer

nimbus/db/capturedb.nim

@@ -0,0 +1,40 @@
+import eth_trie/db, ranges
+
+type
+  CaptureFlags* {.pure.} = enum
+    PersistPut
+    PersistDel
+
+  DB = TrieDatabaseRef
+  BytesRange = Range[byte]
+
+  CaptureDB* = ref object of RootObj
+    srcDb: DB
+    dstDb: DB
+    flags: set[CaptureFlags]
+
+proc get*(db: CaptureDB, key: openArray[byte]): seq[byte] =
+  result = db.dstDb.get(key)
+  if result.len != 0: return
+  result = db.srcDb.get(key)
+  db.dstDb.put(key, result)
+
+proc put*(db: CaptureDB, key, value: openArray[byte]) =
+  db.dstDb.put(key, value)
+  if CaptureFlags.PersistPut in db.flags:
+    db.srcDb.put(key, value)
+
+proc contains*(db: CaptureDB, key: openArray[byte]): bool =
+  result = db.srcDb.contains(key)
+  assert(db.dstDb.contains(key) == result)
+
+proc del*(db: CaptureDB, key: openArray[byte]) =
+  db.dstDb.del(key)
+  if CaptureFlags.PersistDel in db.flags:
+    db.srcDb.del(key)
+
+proc newCaptureDB*(srcDb, dstDb: DB, flags: set[CaptureFlags] = {}): CaptureDB =
+  result.new()
+  result.srcDb = srcDb
+  result.dstDb = dstDb
+  result.flags = flags

nimbus/p2p/chain.nim

@@ -1,8 +1,6 @@
 import ../db/[db_chain, state_db], eth_common, chronicles, ../vm_state, ../vm_types, ../transaction, ranges,
   ../vm/[computation, interpreter_dispatch, message], ../constants, stint, nimcrypto,
-  ../vm_state_transactions,
+  ../vm_state_transactions, sugar, ../utils, eth_trie/db
-  eth_trie/db, eth_trie, rlp,
-  sugar
 
 type
   Chain* = ref object of AbstractChainDB
@@ -32,7 +30,7 @@ method getSuccessorHeader*(c: Chain, h: BlockHeader, output: var BlockHeader): b
 method getBlockBody*(c: Chain, blockHash: KeccakHash): BlockBodyRef =
   result = nil
 
-proc processTransaction(db: var AccountStateDB, t: Transaction, sender: EthAddress, head: BlockHeader, chainDB: BaseChainDB): UInt256 =
+proc processTransaction*(db: var AccountStateDB, t: Transaction, sender: EthAddress, vmState: BaseVMState): UInt256 =
   ## Process the transaction, write the results to db.
   ## Returns amount of ETH to be rewarded to miner
   echo "Sender: ", sender
@@ -70,8 +68,7 @@ proc processTransaction(db: var AccountStateDB, t: Transaction, sender: EthAddre
   else:
     if t.isContractCreation:
       # TODO: re-derive sender in callee for cleaner interface, perhaps
-      var vmState = newBaseVMState(head, chainDB)
+      return applyCreateTransaction(db, t, vmState, sender)
-      return applyCreateTransaction(db, t, head, vmState, sender)
 
     else:
       let code = db.getCode(t.to)
@@ -99,12 +96,6 @@ proc processTransaction(db: var AccountStateDB, t: Transaction, sender: EthAddre
 
   return gasUsed.u256 * t.gasPrice.u256
 
-proc calcTxRoot(transactions: openarray[Transaction]): Hash256 =
-  var tr = initHexaryTrie(newMemoryDB())
-  for i, t in transactions:
-    tr.put(rlp.encode(i).toRange, rlp.encode(t).toRange)
-  return tr.rootHash
-
 method persistBlocks*(c: Chain, headers: openarray[BlockHeader], bodies: openarray[BlockBody]) =
   # Run the VM here
   assert(headers.len == bodies.len)
@@ -135,7 +126,7 @@ method persistBlocks*(c: Chain, headers: openarray[BlockHeader], bodies: openarr
         for t in bodies[i].transactions:
           var sender: EthAddress
           if t.getSender(sender):
-            gasReward += processTransaction(stateDb, t, sender, head, c.db)
+            gasReward += processTransaction(stateDb, t, sender, vmState)
           else:
             assert(false, "Could not get sender")
 

nimbus/rpc/debug.nim

@@ -11,7 +11,7 @@ import
   strutils, hexstrings, eth_p2p, options,
   ../db/[db_chain, state_db, storage_types],
   json_rpc/rpcserver, json, macros, rpc_utils,
-  eth_common
+  eth_common, ../tracer, ../vm_state, ../vm_types
 
 type
   TraceTxOptions = object
@@ -19,10 +19,13 @@ type
     disableMemory: Option[bool]
     disableStack: Option[bool]
 
-proc setupDebugRpc*(chain: BaseChainDB, rpcsrv: RpcServer) =
+proc isTrue(x: Option[bool]): bool =
+  result = x.isSome and x.get() == true
+
+proc setupDebugRpc*(chainDB: BaseChainDB, rpcsrv: RpcServer) =
 
   proc getBlockBody(hash: Hash256): BlockBody =
-    if not chain.getBlockBody(hash, result):
+    if not chainDB.getBlockBody(hash, result):
       raise newException(ValueError, "Error when retrieving block body")
 
   rpcsrv.rpc("debug_traceTransaction") do(data: EthHashStr, options: Option[TraceTxOptions]) -> JsonNode:
@@ -39,7 +42,18 @@ proc setupDebugRpc*(chain: BaseChainDB, rpcsrv: RpcServer) =
     ## * disableStack: BOOL. Setting this to true will disable stack capture (default = false).
     let
       txHash = toHash(data)
-      txDetails = chain.getTransactionKey(txHash)
+      txDetails = chainDB.getTransactionKey(txHash)
-      blockHeader = chain.getBlockHeader(txDetails.blockNumber)
+      blockHeader = chainDB.getBlockHeader(txDetails.blockNumber)
-      blockHash = chain.getBlockHash(txDetails.blockNumber)
+      blockHash = chainDB.getBlockHash(txDetails.blockNumber)
       blockBody = getBlockBody(blockHash)
+
+    var
+      flags: set[TracerFlags]
+
+    if options.isSome:
+      let opts = options.get
+      if opts.disableStorage.isTrue: flags.incl TracerFlags.DisableStorage
+      if opts.disableMemory.isTrue: flags.incl TracerFlags.DisableMemory
+      if opts.disableStack.isTrue: flags.incl TracerFlags.DisableStack
+
+    traceTransaction(chainDB, blockHeader, blockBody, txDetails.index, flags)

nimbus/tracer.nim

@@ -0,0 +1,24 @@
+import
+  db/[db_chain, state_db], eth_common, utils, json,
+  constants, vm_state, vm_types, transaction, p2p/chain
+
+proc traceTransaction*(db: BaseChainDB, header: BlockHeader,
+                       body: BlockBody, txIndex: int, tracerFlags: set[TracerFlags]): JsonNode =
+  let head = db.getCanonicalHead()
+  assert(head.blockNumber == header.blockNumber - 1)
+  var stateDb = newAccountStateDB(db.db, head.stateRoot, db.pruneTrie)
+  assert(body.transactions.calcTxRoot == header.txRoot)
+  if header.txRoot == BLANK_ROOT_HASH: return
+
+  let vmState = newBaseVMState(head, db, tracerFlags + {EnableTracing})
+  assert(body.transactions.len != 0)
+
+  for idx, tx in body.transactions:
+    var sender: EthAddress
+    if tx.getSender(sender):
+      discard processTransaction(stateDb, tx, sender, vmState)
+      if idx == txIndex: break
+    else:
+      assert(false, "Could not get sender")
+
+  vmState.getTracingResult()

nimbus/utils.nim

@@ -0,0 +1,13 @@
+import eth_trie/db, eth_trie, rlp, eth_common
+
+proc calcRootHash[T](items: openArray[T]): Hash256 =
+  var tr = initHexaryTrie(newMemoryDB())
+  for i, t in items:
+    tr.put(rlp.encode(i).toRange, rlp.encode(t).toRange)
+  return tr.rootHash
+
+template calcTxRoot*(transactions: openArray[Transaction]): Hash256 =
+  calcRootHash(transactions)
+
+template calcReceiptRoot*(receipts: openArray[Receipt]): Hash256 =
+  calcRootHash(receipts)

nimbus/vm/computation.nim

@@ -11,7 +11,8 @@ import
   ../constants, ../errors, ../validation, ../vm_state, ../vm_types,
   ./interpreter/[opcode_values, gas_meter, gas_costs, vm_forks],
   ./code_stream, ./memory, ./message, ./stack, ../db/[state_db, db_chain],
-  ../utils/header, byteutils, ranges, eth_keys, precompiles
+  ../utils/header, byteutils, ranges, eth_keys, precompiles,
+  transaction_tracer
 
 logScope:
   topics = "vm computation"
@@ -284,3 +285,12 @@ proc getGasRemaining*(c: BaseComputation): GasInt =
     result = 0
   else:
     result = c.gasMeter.gasRemaining
+
+template tracingEnabled*(c: BaseComputation): bool =
+  c.vmState.tracingEnabled
+
+template traceOpCodeStarted*(c: BaseComputation, op: string) =
+  traceOpCodeStarted(c.vmState.tracer, c, op)
+
+proc traceOpCodeEnded*(c: BaseComputation) =
+  c.vmState.tracer.traceOpCodeEnded(c)

nimbus/vm/interpreter_dispatch.nim

@@ -11,7 +11,7 @@ import
   ./interpreter/[opcode_values, opcodes_impl, vm_forks, gas_costs, gas_meter, utils/macros_gen_opcodes],
   ./code_stream,
   ../vm_types, ../errors, precompiles,
-  ./stack, ./computation, ./transaction_tracer, terminal # Those are only needed for logging
+  ./stack, ./computation, terminal # Those are only needed for logging
 
 func invalidInstruction*(computation: var BaseComputation) {.inline.} =
   raise newException(ValueError, "Invalid instruction, received an opcode not implemented in the current fork.")
@@ -192,19 +192,19 @@ proc opTableToCaseStmt(opTable: array[Op, NimNode], computation: NimNode): NimNo
         if BaseGasCosts[op].kind == GckFixed:
           quote do:
             if `computation`.tracingEnabled:
-              `computation`.tracer.traceOpCodeStarted(`computation`, $`asOp`)
+              `computation`.traceOpCodeStarted($`asOp`)
             `computation`.gasMeter.consumeGas(`computation`.gasCosts[`asOp`].cost, reason = $`asOp`)
             `opImpl`(`computation`)
             if `computation`.tracingEnabled:
-              `computation`.tracer.traceOpCodeEnded(`computation`)
+              `computation`.traceOpCodeEnded()
             `instr` = `computation`.code.next()
         else:
           quote do:
             if `computation`.tracingEnabled:
-              `computation`.tracer.traceOpCodeStarted(`computation`, $`asOp`)
+              `computation`.traceOpCodeStarted($`asOp`)
             `opImpl`(`computation`)
             if `computation`.tracingEnabled:
-              `computation`.tracer.traceOpCodeEnded(`computation`)
+              `computation`.traceOpCodeEnded()
             when `asOp` in {Return, Revert, SelfDestruct}:
               break
             else:

nimbus/vm/transaction_tracer.nim

@@ -3,38 +3,45 @@ import
   eth_common, stint, byteutils,
   ../vm_types, memory, stack
 
-proc initTrace(t: var TransactionTracer) =
+proc initTracer*(tracer: var TransactionTracer, flags: set[TracerFlags] = {}) =
-  t.trace = newJObject()
+  tracer.trace = newJObject()
-  t.trace["structLogs"] = newJArray()
+  tracer.trace["structLogs"] = newJArray()
+  tracer.flags = flags
 
-proc traceOpCodeStarted*(t: var TransactionTracer, c: BaseComputation, op: string) =
+proc traceOpCodeStarted*(tracer: var TransactionTracer, c: BaseComputation, op: string) =
-  if unlikely t.trace.isNil:
+  if unlikely tracer.trace.isNil:
-    t.initTrace()
+    tracer.initTracer()
 
   let j = newJObject()
-  t.trace["structLogs"].add(j)
+  tracer.trace["structLogs"].add(j)
 
   j["op"] = %op.toUpperAscii
   j["pc"] = %(c.code.pc - 1)
   j["depth"] = %1 # stub
   j["gas"] = %c.gasMeter.gasRemaining
-  t.gasRemaining = c.gasMeter.gasRemaining
+  tracer.gasRemaining = c.gasMeter.gasRemaining
 
   # log stack
+  if TracerFlags.DisableStack notin tracer.flags:
     let st = newJArray()
     for v in c.stack.values:
       st.add(%v.dumpHex())
     j["stack"] = st
+
   # log memory
+  if TracerFlags.DisableMemory notin tracer.flags:
     let mem = newJArray()
     const chunkLen = 32
     let numChunks = c.memory.len div chunkLen
     for i in 0 ..< numChunks:
       mem.add(%c.memory.bytes.toOpenArray(i * chunkLen, (i + 1) * chunkLen - 1).toHex())
     j["memory"] = mem
+
   # TODO: log storage
+  if TracerFlags.DisableStorage notin tracer.flags:
+    let storage = newJArray()
+    j["storage"] = storage
 
-proc traceOpCodeEnded*(t: var TransactionTracer, c: BaseComputation) =
+proc traceOpCodeEnded*(tracer: var TransactionTracer, c: BaseComputation) =
-  let j = t.trace["structLogs"].elems[^1]
+  let j = tracer.trace["structLogs"].elems[^1]
-  j["gasCost"] = %(t.gasRemaining - c.gasMeter.gasRemaining)
+  j["gasCost"] = %(tracer.gasRemaining - c.gasMeter.gasRemaining)
-

nimbus/vm_state.nim

@@ -9,20 +9,7 @@ import
   macros, strformat, tables,
   eth_common, eth_trie/db,
   ./constants, ./errors, ./transaction, ./db/[db_chain, state_db],
-  ./utils/header
+  ./utils/header, json, vm_types, vm/transaction_tracer
-
-type
-  BaseVMState* = ref object of RootObj
-    prevHeaders*: seq[BlockHeader]
-    # receipts*:
-    chaindb*: BaseChainDB
-    accessLogs*: AccessLogs
-    blockHeader*: BlockHeader
-    name*: string
-
-  AccessLogs* = ref object
-    reads*: Table[string, string]
-    writes*: Table[string, string]
 
 proc newAccessLogs*: AccessLogs =
   AccessLogs(reads: initTable[string, string](), writes: initTable[string, string]())
@@ -37,13 +24,15 @@ proc `$`*(vmState: BaseVMState): string =
   else:
     result = &"VMState {vmState.name}:\n  header: {vmState.blockHeader}\n  chaindb:  {vmState.chaindb}"
 
-proc newBaseVMState*(header: BlockHeader, chainDB: BaseChainDB): BaseVMState =
+proc newBaseVMState*(header: BlockHeader, chainDB: BaseChainDB, tracerFlags: set[TracerFlags] = {}): BaseVMState =
   new result
   result.prevHeaders = @[]
   result.name = "BaseVM"
   result.accessLogs = newAccessLogs()
   result.blockHeader = header
   result.chaindb = chainDB
+  result.tracer.initTracer(tracerFlags)
+  result.tracingEnabled = TracerFlags.EnableTracing in tracerFlags
 
 method blockhash*(vmState: BaseVMState): Hash256 =
   vmState.blockHeader.hash
@@ -124,3 +113,6 @@ export DbTransaction, commit, rollback, dispose, safeDispose
 proc beginTransaction*(vmState: BaseVMState): DbTransaction =
   vmState.chaindb.db.beginTransaction()
 
+proc getTracingResult*(vmState: BaseVMState): JsonNode =
+  assert(vmState.tracingEnabled)
+  vmState.tracer.trace

nimbus/vm_state_transactions.nim

@@ -33,7 +33,7 @@ proc validateTransaction*(vmState: BaseVMState, transaction: Transaction, sender
     transaction.accountNonce == readOnlyDB.getNonce(sender) and
     readOnlyDB.getBalance(sender) >= gas_cost
 
-proc setupComputation*(header: BlockHeader, vmState: var BaseVMState, transaction: Transaction, sender: EthAddress) : BaseComputation =
+proc setupComputation*(header: BlockHeader, vmState: BaseVMState, transaction: Transaction, sender: EthAddress) : BaseComputation =
   let message = newMessage(
       gas = transaction.gasLimit - transaction.payload.intrinsicGas,
       gasPrice = transaction.gasPrice,
@@ -59,7 +59,7 @@ proc execComputation*(computation: var BaseComputation): bool =
   except ValueError:
     result = false
 
-proc applyCreateTransaction*(db: var AccountStateDB, t: Transaction, head: BlockHeader, vmState: var BaseVMState, sender: EthAddress, useHomestead: bool = false): UInt256 =
+proc applyCreateTransaction*(db: var AccountStateDB, t: Transaction, vmState: BaseVMState, sender: EthAddress, useHomestead: bool = false): UInt256 =
   doAssert t.isContractCreation
   # TODO: clean up params
   echo "Contract creation"
@@ -71,7 +71,7 @@ proc applyCreateTransaction*(db: var AccountStateDB, t: Transaction, head: Block
   let msg = newMessage(t.gasLimit - gasUsed, t.gasPrice, t.to, sender, t.value, @[], t.payload,
                        options = newMessageOptions(origin = sender,
                                                    createAddress = contractAddress))
-  var c = newBaseComputation(vmState, head.blockNumber, msg)
+  var c = newBaseComputation(vmState, vmState.blockNumber, msg)
 
   if execComputation(c):
     db.addBalance(contractAddress, t.value)
@@ -111,7 +111,7 @@ proc applyCreateTransaction*(db: var AccountStateDB, t: Transaction, head: Block
     echo "isError: ", c.isError
     return t.gasLimit.u256 * t.gasPrice.u256
 
-method executeTransaction(vmState: var BaseVMState, transaction: Transaction): (BaseComputation, BlockHeader) {.base.}=
+method executeTransaction(vmState: BaseVMState, transaction: Transaction): (BaseComputation, BlockHeader) {.base.}=
   # Execute the transaction in the vm
   # TODO: introduced here: https://github.com/ethereum/py-evm/commit/21c57f2d56ab91bb62723c3f9ebe291d0b132dde
   # Refactored/Removed here: https://github.com/ethereum/py-evm/commit/cc991bf
@@ -119,7 +119,7 @@ method executeTransaction(vmState: var BaseVMState, transaction: Transaction): (
   raise newException(ValueError, "Must be implemented by subclasses")
 
 
-method addTransaction*(vmState: var BaseVMState, transaction: Transaction, computation: BaseComputation, b: Block): (Block, Table[string, string]) =
+method addTransaction*(vmState: BaseVMState, transaction: Transaction, computation: BaseComputation, b: Block): (Block, Table[string, string]) =
   # Add a transaction to the given block and
   # return `trieData` to store the transaction data in chaindb in VM layer
   # Update the bloomFilter, transaction trie and receipt trie roots, bloom_filter,
@@ -149,7 +149,7 @@ method addTransaction*(vmState: var BaseVMState, transaction: Transaction, compu
   result = (b, initTable[string, string]())
 
 method applyTransaction*(
-    vmState: var BaseVMState,
+    vmState: BaseVMState,
     transaction: Transaction,
     b: Block,
     isStateless: bool): (BaseComputation, Block, Table[string, string]) =

nimbus/vm_types.nim

@@ -6,14 +6,38 @@
 # at your option. This file may not be copied, modified, or distributed except according to those terms.
 
 import
-  tables, json,
+  tables, eth_common,
-  eth_common,
+  ./constants, json,
-  ./constants, ./vm_state,
   ./vm/[memory, stack, code_stream],
-  ./vm/interpreter/[gas_costs, opcode_values] # TODO - will be hidden at a lower layer
+  ./vm/interpreter/[gas_costs, opcode_values], # TODO - will be hidden at a lower layer
-
+  ./db/db_chain
 
 type
+  BaseVMState* = ref object of RootObj
+    prevHeaders*   : seq[BlockHeader]
+    # receipts*:
+    chaindb*       : BaseChainDB
+    accessLogs*    : AccessLogs
+    blockHeader*   : BlockHeader
+    name*          : string
+    tracingEnabled*: bool
+    tracer*        : TransactionTracer
+
+  AccessLogs* = ref object
+    reads*: Table[string, string]
+    writes*: Table[string, string]
+
+  TracerFlags* {.pure.} = enum
+    EnableTracing
+    DisableStorage
+    DisableMemory
+    DisableStack
+
+  TransactionTracer* = object
+    trace*: JsonNode
+    gasRemaining*: GasInt
+    flags*: set[TracerFlags]
+
   OpcodeExecutor* = proc(computation: var BaseComputation)
 
   BaseComputation* = ref object of RootObj
@@ -34,8 +58,6 @@ type
     opcodes*:               Table[Op, proc(computation: var BaseComputation){.nimcall.}]
     gasCosts*:              GasCosts # TODO - will be hidden at a lower layer
     opCodeExec*:            OpcodeExecutor
-    tracingEnabled*:        bool
-    tracer*:                TransactionTracer
 
   Error* = ref object
     info*:                  string
@@ -106,8 +128,3 @@ type
     createAddress*:           EthAddress
     codeAddress*:             EthAddress
     flags*:                   MsgFlags
-
-  TransactionTracer* = object
-    trace*: JsonNode
-    gasRemaining*: GasInt
-