From 5509c66cce83026f0ac030bae01a62bdbc959dda Mon Sep 17 00:00:00 2001 From: andri lim Date: Mon, 3 Dec 2018 17:54:19 +0700 Subject: [PATCH] implement transaction tracer --- nimbus/db/capturedb.nim | 40 +++++++++++++++++++++++ nimbus/p2p/chain.nim | 17 +++------- nimbus/rpc/debug.nim | 26 +++++++++++---- nimbus/tracer.nim | 24 ++++++++++++++ nimbus/utils.nim | 13 ++++++++ nimbus/vm/computation.nim | 18 ++++++++--- nimbus/vm/interpreter_dispatch.nim | 10 +++--- nimbus/vm/transaction_tracer.nim | 51 +++++++++++++++++------------- nimbus/vm_state.nim | 22 ++++--------- nimbus/vm_state_transactions.nim | 12 +++---- nimbus/vm_types.nim | 41 +++++++++++++++++------- 11 files changed, 191 insertions(+), 83 deletions(-) create mode 100644 nimbus/db/capturedb.nim create mode 100644 nimbus/tracer.nim create mode 100644 nimbus/utils.nim diff --git a/nimbus/db/capturedb.nim b/nimbus/db/capturedb.nim new file mode 100644 index 000000000..dfa9e23cf --- /dev/null +++ b/nimbus/db/capturedb.nim @@ -0,0 +1,40 @@ +import eth_trie/db, ranges + +type + CaptureFlags* {.pure.} = enum + PersistPut + PersistDel + + DB = TrieDatabaseRef + BytesRange = Range[byte] + + CaptureDB* = ref object of RootObj + srcDb: DB + dstDb: DB + flags: set[CaptureFlags] + +proc get*(db: CaptureDB, key: openArray[byte]): seq[byte] = + result = db.dstDb.get(key) + if result.len != 0: return + result = db.srcDb.get(key) + db.dstDb.put(key, result) + +proc put*(db: CaptureDB, key, value: openArray[byte]) = + db.dstDb.put(key, value) + if CaptureFlags.PersistPut in db.flags: + db.srcDb.put(key, value) + +proc contains*(db: CaptureDB, key: openArray[byte]): bool = + result = db.srcDb.contains(key) + assert(db.dstDb.contains(key) == result) + +proc del*(db: CaptureDB, key: openArray[byte]) = + db.dstDb.del(key) + if CaptureFlags.PersistDel in db.flags: + db.srcDb.del(key) + +proc newCaptureDB*(srcDb, dstDb: DB, flags: set[CaptureFlags] = {}): CaptureDB = + result.new() + result.srcDb = srcDb + result.dstDb = dstDb + result.flags = flags diff --git a/nimbus/p2p/chain.nim b/nimbus/p2p/chain.nim index 544a02c03..706cc550b 100644 --- a/nimbus/p2p/chain.nim +++ b/nimbus/p2p/chain.nim @@ -1,8 +1,6 @@ import ../db/[db_chain, state_db], eth_common, chronicles, ../vm_state, ../vm_types, ../transaction, ranges, ../vm/[computation, interpreter_dispatch, message], ../constants, stint, nimcrypto, - ../vm_state_transactions, - eth_trie/db, eth_trie, rlp, - sugar + ../vm_state_transactions, sugar, ../utils, eth_trie/db type Chain* = ref object of AbstractChainDB @@ -32,7 +30,7 @@ method getSuccessorHeader*(c: Chain, h: BlockHeader, output: var BlockHeader): b method getBlockBody*(c: Chain, blockHash: KeccakHash): BlockBodyRef = result = nil -proc processTransaction(db: var AccountStateDB, t: Transaction, sender: EthAddress, head: BlockHeader, chainDB: BaseChainDB): UInt256 = +proc processTransaction*(db: var AccountStateDB, t: Transaction, sender: EthAddress, vmState: BaseVMState): UInt256 = ## Process the transaction, write the results to db. ## Returns amount of ETH to be rewarded to miner echo "Sender: ", sender @@ -70,8 +68,7 @@ proc processTransaction(db: var AccountStateDB, t: Transaction, sender: EthAddre else: if t.isContractCreation: # TODO: re-derive sender in callee for cleaner interface, perhaps - var vmState = newBaseVMState(head, chainDB) - return applyCreateTransaction(db, t, head, vmState, sender) + return applyCreateTransaction(db, t, vmState, sender) else: let code = db.getCode(t.to) @@ -99,12 +96,6 @@ proc processTransaction(db: var AccountStateDB, t: Transaction, sender: EthAddre return gasUsed.u256 * t.gasPrice.u256 -proc calcTxRoot(transactions: openarray[Transaction]): Hash256 = - var tr = initHexaryTrie(newMemoryDB()) - for i, t in transactions: - tr.put(rlp.encode(i).toRange, rlp.encode(t).toRange) - return tr.rootHash - method persistBlocks*(c: Chain, headers: openarray[BlockHeader], bodies: openarray[BlockBody]) = # Run the VM here assert(headers.len == bodies.len) @@ -135,7 +126,7 @@ method persistBlocks*(c: Chain, headers: openarray[BlockHeader], bodies: openarr for t in bodies[i].transactions: var sender: EthAddress if t.getSender(sender): - gasReward += processTransaction(stateDb, t, sender, head, c.db) + gasReward += processTransaction(stateDb, t, sender, vmState) else: assert(false, "Could not get sender") diff --git a/nimbus/rpc/debug.nim b/nimbus/rpc/debug.nim index 1f4b1b259..bded5f8ad 100644 --- a/nimbus/rpc/debug.nim +++ b/nimbus/rpc/debug.nim @@ -11,7 +11,7 @@ import strutils, hexstrings, eth_p2p, options, ../db/[db_chain, state_db, storage_types], json_rpc/rpcserver, json, macros, rpc_utils, - eth_common + eth_common, ../tracer, ../vm_state, ../vm_types type TraceTxOptions = object @@ -19,10 +19,13 @@ type disableMemory: Option[bool] disableStack: Option[bool] -proc setupDebugRpc*(chain: BaseChainDB, rpcsrv: RpcServer) = +proc isTrue(x: Option[bool]): bool = + result = x.isSome and x.get() == true + +proc setupDebugRpc*(chainDB: BaseChainDB, rpcsrv: RpcServer) = proc getBlockBody(hash: Hash256): BlockBody = - if not chain.getBlockBody(hash, result): + if not chainDB.getBlockBody(hash, result): raise newException(ValueError, "Error when retrieving block body") rpcsrv.rpc("debug_traceTransaction") do(data: EthHashStr, options: Option[TraceTxOptions]) -> JsonNode: @@ -39,7 +42,18 @@ proc setupDebugRpc*(chain: BaseChainDB, rpcsrv: RpcServer) = ## * disableStack: BOOL. Setting this to true will disable stack capture (default = false). let txHash = toHash(data) - txDetails = chain.getTransactionKey(txHash) - blockHeader = chain.getBlockHeader(txDetails.blockNumber) - blockHash = chain.getBlockHash(txDetails.blockNumber) + txDetails = chainDB.getTransactionKey(txHash) + blockHeader = chainDB.getBlockHeader(txDetails.blockNumber) + blockHash = chainDB.getBlockHash(txDetails.blockNumber) blockBody = getBlockBody(blockHash) + + var + flags: set[TracerFlags] + + if options.isSome: + let opts = options.get + if opts.disableStorage.isTrue: flags.incl TracerFlags.DisableStorage + if opts.disableMemory.isTrue: flags.incl TracerFlags.DisableMemory + if opts.disableStack.isTrue: flags.incl TracerFlags.DisableStack + + traceTransaction(chainDB, blockHeader, blockBody, txDetails.index, flags) diff --git a/nimbus/tracer.nim b/nimbus/tracer.nim new file mode 100644 index 000000000..c3f4862f4 --- /dev/null +++ b/nimbus/tracer.nim @@ -0,0 +1,24 @@ +import + db/[db_chain, state_db], eth_common, utils, json, + constants, vm_state, vm_types, transaction, p2p/chain + +proc traceTransaction*(db: BaseChainDB, header: BlockHeader, + body: BlockBody, txIndex: int, tracerFlags: set[TracerFlags]): JsonNode = + let head = db.getCanonicalHead() + assert(head.blockNumber == header.blockNumber - 1) + var stateDb = newAccountStateDB(db.db, head.stateRoot, db.pruneTrie) + assert(body.transactions.calcTxRoot == header.txRoot) + if header.txRoot == BLANK_ROOT_HASH: return + + let vmState = newBaseVMState(head, db, tracerFlags + {EnableTracing}) + assert(body.transactions.len != 0) + + for idx, tx in body.transactions: + var sender: EthAddress + if tx.getSender(sender): + discard processTransaction(stateDb, tx, sender, vmState) + if idx == txIndex: break + else: + assert(false, "Could not get sender") + + vmState.getTracingResult() diff --git a/nimbus/utils.nim b/nimbus/utils.nim new file mode 100644 index 000000000..5fa9d191e --- /dev/null +++ b/nimbus/utils.nim @@ -0,0 +1,13 @@ +import eth_trie/db, eth_trie, rlp, eth_common + +proc calcRootHash[T](items: openArray[T]): Hash256 = + var tr = initHexaryTrie(newMemoryDB()) + for i, t in items: + tr.put(rlp.encode(i).toRange, rlp.encode(t).toRange) + return tr.rootHash + +template calcTxRoot*(transactions: openArray[Transaction]): Hash256 = + calcRootHash(transactions) + +template calcReceiptRoot*(receipts: openArray[Receipt]): Hash256 = + calcRootHash(receipts) diff --git a/nimbus/vm/computation.nim b/nimbus/vm/computation.nim index 21cc083a4..bee824e30 100644 --- a/nimbus/vm/computation.nim +++ b/nimbus/vm/computation.nim @@ -11,7 +11,8 @@ import ../constants, ../errors, ../validation, ../vm_state, ../vm_types, ./interpreter/[opcode_values, gas_meter, gas_costs, vm_forks], ./code_stream, ./memory, ./message, ./stack, ../db/[state_db, db_chain], - ../utils/header, byteutils, ranges, eth_keys, precompiles + ../utils/header, byteutils, ranges, eth_keys, precompiles, + transaction_tracer logScope: topics = "vm computation" @@ -122,7 +123,7 @@ proc applyMessage(computation: var BaseComputation, opCode: static[Op]) = computation.gasMeter.returnGas(computation.msg.gas) push: 0 return - + newBalance = senderBalance - computation.msg.value computation.vmState.mutateStateDb: db.setBalance(computation.msg.sender, newBalance) @@ -208,10 +209,10 @@ proc generateChildComputation*(fork: Fork, computation: BaseComputation, childMs computation.vmState, computation.vmState.blockHeader.blockNumber, childMsg) - + # Copy the fork op code executor proc (assumes child computation is in the same fork) childComp.opCodeExec = computation.opCodeExec - + if childMsg.isCreate: fork.applyCreateMessage(childComp, opCode) else: @@ -284,3 +285,12 @@ proc getGasRemaining*(c: BaseComputation): GasInt = result = 0 else: result = c.gasMeter.gasRemaining + +template tracingEnabled*(c: BaseComputation): bool = + c.vmState.tracingEnabled + +template traceOpCodeStarted*(c: BaseComputation, op: string) = + traceOpCodeStarted(c.vmState.tracer, c, op) + +proc traceOpCodeEnded*(c: BaseComputation) = + c.vmState.tracer.traceOpCodeEnded(c) diff --git a/nimbus/vm/interpreter_dispatch.nim b/nimbus/vm/interpreter_dispatch.nim index d62a0dcf8..5ebd7afa0 100644 --- a/nimbus/vm/interpreter_dispatch.nim +++ b/nimbus/vm/interpreter_dispatch.nim @@ -11,7 +11,7 @@ import ./interpreter/[opcode_values, opcodes_impl, vm_forks, gas_costs, gas_meter, utils/macros_gen_opcodes], ./code_stream, ../vm_types, ../errors, precompiles, - ./stack, ./computation, ./transaction_tracer, terminal # Those are only needed for logging + ./stack, ./computation, terminal # Those are only needed for logging func invalidInstruction*(computation: var BaseComputation) {.inline.} = raise newException(ValueError, "Invalid instruction, received an opcode not implemented in the current fork.") @@ -192,19 +192,19 @@ proc opTableToCaseStmt(opTable: array[Op, NimNode], computation: NimNode): NimNo if BaseGasCosts[op].kind == GckFixed: quote do: if `computation`.tracingEnabled: - `computation`.tracer.traceOpCodeStarted(`computation`, $`asOp`) + `computation`.traceOpCodeStarted($`asOp`) `computation`.gasMeter.consumeGas(`computation`.gasCosts[`asOp`].cost, reason = $`asOp`) `opImpl`(`computation`) if `computation`.tracingEnabled: - `computation`.tracer.traceOpCodeEnded(`computation`) + `computation`.traceOpCodeEnded() `instr` = `computation`.code.next() else: quote do: if `computation`.tracingEnabled: - `computation`.tracer.traceOpCodeStarted(`computation`, $`asOp`) + `computation`.traceOpCodeStarted($`asOp`) `opImpl`(`computation`) if `computation`.tracingEnabled: - `computation`.tracer.traceOpCodeEnded(`computation`) + `computation`.traceOpCodeEnded() when `asOp` in {Return, Revert, SelfDestruct}: break else: diff --git a/nimbus/vm/transaction_tracer.nim b/nimbus/vm/transaction_tracer.nim index 05e1ad35a..1354d6427 100644 --- a/nimbus/vm/transaction_tracer.nim +++ b/nimbus/vm/transaction_tracer.nim @@ -3,38 +3,45 @@ import eth_common, stint, byteutils, ../vm_types, memory, stack -proc initTrace(t: var TransactionTracer) = - t.trace = newJObject() - t.trace["structLogs"] = newJArray() +proc initTracer*(tracer: var TransactionTracer, flags: set[TracerFlags] = {}) = + tracer.trace = newJObject() + tracer.trace["structLogs"] = newJArray() + tracer.flags = flags -proc traceOpCodeStarted*(t: var TransactionTracer, c: BaseComputation, op: string) = - if unlikely t.trace.isNil: - t.initTrace() +proc traceOpCodeStarted*(tracer: var TransactionTracer, c: BaseComputation, op: string) = + if unlikely tracer.trace.isNil: + tracer.initTracer() let j = newJObject() - t.trace["structLogs"].add(j) + tracer.trace["structLogs"].add(j) j["op"] = %op.toUpperAscii j["pc"] = %(c.code.pc - 1) j["depth"] = %1 # stub j["gas"] = %c.gasMeter.gasRemaining - t.gasRemaining = c.gasMeter.gasRemaining + tracer.gasRemaining = c.gasMeter.gasRemaining # log stack - let st = newJArray() - for v in c.stack.values: - st.add(%v.dumpHex()) - j["stack"] = st + if TracerFlags.DisableStack notin tracer.flags: + let st = newJArray() + for v in c.stack.values: + st.add(%v.dumpHex()) + j["stack"] = st + # log memory - let mem = newJArray() - const chunkLen = 32 - let numChunks = c.memory.len div chunkLen - for i in 0 ..< numChunks: - mem.add(%c.memory.bytes.toOpenArray(i * chunkLen, (i + 1) * chunkLen - 1).toHex()) - j["memory"] = mem + if TracerFlags.DisableMemory notin tracer.flags: + let mem = newJArray() + const chunkLen = 32 + let numChunks = c.memory.len div chunkLen + for i in 0 ..< numChunks: + mem.add(%c.memory.bytes.toOpenArray(i * chunkLen, (i + 1) * chunkLen - 1).toHex()) + j["memory"] = mem + # TODO: log storage + if TracerFlags.DisableStorage notin tracer.flags: + let storage = newJArray() + j["storage"] = storage -proc traceOpCodeEnded*(t: var TransactionTracer, c: BaseComputation) = - let j = t.trace["structLogs"].elems[^1] - j["gasCost"] = %(t.gasRemaining - c.gasMeter.gasRemaining) - +proc traceOpCodeEnded*(tracer: var TransactionTracer, c: BaseComputation) = + let j = tracer.trace["structLogs"].elems[^1] + j["gasCost"] = %(tracer.gasRemaining - c.gasMeter.gasRemaining) diff --git a/nimbus/vm_state.nim b/nimbus/vm_state.nim index 5ebf332a9..2a808a9b9 100644 --- a/nimbus/vm_state.nim +++ b/nimbus/vm_state.nim @@ -9,20 +9,7 @@ import macros, strformat, tables, eth_common, eth_trie/db, ./constants, ./errors, ./transaction, ./db/[db_chain, state_db], - ./utils/header - -type - BaseVMState* = ref object of RootObj - prevHeaders*: seq[BlockHeader] - # receipts*: - chaindb*: BaseChainDB - accessLogs*: AccessLogs - blockHeader*: BlockHeader - name*: string - - AccessLogs* = ref object - reads*: Table[string, string] - writes*: Table[string, string] + ./utils/header, json, vm_types, vm/transaction_tracer proc newAccessLogs*: AccessLogs = AccessLogs(reads: initTable[string, string](), writes: initTable[string, string]()) @@ -37,13 +24,15 @@ proc `$`*(vmState: BaseVMState): string = else: result = &"VMState {vmState.name}:\n header: {vmState.blockHeader}\n chaindb: {vmState.chaindb}" -proc newBaseVMState*(header: BlockHeader, chainDB: BaseChainDB): BaseVMState = +proc newBaseVMState*(header: BlockHeader, chainDB: BaseChainDB, tracerFlags: set[TracerFlags] = {}): BaseVMState = new result result.prevHeaders = @[] result.name = "BaseVM" result.accessLogs = newAccessLogs() result.blockHeader = header result.chaindb = chainDB + result.tracer.initTracer(tracerFlags) + result.tracingEnabled = TracerFlags.EnableTracing in tracerFlags method blockhash*(vmState: BaseVMState): Hash256 = vmState.blockHeader.hash @@ -124,3 +113,6 @@ export DbTransaction, commit, rollback, dispose, safeDispose proc beginTransaction*(vmState: BaseVMState): DbTransaction = vmState.chaindb.db.beginTransaction() +proc getTracingResult*(vmState: BaseVMState): JsonNode = + assert(vmState.tracingEnabled) + vmState.tracer.trace diff --git a/nimbus/vm_state_transactions.nim b/nimbus/vm_state_transactions.nim index ddc94c44e..27a5268d7 100644 --- a/nimbus/vm_state_transactions.nim +++ b/nimbus/vm_state_transactions.nim @@ -33,7 +33,7 @@ proc validateTransaction*(vmState: BaseVMState, transaction: Transaction, sender transaction.accountNonce == readOnlyDB.getNonce(sender) and readOnlyDB.getBalance(sender) >= gas_cost -proc setupComputation*(header: BlockHeader, vmState: var BaseVMState, transaction: Transaction, sender: EthAddress) : BaseComputation = +proc setupComputation*(header: BlockHeader, vmState: BaseVMState, transaction: Transaction, sender: EthAddress) : BaseComputation = let message = newMessage( gas = transaction.gasLimit - transaction.payload.intrinsicGas, gasPrice = transaction.gasPrice, @@ -59,7 +59,7 @@ proc execComputation*(computation: var BaseComputation): bool = except ValueError: result = false -proc applyCreateTransaction*(db: var AccountStateDB, t: Transaction, head: BlockHeader, vmState: var BaseVMState, sender: EthAddress, useHomestead: bool = false): UInt256 = +proc applyCreateTransaction*(db: var AccountStateDB, t: Transaction, vmState: BaseVMState, sender: EthAddress, useHomestead: bool = false): UInt256 = doAssert t.isContractCreation # TODO: clean up params echo "Contract creation" @@ -71,7 +71,7 @@ proc applyCreateTransaction*(db: var AccountStateDB, t: Transaction, head: Block let msg = newMessage(t.gasLimit - gasUsed, t.gasPrice, t.to, sender, t.value, @[], t.payload, options = newMessageOptions(origin = sender, createAddress = contractAddress)) - var c = newBaseComputation(vmState, head.blockNumber, msg) + var c = newBaseComputation(vmState, vmState.blockNumber, msg) if execComputation(c): db.addBalance(contractAddress, t.value) @@ -111,7 +111,7 @@ proc applyCreateTransaction*(db: var AccountStateDB, t: Transaction, head: Block echo "isError: ", c.isError return t.gasLimit.u256 * t.gasPrice.u256 -method executeTransaction(vmState: var BaseVMState, transaction: Transaction): (BaseComputation, BlockHeader) {.base.}= +method executeTransaction(vmState: BaseVMState, transaction: Transaction): (BaseComputation, BlockHeader) {.base.}= # Execute the transaction in the vm # TODO: introduced here: https://github.com/ethereum/py-evm/commit/21c57f2d56ab91bb62723c3f9ebe291d0b132dde # Refactored/Removed here: https://github.com/ethereum/py-evm/commit/cc991bf @@ -119,7 +119,7 @@ method executeTransaction(vmState: var BaseVMState, transaction: Transaction): ( raise newException(ValueError, "Must be implemented by subclasses") -method addTransaction*(vmState: var BaseVMState, transaction: Transaction, computation: BaseComputation, b: Block): (Block, Table[string, string]) = +method addTransaction*(vmState: BaseVMState, transaction: Transaction, computation: BaseComputation, b: Block): (Block, Table[string, string]) = # Add a transaction to the given block and # return `trieData` to store the transaction data in chaindb in VM layer # Update the bloomFilter, transaction trie and receipt trie roots, bloom_filter, @@ -149,7 +149,7 @@ method addTransaction*(vmState: var BaseVMState, transaction: Transaction, compu result = (b, initTable[string, string]()) method applyTransaction*( - vmState: var BaseVMState, + vmState: BaseVMState, transaction: Transaction, b: Block, isStateless: bool): (BaseComputation, Block, Table[string, string]) = diff --git a/nimbus/vm_types.nim b/nimbus/vm_types.nim index 923685173..11c8dd68f 100644 --- a/nimbus/vm_types.nim +++ b/nimbus/vm_types.nim @@ -6,14 +6,38 @@ # at your option. This file may not be copied, modified, or distributed except according to those terms. import - tables, json, - eth_common, - ./constants, ./vm_state, + tables, eth_common, + ./constants, json, ./vm/[memory, stack, code_stream], - ./vm/interpreter/[gas_costs, opcode_values] # TODO - will be hidden at a lower layer - + ./vm/interpreter/[gas_costs, opcode_values], # TODO - will be hidden at a lower layer + ./db/db_chain type + BaseVMState* = ref object of RootObj + prevHeaders* : seq[BlockHeader] + # receipts*: + chaindb* : BaseChainDB + accessLogs* : AccessLogs + blockHeader* : BlockHeader + name* : string + tracingEnabled*: bool + tracer* : TransactionTracer + + AccessLogs* = ref object + reads*: Table[string, string] + writes*: Table[string, string] + + TracerFlags* {.pure.} = enum + EnableTracing + DisableStorage + DisableMemory + DisableStack + + TransactionTracer* = object + trace*: JsonNode + gasRemaining*: GasInt + flags*: set[TracerFlags] + OpcodeExecutor* = proc(computation: var BaseComputation) BaseComputation* = ref object of RootObj @@ -34,8 +58,6 @@ type opcodes*: Table[Op, proc(computation: var BaseComputation){.nimcall.}] gasCosts*: GasCosts # TODO - will be hidden at a lower layer opCodeExec*: OpcodeExecutor - tracingEnabled*: bool - tracer*: TransactionTracer Error* = ref object info*: string @@ -106,8 +128,3 @@ type createAddress*: EthAddress codeAddress*: EthAddress flags*: MsgFlags - - TransactionTracer* = object - trace*: JsonNode - gasRemaining*: GasInt -