nimbus-eth1/nimbus/tracer.nim

282 lines
9.2 KiB
Nim
Raw Normal View History

2018-12-03 10:54:19 +00:00
import
2022-12-02 04:39:12 +00:00
std/[strutils, json],
./common/common,
./db/[accounts_cache, capturedb],
./utils/utils,
"."/[constants, vm_state, vm_types, transaction, core/executor],
nimcrypto/utils as ncrutils,
./rpc/hexstrings, ./launcher,
stew/results
2018-12-03 16:22:08 +00:00
2020-06-10 05:54:15 +00:00
when defined(geth):
import db/geth_db
2022-12-02 04:39:12 +00:00
proc getParentHeader(db: ChainDBRef, header: BlockHeader): BlockHeader =
2020-06-10 05:54:15 +00:00
db.blockHeader(header.blockNumber.truncate(uint64) - 1)
else:
2022-12-02 04:39:12 +00:00
proc getParentHeader(self: ChainDBRef, header: BlockHeader): BlockHeader =
2020-06-10 05:54:15 +00:00
self.getBlockHeader(header.parentHash)
2018-12-03 16:22:08 +00:00
2018-12-25 10:31:51 +00:00
proc `%`(x: openArray[byte]): JsonNode =
result = %toHex(x, false)
proc toJson(receipt: Receipt): JsonNode =
result = newJObject()
result["cumulativeGasUsed"] = %receipt.cumulativeGasUsed
result["bloom"] = %receipt.bloom
result["logs"] = %receipt.logs
if receipt.hasStateRoot:
result["root"] = %($receipt.stateRoot)
else:
result["status"] = %receipt.status
2022-12-02 04:39:12 +00:00
proc dumpReceipts*(chainDB: ChainDBRef, header: BlockHeader): JsonNode =
2018-12-25 10:31:51 +00:00
result = newJArray()
for receipt in chainDB.getReceipts(header.receiptRoot):
2018-12-25 10:31:51 +00:00
result.add receipt.toJson
2018-12-31 03:27:02 +00:00
proc toJson*(receipts: seq[Receipt]): JsonNode =
2018-12-25 10:31:51 +00:00
result = newJArray()
for receipt in receipts:
result.add receipt.toJson
2018-12-03 10:54:19 +00:00
proc captureAccount(n: JsonNode, db: AccountsCache, address: EthAddress, name: string) =
2018-12-12 03:40:37 +00:00
var jaccount = newJObject()
jaccount["name"] = %name
2019-01-11 06:53:18 +00:00
jaccount["address"] = %("0x" & $address)
let nonce = db.getNonce(address)
let balance = db.getBalance(address)
let codeHash = db.getCodeHash(address)
let storageRoot = db.getStorageRoot(address)
jaccount["nonce"] = %(encodeQuantity(nonce).string.toLowerAscii)
jaccount["balance"] = %("0x" & balance.toHex)
2018-12-11 09:53:05 +00:00
let code = db.getCode(address)
jaccount["codeHash"] = %("0x" & ($codeHash).toLowerAscii)
jaccount["code"] = %("0x" & toHex(code, true))
jaccount["storageRoot"] = %("0x" & ($storageRoot).toLowerAscii)
2018-12-11 09:53:05 +00:00
var storage = newJObject()
for key, value in db.storage(address):
2019-01-11 06:53:18 +00:00
storage["0x" & key.dumpHex] = %("0x" & value.dumpHex)
2018-12-12 03:40:37 +00:00
jaccount["storage"] = storage
2018-12-11 09:53:05 +00:00
2018-12-12 03:40:37 +00:00
n.add jaccount
2018-12-11 09:53:05 +00:00
2018-12-12 15:18:46 +00:00
proc dumpMemoryDB*(node: JsonNode, memoryDB: TrieDatabaseRef) =
var n = newJObject()
for k, v in pairsInMemoryDB(memoryDB):
2018-12-25 10:31:51 +00:00
n[k.toHex(false)] = %v
2018-12-12 15:18:46 +00:00
node["state"] = n
2018-12-12 03:40:37 +00:00
const
senderName = "sender"
recipientName = "recipient"
minerName = "miner"
uncleName = "uncle"
2018-12-25 07:31:12 +00:00
internalTxName = "internalTx"
2018-12-11 09:53:05 +00:00
2022-12-02 04:39:12 +00:00
proc traceTransaction*(com: CommonRef, header: BlockHeader,
2018-12-04 11:42:55 +00:00
body: BlockBody, txIndex: int, tracerFlags: set[TracerFlags] = {}): JsonNode =
2018-12-03 16:22:08 +00:00
let
2022-12-02 04:39:12 +00:00
parent = com.db.getParentHeader(header)
# we add a memory layer between backend/lower layer db
2018-12-04 01:53:21 +00:00
# and capture state db snapshot during transaction execution
2018-12-03 16:22:08 +00:00
memoryDB = newMemoryDB()
2022-12-02 04:39:12 +00:00
captureDB = newCaptureDB(com.db.db, memoryDB)
2018-12-03 16:22:08 +00:00
captureTrieDB = trieDB captureDB
Redesign of BaseVMState descriptor (#923) * Redesign of BaseVMState descriptor why: BaseVMState provides an environment for executing transactions. The current descriptor also provides data that cannot generally be known within the execution environment, e.g. the total gasUsed which is available not before after all transactions have finished. Also, the BaseVMState constructor has been replaced by a constructor that does not need pre-initialised input of the account database. also: Previous constructor and some fields are provided with a deprecated annotation (producing a lot of noise.) * Replace legacy directives in production sources * Replace legacy directives in unit test sources * fix CI (missing premix update) * Remove legacy directives * chase CI problem * rebased * Re-introduce 'AccountsCache' constructor optimisation for 'BaseVmState' re-initialisation why: Constructing a new 'AccountsCache' descriptor can be avoided sometimes when the current state root is properly positioned already. Such a feature existed already as the update function 'initStateDB()' for the 'BaseChanDB' where the accounts cache was linked into this desctiptor. The function 'initStateDB()' was removed and re-implemented into the 'BaseVmState' constructor without optimisation. The old version was of restricted use as a wrong accounts cache state would unconditionally throw an exception rather than conceptually ask for a remedy. The optimised 'BaseVmState' re-initialisation has been implemented for the 'persistBlocks()' function. also: moved some test helpers to 'test/replay' folder * Remove unused & undocumented fields from Chain descriptor why: Reduces attack surface in general & improves reading the code.
2022-01-18 16:19:32 +00:00
captureFlags = tracerFlags + {EnableAccount}
2022-12-02 04:39:12 +00:00
captureCom = com.clone(captureTrieDB)
vmState = BaseVMState.new(header, captureCom, captureFlags)
var stateDb = vmState.stateDB
2018-12-26 03:34:16 +00:00
if header.txRoot == EMPTY_ROOT_HASH: return newJNull()
2019-03-13 21:36:54 +00:00
doAssert(body.transactions.calcTxRoot == header.txRoot)
doAssert(body.transactions.len != 0)
2018-12-03 10:54:19 +00:00
2018-12-11 09:53:05 +00:00
var
gasUsed: GasInt
2018-12-12 09:41:18 +00:00
before = newJArray()
after = newJArray()
2018-12-11 09:53:05 +00:00
stateDiff = %{"before": before, "after": after}
2018-12-26 03:34:16 +00:00
beforeRoot: Hash256
2018-12-11 09:53:05 +00:00
let
miner = vmState.coinbase()
2019-04-23 12:50:45 +00:00
2018-12-03 10:54:19 +00:00
for idx, tx in body.transactions:
2018-12-12 03:40:37 +00:00
let sender = tx.getSender
let recipient = tx.getRecipient(sender)
2018-12-11 09:53:05 +00:00
2018-12-12 03:40:37 +00:00
if idx == txIndex:
vmState.enableTracing()
before.captureAccount(stateDb, sender, senderName)
before.captureAccount(stateDb, recipient, recipientName)
before.captureAccount(stateDb, miner, minerName)
stateDb.persist()
2018-12-25 10:31:51 +00:00
stateDiff["beforeRoot"] = %($stateDb.rootHash)
2018-12-26 03:34:16 +00:00
beforeRoot = stateDb.rootHash
2018-12-11 09:53:05 +00:00
let rc = vmState.processTransaction(tx, sender, header)
2022-04-08 04:54:11 +00:00
gasUsed = if rc.isOk: rc.value else: 0
2018-12-12 03:40:37 +00:00
if idx == txIndex:
after.captureAccount(stateDb, sender, senderName)
after.captureAccount(stateDb, recipient, recipientName)
after.captureAccount(stateDb, miner, minerName)
vmState.removeTracedAccounts(sender, recipient, miner)
stateDb.persist()
2018-12-25 10:31:51 +00:00
stateDiff["afterRoot"] = %($stateDb.rootHash)
2018-12-12 03:40:37 +00:00
break
2018-12-03 10:54:19 +00:00
2018-12-25 07:31:12 +00:00
# internal transactions:
2022-12-02 04:39:12 +00:00
var stateBefore = AccountsCache.init(captureTrieDB, beforeRoot, com.pruneTrie)
2018-12-25 07:31:12 +00:00
for idx, acc in tracedAccountsPairs(vmState):
before.captureAccount(stateBefore, acc, internalTxName & $idx)
for idx, acc in tracedAccountsPairs(vmState):
after.captureAccount(stateDb, acc, internalTxName & $idx)
2018-12-03 16:22:08 +00:00
result = vmState.getTracingResult()
result["gas"] = %gasUsed
2018-12-25 10:31:51 +00:00
if TracerFlags.DisableStateDiff notin tracerFlags:
result["stateDiff"] = stateDiff
2018-12-03 16:22:08 +00:00
2018-12-04 01:53:21 +00:00
# now we dump captured state db
if TracerFlags.DisableState notin tracerFlags:
2018-12-12 15:18:46 +00:00
result.dumpMemoryDB(memoryDB)
2018-12-11 10:05:49 +00:00
2022-12-02 04:39:12 +00:00
proc dumpBlockState*(com: CommonRef, header: BlockHeader, body: BlockBody, dumpState = false): JsonNode =
2018-12-12 03:40:37 +00:00
let
2022-12-02 04:39:12 +00:00
parent = com.db.getParentHeader(header)
2018-12-12 03:40:37 +00:00
memoryDB = newMemoryDB()
2022-12-02 04:39:12 +00:00
captureDB = newCaptureDB(com.db.db, memoryDB)
2018-12-12 03:40:37 +00:00
captureTrieDB = trieDB captureDB
2022-12-02 04:39:12 +00:00
captureCom = com.clone(captureTrieDB)
2018-12-12 03:40:37 +00:00
# we only need stack dump if we want to scan for internal transaction address
Redesign of BaseVMState descriptor (#923) * Redesign of BaseVMState descriptor why: BaseVMState provides an environment for executing transactions. The current descriptor also provides data that cannot generally be known within the execution environment, e.g. the total gasUsed which is available not before after all transactions have finished. Also, the BaseVMState constructor has been replaced by a constructor that does not need pre-initialised input of the account database. also: Previous constructor and some fields are provided with a deprecated annotation (producing a lot of noise.) * Replace legacy directives in production sources * Replace legacy directives in unit test sources * fix CI (missing premix update) * Remove legacy directives * chase CI problem * rebased * Re-introduce 'AccountsCache' constructor optimisation for 'BaseVmState' re-initialisation why: Constructing a new 'AccountsCache' descriptor can be avoided sometimes when the current state root is properly positioned already. Such a feature existed already as the update function 'initStateDB()' for the 'BaseChanDB' where the accounts cache was linked into this desctiptor. The function 'initStateDB()' was removed and re-implemented into the 'BaseVmState' constructor without optimisation. The old version was of restricted use as a wrong accounts cache state would unconditionally throw an exception rather than conceptually ask for a remedy. The optimised 'BaseVmState' re-initialisation has been implemented for the 'persistBlocks()' function. also: moved some test helpers to 'test/replay' folder * Remove unused & undocumented fields from Chain descriptor why: Reduces attack surface in general & improves reading the code.
2022-01-18 16:19:32 +00:00
captureFlags = {EnableTracing, DisableMemory, DisableStorage, EnableAccount}
2022-12-02 04:39:12 +00:00
vmState = BaseVMState.new(header, captureCom, captureFlags)
miner = vmState.coinbase()
2018-12-12 03:40:37 +00:00
var
2018-12-12 09:41:18 +00:00
before = newJArray()
after = newJArray()
2022-12-02 04:39:12 +00:00
stateBefore = AccountsCache.init(captureTrieDB, parent.stateRoot, com.pruneTrie)
2018-12-12 03:40:37 +00:00
2018-12-12 09:41:18 +00:00
for idx, tx in body.transactions:
2018-12-12 03:40:37 +00:00
let sender = tx.getSender
let recipient = tx.getRecipient(sender)
2018-12-12 09:41:18 +00:00
before.captureAccount(stateBefore, sender, senderName & $idx)
before.captureAccount(stateBefore, recipient, recipientName & $idx)
2018-12-12 03:40:37 +00:00
before.captureAccount(stateBefore, miner, minerName)
2018-12-12 03:40:37 +00:00
for idx, uncle in body.uncles:
before.captureAccount(stateBefore, uncle.coinbase, uncleName & $idx)
Fearture/poa clique tuning (#765) * Provide API details: API is bundled via clique.nim. * Set extraValidation as default for PoA chains why: This triggers consensus verification and an update of the list of authorised signers. These signers are integral part of the PoA block chain. todo: Option argument to control validation for the nimbus binary. * Fix snapshot state block number why: Using sub-sequence here, so the len() function was wrong. * Optional start where block verification begins why: Can speed up time building loading initial parts of block chain. For PoA, this allows to prove & test that authorised signers can be (correctly) calculated starting at any point on the block chain. todo: On Goerli around blocks #193537..#197568, processing time increases disproportionally -- needs to be understand * For Clique test, get old grouping back (7 transactions per log entry) why: Forgot to change back after troubleshooting * Fix field/function/module-name misunderstanding why: Make compilation work * Use eth_types.blockHash() rather than utils.hash() in Clique modules why: Prefer lib module * Dissolve snapshot_misc.nim details: .. into clique_verify.nim (the other source file clique_unused.nim is inactive) * Hide unused AsyncLock in Clique descriptor details: Unused here but was part of the Go reference implementation * Remove fakeDiff flag from Clique descriptor details: This flag was a kludge in the Go reference implementation used for the canonical tests. The tests have been adapted so there is no need for the fakeDiff flag and its implementation. * Not observing minimum distance from epoch sync point why: For compiling PoA state, the go implementation will walk back to the epoch header with at least 90000 blocks apart from the current header in the absence of other synchronisation points. Here just the nearest epoch header is used. The assumption is that all the checkpoints before have been vetted already regardless of the current branch. details: The behaviour of using the nearest vs the minimum distance epoch is controlled by a flag and can be changed at run time. * Analysing processing time (patch adds some debugging/visualisation support) why: At the first half million blocks of the Goerli replay, blocks on the interval #194854..#196224 take exceptionally long to process, but not due to PoA processing. details: It turns out that much time is spent in p2p/excecutor.processBlock() where the elapsed transaction execution time is significantly greater for many of these blocks. Between the 1371 blocks #194854..#196224 there are 223 blocks with more than 1/2 seconds execution time whereas there are only 4 such blocks before and 13 such after this range up to #504192. * fix debugging symbol in clique_desc (causes CI failing) * Fixing canonical reference tests why: Two errors were introduced earlier but ovelooked: 1. "Remove fakeDiff flag .." patch was incomplete 2. "Not observing minimum distance .." introduced problem w/tests 23/24 details: Fixing 2. needed to revert the behaviour by setting the applySnapsMinBacklog flag for the Clique descriptor. Also a new test was added to lock the new behaviour. * Remove cruft why: Clique/PoA processing was intended to take place somewhere in executor/process_block.processBlock() but was decided later to run from chain/persist_block.persistBlock() instead. * Update API comment * ditto
2021-07-30 14:06:51 +00:00
discard vmState.processBlockNotPoA(header, body)
2018-12-12 03:40:37 +00:00
var stateAfter = vmState.stateDB
2019-02-02 09:20:45 +00:00
2018-12-12 09:41:18 +00:00
for idx, tx in body.transactions:
2018-12-12 03:40:37 +00:00
let sender = tx.getSender
let recipient = tx.getRecipient(sender)
2018-12-12 09:41:18 +00:00
after.captureAccount(stateAfter, sender, senderName & $idx)
after.captureAccount(stateAfter, recipient, recipientName & $idx)
vmState.removeTracedAccounts(sender, recipient)
2018-12-12 03:40:37 +00:00
after.captureAccount(stateAfter, miner, minerName)
vmState.removeTracedAccounts(miner)
2018-12-12 03:40:37 +00:00
for idx, uncle in body.uncles:
after.captureAccount(stateAfter, uncle.coinbase, uncleName & $idx)
vmState.removeTracedAccounts(uncle.coinbase)
2018-12-12 03:40:37 +00:00
2018-12-25 07:31:12 +00:00
# internal transactions:
for idx, acc in tracedAccountsPairs(vmState):
before.captureAccount(stateBefore, acc, internalTxName & $idx)
for idx, acc in tracedAccountsPairs(vmState):
after.captureAccount(stateAfter, acc, internalTxName & $idx)
2018-12-12 03:40:37 +00:00
result = %{"before": before, "after": after}
2018-12-12 04:16:40 +00:00
2018-12-12 15:18:46 +00:00
if dumpState:
result.dumpMemoryDB(memoryDB)
2022-12-02 04:39:12 +00:00
proc traceBlock*(com: CommonRef, header: BlockHeader, body: BlockBody, tracerFlags: set[TracerFlags] = {}): JsonNode =
2018-12-12 04:16:40 +00:00
let
2022-12-02 04:39:12 +00:00
parent = com.db.getParentHeader(header)
2018-12-12 04:16:40 +00:00
memoryDB = newMemoryDB()
2022-12-02 04:39:12 +00:00
captureDB = newCaptureDB(com.db.db, memoryDB)
2018-12-12 04:16:40 +00:00
captureTrieDB = trieDB captureDB
2022-12-02 04:39:12 +00:00
captureCom = com.clone(captureTrieDB)
Redesign of BaseVMState descriptor (#923) * Redesign of BaseVMState descriptor why: BaseVMState provides an environment for executing transactions. The current descriptor also provides data that cannot generally be known within the execution environment, e.g. the total gasUsed which is available not before after all transactions have finished. Also, the BaseVMState constructor has been replaced by a constructor that does not need pre-initialised input of the account database. also: Previous constructor and some fields are provided with a deprecated annotation (producing a lot of noise.) * Replace legacy directives in production sources * Replace legacy directives in unit test sources * fix CI (missing premix update) * Remove legacy directives * chase CI problem * rebased * Re-introduce 'AccountsCache' constructor optimisation for 'BaseVmState' re-initialisation why: Constructing a new 'AccountsCache' descriptor can be avoided sometimes when the current state root is properly positioned already. Such a feature existed already as the update function 'initStateDB()' for the 'BaseChanDB' where the accounts cache was linked into this desctiptor. The function 'initStateDB()' was removed and re-implemented into the 'BaseVmState' constructor without optimisation. The old version was of restricted use as a wrong accounts cache state would unconditionally throw an exception rather than conceptually ask for a remedy. The optimised 'BaseVmState' re-initialisation has been implemented for the 'persistBlocks()' function. also: moved some test helpers to 'test/replay' folder * Remove unused & undocumented fields from Chain descriptor why: Reduces attack surface in general & improves reading the code.
2022-01-18 16:19:32 +00:00
captureFlags = tracerFlags + {EnableTracing}
2022-12-02 04:39:12 +00:00
vmState = BaseVMState.new(header, captureCom, captureFlags)
2018-12-12 04:16:40 +00:00
if header.txRoot == EMPTY_ROOT_HASH: return newJNull()
2019-03-13 21:36:54 +00:00
doAssert(body.transactions.calcTxRoot == header.txRoot)
doAssert(body.transactions.len != 0)
2018-12-12 04:16:40 +00:00
var gasUsed = GasInt(0)
for tx in body.transactions:
let
sender = tx.getSender
rc = vmState.processTransaction(tx, sender, header)
2022-04-08 04:54:11 +00:00
if rc.isOk:
gasUsed = gasUsed + rc.value
2018-12-12 04:16:40 +00:00
result = vmState.getTracingResult()
2018-12-12 04:31:53 +00:00
result["gas"] = %gasUsed
2018-12-12 15:18:46 +00:00
if TracerFlags.DisableState notin tracerFlags:
result.dumpMemoryDB(memoryDB)
2022-12-02 04:39:12 +00:00
proc traceTransactions*(com: CommonRef, header: BlockHeader, blockBody: BlockBody): JsonNode =
2018-12-25 10:31:51 +00:00
result = newJArray()
for i in 0 ..< blockBody.transactions.len:
2022-12-02 04:39:12 +00:00
result.add traceTransaction(com, header, blockBody, i, {DisableState})
2018-12-25 10:31:51 +00:00
2022-12-02 04:39:12 +00:00
proc dumpDebuggingMetaData*(com: CommonRef, header: BlockHeader,
blockBody: BlockBody, vmState: BaseVMState, launchDebugger = true) =
2018-12-25 10:31:51 +00:00
let
blockNumber = header.blockNumber
var
memoryDB = newMemoryDB()
2022-12-02 04:39:12 +00:00
captureDB = newCaptureDB(com.db.db, memoryDB)
2018-12-25 10:31:51 +00:00
captureTrieDB = trieDB captureDB
2022-12-02 04:39:12 +00:00
captureCom = com.clone(captureTrieDB)
bloom = createBloom(vmState.receipts)
let blockSummary = %{
"receiptsRoot": %("0x" & toHex(calcReceiptRoot(vmState.receipts).data)),
"stateRoot": %("0x" & toHex(vmState.stateDB.rootHash.data)),
"logsBloom": %("0x" & toHex(bloom))
}
2018-12-25 10:31:51 +00:00
var metaData = %{
"blockNumber": %blockNumber.toHex,
2022-12-02 04:39:12 +00:00
"txTraces": traceTransactions(captureCom, header, blockBody),
"stateDump": dumpBlockState(captureCom, header, blockBody),
"blockTrace": traceBlock(captureCom, header, blockBody, {DisableState}),
"receipts": toJson(vmState.receipts),
"block": blockSummary
2018-12-25 10:31:51 +00:00
}
metaData.dumpMemoryDB(memoryDB)
2019-01-12 12:48:28 +00:00
let jsonFileName = "debug" & $blockNumber & ".json"
if launchDebugger:
launchPremix(jsonFileName, metaData)
else:
writeFile(jsonFileName, metaData.pretty())