diff --git a/Makefile b/Makefile index bd1ddfe8a..b986a7528 100644 --- a/Makefile +++ b/Makefile @@ -237,9 +237,18 @@ lc-proxy: | build deps lc-proxy-test: | build deps $(ENV_SCRIPT) nim lc_proxy_test $(NIM_PARAMS) nimbus.nims +# builds transition tool +t8n: | build deps + $(ENV_SCRIPT) nim c $(NIM_PARAMS) -d:chronicles_default_output_device=stderr "tools/t8n/$@.nim" + +# builds and runs transition tool test suite +t8n_test: | build deps t8n + $(ENV_SCRIPT) nim c -r $(NIM_PARAMS) -d:chronicles_default_output_device=stderr "tools/t8n/$@.nim" + # usual cleaning clean: | clean-common rm -rf build/{nimbus,fluffy,lc_proxy,$(TOOLS_CSV),all_tests,test_kvstore_rocksdb,test_rpc,all_fluffy_tests,all_fluffy_portal_spec_tests,test_portal_testnet,portalcli,blockwalk,eth_data_exporter,utp_test_app,utp_test,*.dSYM} + rm -rf tools/t8n/{t8n,t8n_test} ifneq ($(USE_LIBBACKTRACE), 0) + $(MAKE) -C vendor/nim-libbacktrace clean $(HANDLE_OUTPUT) endif diff --git a/nimbus/vm2/transaction_tracer.nim b/nimbus/vm2/transaction_tracer.nim index 0560d0c0e..9d4b5bcdc 100644 --- a/nimbus/vm2/transaction_tracer.nim +++ b/nimbus/vm2/transaction_tracer.nim @@ -107,7 +107,7 @@ proc traceOpCodeEnded*(tracer: var TransactionTracer, c: Computation, op: Op, la let gasRemaining = j["gas"].getBiggestInt() j["gasCost"] = %(gasRemaining - c.gasMeter.gasRemaining) - if op in {Return, Revert}: + if op in {Return, Revert} and TracerFlags.DisableReturnData notin tracer.flags: let returnValue = %("0x" & toHex(c.output, true)) j["returnValue"] = returnValue tracer.trace["returnValue"] = returnValue diff --git a/nimbus/vm2/types.nim b/nimbus/vm2/types.nim index d2ea4a3b6..c2be949b2 100644 --- a/nimbus/vm2/types.nim +++ b/nimbus/vm2/types.nim @@ -64,6 +64,7 @@ type DisableState DisableStateDiff EnableAccount + DisableReturnData TransactionTracer* = object trace*: JsonNode diff --git a/tools/t8n/config.nim b/tools/t8n/config.nim new file mode 100644 index 000000000..939d0d244 --- /dev/null +++ b/tools/t8n/config.nim @@ -0,0 +1,149 @@ +import + std/[options, os, strutils], + confutils, + ./types + +export + options + +func combineForks(): string = + for x in low(TestFork)..high(TestFork): + result.add "- " & $x & "\n" + +const + availableForks = combineForks() + +type + HexOrInt* = distinct uint64 + + T8NConf* = object of RootObj + traceEnabled* {. + desc: "Output full trace logs to files trace--.jsonl" + defaultValue: false + name: "trace" }: bool + + traceMemory* {. + desc: "Enable full memory dump in traces" + defaultValue: false + name: "trace.memory" }: bool + + traceNostack* {. + desc: "Disable stack output in traces" + defaultValue: false + name: "trace.nostack" }: bool + + traceReturnData* {. + desc: "Enable return data output in traces" + defaultValue: false + name: "trace.returndata" }: bool + + outputBaseDir* {. + desc: "Specifies where output files are placed. Will be created if it does not exist" + defaultValue: "" + name: "output.basedir" }: string + + outputBody* {. + desc: "If set, the RLP of the transactions (block body) will be written to this file" + defaultValue: "" + name: "output.body" }: string + + outputAlloc* {. + desc: "Determines where to put the `alloc` of the post-state." + longDesc: + "`stdout` - into the stdout output\n" & + "`stderr` - into the stderr output\n" & + " - into the file \n" + defaultValue: "alloc.json" + name: "output.alloc" }: string + + outputResult* {. + desc: "Determines where to put the `result` (stateroot, txroot etc) of the post-state." + longDesc: + "`stdout` - into the stdout output\n" & + "`stderr` - into the stderr output\n" & + " - into the file \n" + defaultValue: "result.json" + name: "output.result" }: string + + inputAlloc* {. + desc: "`stdin` or file name of where to find the prestate alloc to use." + defaultValue: "alloc.json" + name: "input.alloc" }: string + + inputEnv* {. + desc: "`stdin` or file name of where to find the prestate env to use." + defaultValue: "env.json" + name: "input.env" }: string + + inputTxs* {. + desc: "`stdin` or file name of where to find the transactions to apply. " & + "If the file extension is '.rlp', then the data is interpreted as an RLP list of signed transactions. " & + "The '.rlp' format is identical to the output.body format." + defaultValue: "txs.json" + name: "input.txs" }: string + + stateReward* {. + desc: "Mining reward. Set to 0 to disable" + defaultValue: 0 + name: "state.reward" }: HexOrInt + + stateChainId* {. + desc: "ChainID to use" + defaultValue: 1 + name: "state.chainid" }: HexOrInt + + stateFork* {. + desc: "Name of ruleset to use." + longDesc: $availableForks + defaultValue: "GrayGlacier" + name: "state.fork" }: string + + verbosity* {. + desc: "sets the verbosity level" + longDesc: + "0 = silent, 1 = error, 2 = warn, 3 = info, 4 = debug, 5 = detail" + defaultValue: 3 + name: "verbosity" }: int + +proc parseCmdArg*(T: type HexOrInt, p: TaintedString): T = + if startsWith(p.string, "0x"): + parseHexInt(p.string).T + else: + parseInt(p.string).T + +proc completeCmdArg*(T: type HexOrInt, val: TaintedString): seq[string] = + return @[] + +proc notCmd(x: string): bool = + if x.len == 0: return true + x[0] != '-' + +proc convertToNimStyle(cmds: openArray[string]): seq[string] = + # convert something like '--key value' to '--key=value' + var i = 0 + while i < cmds.len: + if notCmd(cmds[i]) or i == cmds.len-1: + result.add cmds[i] + inc i + continue + + if i < cmds.len and notCmd(cmds[i+1]): + result.add cmds[i] & "=" & cmds[i+1] + inc i + else: + result.add cmds[i] + + inc i + +const + Copyright = "Copyright (c) 2022 Status Research & Development GmbH" + Version = "Nimbus-t8n 0.1.0" + +proc init*(_: type T8NConf, cmdLine = commandLineParams()): T8NConf = + {.push warning[ProveInit]: off.} + result = T8NConf.load( + cmdLine.convertToNimStyle, + version = Version, + copyrightBanner = Copyright + ) + {.pop.} diff --git a/tools/t8n/config.nims b/tools/t8n/config.nims new file mode 100644 index 000000000..0eab9fa64 --- /dev/null +++ b/tools/t8n/config.nims @@ -0,0 +1 @@ +switch("define", "chronicles_default_output_device=stderr") diff --git a/tools/t8n/helpers.nim b/tools/t8n/helpers.nim new file mode 100644 index 000000000..93cbc077b --- /dev/null +++ b/tools/t8n/helpers.nim @@ -0,0 +1,424 @@ +import + std/[json, strutils, tables], + stew/byteutils, + stint, + eth/[common, rlp, keys], + ../../nimbus/[chain_config, forks, transaction], + ./types + +func getChainConfig*(network: string): ChainConfig = + let c = ChainConfig() + const + H = high(BlockNumber) + Zero = 0.toBlockNumber + Five = 5.toBlockNumber + + proc assignNumber(c: ChainConfig, + fork: Fork, n: BlockNumber) = + var number: array[Fork, BlockNumber] + var z = low(Fork) + while z < fork: + number[z] = Zero + z = z.succ + number[fork] = n + z = high(Fork) + while z > fork: + number[z] = H + z = z.pred + + c.homesteadBlock = number[FkHomestead] + c.daoForkBlock = number[FkHomestead] + c.eip150Block = number[FkTangerine] + c.eip155Block = number[FkSpurious] + c.eip158Block = number[FkSpurious] + c.byzantiumBlock = number[FkByzantium] + c.constantinopleBlock = number[FkConstantinople] + c.petersburgBlock = number[FkPetersburg] + c.istanbulBlock = number[FkIstanbul] + c.muirGlacierBlock = number[FkBerlin] + c.berlinBlock = number[FkBerlin] + c.londonBlock = number[FkLondon] + c.arrowGlacierBlock = number[FkLondon] + c.grayGlacierBlock = number[FkLondon] + c.mergeForkBlock = number[FkParis] + c.shanghaiBlock = number[FkShanghai] + c.cancunBlock = number[FkCancun] + + c.daoForkSupport = false + c.chainId = 1.ChainId + c.terminalTotalDifficulty = none(UInt256) + + case network + of "Frontier": + c.assignNumber(FkFrontier, Zero) + of "Homestead": + c.assignNumber(FkHomestead, Zero) + of "EIP150": + c.assignNumber(FkTangerine, Zero) + of "EIP158": + c.assignNumber(FkSpurious, Zero) + of "Byzantium": + c.assignNumber(FkByzantium, Zero) + of "Constantinople": + c.assignNumber(FkConstantinople, Zero) + of "ConstantinopleFix": + c.assignNumber(FkPetersburg, Zero) + of "Istanbul": + c.assignNumber(FkIstanbul, Zero) + of "FrontierToHomesteadAt5": + c.assignNumber(FkHomestead, Five) + of "HomesteadToEIP150At5": + c.assignNumber(FkTangerine, Five) + of "HomesteadToDaoAt5": + c.assignNumber(FkHomestead, Zero) + c.daoForkBlock = Five + c.daoForkSupport = true + of "EIP158ToByzantiumAt5": + c.assignNumber(FkByzantium, Five) + of "ByzantiumToConstantinopleAt5": + c.assignNumber(FkPetersburg, Five) + of "ByzantiumToConstantinopleFixAt5": + c.assignNumber(FkPetersburg, Five) + c.constantinopleBlock = Five + of "ConstantinopleFixToIstanbulAt5": + c.assignNumber(FkIstanbul, Five) + of "Berlin": + c.assignNumber(FkBerlin, Zero) + of "BerlinToLondonAt5": + c.assignNumber(FkLondon, Five) + of "London": + c.assignNumber(FkLondon, Zero) + c.arrowGlacierBlock = H + c.grayGlacierBlock = H + of "ArrowGlacier": + c.assignNumber(FkLondon, Zero) + c.grayGlacierBlock = H + of "GrayGlacier": + c.assignNumber(FkLondon, Zero) + c.grayGlacierBlock = Zero + of "Merged": + c.assignNumber(FkParis, Zero) + c.terminalTotalDifficulty = some(0.u256) + of "ArrowGlacierToMergeAtDiffC0000": + c.assignNumber(FkParis, H) + c.terminalTotalDifficulty = some(0xC0000.u256) + else: + raise newError(ErrorConfig, "unsupported network " & network) + + result = c + +proc parseHexOrInt[T](x: string): T = + when T is UInt256: + if x.startsWith("0x"): + UInt256.fromHex(x) + else: + parse(x, UInt256, 10) + else: + if x.startsWith("0x"): + fromHex[T](x) + else: + parseInt(x).T + +template fromJson(T: type EthAddress, n: JsonNode, field: string): EthAddress = + hexToByteArray(n[field].getStr(), sizeof(T)) + +template fromJson(T: type Blob, n: JsonNode, field: string): Blob = + hexToSeqByte(n[field].getStr()) + +proc fromJson(T: type uint64, n: JsonNode, field: string): uint64 = + if n[field].kind == JInt: + n[field].getInt().uint64 + else: + parseHexOrInt[AccountNonce](n[field].getStr()) + +template fromJson(T: type UInt256, n: JsonNode, field: string): UInt256 = + parseHexOrInt[UInt256](n[field].getStr()) + +template fromJson(T: type GasInt, n: JsonNode, field: string): GasInt = + parseHexOrInt[GasInt](n[field].getStr()) + +template fromJson(T: type ChainId, n: JsonNode, field: string): ChainId = + parseHexOrInt[uint64](n[field].getStr()).ChainId + +proc fromJson(T: type Hash256, n: JsonNode, field: string): Hash256 = + var num = n[field].getStr() + num.removePrefix("0x") + if num.len < 64: + num = repeat('0', 64 - num.len) & num + Hash256(data: hexToByteArray(num, 32)) + +template fromJson(T: type EthTime, n: JsonNode, field: string): EthTime = + fromUnix(parseHexOrInt[int64](n[field].getStr())) + +proc fromJson(T: type AccessList, n: JsonNode, field: string): AccessList = + let z = n[field] + if z.kind == JNull: + return + + for x in z: + var ap = AccessPair( + address: EthAddress.fromJson(x, "address") + ) + let sks = x["storageKeys"] + for sk in sks: + ap.storageKeys.add hexToByteArray(sk.getStr(), 32) + result.add ap + +proc fromJson(T: type Ommer, n: JsonNode): Ommer = + Ommer( + delta: fromJson(uint64, n, "delta"), + address: fromJson(EthAddress, n, "address") + ) + +template `gas=`(tx: var Transaction, x: GasInt) = + tx.gasLimit = x + +template `input=`(tx: var Transaction, x: Blob) = + tx.payload = x + +template `v=`(tx: var Transaction, x: int64) = + tx.V = x + +template `r=`(tx: var Transaction, x: UInt256) = + tx.R = x + +template `s=`(tx: var Transaction, x: UInt256) = + tx.S = x + +template `maxPriorityFeePerGas=`(tx: var Transaction, x: GasInt) = + tx.maxPriorityFee = x + +template `maxFeePerGas=`(tx: var Transaction, x: GasInt) = + tx.maxFee = x + +template required(o: untyped, T: type, oField: untyped) = + const fName = astToStr(oField) + if not n.hasKey(fName): + raise newError(ErrorJson, "missing required field '" & fName & "' in transaction") + o.oField = T.fromJson(n, fName) + +template omitZero(o: untyped, T: type, oField: untyped) = + const fName = astToStr(oField) + if n.hasKey(fName): + o.oField = T.fromJson(n, fName) + +template optional(o: untyped, T: type, oField: untyped) = + const fName = astToStr(oField) + if n.hasKey(fName) and n[fName].kind != JNull: + o.oField = some(T.fromJson(n, fName)) + +proc parseAlloc*(ctx: var TransContext, n: JsonNode) = + for accAddr, acc in n: + let address = hexToByteArray[20](accAddr) + var ga = GenesisAccount() + if acc.hasKey("code"): + ga.code = Blob.fromJson(acc, "code") + if acc.hasKey("nonce"): + ga.nonce = AccountNonce.fromJson(acc, "nonce") + if acc.hasKey("balance"): + ga.balance = UInt256.fromJson(acc, "balance") + else: + raise newError(ErrorJson, "GenesisAlloc: balance required") + if acc.hasKey("storage"): + let storage = acc["storage"] + for k, v in storage: + ga.storage[UInt256.fromHex(k)] = UInt256.fromHex(v.getStr()) + ctx.alloc[address] = ga + +proc parseEnv*(ctx: var TransContext, n: JsonNode) = + required(ctx.env, EthAddress, currentCoinbase) + required(ctx.env, GasInt, currentGasLimit) + required(ctx.env, BlockNumber, currentNumber) + required(ctx.env, EthTime, currentTimestamp) + optional(ctx.env, DifficultyInt, currentDifficulty) + optional(ctx.env, Hash256, currentRandom) + optional(ctx.env, DifficultyInt, parentDifficulty) + omitZero(ctx.env, EthTime, parentTimestamp) + optional(ctx.env, UInt256, currentBaseFee) + omitZero(ctx.env, Hash256, parentUncleHash) + + if n.hasKey("blockHashes"): + let w = n["blockHashes"] + for k, v in w: + ctx.env.blockHashes[fromHex[uint64](k)] = Hash256.fromHex(v.getStr()) + + if n.hasKey("ommers"): + let w = n["ommers"] + for v in w: + ctx.env.ommers.add Ommer.fromJson(v) + +proc parseTx(n: JsonNode, chainId: ChainID): Transaction = + var tx: Transaction + if not n.hasKey("type"): + tx.txType = TxLegacy + else: + tx.txType = int64.fromJson(n, "type").TxType + + required(tx, AccountNonce, nonce) + required(tx, GasInt, gas) + required(tx, UInt256, value) + required(tx, Blob, input) + required(tx, int64, v) + required(tx, UInt256, r) + required(tx, UInt256, s) + + if n.hasKey("to"): + tx.to = some(EthAddress.fromJson(n, "to")) + + case tx.txType + of TxLegacy: + required(tx, GasInt, gasPrice) + of TxEip2930: + required(tx, GasInt, gasPrice) + required(tx, ChainId, chainId) + omitZero(tx, AccessList, accessList) + of TxEip1559: + required(tx, ChainId, chainId) + required(tx, GasInt, maxPriorityFeePerGas) + required(tx, GasInt, maxFeePerGas) + omitZero(tx, AccessList, accessList) + + var eip155 = true + if n.hasKey("protected"): + eip155 = n["protected"].bval + + if n.hasKey("secretKey"): + let data = Blob.fromJson(n, "secretKey") + let secretKey = PrivateKey.fromRaw(data).tryGet + signTransaction(tx, secretKey, chainId, eip155) + else: + tx + +proc parseTxs*(ctx: var TransContext, txs: JsonNode, chainId: ChainID) = + if txs.kind == JNull: + return + if txs.kind != JArray: + raise newError(ErrorJson, "Transaction list should be a JSON array, got=" & $txs.kind) + for n in txs: + ctx.txs.add parseTx(n, chainId) + +proc parseTxsRlp*(ctx: var TransContext, hexData: string) = + let data = hexToSeqByte(hexData) + ctx.txs = rlp.decode(data, seq[Transaction]) + +proc parseInputFromStdin*(ctx: var TransContext, chainConfig: ChainConfig) = + let data = stdin.readAll() + let n = json.parseJson(data) + if n.hasKey("alloc"): ctx.parseAlloc(n["alloc"]) + if n.hasKey("env"): ctx.parseEnv(n["env"]) + if n.hasKey("txs"): ctx.parseTxs(n["txs"], chainConfig.chainId) + if n.hasKey("txsRlp"): ctx.parseTxsRlp(n["txsRlp"].getStr()) + +template stripLeadingZeros(value: string): string = + var cidx = 0 + # ignore the last character so we retain '0' on zero value + while cidx < value.len - 1 and value[cidx] == '0': + cidx.inc + value[cidx .. ^1] + +proc `@@`*[K, V](x: Table[K, V]): JsonNode +proc `@@`*[T](x: seq[T]): JsonNode + +proc to0xHex(x: UInt256): string = + "0x" & x.toHex + +proc `@@`(x: uint64 | int64 | int): JsonNode = + let hex = x.toHex.stripLeadingZeros + %("0x" & hex.toLowerAscii) + +proc `@@`(x: UInt256): JsonNode = + %("0x" & x.toHex) + +proc `@@`(x: Hash256): JsonNode = + %("0x" & x.data.toHex) + +proc `@@`*(x: Blob): JsonNode = + %("0x" & x.toHex) + +proc `@@`(x: bool): JsonNode = + %(if x: "0x1" else: "0x0") + +proc `@@`(x: EthAddress): JsonNode = + %("0x" & x.toHex) + +proc `@@`(x: Topic): JsonNode = + %("0x" & x.toHex) + +proc toJson(x: Table[UInt256, UInt256]): JsonNode = + # special case, we need to convert UInt256 into full 32 bytes + # and not shorter + result = newJObject() + for k, v in x: + result["0x" & k.dumpHex] = %("0x" & v.dumpHex) + +proc `@@`(acc: GenesisAccount): JsonNode = + result = newJObject() + if acc.code.len > 0: + result["code"] = @@(acc.code) + result["balance"] = @@(acc.balance) + if acc.nonce > 0: + result["nonce"] = @@(acc.nonce) + if acc.storage.len > 0: + result["storage"] = toJson(acc.storage) + +proc `@@`[K, V](x: Table[K, V]): JsonNode = + result = newJObject() + for k, v in x: + result[k.to0xHex] = @@(v) + +proc `@@`(x: BloomFilter): JsonNode = + %("0x" & toHex[256](x)) + +proc `@@`(x: Log): JsonNode = + result = %{ + "address": @@(x.address), + "topics" : @@(x.topics), + "data" : @@(x.data) + } + +proc `@@`(x: TxReceipt): JsonNode = + result = %{ + "root" : if x.root == Hash256(): %("0x") else: @@(x.root), + "status" : @@(x.status), + "cumulativeGasUsed": @@(x.cumulativeGasUsed), + "logsBloom" : @@(x.logsBloom), + "logs" : if x.logs.len == 0: newJNull() else: @@(x.logs), + "transactionHash" : @@(x.transactionHash), + "contractAddress" : @@(x.contractAddress), + "gasUsed" : @@(x.gasUsed), + "blockHash" : @@(x.blockHash), + "transactionIndex" : @@(x.transactionIndex) + } + if x.txType > TxLegacy: + result["type"] = %("0x" & toHex(x.txType.int, 1)) + +proc `@@`(x: RejectedTx): JsonNode = + result = %{ + "index": %(x.index), + "error": %(x.error) + } + +proc `@@`[T](x: seq[T]): JsonNode = + result = newJArray() + for c in x: + result.add @@(c) + +proc `@@`[T](x: Option[T]): JsonNode = + if x.isNone: + newJNull() + else: + @@(x.get()) + +proc `@@`*(x: ExecutionResult): JsonNode = + result = %{ + "stateRoot" : @@(x.stateRoot), + "txRoot" : @@(x.txRoot), + "receiptsRoot": @@(x.receiptsRoot), + "logsHash" : @@(x.logsHash), + "logsBloom" : @@(x.bloom), + "receipts" : @@(x.receipts), + "currentDifficulty": @@(x.currentDifficulty), + "gasUsed" : @@(x.gasUsed) + } + if x.rejected.len > 0: + result["rejected"] = @@(x.rejected) diff --git a/tools/t8n/readme.md b/tools/t8n/readme.md new file mode 100644 index 000000000..30a007419 --- /dev/null +++ b/tools/t8n/readme.md @@ -0,0 +1,56 @@ +## EVM state transition tool + +The `t8n` tool is a stateless state transition utility. + +### Build instructions + +There are few options to build `t8n` tool like any other nimbus tools. + +1. Use nimble to install dependencies and your system Nim compiler(version <= 1.6.0). + ``` + $> nimble install -y --depsOnly + $> nim c -d:release -d:chronicles_default_output_device=stderr tools/t8n/t8n + $> nim c -r -d:release tools/t8n/t8n_test + ``` +2. Use nimbus shipped Nim compiler and dependencies. + ``` + $> make update + $> ./env.sh nim c -d:release -d:chronicles_default_output_device=stderr tools/t8n/t8n + $> ./env.sh nim c -r -d:release tools/t8n/t8n_test + ``` +3. Use nimbus makefile. + ``` + $> make update + $> make t8n + $> make t8n_test + ``` + +### Command line params + +Available command line params +``` + + --trace Output full trace logs to files trace--.jsonl + --trace.memory Enable full memory dump in traces. + --trace.nostack Disable stack output in traces. + --trace.returndata Enable return data output in traces. + --output.basedir value Specifies where output files are placed. Will be created if it does not exist. + --output.alloc alloc Determines where to put the alloc of the post-state. + `stdout` - into the stdout output + `stderr` - into the stderr output + - into the file + --output.result result Determines where to put the result (stateroot, txroot etc) of the post-state. + `stdout` - into the stdout output + `stderr` - into the stderr output + - into the file + --output.body value If set, the RLP of the transactions (block body) will be written to this file. + --input.txs stdin stdin or file name of where to find the transactions to apply. + If the file extension is '.rlp', then the data is interpreted as an RLP list of signed transactions. + The '.rlp' format is identical to the output.body format. + --input.alloc stdin `stdin` or file name of where to find the prestate alloc to use. + --input.env stdin `stdin` or file name of where to find the prestate env to use. + --state.fork value Name of ruleset to use. + --state.chainid value ChainID to use (default: 1). + --state.reward value Mining reward. Set to 0 to disable (default: 0). + +``` diff --git a/tools/t8n/t8n.nim b/tools/t8n/t8n.nim new file mode 100644 index 000000000..4386a16fb --- /dev/null +++ b/tools/t8n/t8n.nim @@ -0,0 +1,24 @@ +import + "."/[config, transition, types] + +template wrapException(body) = + when wrapExceptionEnabled: + try: + body + except T8NError as e: + stderr.writeLine(e.msg) + quit(e.exitCode.int) + except: + let e = getCurrentException() + stderr.writeLine($e.name & " : " & e.msg) + quit(QuitFailure) + else: + body + +proc main() = + wrapException: + let conf = T8NConf.init() + var ctx = TransContext() + ctx.transitionAction(conf) + +main() diff --git a/tools/t8n/t8n_test.nim b/tools/t8n/t8n_test.nim new file mode 100644 index 000000000..5c05481e1 --- /dev/null +++ b/tools/t8n/t8n_test.nim @@ -0,0 +1,299 @@ +import + std/[os, osproc, strutils, json, tables], + unittest2, + "."/[types] + +type + T8nInput = object + inAlloc : string + inTxs : string + inEnv : string + stFork : string + stReward: string + + T8nOutput = object + alloc : bool + result: bool + body : bool + + TestSpec = object + name : string + base : string + input : T8nInput + output : T8nOutput + expExitCode: int + expOut : string + + JsonComparator = object + path: string + error: string + +proc t8nInput(alloc, txs, env, fork, reward: string): T8nInput = + T8nInput( + inAlloc : alloc, + inTxs : txs, + inEnv : env, + stFork : fork, + stReward: reward + ) + +proc get(opt: T8nInput, base : string): string = + result.add(" --input.alloc " & (base / opt.inAlloc)) + result.add(" --input.txs " & (base / opt.inTxs)) + result.add(" --input.env " & (base / opt.inEnv)) + result.add(" --state.fork " & opt.stFork) + if opt.stReward.len > 0: + result.add(" --state.reward " & opt.stReward) + +proc get(opt: T8nOutput): string = + if opt.alloc: + result.add(" --output.alloc stdout") + else: + result.add(" --output.alloc") + + if opt.result: + result.add(" --output.result stdout") + else: + result.add(" --output.result") + + if opt.body: + result.add(" --output.body stdout") + else: + result.add(" --output.body") + +template exit(jsc: var JsonComparator, msg: string) = + jsc.path = path + jsc.error = msg + return false + +proc cmp(jsc: var JsonComparator; a, b: JsonNode, path: string): bool = + ## Check two nodes for equality + if a.isNil: + if b.isNil: return true + jsc.exit("A nil, but B not nil") + elif b.isNil: + jsc.exit("A not nil, but B nil") + elif a.kind != b.kind: + jsc.exit("A($1) != B($2)" % [$a.kind, $b.kind]) + else: + result = true + case a.kind + of JString: + if a.str != b.str: + jsc.exit("STRING A($1) != B($2)" % [a.str, b.str]) + of JInt: + if a.num != b.num: + jsc.exit("INT A($1) != B($2)" % [$a.num, $b.num]) + of JFloat: + if a.fnum != b.fnum: + jsc.exit("FLOAT A($1) != B($2)" % [$a.fnum, $b.fnum]) + of JBool: + if a.bval != b.bval: + jsc.exit("BOOL A($1) != B($2)" % [$a.bval, $b.bval]) + of JNull: + result = true + of JArray: + for i, x in a.elems: + if not jsc.cmp(x, b.elems[i], path & "/" & $i): + return false + of JObject: + # we cannot use OrderedTable's equality here as + # the order does not matter for equality here. + if a.fields.len != b.fields.len: + jsc.exit("OBJ LEN A($1) != B($2)" % [$a.fields.len, $b.fields.len]) + for key, val in a.fields: + if not b.fields.hasKey(key): + jsc.exit("OBJ FIELD A($1) != B(none)" % [key]) + if not jsc.cmp(val, b.fields[key], path & "/" & key): + return false + +proc notRejectedError(path: string): bool = + # we only check error status, and not the error message + # because each implementation can have different error + # message + not (path.startsWith("root/result/rejected/") and + path.endsWith("/error")) + +proc runTest(appDir: string, spec: TestSpec): bool = + let base = appDir / spec.base + let args = spec.input.get(base) & spec.output.get() + let cmd = appDir / "t8n" & args + let (res, exitCode) = execCmdEx(cmd) + + if exitCode != spec.expExitCode: + echo "test $1: wrong exit code, have $2, want $3" % + [spec.name, $exitCode, $spec.expExitCode] + echo res + return false + + if spec.expOut.len > 0: + let path = base / spec.expOut + let want = json.parseFile(path) + let have = json.parseJson(res) + var jsc = JsonComparator() + if not jsc.cmp(want, have, "root") and notRejectedError(jsc.path): + echo "test $1: output wrong, have \n$2\nwant\n$3\n" % + [spec.name, have.pretty, want.pretty] + echo "path: $1, error: $2" % + [jsc.path, jsc.error] + return false + + return true + +const + testSpec = [ + TestSpec( + name : "Test exit (3) on bad config", + base : "testdata/1", + input : t8nInput( + "alloc.json", "txs.json", "env.json", "Frontier+1346", "", + ), + output: T8nOutput(alloc: true, result: true), + expExitCode: ErrorConfig.int, + ), + TestSpec( + name : "baseline test", + base : "testdata/1", + input : t8nInput( + "alloc.json", "txs.json", "env.json", "Byzantium", "", + ), + output: T8nOutput(alloc: true, result: true), + expOut: "exp.json", + ), + TestSpec( + name : "blockhash test", + base : "testdata/3", + input : t8nInput( + "alloc.json", "txs.json", "env.json", "Berlin", "" + ), + output: T8nOutput(alloc: true, result: true), + expOut: "exp.json" + ), + TestSpec( + name : "missing blockhash test", + base : "testdata/4", + input : t8nInput( + "alloc.json", "txs.json", "env.json", "Berlin", "", + ), + output: T8nOutput(alloc: true, result: true), + expExitCode: ErrorMissingBlockhash.int, + ), + TestSpec( + name : "Uncle test", + base : "testdata/5", + input : t8nInput( + "alloc.json", "txs.json", "env.json", "Byzantium", "0x80", + ), + output: T8nOutput(alloc: true, result: true), + expOut: "exp.json", + ), + TestSpec( + name : "Sign json transactions", + base : "testdata/13", + input : t8nInput( + "alloc.json", "txs.json", "env.json", "London", "", + ), + output: T8nOutput(body: true), + expOut: "exp.json", + ), + TestSpec( + name : "Already signed transactions", + base : "testdata/13", + input : t8nInput( + "alloc.json", "signed_txs.rlp", "env.json", "London", "", + ), + output: T8nOutput(result: true), + expOut: "exp2.json", + ), + TestSpec( + name : "Difficulty calculation - no uncles", + base : "testdata/14", + input : t8nInput( + "alloc.json", "txs.json", "env.json", "London", "", + ), + output: T8nOutput(result: true), + expOut: "exp.json", + ), + TestSpec( + name : "Difficulty calculation - with uncles", + base : "testdata/14", + input : t8nInput( + "alloc.json", "txs.json", "env.uncles.json", "London", "", + ), + output: T8nOutput(result: true), + expOut: "exp2.json", + ), + TestSpec( + name : "Difficulty calculation - with ommers + Berlin", + base : "testdata/14", + input : t8nInput( + "alloc.json", "txs.json", "env.uncles.json", "Berlin", "", + ), + output: T8nOutput(result: true), + expOut: "exp_berlin.json", + ), + TestSpec( + name : "Difficulty calculation on arrow glacier", + base : "testdata/19", + input : t8nInput( + "alloc.json", "txs.json", "env.json", "London", "", + ), + output: T8nOutput(result: true), + expOut: "exp_london.json", + ), + TestSpec( + name : "Difficulty calculation on arrow glacier", + base : "testdata/19", + input : t8nInput( + "alloc.json", "txs.json", "env.json", "ArrowGlacier", "", + ), + output: T8nOutput(result: true), + expOut: "exp_arrowglacier.json", + ), + TestSpec( + name : "Difficulty calculation on gray glacier", + base : "testdata/19", + input : t8nInput( + "alloc.json", "txs.json", "env.json", "GrayGlacier", "", + ), + output: T8nOutput(result: true), + expOut: "exp_grayglacier.json", + ), + TestSpec( + name : "Sign unprotected (pre-EIP155) transaction", + base : "testdata/23", + input : t8nInput( + "alloc.json", "txs.json", "env.json", "Berlin", "", + ), + output: T8nOutput(result: true), + expOut: "exp.json", + ), + TestSpec( + name : "Test post-merge transition", + base : "testdata/24", + input : t8nInput( + "alloc.json", "txs.json", "env.json", "Merged", "", + ), + output: T8nOutput(alloc: true, result: true), + expOut: "exp.json", + ), + TestSpec( + name : "Test post-merge transition where input is missing random", + base : "testdata/24", + input : t8nInput( + "alloc.json", "txs.json", "env-missingrandom.json", "Merged", "", + ), + output: T8nOutput(alloc: false, result: false), + expExitCode: ErrorConfig.int, + ), + ] + +proc main() = + suite "Transition tool (t8n) test suite": + let appDir = getAppDir() + for x in testSpec: + test x.name: + check runTest(appDir, x) + +when isMainModule: + main() diff --git a/tools/t8n/transition.nim b/tools/t8n/transition.nim new file mode 100644 index 000000000..e8dce0118 --- /dev/null +++ b/tools/t8n/transition.nim @@ -0,0 +1,363 @@ +import + std/[json, strutils, times, tables, options, os], + eth/[common, rlp, trie, trie/db], + stint, chronicles, stew/results, + "."/[config, types, helpers], + ../../nimbus/[chain_config, vm_types, vm_state, utils, transaction], + ../../nimbus/db/[db_chain, accounts_cache], + ../../nimbus/utils/difficulty, + ../../nimbus/p2p/dao, + ../../nimbus/p2p/executor/[process_transaction, executor_helpers] + +const + wrapExceptionEnabled* {.booldefine.} = true + stdinSelector = "stdin" + +type + Dispatch = object + stdout: JsonNode + stderr: JsonNode + + ExecOutput = object + result: ExecutionResult + alloc: GenesisAlloc + + TestVMState = ref object of BaseVMState + blockHashes: Table[uint64, Hash256] + hashError: string + +proc init(_: type Dispatch): Dispatch = + result.stdout = newJObject() + result.stderr = newJObject() + +proc dispatch(dis: var Dispatch, baseDir, fName, name: string, obj: JsonNode) = + case fName + of "stdout": + dis.stdout[name] = obj + of "stderr": + dis.stderr[name] = obj + of "": + # don't save + discard + else: + # save to file + let path = if baseDir.len > 0: + baseDir / fName + else: + fName + writeFile(path, obj.pretty) + +proc dispatchOutput(ctx: var TransContext, conf: T8NConf, res: ExecOutput) = + var dis = Dispatch.init() + createDir(conf.outputBaseDir) + + dis.dispatch(conf.outputBaseDir, conf.outputAlloc, "alloc", @@(res.alloc)) + dis.dispatch(conf.outputBaseDir, conf.outputResult, "result", @@(res.result)) + + let body = @@(rlp.encode(ctx.txs)) + dis.dispatch(conf.outputBaseDir, conf.outputBody, "body", body) + + if dis.stdout.len > 0: + stdout.write(dis.stdout.pretty) + stdout.write("\n") + + if dis.stderr.len > 0: + stderr.write(dis.stderr.pretty) + stderr.write("\n") + +proc envToHeader(env: EnvStruct): BlockHeader = + BlockHeader( + coinbase : env.currentCoinbase, + difficulty : env.currentDifficulty.get(0.u256), + mixDigest : env.currentRandom.get(Hash256()), + blockNumber: env.currentNumber, + gasLimit : env.currentGasLimit, + timestamp : env.currentTimestamp, + stateRoot : emptyRlpHash, + fee : env.currentBaseFee + ) + +proc postState(db: AccountsCache, alloc: var GenesisAlloc) = + for accAddr in db.addresses(): + var acc = GenesisAccount( + code: db.getCode(accAddr), + balance: db.getBalance(accAddr), + nonce: db.getNonce(accAddr) + ) + + for k, v in db.storage(accAddr): + acc.storage[k] = v + alloc[accAddr] = acc + +proc genAddress(vmState: BaseVMState, tx: Transaction, sender: EthAddress): EthAddress = + if tx.to.isNone: + let creationNonce = vmState.readOnlyStateDB().getNonce(sender) + result = generateAddress(sender, creationNonce) + +proc toTxReceipt(vmState: BaseVMState, + rec: Receipt, + tx: Transaction, + sender: EthAddress, + txIndex: int, + gasUsed: GasInt): TxReceipt = + + let contractAddress = genAddress(vmState, tx, sender) + TxReceipt( + txType: tx.txType, + root: if rec.isHash: rec.hash else: Hash256(), + status: rec.status, + cumulativeGasUsed: rec.cumulativeGasUsed, + logsBloom: rec.bloom, + logs: rec.logs, + transactionHash: rlpHash(tx), + contractAddress: contractAddress, + gasUsed: gasUsed, + blockHash: Hash256(), + transactionIndex: txIndex + ) + +proc calcLogsHash(receipts: openArray[Receipt]): Hash256 = + var logs: seq[Log] + for rec in receipts: + logs.add rec.logs + rlpHash(logs) + +proc dumpTrace(txIndex: int, txHash: Hash256, traceResult: JsonNode) = + let fName = "trace-$1-$2.jsonl" % [$txIndex, $txHash] + writeFile(fName, traceResult.pretty) + +proc exec(ctx: var TransContext, + vmState: BaseVMState, + blockReward: UInt256, + header: BlockHeader): ExecOutput = + + var + receipts = newSeqOfCap[TxReceipt](ctx.txs.len) + rejected = newSeq[RejectedTx]() + includedTx = newSeq[Transaction]() + + if vmState.chainDB.config.daoForkSupport and + vmState.chainDB.config.daoForkBlock == vmState.blockNumber: + vmState.mutateStateDB: + db.applyDAOHardFork() + + vmState.receipts = newSeqOfCap[Receipt](ctx.txs.len) + vmState.cumulativeGasUsed = 0 + + for txIndex, tx in ctx.txs: + var sender: EthAddress + if not tx.getSender(sender): + rejected.add RejectedTx( + index: txIndex, + error: "Could not get sender" + ) + continue + + let rc = vmState.processTransaction(tx, sender, header) + + if vmState.tracingEnabled: + dumpTrace(txIndex, rlpHash(tx), vmState.getTracingResult) + + if rc.isErr: + rejected.add RejectedTx( + index: txIndex, + error: "processTransaction failed" + ) + continue + + let gasUsed = rc.get() + let rec = vmState.makeReceipt(tx.txType) + vmState.receipts.add rec + receipts.add toTxReceipt( + vmState, rec, + tx, sender, txIndex, gasUsed + ) + includedTx.add tx + + if blockReward > 0.u256: + var mainReward = blockReward + for uncle in ctx.env.ommers: + var uncleReward = 8.u256 - uncle.delta.u256 + uncleReward = uncleReward * blockReward + uncleReward = uncleReward div 8.u256 + vmState.mutateStateDB: + db.addBalance(uncle.address, uncleReward) + mainReward += blockReward div 32.u256 + + vmState.mutateStateDB: + db.addBalance(ctx.env.currentCoinbase, mainReward) + db.persist(clearCache = false) + + let stateDB = vmState.stateDB + stateDB.postState(result.alloc) + result.result = ExecutionResult( + stateRoot : stateDB.rootHash, + txRoot : includedTx.calcTxRoot, + receiptsRoot: calcReceiptRoot(vmState.receipts), + logsHash : calcLogsHash(vmState.receipts), + bloom : createBloom(vmState.receipts), + receipts : system.move(receipts), + rejected : system.move(rejected), + # geth using both vmContext.Difficulty and vmContext.Random + # therefore we cannot use vmState.difficulty + currentDifficulty: ctx.env.currentDifficulty, + gasUsed : vmState.cumulativeGasUsed + ) + +template wrapException(body: untyped) = + when wrapExceptionEnabled: + try: + body + except IOError as e: + raise newError(ErrorIO, e.msg) + except RlpError as e: + raise newError(ErrorRlp, e.msg) + except ValueError as e: + raise newError(ErrorJson, e.msg) + else: + body + +proc setupAlloc(stateDB: AccountsCache, alloc: GenesisAlloc) = + for accAddr, acc in alloc: + stateDB.setNonce(accAddr, acc.nonce) + stateDB.setCode(accAddr, acc.code) + stateDB.setBalance(accAddr, acc.balance) + + for slot, value in acc.storage: + stateDB.setStorage(accAddr, slot, value) + +method getAncestorHash(vmState: TestVMState; blockNumber: BlockNumber): Hash256 {.gcsafe.} = + # we can't raise exception here, it'll mess with EVM exception handler. + # so, store the exception for later using `hashError` + + let num = blockNumber.truncate(uint64) + var h = Hash256() + if vmState.blockHashes.len == 0: + vmState.hashError = "getAncestorHash($1) invoked, no blockhashes provided" % [$num] + return h + + if not vmState.blockHashes.take(num, h): + vmState.hashError = "getAncestorHash($1) invoked, blockhash for that block not provided" % [$num] + + return h + +proc transitionAction*(ctx: var TransContext, conf: T8NConf) = + wrapException: + var tracerFlags = { + TracerFlags.DisableMemory, + TracerFlags.DisableStorage, + TracerFlags.DisableState, + TracerFlags.DisableStateDiff, + TracerFlags.DisableReturnData + } + + if conf.traceEnabled: + tracerFlags.incl TracerFlags.EnableTracing + if conf.traceMemory: tracerFlags.excl TracerFlags.DisableMemory + if conf.traceNostack: tracerFlags.incl TracerFlags.DisableStack + if conf.traceReturnData: tracerFlags.excl TracerFlags.DisableReturnData + + if conf.inputAlloc.len == 0 and conf.inputEnv.len == 0 and conf.inputTxs.len == 0: + raise newError(ErrorConfig, "either one of input is needeed(alloc, txs, or env)") + + let chainConfig = getChainConfig(conf.stateFork) + chainConfig.chainId = conf.stateChainId.ChainId + + # We need to load three things: alloc, env and transactions. + # May be either in stdin input or in files. + + if conf.inputAlloc == stdinSelector or + conf.inputEnv == stdinSelector or + conf.inputTxs == stdinSelector: + ctx.parseInputFromStdin(chainConfig) + + if conf.inputAlloc != stdinSelector and conf.inputAlloc.len > 0: + let n = json.parseFile(conf.inputAlloc) + ctx.parseAlloc(n) + + if conf.inputEnv != stdinSelector and conf.inputEnv.len > 0: + let n = json.parseFile(conf.inputEnv) + ctx.parseEnv(n) + + if conf.inputTxs != stdinSelector and conf.inputTxs.len > 0: + if conf.inputTxs.endsWith(".rlp"): + let data = readFile(conf.inputTxs) + ctx.parseTxsRlp(data.strip(chars={'"'})) + else: + let n = json.parseFile(conf.inputTxs) + ctx.parseTxs(n, chainConfig.chainId) + + let uncleHash = if ctx.env.parentUncleHash == Hash256(): + EMPTY_UNCLE_HASH + else: + ctx.env.parentUncleHash + + let parent = BlockHeader( + stateRoot: emptyRlpHash, + timestamp: ctx.env.parentTimestamp, + difficulty: ctx.env.parentDifficulty.get(0.u256), + ommersHash: uncleHash, + blockNumber: ctx.env.currentNumber - 1.toBlockNumber + ) + + # Sanity check, to not `panic` in state_transition + if chainConfig.isLondon(ctx.env.currentNumber): + if ctx.env.currentBaseFee.isNone: + raise newError(ErrorConfig, "EIP-1559 config but missing 'currentBaseFee' in env section") + + let isMerged = chainConfig.terminalTotalDifficulty.isSome and + chainConfig.terminalTotalDifficulty.get() == 0 + + if isMerged: + if ctx.env.currentRandom.isNone: + raise newError(ErrorConfig, "post-merge requires currentRandom to be defined in env") + + if ctx.env.currentDifficulty.isSome and ctx.env.currentDifficulty.get() != 0: + raise newError(ErrorConfig, "post-merge difficulty must be zero (or omitted) in env") + ctx.env.currentDifficulty = none(DifficultyInt) + + elif ctx.env.currentDifficulty.isNone: + if ctx.env.parentDifficulty.isNone: + raise newError(ErrorConfig, "currentDifficulty was not provided, and cannot be calculated due to missing parentDifficulty") + + if ctx.env.currentNumber == 0.toBlockNumber: + raise newError(ErrorConfig, "currentDifficulty needs to be provided for block number 0") + + if ctx.env.currentTimestamp <= ctx.env.parentTimestamp: + raise newError(ErrorConfig, + "currentDifficulty cannot be calculated -- currentTime ($1) needs to be after parent time ($2)" % + [$ctx.env.currentTimestamp, $ctx.env.parentTimestamp]) + + ctx.env.currentDifficulty = some(calcDifficulty(chainConfig, + ctx.env.currentTimestamp, parent)) + + let + chainDB = newBaseChainDB(newMemoryDb(), chainConfig, pruneTrie = true) + header = envToHeader(ctx.env) + + # set parent total difficulty + chainDB.setScore(parent.blockHash, 0.u256) + + let vmState = TestVMState( + blockHashes: system.move(ctx.env.blockHashes), + hashError: "" + ) + + vmState.init( + parent = parent, + header = header, + chainDB = chainDB, + tracerFlags = (if conf.traceEnabled: tracerFlags else: {}), + pruneTrie = chainDB.pruneTrie + ) + + vmState.mutateStateDB: + db.setupAlloc(ctx.alloc) + db.persist(clearCache = false) + + let res = exec(ctx, vmState, conf.stateReward.uint64.u256, header) + + if vmState.hashError.len > 0: + raise newError(ErrorMissingBlockhash, vmState.hashError) + + ctx.dispatchOutput(conf, res) diff --git a/tools/t8n/types.nim b/tools/t8n/types.nim new file mode 100644 index 000000000..f5e037936 --- /dev/null +++ b/tools/t8n/types.nim @@ -0,0 +1,106 @@ +import + std/[tables], + eth/common, + ../../nimbus/[chain_config] + +type + TestFork* = enum + Frontier + Homestead + EIP150 + EIP158 + Byzantium + Constantinople + ConstantinopleFix + Istanbul + FrontierToHomesteadAt5 + HomesteadToEIP150At5 + HomesteadToDaoAt5 + EIP158ToByzantiumAt5 + ByzantiumToConstantinopleAt5 + ByzantiumToConstantinopleFixAt5 + ConstantinopleFixToIstanbulAt5 + Berlin + BerlinToLondonAt5 + London + ArrowGlacier + GrayGlacier + Merged + + LogLevel* = enum + Silent + Error + Warn + Info + Debug + Detail + + T8NExitCode* = distinct int + + T8NError* = object of CatchableError + exitCode*: T8NExitCode + + Ommer* = object + delta*: uint64 + address*: EthAddress + + EnvStruct* = object + currentCoinbase*: EthAddress + currentDifficulty*: Option[DifficultyInt] + currentRandom*: Option[Hash256] + parentDifficulty*: Option[DifficultyInt] + currentGasLimit*: GasInt + currentNumber*: BlockNumber + currentTimestamp*: EthTime + parentTimestamp*: EthTime + blockHashes*: Table[uint64, Hash256] + ommers*: seq[Ommer] + currentBaseFee*: Option[UInt256] + parentUncleHash*: Hash256 + + TransContext* = object + alloc*: GenesisAlloc + txs*: seq[Transaction] + env*: EnvStruct + + RejectedTx* = object + index*: int + error*: string + + TxReceipt* = object + txType*: TxType + root*: Hash256 + status*: bool + cumulativeGasUsed*: GasInt + logsBloom*: BloomFilter + logs*: seq[Log] + transactionHash*: Hash256 + contractAddress*: EthAddress + gasUsed*: GasInt + blockHash*: Hash256 + transactionIndex*: int + + # ExecutionResult contains the execution status after running a state test, any + # error that might have occurred and a dump of the final state if requested. + ExecutionResult* = object + stateRoot*: Hash256 + txRoot*: Hash256 + receiptsRoot*: Hash256 + logsHash*: Hash256 + bloom*: BloomFilter + receipts*: seq[TxReceipt] + rejected*: seq[RejectedTx] + currentDifficulty*: Option[DifficultyInt] + gasUsed*: GasInt + +const + ErrorEVM* = 2.T8NExitCode + ErrorConfig* = 3.T8NExitCode + ErrorMissingBlockhash* = 4.T8NExitCode + + ErrorJson* = 10.T8NExitCode + ErrorIO* = 11.T8NExitCode + ErrorRlp* = 12.T8NExitCode + +proc newError*(code: T8NExitCode, msg: string): ref T8NError = + (ref T8NError)(exitCode: code, msg: msg)