diff --git a/.gitignore b/.gitignore index f73b9bfce..8a69d8ee6 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,6 @@ *.dll VMTests.md +/debug*.json +/block*.json + diff --git a/Makefile b/Makefile index 84026f5c1..b17cd19e6 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,10 @@ +# Copyright (c) 2018-2019 Status Research & Development GmbH. Licensed under +# either of: +# - Apache License, version 2.0 +# - MIT license +# at your option. This file may not be copied, modified, or distributed except +# according to those terms. + SHELL := bash # the shell used internally by "make" GIT_CLONE := git clone --quiet --recurse-submodules GIT_PULL := git pull --recurse-submodules @@ -57,17 +64,22 @@ GITHUB_REPOS := \ # "foo/bar" -> "$(REPOS_DIR)/bar" REPOS := $(addprefix $(REPOS_DIR)/, $(foreach github_repo,$(GITHUB_REPOS),$(word 2,$(subst /, ,$(github_repo))))) -.PHONY: all deps github-ssh build-nim update status ntags ctags nimbus test clean mrproper fetch-dlls beacon_node validator_keygen clean_eth2_network_simulation_files eth2_network_simulation +.PHONY: all premix persist debug dumper hunter deps github-ssh build-nim update status ntags ctags nimbus test clean mrproper fetch-dlls beacon_node validator_keygen clean_eth2_network_simulation_files eth2_network_simulation # default target, because it's the first one that doesn't start with '.' -all: nimbus +all: premix persist debug dumper hunter nimbus + +# debugging tools +premix persist debug dumper hunter: | build deps + $(ENV_SCRIPT) nim c -o:build/$@ premix/$@.nim && \ + echo -e "\nThe binary is in './build/$@'.\n" #- "--nimbleDir" is ignored for custom tasks: https://github.com/nim-lang/nimble/issues/495 # so we can't run `nimble ... nimbus` or `nimble ... test`. We have to duplicate those custom tasks here. #- a phony target, because teaching `make` how to do conditional recompilation of Nim projects is too complicated nimbus: | build deps $(ENV_SCRIPT) nim c -o:build/nimbus nimbus/nimbus.nim && \ - echo -e "\nThe binary is in './build/nimbus'." + echo -e "\nThe binary is in './build/nimbus'.\n" # dir build: @@ -94,7 +106,7 @@ $(REPOS): $(GIT_CLONE) https://github.com/$(filter %/$(PROJ_NAME),$(GITHUB_REPOS)) $@ && \ rm -rf $(NIMBLE_DIR) -#- clones and builds the Nim compiler and Nimble +# clones and builds the Nim compiler and Nimble $(NIM_DIR): $(GIT_CLONE) --depth 1 https://github.com/status-im/Nim $@ $(BUILD_NIM) diff --git a/README.md b/README.md index 24b0ff3fc..156d1ecd0 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ Join the Status community chats: ## Rationale + [Nimbus: an Ethereum 2.0 Sharding Client](https://our.status.im/nimbus-for-newbies/). The code in this repository is currently focusing on Ethereum 1.0 feature parity, while all 2.0 research and development is happening in parallel in [nim-beacon-chain](https://github.com/status-im/nim-beacon-chain). The two repositories are expected to merge in Q1 2019. ## Development Updates @@ -95,7 +96,9 @@ You can now follow those instructions in the previous section by replacing `make - you can switch the DB backend with a Nim compiler define: `-d:nimbus_db_backend=...` where the (case-insensitive) value is one of - "rocksdb" (the default), "sqlite", "lmdb". + "rocksdb" (the default), "sqlite", "lmdb" + +- the Premix debugging tools are [documented separately](premix/readme.md) ### Troubleshooting @@ -133,7 +136,7 @@ or * Apache License, Version 2.0, ([LICENSE-APACHEv2](LICENSE-APACHEv2) or http://www.apache.org/licenses/LICENSE-2.0) -at your option. This file may not be copied, modified, or distributed except according to those terms. +at your option. These files may not be copied, modified, or distributed except according to those terms. [1]: https://github.com/status-im/nimbus/wiki/Understanding-and-debugging-Nimbus-EVM-JSON-tests [2]: https://github.com/status-im/nimbus/wiki/Debugging-state-reconstruction diff --git a/nimbus.nimble b/nimbus.nimble index 33f1267bb..be50a9d2f 100644 --- a/nimbus.nimble +++ b/nimbus.nimble @@ -43,6 +43,7 @@ task test, "Run tests": exec "nim c premix/persist" exec "nim c premix/debug" exec "nim c premix/dumper" + exec "nim c premix/hunter" task nimbus, "Build Nimbus": buildBinary "nimbus", "nimbus/" diff --git a/nimbus/launcher.nim b/nimbus/launcher.nim index 20d912f67..3563f60d1 100644 --- a/nimbus/launcher.nim +++ b/nimbus/launcher.nim @@ -13,28 +13,35 @@ else: premixExecutable = "premix" browserLauncher = "xdg-open" -proc getPremixDir(): (bool, string) = - var premixDir = [ +proc getFileDir*(file: string): string = + var searchDirs = [ "." , - ".." & DirSep & "premix" + "." / "build" , + "." / "premix" ] - for c in premixDir: - if fileExists(c & DirSep & premixExecutable): - return (true, c) + for dir in searchDirs: + if fileExists(dir / file): + return dir - result = (false, ".") + result = "" + +proc getFilePath(file: string): string = + let dir = getFileDir(file) + if dir.len > 0: + return dir / file + else: + return "" proc launchPremix*(fileName: string, metaData: JsonNode) = - let - (premixAvailable, premixDir) = getPremixDir() - premixExe = premixDir & DirSep & premixExecutable + let premixExe = getFilePath(premixExecutable) - writeFile(premixDir & DirSep & fileName, metaData.pretty) + writeFile(fileName, metaData.pretty) - if premixAvailable: - if execCmd(premixExe & " " & premixDir & DirSep & fileName) == 0: - if execCmd(browserLauncher & " " & premixDir & DirSep & "index.html") != 0: + if premixExe.len > 0: + if execCmd(premixExe & " " & fileName) == 0: + if execCmd(browserLauncher & " " & getFilePath("index.html")) != 0: echo "failed to launch default browser" else: - echo "failed to execute premix debugging tool" + echo "failed to execute the premix debugging tool" + diff --git a/nimbus/p2p/chain.nim b/nimbus/p2p/chain.nim index 26b34d2b4..9ff32f655 100644 --- a/nimbus/p2p/chain.nim +++ b/nimbus/p2p/chain.nim @@ -48,6 +48,7 @@ method persistBlocks*(c: Chain, headers: openarray[BlockHeader], bodies: openarr when not defined(release): if validationResult == ValidationResult.Error: dumpDebuggingMetaData(c.db, headers[i], bodies[i], vmState.receipts) + raise newException(Exception, "Validation error. Debugging metadata dumped.") if validationResult != ValidationResult.OK: result = validationResult diff --git a/premix/premixcore.nim b/premix/premixcore.nim index d28899dde..a1a5518b5 100644 --- a/premix/premixcore.nim +++ b/premix/premixcore.nim @@ -1,7 +1,8 @@ import - json, strutils, stint, parser, downloader, - ../nimbus/tracer, chronicles, eth_common, - js_tracer + json, strutils, os, + stint, chronicles, eth_common, + ../nimbus/tracer, ../nimbus/launcher, + ./js_tracer, ./parser, ./downloader proc fakeAlloc(n: JsonNode) = const @@ -63,7 +64,7 @@ proc generatePremixData*(nimbus, geth: JsonNode) = } var data = "var premixData = " & premixData.pretty & "\n" - writeFile("premixData.js", data) + writeFile(getFileDir("index.html") / "premixData.js", data) proc hasInternalTx(tx: Transaction, blockNumber: Uint256): bool = let diff --git a/premix/readme.md b/premix/readme.md index c8ae4afc3..c3413cbdd 100644 --- a/premix/readme.md +++ b/premix/readme.md @@ -1,111 +1,139 @@ # Premix -> Premix was **pre**mium(subsidized gasoline) **mix**ed [with lubricant oil] -used for two stroke internal combustion engines and it tends to produce a lot -of smoke. +> Premix is **pre**mium gasoline **mix**ed with lubricant oil and it is +used in two-stroke internal combustion engines. It tends to produce a lot of +smoke. -Today premix is a block validation debugging tool targeting at nimbus ethereum -client. Premix will query transaction execution steps from other ethereum -clients and compare it with nimbus'. +This Premix is a block validation debugging tool for the Nimbus Ethereum +client. Premix will query transaction execution steps from other Ethereum +clients and compare them with those generated by Nimbus. It will then produce a +web page to present comparison results that can be inspected by the developer +to pinpoint the faulty instruction. -Premix then will produce a web page to present comparison result that can be -inspected by developer to pinpoint where the faulty instruction located. - -Premix will also produce a test case for the specific problematic transaction -complete with snapshot database to execute transaction validation in isolation. -This test case then can be integrated with nimbus project test suite. +Premix will also produce a test case for the specific problematic transaction, +complete with a database snapshot to execute transaction validation in +isolation. This test case can then be integrated with the Nimbus project's test +suite. ![screenshot](assets/images/premix_screenshot.png) ## Requirements -Before you start to use premix debugging tool there are several things you need to prepare. -The first one is you need to install `geth` from [source](https://github.com/ethereum/go-ethereum/releases) -or [binary](https://ethereum.github.io/go-ethereum/downloads/). Minimum required geth version is 1.8.18. -Then you can run it with this command: + +Before you can use the Premix debugging tool there are several things you need +to prepare. The first requirement is a recent version of `geth` installed from +[source](https://github.com/ethereum/go-ethereum/releases) or +[binary](https://ethereum.github.io/go-ethereum/downloads/). The minimum +required version is 1.8.18. Afterwards, you can run it with this command: ```bash geth --rpc --rpcapi eth,debug --syncmode full --gcmode=archive ``` -You need to run it until it synced past the problematic block you want to debug. -After that you can stop it by pressing `CTRL-C` and rerun it with additional -flags `--maxpeers 0` if you want it to stop syncing or let it run as is if you want keep syncing. +You need to run it until it fully syncs past the problematic block you want to +debug (note that it will first do a fast sync of all blocks before going back +to do the full sync). After that, you can stop it by pressing `CTRL-C` and +rerun it with the additional flag `--maxpeers 0` if you want it to stop syncing +- or just let it run as is if you want to keep syncing. -The next requirement is you should build Nimbus and Premix with latest dependencies: +The next requirement is building Nimbus and Premix: ```bash -nim c nimbus/nimbus -nim c premix/premix +# in the top-level directory: +make ``` -After you successfully build nimbus and premix, you can run nimbus with this command. +After that, you can run Nimbus with this command: ```bash -nimbus --prune:archive +./build/nimbus --prune:archive --port:30304 ``` -Nimbus will try to sync up to problematic block then it will stop and execute Premix. -Premix then will launch browser to display a report page. If premix failed to open your default browser, -you can see the report page by opening `premix/index.html`. +Nimbus will try to sync up to the problematic block, then stop and execute +Premix which will then load a report page in your default browser. If it fails +to do that, you can see the report page by manually opening +`premix/index.html`. -In the browser, you can try to navigate tracing result and find where the problem/bug is. +In your browser, you can explore the tracing result and find where the problem is. ## Tools * Premix -`Premix` is the main tool in this tool set. It produce data that can be viewed with browser and -debug data that can be consumed by `debug` tool. `Premix` consume data produced either by `nimbus`, `persist`, or `dumper`. -You can run `Premix` manually using this command: `premix debugxxx.json` +Premix is the main debugging tool. It produces reports that can be viewed in +a browser and serialised debug data that can be consumed by the `debug` tool. +Premix consumes data produced by either `nimbus`, `persist`, or `dumper`. + +You can run it manually using this command: + +```bash +./build/premix debug*.json +``` * Persist -Because nimbus p2p layer still contains bugs, you may become impatient when try to syncing blocks. -In `/premix` directory, you can find a `persist.nim` tool. -It will help you to sync relatively quicker because it will bypass p2p layer and download blocks from `geth` via `rpc-api`. +Because the Nimbus P2P layer still contains bugs, you may become impatient when +trying to sync blocks. In the `./premix` directory, you can find a `persist` +tool. It will help you sync relatively quicker because it will bypass the P2P +layer and download blocks from `geth` via `rpc-api`. -When it encounter problematic block during syncing, it will stop and produce debugging data like nimbus does. +When it encounters a problematic block during syncing, it will stop and produce +debugging data just like Nimbus does. ```bash -nim c -r premix/persist [--dataDir:your_database_directory] [--head: blockNumber] [--maxBlocks: number] [--numCommits: number] +./build/persist [--dataDir:your_database_directory] [--head: blockNumber] [--maxBlocks: number] [--numCommits: number] ``` * Debug -Premix debugging tool also produce a set of debugging meta data that you can use to quickly -find the bug without the need to run p2p layer or any other unnecessary code. -In `/premix` directory you'll find `debug.nim` tool that you can use to execute -this debug meta data and you'll only need to work with one block and one transaction -at a time instead of multiple confusing blocks or transactions. +In the same `./premix` directory you'll find the `debug` tool that you can use +to process previously generated debugging info in order to work with one block +and one transaction at a time instead of multiple confusing blocks and +transactions. ```bash -nim c -r premix/debug blockxxx.json +./build/debug block*.json ``` -`blockxxx.json` contains database snapshot needed to debug a single block produced by Premix tool. +where `block*.json` contains the database snapshot needed to debug a single +block produced by the Premix tool. * Dumper -`Dumper` was designed specifically to produce debugging data that can be further processed by `Premix` from -information already stored in database. It will create a single block tracing information if the block already persisted. -If you want to produce problematic block debug data, better to use `Persist` tool. `Dumper` produced data -usually used to debug features of `Premix` and it's report page logic. +Dumper was designed specifically to produce debugging data that can be further +processed by Premix from information already stored in database. It will create +tracing information for a single block if that block has been already +persisted. + +If you want to generate debugging data, it's better to use the Persist tool. +The data generated by Dumper is usually used to debug Premix features in +general and the report page logic in particular. ```bash -usage: dumper [--datadir:your_path] --head:blockNumber +# usage: +./build/dumper [--datadir:your_path] --head:blockNumber ``` * Hunter -`Hunter` purpose is to tracking down problematic block and create debugging meta data associated with that block. -`Hunter` will will not access your on disk database, it has it's own prestate construction code. -`Hunter` will download all it needed data from geth, make sure your geth version at least 1.8.18. -`Hunter` depends on `eth_getProof`[(EIP1186)](https://github.com/ethereum/EIPs/issues/1186). -Make sure your installed geth support this functionality(lower version don't have this implemented). +Hunter's purpose is to track down problematic blocks and create debugging info +associated with them. It will not access your on-disk database, because it has +its own prestate construction code. + +Hunter will download all it needs from geth, just make sure your geth version +is at least 1.8.18. + +Hunter depends on +`eth_getProof`[(EIP1186)](https://github.com/ethereum/EIPs/issues/1186). Make +sure your installed `geth` supports this functionality (older versions don't +have this implemented). ```bash -usage: hunter --head:blockNumber --maxBlocks:number +# usage: +./build/hunter --head:blockNumber --maxBlocks:number ``` -`blockNumber` is the starting block where hunting begin. -`maxBlocks` is the number of problematic blocks you want to capture before stop hunting. +`blockNumber` is the starting block where the hunt begins. + +`maxBlocks` is the number of problematic blocks you want to capture before +stopping the hunt. +