Premix-related changes
- build all tools using the Makefile and place them in ./build - add copyright header to Makefile - premix/readme.md edited to fix some errors, improve descriptions and document Makefile usage - link the Premix documentation in the main README.md - also build `hunter` in the nimble tests - refactored the file and directory search so all debugging tools can be run from the top-level dir like this: `./build/<tool> ...` - write all JSON debugging data in the current directory - add JSON files generated in the top-level dir to .gitignore - Nimbus now exits with an exception after dumping debug data and running `premix` on it
This commit is contained in:
Ștefan Talpalaru
zah
parent
85b979ec32
commit
ede45648e7
|
|
@ -18,3 +18,6 @@
|
|||
*.dll
|
||||
|
||||
VMTests.md
|
||||
/debug*.json
|
||||
/block*.json
|
||||
|
||||
|
|
|
|||
20
Makefile
20
Makefile
|
|
@ -1,3 +1,10 @@
|
|||
# Copyright (c) 2018-2019 Status Research & Development GmbH. Licensed under
|
||||
# either of:
|
||||
# - Apache License, version 2.0
|
||||
# - MIT license
|
||||
# at your option. This file may not be copied, modified, or distributed except
|
||||
# according to those terms.
|
||||
|
||||
SHELL := bash # the shell used internally by "make"
|
||||
GIT_CLONE := git clone --quiet --recurse-submodules
|
||||
GIT_PULL := git pull --recurse-submodules
|
||||
|
|
@ -57,17 +64,22 @@ GITHUB_REPOS := \
|
|||
# "foo/bar" -> "$(REPOS_DIR)/bar"
|
||||
REPOS := $(addprefix $(REPOS_DIR)/, $(foreach github_repo,$(GITHUB_REPOS),$(word 2,$(subst /, ,$(github_repo)))))
|
||||
|
||||
.PHONY: all deps github-ssh build-nim update status ntags ctags nimbus test clean mrproper fetch-dlls beacon_node validator_keygen clean_eth2_network_simulation_files eth2_network_simulation
|
||||
.PHONY: all premix persist debug dumper hunter deps github-ssh build-nim update status ntags ctags nimbus test clean mrproper fetch-dlls beacon_node validator_keygen clean_eth2_network_simulation_files eth2_network_simulation
|
||||
|
||||
# default target, because it's the first one that doesn't start with '.'
|
||||
all: nimbus
|
||||
all: premix persist debug dumper hunter nimbus
|
||||
|
||||
# debugging tools
|
||||
premix persist debug dumper hunter: | build deps
|
||||
$(ENV_SCRIPT) nim c -o:build/$@ premix/$@.nim && \
|
||||
echo -e "\nThe binary is in './build/$@'.\n"
|
||||
|
||||
#- "--nimbleDir" is ignored for custom tasks: https://github.com/nim-lang/nimble/issues/495
|
||||
# so we can't run `nimble ... nimbus` or `nimble ... test`. We have to duplicate those custom tasks here.
|
||||
#- a phony target, because teaching `make` how to do conditional recompilation of Nim projects is too complicated
|
||||
nimbus: | build deps
|
||||
$(ENV_SCRIPT) nim c -o:build/nimbus nimbus/nimbus.nim && \
|
||||
echo -e "\nThe binary is in './build/nimbus'."
|
||||
echo -e "\nThe binary is in './build/nimbus'.\n"
|
||||
|
||||
# dir
|
||||
build:
|
||||
|
|
@ -94,7 +106,7 @@ $(REPOS):
|
|||
$(GIT_CLONE) https://github.com/$(filter %/$(PROJ_NAME),$(GITHUB_REPOS)) $@ && \
|
||||
rm -rf $(NIMBLE_DIR)
|
||||
|
||||
#- clones and builds the Nim compiler and Nimble
|
||||
# clones and builds the Nim compiler and Nimble
|
||||
$(NIM_DIR):
|
||||
$(GIT_CLONE) --depth 1 https://github.com/status-im/Nim $@
|
||||
$(BUILD_NIM)
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ Join the Status community chats:
|
|||
|
||||
|
||||
## Rationale
|
||||
|
||||
[Nimbus: an Ethereum 2.0 Sharding Client](https://our.status.im/nimbus-for-newbies/). The code in this repository is currently focusing on Ethereum 1.0 feature parity, while all 2.0 research and development is happening in parallel in [nim-beacon-chain](https://github.com/status-im/nim-beacon-chain). The two repositories are expected to merge in Q1 2019.
|
||||
|
||||
## Development Updates
|
||||
|
|
@ -95,7 +96,9 @@ You can now follow those instructions in the previous section by replacing `make
|
|||
|
||||
- you can switch the DB backend with a Nim compiler define:
|
||||
`-d:nimbus_db_backend=...` where the (case-insensitive) value is one of
|
||||
"rocksdb" (the default), "sqlite", "lmdb".
|
||||
"rocksdb" (the default), "sqlite", "lmdb"
|
||||
|
||||
- the Premix debugging tools are [documented separately](premix/readme.md)
|
||||
|
||||
### Troubleshooting
|
||||
|
||||
|
|
@ -133,7 +136,7 @@ or
|
|||
|
||||
* Apache License, Version 2.0, ([LICENSE-APACHEv2](LICENSE-APACHEv2) or http://www.apache.org/licenses/LICENSE-2.0)
|
||||
|
||||
at your option. This file may not be copied, modified, or distributed except according to those terms.
|
||||
at your option. These files may not be copied, modified, or distributed except according to those terms.
|
||||
|
||||
[1]: https://github.com/status-im/nimbus/wiki/Understanding-and-debugging-Nimbus-EVM-JSON-tests
|
||||
[2]: https://github.com/status-im/nimbus/wiki/Debugging-state-reconstruction
|
||||
|
|
|
|||
|
|
@ -43,6 +43,7 @@ task test, "Run tests":
|
|||
exec "nim c premix/persist"
|
||||
exec "nim c premix/debug"
|
||||
exec "nim c premix/dumper"
|
||||
exec "nim c premix/hunter"
|
||||
|
||||
task nimbus, "Build Nimbus":
|
||||
buildBinary "nimbus", "nimbus/"
|
||||
|
|
|
|||
|
|
@ -13,28 +13,35 @@ else:
|
|||
premixExecutable = "premix"
|
||||
browserLauncher = "xdg-open"
|
||||
|
||||
proc getPremixDir(): (bool, string) =
|
||||
var premixDir = [
|
||||
proc getFileDir*(file: string): string =
|
||||
var searchDirs = [
|
||||
"." ,
|
||||
".." & DirSep & "premix"
|
||||
"." / "build" ,
|
||||
"." / "premix"
|
||||
]
|
||||
|
||||
for c in premixDir:
|
||||
if fileExists(c & DirSep & premixExecutable):
|
||||
return (true, c)
|
||||
for dir in searchDirs:
|
||||
if fileExists(dir / file):
|
||||
return dir
|
||||
|
||||
result = (false, ".")
|
||||
result = ""
|
||||
|
||||
proc getFilePath(file: string): string =
|
||||
let dir = getFileDir(file)
|
||||
if dir.len > 0:
|
||||
return dir / file
|
||||
else:
|
||||
return ""
|
||||
|
||||
proc launchPremix*(fileName: string, metaData: JsonNode) =
|
||||
let
|
||||
(premixAvailable, premixDir) = getPremixDir()
|
||||
premixExe = premixDir & DirSep & premixExecutable
|
||||
let premixExe = getFilePath(premixExecutable)
|
||||
|
||||
writeFile(premixDir & DirSep & fileName, metaData.pretty)
|
||||
writeFile(fileName, metaData.pretty)
|
||||
|
||||
if premixAvailable:
|
||||
if execCmd(premixExe & " " & premixDir & DirSep & fileName) == 0:
|
||||
if execCmd(browserLauncher & " " & premixDir & DirSep & "index.html") != 0:
|
||||
if premixExe.len > 0:
|
||||
if execCmd(premixExe & " " & fileName) == 0:
|
||||
if execCmd(browserLauncher & " " & getFilePath("index.html")) != 0:
|
||||
echo "failed to launch default browser"
|
||||
else:
|
||||
echo "failed to execute premix debugging tool"
|
||||
echo "failed to execute the premix debugging tool"
|
||||
|
||||
|
|
|
|||
|
|
@ -48,6 +48,7 @@ method persistBlocks*(c: Chain, headers: openarray[BlockHeader], bodies: openarr
|
|||
when not defined(release):
|
||||
if validationResult == ValidationResult.Error:
|
||||
dumpDebuggingMetaData(c.db, headers[i], bodies[i], vmState.receipts)
|
||||
raise newException(Exception, "Validation error. Debugging metadata dumped.")
|
||||
|
||||
if validationResult != ValidationResult.OK:
|
||||
result = validationResult
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
import
|
||||
json, strutils, stint, parser, downloader,
|
||||
../nimbus/tracer, chronicles, eth_common,
|
||||
js_tracer
|
||||
json, strutils, os,
|
||||
stint, chronicles, eth_common,
|
||||
../nimbus/tracer, ../nimbus/launcher,
|
||||
./js_tracer, ./parser, ./downloader
|
||||
|
||||
proc fakeAlloc(n: JsonNode) =
|
||||
const
|
||||
|
|
@ -63,7 +64,7 @@ proc generatePremixData*(nimbus, geth: JsonNode) =
|
|||
}
|
||||
|
||||
var data = "var premixData = " & premixData.pretty & "\n"
|
||||
writeFile("premixData.js", data)
|
||||
writeFile(getFileDir("index.html") / "premixData.js", data)
|
||||
|
||||
proc hasInternalTx(tx: Transaction, blockNumber: Uint256): bool =
|
||||
let
|
||||
|
|
|
|||
140
premix/readme.md
140
premix/readme.md
|
|
@ -1,111 +1,139 @@
|
|||
# Premix
|
||||
|
||||
> Premix was **pre**mium(subsidized gasoline) **mix**ed [with lubricant oil]
|
||||
used for two stroke internal combustion engines and it tends to produce a lot
|
||||
of smoke.
|
||||
> Premix is **pre**mium gasoline **mix**ed with lubricant oil and it is
|
||||
used in two-stroke internal combustion engines. It tends to produce a lot of
|
||||
smoke.
|
||||
|
||||
Today premix is a block validation debugging tool targeting at nimbus ethereum
|
||||
client. Premix will query transaction execution steps from other ethereum
|
||||
clients and compare it with nimbus'.
|
||||
This Premix is a block validation debugging tool for the Nimbus Ethereum
|
||||
client. Premix will query transaction execution steps from other Ethereum
|
||||
clients and compare them with those generated by Nimbus. It will then produce a
|
||||
web page to present comparison results that can be inspected by the developer
|
||||
to pinpoint the faulty instruction.
|
||||
|
||||
Premix then will produce a web page to present comparison result that can be
|
||||
inspected by developer to pinpoint where the faulty instruction located.
|
||||
|
||||
Premix will also produce a test case for the specific problematic transaction
|
||||
complete with snapshot database to execute transaction validation in isolation.
|
||||
This test case then can be integrated with nimbus project test suite.
|
||||
Premix will also produce a test case for the specific problematic transaction,
|
||||
complete with a database snapshot to execute transaction validation in
|
||||
isolation. This test case can then be integrated with the Nimbus project's test
|
||||
suite.
|
||||
|
||||
|
||||
|
||||
## Requirements
|
||||
Before you start to use premix debugging tool there are several things you need to prepare.
|
||||
The first one is you need to install `geth` from [source](https://github.com/ethereum/go-ethereum/releases)
|
||||
or [binary](https://ethereum.github.io/go-ethereum/downloads/). Minimum required geth version is 1.8.18.
|
||||
Then you can run it with this command:
|
||||
|
||||
Before you can use the Premix debugging tool there are several things you need
|
||||
to prepare. The first requirement is a recent version of `geth` installed from
|
||||
[source](https://github.com/ethereum/go-ethereum/releases) or
|
||||
[binary](https://ethereum.github.io/go-ethereum/downloads/). The minimum
|
||||
required version is 1.8.18. Afterwards, you can run it with this command:
|
||||
|
||||
```bash
|
||||
geth --rpc --rpcapi eth,debug --syncmode full --gcmode=archive
|
||||
```
|
||||
|
||||
You need to run it until it synced past the problematic block you want to debug.
|
||||
After that you can stop it by pressing `CTRL-C` and rerun it with additional
|
||||
flags `--maxpeers 0` if you want it to stop syncing or let it run as is if you want keep syncing.
|
||||
You need to run it until it fully syncs past the problematic block you want to
|
||||
debug (note that it will first do a fast sync of all blocks before going back
|
||||
to do the full sync). After that, you can stop it by pressing `CTRL-C` and
|
||||
rerun it with the additional flag `--maxpeers 0` if you want it to stop syncing
|
||||
- or just let it run as is if you want to keep syncing.
|
||||
|
||||
The next requirement is you should build Nimbus and Premix with latest dependencies:
|
||||
The next requirement is building Nimbus and Premix:
|
||||
|
||||
```bash
|
||||
nim c nimbus/nimbus
|
||||
nim c premix/premix
|
||||
# in the top-level directory:
|
||||
make
|
||||
```
|
||||
|
||||
After you successfully build nimbus and premix, you can run nimbus with this command.
|
||||
After that, you can run Nimbus with this command:
|
||||
|
||||
```bash
|
||||
nimbus --prune:archive
|
||||
./build/nimbus --prune:archive --port:30304
|
||||
```
|
||||
|
||||
Nimbus will try to sync up to problematic block then it will stop and execute Premix.
|
||||
Premix then will launch browser to display a report page. If premix failed to open your default browser,
|
||||
you can see the report page by opening `premix/index.html`.
|
||||
Nimbus will try to sync up to the problematic block, then stop and execute
|
||||
Premix which will then load a report page in your default browser. If it fails
|
||||
to do that, you can see the report page by manually opening
|
||||
`premix/index.html`.
|
||||
|
||||
In the browser, you can try to navigate tracing result and find where the problem/bug is.
|
||||
In your browser, you can explore the tracing result and find where the problem is.
|
||||
|
||||
## Tools
|
||||
|
||||
* Premix
|
||||
|
||||
`Premix` is the main tool in this tool set. It produce data that can be viewed with browser and
|
||||
debug data that can be consumed by `debug` tool. `Premix` consume data produced either by `nimbus`, `persist`, or `dumper`.
|
||||
You can run `Premix` manually using this command: `premix debugxxx.json`
|
||||
Premix is the main debugging tool. It produces reports that can be viewed in
|
||||
a browser and serialised debug data that can be consumed by the `debug` tool.
|
||||
Premix consumes data produced by either `nimbus`, `persist`, or `dumper`.
|
||||
|
||||
You can run it manually using this command:
|
||||
|
||||
```bash
|
||||
./build/premix debug*.json
|
||||
```
|
||||
|
||||
* Persist
|
||||
|
||||
Because nimbus p2p layer still contains bugs, you may become impatient when try to syncing blocks.
|
||||
In `/premix` directory, you can find a `persist.nim` tool.
|
||||
It will help you to sync relatively quicker because it will bypass p2p layer and download blocks from `geth` via `rpc-api`.
|
||||
Because the Nimbus P2P layer still contains bugs, you may become impatient when
|
||||
trying to sync blocks. In the `./premix` directory, you can find a `persist`
|
||||
tool. It will help you sync relatively quicker because it will bypass the P2P
|
||||
layer and download blocks from `geth` via `rpc-api`.
|
||||
|
||||
When it encounter problematic block during syncing, it will stop and produce debugging data like nimbus does.
|
||||
When it encounters a problematic block during syncing, it will stop and produce
|
||||
debugging data just like Nimbus does.
|
||||
|
||||
```bash
|
||||
nim c -r premix/persist [--dataDir:your_database_directory] [--head: blockNumber] [--maxBlocks: number] [--numCommits: number]
|
||||
./build/persist [--dataDir:your_database_directory] [--head: blockNumber] [--maxBlocks: number] [--numCommits: number]
|
||||
```
|
||||
|
||||
* Debug
|
||||
|
||||
Premix debugging tool also produce a set of debugging meta data that you can use to quickly
|
||||
find the bug without the need to run p2p layer or any other unnecessary code.
|
||||
In `/premix` directory you'll find `debug.nim` tool that you can use to execute
|
||||
this debug meta data and you'll only need to work with one block and one transaction
|
||||
at a time instead of multiple confusing blocks or transactions.
|
||||
In the same `./premix` directory you'll find the `debug` tool that you can use
|
||||
to process previously generated debugging info in order to work with one block
|
||||
and one transaction at a time instead of multiple confusing blocks and
|
||||
transactions.
|
||||
|
||||
```bash
|
||||
nim c -r premix/debug blockxxx.json
|
||||
./build/debug block*.json
|
||||
```
|
||||
|
||||
`blockxxx.json` contains database snapshot needed to debug a single block produced by Premix tool.
|
||||
where `block*.json` contains the database snapshot needed to debug a single
|
||||
block produced by the Premix tool.
|
||||
|
||||
* Dumper
|
||||
|
||||
`Dumper` was designed specifically to produce debugging data that can be further processed by `Premix` from
|
||||
information already stored in database. It will create a single block tracing information if the block already persisted.
|
||||
If you want to produce problematic block debug data, better to use `Persist` tool. `Dumper` produced data
|
||||
usually used to debug features of `Premix` and it's report page logic.
|
||||
Dumper was designed specifically to produce debugging data that can be further
|
||||
processed by Premix from information already stored in database. It will create
|
||||
tracing information for a single block if that block has been already
|
||||
persisted.
|
||||
|
||||
If you want to generate debugging data, it's better to use the Persist tool.
|
||||
The data generated by Dumper is usually used to debug Premix features in
|
||||
general and the report page logic in particular.
|
||||
|
||||
```bash
|
||||
usage: dumper [--datadir:your_path] --head:blockNumber
|
||||
# usage:
|
||||
./build/dumper [--datadir:your_path] --head:blockNumber
|
||||
```
|
||||
|
||||
* Hunter
|
||||
|
||||
`Hunter` purpose is to tracking down problematic block and create debugging meta data associated with that block.
|
||||
`Hunter` will will not access your on disk database, it has it's own prestate construction code.
|
||||
`Hunter` will download all it needed data from geth, make sure your geth version at least 1.8.18.
|
||||
`Hunter` depends on `eth_getProof`[(EIP1186)](https://github.com/ethereum/EIPs/issues/1186).
|
||||
Make sure your installed geth support this functionality(lower version don't have this implemented).
|
||||
Hunter's purpose is to track down problematic blocks and create debugging info
|
||||
associated with them. It will not access your on-disk database, because it has
|
||||
its own prestate construction code.
|
||||
|
||||
Hunter will download all it needs from geth, just make sure your geth version
|
||||
is at least 1.8.18.
|
||||
|
||||
Hunter depends on
|
||||
`eth_getProof`[(EIP1186)](https://github.com/ethereum/EIPs/issues/1186). Make
|
||||
sure your installed `geth` supports this functionality (older versions don't
|
||||
have this implemented).
|
||||
|
||||
```bash
|
||||
usage: hunter --head:blockNumber --maxBlocks:number
|
||||
# usage:
|
||||
./build/hunter --head:blockNumber --maxBlocks:number
|
||||
```
|
||||
|
||||
`blockNumber` is the starting block where hunting begin.
|
||||
`maxBlocks` is the number of problematic blocks you want to capture before stop hunting.
|
||||
`blockNumber` is the starting block where the hunt begins.
|
||||
|
||||
`maxBlocks` is the number of problematic blocks you want to capture before
|
||||
stopping the hunt.
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue