From f8fbe0ff3bbd54cf304a6fabdd40c2fa585f40b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C8=98tefan=20Talpalaru?= Date: Thu, 28 Mar 2019 16:18:59 +0100 Subject: [PATCH] Makefile that only works in a Git submodule (#210) - updated README.md - beacon_chain.nimble: accept compilation flags as params - nim.cfg: enable `--opt:speed` at the top level - simulation scripts: allow overriding GIT_ROOT and do some refactoring --- Makefile | 48 +++++++++++++++++++ README.md | 91 +++++++++++++++++++++++++----------- beacon_chain.nimble | 15 +++--- nim.cfg | 1 + tests/nim.cfg | 2 - tests/simulation/run_node.sh | 7 +-- tests/simulation/start.sh | 21 +++++---- tests/simulation/vars.sh | 21 +++++---- 8 files changed, 150 insertions(+), 56 deletions(-) create mode 100644 Makefile delete mode 100644 tests/nim.cfg diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..3391da3ab --- /dev/null +++ b/Makefile @@ -0,0 +1,48 @@ +# Copyright (c) 2019 Status Research & Development GmbH. Licensed under +# either of: +# - Apache License, version 2.0 +# - MIT license +# at your option. This file may not be copied, modified, or distributed except +# according to those terms. + +# we don't want an error here, so we can explain things later, in the sanity-checks target +-include ../../common.mk + +ENV_SCRIPT := "../../env.sh" + +TOOLS := beacon_node validator_keygen bench_bls_sig_agggregation state_sim +TOOLS_DIRS := beacon_chain benchmarks research +# comma-separated values for the "clean" target +TOOLS_CSV := $(subst $(SPACE),$(COMMA),$(TOOLS)) + +.PHONY: all sanity-checks deps test $(TOOLS) clean_eth2_network_simulation_files eth2_network_simulation + +all: | $(TOOLS) + +sanity-checks: + @ [[ "$$PWD" =~ /vendor/nim-beacon-chain$ && -e ../../Makefile && -e ../../common.mk ]] || \ + { echo "This Makefile can only be used from the corresponding Git submodule in the Nimbus repository."; exit 1; } + +deps: | sanity-checks + @+ $(MAKE) --silent -C ../../ deps + +build: + mkdir $@ + +test: | build deps + ../../nimble.sh test $(NIM_PARAMS) + +$(TOOLS): | build deps + for D in $(TOOLS_DIRS); do [ -e "$${D}/$@.nim" ] && TOOL_DIR="$${D}" && break; done && \ + $(ENV_SCRIPT) nim c $(NIM_PARAMS) -o:build/$@ "$${TOOL_DIR}/$@.nim" && \ + echo -e "\nThe binary is in './build/$@'.\n" + +clean_eth2_network_simulation_files: + rm -rf tests/simulation/data + +eth2_network_simulation: | beacon_node validator_keygen clean_eth2_network_simulation_files + SKIP_BUILDS=1 GIT_ROOT="$$PWD" BUILD_OUTPUTS_DIR="./build" tests/simulation/start.sh + +clean: + rm -rf build/{$(TOOLS_CSV),all_tests,*.exe} nimcache + diff --git a/README.md b/README.md index e8ec16b64..cea85de3d 100644 --- a/README.md +++ b/README.md @@ -17,39 +17,39 @@ You can check where the beacon chain fits in the Ethereum research ecosystem in ## Building and Testing -The beacon chain components require that you have Nim installed - the easiest way to get started is to head over to the main [Nimbus](https://github.com/status-im/nimbus/) repository and follow the build instructions there or just execute the commands below in order. +The beacon chain components need to be built with the Nim compiler - the easiest way to get started is to head over to the main [Nimbus](https://github.com/status-im/nimbus/) repository and follow the build instructions there or just execute the commands below in order. _Note: This is because this repository is actually pulled in as a dependency of Nimbus - the Ethereum 1.0 + 2.0 client - so it makes sense to start from there even if you are only interested in testing the Ethereum 2.0 side of things (contained almost entirely in this repository)._ ```bash -# Clone main nimbus repository +# Clone main nimbus repository: git clone https://github.com/status-im/nimbus.git cd nimbus -# Prep environment -make update deps +# Prep environment (assuming you have 4 CPU cores and want to take advantage of them): +make -j4 deps -# Start a shell that uses the Nimbus compile environment -./env.sh bash - -# You're now in a shell environment that has the right Nim version available. -# Head over to the vendor repo where you should have a checkout of this project +# Head over to the vendor repo where you should have a checkout of this project: cd vendor/nim-beacon-chain # You can now run the test suite: -nim c -d:release -r tests/all_tests +make test ``` ## Beacon node simulation The beacon node simulation is will create a full peer-to-peer network of beacon nodes and validators, and run the beacon chain in real time. To change network parameters such as shard and validator counts, see [start.sh](tests/simulation/start.sh). + ```bash -# Start beacon chain simulation, resuming from the previous state (if any) +# get a shell with the right environment vars set: +../../env.sh bash + +# Start the beacon chain simulation, resuming from a previous state (if any): ./tests/simulation/start.sh # if starting from Nimbus, make sure you're in vendor/nim-beacon-chain! -# Clear data from last run and restart simulation with a new genesis block -rm -rf tests/simulation/data ; ./tests/simulation/start.sh +# Clear data files from your last run and restart the simulation with a new genesis block: +rm -rf tests/simulation/data; ./tests/simulation/start.sh # Run an extra node - by default the network will launch with 9 nodes, each # hosting 10 validators. The last 10 validators are lazy bums that hid from the @@ -58,16 +58,12 @@ rm -rf tests/simulation/data ; ./tests/simulation/start.sh ./tests/simulation/run_node.sh 9 ``` -Alternatively, the Makefile flow is available from the Nimbus parent repo: +Alternatively, a Makefile-based flow is available: ```bash -# In the Nimbus repo root - -# Start beacon chain simulation, resuming from the previous state (if any) +# From "vendor/nim-beacon-chain/", +# clear all data from the last run and restart the simulation with a new genesis block: make eth2_network_simulation - -# Clear data from last run and restart simulation with a new genesis block -make clean_eth2_network_simulation_files eth2_network_simulation ``` You can also separate the output from each beacon node in its own panel, using [multitail](http://www.vanheusden.com/multitail/): @@ -77,22 +73,62 @@ USE_MULTITAIL="yes" ./tests/simulation/start.sh # OR -USE_MULTITAIL="yes" make eth2_network_simulation +make USE_MULTITAIL="yes" eth2_network_simulation ``` You can find out more about it in the [development update](https://our.status.im/nimbus-development-update-2018-12-2/). _Alternatively, fire up our [experimental Vagrant instance with Nim pre-installed](https://our.status.im/setting-up-a-local-vagrant-environment-for-nim-development/) and give us yout feedback about the process!_ +### Makefile tips and tricks for developers + +- build all those tools known to the Makefile: + +```bash +make +``` + +- build a specific tool: + +```bash +make state_sim +``` + +- you can control the Makefile's verbosity with the V variable (defaults to 1): + +```bash +make V=0 # quiet +make V=2 test # more verbose than usual +``` + +- same for the [Chronicles log level](https://github.com/status-im/nim-chronicles#chronicles_log_level): + +```bash +make LOG_LEVEL=DEBUG bench_bls_sig_agggregation # this is the default +make LOG_LEVEL=TRACE beacon_node # log everything +``` + +- pass arbitrary parameters to the Nim compiler: + +```bash +make NIMFLAGS="-d:release" +``` + +- you can freely combine those variables on the `make` command line: + +```bash +make -j8 V=0 NIMFLAGS="-d:release" USE_MULTITAIL=yes eth2_network_simulation +``` + ## State transition simulation The state transition simulator can quickly run the Beacon chain state transition function in isolation and output JSON snapshots of the state. The simulation runs without networking and blocks are processed without slot time delays. ```bash -cd research -# build and run state simulator, then display its help - -d:release speeds it -# up substantially, allowing the simulation of longer runs in reasonable time -nim c -d:release -r state_sim --help +# build and run the state simulator, then display its help ("-d:release" speeds it +# up substantially, allowing the simulation of longer runs in reasonable time) +make V=0 NIMFLAGS="-d:release" state_sim +./build/state_sim --help ``` ## Convention @@ -107,7 +143,7 @@ Nim NEP-1 recommends: - PascalCase for constants - PascalCase for types -To facilitate collaboration and comparison, Nim-beacon-chain uses the Ethereum Foundation convention. +To facilitate collaboration and comparison, nim-beacon-chain uses the Ethereum Foundation convention. ## License @@ -119,4 +155,5 @@ or * Apache License, Version 2.0, ([LICENSE-APACHEv2](LICENSE-APACHEv2) or http://www.apache.org/licenses/LICENSE-2.0) -at your option. This file may not be copied, modified, or distributed except according to those terms. +at your option. These files may not be copied, modified, or distributed except according to those terms. + diff --git a/beacon_chain.nimble b/beacon_chain.nimble index 45a7f311b..ede626503 100644 --- a/beacon_chain.nimble +++ b/beacon_chain.nimble @@ -1,3 +1,5 @@ +mode = ScriptMode.Verbose + import beacon_chain/version as ver @@ -30,14 +32,15 @@ requires "nim >= 0.19.0", "libp2p" ### Helper functions -proc test(name: string, defaultLang = "c") = +proc buildBinary(name: string, srcDir = "./", params = "", lang = "c") = if not dirExists "build": mkDir "build" - --run - switch("out", ("./build/" & name)) - switch("opt", "speed") - setCommand defaultLang, "tests/" & name & ".nim" + # allow something like "nim test --verbosity:0 --hints:off beacon_chain.nims" + var extra_params = params + for i in 2../dev/null || USE_MULTITAIL="no" -COMMANDS=() +# Kill child processes on Ctrl-C by sending SIGTERM to the whole process group, +# passing the negative PID of this shell instance to the "kill" command. +# Trap and ignore SIGTERM, so we don't kill this process along with its children. +if [ "$USE_MULTITAIL" = "no" ]; then + trap '' SIGTERM + trap "kill -- -$$" SIGINT EXIT +fi + +COMMANDS=() LAST_NODE=$(( $NUM_NODES - 1 )) for i in $(seq 0 $LAST_NODE); do @@ -80,7 +85,7 @@ for i in $(seq 0 $LAST_NODE); do done fi - CMD="$SIM_ROOT/run_node.sh $i" + CMD="${SIM_ROOT}/run_node.sh $i" if [ "$USE_MULTITAIL" != "no" ]; then if [ "$i" = "0" ]; then diff --git a/tests/simulation/vars.sh b/tests/simulation/vars.sh index a301b5ced..712a70661 100644 --- a/tests/simulation/vars.sh +++ b/tests/simulation/vars.sh @@ -6,17 +6,18 @@ uname | grep -qi mingw && PWD_CMD="pwd -W" cd $(dirname $0) SIM_ROOT="$($PWD_CMD)" -cd $(git rev-parse --show-toplevel) -GIT_ROOT="$($PWD_CMD)" -# Set a default value for the env vars usually supplied by nimbus Makefile +# Set a default value for the env vars usually supplied by a Makefile +cd $(git rev-parse --show-toplevel) +: ${GIT_ROOT:="$($PWD_CMD)"} +cd - &>/dev/null : ${SKIP_BUILDS:=""} : ${BUILD_OUTPUTS_DIR:="$GIT_ROOT/build"} -SIMULATION_DIR="$SIM_ROOT/data" -VALIDATORS_DIR="$SIM_ROOT/validators" -SNAPSHOT_FILE="$SIMULATION_DIR/state_snapshot.json" -NETWORK_METADATA_FILE="$SIMULATION_DIR/network.json" -BEACON_NODE_BIN=$BUILD_OUTPUTS_DIR/beacon_node -VALIDATOR_KEYGEN_BIN=$BUILD_OUTPUTS_DIR/validator_keygen -MASTER_NODE_ADDRESS_FILE="$SIMULATION_DIR/node-0/beacon_node.address" +SIMULATION_DIR="${SIM_ROOT}/data" +VALIDATORS_DIR="${SIM_ROOT}/validators" +SNAPSHOT_FILE="${SIMULATION_DIR}/state_snapshot.json" +NETWORK_METADATA_FILE="${SIMULATION_DIR}/network.json" +BEACON_NODE_BIN="${BUILD_OUTPUTS_DIR}/beacon_node" +VALIDATOR_KEYGEN_BIN="${BUILD_OUTPUTS_DIR}/validator_keygen" +MASTER_NODE_ADDRESS_FILE="${SIMULATION_DIR}/node-0/beacon_node.address"