Makefile that only works in a Git submodule (#210)

- updated README.md - beacon_chain.nimble: accept compilation flags as params - nim.cfg: enable `--opt:speed` at the top level - simulation scripts: allow overriding GIT_ROOT and do some refactoring
2019-03-28 16:18:59 +01:00 · 2019-03-28 16:18:59 +01:00 · f8fbe0ff3b
parent 44d9f7d6c9
commit f8fbe0ff3b
8 changed files with 150 additions and 56 deletions
--- a/48
+++ b/48
@ -0,0 +1,48 @@
+# Copyright (c) 2019 Status Research & Development GmbH. Licensed under
+# either of:
+# - Apache License, version 2.0
+# - MIT license
+# at your option. This file may not be copied, modified, or distributed except
+# according to those terms.
+
+# we don't want an error here, so we can explain things later, in the sanity-checks target
+-include ../../common.mk
+
+ENV_SCRIPT := "../../env.sh"
+
+TOOLS := beacon_node validator_keygen bench_bls_sig_agggregation state_sim
+TOOLS_DIRS := beacon_chain benchmarks research
+# comma-separated values for the "clean" target
+TOOLS_CSV := $(subst $(SPACE),$(COMMA),$(TOOLS))
+
+.PHONY: all sanity-checks deps test $(TOOLS) clean_eth2_network_simulation_files eth2_network_simulation
+
+all: | $(TOOLS)
+
+sanity-checks:
+	@ [[ "$$PWD" =~ /vendor/nim-beacon-chain$ && -e ../../Makefile && -e ../../common.mk ]] || \
+		{ echo "This Makefile can only be used from the corresponding Git submodule in the Nimbus repository."; exit 1; }
+
+deps: | sanity-checks
+	@+ $(MAKE) --silent -C ../../ deps
+
+build:
+	mkdir $@
+
+test: | build deps
+	../../nimble.sh test $(NIM_PARAMS)
+
+$(TOOLS): | build deps
+	for D in $(TOOLS_DIRS); do [ -e "$${D}/$@.nim" ] && TOOL_DIR="$${D}" && break; done && \
+		$(ENV_SCRIPT) nim c $(NIM_PARAMS) -o:build/$@ "$${TOOL_DIR}/$@.nim" && \
+		echo -e "\nThe binary is in './build/$@'.\n"
+
+clean_eth2_network_simulation_files:
+	rm -rf tests/simulation/data
+
+eth2_network_simulation: | beacon_node validator_keygen clean_eth2_network_simulation_files
+	SKIP_BUILDS=1 GIT_ROOT="$$PWD" BUILD_OUTPUTS_DIR="./build" tests/simulation/start.sh
+
+clean:
+	rm -rf build/{$(TOOLS_CSV),all_tests,*.exe} nimcache
+
--- a/README.md
+++ b/README.md
@ -17,39 +17,39 @@ You can check where the beacon chain fits in the Ethereum research ecosystem in

 ## Building and Testing

-The beacon chain components require that you have Nim installed - the easiest way to get started is to head over to the main [Nimbus](https://github.com/status-im/nimbus/) repository and follow the build instructions there or just execute the commands below in order.
+The beacon chain components need to be built with the Nim compiler - the easiest way to get started is to head over to the main [Nimbus](https://github.com/status-im/nimbus/) repository and follow the build instructions there or just execute the commands below in order.

 _Note: This is because this repository is actually pulled in as a dependency of Nimbus - the Ethereum 1.0 + 2.0 client - so it makes sense to start from there even if you are only interested in testing the Ethereum 2.0 side of things (contained almost entirely in this repository)._

 ```bash
-# Clone main nimbus repository
+# Clone main nimbus repository:
 git clone https://github.com/status-im/nimbus.git
 cd nimbus

-# Prep environment
-make update deps
+# Prep environment (assuming you have 4 CPU cores and want to take advantage of them):
+make -j4 deps

-# Start a shell that uses the Nimbus compile environment
-./env.sh bash
-
-# You're now in a shell environment that has the right Nim version available.
-# Head over to the vendor repo where you should have a checkout of this project
+# Head over to the vendor repo where you should have a checkout of this project:
 cd vendor/nim-beacon-chain

 # You can now run the test suite:
-nim c -d:release -r tests/all_tests
+make test
 ```

 ## Beacon node simulation

 The beacon node simulation is will create a full peer-to-peer network of beacon nodes and validators, and run the beacon chain in real time. To change network parameters such as shard and validator counts, see [start.sh](tests/simulation/start.sh).

+
 ```bash
-# Start beacon chain simulation, resuming from the previous state (if any)
+# get a shell with the right environment vars set:
+../../env.sh bash
+
+# Start the beacon chain simulation, resuming from a previous state (if any):
 ./tests/simulation/start.sh # if starting from Nimbus, make sure you're in vendor/nim-beacon-chain!

-# Clear data from last run and restart simulation with a new genesis block
-rm -rf tests/simulation/data ; ./tests/simulation/start.sh
+# Clear data files from your last run and restart the simulation with a new genesis block:
+rm -rf tests/simulation/data; ./tests/simulation/start.sh

 # Run an extra node - by default the network will launch with 9 nodes, each
 # hosting 10 validators. The last 10 validators are lazy bums that hid from the
@ -58,16 +58,12 @@ rm -rf tests/simulation/data ; ./tests/simulation/start.sh
 ./tests/simulation/run_node.sh 9
 ```

-Alternatively, the Makefile flow is available from the Nimbus parent repo:
+Alternatively, a Makefile-based flow is available:

 ```bash
-# In the Nimbus repo root
-
-# Start beacon chain simulation, resuming from the previous state (if any)
+# From "vendor/nim-beacon-chain/",
+# clear all data from the last run and restart the simulation with a new genesis block:
 make eth2_network_simulation
-
-# Clear data from last run and restart simulation with a new genesis block
-make clean_eth2_network_simulation_files eth2_network_simulation
 ```

 You can also separate the output from each beacon node in its own panel, using [multitail](http://www.vanheusden.com/multitail/):
@ -77,22 +73,62 @@ USE_MULTITAIL="yes" ./tests/simulation/start.sh

 # OR

-USE_MULTITAIL="yes" make eth2_network_simulation
+make USE_MULTITAIL="yes" eth2_network_simulation
 ```

 You can find out more about it in the [development update](https://our.status.im/nimbus-development-update-2018-12-2/).

 _Alternatively, fire up our [experimental Vagrant instance with Nim pre-installed](https://our.status.im/setting-up-a-local-vagrant-environment-for-nim-development/) and give us yout feedback about the process!_

+### Makefile tips and tricks for developers
+
+- build all those tools known to the Makefile:
+
+```bash
+make
+```
+
+- build a specific tool:
+
+```bash
+make state_sim
+```
+
+- you can control the Makefile's verbosity with the V variable (defaults to 1):
+
+```bash
+make V=0 # quiet
+make V=2 test # more verbose than usual
+```
+
+- same for the [Chronicles log level](https://github.com/status-im/nim-chronicles#chronicles_log_level):
+
+```bash
+make LOG_LEVEL=DEBUG bench_bls_sig_agggregation # this is the default
+make LOG_LEVEL=TRACE beacon_node # log everything
+```
+
+- pass arbitrary parameters to the Nim compiler:
+
+```bash
+make NIMFLAGS="-d:release"
+```
+
+- you can freely combine those variables on the `make` command line:
+
+```bash
+make -j8 V=0 NIMFLAGS="-d:release" USE_MULTITAIL=yes eth2_network_simulation
+```
+
 ## State transition simulation

 The state transition simulator can quickly run the Beacon chain state transition function in isolation and output JSON snapshots of the state. The simulation runs without networking and blocks are processed without slot time delays.

 ```bash
-cd research
-# build and run state simulator, then display its help - -d:release speeds it
-# up substantially, allowing the simulation of longer runs in reasonable time
-nim c -d:release -r state_sim --help
+# build and run the state simulator, then display its help ("-d:release" speeds it
+# up substantially, allowing the simulation of longer runs in reasonable time)
+make V=0 NIMFLAGS="-d:release" state_sim
+./build/state_sim --help
 ```

 ## Convention
@ -107,7 +143,7 @@ Nim NEP-1 recommends:
  - PascalCase for constants
  - PascalCase for types

-To facilitate collaboration and comparison, Nim-beacon-chain uses the Ethereum Foundation convention.
+To facilitate collaboration and comparison, nim-beacon-chain uses the Ethereum Foundation convention.

 ## License

@ -119,4 +155,5 @@ or

 * Apache License, Version 2.0, ([LICENSE-APACHEv2](LICENSE-APACHEv2) or http://www.apache.org/licenses/LICENSE-2.0)

-at your option. This file may not be copied, modified, or distributed except according to those terms.
+at your option. These files may not be copied, modified, or distributed except according to those terms.
+
--- a/beacon_chain.nimble
+++ b/beacon_chain.nimble
@ -1,3 +1,5 @@
+mode = ScriptMode.Verbose
+
 import
  beacon_chain/version as ver

@ -30,14 +32,15 @@ requires "nim >= 0.19.0",
  "libp2p"

 ### Helper functions
-proc test(name: string, defaultLang = "c") =
+proc buildBinary(name: string, srcDir = "./", params = "", lang = "c") =
  if not dirExists "build":
    mkDir "build"
-  --run
-  switch("out", ("./build/" & name))
-  switch("opt", "speed")
-  setCommand defaultLang, "tests/" & name & ".nim"
+  # allow something like "nim test --verbosity:0 --hints:off beacon_chain.nims"
+  var extra_params = params
+  for i in 2..<paramCount():
+    extra_params &= " " & paramStr(i)
+  exec "nim " & lang & " --out:./build/" & name & " " & extra_params & " " & srcDir & name & ".nim"

 ### tasks
 task test, "Run all tests":
-  test "all_tests"
+  buildBinary "all_tests", "tests/", "-r -d:release -d:chronicles_log_level=ERROR"
--- a/nim.cfg
+++ b/nim.cfg
@ -1,5 +1,6 @@
 # https://github.com/nim-lang/Nim/issues/8294#issuecomment-454556051
 --threads:on
+--opt:speed

@if windows:
  -d:"chronicles_colors=NoColors"
--- a/tests/nim.cfg
+++ b/tests/nim.cfg
@ -1,2 +0,0 @@
--threads:on
-
--- a/tests/simulation/run_node.sh
+++ b/tests/simulation/run_node.sh
@ -1,12 +1,13 @@
 #!/bin/bash

-set -eux
+set -eu

 . $(dirname $0)/vars.sh
+cd "$GIT_ROOT"

-DATA_DIR=$SIMULATION_DIR/node-${1}
+DATA_DIR="${SIMULATION_DIR}/node-${1}"

-V_PREFIX="$VALIDATORS_DIR/v$(printf '%06d' ${1})"
+V_PREFIX="${VALIDATORS_DIR}/v$(printf '%06d' ${1})"
 PORT=$(printf '5%04d' ${1})

 NAT_FLAG=""
--- a/tests/simulation/start.sh
+++ b/tests/simulation/start.sh
@ -1,9 +1,6 @@
 #!/bin/bash

-set -eux
-
-# Kill children on ctrl-c
-trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM EXIT
+set -eu

 # Read in variables
 . $(dirname $0)/vars.sh
@ -13,11 +10,11 @@ trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM EXIT
 NUM_VALIDATORS=${VALIDATORS:-100}
 NUM_NODES=${NODES:-9}

-cd $SIM_ROOT
+cd "$SIM_ROOT"
 mkdir -p "$SIMULATION_DIR"
 mkdir -p "$VALIDATORS_DIR"

-cd $GIT_ROOT
+cd "$GIT_ROOT"
 mkdir -p $BUILD_OUTPUTS_DIR

 # Run with "SHARD_COUNT=4 ./start.sh" to change these
@ -66,8 +63,16 @@ fi
 MULTITAIL="${MULTITAIL:-multitail}" # to allow overriding the program name
 USE_MULTITAIL="${USE_MULTITAIL:-no}" # make it an opt-in
 type "$MULTITAIL" &>/dev/null || USE_MULTITAIL="no"
-COMMANDS=()

+# Kill child processes on Ctrl-C by sending SIGTERM to the whole process group,
+# passing the negative PID of this shell instance to the "kill" command.
+# Trap and ignore SIGTERM, so we don't kill this process along with its children.
+if [ "$USE_MULTITAIL" = "no" ]; then
+  trap '' SIGTERM
+  trap "kill -- -$$" SIGINT EXIT
+fi
+
+COMMANDS=()
 LAST_NODE=$(( $NUM_NODES - 1 ))

 for i in $(seq 0 $LAST_NODE); do
@ -80,7 +85,7 @@ for i in $(seq 0 $LAST_NODE); do
    done
  fi

-  CMD="$SIM_ROOT/run_node.sh $i"
+  CMD="${SIM_ROOT}/run_node.sh $i"

  if [ "$USE_MULTITAIL" != "no" ]; then
    if [ "$i" = "0" ]; then
--- a/tests/simulation/vars.sh
+++ b/tests/simulation/vars.sh
@ -6,17 +6,18 @@ uname | grep -qi mingw && PWD_CMD="pwd -W"

 cd $(dirname $0)
 SIM_ROOT="$($PWD_CMD)"
-cd $(git rev-parse --show-toplevel)
-GIT_ROOT="$($PWD_CMD)"

-# Set a default value for the env vars usually supplied by nimbus Makefile
+# Set a default value for the env vars usually supplied by a Makefile
+cd $(git rev-parse --show-toplevel)
+: ${GIT_ROOT:="$($PWD_CMD)"}
+cd - &>/dev/null
 : ${SKIP_BUILDS:=""}
 : ${BUILD_OUTPUTS_DIR:="$GIT_ROOT/build"}

-SIMULATION_DIR="$SIM_ROOT/data"
-VALIDATORS_DIR="$SIM_ROOT/validators"
-SNAPSHOT_FILE="$SIMULATION_DIR/state_snapshot.json"
-NETWORK_METADATA_FILE="$SIMULATION_DIR/network.json"
-BEACON_NODE_BIN=$BUILD_OUTPUTS_DIR/beacon_node
-VALIDATOR_KEYGEN_BIN=$BUILD_OUTPUTS_DIR/validator_keygen
-MASTER_NODE_ADDRESS_FILE="$SIMULATION_DIR/node-0/beacon_node.address"
+SIMULATION_DIR="${SIM_ROOT}/data"
+VALIDATORS_DIR="${SIM_ROOT}/validators"
+SNAPSHOT_FILE="${SIMULATION_DIR}/state_snapshot.json"
+NETWORK_METADATA_FILE="${SIMULATION_DIR}/network.json"
+BEACON_NODE_BIN="${BUILD_OUTPUTS_DIR}/beacon_node"
+VALIDATOR_KEYGEN_BIN="${BUILD_OUTPUTS_DIR}/validator_keygen"
+MASTER_NODE_ADDRESS_FILE="${SIMULATION_DIR}/node-0/beacon_node.address"