Makefile that only works in a Git submodule (#210)
- updated README.md - beacon_chain.nimble: accept compilation flags as params - nim.cfg: enable `--opt:speed` at the top level - simulation scripts: allow overriding GIT_ROOT and do some refactoring
This commit is contained in:
parent
44d9f7d6c9
commit
f8fbe0ff3b
|
@ -0,0 +1,48 @@
|
||||||
|
# Copyright (c) 2019 Status Research & Development GmbH. Licensed under
|
||||||
|
# either of:
|
||||||
|
# - Apache License, version 2.0
|
||||||
|
# - MIT license
|
||||||
|
# at your option. This file may not be copied, modified, or distributed except
|
||||||
|
# according to those terms.
|
||||||
|
|
||||||
|
# we don't want an error here, so we can explain things later, in the sanity-checks target
|
||||||
|
-include ../../common.mk
|
||||||
|
|
||||||
|
ENV_SCRIPT := "../../env.sh"
|
||||||
|
|
||||||
|
TOOLS := beacon_node validator_keygen bench_bls_sig_agggregation state_sim
|
||||||
|
TOOLS_DIRS := beacon_chain benchmarks research
|
||||||
|
# comma-separated values for the "clean" target
|
||||||
|
TOOLS_CSV := $(subst $(SPACE),$(COMMA),$(TOOLS))
|
||||||
|
|
||||||
|
.PHONY: all sanity-checks deps test $(TOOLS) clean_eth2_network_simulation_files eth2_network_simulation
|
||||||
|
|
||||||
|
all: | $(TOOLS)
|
||||||
|
|
||||||
|
sanity-checks:
|
||||||
|
@ [[ "$$PWD" =~ /vendor/nim-beacon-chain$ && -e ../../Makefile && -e ../../common.mk ]] || \
|
||||||
|
{ echo "This Makefile can only be used from the corresponding Git submodule in the Nimbus repository."; exit 1; }
|
||||||
|
|
||||||
|
deps: | sanity-checks
|
||||||
|
@+ $(MAKE) --silent -C ../../ deps
|
||||||
|
|
||||||
|
build:
|
||||||
|
mkdir $@
|
||||||
|
|
||||||
|
test: | build deps
|
||||||
|
../../nimble.sh test $(NIM_PARAMS)
|
||||||
|
|
||||||
|
$(TOOLS): | build deps
|
||||||
|
for D in $(TOOLS_DIRS); do [ -e "$${D}/$@.nim" ] && TOOL_DIR="$${D}" && break; done && \
|
||||||
|
$(ENV_SCRIPT) nim c $(NIM_PARAMS) -o:build/$@ "$${TOOL_DIR}/$@.nim" && \
|
||||||
|
echo -e "\nThe binary is in './build/$@'.\n"
|
||||||
|
|
||||||
|
clean_eth2_network_simulation_files:
|
||||||
|
rm -rf tests/simulation/data
|
||||||
|
|
||||||
|
eth2_network_simulation: | beacon_node validator_keygen clean_eth2_network_simulation_files
|
||||||
|
SKIP_BUILDS=1 GIT_ROOT="$$PWD" BUILD_OUTPUTS_DIR="./build" tests/simulation/start.sh
|
||||||
|
|
||||||
|
clean:
|
||||||
|
rm -rf build/{$(TOOLS_CSV),all_tests,*.exe} nimcache
|
||||||
|
|
91
README.md
91
README.md
|
@ -17,39 +17,39 @@ You can check where the beacon chain fits in the Ethereum research ecosystem in
|
||||||
|
|
||||||
## Building and Testing
|
## Building and Testing
|
||||||
|
|
||||||
The beacon chain components require that you have Nim installed - the easiest way to get started is to head over to the main [Nimbus](https://github.com/status-im/nimbus/) repository and follow the build instructions there or just execute the commands below in order.
|
The beacon chain components need to be built with the Nim compiler - the easiest way to get started is to head over to the main [Nimbus](https://github.com/status-im/nimbus/) repository and follow the build instructions there or just execute the commands below in order.
|
||||||
|
|
||||||
_Note: This is because this repository is actually pulled in as a dependency of Nimbus - the Ethereum 1.0 + 2.0 client - so it makes sense to start from there even if you are only interested in testing the Ethereum 2.0 side of things (contained almost entirely in this repository)._
|
_Note: This is because this repository is actually pulled in as a dependency of Nimbus - the Ethereum 1.0 + 2.0 client - so it makes sense to start from there even if you are only interested in testing the Ethereum 2.0 side of things (contained almost entirely in this repository)._
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Clone main nimbus repository
|
# Clone main nimbus repository:
|
||||||
git clone https://github.com/status-im/nimbus.git
|
git clone https://github.com/status-im/nimbus.git
|
||||||
cd nimbus
|
cd nimbus
|
||||||
|
|
||||||
# Prep environment
|
# Prep environment (assuming you have 4 CPU cores and want to take advantage of them):
|
||||||
make update deps
|
make -j4 deps
|
||||||
|
|
||||||
# Start a shell that uses the Nimbus compile environment
|
# Head over to the vendor repo where you should have a checkout of this project:
|
||||||
./env.sh bash
|
|
||||||
|
|
||||||
# You're now in a shell environment that has the right Nim version available.
|
|
||||||
# Head over to the vendor repo where you should have a checkout of this project
|
|
||||||
cd vendor/nim-beacon-chain
|
cd vendor/nim-beacon-chain
|
||||||
|
|
||||||
# You can now run the test suite:
|
# You can now run the test suite:
|
||||||
nim c -d:release -r tests/all_tests
|
make test
|
||||||
```
|
```
|
||||||
|
|
||||||
## Beacon node simulation
|
## Beacon node simulation
|
||||||
|
|
||||||
The beacon node simulation is will create a full peer-to-peer network of beacon nodes and validators, and run the beacon chain in real time. To change network parameters such as shard and validator counts, see [start.sh](tests/simulation/start.sh).
|
The beacon node simulation is will create a full peer-to-peer network of beacon nodes and validators, and run the beacon chain in real time. To change network parameters such as shard and validator counts, see [start.sh](tests/simulation/start.sh).
|
||||||
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Start beacon chain simulation, resuming from the previous state (if any)
|
# get a shell with the right environment vars set:
|
||||||
|
../../env.sh bash
|
||||||
|
|
||||||
|
# Start the beacon chain simulation, resuming from a previous state (if any):
|
||||||
./tests/simulation/start.sh # if starting from Nimbus, make sure you're in vendor/nim-beacon-chain!
|
./tests/simulation/start.sh # if starting from Nimbus, make sure you're in vendor/nim-beacon-chain!
|
||||||
|
|
||||||
# Clear data from last run and restart simulation with a new genesis block
|
# Clear data files from your last run and restart the simulation with a new genesis block:
|
||||||
rm -rf tests/simulation/data ; ./tests/simulation/start.sh
|
rm -rf tests/simulation/data; ./tests/simulation/start.sh
|
||||||
|
|
||||||
# Run an extra node - by default the network will launch with 9 nodes, each
|
# Run an extra node - by default the network will launch with 9 nodes, each
|
||||||
# hosting 10 validators. The last 10 validators are lazy bums that hid from the
|
# hosting 10 validators. The last 10 validators are lazy bums that hid from the
|
||||||
|
@ -58,16 +58,12 @@ rm -rf tests/simulation/data ; ./tests/simulation/start.sh
|
||||||
./tests/simulation/run_node.sh 9
|
./tests/simulation/run_node.sh 9
|
||||||
```
|
```
|
||||||
|
|
||||||
Alternatively, the Makefile flow is available from the Nimbus parent repo:
|
Alternatively, a Makefile-based flow is available:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# In the Nimbus repo root
|
# From "vendor/nim-beacon-chain/",
|
||||||
|
# clear all data from the last run and restart the simulation with a new genesis block:
|
||||||
# Start beacon chain simulation, resuming from the previous state (if any)
|
|
||||||
make eth2_network_simulation
|
make eth2_network_simulation
|
||||||
|
|
||||||
# Clear data from last run and restart simulation with a new genesis block
|
|
||||||
make clean_eth2_network_simulation_files eth2_network_simulation
|
|
||||||
```
|
```
|
||||||
|
|
||||||
You can also separate the output from each beacon node in its own panel, using [multitail](http://www.vanheusden.com/multitail/):
|
You can also separate the output from each beacon node in its own panel, using [multitail](http://www.vanheusden.com/multitail/):
|
||||||
|
@ -77,22 +73,62 @@ USE_MULTITAIL="yes" ./tests/simulation/start.sh
|
||||||
|
|
||||||
# OR
|
# OR
|
||||||
|
|
||||||
USE_MULTITAIL="yes" make eth2_network_simulation
|
make USE_MULTITAIL="yes" eth2_network_simulation
|
||||||
```
|
```
|
||||||
|
|
||||||
You can find out more about it in the [development update](https://our.status.im/nimbus-development-update-2018-12-2/).
|
You can find out more about it in the [development update](https://our.status.im/nimbus-development-update-2018-12-2/).
|
||||||
|
|
||||||
_Alternatively, fire up our [experimental Vagrant instance with Nim pre-installed](https://our.status.im/setting-up-a-local-vagrant-environment-for-nim-development/) and give us yout feedback about the process!_
|
_Alternatively, fire up our [experimental Vagrant instance with Nim pre-installed](https://our.status.im/setting-up-a-local-vagrant-environment-for-nim-development/) and give us yout feedback about the process!_
|
||||||
|
|
||||||
|
### Makefile tips and tricks for developers
|
||||||
|
|
||||||
|
- build all those tools known to the Makefile:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make
|
||||||
|
```
|
||||||
|
|
||||||
|
- build a specific tool:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make state_sim
|
||||||
|
```
|
||||||
|
|
||||||
|
- you can control the Makefile's verbosity with the V variable (defaults to 1):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make V=0 # quiet
|
||||||
|
make V=2 test # more verbose than usual
|
||||||
|
```
|
||||||
|
|
||||||
|
- same for the [Chronicles log level](https://github.com/status-im/nim-chronicles#chronicles_log_level):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make LOG_LEVEL=DEBUG bench_bls_sig_agggregation # this is the default
|
||||||
|
make LOG_LEVEL=TRACE beacon_node # log everything
|
||||||
|
```
|
||||||
|
|
||||||
|
- pass arbitrary parameters to the Nim compiler:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make NIMFLAGS="-d:release"
|
||||||
|
```
|
||||||
|
|
||||||
|
- you can freely combine those variables on the `make` command line:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make -j8 V=0 NIMFLAGS="-d:release" USE_MULTITAIL=yes eth2_network_simulation
|
||||||
|
```
|
||||||
|
|
||||||
## State transition simulation
|
## State transition simulation
|
||||||
|
|
||||||
The state transition simulator can quickly run the Beacon chain state transition function in isolation and output JSON snapshots of the state. The simulation runs without networking and blocks are processed without slot time delays.
|
The state transition simulator can quickly run the Beacon chain state transition function in isolation and output JSON snapshots of the state. The simulation runs without networking and blocks are processed without slot time delays.
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cd research
|
# build and run the state simulator, then display its help ("-d:release" speeds it
|
||||||
# build and run state simulator, then display its help - -d:release speeds it
|
# up substantially, allowing the simulation of longer runs in reasonable time)
|
||||||
# up substantially, allowing the simulation of longer runs in reasonable time
|
make V=0 NIMFLAGS="-d:release" state_sim
|
||||||
nim c -d:release -r state_sim --help
|
./build/state_sim --help
|
||||||
```
|
```
|
||||||
|
|
||||||
## Convention
|
## Convention
|
||||||
|
@ -107,7 +143,7 @@ Nim NEP-1 recommends:
|
||||||
- PascalCase for constants
|
- PascalCase for constants
|
||||||
- PascalCase for types
|
- PascalCase for types
|
||||||
|
|
||||||
To facilitate collaboration and comparison, Nim-beacon-chain uses the Ethereum Foundation convention.
|
To facilitate collaboration and comparison, nim-beacon-chain uses the Ethereum Foundation convention.
|
||||||
|
|
||||||
## License
|
## License
|
||||||
|
|
||||||
|
@ -119,4 +155,5 @@ or
|
||||||
|
|
||||||
* Apache License, Version 2.0, ([LICENSE-APACHEv2](LICENSE-APACHEv2) or http://www.apache.org/licenses/LICENSE-2.0)
|
* Apache License, Version 2.0, ([LICENSE-APACHEv2](LICENSE-APACHEv2) or http://www.apache.org/licenses/LICENSE-2.0)
|
||||||
|
|
||||||
at your option. This file may not be copied, modified, or distributed except according to those terms.
|
at your option. These files may not be copied, modified, or distributed except according to those terms.
|
||||||
|
|
||||||
|
|
|
@ -1,3 +1,5 @@
|
||||||
|
mode = ScriptMode.Verbose
|
||||||
|
|
||||||
import
|
import
|
||||||
beacon_chain/version as ver
|
beacon_chain/version as ver
|
||||||
|
|
||||||
|
@ -30,14 +32,15 @@ requires "nim >= 0.19.0",
|
||||||
"libp2p"
|
"libp2p"
|
||||||
|
|
||||||
### Helper functions
|
### Helper functions
|
||||||
proc test(name: string, defaultLang = "c") =
|
proc buildBinary(name: string, srcDir = "./", params = "", lang = "c") =
|
||||||
if not dirExists "build":
|
if not dirExists "build":
|
||||||
mkDir "build"
|
mkDir "build"
|
||||||
--run
|
# allow something like "nim test --verbosity:0 --hints:off beacon_chain.nims"
|
||||||
switch("out", ("./build/" & name))
|
var extra_params = params
|
||||||
switch("opt", "speed")
|
for i in 2..<paramCount():
|
||||||
setCommand defaultLang, "tests/" & name & ".nim"
|
extra_params &= " " & paramStr(i)
|
||||||
|
exec "nim " & lang & " --out:./build/" & name & " " & extra_params & " " & srcDir & name & ".nim"
|
||||||
|
|
||||||
### tasks
|
### tasks
|
||||||
task test, "Run all tests":
|
task test, "Run all tests":
|
||||||
test "all_tests"
|
buildBinary "all_tests", "tests/", "-r -d:release -d:chronicles_log_level=ERROR"
|
||||||
|
|
1
nim.cfg
1
nim.cfg
|
@ -1,5 +1,6 @@
|
||||||
# https://github.com/nim-lang/Nim/issues/8294#issuecomment-454556051
|
# https://github.com/nim-lang/Nim/issues/8294#issuecomment-454556051
|
||||||
--threads:on
|
--threads:on
|
||||||
|
--opt:speed
|
||||||
|
|
||||||
@if windows:
|
@if windows:
|
||||||
-d:"chronicles_colors=NoColors"
|
-d:"chronicles_colors=NoColors"
|
||||||
|
|
|
@ -1,2 +0,0 @@
|
||||||
--threads:on
|
|
||||||
|
|
|
@ -1,12 +1,13 @@
|
||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
|
|
||||||
set -eux
|
set -eu
|
||||||
|
|
||||||
. $(dirname $0)/vars.sh
|
. $(dirname $0)/vars.sh
|
||||||
|
cd "$GIT_ROOT"
|
||||||
|
|
||||||
DATA_DIR=$SIMULATION_DIR/node-${1}
|
DATA_DIR="${SIMULATION_DIR}/node-${1}"
|
||||||
|
|
||||||
V_PREFIX="$VALIDATORS_DIR/v$(printf '%06d' ${1})"
|
V_PREFIX="${VALIDATORS_DIR}/v$(printf '%06d' ${1})"
|
||||||
PORT=$(printf '5%04d' ${1})
|
PORT=$(printf '5%04d' ${1})
|
||||||
|
|
||||||
NAT_FLAG=""
|
NAT_FLAG=""
|
||||||
|
|
|
@ -1,9 +1,6 @@
|
||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
|
|
||||||
set -eux
|
set -eu
|
||||||
|
|
||||||
# Kill children on ctrl-c
|
|
||||||
trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM EXIT
|
|
||||||
|
|
||||||
# Read in variables
|
# Read in variables
|
||||||
. $(dirname $0)/vars.sh
|
. $(dirname $0)/vars.sh
|
||||||
|
@ -13,11 +10,11 @@ trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM EXIT
|
||||||
NUM_VALIDATORS=${VALIDATORS:-100}
|
NUM_VALIDATORS=${VALIDATORS:-100}
|
||||||
NUM_NODES=${NODES:-9}
|
NUM_NODES=${NODES:-9}
|
||||||
|
|
||||||
cd $SIM_ROOT
|
cd "$SIM_ROOT"
|
||||||
mkdir -p "$SIMULATION_DIR"
|
mkdir -p "$SIMULATION_DIR"
|
||||||
mkdir -p "$VALIDATORS_DIR"
|
mkdir -p "$VALIDATORS_DIR"
|
||||||
|
|
||||||
cd $GIT_ROOT
|
cd "$GIT_ROOT"
|
||||||
mkdir -p $BUILD_OUTPUTS_DIR
|
mkdir -p $BUILD_OUTPUTS_DIR
|
||||||
|
|
||||||
# Run with "SHARD_COUNT=4 ./start.sh" to change these
|
# Run with "SHARD_COUNT=4 ./start.sh" to change these
|
||||||
|
@ -66,8 +63,16 @@ fi
|
||||||
MULTITAIL="${MULTITAIL:-multitail}" # to allow overriding the program name
|
MULTITAIL="${MULTITAIL:-multitail}" # to allow overriding the program name
|
||||||
USE_MULTITAIL="${USE_MULTITAIL:-no}" # make it an opt-in
|
USE_MULTITAIL="${USE_MULTITAIL:-no}" # make it an opt-in
|
||||||
type "$MULTITAIL" &>/dev/null || USE_MULTITAIL="no"
|
type "$MULTITAIL" &>/dev/null || USE_MULTITAIL="no"
|
||||||
COMMANDS=()
|
|
||||||
|
|
||||||
|
# Kill child processes on Ctrl-C by sending SIGTERM to the whole process group,
|
||||||
|
# passing the negative PID of this shell instance to the "kill" command.
|
||||||
|
# Trap and ignore SIGTERM, so we don't kill this process along with its children.
|
||||||
|
if [ "$USE_MULTITAIL" = "no" ]; then
|
||||||
|
trap '' SIGTERM
|
||||||
|
trap "kill -- -$$" SIGINT EXIT
|
||||||
|
fi
|
||||||
|
|
||||||
|
COMMANDS=()
|
||||||
LAST_NODE=$(( $NUM_NODES - 1 ))
|
LAST_NODE=$(( $NUM_NODES - 1 ))
|
||||||
|
|
||||||
for i in $(seq 0 $LAST_NODE); do
|
for i in $(seq 0 $LAST_NODE); do
|
||||||
|
@ -80,7 +85,7 @@ for i in $(seq 0 $LAST_NODE); do
|
||||||
done
|
done
|
||||||
fi
|
fi
|
||||||
|
|
||||||
CMD="$SIM_ROOT/run_node.sh $i"
|
CMD="${SIM_ROOT}/run_node.sh $i"
|
||||||
|
|
||||||
if [ "$USE_MULTITAIL" != "no" ]; then
|
if [ "$USE_MULTITAIL" != "no" ]; then
|
||||||
if [ "$i" = "0" ]; then
|
if [ "$i" = "0" ]; then
|
||||||
|
|
|
@ -6,17 +6,18 @@ uname | grep -qi mingw && PWD_CMD="pwd -W"
|
||||||
|
|
||||||
cd $(dirname $0)
|
cd $(dirname $0)
|
||||||
SIM_ROOT="$($PWD_CMD)"
|
SIM_ROOT="$($PWD_CMD)"
|
||||||
cd $(git rev-parse --show-toplevel)
|
|
||||||
GIT_ROOT="$($PWD_CMD)"
|
|
||||||
|
|
||||||
# Set a default value for the env vars usually supplied by nimbus Makefile
|
# Set a default value for the env vars usually supplied by a Makefile
|
||||||
|
cd $(git rev-parse --show-toplevel)
|
||||||
|
: ${GIT_ROOT:="$($PWD_CMD)"}
|
||||||
|
cd - &>/dev/null
|
||||||
: ${SKIP_BUILDS:=""}
|
: ${SKIP_BUILDS:=""}
|
||||||
: ${BUILD_OUTPUTS_DIR:="$GIT_ROOT/build"}
|
: ${BUILD_OUTPUTS_DIR:="$GIT_ROOT/build"}
|
||||||
|
|
||||||
SIMULATION_DIR="$SIM_ROOT/data"
|
SIMULATION_DIR="${SIM_ROOT}/data"
|
||||||
VALIDATORS_DIR="$SIM_ROOT/validators"
|
VALIDATORS_DIR="${SIM_ROOT}/validators"
|
||||||
SNAPSHOT_FILE="$SIMULATION_DIR/state_snapshot.json"
|
SNAPSHOT_FILE="${SIMULATION_DIR}/state_snapshot.json"
|
||||||
NETWORK_METADATA_FILE="$SIMULATION_DIR/network.json"
|
NETWORK_METADATA_FILE="${SIMULATION_DIR}/network.json"
|
||||||
BEACON_NODE_BIN=$BUILD_OUTPUTS_DIR/beacon_node
|
BEACON_NODE_BIN="${BUILD_OUTPUTS_DIR}/beacon_node"
|
||||||
VALIDATOR_KEYGEN_BIN=$BUILD_OUTPUTS_DIR/validator_keygen
|
VALIDATOR_KEYGEN_BIN="${BUILD_OUTPUTS_DIR}/validator_keygen"
|
||||||
MASTER_NODE_ADDRESS_FILE="$SIMULATION_DIR/node-0/beacon_node.address"
|
MASTER_NODE_ADDRESS_FILE="${SIMULATION_DIR}/node-0/beacon_node.address"
|
||||||
|
|
Loading…
Reference in New Issue