2020-01-20 22:28:48 +00:00
#!/bin/bash
2020-02-18 23:33:26 +00:00
# Copyright (c) 2020 Status Research & Development GmbH. Licensed under
# either of:
# - Apache License, version 2.0
# - MIT license
# at your option. This file may not be copied, modified, or distributed except
# according to those terms.
2020-01-20 22:28:48 +00:00
# Mostly a duplication of "tests/simulation/{start.sh,run_node.sh}", but with a focus on
2020-02-13 12:19:12 +00:00
# replicating testnets as closely as possible, which means following the Docker execution labyrinth.
2020-01-20 22:28:48 +00:00
set -e
cd " $( dirname " ${ BASH_SOURCE [0] } " ) " /..
2020-02-18 23:33:26 +00:00
####################
# argument parsing #
####################
! getopt --test > /dev/null
if [ ${ PIPESTATUS [0] } != 4 ] ; then
echo '`getopt --test` failed in this environment.'
exit 1
fi
OPTS = "ht:n:d:"
2020-05-05 22:02:39 +00:00
LONGOPTS = "help,testnet:,nodes:,data-dir:,disable-htop,log-level:,grafana"
2020-01-20 22:28:48 +00:00
2020-02-18 23:33:26 +00:00
# default values
TESTNET = "1"
NUM_NODES = "10"
2020-01-20 22:28:48 +00:00
DATA_DIR = "local_testnet_data"
2020-02-18 23:33:26 +00:00
USE_HTOP = "1"
2020-03-09 00:32:58 +00:00
LOG_LEVEL = "DEBUG"
2020-05-05 22:02:39 +00:00
ENABLE_GRAFANA = "0"
2020-02-18 23:33:26 +00:00
print_help( ) {
cat <<EOF
Usage: $( basename $0 ) --testnet <testnet number> [ OTHER OPTIONS] -- [ BEACON NODE OPTIONS]
E.g.: $( basename $0 ) --testnet ${ TESTNET } --nodes ${ NUM_NODES } --data-dir " ${ DATA_DIR } " # defaults
CI run: $( basename $0 ) --disable-htop -- --verify-finalization --stop-at-epoch= 5
-h, --help this help message
-t, --testnet testnet number ( default: ${ TESTNET } )
-n, --nodes number of nodes to launch ( default: ${ NUM_NODES } )
-d, --data-dir directory where all the node data and logs will end up
( default: " ${ DATA_DIR } " )
--disable-htop don' t use "htop" to see the beacon_node processes
2020-03-09 00:32:58 +00:00
--log-level set the log level ( default: ${ LOG_LEVEL } )
2020-05-05 22:02:39 +00:00
--grafana generate Grafana dashboards ( and Prometheus config file)
2020-02-18 23:33:26 +00:00
EOF
}
! PARSED = $( getopt --options= ${ OPTS } --longoptions= ${ LONGOPTS } --name " $0 " -- " $@ " )
if [ ${ PIPESTATUS [0] } != 0 ] ; then
# getopt has complained about wrong arguments to stdout
exit 1
fi
# read getopt's output this way to handle the quoting right
eval set -- " $PARSED "
while true; do
case " $1 " in
-h| --help)
print_help
exit
; ;
-t| --testnet)
TESTNET = " $2 "
shift 2
; ;
-n| --nodes)
NUM_NODES = " $2 "
shift 2
; ;
-d| --data-dir)
DATA_DIR = " $2 "
shift 2
; ;
--disable-htop)
USE_HTOP = "0"
shift
; ;
2020-03-09 00:32:58 +00:00
--log-level)
LOG_LEVEL = " $2 "
shift 2
; ;
2020-05-05 22:02:39 +00:00
--grafana)
ENABLE_GRAFANA = "1"
shift
; ;
2020-02-18 23:33:26 +00:00
--)
shift
break
; ;
*)
echo "argument parsing error"
print_help
exit 1
esac
done
2020-02-26 00:14:20 +00:00
# when sourcing env.sh, it will try to execute $@, so empty it
EXTRA_ARGS = " $@ "
if [ [ $# != 0 ] ] ; then
shift $#
fi
2020-02-18 23:33:26 +00:00
NETWORK = " testnet ${ TESTNET } "
2020-05-05 22:02:39 +00:00
BASE_METRICS_PORT = 8008
2020-02-18 23:33:26 +00:00
2020-01-20 22:28:48 +00:00
rm -rf " ${ DATA_DIR } "
DEPOSITS_DIR = " ${ DATA_DIR } /deposits_dir "
mkdir -p " ${ DEPOSITS_DIR } "
NETWORK_DIR = " ${ DATA_DIR } /network_dir "
mkdir -p " ${ NETWORK_DIR } "
set -a
source " scripts/ ${ NETWORK } .env "
set +a
2020-02-18 23:33:26 +00:00
# Windows detection
if uname | grep -qiE "mingw|msys" ; then
MAKE = "mingw32-make"
else
MAKE = "make"
fi
2020-01-20 22:28:48 +00:00
NETWORK_NIM_FLAGS = $( scripts/load-testnet-nim-flags.sh ${ NETWORK } )
2020-03-09 00:32:58 +00:00
$MAKE LOG_LEVEL = " ${ LOG_LEVEL } " NIMFLAGS = " -d:insecure -d:testnet_servers_image ${ NETWORK_NIM_FLAGS } " beacon_node
2020-01-20 22:28:48 +00:00
rm -rf " ${ DEPOSITS_DIR } "
./build/beacon_node makeDeposits \
2020-02-18 23:33:26 +00:00
--quickstart-deposits= ${ QUICKSTART_VALIDATORS } \
--random-deposits= ${ RANDOM_VALIDATORS } \
--deposits-dir= " ${ DEPOSITS_DIR } "
2020-01-20 22:28:48 +00:00
TOTAL_VALIDATORS = " $(( $QUICKSTART_VALIDATORS + $RANDOM_VALIDATORS )) "
BOOTSTRAP_IP = "127.0.0.1"
./build/beacon_node createTestnet \
2020-02-18 23:33:26 +00:00
--data-dir= " ${ DATA_DIR } /node0 " \
--validators-dir= " ${ DEPOSITS_DIR } " \
--total-validators= ${ TOTAL_VALIDATORS } \
--last-user-validator= ${ QUICKSTART_VALIDATORS } \
--output-genesis= " ${ NETWORK_DIR } /genesis.ssz " \
--output-bootstrap-file= " ${ NETWORK_DIR } /bootstrap_nodes.txt " \
--bootstrap-address= ${ BOOTSTRAP_IP } \
--bootstrap-port= ${ BOOTSTRAP_PORT } \
--genesis-offset= 5 # Delay in seconds
2020-01-20 22:28:48 +00:00
2020-05-05 22:02:39 +00:00
if [ [ " $ENABLE_GRAFANA " = = "1" ] ] ; then
# Prometheus config
cat > " ${ DATA_DIR } /prometheus.yml " <<EOF
global:
scrape_interval: 1s
scrape_configs:
- job_name: "nimbus"
static_configs:
EOF
for NUM_NODE in $( seq 0 $(( ${ NUM_NODES } - 1 )) ) ; do
cat >> " ${ DATA_DIR } /prometheus.yml " <<EOF
- targets: [ '127.0.0.1:$(( BASE_METRICS_PORT + NUM_NODE ))' ]
labels:
node: '$NUM_NODE'
EOF
done
# use the exported Grafana dashboard for a single node to create one for all nodes
PROCESS_DASHBOARD_BIN = "build/process_dashboard"
if [ [ ! -f " $PROCESS_DASHBOARD_BIN " ] ] ; then
$MAKE process_dashboard
fi
" ${ PROCESS_DASHBOARD_BIN } " \
--nodes= ${ NUM_NODES } \
--in= "tests/simulation/beacon-chain-sim-node0-Grafana-dashboard.json" \
--out= " ${ DATA_DIR } /local-testnet-all-nodes-Grafana-dashboard.json "
fi
2020-04-30 13:59:57 +00:00
# Kill child processes on Ctrl-C/SIGTERM/exit, passing the PID of this shell
# instance as the parent and the target process name as a pattern to the
# "pkill" command.
2020-01-20 22:28:48 +00:00
cleanup( ) {
2020-04-30 13:59:57 +00:00
pkill -P $$ beacon_node & >/dev/null || true
2020-01-20 22:28:48 +00:00
sleep 2
2020-04-30 13:59:57 +00:00
pkill -9 -P $$ beacon_node & >/dev/null || true
2020-01-20 22:28:48 +00:00
}
2020-04-30 13:59:57 +00:00
trap 'cleanup' SIGINT SIGTERM EXIT
2020-01-20 22:28:48 +00:00
2020-05-01 22:09:04 +00:00
dump_logs( ) {
LOG_LINES = 20
for LOG in " ${ DATA_DIR } " /log*.txt; do
echo " Last ${ LOG_LINES } lines of ${ LOG } : "
tail -n ${ LOG_LINES } " ${ LOG } "
echo "======"
done
}
2020-01-20 22:28:48 +00:00
PIDS = ""
2020-02-21 13:19:34 +00:00
NODES_WITH_VALIDATORS = ${ NODES_WITH_VALIDATORS :- 4 }
VALIDATORS_PER_NODE = $(( $RANDOM_VALIDATORS / $NODES_WITH_VALIDATORS ))
2020-05-01 18:57:12 +00:00
BOOTSTRAP_TIMEOUT = 10 # in seconds
2020-02-21 13:19:34 +00:00
2020-01-20 22:28:48 +00:00
for NUM_NODE in $( seq 0 $(( ${ NUM_NODES } - 1 )) ) ; do
if [ [ ${ NUM_NODE } = = 0 ] ] ; then
BOOTSTRAP_ARG = ""
else
BOOTSTRAP_ARG = " --bootstrap-file= ${ NETWORK_DIR } /bootstrap_nodes.txt "
# Wait for the master node to write out its address file
2020-05-01 18:57:12 +00:00
START_TIMESTAMP = $( date +%s)
2020-01-20 22:28:48 +00:00
while [ ! -f " ${ DATA_DIR } /node0/beacon_node.address " ] ; do
sleep 0.1
2020-05-01 18:57:12 +00:00
NOW_TIMESTAMP = $( date +%s)
if [ [ " $(( NOW_TIMESTAMP - START_TIMESTAMP )) " -ge " $BOOTSTRAP_TIMEOUT " ] ] ; then
echo " Bootstrap node failed to start in ${ BOOTSTRAP_TIMEOUT } seconds. Aborting. "
2020-05-01 22:09:04 +00:00
dump_logs
2020-05-01 18:57:12 +00:00
exit 1
fi
2020-01-20 22:28:48 +00:00
done
fi
2020-02-21 13:19:34 +00:00
# Copy validators to individual nodes.
# The first $NODES_WITH_VALIDATORS nodes split them equally between them, after skipping the first $QUICKSTART_VALIDATORS.
NODE_DATA_DIR = " ${ DATA_DIR } /node ${ NUM_NODE } "
mkdir -p " ${ NODE_DATA_DIR } /validators "
if [ [ $NUM_NODE -lt $NODES_WITH_VALIDATORS ] ] ; then
for KEYFILE in $( ls ${ DEPOSITS_DIR } /*.privkey | tail -n +$(( $QUICKSTART_VALIDATORS + ( $VALIDATORS_PER_NODE * $NUM_NODE ) + 1 )) | head -n $VALIDATORS_PER_NODE ) ; do
cp -a " $KEYFILE " " ${ NODE_DATA_DIR } /validators/ "
done
fi
2020-04-30 13:59:57 +00:00
./build/beacon_node \
2020-03-11 09:07:39 +00:00
--nat:extip:127.0.0.1 \
2020-03-09 00:32:58 +00:00
--log-level= " ${ LOG_LEVEL } " \
2020-05-01 18:57:12 +00:00
--tcp-port= $(( BOOTSTRAP_PORT + NUM_NODE )) \
--udp-port= $(( BOOTSTRAP_PORT + NUM_NODE )) \
2020-02-21 13:19:34 +00:00
--data-dir= " ${ NODE_DATA_DIR } " \
2020-01-20 22:28:48 +00:00
${ BOOTSTRAP_ARG } \
--state-snapshot= " ${ NETWORK_DIR } /genesis.ssz " \
2020-05-05 22:02:39 +00:00
--metrics \
--metrics-address= "127.0.0.1" \
--metrics-port= " $(( BASE_METRICS_PORT + NUM_NODE )) " \
2020-02-26 00:14:20 +00:00
${ EXTRA_ARGS } \
2020-01-20 22:28:48 +00:00
> " ${ DATA_DIR } /log ${ NUM_NODE } .txt " 2>& 1 &
2020-05-01 18:57:12 +00:00
2020-01-20 22:28:48 +00:00
if [ [ " ${ PIDS } " = = "" ] ] ; then
PIDS = " $! "
else
PIDS = " ${ PIDS } , $! "
fi
done
2020-05-01 18:57:12 +00:00
# give the regular nodes time to crash
sleep 5
BG_JOBS = " $( jobs | wc -l) "
if [ [ " $BG_JOBS " != " $NUM_NODES " ] ] ; then
echo " $(( NUM_NODES - BG_JOBS)) beacon_node instance(s) exited early. Aborting. "
2020-05-01 22:09:04 +00:00
dump_logs
2020-05-01 18:57:12 +00:00
exit 1
fi
2020-02-18 23:33:26 +00:00
if [ [ " $USE_HTOP " = = "1" ] ] ; then
htop -p " $PIDS "
cleanup
else
FAILED = 0
for PID in $( echo " $PIDS " | tr ',' ' ' ) ; do
wait $PID || FAILED = " $(( FAILED += 1 )) "
done
if [ [ " $FAILED " != "0" ] ] ; then
echo " ${ FAILED } child processes had non-zero exit codes (or exited early). "
2020-05-01 22:09:04 +00:00
dump_logs
2020-02-18 23:33:26 +00:00
exit 1
fi
fi
2020-01-20 22:28:48 +00:00