2020-01-20 22:28:48 +00:00
|
|
|
#!/bin/bash
|
|
|
|
|
2020-02-18 23:33:26 +00:00
|
|
|
# Copyright (c) 2020 Status Research & Development GmbH. Licensed under
|
|
|
|
# either of:
|
|
|
|
# - Apache License, version 2.0
|
|
|
|
# - MIT license
|
|
|
|
# at your option. This file may not be copied, modified, or distributed except
|
|
|
|
# according to those terms.
|
|
|
|
|
2020-01-20 22:28:48 +00:00
|
|
|
# Mostly a duplication of "tests/simulation/{start.sh,run_node.sh}", but with a focus on
|
2020-02-13 12:19:12 +00:00
|
|
|
# replicating testnets as closely as possible, which means following the Docker execution labyrinth.
|
2020-01-20 22:28:48 +00:00
|
|
|
|
|
|
|
set -e
|
|
|
|
|
|
|
|
cd "$(dirname "${BASH_SOURCE[0]}")"/..
|
|
|
|
|
2020-02-18 23:33:26 +00:00
|
|
|
####################
|
|
|
|
# argument parsing #
|
|
|
|
####################
|
|
|
|
! getopt --test > /dev/null
|
|
|
|
if [ ${PIPESTATUS[0]} != 4 ]; then
|
|
|
|
echo '`getopt --test` failed in this environment.'
|
|
|
|
exit 1
|
|
|
|
fi
|
|
|
|
|
2020-06-22 18:22:45 +00:00
|
|
|
OPTS="hgt:n:d:"
|
|
|
|
LONGOPTS="help,testnet:,nodes:,data-dir:,disable-htop,log-level:,base-port:,base-metrics-port:,with-ganache"
|
2020-01-20 22:28:48 +00:00
|
|
|
|
2020-02-18 23:33:26 +00:00
|
|
|
# default values
|
|
|
|
TESTNET="1"
|
|
|
|
NUM_NODES="10"
|
2020-01-20 22:28:48 +00:00
|
|
|
DATA_DIR="local_testnet_data"
|
2020-02-18 23:33:26 +00:00
|
|
|
USE_HTOP="1"
|
2020-06-22 18:22:45 +00:00
|
|
|
USE_GANACHE="0"
|
2020-03-09 00:32:58 +00:00
|
|
|
LOG_LEVEL="DEBUG"
|
2020-05-14 17:34:47 +00:00
|
|
|
BASE_PORT="9000"
|
|
|
|
BASE_METRICS_PORT="8008"
|
2020-02-18 23:33:26 +00:00
|
|
|
|
|
|
|
print_help() {
|
|
|
|
cat <<EOF
|
|
|
|
Usage: $(basename $0) --testnet <testnet number> [OTHER OPTIONS] -- [BEACON NODE OPTIONS]
|
|
|
|
E.g.: $(basename $0) --testnet ${TESTNET} --nodes ${NUM_NODES} --data-dir "${DATA_DIR}" # defaults
|
|
|
|
CI run: $(basename $0) --disable-htop -- --verify-finalization --stop-at-epoch=5
|
|
|
|
|
2020-05-14 17:34:47 +00:00
|
|
|
-h, --help this help message
|
|
|
|
-t, --testnet testnet number (default: ${TESTNET})
|
|
|
|
-n, --nodes number of nodes to launch (default: ${NUM_NODES})
|
2020-06-22 18:22:45 +00:00
|
|
|
-g, --with-ganache simulate a genesis event based on a deposit contract
|
2020-05-14 17:34:47 +00:00
|
|
|
-d, --data-dir directory where all the node data and logs will end up
|
2020-06-22 18:22:45 +00:00
|
|
|
(default: "${DATA_DIR}")
|
2020-05-14 17:34:47 +00:00
|
|
|
--base-port bootstrap node's Eth2 traffic port (default: ${BASE_PORT})
|
|
|
|
--base-metrics-port bootstrap node's metrics server port (default: ${BASE_METRICS_PORT})
|
|
|
|
--disable-htop don't use "htop" to see the beacon_node processes
|
|
|
|
--log-level set the log level (default: ${LOG_LEVEL})
|
2020-02-18 23:33:26 +00:00
|
|
|
EOF
|
|
|
|
}
|
|
|
|
|
|
|
|
! PARSED=$(getopt --options=${OPTS} --longoptions=${LONGOPTS} --name "$0" -- "$@")
|
|
|
|
if [ ${PIPESTATUS[0]} != 0 ]; then
|
|
|
|
# getopt has complained about wrong arguments to stdout
|
|
|
|
exit 1
|
|
|
|
fi
|
|
|
|
|
|
|
|
# read getopt's output this way to handle the quoting right
|
|
|
|
eval set -- "$PARSED"
|
|
|
|
while true; do
|
|
|
|
case "$1" in
|
|
|
|
-h|--help)
|
|
|
|
print_help
|
|
|
|
exit
|
|
|
|
;;
|
|
|
|
-t|--testnet)
|
|
|
|
TESTNET="$2"
|
|
|
|
shift 2
|
|
|
|
;;
|
|
|
|
-n|--nodes)
|
|
|
|
NUM_NODES="$2"
|
|
|
|
shift 2
|
|
|
|
;;
|
|
|
|
-d|--data-dir)
|
|
|
|
DATA_DIR="$2"
|
|
|
|
shift 2
|
|
|
|
;;
|
|
|
|
--disable-htop)
|
|
|
|
USE_HTOP="0"
|
|
|
|
shift
|
|
|
|
;;
|
2020-06-22 18:22:45 +00:00
|
|
|
-g|--with-ganache)
|
|
|
|
USE_GANACHE="1"
|
|
|
|
shift
|
|
|
|
;;
|
2020-03-09 00:32:58 +00:00
|
|
|
--log-level)
|
|
|
|
LOG_LEVEL="$2"
|
|
|
|
shift 2
|
|
|
|
;;
|
2020-05-14 17:34:47 +00:00
|
|
|
--base-port)
|
|
|
|
BASE_PORT="$2"
|
|
|
|
shift 2
|
|
|
|
;;
|
|
|
|
--base-metrics-port)
|
|
|
|
BASE_METRICS_PORT="$2"
|
|
|
|
shift 2
|
|
|
|
;;
|
2020-02-18 23:33:26 +00:00
|
|
|
--)
|
|
|
|
shift
|
|
|
|
break
|
|
|
|
;;
|
|
|
|
*)
|
|
|
|
echo "argument parsing error"
|
|
|
|
print_help
|
|
|
|
exit 1
|
|
|
|
esac
|
|
|
|
done
|
|
|
|
|
2020-02-26 00:14:20 +00:00
|
|
|
# when sourcing env.sh, it will try to execute $@, so empty it
|
|
|
|
EXTRA_ARGS="$@"
|
|
|
|
if [[ $# != 0 ]]; then
|
|
|
|
shift $#
|
|
|
|
fi
|
2020-02-18 23:33:26 +00:00
|
|
|
NETWORK="testnet${TESTNET}"
|
|
|
|
|
2020-01-20 22:28:48 +00:00
|
|
|
rm -rf "${DATA_DIR}"
|
2020-06-01 19:48:20 +00:00
|
|
|
|
2020-01-20 22:28:48 +00:00
|
|
|
DEPOSITS_DIR="${DATA_DIR}/deposits_dir"
|
|
|
|
mkdir -p "${DEPOSITS_DIR}"
|
2020-06-01 19:48:20 +00:00
|
|
|
|
|
|
|
SECRETS_DIR="${DATA_DIR}/secrets"
|
|
|
|
mkdir -p "${SECRETS_DIR}"
|
|
|
|
|
2020-01-20 22:28:48 +00:00
|
|
|
NETWORK_DIR="${DATA_DIR}/network_dir"
|
|
|
|
mkdir -p "${NETWORK_DIR}"
|
|
|
|
|
|
|
|
set -a
|
|
|
|
source "scripts/${NETWORK}.env"
|
|
|
|
set +a
|
|
|
|
|
2020-02-18 23:33:26 +00:00
|
|
|
# Windows detection
|
|
|
|
if uname | grep -qiE "mingw|msys"; then
|
|
|
|
MAKE="mingw32-make"
|
|
|
|
else
|
|
|
|
MAKE="make"
|
|
|
|
fi
|
|
|
|
|
2020-01-20 22:28:48 +00:00
|
|
|
NETWORK_NIM_FLAGS=$(scripts/load-testnet-nim-flags.sh ${NETWORK})
|
2020-06-10 15:21:32 +00:00
|
|
|
$MAKE LOG_LEVEL="${LOG_LEVEL}" NIMFLAGS="-d:insecure -d:testnet_servers_image ${NETWORK_NIM_FLAGS}" beacon_node
|
2020-01-20 22:28:48 +00:00
|
|
|
|
2020-06-22 18:22:45 +00:00
|
|
|
PIDS=""
|
|
|
|
WEB3_ARG=""
|
|
|
|
STATE_SNAPSHOT_ARG=""
|
|
|
|
BOOTSTRAP_TIMEOUT=10 # in seconds
|
2020-07-09 22:08:54 +00:00
|
|
|
DEPOSIT_CONTRACT_ADDRESS="0x0000000000000000000000000000000000000000"
|
|
|
|
DEPOSIT_CONTRACT_BLOCK="0x0000000000000000000000000000000000000000000000000000000000000000"
|
|
|
|
NETWORK_METADATA_FILE="${DATA_DIR}/network.json"
|
2020-06-22 18:22:45 +00:00
|
|
|
|
2020-06-15 22:20:31 +00:00
|
|
|
./build/beacon_node deposits create \
|
2020-06-01 19:48:20 +00:00
|
|
|
--count=${TOTAL_VALIDATORS} \
|
2020-06-16 18:38:51 +00:00
|
|
|
--out-deposits-dir="${DEPOSITS_DIR}" \
|
2020-06-15 22:20:31 +00:00
|
|
|
--out-secrets-dir="${SECRETS_DIR}" \
|
|
|
|
--dont-send
|
2020-01-20 22:28:48 +00:00
|
|
|
|
2020-06-22 18:22:45 +00:00
|
|
|
if [[ $USE_GANACHE == "0" ]]; then
|
|
|
|
GENESIS_OFFSET=30
|
|
|
|
BOOTSTRAP_IP="127.0.0.1"
|
|
|
|
|
|
|
|
./build/beacon_node createTestnet \
|
|
|
|
--data-dir="${DATA_DIR}/node0" \
|
|
|
|
--validators-dir="${DEPOSITS_DIR}" \
|
|
|
|
--total-validators=${TOTAL_VALIDATORS} \
|
|
|
|
--last-user-validator=${USER_VALIDATORS} \
|
|
|
|
--output-genesis="${NETWORK_DIR}/genesis.ssz" \
|
|
|
|
--output-bootstrap-file="${NETWORK_DIR}/bootstrap_nodes.txt" \
|
|
|
|
--bootstrap-address=${BOOTSTRAP_IP} \
|
|
|
|
--bootstrap-port=${BASE_PORT} \
|
|
|
|
--genesis-offset=${GENESIS_OFFSET} # Delay in seconds
|
|
|
|
|
|
|
|
STATE_SNAPSHOT_ARG="--state-snapshot=${NETWORK_DIR}/genesis.ssz"
|
|
|
|
else
|
|
|
|
make deposit_contract
|
|
|
|
|
2020-06-25 09:42:17 +00:00
|
|
|
echo "Launching ganache"
|
2020-06-22 18:22:45 +00:00
|
|
|
ganache-cli --blockTime 17 --gasLimit 100000000 -e 100000 --verbose > "${DATA_DIR}/log_ganache.txt" 2>&1 &
|
|
|
|
PIDS="${PIDS},$!"
|
|
|
|
|
|
|
|
WEB3_ARG="--web3-url=ws://localhost:8545"
|
2020-07-09 22:08:54 +00:00
|
|
|
|
|
|
|
echo "Deploying deposit contract"
|
|
|
|
DEPLOY_CMD_OUTPUT=$(./build/deposit_contract deploy $WEB3_ARG)
|
|
|
|
# https://stackoverflow.com/questions/918886/how-do-i-split-a-string-on-a-delimiter-in-bash
|
|
|
|
OUTPUT_PIECES=(${DEPLOY_CMD_OUTPUT//;/ })
|
|
|
|
DEPOSIT_CONTRACT_ADDRESS=${OUTPUT_PIECES[0]}
|
|
|
|
DEPOSIT_CONTRACT_BLOCK=${OUTPUT_PIECES[1]}
|
|
|
|
|
|
|
|
echo Contract deployed at $DEPOSIT_CONTRACT_ADDRESS:$DEPOSIT_CONTRACT_BLOCK
|
2020-06-11 13:31:35 +00:00
|
|
|
|
2020-06-22 18:22:45 +00:00
|
|
|
MIN_DELAY=1
|
|
|
|
MAX_DELAY=5
|
|
|
|
|
|
|
|
BOOTSTRAP_TIMEOUT=$(( MAX_DELAY * TOTAL_VALIDATORS ))
|
|
|
|
|
|
|
|
./build/beacon_node deposits send \
|
|
|
|
--non-interactive \
|
|
|
|
--deposits-dir="${DEPOSITS_DIR}" \
|
|
|
|
--min-delay=$MIN_DELAY --max-delay=$MAX_DELAY \
|
|
|
|
$WEB3_ARG \
|
2020-07-09 22:08:54 +00:00
|
|
|
--deposit-contract=${DEPOSIT_CONTRACT_ADDRESS} > "${DATA_DIR}/log_deposit_maker.txt" 2>&1 &
|
2020-06-22 18:22:45 +00:00
|
|
|
|
|
|
|
PIDS="${PIDS},$!"
|
|
|
|
fi
|
2020-01-20 22:28:48 +00:00
|
|
|
|
2020-06-10 15:21:32 +00:00
|
|
|
./scripts/make_prometheus_config.sh \
|
2020-06-22 18:22:45 +00:00
|
|
|
--nodes ${NUM_NODES} \
|
|
|
|
--base-metrics-port ${BASE_METRICS_PORT} \
|
2020-06-25 09:42:17 +00:00
|
|
|
--config-file "${DATA_DIR}/prometheus.yml" || true # TODO: this currently fails on macOS,
|
|
|
|
# but it can be considered non-critical
|
2020-05-05 22:02:39 +00:00
|
|
|
|
2020-07-09 22:08:54 +00:00
|
|
|
echo Wrote $NETWORK_METADATA_FILE:
|
|
|
|
tee "$NETWORK_METADATA_FILE" <<EOF
|
|
|
|
{
|
|
|
|
"runtimePreset": {
|
|
|
|
"MIN_GENESIS_ACTIVE_VALIDATOR_COUNT": ${TOTAL_VALIDATORS},
|
|
|
|
"MIN_GENESIS_TIME": 0,
|
|
|
|
"GENESIS_DELAY": 10,
|
|
|
|
"GENESIS_FORK_VERSION": "0x00000000"
|
|
|
|
},
|
|
|
|
"depositContractAddress": "${DEPOSIT_CONTRACT_ADDRESS}",
|
|
|
|
"depositContractDeployedAt": "${DEPOSIT_CONTRACT_BLOCK}"
|
|
|
|
}
|
|
|
|
EOF
|
|
|
|
|
2020-04-30 13:59:57 +00:00
|
|
|
# Kill child processes on Ctrl-C/SIGTERM/exit, passing the PID of this shell
|
|
|
|
# instance as the parent and the target process name as a pattern to the
|
|
|
|
# "pkill" command.
|
2020-01-20 22:28:48 +00:00
|
|
|
cleanup() {
|
2020-04-30 13:59:57 +00:00
|
|
|
pkill -P $$ beacon_node &>/dev/null || true
|
2020-01-20 22:28:48 +00:00
|
|
|
sleep 2
|
2020-04-30 13:59:57 +00:00
|
|
|
pkill -9 -P $$ beacon_node &>/dev/null || true
|
2020-01-20 22:28:48 +00:00
|
|
|
}
|
2020-04-30 13:59:57 +00:00
|
|
|
trap 'cleanup' SIGINT SIGTERM EXIT
|
2020-01-20 22:28:48 +00:00
|
|
|
|
2020-05-01 22:09:04 +00:00
|
|
|
dump_logs() {
|
|
|
|
LOG_LINES=20
|
|
|
|
for LOG in "${DATA_DIR}"/log*.txt; do
|
|
|
|
echo "Last ${LOG_LINES} lines of ${LOG}:"
|
|
|
|
tail -n ${LOG_LINES} "${LOG}"
|
|
|
|
echo "======"
|
|
|
|
done
|
|
|
|
}
|
|
|
|
|
2020-02-21 13:19:34 +00:00
|
|
|
NODES_WITH_VALIDATORS=${NODES_WITH_VALIDATORS:-4}
|
2020-06-22 18:22:45 +00:00
|
|
|
BOOTSTRAP_NODE=$(( NUM_NODES - 1 ))
|
2020-06-03 11:40:14 +00:00
|
|
|
SYSTEM_VALIDATORS=$(( TOTAL_VALIDATORS - USER_VALIDATORS ))
|
|
|
|
VALIDATORS_PER_NODE=$(( SYSTEM_VALIDATORS / NODES_WITH_VALIDATORS ))
|
2020-02-21 13:19:34 +00:00
|
|
|
|
2020-06-22 18:22:45 +00:00
|
|
|
for NUM_NODE in $(seq $BOOTSTRAP_NODE -1 0); do
|
|
|
|
if [[ ${NUM_NODE} == ${BOOTSTRAP_NODE} ]]; then
|
2020-01-20 22:28:48 +00:00
|
|
|
BOOTSTRAP_ARG=""
|
|
|
|
else
|
2020-06-22 18:22:45 +00:00
|
|
|
BOOTSTRAP_ENR="${DATA_DIR}/node${BOOTSTRAP_NODE}/beacon_node.enr"
|
|
|
|
BOOTSTRAP_ARG="--bootstrap-file=${BOOTSTRAP_ENR}"
|
2020-01-20 22:28:48 +00:00
|
|
|
# Wait for the master node to write out its address file
|
2020-05-01 18:57:12 +00:00
|
|
|
START_TIMESTAMP=$(date +%s)
|
2020-06-22 18:22:45 +00:00
|
|
|
while [ ! -f "${BOOTSTRAP_ENR}" ]; do
|
2020-01-20 22:28:48 +00:00
|
|
|
sleep 0.1
|
2020-05-01 18:57:12 +00:00
|
|
|
NOW_TIMESTAMP=$(date +%s)
|
2020-06-11 13:31:35 +00:00
|
|
|
if [[ "$(( NOW_TIMESTAMP - START_TIMESTAMP - GENESIS_OFFSET ))" -ge "$BOOTSTRAP_TIMEOUT" ]]; then
|
2020-05-01 18:57:12 +00:00
|
|
|
echo "Bootstrap node failed to start in ${BOOTSTRAP_TIMEOUT} seconds. Aborting."
|
2020-05-01 22:09:04 +00:00
|
|
|
dump_logs
|
2020-05-01 18:57:12 +00:00
|
|
|
exit 1
|
|
|
|
fi
|
2020-01-20 22:28:48 +00:00
|
|
|
done
|
|
|
|
fi
|
|
|
|
|
2020-02-21 13:19:34 +00:00
|
|
|
# Copy validators to individual nodes.
|
2020-06-01 19:48:20 +00:00
|
|
|
# The first $NODES_WITH_VALIDATORS nodes split them equally between them, after skipping the first $USER_VALIDATORS.
|
2020-02-21 13:19:34 +00:00
|
|
|
NODE_DATA_DIR="${DATA_DIR}/node${NUM_NODE}"
|
|
|
|
mkdir -p "${NODE_DATA_DIR}/validators"
|
2020-06-02 19:59:51 +00:00
|
|
|
mkdir -p "${NODE_DATA_DIR}/secrets"
|
|
|
|
|
2020-02-21 13:19:34 +00:00
|
|
|
if [[ $NUM_NODE -lt $NODES_WITH_VALIDATORS ]]; then
|
2020-06-03 11:40:14 +00:00
|
|
|
for VALIDATOR in $(ls ${DEPOSITS_DIR} | tail -n +$(( $USER_VALIDATORS + ($VALIDATORS_PER_NODE * $NUM_NODE) + 1 )) | head -n $VALIDATORS_PER_NODE); do
|
|
|
|
cp -ar "${DEPOSITS_DIR}/$VALIDATOR" "${NODE_DATA_DIR}/validators/"
|
2020-06-03 11:52:36 +00:00
|
|
|
cp -a "${SECRETS_DIR}/${VALIDATOR}" "${NODE_DATA_DIR}/secrets/"
|
2020-02-21 13:19:34 +00:00
|
|
|
done
|
|
|
|
fi
|
|
|
|
|
2020-04-30 13:59:57 +00:00
|
|
|
./build/beacon_node \
|
2020-06-03 11:40:14 +00:00
|
|
|
--non-interactive \
|
2020-06-22 18:22:45 +00:00
|
|
|
--nat:extip:127.0.0.1 \
|
2020-07-09 22:08:54 +00:00
|
|
|
--network="${NETWORK_METADATA_FILE}" \
|
2020-03-09 00:32:58 +00:00
|
|
|
--log-level="${LOG_LEVEL}" \
|
2020-05-14 17:34:47 +00:00
|
|
|
--tcp-port=$(( BASE_PORT + NUM_NODE )) \
|
|
|
|
--udp-port=$(( BASE_PORT + NUM_NODE )) \
|
2020-02-21 13:19:34 +00:00
|
|
|
--data-dir="${NODE_DATA_DIR}" \
|
2020-01-20 22:28:48 +00:00
|
|
|
${BOOTSTRAP_ARG} \
|
2020-06-22 18:22:45 +00:00
|
|
|
${STATE_SNAPSHOT_ARG} \
|
|
|
|
${WEB3_ARG} \
|
2020-05-05 22:02:39 +00:00
|
|
|
--metrics \
|
|
|
|
--metrics-address="127.0.0.1" \
|
|
|
|
--metrics-port="$(( BASE_METRICS_PORT + NUM_NODE ))" \
|
2020-02-26 00:14:20 +00:00
|
|
|
${EXTRA_ARGS} \
|
2020-01-20 22:28:48 +00:00
|
|
|
> "${DATA_DIR}/log${NUM_NODE}.txt" 2>&1 &
|
2020-05-01 18:57:12 +00:00
|
|
|
|
2020-01-20 22:28:48 +00:00
|
|
|
if [[ "${PIDS}" == "" ]]; then
|
|
|
|
PIDS="$!"
|
|
|
|
else
|
|
|
|
PIDS="${PIDS},$!"
|
|
|
|
fi
|
|
|
|
done
|
|
|
|
|
2020-05-01 18:57:12 +00:00
|
|
|
# give the regular nodes time to crash
|
|
|
|
sleep 5
|
|
|
|
BG_JOBS="$(jobs | wc -l)"
|
|
|
|
if [[ "$BG_JOBS" != "$NUM_NODES" ]]; then
|
|
|
|
echo "$((NUM_NODES - BG_JOBS)) beacon_node instance(s) exited early. Aborting."
|
2020-05-01 22:09:04 +00:00
|
|
|
dump_logs
|
2020-05-01 18:57:12 +00:00
|
|
|
exit 1
|
|
|
|
fi
|
|
|
|
|
2020-02-18 23:33:26 +00:00
|
|
|
if [[ "$USE_HTOP" == "1" ]]; then
|
|
|
|
htop -p "$PIDS"
|
|
|
|
cleanup
|
|
|
|
else
|
|
|
|
FAILED=0
|
|
|
|
for PID in $(echo "$PIDS" | tr ',' ' '); do
|
|
|
|
wait $PID || FAILED="$(( FAILED += 1 ))"
|
|
|
|
done
|
|
|
|
if [[ "$FAILED" != "0" ]]; then
|
|
|
|
echo "${FAILED} child processes had non-zero exit codes (or exited early)."
|
2020-05-01 22:09:04 +00:00
|
|
|
dump_logs
|
2020-02-18 23:33:26 +00:00
|
|
|
exit 1
|
|
|
|
fi
|
|
|
|
fi
|
2020-01-20 22:28:48 +00:00
|
|
|
|