From 96e1a5d70eb460a509374e731528d102b50a5fe9 Mon Sep 17 00:00:00 2001 From: Viktor Kirilov Date: Tue, 1 Sep 2020 16:38:34 +0300 Subject: [PATCH] 50/50 bn/vc split for the validator keys ON by default for the testnet scripts --- Jenkinsfile | 8 ++-- README.md | 15 ++---- beacon_chain/beacon_node.nim | 11 +---- beacon_chain/conf.nim | 5 ++ beacon_chain/nimbus_binary_common.nim | 14 +++++- beacon_chain/validator_client.nim | 3 ++ scripts/launch_local_testnet.sh | 67 ++++++++++++++++++++++++--- tests/simulation/vars.sh | 2 +- 8 files changed, 91 insertions(+), 34 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index f71946817..5f45b4cca 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -47,16 +47,16 @@ def runStages() { // EXECUTOR_NUMBER will be 0 or 1, since we have 2 executors per Jenkins node sh """#!/bin/bash set -e - ./scripts/launch_local_testnet.sh --testnet 0 --nodes 4 --log-level DEBUG --disable-htop --enable-logtrace --data-dir local_testnet0_data --base-port \$(( 9000 + EXECUTOR_NUMBER * 100 )) --base-metrics-port \$(( 8008 + EXECUTOR_NUMBER * 100 )) -- --verify-finalization --stop-at-epoch=5 --discv5:no - ./scripts/launch_local_testnet.sh --testnet 1 --nodes 4 --log-level DEBUG --disable-htop --enable-logtrace --data-dir local_testnet1_data --base-port \$(( 9000 + EXECUTOR_NUMBER * 100 )) --base-metrics-port \$(( 8008 + EXECUTOR_NUMBER * 100 )) -- --verify-finalization --stop-at-epoch=5 --discv5:no + ./scripts/launch_local_testnet.sh --testnet 0 --nodes 4 --stop-at-epoch 5 --log-level DEBUG --disable-htop --enable-logtrace --data-dir local_testnet0_data --base-port \$(( 9000 + EXECUTOR_NUMBER * 100 )) --base-rpc-port \$(( 7000 + EXECUTOR_NUMBER * 100 )) --base-metrics-port \$(( 8008 + EXECUTOR_NUMBER * 100 )) -- --verify-finalization --discv5:no + ./scripts/launch_local_testnet.sh --testnet 1 --nodes 4 --stop-at-epoch 5 --log-level DEBUG --disable-htop --enable-logtrace --data-dir local_testnet1_data --base-port \$(( 9000 + EXECUTOR_NUMBER * 100 )) --base-rpc-port \$(( 7000 + EXECUTOR_NUMBER * 100 )) --base-metrics-port \$(( 8008 + EXECUTOR_NUMBER * 100 )) -- --verify-finalization --discv5:no """ } // stage("testnet finalization - Miracl/Milagro fallback") { // // EXECUTOR_NUMBER will be 0 or 1, since we have 2 executors per Jenkins node // sh """#!/bin/bash // set -e - // NIMFLAGS="-d:BLS_FORCE_BACKEND=miracl" ./scripts/launch_local_testnet.sh --testnet 0 --nodes 4 --log-level INFO --disable-htop --data-dir local_testnet0_data --base-port \$(( 9000 + EXECUTOR_NUMBER * 100 )) --base-metrics-port \$(( 8008 + EXECUTOR_NUMBER * 100 )) -- --verify-finalization --stop-at-epoch=5 - // NIMFLAGS="-d:BLS_FORCE_BACKEND=miracl" ./scripts/launch_local_testnet.sh --testnet 1 --nodes 4 --log-level INFO --disable-htop --data-dir local_testnet1_data --base-port \$(( 9000 + EXECUTOR_NUMBER * 100 )) --base-metrics-port \$(( 8008 + EXECUTOR_NUMBER * 100 )) -- --verify-finalization --stop-at-epoch=5 + // NIMFLAGS="-d:BLS_FORCE_BACKEND=miracl" ./scripts/launch_local_testnet.sh --testnet 0 --nodes 4 --stop-at-epoch 5 --log-level INFO --disable-htop --data-dir local_testnet0_data --base-port \$(( 9000 + EXECUTOR_NUMBER * 100 )) --base-rpc-port \$(( 7000 + EXECUTOR_NUMBER * 100 )) --base-metrics-port \$(( 8008 + EXECUTOR_NUMBER * 100 )) -- --verify-finalization + // NIMFLAGS="-d:BLS_FORCE_BACKEND=miracl" ./scripts/launch_local_testnet.sh --testnet 1 --nodes 4 --stop-at-epoch 5 --log-level INFO --disable-htop --data-dir local_testnet1_data --base-port \$(( 9000 + EXECUTOR_NUMBER * 100 )) --base-rpc-port \$(( 7000 + EXECUTOR_NUMBER * 100 )) --base-metrics-port \$(( 8008 + EXECUTOR_NUMBER * 100 )) -- --verify-finalization // """ // } } diff --git a/README.md b/README.md index 71f4d7924..011d1123c 100644 --- a/README.md +++ b/README.md @@ -214,23 +214,14 @@ make VALIDATORS=192 NODES=6 USER_NODES=1 eth2_network_simulation # looks like from a single nodes' perspective. ``` -By default, all validators are loaded within the beacon nodes, but if you want to use -external processes as validator clients you can pass `BN_VC_VALIDATOR_SPLIT=yes` as an -additional argument to the `make eth2_network_simulation` command and that will split -the `VALIDATORS` between beacon nodes and validator clients - for example with `192` -validators and `6` nodes you will end up with 6 beacon node and 6 validator client -processes, where each of them will handle 16 validators. - - +want to have all the validators handled by the beacon nodes you may use +`BN_VC_VALIDATOR_SPLIT=no` as an additional argument to `make eth2_network_simulation`. By default, the simulation will start from a pre-generated genesis state. If you wish to simulate the bootstrap process with a Ethereum 1.0 validator deposit contract, start the diff --git a/beacon_chain/beacon_node.nim b/beacon_chain/beacon_node.nim index af058c80f..59412b47c 100644 --- a/beacon_chain/beacon_node.nim +++ b/beacon_chain/beacon_node.nim @@ -422,16 +422,7 @@ proc onSlotStart(node: BeaconNode, lastSlot, scheduledSlot: Slot) {.gcsafe, asyn finalizedEpoch = shortLog(finalizedEpoch) # Check before any re-scheduling of onSlotStart() - # Offset backwards slightly to allow this epoch's finalization check to occur - if scheduledSlot > 3 and node.config.stopAtEpoch > 0'u64 and - (scheduledSlot - 3).compute_epoch_at_slot() >= node.config.stopAtEpoch: - info "Stopping at pre-chosen epoch", - chosenEpoch = node.config.stopAtEpoch, - epoch = scheduledSlot.compute_epoch_at_slot(), - slot = scheduledSlot - - # Brute-force, but ensure it's reliable enough to run in CI. - quit(0) + checkIfShouldStopAtEpoch(scheduledSlot, node.config.stopAtEpoch) if not wallSlot.afterGenesis or (wallSlot.slot < lastSlot): let diff --git a/beacon_chain/conf.nim b/beacon_chain/conf.nim index 4eee20476..315a843c9 100644 --- a/beacon_chain/conf.nim +++ b/beacon_chain/conf.nim @@ -366,6 +366,11 @@ type "You can use a 0x-prefixed hex encoded string to specify raw bytes." name: "graffiti" }: Option[GraffitiBytes] + stopAtEpoch* {. + defaultValue: 0 + desc: "A positive epoch selects the epoch at which to stop" + name: "stop-at-epoch" }: uint64 + rpcPort* {. defaultValue: defaultEth2RpcPort desc: "HTTP port of the server to connect to for RPC" diff --git a/beacon_chain/nimbus_binary_common.nim b/beacon_chain/nimbus_binary_common.nim index 7a0246a51..113ddd7b9 100644 --- a/beacon_chain/nimbus_binary_common.nim +++ b/beacon_chain/nimbus_binary_common.nim @@ -16,7 +16,7 @@ import chronicles, chronicles/helpers as chroniclesHelpers, # Local modules - spec/[datatypes, crypto], eth2_network, time + spec/[datatypes, crypto, helpers], eth2_network, time proc setupLogging*(logLevel: string, logFile: Option[OutFile]) = when compiles(defaultChroniclesStream.output.writer): @@ -88,3 +88,15 @@ proc sleepToSlotOffset*(clock: BeaconClock, extra: chronos.Duration, await sleepAsync(fromNow.offset) return true return false + +proc checkIfShouldStopAtEpoch*(scheduledSlot: Slot, stopAtEpoch: uint64) = + # Offset backwards slightly to allow this epoch's finalization check to occur + if scheduledSlot > 3 and stopAtEpoch > 0'u64 and + (scheduledSlot - 3).compute_epoch_at_slot() >= stopAtEpoch: + info "Stopping at pre-chosen epoch", + chosenEpoch = stopAtEpoch, + epoch = scheduledSlot.compute_epoch_at_slot(), + slot = scheduledSlot + + # Brute-force, but ensure it's reliable enough to run in CI. + quit(0) diff --git a/beacon_chain/validator_client.nim b/beacon_chain/validator_client.nim index b479561e7..92e10e225 100644 --- a/beacon_chain/validator_client.nim +++ b/beacon_chain/validator_client.nim @@ -117,6 +117,9 @@ proc onSlotStart(vc: ValidatorClient, lastSlot, scheduledSlot: Slot) {.gcsafe, a beaconTime = shortLog(beaconTime), portBN = vc.config.rpcPort + # Check before any re-scheduling of onSlotStart() + checkIfShouldStopAtEpoch(scheduledSlot, vc.config.stopAtEpoch) + try: # at the start of each epoch - request all validator duties # TODO perhaps call this not on the first slot of each Epoch but perhaps diff --git a/scripts/launch_local_testnet.sh b/scripts/launch_local_testnet.sh index 9c99547af..fa9d23c2a 100755 --- a/scripts/launch_local_testnet.sh +++ b/scripts/launch_local_testnet.sh @@ -32,35 +32,41 @@ if [ ${PIPESTATUS[0]} != 4 ]; then fi OPTS="hgt:n:d:" -LONGOPTS="help,testnet:,nodes:,data-dir:,disable-htop,enable-logtrace,log-level:,base-port:,base-metrics-port:,with-ganache,reuse-existing-data-dir" +LONGOPTS="help,testnet:,nodes:,data-dir:,stop-at-epoch:,disable-htop,disable-vc,enable-logtrace,log-level:,base-port:,base-rpc-port:,base-metrics-port:,with-ganache,reuse-existing-data-dir" # default values TESTNET="1" NUM_NODES="10" DATA_DIR="local_testnet_data" USE_HTOP="1" +USE_VC="1" USE_GANACHE="0" LOG_LEVEL="DEBUG" BASE_PORT="9000" BASE_METRICS_PORT="8008" +BASE_RPC_PORT="7000" REUSE_EXISTING_DATA_DIR="0" ENABLE_LOGTRACE="0" +STOP_AT_EPOCH_FLAG="" print_help() { cat < [OTHER OPTIONS] -- [BEACON NODE OPTIONS] -E.g.: $(basename "$0") --testnet ${TESTNET} --nodes ${NUM_NODES} --data-dir "${DATA_DIR}" # defaults -CI run: $(basename "$0") --disable-htop -- --verify-finalization --stop-at-epoch=5 +E.g.: $(basename "$0") --testnet ${TESTNET} --nodes ${NUM_NODES} --stop-at-epoch 5 --data-dir "${DATA_DIR}" # defaults +CI run: $(basename "$0") --disable-htop -- --verify-finalization -h, --help this help message -t, --testnet testnet number (default: ${TESTNET}) -n, --nodes number of nodes to launch (default: ${NUM_NODES}) -g, --with-ganache simulate a genesis event based on a deposit contract + -s, --stop-at-epoch stop simulation at epoch (default: infinite) -d, --data-dir directory where all the node data and logs will end up (default: "${DATA_DIR}") --base-port bootstrap node's Eth2 traffic port (default: ${BASE_PORT}) + --base-rpc-port bootstrap node's RPC port (default: ${BASE_RPC_PORT}) --base-metrics-port bootstrap node's metrics server port (default: ${BASE_METRICS_PORT}) --disable-htop don't use "htop" to see the beacon_node processes + --disable-vc don't use validator client binaries for validators (by default validators are split 50/50 between beacon nodes and validator clients) --enable-logtrace display logtrace asr analysis --log-level set the log level (default: ${LOG_LEVEL}) --reuse-existing-data-dir instead of deleting and recreating the data dir, keep it and reuse everything we can from it @@ -85,6 +91,10 @@ while true; do TESTNET="$2" shift 2 ;; + -n|--stop-at-epoch) + STOP_AT_EPOCH_FLAG="--stop-at-epoch=$2" + shift 2 + ;; -n|--nodes) NUM_NODES="$2" shift 2 @@ -97,6 +107,10 @@ while true; do USE_HTOP="0" shift ;; + --disable-vc) + USE_VC="0" + shift + ;; -g|--with-ganache) USE_GANACHE="1" shift @@ -109,6 +123,10 @@ while true; do BASE_PORT="$2" shift 2 ;; + --base-rpc-port) + BASE_RPC_PORT="$2" + shift 2 + ;; --base-metrics-port) BASE_METRICS_PORT="$2" shift 2 @@ -166,7 +184,7 @@ else fi NETWORK_NIM_FLAGS=$(scripts/load-testnet-nim-flags.sh "${NETWORK}") -$MAKE -j2 LOG_LEVEL="${LOG_LEVEL}" NIMFLAGS="${NIMFLAGS} -d:insecure -d:testnet_servers_image -d:local_testnet ${NETWORK_NIM_FLAGS}" beacon_node deposit_contract +$MAKE -j2 LOG_LEVEL="${LOG_LEVEL}" NIMFLAGS="${NIMFLAGS} -d:insecure -d:testnet_servers_image -d:local_testnet ${NETWORK_NIM_FLAGS}" beacon_node validator_client deposit_contract if [[ "$ENABLE_LOGTRACE" == "1" ]]; then $MAKE LOG_LEVEL="${LOG_LEVEL}" NIMFLAGS="${NIMFLAGS} -d:insecure -d:testnet_servers_image -d:local_testnet ${NETWORK_NIM_FLAGS}" logtrace fi @@ -178,6 +196,7 @@ BOOTSTRAP_TIMEOUT=30 # in seconds DEPOSIT_CONTRACT_ADDRESS="0x0000000000000000000000000000000000000000" DEPOSIT_CONTRACT_BLOCK="0x0000000000000000000000000000000000000000000000000000000000000000" NETWORK_METADATA_FILE="${DATA_DIR}/network.json" +NUM_JOBS=${NUM_NODES} if [[ "$REUSE_EXISTING_DATA_DIR" == "0" ]]; then ./build/deposit_contract generateSimulationDeposits \ @@ -257,8 +276,10 @@ EOF # "pkill" command. cleanup() { pkill -P $$ beacon_node &>/dev/null || true + pkill -P $$ validator_client &>/dev/null || true sleep 2 pkill -9 -P $$ beacon_node &>/dev/null || true + pkill -9 -P $$ validator_client &>/dev/null || true } trap 'cleanup' SIGINT SIGTERM EXIT @@ -281,6 +302,15 @@ NODES_WITH_VALIDATORS=${NODES_WITH_VALIDATORS:-4} BOOTSTRAP_NODE=0 SYSTEM_VALIDATORS=$(( TOTAL_VALIDATORS - USER_VALIDATORS )) VALIDATORS_PER_NODE=$(( SYSTEM_VALIDATORS / NODES_WITH_VALIDATORS )) +if [ "${USE_VC:-}" == "1" ]; then + # if using validator client binaries in addition to beacon nodes we will + # split the keys for this instance in half between the BN and the VC + # and the validators for the BNs will be from the first half of all validators + VALIDATORS_PER_NODE=$((VALIDATORS_PER_NODE / 2 )) + NUM_JOBS=$((NUM_JOBS * 2 )) +fi +VALIDATORS_PER_VALIDATOR=$(( (SYSTEM_VALIDATORS / NODES_WITH_VALIDATORS) / 2 )) +VALIDATOR_OFFSET=$((SYSTEM_VALIDATORS / 2)) BOOTSTRAP_ENR="${DATA_DIR}/node${BOOTSTRAP_NODE}/beacon_node.enr" for NUM_NODE in $(seq 0 $(( NUM_NODES - 1 ))); do @@ -309,6 +339,18 @@ for NUM_NODE in $(seq 0 $(( NUM_NODES - 1 ))); do mkdir -p "${NODE_DATA_DIR}/secrets" if [[ $NUM_NODE -lt $NODES_WITH_VALIDATORS ]]; then + if [ "${USE_VC:-}" == "1" ]; then + VALIDATOR_DATA_DIR="${DATA_DIR}/validator${NUM_NODE}" + rm -rf "${VALIDATOR_DATA_DIR}" + mkdir -p "${VALIDATOR_DATA_DIR}/validators" + mkdir -p "${VALIDATOR_DATA_DIR}/secrets" + + for VALIDATOR in $(ls "${VALIDATORS_DIR}" | tail -n +$(( $USER_VALIDATORS + ($VALIDATORS_PER_VALIDATOR * $NUM_NODE) + 1 + $VALIDATOR_OFFSET )) | head -n $VALIDATORS_PER_VALIDATOR); do + cp -a "${VALIDATORS_DIR}/$VALIDATOR" "${VALIDATOR_DATA_DIR}/validators/" + cp -a "${SECRETS_DIR}/${VALIDATOR}" "${VALIDATOR_DATA_DIR}/secrets/" + done + fi + for VALIDATOR in $(ls "${VALIDATORS_DIR}" | tail -n +$(( $USER_VALIDATORS + ($VALIDATORS_PER_NODE * $NUM_NODE) + 1 )) | head -n $VALIDATORS_PER_NODE); do cp -a "${VALIDATORS_DIR}/$VALIDATOR" "${NODE_DATA_DIR}/validators/" cp -a "${SECRETS_DIR}/${VALIDATOR}" "${NODE_DATA_DIR}/secrets/" @@ -326,6 +368,10 @@ for NUM_NODE in $(seq 0 $(( NUM_NODES - 1 ))); do ${BOOTSTRAP_ARG} \ ${STATE_SNAPSHOT_ARG} \ ${WEB3_ARG} \ + ${STOP_AT_EPOCH_FLAG} \ + --rpc \ + --rpc-address="127.0.0.1" \ + --rpc-port="$(( BASE_RPC_PORT + NUM_NODE ))" \ --metrics \ --metrics-address="127.0.0.1" \ --metrics-port="$(( BASE_METRICS_PORT + NUM_NODE ))" \ @@ -337,13 +383,22 @@ for NUM_NODE in $(seq 0 $(( NUM_NODES - 1 ))); do else PIDS="${PIDS},$!" fi + + if [ "${USE_VC:-}" == "1" ]; then + ./build/validator_client \ + --log-level="${LOG_LEVEL}" \ + ${STOP_AT_EPOCH_FLAG} \ + --data-dir="${VALIDATOR_DATA_DIR}" \ + --rpc-port="$(( BASE_RPC_PORT + NUM_NODE ))" \ + > "${DATA_DIR}/log_val${NUM_NODE}.txt" 2>&1 & PIDS="${PIDS},$!" + fi done # give the regular nodes time to crash sleep 5 BG_JOBS="$(jobs | wc -l | tr -d ' ')" -if [[ "$BG_JOBS" != "$NUM_NODES" ]]; then - echo "$((NUM_NODES - BG_JOBS)) beacon_node instance(s) exited early. Aborting." +if [[ "$BG_JOBS" != "$NUM_JOBS" ]]; then + echo "$(( NUM_JOBS - BG_JOBS )) beacon_node/validator_client instance(s) exited early. Aborting." dump_logs dump_logtrace exit 1 diff --git a/tests/simulation/vars.sh b/tests/simulation/vars.sh index d7d011310..79a573d16 100644 --- a/tests/simulation/vars.sh +++ b/tests/simulation/vars.sh @@ -24,7 +24,7 @@ TOTAL_NODES=${NODES:-4} TOTAL_USER_NODES=${USER_NODES:-0} TOTAL_SYSTEM_NODES=$(( TOTAL_NODES - TOTAL_USER_NODES )) BOOTSTRAP_NODE=$(( TOTAL_NODES - 1 )) -USE_BN_VC_VALIDATOR_SPLIT=${BN_VC_VALIDATOR_SPLIT:-no} +USE_BN_VC_VALIDATOR_SPLIT=${BN_VC_VALIDATOR_SPLIT:-yes} SIMULATION_DIR="${SIM_ROOT}/data" METRICS_DIR="${SIM_ROOT}/prometheus"