50/50 bn/vc split for the validator keys ON by default for the testnet scripts

This commit is contained in:
Viktor Kirilov 2020-09-01 16:38:34 +03:00
parent 6000a49e84
commit 65d7787b1e
8 changed files with 91 additions and 34 deletions

8
Jenkinsfile vendored
View File

@ -47,16 +47,16 @@ def runStages() {
// EXECUTOR_NUMBER will be 0 or 1, since we have 2 executors per Jenkins node // EXECUTOR_NUMBER will be 0 or 1, since we have 2 executors per Jenkins node
sh """#!/bin/bash sh """#!/bin/bash
set -e set -e
./scripts/launch_local_testnet.sh --testnet 0 --nodes 4 --log-level DEBUG --disable-htop --enable-logtrace --data-dir local_testnet0_data --base-port \$(( 9000 + EXECUTOR_NUMBER * 100 )) --base-metrics-port \$(( 8008 + EXECUTOR_NUMBER * 100 )) -- --verify-finalization --stop-at-epoch=5 --discv5:no ./scripts/launch_local_testnet.sh --testnet 0 --nodes 4 --stop-at-epoch 5 --log-level DEBUG --disable-htop --enable-logtrace --data-dir local_testnet0_data --base-port \$(( 9000 + EXECUTOR_NUMBER * 100 )) --base-rpc-port \$(( 7000 + EXECUTOR_NUMBER * 100 )) --base-metrics-port \$(( 8008 + EXECUTOR_NUMBER * 100 )) -- --verify-finalization --discv5:no
./scripts/launch_local_testnet.sh --testnet 1 --nodes 4 --log-level DEBUG --disable-htop --enable-logtrace --data-dir local_testnet1_data --base-port \$(( 9000 + EXECUTOR_NUMBER * 100 )) --base-metrics-port \$(( 8008 + EXECUTOR_NUMBER * 100 )) -- --verify-finalization --stop-at-epoch=5 --discv5:no ./scripts/launch_local_testnet.sh --testnet 1 --nodes 4 --stop-at-epoch 5 --log-level DEBUG --disable-htop --enable-logtrace --data-dir local_testnet1_data --base-port \$(( 9000 + EXECUTOR_NUMBER * 100 )) --base-rpc-port \$(( 7000 + EXECUTOR_NUMBER * 100 )) --base-metrics-port \$(( 8008 + EXECUTOR_NUMBER * 100 )) -- --verify-finalization --discv5:no
""" """
} }
// stage("testnet finalization - Miracl/Milagro fallback") { // stage("testnet finalization - Miracl/Milagro fallback") {
// // EXECUTOR_NUMBER will be 0 or 1, since we have 2 executors per Jenkins node // // EXECUTOR_NUMBER will be 0 or 1, since we have 2 executors per Jenkins node
// sh """#!/bin/bash // sh """#!/bin/bash
// set -e // set -e
// NIMFLAGS="-d:BLS_FORCE_BACKEND=miracl" ./scripts/launch_local_testnet.sh --testnet 0 --nodes 4 --log-level INFO --disable-htop --data-dir local_testnet0_data --base-port \$(( 9000 + EXECUTOR_NUMBER * 100 )) --base-metrics-port \$(( 8008 + EXECUTOR_NUMBER * 100 )) -- --verify-finalization --stop-at-epoch=5 // NIMFLAGS="-d:BLS_FORCE_BACKEND=miracl" ./scripts/launch_local_testnet.sh --testnet 0 --nodes 4 --stop-at-epoch 5 --log-level INFO --disable-htop --data-dir local_testnet0_data --base-port \$(( 9000 + EXECUTOR_NUMBER * 100 )) --base-rpc-port \$(( 7000 + EXECUTOR_NUMBER * 100 )) --base-metrics-port \$(( 8008 + EXECUTOR_NUMBER * 100 )) -- --verify-finalization
// NIMFLAGS="-d:BLS_FORCE_BACKEND=miracl" ./scripts/launch_local_testnet.sh --testnet 1 --nodes 4 --log-level INFO --disable-htop --data-dir local_testnet1_data --base-port \$(( 9000 + EXECUTOR_NUMBER * 100 )) --base-metrics-port \$(( 8008 + EXECUTOR_NUMBER * 100 )) -- --verify-finalization --stop-at-epoch=5 // NIMFLAGS="-d:BLS_FORCE_BACKEND=miracl" ./scripts/launch_local_testnet.sh --testnet 1 --nodes 4 --stop-at-epoch 5 --log-level INFO --disable-htop --data-dir local_testnet1_data --base-port \$(( 9000 + EXECUTOR_NUMBER * 100 )) --base-rpc-port \$(( 7000 + EXECUTOR_NUMBER * 100 )) --base-metrics-port \$(( 8008 + EXECUTOR_NUMBER * 100 )) -- --verify-finalization
// """ // """
// } // }
} }

View File

@ -214,23 +214,14 @@ make VALIDATORS=192 NODES=6 USER_NODES=1 eth2_network_simulation
# looks like from a single nodes' perspective. # looks like from a single nodes' perspective.
``` ```
By default, all validators are loaded within the beacon nodes, but if you want to use
external processes as validator clients you can pass `BN_VC_VALIDATOR_SPLIT=yes` as an
additional argument to the `make eth2_network_simulation` command and that will split
the `VALIDATORS` between beacon nodes and validator clients - for example with `192`
validators and `6` nodes you will end up with 6 beacon node and 6 validator client
processes, where each of them will handle 16 validators.
<!--
By default, validators will be split in half between beacon node and validator By default, validators will be split in half between beacon node and validator
client processes, communicating through the client processes (50/50), communicating through the
[official validator API](https://ethereum.github.io/eth2.0-APIs/#/ValidatorRequiredApi) [official validator API](https://ethereum.github.io/eth2.0-APIs/#/ValidatorRequiredApi)
(for example with `192` validators and `6` nodes you will roughly end up with 6 (for example with `192` validators and `6` nodes you will roughly end up with 6
beacon node and 6 validator client processes, where each of them will handle 16 beacon node and 6 validator client processes, where each of them will handle 16
validators), but if you don't want to use external validator clients and instead validators), but if you don't want to use external validator clients and instead
want to have all the validators handled in-process by the beacon nodes you may want to have all the validators handled by the beacon nodes you may use
use `BN_VC_VALIDATOR_SPLIT=no` as an additional argument to `make eth2_network_simulation`. `BN_VC_VALIDATOR_SPLIT=no` as an additional argument to `make eth2_network_simulation`.
-->
By default, the simulation will start from a pre-generated genesis state. If you wish to By default, the simulation will start from a pre-generated genesis state. If you wish to
simulate the bootstrap process with a Ethereum 1.0 validator deposit contract, start the simulate the bootstrap process with a Ethereum 1.0 validator deposit contract, start the

View File

@ -422,16 +422,7 @@ proc onSlotStart(node: BeaconNode, lastSlot, scheduledSlot: Slot) {.gcsafe, asyn
finalizedEpoch = shortLog(finalizedEpoch) finalizedEpoch = shortLog(finalizedEpoch)
# Check before any re-scheduling of onSlotStart() # Check before any re-scheduling of onSlotStart()
# Offset backwards slightly to allow this epoch's finalization check to occur checkIfShouldStopAtEpoch(scheduledSlot, node.config.stopAtEpoch)
if scheduledSlot > 3 and node.config.stopAtEpoch > 0'u64 and
(scheduledSlot - 3).compute_epoch_at_slot() >= node.config.stopAtEpoch:
info "Stopping at pre-chosen epoch",
chosenEpoch = node.config.stopAtEpoch,
epoch = scheduledSlot.compute_epoch_at_slot(),
slot = scheduledSlot
# Brute-force, but ensure it's reliable enough to run in CI.
quit(0)
if not wallSlot.afterGenesis or (wallSlot.slot < lastSlot): if not wallSlot.afterGenesis or (wallSlot.slot < lastSlot):
let let

View File

@ -366,6 +366,11 @@ type
"You can use a 0x-prefixed hex encoded string to specify raw bytes." "You can use a 0x-prefixed hex encoded string to specify raw bytes."
name: "graffiti" }: Option[GraffitiBytes] name: "graffiti" }: Option[GraffitiBytes]
stopAtEpoch* {.
defaultValue: 0
desc: "A positive epoch selects the epoch at which to stop"
name: "stop-at-epoch" }: uint64
rpcPort* {. rpcPort* {.
defaultValue: defaultEth2RpcPort defaultValue: defaultEth2RpcPort
desc: "HTTP port of the server to connect to for RPC" desc: "HTTP port of the server to connect to for RPC"

View File

@ -16,7 +16,7 @@ import
chronicles, chronicles/helpers as chroniclesHelpers, chronicles, chronicles/helpers as chroniclesHelpers,
# Local modules # Local modules
spec/[datatypes, crypto], eth2_network, time spec/[datatypes, crypto, helpers], eth2_network, time
proc setupLogging*(logLevel: string, logFile: Option[OutFile]) = proc setupLogging*(logLevel: string, logFile: Option[OutFile]) =
when compiles(defaultChroniclesStream.output.writer): when compiles(defaultChroniclesStream.output.writer):
@ -88,3 +88,15 @@ proc sleepToSlotOffset*(clock: BeaconClock, extra: chronos.Duration,
await sleepAsync(fromNow.offset) await sleepAsync(fromNow.offset)
return true return true
return false return false
proc checkIfShouldStopAtEpoch*(scheduledSlot: Slot, stopAtEpoch: uint64) =
# Offset backwards slightly to allow this epoch's finalization check to occur
if scheduledSlot > 3 and stopAtEpoch > 0'u64 and
(scheduledSlot - 3).compute_epoch_at_slot() >= stopAtEpoch:
info "Stopping at pre-chosen epoch",
chosenEpoch = stopAtEpoch,
epoch = scheduledSlot.compute_epoch_at_slot(),
slot = scheduledSlot
# Brute-force, but ensure it's reliable enough to run in CI.
quit(0)

View File

@ -117,6 +117,9 @@ proc onSlotStart(vc: ValidatorClient, lastSlot, scheduledSlot: Slot) {.gcsafe, a
beaconTime = shortLog(beaconTime), beaconTime = shortLog(beaconTime),
portBN = vc.config.rpcPort portBN = vc.config.rpcPort
# Check before any re-scheduling of onSlotStart()
checkIfShouldStopAtEpoch(scheduledSlot, vc.config.stopAtEpoch)
try: try:
# at the start of each epoch - request all validator duties # at the start of each epoch - request all validator duties
# TODO perhaps call this not on the first slot of each Epoch but perhaps # TODO perhaps call this not on the first slot of each Epoch but perhaps

View File

@ -32,35 +32,41 @@ if [ ${PIPESTATUS[0]} != 4 ]; then
fi fi
OPTS="hgt:n:d:" OPTS="hgt:n:d:"
LONGOPTS="help,testnet:,nodes:,data-dir:,disable-htop,enable-logtrace,log-level:,base-port:,base-metrics-port:,with-ganache,reuse-existing-data-dir" LONGOPTS="help,testnet:,nodes:,data-dir:,stop-at-epoch:,disable-htop,disable-vc,enable-logtrace,log-level:,base-port:,base-rpc-port:,base-metrics-port:,with-ganache,reuse-existing-data-dir"
# default values # default values
TESTNET="1" TESTNET="1"
NUM_NODES="10" NUM_NODES="10"
DATA_DIR="local_testnet_data" DATA_DIR="local_testnet_data"
USE_HTOP="1" USE_HTOP="1"
USE_VC="1"
USE_GANACHE="0" USE_GANACHE="0"
LOG_LEVEL="DEBUG" LOG_LEVEL="DEBUG"
BASE_PORT="9000" BASE_PORT="9000"
BASE_METRICS_PORT="8008" BASE_METRICS_PORT="8008"
BASE_RPC_PORT="7000"
REUSE_EXISTING_DATA_DIR="0" REUSE_EXISTING_DATA_DIR="0"
ENABLE_LOGTRACE="0" ENABLE_LOGTRACE="0"
STOP_AT_EPOCH_FLAG=""
print_help() { print_help() {
cat <<EOF cat <<EOF
Usage: $(basename "$0") --testnet <testnet number> [OTHER OPTIONS] -- [BEACON NODE OPTIONS] Usage: $(basename "$0") --testnet <testnet number> [OTHER OPTIONS] -- [BEACON NODE OPTIONS]
E.g.: $(basename "$0") --testnet ${TESTNET} --nodes ${NUM_NODES} --data-dir "${DATA_DIR}" # defaults E.g.: $(basename "$0") --testnet ${TESTNET} --nodes ${NUM_NODES} --stop-at-epoch 5 --data-dir "${DATA_DIR}" # defaults
CI run: $(basename "$0") --disable-htop -- --verify-finalization --stop-at-epoch=5 CI run: $(basename "$0") --disable-htop -- --verify-finalization
-h, --help this help message -h, --help this help message
-t, --testnet testnet number (default: ${TESTNET}) -t, --testnet testnet number (default: ${TESTNET})
-n, --nodes number of nodes to launch (default: ${NUM_NODES}) -n, --nodes number of nodes to launch (default: ${NUM_NODES})
-g, --with-ganache simulate a genesis event based on a deposit contract -g, --with-ganache simulate a genesis event based on a deposit contract
-s, --stop-at-epoch stop simulation at epoch (default: infinite)
-d, --data-dir directory where all the node data and logs will end up -d, --data-dir directory where all the node data and logs will end up
(default: "${DATA_DIR}") (default: "${DATA_DIR}")
--base-port bootstrap node's Eth2 traffic port (default: ${BASE_PORT}) --base-port bootstrap node's Eth2 traffic port (default: ${BASE_PORT})
--base-rpc-port bootstrap node's RPC port (default: ${BASE_RPC_PORT})
--base-metrics-port bootstrap node's metrics server port (default: ${BASE_METRICS_PORT}) --base-metrics-port bootstrap node's metrics server port (default: ${BASE_METRICS_PORT})
--disable-htop don't use "htop" to see the beacon_node processes --disable-htop don't use "htop" to see the beacon_node processes
--disable-vc don't use validator client binaries for validators (by default validators are split 50/50 between beacon nodes and validator clients)
--enable-logtrace display logtrace asr analysis --enable-logtrace display logtrace asr analysis
--log-level set the log level (default: ${LOG_LEVEL}) --log-level set the log level (default: ${LOG_LEVEL})
--reuse-existing-data-dir instead of deleting and recreating the data dir, keep it and reuse everything we can from it --reuse-existing-data-dir instead of deleting and recreating the data dir, keep it and reuse everything we can from it
@ -85,6 +91,10 @@ while true; do
TESTNET="$2" TESTNET="$2"
shift 2 shift 2
;; ;;
-n|--stop-at-epoch)
STOP_AT_EPOCH_FLAG="--stop-at-epoch=$2"
shift 2
;;
-n|--nodes) -n|--nodes)
NUM_NODES="$2" NUM_NODES="$2"
shift 2 shift 2
@ -97,6 +107,10 @@ while true; do
USE_HTOP="0" USE_HTOP="0"
shift shift
;; ;;
--disable-vc)
USE_VC="0"
shift
;;
-g|--with-ganache) -g|--with-ganache)
USE_GANACHE="1" USE_GANACHE="1"
shift shift
@ -109,6 +123,10 @@ while true; do
BASE_PORT="$2" BASE_PORT="$2"
shift 2 shift 2
;; ;;
--base-rpc-port)
BASE_RPC_PORT="$2"
shift 2
;;
--base-metrics-port) --base-metrics-port)
BASE_METRICS_PORT="$2" BASE_METRICS_PORT="$2"
shift 2 shift 2
@ -166,7 +184,7 @@ else
fi fi
NETWORK_NIM_FLAGS=$(scripts/load-testnet-nim-flags.sh "${NETWORK}") NETWORK_NIM_FLAGS=$(scripts/load-testnet-nim-flags.sh "${NETWORK}")
$MAKE -j2 LOG_LEVEL="${LOG_LEVEL}" NIMFLAGS="${NIMFLAGS} -d:insecure -d:testnet_servers_image -d:local_testnet ${NETWORK_NIM_FLAGS}" beacon_node deposit_contract $MAKE -j2 LOG_LEVEL="${LOG_LEVEL}" NIMFLAGS="${NIMFLAGS} -d:insecure -d:testnet_servers_image -d:local_testnet ${NETWORK_NIM_FLAGS}" beacon_node validator_client deposit_contract
if [[ "$ENABLE_LOGTRACE" == "1" ]]; then if [[ "$ENABLE_LOGTRACE" == "1" ]]; then
$MAKE LOG_LEVEL="${LOG_LEVEL}" NIMFLAGS="${NIMFLAGS} -d:insecure -d:testnet_servers_image -d:local_testnet ${NETWORK_NIM_FLAGS}" logtrace $MAKE LOG_LEVEL="${LOG_LEVEL}" NIMFLAGS="${NIMFLAGS} -d:insecure -d:testnet_servers_image -d:local_testnet ${NETWORK_NIM_FLAGS}" logtrace
fi fi
@ -178,6 +196,7 @@ BOOTSTRAP_TIMEOUT=30 # in seconds
DEPOSIT_CONTRACT_ADDRESS="0x0000000000000000000000000000000000000000" DEPOSIT_CONTRACT_ADDRESS="0x0000000000000000000000000000000000000000"
DEPOSIT_CONTRACT_BLOCK="0x0000000000000000000000000000000000000000000000000000000000000000" DEPOSIT_CONTRACT_BLOCK="0x0000000000000000000000000000000000000000000000000000000000000000"
NETWORK_METADATA_FILE="${DATA_DIR}/network.json" NETWORK_METADATA_FILE="${DATA_DIR}/network.json"
NUM_JOBS=${NUM_NODES}
if [[ "$REUSE_EXISTING_DATA_DIR" == "0" ]]; then if [[ "$REUSE_EXISTING_DATA_DIR" == "0" ]]; then
./build/deposit_contract generateSimulationDeposits \ ./build/deposit_contract generateSimulationDeposits \
@ -257,8 +276,10 @@ EOF
# "pkill" command. # "pkill" command.
cleanup() { cleanup() {
pkill -P $$ beacon_node &>/dev/null || true pkill -P $$ beacon_node &>/dev/null || true
pkill -P $$ validator_client &>/dev/null || true
sleep 2 sleep 2
pkill -9 -P $$ beacon_node &>/dev/null || true pkill -9 -P $$ beacon_node &>/dev/null || true
pkill -9 -P $$ validator_client &>/dev/null || true
} }
trap 'cleanup' SIGINT SIGTERM EXIT trap 'cleanup' SIGINT SIGTERM EXIT
@ -281,6 +302,15 @@ NODES_WITH_VALIDATORS=${NODES_WITH_VALIDATORS:-4}
BOOTSTRAP_NODE=0 BOOTSTRAP_NODE=0
SYSTEM_VALIDATORS=$(( TOTAL_VALIDATORS - USER_VALIDATORS )) SYSTEM_VALIDATORS=$(( TOTAL_VALIDATORS - USER_VALIDATORS ))
VALIDATORS_PER_NODE=$(( SYSTEM_VALIDATORS / NODES_WITH_VALIDATORS )) VALIDATORS_PER_NODE=$(( SYSTEM_VALIDATORS / NODES_WITH_VALIDATORS ))
if [ "${USE_VC:-}" == "1" ]; then
# if using validator client binaries in addition to beacon nodes we will
# split the keys for this instance in half between the BN and the VC
# and the validators for the BNs will be from the first half of all validators
VALIDATORS_PER_NODE=$((VALIDATORS_PER_NODE / 2 ))
NUM_JOBS=$((NUM_JOBS * 2 ))
fi
VALIDATORS_PER_VALIDATOR=$(( (SYSTEM_VALIDATORS / NODES_WITH_VALIDATORS) / 2 ))
VALIDATOR_OFFSET=$((SYSTEM_VALIDATORS / 2))
BOOTSTRAP_ENR="${DATA_DIR}/node${BOOTSTRAP_NODE}/beacon_node.enr" BOOTSTRAP_ENR="${DATA_DIR}/node${BOOTSTRAP_NODE}/beacon_node.enr"
for NUM_NODE in $(seq 0 $(( NUM_NODES - 1 ))); do for NUM_NODE in $(seq 0 $(( NUM_NODES - 1 ))); do
@ -309,6 +339,18 @@ for NUM_NODE in $(seq 0 $(( NUM_NODES - 1 ))); do
mkdir -p "${NODE_DATA_DIR}/secrets" mkdir -p "${NODE_DATA_DIR}/secrets"
if [[ $NUM_NODE -lt $NODES_WITH_VALIDATORS ]]; then if [[ $NUM_NODE -lt $NODES_WITH_VALIDATORS ]]; then
if [ "${USE_VC:-}" == "1" ]; then
VALIDATOR_DATA_DIR="${DATA_DIR}/validator${NUM_NODE}"
rm -rf "${VALIDATOR_DATA_DIR}"
mkdir -p "${VALIDATOR_DATA_DIR}/validators"
mkdir -p "${VALIDATOR_DATA_DIR}/secrets"
for VALIDATOR in $(ls "${VALIDATORS_DIR}" | tail -n +$(( $USER_VALIDATORS + ($VALIDATORS_PER_VALIDATOR * $NUM_NODE) + 1 + $VALIDATOR_OFFSET )) | head -n $VALIDATORS_PER_VALIDATOR); do
cp -a "${VALIDATORS_DIR}/$VALIDATOR" "${VALIDATOR_DATA_DIR}/validators/"
cp -a "${SECRETS_DIR}/${VALIDATOR}" "${VALIDATOR_DATA_DIR}/secrets/"
done
fi
for VALIDATOR in $(ls "${VALIDATORS_DIR}" | tail -n +$(( $USER_VALIDATORS + ($VALIDATORS_PER_NODE * $NUM_NODE) + 1 )) | head -n $VALIDATORS_PER_NODE); do for VALIDATOR in $(ls "${VALIDATORS_DIR}" | tail -n +$(( $USER_VALIDATORS + ($VALIDATORS_PER_NODE * $NUM_NODE) + 1 )) | head -n $VALIDATORS_PER_NODE); do
cp -a "${VALIDATORS_DIR}/$VALIDATOR" "${NODE_DATA_DIR}/validators/" cp -a "${VALIDATORS_DIR}/$VALIDATOR" "${NODE_DATA_DIR}/validators/"
cp -a "${SECRETS_DIR}/${VALIDATOR}" "${NODE_DATA_DIR}/secrets/" cp -a "${SECRETS_DIR}/${VALIDATOR}" "${NODE_DATA_DIR}/secrets/"
@ -326,6 +368,10 @@ for NUM_NODE in $(seq 0 $(( NUM_NODES - 1 ))); do
${BOOTSTRAP_ARG} \ ${BOOTSTRAP_ARG} \
${STATE_SNAPSHOT_ARG} \ ${STATE_SNAPSHOT_ARG} \
${WEB3_ARG} \ ${WEB3_ARG} \
${STOP_AT_EPOCH_FLAG} \
--rpc \
--rpc-address="127.0.0.1" \
--rpc-port="$(( BASE_RPC_PORT + NUM_NODE ))" \
--metrics \ --metrics \
--metrics-address="127.0.0.1" \ --metrics-address="127.0.0.1" \
--metrics-port="$(( BASE_METRICS_PORT + NUM_NODE ))" \ --metrics-port="$(( BASE_METRICS_PORT + NUM_NODE ))" \
@ -337,13 +383,22 @@ for NUM_NODE in $(seq 0 $(( NUM_NODES - 1 ))); do
else else
PIDS="${PIDS},$!" PIDS="${PIDS},$!"
fi fi
if [ "${USE_VC:-}" == "1" ]; then
./build/validator_client \
--log-level="${LOG_LEVEL}" \
${STOP_AT_EPOCH_FLAG} \
--data-dir="${VALIDATOR_DATA_DIR}" \
--rpc-port="$(( BASE_RPC_PORT + NUM_NODE ))" \
> "${DATA_DIR}/log_val${NUM_NODE}.txt" 2>&1 & PIDS="${PIDS},$!"
fi
done done
# give the regular nodes time to crash # give the regular nodes time to crash
sleep 5 sleep 5
BG_JOBS="$(jobs | wc -l | tr -d ' ')" BG_JOBS="$(jobs | wc -l | tr -d ' ')"
if [[ "$BG_JOBS" != "$NUM_NODES" ]]; then if [[ "$BG_JOBS" != "$NUM_JOBS" ]]; then
echo "$((NUM_NODES - BG_JOBS)) beacon_node instance(s) exited early. Aborting." echo "$(( NUM_JOBS - BG_JOBS )) beacon_node/validator_client instance(s) exited early. Aborting."
dump_logs dump_logs
dump_logtrace dump_logtrace
exit 1 exit 1

View File

@ -24,7 +24,7 @@ TOTAL_NODES=${NODES:-4}
TOTAL_USER_NODES=${USER_NODES:-0} TOTAL_USER_NODES=${USER_NODES:-0}
TOTAL_SYSTEM_NODES=$(( TOTAL_NODES - TOTAL_USER_NODES )) TOTAL_SYSTEM_NODES=$(( TOTAL_NODES - TOTAL_USER_NODES ))
BOOTSTRAP_NODE=$(( TOTAL_NODES - 1 )) BOOTSTRAP_NODE=$(( TOTAL_NODES - 1 ))
USE_BN_VC_VALIDATOR_SPLIT=${BN_VC_VALIDATOR_SPLIT:-no} USE_BN_VC_VALIDATOR_SPLIT=${BN_VC_VALIDATOR_SPLIT:-yes}
SIMULATION_DIR="${SIM_ROOT}/data" SIMULATION_DIR="${SIM_ROOT}/data"
METRICS_DIR="${SIM_ROOT}/prometheus" METRICS_DIR="${SIM_ROOT}/prometheus"