50/50 bn/vc split for the validator keys ON by default for the testnet scripts

Jenkinsfile

@@ -47,16 +47,16 @@ def runStages() {
 						// EXECUTOR_NUMBER will be 0 or 1, since we have 2 executors per Jenkins node
 						sh """#!/bin/bash
 						set -e
-						./scripts/launch_local_testnet.sh --testnet 0 --nodes 4 --log-level DEBUG --disable-htop --enable-logtrace --data-dir local_testnet0_data --base-port \$(( 9000 + EXECUTOR_NUMBER * 100 )) --base-metrics-port \$(( 8008 + EXECUTOR_NUMBER * 100 )) -- --verify-finalization --stop-at-epoch=5 --discv5:no
+						./scripts/launch_local_testnet.sh --testnet 0 --nodes 4 --stop-at-epoch 5 --log-level DEBUG --disable-htop --enable-logtrace --data-dir local_testnet0_data --base-port \$(( 9000 + EXECUTOR_NUMBER * 100 )) --base-rpc-port \$(( 7000 + EXECUTOR_NUMBER * 100 )) --base-metrics-port \$(( 8008 + EXECUTOR_NUMBER * 100 )) -- --verify-finalization --discv5:no
-						./scripts/launch_local_testnet.sh --testnet 1 --nodes 4 --log-level DEBUG --disable-htop --enable-logtrace --data-dir local_testnet1_data --base-port \$(( 9000 + EXECUTOR_NUMBER * 100 )) --base-metrics-port \$(( 8008 + EXECUTOR_NUMBER * 100 )) -- --verify-finalization --stop-at-epoch=5 --discv5:no
+						./scripts/launch_local_testnet.sh --testnet 1 --nodes 4 --stop-at-epoch 5 --log-level DEBUG --disable-htop --enable-logtrace --data-dir local_testnet1_data --base-port \$(( 9000 + EXECUTOR_NUMBER * 100 )) --base-rpc-port \$(( 7000 + EXECUTOR_NUMBER * 100 )) --base-metrics-port \$(( 8008 + EXECUTOR_NUMBER * 100 )) -- --verify-finalization --discv5:no
 						"""
 					}
 					// stage("testnet finalization - Miracl/Milagro fallback") {
 					// 	// EXECUTOR_NUMBER will be 0 or 1, since we have 2 executors per Jenkins node
 					// 	sh """#!/bin/bash
 					// 	set -e
-					// 	NIMFLAGS="-d:BLS_FORCE_BACKEND=miracl" ./scripts/launch_local_testnet.sh --testnet 0 --nodes 4 --log-level INFO --disable-htop --data-dir local_testnet0_data --base-port \$(( 9000 + EXECUTOR_NUMBER * 100 )) --base-metrics-port \$(( 8008 + EXECUTOR_NUMBER * 100 )) -- --verify-finalization --stop-at-epoch=5
+					// 	NIMFLAGS="-d:BLS_FORCE_BACKEND=miracl" ./scripts/launch_local_testnet.sh --testnet 0 --nodes 4 --stop-at-epoch 5 --log-level INFO --disable-htop --data-dir local_testnet0_data --base-port \$(( 9000 + EXECUTOR_NUMBER * 100 )) --base-rpc-port \$(( 7000 + EXECUTOR_NUMBER * 100 )) --base-metrics-port \$(( 8008 + EXECUTOR_NUMBER * 100 )) -- --verify-finalization
-					// 	NIMFLAGS="-d:BLS_FORCE_BACKEND=miracl" ./scripts/launch_local_testnet.sh --testnet 1 --nodes 4 --log-level INFO --disable-htop --data-dir local_testnet1_data --base-port \$(( 9000 + EXECUTOR_NUMBER * 100 )) --base-metrics-port \$(( 8008 + EXECUTOR_NUMBER * 100 )) -- --verify-finalization --stop-at-epoch=5
+					// 	NIMFLAGS="-d:BLS_FORCE_BACKEND=miracl" ./scripts/launch_local_testnet.sh --testnet 1 --nodes 4 --stop-at-epoch 5 --log-level INFO --disable-htop --data-dir local_testnet1_data --base-port \$(( 9000 + EXECUTOR_NUMBER * 100 )) --base-rpc-port \$(( 7000 + EXECUTOR_NUMBER * 100 )) --base-metrics-port \$(( 8008 + EXECUTOR_NUMBER * 100 )) -- --verify-finalization
 					// 	"""
 					// }
 				}

README.md

@@ -214,23 +214,14 @@ make VALIDATORS=192 NODES=6 USER_NODES=1 eth2_network_simulation
 # looks like from a single nodes' perspective.
 ```
 
-By default, all validators are loaded within the beacon nodes, but if you want to use
-external processes as validator clients you can pass `BN_VC_VALIDATOR_SPLIT=yes` as an
-additional argument to the `make eth2_network_simulation` command and that will split
-the `VALIDATORS` between beacon nodes and validator clients - for example with `192`
-validators and `6` nodes you will end up with 6 beacon node and 6 validator client
-processes, where each of them will handle 16 validators.
-
-<!--
 By default, validators will be split in half between beacon node and validator
-client processes, communicating through the
+client processes (50/50), communicating through the
 [official validator API](https://ethereum.github.io/eth2.0-APIs/#/ValidatorRequiredApi)
 (for example with `192` validators and `6` nodes you will roughly end up with 6
 beacon node and 6 validator client processes, where each of them will handle 16
 validators), but if you don't want to use external validator clients and instead
-want to have all the validators handled in-process by the beacon nodes you may
+want to have all the validators handled by the beacon nodes you may use
-use `BN_VC_VALIDATOR_SPLIT=no` as an additional argument to `make eth2_network_simulation`.
+`BN_VC_VALIDATOR_SPLIT=no` as an additional argument to `make eth2_network_simulation`.
--->
 
 By default, the simulation will start from a pre-generated genesis state. If you wish to
 simulate the bootstrap process with a Ethereum 1.0 validator deposit contract, start the

beacon_chain/beacon_node.nim

@@ -422,16 +422,7 @@ proc onSlotStart(node: BeaconNode, lastSlot, scheduledSlot: Slot) {.gcsafe, asyn
     finalizedEpoch = shortLog(finalizedEpoch)
 
   # Check before any re-scheduling of onSlotStart()
-  # Offset backwards slightly to allow this epoch's finalization check to occur
+  checkIfShouldStopAtEpoch(scheduledSlot, node.config.stopAtEpoch)
-  if scheduledSlot > 3 and node.config.stopAtEpoch > 0'u64 and
-      (scheduledSlot - 3).compute_epoch_at_slot() >= node.config.stopAtEpoch:
-    info "Stopping at pre-chosen epoch",
-      chosenEpoch = node.config.stopAtEpoch,
-      epoch = scheduledSlot.compute_epoch_at_slot(),
-      slot = scheduledSlot
-
-    # Brute-force, but ensure it's reliable enough to run in CI.
-    quit(0)
 
   if not wallSlot.afterGenesis or (wallSlot.slot < lastSlot):
     let

beacon_chain/conf.nim

@@ -366,6 +366,11 @@ type
               "You can use a 0x-prefixed hex encoded string to specify raw bytes."
         name: "graffiti" }: Option[GraffitiBytes]
 
+      stopAtEpoch* {.
+        defaultValue: 0
+        desc: "A positive epoch selects the epoch at which to stop"
+        name: "stop-at-epoch" }: uint64
+
       rpcPort* {.
         defaultValue: defaultEth2RpcPort
         desc: "HTTP port of the server to connect to for RPC"

beacon_chain/nimbus_binary_common.nim

@@ -16,7 +16,7 @@ import
   chronicles, chronicles/helpers as chroniclesHelpers,
 
   # Local modules
-  spec/[datatypes, crypto], eth2_network, time
+  spec/[datatypes, crypto, helpers], eth2_network, time
 
 proc setupLogging*(logLevel: string, logFile: Option[OutFile]) =
   when compiles(defaultChroniclesStream.output.writer):
@@ -88,3 +88,15 @@ proc sleepToSlotOffset*(clock: BeaconClock, extra: chronos.Duration,
     await sleepAsync(fromNow.offset)
     return true
   return false
+
+proc checkIfShouldStopAtEpoch*(scheduledSlot: Slot, stopAtEpoch: uint64) =
+  # Offset backwards slightly to allow this epoch's finalization check to occur
+  if scheduledSlot > 3 and stopAtEpoch > 0'u64 and
+      (scheduledSlot - 3).compute_epoch_at_slot() >= stopAtEpoch:
+    info "Stopping at pre-chosen epoch",
+      chosenEpoch = stopAtEpoch,
+      epoch = scheduledSlot.compute_epoch_at_slot(),
+      slot = scheduledSlot
+
+    # Brute-force, but ensure it's reliable enough to run in CI.
+    quit(0)

beacon_chain/validator_client.nim

@@ -117,6 +117,9 @@ proc onSlotStart(vc: ValidatorClient, lastSlot, scheduledSlot: Slot) {.gcsafe, a
     beaconTime = shortLog(beaconTime),
     portBN = vc.config.rpcPort
 
+  # Check before any re-scheduling of onSlotStart()
+  checkIfShouldStopAtEpoch(scheduledSlot, vc.config.stopAtEpoch)
+
   try:
     # at the start of each epoch - request all validator duties
     # TODO perhaps call this not on the first slot of each Epoch but perhaps

scripts/launch_local_testnet.sh

@@ -32,35 +32,41 @@ if [ ${PIPESTATUS[0]} != 4 ]; then
 fi
 
 OPTS="hgt:n:d:"
-LONGOPTS="help,testnet:,nodes:,data-dir:,disable-htop,enable-logtrace,log-level:,base-port:,base-metrics-port:,with-ganache,reuse-existing-data-dir"
+LONGOPTS="help,testnet:,nodes:,data-dir:,stop-at-epoch:,disable-htop,disable-vc,enable-logtrace,log-level:,base-port:,base-rpc-port:,base-metrics-port:,with-ganache,reuse-existing-data-dir"
 
 # default values
 TESTNET="1"
 NUM_NODES="10"
 DATA_DIR="local_testnet_data"
 USE_HTOP="1"
+USE_VC="1"
 USE_GANACHE="0"
 LOG_LEVEL="DEBUG"
 BASE_PORT="9000"
 BASE_METRICS_PORT="8008"
+BASE_RPC_PORT="7000"
 REUSE_EXISTING_DATA_DIR="0"
 ENABLE_LOGTRACE="0"
+STOP_AT_EPOCH_FLAG=""
 
 print_help() {
   cat <<EOF
 Usage: $(basename "$0") --testnet <testnet number> [OTHER OPTIONS] -- [BEACON NODE OPTIONS]
-E.g.: $(basename "$0") --testnet ${TESTNET} --nodes ${NUM_NODES} --data-dir "${DATA_DIR}" # defaults
+E.g.: $(basename "$0") --testnet ${TESTNET} --nodes ${NUM_NODES} --stop-at-epoch 5 --data-dir "${DATA_DIR}" # defaults
-CI run: $(basename "$0") --disable-htop -- --verify-finalization --stop-at-epoch=5
+CI run: $(basename "$0") --disable-htop -- --verify-finalization
 
   -h, --help                  this help message
   -t, --testnet               testnet number (default: ${TESTNET})
   -n, --nodes                 number of nodes to launch (default: ${NUM_NODES})
   -g, --with-ganache          simulate a genesis event based on a deposit contract
+  -s, --stop-at-epoch         stop simulation at epoch (default: infinite)
   -d, --data-dir              directory where all the node data and logs will end up
                               (default: "${DATA_DIR}")
   --base-port                 bootstrap node's Eth2 traffic port (default: ${BASE_PORT})
+  --base-rpc-port             bootstrap node's RPC port (default: ${BASE_RPC_PORT})
   --base-metrics-port         bootstrap node's metrics server port (default: ${BASE_METRICS_PORT})
   --disable-htop              don't use "htop" to see the beacon_node processes
+  --disable-vc                don't use validator client binaries for validators (by default validators are split 50/50 between beacon nodes and validator clients)
   --enable-logtrace           display logtrace asr analysis
   --log-level                 set the log level (default: ${LOG_LEVEL})
   --reuse-existing-data-dir   instead of deleting and recreating the data dir, keep it and reuse everything we can from it
@@ -85,6 +91,10 @@ while true; do
       TESTNET="$2"
       shift 2
       ;;
+    -n|--stop-at-epoch)
+      STOP_AT_EPOCH_FLAG="--stop-at-epoch=$2"
+      shift 2
+      ;;
     -n|--nodes)
       NUM_NODES="$2"
       shift 2
@@ -97,6 +107,10 @@ while true; do
       USE_HTOP="0"
       shift
       ;;
+    --disable-vc)
+      USE_VC="0"
+      shift
+      ;;
     -g|--with-ganache)
       USE_GANACHE="1"
       shift
@@ -109,6 +123,10 @@ while true; do
       BASE_PORT="$2"
       shift 2
       ;;
+    --base-rpc-port)
+      BASE_RPC_PORT="$2"
+      shift 2
+      ;;
     --base-metrics-port)
       BASE_METRICS_PORT="$2"
       shift 2
@@ -166,7 +184,7 @@ else
 fi
 
 NETWORK_NIM_FLAGS=$(scripts/load-testnet-nim-flags.sh "${NETWORK}")
-$MAKE -j2 LOG_LEVEL="${LOG_LEVEL}" NIMFLAGS="${NIMFLAGS} -d:insecure -d:testnet_servers_image -d:local_testnet ${NETWORK_NIM_FLAGS}" beacon_node deposit_contract
+$MAKE -j2 LOG_LEVEL="${LOG_LEVEL}" NIMFLAGS="${NIMFLAGS} -d:insecure -d:testnet_servers_image -d:local_testnet ${NETWORK_NIM_FLAGS}" beacon_node validator_client deposit_contract
 if [[ "$ENABLE_LOGTRACE" == "1" ]]; then
   $MAKE LOG_LEVEL="${LOG_LEVEL}" NIMFLAGS="${NIMFLAGS} -d:insecure -d:testnet_servers_image -d:local_testnet ${NETWORK_NIM_FLAGS}" logtrace
 fi
@@ -178,6 +196,7 @@ BOOTSTRAP_TIMEOUT=30 # in seconds
 DEPOSIT_CONTRACT_ADDRESS="0x0000000000000000000000000000000000000000"
 DEPOSIT_CONTRACT_BLOCK="0x0000000000000000000000000000000000000000000000000000000000000000"
 NETWORK_METADATA_FILE="${DATA_DIR}/network.json"
+NUM_JOBS=${NUM_NODES}
 
 if [[ "$REUSE_EXISTING_DATA_DIR" == "0" ]]; then
   ./build/deposit_contract generateSimulationDeposits \
@@ -257,8 +276,10 @@ EOF
 # "pkill" command.
 cleanup() {
   pkill -P $$ beacon_node &>/dev/null || true
+  pkill -P $$ validator_client &>/dev/null || true
   sleep 2
   pkill -9 -P $$ beacon_node &>/dev/null || true
+  pkill -9 -P $$ validator_client &>/dev/null || true
 }
 trap 'cleanup' SIGINT SIGTERM EXIT
 
@@ -281,6 +302,15 @@ NODES_WITH_VALIDATORS=${NODES_WITH_VALIDATORS:-4}
 BOOTSTRAP_NODE=0
 SYSTEM_VALIDATORS=$(( TOTAL_VALIDATORS - USER_VALIDATORS ))
 VALIDATORS_PER_NODE=$(( SYSTEM_VALIDATORS / NODES_WITH_VALIDATORS ))
+if [ "${USE_VC:-}" == "1" ]; then
+  # if using validator client binaries in addition to beacon nodes we will
+  # split the keys for this instance in half between the BN and the VC
+  # and the validators for the BNs will be from the first half of all validators
+  VALIDATORS_PER_NODE=$((VALIDATORS_PER_NODE / 2 ))
+  NUM_JOBS=$((NUM_JOBS * 2 ))
+fi
+VALIDATORS_PER_VALIDATOR=$(( (SYSTEM_VALIDATORS / NODES_WITH_VALIDATORS) / 2 ))
+VALIDATOR_OFFSET=$((SYSTEM_VALIDATORS / 2))
 BOOTSTRAP_ENR="${DATA_DIR}/node${BOOTSTRAP_NODE}/beacon_node.enr"
 
 for NUM_NODE in $(seq 0 $(( NUM_NODES - 1 ))); do
@@ -309,6 +339,18 @@ for NUM_NODE in $(seq 0 $(( NUM_NODES - 1 ))); do
   mkdir -p "${NODE_DATA_DIR}/secrets"
 
   if [[ $NUM_NODE -lt $NODES_WITH_VALIDATORS ]]; then
+    if [ "${USE_VC:-}" == "1" ]; then
+      VALIDATOR_DATA_DIR="${DATA_DIR}/validator${NUM_NODE}"
+      rm -rf "${VALIDATOR_DATA_DIR}"
+      mkdir -p "${VALIDATOR_DATA_DIR}/validators"
+      mkdir -p "${VALIDATOR_DATA_DIR}/secrets"
+
+      for VALIDATOR in $(ls "${VALIDATORS_DIR}" | tail -n +$(( $USER_VALIDATORS + ($VALIDATORS_PER_VALIDATOR * $NUM_NODE) + 1 + $VALIDATOR_OFFSET )) | head -n $VALIDATORS_PER_VALIDATOR); do
+        cp -a "${VALIDATORS_DIR}/$VALIDATOR" "${VALIDATOR_DATA_DIR}/validators/"
+        cp -a "${SECRETS_DIR}/${VALIDATOR}" "${VALIDATOR_DATA_DIR}/secrets/"
+      done
+    fi
+
     for VALIDATOR in $(ls "${VALIDATORS_DIR}" | tail -n +$(( $USER_VALIDATORS + ($VALIDATORS_PER_NODE * $NUM_NODE) + 1 )) | head -n $VALIDATORS_PER_NODE); do
       cp -a "${VALIDATORS_DIR}/$VALIDATOR" "${NODE_DATA_DIR}/validators/"
       cp -a "${SECRETS_DIR}/${VALIDATOR}" "${NODE_DATA_DIR}/secrets/"
@@ -326,6 +368,10 @@ for NUM_NODE in $(seq 0 $(( NUM_NODES - 1 ))); do
     ${BOOTSTRAP_ARG} \
     ${STATE_SNAPSHOT_ARG} \
     ${WEB3_ARG} \
+    ${STOP_AT_EPOCH_FLAG} \
+    --rpc \
+    --rpc-address="127.0.0.1" \
+    --rpc-port="$(( BASE_RPC_PORT + NUM_NODE ))" \
     --metrics \
     --metrics-address="127.0.0.1" \
     --metrics-port="$(( BASE_METRICS_PORT + NUM_NODE ))" \
@@ -337,13 +383,22 @@ for NUM_NODE in $(seq 0 $(( NUM_NODES - 1 ))); do
   else
     PIDS="${PIDS},$!"
   fi
+
+  if [ "${USE_VC:-}" == "1" ]; then
+    ./build/validator_client \
+      --log-level="${LOG_LEVEL}" \
+      ${STOP_AT_EPOCH_FLAG} \
+      --data-dir="${VALIDATOR_DATA_DIR}" \
+      --rpc-port="$(( BASE_RPC_PORT + NUM_NODE ))" \
+      > "${DATA_DIR}/log_val${NUM_NODE}.txt" 2>&1 & PIDS="${PIDS},$!"
+  fi
 done
 
 # give the regular nodes time to crash
 sleep 5
 BG_JOBS="$(jobs | wc -l | tr -d ' ')"
-if [[ "$BG_JOBS" != "$NUM_NODES" ]]; then
+if [[ "$BG_JOBS" != "$NUM_JOBS" ]]; then
-  echo "$((NUM_NODES - BG_JOBS)) beacon_node instance(s) exited early. Aborting."
+  echo "$(( NUM_JOBS - BG_JOBS )) beacon_node/validator_client instance(s) exited early. Aborting."
   dump_logs
   dump_logtrace
   exit 1

tests/simulation/vars.sh

@@ -24,7 +24,7 @@ TOTAL_NODES=${NODES:-4}
 TOTAL_USER_NODES=${USER_NODES:-0}
 TOTAL_SYSTEM_NODES=$(( TOTAL_NODES - TOTAL_USER_NODES ))
 BOOTSTRAP_NODE=$(( TOTAL_NODES - 1 ))
-USE_BN_VC_VALIDATOR_SPLIT=${BN_VC_VALIDATOR_SPLIT:-no}
+USE_BN_VC_VALIDATOR_SPLIT=${BN_VC_VALIDATOR_SPLIT:-yes}
 
 SIMULATION_DIR="${SIM_ROOT}/data"
 METRICS_DIR="${SIM_ROOT}/prometheus"