Allow running local nodes without attached validators

The number of user nodes is now specified with `USER_NODES`.

To make the instructions more stable, the "numeric id" of the user
nodes will be starting from 0 (so you can always use `run_node.sh 0`
to start a user node).

If you specify a node index above the total number of nodes, you'll
launch a node without any validators attached (this is useful for
testing the sync for example).

README.md

@@ -186,15 +186,15 @@ The beacon node simulation will create a full peer-to-peer network of beacon nod
 
 ```bash
 # Clear data files from your last run and start the simulation with a new genesis block:
-make VALIDATORS=192 NODES=6 MISSING_NODES=1 eth2_network_simulation
+make VALIDATORS=192 NODES=6 USER_NODES=1 eth2_network_simulation
 
 # In another terminal, get a shell with the right environment variables set:
 ./env.sh bash
 
 # In the above example, the network is prepared for 7 beacon nodes but one of
-# them is not started by default (`MISSING_NODES`) - you can start it separately
+# them is not started by default (`USER_NODES`) - you can start it separately
 # by running:
-./tests/simulation/run_node.sh 6 # (or the 0-based node number of the missing node)
+./tests/simulation/run_node.sh 0 # (or the 0-based node number of the missing node)
 
 # Running a separate node allows you to test sync as well as see what the action
 # looks like from a single nodes' perspective.
@@ -219,7 +219,7 @@ Specific steps:
 ```bash
 # This will generate the Prometheus config and the Grafana dashboard on the fly,
 # based on the number of nodes (which you can control by passing something like NODES=6 to `make`).
-make VALIDATORS=192 NODES=6 MISSING_NODES=0 eth2_network_simulation
+make VALIDATORS=192 NODES=6 USER_NODES=0 eth2_network_simulation
 
 # In another terminal tab, after the sim started:
 cd tests/simulation/prometheus

beacon_chain/sync_protocol.nim

@@ -10,6 +10,9 @@ when networkBackend == rlpxBackend:
 
 declarePublicGauge libp2p_peers, "Number of libp2p peers"
 
+logScope:
+  topics = "sync"
+
 type
   ValidatorSetDeltaFlags {.pure.} = enum
     Activation = 0
@@ -135,6 +138,7 @@ p2pProtocol BeaconSync(version = 1,
             count: uint64,
             step: uint64) {.
             libp2pProtocol("beacon_blocks_by_range", 1).} =
+      trace "got range request", peer, count, startSlot, headBlockRoot, step
 
       if count > 0'u64:
         let count = if step != 0: min(count, MAX_REQUESTED_BLOCKS.uint64) else: 1
@@ -145,6 +149,7 @@ p2pProtocol BeaconSync(version = 1,
           firstPos = pool.getBlockRange(headBlockRoot, startSlot, step,
                                         results.toOpenArray(0, lastPos))
         for i in firstPos.int .. lastPos.int:
+          trace "wrote response block", slot = results[i].slot
           await response.write(pool.get(results[i]).data)
 
     proc beaconBlocksByRoot(

scripts/README.md

@@ -14,7 +14,7 @@ The `beacon_node` binary has a `createTestnet` command.
 
 ```bash
   nim c -r beacon_chain/beacon_node \
-    --data-dir=$DATA_DIR/node-0 \
+    --data-dir=$NETWORK_DIR/data \
     createTestnet \
     --validators-dir=$NETWORK_DIR \
     --total-validators=$VALIDATOR_COUNT \

tests/simulation/run_node.sh

@@ -14,24 +14,24 @@ source "${SIM_ROOT}/../../env.sh"
 cd "$GIT_ROOT"
 
 DATA_DIR="${SIMULATION_DIR}/node-$NODE_ID"
-
+PORT=$(( BASE_P2P_PORT + NODE_ID ))
-V_PREFIX="${VALIDATORS_DIR}/v$(printf '%06d' $NODE_ID)"
-PORT=$(printf '5%04d' $NODE_ID)
 
 NAT_FLAG="--nat:none"
 if [ "${NAT:-}" == "1" ]; then
   NAT_FLAG="--nat:any"
 fi
 
-FIRST_VALIDATOR_IDX=$(( (NUM_VALIDATORS / ($NUM_NODES + $NUM_MISSING_NODES)) * $NODE_ID ))
-LAST_VALIDATOR_IDX=$(( (NUM_VALIDATORS / ($NUM_NODES + $NUM_MISSING_NODES)) * ($NODE_ID + 1) - 1 ))
-
 mkdir -p $DATA_DIR/validators
 rm -f $DATA_DIR/validators/*
 
-pushd $VALIDATORS_DIR >/dev/null
+if [[ $NODE_ID -lt $TOTAL_NODES ]]; then
+  FIRST_VALIDATOR_IDX=$(( (NUM_VALIDATORS / TOTAL_NODES) * NODE_ID ))
+  LAST_VALIDATOR_IDX=$(( (NUM_VALIDATORS / TOTAL_NODES) * (NODE_ID + 1) - 1 ))
+
+  pushd $VALIDATORS_DIR >/dev/null
     cp $(seq -s " " -f v%07g.privkey $FIRST_VALIDATOR_IDX $LAST_VALIDATOR_IDX) $DATA_DIR/validators
-popd >/dev/null
+  popd >/dev/null
+fi
 
 $BEACON_NODE_BIN \
   --bootstrap-file=$NETWORK_BOOTSTRAP_FILE \
@@ -46,5 +46,5 @@ $BEACON_NODE_BIN \
   --metrics-server=on \
   --metrics-server-address="127.0.0.1" \
   --metrics-server-port="$(( $BASE_METRICS_PORT + $NODE_ID ))" \
-  "$@"
+  "$@" | tee "$DATA_DIR/node.log"
 

tests/simulation/start.sh

@@ -19,9 +19,9 @@ NIMFLAGS="-d:chronicles_log_level=DEBUG --hints:off --warnings:off --verbosity:0
 # Run with "SHARD_COUNT=4 ./start.sh" to change these
 DEFS=""
 
-DEFS+="-d:SHARD_COUNT=${SHARD_COUNT:-16} "      # Spec default: 1024
+DEFS+="-d:SHARD_COUNT=${SHARD_COUNT:-8} "      # Spec default: 1024
-DEFS+="-d:SLOTS_PER_EPOCH=${SLOTS_PER_EPOCH:-16} "   # Spec default: 64
+DEFS+="-d:SLOTS_PER_EPOCH=${SLOTS_PER_EPOCH:-8} "   # Spec default: 64
-DEFS+="-d:SECONDS_PER_SLOT=${SECONDS_PER_SLOT:-6} "  # Spec default: 6
+DEFS+="-d:SECONDS_PER_SLOT=${SECONDS_PER_SLOT:-4} "  # Spec default: 6
 
 LAST_VALIDATOR_NUM=$(( NUM_VALIDATORS - 1 ))
 LAST_VALIDATOR="$VALIDATORS_DIR/v$(printf '%07d' $LAST_VALIDATOR_NUM).deposit.json"
@@ -47,14 +47,14 @@ fi
 
 if [ ! -f "${SNAPSHOT_FILE}" ]; then
   $BEACON_NODE_BIN \
-    --data-dir="${SIMULATION_DIR}/node-0" \
+    --data-dir="${SIMULATION_DIR}/node-$MASTER_NODE" \
     createTestnet \
     --validators-dir="${VALIDATORS_DIR}" \
     --total-validators="${NUM_VALIDATORS}" \
     --output-genesis="${SNAPSHOT_FILE}" \
     --output-bootstrap-file="${NETWORK_BOOTSTRAP_FILE}" \
     --bootstrap-address=127.0.0.1 \
-    --bootstrap-port=50000 \
+    --bootstrap-port=$(( BASE_P2P_PORT + MASTER_NODE )) \
     --genesis-offset=5 # Delay in seconds
 fi
 
@@ -87,7 +87,7 @@ fi
 
 # use the exported Grafana dashboard for a single node to create one for all nodes
 "${SIM_ROOT}/../../build/process_dashboard" \
-  --nodes=${NUM_NODES} \
+  --nodes=${TOTAL_NODES} \
   --in="${SIM_ROOT}/beacon-chain-sim-node0-Grafana-dashboard.json" \
   --out="${SIM_ROOT}/beacon-chain-sim-all-nodes-Grafana-dashboard.json"
 
@@ -100,10 +100,9 @@ if [ "$USE_MULTITAIL" = "no" ]; then
 fi
 
 COMMANDS=()
-LAST_NODE=$(( NUM_NODES - 1 ))
 
-for i in $(seq 0 $LAST_NODE); do
+for i in $(seq $MASTER_NODE $TOTAL_USER_NODES); do
-  if [[ "$i" != "0" && "$USE_MULTITAIL" == "no" ]]; then
+  if [[ "$i" != "$MASTER_NODE" && "$USE_MULTITAIL" == "no" ]]; then
     # Wait for the master node to write out its address file
     while [ ! -f "${MASTER_NODE_ADDRESS_FILE}" ]; do
       sleep 0.1

tests/simulation/tmux_demo.sh

@@ -14,7 +14,7 @@ tmux set-option -g history-limit 999999
 tmux set -g mouse on
 
 tmux send-keys -t 0 './start.sh' Enter
-tmux new-window -n "demo_node" "./wait_master_node.sh && ./run_node.sh $NUM_NODES"
+tmux new-window -n "demo_node" "./wait_master_node.sh && ./run_node.sh 0"
 
 tmux attach-session -d
 

tests/simulation/vars.sh

@@ -16,8 +16,10 @@ cd - &>/dev/null
 # When changing these, also update the readme section on running simulation
 # so that the run_node example is correct!
 NUM_VALIDATORS=${VALIDATORS:-192}
-NUM_NODES=${NODES:-6}
+TOTAL_NODES=${NODES:-4}
-NUM_MISSING_NODES=${MISSING_NODES:-1}
+TOTAL_USER_NODES=${USER_NODES:-0}
+TOTAL_SYSTEM_NODES=$(( TOTAL_NODES - TOTAL_USER_NODES ))
+MASTER_NODE=$(( TOTAL_NODES - 1 ))
 
 SIMULATION_DIR="${SIM_ROOT}/data"
 METRICS_DIR="${SIM_ROOT}/prometheus"
@@ -26,7 +28,9 @@ SNAPSHOT_FILE="${SIMULATION_DIR}/state_snapshot.ssz"
 NETWORK_BOOTSTRAP_FILE="${SIMULATION_DIR}/bootstrap_nodes.txt"
 BEACON_NODE_BIN="${SIMULATION_DIR}/beacon_node"
 DEPLOY_DEPOSIT_CONTRACT_BIN="${SIMULATION_DIR}/deploy_deposit_contract"
-MASTER_NODE_ADDRESS_FILE="${SIMULATION_DIR}/node-0/beacon_node.address"
+MASTER_NODE_ADDRESS_FILE="${SIMULATION_DIR}/node-${MASTER_NODE}/beacon_node.address"
+
+BASE_P2P_PORT=30000
 BASE_METRICS_PORT=8008
 # Set DEPOSIT_WEB3_URL_ARG to empty to get genesis state from file, not using web3
 # DEPOSIT_WEB3_URL_ARG=--web3-url=ws://localhost:8545