From f5e9b9a922678a13bfaac38580b9f8a8c85bf7f5 Mon Sep 17 00:00:00 2001 From: Zahary Karadjov Date: Mon, 11 Nov 2019 01:28:13 +0000 Subject: [PATCH] Allow running local nodes without attached validators The number of user nodes is now specified with `USER_NODES`. To make the instructions more stable, the "numeric id" of the user nodes will be starting from 0 (so you can always use `run_node.sh 0` to start a user node). If you specify a node index above the total number of nodes, you'll launch a node without any validators attached (this is useful for testing the sync for example). --- README.md | 8 ++++---- beacon_chain/sync_protocol.nim | 5 +++++ scripts/README.md | 2 +- tests/simulation/run_node.sh | 20 ++++++++++---------- tests/simulation/start.sh | 17 ++++++++--------- tests/simulation/tmux_demo.sh | 2 +- tests/simulation/vars.sh | 10 +++++++--- 7 files changed, 36 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 763232bc3..e8b4868bc 100644 --- a/README.md +++ b/README.md @@ -186,15 +186,15 @@ The beacon node simulation will create a full peer-to-peer network of beacon nod ```bash # Clear data files from your last run and start the simulation with a new genesis block: -make VALIDATORS=192 NODES=6 MISSING_NODES=1 eth2_network_simulation +make VALIDATORS=192 NODES=6 USER_NODES=1 eth2_network_simulation # In another terminal, get a shell with the right environment variables set: ./env.sh bash # In the above example, the network is prepared for 7 beacon nodes but one of -# them is not started by default (`MISSING_NODES`) - you can start it separately +# them is not started by default (`USER_NODES`) - you can start it separately # by running: -./tests/simulation/run_node.sh 6 # (or the 0-based node number of the missing node) +./tests/simulation/run_node.sh 0 # (or the 0-based node number of the missing node) # Running a separate node allows you to test sync as well as see what the action # looks like from a single nodes' perspective. @@ -219,7 +219,7 @@ Specific steps: ```bash # This will generate the Prometheus config and the Grafana dashboard on the fly, # based on the number of nodes (which you can control by passing something like NODES=6 to `make`). -make VALIDATORS=192 NODES=6 MISSING_NODES=0 eth2_network_simulation +make VALIDATORS=192 NODES=6 USER_NODES=0 eth2_network_simulation # In another terminal tab, after the sim started: cd tests/simulation/prometheus diff --git a/beacon_chain/sync_protocol.nim b/beacon_chain/sync_protocol.nim index 437777470..eaf2079c2 100644 --- a/beacon_chain/sync_protocol.nim +++ b/beacon_chain/sync_protocol.nim @@ -10,6 +10,9 @@ when networkBackend == rlpxBackend: declarePublicGauge libp2p_peers, "Number of libp2p peers" +logScope: + topics = "sync" + type ValidatorSetDeltaFlags {.pure.} = enum Activation = 0 @@ -135,6 +138,7 @@ p2pProtocol BeaconSync(version = 1, count: uint64, step: uint64) {. libp2pProtocol("beacon_blocks_by_range", 1).} = + trace "got range request", peer, count, startSlot, headBlockRoot, step if count > 0'u64: let count = if step != 0: min(count, MAX_REQUESTED_BLOCKS.uint64) else: 1 @@ -145,6 +149,7 @@ p2pProtocol BeaconSync(version = 1, firstPos = pool.getBlockRange(headBlockRoot, startSlot, step, results.toOpenArray(0, lastPos)) for i in firstPos.int .. lastPos.int: + trace "wrote response block", slot = results[i].slot await response.write(pool.get(results[i]).data) proc beaconBlocksByRoot( diff --git a/scripts/README.md b/scripts/README.md index 84d03d786..ab025bfec 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -14,7 +14,7 @@ The `beacon_node` binary has a `createTestnet` command. ```bash nim c -r beacon_chain/beacon_node \ - --data-dir=$DATA_DIR/node-0 \ + --data-dir=$NETWORK_DIR/data \ createTestnet \ --validators-dir=$NETWORK_DIR \ --total-validators=$VALIDATOR_COUNT \ diff --git a/tests/simulation/run_node.sh b/tests/simulation/run_node.sh index b6cf08da0..f4afa6074 100755 --- a/tests/simulation/run_node.sh +++ b/tests/simulation/run_node.sh @@ -14,24 +14,24 @@ source "${SIM_ROOT}/../../env.sh" cd "$GIT_ROOT" DATA_DIR="${SIMULATION_DIR}/node-$NODE_ID" - -V_PREFIX="${VALIDATORS_DIR}/v$(printf '%06d' $NODE_ID)" -PORT=$(printf '5%04d' $NODE_ID) +PORT=$(( BASE_P2P_PORT + NODE_ID )) NAT_FLAG="--nat:none" if [ "${NAT:-}" == "1" ]; then NAT_FLAG="--nat:any" fi -FIRST_VALIDATOR_IDX=$(( (NUM_VALIDATORS / ($NUM_NODES + $NUM_MISSING_NODES)) * $NODE_ID )) -LAST_VALIDATOR_IDX=$(( (NUM_VALIDATORS / ($NUM_NODES + $NUM_MISSING_NODES)) * ($NODE_ID + 1) - 1 )) - mkdir -p $DATA_DIR/validators rm -f $DATA_DIR/validators/* -pushd $VALIDATORS_DIR >/dev/null - cp $(seq -s " " -f v%07g.privkey $FIRST_VALIDATOR_IDX $LAST_VALIDATOR_IDX) $DATA_DIR/validators -popd >/dev/null +if [[ $NODE_ID -lt $TOTAL_NODES ]]; then + FIRST_VALIDATOR_IDX=$(( (NUM_VALIDATORS / TOTAL_NODES) * NODE_ID )) + LAST_VALIDATOR_IDX=$(( (NUM_VALIDATORS / TOTAL_NODES) * (NODE_ID + 1) - 1 )) + + pushd $VALIDATORS_DIR >/dev/null + cp $(seq -s " " -f v%07g.privkey $FIRST_VALIDATOR_IDX $LAST_VALIDATOR_IDX) $DATA_DIR/validators + popd >/dev/null +fi $BEACON_NODE_BIN \ --bootstrap-file=$NETWORK_BOOTSTRAP_FILE \ @@ -46,5 +46,5 @@ $BEACON_NODE_BIN \ --metrics-server=on \ --metrics-server-address="127.0.0.1" \ --metrics-server-port="$(( $BASE_METRICS_PORT + $NODE_ID ))" \ - "$@" + "$@" | tee "$DATA_DIR/node.log" diff --git a/tests/simulation/start.sh b/tests/simulation/start.sh index 7b9189c11..d653c6774 100755 --- a/tests/simulation/start.sh +++ b/tests/simulation/start.sh @@ -19,9 +19,9 @@ NIMFLAGS="-d:chronicles_log_level=DEBUG --hints:off --warnings:off --verbosity:0 # Run with "SHARD_COUNT=4 ./start.sh" to change these DEFS="" -DEFS+="-d:SHARD_COUNT=${SHARD_COUNT:-16} " # Spec default: 1024 -DEFS+="-d:SLOTS_PER_EPOCH=${SLOTS_PER_EPOCH:-16} " # Spec default: 64 -DEFS+="-d:SECONDS_PER_SLOT=${SECONDS_PER_SLOT:-6} " # Spec default: 6 +DEFS+="-d:SHARD_COUNT=${SHARD_COUNT:-8} " # Spec default: 1024 +DEFS+="-d:SLOTS_PER_EPOCH=${SLOTS_PER_EPOCH:-8} " # Spec default: 64 +DEFS+="-d:SECONDS_PER_SLOT=${SECONDS_PER_SLOT:-4} " # Spec default: 6 LAST_VALIDATOR_NUM=$(( NUM_VALIDATORS - 1 )) LAST_VALIDATOR="$VALIDATORS_DIR/v$(printf '%07d' $LAST_VALIDATOR_NUM).deposit.json" @@ -47,14 +47,14 @@ fi if [ ! -f "${SNAPSHOT_FILE}" ]; then $BEACON_NODE_BIN \ - --data-dir="${SIMULATION_DIR}/node-0" \ + --data-dir="${SIMULATION_DIR}/node-$MASTER_NODE" \ createTestnet \ --validators-dir="${VALIDATORS_DIR}" \ --total-validators="${NUM_VALIDATORS}" \ --output-genesis="${SNAPSHOT_FILE}" \ --output-bootstrap-file="${NETWORK_BOOTSTRAP_FILE}" \ --bootstrap-address=127.0.0.1 \ - --bootstrap-port=50000 \ + --bootstrap-port=$(( BASE_P2P_PORT + MASTER_NODE )) \ --genesis-offset=5 # Delay in seconds fi @@ -87,7 +87,7 @@ fi # use the exported Grafana dashboard for a single node to create one for all nodes "${SIM_ROOT}/../../build/process_dashboard" \ - --nodes=${NUM_NODES} \ + --nodes=${TOTAL_NODES} \ --in="${SIM_ROOT}/beacon-chain-sim-node0-Grafana-dashboard.json" \ --out="${SIM_ROOT}/beacon-chain-sim-all-nodes-Grafana-dashboard.json" @@ -100,10 +100,9 @@ if [ "$USE_MULTITAIL" = "no" ]; then fi COMMANDS=() -LAST_NODE=$(( NUM_NODES - 1 )) -for i in $(seq 0 $LAST_NODE); do - if [[ "$i" != "0" && "$USE_MULTITAIL" == "no" ]]; then +for i in $(seq $MASTER_NODE $TOTAL_USER_NODES); do + if [[ "$i" != "$MASTER_NODE" && "$USE_MULTITAIL" == "no" ]]; then # Wait for the master node to write out its address file while [ ! -f "${MASTER_NODE_ADDRESS_FILE}" ]; do sleep 0.1 diff --git a/tests/simulation/tmux_demo.sh b/tests/simulation/tmux_demo.sh index c0baa2d0f..ca290804b 100755 --- a/tests/simulation/tmux_demo.sh +++ b/tests/simulation/tmux_demo.sh @@ -14,7 +14,7 @@ tmux set-option -g history-limit 999999 tmux set -g mouse on tmux send-keys -t 0 './start.sh' Enter -tmux new-window -n "demo_node" "./wait_master_node.sh && ./run_node.sh $NUM_NODES" +tmux new-window -n "demo_node" "./wait_master_node.sh && ./run_node.sh 0" tmux attach-session -d diff --git a/tests/simulation/vars.sh b/tests/simulation/vars.sh index 3181740c7..7396a71dd 100644 --- a/tests/simulation/vars.sh +++ b/tests/simulation/vars.sh @@ -16,8 +16,10 @@ cd - &>/dev/null # When changing these, also update the readme section on running simulation # so that the run_node example is correct! NUM_VALIDATORS=${VALIDATORS:-192} -NUM_NODES=${NODES:-6} -NUM_MISSING_NODES=${MISSING_NODES:-1} +TOTAL_NODES=${NODES:-4} +TOTAL_USER_NODES=${USER_NODES:-0} +TOTAL_SYSTEM_NODES=$(( TOTAL_NODES - TOTAL_USER_NODES )) +MASTER_NODE=$(( TOTAL_NODES - 1 )) SIMULATION_DIR="${SIM_ROOT}/data" METRICS_DIR="${SIM_ROOT}/prometheus" @@ -26,7 +28,9 @@ SNAPSHOT_FILE="${SIMULATION_DIR}/state_snapshot.ssz" NETWORK_BOOTSTRAP_FILE="${SIMULATION_DIR}/bootstrap_nodes.txt" BEACON_NODE_BIN="${SIMULATION_DIR}/beacon_node" DEPLOY_DEPOSIT_CONTRACT_BIN="${SIMULATION_DIR}/deploy_deposit_contract" -MASTER_NODE_ADDRESS_FILE="${SIMULATION_DIR}/node-0/beacon_node.address" +MASTER_NODE_ADDRESS_FILE="${SIMULATION_DIR}/node-${MASTER_NODE}/beacon_node.address" + +BASE_P2P_PORT=30000 BASE_METRICS_PORT=8008 # Set DEPOSIT_WEB3_URL_ARG to empty to get genesis state from file, not using web3 # DEPOSIT_WEB3_URL_ARG=--web3-url=ws://localhost:8545