launch_local_testnet.sh: avoid infinite waits

and tail the logs on failure

scripts/launch_local_testnet.sh

@@ -153,6 +153,7 @@ trap 'cleanup' SIGINT SIGTERM EXIT
 PIDS=""
 NODES_WITH_VALIDATORS=${NODES_WITH_VALIDATORS:-4}
 VALIDATORS_PER_NODE=$(( $RANDOM_VALIDATORS / $NODES_WITH_VALIDATORS ))
+BOOTSTRAP_TIMEOUT=10 # in seconds
 
 for NUM_NODE in $(seq 0 $(( ${NUM_NODES} - 1 ))); do
 	if [[ ${NUM_NODE} == 0 ]]; then
@@ -160,8 +161,14 @@ for NUM_NODE in $(seq 0 $(( ${NUM_NODES} - 1 ))); do
 	else
 		BOOTSTRAP_ARG="--bootstrap-file=${NETWORK_DIR}/bootstrap_nodes.txt"
 		# Wait for the master node to write out its address file
+		START_TIMESTAMP=$(date +%s)
 		while [ ! -f "${DATA_DIR}/node0/beacon_node.address" ]; do
 			sleep 0.1
+			NOW_TIMESTAMP=$(date +%s)
+			if [[ "$(( NOW_TIMESTAMP - START_TIMESTAMP ))" -ge "$BOOTSTRAP_TIMEOUT" ]]; then
+				echo "Bootstrap node failed to start in ${BOOTSTRAP_TIMEOUT} seconds. Aborting."
+				exit 1
+			fi
 		done
 	fi
 
@@ -178,13 +185,14 @@ for NUM_NODE in $(seq 0 $(( ${NUM_NODES} - 1 ))); do
 	./build/beacon_node \
 		--nat:extip:127.0.0.1 \
 		--log-level="${LOG_LEVEL}" \
-		--tcp-port=$(( ${BOOTSTRAP_PORT} + ${NUM_NODE} )) \
-		--udp-port=$(( ${BOOTSTRAP_PORT} + ${NUM_NODE} )) \
+		--tcp-port=$(( BOOTSTRAP_PORT + NUM_NODE )) \
+		--udp-port=$(( BOOTSTRAP_PORT + NUM_NODE )) \
 		--data-dir="${NODE_DATA_DIR}" \
 		${BOOTSTRAP_ARG} \
 		--state-snapshot="${NETWORK_DIR}/genesis.ssz" \
 		${EXTRA_ARGS} \
 		> "${DATA_DIR}/log${NUM_NODE}.txt" 2>&1 &
+
 	if [[ "${PIDS}" == "" ]]; then
 		PIDS="$!"
 	else
@@ -192,6 +200,14 @@ for NUM_NODE in $(seq 0 $(( ${NUM_NODES} - 1 ))); do
 	fi
 done
 
+# give the regular nodes time to crash
+sleep 5
+BG_JOBS="$(jobs | wc -l)"
+if [[ "$BG_JOBS" != "$NUM_NODES" ]]; then
+	echo "$((NUM_NODES - BG_JOBS)) beacon_node instance(s) exited early. Aborting."
+	exit 1
+fi
+
 if [[ "$USE_HTOP" == "1" ]]; then
 	htop -p "$PIDS"
 	cleanup
@@ -202,6 +218,12 @@ else
 	done
 	if [[ "$FAILED" != "0" ]]; then
 		echo "${FAILED} child processes had non-zero exit codes (or exited early)."
+		LOG_LINES=20
+		for NUM_NODE in $(seq 0 $(( ${NUM_NODES} - 1 ))); do
+			echo "Last ${LOG_LINES} lines of ${DATA_DIR}/log${NUM_NODE}.txt:"
+			tail -n ${LOG_LINES} "${DATA_DIR}/log${NUM_NODE}.txt"
+			echo "======"
+		done
 		exit 1
 	fi
 fi