From 8833acbe231219d7fa8c5a86abfe10844499e31d Mon Sep 17 00:00:00 2001 From: zah Date: Thu, 25 May 2023 18:05:38 +0300 Subject: [PATCH] Add support for using custom remote signers in local sim (#4989) * Add support for using custom remote signers in local sim Other changes: * Enable the Nimbus remote signer in the minimal simulation * Move all log files into the `logs` folder of the simulation * Create PID files for all processes and use them during the clean-up phase instead of the previous more fragile methods for killing the remaining processes. --- Makefile | 9 +- ci/Jenkinsfile | 4 +- scripts/launch_local_testnet.sh | 191 +++++++++++-------------------- scripts/signers/custom.sh | 54 +++++++++ scripts/signers/nimbus.sh | 18 +++ scripts/signers/web3signer.sh | 47 ++++++++ scripts/start_geth_nodes.sh | 18 ++- scripts/start_nimbus_el_nodes.sh | 13 ++- 8 files changed, 216 insertions(+), 138 deletions(-) create mode 100755 scripts/signers/custom.sh create mode 100755 scripts/signers/nimbus.sh create mode 100755 scripts/signers/web3signer.sh diff --git a/Makefile b/Makefile index 33e658706..b964698b1 100644 --- a/Makefile +++ b/Makefile @@ -185,8 +185,8 @@ libbacktrace: # - --base-metrics-port + [0, --nodes) # - --base-vc-keymanager-port + [0, --nodes) # - --base-vc-metrics-port + [0, --nodes] -# - --base-remote-signer-port + [0, --nimbus-signer-nodes | --web3signer-nodes) -# - --base-remote-signer-metrics-port + [0, --nimbus-signer-node | --web3signer-nodes) +# - --base-remote-signer-port + [0, --signer-nodes) +# - --base-remote-signer-metrics-port + [0, --signer-nodes) # # Local testnets with --run-geth or --run-nimbus (only these ports): # - --base-el-net-port + --el-port-offset * [0, --nodes + --light-clients) @@ -204,11 +204,16 @@ restapi-test: --resttest-delay 30 \ --kill-old-processes +SIGNER_TYPE := nimbus + local-testnet-minimal: ./scripts/launch_local_testnet.sh \ --data-dir $@ \ --preset minimal \ --nodes 2 \ + --signer-nodes 1 \ + --remote-validators-count 1024 \ + --signer-type $(SIGNER_TYPE) \ --capella-fork-epoch 3 \ --deneb-fork-epoch 20 \ --stop-at-epoch 6 \ diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile index 0fdb2a8be..3e9d9e543 100644 --- a/ci/Jenkinsfile +++ b/ci/Jenkinsfile @@ -111,7 +111,7 @@ pipeline { sh 'make local-testnet-minimal' } } post { always { - sh 'tar cjf local-testnet-minimal.tar.gz local-testnet-minimal/*.txt' + sh 'tar cjf local-testnet-minimal.tar.gz local-testnet-minimal/logs/*' } } } @@ -120,7 +120,7 @@ pipeline { sh 'make local-testnet-mainnet' } } post { always { - sh 'tar cjf local-testnet-mainnet.tar.gz local-testnet-mainnet/*.txt' + sh 'tar cjf local-testnet-mainnet.tar.gz local-testnet-mainnet/logs/*' } } } } diff --git a/scripts/launch_local_testnet.sh b/scripts/launch_local_testnet.sh index 6fef08aed..8d73f1d4e 100755 --- a/scripts/launch_local_testnet.sh +++ b/scripts/launch_local_testnet.sh @@ -27,7 +27,7 @@ log() { source "$SCRIPTS_DIR/detect_platform.sh" # Created processed that will be cleaned up when the script exits -PIDS="" +PIDS_TO_WAIT="" #################### # argument parsing # @@ -51,7 +51,7 @@ CURL_BINARY="$(command -v curl)" || { echo "Curl not installed. Aborting."; exit JQ_BINARY="$(command -v jq)" || { echo "Jq not installed. Aborting."; exit 1; } OPTS="ht:n:d:g" -LONGOPTS="help,preset:,nodes:,data-dir:,remote-validators-count:,threshold:,nimbus-signer-nodes:,web3signer-nodes:,with-ganache,stop-at-epoch:,disable-htop,use-vc:,disable-vc,enable-payload-builder,enable-logtrace,log-level:,base-port:,base-rest-port:,base-metrics-port:,base-vc-metrics-port:,base-vc-keymanager-port:,base-remote-signer-port:,base-remote-signer-metrics-port:,base-el-net-port:,base-el-rpc-port:,base-el-ws-port:,base-el-auth-rpc-port:,el-port-offset:,reuse-existing-data-dir,reuse-binaries,timeout:,kill-old-processes,eth2-docker-image:,lighthouse-vc-nodes:,run-geth,dl-geth,dl-nimbus-eth1,dl-nimbus-eth2,light-clients:,run-nimbus-eth1,verbose,altair-fork-epoch:,bellatrix-fork-epoch:,capella-fork-epoch:,deneb-fork-epoch:" +LONGOPTS="help,preset:,nodes:,data-dir:,remote-validators-count:,threshold:,signer-nodes:,signer-type:,with-ganache,stop-at-epoch:,disable-htop,use-vc:,disable-vc,enable-payload-builder,enable-logtrace,log-level:,base-port:,base-rest-port:,base-metrics-port:,base-vc-metrics-port:,base-vc-keymanager-port:,base-remote-signer-port:,base-remote-signer-metrics-port:,base-el-net-port:,base-el-rpc-port:,base-el-ws-port:,base-el-auth-rpc-port:,el-port-offset:,reuse-existing-data-dir,reuse-binaries,timeout:,kill-old-processes,eth2-docker-image:,lighthouse-vc-nodes:,run-geth,dl-geth,dl-nimbus-eth1,dl-nimbus-eth2,light-clients:,run-nimbus-eth1,verbose,altair-fork-epoch:,bellatrix-fork-epoch:,capella-fork-epoch:,deneb-fork-epoch:" # default values BINARIES="" @@ -88,7 +88,6 @@ TIMEOUT_DURATION="0" CONST_PRESET="mainnet" KILL_OLD_PROCESSES="0" ETH2_DOCKER_IMAGE="" -NIMBUS_SIGNER_NODES=0 REMOTE_SIGNER_THRESHOLD=1 REMOTE_VALIDATORS_COUNT=0 LC_NODES=1 @@ -111,8 +110,8 @@ RUN_NIMBUS_ETH1="0" : ${WEB3SIGNER_VERSION:=22.11.0} : ${WEB3SIGNER_DIR:="${BUILD_DIR}/downloads/web3signer-${WEB3SIGNER_VERSION}"} : ${WEB3SIGNER_BINARY:="${WEB3SIGNER_DIR}/bin/web3signer$BAT_EXTENSION"} -WEB3SIGNER_NODES=0 -PROCS_TO_KILL=("nimbus_beacon_node" "nimbus_validator_client" "nimbus_signing_node" "nimbus_light_client") +: ${SIGNER_NODES:=0} +: ${SIGNER_TYPE:="nimbus"} PORTS_TO_KILL=() WEB3_ARG=() CLEANUP_DIRS=() @@ -158,8 +157,8 @@ CI run: $(basename "$0") --disable-htop -- --verify-finalization --remote-validators-count number of remote validators which will be generated --threshold used by a threshold secret sharing mechanism and determine how many shares are need to restore signature of the original secret key - --web3signer-nodes number of remote web3signer nodes - --nimbus-signer-nodes number of remote nimbus signing nodes + --signer-nodes number of remote signer nodes + --signer-type a script in the scripts/signers directory, used to launch remote signers --light-clients number of light clients --run-nimbus-eth1 Run nimbush-eth1 as EL --run-geth Run geth EL clients @@ -187,12 +186,12 @@ while true; do NUM_NODES="$2" shift 2 ;; - --web3signer-nodes) - WEB3SIGNER_NODES=$2 + --signer-nodes) + SIGNER_NODES=$2 shift 2 ;; - --nimbus-signer-nodes) - NIMBUS_SIGNER_NODES=$2 + --signer-type) + SIGNER_TYPE=$2 shift 2 ;; --remote-validators-count) @@ -381,6 +380,9 @@ if [[ "$REUSE_EXISTING_DATA_DIR" == "0" ]]; then rm -rf "${DATA_DIR}" fi +rm -rf "${DATA_DIR}/pids/*" +mkdir -p "${DATA_DIR}/pids" "${DATA_DIR}/logs" + if [[ "${LIGHTHOUSE_VC_NODES}" != "0" && "${CONST_PRESET}" != "mainnet" ]]; then echo "The prebuilt Lighthouse binary we're using only supports mainnet. Aborting." exit 1 @@ -445,8 +447,7 @@ kill_by_port() { GETH_NUM_NODES="$(( NUM_NODES + LC_NODES ))" NIMBUS_ETH1_NUM_NODES="$(( NUM_NODES + LC_NODES ))" -REMOTE_SIGNER_NODES=$(( NIMBUS_SIGNER_NODES + WEB3SIGNER_NODES )) -LAST_REMOTE_SIGNER_NODE_IDX=$(( REMOTE_SIGNER_NODES - 1 )) +LAST_SIGNER_NODE_IDX=$(( SIGNER_NODES - 1 )) if [[ "${RUN_GETH}" == "1" ]]; then source "${SCRIPTS_DIR}/geth_binaries.sh" @@ -502,7 +503,7 @@ if [[ "${OS}" != "windows" ]]; then fi # Stop Remote Signers - for NUM_REMOTE in $(seq 0 $LAST_REMOTE_SIGNER_NODE_IDX); do + for NUM_REMOTE in $(seq 0 $LAST_SIGNER_NODE_IDX); do for PORT in $(( BASE_REMOTE_SIGNER_PORT + NUM_REMOTE )) \ $(( BASE_REMOTE_SIGNER_METRICS_PORT + NUM_REMOTE )) ; do PORTS_TO_KILL+=("${PORT}") @@ -667,8 +668,12 @@ if [[ "$LC_NODES" -ge "1" ]]; then BINARIES="${BINARIES} nimbus_light_client" fi -if [[ "$NIMBUS_SIGNER_NODES" -gt "0" ]]; then - BINARIES="${BINARIES} nimbus_signing_node" +if [[ "$SIGNER_NODES" -gt "0" ]]; then + if [[ "$SIGNER_TYPE" == "nimbus" ]]; then + BINARIES="${BINARIES} nimbus_signing_node" + elif [[ "$SIGNER_TYPE" == "web3signer" ]]; then + download_web3signer + fi fi # Don't build binaries if we are downloading them @@ -682,15 +687,6 @@ if [[ "${DL_NIMBUS_ETH2}" != "1" ]]; then BINARIES="${BINARIES} nimbus_beacon_node" fi -if [[ "$WEB3SIGNER_NODES" -gt "0" ]]; then - download_web3signer -fi - -if [[ "$WEB3SIGNER_NODES" -gt "0" && "$NIMBUS_SIGNER_NODES" -gt "0" ]]; then - echo "You can use either --web3signer-nodes or --nimbus-signer-nodes, but not together" - exit 1 -fi - if [[ -n "${ETH2_DOCKER_IMAGE}" ]]; then DATA_DIR_FULL_PATH="$(cd "${DATA_DIR}"; pwd)" # CONTAINER_DATA_DIR must be used everywhere where paths are supplied to BEACON_NODE_COMMAND executions. @@ -739,28 +735,18 @@ cleanup() { PKILL_ECHO_FLAG='-l' fi - echo "Existing processes:" - for proc in "${PROCS_TO_KILL[@]}"; do - PROC_NAME=$(basename "$proc") - pgrep -al "${PROC_NAME}" || true - done + PIDS_TO_KILL=$(find "${DATA_DIR}/pids" -type f -exec cat {} \+ 2>/dev/null) - echo "Terminating:" - for proc in "${PROCS_TO_KILL[@]}"; do - PROC_NAME=$(basename "$proc") - echo -n "Terminating ${PROC_NAME}: " >&2 - # WARNING: The '-P $$' avoids killing unrelated processes. - pkill -SIGTERM "${PKILL_ECHO_FLAG}" -P $$ "${PROC_NAME}" || true + echo Terminating processes... + for PID in $PIDS_TO_KILL; do + kill -SIGTERM $PID 2>/dev/null || true done sleep 2 - echo "Killing:" - for proc in "${PROCS_TO_KILL[@]}"; do - PROC_NAME=$(basename "$proc") - echo -n "Killing ${PROC_NAME}: " >&2 - # WARNING: The '-P $$' avoids killing unrelated processes. - pkill -SIGKILL "${PKILL_ECHO_FLAG}" -P $$ "${PROC_NAME}" || true + echo Killing processes... + for PID in $PIDS_TO_KILL; do + kill -SIGKILL $PID 2>/dev/null || true done # Delete all binaries we just built, because these are unusable outside this @@ -803,7 +789,7 @@ fi REMOTE_URLS="" -for NUM_REMOTE in $(seq 0 $LAST_REMOTE_SIGNER_NODE_IDX); do +for NUM_REMOTE in $(seq 0 $LAST_SIGNER_NODE_IDX); do REMOTE_PORT=$(( BASE_REMOTE_SIGNER_PORT + NUM_REMOTE )) REMOTE_URLS="${REMOTE_URLS} --remote-signer=http://127.0.0.1:${REMOTE_PORT}" done @@ -817,6 +803,11 @@ DEPOSITS_FILE="${DATA_DIR}/deposits.json" CONTAINER_DEPOSITS_FILE="${CONTAINER_DATA_DIR}/deposits.json" CONTAINER_DEPOSIT_TREE_SNAPSHOT_FILE="${CONTAINER_DATA_DIR}/deposit_tree_snapshot.ssz" +if command -v ulimit; then + echo "Raising limits" + ulimit -n $((TOTAL_VALIDATORS * 10)) +fi + if [[ "$REUSE_EXISTING_DATA_DIR" == "0" ]]; then ./build/ncli_testnet generateDeposits \ --count=${TOTAL_VALIDATORS} \ @@ -862,7 +853,6 @@ if [[ "${RUN_GETH}" == "1" ]]; then source "./scripts/start_geth_nodes.sh" - PROCS_TO_KILL+=("${GETH_BINARY}") CLEANUP_DIRS+=("${GETH_DATA_DIRS[@]}") MAIN_WEB3_URL="http://127.0.0.1:${GETH_RPC_PORTS[0]}" get_execution_genesis_block "${MAIN_WEB3_URL}" > "$EXECUTION_GENESIS_BLOCK_JSON" @@ -876,7 +866,6 @@ if [[ "${RUN_NIMBUS_ETH1}" == "1" ]]; then source "./scripts/start_nimbus_el_nodes.sh" - PROCS_TO_KILL+=("${NIMBUS_ETH1_BINARY}") CLEANUP_DIRS+=("${NIMBUS_ETH1_DATA_DIRS[@]}") MAIN_WEB3_URL="http://127.0.0.1:${NIMBUS_ETH1_RPC_PORTS[0]}" @@ -942,7 +931,7 @@ fi dump_logs() { LOG_LINES=20 - for LOG in "${DATA_DIR}"/log*.txt; do + for LOG in "${DATA_DIR}"/logs/*; do echo "Last ${LOG_LINES} lines of ${LOG}:" tail -n ${LOG_LINES} "${LOG}" echo "======" @@ -951,7 +940,7 @@ dump_logs() { dump_logtrace() { if [[ "$ENABLE_LOGTRACE" == "1" ]]; then - find "${DATA_DIR}" -maxdepth 1 -type f -regex '.*/log[0-9]+.txt' | sed -e"s/${DATA_DIR}\//--nodes=/" | sort | xargs ./build/ncli_testnet analyzeLogs --log-dir="${DATA_DIR}" --const-preset=${CONST_PRESET} || true + find "${DATA_DIR}/logs" -maxdepth 1 -type f -regex 'nimbus_beacon_node[0-9]+.jsonl' | sed -e"s/${DATA_DIR}\//--nodes=/" | sort | xargs ./build/ncli_testnet analyzeLogs --log-dir="${DATA_DIR}" --const-preset=${CONST_PRESET} || true fi } @@ -967,8 +956,8 @@ if [[ "${USE_VC}" == "1" ]]; then NUM_JOBS=$(( NUM_JOBS * 2 )) fi -if [[ "$REMOTE_SIGNER_NODES" -ge "0" ]]; then - NUM_JOBS=$(( NUM_JOBS + REMOTE_SIGNER_NODES )) +if [[ "$SIGNER_NODES" -ge "0" ]]; then + NUM_JOBS=$(( NUM_JOBS + SIGNER_NODES )) fi if [[ "$LC_NODES" -ge "1" ]]; then @@ -1062,74 +1051,19 @@ metrics = true metrics-address = "127.0.0.1" END_CLI_CONFIG +# Export some variables that can be used by the signer launch scripts +export DATA_DIR +export BASE_REMOTE_SIGNER_PORT +export WEB3SIGNER_BINARY +export RUNTIME_CONFIG_FILE + # https://ss64.com/osx/seq.html documents that at macOS seq(1) counts backwards # as probably do some others -if ((NIMBUS_SIGNER_NODES > 0)); then - launch_nimbus_signing_node() { - SIGNING_NODE_IDX=$1 - ./build/nimbus_signing_node \ - --validators-dir="${DATA_DIR}/validators_shares/${SIGNING_NODE_IDX}" \ - --secrets-dir="${DATA_DIR}/secrets_shares/${SIGNING_NODE_IDX}" \ - --bind-port=$(( BASE_REMOTE_SIGNER_PORT + SIGNING_NODE_IDX - 1 )) - echo "Signing not exited with code $?" - } - - for NUM_REMOTE in $(seq 1 $NIMBUS_SIGNER_NODES); do +if ((SIGNER_NODES > 0)); then + for NUM_REMOTE in $(seq 0 $LAST_SIGNER_NODE_IDX); do # TODO find some way for this and other background-launched processes to # still participate in set -e, ideally - launch_nimbus_signing_node $NUM_REMOTE > "${DATA_DIR}/log_nimbus_signing_node_${NUM_REMOTE}.txt" & - done - - PROCS_TO_KILL+=("nimbus_signing_node") -fi - -if ((WEB3SIGNER_NODES > 0)); then - if ! command javac > /dev/null || ! javac -version > /dev/null; then - # On macOS, homebrew doesn't make java available in your PATH by default. - # Instead, macOS ships with a stub executable that displays a message that - # Java is not installed (javac -version exits with an error code 1). - # If the user is running under these default settings, but a homebrew - # installation is disovered, we are happy to use it just in this script: - if [[ -d /opt/homebrew/opt/openjdk/bin ]]; then - export PATH="/opt/homebrew/opt/openjdk/bin:$PATH" - fi - fi - - launch_web3signer() { - WEB3SIGNER_NODE_IDX=$1 - - local secrets_dir="${DATA_DIR}/secrets_shares/${WEB3SIGNER_NODE_IDX}" - local keystores_dir="${DATA_DIR}/validators_shares/${WEB3SIGNER_NODE_IDX}" - - # We re-arrange the keystore files to match the layout expected by the Web3Signer - # TODO generateSimulationDeposits can be refactored to produce the right layout from the start - for validator_pubkey in $(ls "$secrets_dir") - do - mv "$secrets_dir/$validator_pubkey" "$secrets_dir/$validator_pubkey.txt" - mv "$keystores_dir/$validator_pubkey/keystore.json" "$keystores_dir/$validator_pubkey.json" - done - - # still participate in set -e, ideally - # TODO find some way for this and other background-launched processes to - "${WEB3SIGNER_BINARY}" \ - --http-listen-port=$(( BASE_REMOTE_SIGNER_PORT + WEB3SIGNER_NODE_IDX - 1 )) \ - --logging=DEBUG \ - --metrics-enabled=true \ - --metrics-port=$(( BASE_REMOTE_SIGNER_METRICS_PORT + WEB3SIGNER_NODE_IDX - 1 )) \ - eth2 \ - --slashing-protection-enabled=false \ - --keystores-passwords-path="${secrets_dir}" \ - --keystores-path="${keystores_dir}" \ - --network="${RUNTIME_CONFIG_FILE}" - - echo "Web3Signer exited with code $?" - } - - PROCS_TO_KILL+=("${WEB3SIGNER_BINARY}") - PROCS_TO_KILL+=("java") - - for NUM_REMOTE in $(seq 1 $WEB3SIGNER_NODES); do - launch_web3signer $NUM_REMOTE > "${DATA_DIR}/log_web3signer_${NUM_REMOTE}.txt" & + source "${SCRIPTS_DIR}/signers/${SIGNER_TYPE}.sh" $NUM_REMOTE done fi @@ -1210,9 +1144,10 @@ for NUM_NODE in $(seq 1 $NUM_NODES); do --metrics-port="$(( BASE_METRICS_PORT + NUM_NODE - 1 ))" \ --doppelganger-detection=off \ ${EXTRA_ARGS} \ - &> "${DATA_DIR}/log${NUM_NODE}.txt" & - - PIDS="${PIDS},$!" + &> "${DATA_DIR}/logs/nimbus_beacon_node.${NUM_NODE}.jsonl" & + PID=$! + PIDS_TO_WAIT="${PIDS_TO_WAIT},$!" + echo $PID > "$DATA_DIR/pids/nimbus_beacon_node.${NUM_NODE}" if [[ "${USE_VC}" == "1" ]]; then if [[ "${LIGHTHOUSE_VC_NODES}" -ge "${NUM_NODE}" ]]; then @@ -1232,9 +1167,8 @@ for NUM_NODE in $(seq 1 $NUM_NODES); do --beacon-nodes "http://127.0.0.1:$((BASE_REST_PORT + NUM_NODE))" \ --testnet-dir "${DATA_DIR}" \ --init-slashing-protection \ - &> "${DATA_DIR}/log_val${NUM_NODE}.txt" & - # No "--stop-at-epoch" equivalent here, so we let these VC processes be - # killed the ugly way, when the script exits. + &> "${DATA_DIR}/logs/lighthouse_vc.${NUM_NODE}.txt" & + echo $! > "$DATA_DIR/pids/lighthouse_vc.${NUM_NODE}" else ./build/nimbus_validator_client \ --log-level="${LOG_LEVEL}" \ @@ -1247,8 +1181,10 @@ for NUM_NODE in $(seq 1 $NUM_NODES); do --keymanager-port=$(( BASE_VC_KEYMANAGER_PORT + NUM_NODE - 1 )) \ --keymanager-token-file="${DATA_DIR}/keymanager-token" \ --beacon-node="http://127.0.0.1:$(( BASE_REST_PORT + NUM_NODE - 1 ))" \ - &> "${DATA_DIR}/log_val${NUM_NODE}.txt" & - PIDS="${PIDS},$!" + &> "${DATA_DIR}/logs/nimbus_validator_client.${NUM_NODE}.jsonl" & + PID=$! + PIDS_TO_WAIT="${PIDS_TO_WAIT},$PID" + echo $PID > "$DATA_DIR/pids/nimbus_validator_client.${NUM_NODE}" fi fi done @@ -1311,8 +1247,10 @@ if [ "$LC_NODES" -ge "1" ]; then --jwt-secret="${JWT_FILE}" \ "${WEB3_ARG[@]}" \ ${STOP_AT_EPOCH_FLAG} \ - &> "${DATA_DIR}/log_lc${NUM_LC}.txt" & - PIDS="${PIDS},$!" + &> "${DATA_DIR}/logs/nimbus_light_client.${NUM_LC}.jsonl" & + PID=$! + PIDS_TO_WAIT="${PIDS_TO_WAIT},${PID}" + echo $PID > "${DATA_DIR}/pids/nimbus_light_client.${NUM_LC}" done fi @@ -1322,6 +1260,7 @@ BG_JOBS="$(jobs | wc -l | tr -d ' ')" if [[ "${TIMEOUT_DURATION}" != "0" ]]; then BG_JOBS=$(( BG_JOBS - 1 )) # minus the timeout bg job fi + if [[ "$BG_JOBS" != "$NUM_JOBS" ]]; then echo "$(( NUM_JOBS - BG_JOBS )) nimbus_beacon_node/nimbus_validator_client/nimbus_light_client instance(s) exited early. Aborting." dump_logs @@ -1329,15 +1268,15 @@ if [[ "$BG_JOBS" != "$NUM_JOBS" ]]; then exit 1 fi -echo "About to wait for the following sub-processes: " $PIDS +echo "About to wait for the following sub-processes: " $PIDS_TO_WAIT # launch "htop" or wait for background jobs if [[ "$USE_HTOP" == "1" ]]; then - htop -p "$PIDS" + htop -p "$PIDS_TO_WAIT" # Cleanup is done when this script exists, since we listen to the EXIT signal. else FAILED=0 - for PID in $(echo "$PIDS" | tr ',' ' '); do + for PID in $(echo "$PIDS_TO_WAIT" | tr ',' ' '); do wait "$PID" || FAILED="$(( FAILED += 1 ))" echo $PID has completed done diff --git a/scripts/signers/custom.sh b/scripts/signers/custom.sh new file mode 100755 index 000000000..878a934c8 --- /dev/null +++ b/scripts/signers/custom.sh @@ -0,0 +1,54 @@ +#!/usr/bin/bash + +# Copyright (c) 2023 Status Research & Development GmbH. +# Licensed under either of: +# - Apache License, version 2.0 +# - MIT license +# at your option. This file may not be copied, modified, or distributed +# except according to those terms. + +# This script will receive a single numeric argument representing the +# instance ID of the signer. It will range from 0 to 9. +# Usually, only a single signer will be launched to produce signatures +# for all validators, but if threshold signing is enabled through the +# `--signer-nodes N` parameter, the simulation script will launch +# multiple instances working with partial validator keys. +SIGNER_NODE_IDX=$1 + +# These directories store the keystores and secrets generated by the +# simulation script. These are either full validator keys or partial +# validator keys depending on the `--signer-nodes` parameter (see above). +local secrets_dir="${DATA_DIR}/secrets_shares/$((SIGNER_NODE_IDX + 1))" +local keystores_dir="${DATA_DIR}/validators_shares/$((SIGNER_NODE_IDX + 1))" + +# You can re-arrange the keystore files to match the layout expected by +# your signer. The example below demonstrates how this is done when working +# with the Consensys Web3Signer: +# +# for validator_pubkey in $(ls "$secrets_dir") +# do +# mv "$secrets_dir/$validator_pubkey" "$secrets_dir/$validator_pubkey.txt" +# mv "$keystores_dir/$validator_pubkey/keystore.json" "$keystores_dir/$validator_pubkey.json" +# done + +# Here you need to launch your signer server process. +# You must make sure that it will listen on the `$((BASE_REMOTE_SIGNER_PORT + SIGNER_NODE_IDX))` port. +# The new process must be launched in the background. +# Preferrably, you will also create a log file in the `${DATA_DIR}/logs` directory. + +# Here is an example way to achieve the above with the web3signer binary: +# +# web3signer \ +# --http-listen-port=$(( BASE_REMOTE_SIGNER_PORT + SIGNER_NODE_IDX )) \ +# --logging=DEBUG \ +# --metrics-enabled=true \ +# --metrics-port=$(( BASE_REMOTE_SIGNER_METRICS_PORT + SIGNER_NODE_IDX )) \ +# eth2 \ +# --slashing-protection-enabled=false \ +# --keystores-passwords-path="${secrets_dir}" \ +# --keystores-path="${keystores_dir}" \ +# --network="${RUNTIME_CONFIG_FILE}" &> "${DATA_DIR}/web3signer.log" & + +# Finally, you must write the PIDs of any created processes in the `pids` directory +# The names of the PID files can be arbitrary, but make sure they are unique for each launched instance +echo $! > "${DATA_DIR}/pids/my-custom-signer.${SIGNER_NODE_IDX}" diff --git a/scripts/signers/nimbus.sh b/scripts/signers/nimbus.sh new file mode 100755 index 000000000..a3e25a0dc --- /dev/null +++ b/scripts/signers/nimbus.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +# Copyright (c) 2023 Status Research & Development GmbH. +# Licensed under either of: +# - Apache License, version 2.0 +# - MIT license +# at your option. This file may not be copied, modified, or distributed +# except according to those terms. + +SIGNING_NODE_IDX=$1 + +./build/nimbus_signing_node \ + --log-level=DEBUG \ + --validators-dir="${DATA_DIR}/validators_shares/$(( SIGNING_NODE_IDX + 1 ))" \ + --secrets-dir="${DATA_DIR}/secrets_shares/$(( SIGNING_NODE_IDX + 1 ))" \ + --bind-port=$(( BASE_REMOTE_SIGNER_PORT + SIGNING_NODE_IDX )) &> "${DATA_DIR}/logs/nimbus_signing_node.${SIGNING_NODE_IDX}.jsonl" & + +echo $! > "${DATA_DIR}/pids/nimbus_signing_node.${SIGNING_NODE_IDX}" diff --git a/scripts/signers/web3signer.sh b/scripts/signers/web3signer.sh new file mode 100755 index 000000000..d98342513 --- /dev/null +++ b/scripts/signers/web3signer.sh @@ -0,0 +1,47 @@ +#!/usr/bin/bash + +# Copyright (c) 2023 Status Research & Development GmbH. +# Licensed under either of: +# - Apache License, version 2.0 +# - MIT license +# at your option. This file may not be copied, modified, or distributed +# except according to those terms. + +if ! command javac > /dev/null || ! javac -version > /dev/null; then + # On macOS, homebrew doesn't make java available in your PATH by default. + # Instead, macOS ships with a stub executable that displays a message that + # Java is not installed (javac -version exits with an error code 1). + # If the user is running under these default settings, but a homebrew + # installation is disovered, we are happy to use it just in this script: + if [[ -d /opt/homebrew/opt/openjdk/bin ]]; then + export PATH="/opt/homebrew/opt/openjdk/bin:$PATH" + fi +fi + +WEB3SIGNER_NODE_IDX=$1 + +local secrets_dir="${DATA_DIR}/secrets_shares/$((WEB3SIGNER_NODE_IDX + 1))" +local keystores_dir="${DATA_DIR}/validators_shares/$((WEB3SIGNER_NODE_IDX + 1))" + +# We re-arrange the keystore files to match the layout expected by the Web3Signer +# TODO generateSimulationDeposits can be refactored to produce the right layout from the start +for validator_pubkey in $(ls "$secrets_dir") +do + mv "$secrets_dir/$validator_pubkey" "$secrets_dir/$validator_pubkey.txt" + mv "$keystores_dir/$validator_pubkey/keystore.json" "$keystores_dir/$validator_pubkey.json" +done + +# still participate in set -e, ideally +# TODO find some way for this and other background-launched processes to +"${WEB3SIGNER_BINARY}" \ + --http-listen-port=$(( BASE_REMOTE_SIGNER_PORT + WEB3SIGNER_NODE_IDX )) \ + --logging=DEBUG \ + --metrics-enabled=true \ + --metrics-port=$(( BASE_REMOTE_SIGNER_METRICS_PORT + WEB3SIGNER_NODE_IDX )) \ + eth2 \ + --slashing-protection-enabled=false \ + --keystores-passwords-path="${secrets_dir}" \ + --keystores-path="${keystores_dir}" \ + --network="${RUNTIME_CONFIG_FILE}" &> "${DATA_DIR}/logs/web3signer.${WEB3SIGNER_NODE_IDX}.log" & + +echo $! > "${DATA_DIR}/pids/web3signer.${WEB3SIGNER_NODE_IDX}" diff --git a/scripts/start_geth_nodes.sh b/scripts/start_geth_nodes.sh index cd2f72dec..d7349254f 100755 --- a/scripts/start_geth_nodes.sh +++ b/scripts/start_geth_nodes.sh @@ -1,5 +1,12 @@ #!/usr/bin/env bash +# Copyright (c) 2023 Status Research & Development GmbH. +# Licensed under either of: +# - Apache License, version 2.0 +# - MIT license +# at your option. This file may not be copied, modified, or distributed +# except according to those terms. + set -euo pipefail SCRIPTS_DIR="$(dirname "${BASH_SOURCE[0]}")" @@ -14,9 +21,9 @@ log "Using ${GETH_BINARY}" for GETH_NODE_IDX in $(seq 0 $GETH_LAST_NODE_IDX); do mkdir -p "${GETH_DATA_DIRS[GETH_NODE_IDX]}" - set -x - ${GETH_BINARY} version - ${GETH_BINARY} --datadir "${GETH_DATA_DIRS[GETH_NODE_IDX]}" init "${EXECUTION_GENESIS_JSON}" + GETH_LOG="${DATA_DIR}/logs/geth.${GETH_NODE_IDX}.txt" + ${GETH_BINARY} version > "$GETH_LOG" + ${GETH_BINARY} --datadir "${GETH_DATA_DIRS[GETH_NODE_IDX]}" init "${EXECUTION_GENESIS_JSON}" >> "$GETH_LOG" 2>&1 ${GETH_BINARY} \ --syncmode full \ --datadir "${GETH_DATA_DIRS[GETH_NODE_IDX]}" \ @@ -26,8 +33,9 @@ for GETH_NODE_IDX in $(seq 0 $GETH_LAST_NODE_IDX); do --port ${GETH_NET_PORTS[GETH_NODE_IDX]} \ --authrpc.port ${GETH_AUTH_RPC_PORTS[GETH_NODE_IDX]} \ --authrpc.jwtsecret "${JWT_FILE}" \ - &> "${DATA_DIR}/geth-log${GETH_NODE_IDX}.txt" & - set +x + >> "${GETH_LOG}" 2>&1 & + PID=$! + echo $PID > "${DATA_DIR}/pids/geth.${GETH_NODE_IDX}" done for GETH_NODE_IDX in $(seq 0 $GETH_LAST_NODE_IDX); do diff --git a/scripts/start_nimbus_el_nodes.sh b/scripts/start_nimbus_el_nodes.sh index 2973bdcc7..aadac7214 100755 --- a/scripts/start_nimbus_el_nodes.sh +++ b/scripts/start_nimbus_el_nodes.sh @@ -1,5 +1,12 @@ #!/usr/bin/env bash +# Copyright (c) 2023 Status Research & Development GmbH. +# Licensed under either of: +# - Apache License, version 2.0 +# - MIT license +# at your option. This file may not be copied, modified, or distributed +# except according to those terms. + set -euo pipefail SCRIPTS_DIR="$(dirname "${BASH_SOURCE[0]}")" @@ -29,8 +36,6 @@ if [ -d /opt/homebrew/lib ]; then # See https://github.com/Homebrew/brew/issues/13481 for more details fi -PROCS_TO_KILL+="(${NIMBUS_ETH1_BINARY})" - for NIMBUS_ETH1_NODE_IDX in $(seq 0 $NIMBUS_ETH1_LAST_NODE_IDX); do NIMBUS_ETH1_DATA_DIR=$(mktemp -d "${DATA_DIR}/nimbus-eth1-data-XXXXXX") NIMBUS_ETH1_DATA_DIRS+=("${NIMBUS_ETH1_DATA_DIR}") @@ -43,7 +48,9 @@ for NIMBUS_ETH1_NODE_IDX in $(seq 0 $NIMBUS_ETH1_LAST_NODE_IDX); do --jwt-secret="${JWT_FILE}" \ --engine-api --engine-api-port="${NIMBUS_ETH1_AUTH_RPC_PORTS[NIMBUS_ETH1_NODE_IDX]}" \ --rpc --rpc-port="${NIMBUS_ETH1_RPC_PORTS[NIMBUS_ETH1_NODE_IDX]}" \ - &> "${DATA_DIR}/nimbus_eth1_log${NIMBUS_ETH1_NODE_IDX}.txt" & + &> "${DATA_DIR}/logs/nimbus_eth1.${NIMBUS_ETH1_NODE_IDX}.txt" & + PID=$! + echo $PID > "${DATA_DIR}/pids/nimbus_eth1.${NIMBUS_ETH1_NODE_IDX}" done echo "Waiting for the Nimbus ETH1 nodes to come online..."