Add support for using custom remote signers in local sim (#4989)

* Add support for using custom remote signers in local sim Other changes: * Enable the Nimbus remote signer in the minimal simulation * Move all log files into the `logs` folder of the simulation * Create PID files for all processes and use them during the clean-up phase instead of the previous more fragile methods for killing the remaining processes.
2025-01-11 14:54:12 +00:00 · 2023-05-25 18:05:38 +03:00 · 2023-05-25 18:05:38 +03:00 · 8833acbe23
commit 8833acbe23
parent 00728e9bb7
8 changed files with 216 additions and 138 deletions
--- a/9
+++ b/9
@ -185,8 +185,8 @@ libbacktrace:
 # - --base-metrics-port + [0, --nodes)
 # - --base-vc-keymanager-port + [0, --nodes)
 # - --base-vc-metrics-port + [0, --nodes]
-# - --base-remote-signer-port + [0, --nimbus-signer-nodes | --web3signer-nodes)
-# - --base-remote-signer-metrics-port + [0, --nimbus-signer-node | --web3signer-nodes)
+# - --base-remote-signer-port + [0, --signer-nodes)
+# - --base-remote-signer-metrics-port + [0, --signer-nodes)
 #
 # Local testnets with --run-geth or --run-nimbus (only these ports):
 # - --base-el-net-port + --el-port-offset * [0, --nodes + --light-clients)
@ -204,11 +204,16 @@ restapi-test:
 		--resttest-delay 30 \
 		--kill-old-processes

+SIGNER_TYPE := nimbus
+
 local-testnet-minimal:
 	./scripts/launch_local_testnet.sh \
 		--data-dir $@ \
 		--preset minimal \
 		--nodes 2 \
+		--signer-nodes 1 \
+		--remote-validators-count 1024 \
+		--signer-type $(SIGNER_TYPE) \
 		--capella-fork-epoch 3 \
 		--deneb-fork-epoch 20 \
 		--stop-at-epoch 6 \
--- a/ci/Jenkinsfile
+++ b/ci/Jenkinsfile
@ -111,7 +111,7 @@ pipeline {
            sh 'make local-testnet-minimal'
          } }
          post { always {
-            sh 'tar cjf local-testnet-minimal.tar.gz local-testnet-minimal/*.txt'
+            sh 'tar cjf local-testnet-minimal.tar.gz local-testnet-minimal/logs/*'
          } }
        }

@ -120,7 +120,7 @@ pipeline {
            sh 'make local-testnet-mainnet'
          } }
          post { always {
-            sh 'tar cjf local-testnet-mainnet.tar.gz local-testnet-mainnet/*.txt'
+            sh 'tar cjf local-testnet-mainnet.tar.gz local-testnet-mainnet/logs/*'
          } }
        }
      }
--- a/scripts/launch_local_testnet.sh
+++ b/scripts/launch_local_testnet.sh
@ -27,7 +27,7 @@ log() {
 source "$SCRIPTS_DIR/detect_platform.sh"

 # Created processed that will be cleaned up when the script exits
-PIDS=""
+PIDS_TO_WAIT=""

 ####################
 # argument parsing #
@ -51,7 +51,7 @@ CURL_BINARY="$(command -v curl)" || { echo "Curl not installed. Aborting."; exit
 JQ_BINARY="$(command -v jq)" || { echo "Jq not installed. Aborting."; exit 1; }

 OPTS="ht:n:d:g"
-LONGOPTS="help,preset:,nodes:,data-dir:,remote-validators-count:,threshold:,nimbus-signer-nodes:,web3signer-nodes:,with-ganache,stop-at-epoch:,disable-htop,use-vc:,disable-vc,enable-payload-builder,enable-logtrace,log-level:,base-port:,base-rest-port:,base-metrics-port:,base-vc-metrics-port:,base-vc-keymanager-port:,base-remote-signer-port:,base-remote-signer-metrics-port:,base-el-net-port:,base-el-rpc-port:,base-el-ws-port:,base-el-auth-rpc-port:,el-port-offset:,reuse-existing-data-dir,reuse-binaries,timeout:,kill-old-processes,eth2-docker-image:,lighthouse-vc-nodes:,run-geth,dl-geth,dl-nimbus-eth1,dl-nimbus-eth2,light-clients:,run-nimbus-eth1,verbose,altair-fork-epoch:,bellatrix-fork-epoch:,capella-fork-epoch:,deneb-fork-epoch:"
+LONGOPTS="help,preset:,nodes:,data-dir:,remote-validators-count:,threshold:,signer-nodes:,signer-type:,with-ganache,stop-at-epoch:,disable-htop,use-vc:,disable-vc,enable-payload-builder,enable-logtrace,log-level:,base-port:,base-rest-port:,base-metrics-port:,base-vc-metrics-port:,base-vc-keymanager-port:,base-remote-signer-port:,base-remote-signer-metrics-port:,base-el-net-port:,base-el-rpc-port:,base-el-ws-port:,base-el-auth-rpc-port:,el-port-offset:,reuse-existing-data-dir,reuse-binaries,timeout:,kill-old-processes,eth2-docker-image:,lighthouse-vc-nodes:,run-geth,dl-geth,dl-nimbus-eth1,dl-nimbus-eth2,light-clients:,run-nimbus-eth1,verbose,altair-fork-epoch:,bellatrix-fork-epoch:,capella-fork-epoch:,deneb-fork-epoch:"

 # default values
 BINARIES=""
@ -88,7 +88,6 @@ TIMEOUT_DURATION="0"
 CONST_PRESET="mainnet"
 KILL_OLD_PROCESSES="0"
 ETH2_DOCKER_IMAGE=""
-NIMBUS_SIGNER_NODES=0
 REMOTE_SIGNER_THRESHOLD=1
 REMOTE_VALIDATORS_COUNT=0
 LC_NODES=1
@ -111,8 +110,8 @@ RUN_NIMBUS_ETH1="0"
 : ${WEB3SIGNER_VERSION:=22.11.0}
 : ${WEB3SIGNER_DIR:="${BUILD_DIR}/downloads/web3signer-${WEB3SIGNER_VERSION}"}
 : ${WEB3SIGNER_BINARY:="${WEB3SIGNER_DIR}/bin/web3signer$BAT_EXTENSION"}
-WEB3SIGNER_NODES=0
-PROCS_TO_KILL=("nimbus_beacon_node" "nimbus_validator_client" "nimbus_signing_node" "nimbus_light_client")
+: ${SIGNER_NODES:=0}
+: ${SIGNER_TYPE:="nimbus"}
 PORTS_TO_KILL=()
 WEB3_ARG=()
 CLEANUP_DIRS=()
@ -158,8 +157,8 @@ CI run: $(basename "$0") --disable-htop -- --verify-finalization
  --remote-validators-count   number of remote validators which will be generated
  --threshold                 used by a threshold secret sharing mechanism and determine how many shares are need to
                              restore signature of the original secret key
-  --web3signer-nodes          number of remote web3signer nodes
-  --nimbus-signer-nodes       number of remote nimbus signing nodes
+  --signer-nodes              number of remote signer nodes
+  --signer-type               a script in the scripts/signers directory, used to launch remote signers
  --light-clients             number of light clients
  --run-nimbus-eth1           Run nimbush-eth1 as EL
  --run-geth                  Run geth EL clients
@ -187,12 +186,12 @@ while true; do
      NUM_NODES="$2"
      shift 2
      ;;
-    --web3signer-nodes)
-      WEB3SIGNER_NODES=$2
+    --signer-nodes)
+      SIGNER_NODES=$2
      shift 2
      ;;
-    --nimbus-signer-nodes)
-      NIMBUS_SIGNER_NODES=$2
+    --signer-type)
+      SIGNER_TYPE=$2
      shift 2
      ;;
    --remote-validators-count)
@ -381,6 +380,9 @@ if [[ "$REUSE_EXISTING_DATA_DIR" == "0" ]]; then
  rm -rf "${DATA_DIR}"
 fi

+rm -rf "${DATA_DIR}/pids/*"
+mkdir -p "${DATA_DIR}/pids" "${DATA_DIR}/logs"
+
 if [[ "${LIGHTHOUSE_VC_NODES}" != "0" && "${CONST_PRESET}" != "mainnet" ]]; then
  echo "The prebuilt Lighthouse binary we're using only supports mainnet. Aborting."
  exit 1
@ -445,8 +447,7 @@ kill_by_port() {

 GETH_NUM_NODES="$(( NUM_NODES + LC_NODES ))"
 NIMBUS_ETH1_NUM_NODES="$(( NUM_NODES + LC_NODES ))"
-REMOTE_SIGNER_NODES=$(( NIMBUS_SIGNER_NODES + WEB3SIGNER_NODES ))
-LAST_REMOTE_SIGNER_NODE_IDX=$(( REMOTE_SIGNER_NODES - 1 ))
+LAST_SIGNER_NODE_IDX=$(( SIGNER_NODES - 1 ))

 if [[ "${RUN_GETH}" == "1" ]]; then
  source "${SCRIPTS_DIR}/geth_binaries.sh"
@ -502,7 +503,7 @@ if [[ "${OS}" != "windows" ]]; then
  fi

  # Stop Remote Signers
-  for NUM_REMOTE in $(seq 0 $LAST_REMOTE_SIGNER_NODE_IDX); do
+  for NUM_REMOTE in $(seq 0 $LAST_SIGNER_NODE_IDX); do
    for PORT in $(( BASE_REMOTE_SIGNER_PORT + NUM_REMOTE )) \
                $(( BASE_REMOTE_SIGNER_METRICS_PORT + NUM_REMOTE )) ; do
      PORTS_TO_KILL+=("${PORT}")
@ -667,8 +668,12 @@ if [[ "$LC_NODES" -ge "1" ]]; then
  BINARIES="${BINARIES} nimbus_light_client"
 fi

-if [[ "$NIMBUS_SIGNER_NODES" -gt "0" ]]; then
-  BINARIES="${BINARIES} nimbus_signing_node"
+if [[ "$SIGNER_NODES" -gt "0" ]]; then
+  if [[ "$SIGNER_TYPE" == "nimbus" ]]; then
+    BINARIES="${BINARIES} nimbus_signing_node"
+  elif [[ "$SIGNER_TYPE" == "web3signer" ]]; then
+    download_web3signer
+  fi
 fi

 # Don't build binaries if we are downloading them
@ -682,15 +687,6 @@ if [[ "${DL_NIMBUS_ETH2}" != "1" ]]; then
  BINARIES="${BINARIES} nimbus_beacon_node"
 fi

-if [[ "$WEB3SIGNER_NODES" -gt "0" ]]; then
-  download_web3signer
-fi
-
-if [[ "$WEB3SIGNER_NODES" -gt "0" && "$NIMBUS_SIGNER_NODES" -gt "0" ]]; then
-  echo "You can use either --web3signer-nodes or --nimbus-signer-nodes, but not together"
-  exit 1
-fi
-
 if [[ -n "${ETH2_DOCKER_IMAGE}" ]]; then
  DATA_DIR_FULL_PATH="$(cd "${DATA_DIR}"; pwd)"
  # CONTAINER_DATA_DIR must be used everywhere where paths are supplied to BEACON_NODE_COMMAND executions.
@ -739,28 +735,18 @@ cleanup() {
    PKILL_ECHO_FLAG='-l'
  fi

-  echo "Existing processes:"
-  for proc in "${PROCS_TO_KILL[@]}"; do
-    PROC_NAME=$(basename "$proc")
-    pgrep -al "${PROC_NAME}" || true
-  done
+  PIDS_TO_KILL=$(find "${DATA_DIR}/pids" -type f -exec cat {} \+ 2>/dev/null)

-  echo "Terminating:"
-  for proc in "${PROCS_TO_KILL[@]}"; do
-    PROC_NAME=$(basename "$proc")
-    echo -n "Terminating ${PROC_NAME}: " >&2
-    # WARNING: The '-P $$' avoids killing unrelated processes.
-    pkill -SIGTERM "${PKILL_ECHO_FLAG}" -P $$ "${PROC_NAME}" || true
+  echo Terminating processes...
+  for PID in $PIDS_TO_KILL; do
+    kill -SIGTERM $PID 2>/dev/null || true
  done

  sleep 2

-  echo "Killing:"
-  for proc in "${PROCS_TO_KILL[@]}"; do
-    PROC_NAME=$(basename "$proc")
-    echo -n "Killing ${PROC_NAME}: " >&2
-    # WARNING: The '-P $$' avoids killing unrelated processes.
-    pkill -SIGKILL "${PKILL_ECHO_FLAG}" -P $$ "${PROC_NAME}" || true
+  echo Killing processes...
+  for PID in $PIDS_TO_KILL; do
+    kill -SIGKILL $PID 2>/dev/null || true
  done

  # Delete all binaries we just built, because these are unusable outside this
@ -803,7 +789,7 @@ fi

 REMOTE_URLS=""

-for NUM_REMOTE in $(seq 0 $LAST_REMOTE_SIGNER_NODE_IDX); do
+for NUM_REMOTE in $(seq 0 $LAST_SIGNER_NODE_IDX); do
  REMOTE_PORT=$(( BASE_REMOTE_SIGNER_PORT + NUM_REMOTE ))
  REMOTE_URLS="${REMOTE_URLS} --remote-signer=http://127.0.0.1:${REMOTE_PORT}"
 done
@ -817,6 +803,11 @@ DEPOSITS_FILE="${DATA_DIR}/deposits.json"
 CONTAINER_DEPOSITS_FILE="${CONTAINER_DATA_DIR}/deposits.json"
 CONTAINER_DEPOSIT_TREE_SNAPSHOT_FILE="${CONTAINER_DATA_DIR}/deposit_tree_snapshot.ssz"

+if command -v ulimit; then
+  echo "Raising limits"
+  ulimit -n $((TOTAL_VALIDATORS * 10))
+fi
+
 if [[ "$REUSE_EXISTING_DATA_DIR" == "0" ]]; then
  ./build/ncli_testnet generateDeposits \
    --count=${TOTAL_VALIDATORS} \
@ -862,7 +853,6 @@ if [[ "${RUN_GETH}" == "1" ]]; then

  source "./scripts/start_geth_nodes.sh"

-  PROCS_TO_KILL+=("${GETH_BINARY}")
  CLEANUP_DIRS+=("${GETH_DATA_DIRS[@]}")
  MAIN_WEB3_URL="http://127.0.0.1:${GETH_RPC_PORTS[0]}"
  get_execution_genesis_block "${MAIN_WEB3_URL}" >  "$EXECUTION_GENESIS_BLOCK_JSON"
@ -876,7 +866,6 @@ if [[ "${RUN_NIMBUS_ETH1}" == "1" ]]; then

  source "./scripts/start_nimbus_el_nodes.sh"

-  PROCS_TO_KILL+=("${NIMBUS_ETH1_BINARY}")
  CLEANUP_DIRS+=("${NIMBUS_ETH1_DATA_DIRS[@]}")

  MAIN_WEB3_URL="http://127.0.0.1:${NIMBUS_ETH1_RPC_PORTS[0]}"
@ -942,7 +931,7 @@ fi

 dump_logs() {
  LOG_LINES=20
-  for LOG in "${DATA_DIR}"/log*.txt; do
+  for LOG in "${DATA_DIR}"/logs/*; do
    echo "Last ${LOG_LINES} lines of ${LOG}:"
    tail -n ${LOG_LINES} "${LOG}"
    echo "======"
@ -951,7 +940,7 @@ dump_logs() {

 dump_logtrace() {
  if [[ "$ENABLE_LOGTRACE" == "1" ]]; then
-    find "${DATA_DIR}" -maxdepth 1 -type f -regex '.*/log[0-9]+.txt' | sed -e"s/${DATA_DIR}\//--nodes=/" | sort | xargs ./build/ncli_testnet analyzeLogs --log-dir="${DATA_DIR}" --const-preset=${CONST_PRESET} || true
+    find "${DATA_DIR}/logs" -maxdepth 1 -type f -regex 'nimbus_beacon_node[0-9]+.jsonl' | sed -e"s/${DATA_DIR}\//--nodes=/" | sort | xargs ./build/ncli_testnet analyzeLogs --log-dir="${DATA_DIR}" --const-preset=${CONST_PRESET} || true
  fi
 }

@ -967,8 +956,8 @@ if [[ "${USE_VC}" == "1" ]]; then
  NUM_JOBS=$(( NUM_JOBS * 2 ))
 fi

-if [[ "$REMOTE_SIGNER_NODES" -ge "0" ]]; then
-  NUM_JOBS=$(( NUM_JOBS + REMOTE_SIGNER_NODES ))
+if [[ "$SIGNER_NODES" -ge "0" ]]; then
+  NUM_JOBS=$(( NUM_JOBS + SIGNER_NODES ))
 fi

 if [[ "$LC_NODES" -ge "1" ]]; then
@ -1062,74 +1051,19 @@ metrics = true
 metrics-address = "127.0.0.1"
 END_CLI_CONFIG

+# Export some variables that can be used by the signer launch scripts
+export DATA_DIR
+export BASE_REMOTE_SIGNER_PORT
+export WEB3SIGNER_BINARY
+export RUNTIME_CONFIG_FILE
+
 # https://ss64.com/osx/seq.html documents that at macOS seq(1) counts backwards
 # as probably do some others
-if ((NIMBUS_SIGNER_NODES > 0)); then
-  launch_nimbus_signing_node() {
-    SIGNING_NODE_IDX=$1
-    ./build/nimbus_signing_node \
-      --validators-dir="${DATA_DIR}/validators_shares/${SIGNING_NODE_IDX}" \
-      --secrets-dir="${DATA_DIR}/secrets_shares/${SIGNING_NODE_IDX}" \
-      --bind-port=$(( BASE_REMOTE_SIGNER_PORT + SIGNING_NODE_IDX - 1 ))
-    echo "Signing not exited with code $?"
-  }
-
-  for NUM_REMOTE in $(seq 1 $NIMBUS_SIGNER_NODES); do
+if ((SIGNER_NODES > 0)); then
+  for NUM_REMOTE in $(seq 0 $LAST_SIGNER_NODE_IDX); do
    # TODO find some way for this and other background-launched processes to
    # still participate in set -e, ideally
-    launch_nimbus_signing_node $NUM_REMOTE > "${DATA_DIR}/log_nimbus_signing_node_${NUM_REMOTE}.txt" &
-  done
-
-  PROCS_TO_KILL+=("nimbus_signing_node")
-fi
-
-if ((WEB3SIGNER_NODES > 0)); then
-  if ! command javac > /dev/null || ! javac -version > /dev/null; then
-    # On macOS, homebrew doesn't make java available in your PATH by default.
-    # Instead, macOS ships with a stub executable that displays a message that
-    # Java is not installed (javac -version exits with an error code 1).
-    # If the user is running under these default settings, but a homebrew
-    # installation is disovered, we are happy to use it just in this script:
-    if [[ -d /opt/homebrew/opt/openjdk/bin ]]; then
-      export PATH="/opt/homebrew/opt/openjdk/bin:$PATH"
-    fi
-  fi
-
-  launch_web3signer() {
-    WEB3SIGNER_NODE_IDX=$1
-
-    local secrets_dir="${DATA_DIR}/secrets_shares/${WEB3SIGNER_NODE_IDX}"
-    local keystores_dir="${DATA_DIR}/validators_shares/${WEB3SIGNER_NODE_IDX}"
-
-    # We re-arrange the keystore files to match the layout expected by the Web3Signer
-    # TODO generateSimulationDeposits can be refactored to produce the right layout from the start
-    for validator_pubkey in $(ls "$secrets_dir")
-    do
-      mv "$secrets_dir/$validator_pubkey" "$secrets_dir/$validator_pubkey.txt"
-      mv "$keystores_dir/$validator_pubkey/keystore.json" "$keystores_dir/$validator_pubkey.json"
-    done
-
-    # still participate in set -e, ideally
-    # TODO find some way for this and other background-launched processes to
-    "${WEB3SIGNER_BINARY}" \
-      --http-listen-port=$(( BASE_REMOTE_SIGNER_PORT + WEB3SIGNER_NODE_IDX - 1 )) \
-      --logging=DEBUG \
-      --metrics-enabled=true \
-      --metrics-port=$(( BASE_REMOTE_SIGNER_METRICS_PORT + WEB3SIGNER_NODE_IDX - 1 )) \
-      eth2 \
-      --slashing-protection-enabled=false \
-      --keystores-passwords-path="${secrets_dir}" \
-      --keystores-path="${keystores_dir}" \
-      --network="${RUNTIME_CONFIG_FILE}"
-
-    echo "Web3Signer exited with code $?"
-  }
-
-  PROCS_TO_KILL+=("${WEB3SIGNER_BINARY}")
-  PROCS_TO_KILL+=("java")
-
-  for NUM_REMOTE in $(seq 1 $WEB3SIGNER_NODES); do
-    launch_web3signer $NUM_REMOTE > "${DATA_DIR}/log_web3signer_${NUM_REMOTE}.txt" &
+    source "${SCRIPTS_DIR}/signers/${SIGNER_TYPE}.sh" $NUM_REMOTE
  done
 fi

@ -1210,9 +1144,10 @@ for NUM_NODE in $(seq 1 $NUM_NODES); do
    --metrics-port="$(( BASE_METRICS_PORT + NUM_NODE - 1 ))" \
    --doppelganger-detection=off \
    ${EXTRA_ARGS} \
-    &> "${DATA_DIR}/log${NUM_NODE}.txt" &
-
-  PIDS="${PIDS},$!"
+    &> "${DATA_DIR}/logs/nimbus_beacon_node.${NUM_NODE}.jsonl" &
+  PID=$!
+  PIDS_TO_WAIT="${PIDS_TO_WAIT},$!"
+  echo $PID > "$DATA_DIR/pids/nimbus_beacon_node.${NUM_NODE}"

  if [[ "${USE_VC}" == "1" ]]; then
    if [[ "${LIGHTHOUSE_VC_NODES}" -ge "${NUM_NODE}" ]]; then
@ -1232,9 +1167,8 @@ for NUM_NODE in $(seq 1 $NUM_NODES); do
        --beacon-nodes "http://127.0.0.1:$((BASE_REST_PORT + NUM_NODE))" \
        --testnet-dir "${DATA_DIR}" \
        --init-slashing-protection \
-        &> "${DATA_DIR}/log_val${NUM_NODE}.txt" &
-      # No "--stop-at-epoch" equivalent here, so we let these VC processes be
-      # killed the ugly way, when the script exits.
+        &> "${DATA_DIR}/logs/lighthouse_vc.${NUM_NODE}.txt" &
+      echo $! > "$DATA_DIR/pids/lighthouse_vc.${NUM_NODE}"
    else
      ./build/nimbus_validator_client \
        --log-level="${LOG_LEVEL}" \
@ -1247,8 +1181,10 @@ for NUM_NODE in $(seq 1 $NUM_NODES); do
        --keymanager-port=$(( BASE_VC_KEYMANAGER_PORT + NUM_NODE - 1 )) \
        --keymanager-token-file="${DATA_DIR}/keymanager-token" \
        --beacon-node="http://127.0.0.1:$(( BASE_REST_PORT + NUM_NODE - 1 ))" \
-        &> "${DATA_DIR}/log_val${NUM_NODE}.txt" &
-      PIDS="${PIDS},$!"
+        &> "${DATA_DIR}/logs/nimbus_validator_client.${NUM_NODE}.jsonl" &
+      PID=$!
+      PIDS_TO_WAIT="${PIDS_TO_WAIT},$PID"
+      echo $PID > "$DATA_DIR/pids/nimbus_validator_client.${NUM_NODE}"
    fi
  fi
 done
@ -1311,8 +1247,10 @@ if [ "$LC_NODES" -ge "1" ]; then
      --jwt-secret="${JWT_FILE}" \
      "${WEB3_ARG[@]}" \
      ${STOP_AT_EPOCH_FLAG} \
-      &> "${DATA_DIR}/log_lc${NUM_LC}.txt" &
-    PIDS="${PIDS},$!"
+      &> "${DATA_DIR}/logs/nimbus_light_client.${NUM_LC}.jsonl" &
+    PID=$!
+    PIDS_TO_WAIT="${PIDS_TO_WAIT},${PID}"
+    echo $PID > "${DATA_DIR}/pids/nimbus_light_client.${NUM_LC}"
  done
 fi

@ -1322,6 +1260,7 @@ BG_JOBS="$(jobs | wc -l | tr -d ' ')"
 if [[ "${TIMEOUT_DURATION}" != "0" ]]; then
  BG_JOBS=$(( BG_JOBS - 1 )) # minus the timeout bg job
 fi
+
 if [[ "$BG_JOBS" != "$NUM_JOBS" ]]; then
  echo "$(( NUM_JOBS - BG_JOBS )) nimbus_beacon_node/nimbus_validator_client/nimbus_light_client instance(s) exited early. Aborting."
  dump_logs
@ -1329,15 +1268,15 @@ if [[ "$BG_JOBS" != "$NUM_JOBS" ]]; then
  exit 1
 fi

-echo "About to wait for the following sub-processes: " $PIDS
+echo "About to wait for the following sub-processes: " $PIDS_TO_WAIT

 # launch "htop" or wait for background jobs
 if [[ "$USE_HTOP" == "1" ]]; then
-  htop -p "$PIDS"
+  htop -p "$PIDS_TO_WAIT"
  # Cleanup is done when this script exists, since we listen to the EXIT signal.
 else
  FAILED=0
-  for PID in $(echo "$PIDS" | tr ',' ' '); do
+  for PID in $(echo "$PIDS_TO_WAIT" | tr ',' ' '); do
    wait "$PID" || FAILED="$(( FAILED += 1 ))"
    echo $PID has completed
  done
--- a/scripts/signers/custom.sh
+++ b/scripts/signers/custom.sh
@ -0,0 +1,54 @@
+#!/usr/bin/bash
+
+# Copyright (c) 2023 Status Research & Development GmbH.
+# Licensed under either of:
+# - Apache License, version 2.0
+# - MIT license
+# at your option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+# This script will receive a single numeric argument representing the
+# instance ID of the signer. It will range from 0 to 9.
+# Usually, only a single signer will be launched to produce signatures
+# for all validators, but if threshold signing is enabled through the
+# `--signer-nodes N` parameter, the simulation script will launch
+# multiple instances working with partial validator keys.
+SIGNER_NODE_IDX=$1
+
+# These directories store the keystores and secrets generated by the
+# simulation script. These are either full validator keys or partial
+# validator keys depending on the `--signer-nodes` parameter (see above).
+local secrets_dir="${DATA_DIR}/secrets_shares/$((SIGNER_NODE_IDX + 1))"
+local keystores_dir="${DATA_DIR}/validators_shares/$((SIGNER_NODE_IDX + 1))"
+
+# You can re-arrange the keystore files to match the layout expected by
+# your signer. The example below demonstrates how this is done when working
+# with the Consensys Web3Signer:
+#
+# for validator_pubkey in $(ls "$secrets_dir")
+# do
+#   mv "$secrets_dir/$validator_pubkey" "$secrets_dir/$validator_pubkey.txt"
+#   mv "$keystores_dir/$validator_pubkey/keystore.json" "$keystores_dir/$validator_pubkey.json"
+# done
+
+# Here you need to launch your signer server process.
+# You must make sure that it will listen on the `$((BASE_REMOTE_SIGNER_PORT + SIGNER_NODE_IDX))` port.
+# The new process must be launched in the background.
+# Preferrably, you will also create a log file in the `${DATA_DIR}/logs` directory.
+
+# Here is an example way to achieve the above with the web3signer binary:
+#
+# web3signer \
+#   --http-listen-port=$(( BASE_REMOTE_SIGNER_PORT + SIGNER_NODE_IDX )) \
+#   --logging=DEBUG \
+#   --metrics-enabled=true \
+#   --metrics-port=$(( BASE_REMOTE_SIGNER_METRICS_PORT + SIGNER_NODE_IDX )) \
+#   eth2 \
+#   --slashing-protection-enabled=false \
+#   --keystores-passwords-path="${secrets_dir}" \
+#   --keystores-path="${keystores_dir}" \
+#   --network="${RUNTIME_CONFIG_FILE}" &> "${DATA_DIR}/web3signer.log" &
+
+# Finally, you must write the PIDs of any created processes in the `pids` directory
+# The names of the PID files can be arbitrary, but make sure they are unique for each launched instance
+echo $! > "${DATA_DIR}/pids/my-custom-signer.${SIGNER_NODE_IDX}"
--- a/scripts/signers/nimbus.sh
+++ b/scripts/signers/nimbus.sh
@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2023 Status Research & Development GmbH.
+# Licensed under either of:
+# - Apache License, version 2.0
+# - MIT license
+# at your option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+SIGNING_NODE_IDX=$1
+
+./build/nimbus_signing_node \
+  --log-level=DEBUG \
+  --validators-dir="${DATA_DIR}/validators_shares/$(( SIGNING_NODE_IDX + 1 ))" \
+  --secrets-dir="${DATA_DIR}/secrets_shares/$(( SIGNING_NODE_IDX + 1 ))" \
+  --bind-port=$(( BASE_REMOTE_SIGNER_PORT + SIGNING_NODE_IDX )) &> "${DATA_DIR}/logs/nimbus_signing_node.${SIGNING_NODE_IDX}.jsonl" &
+
+echo $! > "${DATA_DIR}/pids/nimbus_signing_node.${SIGNING_NODE_IDX}"
--- a/scripts/signers/web3signer.sh
+++ b/scripts/signers/web3signer.sh
@ -0,0 +1,47 @@
+#!/usr/bin/bash
+
+# Copyright (c) 2023 Status Research & Development GmbH.
+# Licensed under either of:
+# - Apache License, version 2.0
+# - MIT license
+# at your option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+if ! command javac > /dev/null || ! javac -version > /dev/null; then
+  # On macOS, homebrew doesn't make java available in your PATH by default.
+  # Instead, macOS ships with a stub executable that displays a message that
+  # Java is not installed (javac -version exits with an error code 1).
+  # If the user is running under these default settings, but a homebrew
+  # installation is disovered, we are happy to use it just in this script:
+  if [[ -d /opt/homebrew/opt/openjdk/bin ]]; then
+    export PATH="/opt/homebrew/opt/openjdk/bin:$PATH"
+  fi
+fi
+
+WEB3SIGNER_NODE_IDX=$1
+
+local secrets_dir="${DATA_DIR}/secrets_shares/$((WEB3SIGNER_NODE_IDX + 1))"
+local keystores_dir="${DATA_DIR}/validators_shares/$((WEB3SIGNER_NODE_IDX + 1))"
+
+# We re-arrange the keystore files to match the layout expected by the Web3Signer
+# TODO generateSimulationDeposits can be refactored to produce the right layout from the start
+for validator_pubkey in $(ls "$secrets_dir")
+do
+  mv "$secrets_dir/$validator_pubkey" "$secrets_dir/$validator_pubkey.txt"
+  mv "$keystores_dir/$validator_pubkey/keystore.json" "$keystores_dir/$validator_pubkey.json"
+done
+
+# still participate in set -e, ideally
+# TODO find some way for this and other background-launched processes to
+"${WEB3SIGNER_BINARY}" \
+  --http-listen-port=$(( BASE_REMOTE_SIGNER_PORT + WEB3SIGNER_NODE_IDX )) \
+  --logging=DEBUG \
+  --metrics-enabled=true \
+  --metrics-port=$(( BASE_REMOTE_SIGNER_METRICS_PORT + WEB3SIGNER_NODE_IDX )) \
+  eth2 \
+  --slashing-protection-enabled=false \
+  --keystores-passwords-path="${secrets_dir}" \
+  --keystores-path="${keystores_dir}" \
+  --network="${RUNTIME_CONFIG_FILE}" &> "${DATA_DIR}/logs/web3signer.${WEB3SIGNER_NODE_IDX}.log" &
+
+echo $! > "${DATA_DIR}/pids/web3signer.${WEB3SIGNER_NODE_IDX}"
--- a/scripts/start_geth_nodes.sh
+++ b/scripts/start_geth_nodes.sh
@ -1,5 +1,12 @@
 #!/usr/bin/env bash

+# Copyright (c) 2023 Status Research & Development GmbH.
+# Licensed under either of:
+# - Apache License, version 2.0
+# - MIT license
+# at your option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
 set -euo pipefail

 SCRIPTS_DIR="$(dirname "${BASH_SOURCE[0]}")"
@ -14,9 +21,9 @@ log "Using ${GETH_BINARY}"

 for GETH_NODE_IDX in $(seq 0 $GETH_LAST_NODE_IDX); do
  mkdir -p "${GETH_DATA_DIRS[GETH_NODE_IDX]}"
-  set -x
-  ${GETH_BINARY} version
-  ${GETH_BINARY} --datadir "${GETH_DATA_DIRS[GETH_NODE_IDX]}" init "${EXECUTION_GENESIS_JSON}"
+  GETH_LOG="${DATA_DIR}/logs/geth.${GETH_NODE_IDX}.txt"
+  ${GETH_BINARY} version > "$GETH_LOG"
+  ${GETH_BINARY} --datadir "${GETH_DATA_DIRS[GETH_NODE_IDX]}" init "${EXECUTION_GENESIS_JSON}" >> "$GETH_LOG" 2>&1
  ${GETH_BINARY} \
    --syncmode full \
    --datadir "${GETH_DATA_DIRS[GETH_NODE_IDX]}" \
@ -26,8 +33,9 @@ for GETH_NODE_IDX in $(seq 0 $GETH_LAST_NODE_IDX); do
    --port ${GETH_NET_PORTS[GETH_NODE_IDX]} \
    --authrpc.port ${GETH_AUTH_RPC_PORTS[GETH_NODE_IDX]} \
    --authrpc.jwtsecret "${JWT_FILE}" \
-        &> "${DATA_DIR}/geth-log${GETH_NODE_IDX}.txt" &
-  set +x
+        >> "${GETH_LOG}" 2>&1 &
+  PID=$!
+  echo $PID > "${DATA_DIR}/pids/geth.${GETH_NODE_IDX}"
 done

 for GETH_NODE_IDX in $(seq 0 $GETH_LAST_NODE_IDX); do
--- a/scripts/start_nimbus_el_nodes.sh
+++ b/scripts/start_nimbus_el_nodes.sh
@ -1,5 +1,12 @@
 #!/usr/bin/env bash

+# Copyright (c) 2023 Status Research & Development GmbH.
+# Licensed under either of:
+# - Apache License, version 2.0
+# - MIT license
+# at your option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
 set -euo pipefail

 SCRIPTS_DIR="$(dirname "${BASH_SOURCE[0]}")"
@ -29,8 +36,6 @@ if [ -d /opt/homebrew/lib ]; then
  # See https://github.com/Homebrew/brew/issues/13481 for more details
 fi

-PROCS_TO_KILL+="(${NIMBUS_ETH1_BINARY})"
-
 for NIMBUS_ETH1_NODE_IDX in $(seq 0 $NIMBUS_ETH1_LAST_NODE_IDX); do
  NIMBUS_ETH1_DATA_DIR=$(mktemp -d "${DATA_DIR}/nimbus-eth1-data-XXXXXX")
  NIMBUS_ETH1_DATA_DIRS+=("${NIMBUS_ETH1_DATA_DIR}")
@ -43,7 +48,9 @@ for NIMBUS_ETH1_NODE_IDX in $(seq 0 $NIMBUS_ETH1_LAST_NODE_IDX); do
    --jwt-secret="${JWT_FILE}" \
    --engine-api --engine-api-port="${NIMBUS_ETH1_AUTH_RPC_PORTS[NIMBUS_ETH1_NODE_IDX]}" \
    --rpc --rpc-port="${NIMBUS_ETH1_RPC_PORTS[NIMBUS_ETH1_NODE_IDX]}" \
-      &> "${DATA_DIR}/nimbus_eth1_log${NIMBUS_ETH1_NODE_IDX}.txt" &
+      &> "${DATA_DIR}/logs/nimbus_eth1.${NIMBUS_ETH1_NODE_IDX}.txt" &
+  PID=$!
+  echo $PID > "${DATA_DIR}/pids/nimbus_eth1.${NIMBUS_ETH1_NODE_IDX}"
 done

 echo "Waiting for the Nimbus ETH1 nodes to come online..."