fix concurrent Jenkins stages (#3904)

The ports for the concurrently executing REST and Minimal testnet clash,
leading to some CI failures since #3827 introduced further concurrency.
Adjusting the ports to be distinct across various tests should fix this.

Makefile

@@ -148,12 +148,29 @@ update: | update-common
 libbacktrace:
 	+ "$(MAKE)" -C vendor/nim-libbacktrace --no-print-directory BUILD_CXX_LIB=0
 
+# Make sure ports don't overlap to support concurrent execution of tests
+# Avoid selecting ephemeral ports that may be used by others; safe = 5001-9999
+#
+# EXECUTOR_NUMBER: [0, 1] (depends on max number of concurrent CI jobs)
+#
+# The following port ranges are allocated (entire continuous range):
+# - --base-port + [0, --nodes + --light-clients)
+# - --base-rest-port + [0, --nodes)
+# - --base-metrics-port + [0, --nodes)
+# - --base-remote-signer-port + [0, --remote-signers)
+#
+# If --run-geth or --run-nimbus is specified (only these ports):
+# - --base-el-net-port + --el-port-offset * [0, --nodes + --light-clients)
+# - --base-el-http-port + --el-port-offset * [0, --nodes + --light-clients)
+# - --base-el-ws-port + --el-port-offset * [0, --nodes + --light-clients)
+# - --base-el-auth-rpc-port + --el-port-offset * [0, --nodes + --light-clients)
+
 restapi-test:
 	./tests/simulation/restapi.sh \
 		--data-dir resttest0_data \
-		--base-port $$(( 9100 + EXECUTOR_NUMBER * 100 )) \
+		--base-port $$(( 5001 + EXECUTOR_NUMBER * 500 )) \
-		--base-rest-port $$(( 7100 + EXECUTOR_NUMBER * 100 )) \
+		--base-rest-port $$(( 5031 + EXECUTOR_NUMBER * 500 )) \
-		--base-metrics-port $$(( 8108 + EXECUTOR_NUMBER * 100 )) \
+		--base-metrics-port $$(( 5061 + EXECUTOR_NUMBER * 500 )) \
 		--resttest-delay 30 \
 		--kill-old-processes
 
@@ -165,12 +182,17 @@ local-testnet-minimal:
 		--stop-at-epoch 5 \
 		--disable-htop \
 		--enable-logtrace \
-		--base-port $$(( 9100 + EXECUTOR_NUMBER * 100 )) \
+		--base-port $$(( 6001 + EXECUTOR_NUMBER * 500 )) \
-		--base-rest-port $$(( 7100 + EXECUTOR_NUMBER * 100 )) \
+		--base-rest-port $$(( 6031 + EXECUTOR_NUMBER * 500 )) \
-		--base-metrics-port $$(( 8108 + EXECUTOR_NUMBER * 100 )) \
+		--base-metrics-port $$(( 6061 + EXECUTOR_NUMBER * 500 )) \
+		--base-remote-signer-port $$(( 6101 + EXECUTOR_NUMBER * 500 )) \
+		--base-el-net-port $$(( 6201 + EXECUTOR_NUMBER * 500 )) \
+		--base-el-http-port $$(( 6202 + EXECUTOR_NUMBER * 500 )) \
+		--base-el-ws-port $$(( 6203 + EXECUTOR_NUMBER * 500 )) \
+		--base-el-auth-rpc-port $$(( 6204 + EXECUTOR_NUMBER * 500 )) \
+		--el-port-offset 5 \
 		--timeout 600 \
 		--kill-old-processes \
-		--light-clients 1 \
 		-- \
 		--verify-finalization \
 		--discv5:no
@@ -182,12 +204,17 @@ local-testnet-mainnet:
 		--stop-at-epoch 5 \
 		--disable-htop \
 		--enable-logtrace \
-		--base-port $$(( 9100 + EXECUTOR_NUMBER * 100 )) \
+		--base-port $$(( 7001 + EXECUTOR_NUMBER * 500 )) \
-		--base-rest-port $$(( 7100 + EXECUTOR_NUMBER * 100 )) \
+		--base-rest-port $$(( 7031 + EXECUTOR_NUMBER * 500 )) \
-		--base-metrics-port $$(( 8108 + EXECUTOR_NUMBER * 100 )) \
+		--base-metrics-port $$(( 7061 + EXECUTOR_NUMBER * 500 )) \
+		--base-remote-signer-port $$(( 7101 + EXECUTOR_NUMBER * 500 )) \
+		--base-el-net-port $$(( 7201 + EXECUTOR_NUMBER * 500 )) \
+		--base-el-http-port $$(( 7202 + EXECUTOR_NUMBER * 500 )) \
+		--base-el-ws-port $$(( 7203 + EXECUTOR_NUMBER * 500 )) \
+		--base-el-auth-rpc-port $$(( 7204 + EXECUTOR_NUMBER * 500 )) \
+		--el-port-offset 5 \
 		--timeout 2400 \
 		--kill-old-processes \
-		--light-clients 1 \
 		-- \
 		--verify-finalization \
 		--discv5:no

ci/Jenkinsfile

@@ -12,6 +12,7 @@ pipeline {
 
   options {
     timestamps()
+    ansiColor('xterm')
     /* This also includes wait time in the queue. */
     timeout(time: 24, unit: 'HOURS')
     /* Limit builds retained. */
@@ -72,8 +73,7 @@ pipeline {
     }
 
     stage('Finalizations') {
-      /* TODO: Try in parallel. */
+      stages {  /* parallel builds of minimal / mainnet not yet supported */
-      stages {
         stage('minimal') {
           steps { script { timeout(15) {
             launchLocalTestnet('minimal')

scripts/geth_vars.sh

@@ -1,12 +1,17 @@
-GETH_PORT_OFFSET=100
+# beacon_chain
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
 GETH_BINARY="${GETH_BINARY:-"${HOME}/go-ethereum/build/bin/geth"}"
 GETH_NUM_NODES="${GETH_NUM_NODES:-4}"
 GETH_BINARY="${GETH_BINARY:-${HOME}/go-ethereum/build/bin/geth}"
-GETH_BASE_HTTP_PORT="${GETH_BASE_HTTP_PORT:-8550}"
+GETH_BASE_NET_PORT="${BASE_EL_NET_PORT:-30303}"
-GETH_NET_BASE_PORT="${GETH_NET_BASE_PORT:-30303}"
+GETH_BASE_HTTP_PORT="${BASE_EL_HTTP_PORT:-8545}"
-GETH_HTTP_BASE_PORT="${GETH_HTTP_BASE_PORT:-8545}"
+GETH_BASE_WS_PORT="${BASE_EL_WS_PORT:-8546}"
-GETH_WS_BASE_PORT="${GETH_WS_BASE_PORT:-8546}"
+GETH_BASE_AUTH_RPC_PORT="${BASE_EL_AUTH_RPC_PORT:-8551}"
-GETH_AUTH_RPC_PORT_BASE="${GETH_AUTH_RPC_PORT_BASE:-8551}"
+GETH_PORT_OFFSET="${EL_PORT_OFFSET:-10}"
-PORT_OFFSET="${PORT_OFFSET:-100}"
 GENESISJSON="${GENESISJSON:-${BASEDIR}/geth_genesis.json}"
 DISCOVER="--nodiscover"

scripts/launch_local_testnet.sh

@@ -55,7 +55,7 @@ CURL_BINARY="$(command -v curl)" || { echo "Curl not installed. Aborting."; exit
 JQ_BINARY="$(command -v jq)" || { echo "Jq not installed. Aborting."; exit 1; }
 
 OPTS="ht:n:d:g"
-LONGOPTS="help,preset:,nodes:,data-dir:,remote-validators-count:,threshold:,remote-signers:,with-ganache,stop-at-epoch:,disable-htop,disable-vc,enable-logtrace,log-level:,base-port:,base-rest-port:,base-metrics-port:,reuse-existing-data-dir,reuse-binaries,timeout:,kill-old-processes,eth2-docker-image:,lighthouse-vc-nodes:,run-geth,dl-geth,light-clients:,run-nimbus-el,verbose"
+LONGOPTS="help,preset:,nodes:,data-dir:,remote-validators-count:,threshold:,remote-signers:,with-ganache,stop-at-epoch:,disable-htop,disable-vc,enable-logtrace,log-level:,base-port:,base-rest-port:,base-metrics-port:,base-remote-signer-port:,base-el-net-port:,base-el-http-port:,base-el-ws-port:,base-el-auth-rpc-port:,el-port-offset:,reuse-existing-data-dir,reuse-binaries,timeout:,kill-old-processes,eth2-docker-image:,lighthouse-vc-nodes:,run-geth,dl-geth,light-clients:,run-nimbus-el,verbose"
 
 # default values
 NIMFLAGS="${NIMFLAGS:-""}"
@@ -70,6 +70,11 @@ BASE_PORT="9000"
 BASE_REMOTE_SIGNER_PORT="6000"
 BASE_METRICS_PORT="8008"
 BASE_REST_PORT="7500"
+BASE_EL_NET_PORT="30303"
+BASE_EL_HTTP_PORT="8545"
+BASE_EL_WS_PORT="8546"
+BASE_EL_AUTH_RPC_PORT="8551"
+EL_PORT_OFFSET="10"
 REUSE_EXISTING_DATA_DIR="0"
 REUSE_BINARIES="0"
 NIMFLAGS=""
@@ -112,6 +117,12 @@ CI run: $(basename "$0") --disable-htop -- --verify-finalization
   --base-port                 bootstrap node's Eth2 traffic port (default: ${BASE_PORT})
   --base-rest-port            bootstrap node's REST port (default: ${BASE_REST_PORT})
   --base-metrics-port         bootstrap node's metrics server port (default: ${BASE_METRICS_PORT})
+  --base-remote-signer-port   first remote signing node's port (default: ${BASE_REMOTE_SIGNER_PORT})
+  --base-el-net-port          first EL's network traffic port (default: ${BASE_EL_NET_PORT})
+  --base-el-http-port         first EL's HTTP web3 port (default: ${BASE_EL_HTTP_PORT})
+  --base-el-ws-port           first EL's WebSocket web3 port (default: ${BASE_EL_WS_PORT})
+  --base-el-auth-rpc-port     first EL's authenticated engine API port (default: ${BASE_EL_AUTH_RPC_PORT})
+  --el-port-offset            offset to apply between ports of multiple ELs (default: ${EL_PORT_OFFSET})
   --disable-htop              don't use "htop" to see the nimbus_beacon_node processes
   --disable-vc                don't use validator client binaries for validators
                               (by default validators are split 50/50 between beacon nodes
@@ -211,6 +222,30 @@ while true; do
       BASE_METRICS_PORT="$2"
       shift 2
       ;;
+    --base-remote-signer-port)
+      BASE_REMOTE_SIGNER_PORT="$2"
+      shift 2
+      ;;
+    --base-el-net-port)
+      BASE_EL_NET_PORT="$2"
+      shift 2
+      ;;
+    --base-el-http-port)
+      BASE_EL_HTTP_PORT="$2"
+      shift 2
+      ;;
+    --base-el-ws-port)
+      BASE_EL_WS_PORT="$2"
+      shift 2
+      ;;
+    --base-el-auth-rpc-port)
+      BASE_EL_AUTH_RPC_PORT="$2"
+      shift 2
+      ;;
+    --el-port-offset)
+      EL_PORT_OFFSET="$2"
+      shift 2
+      ;;
     --reuse-existing-data-dir)
       REUSE_EXISTING_DATA_DIR="1"
       shift
@@ -342,8 +377,8 @@ if [[ "${OS}" != "windows" ]]; then
   #Stop geth nodes
   if [[ "${RUN_GETH}" == "1" ]]; then
     for NUM_NODE in $(seq 0 $(( NUM_NODES - 1 ))); do
-      for PORT in $(( NUM_NODE * GETH_PORT_OFFSET + GETH_NET_BASE_PORT )) $(( NUM_NODE * GETH_PORT_OFFSET + GETH_HTTP_BASE_PORT )) \
+      for PORT in $(( NUM_NODE * GETH_PORT_OFFSET + GETH_BASE_NET_PORT )) $(( NUM_NODE * GETH_PORT_OFFSET + GETH_BASE_HTTP_PORT )) \
-                  $(( NUM_NODE * GETH_PORT_OFFSET + GETH_WS_BASE_PORT )) $(( NUM_NODE * GETH_PORT_OFFSET + GETH_AUTH_RPC_PORT_BASE )); do
+                  $(( NUM_NODE * GETH_PORT_OFFSET + GETH_BASE_WS_PORT )) $(( NUM_NODE * GETH_PORT_OFFSET + GETH_BASE_AUTH_RPC_PORT )); do
         for PID in $(lsof -n -i tcp:${PORT} -sTCP:LISTEN -t); do
           echo -n "Found old geth processes listening on port ${PORT}, with PID ${PID}. "
           if [[ "${KILL_OLD_PROCESSES}" == "1" ]]; then
@@ -360,8 +395,8 @@ if [[ "${OS}" != "windows" ]]; then
 
   if [[ "${RUN_NIMBUS}" == "1" ]]; then
     for NUM_NODE in $(seq 0 $(( NUM_NODES - 1 ))); do
-      for PORT in $(( NUM_NODE * NIMBUSEL_PORT_OFFSET + NIMBUSEL_NET_BASE_PORT )) $(( NUM_NODE * NIMBUSEL_PORT_OFFSET + NIMBUSEL_HTTP_BASE_PORT )) \
+      for PORT in $(( NUM_NODE * NIMBUSEL_PORT_OFFSET + NIMBUSEL_BASE_NET_PORT )) $(( NUM_NODE * NIMBUSEL_PORT_OFFSET + NIMBUSEL_BASE_HTTP_PORT )) \
-                  $(( NUM_NODE * NIMBUSEL_PORT_OFFSET + NIMBUSEL_WS_BASE_PORT )) $(( NUM_NODE * NIMBUSEL_PORT_OFFSET + NIMBUSEL_AUTH_RPC_PORT_BASE )); do
+                  $(( NUM_NODE * NIMBUSEL_PORT_OFFSET + NIMBUSEL_BASE_WS_PORT )) $(( NUM_NODE * NIMBUSEL_PORT_OFFSET + NIMBUSEL_BASE_AUTH_RPC_PORT )); do
         for PID in $(lsof -n -i tcp:${PORT} -sTCP:LISTEN -t); do
           echo -n "Found old nimbus EL processes listening on port ${PORT}, with PID ${PID}. "
           if [[ "${KILL_OLD_PROCESSES}" == "1" ]]; then

scripts/nimbus_el_vars.sh

@@ -1,15 +1,20 @@
+# beacon_chain
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
 NIMBUSEL_DISCOVERY="--discovery=None"
 
-NIMBUSEL_PORT_OFFSET="${PORT_OFFSET:-100}"
 NIMBUSEL_BINARY="${NIMBUSEL_BINARY:-"${HOME}/work/nimbus-eth1/build/nimbus"}"
 NIMBUSEL_GENESIS="${NIMBUSEL_GENESIS:-"${HOME}/work/nimbus-eth2/scripts/nimbusel_genesis.json"}"
 NIMBUSEL_NUM_NODES="${NIMBUSEL_NUM_NODES:-4}"
 NIMBUSEL_BINARY="${NIMBUSEL_BINARY:-${HOME}/go-ethereum/build/bin/geth}"
-NIMBUSEL_BASE_HTTP_PORT="${NIMBUSEL_BASE_HTTP_PORT:-8550}"
+NIMBUSEL_BASE_NET_PORT="${BASE_EL_NET_PORT:-30303}"
-NIMBUSEL_NET_BASE_PORT="${NIMBUSEL_NET_BASE_PORT:-30303}"
+NIMBUSEL_BASE_HTTP_PORT="${BASE_EL_HTTP_PORT:-8545}"
-NIMBUSEL_HTTP_BASE_PORT="${NIMBUSEL_HTTP_BASE_PORT:-8545}"
+NIMBUSEL_BASE_WS_PORT="${BASE_EL_WS_PORT:-8546}"
-NIMBUSEL_WS_BASE_PORT="${NIMBUSEL_WS_BASE_PORT:-8546}"
+NIMBUSEL_BASE_AUTH_RPC_PORT="${BASE_EL_AUTH_RPC_PORT:-8551}"
-NIMBUSEL_AUTH_RPC_PORT_BASE="${NIMBUSEL_AUTH_RPC_PORT_BASE:-8551}"
+NIMBUSEL_PORT_OFFSET="${EL_PORT_OFFSET:-10}"
 CURL_BINARY=${CURL_BINARY:-curl}
 JQ_BINARY=${JQ_BINARY:-jq}
-

scripts/start_geth_nodes.sh

@@ -17,10 +17,10 @@ GETH_DATA_DIRS=()
 log "Using ${GETH_BINARY}"
 
 for GETH_NUM_NODE in $(seq 0 $(( GETH_NUM_NODES - 1 ))); do
-    GETH_NET_PORT=$(( GETH_NUM_NODE * GETH_PORT_OFFSET + GETH_NET_BASE_PORT ))
+    GETH_NET_PORT=$(( GETH_NUM_NODE * GETH_PORT_OFFSET + GETH_BASE_NET_PORT ))
-    GETH_HTTP_PORT=$(( GETH_NUM_NODE * GETH_PORT_OFFSET + GETH_HTTP_BASE_PORT ))
+    GETH_HTTP_PORT=$(( GETH_NUM_NODE * GETH_PORT_OFFSET + GETH_BASE_HTTP_PORT ))
-    GETH_WS_PORT=$(( GETH_NUM_NODE * GETH_PORT_OFFSET + GETH_WS_BASE_PORT ))
+    GETH_WS_PORT=$(( GETH_NUM_NODE * GETH_PORT_OFFSET + GETH_BASE_WS_PORT ))
-    GETH_AUTH_RPC_PORT=$(( GETH_NUM_NODE * GETH_PORT_OFFSET + GETH_AUTH_RPC_PORT_BASE ))
+    GETH_AUTH_RPC_PORT=$(( GETH_NUM_NODE * GETH_PORT_OFFSET + GETH_BASE_AUTH_RPC_PORT ))
     log "Starting geth node ${GETH_NUM_NODE} on net port ${GETH_NET_PORT} HTTP port ${GETH_HTTP_PORT} WS port ${GETH_WS_PORT}"
     GETHDATADIR=$(mktemp -d "${DATA_DIR}"/geth-data-XXX)
     GETH_DATA_DIRS+=(${GETHDATADIR})

scripts/start_nimbus_el_nodes.sh

@@ -17,10 +17,10 @@ NIMBUSEL_DATA_DIRS=()
 log "Using ${NIMBUSEL_BINARY}"
 
 for NUM_NODE in $(seq 0 $(( NIMBUSEL_NUM_NODES - 1 ))); do
-    NIMBUSEL_NET_PORT=$(( NUM_NODE * NIMBUSEL_PORT_OFFSET + NIMBUSEL_NET_BASE_PORT ))
+    NIMBUSEL_NET_PORT=$(( NUM_NODE * NIMBUSEL_PORT_OFFSET + NIMBUSEL_BASE_NET_PORT ))
-    NIMBUSEL_HTTP_PORT=$(( NUM_NODE * NIMBUSEL_PORT_OFFSET + NIMBUSEL_HTTP_BASE_PORT ))
+    NIMBUSEL_HTTP_PORT=$(( NUM_NODE * NIMBUSEL_PORT_OFFSET + NIMBUSEL_BASE_HTTP_PORT ))
-    NIMBUSEL_WS_PORT=$(( NUM_NODE * NIMBUSEL_PORT_OFFSET + NIMBUSEL_WS_BASE_PORT ))
+    NIMBUSEL_WS_PORT=$(( NUM_NODE * NIMBUSEL_PORT_OFFSET + NIMBUSEL_BASE_WS_PORT ))
-    NIMBUSEL_AUTH_RPC_PORT=$(( NUM_NODE * NIMBUSEL_PORT_OFFSET + NIMBUSEL_AUTH_RPC_PORT_BASE ))
+    NIMBUSEL_AUTH_RPC_PORT=$(( NUM_NODE * NIMBUSEL_PORT_OFFSET + NIMBUSEL_BASE_AUTH_RPC_PORT ))
     log "Starting nimbus EL node ${NUM_NODE} on net port ${NIMBUSEL_NET_PORT} HTTP port ${NIMBUSEL_HTTP_PORT} WS port ${NIMBUSEL_WS_PORT}"
     NIMBUSEL_DATADIR=$(mktemp -d nimbusel-data-XXX)
     NIMBUSEL_DATA_DIRS+=("${NIMBUSEL_DATADIR}")