simplify startup, describe a few startup tricks

beacon_chain/beacon_node.nim

@@ -1,8 +1,8 @@
 import
-  net, sequtils, options, tables, osproc, random, strutils, times, strformat,
+  net, sequtils, tables, osproc, random, strutils, times, strformat,
   stew/shims/os, stew/[objects, bitseqs],
   chronos, chronicles, confutils, metrics,
-  json_serialization/std/sets, serialization/errors,
+  json_serialization/std/[options, sets], serialization/errors,
   eth/trie/db, eth/trie/backends/rocksdb_backend, eth/async_utils,
   spec/[datatypes, digest, crypto, beaconstate, helpers, validator,
   state_transition_block],
@@ -881,7 +881,10 @@ when isMainModule:
 
       testnetMetadata = NetworkMetadata(
         networkGeneration: semanticVersion,
-        genesisRoot: hash_tree_root(initialState),
+        genesisRoot:
+          if config.withGenesisRoot:
+            some(hash_tree_root(initialState))
+          else: none(Eth2Digest),
         bootstrapNodes: @[bootstrapAddress],
         numShards: SHARD_COUNT,
         slotDuration: SECONDS_PER_SLOT,

beacon_chain/beacon_node_types.nim

@@ -1,5 +1,5 @@
 import
-  sets, deques, tables,
+  sets, deques, tables, options,
   eth/keys, stew/[bitseqs, endians2],
   spec/[datatypes, crypto, digest],
   beacon_chain_db, conf, mainchain_monitor, eth2_network, time
@@ -245,7 +245,7 @@ type
 
   NetworkMetadata* = object
     networkGeneration*: uint64
-    genesisRoot*: Eth2Digest
+    genesisRoot*: Option[Eth2Digest]
     bootstrapNodes*: seq[BootstrapAddr]
     numShards*: uint64
     slotDuration*: uint64

beacon_chain/conf.nim

@@ -137,6 +137,9 @@ type
       outputNetwork* {.
         desc: "Output file where to write the initial state snapshot".}: OutFile
 
+      withGenesisRoot* {.
+        desc: "Include a genesis root in network.json", defaultValue: false.}: bool
+
     of importValidator:
       keyFiles* {.
         longform: "keyfile"

multinet/README.md

@@ -4,6 +4,10 @@ This folder contains scripts for launching the nimbus beacon chain node in a con
 
 In general, follow the build instructions of `nim-beacon-chain` as documented in the main repo - make sure to set up your build environment with all necessary system libraries as documented there:
 
+### Prerequisites
+
+:warning: To build nimbus, you need to have `rocksdb` and `pcre` installed - see [../](main repo) for instructions.
+
 ```bash
 # Clone repo
 
@@ -21,13 +25,19 @@ make update deps # build dependencies
 Look in the scripts for options - the default config is a small setup using the `minimal` state spec.
 
 ```
-cd interop
+cd multinet
 
-# Clear data from previous run, then start a new simulation
+# Create a new genesis 10s in the future
-rm -rf data; ./start.sh
+./make_genesis.sh
 
-# In a separate terminal, can run another beacon node, such as lighthouse:
+# You can now start the clients
+./run_nimbus.sh
+./run_trinity.sh
 ./run_lighthouse.sh
+
+# Or do all in one step, with multitail
+USE_MULTITAIL=1 ./run_all.sh
+
 ```
 
 ## Diagnostics

multinet/make_genesis.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+
+set -eo pipefail
+
+# Read in variables
+source "$(dirname "$0")/vars.sh"
+
+# set up the environment
+source "${SIM_ROOT}/../env.sh"
+
+cd "$SIM_ROOT"
+
+rm -rf "$SIMULATION_DIR"
+mkdir -p "$SIMULATION_DIR"
+mkdir -p "$VALIDATORS_DIR"
+
+cd "$GIT_ROOT"
+
+make update deps
+
+NIMFLAGS="-d:chronicles_log_level=DEBUG --warnings:off --hints:off --opt:speed"
+
+# For interop, we run the minimal config
+DEFS="-d:const_preset=minimal"
+
+LAST_VALIDATOR_NUM=$(( NUM_VALIDATORS - 1 ))
+LAST_VALIDATOR="$VALIDATORS_DIR/v$(printf '%07d' $LAST_VALIDATOR_NUM).deposit.json"
+
+[[ -x "$BEACON_NODE_BIN" ]] || {
+  echo "Building $BEACON_NODE_BIN ($DEFS)"
+  nim c -o:"$BEACON_NODE_BIN" $NIMFLAGS $DEFS beacon_chain/beacon_node
+}
+
+if [ ! -f "${LAST_VALIDATOR}" ]; then
+  $BEACON_NODE_BIN makeDeposits \
+    --totalDeposits="${NUM_VALIDATORS}" \
+    --depositDir="$VALIDATORS_DIR" \
+    --randomKeys=false
+fi
+
+if [ ! -f "${SNAPSHOT_FILE}" ]; then
+  $BEACON_NODE_BIN \
+    --dataDir="${SIMULATION_DIR}/node-0" \
+    createTestnet \
+    --validatorsDir="${VALIDATORS_DIR}" \
+    --totalValidators="${NUM_VALIDATORS}" \
+    --outputGenesis="${SNAPSHOT_FILE}" \
+    --outputNetwork="${NETWORK_METADATA_FILE}" \
+    --bootstrapAddress=127.0.0.1 \
+    --bootstrapPort=50000 \
+    --genesisOffset=10 # Delay in seconds
+fi
+
+# Delete any leftover address files from a previous session
+if [ -f "${MASTER_NODE_ADDRESS_FILE}" ]; then
+  rm "${MASTER_NODE_ADDRESS_FILE}"
+fi
+

multinet/run_all.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+# Kill child processes on Ctrl-C by sending SIGTERM to the whole process group,
+# passing the negative PID of this shell instance to the "kill" command.
+# Trap and ignore SIGTERM, so we don't kill this process along with its children.
+trap '' SIGTERM
+trap 'kill -- -$$' SIGINT EXIT
+
+./make_genesis.sh
+
+# multitail support
+MULTITAIL="${MULTITAIL:-multitail}" # to allow overriding the program name
+USE_MULTITAIL="${USE_MULTITAIL:-no}" # make it an opt-in
+type "$MULTITAIL" &>/dev/null || USE_MULTITAIL="no"
+
+# Kill child processes on Ctrl-C by sending SIGTERM to the whole process group,
+# passing the negative PID of this shell instance to the "kill" command.
+# Trap and ignore SIGTERM, so we don't kill this process along with its children.
+if [ "$USE_MULTITAIL" = "no" ]; then
+  trap '' SIGTERM
+  trap 'kill -- -$$' SIGINT EXIT
+fi
+
+if [ "$USE_MULTITAIL" != "no" ]; then
+  COMMANDS=()
+  # "multitail" closes the corresponding panel when a command exits, so let's make sure it doesn't exit
+  COMMANDS+=( " -cT ansi -t 'nimbus' -l './run_nimbus.sh 0; echo [node execution completed]; while true; do sleep 100; done'" )
+  COMMANDS+=( " -cT ansi -t 'trinity' -l 'sleep 3; ./run_trinity.sh; echo [node execution completed]; while true; do sleep 100; done'" )
+  COMMANDS+=( " -cT ansi -t 'lighthouse' -l 'sleep 3; ./run_lighthouse.sh; echo [node execution completed]; while true; do sleep 100; done'" )
+  eval $MULTITAIL -s 3 -M 0 -x \"Multichain\" "${COMMANDS[@]}"
+else
+  ./run_nimbus.sh 0 &
+  sleep 2
+  ./run_trinity.sh &
+  ./run_lighthouse.sh &
+  wait
+fi

multinet/run_lighthouse.sh

@@ -47,6 +47,11 @@ LIGHTHOUSE=${LIGHTHOSE_PATH:-"lighthouse"}
   popd
 }
 
+
+pushd "$LIGHTHOUSE"
+cargo build --release
+popd
+
 # Fetch genesis time, as set up by start.sh
 if command -v jq; then
   genesis_time=$(jq '.genesis_time' data/state_snapshot.json)
@@ -56,10 +61,7 @@ fi
 
 echo Genesis time was $genesis_time
 
-cd "$LIGHTHOUSE"
+cd "$LIGHTHOUSE/target/release"
-cargo build
-
-cd target/debug
 
 #$export RUST_LOG=libp2p=trace,multistream=trace,gossipsub=trace
 

multinet/run_nimbus.sh

@@ -5,10 +5,10 @@ set -eu
 . $(dirname $0)/vars.sh
 cd "$GIT_ROOT"
 
-DATA_DIR="${SIMULATION_DIR}/node-${1}"
+DATA_DIR="${SIMULATION_DIR}/node-0"
 
-V_PREFIX="${VALIDATORS_DIR}/v$(printf '%06d' ${1})"
+V_PREFIX="${VALIDATORS_DIR}/v$(printf '%06d' 0)"
-PORT=$(printf '5%04d' ${1})
+PORT=$(printf '5%04d' 0)
 
 NAT_FLAG="--nat:none"
 if [ "${NAT:-}" == "1" ]; then
@@ -28,7 +28,7 @@ popd >/dev/null
 $BEACON_NODE_BIN \
   --network:$NETWORK_METADATA_FILE \
   --dataDir:$DATA_DIR \
-  --nodename:${1} \
+  --nodename:0 \
   --tcpPort:$PORT \
   --udpPort:$PORT \
   $NAT_FLAG \

multinet/start.sh

@@ -1,102 +0,0 @@
-#!/bin/bash
-
-set -eo pipefail
-
-# Read in variables
-source "$(dirname "$0")/vars.sh"
-
-# set up the environment
-source "${SIM_ROOT}/../env.sh"
-
-cd "$SIM_ROOT"
-mkdir -p "$SIMULATION_DIR"
-mkdir -p "$VALIDATORS_DIR"
-
-cd "$GIT_ROOT"
-
-make update deps
-
-NIMFLAGS="-d:chronicles_log_level=DEBUG --hints:off --warnings:off --opt:speed --debuginfo"
-
-# For interop, we run the minimal config
-DEFS="-d:const_preset=minimal"
-
-LAST_VALIDATOR_NUM=$(( NUM_VALIDATORS - 1 ))
-LAST_VALIDATOR="$VALIDATORS_DIR/v$(printf '%07d' $LAST_VALIDATOR_NUM).deposit.json"
-
-[[ -x "$BEACON_NODE_BIN" ]] || {
-  echo "Building $BEACON_NODE_BIN ($DEFS)"
-  nim c -o:"$BEACON_NODE_BIN" $NIMFLAGS $DEFS beacon_chain/beacon_node
-}
-
-if [ ! -f "${LAST_VALIDATOR}" ]; then
-  $BEACON_NODE_BIN makeDeposits \
-    --totalDeposits="${NUM_VALIDATORS}" \
-    --depositDir="$VALIDATORS_DIR" \
-    --randomKeys=false
-fi
-
-if [ ! -f "${SNAPSHOT_FILE}" ]; then
-  $BEACON_NODE_BIN \
-    --dataDir="${SIMULATION_DIR}/node-0" \
-    createTestnet \
-    --validatorsDir="${VALIDATORS_DIR}" \
-    --totalValidators="${NUM_VALIDATORS}" \
-    --outputGenesis="${SNAPSHOT_FILE}" \
-    --outputNetwork="${NETWORK_METADATA_FILE}" \
-    --bootstrapAddress=127.0.0.1 \
-    --bootstrapPort=50000 \
-    --genesisOffset=5 # Delay in seconds
-fi
-
-# Delete any leftover address files from a previous session
-if [ -f "${MASTER_NODE_ADDRESS_FILE}" ]; then
-  rm "${MASTER_NODE_ADDRESS_FILE}"
-fi
-
-# multitail support
-MULTITAIL="${MULTITAIL:-multitail}" # to allow overriding the program name
-USE_MULTITAIL="${USE_MULTITAIL:-no}" # make it an opt-in
-type "$MULTITAIL" &>/dev/null || USE_MULTITAIL="no"
-
-# Kill child processes on Ctrl-C by sending SIGTERM to the whole process group,
-# passing the negative PID of this shell instance to the "kill" command.
-# Trap and ignore SIGTERM, so we don't kill this process along with its children.
-if [ "$USE_MULTITAIL" = "no" ]; then
-  trap '' SIGTERM
-  trap 'kill -- -$$' SIGINT EXIT
-fi
-
-COMMANDS=()
-LAST_NODE=$(( NUM_NODES - 1 ))
-
-for i in $(seq 0 $LAST_NODE); do
-  if [[ "$i" == "0" ]]; then
-    sleep 0
-  elif [ "$USE_MULTITAIL" = "no" ]; then
-    # Wait for the master node to write out its address file
-    while [ ! -f "${MASTER_NODE_ADDRESS_FILE}" ]; do
-      sleep 0.1
-    done
-  fi
-
-  CMD="${SIM_ROOT}/run_node.sh $i"
-
-  if [ "$USE_MULTITAIL" != "no" ]; then
-    if [ "$i" = "0" ]; then
-      SLEEP="0"
-    else
-      SLEEP="2"
-    fi
-    # "multitail" closes the corresponding panel when a command exits, so let's make sure it doesn't exit
-    COMMANDS+=( " -cT ansi -t 'node #$i' -l 'sleep $SLEEP; $CMD; echo [node execution completed]; while true; do sleep 100; done'" )
-  else
-    eval "${CMD}" &
-  fi
-done
-
-if [ "$USE_MULTITAIL" != "no" ]; then
-  eval $MULTITAIL -s 3 -M 0 -x \"Nimbus beacon chain\" "${COMMANDS[@]}"
-else
-  wait # Stop when all nodes have gone down
-fi