Add support for testnets with mixed quickstart/random deposits

.gitignore

@@ -29,7 +29,3 @@ build/
 # State sim # TODO - move in another folder
 0000-*.json
 
-# Testnet resetting working directories
-testnet-reset-data/
-nim-eth2-testnet-data/
-

beacon_chain/beacon_node.nim

@@ -1002,9 +1002,17 @@ when isMainModule:
       node.start(node.stateCache.data.data)
 
   of makeDeposits:
-    let deposits = generateDeposits(
+    let
-      config.totalDeposits, config.depositsDir, config.randomKeys)
+      quickstartDeposits = generateDeposits(
+        config.totalQuickstartDeposits, config.depositsDir, false)
 
-    if config.depositWeb3Url.len() > 0 and config.depositContractAddress.len() > 0:
+      randomDeposits = generateDeposits(
+        config.totalRandomDeposits, config.depositsDir, true,
+        firstIdx = config.totalQuickstartDeposits)
+
+    if config.depositWeb3Url.len > 0 and config.depositContractAddress.len > 0:
       waitFor sendDeposits(
-        deposits, config.depositWeb3Url, config.depositContractAddress)
+        quickstartDeposits & randomDeposits,
+        config.depositWeb3Url,
+        config.depositContractAddress)
+

beacon_chain/conf.nim

@@ -186,20 +186,21 @@ type
         longform: "keyfile" }: seq[ValidatorKeyPath]
 
     of makeDeposits:
-      totalDeposits* {.
+      totalQuickstartDeposits* {.
-        desc: "Total number of deposits and keys to generate."
+        desc: "Number of quick-start deposits to generate."
-        longform: "total-deposits" }: int
+        defaultValue: 0
+        longform: "quickstart-deposits" }: int
+
+      totalRandomDeposits* {.
+        desc: "Number of secure random deposits to generate."
+        defaultValue: 0
+        longform: "random-deposits" }: int
 
       depositsDir* {.
         desc: "Folder to write deposits to."
         defaultValue: "validators"
         longform: "deposits-dir" }: string
 
-      randomKeys* {.
-        desc: "Use random keys (instead of interop keys)."
-        defaultValue: false
-        longform: "random-keys" }: bool
-
 proc defaultPort*(config: BeaconNodeConf): int =
   9000
 

beacon_chain/validator_keygen.nim

@@ -18,12 +18,14 @@ proc writeFile(filename: string, value: auto) =
 proc ethToWei(eth: UInt256): UInt256 =
   eth * 1000000000000000000.u256
 
-proc generateDeposits*(
+proc generateDeposits*(totalValidators: int,
-    totalValidators: int, outputDir: string, randomKeys: bool): seq[Deposit] =
+                       outputDir: string,
+                       randomKeys: bool,
+                       firstIdx = 0): seq[Deposit] =
   info "Generating deposits", totalValidators, outputDir, randomKeys
   for i in 0 ..< totalValidators:
     let
-      v = validatorFileBaseName(i)
+      v = validatorFileBaseName(firstIdx + i)
       depositFn = outputDir / v & ".deposit.json"
       privKeyFn = outputDir / v & ".privkey"
 

multinet/make_genesis.sh

@@ -33,9 +33,8 @@ LAST_VALIDATOR="$VALIDATORS_DIR/v$(printf '%07d' $LAST_VALIDATOR_NUM).deposit.js
 
 if [ ! -f "${LAST_VALIDATOR}" ]; then
   $BEACON_NODE_BIN makeDeposits \
-    --total-deposits="${NUM_VALIDATORS}" \
+    --quickstart-deposits="${NUM_VALIDATORS}" \
-    --deposits-dir="$VALIDATORS_DIR" \
+    --deposits-dir="$VALIDATORS_DIR"
-    --random-keys=no
 fi
 
 if [ ! -f "${SNAPSHOT_FILE}" ]; then

scripts/download_validator_keys.sh

@@ -1,18 +0,0 @@
-#!/bin/bash
-
-set -eu
-
-NETWORK_NAME=$1
-FIRST_VALIDATOR=$2
-LAST_VALIDATOR=$3
-DATA_DIR=${4:-~/.cache/nimbus/BeaconNode/$NETWORK_NAME}
-VALIDATORS_DIR=$DATA_DIR/validators
-
-mkdir -p $VALIDATORS_DIR
-
-FIRST_IDX=$(printf '%07d' $FIRST_VALIDATOR)
-LAST_IDX=$(printf '%07d' $LAST_VALIDATOR)
-
-curl "https://raw.githubusercontent.com/status-im/nim-eth2-testnet-data/master/www/${NETWORK_NAME}/v[$FIRST_IDX-$LAST_IDX].privkey" \
-  -o "$VALIDATORS_DIR/v#1.privkey" -s -w '%{url_effective} > %{filename_effective}\n'
-

scripts/reset_testnet.sh

@@ -10,7 +10,7 @@ source "$NETWORK_NAME.env"
 cd ..
 
 if [ -f .env ]; then
-  # allow server overrides for ETH2_TESTNET_DATA_DIR, DATA_DIR and ETH1_PRIVATE_KEY
+  # allow server overrides for ETH2_TESTNETS, DATA_DIR and ETH1_PRIVATE_KEY
   source .env
 fi
 
@@ -19,8 +19,8 @@ echo ${BOOTSTRAP_HOST:="master-01.do-ams3.nimbus.test.statusim.net"} > /dev/null
 echo Execution plan:
 
 echo "Testnet name          : $NETWORK_NAME"
-echo "Testnet files repo    : ${ETH2_TESTNET_DATA_DIR:="nim-eth2-testnet-data"}"
+echo "Testnet files repo    : ${ETH2_TESTNETS:="build/eth2-testnets"}"
-echo "Beacon node data dir  : ${DATA_DIR:="testnet-reset-data"}"
+echo "Beacon node data dir  : ${DATA_DIR:="build/testnet-reset-data"}"
 echo "Bootstrap node ip     : ${BOOTSTRAP_IP:="$(dig +short $BOOTSTRAP_HOST)"}"
 echo "Reset testnet at end  : ${PUBLISH_TESTNET_RESETS:="1"}"
 
@@ -33,13 +33,13 @@ while true; do
     esac
 done
 
-if [[ ! -d "$ETH2_TESTNET_DATA_DIR"  ]]; then
+if [[ ! -d "$ETH2_TESTNETS"  ]]; then
-  git clone git@github.com:status-im/nim-eth2-testnet-data "$ETH2_TESTNET_DATA_DIR"
+  git clone git@github.com:zah/eth2-testnets "$ETH2_TESTNETS"
 fi
 
-ETH2_TESTNET_DATA_DIR_ABS=$(cd "$ETH2_TESTNET_DATA_DIR"; pwd)
+ETH2_TESTNETS_ABS=$(cd "$ETH2_TESTNETS"; pwd)
 DATA_DIR_ABS=$(mkdir -p "$DATA_DIR"; cd "$DATA_DIR"; pwd)
-NETWORK_DIR_ABS="$ETH2_TESTNET_DATA_DIR_ABS/www/$NETWORK_NAME"
+NETWORK_DIR_ABS="$ETH2_TESTNETS_ABS/nimbus/$NETWORK_NAME"
 
 if [ "$WEB3_URL" != "" ]; then
   WEB3_URL_ARG="--web3-url=$WEB3_URL"
@@ -62,30 +62,36 @@ make build
 if [ ! -f $NETWORK_DIR_ABS/genesis.ssz ]; then
   rm -f $NETWORK_DIR_ABS/*
   $DOCKER_BEACON_NODE makeDeposits \
-    --total-deposits=$VALIDATOR_COUNT \
+    --quickstart-deposits=$QUICKSTART_VALIDATORS \
-    --deposits-dir=/network_dir \
+    --random-deposits=$RANDOM_VALIDATORS \
-    --random-keys=no
+    --deposits-dir=/network_dir
 fi
 
+TOTAL_VALIDATORS="$(( $QUICKSTART_VALIDATORS + $RANDOM_VALIDATORS_COUNT ))"
+
 $DOCKER_BEACON_NODE \
   --network=$NETWORK_NAME \
   --data-dir=/data_dir \
   createTestnet \
   --validators-dir=/network_dir \
-  --total-validators=$VALIDATOR_COUNT \
+  --total-validators=$TOTAL_VALIDATORS \
   --last-user-validator=$LAST_USER_VALIDATOR \
-  --output-genesis=/network_dir/genesis.json \
+  --output-genesis=/network_dir/genesis.ssz \
   --output-bootstrap-file=/network_dir/bootstrap_nodes.txt \
   --bootstrap-address=$BOOTSTRAP_IP \
   --bootstrap-port=$BOOTSTRAP_PORT \
   $WEB3_URL_ARG $DEPOSIT_CONTRACT_ADDRESS_ARG \
   --genesis-offset=60 # Delay in seconds
 
+if [[ ! -z "$DEPOSIT_CONTRACT_ADDRESS" ]]; then
+  echo $DEPOSIT_CONTRACT_ADDRESS > "$ETH2_TESTNETS_ABS/deposit_contract.txt"
+fi
+
 if [[ $PUBLISH_TESTNET_RESETS != "0" ]]; then
   echo Persisting testnet data to git...
-  pushd "$ETH2_TESTNET_DATA_DIR_ABS"
+  pushd "$ETH2_TESTNETS_ABS"
     git add --all
-    git commit -m "Testnet reset"
+    git commit -m "Reset of Nimbus $NETWORK_NAME"
     git push
   popd
 

scripts/testnet0.env

@@ -3,7 +3,7 @@ NETWORK_TYPE=libp2p_daemon
 SHARD_COUNT=16
 SLOTS_PER_EPOCH=16
 SECONDS_PER_SLOT=30
-VALIDATOR_COUNT=1000
+QUICKSTART_VALIDATORS=192
-LAST_USER_VALIDATOR=199
+RANDOM_VALIDATORS=0
 BOOTSTRAP_PORT=9000
 WEB3_URL=wss://goerli.infura.io/ws/v3/809a18497dd74102b5f37d25aae3c85a

scripts/testnet1.env

@@ -3,7 +3,7 @@ NETWORK_TYPE=libp2p_daemon
 SHARD_COUNT=8
 SLOTS_PER_EPOCH=8
 SECONDS_PER_SLOT=6
-VALIDATOR_COUNT=1000
+QUICKSTART_VALIDATORS=0
-LAST_USER_VALIDATOR=199
+RANDOM_VALIDATORS=1000
 BOOTSTRAP_PORT=9100
 WEB3_URL=wss://goerli.infura.io/ws/v3/809a18497dd74102b5f37d25aae3c85a

tests/simulation/start.sh

@@ -44,9 +44,8 @@ if [ ! -f "${LAST_VALIDATOR}" ]; then
   fi
 
   $BEACON_NODE_BIN makeDeposits \
-    --total-deposits="${NUM_VALIDATORS}" \
+    --quickstart-deposits="${NUM_VALIDATORS}" \
-    --deposits-dir="$VALIDATORS_DIR" \
+    --deposits-dir="$VALIDATORS_DIR"
-    --random-keys=no \
     $DEPOSIT_WEB3_URL_ARG \
     --deposit-contract="${DEPOSIT_CONTRACT_ADDRESS}"
 fi