Restore the local sim to a working state

beacon_chain/beacon_node.nim

@@ -997,9 +997,16 @@ programMain:
   case config.cmd
   of createTestnet:
     var deposits: seq[Deposit]
-    for i in config.firstValidator.int ..< config.totalValidators.int:
+    var i = -1
-      let depositFile = config.testnetDepositsDir /
+    for kind, dir in walkDir(config.testnetDepositsDir.string):
-                        validatorFileBaseName(i) & ".deposit.json"
+      if kind != pcDir:
+        continue
+
+      inc i
+      if i < config.firstValidator.int:
+        continue
+
+      let depositFile = dir / "deposit.json"
       try:
         deposits.add Json.loadFile(depositFile, Deposit)
       except SerializationError as err:
@@ -1097,6 +1104,7 @@ programMain:
 
   of makeDeposits:
     createDir(config.outValidatorsDir)
+    createDir(config.outSecretsDir)
 
     let
       deposits = generateDeposits(

beacon_chain/conf.nim

@@ -1,7 +1,7 @@
 {.push raises: [Defect].}
 
 import
-  os, options, strformat,
+  os, options,
   chronicles, confutils, json_serialization,
   confutils/defs, confutils/std/net,
   chronicles/options as chroniclesOptions,
@@ -106,11 +106,11 @@ type
         name: "validator" }: seq[ValidatorKeyPath]
 
       validatorsDir* {.
-        desc: "A directory containing validator keystores"
+        desc: "A directory containing validator keystores."
         name: "validators-dir" }: Option[InputDir]
 
       secretsDir* {.
-        desc: "A directory containing validator keystore passwords"
+        desc: "A directory containing validator keystore passwords."
         name: "secrets-dir" }: Option[InputDir]
 
       stateSnapshot* {.
@@ -304,11 +304,11 @@ type
         name: "validator" }: seq[ValidatorKeyPath]
 
       validatorsDir* {.
-        desc: "A directory containing validator keystores"
+        desc: "A directory containing validator keystores."
         name: "validators-dir" }: Option[InputDir]
 
       secretsDir* {.
-        desc: "A directory containing validator keystore passwords"
+        desc: "A directory containing validator keystore passwords."
         name: "secrets-dir" }: Option[InputDir]
 
 proc defaultDataDir*(conf: BeaconNodeConf|ValidatorClientConf): string =
@@ -321,13 +321,6 @@ proc defaultDataDir*(conf: BeaconNodeConf|ValidatorClientConf): string =
 
   getHomeDir() / dataDir / "BeaconNode"
 
-proc validatorFileBaseName*(validatorIdx: int): string =
-  # there can apparently be tops 4M validators so we use 7 digits..
-  try:
-    fmt"v{validatorIdx:07}"
-  except ValueError as e:
-    raiseAssert e.msg
-
 func dumpDir*(conf: BeaconNodeConf|ValidatorClientConf): string =
   conf.dataDir / "dump"
 

beacon_chain/spec/keystore.nim

@@ -7,8 +7,9 @@
 
 import
   json, math, strutils, strformat,
-  eth/keyfile/uuid, stew/[results, byteutils, bitseqs, bitops2],
+  stew/[results, byteutils, bitseqs, bitops2], stew/shims/macros,
-  nimcrypto/[sha2, rijndael, pbkdf2, bcmode, hash, sysrand], blscurve,
+  eth/keyfile/uuid, blscurve,
+  nimcrypto/[sha2, rijndael, pbkdf2, bcmode, hash, sysrand],
   datatypes, crypto, digest, helpers
 
 export
@@ -102,7 +103,16 @@ const
 
   # https://github.com/bitcoin/bips/blob/master/bip-0039/bip-0039-wordlists.md
   wordListLen = 2048
-  englishWords = split slurp("english_word_list.txt")
+
+macro wordListArray(filename: static string): array[wordListLen, cstring] =
+  result = newTree(nnkBracket)
+  var words = slurp(filename).split()
+  words.setLen wordListLen
+  for word in words:
+    result.add newCall("cstring", newLit(word))
+
+const
+  englishWords = wordListArray "english_word_list.txt"
 
 iterator pathNodesImpl(path: string): Natural
                       {.raises: [ValueError].} =
@@ -142,7 +152,7 @@ func getSeed*(mnemonic: Mnemonic, password: KeyStorePass): KeySeed =
   let salt = "mnemonic-" & password.string
   KeySeed sha512.pbkdf2(mnemonic.string, salt, 2048, 64)
 
-proc generateMnemonic*(words: openarray[string],
+proc generateMnemonic*(words: openarray[cstring],
                        entropyParam: openarray[byte] = @[]): Mnemonic =
   # https://github.com/bitcoin/bips/blob/master/bip-0039.mediawiki#generating-the-mnemonic
   doAssert words.len == wordListLen
@@ -163,7 +173,9 @@ proc generateMnemonic*(words: openarray[string],
 
   entropy.add byte(checksum.data.getBitsBE(0 ..< checksumBits))
 
-  var res = words[entropy.getBitsBE(0..10)]
+  var res = ""
+  res.add words[entropy.getBitsBE(0..10)]
+
   for i in 1 ..< mnemonicWordCount:
     let
       firstBit = i*11
@@ -276,7 +288,7 @@ proc encryptKeystore*(T: type[KdfParams],
                       path = KeyPath "",
                       salt: openarray[byte] = @[],
                       iv: openarray[byte] = @[],
-                      ugly = true): KeyStoreContent =
+                      pretty = true): KeyStoreContent =
   var
     secret = privKey.toRaw[^32..^1]
     decKey: seq[byte]
@@ -334,8 +346,8 @@ proc encryptKeystore*(T: type[KdfParams],
       uuid: $uuid,
       version: 4)
 
-  KeyStoreContent if ugly: $(%keystore)
+  KeyStoreContent if pretty: json.pretty(%keystore, indent=4)
-                  else: pretty(%keystore, indent=4)
+                  else: $(%keystore)
 
 proc restoreCredentials*(mnemonic: Mnemonic,
                          password = KeyStorePass ""): Credentials =

beacon_chain/validator_keygen.nim

@@ -52,7 +52,7 @@ proc loadKeyStore(conf: BeaconNodeConf|ValidatorClientConf,
     return
 
   var remainingAttempts = 3
-  var prompt = "Please enter passphrase for key \"" & validatorsDir/keyName & "\""
+  var prompt = "Please enter passphrase for key \"" & validatorsDir/keyName & "\"\n"
   while remainingAttempts > 0:
     let passphrase = KeyStorePass:
       try: readPasswordFromStdin(prompt)

docker/manage_testnet_hosts.nims

@@ -1,5 +1,5 @@
 import
-  strformat, os, confutils
+  strformat, os, confutils, algorightm
 
 type
   Command = enum
@@ -22,9 +22,6 @@ type
         defaultValue: "data"
         name: "network-data-dir" }: string
 
-      totalValidators {.
-        name: "total-validators" }: int
-
       totalUserValidators {.
         defaultValue: 0
         name: "user-validators" }: int
@@ -39,6 +36,9 @@ var conf = load CliConfig
 var
   serverCount = 10
   instancesCount = 2
+  validators = listDirs(conf.depositsDir)
+
+sort(validators)
 
 proc findOrDefault[K, V](tupleList: openarray[(K, V)], key: K, default: V): V =
   for t in tupleList:
@@ -60,7 +60,7 @@ iterator nodes: Node =
 
 iterator validatorAssignments: tuple[node: Node; firstValidator, lastValidator: int] =
   let
-    systemValidators = conf.totalValidators - conf.totalUserValidators
+    systemValidators = validators.len - conf.totalUserValidators
 
     defaultValidatorAssignment = proc (nodeIdx: int): int =
       (systemValidators div serverCount) div instancesCount
@@ -110,26 +110,25 @@ of restart_nodes:
 of reset_network:
   for n, firstValidator, lastValidator in validatorAssignments():
     var
-      keysList = ""
+      validatorDirs = ""
       networkDataFiles = conf.networkDataDir & "/{genesis.ssz,bootstrap_nodes.txt}"
 
     for i in firstValidator ..< lastValidator:
-      let validatorKey = fmt"v{i:07}.privkey"
+      validatorDirs.add " "
-      keysList.add " "
+      validatorDirs.add conf.depositsDir / validators[i]
-      keysList.add conf.depositsDir / validatorKey
 
     let dockerPath = &"/docker/{n.container}/data/BeaconNode"
     echo &"echo Syncing {lastValidator - firstValidator} keys starting from {firstValidator} to container {n.container}@{n.server} ... && \\"
     echo &"  ssh {n.server} 'sudo rm -rf /tmp/nimbus && mkdir -p /tmp/nimbus/' && \\"
     echo &"  rsync -a -zz {networkDataFiles} {n.server}:/tmp/nimbus/net-data/ && \\"
-    if keysList.len > 0:
+    if validatorDirs.len > 0:
-      echo &"  rsync -a -zz {keysList} {n.server}:/tmp/nimbus/keys/ && \\"
+      echo &"  rsync -a -zz {validatorDirs} {n.server}:/tmp/nimbus/keys/ && \\"
 
     echo &"  ssh {n.server} 'sudo docker container stop {n.container}; " &
                          &"sudo mkdir -p {dockerPath}/validators && " &
                          &"sudo rm -rf {dockerPath}/validators/* && " &
                          &"sudo rm -rf {dockerPath}/db && " &
-                         (if keysList.len > 0: &"sudo mv /tmp/nimbus/keys/* {dockerPath}/validators/ && " else: "") &
+                         (if validatorDirs.len > 0: &"sudo mv /tmp/nimbus/keys/* {dockerPath}/validators/ && " else: "") &
                          &"sudo mv /tmp/nimbus/net-data/* {dockerPath}/ && " &
                          &"sudo chown dockremap:docker -R {dockerPath}'"
 

scripts/attach_validators.sh

@@ -1,24 +0,0 @@
-#!/bin/bash
-
-set -eu
-
-NETWORK_NAME=$1
-NODE_ID=$2
-FIRST_VALIDATOR=$3
-LAST_VALIDATOR=$4
-
-cd $(dirname "$0")
-cd ..
-
-if [ -f .env ]; then
-  source .env
-fi
-
-NETWORK_DIR=$WWW_DIR/$NETWORK_NAME
-
-for i in $(seq $FIRST_VALIDATOR $LAST_VALIDATOR); do
-  VALIDATOR=v$(printf '%07d' $i)
-  beacon_chain/beacon_node --data-dir="$DATA_DIR/node-$NODE_ID" importValidator \
-    --keyfile="$NETWORK_DIR/$VALIDATOR.privkey"
-done
-

scripts/connect_to_testnet.nims

@@ -137,7 +137,7 @@ cli do (skipGoerliKey {.
       mkDir validatorsDir
       mode = Verbose
       exec replace(&"""{beaconNodeBinary} makeDeposits
-        --random-deposits=1
+        --count=1
         --out-validators-dir="{validatorsDir}"
         --out-secrets-dir="{secretsDir}"
         --deposit-private-key={privKey}

scripts/launch_local_testnet.sh

@@ -141,7 +141,7 @@ $MAKE LOG_LEVEL="${LOG_LEVEL}" NIMFLAGS="-d:insecure -d:testnet_servers_image ${
 ./build/beacon_node makeDeposits \
 	--count=${TOTAL_VALIDATORS} \
 	--out-validators-dir="${DEPOSITS_DIR}" \
-  --out-secrets-dir="${SECRETS_DIR}"
+	--out-secrets-dir="${SECRETS_DIR}"
 
 BOOTSTRAP_IP="127.0.0.1"
 ./build/beacon_node createTestnet \
@@ -206,10 +206,12 @@ for NUM_NODE in $(seq 0 $(( ${NUM_NODES} - 1 ))); do
 	# The first $NODES_WITH_VALIDATORS nodes split them equally between them, after skipping the first $USER_VALIDATORS.
 	NODE_DATA_DIR="${DATA_DIR}/node${NUM_NODE}"
 	mkdir -p "${NODE_DATA_DIR}/validators"
+	mkdir -p "${NODE_DATA_DIR}/secrets"
+
 	if [[ $NUM_NODE -lt $NODES_WITH_VALIDATORS ]]; then
-		# TODO: There are no longer privkey files
+		for VALIDATOR in $(ls ${DEPOSITS_DIR}/* | tail -n +$(( $USER_VALIDATORS + ($VALIDATORS_PER_NODE * $NUM_NODE) + 1 )) | head -n $VALIDATORS_PER_NODE); do
-    for KEYFILE in $(ls ${DEPOSITS_DIR}/*.privkey | tail -n +$(( $USER_VALIDATORS + ($VALIDATORS_PER_NODE * $NUM_NODE) + 1 )) | head -n $VALIDATORS_PER_NODE); do
+			cp -ar "$VALIDATOR" "${NODE_DATA_DIR}/validators/"
-			cp -a "$KEYFILE" "${NODE_DATA_DIR}/validators/"
+			cp -a "${SECRETS_DIR}/${VALIDATOR}" "${NODE_DATA_DIR}/secrets"
 		done
 	fi
 

scripts/reset_testnet.sh

@@ -117,7 +117,6 @@ if [[ $PUBLISH_TESTNET_RESETS != "0" ]]; then
     --deposits-dir="$DEPOSITS_DIR_ABS" \
     --network-data-dir="$NETWORK_DIR_ABS" \
     --user-validators=$USER_VALIDATORS \
-    --total-validators=$TOTAL_VALIDATORS \
     > /tmp/reset-network.sh
 
   bash /tmp/reset-network.sh

tests/simulation/run_node.sh

@@ -31,7 +31,10 @@ source "${SIM_ROOT}/../../env.sh"
 
 cd "$GIT_ROOT"
 
-DATA_DIR="${SIMULATION_DIR}/node-$NODE_ID"
+NODE_DATA_DIR="${SIMULATION_DIR}/node-$NODE_ID"
+NODE_VALIDATORS_DIR=$NODE_DATA_DIR/validators/
+NODE_SECRETS_DIR=$NODE_DATA_DIR/secrets/
+
 PORT=$(( BASE_P2P_PORT + NODE_ID ))
 
 NAT_ARG="--nat:extip:127.0.0.1"
@@ -39,41 +42,47 @@ if [ "${NAT:-}" == "1" ]; then
   NAT_ARG="--nat:any"
 fi
 
-mkdir -p "$DATA_DIR/validators"
+rm -rf "$NODE_VALIDATORS_DIR"
-rm -f $DATA_DIR/validators/*
+mkdir -p "$NODE_VALIDATORS_DIR"
+
+rm -rf "$NODE_SECRETS_DIR"
+mkdir -p "$NODE_SECRETS_DIR"
+
+VALIDATORS_PER_NODE=$((NUM_VALIDATORS / TOTAL_NODES))
 
 if [[ $NODE_ID -lt $TOTAL_NODES ]]; then
-  VALIDATORS_PER_NODE=$((NUM_VALIDATORS / TOTAL_NODES))
-  VALIDATORS_PER_NODE_HALF=$((VALIDATORS_PER_NODE / 2))
-  FIRST_VALIDATOR_IDX=$(( VALIDATORS_PER_NODE * NODE_ID ))
   # if using validator client binaries in addition to beacon nodes
   # we will split the keys for this instance in half between the BN and the VC
   if [ "${SPLIT_VALIDATORS_BETWEEN_BN_AND_VC:-}" == "yes" ]; then
-    LAST_VALIDATOR_IDX=$(( FIRST_VALIDATOR_IDX + VALIDATORS_PER_NODE_HALF - 1 ))
+    ATTACHED_VALIDATORS=$((VALIDATORS_PER_NODE / 2))
   else
-    LAST_VALIDATOR_IDX=$(( FIRST_VALIDATOR_IDX + VALIDATORS_PER_NODE - 1 ))
+    ATTACHED_VALIDATORS=$VALIDATORS_PER_NODE
   fi
 
   pushd "$VALIDATORS_DIR" >/dev/null
-    cp $(seq -s " " -f v%07g.privkey $FIRST_VALIDATOR_IDX $LAST_VALIDATOR_IDX) "$DATA_DIR/validators"
+  for VALIDATOR in $(ls | tail -n +$(( ($VALIDATORS_PER_NODE * $NODE_ID) + 1 )) | head -n $ATTACHED_VALIDATORS); do
+      cp -ar "$VALIDATOR" "$NODE_VALIDATORS_DIR"
+      cp -a "$SECRETS_DIR/$VALIDATOR" "$NODE_SECRETS_DIR"
+    done
   popd >/dev/null
 fi
 
-rm -rf "$DATA_DIR/dump"
+rm -rf "$NODE_DATA_DIR/dump"
-mkdir -p "$DATA_DIR/dump"
+mkdir -p "$NODE_DATA_DIR/dump"
 
 SNAPSHOT_ARG=""
 if [ -f "${SNAPSHOT_FILE}" ]; then
   SNAPSHOT_ARG="--state-snapshot=${SNAPSHOT_FILE}"
 fi
 
-cd "$DATA_DIR"
+cd "$NODE_DATA_DIR"
 
 # if you want tracing messages, add "--log-level=TRACE" below
 $BEACON_NODE_BIN \
   --log-level=${LOG_LEVEL:-DEBUG} \
   --bootstrap-file=$BOOTSTRAP_ADDRESS_FILE \
-  --data-dir=$DATA_DIR \
+  --data-dir=$NODE_DATA_DIR \
+  --secrets-dir=$NODE_SECRETS_DIR \
   --node-name=$NODE_ID \
   --tcp-port=$PORT \
   --udp-port=$PORT \

tests/simulation/start.sh

@@ -21,9 +21,6 @@ DEFS+="-d:MAX_COMMITTEES_PER_SLOT=${MAX_COMMITTEES_PER_SLOT:-1} "      # Spec de
 DEFS+="-d:SLOTS_PER_EPOCH=${SLOTS_PER_EPOCH:-6} "   # Spec default: 32
 DEFS+="-d:SECONDS_PER_SLOT=${SECONDS_PER_SLOT:-6} "  # Spec default: 12
 
-LAST_VALIDATOR_NUM=$(( NUM_VALIDATORS - 1 ))
-LAST_VALIDATOR="$VALIDATORS_DIR/v$(printf '%07d' $LAST_VALIDATOR_NUM).deposit.json"
-
 # Windows detection
 if uname | grep -qiE "mingw|msys"; then
   MAKE="mingw32-make"
@@ -96,10 +93,20 @@ if [[ "$USE_TMUX" != "no" ]]; then
   $TMUX select-window -t "${TMUX_SESSION_NAME}:sim"
 fi
 
-$MAKE -j3 --no-print-directory NIMFLAGS="$CUSTOM_NIMFLAGS $DEFS" LOG_LEVEL="${LOG_LEVEL:-DEBUG}" beacon_node validator_client deposit_contract
+$MAKE -j3 --no-print-directory NIMFLAGS="$CUSTOM_NIMFLAGS $DEFS" LOG_LEVEL="${LOG_LEVEL:-DEBUG}" beacon_node validator_client
+
+count_files () {
+  { ls -1q $1 2> /dev/null || true ; } | wc -l
+}
+
+EXISTING_VALIDATORS=$(count_files "$VALIDATORS_DIR/*/deposit.json")
+
+if [[ $EXISTING_VALIDATORS -lt $NUM_VALIDATORS ]]; then
+  rm -rf "$VALIDATORS_DIR"
+  rm -rf "$SECRETS_DIR"
 
-if [ ! -f "${LAST_VALIDATOR}" ]; then
   if [ "$WEB3_ARG" != "" ]; then
+    make deposit_contract
     echo Deploying the validator deposit contract...
     DEPOSIT_CONTRACT_ADDRESS=$($DEPLOY_DEPOSIT_CONTRACT_BIN deploy $WEB3_ARG)
     echo Contract deployed at $DEPOSIT_CONTRACT_ADDRESS