status-im
/
nimbus-eth2
mirror of https://github.com/status-im/nimbus-eth2.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Restore the local sim to a working state

This commit is contained in:
Zahary Karadjov 2020-06-02 22:59:51 +03:00 committed by zah
parent 17343442ea
commit a8113cf2bc
11 changed files with 89 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
beacon_chain/beacon_node.nim
View File

@ -997,9 +997,16 @@ programMain:
case config.cmd
of createTestnet:
var deposits: seq[Deposit]
for i in config.firstValidator.int ..< config.totalValidators.int:
let depositFile = config.testnetDepositsDir /
validatorFileBaseName(i) & ".deposit.json"
var i = -1
for kind, dir in walkDir(config.testnetDepositsDir.string):
if kind != pcDir:
continue
inc i
if i < config.firstValidator.int:
continue
let depositFile = dir / "deposit.json"
try:
deposits.add Json.loadFile(depositFile, Deposit)
except SerializationError as err:
@ -1097,6 +1104,7 @@ programMain:
of makeDeposits:
createDir(config.outValidatorsDir)
createDir(config.outSecretsDir)
let
deposits = generateDeposits(

17
beacon_chain/conf.nim
View File

@ -1,7 +1,7 @@
{.push raises: [Defect].}
import
os, options, strformat,
os, options,
chronicles, confutils, json_serialization,
confutils/defs, confutils/std/net,
chronicles/options as chroniclesOptions,
@ -106,11 +106,11 @@ type
name: "validator" }: seq[ValidatorKeyPath]
validatorsDir* {.
desc: "A directory containing validator keystores"
desc: "A directory containing validator keystores."
name: "validators-dir" }: Option[InputDir]
secretsDir* {.
desc: "A directory containing validator keystore passwords"
desc: "A directory containing validator keystore passwords."
name: "secrets-dir" }: Option[InputDir]
stateSnapshot* {.
@ -304,11 +304,11 @@ type
name: "validator" }: seq[ValidatorKeyPath]
validatorsDir* {.
desc: "A directory containing validator keystores"
desc: "A directory containing validator keystores."
name: "validators-dir" }: Option[InputDir]
secretsDir* {.
desc: "A directory containing validator keystore passwords"
desc: "A directory containing validator keystore passwords."
name: "secrets-dir" }: Option[InputDir]
proc defaultDataDir*(conf: BeaconNodeConf|ValidatorClientConf): string =
@ -321,13 +321,6 @@ proc defaultDataDir*(conf: BeaconNodeConf|ValidatorClientConf): string =
getHomeDir() / dataDir / "BeaconNode"
proc validatorFileBaseName*(validatorIdx: int): string =
# there can apparently be tops 4M validators so we use 7 digits..
try:
fmt"v{validatorIdx:07}"
except ValueError as e:
raiseAssert e.msg
func dumpDir*(conf: BeaconNodeConf|ValidatorClientConf): string =
conf.dataDir / "dump"

28
beacon_chain/spec/keystore.nim
View File

@ -7,8 +7,9 @@
import
json, math, strutils, strformat,
eth/keyfile/uuid, stew/[results, byteutils, bitseqs, bitops2],
nimcrypto/[sha2, rijndael, pbkdf2, bcmode, hash, sysrand], blscurve,
stew/[results, byteutils, bitseqs, bitops2], stew/shims/macros,
eth/keyfile/uuid, blscurve,
nimcrypto/[sha2, rijndael, pbkdf2, bcmode, hash, sysrand],
datatypes, crypto, digest, helpers
export
@ -102,7 +103,16 @@ const
# https://github.com/bitcoin/bips/blob/master/bip-0039/bip-0039-wordlists.md
wordListLen = 2048
englishWords = split slurp("english_word_list.txt")
macro wordListArray(filename: static string): array[wordListLen, cstring] =
result = newTree(nnkBracket)
var words = slurp(filename).split()
words.setLen wordListLen
for word in words:
result.add newCall("cstring", newLit(word))
const
englishWords = wordListArray "english_word_list.txt"
iterator pathNodesImpl(path: string): Natural
{.raises: [ValueError].} =
@ -142,7 +152,7 @@ func getSeed*(mnemonic: Mnemonic, password: KeyStorePass): KeySeed =
let salt = "mnemonic-" & password.string
KeySeed sha512.pbkdf2(mnemonic.string, salt, 2048, 64)
proc generateMnemonic*(words: openarray[string],
proc generateMnemonic*(words: openarray[cstring],
entropyParam: openarray[byte] = @[]): Mnemonic =
# https://github.com/bitcoin/bips/blob/master/bip-0039.mediawiki#generating-the-mnemonic
doAssert words.len == wordListLen
@ -163,7 +173,9 @@ proc generateMnemonic*(words: openarray[string],
entropy.add byte(checksum.data.getBitsBE(0 ..< checksumBits))
var res = words[entropy.getBitsBE(0..10)]
var res = ""
res.add words[entropy.getBitsBE(0..10)]
for i in 1 ..< mnemonicWordCount:
let
firstBit = i*11
@ -276,7 +288,7 @@ proc encryptKeystore*(T: type[KdfParams],
path = KeyPath "",
salt: openarray[byte] = @[],
iv: openarray[byte] = @[],
ugly = true): KeyStoreContent =
pretty = true): KeyStoreContent =
var
secret = privKey.toRaw[^32..^1]
decKey: seq[byte]
@ -334,8 +346,8 @@ proc encryptKeystore*(T: type[KdfParams],
uuid: $uuid,
version: 4)
KeyStoreContent if ugly: $(%keystore)
else: pretty(%keystore, indent=4)
KeyStoreContent if pretty: json.pretty(%keystore, indent=4)
else: $(%keystore)
proc restoreCredentials*(mnemonic: Mnemonic,
password = KeyStorePass ""): Credentials =

2
beacon_chain/validator_keygen.nim
View File

@ -52,7 +52,7 @@ proc loadKeyStore(conf: BeaconNodeConf|ValidatorClientConf,
return
var remainingAttempts = 3
var prompt = "Please enter passphrase for key \"" & validatorsDir/keyName & "\""
var prompt = "Please enter passphrase for key \"" & validatorsDir/keyName & "\"\n"
while remainingAttempts > 0:
let passphrase = KeyStorePass:
try: readPasswordFromStdin(prompt)

23
docker/manage_testnet_hosts.nims
View File

@ -1,5 +1,5 @@
import
strformat, os, confutils
strformat, os, confutils, algorightm
type
Command = enum
@ -22,9 +22,6 @@ type
defaultValue: "data"
name: "network-data-dir" }: string
totalValidators {.
name: "total-validators" }: int
totalUserValidators {.
defaultValue: 0
name: "user-validators" }: int
@ -39,6 +36,9 @@ var conf = load CliConfig
var
serverCount = 10
instancesCount = 2
validators = listDirs(conf.depositsDir)
sort(validators)
proc findOrDefault[K, V](tupleList: openarray[(K, V)], key: K, default: V): V =
for t in tupleList:
@ -60,7 +60,7 @@ iterator nodes: Node =
iterator validatorAssignments: tuple[node: Node; firstValidator, lastValidator: int] =
let
systemValidators = conf.totalValidators - conf.totalUserValidators
systemValidators = validators.len - conf.totalUserValidators
defaultValidatorAssignment = proc (nodeIdx: int): int =
(systemValidators div serverCount) div instancesCount
@ -110,26 +110,25 @@ of restart_nodes:
of reset_network:
for n, firstValidator, lastValidator in validatorAssignments():
var
keysList = ""
validatorDirs = ""
networkDataFiles = conf.networkDataDir & "/{genesis.ssz,bootstrap_nodes.txt}"
for i in firstValidator ..< lastValidator:
let validatorKey = fmt"v{i:07}.privkey"
keysList.add " "
keysList.add conf.depositsDir / validatorKey
validatorDirs.add " "
validatorDirs.add conf.depositsDir / validators[i]
let dockerPath = &"/docker/{n.container}/data/BeaconNode"
echo &"echo Syncing {lastValidator - firstValidator} keys starting from {firstValidator} to container {n.container}@{n.server} ... && \\"
echo &" ssh {n.server} 'sudo rm -rf /tmp/nimbus && mkdir -p /tmp/nimbus/' && \\"
echo &" rsync -a -zz {networkDataFiles} {n.server}:/tmp/nimbus/net-data/ && \\"
if keysList.len > 0:
echo &" rsync -a -zz {keysList} {n.server}:/tmp/nimbus/keys/ && \\"
if validatorDirs.len > 0:
echo &" rsync -a -zz {validatorDirs} {n.server}:/tmp/nimbus/keys/ && \\"
echo &" ssh {n.server} 'sudo docker container stop {n.container}; " &
&"sudo mkdir -p {dockerPath}/validators && " &
&"sudo rm -rf {dockerPath}/validators/* && " &
&"sudo rm -rf {dockerPath}/db && " &
(if keysList.len > 0: &"sudo mv /tmp/nimbus/keys/* {dockerPath}/validators/ && " else: "") &
(if validatorDirs.len > 0: &"sudo mv /tmp/nimbus/keys/* {dockerPath}/validators/ && " else: "") &
&"sudo mv /tmp/nimbus/net-data/* {dockerPath}/ && " &
&"sudo chown dockremap:docker -R {dockerPath}'"

24
scripts/attach_validators.sh
View File

@ -1,24 +0,0 @@
#!/bin/bash
set -eu
NETWORK_NAME=$1
NODE_ID=$2
FIRST_VALIDATOR=$3
LAST_VALIDATOR=$4
cd $(dirname "$0")
cd ..
if [ -f .env ]; then
source .env
fi
NETWORK_DIR=$WWW_DIR/$NETWORK_NAME
for i in $(seq $FIRST_VALIDATOR $LAST_VALIDATOR); do
VALIDATOR=v$(printf '%07d' $i)
beacon_chain/beacon_node --data-dir="$DATA_DIR/node-$NODE_ID" importValidator \
--keyfile="$NETWORK_DIR/$VALIDATOR.privkey"
done

2
scripts/connect_to_testnet.nims
View File

@ -137,7 +137,7 @@ cli do (skipGoerliKey {.
mkDir validatorsDir
mode = Verbose
exec replace(&"""{beaconNodeBinary} makeDeposits
--random-deposits=1
--count=1
--out-validators-dir="{validatorsDir}"
--out-secrets-dir="{secretsDir}"
--deposit-private-key={privKey}

10
scripts/launch_local_testnet.sh
View File

@ -141,7 +141,7 @@ $MAKE LOG_LEVEL="${LOG_LEVEL}" NIMFLAGS="-d:insecure -d:testnet_servers_image ${
./build/beacon_node makeDeposits \
--count=${TOTAL_VALIDATORS} \
--out-validators-dir="${DEPOSITS_DIR}" \
--out-secrets-dir="${SECRETS_DIR}"
--out-secrets-dir="${SECRETS_DIR}"
BOOTSTRAP_IP="127.0.0.1"
./build/beacon_node createTestnet \
@ -206,10 +206,12 @@ for NUM_NODE in $(seq 0 $(( ${NUM_NODES} - 1 ))); do
# The first $NODES_WITH_VALIDATORS nodes split them equally between them, after skipping the first $USER_VALIDATORS.
NODE_DATA_DIR="${DATA_DIR}/node${NUM_NODE}"
mkdir -p "${NODE_DATA_DIR}/validators"
mkdir -p "${NODE_DATA_DIR}/secrets"
if [[ $NUM_NODE -lt $NODES_WITH_VALIDATORS ]]; then
# TODO: There are no longer privkey files
for KEYFILE in $(ls ${DEPOSITS_DIR}/*.privkey | tail -n +$(( $USER_VALIDATORS + ($VALIDATORS_PER_NODE * $NUM_NODE) + 1 )) | head -n $VALIDATORS_PER_NODE); do
cp -a "$KEYFILE" "${NODE_DATA_DIR}/validators/"
for VALIDATOR in $(ls ${DEPOSITS_DIR}/* | tail -n +$(( $USER_VALIDATORS + ($VALIDATORS_PER_NODE * $NUM_NODE) + 1 )) | head -n $VALIDATORS_PER_NODE); do
cp -ar "$VALIDATOR" "${NODE_DATA_DIR}/validators/"
cp -a "${SECRETS_DIR}/${VALIDATOR}" "${NODE_DATA_DIR}/secrets"
done
fi

1
scripts/reset_testnet.sh
View File

@ -117,7 +117,6 @@ if [[ $PUBLISH_TESTNET_RESETS != "0" ]]; then
--deposits-dir="$DEPOSITS_DIR_ABS" \
--network-data-dir="$NETWORK_DIR_ABS" \
--user-validators=$USER_VALIDATORS \
--total-validators=$TOTAL_VALIDATORS \
> /tmp/reset-network.sh
bash /tmp/reset-network.sh

35
tests/simulation/run_node.sh
View File

@ -31,7 +31,10 @@ source "${SIM_ROOT}/../../env.sh"
cd "$GIT_ROOT"
DATA_DIR="${SIMULATION_DIR}/node-$NODE_ID"
NODE_DATA_DIR="${SIMULATION_DIR}/node-$NODE_ID"
NODE_VALIDATORS_DIR=$NODE_DATA_DIR/validators/
NODE_SECRETS_DIR=$NODE_DATA_DIR/secrets/
PORT=$(( BASE_P2P_PORT + NODE_ID ))
NAT_ARG="--nat:extip:127.0.0.1"
@ -39,41 +42,47 @@ if [ "${NAT:-}" == "1" ]; then
NAT_ARG="--nat:any"
fi
mkdir -p "$DATA_DIR/validators"
rm -f $DATA_DIR/validators/*
rm -rf "$NODE_VALIDATORS_DIR"
mkdir -p "$NODE_VALIDATORS_DIR"
rm -rf "$NODE_SECRETS_DIR"
mkdir -p "$NODE_SECRETS_DIR"
VALIDATORS_PER_NODE=$((NUM_VALIDATORS / TOTAL_NODES))
if [[ $NODE_ID -lt $TOTAL_NODES ]]; then
VALIDATORS_PER_NODE=$((NUM_VALIDATORS / TOTAL_NODES))
VALIDATORS_PER_NODE_HALF=$((VALIDATORS_PER_NODE / 2))
FIRST_VALIDATOR_IDX=$(( VALIDATORS_PER_NODE * NODE_ID ))
# if using validator client binaries in addition to beacon nodes
# we will split the keys for this instance in half between the BN and the VC
if [ "${SPLIT_VALIDATORS_BETWEEN_BN_AND_VC:-}" == "yes" ]; then
LAST_VALIDATOR_IDX=$(( FIRST_VALIDATOR_IDX + VALIDATORS_PER_NODE_HALF - 1 ))
ATTACHED_VALIDATORS=$((VALIDATORS_PER_NODE / 2))
else
LAST_VALIDATOR_IDX=$(( FIRST_VALIDATOR_IDX + VALIDATORS_PER_NODE - 1 ))
ATTACHED_VALIDATORS=$VALIDATORS_PER_NODE
fi
pushd "$VALIDATORS_DIR" >/dev/null
cp $(seq -s " " -f v%07g.privkey $FIRST_VALIDATOR_IDX $LAST_VALIDATOR_IDX) "$DATA_DIR/validators"
for VALIDATOR in $(ls | tail -n +$(( ($VALIDATORS_PER_NODE * $NODE_ID) + 1 )) | head -n $ATTACHED_VALIDATORS); do
cp -ar "$VALIDATOR" "$NODE_VALIDATORS_DIR"
cp -a "$SECRETS_DIR/$VALIDATOR" "$NODE_SECRETS_DIR"
done
popd >/dev/null
fi
rm -rf "$DATA_DIR/dump"
mkdir -p "$DATA_DIR/dump"
rm -rf "$NODE_DATA_DIR/dump"
mkdir -p "$NODE_DATA_DIR/dump"
SNAPSHOT_ARG=""
if [ -f "${SNAPSHOT_FILE}" ]; then
SNAPSHOT_ARG="--state-snapshot=${SNAPSHOT_FILE}"
fi
cd "$DATA_DIR"
cd "$NODE_DATA_DIR"
# if you want tracing messages, add "--log-level=TRACE" below
$BEACON_NODE_BIN \
--log-level=${LOG_LEVEL:-DEBUG} \
--bootstrap-file=$BOOTSTRAP_ADDRESS_FILE \
--data-dir=$DATA_DIR \
--data-dir=$NODE_DATA_DIR \
--secrets-dir=$NODE_SECRETS_DIR \
--node-name=$NODE_ID \
--tcp-port=$PORT \
--udp-port=$PORT \

17
tests/simulation/start.sh
View File

@ -21,9 +21,6 @@ DEFS+="-d:MAX_COMMITTEES_PER_SLOT=${MAX_COMMITTEES_PER_SLOT:-1} " # Spec de
DEFS+="-d:SLOTS_PER_EPOCH=${SLOTS_PER_EPOCH:-6} " # Spec default: 32
DEFS+="-d:SECONDS_PER_SLOT=${SECONDS_PER_SLOT:-6} " # Spec default: 12
LAST_VALIDATOR_NUM=$(( NUM_VALIDATORS - 1 ))
LAST_VALIDATOR="$VALIDATORS_DIR/v$(printf '%07d' $LAST_VALIDATOR_NUM).deposit.json"
# Windows detection
if uname | grep -qiE "mingw|msys"; then
MAKE="mingw32-make"
@ -96,10 +93,20 @@ if [[ "$USE_TMUX" != "no" ]]; then
$TMUX select-window -t "${TMUX_SESSION_NAME}:sim"
fi
$MAKE -j3 --no-print-directory NIMFLAGS="$CUSTOM_NIMFLAGS $DEFS" LOG_LEVEL="${LOG_LEVEL:-DEBUG}" beacon_node validator_client deposit_contract
$MAKE -j3 --no-print-directory NIMFLAGS="$CUSTOM_NIMFLAGS $DEFS" LOG_LEVEL="${LOG_LEVEL:-DEBUG}" beacon_node validator_client
count_files () {
{ ls -1q $1 2> /dev/null || true ; } | wc -l
}
EXISTING_VALIDATORS=$(count_files "$VALIDATORS_DIR/*/deposit.json")
if [[ $EXISTING_VALIDATORS -lt $NUM_VALIDATORS ]]; then
rm -rf "$VALIDATORS_DIR"
rm -rf "$SECRETS_DIR"
if [ ! -f "${LAST_VALIDATOR}" ]; then
if [ "$WEB3_ARG" != "" ]; then
make deposit_contract
echo Deploying the validator deposit contract...
DEPOSIT_CONTRACT_ADDRESS=$($DEPLOY_DEPOSIT_CONTRACT_BIN deploy $WEB3_ARG)
echo Contract deployed at $DEPOSIT_CONTRACT_ADDRESS