status-im
/
nimbus-eth2
mirror of https://github.com/status-im/nimbus-eth2.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Add simulation mode that bootstraps from an Eth1 ganache instance

This commit is contained in:
Zahary Karadjov 2020-06-16 01:20:31 +03:00 committed by zah
parent 4275abbefe
commit e9d68e2f7b
13 changed files with 216 additions and 122 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
beacon_chain/beacon_node.nim
View File

@ -1046,7 +1046,7 @@ programMain:
except SerializationError as err:
stderr.write "Error while loading a deposit file:\n"
stderr.write err.formatMsg(depositFile), "\n"
stderr.write "Please regenerate the deposit files by running makeDeposits again\n"
stderr.write "Please regenerate the deposit files by running 'beacon_node deposits create' again\n"
quit 1
let
@ -1134,21 +1134,24 @@ programMain:
else:
node.start()
of makeDeposits:
createDir(config.outValidatorsDir)
createDir(config.outSecretsDir)
of deposits:
case config.depositsCmd
of DepositsCmd.create:
createDir(config.outValidatorsDir)
createDir(config.outSecretsDir)
let
deposits = generateDeposits(
discard generateDeposits(
config.totalDeposits,
config.outValidatorsDir,
config.outSecretsDir).tryGet
config.outSecretsDir)
if config.web3Url.len > 0 and config.depositContractAddress.len > 0:
of DepositsCmd.send:
if config.minDelay > config.maxDelay:
echo "The minimum delay should not be larger than the maximum delay"
quit 1
let deposits = loadDeposits(config.depositsDir)
var delayGenerator: DelayGenerator
if config.maxDelay > 0.0:
delayGenerator = proc (): chronos.Duration {.gcsafe.} =

79
beacon_chain/conf.nim
View File

@ -17,7 +17,14 @@ type
noCommand
importValidator
createTestnet
makeDeposits
deposits
DepositsCmd* = enum
create = "Create validator keystores and deposits"
send = "Send prepared deposits to the validator deposit contract"
# TODO
# status = "Display status information about all deposits"
VCStartUpCmd* = enum
VCNoCommand
@ -101,7 +108,7 @@ type
validators* {.
required
desc: "Path to a validator private key, as generated by makeDeposits."
desc: "Path to a validator keystore"
abbr: "v"
name: "validator" }: seq[ValidatorKeyPath]
@ -235,37 +242,53 @@ type
desc: "File with validator key to be imported (in hex form)."
name: "keyfile" }: seq[ValidatorKeyPath]
of makeDeposits:
totalDeposits* {.
defaultValue: 1
desc: "Number of deposits to generate."
name: "count" }: int
of deposits:
case depositsCmd*: DepositsCmd
of create:
totalDeposits* {.
defaultValue: 1
desc: "Number of deposits to generate."
name: "count" }: int
outValidatorsDir* {.
defaultValue: "validators"
desc: "Output folder for validator keystores and deposits."
name: "out-validators-dir" }: string
outValidatorsDir* {.
defaultValue: "validators"
desc: "Output folder for validator keystores and deposits."
name: "out-validators-dir" }: string
outSecretsDir* {.
defaultValue: "secrets"
desc: "Output folder for randomly generated keystore passphrases."
name: "out-secrets-dir" }: string
outSecretsDir* {.
defaultValue: "secrets"
desc: "Output folder for randomly generated keystore passphrases."
name: "out-secrets-dir" }: string
depositPrivateKey* {.
defaultValue: ""
desc: "Private key of the controlling (sending) account.",
name: "deposit-private-key" }: string
depositPrivateKey* {.
defaultValue: ""
desc: "Private key of the controlling (sending) account.",
name: "deposit-private-key" }: string
minDelay* {.
defaultValue: 0.0
desc: "Minimum possible delay between making two deposits (in seconds)."
name: "min-delay" }: float
dontSend* {.
defaultValue: false,
desc: "By default, all created deposits are also immediately sent " &
"to the validator deposit contract. You can use this option to " &
"prevent this behavior. Use the `deposits send` command to send " &
"the deposit transactions at your convenience later."
name: "dont-send" .}: bool
maxDelay* {.
defaultValue: 0.0
desc: "Maximum possible delay between making two deposits (in seconds)."
name: "max-delay" }: float
of send:
depositsDir* {.
defaultValue: "validators"
desc: "A folder with validator metadata created by the `deposits create` command."
name: "out-validators-dir" }: string
minDelay* {.
defaultValue: 0.0
desc: "Minimum possible delay between making two deposits (in seconds)."
name: "min-delay" }: float
maxDelay* {.
defaultValue: 0.0
desc: "Maximum possible delay between making two deposits (in seconds)."
name: "max-delay" }: float
ValidatorClientConf* = object
logLevel* {.
defaultValue: "DEBUG"
@ -299,7 +322,7 @@ type
validators* {.
required
desc: "Path to a validator key store, as generated by makeDeposits."
desc: "Attach a validator by supplying a keystore path."
abbr: "v"
name: "validator" }: seq[ValidatorKeyPath]

18
beacon_chain/keystore_management.nim
View File

@ -139,6 +139,24 @@ proc generateDeposits*(totalValidators: int,
ok deposits
proc loadDeposits*(depositsDir: string): seq[Deposit] =
try:
for kind, dir in walkDir(depositsDir):
if kind == pcDir:
let depositFile = dir / depositFileName
try:
result.add Json.loadFile(depositFile, Deposit)
except IOError as err:
error "Failed to open deposit file", depositFile, err = err.msg
quit 1
except SerializationError as err:
error "Invalid deposit file", error = formatMsg(err, depositFile)
quit 1
except OSError as err:
error "Deposits directory not accessible",
path = depositsDir, err = err.msg
quit 1
{.pop.}
proc sendDeposits*(deposits: seq[Deposit],

35
beacon_chain/spec/crypto.nim
View File

@ -260,30 +260,47 @@ template hash*(x: BlsCurveType): Hash =
# Serialization
# ----------------------------------------------------------------------
{.pragma: serializationRaises, raises: [SerializationError, IOError, Defect].}
proc writeValue*(writer: var JsonWriter, value: ValidatorPubKey) {.
inline, raises: [IOError, Defect].} =
writer.writeValue(value.toHex())
proc readValue*(reader: var JsonReader, value: var ValidatorPubKey) {.
inline, raises: [Exception].} =
value = ValidatorPubKey.fromHex(reader.readValue(string)).tryGet()
proc readValue*(reader: var JsonReader, value: var ValidatorPubKey)
{.serializationRaises.} =
let key = ValidatorPubKey.fromHex(reader.readValue(string))
if key.isOk:
value = key.get
else:
# TODO: Can we provide better diagnostic?
raiseUnexpectedValue(reader, "Valid hex-encoded public key expected")
proc writeValue*(writer: var JsonWriter, value: ValidatorSig) {.
inline, raises: [IOError, Defect].} =
# Workaround: https://github.com/status-im/nim-beacon-chain/issues/374
writer.writeValue(value.toHex())
proc readValue*(reader: var JsonReader, value: var ValidatorSig) {.
inline, raises: [Exception].} =
value = ValidatorSig.fromHex(reader.readValue(string)).tryGet()
proc readValue*(reader: var JsonReader, value: var ValidatorSig)
{.serializationRaises.} =
let sig = ValidatorSig.fromHex(reader.readValue(string))
if sig.isOk:
value = sig.get
else:
# TODO: Can we provide better diagnostic?
raiseUnexpectedValue(reader, "Valid hex-encoded signature expected")
proc writeValue*(writer: var JsonWriter, value: ValidatorPrivKey) {.
inline, raises: [IOError, Defect].} =
writer.writeValue(value.toHex())
proc readValue*(reader: var JsonReader, value: var ValidatorPrivKey) {.
inline, raises: [Exception].} =
value = ValidatorPrivKey.fromHex(reader.readValue(string)).tryGet()
proc readValue*(reader: var JsonReader, value: var ValidatorPrivKey)
{.serializationRaises.} =
let key = ValidatorPrivKey.fromHex(reader.readValue(string))
if key.isOk:
value = key.get
else:
# TODO: Can we provide better diagnostic?
raiseUnexpectedValue(reader, "Valid hex-encoded private key expected")
template fromSszBytes*(T: type BlsValue, bytes: openArray[byte]): auto =
let v = fromRaw(T, bytes)

10
beacon_chain/spec/datatypes.nim
View File

@ -408,8 +408,6 @@ type
committee_count_cache*: Table[Epoch, uint64]
beacon_proposer_indices*: Table[Slot, Option[ValidatorIndex]]
JsonError = jsonTypes.JsonError
# https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#signingdata
# TODO move back into big `type` block
when ETH2_SPEC == "v0.12.1":
@ -485,7 +483,7 @@ template ethTimeUnit(typ: type) {.dirty.} =
writeValue(writer, uint64 value)
proc readValue*(reader: var JsonReader, value: var typ)
{.raises: [IOError, JsonError, Defect].} =
{.raises: [IOError, SerializationError, Defect].} =
value = typ reader.readValue(uint64)
proc writeValue*(writer: var JsonWriter, value: ValidatorIndex)
@ -493,14 +491,14 @@ proc writeValue*(writer: var JsonWriter, value: ValidatorIndex)
writeValue(writer, uint32 value)
proc readValue*(reader: var JsonReader, value: var ValidatorIndex)
{.raises: [IOError, JsonError, Defect].} =
{.raises: [IOError, SerializationError, Defect].} =
value = ValidatorIndex reader.readValue(uint32)
template writeValue*(writer: var JsonWriter, value: Version | ForkDigest) =
writeValue(writer, $value)
proc readValue*(reader: var JsonReader, value: var Version)
{.raises: [IOError, JsonError, Defect].} =
{.raises: [IOError, SerializationError, Defect].} =
let hex = reader.readValue(string)
try:
hexToByteArray(hex, array[4, byte](value))
@ -508,7 +506,7 @@ proc readValue*(reader: var JsonReader, value: var Version)
raiseUnexpectedValue(reader, "Hex string of 4 bytes expected")
proc readValue*(reader: var JsonReader, value: var ForkDigest)
{.raises: [IOError, JsonError, Defect].} =
{.raises: [IOError, SerializationError, Defect].} =
let hex = reader.readValue(string)
try:
hexToByteArray(hex, array[4, byte](value))

2
scripts/connect_to_testnet.nims
View File

@ -145,7 +145,7 @@ cli do (skipGoerliKey {.
if privKey.len > 0:
mkDir validatorsDir
mode = Verbose
exec replace(&"""{beaconNodeBinary} makeDeposits
exec replace(&"""{beaconNodeBinary} deposits create
--count=1
--out-validators-dir="{validatorsDir}"
--out-secrets-dir="{secretsDir}"

5
scripts/launch_local_testnet.sh
View File

@ -138,10 +138,11 @@ fi
NETWORK_NIM_FLAGS=$(scripts/load-testnet-nim-flags.sh ${NETWORK})
$MAKE LOG_LEVEL="${LOG_LEVEL}" NIMFLAGS="-d:insecure -d:testnet_servers_image ${NETWORK_NIM_FLAGS}" beacon_node
./build/beacon_node makeDeposits \
./build/beacon_node deposits create \
--count=${TOTAL_VALIDATORS} \
--out-validators-dir="${DEPOSITS_DIR}" \
--out-secrets-dir="${SECRETS_DIR}"
--out-secrets-dir="${SECRETS_DIR}" \
--dont-send
GENESIS_OFFSET=30

7
scripts/reset_testnet.sh
View File

@ -65,7 +65,7 @@ if [ "$ETH1_PRIVATE_KEY" != "" ]; then
echo "Done: $DEPOSIT_CONTRACT_ADDRESS"
fi
echo "Building a local beacon_node instance for 'makeDeposits' and 'createTestnet'"
echo "Building a local beacon_node instance for 'deposits create' and 'createTestnet'"
make -j2 NIMFLAGS="-d:insecure -d:testnet_servers_image ${NETWORK_NIM_FLAGS}" beacon_node process_dashboard
echo "Generating Grafana dashboards for remote testnet servers"
@ -83,10 +83,11 @@ echo "Building Docker image..."
# in docker/Makefile, and are enabled by default.
make build
../build/beacon_node makeDeposits \
../build/beacon_node deposits create \
--count=$TOTAL_VALIDATORS \
--out-validators-dir="$DEPOSITS_DIR_ABS" \
--out-secrets-dir="$SECRETS_DIR_ABS"
--out-secrets-dir="$SECRETS_DIR_ABS" \
--dont-send
../build/beacon_node createTestnet \
--data-dir="$DATA_DIR_ABS" \

155
tests/simulation/start.sh
View File

@ -2,6 +2,31 @@
set -eo pipefail
# To allow overriding the program names
MULTITAIL="${MULTITAIL:-multitail}"
TMUX="${TMUX:-tmux}"
GANACHE="${GANACHE:-ganache-cli}"
PROMETHEUS="${PROMETHEUS:-prometheus}"
CTAIL="${CTAIL:-ctail}"
# Using tmux or multitail is an opt-in
USE_MULTITAIL="${USE_MULTITAIL:-no}"
type "$MULTITAIL" &>/dev/null || { echo "${MULTITAIL}" is missing; USE_MULTITAIL="no"; }
USE_TMUX="${USE_TMUX:-no}"
type "$TMUX" &>/dev/null || { echo "${TMUX}" is missing; USE_TMUX="no"; }
WAIT_GENESIS="${WAIT_GENESIS:-no}"
USE_GANACHE="${USE_GANACHE:-yes}"
type "$GANACHE" &>/dev/null || { echo $GANACHE is missing; USE_GANACHE="no"; WAIT_GENESIS="no"; }
USE_PROMETHEUS="${USE_PROMETHEUS:-yes}"
type "$PROMETHEUS" &>/dev/null || { echo $PROMETHEUS is missing; USE_PROMETHEUS="no"; }
USE_CTAIL="${USE_CTAIL:-yes}"
type "$CTAIL" &>/dev/null || { echo $CTAIL is missing; USE_CTAIL="no"; }
# Read in variables
# shellcheck source=/dev/null
source "$(dirname "$0")/vars.sh"
@ -28,11 +53,6 @@ else
MAKE="make"
fi
# to allow overriding the program names
MULTITAIL="${MULTITAIL:-multitail}"
TMUX="${TMUX:-tmux}"
GANACHE="${GANACHE:-ganache-cli}"
PROMETHEUS="${PROMETHEUS:-prometheus}"
TMUX_SESSION_NAME="${TMUX_SESSION_NAME:-nbc-sim}"
WAIT_GENESIS="${WAIT_GENESIS:-no}"
@ -91,7 +111,8 @@ fi
if [[ "$USE_PROMETHEUS" != "no" ]]; then
if [[ "$USE_TMUX" != "no" ]]; then
$TMUX new-window -d -t $TMUX_SESSION_NAME -n "$PROMETHEUS" "cd '$METRICS_DIR' && $PROMETHEUS"
PROMETHEUS_FLAGS="--config.file=./prometheus.yml --storage.tsdb.path=./data"
$TMUX new-window -d -t $TMUX_SESSION_NAME -n "$PROMETHEUS" "cd '$METRICS_DIR' && $PROMETHEUS $PROMETHEUS_FLAGS"
else
echo NOTICE: $PROMETHEUS will be started automatically only with USE_TMUX=1
USE_PROMETHEUS="no"
@ -114,68 +135,16 @@ if [[ $EXISTING_VALIDATORS -lt $NUM_VALIDATORS ]]; then
rm -rf "$VALIDATORS_DIR"
rm -rf "$SECRETS_DIR"
if [ "$WEB3_ARG" != "" ]; then
make deposit_contract
echo Deploying the validator deposit contract...
DEPOSIT_CONTRACT_ADDRESS=$($DEPLOY_DEPOSIT_CONTRACT_BIN deploy $WEB3_ARG)
echo Contract deployed at $DEPOSIT_CONTRACT_ADDRESS
export DEPOSIT_CONTRACT_ADDRESS
fi
DELAY_ARGS=""
# Uncomment this line to slow down the initial deposits.
# This will spread them across multiple blocks which is
# a more realistic scenario.
DELAY_ARGS="--min-delay=1 --max-delay=5"
MAKE_DEPOSITS_WEB3_ARG=$WEB3_ARG
if [[ "$WAIT_GENESIS" == "no" ]]; then
MAKE_DEPOSITS_WEB3_ARG=""
fi
$BEACON_NODE_BIN makeDeposits \
$BEACON_NODE_BIN deposits create \
--count="${NUM_VALIDATORS}" \
--non-interactive \
--out-validators-dir="$VALIDATORS_DIR" \
--out-secrets-dir="$SECRETS_DIR" \
$MAKE_DEPOSITS_WEB3_ARG $DELAY_ARGS \
--deposit-contract="${DEPOSIT_CONTRACT_ADDRESS}"
--dont-send
echo "All deposits prepared"
fi
if [ ! -f "${SNAPSHOT_FILE}" ]; then
if [[ "${WAIT_GENESIS}" == "no" ]]; then
echo Creating testnet genesis...
$BEACON_NODE_BIN \
--data-dir="${SIMULATION_DIR}/node-$MASTER_NODE" \
createTestnet \
--validators-dir="${VALIDATORS_DIR}" \
--total-validators="${NUM_VALIDATORS}" \
--output-genesis="${SNAPSHOT_FILE}" \
--output-bootstrap-file="${NETWORK_BOOTSTRAP_FILE}" \
--bootstrap-address=127.0.0.1 \
--bootstrap-port=$(( BASE_P2P_PORT + MASTER_NODE )) \
--genesis-offset=15 # Delay in seconds
fi
fi
rm -f beacon_node.log
# Delete any leftover address files from a previous session
if [ -f "${MASTER_NODE_ADDRESS_FILE}" ]; then
rm "${MASTER_NODE_ADDRESS_FILE}"
fi
# Kill child processes on Ctrl-C/SIGTERM/exit, passing the PID of this shell
# instance as the parent and the target process name as a pattern to the
# "pkill" command.
if [[ "$USE_MULTITAIL" == "no" && "$USE_TMUX" == "no" ]]; then
trap 'pkill -P $$ beacon_node' SIGINT EXIT
fi
LAST_WAITING_NODE=0
function run_cmd {
i=$1
CMD=$2
@ -198,6 +167,61 @@ function run_cmd {
fi
}
if [ "$WEB3_ARG" != "" ]; then
make deposit_contract
echo Deploying the validator deposit contract...
echo $DEPLOY_DEPOSIT_CONTRACT_BIN deploy $WEB3_ARG
DEPOSIT_CONTRACT_ADDRESS=$($DEPLOY_DEPOSIT_CONTRACT_BIN deploy $WEB3_ARG)
echo Contract deployed at $DEPOSIT_CONTRACT_ADDRESS
export DEPOSIT_CONTRACT_ADDRESS
if [[ "$WAIT_GENESIS" != "no" ]]; then
echo "(deposit maker)" "$BEACON_NODE_BIN deposits send \
--non-interactive \
--validators-dir='$VALIDATORS_DIR' \
--min-delay=1 --max-delay=5 \
$WEB3_ARG \
--deposit-contract=${DEPOSIT_CONTRACT_ADDRESS}"
run_cmd "(deposit maker)" "$BEACON_NODE_BIN deposits send \
--non-interactive \
--validators-dir='$VALIDATORS_DIR' \
--min-delay=1 --max-delay=5 \
$WEB3_ARG \
--deposit-contract=${DEPOSIT_CONTRACT_ADDRESS}"
fi
fi
if [ ! -f "${SNAPSHOT_FILE}" ]; then
if [[ "${WAIT_GENESIS}" == "no" ]]; then
echo Creating testnet genesis...
$BEACON_NODE_BIN \
--data-dir="${SIMULATION_DIR}/node-$MASTER_NODE" \
createTestnet \
--validators-dir="${VALIDATORS_DIR}" \
--total-validators="${NUM_VALIDATORS}" \
--output-genesis="${SNAPSHOT_FILE}" \
--output-bootstrap-file="${NETWORK_BOOTSTRAP_FILE}" \
--bootstrap-address=127.0.0.1 \
--bootstrap-port=$(( BASE_P2P_PORT + MASTER_NODE )) \
--genesis-offset=15 # Delay in seconds
fi
fi
# Delete any leftover address files from a previous session
if [ -f "${MASTER_NODE_ADDRESS_FILE}" ]; then
rm "${MASTER_NODE_ADDRESS_FILE}"
fi
# Kill child processes on Ctrl-C/SIGTERM/exit, passing the PID of this shell
# instance as the parent and the target process name as a pattern to the
# "pkill" command.
if [[ "$USE_MULTITAIL" == "no" && "$USE_TMUX" == "no" ]]; then
trap 'pkill -P $$ beacon_node' SIGINT EXIT
fi
LAST_WAITING_NODE=0
for i in $(seq $MASTER_NODE -1 $TOTAL_USER_NODES); do
if [[ "$i" != "$MASTER_NODE" && "$USE_MULTITAIL" == "no" ]]; then
# Wait for the master node to write out its address file
@ -218,6 +242,15 @@ for i in $(seq $MASTER_NODE -1 $TOTAL_USER_NODES); do
fi
done
if [[ "$USE_CTAIL" != "no" ]]; then
if [[ "$USE_TMUX" != "no" ]]; then
$TMUX new-window -d -t $TMUX_SESSION_NAME -n "$CTAIL" "$CTAIL tail -q -n +1 -f ${SIMULATION_DIR}/node-*/beacon_node.log"
else
echo NOTICE: $CTAIL will be started automatically only with USE_TMUX=1
USE_CTAIL="no"
fi
fi
if [[ "$USE_TMUX" != "no" ]]; then
# kill the console window in the pane where the simulation is running
$TMUX kill-pane -t $TMUX_SESSION_NAME:sim.0

2
vendor/nim-chronicles-tail vendored

@ -1 +1 @@
Subproject commit 66e95a2ec6353dcb98d08aabdc46f6f0d1ea120b
Subproject commit 7064429aa541a7e08c8febd40470b30ae9c7c2e6

2
vendor/nim-eth vendored

@ -1 +1 @@
Subproject commit 4d0a7a46ba38947b8daecb1b5ae817c82c8e16c5
Subproject commit c13e59adaf988f9bb754dba05b0efa9176ffe664

2
vendor/nim-json-serialization vendored

@ -1 +1 @@
Subproject commit c478b7bbbab6ee298a25c29c7357783d94aaecaa
Subproject commit 6e92113a06224b942a6e00b4da08bb862ee67952

2
vendor/nim-serialization vendored

@ -1 +1 @@
Subproject commit d8ca3daf3a788f357bf96b8e61b3ffce9a018ea2
Subproject commit 5498b62dbd99afe0add0d763d94bbfad640e766b