From 13068c6b57db654cc4623c7bee99581a9c14eb8b Mon Sep 17 00:00:00 2001 From: Zahary Karadjov Date: Tue, 5 Nov 2019 13:00:11 +0000 Subject: [PATCH] Safer testnet resets * Delete the node database on all containers when resetting a testnet * Add a simple mechanism for switching the URL of the eth2-testnets repo * More flexible code for validator assignments --- docker/manage_testnet_hosts.nims | 64 ++++++++++++++++++++++---------- scripts/connect_to_testnet.nims | 8 ++-- scripts/reset_testnet.sh | 20 +++++----- 3 files changed, 60 insertions(+), 32 deletions(-) diff --git a/docker/manage_testnet_hosts.nims b/docker/manage_testnet_hosts.nims index 41ff9dff8..f3f21312e 100644 --- a/docker/manage_testnet_hosts.nims +++ b/docker/manage_testnet_hosts.nims @@ -4,7 +4,7 @@ import type Command = enum restart_nodes - redist_validators + reset_network CliConfig = object network: string @@ -13,14 +13,20 @@ type of restart_nodes: discard - of redist_validators: + of reset_network: depositsDir {. + defaultValue: "deposits" longform: "deposits-dir" }: string + networkDataDir {. + defaultValue: "data" longform: "network-data-dir"}: string + totalValidators {. longform: "total-validators" }: int + totalUserValidators {. + defaultValue: 0 longform: "user-validators" }: int var conf = load CliConfig @@ -28,37 +34,53 @@ var conf = load CliConfig var serverCount = 10 instancesCount = 2 - systemValidators = conf.totalValidators - conf.totalUserValidators - validatorsPerServer = systemValidators div serverCount - validatorsPerNode = validatorsPerServer div instancesCount + +let customValidatorAssignments = { + "testnet0": proc (nodeIdx: int): int = + if nodeidx < 2: + systemValidators div 2 + else: + 0 +} + +proc findOrDefault[K, V](tupleList: openarray[(K, V)], key: K, default: V): V = + for t in tupleList: + if t[0] == key: + return t[1] + + return default + +let defaultValidatorAssignment = proc (nodeIdx: int): int = + (systemValidators div serverCount) div instancesCount iterator nodes: tuple[server, container: string, firstValidator, lastValidator: int] = + var nextValidatorIdx = conf.totalUserValidators for i in 0 ..< serverCount: let - baseIdx = conf.totalUserValidators + i * validatorsPerServer nodeName = if i == 0: "master-01" else: &"node-0{i}" server = &"{nodeName}.do-ams3.nimbus.test.statusim.net" for j in 0 ..< instancesCount: - var firstIdx, lastIdx: int - if conf.network == "testnet0" and i == 0 and j == 0: - firstIdx = conf.totalUserValidators - lastIdx = conf.totalValidators - elif true: - firstIdx = 0 - lastIdx = 0 - else: - firstIdx = baseIdx + j * validatorsPerNode - lastIdx = firstIdx + validatorsPerNode - yield (server, &"beacon-node-{conf.network}-{j+1}", firstIdx, lastIdx) + let + globalNodeIdx = i*instancesCount + j + validatorAssignmentFn = customValidatorAssignments.findOrDefault( + conf.network, defaultValidatorAssignment) + nodeValidatorCount = validatorAssignmentFn(globalNodeIdx) + + yield (server, + &"beacon-node-{conf.network}-{j+1}", + nextValidatorIdx, + nextValidatorIdx + nodeValidatorCount) + + inc nextValidatorIdx, nodeValidatorCount case conf.cmd of restart_nodes: for n in nodes(): echo &"ssh {n.server} docker restart {n.container}" -of redist_validators: +of reset_network: for n in nodes(): var keysList = "" @@ -74,7 +96,11 @@ of redist_validators: echo &" ssh {n.server} 'sudo rm -rf /tmp/nimbus && mkdir -p /tmp/nimbus/' && \\" echo &" rsync {networkDataFiles} {n.server}:/tmp/nimbus/net-data/ && \\" if keysList.len > 0: echo &" rsync {keysList} {n.server}:/tmp/nimbus/keys/ && \\" - echo &" ssh {n.server} 'sudo mkdir -p {dockerPath}/validators && sudo rm -f {dockerPath}/validators/* && " & + + echo &" ssh {n.server} 'sudo docker container stop {n.container} && " & + &"sudo mkdir -p {dockerPath}/validators && " & + &"sudo rm -f {dockerPath}/validators/* && " & + &"sudo rm -f {dockerPath}/db" & (if keysList.len > 0: &"sudo mv /tmp/nimbus/keys/* {dockerPath}/validators/ && " else: "") & &"sudo mv /tmp/nimbus/net-data/* {dockerPath}/ && " & &"sudo chown dockremap:docker -R {dockerPath}'" diff --git a/scripts/connect_to_testnet.nims b/scripts/connect_to_testnet.nims index ba0c8493b..579698700 100644 --- a/scripts/connect_to_testnet.nims +++ b/scripts/connect_to_testnet.nims @@ -7,9 +7,11 @@ const depositContractFile = "deposit_contract.txt" genesisFile = "genesis.ssz" configFile = "config.yaml" - clientsOrg = "eth2-clients" testnetsRepo = "eth2-testnets" - testnetsRepoGitUrl = "https://github.com/" & clientsOrg & "/" & testnetsRepo + +let + testnetsOrg = getEnv("ETH2_TESTNETS_ORG", "eth2-testnets") + testnetsGitUrl = getEnv("ETH2_TESTNETS_GIT_URL", "https://github.com/" & testnetsOrg & "/" & testnetsRepo) proc validateTestnetName(parts: openarray[string]): auto = if parts.len != 2: @@ -29,7 +31,7 @@ cli do (testnetName {.argument.}: string): rmDir(allTestnetsDir) cd buildDir - exec &"git clone --quiet {testnetsRepoGitUrl}" + exec &"git clone --quiet --depth=1 {testnetsGitUrl}" let testnetDir = allTestnetsDir / team / testnet if not dirExists(testnetDir): diff --git a/scripts/reset_testnet.sh b/scripts/reset_testnet.sh index 8e1d35b60..cd238956e 100755 --- a/scripts/reset_testnet.sh +++ b/scripts/reset_testnet.sh @@ -14,15 +14,15 @@ if [ -f .env ]; then source .env fi -echo ${BOOTSTRAP_HOST:="master-01.do-ams3.nimbus.test.statusim.net"} > /dev/null - echo Execution plan: -echo "Testnet name : $NETWORK_NAME" -echo "Testnet files repo : ${ETH2_TESTNETS:="build/eth2-testnets"}" -echo "Beacon node data dir : ${DATA_DIR:="build/testnet-reset-data"}" -echo "Bootstrap node ip : ${BOOTSTRAP_IP:="$(dig +short $BOOTSTRAP_HOST)"}" -echo "Reset testnet at end : ${PUBLISH_TESTNET_RESETS:="1"}" +echo "Testnet name : $NETWORK_NAME" +echo "Bootstrap node hostname : ${BOOTSTRAP_HOST:="master-01.do-ams3.nimbus.test.statusim.net"}" +echo "Bootstrap node ip : ${BOOTSTRAP_IP:="$(dig +short $BOOTSTRAP_HOST)"}" +echo "Reset testnet at end : ${PUBLISH_TESTNET_RESETS:="1"}" +echo "Testnet metadata repo : ${ETH2_TESTNETS_GIT_URL:="git@github.com:${ETH2_TESTNETS_ORG:=eth2-clients}/eth2-testnets"}" +echo "Testnet metadata dir : ${ETH2_TESTNETS:="build/eth2-testnets"}" +echo "Beacon node data dir : ${DATA_DIR:="build/testnet-reset-data"}" while true; do read -p "Continue?" yn @@ -34,8 +34,7 @@ while true; do done rm -rf "$ETH2_TESTNETS" -# "--depth 1" does not make it faster, for such a small repo -git clone --quiet git@github.com:eth2-clients/eth2-testnets "$ETH2_TESTNETS" +git clone --quiet --depth=1 $ETH2_TESTNETS_GIT_URL "$ETH2_TESTNETS" ETH2_TESTNETS_ABS=$(cd "$ETH2_TESTNETS"; pwd) NETWORK_DIR_ABS="$ETH2_TESTNETS_ABS/nimbus/$NETWORK_NAME" @@ -98,7 +97,8 @@ if [[ $PUBLISH_TESTNET_RESETS != "0" ]]; then echo Redistributing validator keys to server nodes... # TODO If we try to use direct piping here, bash doesn't execute all of the commands. # The reasons for this are unclear at the moment. - nim --verbosity:0 manage_testnet_hosts.nims redist_validators \ + + nim --verbosity:0 manage_testnet_hosts.nims reset_network \ --network=$NETWORK_NAME \ --deposits-dir="$DEPOSITS_DIR_ABS" \ --network-data-dir="$NETWORK_DIR_ABS" \