Merge pull request #460 from status-im/testnet-maintaince-scripts

Testnet maintaince scripts
2019-09-27 15:27:59 +02:00 · 2019-09-27 15:27:59 +02:00 · 73a8456d67
parent c1918d1253 df7531f9bf
commit 73a8456d67
13 changed files with 195 additions and 47 deletions
--- a/.gitignore
+++ b/.gitignore
@ -28,3 +28,8 @@ build/

 # State sim # TODO - move in another folder
 0000-*.json
+
+# Testnet resetting working directories
+testnet-reset-data/
+nim-eth2-testnet-data/
+
--- a/beacon_chain/beacon_node.nim
+++ b/beacon_chain/beacon_node.nim
@ -907,8 +907,8 @@ when isMainModule:
        totalValidators: config.totalValidators,
        lastUserValidator: config.lastUserValidator)

-    Json.saveFile(config.outputNetwork.string, testnetMetadata, pretty = true)
-    echo "Wrote ", config.outputNetwork.string
+    Json.saveFile(config.outputNetworkMetadata.string, testnetMetadata, pretty = true)
+    echo "Wrote ", config.outputNetworkMetadata.string

  of updateTestnet:
    discard waitFor updateTestnetMetadata(config)
@ -950,7 +950,7 @@ when isMainModule:

  of makeDeposits:
    let deposits = generateDeposits(
-      config.totalDeposits, config.depositDir, config.randomKeys)
+      config.totalDeposits, config.depositsDir, config.randomKeys)

    if config.depositWeb3Url.len() > 0 and config.depositContractAddress.len() > 0:
      waitFor sendDeposits(
--- a/beacon_chain/conf.nim
+++ b/beacon_chain/conf.nim
@ -134,12 +134,15 @@ type
      outputGenesis* {.
        desc: "Output file where to write the initial state snapshot".}: OutFile

-      outputNetwork* {.
+      outputNetworkMetadata* {.
        desc: "Output file where to write the initial state snapshot".}: OutFile

      withGenesisRoot* {.
        desc: "Include a genesis root in network.json", defaultValue: false.}: bool

+      outputBootstrapNodes* {.
+        desc: "Output file with list of bootstrap nodes for the network".}: OutFile
+
    of importValidator:
      keyFiles* {.
        longform: "keyfile"
@ -152,7 +155,7 @@ type
      totalDeposits* {.
        desc: "Total number of deposits and keys to generate".}: int

-      depositDir* {.
+      depositsDir* {.
        desc: "Folder to write deposits to", defaultValue: "validators".}: string

      randomKeys* {.
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@ -16,31 +16,39 @@ ARG GIT_REVISION

 RUN export GOROOT=/usr/local/go \
 && export PATH=$GOROOT/bin:$PATH \
- && git clone https://github.com/status-im/nimbus.git \
- && cd nimbus \
+ && git clone https://github.com/status-im/nim-beacon-chain.git \
+ && cd nim-beacon-chain \
 && git reset --hard ${GIT_REVISION} \
- && make update deps vendor/go/bin/p2pd nat-libs \
+ && git submodule update --init --recursive
+
+RUN cd nim-beacon-chain \
+ && make build-system-checks
+
+# TODO: The command above exits with 1, so we cannot chain it cleanly
+RUN cd nim-beacon-chain \
+ && export PATH=/usr/local/go/bin:$PATH \
+ && make update deps \
 && cp vendor/go/bin/p2pd /usr/bin/p2pd \
- && cp docker/beacon_node/run_in_docker.sh /usr/bin/run_beacon_node.sh
+ && cp docker/run_in_docker.sh /usr/bin/run_beacon_node.sh

 ARG NETWORK
 ARG NETWORK_TYPE

-RUN cd nimbus \
+RUN cd nim-beacon-chain \
 && set -a \
- && . vendor/nim-beacon-chain/scripts/${NETWORK}.env \
+ && . scripts/${NETWORK}.env \
 && ./env.sh nim \
      -o:/usr/bin/beacon_node \
      -d:release \
      --debugger:native \
      --debugInfo \
      -d:"network_type=${NETWORK_TYPE}" \
-      -d:SHARD_COUNT=${SHARD_COUNT} \
-      -d:SLOTS_PER_EPOCH=${SLOTS_PER_EPOCH} \
-      -d:SECONDS_PER_SLOT=${SECONDS_PER_SLOT} \
-      -d:chronicles_log_level=DEBUG \
-      -d:chronicles_sinks=json \
-      c vendor/nim-beacon-chain/beacon_chain/beacon_node.nim
+      -d:"SHARD_COUNT=${SHARD_COUNT}" \
+      -d:"SLOTS_PER_EPOCH=${SLOTS_PER_EPOCH}" \
+      -d:"SECONDS_PER_SLOT=${SECONDS_PER_SLOT}" \
+      -d:"chronicles_log_level=DEBUG" \
+      -d:"chronicles_sinks=json" \
+      c beacon_chain/beacon_node.nim

 # --------------------------------- #
 # Starting new image to reduce size #
@ -48,7 +56,7 @@ RUN cd nimbus \
 FROM debian:9-slim

 RUN apt update \
- && apt install -y librocksdb-dev curl \
+ && apt install -y librocksdb-dev curl psmisc \
 && apt clean \
 && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*

@ -59,4 +67,9 @@ COPY --from=build /usr/bin/p2pd /usr/bin/p2pd
 MAINTAINER Zahary Karadjov <zahary@status.im>
 LABEL description="Nimbus installation that can act as an ETH2 network bootstrap node."

+RUN echo Built from Git revision: ${GIT_REVISION}
+
+# TODO: This custom entry script is necessary only because we must clean up
+#       temporary files left by previous executions of the Go daeamon.
+#       We should be able to remove it once we have a native LibP2P impl.
 ENTRYPOINT ["/usr/bin/run_beacon_node.sh"]
--- a/docker/manage_testnet_hosts.nims
+++ b/docker/manage_testnet_hosts.nims
@ -0,0 +1,46 @@
+import
+  strformat
+
+var
+  serverCount = 10
+  instancesCount = 2
+
+  totalValidators = 1000
+  userValidators = 200
+
+  systemValidators = totalValidators - userValidators
+  validatorsPerServer = systemValidators div serverCount
+  validatorsPerNode = validatorsPerServer div instancesCount
+
+if paramCount() < 4:
+  echo "Usage: nim --verbosity:0 manage_testnet_hosts.nim NETWORK COMMAND"
+  quit 1
+
+let
+  network = paramStr(3)
+  cmd = paramStr(4)
+
+iterator nodes: tuple[server, container: string, firstValidator, lastValidator: int] =
+  for i in 0 ..< serverCount:
+    let
+      baseIdx = userValidators + i * validatorsPerServer
+      nodeName = if i == 0: "master-01" else: &"node-0{i}"
+      server = &"{nodeName}.do-ams3.nimbus.test.statusim.net"
+
+    for j in 1 .. instancesCount:
+      let firstIdx = baseIdx + j * validatorsPerNode
+      let lastIdx = firstIdx + validatorsPerNode - 1
+      yield (server, &"beacon-node-{network}-{j}", firstIdx, lastIdx)
+
+case cmd
+of "restart-nodes":
+  for n in nodes():
+    echo &"ssh {n.server} docker restart {n.container}"
+of "redist-validators":
+  for n in nodes():
+    let dockerPath = &"/docker/{n.container}/data/BeaconNode/{network}"
+    echo &"ssh {n.server} 'sudo mkdir -p {dockerPath}/validators && sudo rm -f {dockerPath}/validators/* && " &
+                         &"sudo ~/nimbus/vendor/nim-beacon-chain/scripts/download_validator_keys.sh {network} {n.firstValidator} {n.lastValidator} {dockerPath} && " &
+                         &"sudo chown dockremap:docker -R {dockerPath}'"
+else:
+  echo "Unrecognized command: ", cmd
--- a/docker/run_in_docker.sh
+++ b/docker/run_in_docker.sh
@ -1,7 +1,19 @@
+
 #!/bin/bash

+# Deal with previous execution of the deamon leaving behind
+# socket files that prevent the deamon from launching again
+# inside the container.
 killall p2pd
 rm -rf /tmp/*

-beacon_node --nat:none $*
-
+if [[ "$2" == "" ]]; then
+  # TODO This is a normal execution of a long-running testnet node.
+  # If the nat is enabled at the moment, the node fails to start.
+  beacon_node --nat:none
+else
+  # This is a one-off command such as createTestnet.
+  # We cannot reuse the command above, because the --nat option
+  # is not compatible with most of the commands.
+  beacon_node "$@"
+fi
--- a/multinet/make_genesis.sh
+++ b/multinet/make_genesis.sh
@ -34,7 +34,7 @@ LAST_VALIDATOR="$VALIDATORS_DIR/v$(printf '%07d' $LAST_VALIDATOR_NUM).deposit.js
 if [ ! -f "${LAST_VALIDATOR}" ]; then
  $BEACON_NODE_BIN makeDeposits \
    --totalDeposits="${NUM_VALIDATORS}" \
-    --depositDir="$VALIDATORS_DIR" \
+    --depositsDir="$VALIDATORS_DIR" \
    --randomKeys=false
 fi

@ -45,7 +45,8 @@ if [ ! -f "${SNAPSHOT_FILE}" ]; then
    --validatorsDir="${VALIDATORS_DIR}" \
    --totalValidators="${NUM_VALIDATORS}" \
    --outputGenesis="${SNAPSHOT_FILE}" \
-    --outputNetwork="${NETWORK_METADATA_FILE}" \
+    --outputNetworkMetadata="${NETWORK_METADATA_FILE}" \
+    --outputBootstrapNodes="${SIMULATION_DIR}/bootstrap_nodes.txt" \
    --bootstrapAddress=127.0.0.1 \
    --bootstrapPort=50000 \
    --genesisOffset=10 # Delay in seconds
--- a/scripts/README.md
+++ b/scripts/README.md
@ -21,7 +21,8 @@ The `beacon_node` binary has a `createTestnet` command.
    --totalValidators=$VALIDATOR_COUNT \
    --lastUserValidator=$LAST_USER_VALIDATOR \
    --outputGenesis=$NETWORK_DIR/genesis.json \
-    --outputNetwork=$NETWORK_DIR/network.json \
+    --outputNetworkMetadata=$NETWORK_DIR/network.json \
+    --outputBootstrapNodes=$NETWORK_DIR/bootstrap_nodes.txt \
    --bootstrapAddress=$PUBLIC_IP \
    --genesisOffset=600 # Delay in seconds
 ```
@ -29,3 +30,8 @@ The `beacon_node` binary has a `createTestnet` command.
 Replace ENV vars with values that make sense to you.

 Full tutorial coming soon.
+
+## Maintaining the Status testnets
+
+For detailed instructions, please see https://github.com/status-im/nimbus-private/blob/master/testnets-maintenance.md
+
--- a/scripts/build_testnet_node.sh
+++ b/scripts/build_testnet_node.sh
@ -14,7 +14,7 @@ cd ..

 # the NIM_PARAMS env var will be set in the Makefile, based on NIMFLAGS passed on the `make` command line
 # (i.e.: make NIMFLAGS="--stackTrace:on" testnet1).
-OUR_NIM_FLAGS="-d:release --lineTrace:on -d:chronicles_log_level=DEBUG -d:network_type=$NETWORK_TYPE -d:SECONDS_PER_SLOT=$SECONDS_PER_SLOT -d:SHARD_COUNT=$SHARD_COUNT -d:SLOTS_PER_EPOCH=$SLOTS_PER_EPOCH -d:DEFAULT_NETWORK=$NETWORK_NAME $NIM_PARAMS"
+OUR_NIM_FLAGS="-d:release --lineTrace:on -d:chronicles_log_level=DEBUG -d:const_preset=$CONST_PRESET -d:network_type=$NETWORK_TYPE -d:SECONDS_PER_SLOT=$SECONDS_PER_SLOT -d:SHARD_COUNT=$SHARD_COUNT -d:SLOTS_PER_EPOCH=$SLOTS_PER_EPOCH -d:DEFAULT_NETWORK=$NETWORK_NAME $NIM_PARAMS"

 BEACON_NODE_BIN="build/${NETWORK_NAME}_node"

--- a/scripts/reset_testnet.sh
+++ b/scripts/reset_testnet.sh
@ -10,34 +10,93 @@ source "$NETWORK_NAME.env"
 cd ..

 if [ -f .env ]; then
-  # allow server overrides for WWW_DIR and DATA_DIR
+  # allow server overrides for ETH2_TESTNET_DATA_DIR and DATA_DIR
  source .env
 fi

-PUBLIC_IP=$(curl -s ifconfig.me)
-NETWORK_DIR=$WWW_DIR/$NETWORK_NAME
+echo ${BOOTSTRAP_HOST:="master-01.do-ams3.nimbus.test.statusim.net"} > /dev/null

-NIM_FLAGS="-d:release -d:SECONDS_PER_SLOT=$SECONDS_PER_SLOT -d:SHARD_COUNT=$SHARD_COUNT -d:SLOTS_PER_EPOCH=$SLOTS_PER_EPOCH ${2:-}"
+echo Execution plan:

-nim c -d:"network_type=$NETWORK_TYPE" $NIM_FLAGS beacon_chain/beacon_node
+echo "Testnet name          : $NETWORK_NAME"
+echo "Testnet files repo    : ${ETH2_TESTNET_DATA_DIR:="nim-eth2-testnet-data"}"
+echo "Beacon node data dir  : ${DATA_DIR:="testnet-reset-data"}"
+echo "Bootstrap node ip     : ${BOOTSTRAP_IP:="$(dig +short $BOOTSTRAP_HOST)"}"
+echo "Reset testnet at end  : ${PUBLISH_TESTNET_RESETS:="1"}"

-if [ ! -f $NETWORK_DIR/genesis.json ]; then
-  rm -f $NETWORK_DIR/*
-  beacon_chain/beacon_node makeDeposits \
-    --totalDeposits=$VALIDATOR_COUNT \
-    --depositDir="$NETWORK_DIR" \
-    --randomKeys=true
+while true; do
+    read -p "Continue?" yn
+    case $yn in
+        [Yy]* ) break;;
+        [Nn]* ) exit 1;;
+        * ) echo "Please answer yes or no.";;
+    esac
+done
+
+if [[ ! -d "$ETH2_TESTNET_DATA_DIR"  ]]; then
+  git clone git@github.com:status-im/nim-eth2-testnet-data "$ETH2_TESTNET_DATA_DIR"
 fi

-beacon_chain/beacon_node \
+ETH2_TESTNET_DATA_DIR_ABS=$(cd "$ETH2_TESTNET_DATA_DIR"; pwd)
+DATA_DIR_ABS=$(mkdir -p "$DATA_DIR"; cd "$DATA_DIR"; pwd)
+NETWORK_DIR_ABS="$ETH2_TESTNET_DATA_DIR_ABS/www/$NETWORK_NAME"
+
+DOCKER_BEACON_NODE="docker run -v $NETWORK_DIR_ABS:/network_dir -v $DATA_DIR_ABS:/data_dir statusteam/nimbus_beacon_node:$NETWORK_NAME"
+
+cd docker
+
+export GIT_REVISION=$(git rev-parse HEAD)
+make build
+
+if [ ! -f $NETWORK_DIR_ABS/genesis.ssz ]; then
+  rm -f $NETWORK_DIR_ABS/*
+  $DOCKER_BEACON_NODE makeDeposits \
+    --totalDeposits=$VALIDATOR_COUNT \
+    --depositsDir=/network_dir \
+    --randomKeys=false
+fi
+
+$DOCKER_BEACON_NODE \
  --network=$NETWORK_NAME \
-  --dataDir=$DATA_DIR/node-0 \
+  --dataDir=/data_dir \
  createTestnet \
-  --validatorsDir=$NETWORK_DIR \
+  --validatorsDir=/network_dir \
  --totalValidators=$VALIDATOR_COUNT \
  --lastUserValidator=$LAST_USER_VALIDATOR \
-  --outputGenesis=$NETWORK_DIR/genesis.json \
-  --outputNetwork=$NETWORK_DIR/network.json \
-  --bootstrapAddress=$PUBLIC_IP \
+  --outputGenesis=/network_dir/genesis.json \
+  --outputBootstrapNodes=/network_dir/bootstrap_nodes.txt \
+  --outputNetworkMetadata=/network_dir/network.json \
+  --bootstrapAddress=$BOOTSTRAP_IP \
  --bootstrapPort=$BOOTSTRAP_PORT \
-  --genesisOffset=600 # Delay in seconds
+  --genesisOffset=60 # Delay in seconds
+
+if [[ $PUBLISH_TESTNET_RESETS != "0" ]]; then
+  echo Persisting testnet data to git...
+  pushd "$ETH2_TESTNET_DATA_DIR_ABS"
+    git add --all
+    git commit -m "Testnet reset"
+    git push
+  popd
+
+  echo Updating https://serenity-testnets.status.im/${NETWORK_NAME}...
+  ssh $BOOTSTRAP_HOST <<-SSH
+    cd /opt/nim-eth2-testnet-data
+    git reset --hard HEAD
+    git checkout master
+    git pull
+SSH
+
+  echo Redistributing validator keys to server nodes...
+  # TODO If we try to use direct piping here, bash doesn't execute all of the commands.
+  #      The reasons for this are unclear at the moment.
+  nim --verbosity:0 manage_testnet_hosts.nims $NETWORK_NAME redist-validators > /tmp/reset-network.sh
+  bash /tmp/reset-network.sh
+
+  echo Uploading bootstrap node network key
+  BOOTSTRAP_NODE_DOCKER_PATH=/docker/beacon-node-$NETWORK_NAME-1/data/BeaconNode/$NETWORK_NAME/
+  scp "$DATA_DIR_ABS/privkey.protobuf" $BOOTSTRAP_HOST:/tmp/
+  ssh $BOOTSTRAP_HOST "sudo install -o dockremap -g docker /tmp/privkey.protobuf $BOOTSTRAP_NODE_DOCKER_PATH"
+
+  echo Publishing docker image...
+  make push
+fi
--- a/scripts/testnet0.env
+++ b/scripts/testnet0.env
@ -1,3 +1,4 @@
+CONST_PRESET=minimal
 NETWORK_ID=10
 NETWORK_TYPE=rlpx
 SHARD_COUNT=16
--- a/scripts/testnet1.env
+++ b/scripts/testnet1.env
@ -1,8 +1,9 @@
+CONST_PRESET=minimal
 NETWORK_ID=20
 NETWORK_TYPE=libp2p
-SHARD_COUNT=16
-SLOTS_PER_EPOCH=16
-SECONDS_PER_SLOT=30
+SHARD_COUNT=8
+SLOTS_PER_EPOCH=8
+SECONDS_PER_SLOT=6
 VALIDATOR_COUNT=1000
 LAST_USER_VALIDATOR=199
 BOOTSTRAP_PORT=9100
--- a/tests/simulation/start.sh
+++ b/tests/simulation/start.sh
@ -45,7 +45,7 @@ if [ ! -f "${LAST_VALIDATOR}" ]; then

  $BEACON_NODE_BIN makeDeposits \
    --totalDeposits="${NUM_VALIDATORS}" \
-    --depositDir="$VALIDATORS_DIR" \
+    --depositsDir="$VALIDATORS_DIR" \
    --randomKeys=false \
    $DEPOSIT_WEB3_URL_ARG \
    --depositContractAddress="${DEPOSIT_CONTRACT_ADDRESS}"
@ -58,7 +58,8 @@ if [ ! -f "${SNAPSHOT_FILE}" ]; then
    --validatorsDir="${VALIDATORS_DIR}" \
    --totalValidators="${NUM_VALIDATORS}" \
    --outputGenesis="${SNAPSHOT_FILE}" \
-    --outputNetwork="${NETWORK_METADATA_FILE}" \
+    --outputNetworkMetadata="${NETWORK_METADATA_FILE}" \
+    --outputBootstrapNodes="${SIMULATION_DIR}/bootstrap_nodes.txt" \
    --bootstrapAddress=127.0.0.1 \
    --bootstrapPort=50000 \
    --genesisOffset=5 # Delay in seconds