Merge pull request #460 from status-im/testnet-maintaince-scripts

Testnet maintaince scripts

.gitignore

@@ -28,3 +28,8 @@ build/
 
 # State sim # TODO - move in another folder
 0000-*.json
+
+# Testnet resetting working directories
+testnet-reset-data/
+nim-eth2-testnet-data/
+

beacon_chain/beacon_node.nim

@@ -907,8 +907,8 @@ when isMainModule:
         totalValidators: config.totalValidators,
         lastUserValidator: config.lastUserValidator)
 
-    Json.saveFile(config.outputNetwork.string, testnetMetadata, pretty = true)
+    Json.saveFile(config.outputNetworkMetadata.string, testnetMetadata, pretty = true)
-    echo "Wrote ", config.outputNetwork.string
+    echo "Wrote ", config.outputNetworkMetadata.string
 
   of updateTestnet:
     discard waitFor updateTestnetMetadata(config)
@@ -950,7 +950,7 @@ when isMainModule:
 
   of makeDeposits:
     let deposits = generateDeposits(
-      config.totalDeposits, config.depositDir, config.randomKeys)
+      config.totalDeposits, config.depositsDir, config.randomKeys)
 
     if config.depositWeb3Url.len() > 0 and config.depositContractAddress.len() > 0:
       waitFor sendDeposits(

beacon_chain/conf.nim

@@ -134,12 +134,15 @@ type
       outputGenesis* {.
         desc: "Output file where to write the initial state snapshot".}: OutFile
 
-      outputNetwork* {.
+      outputNetworkMetadata* {.
         desc: "Output file where to write the initial state snapshot".}: OutFile
 
       withGenesisRoot* {.
         desc: "Include a genesis root in network.json", defaultValue: false.}: bool
 
+      outputBootstrapNodes* {.
+        desc: "Output file with list of bootstrap nodes for the network".}: OutFile
+
     of importValidator:
       keyFiles* {.
         longform: "keyfile"
@@ -152,7 +155,7 @@ type
       totalDeposits* {.
         desc: "Total number of deposits and keys to generate".}: int
 
-      depositDir* {.
+      depositsDir* {.
         desc: "Folder to write deposits to", defaultValue: "validators".}: string
 
       randomKeys* {.

docker/Dockerfile

@@ -16,31 +16,39 @@ ARG GIT_REVISION
 
 RUN export GOROOT=/usr/local/go \
  && export PATH=$GOROOT/bin:$PATH \
- && git clone https://github.com/status-im/nimbus.git \
+ && git clone https://github.com/status-im/nim-beacon-chain.git \
- && cd nimbus \
+ && cd nim-beacon-chain \
  && git reset --hard ${GIT_REVISION} \
- && make update deps vendor/go/bin/p2pd nat-libs \
+ && git submodule update --init --recursive
+
+RUN cd nim-beacon-chain \
+ && make build-system-checks
+
+# TODO: The command above exits with 1, so we cannot chain it cleanly
+RUN cd nim-beacon-chain \
+ && export PATH=/usr/local/go/bin:$PATH \
+ && make update deps \
  && cp vendor/go/bin/p2pd /usr/bin/p2pd \
- && cp docker/beacon_node/run_in_docker.sh /usr/bin/run_beacon_node.sh
+ && cp docker/run_in_docker.sh /usr/bin/run_beacon_node.sh
 
 ARG NETWORK
 ARG NETWORK_TYPE
 
-RUN cd nimbus \
+RUN cd nim-beacon-chain \
  && set -a \
- && . vendor/nim-beacon-chain/scripts/${NETWORK}.env \
+ && . scripts/${NETWORK}.env \
  && ./env.sh nim \
       -o:/usr/bin/beacon_node \
       -d:release \
       --debugger:native \
       --debugInfo \
       -d:"network_type=${NETWORK_TYPE}" \
-      -d:SHARD_COUNT=${SHARD_COUNT} \
+      -d:"SHARD_COUNT=${SHARD_COUNT}" \
-      -d:SLOTS_PER_EPOCH=${SLOTS_PER_EPOCH} \
+      -d:"SLOTS_PER_EPOCH=${SLOTS_PER_EPOCH}" \
-      -d:SECONDS_PER_SLOT=${SECONDS_PER_SLOT} \
+      -d:"SECONDS_PER_SLOT=${SECONDS_PER_SLOT}" \
-      -d:chronicles_log_level=DEBUG \
+      -d:"chronicles_log_level=DEBUG" \
-      -d:chronicles_sinks=json \
+      -d:"chronicles_sinks=json" \
-      c vendor/nim-beacon-chain/beacon_chain/beacon_node.nim
+      c beacon_chain/beacon_node.nim
 
 # --------------------------------- #
 # Starting new image to reduce size #
@@ -48,7 +56,7 @@ RUN cd nimbus \
 FROM debian:9-slim
 
 RUN apt update \
- && apt install -y librocksdb-dev curl \
+ && apt install -y librocksdb-dev curl psmisc \
  && apt clean \
  && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
@@ -59,4 +67,9 @@ COPY --from=build /usr/bin/p2pd /usr/bin/p2pd
 MAINTAINER Zahary Karadjov <zahary@status.im>
 LABEL description="Nimbus installation that can act as an ETH2 network bootstrap node."
 
+RUN echo Built from Git revision: ${GIT_REVISION}
+
+# TODO: This custom entry script is necessary only because we must clean up
+#       temporary files left by previous executions of the Go daeamon.
+#       We should be able to remove it once we have a native LibP2P impl.
 ENTRYPOINT ["/usr/bin/run_beacon_node.sh"]

docker/manage_testnet_hosts.nims

@@ -0,0 +1,46 @@
+import
+  strformat
+
+var
+  serverCount = 10
+  instancesCount = 2
+
+  totalValidators = 1000
+  userValidators = 200
+
+  systemValidators = totalValidators - userValidators
+  validatorsPerServer = systemValidators div serverCount
+  validatorsPerNode = validatorsPerServer div instancesCount
+
+if paramCount() < 4:
+  echo "Usage: nim --verbosity:0 manage_testnet_hosts.nim NETWORK COMMAND"
+  quit 1
+
+let
+  network = paramStr(3)
+  cmd = paramStr(4)
+
+iterator nodes: tuple[server, container: string, firstValidator, lastValidator: int] =
+  for i in 0 ..< serverCount:
+    let
+      baseIdx = userValidators + i * validatorsPerServer
+      nodeName = if i == 0: "master-01" else: &"node-0{i}"
+      server = &"{nodeName}.do-ams3.nimbus.test.statusim.net"
+
+    for j in 1 .. instancesCount:
+      let firstIdx = baseIdx + j * validatorsPerNode
+      let lastIdx = firstIdx + validatorsPerNode - 1
+      yield (server, &"beacon-node-{network}-{j}", firstIdx, lastIdx)
+
+case cmd
+of "restart-nodes":
+  for n in nodes():
+    echo &"ssh {n.server} docker restart {n.container}"
+of "redist-validators":
+  for n in nodes():
+    let dockerPath = &"/docker/{n.container}/data/BeaconNode/{network}"
+    echo &"ssh {n.server} 'sudo mkdir -p {dockerPath}/validators && sudo rm -f {dockerPath}/validators/* && " &
+                         &"sudo ~/nimbus/vendor/nim-beacon-chain/scripts/download_validator_keys.sh {network} {n.firstValidator} {n.lastValidator} {dockerPath} && " &
+                         &"sudo chown dockremap:docker -R {dockerPath}'"
+else:
+  echo "Unrecognized command: ", cmd

docker/run_in_docker.sh

@@ -1,7 +1,19 @@
+
 #!/bin/bash
 
+# Deal with previous execution of the deamon leaving behind
+# socket files that prevent the deamon from launching again
+# inside the container.
 killall p2pd
 rm -rf /tmp/*
 
-beacon_node --nat:none $*
+if [[ "$2" == "" ]]; then
-
+  # TODO This is a normal execution of a long-running testnet node.
+  # If the nat is enabled at the moment, the node fails to start.
+  beacon_node --nat:none
+else
+  # This is a one-off command such as createTestnet.
+  # We cannot reuse the command above, because the --nat option
+  # is not compatible with most of the commands.
+  beacon_node "$@"
+fi

multinet/make_genesis.sh

@@ -34,7 +34,7 @@ LAST_VALIDATOR="$VALIDATORS_DIR/v$(printf '%07d' $LAST_VALIDATOR_NUM).deposit.js
 if [ ! -f "${LAST_VALIDATOR}" ]; then
   $BEACON_NODE_BIN makeDeposits \
     --totalDeposits="${NUM_VALIDATORS}" \
-    --depositDir="$VALIDATORS_DIR" \
+    --depositsDir="$VALIDATORS_DIR" \
     --randomKeys=false
 fi
 
@@ -45,7 +45,8 @@ if [ ! -f "${SNAPSHOT_FILE}" ]; then
     --validatorsDir="${VALIDATORS_DIR}" \
     --totalValidators="${NUM_VALIDATORS}" \
     --outputGenesis="${SNAPSHOT_FILE}" \
-    --outputNetwork="${NETWORK_METADATA_FILE}" \
+    --outputNetworkMetadata="${NETWORK_METADATA_FILE}" \
+    --outputBootstrapNodes="${SIMULATION_DIR}/bootstrap_nodes.txt" \
     --bootstrapAddress=127.0.0.1 \
     --bootstrapPort=50000 \
     --genesisOffset=10 # Delay in seconds

scripts/README.md

@@ -21,7 +21,8 @@ The `beacon_node` binary has a `createTestnet` command.
     --totalValidators=$VALIDATOR_COUNT \
     --lastUserValidator=$LAST_USER_VALIDATOR \
     --outputGenesis=$NETWORK_DIR/genesis.json \
-    --outputNetwork=$NETWORK_DIR/network.json \
+    --outputNetworkMetadata=$NETWORK_DIR/network.json \
+    --outputBootstrapNodes=$NETWORK_DIR/bootstrap_nodes.txt \
     --bootstrapAddress=$PUBLIC_IP \
     --genesisOffset=600 # Delay in seconds
 ```
@@ -29,3 +30,8 @@ The `beacon_node` binary has a `createTestnet` command.
 Replace ENV vars with values that make sense to you.
 
 Full tutorial coming soon.
+
+## Maintaining the Status testnets
+
+For detailed instructions, please see https://github.com/status-im/nimbus-private/blob/master/testnets-maintenance.md
+

scripts/build_testnet_node.sh

@@ -14,7 +14,7 @@ cd ..
 
 # the NIM_PARAMS env var will be set in the Makefile, based on NIMFLAGS passed on the `make` command line
 # (i.e.: make NIMFLAGS="--stackTrace:on" testnet1).
-OUR_NIM_FLAGS="-d:release --lineTrace:on -d:chronicles_log_level=DEBUG -d:network_type=$NETWORK_TYPE -d:SECONDS_PER_SLOT=$SECONDS_PER_SLOT -d:SHARD_COUNT=$SHARD_COUNT -d:SLOTS_PER_EPOCH=$SLOTS_PER_EPOCH -d:DEFAULT_NETWORK=$NETWORK_NAME $NIM_PARAMS"
+OUR_NIM_FLAGS="-d:release --lineTrace:on -d:chronicles_log_level=DEBUG -d:const_preset=$CONST_PRESET -d:network_type=$NETWORK_TYPE -d:SECONDS_PER_SLOT=$SECONDS_PER_SLOT -d:SHARD_COUNT=$SHARD_COUNT -d:SLOTS_PER_EPOCH=$SLOTS_PER_EPOCH -d:DEFAULT_NETWORK=$NETWORK_NAME $NIM_PARAMS"
 
 BEACON_NODE_BIN="build/${NETWORK_NAME}_node"
 

scripts/reset_testnet.sh

@@ -10,34 +10,93 @@ source "$NETWORK_NAME.env"
 cd ..
 
 if [ -f .env ]; then
-  # allow server overrides for WWW_DIR and DATA_DIR
+  # allow server overrides for ETH2_TESTNET_DATA_DIR and DATA_DIR
   source .env
 fi
 
-PUBLIC_IP=$(curl -s ifconfig.me)
+echo ${BOOTSTRAP_HOST:="master-01.do-ams3.nimbus.test.statusim.net"} > /dev/null
-NETWORK_DIR=$WWW_DIR/$NETWORK_NAME
 
-NIM_FLAGS="-d:release -d:SECONDS_PER_SLOT=$SECONDS_PER_SLOT -d:SHARD_COUNT=$SHARD_COUNT -d:SLOTS_PER_EPOCH=$SLOTS_PER_EPOCH ${2:-}"
+echo Execution plan:
 
-nim c -d:"network_type=$NETWORK_TYPE" $NIM_FLAGS beacon_chain/beacon_node
+echo "Testnet name          : $NETWORK_NAME"
+echo "Testnet files repo    : ${ETH2_TESTNET_DATA_DIR:="nim-eth2-testnet-data"}"
+echo "Beacon node data dir  : ${DATA_DIR:="testnet-reset-data"}"
+echo "Bootstrap node ip     : ${BOOTSTRAP_IP:="$(dig +short $BOOTSTRAP_HOST)"}"
+echo "Reset testnet at end  : ${PUBLISH_TESTNET_RESETS:="1"}"
 
-if [ ! -f $NETWORK_DIR/genesis.json ]; then
+while true; do
-  rm -f $NETWORK_DIR/*
+    read -p "Continue?" yn
-  beacon_chain/beacon_node makeDeposits \
+    case $yn in
-    --totalDeposits=$VALIDATOR_COUNT \
+        [Yy]* ) break;;
-    --depositDir="$NETWORK_DIR" \
+        [Nn]* ) exit 1;;
-    --randomKeys=true
+        * ) echo "Please answer yes or no.";;
+    esac
+done
+
+if [[ ! -d "$ETH2_TESTNET_DATA_DIR"  ]]; then
+  git clone git@github.com:status-im/nim-eth2-testnet-data "$ETH2_TESTNET_DATA_DIR"
 fi
 
-beacon_chain/beacon_node \
+ETH2_TESTNET_DATA_DIR_ABS=$(cd "$ETH2_TESTNET_DATA_DIR"; pwd)
+DATA_DIR_ABS=$(mkdir -p "$DATA_DIR"; cd "$DATA_DIR"; pwd)
+NETWORK_DIR_ABS="$ETH2_TESTNET_DATA_DIR_ABS/www/$NETWORK_NAME"
+
+DOCKER_BEACON_NODE="docker run -v $NETWORK_DIR_ABS:/network_dir -v $DATA_DIR_ABS:/data_dir statusteam/nimbus_beacon_node:$NETWORK_NAME"
+
+cd docker
+
+export GIT_REVISION=$(git rev-parse HEAD)
+make build
+
+if [ ! -f $NETWORK_DIR_ABS/genesis.ssz ]; then
+  rm -f $NETWORK_DIR_ABS/*
+  $DOCKER_BEACON_NODE makeDeposits \
+    --totalDeposits=$VALIDATOR_COUNT \
+    --depositsDir=/network_dir \
+    --randomKeys=false
+fi
+
+$DOCKER_BEACON_NODE \
   --network=$NETWORK_NAME \
-  --dataDir=$DATA_DIR/node-0 \
+  --dataDir=/data_dir \
   createTestnet \
-  --validatorsDir=$NETWORK_DIR \
+  --validatorsDir=/network_dir \
   --totalValidators=$VALIDATOR_COUNT \
   --lastUserValidator=$LAST_USER_VALIDATOR \
-  --outputGenesis=$NETWORK_DIR/genesis.json \
+  --outputGenesis=/network_dir/genesis.json \
-  --outputNetwork=$NETWORK_DIR/network.json \
+  --outputBootstrapNodes=/network_dir/bootstrap_nodes.txt \
-  --bootstrapAddress=$PUBLIC_IP \
+  --outputNetworkMetadata=/network_dir/network.json \
+  --bootstrapAddress=$BOOTSTRAP_IP \
   --bootstrapPort=$BOOTSTRAP_PORT \
-  --genesisOffset=600 # Delay in seconds
+  --genesisOffset=60 # Delay in seconds
+
+if [[ $PUBLISH_TESTNET_RESETS != "0" ]]; then
+  echo Persisting testnet data to git...
+  pushd "$ETH2_TESTNET_DATA_DIR_ABS"
+    git add --all
+    git commit -m "Testnet reset"
+    git push
+  popd
+
+  echo Updating https://serenity-testnets.status.im/${NETWORK_NAME}...
+  ssh $BOOTSTRAP_HOST <<-SSH
+    cd /opt/nim-eth2-testnet-data
+    git reset --hard HEAD
+    git checkout master
+    git pull
+SSH
+
+  echo Redistributing validator keys to server nodes...
+  # TODO If we try to use direct piping here, bash doesn't execute all of the commands.
+  #      The reasons for this are unclear at the moment.
+  nim --verbosity:0 manage_testnet_hosts.nims $NETWORK_NAME redist-validators > /tmp/reset-network.sh
+  bash /tmp/reset-network.sh
+
+  echo Uploading bootstrap node network key
+  BOOTSTRAP_NODE_DOCKER_PATH=/docker/beacon-node-$NETWORK_NAME-1/data/BeaconNode/$NETWORK_NAME/
+  scp "$DATA_DIR_ABS/privkey.protobuf" $BOOTSTRAP_HOST:/tmp/
+  ssh $BOOTSTRAP_HOST "sudo install -o dockremap -g docker /tmp/privkey.protobuf $BOOTSTRAP_NODE_DOCKER_PATH"
+
+  echo Publishing docker image...
+  make push
+fi

scripts/testnet0.env

@@ -1,3 +1,4 @@
+CONST_PRESET=minimal
 NETWORK_ID=10
 NETWORK_TYPE=rlpx
 SHARD_COUNT=16

scripts/testnet1.env

@@ -1,8 +1,9 @@
+CONST_PRESET=minimal
 NETWORK_ID=20
 NETWORK_TYPE=libp2p
-SHARD_COUNT=16
+SHARD_COUNT=8
-SLOTS_PER_EPOCH=16
+SLOTS_PER_EPOCH=8
-SECONDS_PER_SLOT=30
+SECONDS_PER_SLOT=6
 VALIDATOR_COUNT=1000
 LAST_USER_VALIDATOR=199
 BOOTSTRAP_PORT=9100

tests/simulation/start.sh

@@ -45,7 +45,7 @@ if [ ! -f "${LAST_VALIDATOR}" ]; then
 
   $BEACON_NODE_BIN makeDeposits \
     --totalDeposits="${NUM_VALIDATORS}" \
-    --depositDir="$VALIDATORS_DIR" \
+    --depositsDir="$VALIDATORS_DIR" \
     --randomKeys=false \
     $DEPOSIT_WEB3_URL_ARG \
     --depositContractAddress="${DEPOSIT_CONTRACT_ADDRESS}"
@@ -58,7 +58,8 @@ if [ ! -f "${SNAPSHOT_FILE}" ]; then
     --validatorsDir="${VALIDATORS_DIR}" \
     --totalValidators="${NUM_VALIDATORS}" \
     --outputGenesis="${SNAPSHOT_FILE}" \
-    --outputNetwork="${NETWORK_METADATA_FILE}" \
+    --outputNetworkMetadata="${NETWORK_METADATA_FILE}" \
+    --outputBootstrapNodes="${SIMULATION_DIR}/bootstrap_nodes.txt" \
     --bootstrapAddress=127.0.0.1 \
     --bootstrapPort=50000 \
     --genesisOffset=5 # Delay in seconds