Merge branch 'libp2p-testnet' into devel

.gitmodules

@@ -138,3 +138,7 @@
 	url = https://github.com/status-im/nim-nat-traversal.git
 	ignore = dirty
 	branch = master
+[submodule "vendor/news"]
+	path = vendor/news
+	url = https://github.com/tormund/news
+	branch = master

Makefile

@@ -105,11 +105,8 @@ endif
 #  for runtime path (i.e.: the second line in $(NIMBLE_DIR)/pkgs/*/*.nimble-link)
 $(NIMBLE_DIR): | $(NIM_BINARY)
 	mkdir -p $(NIMBLE_DIR)/pkgs
-	git submodule foreach --quiet '\
-		[ `ls -1 *.nimble 2>/dev/null | wc -l ` -gt 0 ] && { \
-			mkdir -p $$toplevel/$(NIMBLE_DIR)/pkgs/$${sm_path#*/}-#head;\
-			echo -e "$$($(PWD))\n$$($(PWD))" > $$toplevel/$(NIMBLE_DIR)/pkgs/$${sm_path#*/}-#head/$${sm_path#*/}.nimble-link;\
-		} || true'
+	NIMBLE_DIR="$(CURDIR)/$(NIMBLE_DIR)" PWD_CMD="$(PWD)" \
+		git submodule foreach --quiet 'bash $(CURDIR)/scripts/create_nimble_link.sh "$$sm_path"'
 
 # symlink
 nimbus.nims:

docker/beacon_node/Dockerfile

@@ -1,19 +1,30 @@
 FROM statusteam/nim-base AS build
 
 RUN apt update \
- && apt install -y build-essential make \
+ && apt install -y build-essential make wget \
  && apt clean \
  && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
+RUN export GO_TAR_GZ=go1.12.6.linux-amd64.tar.gz \
+ && cd /tmp/ \
+ && wget https://dl.google.com/go/$GO_TAR_GZ \
+ && tar -xvf $GO_TAR_GZ \
+ && mv go /usr/local \
+ && rm $GO_TAR_GZ
+
 ARG GIT_REVISION
 
-RUN git clone https://github.com/status-im/nimbus.git \
+RUN export GOROOT=/usr/local/go \
+ && export PATH=$GOROOT/bin:$PATH \
+ && git clone https://github.com/status-im/nimbus.git \
  && cd nimbus \
  && git reset --hard ${GIT_REVISION} \
- && make update deps nat-libs
+ && make update deps vendor/go/bin/p2pd nat-libs \
+ && cp vendor/go/bin/p2pd /usr/bin/p2pd \
+ && cp docker/beacon_node/run_in_docker.sh /usr/bin/run_beacon_node.sh
 
 ARG NETWORK
-ARG NETWORK_BACKEND
+ARG NETWORK_TYPE
 
 RUN cd nimbus \
  && set -a \
@@ -23,7 +34,7 @@ RUN cd nimbus \
       -d:release \
       --debugger:native \
       --debugInfo \
-      -d:with${NETWORK_BACKEND} \
+      -d:"network_type=${NETWORK_TYPE}" \
       -d:SHARD_COUNT=${SHARD_COUNT} \
       -d:SLOTS_PER_EPOCH=${SLOTS_PER_EPOCH} \
       -d:SECONDS_PER_SLOT=${SECONDS_PER_SLOT} \
@@ -41,9 +52,11 @@ RUN apt update \
  && apt clean \
  && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
-COPY --from=build /usr/bin/beacon_node /usr/bin/beacon_node 
+COPY --from=build /usr/bin/beacon_node /usr/bin/beacon_node
+COPY --from=build /usr/bin/run_beacon_node.sh /usr/bin/run_beacon_node.sh
+COPY --from=build /usr/bin/p2pd /usr/bin/p2pd
 
 MAINTAINER Zahary Karadjov <zahary@status.im>
 LABEL description="Nimbus installation that can act as an ETH2 network bootstrap node."
 
-ENTRYPOINT ["/usr/bin/beacon_node"]
+ENTRYPOINT ["/usr/bin/run_beacon_node.sh"]

docker/beacon_node/Makefile

@@ -2,17 +2,17 @@
 
 GIT_REVISION    ?= $(git rev-parse HEAD)
 
-NETWORK         ?= testnet0
-NETWORK_BACKEND ?= rlpx
+NETWORK         ?= testnet1
+NETWORK_TYPE    ?= libp2p_spec
 
-IMAGE_TAG       ?= $(NETWORK)-$(NETWORK_BACKEND)
+IMAGE_TAG       ?= $(NETWORK)
 IMAGE_NAME      ?= statusteam/nimbus_beacon_node:$(IMAGE_TAG)
 
 build: $(NIX_INSTALL)
 	docker build \
 		--build-arg="GIT_REVISION=$(GIT_REVISION)" \
 		--build-arg="NETWORK=$(NETWORK)" \
-		--build-arg="NETWORK_BACKEND=$(NETWORK_BACKEND)" \
+		--build-arg="NETWORK_TYPE=$(NETWORK_TYPE)" \
 		-t $(IMAGE_NAME) .
 
 push: build

docker/beacon_node/run_in_docker.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+killall p2pd
+rm -rf /tmp/*
+
+beacon_node $*
+

docker/build_beacon_node.sh

@@ -6,7 +6,7 @@ cd $(dirname "$0")
 
 export GIT_REVISION=$(git rev-parse HEAD)
 
-NETWORK=testnet0
+NETWORK=testnet1
 
 if [[ $(git rev-parse --abbrev-ref HEAD) == "devel" ]]; then
   NETWORK=testnet1
@@ -14,10 +14,11 @@ fi
 
 buildAndPush() {
   export NETWORK=$1
-  export NETWORK_BACKEND=$2
+  export NETWORK_TYPE=$2
 
   (cd beacon_node && make push)
 }
 
-buildAndPush $NETWORK rlpx
-#buildAndPush testnet0 libp2p
+# buildAndPush $NETWORK rlpx
+buildAndPush $NETWORK libp2p_spec
+

docker/hosts.nims

@@ -0,0 +1,44 @@
+import
+  strformat
+
+var
+  serverCount = 10
+  instancesCount = 2
+
+  totalValidators = 1000
+  userValidators = 200
+
+  systemValidators = totalValidators - userValidators
+  validatorsPerServer = systemValidators div serverCount
+  validatorsPerNode = validatorsPerServer div instancesCount
+
+  network = "testnet0"
+
+type
+  ContainerCmd = enum
+    restart
+    stop
+
+iterator containers: tuple[cmd: ContainerCmd, node, container: string, firstValidator, lastValidator: int] =
+  for i in 0 ..< serverCount:
+    let baseIdx = userValidators + i * validatorsPerServer
+    for j in 1 .. instancesCount:
+      let firstIdx = baseIdx + j * validatorsPerNode
+      let lastIdx = firstIdx + validatorsPerNode - 1
+      yield (restart, &"nimbus-node-0{i}", &"beacon-node-{network}-{j}", firstIdx, lastIdx)
+
+when false:
+  for i in 0 ..< serverCount:
+    let baseIdx = userValidators + i * validatorsPerServer
+    for j in 1 .. instancesCount:
+      let firstIdx = baseIdx + (j - 1) * validatorsPerNode
+      let lastIdx = firstIdx + validatorsPerNode - 1
+      let dockerPath = &"/docker/beacon-node-{network}-{j}/data/BeaconNode/{network}"
+      
+      echo &"ssh nimbus-node-0{i} 'sudo mkdir -p {dockerPath}/validators && sudo rm -f {dockerPath}/validators/* && " &
+                                 &"sudo ~/nimbus/vendor/nim-beacon-chain/scripts/download_validator_keys.sh {network} {firstIdx} {lastIdx} {dockerPath} && " &
+                                 &"sudo chown dockremap:docker -R {dockerPath}'"
+
+for c in containers():
+  echo &"ssh {c.node} docker {c.cmd} {c.container}"
+

scripts/create_nimble_link.sh

@@ -0,0 +1,13 @@
+set -u
+
+module_name="${1#*/}"
+
+if [ `ls -1 *.nimble 2>/dev/null | wc -l ` -gt 0 ]; then
+  mkdir -p "${NIMBLE_DIR}/pkgs/${module_name}-#head"
+  PKG_DIR="$(${PWD_CMD})"
+  if [ -d src ]; then
+    PKG_DIR="${PKG_DIR}/src"
+  fi
+  echo -e "${PKG_DIR}\n${PKG_DIR}" > "${NIMBLE_DIR}/pkgs/${module_name}-#head/${module_name}.nimble-link"
+fi
+

vendor/news

@@ -0,0 +1 @@
+Subproject commit ca34a894331c37bf71a43e0f0732df1cac01746d

vendor/nim-beacon-chain

@@ -1 +1 @@
-Subproject commit 10c7920b274f61e70531a5318e1ea0e4f61e65f2
+Subproject commit c7512851125030cb87871c34722a9143742e375a

vendor/nim-confutils

@@ -1 +1 @@
-Subproject commit da24be1a9d37aa2c5c5ae9b772282834d04beecf
+Subproject commit faca1407a5871c71a15df601703326c18ebefc0a

vendor/nim-eth

@@ -1 +1 @@
-Subproject commit d535bb1ade320d304cf1bfb17c4b4ec027d467c7
+Subproject commit 07484bfc88c2142df8073f6636af81179a261d2c

vendor/nim-faststreams

@@ -1 +1 @@
-Subproject commit 6e839bb5fa3373d23b8358652ee15ed05574d22f
+Subproject commit 1d693484f6815ba7706aac30657e4d9050ac5e2f

vendor/nim-json-rpc

@@ -1 +1 @@
-Subproject commit 707567c7ab775107a6e7eb47ff728e3596079d62
+Subproject commit 9d0326aad41c0efdc83b5612e76f7a493d82f27d

vendor/nim-json-serialization

@@ -1 +1 @@
-Subproject commit f3d8f822bddf1135233b82bd0f9b0f6495ce64ea
+Subproject commit a1baedeb1b52bc8016b6f2be519801d96f26da0d

vendor/nim-libp2p

@@ -1 +1 @@
-Subproject commit d685147c52b189d5da22e00c48b7fb4ea8a22ea8
+Subproject commit 3493b9ff7f02b34ea195893258a23c19e751c5f6

vendor/nim-serialization

@@ -1 +1 @@
-Subproject commit 6804ea25372de5919ef87cf13c7544770aacba2f
+Subproject commit 060c8fd9bf458e4710f139bdc923afbfdf13163d

vendor/nim-std-shims

@@ -1 +1 @@
-Subproject commit 58a313d8eb5750dc9b1c9f7f0bdebb08fb5023d3
+Subproject commit 7fa381eb8d12a21bd1c7a1b99217b4cf8acacc46