diff --git a/.gitmodules b/.gitmodules index bf918097e..9f19c1eb8 100644 --- a/.gitmodules +++ b/.gitmodules @@ -138,3 +138,7 @@ url = https://github.com/status-im/nim-nat-traversal.git ignore = dirty branch = master +[submodule "vendor/news"] + path = vendor/news + url = https://github.com/tormund/news + branch = master diff --git a/Makefile b/Makefile index 8ef08864d..9f9b63157 100644 --- a/Makefile +++ b/Makefile @@ -105,11 +105,8 @@ endif # for runtime path (i.e.: the second line in $(NIMBLE_DIR)/pkgs/*/*.nimble-link) $(NIMBLE_DIR): | $(NIM_BINARY) mkdir -p $(NIMBLE_DIR)/pkgs - git submodule foreach --quiet '\ - [ `ls -1 *.nimble 2>/dev/null | wc -l ` -gt 0 ] && { \ - mkdir -p $$toplevel/$(NIMBLE_DIR)/pkgs/$${sm_path#*/}-#head;\ - echo -e "$$($(PWD))\n$$($(PWD))" > $$toplevel/$(NIMBLE_DIR)/pkgs/$${sm_path#*/}-#head/$${sm_path#*/}.nimble-link;\ - } || true' + NIMBLE_DIR="$(CURDIR)/$(NIMBLE_DIR)" PWD_CMD="$(PWD)" \ + git submodule foreach --quiet 'bash $(CURDIR)/scripts/create_nimble_link.sh "$$sm_path"' # symlink nimbus.nims: diff --git a/docker/beacon_node/Dockerfile b/docker/beacon_node/Dockerfile index 2e48bef09..d3b5a2ed8 100644 --- a/docker/beacon_node/Dockerfile +++ b/docker/beacon_node/Dockerfile @@ -1,19 +1,30 @@ FROM statusteam/nim-base AS build RUN apt update \ - && apt install -y build-essential make \ + && apt install -y build-essential make wget \ && apt clean \ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* +RUN export GO_TAR_GZ=go1.12.6.linux-amd64.tar.gz \ + && cd /tmp/ \ + && wget https://dl.google.com/go/$GO_TAR_GZ \ + && tar -xvf $GO_TAR_GZ \ + && mv go /usr/local \ + && rm $GO_TAR_GZ + ARG GIT_REVISION -RUN git clone https://github.com/status-im/nimbus.git \ +RUN export GOROOT=/usr/local/go \ + && export PATH=$GOROOT/bin:$PATH \ + && git clone https://github.com/status-im/nimbus.git \ && cd nimbus \ && git reset --hard ${GIT_REVISION} \ - && make update deps nat-libs + && make update deps vendor/go/bin/p2pd nat-libs \ + && cp vendor/go/bin/p2pd /usr/bin/p2pd \ + && cp docker/beacon_node/run_in_docker.sh /usr/bin/run_beacon_node.sh ARG NETWORK -ARG NETWORK_BACKEND +ARG NETWORK_TYPE RUN cd nimbus \ && set -a \ @@ -23,7 +34,7 @@ RUN cd nimbus \ -d:release \ --debugger:native \ --debugInfo \ - -d:with${NETWORK_BACKEND} \ + -d:"network_type=${NETWORK_TYPE}" \ -d:SHARD_COUNT=${SHARD_COUNT} \ -d:SLOTS_PER_EPOCH=${SLOTS_PER_EPOCH} \ -d:SECONDS_PER_SLOT=${SECONDS_PER_SLOT} \ @@ -41,9 +52,11 @@ RUN apt update \ && apt clean \ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* -COPY --from=build /usr/bin/beacon_node /usr/bin/beacon_node +COPY --from=build /usr/bin/beacon_node /usr/bin/beacon_node +COPY --from=build /usr/bin/run_beacon_node.sh /usr/bin/run_beacon_node.sh +COPY --from=build /usr/bin/p2pd /usr/bin/p2pd MAINTAINER Zahary Karadjov LABEL description="Nimbus installation that can act as an ETH2 network bootstrap node." -ENTRYPOINT ["/usr/bin/beacon_node"] +ENTRYPOINT ["/usr/bin/run_beacon_node.sh"] diff --git a/docker/beacon_node/Makefile b/docker/beacon_node/Makefile index fb0d4a009..62193f2e4 100644 --- a/docker/beacon_node/Makefile +++ b/docker/beacon_node/Makefile @@ -2,17 +2,17 @@ GIT_REVISION ?= $(git rev-parse HEAD) -NETWORK ?= testnet0 -NETWORK_BACKEND ?= rlpx +NETWORK ?= testnet1 +NETWORK_TYPE ?= libp2p_spec -IMAGE_TAG ?= $(NETWORK)-$(NETWORK_BACKEND) +IMAGE_TAG ?= $(NETWORK) IMAGE_NAME ?= statusteam/nimbus_beacon_node:$(IMAGE_TAG) build: $(NIX_INSTALL) docker build \ --build-arg="GIT_REVISION=$(GIT_REVISION)" \ --build-arg="NETWORK=$(NETWORK)" \ - --build-arg="NETWORK_BACKEND=$(NETWORK_BACKEND)" \ + --build-arg="NETWORK_TYPE=$(NETWORK_TYPE)" \ -t $(IMAGE_NAME) . push: build diff --git a/docker/beacon_node/run_in_docker.sh b/docker/beacon_node/run_in_docker.sh new file mode 100755 index 000000000..16ec78a0a --- /dev/null +++ b/docker/beacon_node/run_in_docker.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +killall p2pd +rm -rf /tmp/* + +beacon_node $* + diff --git a/docker/build_beacon_node.sh b/docker/build_beacon_node.sh index 812fea74b..fd2186e8d 100755 --- a/docker/build_beacon_node.sh +++ b/docker/build_beacon_node.sh @@ -6,7 +6,7 @@ cd $(dirname "$0") export GIT_REVISION=$(git rev-parse HEAD) -NETWORK=testnet0 +NETWORK=testnet1 if [[ $(git rev-parse --abbrev-ref HEAD) == "devel" ]]; then NETWORK=testnet1 @@ -14,10 +14,11 @@ fi buildAndPush() { export NETWORK=$1 - export NETWORK_BACKEND=$2 + export NETWORK_TYPE=$2 (cd beacon_node && make push) } -buildAndPush $NETWORK rlpx -#buildAndPush testnet0 libp2p +# buildAndPush $NETWORK rlpx +buildAndPush $NETWORK libp2p_spec + diff --git a/docker/hosts.nims b/docker/hosts.nims new file mode 100644 index 000000000..b652fd9d0 --- /dev/null +++ b/docker/hosts.nims @@ -0,0 +1,44 @@ +import + strformat + +var + serverCount = 10 + instancesCount = 2 + + totalValidators = 1000 + userValidators = 200 + + systemValidators = totalValidators - userValidators + validatorsPerServer = systemValidators div serverCount + validatorsPerNode = validatorsPerServer div instancesCount + + network = "testnet0" + +type + ContainerCmd = enum + restart + stop + +iterator containers: tuple[cmd: ContainerCmd, node, container: string, firstValidator, lastValidator: int] = + for i in 0 ..< serverCount: + let baseIdx = userValidators + i * validatorsPerServer + for j in 1 .. instancesCount: + let firstIdx = baseIdx + j * validatorsPerNode + let lastIdx = firstIdx + validatorsPerNode - 1 + yield (restart, &"nimbus-node-0{i}", &"beacon-node-{network}-{j}", firstIdx, lastIdx) + +when false: + for i in 0 ..< serverCount: + let baseIdx = userValidators + i * validatorsPerServer + for j in 1 .. instancesCount: + let firstIdx = baseIdx + (j - 1) * validatorsPerNode + let lastIdx = firstIdx + validatorsPerNode - 1 + let dockerPath = &"/docker/beacon-node-{network}-{j}/data/BeaconNode/{network}" + + echo &"ssh nimbus-node-0{i} 'sudo mkdir -p {dockerPath}/validators && sudo rm -f {dockerPath}/validators/* && " & + &"sudo ~/nimbus/vendor/nim-beacon-chain/scripts/download_validator_keys.sh {network} {firstIdx} {lastIdx} {dockerPath} && " & + &"sudo chown dockremap:docker -R {dockerPath}'" + +for c in containers(): + echo &"ssh {c.node} docker {c.cmd} {c.container}" + diff --git a/scripts/create_nimble_link.sh b/scripts/create_nimble_link.sh new file mode 100644 index 000000000..84745207d --- /dev/null +++ b/scripts/create_nimble_link.sh @@ -0,0 +1,13 @@ +set -u + +module_name="${1#*/}" + +if [ `ls -1 *.nimble 2>/dev/null | wc -l ` -gt 0 ]; then + mkdir -p "${NIMBLE_DIR}/pkgs/${module_name}-#head" + PKG_DIR="$(${PWD_CMD})" + if [ -d src ]; then + PKG_DIR="${PKG_DIR}/src" + fi + echo -e "${PKG_DIR}\n${PKG_DIR}" > "${NIMBLE_DIR}/pkgs/${module_name}-#head/${module_name}.nimble-link" +fi + diff --git a/vendor/news b/vendor/news new file mode 160000 index 000000000..ca34a8943 --- /dev/null +++ b/vendor/news @@ -0,0 +1 @@ +Subproject commit ca34a894331c37bf71a43e0f0732df1cac01746d diff --git a/vendor/nim-beacon-chain b/vendor/nim-beacon-chain index 10c7920b2..c75128511 160000 --- a/vendor/nim-beacon-chain +++ b/vendor/nim-beacon-chain @@ -1 +1 @@ -Subproject commit 10c7920b274f61e70531a5318e1ea0e4f61e65f2 +Subproject commit c7512851125030cb87871c34722a9143742e375a diff --git a/vendor/nim-confutils b/vendor/nim-confutils index da24be1a9..faca1407a 160000 --- a/vendor/nim-confutils +++ b/vendor/nim-confutils @@ -1 +1 @@ -Subproject commit da24be1a9d37aa2c5c5ae9b772282834d04beecf +Subproject commit faca1407a5871c71a15df601703326c18ebefc0a diff --git a/vendor/nim-eth b/vendor/nim-eth index d535bb1ad..07484bfc8 160000 --- a/vendor/nim-eth +++ b/vendor/nim-eth @@ -1 +1 @@ -Subproject commit d535bb1ade320d304cf1bfb17c4b4ec027d467c7 +Subproject commit 07484bfc88c2142df8073f6636af81179a261d2c diff --git a/vendor/nim-faststreams b/vendor/nim-faststreams index 6e839bb5f..1d693484f 160000 --- a/vendor/nim-faststreams +++ b/vendor/nim-faststreams @@ -1 +1 @@ -Subproject commit 6e839bb5fa3373d23b8358652ee15ed05574d22f +Subproject commit 1d693484f6815ba7706aac30657e4d9050ac5e2f diff --git a/vendor/nim-json-rpc b/vendor/nim-json-rpc index 707567c7a..9d0326aad 160000 --- a/vendor/nim-json-rpc +++ b/vendor/nim-json-rpc @@ -1 +1 @@ -Subproject commit 707567c7ab775107a6e7eb47ff728e3596079d62 +Subproject commit 9d0326aad41c0efdc83b5612e76f7a493d82f27d diff --git a/vendor/nim-json-serialization b/vendor/nim-json-serialization index f3d8f822b..a1baedeb1 160000 --- a/vendor/nim-json-serialization +++ b/vendor/nim-json-serialization @@ -1 +1 @@ -Subproject commit f3d8f822bddf1135233b82bd0f9b0f6495ce64ea +Subproject commit a1baedeb1b52bc8016b6f2be519801d96f26da0d diff --git a/vendor/nim-libp2p b/vendor/nim-libp2p index d685147c5..3493b9ff7 160000 --- a/vendor/nim-libp2p +++ b/vendor/nim-libp2p @@ -1 +1 @@ -Subproject commit d685147c52b189d5da22e00c48b7fb4ea8a22ea8 +Subproject commit 3493b9ff7f02b34ea195893258a23c19e751c5f6 diff --git a/vendor/nim-serialization b/vendor/nim-serialization index 6804ea253..060c8fd9b 160000 --- a/vendor/nim-serialization +++ b/vendor/nim-serialization @@ -1 +1 @@ -Subproject commit 6804ea25372de5919ef87cf13c7544770aacba2f +Subproject commit 060c8fd9bf458e4710f139bdc923afbfdf13163d diff --git a/vendor/nim-std-shims b/vendor/nim-std-shims index 58a313d8e..7fa381eb8 160000 --- a/vendor/nim-std-shims +++ b/vendor/nim-std-shims @@ -1 +1 @@ -Subproject commit 58a313d8eb5750dc9b1c9f7f0bdebb08fb5023d3 +Subproject commit 7fa381eb8d12a21bd1c7a1b99217b4cf8acacc46