status-im
/
nimbus-eth2
mirror of https://github.com/status-im/nimbus-eth2.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Windows binary release 

CI: use both cores on GitHub Actions and set timeouts for the local testnet tests
This commit is contained in:
Ștefan Talpalaru 2021-02-02 23:31:01 +01:00 committed by zah
parent 67e4a045a3
commit 80c11546ff
21 changed files with 228 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
.github/workflows/ci.yml vendored
View File

@ -122,6 +122,21 @@ jobs:
echo "CFLAGS=$CFLAGS" >> $GITHUB_ENV
fi
ncpu=
case '${{ runner.os }}' in
'Linux')
ncpu=$(nproc)
;;
'macOS')
ncpu=$(sysctl -n hw.ncpu)
;;
'Windows')
ncpu=$NUMBER_OF_PROCESSORS
;;
esac
[[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
echo "ncpu=$ncpu" >> $GITHUB_ENV
- name: Install build dependencies (Linux i386)
if: runner.os == 'Linux' && matrix.target.cpu == 'i386'
run: |
@ -218,23 +233,6 @@ jobs:
shell: bash
working-directory: nim-beacon-chain
run: |
ncpu=
ext=
case '${{ runner.os }}' in
'Linux')
ncpu=$(nproc)
;;
'macOS')
ncpu=$(sysctl -n hw.ncpu)
;;
'Windows')
ncpu=$NUMBER_OF_PROCESSORS
ext=.exe
;;
esac
[[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
echo "::set-output name=ncpu::$ncpu"
make -j$ncpu ARCH_OVERRIDE=$PLATFORM CI_CACHE=NimBinaries V=1 update
- name: Get latest fixtures commit hash
@ -284,11 +282,11 @@ jobs:
shell: bash
working-directory: nim-beacon-chain
run: |
./scripts/launch_local_testnet.sh --testnet 0 --nodes 4 --stop-at-epoch 5 --log-level DEBUG --disable-htop --enable-logtrace --data-dir local_testnet0_data --base-port 9000 --base-rpc-port 7000 --base-metrics-port 8008 -- --verify-finalization --discv5:no
./scripts/launch_local_testnet.sh --testnet 0 --nodes 4 --stop-at-epoch 5 --log-level DEBUG --disable-htop --enable-logtrace --data-dir local_testnet0_data --base-port 9000 --base-rpc-port 7000 --base-metrics-port 8008 --timeout 600 -- --verify-finalization --discv5:no
- name: Run nim-beacon-chain testnet1 (mainnet)
if: matrix.target.TEST_KIND == 'finalization-mainnet'
shell: bash
working-directory: nim-beacon-chain
run: |
./scripts/launch_local_testnet.sh --testnet 1 --nodes 4 --stop-at-epoch 5 --log-level DEBUG --disable-htop --enable-logtrace --data-dir local_testnet0_data --base-port 9000 --base-rpc-port 7000 --base-metrics-port 8008 -- --verify-finalization --discv5:no
./scripts/launch_local_testnet.sh --testnet 1 --nodes 4 --stop-at-epoch 5 --log-level DEBUG --disable-htop --enable-logtrace --data-dir local_testnet0_data --base-port 9000 --base-rpc-port 7000 --base-metrics-port 8008 --timeout 2400 -- --verify-finalization --discv5:no

36
.github/workflows/release.yml vendored
View File

@ -7,7 +7,7 @@ name: Upload Release Asset
jobs:
build-amd64:
name: AMD64 release asset
name: Linux AMD64 release asset
runs-on: ubuntu-latest
steps:
- name: Create release
@ -54,7 +54,7 @@ jobs:
docker push statusim/nimbus-eth2:amd64-${TAG}
docker push statusim/nimbus-eth2:amd64-latest
build-arm64:
name: ARM64 release asset
name: Linux ARM64 release asset
runs-on: ubuntu-latest
steps:
- name: Install packages
@ -91,7 +91,7 @@ jobs:
asset_name: ${{ steps.make_dist.outputs.linux_arm64_archive }}
asset_content_type: application/gzip
build-arm:
name: ARM release asset
name: Linux ARM release asset
runs-on: ubuntu-latest
steps:
- name: Install packages
@ -127,4 +127,34 @@ jobs:
asset_path: ./dist/${{ steps.make_dist.outputs.linux_arm_archive }}
asset_name: ${{ steps.make_dist.outputs.linux_arm_archive }}
asset_content_type: application/gzip
build-win64:
name: Windows AMD64 release asset
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v2
- name: Build project
id: make_dist
run: |
make dist-win64
cd dist
echo "::set-output name=windows_amd64_archive::"$(echo nimbus-eth2_Windows_amd64_*.tar.gz)
- name: Fetch latest release
id: fetch_release
uses: InsonusK/get-latest-release@v1.0.1
with:
myToken: ${{ github.token }}
view_top: 1
- name: Check release version
run: |
[[ "refs/tags/${{ steps.fetch_release.outputs.tag_name }}" == "${{ github.ref }}" ]]
- name: Upload release asset
uses: actions/upload-release-asset@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: https://uploads.github.com/repos/${{ github.repository }}/releases/${{ steps.fetch_release.outputs.id }}/assets{?name,label}
asset_path: ./dist/${{ steps.make_dist.outputs.windows_amd64_archive }}
asset_name: ${{ steps.make_dist.outputs.windows_amd64_archive }}
asset_content_type: application/gzip

8
Makefile
View File

@ -1,4 +1,4 @@
# Copyright (c) 2019-2020 Status Research & Development GmbH. Licensed under
# Copyright (c) 2019-2021 Status Research & Development GmbH. Licensed under
# either of:
# - Apache License, version 2.0
# - MIT license
@ -81,6 +81,7 @@ TOOLS_CSV := $(subst $(SPACE),$(COMMA),$(TOOLS))
dist-amd64 \
dist-arm64 \
dist-arm \
dist-win64 \
dist \
benchmarks
@ -396,10 +397,15 @@ dist-arm:
MAKE="$(MAKE)" \
scripts/make_dist.sh arm
dist-win64:
MAKE="$(MAKE)" \
scripts/make_dist.sh win64
dist:
$(MAKE) dist-amd64
$(MAKE) dist-arm64
$(MAKE) dist-arm
$(MAKE) dist-win64
#- this simple test will show any missing dynamically-linked Glibc symbols in the target distro
dist-test:

30
beacon_chain/network_metadata.nim
View File

@ -1,3 +1,9 @@
# Copyright (c) 2020-2021 Status Research & Development GmbH
# Licensed and distributed under either of
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
# at your option. This file may not be copied, modified, or distributed except according to those terms.
import
tables, strutils, os,
stew/shims/macros, nimcrypto/hash,
@ -68,7 +74,7 @@ type
incompatibilityDesc*: string
const
eth2testnetsDir = currentSourcePath.parentDir / ".." / "vendor" / "eth2-testnets"
eth2testnetsDir = currentSourcePath.parentDir.replace('\\', '/') & "/../vendor/eth2-testnets"
const presetValueLoaders = genExpr(nnkBracket):
for constName in PresetValue:
@ -107,12 +113,12 @@ proc loadEth2NetworkMetadata*(path: string): Eth2NetworkMetadata
{.raises: [CatchableError, Defect].} =
try:
let
genesisPath = path / "genesis.ssz"
genesisDepositsSnapshotPath = path / "genesis_deposit_contract_snapshot.ssz"
configPath = path / "config.yaml"
depositContractPath = path / "deposit_contract.txt"
depositContractBlockPath = path / "deposit_contract_block.txt"
bootstrapNodesPath = path / "bootstrap_nodes.txt"
genesisPath = path & "/genesis.ssz"
genesisDepositsSnapshotPath = path & "/genesis_deposit_contract_snapshot.ssz"
configPath = path & "/config.yaml"
depositContractPath = path & "/deposit_contract.txt"
depositContractBlockPath = path & "/deposit_contract_block.txt"
bootstrapNodesPath = path & "/bootstrap_nodes.txt"
runtimePreset = if fileExists(configPath):
extractRuntimePreset(configPath, readPresetFile(configPath))
@ -164,7 +170,7 @@ proc loadEth2NetworkMetadata*(path: string): Eth2NetworkMetadata
incompatibilityDesc: err.msg)
const
mainnetMetadataDir = eth2testnetsDir / "shared" / "mainnet"
mainnetMetadataDir = eth2testnetsDir & "/shared/mainnet"
mainnetMetadata* = when const_preset == "mainnet":
Eth2NetworkMetadata(
@ -172,11 +178,11 @@ const
# that there are no constant overrides
eth1Network: some mainnet,
runtimePreset: mainnetRuntimePreset,
bootstrapNodes: readFile(mainnetMetadataDir / "bootstrap_nodes.txt").splitLines,
bootstrapNodes: readFile(mainnetMetadataDir & "/bootstrap_nodes.txt").splitLines,
depositContractAddress: Eth1Address.fromHex "0x00000000219ab540356cBB839Cbe05303d7705Fa",
depositContractDeployedAt: BlockHashOrNumber.init "11052984",
genesisData: readFile(mainnetMetadataDir / "genesis.ssz"),
genesisDepositsSnapshot: readFile(mainnetMetadataDir / "genesis_deposit_contract_snapshot.ssz"))
genesisData: readFile(mainnetMetadataDir & "/genesis.ssz"),
genesisDepositsSnapshot: readFile(mainnetMetadataDir & "/genesis_deposit_contract_snapshot.ssz"))
else:
Eth2NetworkMetadata(
incompatible: true,
@ -184,7 +190,7 @@ const
"It's not compatible with mainnet")
template eth2testnet(path: string): Eth2NetworkMetadata =
loadEth2NetworkMetadata(eth2testnetsDir / path)
loadEth2NetworkMetadata(eth2testnetsDir & "/" & path)
const
pyrmontMetadata* = eth2testnet "shared/pyrmont"

2
beacon_chain/spec/keystore.nim
View File

@ -217,7 +217,7 @@ macro wordListArray*(filename: static string,
minWordLen: static int = 0,
maxWordLen: static int = high(int)): untyped =
result = newTree(nnkBracket)
var words = slurp(filename).splitLines()
var words = slurp(filename.replace('\\', '/')).splitLines()
for word in words:
if word.len >= minWordLen and word.len <= maxWordLen:
result.add newCall("cstring", newLit(word))

18
docker/dist/Dockerfile.win64 vendored Normal file
View File

@ -0,0 +1,18 @@
# The build is reproducible only if this base image stays the same.
FROM statusteam/nimbus_beacon_node:dist_base_20210202230118_win64@sha256:47b8ea60a35bd45355f0490677c04a9b42df56b5041cbf64da8ad233511a55d5
SHELL ["/bin/bash", "-c"]
ARG USER_ID
ARG GROUP_ID
RUN addgroup --gid ${GROUP_ID} user; \
adduser --disabled-password --gecos '' --uid ${USER_ID} --gid ${GROUP_ID} user;
USER user
STOPSIGNAL SIGINT
COPY "entry_point.sh" "/home/user/"
ENTRYPOINT ["/home/user/entry_point.sh", "Windows_amd64"]

7
docker/dist/README-Windows.md vendored Normal file
View File

@ -0,0 +1,7 @@
# Windows-specific requirements
Run the wrapper scripts from a [Git for Windows](https://gitforwindows.org/) Bash shell.
If you run the beacon node binary directly, prefix it with "winpty -- ". It
will increase the chance of Ctrl+C working inside that "mintty" terminal emulator.

23
docker/dist/base_image/Dockerfile.win64 vendored Normal file
View File

@ -0,0 +1,23 @@
# This Docker image can change from one build to another, because the upstream
# Debian/Ubuntu package index is continuously updated and we have to run
# `apt-get update` in here.
#
# The only way to make this a part of our reproducible build system is to build
# it once, upload it to Docker Hub and make sure it's being pulled regularly so
# it's not deleted after 6 months of inactivity.
FROM ubuntu:20.04
SHELL ["/bin/bash", "-c"]
ENV DEBIAN_FRONTEND=noninteractive TZ="Etc/UTC"
RUN \
apt-get -qq update \
&& apt-get -qq -y install git gnupg software-properties-common lsb &>/dev/null \
&& apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 86B72ED9 \
&& add-apt-repository "deb [arch=amd64] https://pkg.mxe.cc/repos/apt $(lsb_release -sc) main" \
&& apt-get -qq update \
&& apt-get -qq -y install mxe-x86-64-w64-mingw32.static-cc &>/dev/null \
&& apt-get -qq clean \
&& rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*

10
docker/dist/base_image/Makefile vendored
View File

@ -7,9 +7,11 @@ IMAGE_NAME := statusteam/nimbus_beacon_node:$(IMAGE_TAG)
build-amd64 \
build-arm64 \
build-arm \
build-win64 \
push-amd64 \
push-arm64 \
push-arm
push-arm \
push-win64
build-amd64:
$(CURDIR)/make_base_image.sh amd64 "$(IMAGE_NAME)"
@ -20,6 +22,9 @@ build-arm64:
build-arm:
$(CURDIR)/make_base_image.sh arm "$(IMAGE_NAME)_arm"
build-win64:
$(CURDIR)/make_base_image.sh win64 "$(IMAGE_NAME)_win64"
# You probably don't want to recreate and push these base images to Docker Hub,
# because when older images expire and get deleted, it will no longer be possible
# to reproduce old releases.
@ -33,3 +38,6 @@ build-arm:
#push-arm: build-arm
# docker push $(IMAGE_NAME)_arm
#push-win64: build-win64
# docker push $(IMAGE_NAME)_win64

6
docker/dist/base_image/make_base_image.sh vendored
View File

@ -1,6 +1,6 @@
#!/bin/bash
# Copyright (c) 2020 Status Research & Development GmbH. Licensed under
# Copyright (c) 2020-2021 Status Research & Development GmbH. Licensed under
# either of:
# - Apache License, version 2.0
# - MIT license
@ -23,7 +23,7 @@ fi
ARCH="${1}"
DOCKER_TAG="${2}"
if [[ "${ARCH}" == "amd64" ]]; then
if [[ "${ARCH}" == "amd64" || "${ARCH}" == "win64" ]]; then
USE_QEMU=0
else
USE_QEMU=1
@ -63,8 +63,6 @@ DOCKER_BUILDKIT=1 \
docker build \
-t ${DOCKER_TAG} \
--progress=plain \
--build-arg USER_ID=$(id -u) \
--build-arg GROUP_ID=$(id -g) \
${DOCKER_EXTRA_ARGS} \
-f Dockerfile.${ARCH} .

51
docker/dist/entry_point.sh vendored
View File

@ -1,5 +1,12 @@
#!/bin/bash
# Copyright (c) 2020-2021 Status Research & Development GmbH. Licensed under
# either of:
# - Apache License, version 2.0
# - MIT license
# at your option. This file may not be copied, modified, or distributed except
# according to those terms.
set -e
cd /home/user/nimbus-eth2
@ -15,12 +22,46 @@ BINARIES="nimbus_beacon_node nimbus_signing_process"
#- we need to build everything against libraries available inside this container, including the Nim compiler
#- we disable the log file and log colours; the user only has to worry about logging stdout now
make clean
if [[ "${PLATFORM}" == "Windows_amd64" ]]; then
# Cross-compilation using the MXE distribution of Mingw-w64
export PATH="/usr/lib/mxe/usr/bin:${PATH}"
make \
-j$(nproc) \
USE_LIBBACKTRACE=0 \
deps
make \
-C vendor/nim-nat-traversal/vendor/miniupnp/miniupnpc \
-f Makefile.mingw \
clean &>/dev/null
make \
-j$(nproc) \
-C vendor/nim-nat-traversal/vendor/miniupnp/miniupnpc \
-f Makefile.mingw \
CC=x86_64-w64-mingw32.static-gcc \
libminiupnpc.a &>/dev/null
make \
-C vendor/nim-nat-traversal/vendor/libnatpmp-upstream \
clean &>/dev/null
make \
-j$(nproc) \
-C vendor/nim-nat-traversal/vendor/libnatpmp-upstream \
CC=x86_64-w64-mingw32.static-gcc \
CFLAGS="-Wall -Os -DWIN32 -DNATPMP_STATICLIB -DENABLE_STRNATPMPERR -DNATPMP_MAX_RETRIES=4 ${CFLAGS}" \
libnatpmp.a &>/dev/null
make \
-j$(nproc) \
USE_LIBBACKTRACE=0 \
LOG_LEVEL="TRACE" \
NIMFLAGS="-d:disableMarchNative -d:chronicles_sinks=textlines -d:chronicles_colors=none --os:windows --gcc.exe=x86_64-w64-mingw32.static-gcc --gcc.linkerexe=x86_64-w64-mingw32.static-gcc --passL:-static" \
${BINARIES}
else
make \
-j$(nproc) \
LOG_LEVEL="TRACE" \
NIMFLAGS="-d:disableMarchNative -d:chronicles_sinks=textlines -d:chronicles_colors=none" \
PARTIAL_STATIC_LINKING=1 \
${BINARIES}
fi
# archive directory (we need the Nim compiler in here)
PREFIX="nimbus-eth2_${PLATFORM}_"
@ -40,12 +81,18 @@ mkdir "${DIST_PATH}/build"
# copy and checksum binaries, copy scripts and docs
for BINARY in ${BINARIES}; do
cp -a ./build/${BINARY} "${DIST_PATH}/build/"
cp -a "./build/${BINARY}" "${DIST_PATH}/build/"
cd "${DIST_PATH}/build"
sha512sum ${BINARY} > ${BINARY}.sha512sum
sha512sum "${BINARY}" > "${BINARY}.sha512sum"
if [[ "${PLATFORM}" == "Windows_amd64" ]]; then
mv "${BINARY}" "${BINARY}.exe"
fi
cd - >/dev/null
done
sed -e "s/GIT_COMMIT/${GIT_COMMIT}/" docker/dist/README.md > "${DIST_PATH}/README.md"
if [[ "${PLATFORM}" == "Windows_amd64" ]]; then
cp -a docker/dist/README-Windows.md "${DIST_PATH}/"
fi
cp -a scripts/run-beacon-node.sh "${DIST_PATH}/scripts"
cp -a ./run-*-beacon-node.sh "${DIST_PATH}/"

7
run-mainnet-beacon-node.sh
View File

@ -1,5 +1,12 @@
#!/bin/bash
# Copyright (c) 2020-2021 Status Research & Development GmbH. Licensed under
# either of:
# - Apache License, version 2.0
# - MIT license
# at your option. This file may not be copied, modified, or distributed except
# according to those terms.
cd "$(dirname $0)"
# Allow the binary to receive signals directly.
exec scripts/run-beacon-node.sh nimbus_beacon_node mainnet $@

7
run-pyrmont-beacon-node.sh
View File

@ -1,5 +1,12 @@
#!/bin/bash
# Copyright (c) 2020-2021 Status Research & Development GmbH. Licensed under
# either of:
# - Apache License, version 2.0
# - MIT license
# at your option. This file may not be copied, modified, or distributed except
# according to those terms.
cd "$(dirname $0)"
# Allow the binary to receive signals directly.
exec scripts/run-beacon-node.sh nimbus_beacon_node pyrmont $@

4
scripts/make_dist.sh
View File

@ -1,6 +1,6 @@
#!/bin/bash
# Copyright (c) 2020 Status Research & Development GmbH. Licensed under
# Copyright (c) 2020-2021 Status Research & Development GmbH. Licensed under
# either of:
# - Apache License, version 2.0
# - MIT license
@ -16,7 +16,7 @@ cd "$(dirname "${BASH_SOURCE[0]}")"/..
CURDIR="${PWD}"
ARCH="${1:-amd64}"
if [[ "${ARCH}" == "amd64" ]]; then
if [[ "${ARCH}" == "amd64" || "${ARCH}" == "win64" ]]; then
USE_QEMU=0
else
USE_QEMU=1

11
scripts/run-beacon-node.sh
View File

@ -1,5 +1,12 @@
#!/bin/bash
# Copyright (c) 2020-2021 Status Research & Development GmbH. Licensed under
# either of:
# - Apache License, version 2.0
# - MIT license
# at your option. This file may not be copied, modified, or distributed except
# according to those terms.
set -e
cd "$(dirname $0)/.."
@ -32,8 +39,10 @@ fi
# Windows detection
if uname | grep -qiE "mingw|msys"; then
MAKE="mingw32-make"
WINPTY="winpty --"
else
MAKE="make"
WINPTY=""
fi
if [[ ! -f build/${NBC_BINARY} ]]; then
@ -66,7 +75,7 @@ WEB3_HELP
fi
# Allow the binary to receive signals directly.
exec build/${NBC_BINARY} \
exec ${WINPTY} build/${NBC_BINARY} \
--network=${NETWORK} \
--data-dir="${DATA_DIR}" \
--log-file="${DATA_DIR}/nbc_bn_$(date +"%Y%m%d%H%M%S").log" \

14
tools/generate_makefile.nim
View File

@ -1,4 +1,4 @@
# Copyright (c) 2020 Status Research & Development GmbH
# Copyright (c) 2020-2021 Status Research & Development GmbH
# Licensed and distributed under either of
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
@ -63,10 +63,10 @@ proc main() =
cmd: string
for compile in data["compile"]:
cmd = compile[1].getStr()
cmd = compile[1].getStr().replace('\\', '/')
objectPath = ""
found = false
for token in split(cmd):
for token in split(cmd, Whitespace + {'\''}):
if found and token.len > 0 and token.endsWith(".o"):
objectPath = token
break
@ -75,20 +75,20 @@ proc main() =
if found == false or objectPath == "":
echo "Could not find the object file in this command: ", cmd
quit(QuitFailure)
makefile.writeLine("$#: $#" % [objectPath, compile[0].getStr()])
makefile.writeLine("$#: $#" % [objectPath.replace('\\', '/'), compile[0].getStr().replace('\\', '/')])
makefile.writeLine("\t+ $#\n" % cmd)
var objects: seq[string]
for obj in data["link"]:
objects.add(obj.getStr())
objects.add(obj.getStr().replace('\\', '/'))
makefile.writeLine("OBJECTS := $#\n" % objects.join(" \\\n"))
makefile.writeLine(".PHONY: build")
makefile.writeLine("build: $(OBJECTS)")
makefile.writeLine("\t+ $#" % processLinkCmd(data["linkcmd"].getStr(), makefilePath & ".linkerArgs"))
makefile.writeLine("\t+ $#" % processLinkCmd(data["linkcmd"].getStr().replace('\\', '/'), makefilePath & ".linkerArgs"))
if data.hasKey("extraCmds"):
for cmd in data["extraCmds"]:
makefile.writeLine("\t+ $#" % cmd.getStr())
makefile.writeLine("\t+ $#" % cmd.getStr().replace('\\', '/'))
main()

2
vendor/nim-bearssl vendored

@ -1 +1 @@
Subproject commit eebf730ccda5b5fade2a8f48b3da1496f2c47ba5
Subproject commit 5f226c97ec5832ce4a316b7b3ef0d8c2535c8972

2
vendor/nim-blscurve vendored

@ -1 +1 @@
Subproject commit f7d0341f8044fec7603c4acec6774b529b036b32
Subproject commit 7e8de1e748e75a977ff4ca9aed798fc6f976fc1c

2
vendor/nim-json-rpc vendored

@ -1 +1 @@
Subproject commit ded863fcb1741483f285e3bbf27541efdc32c78c
Subproject commit b0e82cb70a2055c133f118044cb7510c778903e7

2
vendor/nim-secp256k1 vendored

@ -1 +1 @@
Subproject commit ac96054870eb7cbc2cf72dd03fe5d5336b2e5dd9
Subproject commit 67ebdfa8014e1c6011f14d7fc5eb341684eb3b32

2
vendor/nim-web3 vendored

@ -1 +1 @@
Subproject commit dde382f70e811d964a000bdd4d86151615f9d4c0
Subproject commit 21b465fcd58460e6018dcb1048254f2514696778