Windows binary release

CI: use both cores on GitHub Actions and set timeouts for the local testnet tests
This commit is contained in:
Ștefan Talpalaru 2021-02-02 23:31:01 +01:00 committed by zah
parent 67e4a045a3
commit 80c11546ff
21 changed files with 228 additions and 64 deletions

View File

@ -122,6 +122,21 @@ jobs:
echo "CFLAGS=$CFLAGS" >> $GITHUB_ENV
fi
ncpu=
case '${{ runner.os }}' in
'Linux')
ncpu=$(nproc)
;;
'macOS')
ncpu=$(sysctl -n hw.ncpu)
;;
'Windows')
ncpu=$NUMBER_OF_PROCESSORS
;;
esac
[[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
echo "ncpu=$ncpu" >> $GITHUB_ENV
- name: Install build dependencies (Linux i386)
if: runner.os == 'Linux' && matrix.target.cpu == 'i386'
run: |
@ -218,23 +233,6 @@ jobs:
shell: bash
working-directory: nim-beacon-chain
run: |
ncpu=
ext=
case '${{ runner.os }}' in
'Linux')
ncpu=$(nproc)
;;
'macOS')
ncpu=$(sysctl -n hw.ncpu)
;;
'Windows')
ncpu=$NUMBER_OF_PROCESSORS
ext=.exe
;;
esac
[[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
echo "::set-output name=ncpu::$ncpu"
make -j$ncpu ARCH_OVERRIDE=$PLATFORM CI_CACHE=NimBinaries V=1 update
- name: Get latest fixtures commit hash
@ -284,11 +282,11 @@ jobs:
shell: bash
working-directory: nim-beacon-chain
run: |
./scripts/launch_local_testnet.sh --testnet 0 --nodes 4 --stop-at-epoch 5 --log-level DEBUG --disable-htop --enable-logtrace --data-dir local_testnet0_data --base-port 9000 --base-rpc-port 7000 --base-metrics-port 8008 -- --verify-finalization --discv5:no
./scripts/launch_local_testnet.sh --testnet 0 --nodes 4 --stop-at-epoch 5 --log-level DEBUG --disable-htop --enable-logtrace --data-dir local_testnet0_data --base-port 9000 --base-rpc-port 7000 --base-metrics-port 8008 --timeout 600 -- --verify-finalization --discv5:no
- name: Run nim-beacon-chain testnet1 (mainnet)
if: matrix.target.TEST_KIND == 'finalization-mainnet'
shell: bash
working-directory: nim-beacon-chain
run: |
./scripts/launch_local_testnet.sh --testnet 1 --nodes 4 --stop-at-epoch 5 --log-level DEBUG --disable-htop --enable-logtrace --data-dir local_testnet0_data --base-port 9000 --base-rpc-port 7000 --base-metrics-port 8008 -- --verify-finalization --discv5:no
./scripts/launch_local_testnet.sh --testnet 1 --nodes 4 --stop-at-epoch 5 --log-level DEBUG --disable-htop --enable-logtrace --data-dir local_testnet0_data --base-port 9000 --base-rpc-port 7000 --base-metrics-port 8008 --timeout 2400 -- --verify-finalization --discv5:no

View File

@ -7,7 +7,7 @@ name: Upload Release Asset
jobs:
build-amd64:
name: AMD64 release asset
name: Linux AMD64 release asset
runs-on: ubuntu-latest
steps:
- name: Create release
@ -54,7 +54,7 @@ jobs:
docker push statusim/nimbus-eth2:amd64-${TAG}
docker push statusim/nimbus-eth2:amd64-latest
build-arm64:
name: ARM64 release asset
name: Linux ARM64 release asset
runs-on: ubuntu-latest
steps:
- name: Install packages
@ -91,7 +91,7 @@ jobs:
asset_name: ${{ steps.make_dist.outputs.linux_arm64_archive }}
asset_content_type: application/gzip
build-arm:
name: ARM release asset
name: Linux ARM release asset
runs-on: ubuntu-latest
steps:
- name: Install packages
@ -127,4 +127,34 @@ jobs:
asset_path: ./dist/${{ steps.make_dist.outputs.linux_arm_archive }}
asset_name: ${{ steps.make_dist.outputs.linux_arm_archive }}
asset_content_type: application/gzip
build-win64:
name: Windows AMD64 release asset
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v2
- name: Build project
id: make_dist
run: |
make dist-win64
cd dist
echo "::set-output name=windows_amd64_archive::"$(echo nimbus-eth2_Windows_amd64_*.tar.gz)
- name: Fetch latest release
id: fetch_release
uses: InsonusK/get-latest-release@v1.0.1
with:
myToken: ${{ github.token }}
view_top: 1
- name: Check release version
run: |
[[ "refs/tags/${{ steps.fetch_release.outputs.tag_name }}" == "${{ github.ref }}" ]]
- name: Upload release asset
uses: actions/upload-release-asset@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: https://uploads.github.com/repos/${{ github.repository }}/releases/${{ steps.fetch_release.outputs.id }}/assets{?name,label}
asset_path: ./dist/${{ steps.make_dist.outputs.windows_amd64_archive }}
asset_name: ${{ steps.make_dist.outputs.windows_amd64_archive }}
asset_content_type: application/gzip

View File

@ -1,4 +1,4 @@
# Copyright (c) 2019-2020 Status Research & Development GmbH. Licensed under
# Copyright (c) 2019-2021 Status Research & Development GmbH. Licensed under
# either of:
# - Apache License, version 2.0
# - MIT license
@ -81,6 +81,7 @@ TOOLS_CSV := $(subst $(SPACE),$(COMMA),$(TOOLS))
dist-amd64 \
dist-arm64 \
dist-arm \
dist-win64 \
dist \
benchmarks
@ -396,10 +397,15 @@ dist-arm:
MAKE="$(MAKE)" \
scripts/make_dist.sh arm
dist-win64:
MAKE="$(MAKE)" \
scripts/make_dist.sh win64
dist:
$(MAKE) dist-amd64
$(MAKE) dist-arm64
$(MAKE) dist-arm
$(MAKE) dist-win64
#- this simple test will show any missing dynamically-linked Glibc symbols in the target distro
dist-test:

View File

@ -1,3 +1,9 @@
# Copyright (c) 2020-2021 Status Research & Development GmbH
# Licensed and distributed under either of
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
# at your option. This file may not be copied, modified, or distributed except according to those terms.
import
tables, strutils, os,
stew/shims/macros, nimcrypto/hash,
@ -68,7 +74,7 @@ type
incompatibilityDesc*: string
const
eth2testnetsDir = currentSourcePath.parentDir / ".." / "vendor" / "eth2-testnets"
eth2testnetsDir = currentSourcePath.parentDir.replace('\\', '/') & "/../vendor/eth2-testnets"
const presetValueLoaders = genExpr(nnkBracket):
for constName in PresetValue:
@ -107,12 +113,12 @@ proc loadEth2NetworkMetadata*(path: string): Eth2NetworkMetadata
{.raises: [CatchableError, Defect].} =
try:
let
genesisPath = path / "genesis.ssz"
genesisDepositsSnapshotPath = path / "genesis_deposit_contract_snapshot.ssz"
configPath = path / "config.yaml"
depositContractPath = path / "deposit_contract.txt"
depositContractBlockPath = path / "deposit_contract_block.txt"
bootstrapNodesPath = path / "bootstrap_nodes.txt"
genesisPath = path & "/genesis.ssz"
genesisDepositsSnapshotPath = path & "/genesis_deposit_contract_snapshot.ssz"
configPath = path & "/config.yaml"
depositContractPath = path & "/deposit_contract.txt"
depositContractBlockPath = path & "/deposit_contract_block.txt"
bootstrapNodesPath = path & "/bootstrap_nodes.txt"
runtimePreset = if fileExists(configPath):
extractRuntimePreset(configPath, readPresetFile(configPath))
@ -164,7 +170,7 @@ proc loadEth2NetworkMetadata*(path: string): Eth2NetworkMetadata
incompatibilityDesc: err.msg)
const
mainnetMetadataDir = eth2testnetsDir / "shared" / "mainnet"
mainnetMetadataDir = eth2testnetsDir & "/shared/mainnet"
mainnetMetadata* = when const_preset == "mainnet":
Eth2NetworkMetadata(
@ -172,11 +178,11 @@ const
# that there are no constant overrides
eth1Network: some mainnet,
runtimePreset: mainnetRuntimePreset,
bootstrapNodes: readFile(mainnetMetadataDir / "bootstrap_nodes.txt").splitLines,
bootstrapNodes: readFile(mainnetMetadataDir & "/bootstrap_nodes.txt").splitLines,
depositContractAddress: Eth1Address.fromHex "0x00000000219ab540356cBB839Cbe05303d7705Fa",
depositContractDeployedAt: BlockHashOrNumber.init "11052984",
genesisData: readFile(mainnetMetadataDir / "genesis.ssz"),
genesisDepositsSnapshot: readFile(mainnetMetadataDir / "genesis_deposit_contract_snapshot.ssz"))
genesisData: readFile(mainnetMetadataDir & "/genesis.ssz"),
genesisDepositsSnapshot: readFile(mainnetMetadataDir & "/genesis_deposit_contract_snapshot.ssz"))
else:
Eth2NetworkMetadata(
incompatible: true,
@ -184,7 +190,7 @@ const
"It's not compatible with mainnet")
template eth2testnet(path: string): Eth2NetworkMetadata =
loadEth2NetworkMetadata(eth2testnetsDir / path)
loadEth2NetworkMetadata(eth2testnetsDir & "/" & path)
const
pyrmontMetadata* = eth2testnet "shared/pyrmont"

View File

@ -217,7 +217,7 @@ macro wordListArray*(filename: static string,
minWordLen: static int = 0,
maxWordLen: static int = high(int)): untyped =
result = newTree(nnkBracket)
var words = slurp(filename).splitLines()
var words = slurp(filename.replace('\\', '/')).splitLines()
for word in words:
if word.len >= minWordLen and word.len <= maxWordLen:
result.add newCall("cstring", newLit(word))

18
docker/dist/Dockerfile.win64 vendored Normal file
View File

@ -0,0 +1,18 @@
# The build is reproducible only if this base image stays the same.
FROM statusteam/nimbus_beacon_node:dist_base_20210202230118_win64@sha256:47b8ea60a35bd45355f0490677c04a9b42df56b5041cbf64da8ad233511a55d5
SHELL ["/bin/bash", "-c"]
ARG USER_ID
ARG GROUP_ID
RUN addgroup --gid ${GROUP_ID} user; \
adduser --disabled-password --gecos '' --uid ${USER_ID} --gid ${GROUP_ID} user;
USER user
STOPSIGNAL SIGINT
COPY "entry_point.sh" "/home/user/"
ENTRYPOINT ["/home/user/entry_point.sh", "Windows_amd64"]

7
docker/dist/README-Windows.md vendored Normal file
View File

@ -0,0 +1,7 @@
# Windows-specific requirements
Run the wrapper scripts from a [Git for Windows](https://gitforwindows.org/) Bash shell.
If you run the beacon node binary directly, prefix it with "winpty -- ". It
will increase the chance of Ctrl+C working inside that "mintty" terminal emulator.

23
docker/dist/base_image/Dockerfile.win64 vendored Normal file
View File

@ -0,0 +1,23 @@
# This Docker image can change from one build to another, because the upstream
# Debian/Ubuntu package index is continuously updated and we have to run
# `apt-get update` in here.
#
# The only way to make this a part of our reproducible build system is to build
# it once, upload it to Docker Hub and make sure it's being pulled regularly so
# it's not deleted after 6 months of inactivity.
FROM ubuntu:20.04
SHELL ["/bin/bash", "-c"]
ENV DEBIAN_FRONTEND=noninteractive TZ="Etc/UTC"
RUN \
apt-get -qq update \
&& apt-get -qq -y install git gnupg software-properties-common lsb &>/dev/null \
&& apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 86B72ED9 \
&& add-apt-repository "deb [arch=amd64] https://pkg.mxe.cc/repos/apt $(lsb_release -sc) main" \
&& apt-get -qq update \
&& apt-get -qq -y install mxe-x86-64-w64-mingw32.static-cc &>/dev/null \
&& apt-get -qq clean \
&& rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*

View File

@ -7,9 +7,11 @@ IMAGE_NAME := statusteam/nimbus_beacon_node:$(IMAGE_TAG)
build-amd64 \
build-arm64 \
build-arm \
build-win64 \
push-amd64 \
push-arm64 \
push-arm
push-arm \
push-win64
build-amd64:
$(CURDIR)/make_base_image.sh amd64 "$(IMAGE_NAME)"
@ -20,6 +22,9 @@ build-arm64:
build-arm:
$(CURDIR)/make_base_image.sh arm "$(IMAGE_NAME)_arm"
build-win64:
$(CURDIR)/make_base_image.sh win64 "$(IMAGE_NAME)_win64"
# You probably don't want to recreate and push these base images to Docker Hub,
# because when older images expire and get deleted, it will no longer be possible
# to reproduce old releases.
@ -33,3 +38,6 @@ build-arm:
#push-arm: build-arm
# docker push $(IMAGE_NAME)_arm
#push-win64: build-win64
# docker push $(IMAGE_NAME)_win64

View File

@ -1,6 +1,6 @@
#!/bin/bash
# Copyright (c) 2020 Status Research & Development GmbH. Licensed under
# Copyright (c) 2020-2021 Status Research & Development GmbH. Licensed under
# either of:
# - Apache License, version 2.0
# - MIT license
@ -23,7 +23,7 @@ fi
ARCH="${1}"
DOCKER_TAG="${2}"
if [[ "${ARCH}" == "amd64" ]]; then
if [[ "${ARCH}" == "amd64" || "${ARCH}" == "win64" ]]; then
USE_QEMU=0
else
USE_QEMU=1
@ -63,8 +63,6 @@ DOCKER_BUILDKIT=1 \
docker build \
-t ${DOCKER_TAG} \
--progress=plain \
--build-arg USER_ID=$(id -u) \
--build-arg GROUP_ID=$(id -g) \
${DOCKER_EXTRA_ARGS} \
-f Dockerfile.${ARCH} .

View File

@ -1,5 +1,12 @@
#!/bin/bash
# Copyright (c) 2020-2021 Status Research & Development GmbH. Licensed under
# either of:
# - Apache License, version 2.0
# - MIT license
# at your option. This file may not be copied, modified, or distributed except
# according to those terms.
set -e
cd /home/user/nimbus-eth2
@ -15,12 +22,46 @@ BINARIES="nimbus_beacon_node nimbus_signing_process"
#- we need to build everything against libraries available inside this container, including the Nim compiler
#- we disable the log file and log colours; the user only has to worry about logging stdout now
make clean
make \
-j$(nproc) \
LOG_LEVEL="TRACE" \
NIMFLAGS="-d:disableMarchNative -d:chronicles_sinks=textlines -d:chronicles_colors=none" \
PARTIAL_STATIC_LINKING=1 \
${BINARIES}
if [[ "${PLATFORM}" == "Windows_amd64" ]]; then
# Cross-compilation using the MXE distribution of Mingw-w64
export PATH="/usr/lib/mxe/usr/bin:${PATH}"
make \
-j$(nproc) \
USE_LIBBACKTRACE=0 \
deps
make \
-C vendor/nim-nat-traversal/vendor/miniupnp/miniupnpc \
-f Makefile.mingw \
clean &>/dev/null
make \
-j$(nproc) \
-C vendor/nim-nat-traversal/vendor/miniupnp/miniupnpc \
-f Makefile.mingw \
CC=x86_64-w64-mingw32.static-gcc \
libminiupnpc.a &>/dev/null
make \
-C vendor/nim-nat-traversal/vendor/libnatpmp-upstream \
clean &>/dev/null
make \
-j$(nproc) \
-C vendor/nim-nat-traversal/vendor/libnatpmp-upstream \
CC=x86_64-w64-mingw32.static-gcc \
CFLAGS="-Wall -Os -DWIN32 -DNATPMP_STATICLIB -DENABLE_STRNATPMPERR -DNATPMP_MAX_RETRIES=4 ${CFLAGS}" \
libnatpmp.a &>/dev/null
make \
-j$(nproc) \
USE_LIBBACKTRACE=0 \
LOG_LEVEL="TRACE" \
NIMFLAGS="-d:disableMarchNative -d:chronicles_sinks=textlines -d:chronicles_colors=none --os:windows --gcc.exe=x86_64-w64-mingw32.static-gcc --gcc.linkerexe=x86_64-w64-mingw32.static-gcc --passL:-static" \
${BINARIES}
else
make \
-j$(nproc) \
LOG_LEVEL="TRACE" \
NIMFLAGS="-d:disableMarchNative -d:chronicles_sinks=textlines -d:chronicles_colors=none" \
PARTIAL_STATIC_LINKING=1 \
${BINARIES}
fi
# archive directory (we need the Nim compiler in here)
PREFIX="nimbus-eth2_${PLATFORM}_"
@ -40,12 +81,18 @@ mkdir "${DIST_PATH}/build"
# copy and checksum binaries, copy scripts and docs
for BINARY in ${BINARIES}; do
cp -a ./build/${BINARY} "${DIST_PATH}/build/"
cp -a "./build/${BINARY}" "${DIST_PATH}/build/"
cd "${DIST_PATH}/build"
sha512sum ${BINARY} > ${BINARY}.sha512sum
sha512sum "${BINARY}" > "${BINARY}.sha512sum"
if [[ "${PLATFORM}" == "Windows_amd64" ]]; then
mv "${BINARY}" "${BINARY}.exe"
fi
cd - >/dev/null
done
sed -e "s/GIT_COMMIT/${GIT_COMMIT}/" docker/dist/README.md > "${DIST_PATH}/README.md"
if [[ "${PLATFORM}" == "Windows_amd64" ]]; then
cp -a docker/dist/README-Windows.md "${DIST_PATH}/"
fi
cp -a scripts/run-beacon-node.sh "${DIST_PATH}/scripts"
cp -a ./run-*-beacon-node.sh "${DIST_PATH}/"

View File

@ -1,5 +1,12 @@
#!/bin/bash
# Copyright (c) 2020-2021 Status Research & Development GmbH. Licensed under
# either of:
# - Apache License, version 2.0
# - MIT license
# at your option. This file may not be copied, modified, or distributed except
# according to those terms.
cd "$(dirname $0)"
# Allow the binary to receive signals directly.
exec scripts/run-beacon-node.sh nimbus_beacon_node mainnet $@

View File

@ -1,5 +1,12 @@
#!/bin/bash
# Copyright (c) 2020-2021 Status Research & Development GmbH. Licensed under
# either of:
# - Apache License, version 2.0
# - MIT license
# at your option. This file may not be copied, modified, or distributed except
# according to those terms.
cd "$(dirname $0)"
# Allow the binary to receive signals directly.
exec scripts/run-beacon-node.sh nimbus_beacon_node pyrmont $@

View File

@ -1,6 +1,6 @@
#!/bin/bash
# Copyright (c) 2020 Status Research & Development GmbH. Licensed under
# Copyright (c) 2020-2021 Status Research & Development GmbH. Licensed under
# either of:
# - Apache License, version 2.0
# - MIT license
@ -16,7 +16,7 @@ cd "$(dirname "${BASH_SOURCE[0]}")"/..
CURDIR="${PWD}"
ARCH="${1:-amd64}"
if [[ "${ARCH}" == "amd64" ]]; then
if [[ "${ARCH}" == "amd64" || "${ARCH}" == "win64" ]]; then
USE_QEMU=0
else
USE_QEMU=1

View File

@ -1,5 +1,12 @@
#!/bin/bash
# Copyright (c) 2020-2021 Status Research & Development GmbH. Licensed under
# either of:
# - Apache License, version 2.0
# - MIT license
# at your option. This file may not be copied, modified, or distributed except
# according to those terms.
set -e
cd "$(dirname $0)/.."
@ -32,8 +39,10 @@ fi
# Windows detection
if uname | grep -qiE "mingw|msys"; then
MAKE="mingw32-make"
WINPTY="winpty --"
else
MAKE="make"
WINPTY=""
fi
if [[ ! -f build/${NBC_BINARY} ]]; then
@ -66,7 +75,7 @@ WEB3_HELP
fi
# Allow the binary to receive signals directly.
exec build/${NBC_BINARY} \
exec ${WINPTY} build/${NBC_BINARY} \
--network=${NETWORK} \
--data-dir="${DATA_DIR}" \
--log-file="${DATA_DIR}/nbc_bn_$(date +"%Y%m%d%H%M%S").log" \

View File

@ -1,4 +1,4 @@
# Copyright (c) 2020 Status Research & Development GmbH
# Copyright (c) 2020-2021 Status Research & Development GmbH
# Licensed and distributed under either of
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
@ -63,10 +63,10 @@ proc main() =
cmd: string
for compile in data["compile"]:
cmd = compile[1].getStr()
cmd = compile[1].getStr().replace('\\', '/')
objectPath = ""
found = false
for token in split(cmd):
for token in split(cmd, Whitespace + {'\''}):
if found and token.len > 0 and token.endsWith(".o"):
objectPath = token
break
@ -75,20 +75,20 @@ proc main() =
if found == false or objectPath == "":
echo "Could not find the object file in this command: ", cmd
quit(QuitFailure)
makefile.writeLine("$#: $#" % [objectPath, compile[0].getStr()])
makefile.writeLine("$#: $#" % [objectPath.replace('\\', '/'), compile[0].getStr().replace('\\', '/')])
makefile.writeLine("\t+ $#\n" % cmd)
var objects: seq[string]
for obj in data["link"]:
objects.add(obj.getStr())
objects.add(obj.getStr().replace('\\', '/'))
makefile.writeLine("OBJECTS := $#\n" % objects.join(" \\\n"))
makefile.writeLine(".PHONY: build")
makefile.writeLine("build: $(OBJECTS)")
makefile.writeLine("\t+ $#" % processLinkCmd(data["linkcmd"].getStr(), makefilePath & ".linkerArgs"))
makefile.writeLine("\t+ $#" % processLinkCmd(data["linkcmd"].getStr().replace('\\', '/'), makefilePath & ".linkerArgs"))
if data.hasKey("extraCmds"):
for cmd in data["extraCmds"]:
makefile.writeLine("\t+ $#" % cmd.getStr())
makefile.writeLine("\t+ $#" % cmd.getStr().replace('\\', '/'))
main()

2
vendor/nim-bearssl vendored

@ -1 +1 @@
Subproject commit eebf730ccda5b5fade2a8f48b3da1496f2c47ba5
Subproject commit 5f226c97ec5832ce4a316b7b3ef0d8c2535c8972

2
vendor/nim-blscurve vendored

@ -1 +1 @@
Subproject commit f7d0341f8044fec7603c4acec6774b529b036b32
Subproject commit 7e8de1e748e75a977ff4ca9aed798fc6f976fc1c

2
vendor/nim-json-rpc vendored

@ -1 +1 @@
Subproject commit ded863fcb1741483f285e3bbf27541efdc32c78c
Subproject commit b0e82cb70a2055c133f118044cb7510c778903e7

@ -1 +1 @@
Subproject commit ac96054870eb7cbc2cf72dd03fe5d5336b2e5dd9
Subproject commit 67ebdfa8014e1c6011f14d7fc5eb341684eb3b32

2
vendor/nim-web3 vendored

@ -1 +1 @@
Subproject commit dde382f70e811d964a000bdd4d86151615f9d4c0
Subproject commit 21b465fcd58460e6018dcb1048254f2514696778