diff --git a/.gitignore b/.gitignore index 1c3ea4783..706716312 100644 --- a/.gitignore +++ b/.gitignore @@ -44,3 +44,5 @@ build/ # generated during Nim compilation beacon_chain/sync_protocol.nim.generated.nim +/dist + diff --git a/Makefile b/Makefile index a8643b602..4015cbda1 100644 --- a/Makefile +++ b/Makefile @@ -74,7 +74,8 @@ TOOLS_CSV := $(subst $(SPACE),$(COMMA),$(TOOLS)) clean \ libbacktrace \ book \ - publish-book + publish-book \ + dist ifeq ($(NIM_PARAMS),) # "variables.mk" was not included, so we update the submodules. @@ -420,4 +421,13 @@ publish-book: | book auditors-book git worktree remove -f tmp-book && \ rm -rf tmp-book +#- we rebuild everything inside the container, so we need to clean up afterwards +dist: + docker rm nimbus-eth2-dist $(HANDLE_OUTPUT) || true + cd docker/dist && \ + DOCKER_BUILDKIT=1 docker build -t nimbus-eth2-dist --progress=plain --build-arg USER_ID=$$(id -u) --build-arg GROUP_ID=$$(id -g) . && \ + docker run --rm --name nimbus-eth2-dist -v $(CURDIR):/home/user/nimbus-eth2 nimbus-eth2-dist + ls -l dist + $(MAKE) clean + endif # "variables.mk" was not included diff --git a/README.md b/README.md index a00cd5a89..bb6d7ee15 100644 --- a/README.md +++ b/README.md @@ -489,6 +489,12 @@ make NIMFLAGS="--passL:-static" beacon_node make publish-book ``` +- create a binary distribution + +```bash +make dist +``` + ### CI setup Local testnets run for 4 epochs each, to test finalization. That happens only on Jenkins Linux hosts, and their logs are available for download as artifacts, from the job's page. Don't expect these artifacts to be kept more than a day after the corresponding branch is deleted. diff --git a/docker/dist/Dockerfile b/docker/dist/Dockerfile new file mode 100644 index 000000000..a3f1044b1 --- /dev/null +++ b/docker/dist/Dockerfile @@ -0,0 +1,18 @@ +# The build is reproducible only if this base image stays the same. +FROM statusteam/nimbus_beacon_node:dist_base_20201008172919@sha256:7190a4d54e56aa4a45fa3032381a43dcec89f23e6d7ed5dfb0f98967f92bf522 + +SHELL ["/bin/bash", "-c"] + +ARG USER_ID +ARG GROUP_ID + +RUN addgroup --gid ${GROUP_ID} user; \ + adduser --disabled-password --gecos '' --uid ${USER_ID} --gid ${GROUP_ID} user; + +USER user + +STOPSIGNAL SIGINT + +COPY "entry_point.sh" "/home/user/" +ENTRYPOINT ["/home/user/entry_point.sh"] + diff --git a/docker/dist/README.md b/docker/dist/README.md new file mode 100644 index 000000000..a61141745 --- /dev/null +++ b/docker/dist/README.md @@ -0,0 +1,22 @@ +# Binary Nimbus beacon_node distribution + +This binary distribution was created from https://github.com/status-im/nimbus-eth2 + +Tarball naming scheme: "nimbus-eth2\_Linux\_amd64\_\_.tar.gz" (the date is in UTC). + +## Reproducing the build + +Besides the generic build requirements, you also need [Docker](https://www.docker.com/). + +```bash +git clone https://github.com/status-im/nimbus-eth2.git +cd nimbus-eth2 +git checkout GIT_COMMIT +make update +make dist +``` + +## Significant differences from self-built binaries + +No `-march=native` and no metrics support. + diff --git a/docker/dist/base_image/Dockerfile b/docker/dist/base_image/Dockerfile new file mode 100644 index 000000000..f7e0fc4fb --- /dev/null +++ b/docker/dist/base_image/Dockerfile @@ -0,0 +1,17 @@ +# This Docker image can change from one build to another, because the upstream +# Debian package index is continuously updated and we have to run `apt-get +# update` in here. +# +# The only way to make this a part of our reproducible build system is to build +# it once, upload it to Docker Hub and make sure it's being pulled regularly so +# it's not deleted after 6 months of inactivity. + +FROM debian:bullseye-slim + +SHELL ["/bin/bash", "-c"] + +RUN apt-get -qq update \ + && apt-get -qq -y install build-essential libpcre3-dev git &>/dev/null \ + && apt-get -qq clean \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + diff --git a/docker/dist/base_image/Makefile b/docker/dist/base_image/Makefile new file mode 100644 index 000000000..dca0928d3 --- /dev/null +++ b/docker/dist/base_image/Makefile @@ -0,0 +1,17 @@ +SHELL := bash + +IMAGE_TAG := dist_base_$(shell date --utc +"%Y%m%d%H%M%S") +IMAGE_NAME := statusteam/nimbus_beacon_node:$(IMAGE_TAG) + +.PHONY: build push + +build: + @ DOCKER_BUILDKIT=1 \ + docker build \ + -t $(IMAGE_NAME) \ + --progress=plain \ + . + +push: build + docker push $(IMAGE_NAME) + diff --git a/docker/dist/entry_point.sh b/docker/dist/entry_point.sh new file mode 100755 index 000000000..3dd00a3f4 --- /dev/null +++ b/docker/dist/entry_point.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +set -e + +cd /home/user/nimbus-eth2 + +PREFIX="nimbus-eth2_Linux_amd64_" +GIT_COMMIT="$(git rev-parse --short HEAD)" +DIR="${PREFIX}${GIT_COMMIT}_$(date --utc +'%Y%m%d%H%M%S')" +DIST_PATH="dist/${DIR}" +BINARIES="beacon_node" + +# delete old artefacts +rm -rf "dist/${PREFIX}"* + +mkdir -p "${DIST_PATH}" + +# we need to build everything against libraries available inside this container, including the Nim compiler +make clean +make -j$(nproc) NIMFLAGS="-d:disableMarchNative" PARTIAL_STATIC_LINKING=1 ${BINARIES} +for BINARY in ${BINARIES}; do + cp -a ./build/${BINARY} "${DIST_PATH}/" + cd "${DIST_PATH}" + md5sum ${BINARY} > ${BINARY}.md5sum + sha512sum ${BINARY} > ${BINARY}.sha512sum + cd - >/dev/null +done +sed -e "s/GIT_COMMIT/${GIT_COMMIT}/" docker/dist/README.md > "${DIST_PATH}/README.md" + +# create the tarball +cd dist +tar czf "${DIR}.tar.gz" "${DIR}" +# don't leave the directory hanging around +rm -rf "${DIR}" +cd - >/dev/null + diff --git a/vendor/nimbus-build-system b/vendor/nimbus-build-system index 4b662df1e..c278412db 160000 --- a/vendor/nimbus-build-system +++ b/vendor/nimbus-build-system @@ -1 +1 @@ -Subproject commit 4b662df1e95b5bebac43b6d2faa297f135bc90d5 +Subproject commit c278412dbf7fce294ca1fabb8264607c2971ea8f