CI release: ARM and ARM64 builds (#2213)

.github/workflows/release.yml

@@ -6,21 +6,11 @@ on:
 name: Upload Release Asset
 
 jobs:
-  build:
+  build-amd64:
-    name: Upload Release Asset
+    name: AMD64 release asset
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout code
+      - name: Create release
-        uses: actions/checkout@v2
-      - name: Build project
-        id: build_linux
-        run: |
-          #echo "::set-output name=tag::"$(echo ${{ github.ref }} | sed 's%refs/tags/%%')
-          #curl -sSOL https://github.com/status-im/nimbus-eth2/archive/$(echo ${{ github.ref }} | sed 's%refs/tags/%%').tar.gz
-          make dist
-          cd dist
-          echo "::set-output name=linux_amd64_archive::"$(echo nimbus-eth2_Linux_amd64_*.tar.gz)
-      - name: Create Release
         id: create_release
         uses: actions/create-release@v1
         env:
@@ -30,22 +20,95 @@ jobs:
           release_name: Release ${{ github.ref }}
           draft: true
           prerelease: false
-      #- name: Upload Release Tarball
+      - name: Checkout code
-        #uses: actions/upload-release-asset@v1
+        uses: actions/checkout@v2
-        #env:
+      - name: Build project
-          #GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        id: make_dist
-        #with:
+        run: |
-          #upload_url: ${{ steps.create_release.outputs.upload_url }}
+          make dist-amd64
-          #asset_path: ${{ steps.build_linux.outputs.tag }}.tar.gz
+          cd dist
-          #asset_name: nimbus-eth2-${{ steps.build_linux.outputs.tag }}.tar.gz
+          echo "::set-output name=linux_amd64_archive::"$(echo nimbus-eth2_Linux_amd64_*.tar.gz)
-          #asset_content_type: application/gzip
+      - name: Upload release asset
-      - name: Upload Release Asset
         uses: actions/upload-release-asset@v1
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
           upload_url: ${{ steps.create_release.outputs.upload_url }}
-          asset_path: ./dist/${{ steps.build_linux.outputs.linux_amd64_archive }}
+          asset_path: ./dist/${{ steps.make_dist.outputs.linux_amd64_archive }}
-          asset_name: ${{ steps.build_linux.outputs.linux_amd64_archive }}
+          asset_name: ${{ steps.make_dist.outputs.linux_amd64_archive }}
+          asset_content_type: application/gzip
+  build-arm64:
+    name: ARM64 release asset
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install packages
+        env:
+          DEBIAN_FRONTEND: "noninteractive"
+          TZ: "Etc/UTC"
+        run: |
+          sudo apt-get -qq update
+          sudo apt-get -qq -y install binfmt-support qemu-user-static
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Build project
+        id: make_dist
+        run: |
+          make dist-arm64
+          cd dist
+          echo "::set-output name=linux_arm64_archive::"$(echo nimbus-eth2_Linux_arm64v8_*.tar.gz)
+      - name: Fetch latest release
+        id: fetch_release
+        uses: InsonusK/get-latest-release@v1.0.1
+        with:
+          myToken: ${{ github.token }}
+          view_top: 1
+      - name: Check release version
+        run: |
+          [[ "refs/tags/${{ steps.fetch_release.outputs.tag_name }}" == "${{ github.ref }}" ]]
+      - name: Upload release asset
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: https://uploads.github.com/repos/${{ github.repository }}/releases/${{ steps.fetch_release.outputs.id }}/assets{?name,label}
+          asset_path: ./dist/${{ steps.make_dist.outputs.linux_arm64_archive }}
+          asset_name: ${{ steps.make_dist.outputs.linux_arm64_archive }}
+          asset_content_type: application/gzip
+  build-arm:
+    name: ARM release asset
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install packages
+        env:
+          DEBIAN_FRONTEND: "noninteractive"
+          TZ: "Etc/UTC"
+        run: |
+          sudo apt-get -qq update
+          sudo apt-get -qq -y install binfmt-support qemu-user-static
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Build project
+        id: make_dist
+        run: |
+          make dist-arm
+          cd dist
+          echo "::set-output name=linux_arm_archive::"$(echo nimbus-eth2_Linux_arm32v7_*.tar.gz)
+      - name: Fetch latest release
+        id: fetch_release
+        uses: InsonusK/get-latest-release@v1.0.1
+        with:
+          myToken: ${{ github.token }}
+          view_top: 1
+      - name: Check release version
+        run: |
+          [[ "refs/tags/${{ steps.fetch_release.outputs.tag_name }}" == "${{ github.ref }}" ]]
+      - name: Upload release asset
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: https://uploads.github.com/repos/${{ github.repository }}/releases/${{ steps.fetch_release.outputs.id }}/assets{?name,label}
+          asset_path: ./dist/${{ steps.make_dist.outputs.linux_arm_archive }}
+          asset_name: ${{ steps.make_dist.outputs.linux_arm_archive }}
           asset_content_type: application/gzip
 

CHANGELOG.md

@@ -1,3 +1,5 @@
+* ARM and ARM64 binaries are now produced in CI and added to the release
+
 2020-12-16 v1.0.4
 =================
 

Makefile

@@ -78,6 +78,9 @@ TOOLS_CSV := $(subst $(SPACE),$(COMMA),$(TOOLS))
 	libbacktrace \
 	book \
 	publish-book \
+	dist-amd64 \
+	dist-arm64 \
+	dist-arm \
 	dist \
 	benchmarks
 
@@ -390,14 +393,22 @@ publish-book: | book auditors-book
 	git worktree remove -f tmp-book && \
 	rm -rf tmp-book
 
-#- we rebuild everything inside the container, so we need to clean up afterwards
+dist-amd64:
+	MAKE="$(MAKE)" \
+		scripts/make_dist.sh amd64
+
+dist-arm64:
+	MAKE="$(MAKE)" \
+		scripts/make_dist.sh arm64
+
+dist-arm:
+	MAKE="$(MAKE)" \
+		scripts/make_dist.sh arm
+
 dist:
-	docker rm nimbus-eth2-dist $(HANDLE_OUTPUT) || true
+	$(MAKE) dist-amd64
-	cd docker/dist && \
+	$(MAKE) dist-arm64
-		DOCKER_BUILDKIT=1 docker build -t nimbus-eth2-dist --progress=plain --build-arg USER_ID=$$(id -u) --build-arg GROUP_ID=$$(id -g) . && \
+	$(MAKE) dist-arm
-		docker run --rm --name nimbus-eth2-dist -v $(CURDIR):/home/user/nimbus-eth2 nimbus-eth2-dist
-	ls -l dist
-	$(MAKE) clean
 
 #- this simple test will show any missing dynamically-linked Glibc symbols in the target distro
 dist-test:

config.nims

@@ -41,6 +41,7 @@ if defined(windows):
 # engineering a more portable binary release, this should be tweaked but still
 # use at least -msse2 or -msse3.
 if defined(disableMarchNative):
+  if defined(i386) or defined(amd64):
     switch("passC", "-msse3")
     switch("passL", "-msse3")
 else:

docker/dist/Dockerfile.amd64

@@ -14,5 +14,5 @@ USER user
 STOPSIGNAL SIGINT
 
 COPY "entry_point.sh" "/home/user/"
-ENTRYPOINT ["/home/user/entry_point.sh"]
+ENTRYPOINT ["/home/user/entry_point.sh", "Linux_amd64"]
 

docker/dist/Dockerfile.arm

@@ -0,0 +1,26 @@
+# The build is reproducible only if this base image stays the same.
+FROM statusteam/nimbus_beacon_node:dist_base_20210105213024_arm@sha256:2096aeba814f15d25dcf96b105197d5389d4bc9443fe20fec775868fac21d47a
+
+SHELL ["/bin/bash", "-c"]
+
+ARG QEMU_NAME
+ARG QEMU_DIR
+
+# We need the host's registered binfmt_misc "interpreter" inside the container,
+# for that transparent virtualisation to work.
+# Don't bother deleting it at the end, because this image is not being pushed to the hub.
+COPY $QEMU_NAME $QEMU_DIR
+
+ARG USER_ID
+ARG GROUP_ID
+
+RUN addgroup --gid ${GROUP_ID} user; \
+	adduser --disabled-password --gecos '' --uid ${USER_ID} --gid ${GROUP_ID} user;
+
+USER user
+
+STOPSIGNAL SIGINT
+
+COPY "entry_point.sh" "/home/user/"
+ENTRYPOINT ["/home/user/entry_point.sh", "Linux_arm32v7"]
+

docker/dist/Dockerfile.arm64

@@ -0,0 +1,26 @@
+# The build is reproducible only if this base image stays the same.
+FROM statusteam/nimbus_beacon_node:dist_base_20210105215256_arm64@sha256:472b1625f9d0fbdff3edc7543490980e128bef26ca4e7768a5f1b43fbf85e941
+
+SHELL ["/bin/bash", "-c"]
+
+ARG QEMU_NAME
+ARG QEMU_DIR
+
+# We need the host's registered binfmt_misc "interpreter" inside the container,
+# for that transparent virtualisation to work.
+# Don't bother deleting it at the end, because this image is not being pushed to the hub.
+COPY $QEMU_NAME $QEMU_DIR
+
+ARG USER_ID
+ARG GROUP_ID
+
+RUN addgroup --gid ${GROUP_ID} user; \
+	adduser --disabled-password --gecos '' --uid ${USER_ID} --gid ${GROUP_ID} user;
+
+USER user
+
+STOPSIGNAL SIGINT
+
+COPY "entry_point.sh" "/home/user/"
+ENTRYPOINT ["/home/user/entry_point.sh", "Linux_arm64v8"]
+

docker/dist/base_image/Dockerfile.amd64

@@ -10,6 +10,7 @@ FROM ubuntu:20.04
 
 SHELL ["/bin/bash", "-c"]
 
+ENV DEBIAN_FRONTEND=noninteractive TZ="Etc/UTC"
 RUN apt-get -qq update \
  && apt-get -qq -y install build-essential git &>/dev/null \
  && apt-get -qq clean \

docker/dist/base_image/Dockerfile.arm

@@ -0,0 +1,28 @@
+# This Docker image can change from one build to another, because the upstream
+# Debian/Ubuntu package index is continuously updated and we have to run
+# `apt-get update` in here.
+#
+# The only way to make this a part of our reproducible build system is to build
+# it once, upload it to Docker Hub and make sure it's being pulled regularly so
+# it's not deleted after 6 months of inactivity.
+
+# >=glibc-2.28 breaks things for us: https://bugs.launchpad.net/qemu/+bug/1805913
+FROM arm32v7/ubuntu:18.04
+
+SHELL ["/bin/bash", "-c"]
+
+ARG QEMU_NAME
+ARG QEMU_DIR
+
+# We need the host's registered binfmt_misc "interpreter" inside the container,
+# for that transparent virtualisation to work.
+COPY $QEMU_NAME $QEMU_DIR
+
+# We delete the static Qemu binary when we're done with it, because it might
+# not run on other systems. Let the child image copy it again.
+ENV DEBIAN_FRONTEND=noninteractive TZ="Etc/UTC"
+RUN apt-get -qq update \
+ && apt-get -qq -y install build-essential git &>/dev/null \
+ && apt-get -qq clean \
+ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* "${QEMU_DIR}/${QEMU_NAME}"
+

docker/dist/base_image/Dockerfile.arm64

@@ -0,0 +1,27 @@
+# This Docker image can change from one build to another, because the upstream
+# Debian/Ubuntu package index is continuously updated and we have to run
+# `apt-get update` in here.
+#
+# The only way to make this a part of our reproducible build system is to build
+# it once, upload it to Docker Hub and make sure it's being pulled regularly so
+# it's not deleted after 6 months of inactivity.
+
+FROM arm64v8/ubuntu:20.04
+
+SHELL ["/bin/bash", "-c"]
+
+ARG QEMU_NAME
+ARG QEMU_DIR
+
+# We need the host's registered binfmt_misc "interpreter" inside the container,
+# for that transparent virtualisation to work.
+COPY $QEMU_NAME $QEMU_DIR
+
+# We delete the static Qemu binary when we're done with it, because it might
+# not run on other systems. Let the child image copy it again.
+ENV DEBIAN_FRONTEND=noninteractive TZ="Etc/UTC"
+RUN apt-get -qq update \
+ && apt-get -qq -y install build-essential git &>/dev/null \
+ && apt-get -qq clean \
+ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* "${QEMU_DIR}/${QEMU_NAME}"
+

docker/dist/base_image/Makefile

@@ -3,15 +3,33 @@ SHELL := bash
 IMAGE_TAG := dist_base_$(shell date --utc +"%Y%m%d%H%M%S")
 IMAGE_NAME := statusteam/nimbus_beacon_node:$(IMAGE_TAG)
 
-.PHONY: build push
+.PHONY: \
+	build-amd64 \
+	build-arm64 \
+	build-arm \
+	push-amd64 \
+	push-arm64 \
+	push-arm
 
-build:
+build-amd64:
-	@ DOCKER_BUILDKIT=1 \
+	$(CURDIR)/make_base_image.sh amd64 "$(IMAGE_NAME)"
-	docker build \
-		-t $(IMAGE_NAME) \
-		--progress=plain \
-		.
 
-push: build
+build-arm64:
-	docker push $(IMAGE_NAME)
+	$(CURDIR)/make_base_image.sh arm64 "$(IMAGE_NAME)_arm64"
+
+build-arm:
+	$(CURDIR)/make_base_image.sh arm "$(IMAGE_NAME)_arm"
+
+# You probably don't want to recreate and push these base images to Docker Hub,
+# because when older images expire and get deleted, it will no longer be possible
+# to reproduce old releases.
+
+#push-amd64: build-amd64
+#	docker push $(IMAGE_NAME)
+
+#push-arm64: build-arm64
+#	docker push $(IMAGE_NAME)_arm64
+
+#push-arm: build-arm
+#	docker push $(IMAGE_NAME)_arm
 

docker/dist/base_image/README.md

@@ -0,0 +1,4 @@
+You probably don't want to recreate and push these base images to Docker Hub,
+because when older images expire and get deleted, it will no longer be possible
+to reproduce old releases.
+

docker/dist/base_image/make_base_image.sh

@@ -0,0 +1,74 @@
+#!/bin/bash
+
+# Copyright (c) 2020 Status Research & Development GmbH. Licensed under
+# either of:
+# - Apache License, version 2.0
+# - MIT license
+# at your option. This file may not be copied, modified, or distributed except
+# according to those terms.
+
+# Build base Docker images for making distributable binaries, using Qemu for
+# foreign architectures.
+# Should be used from "build-*" Make targets, passing the target architecture's
+# name and Docker image tag as parameters.
+
+set -e
+
+cd "$(dirname "${BASH_SOURCE[0]}")"
+
+if [[ -z "${2}" ]]; then
+  echo "Usage: $(basename ${0}) ARCH DOCKER_TAG"
+  exit 1
+fi
+ARCH="${1}"
+DOCKER_TAG="${2}"
+
+if [[ "${ARCH}" == "amd64" ]]; then
+  USE_QEMU=0
+else
+  USE_QEMU=1
+  if [[ "${ARCH}" == "arm64" ]]; then
+    BINFMT_NAME="aarch64"
+  elif [[ "${ARCH}" == "arm" ]]; then
+    BINFMT_NAME="arm"
+  fi
+fi
+
+DOCKER_EXTRA_ARGS=""
+if [[ "${USE_QEMU}" == "1" ]]; then
+  # We need qemu-user-static installed and registered in binfmt_misc, on the host
+  # (`apt install qemu binfmt-support qemu-user-static` should do the trick on Debian-based distros).
+  # The actual binary name varies from one distro to another and we need a copy inside the container.
+
+  if [[ ! -f /proc/sys/fs/binfmt_misc/qemu-${BINFMT_NAME} ]]; then
+    echo "binfmt_misc not set up properly. Aborting."
+    echo "You may have to run 'apt install qemu binfmt-support qemu-user-static' on a Debian-based distro."
+    exit 1
+  fi
+
+  QEMU_PATH="$(grep '^interpreter' /proc/sys/fs/binfmt_misc/qemu-${BINFMT_NAME} | cut -d ' ' -f 2)"
+  QEMU_NAME="$(basename ${QEMU_PATH})"
+  QEMU_DIR="$(dirname ${QEMU_PATH})"
+  DOCKER_EXTRA_ARGS="\
+    --build-arg QEMU_NAME=${QEMU_NAME} \
+    --build-arg QEMU_DIR=${QEMU_DIR} \
+  "
+fi
+
+if [[ "${USE_QEMU}" == "1" ]]; then
+  cp -a "${QEMU_PATH}" .
+fi
+
+DOCKER_BUILDKIT=1 \
+  docker build \
+  -t ${DOCKER_TAG} \
+  --progress=plain \
+  --build-arg USER_ID=$(id -u) \
+  --build-arg GROUP_ID=$(id -g) \
+  ${DOCKER_EXTRA_ARGS} \
+  -f Dockerfile.${ARCH} .
+
+if [[ "${USE_QEMU}" == "1" ]]; then
+  rm "${QEMU_NAME}"
+fi
+

docker/dist/entry_point.sh

@@ -5,17 +5,27 @@ set -e
 cd /home/user/nimbus-eth2
 git config --global core.abbrev 8
 
+if [[ -z "${1}" ]]; then
+  echo "Usage: $(basename ${0}) PLATFORM"
+  exit 1
+fi
+PLATFORM="${1}"
 BINARIES="nimbus_beacon_node nimbus_signing_process"
 
 # we need to build everything against libraries available inside this container, including the Nim compiler
 make clean
-make -j$(nproc) LOG_LEVEL="TRACE" NIMFLAGS="-d:disableMarchNative" PARTIAL_STATIC_LINKING=1 ${BINARIES}
+make \
+  -j$(nproc) \
+  LOG_LEVEL="TRACE" \
+  NIMFLAGS="-d:disableMarchNative" \
+  PARTIAL_STATIC_LINKING=1 \
+  QUICK_AND_DIRTY_COMPILER=1 \
+  ${BINARIES}
 
 # archive directory (we need the Nim compiler in here)
-PREFIX="nimbus-eth2_Linux_amd64_"
+PREFIX="nimbus-eth2_${PLATFORM}_"
 GIT_COMMIT="$(git rev-parse --short HEAD)"
 VERSION="$(./env.sh nim --verbosity:0 --hints:off --warnings:off scripts/print_version.nims)"
-#TIMESTAMP="$(date --utc +'%Y%m%d%H%M%S')"
 DIR="${PREFIX}${VERSION}_${GIT_COMMIT}"
 DIST_PATH="dist/${DIR}"
 # delete old artefacts
@@ -38,7 +48,6 @@ done
 sed -e "s/GIT_COMMIT/${GIT_COMMIT}/" docker/dist/README.md > "${DIST_PATH}/README.md"
 cp -a scripts/run-beacon-node.sh "${DIST_PATH}/scripts"
 cp -a ./run-*-beacon-node.sh "${DIST_PATH}/"
-#cp -a docs/the_nimbus_book "${DIST_PATH}/"
 
 # create the tarball
 cd dist

scripts/make_dist.sh

@@ -0,0 +1,81 @@
+#!/bin/bash
+
+# Copyright (c) 2020 Status Research & Development GmbH. Licensed under
+# either of:
+# - Apache License, version 2.0
+# - MIT license
+# at your option. This file may not be copied, modified, or distributed except
+# according to those terms.
+
+# Build release binaries fit for public distribution, using Docker and Qemu.
+# Should be used from "dist-*" Make targets, passing the target architecture's name as a parameter.
+
+set -e
+
+cd "$(dirname "${BASH_SOURCE[0]}")"/..
+CURDIR="${PWD}"
+
+ARCH="${1:-amd64}"
+if [[ "${ARCH}" == "amd64" ]]; then
+  USE_QEMU=0
+else
+  USE_QEMU=1
+  if [[ "${ARCH}" == "arm64" ]]; then
+    BINFMT_NAME="aarch64"
+  elif [[ "${ARCH}" == "arm" ]]; then
+    BINFMT_NAME="arm"
+  fi
+fi
+DOCKER_TAG="nimbus-eth2-dist-${ARCH}"
+
+DOCKER_EXTRA_ARGS=""
+if [[ "${USE_QEMU}" == "1" ]]; then
+  # We need qemu-user-static installed and registered in binfmt_misc, on the host
+  # (`apt install qemu binfmt-support qemu-user-static` should do the trick on Debian-based distros).
+  # The actual binary name varies from one distro to another and we need a copy inside the container.
+
+  if [[ ! -f /proc/sys/fs/binfmt_misc/qemu-${BINFMT_NAME} ]]; then
+    echo "binfmt_misc not set up properly. Aborting."
+    echo "You may have to run 'apt install qemu binfmt-support qemu-user-static' on a Debian-based distro."
+    exit 1
+  fi
+
+  QEMU_PATH="$(grep '^interpreter' /proc/sys/fs/binfmt_misc/qemu-${BINFMT_NAME} | cut -d ' ' -f 2)"
+  QEMU_NAME="$(basename ${QEMU_PATH})"
+  QEMU_DIR="$(dirname ${QEMU_PATH})"
+  DOCKER_EXTRA_ARGS="\
+    --build-arg QEMU_NAME=${QEMU_NAME} \
+    --build-arg QEMU_DIR=${QEMU_DIR} \
+  "
+fi
+
+docker rm ${DOCKER_TAG} &>/dev/null || true
+
+cd docker/dist
+
+if [[ "${USE_QEMU}" == "1" ]]; then
+  cp -a "${QEMU_PATH}" .
+fi
+
+DOCKER_BUILDKIT=1 \
+  docker build \
+  -t ${DOCKER_TAG} \
+  --progress=plain \
+  --build-arg USER_ID=$(id -u) \
+  --build-arg GROUP_ID=$(id -g) \
+  ${DOCKER_EXTRA_ARGS} \
+  -f Dockerfile.${ARCH} .
+
+docker run --rm --name ${DOCKER_TAG} -v ${CURDIR}:/home/user/nimbus-eth2 ${DOCKER_TAG}
+
+if [[ "${USE_QEMU}" == "1" ]]; then
+  rm "${QEMU_NAME}"
+fi
+
+cd - &>/dev/null
+
+ls -l dist
+
+# We rebuild everything inside the container, so we need to clean up afterwards.
+${MAKE} --no-print-directory clean
+

vendor/nimbus-build-system

@@ -1 +1 @@
-Subproject commit e2de003ce634deca72dab3b3c79426698e7b8579
+Subproject commit f7c732a150bb4dad68eb8bb99a727b5f42083201