From d07d7b1c48389411e6b4282890fe01bc708fedb3 Mon Sep 17 00:00:00 2001 From: Nikolay Mitev Date: Thu, 11 Aug 2022 17:57:44 +0300 Subject: [PATCH] Include auxiliary tools to nightly builds Create stable link (withouth hash) so that scripts can always get the latest nightly build from the same url --- .github/workflows/nightly_build.yml | 70 +++++++++++++++++++---------- docker/dist/Dockerfile.amd64 | 4 ++ docker/dist/Dockerfile.arm | 2 + docker/dist/Dockerfile.arm64 | 2 + docker/dist/Dockerfile.macos | 2 + docker/dist/Dockerfile.macos-arm64 | 2 + docker/dist/Dockerfile.win64 | 2 + docker/dist/entry_point.sh | 8 ++++ scripts/make_dist.sh | 7 ++- 9 files changed, 74 insertions(+), 25 deletions(-) diff --git a/.github/workflows/nightly_build.yml b/.github/workflows/nightly_build.yml index 43b8257fc..7bc1f8fc6 100644 --- a/.github/workflows/nightly_build.yml +++ b/.github/workflows/nightly_build.yml @@ -5,6 +5,10 @@ on: workflow_dispatch: #pull_request: +env: + # Build and include deposit_contract nimbus_signing_node nimbus_light_client logtrace in archive + BUILD_TOOLS: 1 + jobs: build-amd64: name: Linux AMD64 release asset @@ -25,25 +29,28 @@ jobs: NEW_ARCHIVE_DIR="nimbus-eth2_Linux_amd64_$(date +%Y%m%d)_$(git rev-parse --short=8 HEAD)" mv ${ARCHIVE%.tar.gz} ${NEW_ARCHIVE_DIR} tar -czf ${NEW_ARCHIVE_DIR}.tar.gz ${NEW_ARCHIVE_DIR} + cp ${NEW_ARCHIVE_DIR}.tar.gz nimbus-eth2_Linux_amd64_nightly_latest.tar.gz echo "::set-output name=archive::"${NEW_ARCHIVE_DIR}.tar.gz echo "::set-output name=archive_dir::"${NEW_ARCHIVE_DIR} - name: Upload archive artefact - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: Linux_amd64_archive - path: ./dist/${{ steps.make_dist.outputs.archive }} + path: | + ./dist/${{ steps.make_dist.outputs.archive }} + ./dist/nimbus-eth2_Linux_amd64_nightly_latest.tar.gz retention-days: 2 - name: Upload BN checksum artefact - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: Linux_amd64_checksum path: ./dist/${{ steps.make_dist.outputs.archive_dir }}/build/nimbus_beacon_node.sha512sum retention-days: 2 - name: Upload VC checksum artefact - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: Linux_amd64_checksum path: ./dist/${{ steps.make_dist.outputs.archive_dir }}/build/nimbus_validator_client.sha512sum @@ -76,25 +83,28 @@ jobs: NEW_ARCHIVE_DIR="nimbus-eth2_Linux_arm64v8_$(date +%Y%m%d)_$(git rev-parse --short=8 HEAD)" mv ${ARCHIVE%.tar.gz} ${NEW_ARCHIVE_DIR} tar -czf ${NEW_ARCHIVE_DIR}.tar.gz ${NEW_ARCHIVE_DIR} + cp ${NEW_ARCHIVE_DIR}.tar.gz nimbus-eth2_Linux_arm64v8_nightly_latest.tar.gz echo "::set-output name=archive::"${NEW_ARCHIVE_DIR}.tar.gz echo "::set-output name=archive_dir::"${NEW_ARCHIVE_DIR} - name: Upload archive artefact - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: Linux_arm64_archive - path: ./dist/${{ steps.make_dist.outputs.archive }} + path: | + ./dist/${{ steps.make_dist.outputs.archive }} + ./dist/nimbus-eth2_Linux_arm64v8_nightly_latest.tar.gz retention-days: 2 - name: Upload BN checksum artefact - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: Linux_arm64_checksum path: ./dist/${{ steps.make_dist.outputs.archive_dir }}/build/nimbus_beacon_node.sha512sum retention-days: 2 - name: Upload VC checksum artefact - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: Linux_arm64_checksum path: ./dist/${{ steps.make_dist.outputs.archive_dir }}/build/nimbus_validator_client.sha512sum @@ -127,25 +137,28 @@ jobs: NEW_ARCHIVE_DIR="nimbus-eth2_Linux_arm32v7_$(date +%Y%m%d)_$(git rev-parse --short=8 HEAD)" mv ${ARCHIVE%.tar.gz} ${NEW_ARCHIVE_DIR} tar -czf ${NEW_ARCHIVE_DIR}.tar.gz ${NEW_ARCHIVE_DIR} + cp ${NEW_ARCHIVE_DIR}.tar.gz nimbus-eth2_Linux_arm32v7_nightly_latest.tar.gz echo "::set-output name=archive::"${NEW_ARCHIVE_DIR}.tar.gz echo "::set-output name=archive_dir::"${NEW_ARCHIVE_DIR} - name: Upload archive artefact - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: Linux_arm_archive - path: ./dist/${{ steps.make_dist.outputs.archive }} + path: | + ./dist/${{ steps.make_dist.outputs.archive }} + ./dist/nimbus-eth2_Linux_arm32v7_nightly_latest.tar.gz retention-days: 2 - name: Upload BN checksum artefact - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: Linux_arm_checksum path: ./dist/${{ steps.make_dist.outputs.archive_dir }}/build/nimbus_beacon_node.sha512sum retention-days: 2 - name: Upload VC checksum artefact - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: Linux_arm_checksum path: ./dist/${{ steps.make_dist.outputs.archive_dir }}/build/nimbus_validator_client.sha512sum @@ -170,25 +183,28 @@ jobs: NEW_ARCHIVE_DIR="nimbus-eth2_Windows_amd64_$(date +%Y%m%d)_$(git rev-parse --short=8 HEAD)" mv ${ARCHIVE%.tar.gz} ${NEW_ARCHIVE_DIR} tar -czf ${NEW_ARCHIVE_DIR}.tar.gz ${NEW_ARCHIVE_DIR} + cp ${NEW_ARCHIVE_DIR}.tar.gz nimbus-eth2_Windows_amd64_nightly_latest.tar.gz echo "::set-output name=archive::"${NEW_ARCHIVE_DIR}.tar.gz echo "::set-output name=archive_dir::"${NEW_ARCHIVE_DIR} - name: Upload archive artefact - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: Windows_amd64_archive - path: ./dist/${{ steps.make_dist.outputs.archive }} + path: | + ./dist/${{ steps.make_dist.outputs.archive }} + ./dist/nimbus-eth2_Windows_amd64_nightly_latest.tar.gz retention-days: 2 - name: Upload BN checksum artefact - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: Windows_amd64_checksum path: ./dist/${{ steps.make_dist.outputs.archive_dir }}/build/nimbus_beacon_node.sha512sum retention-days: 2 - name: Upload VC checksum artefact - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: Windows_amd64_checksum path: ./dist/${{ steps.make_dist.outputs.archive_dir }}/build/nimbus_validator_client.sha512sum @@ -213,25 +229,28 @@ jobs: NEW_ARCHIVE_DIR="nimbus-eth2_macOS_amd64_$(date +%Y%m%d)_$(git rev-parse --short=8 HEAD)" mv ${ARCHIVE%.tar.gz} ${NEW_ARCHIVE_DIR} tar -czf ${NEW_ARCHIVE_DIR}.tar.gz ${NEW_ARCHIVE_DIR} + cp ${NEW_ARCHIVE_DIR}.tar.gz nimbus-eth2_macOS_amd64_nightly_latest.tar.gz echo "::set-output name=archive::"${NEW_ARCHIVE_DIR}.tar.gz echo "::set-output name=archive_dir::"${NEW_ARCHIVE_DIR} - name: Upload archive artefact - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: macOS_amd64_archive - path: ./dist/${{ steps.make_dist.outputs.archive }} + path: | + ./dist/${{ steps.make_dist.outputs.archive }} + ./dist/nimbus-eth2_macOS_amd64_nightly_latest.tar.gz retention-days: 2 - name: Upload BN checksum artefact - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: macOS_amd64_checksum path: ./dist/${{ steps.make_dist.outputs.archive_dir }}/build/nimbus_beacon_node.sha512sum retention-days: 2 - name: Upload VC checksum artefact - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: macOS_amd64_checksum path: ./dist/${{ steps.make_dist.outputs.archive_dir }}/build/nimbus_validator_client.sha512sum @@ -256,25 +275,28 @@ jobs: NEW_ARCHIVE_DIR="nimbus-eth2_macOS_arm64_$(date +%Y%m%d)_$(git rev-parse --short=8 HEAD)" mv ${ARCHIVE%.tar.gz} ${NEW_ARCHIVE_DIR} tar -czf ${NEW_ARCHIVE_DIR}.tar.gz ${NEW_ARCHIVE_DIR} + cp ${NEW_ARCHIVE_DIR}.tar.gz nimbus-eth2_macOS_arm64_nightly_latest.tar.gz echo "::set-output name=archive::"${NEW_ARCHIVE_DIR}.tar.gz echo "::set-output name=archive_dir::"${NEW_ARCHIVE_DIR} - name: Upload archive artefact - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: macOS_arm64_archive - path: ./dist/${{ steps.make_dist.outputs.archive }} + path: | + ./dist/${{ steps.make_dist.outputs.archive }} + ./dist/nimbus-eth2_macOS_arm64_nightly_latest.tar.gz retention-days: 2 - name: Upload BN checksum artefact - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: macOS_arm64_checksum path: ./dist/${{ steps.make_dist.outputs.archive_dir }}/build/nimbus_beacon_node.sha512sum retention-days: 2 - name: Upload VC checksum artefact - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: macOS_arm64_checksum path: ./dist/${{ steps.make_dist.outputs.archive_dir }}/build/nimbus_validator_client.sha512sum diff --git a/docker/dist/Dockerfile.amd64 b/docker/dist/Dockerfile.amd64 index b5a0503d6..aba00a080 100644 --- a/docker/dist/Dockerfile.amd64 +++ b/docker/dist/Dockerfile.amd64 @@ -6,6 +6,9 @@ SHELL ["/bin/bash", "-c"] ARG USER_ID ARG GROUP_ID +ENV BUILD_TOOLS=${BUILD_TOOLS} + +RUN echo "BUILD_TOOLS=${BUILD_TOOLS}" RUN addgroup --gid ${GROUP_ID} user; \ adduser --disabled-password --gecos '' --uid ${USER_ID} --gid ${GROUP_ID} user; @@ -16,3 +19,4 @@ STOPSIGNAL SIGINT COPY "entry_point.sh" "/home/user/" ENTRYPOINT ["/home/user/entry_point.sh", "Linux_amd64"] + diff --git a/docker/dist/Dockerfile.arm b/docker/dist/Dockerfile.arm index 793a774b0..e528f3cce 100644 --- a/docker/dist/Dockerfile.arm +++ b/docker/dist/Dockerfile.arm @@ -6,6 +6,8 @@ SHELL ["/bin/bash", "-c"] ARG USER_ID ARG GROUP_ID +ENV BUILD_TOOLS=${BUILD_TOOLS} + RUN addgroup --gid ${GROUP_ID} user; \ adduser --disabled-password --gecos '' --uid ${USER_ID} --gid ${GROUP_ID} user; diff --git a/docker/dist/Dockerfile.arm64 b/docker/dist/Dockerfile.arm64 index 6ca157dee..364b111d0 100644 --- a/docker/dist/Dockerfile.arm64 +++ b/docker/dist/Dockerfile.arm64 @@ -6,6 +6,8 @@ SHELL ["/bin/bash", "-c"] ARG USER_ID ARG GROUP_ID +ENV BUILD_TOOLS=${BUILD_TOOLS} + RUN addgroup --gid ${GROUP_ID} user; \ adduser --disabled-password --gecos '' --uid ${USER_ID} --gid ${GROUP_ID} user; diff --git a/docker/dist/Dockerfile.macos b/docker/dist/Dockerfile.macos index 37b30035b..907915a9b 100644 --- a/docker/dist/Dockerfile.macos +++ b/docker/dist/Dockerfile.macos @@ -6,6 +6,8 @@ SHELL ["/bin/bash", "-c"] ARG USER_ID ARG GROUP_ID +ENV BUILD_TOOLS=${BUILD_TOOLS} + RUN addgroup --gid ${GROUP_ID} user; \ adduser --disabled-password --gecos '' --uid ${USER_ID} --gid ${GROUP_ID} user; diff --git a/docker/dist/Dockerfile.macos-arm64 b/docker/dist/Dockerfile.macos-arm64 index 42bb8f85c..6a18568c3 100644 --- a/docker/dist/Dockerfile.macos-arm64 +++ b/docker/dist/Dockerfile.macos-arm64 @@ -6,6 +6,8 @@ SHELL ["/bin/bash", "-c"] ARG USER_ID ARG GROUP_ID +ENV BUILD_TOOLS=${BUILD_TOOLS} + RUN addgroup --gid ${GROUP_ID} user; \ adduser --disabled-password --gecos '' --uid ${USER_ID} --gid ${GROUP_ID} user; diff --git a/docker/dist/Dockerfile.win64 b/docker/dist/Dockerfile.win64 index 319846bec..dcbf02668 100644 --- a/docker/dist/Dockerfile.win64 +++ b/docker/dist/Dockerfile.win64 @@ -6,6 +6,8 @@ SHELL ["/bin/bash", "-c"] ARG USER_ID ARG GROUP_ID +ENV BUILD_TOOLS=${BUILD_TOOLS} + RUN addgroup --gid ${GROUP_ID} user; \ adduser --disabled-password --gecos '' --uid ${USER_ID} --gid ${GROUP_ID} user; diff --git a/docker/dist/entry_point.sh b/docker/dist/entry_point.sh index bf74c2dbb..1b75172b5 100755 --- a/docker/dist/entry_point.sh +++ b/docker/dist/entry_point.sh @@ -19,6 +19,14 @@ fi PLATFORM="${1}" BINARIES="nimbus_beacon_node nimbus_validator_client" +echo "==================STARTING BUILD==================" +echo "Build Tools = ${BUILD_TOOLS}" + +if [[ "${BUILD_TOOLS}" == "1" ]]; then + echo "Including tools in distribution" + BINARIES="${BINARIES} deposit_contract nimbus_signing_node nimbus_light_client logtrace" +fi + echo -e "\nPLATFORM=${PLATFORM}" #- we need to build everything against libraries available inside this container, including the Nim compiler diff --git a/scripts/make_dist.sh b/scripts/make_dist.sh index f8a229e12..fd3e2fe7e 100755 --- a/scripts/make_dist.sh +++ b/scripts/make_dist.sh @@ -22,6 +22,11 @@ docker rm ${DOCKER_TAG} &>/dev/null || true cd docker/dist +BUILD_TOOLS_ENV="--env BUILD_TOOLS=${BUILD_TOOLS:-"0"}" + +echo "${BUILD_TOOLS_ENV}" + + DOCKER_BUILDKIT=1 \ docker build \ -t ${DOCKER_TAG} \ @@ -31,7 +36,7 @@ DOCKER_BUILDKIT=1 \ -f Dockerfile.${ARCH} . # seccomp can have some serious overhead, so we disable it with "--privileged" - https://pythonspeed.com/articles/docker-performance-overhead/ -docker run --privileged --rm --name ${DOCKER_TAG} -v ${REPO_DIR}:/home/user/nimbus-eth2 ${DOCKER_TAG} +docker run --privileged ${BUILD_TOOLS_ENV} --rm --name ${DOCKER_TAG} -v ${REPO_DIR}:/home/user/nimbus-eth2 ${DOCKER_TAG} cd - &>/dev/null