docs: binary distribution internals
This commit is contained in:
parent
93cfc2d446
commit
979bb39b41
|
@ -17,7 +17,7 @@ jobs:
|
||||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||||
with:
|
with:
|
||||||
tag_name: ${{ github.ref }}
|
tag_name: ${{ github.ref }}
|
||||||
release_name: Release ${{ github.ref }}
|
release_name: ${{ github.ref }}
|
||||||
draft: true
|
draft: true
|
||||||
prerelease: false
|
prerelease: false
|
||||||
- name: Checkout code
|
- name: Checkout code
|
||||||
|
|
|
@ -90,9 +90,18 @@ for BINARY in ${BINARIES}; do
|
||||||
cd - >/dev/null
|
cd - >/dev/null
|
||||||
done
|
done
|
||||||
sed -e "s/GIT_COMMIT/${GIT_COMMIT}/" docker/dist/README.md > "${DIST_PATH}/README.md"
|
sed -e "s/GIT_COMMIT/${GIT_COMMIT}/" docker/dist/README.md > "${DIST_PATH}/README.md"
|
||||||
if [[ "${PLATFORM}" == "Windows_amd64" ]]; then
|
|
||||||
|
if [[ "${PLATFORM}" == "Linux_amd64" ]]; then
|
||||||
|
sed -i -e 's/^make dist$/make dist-amd64/' "${DIST_PATH}/README.md"
|
||||||
|
elif [[ "${PLATFORM}" == "Linux_arm32v7" ]]; then
|
||||||
|
sed -i -e 's/^make dist$/make dist-arm/' "${DIST_PATH}/README.md"
|
||||||
|
elif [[ "${PLATFORM}" == "Linux_arm64v8" ]]; then
|
||||||
|
sed -i -e 's/^make dist$/make dist-arm64/' "${DIST_PATH}/README.md"
|
||||||
|
elif [[ "${PLATFORM}" == "Windows_amd64" ]]; then
|
||||||
|
sed -i -e 's/^make dist$/make dist-win64/' "${DIST_PATH}/README.md"
|
||||||
cp -a docker/dist/README-Windows.md "${DIST_PATH}/"
|
cp -a docker/dist/README-Windows.md "${DIST_PATH}/"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
cp -a scripts/run-beacon-node.sh "${DIST_PATH}/scripts"
|
cp -a scripts/run-beacon-node.sh "${DIST_PATH}/scripts"
|
||||||
cp -a ./run-*-beacon-node.sh "${DIST_PATH}/"
|
cp -a ./run-*-beacon-node.sh "${DIST_PATH}/"
|
||||||
|
|
||||||
|
|
|
@ -40,3 +40,5 @@
|
||||||
- [FAQ](./faq.md)
|
- [FAQ](./faq.md)
|
||||||
- [Contribute](./contribute.md)
|
- [Contribute](./contribute.md)
|
||||||
- [Resources](./resources.md)
|
- [Resources](./resources.md)
|
||||||
|
- [Binary distribution internals](./distribution_internals.md)
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,57 @@
|
||||||
|
# Binary distribution internals
|
||||||
|
|
||||||
|
## Reproducibility
|
||||||
|
|
||||||
|
The binaries we build in GitHub Actions and distribute in GitHub releases come
|
||||||
|
from a convoluted process meant to ensure [reproducibility](https://reproducible-builds.org/).
|
||||||
|
|
||||||
|
While being able to produce the same exact binaries from the corresponding Git
|
||||||
|
commits is a good idea for any open source project, it becomes a requirement
|
||||||
|
for software that deals with digital tokens.
|
||||||
|
|
||||||
|
## Docker containers for internal use
|
||||||
|
|
||||||
|
The easiest way to guarantee that users will be able to replicate those
|
||||||
|
binaries is to give them the same software environment we used in CI. Docker
|
||||||
|
containers fit the bill, so everything starts with the architecture- and
|
||||||
|
OS-specific containers in "docker/dist/base\_image/".
|
||||||
|
|
||||||
|
These images have all the packages we need, are built and published once (to
|
||||||
|
Docker Hub), then reused for a long time as the basis for temporary Docker
|
||||||
|
images where the nimbus-eth2 build is done.
|
||||||
|
|
||||||
|
These temporary images are controlled by Dockerfiles in "docker/dist/" and,
|
||||||
|
since we're not publishing them anywhere, we can customize them to the system
|
||||||
|
they run on, by making them use the host's UID/GID, the host's QEMU static
|
||||||
|
binaries, etc. They get access to the source code through the use of external volumes.
|
||||||
|
|
||||||
|
## Build process
|
||||||
|
|
||||||
|
It all starts from the GitHub actions in ".github/workflows/release.yml". There
|
||||||
|
is a different job for each supported OS-architecture combination and they all
|
||||||
|
run in parallel (ideally).
|
||||||
|
|
||||||
|
The "build-amd64" CI job is special, because it's the one that creates a new
|
||||||
|
GitHub release draft, as soon as possible. All the other jobs will upload their
|
||||||
|
binary distribution tarballs to this draft release but, since it's not feasible
|
||||||
|
to communicate between CI jobs, they'll just use GitHub APIs to find out what
|
||||||
|
the latest release is, check that it has the right Git tag, and use it as their
|
||||||
|
last step.
|
||||||
|
|
||||||
|
The build itself is triggered by a Make target: `make dist-amd64`. This invokes
|
||||||
|
"scripts/make\_dist.sh" which builds the corresponding Docker container from
|
||||||
|
"docker/dist/" and runs it with the Git repository's top dir as an external
|
||||||
|
volume.
|
||||||
|
|
||||||
|
The entry point for that container is "docker/dist/entry\_point.sh" and that's
|
||||||
|
where you'll find the Make invocations needed to finally build the software and
|
||||||
|
create distributable tarballs.
|
||||||
|
|
||||||
|
## Docker images for end users
|
||||||
|
|
||||||
|
Also in ".github/workflows/release.yml", but only for the "build-amd64" job, we
|
||||||
|
unpack the distribution tarball and copy its content into a third type of Docker
|
||||||
|
image - this one meant for end users and defined by "docker/dist/binaries/Dockerfile.amd64".
|
||||||
|
|
||||||
|
We publish that to [Docker Hub](https://hub.docker.com/r/statusim/nimbus-eth2).
|
||||||
|
|
|
@ -13,7 +13,7 @@
|
||||||
set -e
|
set -e
|
||||||
|
|
||||||
cd "$(dirname "${BASH_SOURCE[0]}")"/..
|
cd "$(dirname "${BASH_SOURCE[0]}")"/..
|
||||||
CURDIR="${PWD}"
|
REPO_DIR="${PWD}"
|
||||||
|
|
||||||
ARCH="${1:-amd64}"
|
ARCH="${1:-amd64}"
|
||||||
if [[ "${ARCH}" == "amd64" || "${ARCH}" == "win64" ]]; then
|
if [[ "${ARCH}" == "amd64" || "${ARCH}" == "win64" ]]; then
|
||||||
|
@ -66,7 +66,7 @@ DOCKER_BUILDKIT=1 \
|
||||||
${DOCKER_EXTRA_ARGS} \
|
${DOCKER_EXTRA_ARGS} \
|
||||||
-f Dockerfile.${ARCH} .
|
-f Dockerfile.${ARCH} .
|
||||||
|
|
||||||
docker run --rm --name ${DOCKER_TAG} -v ${CURDIR}:/home/user/nimbus-eth2 ${DOCKER_TAG}
|
docker run --rm --name ${DOCKER_TAG} -v ${REPO_DIR}:/home/user/nimbus-eth2 ${DOCKER_TAG}
|
||||||
|
|
||||||
if [[ "${USE_QEMU}" == "1" ]]; then
|
if [[ "${USE_QEMU}" == "1" ]]; then
|
||||||
rm "${QEMU_NAME}"
|
rm "${QEMU_NAME}"
|
||||||
|
|
|
@ -39,6 +39,7 @@ fi
|
||||||
# Windows detection
|
# Windows detection
|
||||||
if uname | grep -qiE "mingw|msys"; then
|
if uname | grep -qiE "mingw|msys"; then
|
||||||
MAKE="mingw32-make"
|
MAKE="mingw32-make"
|
||||||
|
# This "winpty" wrapper is needed to make Ctrl+C work, on some systems.
|
||||||
WINPTY="winpty --"
|
WINPTY="winpty --"
|
||||||
else
|
else
|
||||||
MAKE="make"
|
MAKE="make"
|
||||||
|
|
Loading…
Reference in New Issue