From 63b340ea339caa1c7203d83a475b97041073b416 Mon Sep 17 00:00:00 2001 From: Pedro Pombeiro Date: Wed, 23 Jan 2019 11:11:31 +0100 Subject: [PATCH] Fix reproducible builds in Docker desktop images Signed-off-by: Pedro Pombeiro --- ci/Jenkinsfile.android | 3 +- ci/Jenkinsfile.linux | 3 +- ci/Jenkinsfile.windows | 3 +- docker/android/Dockerfile | 6 ++-- docker/android/Makefile | 25 +++++++++-------- docker/base/Makefile | 6 +++- docker/linux/Dockerfile | 9 ++++-- docker/linux/Makefile | 15 ++++++++-- docker/windows/Dockerfile | 4 ++- docker/windows/Makefile | 14 ++++++++-- scripts/gen-deps-hash.sh | 47 ++++++++++++++++++++++++++++++++ scripts/prepare-for-platform.sh | 2 ++ scripts/run-environment-check.sh | 2 +- scripts/toolversion | 6 ++-- 14 files changed, 115 insertions(+), 30 deletions(-) create mode 100755 scripts/gen-deps-hash.sh diff --git a/ci/Jenkinsfile.android b/ci/Jenkinsfile.android index 2e7d3b13f7..06110e57a0 100644 --- a/ci/Jenkinsfile.android +++ b/ci/Jenkinsfile.android @@ -2,7 +2,8 @@ pipeline { agent { docker { label 'linux' - image 'statusteam/status-build-android:1.1.0' + /* WARNING: remember to keep this up-to-date with the value in docker/android/Makefile */ + image 'statusteam/status-build-android:1.1.0-16a42e06' args ( "-v /home/jenkins/tmp:/var/tmp:rw "+ "-v /home/jenkins/status-im.keystore:/tmp/status-im.keystore:ro" diff --git a/ci/Jenkinsfile.linux b/ci/Jenkinsfile.linux index 4eeb374032..a438b20d94 100644 --- a/ci/Jenkinsfile.linux +++ b/ci/Jenkinsfile.linux @@ -3,7 +3,8 @@ pipeline { /* privileged mode is necessary for fuse */ docker { label 'linux-new' - image 'statusteam/status-build-linux:1.1.0' + /* WARNING: remember to keep this up-to-date with the value in docker/linux/Makefile */ + image 'statusteam/status-build-linux:1.1.0-f653fffb' args ( "--privileged "+ "-v /dev/fuse:/dev/fuse "+ diff --git a/ci/Jenkinsfile.windows b/ci/Jenkinsfile.windows index 6778e04e2e..f3793da88a 100644 --- a/ci/Jenkinsfile.windows +++ b/ci/Jenkinsfile.windows @@ -3,7 +3,8 @@ pipeline { /* privileged mode is necessary for fuse */ docker { label 'linux-new' - image 'statusteam/status-build-windows:1.1.0' + /* WARNING: remember to keep this up-to-date with the value in docker/windows/Makefile */ + image 'statusteam/status-build-windows:1.1.0-c37b3fa2' args ( "--privileged "+ "-v /dev/fuse:/dev/fuse "+ diff --git a/docker/android/Dockerfile b/docker/android/Dockerfile index 309dd401b6..49765e9c2a 100644 --- a/docker/android/Dockerfile +++ b/docker/android/Dockerfile @@ -1,5 +1,7 @@ # This image is only for extracting and cleaning up NDK and SDK -FROM statusteam/status-build-base:1.1.0 AS sdk_and_ndk +ARG BASE_IMAGE_TAG + +FROM statusteam/status-build-base:1.1.0-${BASE_IMAGE_TAG} AS sdk_and_ndk ARG ANDROID_NDK_VERSION ARG ANDROID_SDK_VERSION @@ -38,7 +40,7 @@ RUN cd /usr/lib/android-ndk && rm -fr docs tests samples \ && find sources -mindepth 2 -maxdepth 2 | grep -v 'gnu-libstdc' | xargs rm -fr ################################################################################ -FROM statusteam/status-build-base:1.1.0 +FROM statusteam/status-build-base:1.1.0-${BASE_IMAGE_TAG} ARG ANDROID_NDK_VERSION ARG ANDROID_SDK_VERSION diff --git a/docker/android/Makefile b/docker/android/Makefile index ed4094f0f5..8a2334515e 100644 --- a/docker/android/Makefile +++ b/docker/android/Makefile @@ -17,17 +17,20 @@ SDK_PLATFORM_VERSION=$(call __toolversion, android-sdk-platform) SDK_BUILD_TOOLS_VERSION=$(call __toolversion, android-sdk-build-tools) # WARNING: Remember to change the tag when updating the image -IMAGE_TAG = 1.1.0 +BASE_IMAGE_TAG = $(shell cd ../base && make get-image-tag) +DEPS_HASH = $(shell $(GIT_ROOT)/scripts/gen-deps-hash.sh -b $(BASE_IMAGE_TAG) -d android-ndk -d android-sdk -d android-sdk-platform -d android-sdk-build-tools) +IMAGE_TAG = 1.1.0-$(DEPS_HASH) IMAGE_NAME = statusteam/status-build-android:$(IMAGE_TAG) build: $(ANDROID_NDK_ARCHIVE) $(ANDROID_SDK_ARCHIVE) docker build \ - --build-arg="ANDROID_NDK_VERSION=$(ANDROID_NDK_VERSION)" \ - --build-arg="ANDROID_SDK_VERSION=$(ANDROID_SDK_VERSION)" \ - --build-arg="SDK_PLATFORM_VERSION=$(SDK_PLATFORM_VERSION)" \ - --build-arg="SDK_BUILD_TOOLS_VERSION=$(SDK_BUILD_TOOLS_VERSION)" \ - --label="commit=$(GIT_COMMIT)" \ - -t $(IMAGE_NAME) . + --build-arg="BASE_IMAGE_TAG=$(BASE_IMAGE_TAG)" \ + --build-arg="ANDROID_NDK_VERSION=$(ANDROID_NDK_VERSION)" \ + --build-arg="ANDROID_SDK_VERSION=$(ANDROID_SDK_VERSION)" \ + --build-arg="SDK_PLATFORM_VERSION=$(SDK_PLATFORM_VERSION)" \ + --build-arg="SDK_BUILD_TOOLS_VERSION=$(SDK_BUILD_TOOLS_VERSION)" \ + --label="commit=$(GIT_COMMIT)" \ + -t $(IMAGE_NAME) . $(ANDROID_NDK_ARCHIVE): wget -q "$(ANDROID_NDK_URL)" -O "$(ANDROID_NDK_ARCHIVE)" @@ -39,11 +42,11 @@ $(ANDROID_SDK_ARCHIVE): test: ## Run build inside the image as a test docker run -u $(shell id -u):$(shell id -g) \ - --name android-test --rm \ + --name android-test --rm \ --tmpfs /var/tmp:rw,size=1G,exec,mode=1777 \ - -v $(GIT_ROOT):/repo:rw \ - -w /repo $(IMAGE_NAME) \ - docker/android/build.sh + -v $(GIT_ROOT):/repo:rw \ + -w /repo $(IMAGE_NAME) \ + docker/android/build.sh push: build docker push $(IMAGE_NAME) diff --git a/docker/base/Makefile b/docker/base/Makefile index 928d4d1e37..a374817154 100644 --- a/docker/base/Makefile +++ b/docker/base/Makefile @@ -4,9 +4,13 @@ GIT_COMMIT = $(shell git rev-parse --short HEAD) GIT_ROOT = $(shell git rev-parse --show-toplevel) # WARNING: Remember to change the tag when updating the image -IMAGE_TAG = 1.1.0 +DEPS_HASH = $(shell $(GIT_ROOT)/scripts/gen-deps-hash.sh -d leiningen -d node -d yarn) +IMAGE_TAG = 1.1.0-$(DEPS_HASH) IMAGE_NAME = statusteam/status-build-base:$(IMAGE_TAG) +get-image-tag: + @echo $(DEPS_HASH) + build: nvm_install.sh docker build \ --build-arg="LEIN_VERSION=$(call __toolversion, leiningen)" \ diff --git a/docker/linux/Dockerfile b/docker/linux/Dockerfile index 1e4b2b458e..267cc231be 100644 --- a/docker/linux/Dockerfile +++ b/docker/linux/Dockerfile @@ -1,4 +1,6 @@ -FROM statusteam/status-build-base:1.1.0 AS qt_build +ARG BASE_IMAGE_TAG + +FROM statusteam/status-build-base:1.1.0-${BASE_IMAGE_TAG} AS qt_build ARG QT_VERSION ARG RNATIVE_VERSION @@ -36,7 +38,7 @@ RUN mkdir -p /tmp/qtci \ '.*\/(qdoc|qgltf|linguist|designer|assistant|qhelpconverter|qmlprofiler)$' -delete \ && rm -fr ${XDG_RUNTIME_DIR} -FROM statusteam/status-build-base:1.1.0 +FROM statusteam/status-build-base:1.1.0-${BASE_IMAGE_TAG} ARG QT_VERSION ARG CMAKE_VERSION @@ -47,7 +49,8 @@ RUN ln -s /opt/qt/mkspecs /usr/local/mkspecs \ # We have to do this because Jenkins doesn't let us # https://issues.jenkins-ci.org/browse/JENKINS-49076 -ENV PATH /opt/qt/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +ARG NODE_VERSION +ENV PATH=/opt/qt/bin:/home/jenkins/.nvm/versions/node/v${NODE_VERSION}/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin RUN apt-get update \ && add-apt-repository -y ppa:git-core/ppa \ diff --git a/docker/linux/Makefile b/docker/linux/Makefile index c111c19fef..b3bd48b949 100644 --- a/docker/linux/Makefile +++ b/docker/linux/Makefile @@ -8,18 +8,27 @@ QT_MD5SUM = $(call __toolversion, qt_md5) QT_ARCHIVE = qt-opensource-linux-x64-$(QT_VERSION).run QT_URL = https://download.qt.io/archive/qt -# WARNING: Remember to change the tag when updating the image -IMAGE_TAG = 1.1.0 +# WARNING: Remember to update `ci/Jenkinsfile.*` with the same IMAGE_TAG value +BASE_IMAGE_TAG = $(shell cd ../base && make get-image-tag) +DEPS_HASH = $(shell $(GIT_ROOT)/scripts/gen-deps-hash.sh -b $(BASE_IMAGE_TAG) -d cmake -d node -d qt -d qtci -d react_native_desktop) +IMAGE_TAG = 1.1.0-$(DEPS_HASH) IMAGE_NAME = statusteam/status-build-linux:$(IMAGE_TAG) build: $(QT_ARCHIVE) + @ if [ "${DEPS_HASH}" = "" ]; then \ + echo "DEPS_HASH not set"; \ + exit 1; \ + fi + docker build \ + --build-arg="BASE_IMAGE_TAG=$(BASE_IMAGE_TAG)" \ + --build-arg="NODE_VERSION=$(call __toolversion, node)" \ --build-arg="QT_VERSION=$(QT_VERSION)" \ --build-arg="QT_CI_COMMIT=$(call __toolversion, qtci)" \ --build-arg="RNATIVE_VERSION=$(call __toolversion, react_native_desktop)" \ --build-arg="CMAKE_VERSION=$(call __toolversion, cmake)" \ --label="commit=$(GIT_COMMIT)" \ - -t $(IMAGE_NAME) .; \ + -t $(IMAGE_NAME) . $(QT_ARCHIVE): wget $(QT_URL)/$(call __major_version__, $(QT_VERSION))/$(QT_VERSION)/$(QT_ARCHIVE) diff --git a/docker/windows/Dockerfile b/docker/windows/Dockerfile index cf50a40e34..d9d08a13ad 100644 --- a/docker/windows/Dockerfile +++ b/docker/windows/Dockerfile @@ -1,4 +1,6 @@ -FROM statusteam/status-build-base:1.1.0 +ARG BASE_IMAGE_TAG + +FROM statusteam/status-build-base:1.1.0-${BASE_IMAGE_TAG} ARG RNATIVE_VERSION ARG CONAN_VERSION diff --git a/docker/windows/Makefile b/docker/windows/Makefile index 06c179688c..9c5c0db230 100644 --- a/docker/windows/Makefile +++ b/docker/windows/Makefile @@ -3,17 +3,25 @@ __toolversion = $(shell $(GIT_ROOT)/scripts/toolversion $(1)) GIT_COMMIT = $(shell git rev-parse --short HEAD) GIT_ROOT = $(shell git rev-parse --show-toplevel) -# WARNING: Remember to change the tag when updating the image -IMAGE_TAG = 1.1.0 +# WARNING: Remember to update `ci/Jenkinsfile.*` with the same IMAGE_TAG value +BASE_IMAGE_TAG = $(shell cd ../base && make get-image-tag) +DEPS_HASH = $(shell $(GIT_ROOT)/scripts/gen-deps-hash.sh -b $(BASE_IMAGE_TAG) -d cmake -d conan -d react_native_desktop) +IMAGE_TAG = 1.1.0-$(DEPS_HASH) IMAGE_NAME = statusteam/status-build-windows:$(IMAGE_TAG) build: + @ if [ "${DEPS_HASH}" = "" ]; then \ + echo "DEPS_HASH not set"; \ + exit 1; \ + fi + docker build \ + --build-arg="BASE_IMAGE_TAG=$(BASE_IMAGE_TAG)" \ --build-arg="RNATIVE_VERSION=$(call __toolversion, react_native_desktop)" \ --build-arg="CONAN_VERSION=$(call __toolversion, conan)" \ --build-arg="CMAKE_VERSION=$(call __toolversion, cmake)" \ --label="commit=$(GIT_COMMIT)" \ - -t $(IMAGE_NAME) .; \ + -t $(IMAGE_NAME) . push: build docker push $(IMAGE_NAME) diff --git a/scripts/gen-deps-hash.sh b/scripts/gen-deps-hash.sh new file mode 100755 index 0000000000..4dcdc4fd8c --- /dev/null +++ b/scripts/gen-deps-hash.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash + +################################################################################ +# This tool fetches versions of build tools from the .TOOLVERSIONS +# file in project root and calculates a single hash that represents +# the combined versions of all the specified tools. +################################################################################ + +GIT_ROOT=$(git rev-parse --show-toplevel) +toolversion="${GIT_ROOT}/scripts/toolversion" + +usage () { + echo "Usage: gen-deps-hash [-b ] -d [-d ]" >&2 + echo + echo "This script calculates a hash representing the required versions of the specified tools" + exit 0 +} + +# some options parsing +deps=() +while getopts "hb:d:" opt; do + case $opt in + b) base_hash="$OPTARG";; + d) + version=$($toolversion "$OPTARG") + if [ $? -ne 0 ]; then + echo "ERROR: $OPTARG not found in .TOOLVERSIONS" + exit 1 + fi + deps+=("$OPTARG $version") + ;; + h) usage;; + \?) echo "Invalid option: -$OPTARG" >&2; exit 1;; + esac +done + +if [ ${#deps[@]} -eq 0 ]; then + echo "ERROR: No dependencies specified" + echo + usage +fi + +IFS=$'\n' sorted_deps=($(sort <<<"${deps[*]}")) +unset IFS + +hash=$(echo "${base_hash}${sorted_deps[@]}" | md5sum | cut -f1 -d" ") +echo "${hash:0:8}" diff --git a/scripts/prepare-for-platform.sh b/scripts/prepare-for-platform.sh index 789e898c63..ecb47f9844 100755 --- a/scripts/prepare-for-platform.sh +++ b/scripts/prepare-for-platform.sh @@ -1,5 +1,7 @@ #!/bin/bash +set -e + RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' diff --git a/scripts/run-environment-check.sh b/scripts/run-environment-check.sh index e977c69ae0..d128a110a7 100755 --- a/scripts/run-environment-check.sh +++ b/scripts/run-environment-check.sh @@ -56,7 +56,7 @@ fi if [[ $PLATFORM == 'android' ]]; then _localPropertiesPath=./android/local.properties - if ! grep -Fq "ndk.dir" $_localPropertiesPath; then + if ! grep -Fq "ndk.dir" $_localPropertiesPath > /dev/null; then if [ -z $ANDROID_NDK_HOME ]; then echo -e "${GREEN}NDK directory not configured, please run 'make setup' or add the line to ${_localPropertiesPath}!${NC}" exit 1 diff --git a/scripts/toolversion b/scripts/toolversion index 1f462a636a..afb26df2a7 100755 --- a/scripts/toolversion +++ b/scripts/toolversion @@ -14,7 +14,7 @@ TOOL_VERSIONS_FILE="${GIT_ROOT}/.TOOLVERSIONS" usage () { echo "Usage: toolversion [-c] " >&2 echo - echo "This script extract tooling versions from ${TOOL_VERSIONS_FILE}" + echo "This script extracts tooling versions from ${TOOL_VERSIONS_FILE}" exit 0 } @@ -33,7 +33,9 @@ if [[ -z "${1}" ]]; then usage; fi NAME=${1} getColumn () { - awk -F';' "/^${NAME};/{print \$${1}}" "${TOOL_VERSIONS_FILE}" + local out=$(awk -F';' "/^${NAME};/{print \$${1}}" "${TOOL_VERSIONS_FILE}") + [ -z "$out" ] && exit 1 + echo "$out" } if [[ $CHECKSUM ]]; then