From 85a3557f1e89e95155283b3fd5648b792546c965 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Soko=C5=82owski?= Date: Thu, 18 Jun 2020 18:44:22 +0200 Subject: [PATCH] add Jenkinsfiles and Dockerfile for CI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Changes: - Adds `ci/Dockerfile` for creating `statusteam/nim-status-client-build:latest` used in builds - Adds `ci/Jenkinsfile.linux` and `ci/Jenkinsfile.macos` for respective platforms - Simplifies MacOS signing by adding `scripts/sign-macos-pkg.sh` script - Makes `Makefile` use `scripts/sign-macos-pkg.sh` to make the DMG - Makes `APPIMAGE` and `DMG` in `Makefile` modifiable by environment - Adds `--passL:"-headerpad_max_install_names"` to `NIM_PARAMS` to fix MacOS signing issues Signed-off-by: Jakub SokoĊ‚owski --- Makefile | 86 +++++++++++++++----------------------- ci/Dockerfile | 37 ++++++++++++++++ ci/Jenkinsfile.linux | 72 ++++++++++++++++++++++++++++++++ ci/Jenkinsfile.macos | 83 ++++++++++++++++++++++++++++++++++++ ci/README.md | 28 +++++++++++++ ci/lib.groovy | 24 +++++++++++ docs/README.md | 4 ++ scripts/sign-macos-pkg.sh | 88 +++++++++++++++++++++++++++++++++++++++ 8 files changed, 369 insertions(+), 53 deletions(-) create mode 100644 ci/Dockerfile create mode 100644 ci/Jenkinsfile.linux create mode 100644 ci/Jenkinsfile.macos create mode 100644 ci/README.md create mode 100644 ci/lib.groovy create mode 100755 scripts/sign-macos-pkg.sh diff --git a/Makefile b/Makefile index 2fcfae76f3..73b05b1309 100644 --- a/Makefile +++ b/Makefile @@ -25,6 +25,7 @@ BUILD_SYSTEM_DIR := vendor/nimbus-build-system pkg-linux \ pkg-macos \ run \ + status-go \ update ifeq ($(NIM_PARAMS),) @@ -95,7 +96,9 @@ bottles-macos: | $(BOTTLE_OPENSSL) $(BOTTLE_PCRE) rm -rf bottles/Downloads ifeq ($(detected_OS), Darwin) - NIM_PARAMS := $(NIM_PARAMS) -L:"-framework Foundation -framework Security -framework IOKit -framework CoreServices" + NIM_PARAMS += -L:"-framework Foundation -framework Security -framework IOKit -framework CoreServices" + # Fix for failures due to 'can't allocate code signature data for' + NIM_PARAMS += --passL:"-headerpad_max_install_names" endif DOTHERSIDE := vendor/DOtherSide/build/lib/libDOtherSideStatic.a @@ -105,7 +108,7 @@ QT5_PCFILEDIR := $(shell pkg-config --variable=pcfiledir Qt5Core 2>/dev/null) QT5_LIBDIR := $(shell pkg-config --variable=libdir Qt5Core 2>/dev/null) ifeq ($(QT5_PCFILEDIR),) ifeq ($(QTDIR),) - $(error Can't find your Qt5 installation. Please run "$(MAKE) QTDIR=/path/to/your/Qt5/installation/prefix ...") + $(error Cannot find your Qt5 installation. Please run "$(MAKE) QTDIR=/path/to/your/Qt5/installation/prefix ...") else ifeq ($(detected_OS), Darwin) QT5_PCFILEDIR := $(QTDIR)/lib/pkgconfig @@ -140,11 +143,18 @@ $(DOTHERSIDE): | deps mkdir -p build && \ cd build && \ rm -f CMakeCache.txt && \ - cmake -DCMAKE_BUILD_TYPE=Release -DENABLE_DOCS=OFF -DENABLE_TESTS=OFF -DENABLE_DYNAMIC_LIBS=OFF -DENABLE_STATIC_LIBS=ON .. $(HANDLE_OUTPUT) && \ + cmake \ + -DCMAKE_BUILD_TYPE=Release \ + -DENABLE_DOCS=OFF \ + -DENABLE_TESTS=OFF \ + -DENABLE_DYNAMIC_LIBS=OFF \ + -DENABLE_STATIC_LIBS=ON \ + .. $(HANDLE_OUTPUT) && \ $(MAKE) VERBOSE=$(V) $(HANDLE_OUTPUT) STATUSGO := vendor/status-go/build/bin/libstatus.a +status-go: $(STATUSGO) $(STATUSGO): | deps echo -e $(BUILD_MSG) "status-go" + cd vendor/status-go && \ @@ -171,9 +181,9 @@ $(APPIMAGE_TOOL): mv $(_APPIMAGE_TOOL) tmp/linux/tools/ chmod +x $(APPIMAGE_TOOL) -APPIMAGE := pkg/NimStatusClient-x86_64.AppImage +STATUS_CLIENT_APPIMAGE ?= pkg/NimStatusClient-x86_64.AppImage -$(APPIMAGE): nim_status_client $(APPIMAGE_TOOL) nim-status.desktop +$(STATUS_CLIENT_APPIMAGE): nim_status_client $(APPIMAGE_TOOL) nim-status.desktop rm -rf pkg/*.AppImage mkdir -p tmp/linux/dist/usr/bin mkdir -p tmp/linux/dist/usr/lib @@ -192,7 +202,7 @@ $(APPIMAGE): nim_status_client $(APPIMAGE_TOOL) nim-status.desktop cp AppRun tmp/linux/dist/. mkdir -p pkg - $(APPIMAGE_TOOL) tmp/linux/dist $(APPIMAGE) + $(APPIMAGE_TOOL) tmp/linux/dist $(STATUS_CLIENT_APPIMAGE) DMG_TOOL := node_modules/.bin/create-dmg @@ -202,17 +212,9 @@ $(DMG_TOOL): MACOS_OUTER_BUNDLE := tmp/macos/dist/Status.app MACOS_INNER_BUNDLE := $(MACOS_OUTER_BUNDLE)/Contents/Frameworks/QtWebEngineCore.framework/Versions/Current/Helpers/QtWebEngineProcess.app -DMG := pkg/Status.dmg +STATUS_CLIENT_DMG ?= pkg/Status.dmg -# it's not required to set MACOS_KEYCHAIN if MACOS_CODESIGN_IDENT can be found -# in e.g. your login keychain; this environment variable is primarily useful -# for CI; when specified MACOS_KEYCHAIN should be the path to a preferred -# keychain database file -ifneq ($(MACOS_KEYCHAIN),) - MACOS_KEYCHAIN_OPT := --keychain "$(MACOS_KEYCHAIN)" -endif - -$(DMG): nim_status_client $(DMG_TOOL) +$(STATUS_CLIENT_DMG): nim_status_client $(DMG_TOOL) rm -rf tmp/macos pkg/*.dmg mkdir -p $(MACOS_OUTER_BUNDLE)/Contents/MacOS mkdir -p $(MACOS_OUTER_BUNDLE)/Contents/Resources @@ -234,26 +236,11 @@ $(DMG): nim_status_client $(DMG_TOOL) # if MACOS_CODESIGN_IDENT is not set then the outer and inner .app # bundles are not signed - [ -z "$(MACOS_CODESIGN_IDENT)" ] || \ - codesign \ - --sign "$(MACOS_CODESIGN_IDENT)" \ - $(MACOS_KEYCHAIN_OPT) \ - --options runtime \ - --deep \ - --force \ - --verbose=4 \ - $(MACOS_OUTER_BUNDLE) - [ -z "$(MACOS_CODESIGN_IDENT)" ] || \ - codesign \ - --sign "$(MACOS_CODESIGN_IDENT)" \ - $(MACOS_KEYCHAIN_OPT) \ - --entitlements QtWebEngineProcess.plist \ - --options runtime \ - --deep \ - --force \ - --verbose=4 \ - $(MACOS_INNER_BUNDLE) - +ifdef MACOS_CODESIGN_IDENT + scripts/sign-macos-pkg.sh $(MACOS_OUTER_BUNDLE) $(MACOS_CODESIGN_IDENT) + scripts/sign-macos-pkg.sh $(MACOS_INNER_BUNDLE) $(MACOS_CODESIGN_IDENT) \ + --entitlements QtWebEngineProcess.plist +endif mkdir -p pkg # See: https://github.com/sindresorhus/create-dmg#dmg-icon # GraphicsMagick must be installed for create-dmg to make the custom @@ -262,28 +249,21 @@ $(DMG): nim_status_client $(DMG_TOOL) --identity="NOBODY" \ $(MACOS_OUTER_BUNDLE) \ pkg || true - # `|| true` is used above because code signing will be done manually - # below (to allow for MACOS_KEYCHAIN_OPT) but create-dmg doesn't have - # an option to not attempt signing. To work around that limitation an - # unlikely identity (NOBODY) is specified; this results in a non-zero - # exit code even though the .dmg is created successfully (just not code - # signed); if the above command failed to create a .dmg then the - # following command should result in a non-zero exit code - mv "`ls pkg/*.dmg`" pkg/Status.dmg + # We ignore failure above create-dmg can't skip signing. + # To work around that a dummy identity - 'NOBODY' - is specified. + # This causes non-zero exit code despite DMG being created. + # It is just not signed, hence the next command should succeed. + mv "`ls pkg/*.dmg`" $(STATUS_CLIENT_DMG) - # if MACOS_CODESIGN_IDENT is not set then the .dmg is not signed - [ -z "$(MACOS_CODESIGN_IDENT)" ] || \ - codesign \ - --sign "$(MACOS_CODESIGN_IDENT)" \ - $(MACOS_KEYCHAIN_OPT) \ - --verbose=4 \ - pkg/Status.dmg +ifdef MACOS_CODESIGN_IDENT + scripts/sign-macos-pkg.sh $(STATUS_CLIENT_DMG) $(MACOS_CODESIGN_IDENT) +endif pkg: $(PKG_TARGET) -pkg-linux: $(APPIMAGE) +pkg-linux: $(STATUS_CLIENT_APPIMAGE) -pkg-macos: $(DMG) +pkg-macos: $(STATUS_CLIENT_DMG) clean: | clean-common rm -rf bin/* node_modules pkg/* tmp/* $(STATUSGO) diff --git a/ci/Dockerfile b/ci/Dockerfile new file mode 100644 index 0000000000..8489db0205 --- /dev/null +++ b/ci/Dockerfile @@ -0,0 +1,37 @@ +FROM a12e/docker-qt:5.14-gcc_64 + +RUN export DEBIAN_FRONTEND=noninteractive \ + && sudo apt update -yq \ + && sudo apt install -yq software-properties-common \ + && sudo add-apt-repository -y ppa:git-core/ppa \ + && sudo apt update -yq \ + && sudo apt install -yq --fix-missing \ + build-essential cmake git libpcre3-dev + +# Installing Golang +RUN GOLANG_SHA256="aed845e4185a0b2a3c3d5e1d0a35491702c55889192bb9c30e67a3de6849c067" \ + && GOLANG_TARBALL="go1.14.4.linux-amd64.tar.gz" \ + && wget -q "https://dl.google.com/go/${GOLANG_TARBALL}" \ + && echo "${GOLANG_SHA256} ${GOLANG_TARBALL}" | sha256sum -c \ + && sudo tar -C /usr/local -xzf "${GOLANG_TARBALL}" \ + && rm "${GOLANG_TARBALL}" \ + && sudo ln -s /usr/local/go/bin/go /usr/local/bin + +# $QT_PATH and $QT_PLATFORM are provided by the docker image +# $QT_PATH/$QT_VERSION/$QT_PLATFORM/bin is already prepended to $PATH +# However $QT_VERSION is not exposed to environment so set it here +ENV QT_VERSION="5.14.0" +ENV QTDIR="${QT_PATH}/${QT_VERSION}" +ENV LD_LIBRARY_PATH="${QTDIR}/${QT_PLATFORM}/lib:${LD_LIBRARY_PATH}" +# $OPENSSL_PREFIX is provided by the docker image +ENV LIBRARY_PATH="${OPENSSL_PREFIX}/lib:${LIBRARY_PATH}" + +# Jenkins user needs a specific UID/GID to work +RUN sudo groupadd -g 1001 jenkins \ + && sudo useradd --create-home -u 1001 -g 1001 jenkins +USER jenkins +ENV HOME="/home/jenkins" + +LABEL maintainer="jakub@status.im" +LABEL source="https://github.com/status-im/nim-status-client" +LABEL description="Build image for the Status Desktop client written in Nim." diff --git a/ci/Jenkinsfile.linux b/ci/Jenkinsfile.linux new file mode 100644 index 0000000000..fd203e80c0 --- /dev/null +++ b/ci/Jenkinsfile.linux @@ -0,0 +1,72 @@ +pipeline { + agent { + docker { + label 'linux' + image 'statusteam/nim-status-client-build:latest' + /* allows jenkins use cat and mounts '/dev/fuse' for linuxdeployqt */ + args '--entrypoint="" --cap-add SYS_ADMIN --security-opt apparmor:unconfined --device /dev/fuse' + } + } + + options { + timestamps() + /* Prevent Jenkins jobs from running forever */ + timeout(time: 10, unit: 'MINUTES') + /* manage how many builds we keep */ + buildDiscarder(logRotator( + numToKeepStr: '20', + daysToKeepStr: '60', + )) + } + + environment { + /* Improve make performance */ + MAKEFLAGS = '-j4' + /* Disable colors in Nim compiler logs */ + NIMFLAGS = '--colors:off' + /* Verbosity of make targets */ + V = "1" + /* Control output the filename */ + STATUS_CLIENT_APPIMAGE = "pkg/${load('ci/lib.groovy').pkgFilename('linux', 'AppImage')}" + } + + stages { + stage('Modules') { + steps { + /* avoid re-compiling Nim by using cache */ + cache(maxCacheSize: 250, caches: [[ + $class: 'ArbitraryFileCache', + includes: '**/*', + path: 'vendor/nimbus-build-system/vendor/Nim/bin' + ]]) { + sh 'make update' + } + } + } + + stage('Deps') { + steps { sh 'make deps' } + } + + stage('status-go') { + steps { sh 'make status-go' } + } + + stage('Client') { + steps { sh 'make nim_status_client' } + } + + stage('Package') { + steps { sh 'make pkg-linux' } + } + + stage('Archive') { + steps { script { + archiveArtifacts(env.STATUS_CLIENT_APPIMAGE) + } } + } + } + post { + always { cleanWs() } + } +} diff --git a/ci/Jenkinsfile.macos b/ci/Jenkinsfile.macos new file mode 100644 index 0000000000..a82b31170d --- /dev/null +++ b/ci/Jenkinsfile.macos @@ -0,0 +1,83 @@ +pipeline { + agent { + label 'macos' + } + + options { + timestamps() + /* Prevent Jenkins jobs from running forever */ + timeout(time: 10, unit: 'MINUTES') + /* manage how many builds we keep */ + buildDiscarder(logRotator( + numToKeepStr: '20', + daysToKeepStr: '60', + )) + } + + environment { + /* Improve make performance */ + MAKEFLAGS = '-j4' + /* Disable colors in Nim compiler logs */ + NIMFLAGS = '--colors:off' + /* Qt location is pre-defined */ + QTDIR = '/usr/local/qt' + PATH = "${env.QTDIR}/clang_64/bin:${env.PATH}" + /* Control output the filename */ + STATUS_CLIENT_DMG = "pkg/${load('ci/lib.groovy').pkgFilename('macos', 'dmg')}" + } + + stages { + stage('Modules') { + steps { + /* avoid re-compiling Nim by using cache */ + cache(maxCacheSize: 250, caches: [[ + $class: 'ArbitraryFileCache', + includes: '**/*', + path: 'vendor/nimbus-build-system/vendor/Nim/bin' + ]]) { + sh 'make update' + } + } + } + + stage('Deps') { + steps { sh 'make deps' } + } + + stage('status-go') { + steps { sh 'make status-go' } + } + + stage('Client') { + steps { sh 'make nim_status_client' } + } + + stage('Package') { steps { + withCredentials([ + string( + credentialsId: 'macos-keychain-identity', + variable: 'MACOS_CODESIGN_IDENT' + ), + string( + credentialsId: 'macos-keychain-pass', + variable: 'MACOS_KEYCHAIN_PASS' + ), + file( + credentialsId: 'macos-keychain-file', + variable: 'MACOS_KEYCHAIN_FILE' + ), + ]) { + sh 'make pkg-macos' + } + } } + + stage('Archive') { + steps { script { + archiveArtifacts(env.STATUS_CLIENT_DMG) + } } + } + } + post { + always { cleanWs() } + } +} diff --git a/ci/README.md b/ci/README.md new file mode 100644 index 0000000000..ada6b41ec3 --- /dev/null +++ b/ci/README.md @@ -0,0 +1,28 @@ +# Description + +These `Jenkinsfile`s are used to run CI jobs in Jenkins. You can find them here: +https://ci.status.im/job/nim-status-client/ + +# Builds + +## Linux + +In order to build the Linux version of the application we use a modified `a12e/docker-qt:5.14-gcc_64` Docker image with the addition of Git and Golang. + +The image is built with [`Dockerfile`](./Dockerfile) using: +``` +docker build -t statusteam/nim-status-client-build:latest . +``` +And pushed to: https://hub.docker.com/r/statusteam/nim-status-client-build + +## MacOS + +The MacOS builds are run on MacOS hosts and expect Command Line Toold and XCode to be installed, as well as QT being available under `/usr/local/qt`. + +It also expects the presence of the following credentials: + +* `macos-keychain-identity` - ID of used signing certificate. +* `macos-keychain-pass` - Password to unlock the keychain. +* `macos-keychain-file` - Keychain file with the MacOS signing certificate. + +You can read about how to create such a keychain [here](https://github.com/status-im/infra-docs/blob/master/articles/macos_signing_keychain.md). diff --git a/ci/lib.groovy b/ci/lib.groovy new file mode 100644 index 0000000000..c2d311c0d4 --- /dev/null +++ b/ci/lib.groovy @@ -0,0 +1,24 @@ +def parentOrCurrentBuild() { + def c = currentBuild.rawBuild.getCause(hudson.model.Cause$UpstreamCause) + if (c == null) { return currentBuild } + return c.getUpstreamRun() +} + +def timestamp() { + /* we use parent if available to make timestmaps consistent */ + def now = new Date(parentOrCurrentBuild().timeInMillis) + return now.format('yyMMdd-HHmmss', TimeZone.getTimeZone('UTC')) +} + +def gitCommit() { + return env.GIT_COMMIT.take(6) +} + +def pkgFilename(type, ext, arch=null) { + /* the grep removes the null arch */ + return [ + "StatusIm", timestamp(), gitCommit(), type, arch, + ].grep().join('-') + ".${ext}" +} + +return this diff --git a/docs/README.md b/docs/README.md index cf97f9c7e9..e4559f1170 100644 --- a/docs/README.md +++ b/docs/README.md @@ -20,5 +20,9 @@ * [Tutorial - how to add a new section](https://github.com/status-im/nim-status-client/blob/master/docs/tutorial_adding_section.md) * [Tutorial - how to create a custom QML component](https://github.com/status-im/nim-status-client/blob/master/docs/tutorial_custom_component.md) +### Continuous Integration +* [CI Readme](./ci/README.md) +* [Jenkins Jobs](https://ci.status.im/job/nim-status-client/) + ### API * [QML Nim-Status-Client API reference](https://github.com/status-im/nim-status-client/blob/master/docs/qml_api.md) diff --git a/scripts/sign-macos-pkg.sh b/scripts/sign-macos-pkg.sh new file mode 100755 index 0000000000..ccf533bbdc --- /dev/null +++ b/scripts/sign-macos-pkg.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash + +set -e + +[[ $(uname) != 'Darwin' ]] && { echo 'This only works on macOS.' >&2; exit 1; } +[[ $# -lt 2 ]] && { echo 'sign-macos-bundle.sh ' >&2; exit 1; } + +# First is the target file/directory to sign +TARGET="${1}" +# Second argument is the signing identity +CODESIGN_ID="${2}" +# Rest are extra command line flags for codesign +shift 2 +CODESIGN_OPTS_EXTRA=("${@}") + +[[ ! -e "${TARGET}" ]] && { echo 'Target file does not exist.' >&2; exit 1; } + +function clean_up { + STATUS=$? + [[ $? -eq 0 ]] || echo -e "\n###### ERROR: See above for details." + set +e + + echo -e "\n###### Cleaning up..." + echo -e "\n### Locking keychain..." + security lock-keychain "${MACOS_KEYCHAIN_FILE}" + echo -e "\n### Restoring default keychain search list..." + security list-keychains -s "" + security list-keychains + + exit $STATUS +} + +# Flags for codesign +CODESIGN_OPTS=( + "--sign ${CODESIGN_ID}" + "--options runtime" + "--verbose=4" + "--force" +) +# Add extra flags provided via command line +CODESIGN_OPTS+=( + ${CODESIGN_OPTS_EXTRA[@]} +) + +# Setting MACOS_KEYCHAIN_FILE nd MACOS_KEYCHAIN_PASS is not required because +# MACOS_CODESIGN_IDENT can be found in e.g. your login keychain. +# Those would normally be specified only in CI. +if [[ -n "${MACOS_KEYCHAIN_FILE}" ]]; then + if [[ -z "${MACOS_KEYCHAIN_PASS}" ]]; then + echo "Unable to unlock the keychain without MACOS_KEYCHAIN_PASS!" >&2 + exit 1 + fi + # The keychain file needs to be locked afterwards + trap clean_up EXIT ERR + + echo -e "\n### Adding keychain to search list..." + security list-keychains -s ${ORIG_KEYCHAIN_LIST} "${MACOS_KEYCHAIN_FILE}" + security list-keychains + echo -e "\n### Unlocking keychain..." + security unlock-keychain -p "${MACOS_KEYCHAIN_PASS}" "${MACOS_KEYCHAIN_FILE}" + + # Add a flag to use the unlocked keychain + CODESIGN_OPTS+=("--keychain ${MACOS_KEYCHAIN_FILE}") +fi + +# If 'TARGET' is a directory, we assume it's an app +# bundle, otherwise we consider it to be a dmg. +if [[ -d "${TARGET}" ]]; then + CODESIGN_OPTS+=("--deep") +fi + +echo -e "\n### Signing target..." +codesign ${CODESIGN_OPTS[@]} "${TARGET}" + +echo -e "\n### Verifying signature..." +codesign --verify --strict=all --deep --verbose=4 "${TARGET}" + +echo -e "\n### Assessing Gatekeeper validation..." +if [[ -d "${TARGET}" ]]; then + spctl --assess --type execute --verbose=2 "${TARGET}" +else + echo "WARNING: The 'open' type security assesment is disabled due to lack of 'Notarization'" + # Issue: https://github.com/status-im/status-react/pull/9172 + # Details: https://developer.apple.com/documentation/security/notarizing_your_app_before_distribution + #spctl --assess --type open --context context:primary-signature --verbose=2 "${OBJECT}" +fi + +echo -e "\n###### DONE"