add Jenkinsfiles and Dockerfile for CI

Changes:
- Adds `ci/Dockerfile` for creating `statusteam/nim-status-client-build:latest` used in builds
- Adds `ci/Jenkinsfile.linux` and `ci/Jenkinsfile.macos` for respective platforms
- Simplifies MacOS signing by adding `scripts/sign-macos-pkg.sh` script
- Makes `Makefile` use `scripts/sign-macos-pkg.sh` to make the DMG
- Makes `APPIMAGE` and `DMG` in `Makefile` modifiable by environment
- Adds `--passL:"-headerpad_max_install_names"` to `NIM_PARAMS` to fix MacOS signing issues

Signed-off-by: Jakub Sokołowski <jakub@status.im>

Makefile

@@ -25,6 +25,7 @@ BUILD_SYSTEM_DIR := vendor/nimbus-build-system
 	pkg-linux \
 	pkg-macos \
 	run \
+	status-go \
 	update
 
 ifeq ($(NIM_PARAMS),)
@@ -95,7 +96,9 @@ bottles-macos: | $(BOTTLE_OPENSSL) $(BOTTLE_PCRE)
 	rm -rf bottles/Downloads
 
 ifeq ($(detected_OS), Darwin)
- NIM_PARAMS := $(NIM_PARAMS) -L:"-framework Foundation -framework Security -framework IOKit -framework CoreServices"
+ NIM_PARAMS += -L:"-framework Foundation -framework Security -framework IOKit -framework CoreServices"
+ # Fix for failures due to 'can't allocate code signature data for'
+ NIM_PARAMS += --passL:"-headerpad_max_install_names"
 endif
 
 DOTHERSIDE := vendor/DOtherSide/build/lib/libDOtherSideStatic.a
@@ -105,7 +108,7 @@ QT5_PCFILEDIR := $(shell pkg-config --variable=pcfiledir Qt5Core 2>/dev/null)
 QT5_LIBDIR := $(shell pkg-config --variable=libdir Qt5Core 2>/dev/null)
 ifeq ($(QT5_PCFILEDIR),)
  ifeq ($(QTDIR),)
-  $(error Can't find your Qt5 installation. Please run "$(MAKE) QTDIR=/path/to/your/Qt5/installation/prefix ...")
+  $(error Cannot find your Qt5 installation. Please run "$(MAKE) QTDIR=/path/to/your/Qt5/installation/prefix ...")
  else
   ifeq ($(detected_OS), Darwin)
    QT5_PCFILEDIR := $(QTDIR)/lib/pkgconfig
@@ -140,11 +143,18 @@ $(DOTHERSIDE): | deps
 		mkdir -p build && \
 		cd build && \
 		rm -f CMakeCache.txt && \
-		cmake -DCMAKE_BUILD_TYPE=Release -DENABLE_DOCS=OFF -DENABLE_TESTS=OFF -DENABLE_DYNAMIC_LIBS=OFF -DENABLE_STATIC_LIBS=ON .. $(HANDLE_OUTPUT) && \
+		cmake \
+			-DCMAKE_BUILD_TYPE=Release \
+			-DENABLE_DOCS=OFF \
+			-DENABLE_TESTS=OFF \
+			-DENABLE_DYNAMIC_LIBS=OFF \
+			-DENABLE_STATIC_LIBS=ON \
+			.. $(HANDLE_OUTPUT) && \
 		$(MAKE) VERBOSE=$(V) $(HANDLE_OUTPUT)
 
 STATUSGO := vendor/status-go/build/bin/libstatus.a
 
+status-go: $(STATUSGO)
 $(STATUSGO): | deps
 	echo -e $(BUILD_MSG) "status-go"
 	+ cd vendor/status-go && \
@@ -171,9 +181,9 @@ $(APPIMAGE_TOOL):
 	mv $(_APPIMAGE_TOOL) tmp/linux/tools/
 	chmod +x $(APPIMAGE_TOOL)
 
-APPIMAGE := pkg/NimStatusClient-x86_64.AppImage
+STATUS_CLIENT_APPIMAGE ?= pkg/NimStatusClient-x86_64.AppImage
 
-$(APPIMAGE): nim_status_client $(APPIMAGE_TOOL) nim-status.desktop
+$(STATUS_CLIENT_APPIMAGE): nim_status_client $(APPIMAGE_TOOL) nim-status.desktop
 	rm -rf pkg/*.AppImage
 	mkdir -p tmp/linux/dist/usr/bin
 	mkdir -p tmp/linux/dist/usr/lib
@@ -192,7 +202,7 @@ $(APPIMAGE): nim_status_client $(APPIMAGE_TOOL) nim-status.desktop
 	cp AppRun tmp/linux/dist/.
 
 	mkdir -p pkg
-	$(APPIMAGE_TOOL) tmp/linux/dist $(APPIMAGE)
+	$(APPIMAGE_TOOL) tmp/linux/dist $(STATUS_CLIENT_APPIMAGE)
 
 DMG_TOOL := node_modules/.bin/create-dmg
 
@@ -202,17 +212,9 @@ $(DMG_TOOL):
 MACOS_OUTER_BUNDLE := tmp/macos/dist/Status.app
 MACOS_INNER_BUNDLE := $(MACOS_OUTER_BUNDLE)/Contents/Frameworks/QtWebEngineCore.framework/Versions/Current/Helpers/QtWebEngineProcess.app
 
-DMG := pkg/Status.dmg
+STATUS_CLIENT_DMG ?= pkg/Status.dmg
 
-# it's not required to set MACOS_KEYCHAIN if MACOS_CODESIGN_IDENT can be found
+$(STATUS_CLIENT_DMG): nim_status_client $(DMG_TOOL)
-# in e.g. your login keychain; this environment variable is primarily useful
-# for CI; when specified MACOS_KEYCHAIN should be the path to a preferred
-# keychain database file
-ifneq ($(MACOS_KEYCHAIN),)
- MACOS_KEYCHAIN_OPT := --keychain "$(MACOS_KEYCHAIN)"
-endif
-
-$(DMG): nim_status_client $(DMG_TOOL)
 	rm -rf tmp/macos pkg/*.dmg
 	mkdir -p $(MACOS_OUTER_BUNDLE)/Contents/MacOS
 	mkdir -p $(MACOS_OUTER_BUNDLE)/Contents/Resources
@@ -234,26 +236,11 @@ $(DMG): nim_status_client $(DMG_TOOL)
 
 	# if MACOS_CODESIGN_IDENT is not set then the outer and inner .app
 	# bundles are not signed
-	[ -z "$(MACOS_CODESIGN_IDENT)" ] || \
+ifdef MACOS_CODESIGN_IDENT
-		codesign \
+	scripts/sign-macos-pkg.sh $(MACOS_OUTER_BUNDLE) $(MACOS_CODESIGN_IDENT)
-			--sign "$(MACOS_CODESIGN_IDENT)" \
+	scripts/sign-macos-pkg.sh $(MACOS_INNER_BUNDLE) $(MACOS_CODESIGN_IDENT) \
-			$(MACOS_KEYCHAIN_OPT) \
+		--entitlements QtWebEngineProcess.plist
-			--options runtime \
+endif
-			--deep \
-			--force \
-			--verbose=4 \
-			$(MACOS_OUTER_BUNDLE)
-	[ -z "$(MACOS_CODESIGN_IDENT)" ] || \
-		codesign \
-			--sign "$(MACOS_CODESIGN_IDENT)" \
-			$(MACOS_KEYCHAIN_OPT) \
-			--entitlements QtWebEngineProcess.plist \
-			--options runtime \
-			--deep \
-			--force \
-			--verbose=4 \
-			$(MACOS_INNER_BUNDLE)
-
 	mkdir -p pkg
 	# See: https://github.com/sindresorhus/create-dmg#dmg-icon
 	# GraphicsMagick must be installed for create-dmg to make the custom
@@ -262,28 +249,21 @@ $(DMG): nim_status_client $(DMG_TOOL)
 		--identity="NOBODY" \
 		$(MACOS_OUTER_BUNDLE) \
 		pkg || true
-	# `|| true` is used above because code signing will be done manually
+	# We ignore failure above create-dmg can't skip signing.
-	# below (to allow for MACOS_KEYCHAIN_OPT) but create-dmg doesn't have
+	# To work around that a dummy identity - 'NOBODY' - is specified.
-	# an option to not attempt signing. To work around that limitation an
+	# This causes non-zero exit code despite DMG being created.
-	# unlikely identity (NOBODY) is specified; this results in a non-zero
+	# It is just not signed, hence the next command should succeed.
-	# exit code even though the .dmg is created successfully (just not code
+	mv "`ls pkg/*.dmg`" $(STATUS_CLIENT_DMG)
-	# signed); if the above command failed to create a .dmg then the
-	# following command should result in a non-zero exit code
-	mv "`ls pkg/*.dmg`" pkg/Status.dmg
 
-	# if MACOS_CODESIGN_IDENT is not set then the .dmg is not signed
+ifdef MACOS_CODESIGN_IDENT
-	[ -z "$(MACOS_CODESIGN_IDENT)" ] || \
+	scripts/sign-macos-pkg.sh $(STATUS_CLIENT_DMG) $(MACOS_CODESIGN_IDENT)
-		codesign \
+endif
-			--sign "$(MACOS_CODESIGN_IDENT)" \
-			$(MACOS_KEYCHAIN_OPT) \
-			--verbose=4 \
-			pkg/Status.dmg
 
 pkg: $(PKG_TARGET)
 
-pkg-linux: $(APPIMAGE)
+pkg-linux: $(STATUS_CLIENT_APPIMAGE)
 
-pkg-macos: $(DMG)
+pkg-macos: $(STATUS_CLIENT_DMG)
 
 clean: | clean-common
 	rm -rf bin/* node_modules pkg/* tmp/* $(STATUSGO)

ci/Dockerfile

@@ -0,0 +1,37 @@
+FROM a12e/docker-qt:5.14-gcc_64
+
+RUN export DEBIAN_FRONTEND=noninteractive \
+ && sudo apt update -yq \
+ && sudo apt install -yq software-properties-common \
+ && sudo add-apt-repository -y ppa:git-core/ppa \
+ && sudo apt update -yq \
+ && sudo apt install -yq --fix-missing \
+        build-essential cmake git libpcre3-dev
+
+# Installing Golang
+RUN GOLANG_SHA256="aed845e4185a0b2a3c3d5e1d0a35491702c55889192bb9c30e67a3de6849c067" \
+ && GOLANG_TARBALL="go1.14.4.linux-amd64.tar.gz" \
+ && wget -q "https://dl.google.com/go/${GOLANG_TARBALL}" \
+ && echo "${GOLANG_SHA256} ${GOLANG_TARBALL}" | sha256sum -c \
+ && sudo tar -C /usr/local -xzf "${GOLANG_TARBALL}" \
+ && rm "${GOLANG_TARBALL}" \
+ && sudo ln -s /usr/local/go/bin/go /usr/local/bin
+
+# $QT_PATH and $QT_PLATFORM are provided by the docker image
+# $QT_PATH/$QT_VERSION/$QT_PLATFORM/bin is already prepended to $PATH
+# However $QT_VERSION is not exposed to environment so set it here
+ENV QT_VERSION="5.14.0"
+ENV QTDIR="${QT_PATH}/${QT_VERSION}"
+ENV LD_LIBRARY_PATH="${QTDIR}/${QT_PLATFORM}/lib:${LD_LIBRARY_PATH}"
+# $OPENSSL_PREFIX is provided by the docker image
+ENV LIBRARY_PATH="${OPENSSL_PREFIX}/lib:${LIBRARY_PATH}"
+
+# Jenkins user needs a specific UID/GID to work
+RUN sudo groupadd -g 1001 jenkins \
+ && sudo useradd --create-home -u 1001 -g 1001 jenkins
+USER jenkins
+ENV HOME="/home/jenkins"
+
+LABEL maintainer="jakub@status.im"
+LABEL source="https://github.com/status-im/nim-status-client"
+LABEL description="Build image for the Status Desktop client written in Nim."

ci/Jenkinsfile.linux

@@ -0,0 +1,72 @@
+pipeline {
+  agent {
+    docker { 
+      label 'linux'
+      image 'statusteam/nim-status-client-build:latest'
+      /* allows jenkins use cat and mounts '/dev/fuse' for linuxdeployqt */
+      args '--entrypoint="" --cap-add SYS_ADMIN --security-opt apparmor:unconfined --device /dev/fuse'
+    }
+  }
+
+  options {
+    timestamps()
+    /* Prevent Jenkins jobs from running forever */
+    timeout(time: 10, unit: 'MINUTES')
+    /* manage how many builds we keep */
+    buildDiscarder(logRotator(
+      numToKeepStr: '20',
+      daysToKeepStr: '60',
+    ))
+  }
+
+  environment {
+    /* Improve make performance */
+    MAKEFLAGS = '-j4'
+    /* Disable colors in Nim compiler logs */
+    NIMFLAGS = '--colors:off'
+    /* Verbosity of make targets */
+    V = "1"
+    /* Control output the filename */
+    STATUS_CLIENT_APPIMAGE = "pkg/${load('ci/lib.groovy').pkgFilename('linux', 'AppImage')}"
+  }
+
+  stages {
+    stage('Modules') {
+      steps {
+        /* avoid re-compiling Nim by using cache */
+        cache(maxCacheSize: 250, caches: [[
+          $class: 'ArbitraryFileCache',
+          includes: '**/*',
+          path: 'vendor/nimbus-build-system/vendor/Nim/bin'
+        ]]) {
+          sh 'make update'
+        }
+      }
+    }
+
+    stage('Deps') {
+      steps { sh 'make deps' }
+    }
+
+    stage('status-go') {
+      steps { sh 'make status-go' }
+    }
+
+    stage('Client') {
+      steps { sh 'make nim_status_client' }
+    }
+
+    stage('Package') {
+      steps { sh 'make pkg-linux' }
+    }
+
+    stage('Archive') {
+      steps { script {
+        archiveArtifacts(env.STATUS_CLIENT_APPIMAGE)
+      } }
+    }
+  }
+  post {
+    always { cleanWs() }
+  }
+}

ci/Jenkinsfile.macos

@@ -0,0 +1,83 @@
+pipeline {
+  agent {
+    label 'macos'
+  }
+
+  options {
+    timestamps()
+    /* Prevent Jenkins jobs from running forever */
+    timeout(time: 10, unit: 'MINUTES')
+    /* manage how many builds we keep */
+    buildDiscarder(logRotator(
+      numToKeepStr: '20',
+      daysToKeepStr: '60',
+    ))
+  }
+
+  environment {
+    /* Improve make performance */
+    MAKEFLAGS = '-j4'
+    /* Disable colors in Nim compiler logs */
+    NIMFLAGS = '--colors:off'
+    /* Qt location is pre-defined */
+    QTDIR = '/usr/local/qt'
+    PATH = "${env.QTDIR}/clang_64/bin:${env.PATH}"
+    /* Control output the filename */
+    STATUS_CLIENT_DMG = "pkg/${load('ci/lib.groovy').pkgFilename('macos', 'dmg')}"
+  }
+
+  stages {
+    stage('Modules') {
+      steps { 
+        /* avoid re-compiling Nim by using cache */
+        cache(maxCacheSize: 250, caches: [[
+          $class: 'ArbitraryFileCache',
+          includes: '**/*',
+          path: 'vendor/nimbus-build-system/vendor/Nim/bin'
+        ]]) {
+          sh 'make update'
+        }
+      }
+    }
+
+    stage('Deps') {
+      steps { sh 'make deps' }
+    }
+
+    stage('status-go') {
+      steps { sh 'make status-go' }
+    }
+
+    stage('Client') {
+      steps { sh 'make nim_status_client' }
+    }
+
+    stage('Package') { steps {
+      withCredentials([
+        string(
+          credentialsId: 'macos-keychain-identity',
+          variable: 'MACOS_CODESIGN_IDENT'
+        ),
+        string(
+          credentialsId: 'macos-keychain-pass',
+          variable: 'MACOS_KEYCHAIN_PASS'
+        ),
+        file(
+          credentialsId: 'macos-keychain-file',
+          variable: 'MACOS_KEYCHAIN_FILE'
+        ),
+      ]) {
+        sh 'make pkg-macos'
+      }
+    } }
+
+    stage('Archive') {
+      steps { script {
+        archiveArtifacts(env.STATUS_CLIENT_DMG)
+      } }
+    }
+  }
+  post {
+    always { cleanWs() }
+  }
+}

ci/README.md

@@ -0,0 +1,28 @@
+# Description
+
+These `Jenkinsfile`s are used to run CI jobs in Jenkins. You can find them here:
+https://ci.status.im/job/nim-status-client/
+
+# Builds
+
+## Linux
+
+In order to build the Linux version of the application we use a modified `a12e/docker-qt:5.14-gcc_64` Docker image with the addition of Git and Golang.
+
+The image is built with [`Dockerfile`](./Dockerfile) using:
+```
+docker build -t statusteam/nim-status-client-build:latest .
+```
+And pushed to: https://hub.docker.com/r/statusteam/nim-status-client-build
+
+## MacOS
+
+The MacOS builds are run on MacOS hosts and expect Command Line Toold and XCode to be installed, as well as QT being available under `/usr/local/qt`.
+
+It also expects the presence of the following credentials:
+
+* `macos-keychain-identity` - ID of used signing certificate.
+* `macos-keychain-pass` - Password to unlock the keychain.
+* `macos-keychain-file` - Keychain file with the MacOS signing certificate.
+
+You can read about how to create such a keychain [here](https://github.com/status-im/infra-docs/blob/master/articles/macos_signing_keychain.md).

ci/lib.groovy

@@ -0,0 +1,24 @@
+def parentOrCurrentBuild() {
+  def c = currentBuild.rawBuild.getCause(hudson.model.Cause$UpstreamCause)
+  if (c == null) { return currentBuild }
+  return c.getUpstreamRun()
+}
+
+def timestamp() {
+  /* we use parent if available to make timestmaps consistent */
+  def now = new Date(parentOrCurrentBuild().timeInMillis)
+  return now.format('yyMMdd-HHmmss', TimeZone.getTimeZone('UTC'))
+}
+
+def gitCommit() {
+  return env.GIT_COMMIT.take(6)
+}
+
+def pkgFilename(type, ext, arch=null) {
+  /* the grep removes the null arch */
+  return [
+    "StatusIm", timestamp(), gitCommit(), type, arch,
+  ].grep().join('-') + ".${ext}"
+}
+
+return this

docs/README.md

@@ -20,5 +20,9 @@
 * [Tutorial - how to add a new section](https://github.com/status-im/nim-status-client/blob/master/docs/tutorial_adding_section.md)
 * [Tutorial - how to create a custom QML component](https://github.com/status-im/nim-status-client/blob/master/docs/tutorial_custom_component.md)
 
+### Continuous Integration
+* [CI Readme](./ci/README.md)
+* [Jenkins Jobs](https://ci.status.im/job/nim-status-client/) 
+
 ### API
 * [QML Nim-Status-Client API reference](https://github.com/status-im/nim-status-client/blob/master/docs/qml_api.md)

scripts/sign-macos-pkg.sh

@@ -0,0 +1,88 @@
+#!/usr/bin/env bash
+
+set -e
+
+[[ $(uname) != 'Darwin' ]] && { echo 'This only works on macOS.' >&2; exit 1; }
+[[ $# -lt 2 ]] && { echo 'sign-macos-bundle.sh <file_to_sign> <sign_identity>' >&2; exit 1; }
+
+# First is the target file/directory to sign
+TARGET="${1}"
+# Second argument is the signing identity
+CODESIGN_ID="${2}"
+# Rest are extra command line flags for codesign
+shift 2
+CODESIGN_OPTS_EXTRA=("${@}")
+
+[[ ! -e "${TARGET}" ]] && { echo 'Target file does not exist.' >&2; exit 1; }
+
+function clean_up {
+    STATUS=$?
+    [[ $? -eq 0 ]] || echo -e "\n###### ERROR: See above for details."
+    set +e
+
+    echo -e "\n###### Cleaning up..."
+    echo -e "\n### Locking keychain..."
+    security lock-keychain "${MACOS_KEYCHAIN_FILE}"
+    echo -e "\n### Restoring default keychain search list..."
+    security list-keychains -s ""
+    security list-keychains
+
+    exit $STATUS
+}
+
+# Flags for codesign
+CODESIGN_OPTS=(
+    "--sign ${CODESIGN_ID}"
+    "--options runtime"
+    "--verbose=4"
+    "--force"
+)
+# Add extra flags provided via command line
+CODESIGN_OPTS+=(
+    ${CODESIGN_OPTS_EXTRA[@]}
+)
+
+# Setting MACOS_KEYCHAIN_FILE nd MACOS_KEYCHAIN_PASS is not required because
+# MACOS_CODESIGN_IDENT can be found in e.g. your login keychain.
+# Those would normally be specified only in CI.
+if [[ -n "${MACOS_KEYCHAIN_FILE}" ]]; then
+    if [[ -z "${MACOS_KEYCHAIN_PASS}" ]]; then
+        echo "Unable to unlock the keychain without MACOS_KEYCHAIN_PASS!" >&2
+        exit 1
+    fi
+    # The keychain file needs to be locked afterwards
+    trap clean_up EXIT ERR
+
+    echo -e "\n### Adding keychain to search list..."
+    security list-keychains -s ${ORIG_KEYCHAIN_LIST} "${MACOS_KEYCHAIN_FILE}"
+    security list-keychains
+    echo -e "\n### Unlocking keychain..."
+    security unlock-keychain -p "${MACOS_KEYCHAIN_PASS}" "${MACOS_KEYCHAIN_FILE}"
+
+    # Add a flag to use the unlocked keychain
+    CODESIGN_OPTS+=("--keychain ${MACOS_KEYCHAIN_FILE}")
+fi
+
+# If 'TARGET' is a directory, we assume it's an app
+# bundle, otherwise we consider it to be a dmg.
+if [[ -d "${TARGET}" ]]; then
+    CODESIGN_OPTS+=("--deep")
+fi
+
+echo -e "\n### Signing target..."
+codesign ${CODESIGN_OPTS[@]} "${TARGET}"
+
+echo -e "\n### Verifying signature..."
+codesign --verify --strict=all --deep --verbose=4 "${TARGET}"
+
+echo -e "\n### Assessing Gatekeeper validation..."
+if [[ -d "${TARGET}" ]]; then
+    spctl --assess --type execute --verbose=2 "${TARGET}"
+else
+    echo "WARNING: The 'open' type security assesment is disabled due to lack of 'Notarization'"
+    # Issue: https://github.com/status-im/status-react/pull/9172
+    # Details: https://developer.apple.com/documentation/security/notarizing_your_app_before_distribution
+    #spctl --assess --type open --context context:primary-signature --verbose=2 "${OBJECT}"
+fi
+
+echo -e "\n###### DONE"