diff --git a/ci/Jenkinsfile.android b/ci/Jenkinsfile.android index 235d524614..079ef22f3f 100644 --- a/ci/Jenkinsfile.android +++ b/ci/Jenkinsfile.android @@ -2,7 +2,7 @@ pipeline { agent { docker { label 'linux' - image 'statusteam/nix:jenkins' + image 'statusteam/nix:jenkins-1.0.0-4a147dc3' args ( "-v /home/jenkins/tmp:/var/tmp:rw "+ "-v /home/jenkins/status-im.keystore:/tmp/status-im.keystore:ro" diff --git a/ci/Jenkinsfile.linux b/ci/Jenkinsfile.linux index 98d81f2265..f9649f1d3c 100644 --- a/ci/Jenkinsfile.linux +++ b/ci/Jenkinsfile.linux @@ -2,7 +2,7 @@ pipeline { agent { docker { label 'linux' - image 'statusteam/nix:jenkins' + image 'statusteam/nix:jenkins-1.0.0-4a147dc3' args ( "-v /tmp/Android/Sdk:/home/jenkins/.status/Android/Sdk:rw "+ "-v /var/tmp/lein:/var/tmp/lein:rw "+ diff --git a/ci/Jenkinsfile.nix.linux b/ci/Jenkinsfile.nix.linux index 65003fdcf1..4333245f1b 100644 --- a/ci/Jenkinsfile.nix.linux +++ b/ci/Jenkinsfile.nix.linux @@ -3,7 +3,7 @@ pipeline { /* the -u is necessary for acces to /nix */ docker { label 'linux' - image 'statusteam/nix:jenkins' + image 'statusteam/nix:jenkins-1.0.0-4a147dc3' } } diff --git a/ci/Jenkinsfile.windows b/ci/Jenkinsfile.windows index ab19e9c0dd..ece4b53550 100644 --- a/ci/Jenkinsfile.windows +++ b/ci/Jenkinsfile.windows @@ -2,7 +2,7 @@ pipeline { agent { docker { label 'linux-new' - image 'statusteam/nix:jenkins' + image 'statusteam/nix:jenkins-1.0.0-4a147dc3' args ( "-v /var/tmp/lein:/var/tmp/lein:rw "+ "-v /var/tmp/npm:/var/tmp/npm:rw " diff --git a/ci/docker/nix/Dockerfile b/ci/docker/nix/Dockerfile new file mode 100644 index 0000000000..406935d612 --- /dev/null +++ b/ci/docker/nix/Dockerfile @@ -0,0 +1,96 @@ +# This image is only for extracting and cleaning up Android SDK +FROM ubuntu:18.04 AS android_sdk + +ARG ANDROID_SDK_VERSION +ARG SDK_PLATFORM_VERSION +ARG SDK_BUILD_TOOLS_VERSION + +# names of packages to install with sdkmanager +ENV SDK_PACKAGES emulator patcher;v4 \ + platforms;android-26 \ + platforms;${SDK_PLATFORM_VERSION} \ + build-tools;${SDK_BUILD_TOOLS_VERSION} \ + extras;android;m2repository extras;google;m2repository + +ENV LANG=en_US.UTF-8 \ + LC_ALL=en_US.UTF-8 \ + LANGUAGE=en_US.UTF-8 + +RUN apt-get update && apt-get -q -y --no-install-recommends install software-properties-common \ + && add-apt-repository -y ppa:webupd8team/java \ + && echo debconf shared/accepted-oracle-license-v1-1 select true | debconf-set-selections \ + && echo debconf shared/accepted-oracle-license-v1-1 seen true | debconf-set-selections \ + && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -q -y --no-install-recommends install \ + wget unzip locales ca-certificates oracle-java8-installer oracle-java8-set-default \ + && locale-gen ${LANG} \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /usr/share/man + +# install Android SDK +COPY sdk-tools-linux-${ANDROID_SDK_VERSION}.zip /tmp/sdk-tools-linux.zip +RUN unzip -q /tmp/sdk-tools-linux.zip -d /usr/lib/android-sdk \ + && mkdir ~/.android && touch ~/.android/repositories.cfg +RUN /usr/lib/android-sdk/tools/bin/sdkmanager --licenses +RUN for PKG in ${SDK_PACKAGES}; do \ + yes | /usr/lib/android-sdk/tools/bin/sdkmanager --install "${PKG}" > /dev/null; done \ + && chmod 777 -R /usr/lib/android-sdk +# cleanup SDK +RUN cd /usr/lib/android-sdk \ + && rm -fr extras emulator ndk-bundle/toolchains/{aarch64-linux-android-4.9,x86-4.9,x86_64-4.9,llvm} + +################################################################################ +FROM ubuntu:18.04 +ENV LANG=en_US.UTF-8 \ + LC_ALL=en_US.UTF-8 \ + LANGUAGE=en_US.UTF-8 + +ARG NIX_VERSION +ARG ANDROID_SDK_VERSION +ARG JENKINS_UID +ARG JENKINS_GID + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive \ + apt-get -q -y --no-install-recommends \ + install file binutils make locales curl git strace jq bzip2 ca-certificates s3cmd \ + && apt-get clean \ + && locale-gen ${LANG} \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /usr/share/man + +ADD nix-install-${NIX_VERSION}.sh /tmp/nix-install.sh + +RUN groupadd --gid ${JENKINS_GID} jenkins \ + && useradd --shell /bin/bash --create-home --password "" \ + --comment "jenkins" --groups root \ + --uid ${JENKINS_UID} --gid ${JENKINS_GID} jenkins + +ADD nix.conf /home/jenkins/.config/nix/nix.conf +ADD default.nix /home/jenkins/default.nix +ADD default.nix /bin/default.nix + +RUN mkdir /nix \ + && chown ${JENKINS_UID}:${JENKINS_GID} -R /nix /home/jenkins + +WORKDIR /home/jenkins +USER jenkins + +# Install Android SDK +RUN mkdir -p /home/jenkins/.status/Android/Sdk +COPY --from=android_sdk /usr/lib/android-sdk /home/jenkins/.status/Android/Sdk +RUN chmod o+w /home/jenkins/.status/Android/Sdk + +RUN export USER=jenkins \ + && /tmp/nix-install.sh \ + && . ~/.nix-profile/etc/profile.d/nix.sh \ + && nix-shell https://github.com/status-im/status-react/tarball/develop --run 'echo' + +# hack to avoid calling login +ENV USER=jenkins +# make nix tools available by default +ENV PATH=/home/jenkins/.nix-profile/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +ENV NIX_PATH=nixpkgs=/home/jenkins/.nix-defexpr/channels/nixpkgs +ENV NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt + +MAINTAINER Jakub Sokolowski "jakub@status.im" +LABEL description="Adjusted Nix Docker image for use with Jenkins" diff --git a/ci/docker/nix/Makefile b/ci/docker/nix/Makefile new file mode 100644 index 0000000000..edd67505ca --- /dev/null +++ b/ci/docker/nix/Makefile @@ -0,0 +1,51 @@ +__toolversion = $(shell $(GIT_ROOT)/scripts/toolversion $(1)) + +GIT_COMMIT = $(shell git rev-parse --short HEAD) +GIT_ROOT = $(shell git rev-parse --show-toplevel) + +ANDROID_SDK_VERSION = $(call __toolversion, android-sdk) +ANDROID_SDK_CHECKSUM = $(call __toolversion, -c android-sdk) +ANDROID_SDK_URL = https://dl.google.com/android/repository/sdk-tools-linux-$(ANDROID_SDK_VERSION).zip +ANDROID_SDK_ARCHIVE = sdk-tools-linux-$(ANDROID_SDK_VERSION).zip +SDK_PLATFORM_VERSION=$(call __toolversion, android-sdk-platform) +SDK_BUILD_TOOLS_VERSION=$(call __toolversion, android-sdk-build-tools) + +JENKINS_UID = 1001 +JENKINS_GID = 1001 + +NIX_VERSION = 2.2.1 +NIX_INSTALL = nix-install-$(NIX_VERSION).sh +NIX_SHA = 0039df605d2324f721f0c99ab76fc58fa60f791d84b30196eed805e8e65b1e51 +NIX_URL = https://nixos.org/releases/nix/nix-$(NIX_VERSION)/install + +# WARNING: Remember to change the tag when updating the image +DEPS_HASH = $(shell $(GIT_ROOT)/scripts/gen-deps-hash.sh -b $(NIX_SHA) \ + -d android-sdk \ + -d android-sdk-platform \ + -d android-sdk-build-tools \ + -d nix) + +IMAGE_TAG = jenkins-1.0.0-$(DEPS_HASH) +IMAGE_NAME = statusteam/nix:$(IMAGE_TAG) + +build: $(NIX_INSTALL) $(ANDROID_SDK_ARCHIVE) + docker build \ + --build-arg="JENKINS_UID=$(JENKINS_UID)" \ + --build-arg="JENKINS_GID=$(JENKINS_GID)" \ + --build-arg="NIX_VERSION=$(NIX_VERSION)" \ + --build-arg="ANDROID_SDK_VERSION=$(ANDROID_SDK_VERSION)" \ + --build-arg="SDK_BUILD_TOOLS_VERSION=$(SDK_BUILD_TOOLS_VERSION)" \ + --build-arg="SDK_PLATFORM_VERSION=$(SDK_PLATFORM_VERSION)" \ + -t $(IMAGE_NAME) . + +$(NIX_INSTALL): + wget -q "$(NIX_URL)" -O "$(NIX_INSTALL)" + echo "$(NIX_SHA) $(NIX_INSTALL)" | sha256sum --check + chmod +x "$(NIX_INSTALL)" + +$(ANDROID_SDK_ARCHIVE): + wget -q "$(ANDROID_SDK_URL)" -O "$(ANDROID_SDK_ARCHIVE)" + echo "$(ANDROID_SDK_CHECKSUM) $(ANDROID_SDK_ARCHIVE)" | md5sum --check + +push: build + docker push $(IMAGE_NAME) diff --git a/ci/docker/nix/README.md b/ci/docker/nix/README.md new file mode 100644 index 0000000000..602225a98f --- /dev/null +++ b/ci/docker/nix/README.md @@ -0,0 +1,5 @@ +# Description + +In order to make Nix work with Jenkins we have to make it possible to use it as Jenkins user. + +For that reason we add `jenkins` user with the right UID and GID matching the ones on CI hosts. diff --git a/ci/docker/nix/default.nix b/ci/docker/nix/default.nix new file mode 100644 index 0000000000..6bc849a124 --- /dev/null +++ b/ci/docker/nix/default.nix @@ -0,0 +1,9 @@ +with import { }; +stdenv.mkDerivation rec { + name = "env"; + env = buildEnv { name = name; paths = buildInputs; }; + buildInputs = with pkgs; [ + bash + git + ]; +} diff --git a/ci/docker/nix/nix-install-2.2.1.sh b/ci/docker/nix/nix-install-2.2.1.sh new file mode 100755 index 0000000000..13eadf9120 --- /dev/null +++ b/ci/docker/nix/nix-install-2.2.1.sh @@ -0,0 +1,67 @@ +#!/bin/sh + +# This script installs the Nix package manager on your system by +# downloading a binary distribution and running its installer script +# (which in turn creates and populates /nix). + +{ # Prevent execution if this script was only partially downloaded +oops() { + echo "$0:" "$@" >&2 + exit 1 +} + +tmpDir="$(mktemp -d -t nix-binary-tarball-unpack.XXXXXXXXXX || \ + oops "Can't create temporary directory for downloading the Nix binary tarball")" +cleanup() { + rm -rf "$tmpDir" +} +trap cleanup EXIT INT QUIT TERM + +require_util() { + type "$1" > /dev/null 2>&1 || command -v "$1" > /dev/null 2>&1 || + oops "you do not have '$1' installed, which I need to $2" +} + +case "$(uname -s).$(uname -m)" in + Linux.x86_64) system=x86_64-linux; hash=e229e28f250cad684c278c9007b07a24eb4ead239280c237ed2245871eca79e0;; + Linux.i?86) system=i686-linux; hash=223c141b7ce322aacd4003dfc79b969ee813396d2ac586cf44c0e787233b1666;; + Linux.aarch64) system=aarch64-linux; hash=94d3d7d2b551b228ec38f83c6f037c5e5b6aaf0805450967f800c02a827b1f41;; + Darwin.x86_64) system=x86_64-darwin; hash=4182a38948c9ce520993f79a5caf8ce3d49636189f4f0f7e8decbc91ddb61cd1;; + *) oops "sorry, there is no binary distribution of Nix for your platform";; +esac + +url="https://nixos.org/releases/nix/nix-2.2.1/nix-2.2.1-$system.tar.bz2" + +tarball="$tmpDir/$(basename "$tmpDir/nix-2.2.1-$system.tar.bz2")" + +require_util curl "download the binary tarball" +require_util bzcat "decompress the binary tarball" +require_util tar "unpack the binary tarball" + +echo "downloading Nix 2.2.1 binary tarball for $system from '$url' to '$tmpDir'..." +curl -L "$url" -o "$tarball" || oops "failed to download '$url'" + +if type sha256sum > /dev/null 2>&1; then + hash2="$(sha256sum -b "$tarball" | cut -c1-64)" +elif type shasum > /dev/null 2>&1; then + hash2="$(shasum -a 256 -b "$tarball" | cut -c1-64)" +elif type openssl > /dev/null 2>&1; then + hash2="$(openssl dgst -r -sha256 "$tarball" | cut -c1-64)" +else + oops "cannot verify the SHA-256 hash of '$url'; you need one of 'shasum', 'sha256sum', or 'openssl'" +fi + +if [ "$hash" != "$hash2" ]; then + oops "SHA-256 hash mismatch in '$url'; expected $hash, got $hash2" +fi + +unpack=$tmpDir/unpack +mkdir -p "$unpack" +< "$tarball" bzcat | tar -xf - -C "$unpack" || oops "failed to unpack '$url'" + +script=$(echo "$unpack"/*/install) + +[ -e "$script" ] || oops "installation script is missing from the binary tarball!" +"$script" "$@" + +} # End of wrapping diff --git a/ci/docker/nix/nix.conf b/ci/docker/nix/nix.conf new file mode 100644 index 0000000000..72a643ceac --- /dev/null +++ b/ci/docker/nix/nix.conf @@ -0,0 +1,3 @@ +sandbox = false +extra-substituters = https://nix-cache.status.im +trusted-public-keys = nix-cache.status.im-1:x/93lOfLU+duPplwMSBR+OlY4+mo+dCN7n0mr4oPwgY= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= diff --git a/scripts/gen-deps-hash.sh b/scripts/gen-deps-hash.sh new file mode 100755 index 0000000000..4dcdc4fd8c --- /dev/null +++ b/scripts/gen-deps-hash.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash + +################################################################################ +# This tool fetches versions of build tools from the .TOOLVERSIONS +# file in project root and calculates a single hash that represents +# the combined versions of all the specified tools. +################################################################################ + +GIT_ROOT=$(git rev-parse --show-toplevel) +toolversion="${GIT_ROOT}/scripts/toolversion" + +usage () { + echo "Usage: gen-deps-hash [-b ] -d [-d ]" >&2 + echo + echo "This script calculates a hash representing the required versions of the specified tools" + exit 0 +} + +# some options parsing +deps=() +while getopts "hb:d:" opt; do + case $opt in + b) base_hash="$OPTARG";; + d) + version=$($toolversion "$OPTARG") + if [ $? -ne 0 ]; then + echo "ERROR: $OPTARG not found in .TOOLVERSIONS" + exit 1 + fi + deps+=("$OPTARG $version") + ;; + h) usage;; + \?) echo "Invalid option: -$OPTARG" >&2; exit 1;; + esac +done + +if [ ${#deps[@]} -eq 0 ]; then + echo "ERROR: No dependencies specified" + echo + usage +fi + +IFS=$'\n' sorted_deps=($(sort <<<"${deps[*]}")) +unset IFS + +hash=$(echo "${base_hash}${sorted_deps[@]}" | md5sum | cut -f1 -d" ") +echo "${hash:0:8}"