add dockerimage for building android

- remove unused android-ndk toolchains
- add build.sh and test target for easy testing of image
- separate generic parts into a base image
- pass keystore credentials via env variables

Signed-off-by: Jakub Sokołowski <jakub@status.im>

.gitignore

@@ -24,6 +24,8 @@ DerivedData
 *.ipa
 *.xcuserstate
 project.xcworkspace
+docker/*/*.zip
+docker/*/*.run
 
 # Android/IJ
 #
@@ -81,6 +83,9 @@ figwheel_server.log
 out
 doo-index.html
 
+# Bundler
+.bundle
+
 # Status
 Statusgo.framework
 status-go-local.aar

android/app/build.gradle

@@ -125,6 +125,12 @@ def getBuildUrl = { ->
     }
 }
 
+/* check if environment variable exists for given variable name first */
+def getEnvOrConfig = { varName ->
+    def val = System.getenv(varName) ? System.getenv(varName) : project.property(varName)
+    return val
+}
+
 android {
     compileSdkVersion 26
     buildToolsVersion "26.0.2"
@@ -171,10 +177,11 @@ android {
     }
     signingConfigs {
         release {
-            storeFile file(STATUS_RELEASE_STORE_FILE.replaceAll("~", System.properties['user.home']))
-            storePassword STATUS_RELEASE_STORE_PASSWORD
-            keyAlias STATUS_RELEASE_KEY_ALIAS
-            keyPassword STATUS_RELEASE_KEY_PASSWORD
+            /* environment variables take precedence over gradle.properties file */
+            storeFile     file(getEnvOrConfig('STATUS_RELEASE_STORE_FILE').replaceAll("~", System.properties['user.home']))
+            storePassword getEnvOrConfig('STATUS_RELEASE_STORE_PASSWORD')
+            keyAlias      getEnvOrConfig('STATUS_RELEASE_KEY_ALIAS')
+            keyPassword   getEnvOrConfig('STATUS_RELEASE_KEY_PASSWORD')
         }
     }
     splits {

ci/Jenkinsfile.android

@@ -1,5 +1,14 @@
 pipeline {
-  agent { label 'linux' }
+  agent { 
+    docker {
+      label 'linux'
+      image 'statusteam/status-build-android:1.0.0'
+      args (
+        "-v /home/jenkins/tmp:/var/tmp:rw "+
+        "-v /home/jenkins/status-im.keystore:/tmp/status-im.keystore:ro"
+      )
+    }
+  }
 
   options {
     timestamps()
@@ -29,11 +38,17 @@ pipeline {
     LC_ALL   = 'en_US.UTF-8'
     FASTLANE_DISABLE_COLORS = 1
     REALM_DISABLE_ANALYTICS = 1
-    BUNDLE_PATH = "${HOME}/.bundle"
+    /* since we are mounting it we need to specify location */
+    STATUS_RELEASE_STORE_FILE = '/tmp/status-im.keystore'
     ANDROID_HOME      = '/usr/lib/android-sdk'
     ANDROID_SDK_ROOT  = '/usr/lib/android-sdk'
     ANDROID_NDK       = '/usr/lib/android-ndk'
     ANDROID_NDK_HOME  = '/usr/lib/android-ndk'
+    /* We use EXECUTOR_NUMBER to avoid multiple instances clashing */
+    LEIN_HOME         = "/var/tmp/lein-${EXECUTOR_NUMBER}"
+    YARN_CACHE_FOLDER = "/var/tmp/yarn-${EXECUTOR_NUMBER}"
+    BUNDLE_PATH       = "/var/tmp/bundle-${EXECUTOR_NUMBER}"
+    GRADLE_USER_HOME  = "/var/tmp/gradle-${EXECUTOR_NUMBER}"
   }
 
   stages {

ci/android.groovy

@@ -7,8 +7,20 @@ def bundle(type = 'nightly') {
     gradleOpt += "-PreleaseVersion='${cmn.version()}'"
   }
   dir('android') {
+    withCredentials([
+      string(
+        credentialsId: 'android-keystore-pass',
+        variable: 'STATUS_RELEASE_STORE_PASSWORD'
+      ),
+      usernamePassword(
+        credentialsId: 'android-keystore-key-pass',
+        usernameVariable: 'STATUS_RELEASE_KEY_ALIAS',
+        passwordVariable: 'STATUS_RELEASE_KEY_PASSWORD'
+      )
+    ]) {
       sh "./gradlew assembleRelease ${gradleOpt}"
     }
+  }
   def pkg = cmn.pkgFilename(type, 'apk')
   sh "cp android/app/build/outputs/apk/release/app-release.apk ${pkg}"
   return pkg

ci/mobile.groovy

@@ -50,7 +50,7 @@ def prep(type = 'nightly') {
   }
   /* install ruby dependencies */
   sh 'bundle install --quiet'
-  /* npm deps and status-go download */
+  /* node deps and status-go download */
   sh "make prepare-${env.BUILD_PLATFORM}"
   /* generate ios/StatusIm.xcworkspace */
   if (env.BUILD_PLATFORM == 'ios') {

docker/android/Dockerfile

@@ -0,0 +1,63 @@
+# This image is only for extracting and cleaning up NDK and SDK
+FROM statusteam/status-build-base:1.0.0 AS sdk_and_ndk
+
+ARG ANDROID_NDK_VERSION
+ARG ANDROID_SDK_VERSION
+
+# names of packages to install with sdkmanager
+ENV SDK_PACKAGES emulator patcher;v4 build-tools;26.0.2 platforms;android-26 platforms;android-27 platforms;android-28 extras;android;m2repository extras;google;m2repository
+
+# install SDK
+COPY sdk-tools-linux-${ANDROID_SDK_VERSION}.zip /tmp/sdk-tools-linux.zip
+RUN unzip -q /tmp/sdk-tools-linux.zip -d /usr/lib/android-sdk \
+ && mkdir ~/.android && touch ~/.android/repositories.cfg \
+ && yes | /usr/lib/android-sdk/tools/bin/sdkmanager --licenses > /dev/null \
+ && for PKG in ${SDK_PACKAGES}; do /usr/lib/android-sdk/tools/bin/sdkmanager --install "${PKG}"; done \
+ && chmod 777 -R /usr/lib/android-sdk
+# cleanup SDK
+RUN cd /usr/lib/android-sdk \
+ && rm -fr extras emulator ndk-bundle/toolchains/{aarch64-linux-android-4.9,x86-4.9,x86_64-4.9,llvm}
+
+# install NDK
+COPY android-ndk-${ANDROID_NDK_VERSION}-linux-x86_64.zip /tmp/android-ndk-linux.zip
+RUN unzip -q /tmp/android-ndk-linux.zip -d /usr/lib \
+ && ln -sf /usr/lib/android-ndk-${ANDROID_NDK_VERSION} /usr/lib/android-ndk \
+ && chmod 777 -R /usr/lib/android-ndk
+# cleanup NDK
+RUN cd /usr/lib/android-ndk && rm -fr docs tests samples \
+ && find toolchains -mindepth 1 -maxdepth 1 | grep -vE '(arm|aarch|x86|mips).*4.(8|9)' | xargs rm -fr \
+ && find prebuilt -mindepth 1 -maxdepth 1 | grep -vE '(android-arm|linux-x86)' | xargs rm -fr \
+ && find platforms -mindepth 1 -maxdepth 1 | grep -v android-21 | xargs rm -fr \
+ && find sources -mindepth 2 -maxdepth 2 | grep -v 'gnu-libstdc' | xargs rm -fr
+
+################################################################################
+FROM statusteam/status-build-base:1.0.0
+
+ARG ANDROID_NDK_VERSION
+ARG ANDROID_SDK_VERSION
+
+# We have to do this because Jenkins doesn't let us
+# https://issues.jenkins-ci.org/browse/JENKINS-49076
+ENV GRADLE_USER_HOME=/var/tmp/gradle \
+    BUNDLE_PATH=/var/tmp/bundle \
+    ANDROID_HOME=/usr/lib/android-sdk \
+    ANDROID_SDK_ROOT=/usr/lib/android-sdk \
+    ANDROID_NDK=/usr/lib/android-ndk \
+    ANDROID_NDK_HOME=/usr/lib/android-ndk
+
+RUN add-apt-repository -y ppa:cwchien/gradle \
+ && apt-get update \
+ && DEBIAN_FRONTEND=noninteractive \
+      apt-get -q -y --no-install-recommends install \
+        gradle ruby ruby-dev ruby-bundler \
+ && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /usr/share/man \
+ && gem install cocoapods bundler fastlane fastlane-plugin-diawi fastlane-plugin-clean_testflight_testers CFPropertyList
+
+# Install Android SDK & NDK
+COPY --from=sdk_and_ndk /usr/lib/android-sdk /usr/lib/android-sdk 
+COPY --from=sdk_and_ndk /usr/lib/android-ndk /usr/lib/android-ndk 
+RUN chmod o+w /usr/lib/android-sdk /usr/lib/android-ndk
+
+LABEL source="https://github.com/status-im/status-react/tree/develop/docker/android" \
+      description="Image for building Android version of Status app." \
+      maintainer="jakub@status.im"

docker/android/Makefile

@@ -0,0 +1,42 @@
+GIT_COMMIT = $(shell git rev-parse --short HEAD)
+GIT_ROOT = $(shell git rev-parse --show-toplevel)
+
+ANDROID_NDK_VERSION = r10e
+ANDROID_NDK_CHECKSUM = 070be287539e3e7706f8dabfb6bf9879
+ANDROID_NDK_ARCHIVE = android-ndk-$(ANDROID_NDK_VERSION)-linux-x86_64.zip
+ANDROID_NDK_URL = https://dl.google.com/android/repository/$(ANDROID_NDK_ARCHIVE)
+
+ANDROID_SDK_VERSION = 4333796
+ANDROID_SDK_CHECKSUM = aa190cfd7299cd6a1c687355bb2764e4
+ANDROID_SDK_URL = https://dl.google.com/android/repository/sdk-tools-linux-$(ANDROID_SDK_VERSION).zip
+ANDROID_SDK_ARCHIVE = sdk-tools-linux-$(ANDROID_SDK_VERSION).zip
+
+# WARNING: Remember to change the tag when updating the image
+IMAGE_TAG = 1.0.0
+IMAGE_NAME = statusteam/status-build-android:$(IMAGE_TAG)
+
+build: $(ANDROID_NDK_ARCHIVE) $(ANDROID_SDK_ARCHIVE)
+	docker build \
+		--build-arg="ANDROID_NDK_VERSION=$(ANDROID_NDK_VERSION)" \
+		--build-arg="ANDROID_SDK_VERSION=$(ANDROID_SDK_VERSION)" \
+		--label="commit=$(GIT_COMMIT)" \
+		-t $(IMAGE_NAME) .
+
+$(ANDROID_NDK_ARCHIVE):
+	wget -q "$(ANDROID_NDK_URL)" -O "$(ANDROID_NDK_ARCHIVE)"
+	echo "$(ANDROID_NDK_CHECKSUM)  $(ANDROID_NDK_ARCHIVE)" | md5sum --check
+
+$(ANDROID_SDK_ARCHIVE):
+	wget -q "$(ANDROID_SDK_URL)" -O "$(ANDROID_SDK_ARCHIVE)"
+	echo "$(ANDROID_SDK_CHECKSUM)  $(ANDROID_SDK_ARCHIVE)" | md5sum --check
+
+test: ## Run build inside the image as a test
+	docker run -u $(shell id -u):$(shell id -g) \
+		--name android-test --rm \
+        --tmpfs /var/tmp:rw,size=1G,exec,mode=1777 \
+		-v $(GIT_ROOT):/repo:rw \
+		-w /repo $(IMAGE_NAME) \
+		docker/android/build.sh
+
+push: build
+	docker push $(IMAGE_NAME)

docker/android/README.md

@@ -0,0 +1,3 @@
+# Description
+
+This dockerfile is used to generate an image based on Ubuntu that will be used by Jenkins to build the Android version of the Status app.

docker/android/build.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+set -e -x
+
+export FASTLANE_DISABLE_COLORS=1
+export REALM_DISABLE_ANALYTICS=1
+export YARN_CACHE_FOLDER=/var/tmp/yarn
+export NPM_CONFIG_CACHE=/var/tmp/npm
+export HOME=/tmp
+
+make clean
+## Prep
+bundle install --quiet
+make prepare-android
+## Lint
+lein cljfmt check
+## Test
+lein test-cljs
+## Build
+lein prod-build-android
+# Compile
+cd android
+./gradlew assembleDebug -Dorg.gradle.daemon=false

docker/base/Dockerfile

@@ -0,0 +1,53 @@
+FROM ubuntu:16.04
+
+ARG NVM_VERSION
+ARG NVM_NODE_VERSION
+ARG LEIN_VERSION
+
+# We have to do this because Jenkins doesn't let us
+# https://issues.jenkins-ci.org/browse/JENKINS-49076
+ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
+    LANG=en_US.UTF-8 \
+    LC_ALL=en_US.UTF-8 \
+    LANGUAGE=en_US.UTF-8 \
+    LEIN_HOME=/var/tmp/lein \
+    YARN_CACHE_FOLDER=/var/tmp/yarn
+
+RUN mkdir /var/tmp/npm /var/tmp/lein \
+ && chmod 777 /var/tmp/npm /var/tmp/lein \
+ && apt-get update && apt-get -q -y --no-install-recommends install curl software-properties-common \
+ && add-apt-repository -y ppa:webupd8team/java \
+ && echo debconf shared/accepted-oracle-license-v1-1 select true | debconf-set-selections \
+ && echo debconf shared/accepted-oracle-license-v1-1 seen true | debconf-set-selections \
+ && curl -sL https://deb.nodesource.com/setup_10.x | bash - \
+ && curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - \
+ && echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list \
+ && apt-get update \
+ && DEBIAN_FRONTEND=noninteractive apt-get -q -y --no-install-recommends install \
+      file git wget zip unzip s3cmd build-essential nodejs yarn locales \
+      ca-certificates oracle-java8-installer oracle-java8-set-default \
+ && locale-gen ${LANG} \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /usr/share/man
+
+# These are the UID and GID values used by Jenkins
+RUN addgroup --gid 1002 jenkins \
+ && adduser --shell /bin/bash \
+      --disabled-password --gecos "" \
+      --uid 1001 --gid 1002 jenkins \
+ && su jenkins -c "git config --global user.email 'jenkins'" \
+ && su jenkins -c "git config --global user.name 'jenkins@status.im'"
+
+# Leiningen setup
+RUN wget -q https://raw.githubusercontent.com/technomancy/leiningen/${LEIN_VERSION}/bin/lein -O /usr/local/bin/lein \
+ && chmod 755 /usr/local/bin/lein \
+ && lein version
+
+# Install NVM for Jenkins
+RUN su jenkins -c "\
+    curl -s -o- https://raw.githubusercontent.com/creationix/nvm/${NVM_VERSION}/install.sh | bash && \
+    source ~/.nvm/nvm.sh && nvm install ${NVM_NODE_VERSION} && nvm alias default ${NVM_NODE_VERSION}"
+
+LABEL source="https://github.com/status-im/status-react/tree/develop/docker/base" \
+      description="Base Image used for building Status app." \
+      maintainer="jakub@status.im"

docker/base/Makefile

@@ -0,0 +1,24 @@
+GIT_COMMIT = $(shell git rev-parse --short HEAD)
+
+# Software Versions, URLs, and Checksums
+NVM_VERSION = v0.33.11
+NVM_INSTALL_SCRIPT = nvm_$(NVM_VERSION)_install.sh
+NVM_NODE_VERSION = v10.14.0
+LEIN_VERSION = stable
+
+GIT_ROOT = $(shell git rev-parse --show-toplevel)
+
+# WARNING: Remember to change the tag when updating the image
+IMAGE_TAG = 1.0.0
+IMAGE_NAME = statusteam/status-build-base:$(IMAGE_TAG)
+
+build:
+	docker build \
+		--build-arg="LEIN_VERSION=$(LEIN_VERSION)" \
+		--build-arg="NVM_VERSION=$(NVM_VERSION)" \
+		--build-arg="NVM_NODE_VERSION=$(NVM_NODE_VERSION)" \
+		--label="commit=$(GIT_COMMIT)" \
+		-t $(IMAGE_NAME) .
+
+push: build
+	docker push $(IMAGE_NAME)

docker/base/README.md

@@ -0,0 +1,12 @@
+# Descirption
+
+This is a base Docker image used by all other images used for building the Status app.
+
+# Packages
+
+It includes:
+
+* Generic utilities: `file`, `zip`, `unzip`, `curl`, `wget`, `s3cmd`
+* Interpreters and compilters: `nodejs`, `java`
+* Package and build mangers: `yarn`, `nvm`, `leiningen`
+* User `jenkins` for use with Jenkins continuous integration system

mobile_files/package.json.orig

@@ -15,7 +15,7 @@
     "assert": "1.4.1",
     "asyncstorage-down": "4.0.1",
     "babel-preset-react-native": "5.0.2",
-    "bignumber.js": "https://github.com/status-im/bignumber.js.git",
+    "bignumber.js": "git+https://github.com/status-im/bignumber.js.git",
     "buffer": "3.6.0",
     "chance": "1.0.12",
     "create-react-class": "15.6.2",
@@ -24,7 +24,7 @@
     "eth-phishing-detect": "1.1.13",
     "events": "1.1.1",
     "homoglyph-finder": "1.1.1",
-    "identicon.js": "https://github.com/status-im/identicon.js.git",
+    "identicon.js": "git+https://github.com/status-im/identicon.js.git",
     "js-sha3": "^0.8.0",
     "level-filesystem": "1.2.0",
     "process": "0.11.10",
@@ -33,40 +33,40 @@
     "querystring-es3": "0.2.1",
     "react": "16.4.1",
     "react-dom": "16.4.2",
-    "react-native": "github:status-im/react-native#status-0.56-1",
+    "react-native": "git+https://github.com/status-im/react-native#status-0.56-1",
     "react-native-background-timer": "2.1.0-alpha.6",
-    "react-native-camera": "https://github.com/status-im/react-native-camera#1.1.5-1",
-    "react-native-config": "https://github.com/status-im/react-native-config#0.11.2-1",
+    "react-native-camera": "git+https://github.com/status-im/react-native-camera.git#1.1.5-1",
+    "react-native-config": "git+https://github.com/status-im/react-native-config.git#0.11.2-1",
     "react-native-crypto": "2.1.1",
     "react-native-dialogs": "1.0.2",
     "react-native-fetch-polyfill": "1.1.2",
-    "react-native-firebase": "https://github.com/status-im/react-native-firebase#5.0.0-rc1",
+    "react-native-firebase": "git+https://github.com/status-im/react-native-firebase.git#5.0.0-rc1",
     "react-native-fs": "2.11.15",
     "react-native-http-bridge": "0.6.1",
     "react-native-i18n": "2.0.15",
     "react-native-image-crop-picker": "0.18.1",
-    "react-native-image-resizer": "https://github.com/status-im/react-native-image-resizer.git#1.0.0-1",
+    "react-native-image-resizer": "git+https://github.com/status-im/react-native-image-resizer.git#1.0.0-1",
     "react-native-invertible-scroll-view": "1.1.0",
-    "react-native-keychain": "https://github.com/status-im/react-native-keychain#v.3.0.0-status",
+    "react-native-keychain": "git+https://github.com/status-im/react-native-keychain.git#v.3.0.0-status",
     "react-native-level-fs": "3.0.1",
-    "react-native-os": "https://github.com/status-im/react-native-os.git#1.1.0-1",
+    "react-native-os": "git+https://github.com/status-im/react-native-os.git#1.1.0-1",
     "react-native-qrcode": "0.2.7",
     "react-native-randombytes": "3.5.0",
     "react-native-safe-area-view": "0.9.0",
-    "react-native-securerandom": "https://github.com/status-im/react-native-securerandom#0.1.1-1",
+    "react-native-securerandom": "git+https://github.com/status-im/react-native-securerandom.git#0.1.1-1",
     "react-native-splash-screen": "3.1.1",
-    "react-native-status-keycard": "https://github.com/status-im/react-native-status-keycard",
+    "react-native-status-keycard": "git+https://github.com/status-im/react-native-status-keycard.git",
     "react-native-svg": "6.5.2",
-    "react-native-tcp": "https://github.com/status-im/react-native-tcp.git#3.3.0-1",
-    "react-native-udp": "https://github.com/status-im/react-native-udp.git#2.3.1-1",
+    "react-native-tcp": "git+https://github.com/status-im/react-native-tcp.git#3.3.0-1",
+    "react-native-udp": "git+https://github.com/status-im/react-native-udp.git#2.3.1-1",
     "react-native-webview-bridge": "git+https://github.com/status-im/react-native-webview-bridge.git#0.33.16-3",
     "react-navigation": "^2.12.1",
     "realm": "2.21.0",
-    "rn-snoopy": "https://github.com/status-im/rn-snoopy.git",
+    "rn-snoopy": "git+https://github.com/status-im/rn-snoopy.git",
     "string_decoder": "0.10.31",
     "text-encoding": "^0.6.4",
     "url": "0.10.3",
-    "web3": "https://github.com/status-im/web3.js.git#features/pairing-message",
+    "web3": "git+https://github.com/status-im/web3.js.git#features/pairing-message",
     "web3-utils": "1.0.0-beta.36"
   }
 }

scripts/build-android.sh

@@ -14,7 +14,7 @@ echo
 
 GRADLE_PROPERTIES="--daemon --parallel -q -b android/build.gradle"
 
-npm install
+yarn install
 
 case $TARGET in
   debug)