build multiple APKs for different architectures

Signed-off-by: Jakub Sokołowski <jakub@status.im>

Makefile

@@ -110,8 +110,9 @@ release-android: export TARGET_OS ?= android
 release-android: export BUILD_ENV ?= prod
 release-android: export BUILD_TYPE ?= nightly
 release-android: export BUILD_NUMBER ?= 9999
-release-android: export NDK_ABI_FILTERS ?= armeabi-v7a;arm64-v8a;x86
 release-android: export STORE_FILE ?= $(HOME)/.gradle/status-im.keystore
+release-android: export ANDROID_ABI_SPLIT ?= false
+release-android: export ANDROID_ABI_INCLUDE ?= armeabi-v7a;arm64-v8a;x86
 release-android: ##@build build release for Android
 	scripts/release-android.sh
 

android/app/build.gradle

@@ -92,16 +92,6 @@ project.ext.react = [
 
 apply from: "../../node_modules/react-native/react.gradle"
 
-/**
- * Set this to true to create two separate APKs instead of one:
- *   - An APK that only works on ARM devices
- *   - An APK that only works on x86 devices
- * The advantage is the size of the APK is reduced by about 4MB.
- * Upload all the APKs to the Play Store and people will download
- * the correct one based on the CPU architecture of their device.
- */
-def enableSeparateBuildPerCPUArchitecture = false
-
 /**
  * Run Proguard to shrink the Java bytecode in release builds.
  */
@@ -184,10 +174,13 @@ android {
         multiDexEnabled true
         versionCode getVersionCode()
         versionName getVersionName()
-        ndk {
-            abiFilters getEnvOrConfig('NDK_ABI_FILTERS').split(';')
-        }
         missingDimensionStrategy 'react-native-camera', 'general'
+        /* this needs to be empty if we want APKs split by ABIs */
+        if (!getEnvOrConfig('ANDROID_ABI_SPLIT').toBoolean()) {
+            ndk {
+                abiFilters getEnvOrConfig('ANDROID_ABI_INCLUDE').split(";")
+            }
+        }
     }
     /**
      * Arbitrary project metadata
@@ -242,9 +235,9 @@ android {
     splits {
         abi {
             reset()
-            enable enableSeparateBuildPerCPUArchitecture
-            universalApk false  // If true, also generate a universal APK
-            include "armeabi-v7a", "arm64-v8a", "x86"
+            enable getEnvOrConfig('ANDROID_ABI_SPLIT').toBoolean()
+            include getEnvOrConfig('ANDROID_ABI_INCLUDE').split(";")
+            universalApk true
         }
     }
     buildTypes {

android/gradle.properties

@@ -38,8 +38,10 @@ STATUS_RELEASE_STORE_PASSWORD=password
 STATUS_RELEASE_KEY_ALIAS=status
 STATUS_RELEASE_KEY_PASSWORD=password
 
-# platforms for which to build the Android bundle
-NDK_ABI_FILTERS=armeabi-v7a;arm64-v8a;x86
+# By default we build a mostly universal APK
+ANDROID_ABI_SPLIT=false
+# Some platforms are excluded though
+ANDROID_ABI_INCLUDE=armeabi-v7a;arm64-v8a;x86
 
 org.gradle.jvmargs=-Xmx8704M
 

ci/Jenkinsfile.android

@@ -90,21 +90,23 @@ pipeline {
           }
           stage('Bundle') {
             steps {
-              script { apk = android.bundle() }
+              script { apks = android.bundle() }
             }
           }
         } }
       }
     }
     stage('Archive') {
-      steps {
-        archiveArtifacts apk
-      }
+      steps { script {
+        apks.each { archiveArtifacts it }
+      } }
     }
     stage('Upload') {
       steps {
         script {
-          env.PKG_URL = cmn.utils.uploadArtifact(apk)
+          def urls = apks.collect { cmn.utils.uploadArtifact(it) }
+          /* return only the universal APK */
+          env.PKG_URL = urls.find { it.contains('universal') }
           /* build type specific */
           switch (btype) {
             case 'release':

ci/android.groovy

@@ -2,21 +2,24 @@ nix = load 'ci/nix.groovy'
 utils = load 'ci/utils.groovy'
 
 def bundle() {
+  /* we use the method because parameter build type does not take e2e into account */
   def btype = utils.getBuildType()
   /* Disable Gradle Daemon https://stackoverflow.com/questions/38710327/jenkins-builds-fail-using-the-gradle-daemon */
   def gradleOpt = "-PbuildUrl='${currentBuild.absoluteUrl}' --console plain "
-  def target = "release"
   /* we don't need x86 for any builds except e2e */
-  env.NDK_ABI_FILTERS="armeabi-v7a;arm64-v8a"
+  env.ANDROID_ABI_INCLUDE="armeabi-v7a;arm64-v8a"
+  env.ANDROID_ABI_SPLIT="false"
 
+  /* some builds tyes require different architectures */
   switch (btype) {
     case 'e2e':
-      env.NDK_ABI_FILTERS="x86"; break
+      env.ANDROID_ABI_INCLUDE="x86" /* e2e builds are used with simulators */
     case 'release':
+      env.ANDROID_ABI_SPLIT="true"
       gradleOpt += "-PreleaseVersion='${utils.getVersion()}'"
   }
 
-  // The Nix script cannot access the user home directory, so best to copy the file to the Nix store and pass that to the Nix script
+  /* credentials necessary to open the keystore and sign the APK */
   withCredentials([
     string(
       credentialsId: 'android-keystore-pass',
@@ -28,36 +31,69 @@ def bundle() {
       passwordVariable: 'STATUS_RELEASE_KEY_PASSWORD'
     )
   ]) {
+    /* Nix target which produces the final APKs */
     nix.build(
+      attr: 'targets.mobile.android.release',
       args: [
         'gradle-opts': gradleOpt,
         'build-number': utils.readBuildNumber(),
-        'build-type': btype
+        'build-type': btype,
       ],
       safeEnv: [
         'STATUS_RELEASE_KEY_ALIAS',
         'STATUS_RELEASE_STORE_PASSWORD',
-        'STATUS_RELEASE_KEY_PASSWORD'
+        'STATUS_RELEASE_KEY_PASSWORD',
       ],
       keep: [
-        'NDK_ABI_FILTERS',
-        'STATUS_RELEASE_STORE_FILE'
+        'ANDROID_ABI_SPLIT',
+        'ANDROID_ABI_INCLUDE',
+        'STATUS_RELEASE_STORE_FILE',
       ],
       sbox: [
-        env.STATUS_RELEASE_STORE_FILE
+        env.STATUS_RELEASE_STORE_FILE,
       ],
-      attr: 'targets.mobile.android.release',
       link: false
     )
   }
-  /* because nix-build was run in `android` dir that's where `result` is */
-  def outApk = "result/app.apk"
-  def pkg = utils.pkgFilename(btype, 'apk')
-  /* rename for upload */
-  sh "cp ${outApk} ${pkg}"
   /* necessary for Fastlane */
-  env.APK_PATH = pkg
-  return pkg
+  def apks = renameAPKs()
+  /* for use with Fastlane */
+  env.APK_PATHS = apks.join(";")
+  return apks
+}
+
+def extractArchFromAPK(name) {
+  def pattern = /app-(.+)-[^-]+.apk/
+  /* extract architecture from filename */
+  def matches = (name =~ pattern)
+  if (matches.size() > 0) {
+    return matches[0][1]
+  }
+  /* non-release builds make universal APKs */
+  return 'universal'
+}
+
+/**
+ * We need more informative filenames for all builds.
+ * For more details on the format see utils.pkgFilename().
+ **/
+def renameAPKs() {
+  /* find all APK files */
+  def apkGlob = 'result/*.apk'
+  def found = findFiles(glob: apkGlob)
+  if (found.size() == 0) {
+    error("APKs not found via glob: ${apkGlob}")
+  }
+  def renamed = []
+  /* rename each for upload & archiving */
+  for (apk in found) {
+    def arch = extractArchFromAPK(apk)
+    def pkg = utils.pkgFilename(btype, 'apk', arch)
+    def newApk = "result/${pkg}"
+    renamed += newApk
+    sh "cp ${apk.path} ${newApk}"
+  }
+  return renamed
 }
 
 def uploadToPlayStore(type = 'nightly') {
@@ -67,7 +103,7 @@ def uploadToPlayStore(type = 'nightly') {
     nix.shell(
       "fastlane android ${type}",
       attr: 'targets.mobile.fastlane.shell',
-      keep: ['FASTLANE_DISABLE_COLORS', 'GOOGLE_PLAY_JSON_KEY']
+      keep: ['FASTLANE_DISABLE_COLORS', 'APK_PATHS', 'GOOGLE_PLAY_JSON_KEY']
     )
   }
 }
@@ -91,7 +127,7 @@ def uploadToSauceLabs() {
       'fastlane android saucelabs',
       attr: 'targets.mobile.fastlane.shell',
       keep: [
-        'FASTLANE_DISABLE_COLORS', 'APK_PATH',
+        'FASTLANE_DISABLE_COLORS', 'APK_PATHS',
         'SAUCE_ACCESS_KEY', 'SAUCE_USERNAME', 'SAUCE_LABS_NAME'
       ]
     )
@@ -106,7 +142,7 @@ def uploadToDiawi() {
     nix.shell(
       'fastlane android upload_diawi',
       attr: 'targets.mobile.fastlane.shell',
-      keep: ['FASTLANE_DISABLE_COLORS', 'APK_PATH', 'DIAWI_TOKEN']
+      keep: ['FASTLANE_DISABLE_COLORS', 'APK_PATHS', 'DIAWI_TOKEN']
     )
   }
   diawiUrl = readFile "${env.WORKSPACE}/fastlane/diawi.out"

ci/ghcmgr.groovy

@@ -54,8 +54,8 @@ def postBuild(success) {
   }
   /* We're not using --fail because it suppresses server response */
   if (!stdout.contains('HTTP_CODE:201')) {
-    error("Notifying GHCMGR failed with: ${httpCode}")
     println("STDOUT:\n${stdout}")
+    error("Notifying GHCMGR failed with: TODO")
   }
 }
 

ci/utils.groovy

@@ -31,8 +31,10 @@ def gitCommit() {
   return GIT_COMMIT.take(6)
 }
 
-def pkgFilename(type, ext) {
-  return "StatusIm-${timestamp()}-${gitCommit()}-${type}.${ext}"
+def pkgFilename(type, ext, arch="universal") {
+  return [
+    "StatusIm", timestamp(), gitCommit(), type, arch,
+  ].join('-') + ".${ext}"
 }
 
 def doGitRebase() {

fastlane/Fastfile

@@ -258,14 +258,14 @@ end
 
 platform :android do
   # Optional env variables
-  APK_PATH = ENV['APK_PATH'] || 'result/app.apk'
+  APK_PATHS = ENV["APK_PATHS"]&.split(";") or ["result/app.apk"]
 
   desc 'Deploy a new internal build to Google Play'
   desc 'expects GOOGLE_PLAY_JSON_KEY environment variable'
   lane :nightly do
     upload_to_play_store(
       track: 'internal',
-      apk: APK_PATH,
+      apk_paths: APK_PATHS,
       json_key_data: ENV['GOOGLE_PLAY_JSON_KEY']
     )
   end
@@ -275,7 +275,7 @@ platform :android do
   lane :release do
     upload_to_play_store(
       track: 'alpha',
-      apk: APK_PATH,
+      apk_paths: APK_PATHS,
       json_key_data: ENV['GOOGLE_PLAY_JSON_KEY']
     )
   end
@@ -299,7 +299,8 @@ platform :android do
   desc '---'
   desc 'Output: writes `fastlane/diawi.out` file url of the uploded file'
   lane :upload_diawi do
-    upload_to_diawi(APK_PATH)
+    uniApk = APK_PATHS.detect { |a| a.include? 'universal' }
+    upload_to_diawi(uniApk)
   end
 
   desc '`fastlane android saucelabs` - upload .apk to sauce labs'
@@ -309,6 +310,7 @@ platform :android do
   desc 'expects to have a saucelabs destination name as SAUCE_LABS_NAME env variable'
   desc "will fails if file isn't there"
   lane :saucelabs do
-    upload_to_saucelabs(APK_PATH)
+    e2eApk = APK_PATHS.detect { |a| a.include? 'x86' }
+    upload_to_saucelabs(e2eApk)
   end
 end

nix/build.sh

@@ -26,7 +26,6 @@ trap cleanup EXIT ERR INT QUIT
 # build output will end up under /nix, we have to extract it
 function extractResults() {
   local nixResultPath="$1"
-  echo "Saving build result: ${nixResultPath}"
   mkdir -p "${resultPath}"
   cp -vfr ${nixResultPath}/* "${resultPath}"
   chmod -R u+w "${resultPath}"
@@ -43,7 +42,8 @@ if [[ -z "${targetAttr}" ]]; then
   exit 1
 fi
 
-# Some defaults flags, --pure could be optional in the future
+# Some defaults flags, --pure could be optional in the future.
+# NOTE: The --keep-failed flag can be used for debugging issues.
 nixOpts=(
   "--pure"
   "--fallback"
@@ -59,6 +59,7 @@ nixOpts=(
 echo "Running: nix-build ${nixOpts[@]}"
 nixResultPath=$(nix-build ${nixOpts[@]})
 
+echo "Extracting result: ${nixResultPath}"
 extractResults "${nixResultPath}"
 
 echo "SUCCESS"

nix/mobile/android/targets/release-android.nix

@@ -23,9 +23,8 @@ let
   envFileName =
     if (build-type == "release" || build-type == "nightly" || build-type == "e2e") then ".env.${build-type}" else
     if build-type != "" then ".env.jenkins" else ".env";
-  buildType' = if (build-type == "pr" || build-type == "e2e") then "pr" else "release"; /* PR builds shouldn't replace normal releases */
-  generatedApkPath = "$sourceRoot/android/app/build/outputs/apk/${buildType'}/app-${buildType'}.apk";
-  outApkName = "app.apk";
+  buildType = if (build-type == "pr" || build-type == "e2e") then "pr" else "release"; /* PR builds shouldn't replace normal releases */
+  apksPath = "$sourceRoot/android/app/build/outputs/apk/${buildType}";
   patchedWatchman = watchmanFactory watchmanSockPath;
 
 in stdenv.mkDerivation {
@@ -104,7 +103,7 @@ in stdenv.mkDerivation {
       exportEnvVars = concatStringsSep ";" (mapAttrsToList (name: value: "export ${name}='${value}'") env);
       unsetEnvVars = concatStringsSep ";" (mapAttrsToList (name: value: "unset ${name}") env);
       adhocEnvVars = optionalString stdenv.isLinux "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${makeLibraryPath [ zlib ]}";
-      capitalizedBuildType = toUpper (substring 0 1 buildType') + substring 1 (-1) buildType';
+      capitalizedBuildType = toUpper (substring 0 1 buildType) + substring 1 (-1) buildType;
     in ''
     export STATUS_REACT_HOME=$PWD
     export HOME=$sourceRoot
@@ -123,10 +122,10 @@ in stdenv.mkDerivation {
   '';
   doCheck = true;
   checkPhase = ''
-    unzip -l ${generatedApkPath} | grep 'assets/index.android.bundle'
+    ls ${apksPath}/*.apk | xargs -n1 unzip -qql | grep 'assets/index.android.bundle'
   '';
   installPhase = ''
     mkdir -p $out
-    cp ${generatedApkPath} $out/${outApkName}
+    cp ${apksPath}/*.apk $out/
   '';
 }

scripts/release-android.sh

@@ -7,7 +7,9 @@ source "$_current_dir/lib/setup/path-support.sh"
 source_lib "platform.sh"
 
 nixOpts=(
-  "--arg env {NDK_ABI_FILTERS=\"${NDK_ABI_FILTERS}\";}"
+  "--arg env {BUILD_ENV=\"${BUILD_ENV}\";}"
+  "--arg env {ANDROID_ABI_SPLIT=\"${ANDROID_ABI_SPLIT}\";}"
+  "--arg env {ANDROID_ABI_INCLUDE=\"${ANDROID_ABI_INCLUDE}\";}"
   "--argstr build-type ${BUILD_TYPE}"
   "--argstr build-number ${BUILD_NUMBER}"
   "--argstr keystore-file ${STORE_FILE}"