From 45f1d58832bbac907e2c84e00e25fff37cc3b26d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Soko=C5=82owski?= Date: Wed, 6 May 2020 15:33:54 +0200 Subject: [PATCH] nix: simplify android release build MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jakub SokoĊ‚owski --- .gitignore | 1 + nix/deps/react-native/default.nix | 29 +++++ nix/deps/react-native/deps.nix | 29 +++++ nix/mobile/android/default.nix | 3 +- .../android/maven-and-npm-deps/default.nix | 30 ++--- .../maven/reactnative-android-native-deps.nix | 48 -------- .../android/targets/release-android.nix | 105 ++++++++---------- nix/mobile/default.nix | 9 +- nix/mobile/ios/default.nix | 9 +- nix/overlay.nix | 1 + 10 files changed, 128 insertions(+), 136 deletions(-) create mode 100644 nix/deps/react-native/default.nix create mode 100644 nix/deps/react-native/deps.nix delete mode 100644 nix/mobile/android/maven-and-npm-deps/maven/reactnative-android-native-deps.nix diff --git a/.gitignore b/.gitignore index 638761f4f6..8c310a04c5 100644 --- a/.gitignore +++ b/.gitignore @@ -129,6 +129,7 @@ fastlane/README.md .dir-locals.el #ignore platform-specific files in the root since they are only symlinks to files in folders 'desktop/js_files' and 'mobile/js_files' +/DEPENDENCIES.md /package-lock.json /package.json /metro.config.js diff --git a/nix/deps/react-native/default.nix b/nix/deps/react-native/default.nix new file mode 100644 index 0000000000..044864d454 --- /dev/null +++ b/nix/deps/react-native/default.nix @@ -0,0 +1,29 @@ +{ stdenvNoCC, fetchurl }: + +let + inherit (builtins) map baseNameOf; + + deps = import ./deps.nix; + fetchTarball = dep: with dep; + fetchurl { + inherit url sha256; + name = "${name}-${baseNameOf url}"; + }; + +in stdenvNoCC.mkDerivation { + name = "reactnative-android-deps"; + phases = [ "unpackPhase" ]; + srcs = map fetchTarball deps; + unpackPhase = '' + # Unpack all source archives. + mkdir -p $out/deps + cd $out/deps + for tarball in $srcs; do + unpackFile "$tarball" 2>/dev/null + done + cd .. + + export sourceRoot=$out/deps + echo $sourceRoot + ''; +} diff --git a/nix/deps/react-native/deps.nix b/nix/deps/react-native/deps.nix new file mode 100644 index 0000000000..03850ccb2e --- /dev/null +++ b/nix/deps/react-native/deps.nix @@ -0,0 +1,29 @@ +# These versions should match values in: +# node_modules/react-native/ReactAndroid/gradle.properties +# Currently this is done manually. +[ + rec { + name = "boost"; + version = "1.63.0"; + url = "https://github.com/react-native-community/boost-for-react-native/releases/download/v${version}-0/${name}_${builtins.replaceStrings ["."] ["_"] version}.tar.gz"; + sha256 = "1hq5h6wzcr3sk1kccfk769d59swwxy1kvfhqd6p7786a1zsavdr6"; + } + rec { + name = "double-conversion"; + version = "1.1.6"; + url = "https://github.com/google/${name}/archive/v${version}.tar.gz"; + sha256 = "0ynnckpyyhpwisb976knk9gr8jklfwr9flic8xj5flc8iv6hm1bb"; + } + rec { + name = "folly"; + version = "2018.10.22.00"; + url = "https://github.com/facebook/${name}/archive/v${version}.tar.gz"; + sha256 = "08cxc8hw34vnyyq45d04spy4dilrgqrv374lwxhg3sx60ww640y4"; + } + rec { + name = "glog"; + version = "0.3.5"; + url = "https://github.com/google/${name}/archive/v${version}.tar.gz"; + sha256 = "1q6ihk2asbx95a56kmyqwysq1x3grrw9jwqllafaidf0l84f903m"; + } +] diff --git a/nix/mobile/android/default.nix b/nix/mobile/android/default.nix index bb656f2881..966dfaec2a 100644 --- a/nix/mobile/android/default.nix +++ b/nix/mobile/android/default.nix @@ -29,7 +29,8 @@ in { shell = mkShell { buildInputs = with pkgs; [ - mavenAndNpmDeps.drv openjdk gradle + openjdk + gradle lsof # used in start-react-native.sh flock # used in reset-node_modules.sh ]; diff --git a/nix/mobile/android/maven-and-npm-deps/default.nix b/nix/mobile/android/maven-and-npm-deps/default.nix index 29f7fe6cf6..b0d3f782e1 100644 --- a/nix/mobile/android/maven-and-npm-deps/default.nix +++ b/nix/mobile/android/maven-and-npm-deps/default.nix @@ -3,21 +3,12 @@ # as well as a local version of the Maven repository required by Gradle scripts # -{ stdenv, lib, callPackage, deps, mkShell, - gradle, bash, file, nodejs, zlib, localMavenRepoBuilder }: +{ stdenv, lib, callPackage, pkgs, deps, mkShell +, localMavenRepoBuilder }: let mavenLocalRepo = callPackage ./maven { inherit localMavenRepoBuilder stdenv; }; - # Import the native dependencies for React Native Android builds - react-native-deps = callPackage ./maven/reactnative-android-native-deps.nix { }; - - createMobileFilesSymlinks = root: '' - ln -sf ${root}/mobile/js_files/package.json ${root}/package.json - ln -sf ${root}/mobile/js_files/metro.config.js ${root}/metro.config.js - ln -sf ${root}/mobile/js_files/yarn.lock ${root}/yarn.lock - ''; - # fake build to pre-download deps into fixed-output derivation drv = let @@ -48,8 +39,8 @@ let }; phases = [ "unpackPhase" "patchPhase" "installPhase" "fixupPhase" ]; nativeBuildInputs = [ deps.nodejs ]; - buildInputs = [ gradle nodejs bash file zlib mavenLocalRepo ]; - propagatedBuildInputs = [ react-native-deps ]; + buildInputs = with pkgs; [ gradle nodejs file zlib mavenLocalRepo ]; + propagatedBuildInputs = [ deps.react-native ]; unpackPhase = '' runHook preUnpack @@ -62,10 +53,8 @@ let # Copy RN maven dependencies and make them writable, otherwise Gradle copy fails (since the top-level directory is read-only, Java isn't smart enough to copy the child files/folders into that target directory) mkdir -p ${mavenRepoDir} cp -a ${mavenLocalRepo}/. ${mavenRepoDir} - cp -a ${react-native-deps}/deps ${reactNativeDepsDir} - for d in `find ${reactNativeDepsDir} -mindepth 1 -maxdepth 1 -type d`; do - chmod -R u+w $d - done + cp -a ${deps.react-native}/deps ${reactNativeDepsDir} + find ${reactNativeDepsDir} -maxdepth 1 -type d -exec chmod -R u+w {} \; # Copy node_modules from Nix store rm -rf ${projectBuildDir}/node_modules @@ -80,9 +69,6 @@ let cp -R ${projectBuildDir}/status-modules/ ${projectBuildDir}/node_modules/status-modules/ - # Set up symlinks to mobile enviroment in project root - ${createMobileFilesSymlinks projectBuildDir} - # Create a dummy VERSION, since we don't want this expression to be invalidated just because the version changed echo '0.0.1' > ${projectBuildDir}/VERSION @@ -144,7 +130,7 @@ let substituteInPlace ${projectBuildDir}/android/app/build.gradle \ --replace \ 'nodeExecutableAndArgs: ["node"' \ - 'nodeExecutableAndArgs: ["${nodejs}/bin/node"' + 'nodeExecutableAndArgs: ["${pkgs.nodejs}/bin/node"' # Fix bugs in Hermes usage (https://github.com/facebook/react-native/issues/25601#issuecomment-510856047) # - Make PR builds also count as release builds @@ -191,8 +177,6 @@ in { inherit drv; shell = mkShell { shellHook = '' - ${createMobileFilesSymlinks "$STATUS_REACT_HOME"} - export STATUSREACT_NIX_MAVEN_REPO="${drv}/.m2/repository" ''; }; diff --git a/nix/mobile/android/maven-and-npm-deps/maven/reactnative-android-native-deps.nix b/nix/mobile/android/maven-and-npm-deps/maven/reactnative-android-native-deps.nix deleted file mode 100644 index 92b6828cd3..0000000000 --- a/nix/mobile/android/maven-and-npm-deps/maven/reactnative-android-native-deps.nix +++ /dev/null @@ -1,48 +0,0 @@ -{ stdenvNoCC, fetchurl }: - -let - # These versions should match node_modules/react-native/ReactAndroid/gradle.properties - react-native-deps-sources = [ - rec { - name = "boost"; - version = "1.63.0"; - url = "https://github.com/react-native-community/boost-for-react-native/releases/download/v${version}-0/${name}_${builtins.replaceStrings ["."] ["_"] version}.tar.gz"; - sha256 = "1hq5h6wzcr3sk1kccfk769d59swwxy1kvfhqd6p7786a1zsavdr6"; - } - rec { - name = "double-conversion"; - version = "1.1.6"; - url = "https://github.com/google/${name}/archive/v${version}.tar.gz"; - sha256 = "0ynnckpyyhpwisb976knk9gr8jklfwr9flic8xj5flc8iv6hm1bb"; - } - rec { - name = "folly"; - version = "2018.10.22.00"; - url = "https://github.com/facebook/${name}/archive/v${version}.tar.gz"; - sha256 = "08cxc8hw34vnyyq45d04spy4dilrgqrv374lwxhg3sx60ww640y4"; - } - rec { - name = "glog"; - version = "0.3.5"; - url = "https://github.com/google/${name}/archive/v${version}.tar.gz"; - sha256 = "1q6ihk2asbx95a56kmyqwysq1x3grrw9jwqllafaidf0l84f903m"; - } - ]; - -in stdenvNoCC.mkDerivation { - name = "reactnative-android-native-deps"; - srcs = builtins.map (d: (fetchurl { inherit (d) url sha256; })) react-native-deps-sources; - phases = [ "unpackPhase" ]; - unpackPhase = '' - # Unpack all source archives. - mkdir -p $out/deps - cd $out/deps - for i in $srcs; do - unpackFile "$i" - done - cd .. - - export sourceRoot=$out/deps - echo $sourceRoot - ''; -} diff --git a/nix/mobile/android/targets/release-android.nix b/nix/mobile/android/targets/release-android.nix index d1cc0de968..fe40028af1 100644 --- a/nix/mobile/android/targets/release-android.nix +++ b/nix/mobile/android/targets/release-android.nix @@ -12,23 +12,13 @@ assert (lib.stringLength watchmanSockPath) > 0 -> stdenv.isDarwin; let - inherit (lib) toLower optionalString getConfig; - - # custom env variables derived from config - env = { - ANDROID_ABI_SPLIT = getConfig "android.abi-split" "false"; - ANDROID_ABI_INCLUDE = getConfig "android.abi-include" "armeabi-v7a;arm64-v8a;x86"; - STATUS_GO_SRC_OVERRIDE = getConfig "nimbus.src-override" null; - }; + inherit (lib) toLower optionalString stringLength getConfig makeLibraryPath; buildType = getConfig "build-type" "prod"; buildNumber = getConfig "build-number" 9999; gradleOpts = getConfig "android.gradle-opts" null; keystorePath = getConfig "android.keystore-path" null; - # Keep the same keystore path for determinism - keystoreLocal = "${gradleHome}/status-im.keystore"; - baseName = "release-android"; name = "status-react-build-${baseName}"; @@ -45,7 +35,7 @@ let then "Pr" else "Release"; # PR builds shouldn't replace normal releases - apksPath = "$sourceRoot/android/app/build/outputs/apk/${toLower gradleBuildType}"; + apksPath = "$PROJECT/android/app/build/outputs/apk/${toLower gradleBuildType}"; patchedWatchman = watchmanFactory watchmanSockPath; in stdenv.mkDerivation rec { @@ -74,41 +64,56 @@ in stdenv.mkDerivation rec { # Used by Clojure at compile time to include JS modules BUILD_ENV = buildEnv; - phases = [ "unpackPhase" "patchPhase" "buildPhase" "checkPhase" "installPhase" ]; + # custom env variables derived from config + STATUS_GO_SRC_OVERRIDE = getConfig "nimbus.src-override" null; + ANDROID_ABI_SPLIT = getConfig "android.abi-split" "false"; + ANDROID_ABI_INCLUDE = getConfig "android.abi-include" "armeabi-v7a;arm64-v8a;x86"; + + ANDROID_SDK_ROOT = "${androidPkgs}"; + ANDROID_NDK_ROOT = "${androidPkgs}/ndk-bundle"; + + # Used by the Android Gradle build script in android/build.gradle + STATUS_GO_ANDROID_LIBDIR = "${status-go}"; + + # Used by the Android Gradle wrapper in android/gradlew + STATUSREACT_NIX_MAVEN_REPO = "${mavenAndNpmDeps.drv}/.m2/repository"; + + phases = [ + "unpackPhase" "patchPhase" "secretPhase" "buildPhase" "checkPhase" "installPhase" + ]; unpackPhase = '' - runHook preUnpack - cp -r $src ./project chmod u+w -R ./project - - export sourceRoot=$PWD/project + export PROJECT=$PWD/project runHook postUnpack ''; postUnpack = assert lib.assertMsg (keystorePath != null) "keystore-file has to be set!"; '' mkdir -p ${gradleHome} - # WARNING: Renaming the keystore will cause 'Keystore was tampered with' error - cp -a --no-preserve=ownership "${keystorePath}" "${keystoreLocal}" + # Keep the same keystore path for determinism + export KEYSTORE_PATH="${gradleHome}/status-im.keystore" + cp -a --no-preserve=ownership "${keystorePath}" "$KEYSTORE_PATH" # Ensure we have the right .env file - cp -f $sourceRoot/${envFileName} $sourceRoot/.env + cp -f $PROJECT/${envFileName} $PROJECT/.env # Copy index.js and app/ input files - cp -ra --no-preserve=ownership ${jsbundle}/* $sourceRoot/ + cp -ra --no-preserve=ownership ${jsbundle}/* $PROJECT/ # Copy android/ directory - cp -a --no-preserve=ownership ${sourceProjectDir}/android/ $sourceRoot/ - chmod u+w $sourceRoot/android - chmod u+w $sourceRoot/android/app - mkdir -p $sourceRoot/android/build - chmod -R u+w $sourceRoot/android/build + cp -a --no-preserve=ownership ${sourceProjectDir}/android/ $PROJECT/ + chmod u+w $PROJECT/android + chmod u+w $PROJECT/android/app + mkdir -p $PROJECT/android/build + chmod -R u+w $PROJECT/android/build # Copy node_modules/ directory - cp -a --no-preserve=ownership ${sourceProjectDir}/node_modules/ $sourceRoot/ + cp -a --no-preserve=ownership ${sourceProjectDir}/node_modules/ $PROJECT/ + # Make android/build directories writable under node_modules - for d in `find $sourceRoot/node_modules -type f -name build.gradle | xargs dirname`; do + for d in `find $PROJECT/node_modules -type f -name build.gradle | xargs dirname`; do chmod -R u+w $d done ''; @@ -116,50 +121,38 @@ in stdenv.mkDerivation rec { prevSet=$- set -e - substituteInPlace $sourceRoot/android/gradlew \ + substituteInPlace $PROJECT/android/gradlew \ --replace \ 'exec gradle' \ "exec gradle -Dmaven.repo.local='${localMavenRepo}' --offline ${toString gradleOpts}" set $prevSet ''; + secretPhase = optionalString (secretsFile != "") '' + source "${secretsFile}" + ''; buildPhase = let - inherit (lib) - stringLength optionalString substring - toInt concatStrings concatStringsSep - catAttrs mapAttrsToList makeLibraryPath; - - # Take the env attribute set and build a couple of scripts - # (one to export the environment variables, and another to unset them) - exportEnvVars = concatStringsSep ";" - (mapAttrsToList (name: value: "export ${name}='${toString value}'") env); adhocEnvVars = optionalString stdenv.isLinux "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${makeLibraryPath [ zlib ]}"; in - assert env.ANDROID_ABI_SPLIT != null && env.ANDROID_ABI_SPLIT != ""; - assert stringLength env.ANDROID_ABI_INCLUDE > 0; + assert ANDROID_ABI_SPLIT != null && ANDROID_ABI_SPLIT != ""; + assert stringLength ANDROID_ABI_INCLUDE > 0; '' - export ANDROID_SDK_ROOT="${androidPkgs}" - export ANDROID_NDK_ROOT="${androidPkgs}/ndk-bundle" - - export KEYSTORE_PATH="${keystoreLocal}" - export STATUS_REACT_HOME=$PWD - export HOME=$sourceRoot + export HOME=$PROJECT - # Used by the Android Gradle build script in android/build.gradle - export STATUS_GO_ANDROID_LIBDIR=${status-go} - - ${exportEnvVars} - ${optionalString (secretsFile != "") "source ${secretsFile}"} - - ${concatStrings (catAttrs "shellHook" [ mavenAndNpmDeps.shell ])} + # create mobile node/yarn symlinks + ln -sf $PROJECT/mobile/js_files/* $PROJECT/ # fix permissions so gradle can create directories - chmod -R +w $sourceRoot/android + chmod -R +w $PROJECT/android - pushd $sourceRoot/android - ${adhocEnvVars} ./gradlew -PversionCode=${toString buildNumber} assemble${gradleBuildType} || exit + pushd $PROJECT/android + ${adhocEnvVars} ./gradlew \ + --stacktrace \ + -PversionCode=${toString buildNumber} \ + assemble${gradleBuildType} \ + || exit 1 popd > /dev/null ''; doCheck = true; diff --git a/nix/mobile/default.nix b/nix/mobile/default.nix index 9e356d16d2..affddd8f7f 100644 --- a/nix/mobile/default.nix +++ b/nix/mobile/default.nix @@ -1,5 +1,4 @@ -{ config, lib, stdenvNoCC, callPackage, mkShell, - status-go, xcodeWrapper }: +{ config, lib, stdenvNoCC, callPackage, mkShell, status-go }: let inherit (lib) catAttrs concatStrings optional unique; @@ -14,7 +13,7 @@ let }; ios = callPackage ./ios { - inherit xcodeWrapper fastlane; + inherit fastlane; status-go = status-go.mobile.ios; }; @@ -29,6 +28,10 @@ let in { shell = mkShell { inputsFrom = (catAttrs "shell" selectedSources); + shellHooks = '' + # create mobile node/yarn symlinks + ln -sf $STATUS_REACT_HOME/mobile/js_files/* $STATUS_REACT_HOME/ + ''; }; # TARGETS diff --git a/nix/mobile/ios/default.nix b/nix/mobile/ios/default.nix index f97250ff16..e0f1ef6271 100644 --- a/nix/mobile/ios/default.nix +++ b/nix/mobile/ios/default.nix @@ -1,6 +1,5 @@ -{ callPackage, lib, mkShell, - xcodeWrapper, projectNodePackage, status-go, - flock, procps, watchman, bundler, fastlane }: +{ callPackage, lib, mkShell, deps, pkgs +, status-go, fastlane }: let inherit (lib) catAttrs unique; @@ -12,7 +11,7 @@ in { inherit pod-shell status-go-shell; shell = mkShell { - buildInputs = [ + buildInputs = with pkgs; [ xcodeWrapper watchman bundler procps flock # used in reset-node_modules.sh ]; @@ -32,7 +31,7 @@ in { ln -sf ./mobile/js_files/yarn.lock ./yarn.lock # check if node modules changed and if so install them - ./nix/mobile/reset-node_modules.sh "${projectNodePackage}" + ./nix/mobile/reset-node_modules.sh "${deps.nodejs}" } popd > /dev/null ''; diff --git a/nix/overlay.nix b/nix/overlay.nix index a33c980c3c..64ecd01686 100644 --- a/nix/overlay.nix +++ b/nix/overlay.nix @@ -24,6 +24,7 @@ in { deps = { clojure = callPackage ./deps/clojure { }; nodejs = callPackage ./deps/nodejs { }; + react-native = callPackage ./deps/react-native { }; }; # Android environement