From 73a8992db7e43d5a3e4791eac4e4bb4468ae2e06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Soko=C5=82owski?= Date: Thu, 7 May 2020 12:21:39 +0200 Subject: [PATCH] nix: refactor loading of node.js modules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Changes: - Drop `nix/mobile/android/maven-and-npm-deps/default.nix` - Replace it with much simpler `nix/tools/patchNodeModules` - Move Gradle patching tool to `nix/pkgs/patch-maven-srcs` - Simplify it by using `gradle.deps` and patched node modules separately - Change `TARGET` for `release-android` to `default` - Move `mobile/reset-node_modules.sh` to `scripts/node_modules.sh` - Move `nix/mobile/android/targets/release-android.nix` to `nix/mobile/android/release.nix` Signed-off-by: Jakub SokoĊ‚owski --- Makefile | 2 +- nix/deps/clojure/default.nix | 2 +- nix/deps/gradle/default.nix | 3 +- nix/deps/gradle/shell.nix | 21 --- nix/mobile/android/default.nix | 23 ++- .../android/maven-and-npm-deps/default.nix | 160 ------------------ .../release-android.nix => release.nix} | 66 +++----- nix/mobile/ios/default.nix | 4 +- nix/overlay.nix | 12 +- nix/pkgs/aapt2/default.nix | 33 +++- .../sdk => pkgs/android-sdk}/default.nix | 0 .../sdk => pkgs/android-sdk}/licensed.nix | 0 .../android/sdk => pkgs/android-sdk}/pkgs.nix | 0 .../sdk => pkgs/android-sdk}/shell.nix | 0 nix/pkgs/patch-maven-srcs/default.nix | 26 +++ .../node_modules.sh} | 19 ++- nix/shells.nix | 4 +- nix/tools/patchNodeModules.nix | 68 ++++++++ 18 files changed, 183 insertions(+), 260 deletions(-) delete mode 100644 nix/deps/gradle/shell.nix delete mode 100644 nix/mobile/android/maven-and-npm-deps/default.nix rename nix/mobile/android/{targets/release-android.nix => release.nix} (72%) rename nix/{mobile/android/sdk => pkgs/android-sdk}/default.nix (100%) rename nix/{mobile/android/sdk => pkgs/android-sdk}/licensed.nix (100%) rename nix/{mobile/android/sdk => pkgs/android-sdk}/pkgs.nix (100%) rename nix/{mobile/android/sdk => pkgs/android-sdk}/shell.nix (100%) create mode 100644 nix/pkgs/patch-maven-srcs/default.nix rename nix/{mobile/reset-node_modules.sh => scripts/node_modules.sh} (80%) create mode 100644 nix/tools/patchNodeModules.nix diff --git a/Makefile b/Makefile index 838247b649..feefa1f9a9 100644 --- a/Makefile +++ b/Makefile @@ -151,7 +151,7 @@ keystore: ##@prepare Generate a Keystore for signing Android APKs #---------------- release: release-android release-ios ##@build build release for Android and iOS -release-android: export TARGET := android +release-android: export TARGET := default release-android: export BUILD_ENV ?= prod release-android: export BUILD_TYPE ?= nightly release-android: export BUILD_NUMBER ?= $(TMP_BUILD_NUMBER) diff --git a/nix/deps/clojure/default.nix b/nix/deps/clojure/default.nix index dfe20ddff1..d3c299b64e 100644 --- a/nix/deps/clojure/default.nix +++ b/nix/deps/clojure/default.nix @@ -1,4 +1,4 @@ -{ stdenv, lib, writeShellScriptBin, fetchurl }: +{ stdenv, lib, fetchurl, writeShellScriptBin }: let inherit (lib) diff --git a/nix/deps/gradle/default.nix b/nix/deps/gradle/default.nix index e0d0c4abc6..3af31eda1a 100644 --- a/nix/deps/gradle/default.nix +++ b/nix/deps/gradle/default.nix @@ -16,9 +16,8 @@ let mkdir -p $out cd $out '' + - (concatMapStrings (dep': + (concatMapStrings (dep: let - dep = { postCopy = ""; } // dep'; url = "${dep.host}/${dep.path}"; pom = { sha1 = attrByPath [ "pom" "sha1" ] "" dep; diff --git a/nix/deps/gradle/shell.nix b/nix/deps/gradle/shell.nix deleted file mode 100644 index ca436f291f..0000000000 --- a/nix/deps/gradle/shell.nix +++ /dev/null @@ -1,21 +0,0 @@ -{ pkgs ? import ../../pkgs.nix { } }: - -let - status-go = pkgs.callPackage ../../status-go { }; - nodeJsDeps = pkgs.callPackage ../nodejs { }; -in pkgs.mkShell { - buildInputs = with pkgs; [ - curl flock # used in reset-node_modules.sh - git gradle jq maven nodejs - ]; - inputsFrom = [ pkgs.androidShell ]; - shellHook = '' - # Gradle checks for this being set - export STATUS_GO_ANDROID_LIBDIR=${status-go.mobile.android} - - # Necessary for reset-node_modules.sh - export STATUS_REACT_HOME=$(realpath ../../../) - - $STATUS_REACT_HOME/nix/mobile/reset-node_modules.sh "${nodeJsDeps}" - ''; -} diff --git a/nix/mobile/android/default.nix b/nix/mobile/android/default.nix index d21fae09d5..3b50c11cd0 100644 --- a/nix/mobile/android/default.nix +++ b/nix/mobile/android/default.nix @@ -1,5 +1,5 @@ -{ lib, pkgs, callPackage, mkShell -, status-go, gradle, androidPkgs, androidShell }: +{ lib, pkgs, deps, callPackage, mkShell +, status-go, androidPkgs, androidShell, patchNodeModules }: let # Import a jsbundle compiled out of clojure codebase @@ -8,29 +8,28 @@ let # Import a patched version of watchman (important for sandboxed builds on macOS) watchmanFactory = callPackage ./watchman.nix { }; - # Import a local patched version of node_modules, together with a local version of the Maven repo - mavenAndNpmDeps = callPackage ./maven-and-npm-deps { }; + # Some node_modules have build.gradle files that reference remote repos. + # This patches them to reference local repos only + nodeJsModules = patchNodeModules deps.nodejs deps.gradle; # TARGETS - release = callPackage ./targets/release-android.nix { - inherit gradle mavenAndNpmDeps jsbundle status-go watchmanFactory; + release = callPackage ./release.nix { + inherit jsbundle status-go watchmanFactory nodeJsModules; }; in { # TARGETS - inherit release jsbundle; + inherit release jsbundle nodeJsModules; shell = mkShell { buildInputs = with pkgs; [ openjdk gradle lsof # used in start-react-native.sh - flock # used in reset-node_modules.sh - mavenAndNpmDeps + flock # used in nix/scripts/node_modules.sh ]; inputsFrom = [ - gradle release androidShell ]; @@ -39,7 +38,7 @@ in { export ANDROID_SDK_ROOT="${androidPkgs}" export ANDROID_NDK_ROOT="${androidPkgs}/ndk-bundle" - export STATUSREACT_NIX_MAVEN_REPO="${mavenAndNpmDeps}/.m2/repository" + export STATUSREACT_NIX_MAVEN_REPO="${deps.gradle}" # required by some makefile targets export STATUS_GO_ANDROID_LIBDIR=${status-go} @@ -51,7 +50,7 @@ in { ln -sf ./mobile/js_files/* ./ # check if node modules changed and if so install them - $STATUS_REACT_HOME/nix/mobile/reset-node_modules.sh ${mavenAndNpmDeps}/project + $STATUS_REACT_HOME/nix/scripts/node_modules.sh ${nodeJsModules} } ''; }; diff --git a/nix/mobile/android/maven-and-npm-deps/default.nix b/nix/mobile/android/maven-and-npm-deps/default.nix deleted file mode 100644 index 7c5e9e1129..0000000000 --- a/nix/mobile/android/maven-and-npm-deps/default.nix +++ /dev/null @@ -1,160 +0,0 @@ -# -# This script prepares a finalized version of node_modules, -# as well as a local version of the Maven repository required by Gradle scripts -# - -{ stdenv, lib, callPackage, pkgs, deps, mkShell }: - -# fake build to pre-download deps into fixed-output derivation -let - # Place build target directories in NIX_BUILD_TOP (normally represents /build) - projectBuildDir = "$NIX_BUILD_TOP/project"; - mavenRepoDir = "$NIX_BUILD_TOP/.m2/repository"; - reactNativeDepsDir = "$NIX_BUILD_TOP/deps"; # Use local writable deps, otherwise (probably due to some interaction between Nix sandboxing and Java) gradle will fail copying directly from the nix store - -in stdenv.mkDerivation { - name = "status-react-patched-npm-gradle-modules"; - src = - let path = ./../../../..; # Import the root /android and /mobile/js_files folders clean of any build artifacts - in builtins.path { # We use builtins.path so that we can name the resulting derivation, otherwise the name would be taken from the checkout directory, which is outside of our control - inherit path; - name = "status-react-source-gradle-install"; - filter = - # Keep this filter as restrictive as possible in order to avoid unnecessary rebuilds and limit closure size - lib.mkFilter { - root = path; - include = [ - "android/.*" "resources/.*" - "mobile/js_files/.*" ".babelrc" - ]; - exclude = [ - ".*.keystore" "node_modules" - ]; - }; - }; - phases = [ "unpackPhase" "patchPhase" "installPhase" ]; - nativeBuildInputs = [ deps.nodejs ]; - buildInputs = with pkgs; [ gradle nodejs file zlib deps.gradle ]; - propagatedBuildInputs = [ deps.react-native ]; - unpackPhase = '' - runHook preUnpack - - # Copy project directory - mkdir -p ${projectBuildDir} - cp -a $src/. ${projectBuildDir} - chmod u+w ${projectBuildDir} - cd ${projectBuildDir} - - # Copy RN maven dependencies and make them writable, otherwise Gradle copy fails (since the top-level directory is read-only, Java isn't smart enough to copy the child files/folders into that target directory) - mkdir -p ${mavenRepoDir} - cp -a ${deps.gradle}/. ${mavenRepoDir} - cp -a ${deps.react-native}/deps ${reactNativeDepsDir} - find ${reactNativeDepsDir} -maxdepth 1 -type d -exec chmod -R u+w {} \; - - # Copy node_modules from Nix store - rm -rf ${projectBuildDir}/node_modules - mkdir -p ${projectBuildDir}/node_modules - cp -a ${deps.nodejs}/node_modules/. ${projectBuildDir}/node_modules/ - - # Ensure that module was correctly installed - [ -d ${projectBuildDir}/node_modules/jsc-android/dist ] || exit 1 - - # Adjust permissions - chmod -R u+w ${projectBuildDir} - - # Create a dummy VERSION, since we don't want this expression to be invalidated just because the version changed - echo '0.0.1' > ${projectBuildDir}/VERSION - - runHook postUnpack - ''; - patchPhase = '' - runHook prePatch - - prevSet=$- - set -e - - patchShebangs ${projectBuildDir} - - function patchMavenSource() { - set +e - - local targetGradleFile="$1" - local source="$2" - local deriv="$3" - grep "$source" $targetGradleFile > /dev/null && \ - substituteInPlace $targetGradleFile --replace "$source" "$deriv" - } - - function patchMavenSources() { - set +e - - local targetGradleFile="$1" - local deriv="$2" - patchMavenSource $targetGradleFile 'mavenCentral()' 'mavenLocal()' - patchMavenSource $targetGradleFile 'google()' 'mavenLocal()' - patchMavenSource $targetGradleFile 'jcenter()' 'mavenLocal()' - grep 'https://maven.google.com' $targetGradleFile > /dev/null && \ - substituteInPlace $targetGradleFile --replace 'https://maven.google.com' "$deriv" - grep 'https://jitpack.io' $targetGradleFile > /dev/null && \ - substituteInPlace $targetGradleFile --replace 'https://jitpack.io' "$deriv" - } - - # Patch maven and google central repositories with our own local directories. This prevents the builder from downloading Maven artifacts - patchMavenSources 'android/build.gradle' '${deps.gradle}' - for f in `find ${projectBuildDir}/node_modules/ -name build.gradle`; do - patchMavenSources $f '${deps.gradle}' - done - - # Do not add a BuildId to the generated libraries, for reproducibility - substituteInPlace ${projectBuildDir}/node_modules/react-native/ReactAndroid/src/main/jni/Application.mk \ - --replace \ - '-Wl,--build-id' \ - '-Wl,--build-id=none' - - # Disable Gradle daemon and caching, since that causes rebuilds (and subsequently errors) anyway due to cache being considered stale - substituteInPlace ${projectBuildDir}/android/gradle.properties \ - --replace \ - 'org.gradle.jvmargs=-Xmx8704M' \ - 'org.gradle.jvmargs=-Xmx8704M - org.gradle.daemon=false - org.gradle.caching=false' - - # Patch the path to nodejs in project.ext.react - substituteInPlace ${projectBuildDir}/android/app/build.gradle \ - --replace \ - 'nodeExecutableAndArgs: ["node"' \ - 'nodeExecutableAndArgs: ["${pkgs.nodejs}/bin/node"' - - # Fix bugs in Hermes usage (https://github.com/facebook/react-native/issues/25601#issuecomment-510856047) - # - Make PR builds also count as release builds - # - Fix issue where hermes command is being called with same input/output file - substituteInPlace ${projectBuildDir}/node_modules/react-native/react.gradle \ - --replace \ - 'targetName.toLowerCase().contains("release")' \ - '!targetName.toLowerCase().contains("debug")' - - # Patch dependencies which are not yet ported to AndroidX - npx jetify - - set $prevSet - - runHook postPatch - ''; - installPhase = '' - rm -rf $out - mkdir -p $out/{project,.m2/repository} - - # TODO: maybe node_modules/react-native/ReactAndroid/build/{tmp,generated} can be discarded? - cp -R ${mavenRepoDir} $out/.m2/ - cp -R ${projectBuildDir}/{android,node_modules}/ $out/project - ''; - - # The ELF types are incompatible with the host platform, so let's not even try - # TODO: Use Android NDK to strip binaries manually - dontPatchELF = true; - dontStripHost = true; - - # Take whole sources into consideration when calculating sha - outputHashMode = "recursive"; - outputHashAlgo = "sha256"; -} diff --git a/nix/mobile/android/targets/release-android.nix b/nix/mobile/android/release.nix similarity index 72% rename from nix/mobile/android/targets/release-android.nix rename to nix/mobile/android/release.nix index a2aa1921e3..d49d7169a4 100644 --- a/nix/mobile/android/targets/release-android.nix +++ b/nix/mobile/android/release.nix @@ -1,6 +1,6 @@ -{ stdenv, lib, config, callPackage, +{ stdenv, lib, config, callPackage, deps, bash, file, gnumake, watchmanFactory, gradle, - androidPkgs, mavenAndNpmDeps, + androidPkgs, patchMavenSources, nodeJsModules, nodejs, openjdk, jsbundle, status-go, unzip, zlib }: { @@ -22,9 +22,6 @@ let baseName = "release-android"; name = "status-react-build-${baseName}"; - gradleHome = "$NIX_BUILD_TOP/.gradle"; - localMavenRepo = "${mavenAndNpmDeps}/.m2/repository"; - sourceProjectDir = "${mavenAndNpmDeps}/project"; envFileName = if (buildType == "release" || buildType == "nightly" || buildType == "e2e") then ".env.${buildType}" else if buildType != "" then ".env.jenkins" @@ -35,12 +32,12 @@ let then "Pr" else "Release"; # PR builds shouldn't replace normal releases - apksPath = "$PROJECT/android/app/build/outputs/apk/${toLower gradleBuildType}"; + apksPath = "./android/app/build/outputs/apk/${toLower gradleBuildType}"; patchedWatchman = watchmanFactory watchmanSockPath; in stdenv.mkDerivation rec { inherit name; - src = let path = ./../../../..; + src = let path = ./../../..; # We use builtins.path so that we can name the resulting derivation in builtins.path { inherit path; @@ -50,7 +47,7 @@ in stdenv.mkDerivation rec { root = path; include = [ "mobile/js_files.*" "resources/.*" "translations/.*" - "modules/react-native-status/android.*" + "modules/react-native-status/android.*" "android/.*" envFileName "VERSION" ".watchmanconfig" "status-go-version.json" "react-native.config.js" ]; @@ -69,6 +66,7 @@ in stdenv.mkDerivation rec { ANDROID_ABI_SPLIT = getConfig "android.abi-split" "false"; ANDROID_ABI_INCLUDE = getConfig "android.abi-include" "armeabi-v7a;arm64-v8a;x86"; + # Android SDK/NDK for use by Gradle ANDROID_SDK_ROOT = "${androidPkgs}"; ANDROID_NDK_ROOT = "${androidPkgs}/ndk-bundle"; @@ -80,39 +78,38 @@ in stdenv.mkDerivation rec { ]; unpackPhase = '' - cp -r $src ./project - chmod u+w -R ./project - export PROJECT=$PWD/project - + cp -ar $src/. ./ + chmod u+w -R ./ runHook postUnpack ''; postUnpack = assert lib.assertMsg (keystorePath != null) "keystore-file has to be set!"; '' - mkdir -p ${gradleHome} - # Keep the same keystore path for determinism - export KEYSTORE_PATH="${gradleHome}/status-im.keystore" + export KEYSTORE_PATH="$PWD/status-im.keystore" cp -a --no-preserve=ownership "${keystorePath}" "$KEYSTORE_PATH" # Ensure we have the right .env file - cp -f $PROJECT/${envFileName} $PROJECT/.env + cp -f ./${envFileName} ./.env + + # create mobile node/yarn symlinks + ln -sf ./mobile/js_files/* ./ # Copy index.js and app/ input files - cp -ra --no-preserve=ownership ${jsbundle}/* $PROJECT/ + cp -ra --no-preserve=ownership ${jsbundle}/* ./ # Copy android/ directory - cp -a --no-preserve=ownership ${sourceProjectDir}/android/ $PROJECT/ - chmod u+w $PROJECT/android - chmod u+w $PROJECT/android/app - mkdir -p $PROJECT/android/build - chmod -R u+w $PROJECT/android/build + mkdir -p ./android/build + chmod -R +w ./android - # Copy node_modules/ directory - cp -a --no-preserve=ownership ${sourceProjectDir}/node_modules/ $PROJECT/ + # Copy node_modules/ directory. The -L is CRUCIAL! + # Otherwise Metro failes to find modules due to symlinks. + cp -aL ${nodeJsModules}/node_modules/ ./ + chmod +w -R ./node_modules - # Make android/build directories writable under node_modules - for d in `find $PROJECT/node_modules -type f -name build.gradle | xargs dirname`; do - chmod -R u+w $d - done + # Patch build.gradle to use local repo + ${patchMavenSources} ./android/build.gradle ${deps.gradle} + + # Patch dependencies which are not yet ported to AndroidX + ${nodejs}/bin/node ./node_modules/jetifier/bin/jetify ''; secretPhase = optionalString (secretsFile != "") '' source "${secretsFile}" @@ -124,19 +121,10 @@ in stdenv.mkDerivation rec { assert ANDROID_ABI_SPLIT != null && ANDROID_ABI_SPLIT != ""; assert stringLength ANDROID_ABI_INCLUDE > 0; '' - export STATUS_REACT_HOME=$PWD - export HOME=$PROJECT - - # create mobile node/yarn symlinks - ln -sf $PROJECT/mobile/js_files/* $PROJECT/ - - # fix permissions so gradle can create directories - chmod -R +w $PROJECT/android - - pushd $PROJECT/android + pushd ./android ${adhocEnvVars} gradle ${toString gradleOpts} \ --offline --stacktrace \ - -Dmaven.repo.local='${localMavenRepo}' \ + -Dmaven.repo.local='${deps.gradle}' \ -PversionCode=${toString buildNumber} \ assemble${gradleBuildType} \ || exit 1 diff --git a/nix/mobile/ios/default.nix b/nix/mobile/ios/default.nix index 5b3f1099e2..a98f124ec7 100644 --- a/nix/mobile/ios/default.nix +++ b/nix/mobile/ios/default.nix @@ -13,7 +13,7 @@ in { shell = mkShell { buildInputs = with pkgs; [ xcodeWrapper watchman bundler procps - flock # used in reset-node_modules.sh + flock # used in nix/scripts/node_modules.sh ]; inputsFrom = [ @@ -30,7 +30,7 @@ in { ln -sf ./mobile/js_files/* ./ # check if node modules changed and if so install them - ./nix/mobile/reset-node_modules.sh "${deps.nodejs}" + ./nix/scripts/node_modules.sh "${deps.nodejs}" } ''; }; diff --git a/nix/overlay.nix b/nix/overlay.nix index 2a2dad0292..8c2f47e939 100644 --- a/nix/overlay.nix +++ b/nix/overlay.nix @@ -27,16 +27,19 @@ in { react-native = callPackage ./deps/react-native { }; }; - # Android environement - androidEnvCustom = callPackage ./mobile/android/sdk { }; - androidPkgs = self.androidEnvCustom.licensedPkgs; - androidShell = self.androidEnvCustom.shell; + # For patching Node.js modules with Gradle repo path + patchNodeModules = callPackage ./tools/patchNodeModules.nix { }; # Package version adjustments xcodeWrapper = super.xcodeenv.composeXcodeWrapper { version = "11.4.1"; }; openjdk = super.pkgs.openjdk8_headless; nodejs = super.pkgs.nodejs-12_x; + # Android environement + androidEnvCustom = callPackage ./pkgs/android-sdk { }; + androidPkgs = self.androidEnvCustom.licensedPkgs; + androidShell = self.androidEnvCustom.shell; + # Custom packages aapt2 = callPackage ./pkgs/aapt2 { }; gomobile = callPackage ./pkgs/gomobile { }; @@ -44,4 +47,5 @@ in { qtkeychain-src = callPackage ./pkgs/qtkeychain-src { }; appimagekit = callPackage ./pkgs/appimagekit { }; linuxdeployqt = callPackage ./pkgs/linuxdeployqt { inherit (self) appimagekit; }; + patchMavenSources = callPackage ./pkgs/patch-maven-srcs { }; } diff --git a/nix/pkgs/aapt2/default.nix b/nix/pkgs/aapt2/default.nix index 202a096068..df9d6fdd6f 100644 --- a/nix/pkgs/aapt2/default.nix +++ b/nix/pkgs/aapt2/default.nix @@ -2,10 +2,12 @@ # It is used by Gradle to package Android app resources. # See: https://developer.android.com/studio/command-line/aapt2 -{ stdenv, deps, pkgs, fetchurl }: +{ lib, stdenv, deps, pkgs, fetchurl }: let - inherit (pkgs) zip unzip; + inherit (lib) getAttr optionals; + inherit (pkgs) zip unzip patchelf; + inherit (stdenv) isLinux isDarwin; pname = "aapt2"; # Warning: This must be the same as gradlePluginVersion android/gradle.properties @@ -14,19 +16,30 @@ let pkgPath = "com/android/tools/build/aapt2"; repoUrl = "https://dl.google.com/dl/android/maven2"; + platform = + if isLinux then "linux" else + if isDarwin then "osx" else + throw "Unknown platform!"; + filenames = { - jar = "${pname}-${version}-linux.jar"; + jar = "${pname}-${version}-${platform}.jar"; pom = "${pname}-${version}.pom"; }; urls = { jar = fetchurl { url = "${repoUrl}/${pkgPath}/${version}/${filenames.jar}"; - sha256 = "05gln93wfj4l5b0zfn6ipkx0i9p0x928ygwkrcfyl58aslxg5gx2"; + sha256 = getAttr platform { + linux = "05gln93wfj4l5b0zfn6ipkx0i9p0x928ygwkrcfyl58aslxg5gx2"; + osx = "0nzc3hq3fm847a3rvwdz26ln3inh50x44ml0dq486yz45qv9f7cs"; + }; }; sha = fetchurl { url = "${repoUrl}/${pkgPath}/${version}/${filenames.jar}.sha1"; - sha256 = "0rr7ly0f3w5jw0q985hmxmv8q2nlw1k72n6kl7kcmj4a7i479q90"; + sha256 = getAttr platform { + linux = "0rr7ly0f3w5jw0q985hmxmv8q2nlw1k72n6kl7kcmj4a7i479q90"; + osx = "0k7j54x627jsnl8zdcfj62jj8z0c857yqs3iwyvn29hd02v1b78m"; + }; }; pom = fetchurl { url = "${repoUrl}/${pkgPath}/${version}/${filenames.pom}"; @@ -38,13 +51,15 @@ in stdenv.mkDerivation { inherit pname version; srcs = with urls; [ jar sha pom ]; - phases = [ "unpackPhase" "patchPhase" ]; + phases = [ "unpackPhase" ] + ++ optionals isLinux [ "patchPhase" ]; # OSX binaries don't need patchelf + buildInputs = [ zip unzip patchelf ]; unpackPhase = '' mkdir -p $out for src in $srcs; do - local filename=$(basename $src) - cp $src $out/''${filename#*-} + filename=$(stripHash $src) + cp $src $out/$filename done ''; @@ -58,7 +73,7 @@ in stdenv.mkDerivation { tmpDir=$(mktemp -d) ${unzip}/bin/unzip $out/${filenames.jar} -d $tmpDir for exe in `find $tmpDir/ -type f -executable`; do - patchelf --set-interpreter "$(cat $NIX_CC/nix-support/dynamic-linker)" $exe + ${patchelf}/bin/patchelf --set-interpreter "$(cat $NIX_CC/nix-support/dynamic-linker)" $exe done # Rebuild the .jar file with patched binaries diff --git a/nix/mobile/android/sdk/default.nix b/nix/pkgs/android-sdk/default.nix similarity index 100% rename from nix/mobile/android/sdk/default.nix rename to nix/pkgs/android-sdk/default.nix diff --git a/nix/mobile/android/sdk/licensed.nix b/nix/pkgs/android-sdk/licensed.nix similarity index 100% rename from nix/mobile/android/sdk/licensed.nix rename to nix/pkgs/android-sdk/licensed.nix diff --git a/nix/mobile/android/sdk/pkgs.nix b/nix/pkgs/android-sdk/pkgs.nix similarity index 100% rename from nix/mobile/android/sdk/pkgs.nix rename to nix/pkgs/android-sdk/pkgs.nix diff --git a/nix/mobile/android/sdk/shell.nix b/nix/pkgs/android-sdk/shell.nix similarity index 100% rename from nix/mobile/android/sdk/shell.nix rename to nix/pkgs/android-sdk/shell.nix diff --git a/nix/pkgs/patch-maven-srcs/default.nix b/nix/pkgs/patch-maven-srcs/default.nix new file mode 100644 index 0000000000..1c63952bf6 --- /dev/null +++ b/nix/pkgs/patch-maven-srcs/default.nix @@ -0,0 +1,26 @@ +# This script patches build.gradle files in node_modules to use +# our local version of Maven dependencies instead of fetching them. + +{ stdenv, writeScript, runtimeShell }: + +writeScript "patch-maven-srcs" ('' + #!${runtimeShell} + # Source setup.sh for substituteInPlace + source ${stdenv}/setup + + function patchMavenSource() { + grep "$source" $1 > /dev/null && \ + substituteInPlace $1 --replace "$2" "$3" 2>/dev/null + } + + gradleFile="$1" + derivation="$2" + + # Some of those find something, some don't, that's fine. + patchMavenSource "$gradleFile" 'mavenCentral()' 'mavenLocal()' + patchMavenSource "$gradleFile" 'google()' 'mavenLocal()' + patchMavenSource "$gradleFile" 'jcenter()' 'mavenLocal()' + patchMavenSource "$gradleFile" 'https://maven.google.com' "$derivation" + patchMavenSource "$gradleFile" 'https://www.jitpack.io' "$derivation" + patchMavenSource "$gradleFile" 'https://jitpack.io' "$derivation" +'') diff --git a/nix/mobile/reset-node_modules.sh b/nix/scripts/node_modules.sh similarity index 80% rename from nix/mobile/reset-node_modules.sh rename to nix/scripts/node_modules.sh index 90e01b250a..910749cd25 100755 --- a/nix/mobile/reset-node_modules.sh +++ b/nix/scripts/node_modules.sh @@ -55,14 +55,19 @@ function replaceNodeModules() { # Replace node_modules if necessary if [ ! -d "$targetNodeModules" ]; then local tmpNodeModules=$(mktemp -d) - echo "Copying node_modules from Nix store (${deps}/node_modules)..." - trap "[ -d \"$tmpNodeModules\" ] && chmod -R u+w \"$tmpNodeModules\" && rm -rf \"$tmpNodeModules\"" ERR INT HUP - time cp -HRf --preserve=all ${deps}/node_modules/. "$tmpNodeModules" - chmod -R u+w "$tmpNodeModules" - mv -f "$tmpNodeModules" "$targetNodeModules" - echo -n "${deps}" > $sentinelFilePath + echo "Copying node_modules from Nix store:" + echo " - ${deps}" + export TIMEFORMAT="Done in: %Es" + trap "[ -d \"${tmpNodeModules}\" ] && chmod -R u+w \"${tmpNodeModules}\" && rm -rf \"${tmpNodeModules}\"" ERR INT HUP + # WARNING: The -L here is crucial to let Metro find modules. + time cp -LRf ${deps}/node_modules/. "${tmpNodeModules}" + chmod -R u+w "${tmpNodeModules}" + # WARNING: We can't de-reference .bin symlinks + cp -Rf ${deps}/node_modules/.bin/. "${tmpNodeModules}/.bin/" + mv -f "${tmpNodeModules}" "${targetNodeModules}" + + echo -n "${deps}" > "${sentinelFilePath}" trap - ERR INT HUP - echo "Done" fi } diff --git a/nix/shells.nix b/nix/shells.nix index 003a12ebba..b9afa19b20 100644 --- a/nix/shells.nix +++ b/nix/shells.nix @@ -18,7 +18,7 @@ let buildInputs = with pkgs; lib.unique ([ # core utilities that should always be present in a shell - bash curl wget file unzip flock + bash curl wget file unzip flock rsync git gnumake jq ncurses gnugrep parallel # build specific utilities clojure maven watchman @@ -48,7 +48,7 @@ let buildInputs = [ pkgs.androidPkgs ]; shellHook = '' export STATUS_REACT_HOME=$(git rev-parse --show-toplevel) - $STATUS_REACT_HOME/nix/mobile/reset-node_modules.sh ${pkgs.deps.nodejs} + $STATUS_REACT_HOME/nix/scripts/node_modules.sh ${pkgs.deps.nodejs} ''; }; diff --git a/nix/tools/patchNodeModules.nix b/nix/tools/patchNodeModules.nix new file mode 100644 index 0000000000..8f1b9d53f4 --- /dev/null +++ b/nix/tools/patchNodeModules.nix @@ -0,0 +1,68 @@ +# This method patches Node.js dependencies by taking the +# result of yarn2nix and symlinking what is fine, and +# copying and modifying what needs to be adjusted. + +{ stdenv, lib, pkgs, patchMavenSources, coreutils }: + +nodePkgs: mavenPkgs: + +stdenv.mkDerivation { + name = "${nodePkgs.name}-patched"; + + phases = [ "unpackPhase" "patchPhase" "installPhase" ]; + + # First symlink all modules as is + # WARNING: Metro has issues when dealing with symlinks! + unpackPhase = '' + mkdir -p ./node_modules/ + for module in $(ls ${nodePkgs}/node_modules); do + ln -s ${nodePkgs}/node_modules/$module ./node_modules/ + done + cp -r ${nodePkgs}/node_modules/.bin ./node_modules/ + ''; + + # Then patch the modules that have build.gradle files + patchPhase = '' + # Patch maven and google central repositories with our own local directories. + # This prevents the builder from downloading Maven artifacts + for modBuildGradle in $(find -L ./node_modules -name build.gradle); do + relativeToNode=''${modBuildGradle#*node_modules/} + moduleName=''${relativeToNode%%/*} + if [[ -L ./node_modules/$moduleName ]]; then + unlink ./node_modules/$moduleName + cp -r ${nodePkgs}/node_modules/$moduleName ./node_modules/ + fi + chmod +w -R ./node_modules/$relativeToNode + ${patchMavenSources} $modBuildGradle '${mavenPkgs}' + done + + patchShebangs ./node_modules + + # Do not add a BuildId to the generated libraries, for reproducibility + substituteInPlace ./node_modules/react-native/ReactAndroid/src/main/jni/Application.mk --replace \ + '-Wl,--build-id' \ + '-Wl,--build-id=none' + + # Fix bugs in Hermes usage: + # https://github.com/facebook/react-native/issues/25601#issuecomment-510856047 + # - Make PR builds also count as release builds + # - Fix issue where hermes command is being called with same input/output file + substituteInPlace ./node_modules/react-native/react.gradle --replace \ + 'targetName.toLowerCase().contains("release")' \ + '!targetName.toLowerCase().contains("debug")' + ''; + + installPhase = '' + mkdir -p $out + cp -R node_modules $out/ + ''; + + # The ELF types are incompatible with the host platform, so let's not even try + # TODO: Use Android NDK to strip binaries manually + dontPatchELF = true; + dontStripHost = true; + + # Take whole sources into consideration when calculating sha + outputHashMode = "recursive"; + outputHashAlgo = "sha256"; +}