nix: refactor loading of node.js modules

Changes:
- Drop `nix/mobile/android/maven-and-npm-deps/default.nix`
- Replace it with much simpler `nix/tools/patchNodeModules`
- Move Gradle patching tool to `nix/pkgs/patch-maven-srcs`
- Simplify it by using `gradle.deps` and patched node modules separately
- Change `TARGET` for `release-android` to `default`
- Move `mobile/reset-node_modules.sh` to `scripts/node_modules.sh`
- Move `nix/mobile/android/targets/release-android.nix` to `nix/mobile/android/release.nix`

Signed-off-by: Jakub Sokołowski <jakub@status.im>

Makefile

@@ -151,7 +151,7 @@ keystore: ##@prepare Generate a Keystore for signing Android APKs
 #----------------
 release: release-android release-ios ##@build build release for Android and iOS
 
-release-android: export TARGET := android
+release-android: export TARGET := default
 release-android: export BUILD_ENV ?= prod
 release-android: export BUILD_TYPE ?= nightly
 release-android: export BUILD_NUMBER ?= $(TMP_BUILD_NUMBER)

nix/deps/clojure/default.nix

@@ -1,4 +1,4 @@
-{ stdenv, lib, writeShellScriptBin, fetchurl }:
+{ stdenv, lib, fetchurl, writeShellScriptBin }:
 
 let
   inherit (lib)

nix/deps/gradle/default.nix

@@ -16,9 +16,8 @@ let
     mkdir -p $out
     cd $out
   '' +
-  (concatMapStrings (dep': 
+  (concatMapStrings (dep: 
     let
-      dep = { postCopy = ""; } // dep';
       url = "${dep.host}/${dep.path}";
       pom = {
         sha1 = attrByPath [ "pom" "sha1" ] "" dep;

nix/deps/gradle/shell.nix

@@ -1,21 +0,0 @@
-{ pkgs ? import ../../pkgs.nix { } }:
-
-let
-  status-go = pkgs.callPackage ../../status-go { };
-  nodeJsDeps = pkgs.callPackage ../nodejs { };
-in pkgs.mkShell {
-  buildInputs = with pkgs; [
-    curl flock # used in reset-node_modules.sh
-    git gradle jq maven nodejs
-  ];
-  inputsFrom = [ pkgs.androidShell ];
-  shellHook = ''
-    # Gradle checks for this being set
-    export STATUS_GO_ANDROID_LIBDIR=${status-go.mobile.android}
-
-    # Necessary for reset-node_modules.sh
-    export STATUS_REACT_HOME=$(realpath ../../../)
-
-    $STATUS_REACT_HOME/nix/mobile/reset-node_modules.sh "${nodeJsDeps}"
-  '';
-}

nix/mobile/android/default.nix

@@ -1,5 +1,5 @@
-{ lib, pkgs, callPackage, mkShell
+{ lib, pkgs, deps, callPackage, mkShell
-, status-go, gradle, androidPkgs, androidShell }:
+, status-go, androidPkgs, androidShell, patchNodeModules }:
 
 let
   # Import a jsbundle compiled out of clojure codebase
@@ -8,29 +8,28 @@ let
   # Import a patched version of watchman (important for sandboxed builds on macOS)
   watchmanFactory = callPackage ./watchman.nix { };
 
-  # Import a local patched version of node_modules, together with a local version of the Maven repo
+  # Some node_modules have build.gradle files that reference remote repos.
-  mavenAndNpmDeps = callPackage ./maven-and-npm-deps { };
+  # This patches them to reference local repos only
+  nodeJsModules = patchNodeModules deps.nodejs deps.gradle;
 
   # TARGETS
-  release = callPackage ./targets/release-android.nix {
+  release = callPackage ./release.nix {
-    inherit gradle mavenAndNpmDeps jsbundle status-go watchmanFactory;
+    inherit jsbundle status-go watchmanFactory nodeJsModules;
   };
 
 in {
   # TARGETS
-  inherit release jsbundle;
+  inherit release jsbundle nodeJsModules;
 
   shell = mkShell {
     buildInputs = with pkgs; [
       openjdk
       gradle
       lsof  # used in start-react-native.sh
-      flock # used in reset-node_modules.sh
+      flock # used in nix/scripts/node_modules.sh
-      mavenAndNpmDeps
     ];
 
     inputsFrom = [
-      gradle
       release
       androidShell
     ];
@@ -39,7 +38,7 @@ in {
       export ANDROID_SDK_ROOT="${androidPkgs}"
       export ANDROID_NDK_ROOT="${androidPkgs}/ndk-bundle"
 
-      export STATUSREACT_NIX_MAVEN_REPO="${mavenAndNpmDeps}/.m2/repository"
+      export STATUSREACT_NIX_MAVEN_REPO="${deps.gradle}"
 
       # required by some makefile targets
       export STATUS_GO_ANDROID_LIBDIR=${status-go}
@@ -51,7 +50,7 @@ in {
         ln -sf ./mobile/js_files/* ./
 
         # check if node modules changed and if so install them
-        $STATUS_REACT_HOME/nix/mobile/reset-node_modules.sh ${mavenAndNpmDeps}/project
+        $STATUS_REACT_HOME/nix/scripts/node_modules.sh ${nodeJsModules}
       }
     '';
   };

nix/mobile/android/maven-and-npm-deps/default.nix

@@ -1,160 +0,0 @@
-#
-# This script prepares a finalized version of node_modules,
-#   as well as a local version of the Maven repository required by Gradle scripts
-#
-
-{ stdenv, lib, callPackage, pkgs, deps, mkShell }:
-
-# fake build to pre-download deps into fixed-output derivation
-let
-  # Place build target directories in NIX_BUILD_TOP (normally represents /build)
-  projectBuildDir = "$NIX_BUILD_TOP/project";
-  mavenRepoDir = "$NIX_BUILD_TOP/.m2/repository";
-  reactNativeDepsDir = "$NIX_BUILD_TOP/deps"; # Use local writable deps, otherwise (probably due to some interaction between Nix sandboxing and Java) gradle will fail copying directly from the nix store
-
-in stdenv.mkDerivation {
-  name = "status-react-patched-npm-gradle-modules";
-  src =
-    let path = ./../../../..; # Import the root /android and /mobile/js_files folders clean of any build artifacts
-    in builtins.path { # We use builtins.path so that we can name the resulting derivation, otherwise the name would be taken from the checkout directory, which is outside of our control
-      inherit path;
-      name = "status-react-source-gradle-install";
-      filter =
-        # Keep this filter as restrictive as possible in order to avoid unnecessary rebuilds and limit closure size
-        lib.mkFilter {
-          root = path;
-          include = [
-            "android/.*" "resources/.*"
-            "mobile/js_files/.*" ".babelrc"
-          ];
-          exclude = [
-            ".*.keystore" "node_modules"
-          ];
-        };
-    };
-  phases = [ "unpackPhase" "patchPhase" "installPhase" ];
-  nativeBuildInputs = [ deps.nodejs ];
-  buildInputs = with pkgs; [ gradle nodejs file zlib deps.gradle ];
-  propagatedBuildInputs = [ deps.react-native ];
-  unpackPhase = ''
-    runHook preUnpack
-
-    # Copy project directory
-    mkdir -p ${projectBuildDir}
-    cp -a $src/. ${projectBuildDir}
-    chmod u+w ${projectBuildDir}
-    cd ${projectBuildDir}
-
-    # Copy RN maven dependencies and make them writable, otherwise Gradle copy fails (since the top-level directory is read-only, Java isn't smart enough to copy the child files/folders into that target directory)
-    mkdir -p ${mavenRepoDir}
-    cp -a ${deps.gradle}/. ${mavenRepoDir}
-    cp -a ${deps.react-native}/deps ${reactNativeDepsDir}
-    find ${reactNativeDepsDir} -maxdepth 1 -type d -exec chmod -R u+w {} \;
-
-    # Copy node_modules from Nix store
-    rm -rf ${projectBuildDir}/node_modules
-    mkdir -p ${projectBuildDir}/node_modules
-    cp -a ${deps.nodejs}/node_modules/. ${projectBuildDir}/node_modules/
-
-    # Ensure that module was correctly installed
-    [ -d ${projectBuildDir}/node_modules/jsc-android/dist ] || exit 1
-
-    # Adjust permissions
-    chmod -R u+w ${projectBuildDir}
-
-    # Create a dummy VERSION, since we don't want this expression to be invalidated just because the version changed
-    echo '0.0.1' > ${projectBuildDir}/VERSION
-
-    runHook postUnpack
-  '';
-  patchPhase = ''
-    runHook prePatch
-
-    prevSet=$-
-    set -e
-
-    patchShebangs ${projectBuildDir}
-
-    function patchMavenSource() {
-      set +e
-
-      local targetGradleFile="$1"
-      local source="$2"
-      local deriv="$3"
-      grep "$source" $targetGradleFile > /dev/null && \
-        substituteInPlace $targetGradleFile --replace "$source" "$deriv"
-    }
-
-    function patchMavenSources() {
-      set +e
-
-      local targetGradleFile="$1"
-      local deriv="$2"
-      patchMavenSource $targetGradleFile 'mavenCentral()' 'mavenLocal()'
-      patchMavenSource $targetGradleFile 'google()' 'mavenLocal()'
-      patchMavenSource $targetGradleFile 'jcenter()' 'mavenLocal()'
-      grep 'https://maven.google.com' $targetGradleFile > /dev/null && \
-        substituteInPlace $targetGradleFile --replace 'https://maven.google.com' "$deriv"
-      grep 'https://jitpack.io' $targetGradleFile > /dev/null && \
-        substituteInPlace $targetGradleFile --replace 'https://jitpack.io' "$deriv"
-    }
-
-    # Patch maven and google central repositories with our own local directories. This prevents the builder from downloading Maven artifacts
-    patchMavenSources 'android/build.gradle' '${deps.gradle}'
-    for f in `find ${projectBuildDir}/node_modules/ -name build.gradle`; do
-      patchMavenSources $f '${deps.gradle}'
-    done
-
-    # Do not add a BuildId to the generated libraries, for reproducibility
-    substituteInPlace ${projectBuildDir}/node_modules/react-native/ReactAndroid/src/main/jni/Application.mk \
-      --replace \
-        '-Wl,--build-id' \
-        '-Wl,--build-id=none'
-
-    # Disable Gradle daemon and caching, since that causes rebuilds (and subsequently errors) anyway due to cache being considered stale
-    substituteInPlace ${projectBuildDir}/android/gradle.properties \
-      --replace \
-        'org.gradle.jvmargs=-Xmx8704M' \
-        'org.gradle.jvmargs=-Xmx8704M
-    org.gradle.daemon=false
-    org.gradle.caching=false'
-
-    # Patch the path to nodejs in project.ext.react
-    substituteInPlace ${projectBuildDir}/android/app/build.gradle \
-      --replace \
-        'nodeExecutableAndArgs: ["node"' \
-        'nodeExecutableAndArgs: ["${pkgs.nodejs}/bin/node"'
-
-    # Fix bugs in Hermes usage (https://github.com/facebook/react-native/issues/25601#issuecomment-510856047)
-    # - Make PR builds also count as release builds
-    # - Fix issue where hermes command is being called with same input/output file
-    substituteInPlace ${projectBuildDir}/node_modules/react-native/react.gradle \
-      --replace \
-        'targetName.toLowerCase().contains("release")' \
-        '!targetName.toLowerCase().contains("debug")'
-
-    # Patch dependencies which are not yet ported to AndroidX
-    npx jetify
-
-    set $prevSet
-
-    runHook postPatch
-  '';
-  installPhase = ''
-    rm -rf $out
-    mkdir -p $out/{project,.m2/repository}
-
-    # TODO: maybe node_modules/react-native/ReactAndroid/build/{tmp,generated} can be discarded?
-    cp -R ${mavenRepoDir} $out/.m2/
-    cp -R ${projectBuildDir}/{android,node_modules}/ $out/project
-  '';
-
-  # The ELF types are incompatible with the host platform, so let's not even try
-  # TODO: Use Android NDK to strip binaries manually
-  dontPatchELF = true;
-  dontStripHost = true;
-
-  # Take whole sources into consideration when calculating sha
-  outputHashMode = "recursive";
-  outputHashAlgo = "sha256";
-}

nix/mobile/android/release.nix

@@ -1,6 +1,6 @@
-{ stdenv, lib, config, callPackage,
+{ stdenv, lib, config, callPackage, deps,
   bash, file, gnumake, watchmanFactory, gradle,
-  androidPkgs, mavenAndNpmDeps,
+  androidPkgs, patchMavenSources, nodeJsModules,
   nodejs, openjdk, jsbundle, status-go, unzip, zlib }:
 
 {
@@ -22,9 +22,6 @@ let
   baseName = "release-android";
   name = "status-react-build-${baseName}";
 
-  gradleHome = "$NIX_BUILD_TOP/.gradle";
-  localMavenRepo = "${mavenAndNpmDeps}/.m2/repository";
-  sourceProjectDir = "${mavenAndNpmDeps}/project";
   envFileName = if (buildType == "release" || buildType == "nightly" || buildType == "e2e")
     then ".env.${buildType}"
     else if buildType != "" then ".env.jenkins"
@@ -35,12 +32,12 @@ let
     then "Pr"
     else "Release"; # PR builds shouldn't replace normal releases
 
-  apksPath = "$PROJECT/android/app/build/outputs/apk/${toLower gradleBuildType}";
+  apksPath = "./android/app/build/outputs/apk/${toLower gradleBuildType}";
   patchedWatchman = watchmanFactory watchmanSockPath;
 
 in stdenv.mkDerivation rec {
   inherit name;
-  src = let path = ./../../../..;
+  src = let path = ./../../..;
   # We use builtins.path so that we can name the resulting derivation
   in builtins.path {
     inherit path;
@@ -50,7 +47,7 @@ in stdenv.mkDerivation rec {
       root = path;
       include = [
         "mobile/js_files.*" "resources/.*" "translations/.*"
-        "modules/react-native-status/android.*"
+        "modules/react-native-status/android.*" "android/.*"
         envFileName "VERSION" ".watchmanconfig"
         "status-go-version.json" "react-native.config.js"
       ];
@@ -69,6 +66,7 @@ in stdenv.mkDerivation rec {
   ANDROID_ABI_SPLIT = getConfig "android.abi-split" "false";
   ANDROID_ABI_INCLUDE = getConfig "android.abi-include" "armeabi-v7a;arm64-v8a;x86";
 
+  # Android SDK/NDK for use by Gradle
   ANDROID_SDK_ROOT = "${androidPkgs}";
   ANDROID_NDK_ROOT = "${androidPkgs}/ndk-bundle";
 
@@ -80,39 +78,38 @@ in stdenv.mkDerivation rec {
   ];
 
   unpackPhase = ''
-    cp -r $src ./project
+    cp -ar $src/. ./
-    chmod u+w -R ./project
+    chmod u+w -R ./
-    export PROJECT=$PWD/project
-
     runHook postUnpack
   '';
   postUnpack = assert lib.assertMsg (keystorePath != null) "keystore-file has to be set!"; ''
-    mkdir -p ${gradleHome}
-
     # Keep the same keystore path for determinism
-    export KEYSTORE_PATH="${gradleHome}/status-im.keystore"
+    export KEYSTORE_PATH="$PWD/status-im.keystore"
     cp -a --no-preserve=ownership "${keystorePath}" "$KEYSTORE_PATH"
 
     # Ensure we have the right .env file
-    cp -f $PROJECT/${envFileName} $PROJECT/.env
+    cp -f ./${envFileName} ./.env
+
+    # create mobile node/yarn symlinks
+    ln -sf ./mobile/js_files/* ./
 
     # Copy index.js and app/ input files
-    cp -ra --no-preserve=ownership ${jsbundle}/* $PROJECT/
+    cp -ra --no-preserve=ownership ${jsbundle}/* ./
 
     # Copy android/ directory
-    cp -a --no-preserve=ownership ${sourceProjectDir}/android/ $PROJECT/
+    mkdir -p ./android/build
-    chmod u+w $PROJECT/android
+    chmod -R +w ./android
-    chmod u+w $PROJECT/android/app
-    mkdir -p $PROJECT/android/build
-    chmod -R u+w $PROJECT/android/build
 
-    # Copy node_modules/ directory
+    # Copy node_modules/ directory. The -L is CRUCIAL!
-    cp -a --no-preserve=ownership ${sourceProjectDir}/node_modules/ $PROJECT/
+    # Otherwise Metro failes to find modules due to symlinks.
+    cp -aL ${nodeJsModules}/node_modules/ ./
+    chmod +w -R ./node_modules
 
-    # Make android/build directories writable under node_modules
+    # Patch build.gradle to use local repo
-    for d in `find $PROJECT/node_modules -type f -name build.gradle | xargs dirname`; do
+    ${patchMavenSources} ./android/build.gradle ${deps.gradle}
-      chmod -R u+w $d
+
-    done
+    # Patch dependencies which are not yet ported to AndroidX
+    ${nodejs}/bin/node ./node_modules/jetifier/bin/jetify
   '';
   secretPhase = optionalString (secretsFile != "") ''
     source "${secretsFile}"
@@ -124,19 +121,10 @@ in stdenv.mkDerivation rec {
     assert ANDROID_ABI_SPLIT != null && ANDROID_ABI_SPLIT != "";
     assert stringLength ANDROID_ABI_INCLUDE > 0;
   ''
-    export STATUS_REACT_HOME=$PWD
+    pushd ./android
-    export HOME=$PROJECT
-
-    # create mobile node/yarn symlinks
-    ln -sf $PROJECT/mobile/js_files/* $PROJECT/
-
-    # fix permissions so gradle can create directories
-    chmod -R +w $PROJECT/android
-
-    pushd $PROJECT/android
     ${adhocEnvVars} gradle ${toString gradleOpts} \
       --offline --stacktrace \
-      -Dmaven.repo.local='${localMavenRepo}' \
+      -Dmaven.repo.local='${deps.gradle}' \
       -PversionCode=${toString buildNumber} \
       assemble${gradleBuildType} \
       || exit 1

nix/mobile/ios/default.nix

@@ -13,7 +13,7 @@ in {
   shell = mkShell {
     buildInputs = with pkgs; [
       xcodeWrapper watchman bundler procps
-      flock # used in reset-node_modules.sh
+      flock # used in nix/scripts/node_modules.sh
     ];
 
     inputsFrom = [
@@ -30,7 +30,7 @@ in {
         ln -sf ./mobile/js_files/* ./
 
         # check if node modules changed and if so install them
-        ./nix/mobile/reset-node_modules.sh "${deps.nodejs}"
+        ./nix/scripts/node_modules.sh "${deps.nodejs}"
       }
     '';
   };

nix/overlay.nix

@@ -27,16 +27,19 @@ in {
     react-native = callPackage ./deps/react-native { };
   };
 
-  # Android environement
+  # For patching Node.js modules with Gradle repo path
-  androidEnvCustom = callPackage ./mobile/android/sdk { };
+  patchNodeModules = callPackage ./tools/patchNodeModules.nix { };
-  androidPkgs = self.androidEnvCustom.licensedPkgs;
-  androidShell = self.androidEnvCustom.shell;
 
   # Package version adjustments
   xcodeWrapper = super.xcodeenv.composeXcodeWrapper { version = "11.4.1"; };
   openjdk = super.pkgs.openjdk8_headless;
   nodejs = super.pkgs.nodejs-12_x;
 
+  # Android environement
+  androidEnvCustom = callPackage ./pkgs/android-sdk { };
+  androidPkgs = self.androidEnvCustom.licensedPkgs;
+  androidShell = self.androidEnvCustom.shell;
+
   # Custom packages
   aapt2 = callPackage ./pkgs/aapt2 { };
   gomobile = callPackage ./pkgs/gomobile { };
@@ -44,4 +47,5 @@ in {
   qtkeychain-src = callPackage ./pkgs/qtkeychain-src { };
   appimagekit = callPackage ./pkgs/appimagekit { };
   linuxdeployqt = callPackage ./pkgs/linuxdeployqt { inherit (self) appimagekit; };
+  patchMavenSources = callPackage ./pkgs/patch-maven-srcs { };
 }

nix/pkgs/aapt2/default.nix

@@ -2,10 +2,12 @@
 # It is used by Gradle to package Android app resources.
 # See: https://developer.android.com/studio/command-line/aapt2
 
-{ stdenv, deps, pkgs, fetchurl }:
+{ lib, stdenv, deps, pkgs, fetchurl }:
 
 let
-  inherit (pkgs) zip unzip;
+  inherit (lib) getAttr optionals;
+  inherit (pkgs) zip unzip patchelf;
+  inherit (stdenv) isLinux isDarwin;
 
   pname = "aapt2";
   # Warning: This must be the same as gradlePluginVersion android/gradle.properties
@@ -14,19 +16,30 @@ let
   pkgPath = "com/android/tools/build/aapt2";
   repoUrl = "https://dl.google.com/dl/android/maven2";
 
+  platform =
+    if isLinux then "linux" else
+    if isDarwin then "osx" else
+    throw "Unknown platform!";
+
   filenames = {
-    jar = "${pname}-${version}-linux.jar";
+    jar = "${pname}-${version}-${platform}.jar";
     pom = "${pname}-${version}.pom";
   };
 
   urls = {
     jar = fetchurl {
       url = "${repoUrl}/${pkgPath}/${version}/${filenames.jar}";
-      sha256 = "05gln93wfj4l5b0zfn6ipkx0i9p0x928ygwkrcfyl58aslxg5gx2";
+      sha256 = getAttr platform {
+        linux = "05gln93wfj4l5b0zfn6ipkx0i9p0x928ygwkrcfyl58aslxg5gx2";
+        osx = "0nzc3hq3fm847a3rvwdz26ln3inh50x44ml0dq486yz45qv9f7cs";
+      };
     };
     sha = fetchurl {
       url = "${repoUrl}/${pkgPath}/${version}/${filenames.jar}.sha1";
-      sha256 = "0rr7ly0f3w5jw0q985hmxmv8q2nlw1k72n6kl7kcmj4a7i479q90";
+      sha256 = getAttr platform {
+        linux = "0rr7ly0f3w5jw0q985hmxmv8q2nlw1k72n6kl7kcmj4a7i479q90";
+        osx = "0k7j54x627jsnl8zdcfj62jj8z0c857yqs3iwyvn29hd02v1b78m";
+      };
     };
     pom = fetchurl {
       url = "${repoUrl}/${pkgPath}/${version}/${filenames.pom}";
@@ -38,13 +51,15 @@ in stdenv.mkDerivation {
   inherit pname version;
 
   srcs = with urls; [ jar sha pom ];
-  phases = [ "unpackPhase" "patchPhase" ];
+  phases = [ "unpackPhase" ]
+    ++ optionals isLinux [ "patchPhase" ]; # OSX binaries don't need patchelf
+  buildInputs = [ zip unzip patchelf ];
 
   unpackPhase = ''
     mkdir -p $out
     for src in $srcs; do
-      local filename=$(basename $src)
+      filename=$(stripHash $src)
-      cp $src $out/''${filename#*-}
+      cp $src $out/$filename
     done
   '';
 
@@ -58,7 +73,7 @@ in stdenv.mkDerivation {
     tmpDir=$(mktemp -d)
     ${unzip}/bin/unzip $out/${filenames.jar} -d $tmpDir
     for exe in `find $tmpDir/ -type f -executable`; do
-      patchelf --set-interpreter "$(cat $NIX_CC/nix-support/dynamic-linker)" $exe
+      ${patchelf}/bin/patchelf --set-interpreter "$(cat $NIX_CC/nix-support/dynamic-linker)" $exe
     done
 
     # Rebuild the .jar file with patched binaries

nix/pkgs/patch-maven-srcs/default.nix

@@ -0,0 +1,26 @@
+# This script patches build.gradle files in node_modules to use
+# our local version of Maven dependencies instead of fetching them.
+
+{ stdenv, writeScript, runtimeShell }:
+
+writeScript "patch-maven-srcs" (''
+  #!${runtimeShell}
+  # Source setup.sh for substituteInPlace
+  source ${stdenv}/setup
+
+  function patchMavenSource() {
+    grep "$source" $1 > /dev/null && \
+      substituteInPlace $1 --replace "$2" "$3" 2>/dev/null
+  }
+
+  gradleFile="$1"
+  derivation="$2"
+
+  # Some of those find something, some don't, that's fine.
+  patchMavenSource "$gradleFile" 'mavenCentral()' 'mavenLocal()'
+  patchMavenSource "$gradleFile" 'google()'       'mavenLocal()'
+  patchMavenSource "$gradleFile" 'jcenter()'      'mavenLocal()'
+  patchMavenSource "$gradleFile" 'https://maven.google.com' "$derivation"
+  patchMavenSource "$gradleFile" 'https://www.jitpack.io'   "$derivation"
+  patchMavenSource "$gradleFile" 'https://jitpack.io'       "$derivation"
+'')

nix/scripts/node_modules.sh

@@ -55,14 +55,19 @@ function replaceNodeModules() {
   # Replace node_modules if necessary
   if [ ! -d "$targetNodeModules" ]; then
     local tmpNodeModules=$(mktemp -d)
-    echo "Copying node_modules from Nix store (${deps}/node_modules)..."
+    echo "Copying node_modules from Nix store:"
-    trap "[ -d \"$tmpNodeModules\" ] && chmod -R u+w \"$tmpNodeModules\" && rm -rf \"$tmpNodeModules\"" ERR INT HUP
+    echo " - ${deps}"
-    time cp -HRf --preserve=all ${deps}/node_modules/. "$tmpNodeModules"
+    export TIMEFORMAT="Done in: %Es"
-    chmod -R u+w "$tmpNodeModules"
+    trap "[ -d \"${tmpNodeModules}\" ] && chmod -R u+w \"${tmpNodeModules}\" && rm -rf \"${tmpNodeModules}\"" ERR INT HUP
-    mv -f "$tmpNodeModules" "$targetNodeModules"
+    # WARNING: The -L here is crucial to let Metro find modules.
-    echo -n "${deps}" > $sentinelFilePath
+    time cp -LRf ${deps}/node_modules/. "${tmpNodeModules}"
+    chmod -R u+w "${tmpNodeModules}"
+    # WARNING: We can't de-reference .bin symlinks
+    cp -Rf ${deps}/node_modules/.bin/. "${tmpNodeModules}/.bin/"
+    mv -f "${tmpNodeModules}" "${targetNodeModules}"
+
+    echo -n "${deps}" > "${sentinelFilePath}"
     trap - ERR INT HUP
-    echo "Done"
   fi
 }
 

nix/shells.nix

@@ -18,7 +18,7 @@ let
 
     buildInputs = with pkgs; lib.unique ([
       # core utilities that should always be present in a shell
-      bash curl wget file unzip flock
+      bash curl wget file unzip flock rsync
       git gnumake jq ncurses gnugrep parallel
       # build specific utilities
       clojure maven watchman
@@ -48,7 +48,7 @@ let
     buildInputs = [ pkgs.androidPkgs ];
     shellHook = ''
       export STATUS_REACT_HOME=$(git rev-parse --show-toplevel)
-      $STATUS_REACT_HOME/nix/mobile/reset-node_modules.sh ${pkgs.deps.nodejs}
+      $STATUS_REACT_HOME/nix/scripts/node_modules.sh ${pkgs.deps.nodejs}
     '';
   };
 

nix/tools/patchNodeModules.nix

@@ -0,0 +1,68 @@
+# This method patches Node.js dependencies by taking the
+# result of yarn2nix and symlinking what is fine, and 
+# copying and modifying what needs to be adjusted.
+
+{ stdenv, lib, pkgs, patchMavenSources, coreutils }:
+
+nodePkgs: mavenPkgs:
+
+stdenv.mkDerivation {
+  name = "${nodePkgs.name}-patched";
+
+  phases = [ "unpackPhase" "patchPhase" "installPhase" ];
+
+  # First symlink all modules as is
+  # WARNING: Metro has issues when dealing with symlinks!
+  unpackPhase = ''
+    mkdir -p ./node_modules/
+    for module in $(ls ${nodePkgs}/node_modules); do
+      ln -s ${nodePkgs}/node_modules/$module ./node_modules/
+    done
+    cp -r ${nodePkgs}/node_modules/.bin ./node_modules/
+  '';
+
+  # Then patch the modules that have build.gradle files
+  patchPhase = ''
+    # Patch maven and google central repositories with our own local directories.
+    # This prevents the builder from downloading Maven artifacts
+    for modBuildGradle in $(find -L ./node_modules -name build.gradle); do
+      relativeToNode=''${modBuildGradle#*node_modules/}
+      moduleName=''${relativeToNode%%/*}
+      if [[ -L ./node_modules/$moduleName ]]; then
+        unlink ./node_modules/$moduleName
+        cp -r ${nodePkgs}/node_modules/$moduleName ./node_modules/
+      fi
+      chmod +w -R ./node_modules/$relativeToNode
+      ${patchMavenSources} $modBuildGradle '${mavenPkgs}'
+    done
+
+    patchShebangs ./node_modules
+
+    # Do not add a BuildId to the generated libraries, for reproducibility
+    substituteInPlace ./node_modules/react-native/ReactAndroid/src/main/jni/Application.mk --replace \
+        '-Wl,--build-id' \
+        '-Wl,--build-id=none'
+
+    # Fix bugs in Hermes usage:
+    # https://github.com/facebook/react-native/issues/25601#issuecomment-510856047
+    # - Make PR builds also count as release builds
+    # - Fix issue where hermes command is being called with same input/output file
+    substituteInPlace ./node_modules/react-native/react.gradle --replace \
+        'targetName.toLowerCase().contains("release")' \
+        '!targetName.toLowerCase().contains("debug")'
+  '';
+
+  installPhase = ''
+    mkdir -p $out
+    cp -R node_modules $out/
+  '';
+
+  # The ELF types are incompatible with the host platform, so let's not even try
+  # TODO: Use Android NDK to strip binaries manually
+  dontPatchELF = true;
+  dontStripHost = true;
+
+  # Take whole sources into consideration when calculating sha
+  outputHashMode = "recursive";
+  outputHashAlgo = "sha256";
+}