nix: improve nix/scripts/node_modules.sh

Changes:
- MOAR functions so it's easier to follow
- Don't use /tmp, put the copied node_modules in repo root first
- Ignore more node_modules sub folders when checking for modifications
  - */unpacked_bin/clj-kondo
  - */scripts/.packager.env

Signed-off-by: Jakub Sokołowski <jakub@status.im>

nix/mobile/android/jsbundle/default.nix

@@ -8,11 +8,15 @@ stdenv.mkDerivation {
   name = "status-react-build-jsbundle-android";
   src =
     let path = ./../../../..;
-    in builtins.path { # We use builtins.path so that we can name the resulting derivation, otherwise the name would be taken from the checkout directory, which is outside of our control
+    # We use builtins.path so that we can name the resulting derivation,
+    # otherwise the name would be taken from the checkout directory,
+    # which is outside of our control inherit path;
+    in builtins.path {
       inherit path;
       name = "status-react-source-jsbundle";
       filter =
-        # Keep this filter as restrictive as possible in order to avoid unnecessary rebuilds and limit closure size
+        # Keep this filter as restrictive as possible in order to avoid
+        # unnecessary rebuilds and limit closure size
         lib.mkFilter {
           root = path;
           ignoreVCS = false;

nix/scripts/node_modules.sh

@@ -17,66 +17,117 @@
 
 set -Eeuo pipefail
 
-function replaceNodeModules() {
+GIT_ROOT=$(cd "${BASH_SOURCE%/*}" && git rev-parse --show-toplevel)
-  local deps="$1"
+source "${GIT_ROOT}/scripts/colors.sh"
-  local targetNodeModules="$2"
+
-  local needCopyModules=1
+# More concise output from 'time'
-  local sentinelFilePath="$targetNodeModules/.copied~"
+export TIMEFORMAT="Done in: %Es"
+
+function removeDir() {
+    [[ ! -d "${tmp}" ]] && return
+    chmod -R u+w "${tmp}"
+    rm -rf "${tmp}"
+}
+
+function copyNodeModules() {
+  local src="${1}"
+  local dst="${2}"
+  # WARNING: The ../ is there to avoid a Nix builtins.path bug:
+  # https://github.com/NixOS/nix/issues/3593
+  local tmp=$(mktemp -d -p "$(dirname ${dst})/../")
+
+  # We use a temporary directory to use mv as "atomic" change
+  trap "removeDir ${tmp}" ERR INT HUP
+
+  # WARNING: The -L here is crucial to let Metro find modules.
+  cp -LRf ${src}/node_modules/. "${tmp}"
+  chmod -R +w "${tmp}"
+
+  # WARNING: We can't de-reference .bin symlinks
+  cp -Rf ${src}/node_modules/.bin/. "${tmp}/.bin/"
+  rm -r "${dst}"
+  mv -f "${tmp}" "${dst}"
+}
+
+# Find files that were modified and should cause a re-copying of node modules.
+# Some files are generated/modified by build processes and should be ignored.
+function findFilesNewerThan() {
+  local sentinel="${1}"
+  local dir="${2}"
+  find ${dir} -type f -writable \
+      -newer ${sentinel} \
+      -not -ipath "*/*android/build/*" -prune \
+      -not -ipath "*/xcuserdata/*" -prune \
+      -not -ipath "*/unpacked_bin/clj-kondo" \
+      -not -ipath "*/scripts/.packager.env" \
+      -print 
+}
+
+function nodeModulesUnchanged() {
+  local src="$1"
+  local dst="$2"
+  local sentinelFile="${dst}/.copied~"
 
   # Check if node_modules exists and is valid
-  if [ -d "$targetNodeModules" ]; then
+  if [[ ! -f "${sentinelFile}" ]]; then
-    if [ -f "$sentinelFilePath" ]; then
+      # node_modules have not been created by this script
-      existingPath="$(cat $sentinelFilePath)"
+      echo -e "${YLW}Node modules not created by Nix${RST}" >&2
-      if [ "${existingPath}" != "${deps}" ]; then
+      return 1
-        echo "Yarn modules changed, copying new version over"
+  fi
-      else
+
-        echo "Checking for modifications in node_modules..."
+  # Sentinel file holds location of the node_modules source in Nix store
-        modifiedFiles=(
+  currentNixSrc="$(cat ${sentinelFile})"
-          $(find $targetNodeModules -writable -type f -newer $sentinelFilePath \
+  
-                -not \( -path "$targetNodeModules/react-native/ReactAndroid/build/*" -prune \
+  if [ "${currentNixSrc}" != "${src}" ]; then
-                      -o -path "$targetNodeModules/*/android/build/*" -prune \) -print) )
+    echo -e "${YLW}Yarn modules changed, copying new version over${RST}" >&2
-        if [ ${#modifiedFiles[@]} -eq 0 ]; then
+    return 1
-          needCopyModules=0
+  fi
-          echo "No modifications detected."
+
-        else
+  # Some build processes modify files in node_modules
-          echo "Modifications detected in ${#modifiedFiles[@]} files:"
+  modifiedFiles=($(findFilesNewerThan "${sentinelFile}" "${dst}"))
-          for f in ${modifiedFiles[@]}; do
+  if [ ${#modifiedFiles[@]} -ne 0 ]; then
-            echo "- $(realpath --relative-to=$STATUS_REACT_HOME $f)"
+    echo -e "${YLW}Changes detected in node_modules:${RST} ${#modifiedFiles[@]}" >&2
-          done
+    # Print files that have changes
-        fi
+    for file in ${modifiedFiles[@]}; do
-      fi
+      echo "- $(realpath --relative-to=${dst} ${file})" >&2
-    fi
+    done
-    if [ $needCopyModules -eq 1 ] && [ -d $targetNodeModules ]; then
+    return 1
-      chmod u+w -R $targetNodeModules
+  fi
-      rm -rf $targetNodeModules
+
-    fi
+  echo -e "${GRN}No changes detected.${RST}" >&2
+  return 0
+}
+
+function replaceNodeModules() {
+  local src="$1"
+  local dst="$2"
+  local sentinelFile="${dst}/.copied~"
+
+  if nodeModulesUnchanged ${src} ${dst}; then
+      return
   fi 
 
   # Replace node_modules if necessary
-  if [ ! -d "$targetNodeModules" ]; then
+  echo "Copying node_modules from Nix store:" >&2
-    local tmpNodeModules=$(mktemp -d)
+  echo " - ${src}" >&2
-    echo "Copying node_modules from Nix store:"
+  copyNodeModules ${src} ${dst}
-    echo " - ${deps}"
+  echo -n "${src}" > "${sentinelFile}"
-    export TIMEFORMAT="Done in: %Es"
-    trap "[ -d \"${tmpNodeModules}\" ] && chmod -R u+w \"${tmpNodeModules}\" && rm -rf \"${tmpNodeModules}\"" ERR INT HUP
-    # WARNING: The -L here is crucial to let Metro find modules.
-    time cp -LRf ${deps}/node_modules/. "${tmpNodeModules}"
-    chmod -R u+w "${tmpNodeModules}"
-    # WARNING: We can't de-reference .bin symlinks
-    cp -Rf ${deps}/node_modules/.bin/. "${tmpNodeModules}/.bin/"
-    mv -f "${tmpNodeModules}" "${targetNodeModules}"
-
-    echo -n "${deps}" > "${sentinelFilePath}"
-    trap - ERR INT HUP
-  fi
 }
 
-deps="$1"
+# Destination folder, Nix sets STATUS_REACT_HOME
-[ -d $deps ] || exit 1
+dst="$STATUS_REACT_HOME/node_modules"
+# Source of Node modules from /nix/store
+src="$1"
 
-nodeModulesDir="$STATUS_REACT_HOME/node_modules"
+if [[ ! -d ${src} ]]; then
+    echo -e "${RED}No such folder:${RST} ${src}" >&2
+    exit 1
+fi
 
-export -f replaceNodeModules
+# Make those available in shell spawned by flock
-mkdir -p "$nodeModulesDir/"
+export -f replaceNodeModules nodeModulesUnchanged copyNodeModules findFilesNewerThan removeDir
-# Leverage flock (file lock) utility to create an exclusive lock on node_modules/ while running replaceNodeModules
+
-flock "$nodeModulesDir/" sh -c "replaceNodeModules $deps $nodeModulesDir"
+mkdir -p "${dst}"
+# Leverage file lock to create an exclusive lock.
+# Otherwise multiple calls to this script would clash.
+flock "${dst}/" sh -c "time replaceNodeModules ${src} ${dst}"

nix/shells.nix

@@ -18,7 +18,7 @@ let
 
     buildInputs = with pkgs; lib.unique ([
       # core utilities that should always be present in a shell
-      bash curl wget file unzip flock rsync
+      bash curl wget file unzip flock
       git gnumake jq ncurses gnugrep parallel
       lsof # used in start-react-native.sh
       # build specific utilities

scripts/colors.sh

@@ -1,11 +1,11 @@
 #!/usr/bin/env bash
 
 # Colors
-YLW='\033[1;33m'
+export YLW='\033[1;33m'
-RED='\033[0;31m'
+export RED='\033[0;31m'
-GRN='\033[0;32m'
+export GRN='\033[0;32m'
-BLD='\033[1m'
+export BLD='\033[1m'
-RST='\033[0m'
+export RST='\033[0m'
 
 # Clear line
-CLR='\033[2K'
+export CLR='\033[2K'