diff --git a/nix/mobile/android/jsbundle/default.nix b/nix/mobile/android/jsbundle/default.nix index 72467560b9..ec61fdfbd1 100644 --- a/nix/mobile/android/jsbundle/default.nix +++ b/nix/mobile/android/jsbundle/default.nix @@ -8,11 +8,15 @@ stdenv.mkDerivation { name = "status-react-build-jsbundle-android"; src = let path = ./../../../..; - in builtins.path { # We use builtins.path so that we can name the resulting derivation, otherwise the name would be taken from the checkout directory, which is outside of our control + # We use builtins.path so that we can name the resulting derivation, + # otherwise the name would be taken from the checkout directory, + # which is outside of our control inherit path; + in builtins.path { inherit path; name = "status-react-source-jsbundle"; filter = - # Keep this filter as restrictive as possible in order to avoid unnecessary rebuilds and limit closure size + # Keep this filter as restrictive as possible in order to avoid + # unnecessary rebuilds and limit closure size lib.mkFilter { root = path; ignoreVCS = false; diff --git a/nix/scripts/node_modules.sh b/nix/scripts/node_modules.sh index 910749cd25..61ea8d15ef 100755 --- a/nix/scripts/node_modules.sh +++ b/nix/scripts/node_modules.sh @@ -17,66 +17,117 @@ set -Eeuo pipefail -function replaceNodeModules() { - local deps="$1" - local targetNodeModules="$2" - local needCopyModules=1 - local sentinelFilePath="$targetNodeModules/.copied~" +GIT_ROOT=$(cd "${BASH_SOURCE%/*}" && git rev-parse --show-toplevel) +source "${GIT_ROOT}/scripts/colors.sh" - # Check if node_modules exists and is valid - if [ -d "$targetNodeModules" ]; then - if [ -f "$sentinelFilePath" ]; then - existingPath="$(cat $sentinelFilePath)" - if [ "${existingPath}" != "${deps}" ]; then - echo "Yarn modules changed, copying new version over" - else - echo "Checking for modifications in node_modules..." - modifiedFiles=( - $(find $targetNodeModules -writable -type f -newer $sentinelFilePath \ - -not \( -path "$targetNodeModules/react-native/ReactAndroid/build/*" -prune \ - -o -path "$targetNodeModules/*/android/build/*" -prune \) -print) ) - if [ ${#modifiedFiles[@]} -eq 0 ]; then - needCopyModules=0 - echo "No modifications detected." - else - echo "Modifications detected in ${#modifiedFiles[@]} files:" - for f in ${modifiedFiles[@]}; do - echo "- $(realpath --relative-to=$STATUS_REACT_HOME $f)" - done - fi - fi - fi - if [ $needCopyModules -eq 1 ] && [ -d $targetNodeModules ]; then - chmod u+w -R $targetNodeModules - rm -rf $targetNodeModules - fi - fi +# More concise output from 'time' +export TIMEFORMAT="Done in: %Es" - # Replace node_modules if necessary - if [ ! -d "$targetNodeModules" ]; then - local tmpNodeModules=$(mktemp -d) - echo "Copying node_modules from Nix store:" - echo " - ${deps}" - export TIMEFORMAT="Done in: %Es" - trap "[ -d \"${tmpNodeModules}\" ] && chmod -R u+w \"${tmpNodeModules}\" && rm -rf \"${tmpNodeModules}\"" ERR INT HUP - # WARNING: The -L here is crucial to let Metro find modules. - time cp -LRf ${deps}/node_modules/. "${tmpNodeModules}" - chmod -R u+w "${tmpNodeModules}" - # WARNING: We can't de-reference .bin symlinks - cp -Rf ${deps}/node_modules/.bin/. "${tmpNodeModules}/.bin/" - mv -f "${tmpNodeModules}" "${targetNodeModules}" - - echo -n "${deps}" > "${sentinelFilePath}" - trap - ERR INT HUP - fi +function removeDir() { + [[ ! -d "${tmp}" ]] && return + chmod -R u+w "${tmp}" + rm -rf "${tmp}" } -deps="$1" -[ -d $deps ] || exit 1 +function copyNodeModules() { + local src="${1}" + local dst="${2}" + # WARNING: The ../ is there to avoid a Nix builtins.path bug: + # https://github.com/NixOS/nix/issues/3593 + local tmp=$(mktemp -d -p "$(dirname ${dst})/../") -nodeModulesDir="$STATUS_REACT_HOME/node_modules" + # We use a temporary directory to use mv as "atomic" change + trap "removeDir ${tmp}" ERR INT HUP -export -f replaceNodeModules -mkdir -p "$nodeModulesDir/" -# Leverage flock (file lock) utility to create an exclusive lock on node_modules/ while running replaceNodeModules -flock "$nodeModulesDir/" sh -c "replaceNodeModules $deps $nodeModulesDir" + # WARNING: The -L here is crucial to let Metro find modules. + cp -LRf ${src}/node_modules/. "${tmp}" + chmod -R +w "${tmp}" + + # WARNING: We can't de-reference .bin symlinks + cp -Rf ${src}/node_modules/.bin/. "${tmp}/.bin/" + rm -r "${dst}" + mv -f "${tmp}" "${dst}" +} + +# Find files that were modified and should cause a re-copying of node modules. +# Some files are generated/modified by build processes and should be ignored. +function findFilesNewerThan() { + local sentinel="${1}" + local dir="${2}" + find ${dir} -type f -writable \ + -newer ${sentinel} \ + -not -ipath "*/*android/build/*" -prune \ + -not -ipath "*/xcuserdata/*" -prune \ + -not -ipath "*/unpacked_bin/clj-kondo" \ + -not -ipath "*/scripts/.packager.env" \ + -print +} + +function nodeModulesUnchanged() { + local src="$1" + local dst="$2" + local sentinelFile="${dst}/.copied~" + + # Check if node_modules exists and is valid + if [[ ! -f "${sentinelFile}" ]]; then + # node_modules have not been created by this script + echo -e "${YLW}Node modules not created by Nix${RST}" >&2 + return 1 + fi + + # Sentinel file holds location of the node_modules source in Nix store + currentNixSrc="$(cat ${sentinelFile})" + + if [ "${currentNixSrc}" != "${src}" ]; then + echo -e "${YLW}Yarn modules changed, copying new version over${RST}" >&2 + return 1 + fi + + # Some build processes modify files in node_modules + modifiedFiles=($(findFilesNewerThan "${sentinelFile}" "${dst}")) + if [ ${#modifiedFiles[@]} -ne 0 ]; then + echo -e "${YLW}Changes detected in node_modules:${RST} ${#modifiedFiles[@]}" >&2 + # Print files that have changes + for file in ${modifiedFiles[@]}; do + echo "- $(realpath --relative-to=${dst} ${file})" >&2 + done + return 1 + fi + + echo -e "${GRN}No changes detected.${RST}" >&2 + return 0 +} + +function replaceNodeModules() { + local src="$1" + local dst="$2" + local sentinelFile="${dst}/.copied~" + + if nodeModulesUnchanged ${src} ${dst}; then + return + fi + + # Replace node_modules if necessary + echo "Copying node_modules from Nix store:" >&2 + echo " - ${src}" >&2 + copyNodeModules ${src} ${dst} + echo -n "${src}" > "${sentinelFile}" +} + +# Destination folder, Nix sets STATUS_REACT_HOME +dst="$STATUS_REACT_HOME/node_modules" +# Source of Node modules from /nix/store +src="$1" + +if [[ ! -d ${src} ]]; then + echo -e "${RED}No such folder:${RST} ${src}" >&2 + exit 1 +fi + +# Make those available in shell spawned by flock +export -f replaceNodeModules nodeModulesUnchanged copyNodeModules findFilesNewerThan removeDir + +mkdir -p "${dst}" +# Leverage file lock to create an exclusive lock. +# Otherwise multiple calls to this script would clash. +flock "${dst}/" sh -c "time replaceNodeModules ${src} ${dst}" diff --git a/nix/shells.nix b/nix/shells.nix index 202e91d365..9c1ad231de 100644 --- a/nix/shells.nix +++ b/nix/shells.nix @@ -18,7 +18,7 @@ let buildInputs = with pkgs; lib.unique ([ # core utilities that should always be present in a shell - bash curl wget file unzip flock rsync + bash curl wget file unzip flock git gnumake jq ncurses gnugrep parallel lsof # used in start-react-native.sh # build specific utilities diff --git a/scripts/colors.sh b/scripts/colors.sh index e8dea24900..06833e57a0 100644 --- a/scripts/colors.sh +++ b/scripts/colors.sh @@ -1,11 +1,11 @@ #!/usr/bin/env bash # Colors -YLW='\033[1;33m' -RED='\033[0;31m' -GRN='\033[0;32m' -BLD='\033[1m' -RST='\033[0m' +export YLW='\033[1;33m' +export RED='\033[0;31m' +export GRN='\033[0;32m' +export BLD='\033[1m' +export RST='\033[0m' # Clear line -CLR='\033[2K' +export CLR='\033[2K'