From 8c7323e1262c78c1d01aaeafb6e022a8fb50da9b Mon Sep 17 00:00:00 2001 From: Pedro Pombeiro Date: Fri, 13 Sep 2019 15:45:24 +0200 Subject: [PATCH] Use flock to create exclusive locks on node_modules/ Signed-off-by: Pedro Pombeiro --- nix/derivation.nix | 2 +- nix/mobile/android/default.nix | 9 +- .../maven-and-npm-deps/maven/shell.nix | 10 +- nix/mobile/ios/default.nix | 8 +- nix/mobile/reset-node_modules.sh | 113 ++++++++---------- scripts/prepare-for-desktop-platform.sh | 4 +- shell.nix | 1 + 7 files changed, 76 insertions(+), 71 deletions(-) diff --git a/nix/derivation.nix b/nix/derivation.nix index 7add40996f..e4151ffdef 100644 --- a/nix/derivation.nix +++ b/nix/derivation.nix @@ -37,7 +37,7 @@ let # TARGETS leiningen-shell = mkShell { - buildInputs = with pkgs; [ clojure leiningen maven nodejs openjdk ]; + buildInputs = with pkgs; [ clojure leiningen flock maven nodejs openjdk ]; shellHook = if target-os == "android" then mobile.android.shellHook else if target-os == "ios" then mobile.ios.shellHook else ""; diff --git a/nix/mobile/android/default.nix b/nix/mobile/android/default.nix index 06b2bf209e..4901c40b5a 100644 --- a/nix/mobile/android/default.nix +++ b/nix/mobile/android/default.nix @@ -1,5 +1,5 @@ { config, stdenv, stdenvNoCC, target-os ? "android", callPackage, mkShell, - mkFilter, androidenv, fetchurl, openjdk, nodejs, bash, maven, zlib, + mkFilter, androidenv, fetchurl, flock, openjdk, nodejs, bash, maven, zlib, status-go, localMavenRepoBuilder, projectNodePackage, jsbundle }: let @@ -25,7 +25,12 @@ let in { inherit (androidEnv) androidComposition; - buildInputs = assert platform.targetAndroid; [ mavenAndNpmDeps.deriv openjdk gradle ]; + buildInputs = assert platform.targetAndroid; [ + mavenAndNpmDeps.deriv + flock # used in reset-node_modules.sh + openjdk + gradle + ]; shellHook = let inherit (stdenv.lib) catAttrs concatStrings; diff --git a/nix/mobile/android/maven-and-npm-deps/maven/shell.nix b/nix/mobile/android/maven-and-npm-deps/maven/shell.nix index f9cf0b2300..d401446f8c 100644 --- a/nix/mobile/android/maven-and-npm-deps/maven/shell.nix +++ b/nix/mobile/android/maven-and-npm-deps/maven/shell.nix @@ -1,9 +1,10 @@ -{ mkShell, curl, git, gradle, jq, maven, nodejs, +{ mkShell, curl, flock, git, gradle, jq, maven, nodejs, projectNodePackage, androidEnvShellHook, status-go }: mkShell { buildInputs = [ curl + flock # used in reset-node_modules.sh git gradle jq @@ -11,10 +12,9 @@ mkShell { nodejs projectNodePackage ]; - shellHook = - androidEnvShellHook + - status-go.shellHook + '' + shellHook = '' + ${androidEnvShellHook} + ${status-go.shellHook} $STATUS_REACT_HOME/nix/mobile/reset-node_modules.sh "${projectNodePackage}" - chmod -R u+w ./node_modules/ ''; } diff --git a/nix/mobile/ios/default.nix b/nix/mobile/ios/default.nix index 9d082d55f2..418689c938 100644 --- a/nix/mobile/ios/default.nix +++ b/nix/mobile/ios/default.nix @@ -1,5 +1,5 @@ { config, stdenv, stdenvNoCC, callPackage, mkShell, - xcodeWrapper, mkFilter, fetchurl, nodejs, bash, zlib, procps, + xcodeWrapper, mkFilter, fetchurl, flock, nodejs, bash, zlib, procps, status-go, projectNodePackage }: let @@ -37,7 +37,11 @@ let in { inherit xcodeWrapper; - buildInputs = unique ([ xcodeWrapper procps ] ++ catAttrs "buildInputs" selectedSources); + buildInputs = unique ([ + xcodeWrapper + flock # used in reset-node_modules.sh + procps + ] ++ catAttrs "buildInputs" selectedSources); shellHook = '' ${status-go.shellHook} diff --git a/nix/mobile/reset-node_modules.sh b/nix/mobile/reset-node_modules.sh index 5095e31fe4..3b1a54c150 100755 --- a/nix/mobile/reset-node_modules.sh +++ b/nix/mobile/reset-node_modules.sh @@ -16,68 +16,61 @@ set -Eeuo pipefail +function replaceNodeModules() { + local deps="$1" + local targetNodeModules="$2" + local needCopyModules=1 + local sentinelFilePath="$targetNodeModules/.copied~" + + # Check if node_modules exists and is valid + if [ -d "$targetNodeModules" ]; then + if [ -f "$sentinelFilePath" ]; then + existingPath="$(cat $sentinelFilePath)" + if [ "${existingPath}" != "${deps}" ]; then + echo "Yarn modules changed, copying new version over" + else + echo "Checking for modifications in node_modules..." + modifiedFiles=( + $(find $targetNodeModules -writable -type f -newer $sentinelFilePath \ + -not \( -path "$targetNodeModules/react-native/ReactAndroid/build/*" -prune \ + -o -path "$targetNodeModules/*/android/build/*" -prune \) -print) ) + if [ ${#modifiedFiles[@]} -eq 0 ]; then + needCopyModules=0 + echo "No modifications detected." + else + echo "Modifications detected in ${#modifiedFiles[@]} files:" + for f in ${modifiedFiles[@]}; do + echo "- $(realpath --relative-to=$STATUS_REACT_HOME $f)" + done + fi + fi + fi + if [ $needCopyModules -eq 1 ] && [ -d $targetNodeModules ]; then + chmod u+w -R $targetNodeModules + rm -rf $targetNodeModules + fi + fi + + # Replace node_modules if necessary + if [ ! -d "$targetNodeModules" ]; then + local tmpNodeModules=$(mktemp -d) + echo "Copying node_modules from Nix store (${deps}/node_modules)..." + trap "[ -d \"$tmpNodeModules\" ] && chmod -R u+w \"$tmpNodeModules\" && rm -rf \"$tmpNodeModules\"" ERR INT HUP + time cp -HRf --preserve=all ${deps}/node_modules/. "$tmpNodeModules" + chmod -R u+w "$tmpNodeModules" + mv -f "$tmpNodeModules" "$targetNodeModules" + echo -n "${deps}" > $sentinelFilePath + trap - ERR INT HUP + echo "Done" + fi +} + deps="$1" [ -d $deps ] || exit 1 nodeModulesDir="$STATUS_REACT_HOME/node_modules" -needCopyModules=1 -sentinelFilePath="$nodeModulesDir/.copied~" - -# Check if node_modules exists and is valid -if [ -d "$nodeModulesDir" ]; then - if [ -f "$sentinelFilePath" ]; then - existingPath="$(cat $sentinelFilePath)" - if [ "${existingPath}" != "${deps}" ]; then - echo "Yarn modules changed, copying new version over" - else - echo "Checking for modifications in node_modules..." - modifiedFiles=( - $(find $nodeModulesDir -writable -type f -newer $sentinelFilePath \ - -not \( -path "$nodeModulesDir/react-native/ReactAndroid/build/*" -prune \ - -o -path "$nodeModulesDir/*/android/build/*" -prune \) -print) ) - if [ ${#modifiedFiles[@]} -eq 0 ]; then - needCopyModules=0 - echo "No modifications detected." - else - echo "Modifications detected in ${#modifiedFiles[@]} files:" - for f in ${modifiedFiles[@]}; do - echo "- $(realpath --relative-to=$STATUS_REACT_HOME $f)" - done - fi - fi - fi - if [ $needCopyModules -eq 1 ] && [ -d $nodeModulesDir ]; then - chmod u+w -R $nodeModulesDir - rm -rf $nodeModulesDir - fi -fi - -# Replace node_modules if necessary -if [ ! -d "$nodeModulesDir" ]; then - if [ -n "${IN_CI_ENVIRONMENT:-''}" ]; then - # CI jobs might run multiple tasks in parallel, so it is possible that another process is setting up node_modules. In that case, let's wait up to 10 seconds for the .tmp dir goes away - for i in {1..10}; do - if [ -d "$nodeModulesDir.tmp" ]; then - echo "$nodeModulesDir.tmp exists, another process is probably setting up node_modules, waiting for 1 second..." - sleep 1 - else - break - fi - done - fi - if [ -d "$nodeModulesDir.tmp" ]; then - echo "$nodeModulesDir.tmp already exists, another shell session is probably currently setting up node_modules. Please try again in a few moments." - ls -al $STATUS_REACT_HOME - exit 1 - elif [ ! -d "$nodeModulesDir" ]; then - echo "Copying node_modules from Nix store (${deps}/node_modules)..." - trap "[ -d \"$nodeModulesDir.tmp\" ] && chmod -R u+w \"$nodeModulesDir.tmp\" && rm -rf \"$nodeModulesDir.tmp\"" ERR INT HUP - time cp -HRf --preserve=all ${deps}/node_modules/. "$nodeModulesDir.tmp" - chmod -R u+w "$nodeModulesDir.tmp" - mv -f "$nodeModulesDir.tmp" $nodeModulesDir - echo -n "${deps}" > $sentinelFilePath - trap - ERR INT HUP - echo "Done" - fi -fi +export -f replaceNodeModules +mkdir -p "$nodeModulesDir/" +# Leverage flock (file lock) utility to create an exclusive lock on node_modules/ while running replaceNodeModules +flock "$nodeModulesDir/" sh -c "replaceNodeModules $deps $nodeModulesDir" diff --git a/scripts/prepare-for-desktop-platform.sh b/scripts/prepare-for-desktop-platform.sh index 5ccc11828a..652c918a86 100755 --- a/scripts/prepare-for-desktop-platform.sh +++ b/scripts/prepare-for-desktop-platform.sh @@ -31,6 +31,8 @@ if [ ! -f package.json ] || [ $(readlink package.json) != "${PLATFORM_FOLDER}/pa ln -sf ${PLATFORM_FOLDER}/metro.config.js metro.config.js fi -yarn install --frozen-lockfile +mkdir -p "$GIT_ROOT/node_modules/" +# Leverage flock (file lock) utility to create an exclusive lock on node_modules/ while running 'yarn install' +flock "$GIT_ROOT/node_modules/" yarn install --frozen-lockfile echo -e "${GREEN}Finished!${NC}" diff --git a/shell.nix b/shell.nix index 4ac6d7c2f3..64ab4f74c2 100644 --- a/shell.nix +++ b/shell.nix @@ -20,6 +20,7 @@ let bash curl file + flock git gnumake jq