Use flock to create exclusive locks on node_modules/

Signed-off-by: Pedro Pombeiro <pombeirp@users.noreply.github.com>
This commit is contained in:
Pedro Pombeiro 2019-09-13 15:45:24 +02:00
parent d3e6414c89
commit 8c7323e126
No known key found for this signature in database
GPG Key ID: C4A24185B2AA48A1
7 changed files with 76 additions and 71 deletions

View File

@ -37,7 +37,7 @@ let
# TARGETS # TARGETS
leiningen-shell = mkShell { leiningen-shell = mkShell {
buildInputs = with pkgs; [ clojure leiningen maven nodejs openjdk ]; buildInputs = with pkgs; [ clojure leiningen flock maven nodejs openjdk ];
shellHook = shellHook =
if target-os == "android" then mobile.android.shellHook else if target-os == "android" then mobile.android.shellHook else
if target-os == "ios" then mobile.ios.shellHook else ""; if target-os == "ios" then mobile.ios.shellHook else "";

View File

@ -1,5 +1,5 @@
{ config, stdenv, stdenvNoCC, target-os ? "android", callPackage, mkShell, { config, stdenv, stdenvNoCC, target-os ? "android", callPackage, mkShell,
mkFilter, androidenv, fetchurl, openjdk, nodejs, bash, maven, zlib, mkFilter, androidenv, fetchurl, flock, openjdk, nodejs, bash, maven, zlib,
status-go, localMavenRepoBuilder, projectNodePackage, jsbundle }: status-go, localMavenRepoBuilder, projectNodePackage, jsbundle }:
let let
@ -25,7 +25,12 @@ let
in { in {
inherit (androidEnv) androidComposition; inherit (androidEnv) androidComposition;
buildInputs = assert platform.targetAndroid; [ mavenAndNpmDeps.deriv openjdk gradle ]; buildInputs = assert platform.targetAndroid; [
mavenAndNpmDeps.deriv
flock # used in reset-node_modules.sh
openjdk
gradle
];
shellHook = shellHook =
let let
inherit (stdenv.lib) catAttrs concatStrings; inherit (stdenv.lib) catAttrs concatStrings;

View File

@ -1,9 +1,10 @@
{ mkShell, curl, git, gradle, jq, maven, nodejs, { mkShell, curl, flock, git, gradle, jq, maven, nodejs,
projectNodePackage, androidEnvShellHook, status-go }: projectNodePackage, androidEnvShellHook, status-go }:
mkShell { mkShell {
buildInputs = [ buildInputs = [
curl curl
flock # used in reset-node_modules.sh
git git
gradle gradle
jq jq
@ -11,10 +12,9 @@ mkShell {
nodejs nodejs
projectNodePackage projectNodePackage
]; ];
shellHook = shellHook = ''
androidEnvShellHook + ${androidEnvShellHook}
status-go.shellHook + '' ${status-go.shellHook}
$STATUS_REACT_HOME/nix/mobile/reset-node_modules.sh "${projectNodePackage}" $STATUS_REACT_HOME/nix/mobile/reset-node_modules.sh "${projectNodePackage}"
chmod -R u+w ./node_modules/
''; '';
} }

View File

@ -1,5 +1,5 @@
{ config, stdenv, stdenvNoCC, callPackage, mkShell, { config, stdenv, stdenvNoCC, callPackage, mkShell,
xcodeWrapper, mkFilter, fetchurl, nodejs, bash, zlib, procps, xcodeWrapper, mkFilter, fetchurl, flock, nodejs, bash, zlib, procps,
status-go, projectNodePackage }: status-go, projectNodePackage }:
let let
@ -37,7 +37,11 @@ let
in { in {
inherit xcodeWrapper; inherit xcodeWrapper;
buildInputs = unique ([ xcodeWrapper procps ] ++ catAttrs "buildInputs" selectedSources); buildInputs = unique ([
xcodeWrapper
flock # used in reset-node_modules.sh
procps
] ++ catAttrs "buildInputs" selectedSources);
shellHook = '' shellHook = ''
${status-go.shellHook} ${status-go.shellHook}

View File

@ -16,68 +16,61 @@
set -Eeuo pipefail set -Eeuo pipefail
function replaceNodeModules() {
local deps="$1"
local targetNodeModules="$2"
local needCopyModules=1
local sentinelFilePath="$targetNodeModules/.copied~"
# Check if node_modules exists and is valid
if [ -d "$targetNodeModules" ]; then
if [ -f "$sentinelFilePath" ]; then
existingPath="$(cat $sentinelFilePath)"
if [ "${existingPath}" != "${deps}" ]; then
echo "Yarn modules changed, copying new version over"
else
echo "Checking for modifications in node_modules..."
modifiedFiles=(
$(find $targetNodeModules -writable -type f -newer $sentinelFilePath \
-not \( -path "$targetNodeModules/react-native/ReactAndroid/build/*" -prune \
-o -path "$targetNodeModules/*/android/build/*" -prune \) -print) )
if [ ${#modifiedFiles[@]} -eq 0 ]; then
needCopyModules=0
echo "No modifications detected."
else
echo "Modifications detected in ${#modifiedFiles[@]} files:"
for f in ${modifiedFiles[@]}; do
echo "- $(realpath --relative-to=$STATUS_REACT_HOME $f)"
done
fi
fi
fi
if [ $needCopyModules -eq 1 ] && [ -d $targetNodeModules ]; then
chmod u+w -R $targetNodeModules
rm -rf $targetNodeModules
fi
fi
# Replace node_modules if necessary
if [ ! -d "$targetNodeModules" ]; then
local tmpNodeModules=$(mktemp -d)
echo "Copying node_modules from Nix store (${deps}/node_modules)..."
trap "[ -d \"$tmpNodeModules\" ] && chmod -R u+w \"$tmpNodeModules\" && rm -rf \"$tmpNodeModules\"" ERR INT HUP
time cp -HRf --preserve=all ${deps}/node_modules/. "$tmpNodeModules"
chmod -R u+w "$tmpNodeModules"
mv -f "$tmpNodeModules" "$targetNodeModules"
echo -n "${deps}" > $sentinelFilePath
trap - ERR INT HUP
echo "Done"
fi
}
deps="$1" deps="$1"
[ -d $deps ] || exit 1 [ -d $deps ] || exit 1
nodeModulesDir="$STATUS_REACT_HOME/node_modules" nodeModulesDir="$STATUS_REACT_HOME/node_modules"
needCopyModules=1 export -f replaceNodeModules
sentinelFilePath="$nodeModulesDir/.copied~" mkdir -p "$nodeModulesDir/"
# Leverage flock (file lock) utility to create an exclusive lock on node_modules/ while running replaceNodeModules
# Check if node_modules exists and is valid flock "$nodeModulesDir/" sh -c "replaceNodeModules $deps $nodeModulesDir"
if [ -d "$nodeModulesDir" ]; then
if [ -f "$sentinelFilePath" ]; then
existingPath="$(cat $sentinelFilePath)"
if [ "${existingPath}" != "${deps}" ]; then
echo "Yarn modules changed, copying new version over"
else
echo "Checking for modifications in node_modules..."
modifiedFiles=(
$(find $nodeModulesDir -writable -type f -newer $sentinelFilePath \
-not \( -path "$nodeModulesDir/react-native/ReactAndroid/build/*" -prune \
-o -path "$nodeModulesDir/*/android/build/*" -prune \) -print) )
if [ ${#modifiedFiles[@]} -eq 0 ]; then
needCopyModules=0
echo "No modifications detected."
else
echo "Modifications detected in ${#modifiedFiles[@]} files:"
for f in ${modifiedFiles[@]}; do
echo "- $(realpath --relative-to=$STATUS_REACT_HOME $f)"
done
fi
fi
fi
if [ $needCopyModules -eq 1 ] && [ -d $nodeModulesDir ]; then
chmod u+w -R $nodeModulesDir
rm -rf $nodeModulesDir
fi
fi
# Replace node_modules if necessary
if [ ! -d "$nodeModulesDir" ]; then
if [ -n "${IN_CI_ENVIRONMENT:-''}" ]; then
# CI jobs might run multiple tasks in parallel, so it is possible that another process is setting up node_modules. In that case, let's wait up to 10 seconds for the .tmp dir goes away
for i in {1..10}; do
if [ -d "$nodeModulesDir.tmp" ]; then
echo "$nodeModulesDir.tmp exists, another process is probably setting up node_modules, waiting for 1 second..."
sleep 1
else
break
fi
done
fi
if [ -d "$nodeModulesDir.tmp" ]; then
echo "$nodeModulesDir.tmp already exists, another shell session is probably currently setting up node_modules. Please try again in a few moments."
ls -al $STATUS_REACT_HOME
exit 1
elif [ ! -d "$nodeModulesDir" ]; then
echo "Copying node_modules from Nix store (${deps}/node_modules)..."
trap "[ -d \"$nodeModulesDir.tmp\" ] && chmod -R u+w \"$nodeModulesDir.tmp\" && rm -rf \"$nodeModulesDir.tmp\"" ERR INT HUP
time cp -HRf --preserve=all ${deps}/node_modules/. "$nodeModulesDir.tmp"
chmod -R u+w "$nodeModulesDir.tmp"
mv -f "$nodeModulesDir.tmp" $nodeModulesDir
echo -n "${deps}" > $sentinelFilePath
trap - ERR INT HUP
echo "Done"
fi
fi

View File

@ -31,6 +31,8 @@ if [ ! -f package.json ] || [ $(readlink package.json) != "${PLATFORM_FOLDER}/pa
ln -sf ${PLATFORM_FOLDER}/metro.config.js metro.config.js ln -sf ${PLATFORM_FOLDER}/metro.config.js metro.config.js
fi fi
yarn install --frozen-lockfile mkdir -p "$GIT_ROOT/node_modules/"
# Leverage flock (file lock) utility to create an exclusive lock on node_modules/ while running 'yarn install'
flock "$GIT_ROOT/node_modules/" yarn install --frozen-lockfile
echo -e "${GREEN}Finished!${NC}" echo -e "${GREEN}Finished!${NC}"

View File

@ -20,6 +20,7 @@ let
bash bash
curl curl
file file
flock
git git
gnumake gnumake
jq jq