nix: optimize garbage collection

In order to prevent `nix-store --gc` from removing too much I've:

- Added the `keep-outputs = true` setting to `nix/nix.conf`
- Fixed `nix/scripts/gcroots.sh` to make symlinks in `/nix/var/nix/gcroots`
- Made `nix/scripts/build.sh` and `nix/scripts/shell.sh` use it

This way when running `make nix-gc` most recently used shells and built
derivations won't be removed along with their dependencies.

Signed-off-by: Jakub Sokołowski <jakub@status.im>

Makefile

@@ -39,6 +39,8 @@ export REACT_SERVER_PORT ?= 5001
 
 # Our custom config is located in nix/nix.conf
 export NIX_CONF_DIR = $(PWD)/nix
+# Location of symlinks to derivations that should not be garbage collected
+export _NIX_GCROOTS = /nix/var/nix/gcroots/per-user/$(USER)/status-react
 # Defines which variables will be kept for Nix pure shell, use semicolon as divider
 export _NIX_KEEP ?= TMPDIR,BUILD_ENV,STATUS_GO_SRC_OVERRIDE,NIMBUS_SRC_OVERRIDE
 
@@ -73,9 +75,14 @@ nix-repl: SHELL := /bin/sh
 nix-repl: ##@nix Start an interactive Nix REPL
 	nix repl default.nix
 
-nix-gc: export TARGET := default
+nix-gc-protected: SHELL := /bin/sh
-nix-gc: ##@nix Garbage collect all packages older than 20 days from /nix/store
+nix-gc-protected:
-	nix-collect-garbage --delete-old --delete-older-than 20d
+	@echo -e "$(YELLOW)The following paths are protected:$(RESET)" && \
+	ls -1 $(_NIX_GCROOTS) | sed 's/^/ - /'
+
+nix-gc: export TARGET := nix
+nix-gc: nix-gc-protected ##@nix Garbage collect all packages older than 20 days from /nix/store
+	nix-store --gc
 
 nix-clean: export TARGET := default
 nix-clean: ##@nix Remove all status-react build artifacts from /nix/store
@@ -85,10 +92,6 @@ nix-purge: SHELL := /bin/sh
 nix-purge: ##@nix Completely remove Nix setup, including /nix directory
 	nix/scripts/purge.sh
 
-nix-add-gcroots: export TARGET := default
-nix-add-gcroots: ##@nix Add Nix GC roots to avoid status-react expressions being garbage collected
-	nix/scripts/gcroots.sh
-
 nix-update-gradle: export TARGET := gradle
 nix-update-gradle: ##@nix Update maven nix expressions based on current gradle setup
 	nix/deps/gradle/generate.sh

nix/DETAILS.md

@@ -89,3 +89,11 @@ Some of those are required which is why just calling:
 nix-build --attr targets.mobile.android.release
 ```
 Would fail.
+
+# Garbage Collection
+
+The `make nix-gc` target calls `nix-store --gc` and normally would remove almost everything, but to prevent that we place symlinks to protected derivations in `/nix/var/nix/gcroots` subfolder. Specifically:
+```sh
+_NIX_GCROOTS="${_NIX_GCROOTS:-/nix/var/nix/gcroots/per-user/${USER}/status-react}"
+```
+Whenever `nix/scripts/build.sh` or `nix/scripts/shell.sh` are called they update symlinks named after given targets in that folder. This in combination with `keep-outputs = true` set in `nix/nix.conf` prevents garbage collection from removing too much.

nix/KNOWN_ISSUES.md

@@ -45,3 +45,10 @@ When building Android on NixOS you might encounter the following error:
 ignoring the user-specified setting 'extra-sandbox-paths', because it is a restricted setting and you are not a trusted user
 ```
 You can mitigate this by setting the [`nix.trustedUsers`](https://nixos.org/nixos/options.html#nix.trustedusers) property.
+
+## NixOS Prioritizes System Config
+
+Currently on NixOS `NIX_CONF_DIR` is being ignored in favor of the default `/etc/nix/nix.conf`.
+This will be possible to fix once Nix `2.4` comes out with support for `NIX_USER_CONF_FILES`.
+
+For more details see https://github.com/NixOS/nix/issues/3723.

nix/README.md

@@ -6,7 +6,7 @@ This folder contains configuration for [Nix](https://nixos.org/), a purely funct
 
 The main config file is [`nix/nix.conf`](/nix/nix.conf) and its main purpose is defining the [binary caches](https://nixos.org/nix/manual/#ch-basic-package-mgmt) which allow download of packages to avoid having to compile them yourself locally.
 
-__NOTE:__ If you are in Asia you might want to add the `https://nix-cache-cn.status.im/` to be first in order of `substituters`. Removing `cache.nixos.org` could also help.
+__NOTE:__ If you are in Asia you might want to add the `https://nix-cache-cn.status.im/` to be first in order of `extra-substituters`. Removing `cache.nixos.org` could also help.
 
 ## Build arguments
 

nix/nix.conf

@@ -1,7 +1,11 @@
-# NOTE: If you are in Asia you might want to add https://nix-cache-cn.status.im/ to substituters
+# NOTE: If you are in Asia you might want to add https://nix-cache-cn.status.im/ to extra-substituters
-substituters = https://nix-cache.status.im/ https://cache.nixos.org/
+extra-substituters = https://nix-cache.status.im/
+substituters = https://cache.nixos.org/
 trusted-public-keys = nix-cache.status.im-1:x/93lOfLU+duPplwMSBR+OlY4+mo+dCN7n0mr4oPwgY= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= nix-cache-cn.status.im:WUiOoTQQurm+rEL/yuAuU/a3TViDtMM9DCMgMx/KkOw= 
 # Some downloads are multiple GB, default is 5 minutes
 stalled-download-timeout = 600
 connect-timeout = 10
 max-jobs = auto
+# Helps avoid removing currently used dependencies via garbage collection
+keep-derivations = true
+keep-outputs = true

nix/scripts/build.sh

@@ -36,10 +36,10 @@ function extractResults() {
   chmod -R u+w "${resultPath}"
 }
 
-targetAttr="${1}"
+TARGET="${1}"
 shift
 
-if [[ -z "${targetAttr}" ]]; then
+if [[ -z "${TARGET}" ]]; then
   echo -e "${RED}First argument is mandatory and has to specify the Nix attribute!${RST}"
   exit 1
 fi
@@ -51,9 +51,12 @@ nixOpts=(
   "--fallback"
   "--no-out-link"
   "--show-trace"
-  "--attr" "${targetAttr}"
+  "--attr" "${TARGET}"
 )
 
+# Save derivation from being garbage collected
+${GIT_ROOT}/nix/scripts/gcroots.sh "${TARGET}"
+
 # Run the actual build
 echo "Running: nix-build "${nixOpts[@]}" "${@}" default.nix"
 nixResultPath=$(nix-build "${nixOpts[@]}" "${@}" default.nix)

nix/scripts/gcroots.sh

@@ -2,14 +2,20 @@
 
 set -Eeu
 
+_NIX_GCROOTS="${_NIX_GCROOTS:-/nix/var/nix/gcroots/per-user/${USER}/status-react}"
+
 GIT_ROOT=$(cd "${BASH_SOURCE%/*}" && git rev-parse --show-toplevel)
 source "${GIT_ROOT}/nix/scripts/source.sh"
+source "${GIT_ROOT}/scripts/colors.sh"
 
-GIT_ROOT=$(cd "${BASH_SOURCE%/*}" && git rev-parse --show-toplevel)
+TARGET="${1}"
+if [[ -z "${TARGET}" ]]; then
+    echo -e "${RED}No target specified for gcroots.sh!${RST}" >&2
+    exit 1
+fi
 
-rm -rf .nix-gcroots
+# Creates a symlink to derivation in _NIX_GCROOTS directory.
-mkdir .nix-gcroots
+# This prevents it from being removed by 'gc-collect-garbage'.
-
+nix-instantiate --attr "${TARGET}" \
-drv=$(nix-instantiate --argstr target all --add-root ${GIT_ROOT}/shell.nix)
+    --add-root "${_NIX_GCROOTS}/${TARGET}" \
-refs=$(nix-store --query --references $drv)
+    "${GIT_ROOT}/default.nix" >/dev/null
-nix-store -r $refs --indirect --add-root $GIT_ROOT/.nix-gcroots/shell.dep

nix/scripts/shell.sh

@@ -23,6 +23,12 @@ if [[ -z "${TARGET}" ]]; then
     TARGET="default"
     echo -e "${YLW}Missing TARGET, assuming default target.${RST} See nix/README.md for more details." 1>&2
 fi
+# Minimal shell with just Nix sourced, useful for `make nix-gc`.
+if [[ "${TARGET}" == "nix" ]]; then
+    eval $@
+    exit
+fi
+
 entryPoint="default.nix"
 nixArgs+=("--attr shells.${TARGET}")
 
@@ -61,6 +67,9 @@ fi
 
 echo -e "${GRN}Configuring ${pureDesc}Nix shell for target '${TARGET}'...${RST}" 1>&2
 
+# Save derivation from being garbage collected
+${GIT_ROOT}/nix/scripts/gcroots.sh "shells.${TARGET}"
+
 # ENTER_NIX_SHELL is the fake command used when `make shell` is run.
 # It is just a special string, not a variable, and a marker to not use `--run`.
 if [[ "${@}" == "ENTER_NIX_SHELL" ]]; then

nix/scripts/source.sh

@@ -5,7 +5,7 @@
 GIT_ROOT=$(cd "${BASH_SOURCE%/*}" && git rev-parse --show-toplevel)
 
 # Location of profile script for Nix that adjusts PATH
-NIX_PROFILE_SH="${HOME}/.nix-profile/etc/profile.d/nix.sh"
+export NIX_PROFILE_SH="${HOME}/.nix-profile/etc/profile.d/nix.sh"
 
 function source_nix() {
   # Just stop if Nix is already available