nix: use overlays instead of packageOverrides

Changes:
- Add `nix/DETAILS.md` for more in-depth info
- Rename some of `config.status-im` variables
- Drop `env` argument for Android build
- Use `overlays` instead of `packageOverrides`
- Move the `pkgs` overlay to `nix/overlay.nix`
- Move `nix/status-go/utils.nix` to `nix/tools`
- Make `shell.nix` use the `shells.default` only
- Use `default.nix` as target for `nix/scripts/shell.sh`
- Make `nix/scripts/shell.sh` use `--attr` instead of `target`
- Drop the `target` argument in favour of using `--attr`
- Drop unnecessary `src` from `nix/mobile/ios/default.nix`
- Move `mkShell` and `mergeSh` under `lib`
- Move `patched-go` package to `nix/pkgs` directory
- Move `gomobile` package to `nix/pkgs` directory
- Move `ANDROID_ABI_SPLIT` to `config.status-im.android.abi-split`
- Move `ANDROID_ABI_INCLUDE to `config.status-im.android.abi-include`

Signed-off-by: Jakub Sokołowski <jakub@status.im>

Makefile

@@ -69,6 +69,10 @@ else
 	@echo "${YELLOW}Nix shell is already active$(RESET)"
 endif
 
+nix-repl: SHELL := /bin/sh
+nix-repl: ##@nix Start an interactive Nix REPL
+	nix repl default.nix
+
 nix-gc: export TARGET := default
 nix-gc: ##@nix Garbage collect all packages older than 20 days from /nix/store
 	nix-collect-garbage --delete-old --delete-older-than 20d

ci/Jenkinsfile.android

@@ -1,4 +1,4 @@
-library 'status-react-jenkins@master'
+library 'status-react-jenkins@v1.1.1'
 
 pipeline {
   agent { label 'linux' }

ci/Jenkinsfile.combined

@@ -1,4 +1,4 @@
-library 'status-react-jenkins@master'
+library 'status-react-jenkins@v1.1.1'
 
 pipeline {
   agent { label 'linux' }

ci/Jenkinsfile.fastlane.clean

@@ -1,4 +1,4 @@
-library 'status-react-jenkins@master'
+library 'status-react-jenkins@v1.1.1'
 
 pipeline {
   agent { label 'macos' }

ci/Jenkinsfile.ios

@@ -1,4 +1,4 @@
-library 'status-react-jenkins@master'
+library 'status-react-jenkins@v1.1.1'
 
 pipeline {
   agent { label 'macos-xcode-11.4.1' }

ci/Jenkinsfile.linux

@@ -1,4 +1,4 @@
-library 'status-react-jenkins@master'
+library 'status-react-jenkins@v1.1.1'
 
 pipeline {
   agent { label 'linux' }

ci/Jenkinsfile.macos

@@ -1,4 +1,4 @@
-library 'status-react-jenkins@master'
+library 'status-react-jenkins@v1.1.1'
 
 pipeline {
   agent { label 'macos-xcode-11.4.1' }

ci/Jenkinsfile.nix-cache

@@ -1,4 +1,4 @@
-library 'status-react-jenkins@master'
+library 'status-react-jenkins@v1.1.1'
 
 pipeline {
   agent { label params.AGENT_LABEL }

ci/Jenkinsfile.windows

@@ -1,4 +1,4 @@
-library 'status-react-jenkins@master'
+library 'status-react-jenkins@v1.1.1'
 
 pipeline {
   agent { label 'linux' }

default.nix

@@ -1,9 +1,9 @@
-{
+# for passing build options, see nix/README.md
-  config ? { status-im = { build-type = ""; }; }, # for passing build options, see nix/README.md
+{ config ? { status-im = { build-type = ""; }; } }:
-}:
 
 let
   main = import ./nix/default.nix { inherit config; };
 in {
+  # this is where the --attr argument selects the shell or target
   inherit (main) pkgs targets shells;
 }

nix/DETAILS.md

@@ -0,0 +1,65 @@
+# Description
+
+This document descripts the layout of our Nix setup.
+
+# Folders
+
+There are four main folders in the `nix` directory:
+
+* `nix/scripts` - Bash scripts for easier usage of Nix
+* `nix/pkgs` - Packages we add to or modify in `nixpkgs`
+* `nix/tools` - Various tools used by our derivations and shells
+* `nix/status-go` - Derivations for building [`status-go`](https://github.com/status-im/status-go) repo
+
+# Files
+
+There are a few main files that define the whole build environment:
+
+* `nix/default.nix` - Entrypoint for both shells and targets
+* `nix/shells.nix` - Definition of Nix shells used in builds
+* `nix/targets.nix` - Hierarchy of main build targets
+* `nix/pkgs.nix` - Definition of a custom `nixpkgs` repo
+* `nix/overlay.nix` - Overrides for `nixpkgs`, custom packages
+
+# Start
+
+The starting point for using our Nix shells and targets is the [`default.nix`](/default.nix) file.
+
+It pulls in all the `pkgs`, `targets` and `shells` defined in [`nix/default.nix`](/nix/default.nix). The point is easy access to them via commands like `nix-build` or `nix-shell`, which you'll see next.
+
+# Usage
+
+We will use the `make jsbundle-android` target as an example of a derivation you can build using Nix:
+
+1. `make jsbundle-android` is called by developer
+2. `make` calls `nix/scripts/build.sh targets.mobile.android.jsbundle`
+3. [`build.sh`](/nix/scripts/build.sh) calls `nix-build --attr targets.mobile.android.jsbundle` with extra arguments
+4. `nix-build` builds the derivation from [`nix/mobile/android/jsbundle/default.nix`](/nix/mobile/android/jsbundle/default.nix)
+
+The same can be done for other targets like `targets.mobile.android.release`.
+Except in that case extra arguments are required which is why the [`scripts/release-android.sh`](/scripts/release-android.sh) is used in the `make release-android` target.
+
+If you run `make release-android` you'll see the `nix-build` command used:
+```
+nix-build \
+  --pure \
+  --fallback \
+  --no-out-link \
+  --show-trace \
+  --attr targets.mobile.android.release \
+  --argstr secrets-file '/tmp/tmp-status-react-559a3a441/tmp.xAnrPuNtAP' \
+  --option extra-sandbox-paths '/home/sochan/.gradle/status-im.keystore /tmp/tmp-status-react-559a3a441/tmp.xAnrPuNtAP' \
+  --arg config '{ \
+    status-im.build-type="nightly";
+    status-im.build-number="2020022418";
+    status-im.android.keystore-file="/home/sochan/.gradle/status-im.keystore";
+    status-im.android.abi-split="false";
+    status-im.android.abi-include="armeabi-v7a;arm64-v8a;x86";
+  }' \
+  default.nix
+```
+Some of those are required which is why just calling:
+```
+nix-build --attr targets.mobile.android.release
+```
+Would fail.

nix/mobile/android/default.nix

@@ -1,4 +1,4 @@
-{ config, lib, callPackage, mkShell, mergeSh, flock, lsof, openjdk, gradle_5,
+{ config, lib, callPackage, mkShell, flock, lsof, openjdk, gradle_5,
   status-go, localMavenRepoBuilder, projectNodePackage, androidPkgs, androidShell }:
 
 let
@@ -6,7 +6,7 @@ let
   leinProjectDeps = import ../../lein/lein-project-deps.nix { };
 
   # Import a jsbundle compiled out of clojure codebase
-  jsbundle = callPackage ./jsbundle/default.nix {
+  jsbundle = callPackage ./jsbundle {
     inherit leinProjectDeps localMavenRepoBuilder projectNodePackage;
   };
 
@@ -37,7 +37,7 @@ in {
   # TARGETS
   inherit release jsbundle generate-maven-and-npm-deps-shell buildInputs;
 
-  shell = mergeSh
+  shell = lib.mergeSh
     (mkShell {
       inherit buildInputs;
       inputsFrom = [ release gradle ];

nix/mobile/android/jsbundle/default.nix

@@ -3,7 +3,7 @@
 #
 
 { target-os ? "android",
-  stdenv, mkFilter, clojure, leiningen, nodejs, bash, git,
+  stdenv, lib, clojure, leiningen, nodejs, bash, git,
   leinProjectDeps, localMavenRepoBuilder, projectNodePackage }:
 
 let
@@ -19,7 +19,7 @@ in stdenv.mkDerivation {
       name = "status-react-source-jsbundle";
       filter =
         # Keep this filter as restrictive as possible in order to avoid unnecessary rebuilds and limit closure size
-        mkFilter {
+        lib.mkFilter {
           root = path;
           ignoreVCS = false;
           include = [ 

nix/mobile/android/maven-and-npm-deps/default.nix

@@ -5,7 +5,7 @@
 
 { stdenv, lib, callPackage, mkShell,
   gradle, bash, file, nodejs, zlib,
-  projectNodePackage, localMavenRepoBuilder, mkFilter }:
+  projectNodePackage, localMavenRepoBuilder }:
 
 let
   mavenLocalRepo = callPackage ./maven { inherit localMavenRepoBuilder stdenv; };
@@ -36,7 +36,7 @@ let
             name = "status-react-source-gradle-install";
             filter =
               # Keep this filter as restrictive as possible in order to avoid unnecessary rebuilds and limit closure size
-              mkFilter {
+              lib.mkFilter {
                 root = path;
                 include = [
                   "android/.*" "translations/.*" "status-modules/.*"

nix/mobile/android/targets/release-android.nix

@@ -1,61 +1,85 @@
-{ stdenv, lib, config, callPackage,
+{ stdenv, lib, config, callPackage, bash, file, gnumake, watchmanFactory, gradle
-  mkFilter, bash, file, gnumake, watchmanFactory, gradle,
+, androidPkgs, mavenAndNpmDeps, nodejs, openjdk, jsbundle, status-go, unzip, zlib }:
-  androidPkgs, mavenAndNpmDeps,
-  nodejs, openjdk, jsbundle, status-go, unzip, zlib }:
 
-{ secrets-file ? "", # Path to the file containing secret environment variables
+{
+  buildEnv ? "prod", # Value for BUILD_ENV checked by Clojure code at compile time
+  secretsFile ? "", # Path to the file containing secret environment variables
   watchmanSockPath ? "", # Path to the socket file exposed by an external watchman instance (workaround needed for building Android on macOS)
-  env ? {} # Attribute set containing environment variables to expose to the build script
 }:
 
-assert (builtins.stringLength watchmanSockPath) > 0 -> stdenv.isDarwin;
+assert (lib.stringLength watchmanSockPath) > 0 -> stdenv.isDarwin;
 
 let
-  inherit (lib) attrByPath hasAttrByPath optionalAttrs;
+  inherit (lib)
-  env' = env // optionalAttrs (hasAttrByPath ["status-im" "status-go" "src-override"] config) {
+    toLower splitString optionalString
-    STATUS_GO_SRC_OVERRIDE = config.status-im.status-go.src-override;
+    attrByPath hasAttrByPath optionalAttrs;
-  } // optionalAttrs (hasAttrByPath ["status-im" "nimbus" "src-override"] config) {
+
-    NIMBUS_SRC_OVERRIDE = config.status-im.nimbus.src-override;
+  # helper for getting config values
+  safeGetConfig = name: default:
+    let path = [ "status-im" ] ++ (splitString "." name);
+    in attrByPath path default config;
+
+  # custom env variables derived from config
+  env = {
+    ANDROID_ABI_SPLIT = safeGetConfig "android.abi-split" false;
+    ANDROID_ABI_INCLUDE = safeGetConfig "android.abi-include" "armeabi-v7a;arm64-v8a;x86";
+    STATUS_GO_SRC_OVERRIDE = safeGetConfig "nimbus.src-override" null;
   };
-  inherit (config.status-im) build-type;
+
-  inherit (config.status-im.status-react) build-number;
+  buildType = safeGetConfig "build-type" "prod";
-  gradle-opts = (attrByPath ["status-im" "status-react" "gradle-opts"] "" config);
+  buildNumber = safeGetConfig "build-number" "9999";
-  # Path to the .keystore file used to sign the Android APK
+  gradleOpts = safeGetConfig "android.gradle-opts" "";
-  keystore-file = (attrByPath ["status-im" "status-react" "keystore-file"] "" config);
+  keystorePath = safeGetConfig "android.keystore-path" "";
+  # Keep the same keystore path for determinism
+  keystoreLocal = "${gradleHome}/status-im.keystore";
+
   baseName = "release-android";
   name = "status-react-build-${baseName}";
+
   gradleHome = "$NIX_BUILD_TOP/.gradle";
   localMavenRepo = "${mavenAndNpmDeps.drv}/.m2/repository";
   sourceProjectDir = "${mavenAndNpmDeps.drv}/project";
-  envFileName =
+  envFileName = if (buildType == "release" || buildType == "nightly" || buildType == "e2e")
-    if (build-type == "release" || build-type == "nightly" || build-type == "e2e") then ".env.${build-type}" else
+    then ".env.${buildType}"
-    if build-type != "" then ".env.jenkins" else ".env";
+    else if buildType != "" then ".env.jenkins"
-  buildType = if (build-type == "pr" || build-type == "e2e") then "pr" else "release"; /* PR builds shouldn't replace normal releases */
+    else ".env";
-  apksPath = "$sourceRoot/android/app/build/outputs/apk/${buildType}";
+
+  # There are only two types of Gradle builds: pr and release
+  gradleBuildType = if (buildType == "pr" || buildType == "e2e")
+    then "Pr"
+    else "Release"; # PR builds shouldn't replace normal releases
+
+  apksPath = "$sourceRoot/android/app/build/outputs/apk/${toLower gradleBuildType}";
   patchedWatchman = watchmanFactory watchmanSockPath;
 
-in stdenv.mkDerivation {
+in stdenv.mkDerivation rec {
   inherit name;
-  src =
+  src = let path = ./../../../..;
-    let path = ./../../../..;
+  # We use builtins.path so that we can name the resulting derivation
-    in builtins.path { # We use builtins.path so that we can name the resulting derivation, otherwise the name would be taken from the checkout directory, which is outside of our control
+  in builtins.path {
-      inherit path;
+    inherit path;
-      name = "status-react-source-${baseName}";
+    name = "status-react-source-${baseName}";
-      filter =
+    # Keep this filter as restrictive as possible in order to avoid unnecessary rebuilds and limit closure size
-        # Keep this filter as restrictive as possible in order to avoid unnecessary rebuilds and limit closure size
+    filter = lib.mkFilter {
-        mkFilter {
+      root = path;
-          root = path;
+      include = [
-          include = [
+        "mobile/js_files.*" "resources/.*"
-            "mobile/js_files.*" "resources/.*"
+        "modules/react-native-status/android.*"
-            "modules/react-native-status/android.*"
+        envFileName "VERSION" ".watchmanconfig"
-            envFileName "VERSION" ".watchmanconfig"
+        "status-go-version.json" "react-native.config.js"
-            "status-go-version.json" "react-native.config.js"
+      ];
-          ];
-        };
     };
-  nativeBuildInputs = [ bash gradle unzip ] ++ lib.optionals stdenv.isDarwin [ file gnumake patchedWatchman ];
+  };
+
   buildInputs = [ nodejs openjdk ];
+  nativeBuildInputs = [ bash gradle unzip ]
+    ++ lib.optionals stdenv.isDarwin [ file gnumake patchedWatchman ];
+
+  # Used by Clojure at compile time to include JS modules
+  BUILD_ENV = buildEnv;
+
   phases = [ "unpackPhase" "patchPhase" "buildPhase" "checkPhase" "installPhase" ];
+
   unpackPhase = ''
     runHook preUnpack
 
@@ -66,10 +90,11 @@ in stdenv.mkDerivation {
 
     runHook postUnpack
   '';
-  postUnpack = ''
+  postUnpack = assert lib.assertMsg (keystorePath != "") "keystore-file has to be set!"; ''
     mkdir -p ${gradleHome}
 
-    ${if keystore-file != "" then "cp -a --no-preserve=ownership ${keystore-file} ${gradleHome}/; export KEYSTORE_PATH=${gradleHome}/$(basename ${keystore-file})" else ""}
+    # WARNING: Renaming the keystore will cause 'Keystore was tampered with' error
+    cp -a --no-preserve=ownership "${keystorePath}" "${keystoreLocal}"
 
     # Ensure we have the right .env file
     cp -f $sourceRoot/${envFileName} $sourceRoot/.env
@@ -81,7 +106,8 @@ in stdenv.mkDerivation {
     cp -a --no-preserve=ownership ${sourceProjectDir}/android/ $sourceRoot/
     chmod u+w $sourceRoot/android
     chmod u+w $sourceRoot/android/app
-    mkdir $sourceRoot/android/build && chmod -R u+w $sourceRoot/android/build
+    mkdir -p $sourceRoot/android/build
+    chmod -R u+w $sourceRoot/android/build
 
     # Copy node_modules/ directory
     cp -a --no-preserve=ownership ${sourceProjectDir}/node_modules/ $sourceRoot/
@@ -97,28 +123,36 @@ in stdenv.mkDerivation {
     substituteInPlace $sourceRoot/android/gradlew \
       --replace \
         'exec gradle' \
-        "exec gradle -Dmaven.repo.local='${localMavenRepo}' --offline ${gradle-opts}"
+        "exec gradle -Dmaven.repo.local='${localMavenRepo}' --offline ${gradleOpts}"
 
     set $prevSet
   '';
-  buildPhase =
+  buildPhase = let
-    let
+    inherit (lib)
-      inherit (lib) catAttrs concatStrings concatStringsSep mapAttrsToList makeLibraryPath optionalString substring toUpper;
+      stringLength optionalString substring
-      # Take the env attribute set and build a couple of scripts
+      concatStrings concatStringsSep
-      #  (one to export the environment variables, and another to unset them)
+      catAttrs mapAttrsToList makeLibraryPath;
-      exportEnvVars = concatStringsSep ";" (mapAttrsToList (name: value: "export ${name}='${value}'") env');
+
-      unsetEnvVars = concatStringsSep ";" (mapAttrsToList (name: value: "unset ${name}") env');
+    # Take the env attribute set and build a couple of scripts
-      adhocEnvVars = optionalString stdenv.isLinux "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${makeLibraryPath [ zlib ]}";
+    #  (one to export the environment variables, and another to unset them)
-      capitalizedBuildType = toUpper (substring 0 1 buildType) + substring 1 (-1) buildType;
+    exportEnvVars = concatStringsSep ";"
-    in ''
+      (mapAttrsToList (name: value: "export ${name}='${toString value}'") env);
+    adhocEnvVars = optionalString stdenv.isLinux
+      "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${makeLibraryPath [ zlib ]}";
+  in 
+    assert stringLength env.ANDROID_ABI_SPLIT > 0;
+    assert stringLength env.ANDROID_ABI_INCLUDE > 0;
+  ''
     export ANDROID_SDK_ROOT="${androidPkgs}"
     export ANDROID_NDK_ROOT="${androidPkgs}/ndk-bundle"
 
+    export KEYSTORE_PATH="${keystoreLocal}"
+
     export STATUS_REACT_HOME=$PWD
     export HOME=$sourceRoot
 
     ${exportEnvVars}
-    ${if secrets-file != "" then "source ${secrets-file}" else ""}
+    ${optionalString (secretsFile != "") "source ${secretsFile}"}
 
     ${concatStrings (catAttrs "shellHook" [ mavenAndNpmDeps.shell status-go.shell ])}
 
@@ -126,10 +160,8 @@ in stdenv.mkDerivation {
     chmod -R +w $sourceRoot/android
 
     pushd $sourceRoot/android
-    ${adhocEnvVars} ./gradlew -PversionCode=${assert build-number != ""; build-number} assemble${capitalizedBuildType} || exit
+    ${adhocEnvVars} ./gradlew -PversionCode=${buildNumber} assemble${gradleBuildType} || exit
     popd > /dev/null
-
-    ${unsetEnvVars}
   '';
   doCheck = true;
   checkPhase = ''

nix/mobile/default.nix

@@ -1,5 +1,5 @@
 { config, lib, stdenvNoCC, callPackage, mkShell,
-  status-go, mergeSh, xcodeWrapper }:
+  status-go, xcodeWrapper }:
 
 let
   inherit (lib) catAttrs concatStrings optional unique;
@@ -31,7 +31,7 @@ let
 in {
   buildInputs = unique (catAttrs "buildInputs" selectedSources);
 
-  shell = mergeSh (mkShell {}) (catAttrs "shell" selectedSources);
+  shell = lib.mergeSh (mkShell {}) (catAttrs "shell" selectedSources);
 
   # TARGETS
   inherit android ios fastlane;

nix/mobile/ios/default.nix

@@ -1,58 +1,41 @@
-{ callPackage, lib, stdenv, mkShell, mergeSh, mkFilter, 
+{ callPackage, lib, mkShell,
   xcodeWrapper, projectNodePackage, status-go,
   flock, procps, watchman, bundler, fastlane }:
 
 let
   inherit (lib) catAttrs unique;
 
-  pod = callPackage ./pod-shell.nix { };
+  pod-shell = callPackage ./pod-shell.nix { };
   status-go-shell = callPackage ./status-go-shell.nix { inherit status-go; };
 
   selectedSources = [ status-go fastlane ];
 
-  src =
-    let path = ./../../..;
-    # We use builtins.path so that we can name the resulting derivation,
-    # otherwise the name would be taken from the checkout directory, which is outside of our control
-    in builtins.path {
-      inherit path;
-      name = "status-react-source-npm-deps";
-      filter =
-        # Keep this filter as restrictive as possible in order to avoid 
-        # unnecessary rebuilds and limit closure size
-        mkFilter {
-          include = [ ".babelrc" "mobile/js_files.*" ];
-          root = path;
-        };
-    };
-
   buildInputs = unique ([
     xcodeWrapper watchman bundler procps
     flock # used in reset-node_modules.sh
   ] ++ catAttrs "buildInputs" selectedSources);
 
-  shellHook = ''
-    pushd "$STATUS_REACT_HOME" > /dev/null
-    {
-      # Set up symlinks to mobile enviroment in project root 
-      ln -sf ./mobile/js_files/metro.config.js ./metro.config.js
-      ln -sf ./mobile/js_files/package.json ./package.json
-      ln -sf ./mobile/js_files/yarn.lock ./yarn.lock
-
-      # check if node modules changed and if so install them
-      ./nix/mobile/reset-node_modules.sh "${projectNodePackage}"
-    }
-    popd > /dev/null
-  '';
-
   localShell = mkShell {
-    inherit buildInputs shellHook;
+    shellHook = ''
+      pushd "$STATUS_REACT_HOME" > /dev/null
+      {
+        # Set up symlinks to mobile enviroment in project root 
+        ln -sf ./mobile/js_files/metro.config.js ./metro.config.js
+        ln -sf ./mobile/js_files/package.json ./package.json
+        ln -sf ./mobile/js_files/yarn.lock ./yarn.lock
+
+        # check if node modules changed and if so install them
+        ./nix/mobile/reset-node_modules.sh "${projectNodePackage}"
+      }
+      popd > /dev/null
+    '';
+    inherit buildInputs;
   };
 
 in {
-  inherit shellHook buildInputs pod;
+  inherit buildInputs pod-shell;
 
-  shell = mergeSh localShell [
+  shell = lib.mergeSh localShell [
-    fastlane.shell status-go-shell pod.shell
+    fastlane.shell status-go-shell pod-shell
   ];
 }

nix/mobile/ios/pod-shell.nix

@@ -5,22 +5,20 @@ let
   podfileLock = "ios/Podfile.lock";
   # current state of pods installed by pod
   manifestLock = "ios/Pods/Manifest.lock";
-in {
+in mkShell {
-  shell = mkShell {
+  buildInputs = [ cocoapods ];
-    buildInputs = [ cocoapods ];
+  shellHook = ''
-    shellHook = ''
+    pushd "$STATUS_REACT_HOME" > /dev/null
-      pushd "$STATUS_REACT_HOME" > /dev/null
+    {
-      { 
+      echo "Checking for modifications in ios/Pods..."
-        echo "Checking for modifications in ios/Pods..."
+      if diff -q ${podfileLock} ${manifestLock}; then
-        if diff -q ${podfileLock} ${manifestLock}; then
+        echo "No modifications detected."
-          echo "No modifications detected."
+      else
-        else
+        # CocoaPods are trash and can't handle other pod instances running
-          # CocoaPods are trash and can't handle other pod instances running
+        ./scripts/wait-for.sh 240 'pod install'
-          ./scripts/wait-for.sh 240 'pod install'
+        (cd ios && pod install)
-          (cd ios && pod install)
+      fi
-        fi
+    }
-      }
+    popd > /dev/null
-      popd > /dev/null
+  '';
-    '';
-  };
 }

nix/mobile/ios/status-go-shell.nix

@@ -1,4 +1,4 @@
-{ mkShell, mergeSh, status-go }:
+{ lib, mkShell, status-go }:
 
 let
   shell = mkShell {
@@ -28,4 +28,4 @@ let
     '';
   };
 in
-  mergeSh status-go.shell [ shell ]
+  lib.mergeSh status-go.shell [ shell ]

nix/overlay.nix

@@ -0,0 +1,37 @@
+#
+# Override some packages and utilities in 'pkgs'
+# and make them available globally via callPackage.
+#
+# For more details see:
+# - https://nixos.wiki/wiki/Overlays
+# - https://nixos.org/nixos/nix-pills/callpackage-design-pattern.html
+#
+
+self: super:
+
+let
+  inherit (super) stdenv stdenvNoCC callPackage;
+in {
+  # Fix for MacOS
+  mkShell = super.mkShell.override { stdenv = stdenvNoCC; };
+
+  # Various utilities
+  utils = callPackage ./tools/utils.nix { };
+  lib = (super.lib or {}) // {
+    mkFilter = callPackage ./tools/mkFilter.nix { };
+    mergeSh = callPackage ./tools/mergeSh.nix { };
+  };
+
+  # Android environement
+  androidEnvCustom = callPackage ./mobile/android/sdk { };
+  androidPkgs = self.androidEnvCustom.licensedPkgs;
+  androidShell = self.androidEnvCustom.shell;
+
+  # Package version adjustments
+  xcodeWrapper = super.xcodeenv.composeXcodeWrapper { version = "11.4.1"; };
+  openjdk = super.pkgs.openjdk8_headless;
+  nodejs = super.pkgs.nodejs-12_x;
+
+  # Custom packages
+  gomobile = callPackage ./pkgs/gomobile { };
+}

nix/patched-go/default.nix

@@ -1,22 +0,0 @@
-#
-# Patch the Go compiler so that we can have a say (using a NIX_GOWORKDIR env variable)
-# as to the temporary directory it uses for linking, since that directory path ends up
-# in the string table and .gnu.version_d ELF header.
-#
-
-{ baseGo }:
-
-let
-  go = baseGo.overrideDerivation(oldAttrs: {
-    postPatch = (oldAttrs.postPatch or "") + ''
-      substituteInPlace "src/cmd/go/internal/work/action.go" --replace \
-        'tmp, err := ioutil.TempDir(os.Getenv("GOTMPDIR"), "go-build")' \
-        'var err error
-		tmp := os.Getenv("NIX_GOWORKDIR")
-		if tmp == "" {
-			tmp, err = ioutil.TempDir(os.Getenv("GOTMPDIR"), "go-build")
-		}'
-    '';
-  });
-
-in go

nix/pkgs.nix

@@ -3,11 +3,12 @@
 { config ? { } }:
 
 let
-  inherit (import <nixpkgs> { }) fetchFromGitHub lib;
+  inherit (import <nixpkgs> { }) fetchFromGitHub;
 
   # For testing local version of nixpkgs
   #nixpkgsSrc = (import <nixpkgs> { }).lib.cleanSource "/home/jakubgs/work/nixpkgs";
 
+  # Our own nixpkgs fork with custom fixes
   nixpkgsSrc = fetchFromGitHub {
     name = "nixpkgs-source";
     owner = "status-im";
@@ -16,38 +17,21 @@ let
     sha256 = "0whwzll9lvrq4gg5j838skg7fqpvb55w4z7y44pzib32k613y2qn";
     # To get the compressed Nix sha256, use:
     # nix-prefetch-url --unpack https://github.com/${ORG}/nixpkgs/archive/${REV}.tar.gz
-    # The last line will be the hash.
   };
 
+  # Override some packages and utilities
+  pkgsOverlay = import ./overlay.nix;
+
+  # we want these packages easily available via callPackage
   defaultConfig = {
     android_sdk.accept_license = true;
     # Android Env still needs old OpenSSL
     permittedInsecurePackages = [ "openssl-1.0.2u" ];
-    # Override some package versions
-    packageOverrides = pkgs: rec {
-      inherit (pkgs) callPackage stdenv stdenvNoCC xcodeenv;
- 
-      # utilities
-      mkFilter = import ./tools/mkFilter.nix { inherit (stdenv) lib; };
-      mkShell = import ./tools/mkShell.nix { inherit pkgs; stdenv = stdenvNoCC; };
-      mergeSh = import ./tools/mergeSh.nix { inherit (stdenv) lib; };
-
-      # android environement
-      androidEnvCustom = callPackage ./mobile/android/sdk { };
-      androidPkgs = androidEnvCustom.licensedPkgs;
-      androidShell = androidEnvCustom.shell;
-
-      # custom packages
-      xcodeWrapper = xcodeenv.composeXcodeWrapper { version = "11.4.1"; };
-      openjdk = pkgs.openjdk8_headless;
-      nodejs = pkgs.nodejs-12_x;
-      yarn = pkgs.yarn.override { inherit nodejs; };
-      go = callPackage ./patched-go { baseGo = pkgs.go_1_14; };
-
-      # custom builders
-      buildGoPackage = pkgs.buildGo114Package.override { inherit go; };
-    };
   };
-  pkgs = (import nixpkgsSrc) { config = defaultConfig // config; };
+
 in
-  pkgs
+  # import nixpkgs with a config override
+  (import nixpkgsSrc) {
+    config = defaultConfig // config;
+    overlays = [ pkgsOverlay ];
+  }

nix/pkgs/gomobile/default.nix

@@ -1,4 +1,4 @@
-{ stdenv, callPackage, utils, fetchgit, buildGoPackage,
+{ stdenv, utils, callPackage, fetchgit, buildGoPackage,
   ncurses5, zlib, makeWrapper, patchelf, androidPkgs, xcodeWrapper
 }:
 

nix/pkgs/patched-go/default.nix

@@ -0,0 +1,18 @@
+# Patch the Go compiler so that we can have a say (using a NIX_GOWORKDIR env variable)
+# as to the temporary directory it uses for linking, since that directory path ends up
+# in the string table and .gnu.version_d ELF header.
+#
+
+{ go }:
+
+go.overrideDerivation (oldAttrs: {
+  postPatch = (oldAttrs.postPatch or "") + ''
+    substituteInPlace "src/cmd/go/internal/work/action.go" --replace \
+      'tmp, err := ioutil.TempDir(os.Getenv("GOTMPDIR"), "go-build")' \
+      'var err error
+      tmp := os.Getenv("NIX_GOWORKDIR")
+      if tmp == "" {
+          tmp, err = ioutil.TempDir(os.Getenv("GOTMPDIR"), "go-build")
+      }'
+  '';
+})

nix/scripts/shell.sh

@@ -3,10 +3,7 @@
 #
 # This script is used by the Makefile to have an implicit nix-shell.
 # The following environment variables modify the script behavior:
-# - TARGET: This attribute is passed as `targets` arg to Nix, limiting the scope
+# - TARGET: This attribute is passed via --attr to Nix, defining the scope.
-#     of the Nix expressions.
-# - _NIX_ATTR: Used to specify an attribute set from inside the expression in `default.nix`.
-#     This allows for drilling down into a specific attribute in Nix expressions.
 # - _NIX_PURE: This variable allows for making the shell pure with the use of --pure.
 #     Take note that this makes Nix tools like `nix-build` unavailable in the shell.
 # - _NIX_KEEP: This variable allows specifying which env vars to keep for Nix pure shell.
@@ -22,11 +19,13 @@ nixArgs=(
   "--show-trace"
 )
 
-if [[ -n "${TARGET}" ]]; then
+if [[ -z "${TARGET}" ]]; then
-    nixArgs+=("--argstr target ${TARGET}")
+    TARGET="default"
-else
+    echo -e "${YLW}Missing TARGET, assuming default target.${RST} See nix/README.md for more details." 1>&2
-    echo -e "${YLW}Env is missing TARGET, assuming default target.${RST} See nix/README.md for more details." 1>&2
 fi
+entryPoint="default.nix"
+nixArgs+=("--attr shells.${TARGET}")
+
 
 if [[ "$TARGET" =~ (linux|windows|darwin|macos) ]]; then
   # This is a dirty workaround because 'yarn install' is an impure operation,
@@ -48,29 +47,24 @@ if [ -n "$config" ]; then
   nixArgs+=("--arg config {$config}")
 fi
 
-# if _NIX_ATTR is specified we shouldn't use shell.nix, the path will be different
+# This variable allows specifying which env vars to keep for Nix pure shell
-entryPoint="shell.nix"
+# The separator is a colon
-if [ -n "${_NIX_ATTR}" ]; then
+if [[ -n "${_NIX_KEEP}" ]]; then
-  nixArgs+=("--attr ${_NIX_ATTR}")
+  nixArgs+=("--keep ${_NIX_KEEP//,/ --keep }")
-  entryPoint="default.nix"
 fi
 
+# Not all builds are ready to be run in a pure environment
+if [[ -n "${_NIX_PURE}" ]]; then
+  nixArgs+=("--pure")
+  pureDesc='pure '
+fi
+
+echo -e "${GRN}Configuring ${pureDesc}Nix shell for target '${TARGET}'...${RST}" 1>&2
+
 # ENTER_NIX_SHELL is the fake command used when `make shell` is run.
 # It is just a special string, not a variable, and a marker to not use `--run`.
-if [[ $@ == "ENTER_NIX_SHELL" ]]; then
+if [[ "${@}" == "ENTER_NIX_SHELL" ]]; then
-  echo -e "${GRN}Configuring ${_NIX_ATTR:-default} Nix shell for target '${TARGET:-default}'...${RST}" 1>&2
   exec nix-shell ${nixArgs[@]} ${entryPoint}
 else
-  # Not all builds are ready to be run in a pure environment
-  if [[ -n "${_NIX_PURE}" ]]; then
-    nixArgs+=("--pure")
-    pureDesc='pure '
-  fi
-  # This variable allows specifying which env vars to keep for Nix pure shell
-  # The separator is a colon
-  if [[ -n "${_NIX_KEEP}" ]]; then
-    nixArgs+=("--keep ${_NIX_KEEP//,/ --keep }")
-  fi
-  echo -e "${GRN}Configuring ${pureDesc}${_NIX_ATTR:-default} Nix shell for target '${TARGET}'...${RST}" 1>&2
   exec nix-shell ${nixArgs[@]} --run "$@" ${entryPoint}
 fi

nix/shells.nix

@@ -6,30 +6,16 @@
 }:
 
 let
+  inherit (pkgs) lib stdenv;
+
   # everything else we define in nix/ dir
   targets = pkgs.callPackage ./targets.nix { inherit config; };
 
-  # for calling lein targets in CI or Makefile
-  leiningen-sh = pkgs.mkShell {
-    buildInputs = with pkgs; [ clojure leiningen flock maven nodejs openjdk ];
-  };
-
-  # for 'make watchman-clean'
-  watchman-sh = pkgs.mkShell {
-    buildInputs = [ pkgs.watchman ];
-  };
-
-  # for running fastlane commands alone
-  fastlane-sh = targets.mobile.fastlane.shell;
-
-  # for 'scripts/generate-keystore.sh'
-  keytool-sh = pkgs.mkShell {
-    buildInputs = [ pkgs.openjdk8 ];
-  };
-
   # the default shell that is used when target is not specified
+  # it is also merged with all the other shells
   default = pkgs.mkShell {
     name = "status-react-shell"; # for identifying all shells
+
     buildInputs = with pkgs; lib.unique ([
       # core utilities that should always be present in a shell
       bash curl wget file unzip flock git gnumake jq ncurses
@@ -42,30 +28,62 @@ let
       ++ lib.optionals (!stdenv.isDarwin) [ gcc8 ]
     );
 
+    # avoid terinal issues
+    TERM="xterm";
+
+    # default locale
+    LANG="en_US.UTF-8";
+    LANGUAGE="en_US.UTF-8";
+
     # just a nicety for easy access to node scripts
     shellHook = ''
+      export STATUS_REACT_HOME=$(git rev-parse --show-toplevel)
       export PATH="$STATUS_REACT_HOME/node_modules/.bin:$PATH"
     '';
   };
 
-# values here can be selected using `nix-shell --argstr target $TARGET`
+  # An attrset for easier merging with default shell
+  shells = rec {
+    inherit default;
+
+    # for calling lein targets in CI or Makefile
+    lein = pkgs.mkShell {
+      buildInputs = with pkgs; [ clojure leiningen flock maven nodejs openjdk ];
+    };
+
+    # for 'make watchman-clean'
+    watchman = pkgs.mkShell {
+      buildInputs = [ pkgs.watchman ];
+    };
+
+    # for running fastlane commands alone
+    fastlane = targets.mobile.fastlane.shell;
+
+    # for 'scripts/generate-keystore.sh'
+    keytool = pkgs.mkShell {
+      buildInputs = [ pkgs.openjdk8 ];
+    };
+
+    # for targets that need 'adb'
+    android-env = targets.mobile.android.env.shell;
+
+    # helpers for use with target argument
+    linux = targets.desktop.linux.shell;
+    macos = targets.desktop.macos.shell;
+    windows = targets.desktop.windows.shell;
+    android = targets.mobile.android.shell;
+    ios = targets.mobile.ios.shell;
+
+    # all shells together depending on host OS
+    all = lib.mergeSh (pkgs.mkShell {}) (lib.unique (
+      lib.optionals stdenv.isLinux  [ android linux windows ] ++
+      lib.optionals stdenv.isDarwin [ android macos ios ]
+    ));
+  };
+
+  # for merging the default shell with others
+  mergeDefaultShell = (name: value: lib.mergeSh default [ value ]);
+
+# values here can be selected using `nix-shell --attr shells.$TARGET default.nix`
 # the nix/scripts/shell.sh wrapper does this for us and expects TARGET to be set
-in with pkgs; rec {
+in lib.mapAttrs mergeDefaultShell shells
-  inherit default;
-  lein = leiningen-sh;
-  watchman = watchman-sh;
-  fastlane = fastlane-sh;
-  keytool = keytool-sh;
-  android-env = targets.mobile.android.env.shell;
-  # helpers for use with target argument
-  linux = targets.desktop.linux.shell;
-  macos = targets.desktop.macos.shell;
-  windows = targets.desktop.windows.shell;
-  android = targets.mobile.android.shell;
-  ios = targets.mobile.ios.shell;
-  # all shells together depending on host OS
-  all = mergeSh (mkShell {}) (lib.unique (
-    lib.optionals stdenv.isLinux  [ android linux windows ] ++
-    lib.optionals stdenv.isDarwin [ android macos ios ]
-  ));
-}

nix/status-go/default.nix

@@ -1,5 +1,5 @@
-{ config, stdenv, callPackage, mkShell, mergeSh,
+{ config, lib, utils, stdenv, callPackage, mkShell,
-  fetchFromGitHub, mkFilter, openjdk, androidPkgs }:
+  fetchFromGitHub, openjdk, androidPkgs }:
 
 let
   inherit (stdenv.lib)
@@ -9,20 +9,13 @@ let
 
   envFlags = callPackage ../tools/envParser.nix { };
   enableNimbus = (attrByPath ["STATUS_GO_ENABLE_NIMBUS"] "0" envFlags) != "0";
-  utils = callPackage ./utils.nix { };
-
-  gomobile = callPackage ./gomobile { inherit utils; };
 
   nimbus =
     if enableNimbus then callPackage ./nimbus { }
     else { wrappers-android = { }; };
 
-  buildStatusGoDesktopLib = callPackage ./desktop {
+  buildStatusGoDesktopLib = callPackage ./desktop { };
-    inherit utils;
+  buildStatusGoMobileLib = callPackage ./mobile { };
-  };
-  buildStatusGoMobileLib = callPackage ./mobile {
-    inherit gomobile utils androidPkgs;
-  };
 
   srcData =
     # If config.status-im.status-go.src-override is defined, instruct Nix to use that path to build status-go
@@ -41,7 +34,7 @@ let
             name = "${repo}-source-${shortRev}";
             filter =
               # Keep this filter as restrictive as possible in order to avoid unnecessary rebuilds and limit closure size
-              mkFilter {
+              lib.mkFilter {
                 root = path;
                 include = [ ".*" ];
                 exclude = [
@@ -193,7 +186,7 @@ let
   platforms = [ android ios desktop ];
 
 in {
-  shell = mergeSh mkShell {} (catAttrs "shell" platforms);
+  shell = lib.mergeSh mkShell {} (catAttrs "shell" platforms);
 
   # CHILD DERIVATIONS
   inherit android ios desktop;

nix/status-go/mobile/default.nix

@@ -1,5 +1,6 @@
 { stdenv, utils, callPackage,
-  buildGoPackage, go, gomobile, androidPkgs, openjdk, unzip, zip, xcodeWrapper }:
+  buildGoPackage, go, gomobile, androidPkgs,
+  openjdk, unzip, zip, xcodeWrapper }:
 
 { owner, repo, rev, cleanVersion, goPackagePath, src, host,
 

nix/tools/envParser.nix

@@ -29,8 +29,8 @@ let
   envFileName =
     if build-type == "release" then ../../.env.release else
     if build-type == "nightly" then ../../.env.nightly else
-    if build-type == "e2e" then ../../.env.e2e else
+    if build-type == "e2e"     then ../../.env.e2e else
-    if ci then ../../.env.jenkins else ../../.env;
+    if build-type == "pr"      then ../../.env.jenkins else ../../.env;
   flags = readFlagsFromFile envFileName; # TODO: Simplify this path search with lib.locateDominatingFile
 
 in flags

scripts/release-android.sh

@@ -31,8 +31,10 @@ if [ -n "${NIMBUS_SRC_OVERRIDE}" ]; then
   config+="status-im.nimbus.src-override=\"${NIMBUS_SRC_OVERRIDE}\";"
 fi
 config+="status-im.build-type=\"$(must_get_env BUILD_TYPE)\";"
-config+="status-im.status-react.build-number=\"$(must_get_env BUILD_NUMBER)\";"
+config+="status-im.build-number=\"$(must_get_env BUILD_NUMBER)\";"
-config+="status-im.status-react.keystore-file=\"$(must_get_env KEYSTORE_PATH)\";"
+config+="status-im.android.keystore-path=\"$(must_get_env KEYSTORE_PATH)\";"
+config+="status-im.android.abi-split=\"$(must_get_env ANDROID_ABI_SPLIT)\";"
+config+="status-im.android.abi-include=\"$(must_get_env ANDROID_ABI_INCLUDE)\";"
 nixOpts=()
 
 # Secrets like this can't be passed via args or they end up in derivation
@@ -42,9 +44,8 @@ trap "rm -f ${SECRETS_FILE_PATH}" EXIT
 append_env_export 'KEYSTORE_PASSWORD'
 append_env_export 'KEYSTORE_ALIAS'
 append_env_export 'KEYSTORE_KEY_PASSWORD'
-nixOpts+=(
+nixOpts+=("--argstr" "secretsFile" "${SECRETS_FILE_PATH}")
-  "--argstr" "secrets-file" "${SECRETS_FILE_PATH}"
+nixOpts+=("--argstr" "buildEnv" "$(must_get_env BUILD_ENV)")
-)
 
 if [[ "$(uname -s)" =~ Darwin ]]; then
   # Start a watchman instance if not started already and store its socket path.
@@ -61,9 +62,6 @@ else
   )
 fi
 
-nixOpts+=(
+nixOpts+=("--arg" "config" "{${config}}")
-  "--arg" "config" "{${config}}"
-  "--arg" "env" "{BUILD_ENV=\"${BUILD_ENV}\";ANDROID_ABI_SPLIT=\"${ANDROID_ABI_SPLIT}\";ANDROID_ABI_INCLUDE=\"${ANDROID_ABI_INCLUDE}\";}"
-)
 
 ${GIT_ROOT}/nix/scripts/build.sh targets.mobile.android.release "${nixOpts[@]}"

shell.nix

@@ -1,11 +1,8 @@
-{
+# for passing build options, see nix/README.md
-  config ? { },      # for passing build options, see nix/README.md
+{ config ? { } }:
-  target ? "default" # see nix/shells.nix for all valid values
-}:
 
 let
   project = import ./default.nix { inherit config; };
 in
-  # this is where the $TARGET env variable affects things
+  # we use the shell combining most shells as default
-  project.pkgs.mergeSh project.shells.default [ project.shells.${target} ]
+  project.shells.default
-  # combining with default shell to include all the standard utilities