diff --git a/default.nix b/default.nix index 66742e2cfb..3b2898217c 100644 --- a/default.nix +++ b/default.nix @@ -1,10 +1,10 @@ # for passing build options, see nix/README.md -{ config ? { status-im = { }; } }: +{ config ? { } }: let - main = import ./nix/default.nix { inherit config; }; + main = import ./nix { inherit config; }; in { # this is where the --attr argument selects the shell or target inherit (main) pkgs targets shells; - inherit config; + inherit (main.pkgs) config; } diff --git a/nix/DETAILS.md b/nix/DETAILS.md index 9769160236..49f5565d2a 100644 --- a/nix/DETAILS.md +++ b/nix/DETAILS.md @@ -1,30 +1,43 @@ # Description -This document descripts the layout of our Nix setup. +This document describes the layout of our Nix setup. # Folders There are four main folders in the `nix` directory: -* `nix/scripts` - Bash scripts for easier usage of Nix -* `nix/pkgs` - Packages we add to or modify in `nixpkgs` -* `nix/deps` - Project dependencies managed by Nix -* `nix/lib` - Our tools we merge into `pkgs.lib` -* `nix/tools` - Various tools used by our derivations and shells -* `nix/status-go` - Derivations for building [`status-go`](https://github.com/status-im/status-go) repo +* [`nix/scripts`](./scripts) - Bash scripts for easier usage of Nix +* [`nix/pkgs`](./pkgs) - Packages we add to or modify in `nixpkgs` +* [`nix/deps`](./deps) - Project dependencies managed by Nix +* [`nix/lib`](./lib) - Our tools we merge into `pkgs.lib` +* [`nix/tools`](./tools) - Various tools used by our derivations and shells +* [`nix/status-go`](./status-go) - Derivations for building [`status-go`](https://github.com/status-im/status-go) repo # Files There are a few main files that define the whole build environment: -* `nix/nix.conf` - Binary cache configuration -* `nix/default.nix` - Entrypoint for both shells and targets -* `nix/shells.nix` - Definition of Nix shells used in builds -* `nix/targets.nix` - Hierarchy of main build targets -* `nix/pkgs.nix` - Definition of a custom `nixpkgs` repo -* `nix/overlay.nix` - Overrides for `nixpkgs`, custom packages +* [`nix/nix.conf`](./nix.conf) - Binary cache configuration +* [`nix/default.nix`](./default.nix) - Entry point for both shells and targets +* [`nix/shell.nix`](./shell.nix) - Definition of the default Nix shell +* [`nix/shells.nix`](./shells.nix) - Definitions of other Nix shells used in builds +* [`nix/targets.nix`](./targets.nix) - Hierarchy of main build targets +* [`nix/pkgs.nix`](./pkgs.nix) - Definition of a custom `nixpkgs` repo +* [`nix/config.nix`](./config.nix) - Default config values for instantiating `nixpkgs` +* [`nix/overlay.nix`](./overlay.nix) - Overrides for `nixpkgs`, custom packages -The `default.nix` and `shell.nix` files at th repo root are just a gateway into the `nix` subfolder. +The [`default.nix`](../default.nix) and [`shell.nix`](../shell.nix) files at th repo root are just a gateway into the `nix` sub folder. + +# Scripts + +There's a few scripts in [`nix/scripts`](./scripts) that make use of Nix simpler: + +* [`nix/scripts/setup.sh`](./scripts/setup.sh) - Installs Nix Package manager +* [`nix/scripts/source.sh`](./scripts/source.sh) - Sources the Nix profile or installs Nix +* [`nix/scripts/build.sh`](./scripts/build.sh) - A wrapper around `nix-build` with sane defaults +* [`nix/scripts/shell.sh`](./scripts/shell.sh) - A wrapper around `nix-shell` for `Makefile` +* [`nix/scripts/clean.sh`](./scripts/clean.sh) - For cleaning Nix store after builds +* [`nix/scripts/purge.sh`](./scripts/purge.sh) - For purging everything Nix related from system # Start @@ -32,7 +45,15 @@ The starting point for using our Nix shells and targets is the [`default.nix`](/ It pulls in all the `pkgs`, `targets` and `shells` defined in [`nix/default.nix`](/nix/default.nix). The point is easy access to them via commands like `nix-build` or `nix-shell`, which you'll see next. -# Usage +# Shells + +Normally shells are started using `make shell TARGET=android`, but that is essentially the same as calling: +```bash +nix-shell -A shells.android default.nix +``` +The [`nix/scripts/shell.sh`](./scripts/shell.sh) script is essentially a wrapper around that command to make it usable as shell for the `Makefile`. + +# Building We will use the `make jsbundle-android` target as an example of a derivation you can build using Nix: diff --git a/nix/README.md b/nix/README.md index 67171f2e9a..a58972c713 100644 --- a/nix/README.md +++ b/nix/README.md @@ -10,26 +10,26 @@ __NOTE:__ If you are in Asia you might want to add the `https://nix-cache-cn.sta ## Build arguments -We leverage the standard nixpkgs `config` argument for our own parameterization of the builds (e.g. to pass a build number or build type). +We leverage the `config` argument of standard `nixpkgs` for our own parameterization of the builds (e.g. to pass a build number or build type). Here is a sample structure of the `config` attribute set: ```nix -config = { +{ status-im = { - build-type = "pr"; # Build type (influences which .env file gets used for feature flags) - build-number = 9999; # Used for versionCode and CFBundleVersion in Android and iOS respectively + build-type = "pr"; # Build type (influences which .env file gets used for feature flags) + build-number = 9999; # Used for versionCode and CFBundleVersion in Android and iOS respectively android = { - gradle-opts = ""; # Gradle options passed for Android builds - keystore-path = ""; # Path to keystore for signing the APK - abi-split = false; # If APKs should be split based on architectures - abi-include = "x86"; # Android architectures to build for + gradle-opts = ""; # Gradle options passed for Android builds + keystore-path = ""; # Path to keystore for signing the APK + abi-split = false; # If APKs should be split based on architectures + abi-include = "x86"; # Android architectures to build for }; status-go = { src-override = "$HOME/my/source/status-go"; # local source override }; }; -}; +} ``` You can see the defaults in [`nix/config.nix`](./config.nix). @@ -59,6 +59,15 @@ export STATUS_GO_SRC_OVERRIDE=$GOPATH/src/github.com/status-im/status-go make release-android ``` +## Resources + +You can learn more about Nix by watching these presentations: + +* [Nix Fundamentals](https://www.youtube.com/watch?v=m4sv2M9jRLg) ([PDF](https://drive.google.com/file/d/1Tt5R7QOubudGiSuZIGxuFWB1OYgcThcL/view?usp=sharing), [src](https://github.com/status-im/infra-docs/tree/master/presentations/nix_basics)) +* [Nix in Status](https://www.youtube.com/watch?v=rEQ1EvRG8Wc) ([PDF](https://drive.google.com/file/d/1Ti0wppMoj40icCPdHy7mJcQj__DeaYBE/view?usp=sharing), [src](https://github.com/status-im/infra-docs/tree/master/presentations/nix_in_status)) + +And you can read [`nix/DETAILS.md`](./DETAILS.md) for more information. + ## Known Issues See [`KNOWN_ISSUES.md`](./KNOWN_ISSUES.md). diff --git a/nix/lib/getConfig.nix b/nix/lib/getConfig.nix index 73d6a1e5b2..5b11685a4c 100644 --- a/nix/lib/getConfig.nix +++ b/nix/lib/getConfig.nix @@ -1,12 +1,11 @@ -# # helper for getting status-im config values in a safe way # { lib, config }: +let inherit (lib) splitString attrByPath; +in name: default: let - inherit (lib) splitString attrByPath; -in - name: default: - let path = [ "status-im" ] ++ (splitString "." name); - in attrByPath path default config + path = [ "status-im" ] ++ (splitString "." name); + value = attrByPath path default config; +in if value != null then value else default diff --git a/nix/mobile/android/release.nix b/nix/mobile/android/release.nix index b6511ad4bf..9deccb40a5 100644 --- a/nix/mobile/android/release.nix +++ b/nix/mobile/android/release.nix @@ -16,7 +16,7 @@ assert (lib.stringLength watchmanSockPath) > 0 -> stdenv.isDarwin; let inherit (lib) - toLower optionalString stringLength + toLower optionalString stringLength assertMsg getConfig makeLibraryPath assertEnvVarSet; buildType = getConfig "build-type" "prod"; @@ -82,7 +82,7 @@ in stdenv.mkDerivation rec { phases = [ "unpackPhase" "secretsPhase" "secretsCheckPhase" - "buildPhase" "checkPhase" "installPhase" + "keystorePhase" "buildPhase" "checkPhase" "installPhase" ]; unpackPhase = '' @@ -90,13 +90,7 @@ in stdenv.mkDerivation rec { chmod u+w -R ./ runHook postUnpack ''; - postUnpack = - assert lib.assertMsg (keystorePath != null) "keystore-file has to be set!"; - '' - # Keep the same keystore path for determinism - export KEYSTORE_PATH="$PWD/status-im.keystore" - cp -a --no-preserve=ownership "${keystorePath}" "$KEYSTORE_PATH" - + postUnpack = '' # Ensure we have the right .env file cp -f ./${envFileName} ./.env @@ -118,10 +112,19 @@ in stdenv.mkDerivation rec { # Patch build.gradle to use local repo ${patchMavenSources} ./android/build.gradle ''; + # if secretsFile is not set we use generate keystore secretsPhase = if (secretsFile != "") then '' source "${secretsFile}" '' else keystore.shellHook; + + # if keystorePath is set copy it into build directory + keystorePhase = + assert assertMsg (keystorePath != null) "keystorePath has to be set!"; + '' + export KEYSTORE_PATH="$PWD/status-im.keystore" + cp -a --no-preserve=ownership "${keystorePath}" "$KEYSTORE_PATH" + ''; secretsCheckPhase = '' ${assertEnvVarSet "KEYSTORE_ALIAS"} ${assertEnvVarSet "KEYSTORE_PASSWORD"} diff --git a/nix/shell.nix b/nix/shell.nix new file mode 100644 index 0000000000..f77b539b38 --- /dev/null +++ b/nix/shell.nix @@ -0,0 +1,39 @@ +# +# Defines the default shell that is used when target is not specified. +# It is also merged with all the other shells for a complete set of tools. +# +{ config ? {} +, pkgs ? import ./pkgs.nix { inherit config; } }: + +let + inherit (pkgs) mkShell; +in mkShell { + name = "status-react-shell"; # for identifying all shells + + buildInputs = with pkgs; lib.unique ([ + # core utilities that should always be present in a shell + bash curl wget file unzip flock + git gnumake jq ncurses gnugrep parallel + lsof # used in start-react-native.sh + # build specific utilities + clojure maven watchman + # other nice to have stuff + yarn nodejs python27 + ] # and some special cases + ++ lib.optionals stdenv.isDarwin [ cocoapods clang ] + ++ lib.optionals (!stdenv.isDarwin) [ gcc8 ] + ); + + # avoid terminal issues + TERM="xterm"; + + # default locale + LANG="en_US.UTF-8"; + LANGUAGE="en_US.UTF-8"; + + # just a nicety for easy access to node scripts + shellHook = '' + export STATUS_REACT_HOME=$(git rev-parse --show-toplevel) + export PATH="$STATUS_REACT_HOME/node_modules/.bin:$PATH" + ''; +} diff --git a/nix/shells.nix b/nix/shells.nix index 9c1ad231de..76a60d55a4 100644 --- a/nix/shells.nix +++ b/nix/shells.nix @@ -1,48 +1,16 @@ # This file defines custom shells as well as shortcuts # for accessing more nested shells. -{ - config ? {}, - pkgs ? import ./pkgs.nix { inherit config; } -}: +{ config ? {} +, pkgs ? import ./pkgs.nix { inherit config; } }: let - inherit (pkgs) lib stdenv mkShell; + inherit (pkgs) lib mkShell callPackage; # everything else we define in nix/ dir - targets = pkgs.callPackage ./targets.nix { inherit config; }; + targets = callPackage ./targets.nix { inherit config; }; - # the default shell that is used when target is not specified - # it is also merged with all the other shells - default = mkShell { - name = "status-react-shell"; # for identifying all shells - - buildInputs = with pkgs; lib.unique ([ - # core utilities that should always be present in a shell - bash curl wget file unzip flock - git gnumake jq ncurses gnugrep parallel - lsof # used in start-react-native.sh - # build specific utilities - clojure maven watchman - # other nice to have stuff - yarn nodejs python27 - ] # and some special cases - ++ lib.optionals stdenv.isDarwin [ cocoapods clang ] - ++ lib.optionals (!stdenv.isDarwin) [ gcc8 ] - ); - - # avoid terinal issues - TERM="xterm"; - - # default locale - LANG="en_US.UTF-8"; - LANGUAGE="en_US.UTF-8"; - - # just a nicety for easy access to node scripts - shellHook = '' - export STATUS_REACT_HOME=$(git rev-parse --show-toplevel) - export PATH="$STATUS_REACT_HOME/node_modules/.bin:$PATH" - ''; - }; + # the default shell with most commonly used tools + default = callPackage ./shell.nix { }; # Combines with many other shells node-sh = mkShell { @@ -55,6 +23,8 @@ let # An attrset for easier merging with default shell shells = { + inherit default; + nodejs = node-sh; # for calling clojure targets in CI or Makefile @@ -106,6 +76,4 @@ let # values here can be selected using `nix-shell --attr shells.$TARGET default.nix` # the nix/scripts/shell.sh wrapper does this for us and expects TARGET to be set -in { - inherit default; -} // lib.mapAttrs mergeDefaultShell shells +in lib.mapAttrs mergeDefaultShell shells diff --git a/shell.nix b/shell.nix index d9030a6d62..fb0162b130 100644 --- a/shell.nix +++ b/shell.nix @@ -2,7 +2,7 @@ { config ? { } }: let - project = import ./default.nix { inherit config; }; + main = import ./nix { inherit config; }; in - # we use the shell combining most shells as default - project.shells.default + # use the default shell when calling nix-shell without arguments + main.shells.default