diff --git a/Makefile b/Makefile index 8337f8e4b4..e438f52c9a 100644 --- a/Makefile +++ b/Makefile @@ -69,6 +69,10 @@ else @echo "${YELLOW}Nix shell is already active$(RESET)" endif +nix-repl: SHELL := /bin/sh +nix-repl: ##@nix Start an interactive Nix REPL + nix repl default.nix + nix-gc: export TARGET := default nix-gc: ##@nix Garbage collect all packages older than 20 days from /nix/store nix-collect-garbage --delete-old --delete-older-than 20d diff --git a/ci/Jenkinsfile.android b/ci/Jenkinsfile.android index 583bc24f34..74e1293b5d 100644 --- a/ci/Jenkinsfile.android +++ b/ci/Jenkinsfile.android @@ -1,4 +1,4 @@ -library 'status-react-jenkins@master' +library 'status-react-jenkins@v1.1.1' pipeline { agent { label 'linux' } diff --git a/ci/Jenkinsfile.combined b/ci/Jenkinsfile.combined index 57a6d033c1..eccfa517eb 100644 --- a/ci/Jenkinsfile.combined +++ b/ci/Jenkinsfile.combined @@ -1,4 +1,4 @@ -library 'status-react-jenkins@master' +library 'status-react-jenkins@v1.1.1' pipeline { agent { label 'linux' } diff --git a/ci/Jenkinsfile.fastlane.clean b/ci/Jenkinsfile.fastlane.clean index 6435dec75e..729c277cf7 100644 --- a/ci/Jenkinsfile.fastlane.clean +++ b/ci/Jenkinsfile.fastlane.clean @@ -1,4 +1,4 @@ -library 'status-react-jenkins@master' +library 'status-react-jenkins@v1.1.1' pipeline { agent { label 'macos' } diff --git a/ci/Jenkinsfile.ios b/ci/Jenkinsfile.ios index 4305d8a657..b298da1c9a 100644 --- a/ci/Jenkinsfile.ios +++ b/ci/Jenkinsfile.ios @@ -1,4 +1,4 @@ -library 'status-react-jenkins@master' +library 'status-react-jenkins@v1.1.1' pipeline { agent { label 'macos-xcode-11.4.1' } diff --git a/ci/Jenkinsfile.linux b/ci/Jenkinsfile.linux index efb1416d80..c4a9a61928 100644 --- a/ci/Jenkinsfile.linux +++ b/ci/Jenkinsfile.linux @@ -1,4 +1,4 @@ -library 'status-react-jenkins@master' +library 'status-react-jenkins@v1.1.1' pipeline { agent { label 'linux' } diff --git a/ci/Jenkinsfile.macos b/ci/Jenkinsfile.macos index be792e5236..7d0306b985 100644 --- a/ci/Jenkinsfile.macos +++ b/ci/Jenkinsfile.macos @@ -1,4 +1,4 @@ -library 'status-react-jenkins@master' +library 'status-react-jenkins@v1.1.1' pipeline { agent { label 'macos-xcode-11.4.1' } diff --git a/ci/Jenkinsfile.nix-cache b/ci/Jenkinsfile.nix-cache index 4e496a792f..bc66122879 100644 --- a/ci/Jenkinsfile.nix-cache +++ b/ci/Jenkinsfile.nix-cache @@ -1,4 +1,4 @@ -library 'status-react-jenkins@master' +library 'status-react-jenkins@v1.1.1' pipeline { agent { label params.AGENT_LABEL } diff --git a/ci/Jenkinsfile.windows b/ci/Jenkinsfile.windows index b3b7c1d93a..997756278a 100644 --- a/ci/Jenkinsfile.windows +++ b/ci/Jenkinsfile.windows @@ -1,4 +1,4 @@ -library 'status-react-jenkins@master' +library 'status-react-jenkins@v1.1.1' pipeline { agent { label 'linux' } diff --git a/default.nix b/default.nix index 8d6380b48f..0d44f5e58e 100644 --- a/default.nix +++ b/default.nix @@ -1,9 +1,9 @@ -{ - config ? { status-im = { build-type = ""; }; }, # for passing build options, see nix/README.md -}: +# for passing build options, see nix/README.md +{ config ? { status-im = { build-type = ""; }; } }: let main = import ./nix/default.nix { inherit config; }; in { + # this is where the --attr argument selects the shell or target inherit (main) pkgs targets shells; } diff --git a/nix/DETAILS.md b/nix/DETAILS.md new file mode 100644 index 0000000000..f273f4ce17 --- /dev/null +++ b/nix/DETAILS.md @@ -0,0 +1,65 @@ +# Description + +This document descripts the layout of our Nix setup. + +# Folders + +There are four main folders in the `nix` directory: + +* `nix/scripts` - Bash scripts for easier usage of Nix +* `nix/pkgs` - Packages we add to or modify in `nixpkgs` +* `nix/tools` - Various tools used by our derivations and shells +* `nix/status-go` - Derivations for building [`status-go`](https://github.com/status-im/status-go) repo + +# Files + +There are a few main files that define the whole build environment: + +* `nix/default.nix` - Entrypoint for both shells and targets +* `nix/shells.nix` - Definition of Nix shells used in builds +* `nix/targets.nix` - Hierarchy of main build targets +* `nix/pkgs.nix` - Definition of a custom `nixpkgs` repo +* `nix/overlay.nix` - Overrides for `nixpkgs`, custom packages + +# Start + +The starting point for using our Nix shells and targets is the [`default.nix`](/default.nix) file. + +It pulls in all the `pkgs`, `targets` and `shells` defined in [`nix/default.nix`](/nix/default.nix). The point is easy access to them via commands like `nix-build` or `nix-shell`, which you'll see next. + +# Usage + +We will use the `make jsbundle-android` target as an example of a derivation you can build using Nix: + +1. `make jsbundle-android` is called by developer +2. `make` calls `nix/scripts/build.sh targets.mobile.android.jsbundle` +3. [`build.sh`](/nix/scripts/build.sh) calls `nix-build --attr targets.mobile.android.jsbundle` with extra arguments +4. `nix-build` builds the derivation from [`nix/mobile/android/jsbundle/default.nix`](/nix/mobile/android/jsbundle/default.nix) + +The same can be done for other targets like `targets.mobile.android.release`. +Except in that case extra arguments are required which is why the [`scripts/release-android.sh`](/scripts/release-android.sh) is used in the `make release-android` target. + +If you run `make release-android` you'll see the `nix-build` command used: +``` +nix-build \ + --pure \ + --fallback \ + --no-out-link \ + --show-trace \ + --attr targets.mobile.android.release \ + --argstr secrets-file '/tmp/tmp-status-react-559a3a441/tmp.xAnrPuNtAP' \ + --option extra-sandbox-paths '/home/sochan/.gradle/status-im.keystore /tmp/tmp-status-react-559a3a441/tmp.xAnrPuNtAP' \ + --arg config '{ \ + status-im.build-type="nightly"; + status-im.build-number="2020022418"; + status-im.android.keystore-file="/home/sochan/.gradle/status-im.keystore"; + status-im.android.abi-split="false"; + status-im.android.abi-include="armeabi-v7a;arm64-v8a;x86"; + }' \ + default.nix +``` +Some of those are required which is why just calling: +``` +nix-build --attr targets.mobile.android.release +``` +Would fail. diff --git a/nix/mobile/android/default.nix b/nix/mobile/android/default.nix index 3ed7a50b2e..9438ed2b42 100644 --- a/nix/mobile/android/default.nix +++ b/nix/mobile/android/default.nix @@ -1,4 +1,4 @@ -{ config, lib, callPackage, mkShell, mergeSh, flock, lsof, openjdk, gradle_5, +{ config, lib, callPackage, mkShell, flock, lsof, openjdk, gradle_5, status-go, localMavenRepoBuilder, projectNodePackage, androidPkgs, androidShell }: let @@ -6,7 +6,7 @@ let leinProjectDeps = import ../../lein/lein-project-deps.nix { }; # Import a jsbundle compiled out of clojure codebase - jsbundle = callPackage ./jsbundle/default.nix { + jsbundle = callPackage ./jsbundle { inherit leinProjectDeps localMavenRepoBuilder projectNodePackage; }; @@ -37,7 +37,7 @@ in { # TARGETS inherit release jsbundle generate-maven-and-npm-deps-shell buildInputs; - shell = mergeSh + shell = lib.mergeSh (mkShell { inherit buildInputs; inputsFrom = [ release gradle ]; diff --git a/nix/mobile/android/jsbundle/default.nix b/nix/mobile/android/jsbundle/default.nix index 6c5b44f657..5f70879261 100644 --- a/nix/mobile/android/jsbundle/default.nix +++ b/nix/mobile/android/jsbundle/default.nix @@ -3,7 +3,7 @@ # { target-os ? "android", - stdenv, mkFilter, clojure, leiningen, nodejs, bash, git, + stdenv, lib, clojure, leiningen, nodejs, bash, git, leinProjectDeps, localMavenRepoBuilder, projectNodePackage }: let @@ -19,7 +19,7 @@ in stdenv.mkDerivation { name = "status-react-source-jsbundle"; filter = # Keep this filter as restrictive as possible in order to avoid unnecessary rebuilds and limit closure size - mkFilter { + lib.mkFilter { root = path; ignoreVCS = false; include = [ diff --git a/nix/mobile/android/maven-and-npm-deps/default.nix b/nix/mobile/android/maven-and-npm-deps/default.nix index 8f5bc5ee2b..e33b40a362 100644 --- a/nix/mobile/android/maven-and-npm-deps/default.nix +++ b/nix/mobile/android/maven-and-npm-deps/default.nix @@ -5,7 +5,7 @@ { stdenv, lib, callPackage, mkShell, gradle, bash, file, nodejs, zlib, - projectNodePackage, localMavenRepoBuilder, mkFilter }: + projectNodePackage, localMavenRepoBuilder }: let mavenLocalRepo = callPackage ./maven { inherit localMavenRepoBuilder stdenv; }; @@ -36,7 +36,7 @@ let name = "status-react-source-gradle-install"; filter = # Keep this filter as restrictive as possible in order to avoid unnecessary rebuilds and limit closure size - mkFilter { + lib.mkFilter { root = path; include = [ "android/.*" "translations/.*" "status-modules/.*" diff --git a/nix/mobile/android/targets/release-android.nix b/nix/mobile/android/targets/release-android.nix index 32948d3d8d..95086c7276 100644 --- a/nix/mobile/android/targets/release-android.nix +++ b/nix/mobile/android/targets/release-android.nix @@ -1,61 +1,85 @@ -{ stdenv, lib, config, callPackage, - mkFilter, bash, file, gnumake, watchmanFactory, gradle, - androidPkgs, mavenAndNpmDeps, - nodejs, openjdk, jsbundle, status-go, unzip, zlib }: +{ stdenv, lib, config, callPackage, bash, file, gnumake, watchmanFactory, gradle +, androidPkgs, mavenAndNpmDeps, nodejs, openjdk, jsbundle, status-go, unzip, zlib }: -{ secrets-file ? "", # Path to the file containing secret environment variables +{ + buildEnv ? "prod", # Value for BUILD_ENV checked by Clojure code at compile time + secretsFile ? "", # Path to the file containing secret environment variables watchmanSockPath ? "", # Path to the socket file exposed by an external watchman instance (workaround needed for building Android on macOS) - env ? {} # Attribute set containing environment variables to expose to the build script }: -assert (builtins.stringLength watchmanSockPath) > 0 -> stdenv.isDarwin; +assert (lib.stringLength watchmanSockPath) > 0 -> stdenv.isDarwin; let - inherit (lib) attrByPath hasAttrByPath optionalAttrs; - env' = env // optionalAttrs (hasAttrByPath ["status-im" "status-go" "src-override"] config) { - STATUS_GO_SRC_OVERRIDE = config.status-im.status-go.src-override; - } // optionalAttrs (hasAttrByPath ["status-im" "nimbus" "src-override"] config) { - NIMBUS_SRC_OVERRIDE = config.status-im.nimbus.src-override; + inherit (lib) + toLower splitString optionalString + attrByPath hasAttrByPath optionalAttrs; + + # helper for getting config values + safeGetConfig = name: default: + let path = [ "status-im" ] ++ (splitString "." name); + in attrByPath path default config; + + # custom env variables derived from config + env = { + ANDROID_ABI_SPLIT = safeGetConfig "android.abi-split" false; + ANDROID_ABI_INCLUDE = safeGetConfig "android.abi-include" "armeabi-v7a;arm64-v8a;x86"; + STATUS_GO_SRC_OVERRIDE = safeGetConfig "nimbus.src-override" null; }; - inherit (config.status-im) build-type; - inherit (config.status-im.status-react) build-number; - gradle-opts = (attrByPath ["status-im" "status-react" "gradle-opts"] "" config); - # Path to the .keystore file used to sign the Android APK - keystore-file = (attrByPath ["status-im" "status-react" "keystore-file"] "" config); + + buildType = safeGetConfig "build-type" "prod"; + buildNumber = safeGetConfig "build-number" "9999"; + gradleOpts = safeGetConfig "android.gradle-opts" ""; + keystorePath = safeGetConfig "android.keystore-path" ""; + # Keep the same keystore path for determinism + keystoreLocal = "${gradleHome}/status-im.keystore"; + baseName = "release-android"; name = "status-react-build-${baseName}"; + gradleHome = "$NIX_BUILD_TOP/.gradle"; localMavenRepo = "${mavenAndNpmDeps.drv}/.m2/repository"; sourceProjectDir = "${mavenAndNpmDeps.drv}/project"; - envFileName = - if (build-type == "release" || build-type == "nightly" || build-type == "e2e") then ".env.${build-type}" else - if build-type != "" then ".env.jenkins" else ".env"; - buildType = if (build-type == "pr" || build-type == "e2e") then "pr" else "release"; /* PR builds shouldn't replace normal releases */ - apksPath = "$sourceRoot/android/app/build/outputs/apk/${buildType}"; + envFileName = if (buildType == "release" || buildType == "nightly" || buildType == "e2e") + then ".env.${buildType}" + else if buildType != "" then ".env.jenkins" + else ".env"; + + # There are only two types of Gradle builds: pr and release + gradleBuildType = if (buildType == "pr" || buildType == "e2e") + then "Pr" + else "Release"; # PR builds shouldn't replace normal releases + + apksPath = "$sourceRoot/android/app/build/outputs/apk/${toLower gradleBuildType}"; patchedWatchman = watchmanFactory watchmanSockPath; -in stdenv.mkDerivation { +in stdenv.mkDerivation rec { inherit name; - src = - let path = ./../../../..; - in builtins.path { # We use builtins.path so that we can name the resulting derivation, otherwise the name would be taken from the checkout directory, which is outside of our control - inherit path; - name = "status-react-source-${baseName}"; - filter = - # Keep this filter as restrictive as possible in order to avoid unnecessary rebuilds and limit closure size - mkFilter { - root = path; - include = [ - "mobile/js_files.*" "resources/.*" - "modules/react-native-status/android.*" - envFileName "VERSION" ".watchmanconfig" - "status-go-version.json" "react-native.config.js" - ]; - }; + src = let path = ./../../../..; + # We use builtins.path so that we can name the resulting derivation + in builtins.path { + inherit path; + name = "status-react-source-${baseName}"; + # Keep this filter as restrictive as possible in order to avoid unnecessary rebuilds and limit closure size + filter = lib.mkFilter { + root = path; + include = [ + "mobile/js_files.*" "resources/.*" + "modules/react-native-status/android.*" + envFileName "VERSION" ".watchmanconfig" + "status-go-version.json" "react-native.config.js" + ]; }; - nativeBuildInputs = [ bash gradle unzip ] ++ lib.optionals stdenv.isDarwin [ file gnumake patchedWatchman ]; + }; + buildInputs = [ nodejs openjdk ]; + nativeBuildInputs = [ bash gradle unzip ] + ++ lib.optionals stdenv.isDarwin [ file gnumake patchedWatchman ]; + + # Used by Clojure at compile time to include JS modules + BUILD_ENV = buildEnv; + phases = [ "unpackPhase" "patchPhase" "buildPhase" "checkPhase" "installPhase" ]; + unpackPhase = '' runHook preUnpack @@ -66,10 +90,11 @@ in stdenv.mkDerivation { runHook postUnpack ''; - postUnpack = '' + postUnpack = assert lib.assertMsg (keystorePath != "") "keystore-file has to be set!"; '' mkdir -p ${gradleHome} - ${if keystore-file != "" then "cp -a --no-preserve=ownership ${keystore-file} ${gradleHome}/; export KEYSTORE_PATH=${gradleHome}/$(basename ${keystore-file})" else ""} + # WARNING: Renaming the keystore will cause 'Keystore was tampered with' error + cp -a --no-preserve=ownership "${keystorePath}" "${keystoreLocal}" # Ensure we have the right .env file cp -f $sourceRoot/${envFileName} $sourceRoot/.env @@ -81,7 +106,8 @@ in stdenv.mkDerivation { cp -a --no-preserve=ownership ${sourceProjectDir}/android/ $sourceRoot/ chmod u+w $sourceRoot/android chmod u+w $sourceRoot/android/app - mkdir $sourceRoot/android/build && chmod -R u+w $sourceRoot/android/build + mkdir -p $sourceRoot/android/build + chmod -R u+w $sourceRoot/android/build # Copy node_modules/ directory cp -a --no-preserve=ownership ${sourceProjectDir}/node_modules/ $sourceRoot/ @@ -97,28 +123,36 @@ in stdenv.mkDerivation { substituteInPlace $sourceRoot/android/gradlew \ --replace \ 'exec gradle' \ - "exec gradle -Dmaven.repo.local='${localMavenRepo}' --offline ${gradle-opts}" + "exec gradle -Dmaven.repo.local='${localMavenRepo}' --offline ${gradleOpts}" set $prevSet ''; - buildPhase = - let - inherit (lib) catAttrs concatStrings concatStringsSep mapAttrsToList makeLibraryPath optionalString substring toUpper; - # Take the env attribute set and build a couple of scripts - # (one to export the environment variables, and another to unset them) - exportEnvVars = concatStringsSep ";" (mapAttrsToList (name: value: "export ${name}='${value}'") env'); - unsetEnvVars = concatStringsSep ";" (mapAttrsToList (name: value: "unset ${name}") env'); - adhocEnvVars = optionalString stdenv.isLinux "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${makeLibraryPath [ zlib ]}"; - capitalizedBuildType = toUpper (substring 0 1 buildType) + substring 1 (-1) buildType; - in '' + buildPhase = let + inherit (lib) + stringLength optionalString substring + concatStrings concatStringsSep + catAttrs mapAttrsToList makeLibraryPath; + + # Take the env attribute set and build a couple of scripts + # (one to export the environment variables, and another to unset them) + exportEnvVars = concatStringsSep ";" + (mapAttrsToList (name: value: "export ${name}='${toString value}'") env); + adhocEnvVars = optionalString stdenv.isLinux + "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${makeLibraryPath [ zlib ]}"; + in + assert stringLength env.ANDROID_ABI_SPLIT > 0; + assert stringLength env.ANDROID_ABI_INCLUDE > 0; + '' export ANDROID_SDK_ROOT="${androidPkgs}" export ANDROID_NDK_ROOT="${androidPkgs}/ndk-bundle" + export KEYSTORE_PATH="${keystoreLocal}" + export STATUS_REACT_HOME=$PWD export HOME=$sourceRoot ${exportEnvVars} - ${if secrets-file != "" then "source ${secrets-file}" else ""} + ${optionalString (secretsFile != "") "source ${secretsFile}"} ${concatStrings (catAttrs "shellHook" [ mavenAndNpmDeps.shell status-go.shell ])} @@ -126,10 +160,8 @@ in stdenv.mkDerivation { chmod -R +w $sourceRoot/android pushd $sourceRoot/android - ${adhocEnvVars} ./gradlew -PversionCode=${assert build-number != ""; build-number} assemble${capitalizedBuildType} || exit + ${adhocEnvVars} ./gradlew -PversionCode=${buildNumber} assemble${gradleBuildType} || exit popd > /dev/null - - ${unsetEnvVars} ''; doCheck = true; checkPhase = '' diff --git a/nix/mobile/default.nix b/nix/mobile/default.nix index 855ea35715..68c8fa1858 100644 --- a/nix/mobile/default.nix +++ b/nix/mobile/default.nix @@ -1,5 +1,5 @@ { config, lib, stdenvNoCC, callPackage, mkShell, - status-go, mergeSh, xcodeWrapper }: + status-go, xcodeWrapper }: let inherit (lib) catAttrs concatStrings optional unique; @@ -31,7 +31,7 @@ let in { buildInputs = unique (catAttrs "buildInputs" selectedSources); - shell = mergeSh (mkShell {}) (catAttrs "shell" selectedSources); + shell = lib.mergeSh (mkShell {}) (catAttrs "shell" selectedSources); # TARGETS inherit android ios fastlane; diff --git a/nix/mobile/ios/default.nix b/nix/mobile/ios/default.nix index 9e81a69f45..b4da58e693 100644 --- a/nix/mobile/ios/default.nix +++ b/nix/mobile/ios/default.nix @@ -1,58 +1,41 @@ -{ callPackage, lib, stdenv, mkShell, mergeSh, mkFilter, +{ callPackage, lib, mkShell, xcodeWrapper, projectNodePackage, status-go, flock, procps, watchman, bundler, fastlane }: let inherit (lib) catAttrs unique; - pod = callPackage ./pod-shell.nix { }; + pod-shell = callPackage ./pod-shell.nix { }; status-go-shell = callPackage ./status-go-shell.nix { inherit status-go; }; selectedSources = [ status-go fastlane ]; - src = - let path = ./../../..; - # We use builtins.path so that we can name the resulting derivation, - # otherwise the name would be taken from the checkout directory, which is outside of our control - in builtins.path { - inherit path; - name = "status-react-source-npm-deps"; - filter = - # Keep this filter as restrictive as possible in order to avoid - # unnecessary rebuilds and limit closure size - mkFilter { - include = [ ".babelrc" "mobile/js_files.*" ]; - root = path; - }; - }; - buildInputs = unique ([ xcodeWrapper watchman bundler procps flock # used in reset-node_modules.sh ] ++ catAttrs "buildInputs" selectedSources); - shellHook = '' - pushd "$STATUS_REACT_HOME" > /dev/null - { - # Set up symlinks to mobile enviroment in project root - ln -sf ./mobile/js_files/metro.config.js ./metro.config.js - ln -sf ./mobile/js_files/package.json ./package.json - ln -sf ./mobile/js_files/yarn.lock ./yarn.lock - - # check if node modules changed and if so install them - ./nix/mobile/reset-node_modules.sh "${projectNodePackage}" - } - popd > /dev/null - ''; - localShell = mkShell { - inherit buildInputs shellHook; + shellHook = '' + pushd "$STATUS_REACT_HOME" > /dev/null + { + # Set up symlinks to mobile enviroment in project root + ln -sf ./mobile/js_files/metro.config.js ./metro.config.js + ln -sf ./mobile/js_files/package.json ./package.json + ln -sf ./mobile/js_files/yarn.lock ./yarn.lock + + # check if node modules changed and if so install them + ./nix/mobile/reset-node_modules.sh "${projectNodePackage}" + } + popd > /dev/null + ''; + inherit buildInputs; }; in { - inherit shellHook buildInputs pod; + inherit buildInputs pod-shell; - shell = mergeSh localShell [ - fastlane.shell status-go-shell pod.shell + shell = lib.mergeSh localShell [ + fastlane.shell status-go-shell pod-shell ]; } diff --git a/nix/mobile/ios/pod-shell.nix b/nix/mobile/ios/pod-shell.nix index ef4eb83b2d..676314e5da 100644 --- a/nix/mobile/ios/pod-shell.nix +++ b/nix/mobile/ios/pod-shell.nix @@ -5,22 +5,20 @@ let podfileLock = "ios/Podfile.lock"; # current state of pods installed by pod manifestLock = "ios/Pods/Manifest.lock"; -in { - shell = mkShell { - buildInputs = [ cocoapods ]; - shellHook = '' - pushd "$STATUS_REACT_HOME" > /dev/null - { - echo "Checking for modifications in ios/Pods..." - if diff -q ${podfileLock} ${manifestLock}; then - echo "No modifications detected." - else - # CocoaPods are trash and can't handle other pod instances running - ./scripts/wait-for.sh 240 'pod install' - (cd ios && pod install) - fi - } - popd > /dev/null - ''; - }; +in mkShell { + buildInputs = [ cocoapods ]; + shellHook = '' + pushd "$STATUS_REACT_HOME" > /dev/null + { + echo "Checking for modifications in ios/Pods..." + if diff -q ${podfileLock} ${manifestLock}; then + echo "No modifications detected." + else + # CocoaPods are trash and can't handle other pod instances running + ./scripts/wait-for.sh 240 'pod install' + (cd ios && pod install) + fi + } + popd > /dev/null + ''; } diff --git a/nix/mobile/ios/status-go-shell.nix b/nix/mobile/ios/status-go-shell.nix index 2fc96bd3f2..5cf5f5e0c6 100644 --- a/nix/mobile/ios/status-go-shell.nix +++ b/nix/mobile/ios/status-go-shell.nix @@ -1,4 +1,4 @@ -{ mkShell, mergeSh, status-go }: +{ lib, mkShell, status-go }: let shell = mkShell { @@ -28,4 +28,4 @@ let ''; }; in - mergeSh status-go.shell [ shell ] + lib.mergeSh status-go.shell [ shell ] diff --git a/nix/overlay.nix b/nix/overlay.nix new file mode 100644 index 0000000000..c26c6497db --- /dev/null +++ b/nix/overlay.nix @@ -0,0 +1,37 @@ +# +# Override some packages and utilities in 'pkgs' +# and make them available globally via callPackage. +# +# For more details see: +# - https://nixos.wiki/wiki/Overlays +# - https://nixos.org/nixos/nix-pills/callpackage-design-pattern.html +# + +self: super: + +let + inherit (super) stdenv stdenvNoCC callPackage; +in { + # Fix for MacOS + mkShell = super.mkShell.override { stdenv = stdenvNoCC; }; + + # Various utilities + utils = callPackage ./tools/utils.nix { }; + lib = (super.lib or {}) // { + mkFilter = callPackage ./tools/mkFilter.nix { }; + mergeSh = callPackage ./tools/mergeSh.nix { }; + }; + + # Android environement + androidEnvCustom = callPackage ./mobile/android/sdk { }; + androidPkgs = self.androidEnvCustom.licensedPkgs; + androidShell = self.androidEnvCustom.shell; + + # Package version adjustments + xcodeWrapper = super.xcodeenv.composeXcodeWrapper { version = "11.4.1"; }; + openjdk = super.pkgs.openjdk8_headless; + nodejs = super.pkgs.nodejs-12_x; + + # Custom packages + gomobile = callPackage ./pkgs/gomobile { }; +} diff --git a/nix/patched-go/default.nix b/nix/patched-go/default.nix deleted file mode 100644 index b1093f9959..0000000000 --- a/nix/patched-go/default.nix +++ /dev/null @@ -1,22 +0,0 @@ -# -# Patch the Go compiler so that we can have a say (using a NIX_GOWORKDIR env variable) -# as to the temporary directory it uses for linking, since that directory path ends up -# in the string table and .gnu.version_d ELF header. -# - -{ baseGo }: - -let - go = baseGo.overrideDerivation(oldAttrs: { - postPatch = (oldAttrs.postPatch or "") + '' - substituteInPlace "src/cmd/go/internal/work/action.go" --replace \ - 'tmp, err := ioutil.TempDir(os.Getenv("GOTMPDIR"), "go-build")' \ - 'var err error - tmp := os.Getenv("NIX_GOWORKDIR") - if tmp == "" { - tmp, err = ioutil.TempDir(os.Getenv("GOTMPDIR"), "go-build") - }' - ''; - }); - -in go diff --git a/nix/pkgs.nix b/nix/pkgs.nix index 388a7ef124..de9dcb51b2 100644 --- a/nix/pkgs.nix +++ b/nix/pkgs.nix @@ -3,11 +3,12 @@ { config ? { } }: let - inherit (import { }) fetchFromGitHub lib; + inherit (import { }) fetchFromGitHub; # For testing local version of nixpkgs #nixpkgsSrc = (import { }).lib.cleanSource "/home/jakubgs/work/nixpkgs"; + # Our own nixpkgs fork with custom fixes nixpkgsSrc = fetchFromGitHub { name = "nixpkgs-source"; owner = "status-im"; @@ -16,38 +17,21 @@ let sha256 = "0whwzll9lvrq4gg5j838skg7fqpvb55w4z7y44pzib32k613y2qn"; # To get the compressed Nix sha256, use: # nix-prefetch-url --unpack https://github.com/${ORG}/nixpkgs/archive/${REV}.tar.gz - # The last line will be the hash. }; + # Override some packages and utilities + pkgsOverlay = import ./overlay.nix; + + # we want these packages easily available via callPackage defaultConfig = { android_sdk.accept_license = true; # Android Env still needs old OpenSSL permittedInsecurePackages = [ "openssl-1.0.2u" ]; - # Override some package versions - packageOverrides = pkgs: rec { - inherit (pkgs) callPackage stdenv stdenvNoCC xcodeenv; - - # utilities - mkFilter = import ./tools/mkFilter.nix { inherit (stdenv) lib; }; - mkShell = import ./tools/mkShell.nix { inherit pkgs; stdenv = stdenvNoCC; }; - mergeSh = import ./tools/mergeSh.nix { inherit (stdenv) lib; }; - - # android environement - androidEnvCustom = callPackage ./mobile/android/sdk { }; - androidPkgs = androidEnvCustom.licensedPkgs; - androidShell = androidEnvCustom.shell; - - # custom packages - xcodeWrapper = xcodeenv.composeXcodeWrapper { version = "11.4.1"; }; - openjdk = pkgs.openjdk8_headless; - nodejs = pkgs.nodejs-12_x; - yarn = pkgs.yarn.override { inherit nodejs; }; - go = callPackage ./patched-go { baseGo = pkgs.go_1_14; }; - - # custom builders - buildGoPackage = pkgs.buildGo114Package.override { inherit go; }; - }; }; - pkgs = (import nixpkgsSrc) { config = defaultConfig // config; }; + in - pkgs + # import nixpkgs with a config override + (import nixpkgsSrc) { + config = defaultConfig // config; + overlays = [ pkgsOverlay ]; + } diff --git a/nix/status-go/gomobile/default.nix b/nix/pkgs/gomobile/default.nix similarity index 98% rename from nix/status-go/gomobile/default.nix rename to nix/pkgs/gomobile/default.nix index f6c80d6056..7a7646e379 100644 --- a/nix/status-go/gomobile/default.nix +++ b/nix/pkgs/gomobile/default.nix @@ -1,4 +1,4 @@ -{ stdenv, callPackage, utils, fetchgit, buildGoPackage, +{ stdenv, utils, callPackage, fetchgit, buildGoPackage, ncurses5, zlib, makeWrapper, patchelf, androidPkgs, xcodeWrapper }: diff --git a/nix/status-go/gomobile/deps.nix b/nix/pkgs/gomobile/deps.nix similarity index 100% rename from nix/status-go/gomobile/deps.nix rename to nix/pkgs/gomobile/deps.nix diff --git a/nix/status-go/gomobile/resolve-nix-android-sdk.patch b/nix/pkgs/gomobile/resolve-nix-android-sdk.patch similarity index 100% rename from nix/status-go/gomobile/resolve-nix-android-sdk.patch rename to nix/pkgs/gomobile/resolve-nix-android-sdk.patch diff --git a/nix/pkgs/patched-go/default.nix b/nix/pkgs/patched-go/default.nix new file mode 100644 index 0000000000..502183b01a --- /dev/null +++ b/nix/pkgs/patched-go/default.nix @@ -0,0 +1,18 @@ +# Patch the Go compiler so that we can have a say (using a NIX_GOWORKDIR env variable) +# as to the temporary directory it uses for linking, since that directory path ends up +# in the string table and .gnu.version_d ELF header. +# + +{ go }: + +go.overrideDerivation (oldAttrs: { + postPatch = (oldAttrs.postPatch or "") + '' + substituteInPlace "src/cmd/go/internal/work/action.go" --replace \ + 'tmp, err := ioutil.TempDir(os.Getenv("GOTMPDIR"), "go-build")' \ + 'var err error + tmp := os.Getenv("NIX_GOWORKDIR") + if tmp == "" { + tmp, err = ioutil.TempDir(os.Getenv("GOTMPDIR"), "go-build") + }' + ''; +}) diff --git a/nix/scripts/shell.sh b/nix/scripts/shell.sh index 20dc15712c..45ece7e3ce 100755 --- a/nix/scripts/shell.sh +++ b/nix/scripts/shell.sh @@ -3,10 +3,7 @@ # # This script is used by the Makefile to have an implicit nix-shell. # The following environment variables modify the script behavior: -# - TARGET: This attribute is passed as `targets` arg to Nix, limiting the scope -# of the Nix expressions. -# - _NIX_ATTR: Used to specify an attribute set from inside the expression in `default.nix`. -# This allows for drilling down into a specific attribute in Nix expressions. +# - TARGET: This attribute is passed via --attr to Nix, defining the scope. # - _NIX_PURE: This variable allows for making the shell pure with the use of --pure. # Take note that this makes Nix tools like `nix-build` unavailable in the shell. # - _NIX_KEEP: This variable allows specifying which env vars to keep for Nix pure shell. @@ -22,11 +19,13 @@ nixArgs=( "--show-trace" ) -if [[ -n "${TARGET}" ]]; then - nixArgs+=("--argstr target ${TARGET}") -else - echo -e "${YLW}Env is missing TARGET, assuming default target.${RST} See nix/README.md for more details." 1>&2 +if [[ -z "${TARGET}" ]]; then + TARGET="default" + echo -e "${YLW}Missing TARGET, assuming default target.${RST} See nix/README.md for more details." 1>&2 fi +entryPoint="default.nix" +nixArgs+=("--attr shells.${TARGET}") + if [[ "$TARGET" =~ (linux|windows|darwin|macos) ]]; then # This is a dirty workaround because 'yarn install' is an impure operation, @@ -48,29 +47,24 @@ if [ -n "$config" ]; then nixArgs+=("--arg config {$config}") fi -# if _NIX_ATTR is specified we shouldn't use shell.nix, the path will be different -entryPoint="shell.nix" -if [ -n "${_NIX_ATTR}" ]; then - nixArgs+=("--attr ${_NIX_ATTR}") - entryPoint="default.nix" +# This variable allows specifying which env vars to keep for Nix pure shell +# The separator is a colon +if [[ -n "${_NIX_KEEP}" ]]; then + nixArgs+=("--keep ${_NIX_KEEP//,/ --keep }") fi +# Not all builds are ready to be run in a pure environment +if [[ -n "${_NIX_PURE}" ]]; then + nixArgs+=("--pure") + pureDesc='pure ' +fi + +echo -e "${GRN}Configuring ${pureDesc}Nix shell for target '${TARGET}'...${RST}" 1>&2 + # ENTER_NIX_SHELL is the fake command used when `make shell` is run. # It is just a special string, not a variable, and a marker to not use `--run`. -if [[ $@ == "ENTER_NIX_SHELL" ]]; then - echo -e "${GRN}Configuring ${_NIX_ATTR:-default} Nix shell for target '${TARGET:-default}'...${RST}" 1>&2 +if [[ "${@}" == "ENTER_NIX_SHELL" ]]; then exec nix-shell ${nixArgs[@]} ${entryPoint} else - # Not all builds are ready to be run in a pure environment - if [[ -n "${_NIX_PURE}" ]]; then - nixArgs+=("--pure") - pureDesc='pure ' - fi - # This variable allows specifying which env vars to keep for Nix pure shell - # The separator is a colon - if [[ -n "${_NIX_KEEP}" ]]; then - nixArgs+=("--keep ${_NIX_KEEP//,/ --keep }") - fi - echo -e "${GRN}Configuring ${pureDesc}${_NIX_ATTR:-default} Nix shell for target '${TARGET}'...${RST}" 1>&2 exec nix-shell ${nixArgs[@]} --run "$@" ${entryPoint} fi diff --git a/nix/shells.nix b/nix/shells.nix index f0856684b6..c31289ee2c 100644 --- a/nix/shells.nix +++ b/nix/shells.nix @@ -6,30 +6,16 @@ }: let + inherit (pkgs) lib stdenv; + # everything else we define in nix/ dir targets = pkgs.callPackage ./targets.nix { inherit config; }; - # for calling lein targets in CI or Makefile - leiningen-sh = pkgs.mkShell { - buildInputs = with pkgs; [ clojure leiningen flock maven nodejs openjdk ]; - }; - - # for 'make watchman-clean' - watchman-sh = pkgs.mkShell { - buildInputs = [ pkgs.watchman ]; - }; - - # for running fastlane commands alone - fastlane-sh = targets.mobile.fastlane.shell; - - # for 'scripts/generate-keystore.sh' - keytool-sh = pkgs.mkShell { - buildInputs = [ pkgs.openjdk8 ]; - }; - # the default shell that is used when target is not specified + # it is also merged with all the other shells default = pkgs.mkShell { name = "status-react-shell"; # for identifying all shells + buildInputs = with pkgs; lib.unique ([ # core utilities that should always be present in a shell bash curl wget file unzip flock git gnumake jq ncurses @@ -42,30 +28,62 @@ let ++ lib.optionals (!stdenv.isDarwin) [ gcc8 ] ); + # avoid terinal issues + TERM="xterm"; + + # default locale + LANG="en_US.UTF-8"; + LANGUAGE="en_US.UTF-8"; + # just a nicety for easy access to node scripts shellHook = '' + export STATUS_REACT_HOME=$(git rev-parse --show-toplevel) export PATH="$STATUS_REACT_HOME/node_modules/.bin:$PATH" ''; }; -# values here can be selected using `nix-shell --argstr target $TARGET` + # An attrset for easier merging with default shell + shells = rec { + inherit default; + + # for calling lein targets in CI or Makefile + lein = pkgs.mkShell { + buildInputs = with pkgs; [ clojure leiningen flock maven nodejs openjdk ]; + }; + + # for 'make watchman-clean' + watchman = pkgs.mkShell { + buildInputs = [ pkgs.watchman ]; + }; + + # for running fastlane commands alone + fastlane = targets.mobile.fastlane.shell; + + # for 'scripts/generate-keystore.sh' + keytool = pkgs.mkShell { + buildInputs = [ pkgs.openjdk8 ]; + }; + + # for targets that need 'adb' + android-env = targets.mobile.android.env.shell; + + # helpers for use with target argument + linux = targets.desktop.linux.shell; + macos = targets.desktop.macos.shell; + windows = targets.desktop.windows.shell; + android = targets.mobile.android.shell; + ios = targets.mobile.ios.shell; + + # all shells together depending on host OS + all = lib.mergeSh (pkgs.mkShell {}) (lib.unique ( + lib.optionals stdenv.isLinux [ android linux windows ] ++ + lib.optionals stdenv.isDarwin [ android macos ios ] + )); + }; + + # for merging the default shell with others + mergeDefaultShell = (name: value: lib.mergeSh default [ value ]); + +# values here can be selected using `nix-shell --attr shells.$TARGET default.nix` # the nix/scripts/shell.sh wrapper does this for us and expects TARGET to be set -in with pkgs; rec { - inherit default; - lein = leiningen-sh; - watchman = watchman-sh; - fastlane = fastlane-sh; - keytool = keytool-sh; - android-env = targets.mobile.android.env.shell; - # helpers for use with target argument - linux = targets.desktop.linux.shell; - macos = targets.desktop.macos.shell; - windows = targets.desktop.windows.shell; - android = targets.mobile.android.shell; - ios = targets.mobile.ios.shell; - # all shells together depending on host OS - all = mergeSh (mkShell {}) (lib.unique ( - lib.optionals stdenv.isLinux [ android linux windows ] ++ - lib.optionals stdenv.isDarwin [ android macos ios ] - )); -} +in lib.mapAttrs mergeDefaultShell shells diff --git a/nix/status-go/default.nix b/nix/status-go/default.nix index ecc45569b6..ceb5377b9d 100644 --- a/nix/status-go/default.nix +++ b/nix/status-go/default.nix @@ -1,5 +1,5 @@ -{ config, stdenv, callPackage, mkShell, mergeSh, - fetchFromGitHub, mkFilter, openjdk, androidPkgs }: +{ config, lib, utils, stdenv, callPackage, mkShell, + fetchFromGitHub, openjdk, androidPkgs }: let inherit (stdenv.lib) @@ -9,20 +9,13 @@ let envFlags = callPackage ../tools/envParser.nix { }; enableNimbus = (attrByPath ["STATUS_GO_ENABLE_NIMBUS"] "0" envFlags) != "0"; - utils = callPackage ./utils.nix { }; - - gomobile = callPackage ./gomobile { inherit utils; }; nimbus = if enableNimbus then callPackage ./nimbus { } else { wrappers-android = { }; }; - buildStatusGoDesktopLib = callPackage ./desktop { - inherit utils; - }; - buildStatusGoMobileLib = callPackage ./mobile { - inherit gomobile utils androidPkgs; - }; + buildStatusGoDesktopLib = callPackage ./desktop { }; + buildStatusGoMobileLib = callPackage ./mobile { }; srcData = # If config.status-im.status-go.src-override is defined, instruct Nix to use that path to build status-go @@ -41,7 +34,7 @@ let name = "${repo}-source-${shortRev}"; filter = # Keep this filter as restrictive as possible in order to avoid unnecessary rebuilds and limit closure size - mkFilter { + lib.mkFilter { root = path; include = [ ".*" ]; exclude = [ @@ -193,7 +186,7 @@ let platforms = [ android ios desktop ]; in { - shell = mergeSh mkShell {} (catAttrs "shell" platforms); + shell = lib.mergeSh mkShell {} (catAttrs "shell" platforms); # CHILD DERIVATIONS inherit android ios desktop; diff --git a/nix/status-go/mobile/default.nix b/nix/status-go/mobile/default.nix index 90021c92fc..1f2fb9f363 100644 --- a/nix/status-go/mobile/default.nix +++ b/nix/status-go/mobile/default.nix @@ -1,5 +1,6 @@ { stdenv, utils, callPackage, - buildGoPackage, go, gomobile, androidPkgs, openjdk, unzip, zip, xcodeWrapper }: + buildGoPackage, go, gomobile, androidPkgs, + openjdk, unzip, zip, xcodeWrapper }: { owner, repo, rev, cleanVersion, goPackagePath, src, host, diff --git a/nix/tools/envParser.nix b/nix/tools/envParser.nix index 72e19a12aa..d6ca3a2fed 100644 --- a/nix/tools/envParser.nix +++ b/nix/tools/envParser.nix @@ -29,8 +29,8 @@ let envFileName = if build-type == "release" then ../../.env.release else if build-type == "nightly" then ../../.env.nightly else - if build-type == "e2e" then ../../.env.e2e else - if ci then ../../.env.jenkins else ../../.env; + if build-type == "e2e" then ../../.env.e2e else + if build-type == "pr" then ../../.env.jenkins else ../../.env; flags = readFlagsFromFile envFileName; # TODO: Simplify this path search with lib.locateDominatingFile in flags diff --git a/nix/status-go/utils.nix b/nix/tools/utils.nix similarity index 100% rename from nix/status-go/utils.nix rename to nix/tools/utils.nix diff --git a/scripts/release-android.sh b/scripts/release-android.sh index d6cb04c1dd..85cd59ee34 100755 --- a/scripts/release-android.sh +++ b/scripts/release-android.sh @@ -31,8 +31,10 @@ if [ -n "${NIMBUS_SRC_OVERRIDE}" ]; then config+="status-im.nimbus.src-override=\"${NIMBUS_SRC_OVERRIDE}\";" fi config+="status-im.build-type=\"$(must_get_env BUILD_TYPE)\";" -config+="status-im.status-react.build-number=\"$(must_get_env BUILD_NUMBER)\";" -config+="status-im.status-react.keystore-file=\"$(must_get_env KEYSTORE_PATH)\";" +config+="status-im.build-number=\"$(must_get_env BUILD_NUMBER)\";" +config+="status-im.android.keystore-path=\"$(must_get_env KEYSTORE_PATH)\";" +config+="status-im.android.abi-split=\"$(must_get_env ANDROID_ABI_SPLIT)\";" +config+="status-im.android.abi-include=\"$(must_get_env ANDROID_ABI_INCLUDE)\";" nixOpts=() # Secrets like this can't be passed via args or they end up in derivation @@ -42,9 +44,8 @@ trap "rm -f ${SECRETS_FILE_PATH}" EXIT append_env_export 'KEYSTORE_PASSWORD' append_env_export 'KEYSTORE_ALIAS' append_env_export 'KEYSTORE_KEY_PASSWORD' -nixOpts+=( - "--argstr" "secrets-file" "${SECRETS_FILE_PATH}" -) +nixOpts+=("--argstr" "secretsFile" "${SECRETS_FILE_PATH}") +nixOpts+=("--argstr" "buildEnv" "$(must_get_env BUILD_ENV)") if [[ "$(uname -s)" =~ Darwin ]]; then # Start a watchman instance if not started already and store its socket path. @@ -61,9 +62,6 @@ else ) fi -nixOpts+=( - "--arg" "config" "{${config}}" - "--arg" "env" "{BUILD_ENV=\"${BUILD_ENV}\";ANDROID_ABI_SPLIT=\"${ANDROID_ABI_SPLIT}\";ANDROID_ABI_INCLUDE=\"${ANDROID_ABI_INCLUDE}\";}" -) +nixOpts+=("--arg" "config" "{${config}}") ${GIT_ROOT}/nix/scripts/build.sh targets.mobile.android.release "${nixOpts[@]}" diff --git a/shell.nix b/shell.nix index 03efc2c7b5..d9030a6d62 100644 --- a/shell.nix +++ b/shell.nix @@ -1,11 +1,8 @@ -{ - config ? { }, # for passing build options, see nix/README.md - target ? "default" # see nix/shells.nix for all valid values -}: +# for passing build options, see nix/README.md +{ config ? { } }: let project = import ./default.nix { inherit config; }; in - # this is where the $TARGET env variable affects things - project.pkgs.mergeSh project.shells.default [ project.shells.${target} ] - # combining with default shell to include all the standard utilities + # we use the shell combining most shells as default + project.shells.default