From d62cda4d665056cf05724a0079484595a4595a42 Mon Sep 17 00:00:00 2001 From: Pedro Pombeiro Date: Mon, 13 Jan 2020 15:28:40 +0100 Subject: [PATCH] nix: Add support for reading .env feature flags --- ci/android.groovy | 9 ++--- ci/nix.groovy | 8 +++++ default.nix | 2 +- nix/README.md | 20 +++++++++++ .../android/targets/release-android.nix | 18 +++++----- nix/scripts/shell.sh | 9 ++--- nix/status-go/build-mobile-status-go.nix | 5 ++- nix/status-go/default.nix | 10 +++--- nix/targets.nix | 4 +-- nix/tools/envParser.nix | 36 +++++++++++++++++++ scripts/release-android.sh | 15 ++++---- shell.nix | 2 +- 12 files changed, 101 insertions(+), 37 deletions(-) create mode 100644 nix/tools/envParser.nix diff --git a/ci/android.groovy b/ci/android.groovy index c4fb63b59c..3cd7f9b43a 100644 --- a/ci/android.groovy +++ b/ci/android.groovy @@ -36,10 +36,11 @@ def bundle() { /* Nix target which produces the final APKs */ nix.build( attr: 'targets.mobile.android.release', - args: [ - 'gradle-opts': gradleOpt, - 'build-number': utils.readBuildNumber(), - 'build-type': btype, + conf: [ + 'status-im.ci': '1', + 'status-im.build-type': btype, + 'status-im.status-react.gradle-opts': gradleOpt, + 'status-im.status-react.build-number': utils.readBuildNumber(), ], safeEnv: [ 'STATUS_RELEASE_KEY_ALIAS', diff --git a/ci/nix.groovy b/ci/nix.groovy index 7f2d70c2fb..614e70c565 100644 --- a/ci/nix.groovy +++ b/ci/nix.groovy @@ -31,6 +31,7 @@ def shell(Map opts = [:], String cmd) { * - pure - Use --pure mode with Nix for more deterministic behaviour * - link - Bu default build creates a `result` directory, you can turn that off * - keep - List of env variables to pass through to Nix build + * - conf - Map of config values to provide to --arg config * - args - Map of arguments to provide to --argstr * - attr - Name of attribute to use with --attr flag * - sbox - List of host file paths to pass to the Nix expression @@ -42,6 +43,7 @@ def build(Map opts = [:]) { link: true, args: ['target': env.TARGET], keep: [], + conf: [:], attr: null, sbox: [], safeEnv: [], @@ -116,11 +118,16 @@ private def _getNixCommandArgs(Map opts = [:], boolean isShell) { envFile.deleteOnExit() } + def configFlag = '' def argsFlags = opts.args.collect { key,val -> "--argstr ${key} \'${val}\'" } def attrFlag = '' if (opts.attr != null) { attrFlag = "--attr '${opts.attr}'" } + if (opts.conf != null && opts.conf != [:]) { + def configFlags = opts.conf.collect { key,val -> "${key}=\"${val}\";" } + configFlag = "--arg config \'{${configFlags.join('')}}\'" + } if (opts.sbox != null && !opts.sbox.isEmpty()) { extraSandboxPathsFlag = "--option extra-sandbox-paths \"${opts.sbox.join(' ')}\"" } @@ -128,6 +135,7 @@ private def _getNixCommandArgs(Map opts = [:], boolean isShell) { return [ opts.pure ? "--pure" : "", opts.link ? "" : "--no-out-link", + configFlag, keepFlags.join(" "), argsFlags.join(" "), extraSandboxPathsFlag, diff --git a/default.nix b/default.nix index 2a336a37a9..8d6380b48f 100644 --- a/default.nix +++ b/default.nix @@ -1,5 +1,5 @@ { - config ? { }, # for passing status_go.src_override + config ? { status-im = { build-type = ""; }; }, # for passing build options, see nix/README.md }: let diff --git a/nix/README.md b/nix/README.md index cf3ebe2bea..c01e2fed0c 100644 --- a/nix/README.md +++ b/nix/README.md @@ -8,6 +8,26 @@ The main config file is [`nix/nix.conf`](/nix/nix.conf) and its main purpose is __NOTE:__ If you are in Asia you might want to add the `https://nix-cache-cn.status.im/` to be first in order of `substituters`. Removing `cache.nixos.org` could also help. +## Build arguments + +We leverage the standard nixpkgs `config` argument for our own parameterization of the builds (e.g. to pass a build number or build type). Here is a sample structure of the `config` attribute set: + +```nix +config = { + status-im = { + ci = "1"; # This flag is present when running in a CI environment + build-type = "pr"; # Build type (influences which .env file gets used for feature flags) + status-go = { + src-override = "$GOPATH/src/github.com/status-im/status-go"; + }; + status-react = { + build-number = "9999"; # Build number to be assigned to the app bundle + gradle-opts = ""; # Gradle options passed for Android builds + }; + }; +}; +``` + ## Shell In order to access an interactive Nix shell a user should run `make shell`. diff --git a/nix/mobile/android/targets/release-android.nix b/nix/mobile/android/targets/release-android.nix index a9561a7989..9b8bf3a272 100644 --- a/nix/mobile/android/targets/release-android.nix +++ b/nix/mobile/android/targets/release-android.nix @@ -3,11 +3,7 @@ androidEnvShellHook, mavenAndNpmDeps, nodejs, openjdk, jsbundle, status-go, unzip, zlib }: -{ build-number, - build-type, # Build type (e.g. nightly, release, e2e). Default is to use .env.nightly file - gradle-opts ? "", - keystore-file ? "", # Path to the .keystore file used to sign the package - secrets-file ? "", # Path to the file containing secret environment variables +{ secrets-file ? "", # Path to the file containing secret environment variables watchmanSockPath ? "", # Path to the socket file exposed by an external watchman instance (workaround needed for building Android on macOS) env ? {} # Attribute set containing environment variables to expose to the build script }: @@ -15,10 +11,14 @@ assert (builtins.stringLength watchmanSockPath) > 0 -> stdenv.isDarwin; let - inherit (lib) hasAttrByPath optionalAttrs; - env' = env // optionalAttrs (hasAttrByPath ["status_go" "src_override"] config) { - STATUS_GO_SRC_OVERRIDE = config.status_go.src_override; + inherit (lib) attrByPath hasAttrByPath optionalAttrs; + env' = env // optionalAttrs (hasAttrByPath ["status-im" "status-go" "src-override"] config) { + STATUS_GO_SRC_OVERRIDE = config.status-im.status-go.src-override; }; + inherit (config.status-im) build-type; + inherit (config.status-im.status-react) build-number; + gradle-opts = (attrByPath ["status-im" "status-react" "gradle-opts"] "" config); + keystore-file = (attrByPath ["status-im" "status-react" "keystore-file"] "" config); # Path to the .keystore file used to sign the package baseName = "release-android"; name = "status-react-build-${baseName}"; gradleHome = "$NIX_BUILD_TOP/.gradle"; @@ -119,7 +119,7 @@ in stdenv.mkDerivation { ${concatStrings (catAttrs "shellHook" [ mavenAndNpmDeps.shell status-go.shell ])} pushd $sourceRoot/android - ${adhocEnvVars} ./gradlew -PversionCode=${build-number} assemble${capitalizedBuildType} || exit + ${adhocEnvVars} ./gradlew -PversionCode=${assert build-number != ""; build-number} assemble${capitalizedBuildType} || exit popd > /dev/null ${unsetEnvVars} diff --git a/nix/scripts/shell.sh b/nix/scripts/shell.sh index e5081fb356..9821ab98c7 100755 --- a/nix/scripts/shell.sh +++ b/nix/scripts/shell.sh @@ -51,13 +51,14 @@ if [[ "$TARGET" =~ (linux|windows|darwin|macos) ]]; then nix-shell ${shellArgs[@]} --run "scripts/prepare-for-desktop-platform.sh" || exit fi -statusGoConfig='' +config='' if [ -n "${STATUS_GO_SRC_OVERRIDE}" ]; then - statusGoConfig+="status_go.src_override=\"${STATUS_GO_SRC_OVERRIDE}\";" + config+="status-im.status-go.src-override=\"${STATUS_GO_SRC_OVERRIDE}\";" fi +config+="status-im.build-type=\"${BUILD_TYPE}\";" -if [ -n "$statusGoConfig" ]; then - shellArgs+=("--arg config {$statusGoConfig}") +if [ -n "$config" ]; then + shellArgs+=("--arg config {$config}") fi # if _NIX_ATTR is specified we shouldn't use shell.nix, the path will be different diff --git a/nix/status-go/build-mobile-status-go.nix b/nix/status-go/build-mobile-status-go.nix index 5f3d9ee53f..da58e06a7a 100644 --- a/nix/status-go/build-mobile-status-go.nix +++ b/nix/status-go/build-mobile-status-go.nix @@ -5,19 +5,18 @@ # mobile-only arguments goBuildFlags, goBuildLdFlags, - config } @ args': + targetConfig } @ args': let inherit (stdenv.lib) concatStringsSep makeBinPath optional; - targetConfig = config; buildStatusGo = callPackage ./build-status-go.nix { inherit buildGoPackage go xcodeWrapper utils; }; # Remove mobile-only arguments from args args = removeAttrs args' [ - "config" "goBuildFlags" "goBuildLdFlags" + "targetConfig" "goBuildFlags" "goBuildLdFlags" ]; buildStatusGoMobileLib = buildStatusGo (args // { diff --git a/nix/status-go/default.nix b/nix/status-go/default.nix index f68f11d0dd..c9d2e170c1 100644 --- a/nix/status-go/default.nix +++ b/nix/status-go/default.nix @@ -12,8 +12,8 @@ let buildStatusGoDesktopLib = callPackage ./build-desktop-status-go.nix { inherit buildGoPackage go xcodeWrapper utils; }; buildStatusGoMobileLib = callPackage ./build-mobile-status-go.nix { inherit buildGoPackage go gomobile xcodeWrapper utils; }; srcData = - # If config.status_go.src_override is defined, instruct Nix to use that path to build status-go - if (attrByPath ["status_go" "src_override"] "" config) != "" then rec { + # If config.status-im.status-go.src-override is defined, instruct Nix to use that path to build status-go + if (attrByPath ["status-im" "status-go" "src-override"] "" config) != "" then rec { owner = "status-im"; repo = "status-go"; rev = "unknown"; @@ -22,7 +22,7 @@ let cleanVersion = rawVersion; goPackagePath = "github.com/${owner}/${repo}"; src = - let path = traceValFn (path: "Using local ${repo} sources from ${path}\n") config.status_go.src_override; + let path = traceValFn (path: "Using local ${repo} sources from ${path}\n") config.status-im.status-go.src-override; in builtins.path { # We use builtins.path so that we can name the resulting derivation, otherwise the name would be taken from the checkout directory, which is outside of our control inherit path; name = "${repo}-source-${shortRev}"; @@ -106,12 +106,12 @@ let android = buildStatusGoMobileLib (statusGoArgs // { host = mobileConfigs.android.name; - config = mobileConfigs.android; + targetConfig = mobileConfigs.android; }); ios = buildStatusGoMobileLib (statusGoArgs // { host = mobileConfigs.ios.name; - config = mobileConfigs.ios; + targetConfig = mobileConfigs.ios; }); }; diff --git a/nix/targets.nix b/nix/targets.nix index 1b1037621b..1da19a1309 100644 --- a/nix/targets.nix +++ b/nix/targets.nix @@ -1,4 +1,4 @@ -{ +{ config ? {}, pkgs ? import ./pkgs.nix { inherit config; } }: @@ -9,7 +9,7 @@ let inherit stdenv; }; - status-go = callPackage ./status-go { + status-go = callPackage ./status-go { inherit (mobile) xcodeWrapper; androidPkgs = mobile.android.androidComposition; }; diff --git a/nix/tools/envParser.nix b/nix/tools/envParser.nix new file mode 100644 index 0000000000..72e19a12aa --- /dev/null +++ b/nix/tools/envParser.nix @@ -0,0 +1,36 @@ +# This Nix expression takes care of reading/parsing the correct .env configuration file and return it as an attr set +{ config, stdenv }: + +let + inherit (builtins) listToAttrs head tail readFile; + inherit (stdenv.lib) attrByPath filter hasPrefix nameValuePair splitString; + + build-type = attrByPath ["status-im" "build-type"] "" config; + ci = (attrByPath ["status-im" "ci"] "" config) != ""; + + readLinesFromFile = + file: + let + lines = splitString "\n" (readFile file); + removeComments = filter (line: line != "" && !(hasPrefix "#" line)); + meaningfulLines = removeComments lines; + in + meaningfulLines; + readFlagsFromFile = + file: + let + lines = readLinesFromFile file; + genAttrs = lines: + listToAttrs (map (line: + let flag = splitString "=" line; + in nameValuePair (head flag) (head (tail flag))) lines); + in + genAttrs lines; + envFileName = + if build-type == "release" then ../../.env.release else + if build-type == "nightly" then ../../.env.nightly else + if build-type == "e2e" then ../../.env.e2e else + if ci then ../../.env.jenkins else ../../.env; + flags = readFlagsFromFile envFileName; # TODO: Simplify this path search with lib.locateDominatingFile + +in flags diff --git a/scripts/release-android.sh b/scripts/release-android.sh index c110549729..3796bccd13 100755 --- a/scripts/release-android.sh +++ b/scripts/release-android.sh @@ -6,23 +6,22 @@ source "$_current_dir/lib/setup/path-support.sh" source_lib "platform.sh" -statusGoConfig='' +config='' +config+="status-im.build-type=\"${BUILD_TYPE}\";" if [ -n "${STATUS_GO_SRC_OVERRIDE}" ]; then - statusGoConfig+="status_go.src_override=\"${STATUS_GO_SRC_OVERRIDE}\";" + config+="status-im.status-go.src-override=\"${STATUS_GO_SRC_OVERRIDE}\";" fi - +config+="status-im.status-react.build-number=\"${BUILD_NUMBER}\";" +config+="status-im.status-react.keystore-file=\"${STORE_FILE}\";" nixOpts=( - "--arg config {${statusGoConfig}}" + "--arg config {${config}}" "--arg env {BUILD_ENV=\"${BUILD_ENV}\";ANDROID_ABI_SPLIT=\"${ANDROID_ABI_SPLIT}\";ANDROID_ABI_INCLUDE=\"${ANDROID_ABI_INCLUDE}\";}" - "--argstr build-type ${BUILD_TYPE}" - "--argstr build-number ${BUILD_NUMBER}" - "--argstr keystore-file ${STORE_FILE}" ) if is_macos; then # Start a watchman instance if not started already and store its socket path. # In order to get access to the right versions of watchman and jq, we start an ad-hoc nix-shell that imports the packages from nix/nixpkgs-bootstrap. - WATCHMAN_SOCKFILE = $(watchman get-sockname --no-pretty | jq -r .sockname) + WATCHMAN_SOCKFILE=$(watchman get-sockname --no-pretty | jq -r .sockname) nixOpts+=( "--argstr watchmanSockPath ${WATCHMAN_SOCKFILE}" "--option extra-sandbox-paths ${STORE_FILE};${WATCHMAN_SOCKFILE}" diff --git a/shell.nix b/shell.nix index e33937594c..03efc2c7b5 100644 --- a/shell.nix +++ b/shell.nix @@ -1,5 +1,5 @@ { - config ? { }, # for passing status_go.src_override + config ? { }, # for passing build options, see nix/README.md target ? "default" # see nix/shells.nix for all valid values }: