From f7384ae338802f38a0a625aa074f32315945b6c8 Mon Sep 17 00:00:00 2001 From: Pedro Pombeiro Date: Wed, 13 Nov 2019 14:12:02 +0100 Subject: [PATCH] Add support for `STATUS_GO_SRC_OVERRIDE` env var Signed-off-by: Pedro Pombeiro --- Makefile | 10 ++-- .../react-native-status/android/build.gradle | 6 ++ .../desktop/CMakeLists.txt | 7 ++- nix/README.md | 35 ++++++++++-- nix/build.sh | 4 +- nix/derivation.nix | 2 +- nix/mobile/android/default.nix | 4 +- .../android/targets/release-android.nix | 12 ++-- nix/status-go/default.nix | 56 ++++++++++++------- scripts/release-android.sh | 1 + 10 files changed, 96 insertions(+), 41 deletions(-) diff --git a/Makefile b/Makefile index eee0124159..faca0da84a 100644 --- a/Makefile +++ b/Makefile @@ -33,7 +33,7 @@ export BUILD_TAG := $(shell git rev-parse --short HEAD) endif # Defines which variables will be kept for Nix pure shell, use semicolon as divider -export _NIX_KEEP ?= TMPDIR,BUILD_ENV +export _NIX_KEEP ?= TMPDIR,BUILD_ENV,STATUS_GO_SRC_OVERRIDE export NIX_CONF_DIR = $(PWD)/nix # We don't want to use /run/user/$UID because it runs out of space too easilly export TMPDIR = /tmp/tmp-status-react-$(BUILD_TAG) @@ -109,7 +109,7 @@ clean: _fix-node-perms _tmpdir-rm ##@prepare Remove all output folders git clean -dxf watchman-clean: export _NIX_ATTR := targets.watchman.shell -watchman-clean: ##@prepare Delete repo directory from watchman +watchman-clean: ##@prepare Delete repo directory from watchman watchman watch-del $${STATUS_REACT_HOME} disable-githooks: SHELL := /bin/sh @@ -167,7 +167,7 @@ prod-build-android: jsbundle-android ##@legacy temporary legacy alias for jsbund jsbundle-android: SHELL := /bin/sh jsbundle-android: export TARGET_OS ?= android jsbundle-android: export BUILD_ENV ?= prod -jsbundle-android: ##@jsbundle Compile JavaScript and Clojure into index.android.js +jsbundle-android: ##@jsbundle Compile JavaScript and Clojure into index.android.js # Call nix-build to build the 'targets.mobile.jsbundle' attribute and copy the index.android.js file to the project root @git clean -dxf ./index.$(TARGET_OS).js nix/build.sh targets.mobile.jsbundle && \ @@ -178,7 +178,7 @@ prod-build-ios: jsbundle-ios ##@legacy temporary legacy alias for jsbundle-ios jsbundle-ios: export TARGET_OS ?= ios jsbundle-ios: export BUILD_ENV ?= prod -jsbundle-ios: ##@jsbundle Compile JavaScript and Clojure into index.ios.js +jsbundle-ios: ##@jsbundle Compile JavaScript and Clojure into index.ios.js @git clean -dxf -f ./index.$(TARGET_OS).js && \ lein jsbundle-ios && \ node prepare-modules.js @@ -188,7 +188,7 @@ prod-build-desktop: jsbundle-desktop ##@legacy temporary legacy alias for jsbund jsbundle-desktop: export TARGET_OS ?= $(HOST_OS) jsbundle-desktop: export BUILD_ENV ?= prod -jsbundle-desktop: ##@jsbundle Compile JavaScript and Clojure into index.desktop.js +jsbundle-desktop: ##@jsbundle Compile JavaScript and Clojure into index.desktop.js git clean -qdxf -f ./index.desktop.js desktop/ && \ lein jsbundle-desktop && \ node prepare-modules.js diff --git a/modules/react-native-status/android/build.gradle b/modules/react-native-status/android/build.gradle index 305c9517ff..4d7c2cac8c 100644 --- a/modules/react-native-status/android/build.gradle +++ b/modules/react-native-status/android/build.gradle @@ -1,6 +1,12 @@ apply plugin: 'com.android.library' def getStatusGoSHA1 = { -> + def statusgoOverridePath = System.getenv("STATUS_GO_SRC_OVERRIDE") + if (statusgoOverridePath != null) { + printf "build.gradle: Using local version of status-go from ${statusgoOverridePath}" + return "unknown" // This value is defined in https://github.com/status-im/status-react/blob/develop/nix/status-go/default.nix, in `srcData.shortRev` + } + def jsonSlurper = new groovy.json.JsonSlurper() def content = new File('../status-go-version.json').text def object = jsonSlurper.parseText(content) diff --git a/modules/react-native-status/desktop/CMakeLists.txt b/modules/react-native-status/desktop/CMakeLists.txt index 3d61bb21ac..2ae6ae6475 100755 --- a/modules/react-native-status/desktop/CMakeLists.txt +++ b/modules/react-native-status/desktop/CMakeLists.txt @@ -29,7 +29,12 @@ if (WIN32) if(NOT version) message(FATAL_ERROR "Version name missing from ${versionJSONFilePath}") endif() - set(commit ${json.commit-sha1}) + if($ENV{STATUS_GO_SRC_OVERRIDE}) + message(INFO "CMake: Using local version of status-go from $ENV{STATUS_GO_SRC_OVERRIDE}") + set(commit "unknown") # This value is defined in https://github.com/status-im/status-react/blob/develop/nix/status-go/default.nix, in `srcData.shortRev` + else() + set(commit ${json.commit-sha1}) + endif() if(NOT commit) message(FATAL_ERROR "Commit SHA1 missing from ${versionJSONFilePath}") endif() diff --git a/nix/README.md b/nix/README.md index 2aa2f2586e..4c17b978c0 100644 --- a/nix/README.md +++ b/nix/README.md @@ -2,13 +2,13 @@ This folder contains configuration for [Nix](https://nixos.org/), a purely functional package manager used by the Status app for its build process. -# Configuration +## Configuration The main config file is [`nix/nix.conf`](/nix/nix.conf) and its main purpose is defining the [binary caches](https://nixos.org/nix/manual/#ch-basic-package-mgmt) which allow download of packages to avoid having to compile them yourself locally. __NOTE:__ If you are in Asia you might want to add the `https://nix-cache-cn.status.im/` to be first in order of `substituters`. Removing `cache.nixos.org` could also help. -# Shell +## Shell In order to access an interactive Nix shell a user should run `make shell`. @@ -19,33 +19,58 @@ By default the shell starts without any specific target platform, if you want to ```bash make shell TARGET_OS=android ``` + This way your shell and all other nix commands should run in a setup that is tailored towards Android development. For valid values you can check the [`nix/platform.nix`](/nix/platform.nix) file. -# Known Issues +## Using a local status-go repository -## MacOS 10.15 "Catalina" +If you need to use a locally checked-out status-go repository as a dependency of status-react, you can achieve that by defining the `STATUS_GO_SRC_OVERRIDE` +environment variable. + +```sh +export STATUS_GO_SRC_OVERRIDE=$GOPATH/src/github.com/status-im/status-go +# Any command that you run from now on +# will use the specified status-go location +make release-android +``` + +or for a one-off build: + +```sh +make release-android STATUS_GO_SRC_OVERRIDE=$GOPATH/src/github.com/status-im/status-go +``` + +## Known Issues + +### MacOS 10.15 "Catalina" There is an unsolved issue with the root(`/`) file system in `10.15` being read-only: https://github.com/NixOS/nix/issues/2925 Our current recommended workaround is putting `/nix` under `/opt/nix` and symlinking it via `/etc/synthetic.conf`: + ```bash sudo mkdir /opt/nix sudo chown ${USER} /opt/nix sudo sh -c "echo 'nix\t/opt/nix' >> /etc/synthetic.conf" reboot ``` + After the system reboots you should see the `/nix` symlink in place: -``` + +```bash % ls -l /nix lrwxr-xr-x 1 root wheel 8 Oct 11 13:53 /nix -> /opt/nix ``` + In order to be able to use Nix with a symlinked `/nix` you need to include this in your shell: + ```bash export NIX_IGNORE_SYMLINK_STORE=1 ``` + Add it to your `.bashrc` or any other shell config file. __NOTE__: Your old `/nix` directory will end up in `/Users/Shared/Relocated Items/Security/nix` after OS upgrade. diff --git a/nix/build.sh b/nix/build.sh index c4372a3574..96c51e5abf 100755 --- a/nix/build.sh +++ b/nix/build.sh @@ -9,7 +9,7 @@ resultPath="${gitRoot}/result/" function cleanup() { # clear trapped signals trap - EXIT ERR INT QUIT - # do the actual cleanup, ignore failure + # do the actual cleanup, ignore failure if ./nix/clean.sh "${nixResultPath}"; then echo "Successful cleanup!" elif [[ -n "${IN_CI_ENVIRONMENT}" ]]; then @@ -38,7 +38,7 @@ targetAttr="${1}" shift if [[ -z "${targetAttr}" ]]; then - echo "First argument is madatory and has to specify the Nix attribute!" + echo "First argument is mandatory and has to specify the Nix attribute!" exit 1 fi diff --git a/nix/derivation.nix b/nix/derivation.nix index a6725ec75d..7d26ca4fe9 100644 --- a/nix/derivation.nix +++ b/nix/derivation.nix @@ -22,7 +22,7 @@ let buildGoPackage = pkgs.buildGoPackage.override { inherit go; }; desktop = pkgs.callPackage ./desktop { inherit target-os stdenv status-go pkgs go nodejs; inherit (pkgs) darwin; }; mobile = pkgs.callPackage ./mobile { inherit target-os config stdenv pkgs mkShell nodejs yarn status-go maven localMavenRepoBuilder mkFilter; inherit (pkgs.xcodeenv) composeXcodeWrapper; }; - status-go = pkgs.callPackage ./status-go { inherit target-os go buildGoPackage; inherit (mobile.ios) xcodeWrapper; androidPkgs = mobile.android.androidComposition; }; + status-go = pkgs.callPackage ./status-go { inherit target-os config go buildGoPackage; inherit (mobile.ios) xcodeWrapper; androidPkgs = mobile.android.androidComposition; }; # mkFilter is a function that allows filtering a directory structure (used for filtering source files being captured in a closure) mkFilter = import ./tools/mkFilter.nix { inherit (stdenv) lib; }; localMavenRepoBuilder = diff --git a/nix/mobile/android/default.nix b/nix/mobile/android/default.nix index 4901c40b5a..ca780f53b7 100644 --- a/nix/mobile/android/default.nix +++ b/nix/mobile/android/default.nix @@ -15,7 +15,7 @@ let mavenAndNpmDeps = callPackage ./maven-and-npm-deps { inherit stdenv stdenvNoCC gradle bash nodejs zlib localMavenRepoBuilder mkFilter projectNodePackage; }; # TARGETS - release = callPackage ./targets/release-android.nix { inherit target-os gradle mavenAndNpmDeps mkFilter nodejs jsbundle status-go zlib watchmanFactory; androidEnvShellHook = androidEnv.shellHook; }; + release = callPackage ./targets/release-android.nix { inherit target-os config gradle mavenAndNpmDeps mkFilter nodejs jsbundle status-go zlib watchmanFactory; androidEnvShellHook = androidEnv.shellHook; }; generate-maven-and-npm-deps-shell = callPackage ./maven-and-npm-deps/maven/shell.nix { inherit mkShell gradle maven nodejs projectNodePackage status-go; androidEnvShellHook = androidEnv.shellHook; }; adb-shell = mkShell { buildInputs = [ androidEnv.licensedAndroidEnv ]; @@ -36,7 +36,7 @@ in { inherit (stdenv.lib) catAttrs concatStrings; in '' ${concatStrings (catAttrs "shellHook" [ mavenAndNpmDeps androidEnv ])} - + $STATUS_REACT_HOME/scripts/generate-keystore.sh $STATUS_REACT_HOME/nix/mobile/reset-node_modules.sh "${mavenAndNpmDeps.deriv}/project" || exit diff --git a/nix/mobile/android/targets/release-android.nix b/nix/mobile/android/targets/release-android.nix index 4fbe2afecc..c7cd349604 100644 --- a/nix/mobile/android/targets/release-android.nix +++ b/nix/mobile/android/targets/release-android.nix @@ -1,4 +1,4 @@ -{ stdenv, stdenvNoCC, lib, target-os, callPackage, +{ stdenv, stdenvNoCC, lib, config, target-os, callPackage, mkFilter, bash, file, gnumake, watchmanFactory, gradle, androidEnvShellHook, mavenAndNpmDeps, nodejs, openjdk, jsbundle, status-go, unzip, zlib }: @@ -9,12 +9,16 @@ keystore-file ? "", # Path to the .keystore file used to sign the package secrets-file ? "", # Path to the file containing secret environment variables watchmanSockPath ? "", # Path to the socket file exposed by an external watchman instance (workaround needed for building Android on macOS) - env ? {} # Attribute set contaning environment variables to expose to the build script + env ? {} # Attribute set containing environment variables to expose to the build script }: assert (builtins.stringLength watchmanSockPath) > 0 -> stdenv.isDarwin; let + inherit (stdenv.lib) hasAttrByPath optionalAttrs; + env' = env // optionalAttrs (hasAttrByPath ["status_go" "src_override"] config) { + STATUS_GO_SRC_OVERRIDE = config.status_go.src_override; + }; baseName = "release-${target-os}"; name = "status-react-build-${baseName}"; gradleHome = "$NIX_BUILD_TOP/.gradle"; @@ -100,8 +104,8 @@ in stdenv.mkDerivation { inherit (lib) catAttrs concatStrings concatStringsSep mapAttrsToList makeLibraryPath optionalString substring toUpper; # Take the env attribute set and build a couple of scripts # (one to export the environment variables, and another to unset them) - exportEnvVars = concatStringsSep ";" (mapAttrsToList (name: value: "export ${name}='${value}'") env); - unsetEnvVars = concatStringsSep ";" (mapAttrsToList (name: value: "unset ${name}") env); + exportEnvVars = concatStringsSep ";" (mapAttrsToList (name: value: "export ${name}='${value}'") env'); + unsetEnvVars = concatStringsSep ";" (mapAttrsToList (name: value: "unset ${name}") env'); adhocEnvVars = optionalString stdenv.isLinux "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${makeLibraryPath [ zlib ]}"; capitalizedBuildType = toUpper (substring 0 1 buildType) + substring 1 (-1) buildType; in '' diff --git a/nix/status-go/default.nix b/nix/status-go/default.nix index 0b09578a59..edeb57ef61 100644 --- a/nix/status-go/default.nix +++ b/nix/status-go/default.nix @@ -1,34 +1,50 @@ -{ target-os, stdenv, callPackage, +{ target-os, config, stdenv, callPackage, buildGoPackage, go, fetchFromGitHub, openjdk, androidPkgs, xcodeWrapper }: let inherit (stdenv.lib) catAttrs concatStrings fileContents importJSON makeBinPath - optional optionalString removePrefix strings attrValues mapAttrs; - + optional optionalString removePrefix strings attrValues mapAttrs attrByPath + traceValFn; + platform = callPackage ../platform.nix { inherit target-os; }; utils = callPackage ../utils.nix { inherit xcodeWrapper; }; gomobile = callPackage ./gomobile { inherit (androidPkgs) platform-tools; inherit target-os xcodeWrapper utils buildGoPackage; }; buildStatusGoDesktopLib = callPackage ./build-desktop-status-go.nix { inherit buildGoPackage go xcodeWrapper utils; }; buildStatusGoMobileLib = callPackage ./build-mobile-status-go.nix { inherit buildGoPackage go gomobile xcodeWrapper utils; }; - versionJSON = importJSON ../../status-go-version.json; # TODO: Simplify this path search with lib.locateDominatingFile - versionRegex = "^v?[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+(-[[:alnum:].]+)$"; - owner = versionJSON.owner; - repo = versionJSON.repo; - version = versionJSON.version; - sha256 = versionJSON.src-sha256; - rev = versionJSON.commit-sha1; - shortRev = strings.substring 0 7 rev; - goPackagePath = "github.com/${owner}/${repo}"; - src = fetchFromGitHub { inherit rev owner repo sha256; name = "${repo}-${shortRev}-source"; }; - # Replace src value with the path to a local status-go repository if you want to perform a build against it, e.g. - #src = /home//go/src/github.com/status-im/status-go; + srcData = + # If config.status_go.src_override is defined, instruct Nix to use that path to build status-go + if (attrByPath ["status_go" "src_override"] "" config) != "" then rec { + owner = "status-im"; + repo = "status-go"; + version = "develop"; + goPackagePath = "github.com/${owner}/${repo}"; + rev = "unknown"; + shortRev = "unknown"; + versionName = "develop"; + src = stdenv.lib.cleanSource "${traceValFn (path: "Using local ${repo} sources from ${path}\n") config.status_go.src_override}"; + } else + # Otherwise grab it from the location defined by status-go-version.json + let + versionJSON = importJSON ../../status-go-version.json; # TODO: Simplify this path search with lib.locateDominatingFile + versionRegex = "^v?[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+(-[[:alnum:].]+)$"; + sha256 = versionJSON.src-sha256; + in rec { + inherit (versionJSON) owner repo version; + goPackagePath = "github.com/${owner}/${repo}"; + rev = versionJSON.commit-sha1; + shortRev = strings.substring 0 7 rev; + versionName = if (builtins.match versionRegex version) != null + then removePrefix "v" version # Geth forces a 'v' prefix + else "develop"; # to reduce metrics cardinality in Prometheus + src = fetchFromGitHub { inherit rev owner repo sha256; name = "${repo}-${srcData.shortRev}-source"; }; + }; mobileConfigs = { android = { name = "android"; - outputFileName = "status-go-${shortRev}.aar"; + outputFileName = "status-go-${srcData.shortRev}.aar"; envVars = [ "ANDROID_HOME=${androidPkgs.androidsdk}/libexec/android-sdk" "ANDROID_NDK_HOME=${androidPkgs.ndk-bundle}/libexec/android-sdk/ndk-bundle" @@ -58,10 +74,8 @@ let goBuildFlags = "-v"; # status-go params to be set at build time, important for About section and metrics goBuildParams = { - GitCommit = rev; - Version = if (builtins.match versionRegex version) != null - then removePrefix "v" version # Geth forces a 'v' prefix - else "develop"; # to reduce metrics cardinality in Prometheus + GitCommit = srcData.rev; + Version = srcData.versionName; }; # These are necessary for status-go to show correct version paramsLdFlags = attrValues (mapAttrs (name: value: @@ -73,7 +87,7 @@ let "-w" # -w disables DWARF debugging information ]; - statusGoArgs = { inherit owner repo rev version goPackagePath src goBuildFlags goBuildLdFlags; }; + statusGoArgs = { inherit (srcData) src owner repo rev version goPackagePath; inherit goBuildFlags goBuildLdFlags; }; status-go-packages = { desktop = buildStatusGoDesktopLib (statusGoArgs // { outputFileName = "libstatus.a"; diff --git a/scripts/release-android.sh b/scripts/release-android.sh index 80a4a4155a..27f5474e08 100755 --- a/scripts/release-android.sh +++ b/scripts/release-android.sh @@ -7,6 +7,7 @@ source "$_current_dir/lib/setup/path-support.sh" source_lib "platform.sh" nixOpts=( + "--arg config {android_sdk.accept_license=true;status_go.src_override=\"${STATUS_GO_SRC_OVERRIDE}\";}" "--arg env {BUILD_ENV=\"${BUILD_ENV}\";ANDROID_ABI_SPLIT=\"${ANDROID_ABI_SPLIT}\";ANDROID_ABI_INCLUDE=\"${ANDROID_ABI_INCLUDE}\";}" "--argstr build-type ${BUILD_TYPE}" "--argstr build-number ${BUILD_NUMBER}"