nix: Use nix/build.sh for release-android make target

Signed-off-by: Jakub Sokołowski <jakub@status.im>
This commit is contained in:
Pedro Pombeiro 2019-07-15 18:34:33 +02:00 committed by Jakub Sokołowski
parent 808aa793e5
commit acf6278383
No known key found for this signature in database
GPG Key ID: 4EF064D0E6D63020
17 changed files with 262 additions and 112 deletions

3
.gitignore vendored
View File

@ -7,6 +7,7 @@
# Xcode
#
result/
build/
*.pbxuser
!default.pbxuser
@ -170,4 +171,4 @@ status-modules/resources
## coverage
/.nyc_output
/coverage-report
/coverage-report

View File

@ -30,9 +30,15 @@ HOST_OS := $(shell uname | tr '[:upper:]' '[:lower:]')
# Defines which variables will be kept for Nix pure shell, use semicolon as divider
export _NIX_KEEP ?= BUILD_ENV
export NIX_CONF_DIR = $(PWD)/nix
# We don't want to use /run/user/$UID because it runs out of space too easilly
export TMPDIR = /tmp
export REACT_SERVER_PORT ?= 5001 # any value different from default 5000 will work; this has to be specified for both the Node.JS server process and the Qt process
#----------------
# Nix targets
#----------------
# WARNING: This has to be located right before the targets
ifdef IN_NIX_SHELL
SHELL := env bash
@ -47,6 +53,29 @@ else
@echo "${YELLOW}Nix shell is already active$(RESET)"
endif
nix-clean: ##@nix Remove all status-react build artifacts from /nix/store
nix/clean.sh
nix-purge: SHELL := /bin/sh
nix-purge: ##@nix Completely remove the complete Nix setup
sudo rm -rf /nix ~/.nix-profile ~/.nix-defexpr ~/.nix-channels ~/.cache/nix ~/.status .nix-gcroots
nix-add-gcroots: SHELL := /bin/sh
nix-add-gcroots: ##@nix Add Nix GC roots to avoid status-react expressions being garbage collected
scripts/add-nix-gcroots.sh
nix-update-npm: SHELL := /bin/sh
nix-update-npm: ##@nix Update node2nix expressions based on current package.json
nix/desktop/realm-node/generate-nix.sh
nix-update-gradle: SHELL := /bin/sh
nix-update-gradle: ##@nix Update maven nix expressions based on current gradle setup
nix/mobile/android/maven-and-npm-deps/maven/generate-nix.sh
nix-update-lein: SHELL := /bin/sh
nix-update-lein: ##@nix Update maven nix expressions based on current lein setup
nix/tools/lein/generate-nix.sh nix/lein
#----------------
# General targets
#----------------
@ -72,40 +101,22 @@ disable-githooks: ##@prepare Disables lein githooks
-e 's|:pre-commit|;; :pre-commit|' project.clj; \
rm project.clj~
#----------------
# Nix targets
#----------------
nix-clean: SHELL := /bin/sh
nix-clean: ##@nix Remove complete nix setup
sudo rm -rf /nix ~/.nix-profile ~/.nix-defexpr ~/.nix-channels ~/.cache/nix ~/.status .nix-gcroots
nix-add-gcroots: SHELL := /bin/sh
nix-add-gcroots: ##@nix Add Nix GC roots to avoid status-react expressions being garbage collected
scripts/add-nix-gcroots.sh
nix-update-npm: SHELL := /bin/sh
nix-update-npm: ##@nix Update node2nix expressions based on current package.json
nix/desktop/realm-node/generate-nix.sh
nix-update-gradle: SHELL := /bin/sh
nix-update-gradle: ##@nix Update maven nix expressions based on current gradle setup
nix/mobile/android/maven-and-npm-deps/maven/generate-nix.sh
nix-update-lein: SHELL := /bin/sh
nix-update-lein: ##@nix Update maven nix expressions based on current lein setup
nix/tools/lein/generate-nix.sh nix/lein
#----------------
# Release builds
#----------------
release: release-android release-ios ##@build build release for Android and iOS
release-android: SHELL := /bin/sh
release-android: export TARGET_OS ?= android
release-android: export BUILD_ENV ?= prod
release-android: export BUILD_TYPE ?= nightly
release-android: export NDK_ABI_FILTERS ?= armeabi-v7a;arm64-v8a;x86
release-android: export STORE_FILE ?= ~/.gradle/status-im.keystore
release-android: ##@build build release for Android
scripts/release-$(TARGET_OS).sh
nix/build.sh targets.mobile.$(TARGET_OS).release \
--arg env '{NDK_ABI_FILTERS="$(NDK_ABI_FILTERS)";}' \
--argstr build-type $(BUILD_TYPE) \
--argstr keystore-file $(STORE_FILE) \
--option extra-sandbox-paths $(STORE_FILE)
release-ios: export TARGET_OS ?= ios
release-ios: export BUILD_ENV ?= prod
@ -139,9 +150,9 @@ jsbundle-android: export TARGET_OS ?= android
jsbundle-android: export BUILD_ENV ?= prod
jsbundle-android: ##@jsbundle Compile JavaScript and Clojure into index.android.js
# Call nix-build to build the 'targets.mobile.jsbundle' attribute and copy the index.android.js file to the project root
@git clean -dxf -f ./index.$(TARGET_OS).js && \
_NIX_RESULT_PATH=$(shell . ~/.nix-profile/etc/profile.d/nix.sh && nix-build --argstr target-os $(TARGET_OS) --pure --no-out-link --show-trace -A targets.mobile.jsbundle) && \
[ -n "$${_NIX_RESULT_PATH}" ] && cp -av $${_NIX_RESULT_PATH}/* .
@git clean -dxf ./index.$(TARGET_OS).js
nix/build.sh targets.mobile.jsbundle && \
mv result/index.$(TARGET_OS).js ./
prod-build-ios: jsbundle-ios ##@legacy temporary legacy alias for jsbundle-ios
@echo "${YELLOW}This a deprecated target name, use jsbundle-ios.$(RESET)"
@ -238,14 +249,17 @@ coverage: ##@test Run tests once in NodeJS generating coverage
# Other
#--------------
react-native-desktop: export TARGET_OS ?= $(HOST_OS)
react-native-desktop: export _NIX_PURE ?= true
react-native-desktop: ##@other Start react native packager
@scripts/start-react-native.sh
react-native-android: export TARGET_OS ?= android
react-native-android: export _NIX_PURE ?= true
react-native-android: ##@other Start react native packager for Android client
@scripts/start-react-native.sh
react-native-ios: export TARGET_OS ?= ios
react-native-ios: export _NIX_PURE ?= true
react-native-ios: ##@other Start react native packager for Android client
@scripts/start-react-native.sh

View File

@ -26,6 +26,7 @@ pipeline {
LC_ALL = "en_US.UTF-8"
LANGUAGE = "en_US.UTF-8"
TARGET_OS = 'android'
BUILD_ENV = 'prod'
NIX_CONF_DIR = "${env.WORKSPACE}/nix"
FASTLANE_DISABLE_COLORS = 1
REALM_DISABLE_ANALYTICS = 1
@ -83,7 +84,9 @@ pipeline {
stage('Build') { stages {
stage('JSBundle') {
steps {
script { cmn.nix.build(attr: 'targets.mobile.jsbundle') }
script {
cmn.nix.shell('make jsbundle-android', pure: false)
}
}
}
stage('Bundle') {

View File

@ -12,7 +12,7 @@ pipeline {
options {
timestamps()
/* Prevent Jenkins jobs from running forever */
timeout(time: 25, unit: 'MINUTES')
timeout(time: 30, unit: 'MINUTES')
/* Limit builds retained */
buildDiscarder(logRotator(
numToKeepStr: '10',

View File

@ -23,73 +23,81 @@ pipeline {
}
stages {
stage('Prep') {
steps { script {
nix = load('ci/nix.groovy')
} }
}
stage('Setup') {
steps {
steps { script {
sh 'scripts/setup'
sh '''
. ~/.nix-profile/etc/profile.d/nix.sh && \
nix-env -i openssh
'''
}
nix.shell('nix-env -i openssh', pure: false)
} }
}
stage('Build status-go') {
steps { script {
['android', 'desktop', 'ios'].each { os ->
sh """
. ~/.nix-profile/etc/profile.d/nix.sh && \
nix-build --argstr target-os all -A targets.status-go.${os}.buildInputs
"""
nix.build(
attr: "targets.status-go.${os}.buildInputs",
args: ['target-os': 'all'],
link: false
)
}
} }
}
stage('Build jsbundle-android') {
steps {
// Run a Nix build to build/fetch everything that is necessary to produce a jsbundle for TARGET_OS=android (e.g. maven and node repos)
sh '''
args="--argstr target-os android --show-trace -A targets.mobile.jsbundle"
. ~/.nix-profile/etc/profile.d/nix.sh && \
nix-build --pure --no-out-link $args
prodBuildDrv=$(nix-instantiate --quiet $args) && \
prodBuildSrcPath=$(nix-store -q --binding src $prodBuildDrv) && \
prodBuildOutPath=$(nix-store -q --outputs $prodBuildDrv) && \
nix-store --delete $prodBuildDrv $prodBuildSrcPath $prodBuildOutPath
'''
}
stage('Build jsbundle') {
steps { script {
/* build/fetch things required to produce a js-bundle for android
* (e.g. maven and node repos) */
nix.build(
attr: 'targets.mobile.jsbundle',
args: ['target-os': 'android'],
pure: false,
link: false
)
} }
}
stage('Build android deps') {
steps { script {
/* build/fetch things required to build jsbundle and android */
nix.build(
attr: 'targets.mobile.android.buildInputs',
args: ['target-os': 'android'],
pure: false,
link: false
)
} }
}
stage('Build nix shell deps') {
steps {
// Run a Nix build to build/fetch everything that is necessary to instantiate shell.nix for TARGET_OS=all
sh '''
. ~/.nix-profile/etc/profile.d/nix.sh && \
nix-shell --argstr target-os all --pure --show-trace shell.nix
'''
}
steps { script {
/* build/fetch things required to instantiate shell.nix for TARGET_OS=all */
nix.shell("nix-build ${args.join(' ')}", pure: false)
nix.build(
attr: 'shell',
args: ['target-os': 'all'],
link: false
)
} }
}
stage('Upload') {
steps {
steps { script {
sshagent(credentials: ['nix-cache-ssh']) {
sh """
. ~/.nix-profile/etc/profile.d/nix.sh && \
find /nix/store/ -mindepth 1 -maxdepth 1 \
-not -name '.links' -and -not -name '*.lock' \
-and -not -name '*-status-react-jsbundle-source' \
-and -not -name '*-status-react-release-android-source' \
-and -not -name '*-jsbundle-*' \
-and -not -name '*-release-android' | \
xargs nix-copy-closure -v --to ${NIX_CACHE_USER}@${NIX_CACHE_HOST}
"""
nix.shell("""
find /nix/store/ -mindepth 1 -maxdepth 1 \
-not -name '.links' -and -not -name '*.lock' \
-and -not -name '*-status-react-*' \
| xargs nix-copy-closure -v --to ${NIX_CACHE_USER}@${NIX_CACHE_HOST}
""",
pure: false
)
}
}
} }
}
}
post {
always {
sh '''
. ~/.nix-profile/etc/profile.d/nix.sh && \
nix-store --optimize
'''
}
always { script {
nix.shell('nix-store --optimize', pure: false)
nix.shell('nix/clean.sh', pure: false)
} }
}
}

View File

@ -54,6 +54,7 @@ def bundle() {
)
}
}
/* because nix-build was run in `android` dir that's where `result` is */
def outApk = "android/result/app.apk"
def pkg = utils.pkgFilename(btype, 'apk')
/* rename for upload */

View File

@ -7,7 +7,7 @@
def shell(Map opts = [:], String cmd) {
def defaults = [
pure: true,
args: ['target-os': env.TARGET_OS],
args: ['target-os': env.TARGET_OS ? env.TARGET_OS : 'none'],
keep: ['LOCALE_ARCHIVE_2_27', 'IN_CI_ENVIRONMENT'],
]
/* merge defaults with received opts */
@ -18,34 +18,40 @@ def shell(Map opts = [:], String cmd) {
if (env.TARGET_OS in ['windows', 'ios']) {
opts.pure = false
}
sh """
set +x
. ~/.nix-profile/etc/profile.d/nix.sh
set -x
IN_CI_ENVIRONMENT=1 \\
nix-shell --run \'${cmd}\' ${_getNixCommandArgs(opts, true)}
"""
def stdOut = sh(
script: """
set +x
. ~/.nix-profile/etc/profile.d/nix.sh
set -x
IN_CI_ENVIRONMENT=1 \\
nix-shell --run \'${cmd}\' ${_getNixCommandArgs(opts, true)}
""",
returnStdout: true
)
return stdOut.trim()
}
/**
* Arguments:
* - pure - Use --pure mode with Nix for more deterministic behaviour
* - link - Bu default build creates a `result` directory, you can turn that off
* - keep - List of env variables to pass through to Nix build
* - args - Map of arguments to provide to --argstr
* - attr - Name of attribute to use with --attr flag
* - safeEnv - Name of env variables to pass securely through to Nix build (they won't get captured in Nix derivation file)
* - sbox - List of host file paths to pass to the Nix expression
* - safeEnv - Name of env variables to pass securely through to Nix build (they won't get captured in Nix derivation file)
**/
def build(Map opts = [:]) {
env.IN_CI_ENVIRONMENT = '1'
def defaults = [
pure: true,
link: true,
args: ['target-os': env.TARGET_OS],
keep: ['IN_CI_ENVIRONMENT'],
safeEnv: [],
attr: null,
sbox: []
sbox: [],
safeEnv: [],
]
/* merge defaults with received opts */
opts = defaults + opts
@ -61,7 +67,7 @@ def build(Map opts = [:]) {
set -x
nix-build ${_getNixCommandArgs(opts, false)}
"""
).trim()
).trim()
}
private makeNixBuildEnvFile(Map opts = [:]) {
@ -115,6 +121,7 @@ private def _getNixCommandArgs(Map opts = [:], boolean isShell) {
return [
opts.pure ? "--pure" : "",
opts.link ? "" : "--no-out-link",
keepFlags.join(" "),
argsFlags.join(" "),
extraSandboxPathsFlag,

View File

@ -256,7 +256,7 @@ platform :ios do
desc "`fastlane ios saucelabs` - upload .app to sauce labs"
desc "also notifies in a GitHub comments"
desc "expects to have an .apk prepared: `android/result/app.apk`"
desc "expects to have an .apk prepared: `result/app.apk`"
desc "expects to have a saucelabs access key as SAUCE_ACCESS_KEY env variable"
desc "expects to have a saucelabs username token as SAUCE_USERNAME env variable"
desc "expects to have a saucelabs destination name as SAUCE_LABS_NAME env variable"
@ -276,7 +276,7 @@ end
platform :android do
# Optional env variables
APK_PATH = ENV["APK_PATH"] || "android/result/app.apk"
APK_PATH = ENV["APK_PATH"] || "result/app.apk"
desc "Deploy a new internal build to Google Play"
desc "expects GOOGLE_PLAY_JSON_KEY environment variable"
@ -311,7 +311,7 @@ platform :android do
end
desc "`fastlane android upload_diawi` - upload .apk to diawi"
desc "expects to have an .apk prepared: `android/result/app.apk`"
desc "expects to have an .apk prepared: `result/app.apk`"
desc "expects to have a diawi token as DIAWI_TOKEN env variable"
desc "expects to have a github token as GITHUB_TOKEN env variable"
desc "will fails if file isn't there"
@ -322,7 +322,7 @@ platform :android do
end
desc "`fastlane android saucelabs` - upload .apk to sauce labs"
desc "expects to have an .apk prepared: `android/result/app.apk`"
desc "expects to have an .apk prepared: `result/app.apk`"
desc "expects to have a saucelabs access key as SAUCE_ACCESS_KEY env variable"
desc "expects to have a saucelabs username token as SAUCE_USERNAME env variable"
desc "expects to have a saucelabs destination name as SAUCE_LABS_NAME env variable"

50
nix/build.sh Executable file
View File

@ -0,0 +1,50 @@
#!/usr/bin/env bash
set -e
# cleanup for artifacts created during builds
function cleanup() {
# clear trapped signals
trap - EXIT ERR INT QUIT
# do the actual cleanup
./nix/clean.sh ${nixOpts[@]}
}
trap cleanup EXIT ERR INT QUIT
# build output will end up under /nix, we have to extract it
function extractResults() {
local nixResultPath="$1"
echo "Saving build result: ${nixResultPath}"
mkdir -p result
cp -vfr ${nixResultPath}/* result/
chmod u+w -R result/
}
targetAttr="${1}"
shift
if [[ -z "${targetAttr}" ]]; then
echo "First argument is madatory and has to specify the Nix attribute!"
exit 1
fi
# Some defaults flags, --pure could be optional in the future
nixOpts=(
"--pure"
"--fallback"
"--no-out-link"
"--show-trace"
"--argstr target-os ${TARGET_OS}"
"--attr ${targetAttr}"
"${@}"
"default.nix"
)
# Run the actual build
echo "Running: nix-build ${nixOpts[@]}"
nixResultPath=$(nix-build ${nixOpts[@]})
extractResults "${nixResultPath}"
echo "SUCCESS"

60
nix/clean.sh Executable file
View File

@ -0,0 +1,60 @@
#!/usr/bin/env bash
set -e
function getSources() {
nix-store --query --binding src "${1}"
}
function getOutputs() {
nix-store --query --outputs "${1}"
}
function getDrvFiles() {
nix-store --query --deriver "${1}"
}
function getReferrers() {
nix-store --query --referrers "${1}"
}
# list of store entries to delete
declare -a toDelete
echo "Cleanup of /nix/store after build..."
# regular expression that should match all status-react build artifacts
searchRegex='.*-status-react-(shell|source|build).*'
# search for matching entries in the store
drvPaths=$(
find /nix/store -maxdepth 1 -type d -regextype egrep -regex "${searchRegex}"
)
# for each entry find the source and derivation
for path in ${drvPaths}; do
toDelete+=("${path}")
if [[ "${path}" =~ .*.chroot ]]; then
echo " ! Chroot: ${path}"
continue
fi
echo " ? Checking: ${path}"
drv=$(getDrvFiles "${path}")
# if drv is unknown-deriver then path is a source
if [[ "${drv}" == "unknown-deriver" ]]; then
drv=$(getReferrers "${path}")
src="${path}"
elif [[ -f "${drv}" ]]; then
src=$(getSources "${drv}")
fi
echo " - Derivation: ${drv}"
echo " - Source: ${src}"
toDelete+=("${drv}" "${src}")
done
# remove dupicates
cleanToDelete=$(echo "${toDelete[@]}" | sort | uniq)
echo "Deleting..."
nix-store --ignore-liveness --delete ${cleanToDelete[@]}

View File

@ -36,7 +36,7 @@ let
let path = ./../../../..; # Import the root /android and /mobile_files folders clean of any build artifacts
in builtins.path { # We use builtins.path so that we can name the resulting derivation, otherwise the name would be taken from the checkout directory, which is outside of our control
inherit path;
name = "status-react-gradle-install-source";
name = "status-react-source-gradle-install";
filter =
# Keep this filter as restrictive as possible in order to avoid unnecessary rebuilds and limit closure size
mkFilter {

View File

@ -1,5 +1,7 @@
{ stdenv, stdenvNoCC, lib, target-os, callPackage,
mkFilter, bash, file, gnumake, watchman, gradle, androidEnvShellHook, mavenAndNpmDeps, nodejs, openjdk, jsbundle, status-go, zlib }:
mkFilter, bash, file, gnumake, watchman, gradle,
androidEnvShellHook, mavenAndNpmDeps,
nodejs, openjdk, jsbundle, status-go, zlib }:
{ build-number ? "9999",
build-type ? "nightly", # Build type (e.g. nightly, release, e2e). Default is to use .env.nightly file
@ -10,7 +12,8 @@
}:
let
name = "release-${target-os}";
baseName = "release-${target-os}";
name = "status-react-build-${baseName}";
gradleHome = "$NIX_BUILD_TOP/.gradle";
localMavenRepo = "${mavenAndNpmDeps.deriv}/.m2/repository";
sourceProjectDir = "${mavenAndNpmDeps.deriv}/project";
@ -27,7 +30,7 @@ in stdenv.mkDerivation {
let path = ./../../../..;
in builtins.path { # We use builtins.path so that we can name the resulting derivation, otherwise the name would be taken from the checkout directory, which is outside of our control
inherit path;
name = "status-react-${name}-source";
name = "status-react-source-${baseName}";
filter =
# Keep this filter as restrictive as possible in order to avoid unnecessary rebuilds and limit closure size
mkFilter {

View File

@ -16,7 +16,7 @@ let
let path = ./../../..;
in builtins.path { # We use builtins.path so that we can name the resulting derivation, otherwise the name would be taken from the checkout directory, which is outside of our control
inherit path;
name = "status-react-npm-deps-source";
name = "status-react-source-npm-deps";
filter =
# Keep this filter as restrictive as possible in order to avoid unnecessary rebuilds and limit closure size
mkFilter {

View File

@ -5,10 +5,11 @@
# The following environment variables modify the script behavior:
# - TARGET_OS: This attribute is passed as `target-os` to Nix, limiting the scope
# of the Nix expressions.
# - _NIX_ATTR: This attribute can be used to specify an attribute set
# from inside the expression in `default.nix`, allowing drilling down into a specific attribute.
# - _NIX_ATTR: Used to specify an attribute set from inside the expression in `default.nix`.
# This allows for drilling down into a specific attribute in Nix expressions.
# - _NIX_PURE: This variable allows for making the shell pure with the use of --pure.
# Take note that this makes Nix tools like `nix-build` unavailable in the shell.
# - _NIX_KEEP: This variable allows specifying which env vars to keep for Nix pure shell.
#
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
@ -58,20 +59,21 @@ if [ -n "${_NIX_ATTR}" ]; then
entryPoint="default.nix"
fi
# this happens if `make shell` is run already in a Nix shell
# ENTER_NIX_SHELL is the fake command used when `make shell` is run.
# It is just a special string, not a variable, and a marker to not use `--run`.
if [[ $@ == "ENTER_NIX_SHELL" ]]; then
echo -e "${GREEN}Configuring ${_NIX_ATTR:-default} Nix shell for target '${TARGET_OS}'...${NC}"
exec nix-shell ${shellArgs[@]} ${entryPoint}
else
# Not all builds are ready to be run in a pure environment
if [[ "${TARGET_OS}" =~ (android|macos|linux) ]]; then
if [[ -n "${_NIX_PURE}" ]]; then
shellArgs+=("--pure")
pureDesc='pure '
fi
# This variable allows specifying which env vars to keep for Nix pure shell
# The separator is a semicolon
if [[ -n "${_NIX_KEEP}" ]]; then
nixShelArgs+=("--keep ${_NIX_KEEP//;/ --keep }")
shellArgs+=("--keep ${_NIX_KEEP//;/ --keep }")
fi
echo -e "${GREEN}Configuring ${pureDesc}${_NIX_ATTR:-default} Nix shell for target '${TARGET_OS}'...${NC}"
exec nix-shell ${shellArgs[@]} --run "$@" ${entryPoint}

View File

@ -11,12 +11,12 @@ let
leinProjectDepsLocalRepo = localMavenRepoBuilder "lein-project-deps" lein-project-deps;
in stdenv.mkDerivation {
name = "jsbundle-${target-os}";
name = "status-react-build-jsbundle-${target-os}";
src =
let path = ./../..;
in builtins.path { # We use builtins.path so that we can name the resulting derivation, otherwise the name would be taken from the checkout directory, which is outside of our control
inherit path;
name = "status-react-jsbundle-source";
name = "status-react-source-jsbundle";
filter =
# Keep this filter as restrictive as possible in order to avoid unnecessary rebuilds and limit closure size
mkFilter {

View File

@ -47,7 +47,7 @@ nixOpts="--option extra-sandbox-paths ${STORE_FILE} \
--argstr build-type ${BUILD_TYPE} \
--argstr keystore-file ${STORE_FILE} \
--show-trace \
$exportedEnvFlag \
${exportedEnvFlag} \
-A targets.mobile.${TARGET_OS}.release"
# Run the build

View File

@ -10,6 +10,7 @@ let
stdenv = pkgs.stdenvNoCC;
in mkShell {
name = "status-react-shell";
buildInputs = with pkgs; [
# utilities
bash