status-im
/
status-react
mirror of https://github.com/status-im/status-react.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

customize TMPDIR and clean it 

Also:
- parallelize upload and achive stages
- fix s3cmd uploads for combined Jenkinsfile
- fix failing TestFlight clean job

Signed-off-by: Jakub Sokołowski <jakub@status.im>
This commit is contained in:
Vitaliy Vlasov 2019-10-09 18:37:58 +03:00 committed by Jakub Sokołowski
parent b8d3ef3aa5
commit f28fd8fc9c
No known key found for this signature in database
GPG Key ID: 4EF064D0E6D63020
11 changed files with 137 additions and 93 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
Makefile
View File

@ -1,4 +1,4 @@
.PHONY: nix-add-gcroots clean nix-clean disable-githooks react-native-android react-native-ios react-native-desktop test release _list _fix-node-perms .PHONY: nix-add-gcroots clean nix-clean disable-githooks react-native-android react-native-ios react-native-desktop test release _list _fix-node-perms _tmpdir-mk _tmpdir-rm
help: SHELL := /bin/sh help: SHELL := /bin/sh
help: ##@other Show this help help: ##@other Show this help
@ -27,13 +27,18 @@ HELP_FUN = \
} }
HOST_OS := $(shell uname | tr '[:upper:]' '[:lower:]') HOST_OS := $(shell uname | tr '[:upper:]' '[:lower:]')
# This can come from Jenkins
ifndef BUILD_TAG
export BUILD_TAG = $(shell git rev-parse --short HEAD)
endif
# Defines which variables will be kept for Nix pure shell, use semicolon as divider # Defines which variables will be kept for Nix pure shell, use semicolon as divider
export _NIX_KEEP ?= BUILD_ENV export _NIX_KEEP ?= TMPDIR,BUILD_ENV
export NIX_CONF_DIR = $(PWD)/nix export NIX_CONF_DIR = $(PWD)/nix
# We don't want to use /run/user/$UID because it runs out of space too easilly # We don't want to use /run/user/$UID because it runs out of space too easilly
export TMPDIR = /tmp export TMPDIR = /tmp/tmp-status-react-$(BUILD_TAG)
# this has to be specified for both the Node.JS server process and the Qt process
export REACT_SERVER_PORT ?= 5001 # any value different from default 5000 will work; this has to be specified for both the Node.JS server process and the Qt process export REACT_SERVER_PORT ?= 5001
#---------------- #----------------
# Nix targets # Nix targets
@ -89,9 +94,19 @@ _fix-node-perms: ##@prepare Fix permissions so that directory can be cleaned
$(shell test -d node_modules && chmod -R 744 node_modules) $(shell test -d node_modules && chmod -R 744 node_modules)
$(shell test -d node_modules.tmp && chmod -R 744 node_modules.tmp) $(shell test -d node_modules.tmp && chmod -R 744 node_modules.tmp)
_tmpdir-mk: SHELL := /bin/sh
_tmpdir-mk: ##@prepare Create a TMPDIR for temporary files
@mkdir -p "$(TMPDIR)"
# Make sure TMPDIR exists every time make is called
-include _tmpdir-mk
_tmpdir-rm: SHELL := /bin/sh
_tmpdir-rm: ##@prepare Remove TMPDIR
rm -fr "$(TMPDIR)"
clean: SHELL := /bin/sh clean: SHELL := /bin/sh
clean: _fix-node-perms ##@prepare Remove all output folders clean: _fix-node-perms _tmpdir-rm ##@prepare Remove all output folders
git clean -dxf -f git clean -dxf
watchman-clean: export _NIX_ATTR := targets.watchman.shell watchman-clean: export _NIX_ATTR := targets.watchman.shell
watchman-clean: ##@prepare Delete repo directory from watchman watchman-clean: ##@prepare Delete repo directory from watchman

61
ci/Jenkinsfile.android
View File

@ -61,7 +61,7 @@ pipeline {
script { cmn.nix.shell('lein deps :tree', attr: 'targets.leiningen.shell') } script { cmn.nix.shell('lein deps :tree', attr: 'targets.leiningen.shell') }
} }
} }
stage('Parallel') { stage('Parallel Assemble') {
parallel { parallel {
stage('Checks') { stages { stage('Checks') { stages {
stage('Lint') { stage('Lint') {
@ -74,11 +74,6 @@ pipeline {
script { cmn.nix.shell('lein test-cljs', attr: 'targets.leiningen.shell') } script { cmn.nix.shell('lein test-cljs', attr: 'targets.leiningen.shell') }
} }
} }
/* stage('Coverage') {
steps {
script { android.coverage() }
}
} */
} } } }
stage('Build') { stages { stage('Build') { stages {
stage('JSBundle') { stage('JSBundle') {
@ -96,35 +91,41 @@ pipeline {
} } } }
} }
} }
stage('Archive') { stage('Parallel Upload') {
steps { script { parallel {
apks.each { archiveArtifacts it } stage('Archive') {
} } steps { script {
} apks.each { archiveArtifacts it }
stage('Upload') { } }
steps { }
script { stage('Upload') {
def urls = apks.collect { cmn.uploadArtifact(it) } steps {
/* return only the universal APK */ script {
if (urls.size() > 1) { def urls = apks.collect { cmn.uploadArtifact(it) }
env.PKG_URL = urls.find { it.contains('universal') } /* return only the universal APK */
} else { /* if no universal is available pick first */ if (urls.size() > 1) {
env.PKG_URL = urls.first() env.PKG_URL = urls.find { it.contains('universal') }
} } else { /* if no universal is available pick first */
/* build type specific */ env.PKG_URL = urls.first()
switch (btype) { }
case 'release': /* build type specific */
android.uploadToPlayStore(); break; switch (btype) {
case 'nightly': case 'release':
env.DIAWI_URL = android.uploadToDiawi(); break; android.uploadToPlayStore(); break;
case 'e2e': case 'nightly':
env.SAUCE_URL = android.uploadToSauceLabs(); break; env.DIAWI_URL = android.uploadToDiawi(); break;
case 'e2e':
env.SAUCE_URL = android.uploadToSauceLabs(); break;
}
}
} }
} }
} }
} }
stage('Cleanup') { stage('Cleanup') {
steps { sh 'make clean' } steps {
sh 'make clean'
}
} }
} }
post { post {

8
ci/Jenkinsfile.combined
View File

@ -31,7 +31,7 @@ pipeline {
osx = cmn.ci.Build('status-react/combined/desktop-macos') osx = cmn.ci.Build('status-react/combined/desktop-macos')
} } } } } }
stage('Linux') { steps { script { stage('Linux') { steps { script {
nix = cmn.ci.Build('status-react/combined/desktop-linux') tux = cmn.ci.Build('status-react/combined/desktop-linux')
} } } } } }
stage('Windows') { steps { script { stage('Windows') { steps { script {
win = cmn.ci.Build('status-react/combined/desktop-windows') win = cmn.ci.Build('status-react/combined/desktop-windows')
@ -55,7 +55,7 @@ pipeline {
sh('rm -f pkg/*') sh('rm -f pkg/*')
if (btype != 'release') { if (btype != 'release') {
cmn.ci.copyArts(osx) cmn.ci.copyArts(osx)
cmn.ci.copyArts(nix) cmn.ci.copyArts(tux)
cmn.ci.copyArts(win) cmn.ci.copyArts(win)
} }
cmn.ci.copyArts(ios) cmn.ci.copyArts(ios)
@ -78,7 +78,7 @@ pipeline {
iOS: cmn.pkgUrl(ios), /*iOSe2e: cmn.pkgUrl(iose2e),*/ iOS: cmn.pkgUrl(ios), /*iOSe2e: cmn.pkgUrl(iose2e),*/
Diawi: cmn.utils.getEnv(ios, 'DIAWI_URL'), Diawi: cmn.utils.getEnv(ios, 'DIAWI_URL'),
/* desktop */ /* desktop */
App: cmn.pkgUrl(nix), Mac: cmn.pkgUrl(osx), Win: cmn.pkgUrl(win), App: cmn.pkgUrl(tux), Mac: cmn.pkgUrl(osx), Win: cmn.pkgUrl(win),
/* upload the sha256 checksums file too */ /* upload the sha256 checksums file too */
SHA: cmn.uploadArtifact(cmn.utils.pkgFind('sha256')), SHA: cmn.uploadArtifact(cmn.utils.pkgFind('sha256')),
] ]
@ -95,7 +95,7 @@ pipeline {
stage('Publish') { stage('Publish') {
steps { script { steps { script {
switch (btype) { switch (btype) {
case 'nightly': build('misc/status.im'); break //case 'nightly': build('misc/status.im'); break
case 'release': gh.publishReleaseMobile(); break case 'release': gh.publishReleaseMobile(); break
} }
} } } }

7
ci/Jenkinsfile.fastlane.clean
View File

@ -7,6 +7,10 @@ pipeline {
LC_ALL = 'en_US.UTF-8' LC_ALL = 'en_US.UTF-8'
TARGET_OS = 'ios' TARGET_OS = 'ios'
FASTLANE_DISABLE_COLORS = 1 FASTLANE_DISABLE_COLORS = 1
/* See nix/README.md */
NIX_IGNORE_SYMLINK_STORE = 1
/* avoid writing to r/o /nix */
GEM_HOME = '~/.rubygems'
} }
options { options {
@ -25,7 +29,8 @@ pipeline {
nix = load('ci/nix.groovy') nix = load('ci/nix.groovy')
nix.shell( nix.shell(
'bundle install --gemfile=fastlane/Gemfile', 'bundle install --gemfile=fastlane/Gemfile',
attr: 'targets.mobile.fastlane.shell') attr: 'targets.mobile.fastlane.shell'
)
} } } }
} }
stage('Clean Users'){ stage('Clean Users'){

37
ci/Jenkinsfile.ios
View File

@ -27,7 +27,8 @@ pipeline {
LANGUAGE = "en_US.UTF-8" LANGUAGE = "en_US.UTF-8"
TARGET_OS = 'ios' TARGET_OS = 'ios'
NIX_CONF_DIR = "${env.WORKSPACE}/nix" NIX_CONF_DIR = "${env.WORKSPACE}/nix"
NIX_IGNORE_SYMLINK_STORE = 1 // https://github.com/NixOS/nix/issues/2925#issuecomment-499544039 /* See nix/README.md */
NIX_IGNORE_SYMLINK_STORE = 1
FASTLANE_DISABLE_COLORS = 1 FASTLANE_DISABLE_COLORS = 1
BUNDLE_PATH = "${HOME}/.bundle" BUNDLE_PATH = "${HOME}/.bundle"
/* We use EXECUTOR_NUMBER to avoid multiple instances clashing */ /* We use EXECUTOR_NUMBER to avoid multiple instances clashing */
@ -49,7 +50,7 @@ pipeline {
} }
} }
} }
stage('Parallel') { stage('Parallel Assemble') {
parallel { parallel {
stage('Checks') { stages { stage('Checks') { stages {
stage('Lint') { stage('Lint') {
@ -77,20 +78,24 @@ pipeline {
} } } }
} }
} }
stage('Archive') { stage('Parallel Upload') {
steps { parallel {
archiveArtifacts api stage('Archive') {
} steps {
} archiveArtifacts api
stage('Upload') { }
steps { }
script { stage('Upload') {
env.PKG_URL = cmn.uploadArtifact(api) steps {
/* e2e builds get tested in SauceLabs */ script {
if (btype == 'e2e') { env.PKG_URL = cmn.uploadArtifact(api)
env.SAUCE_URL = ios.uploadToSauceLabs() /* e2e builds get tested in SauceLabs */
} else { if (btype == 'e2e') {
env.DIAWI_URL = ios.uploadToDiawi() env.SAUCE_URL = ios.uploadToSauceLabs()
} else {
env.DIAWI_URL = ios.uploadToDiawi()
}
}
} }
} }
} }

22
ci/Jenkinsfile.linux
View File

@ -53,7 +53,7 @@ pipeline {
} }
} }
} }
stage('Parallel') { stage('Parallel Assemble') {
parallel { parallel {
stage('Checks') { stages { stage('Checks') { stages {
stage('Lint') { stage('Lint') {
@ -86,14 +86,18 @@ pipeline {
} } } }
} }
} }
stage('Archive') { stage('Parallel Upload') {
steps { parallel {
archiveArtifacts app stage('Archive') {
} steps {
} archiveArtifacts app
stage('Upload') { }
steps { }
script { env.PKG_URL = cmn.uploadArtifact(app) } stage('Upload') {
steps {
script { env.PKG_URL = cmn.uploadArtifact(app) }
}
}
} }
} }
stage('Cleanup') { stage('Cleanup') {

25
ci/Jenkinsfile.macos
View File

@ -27,7 +27,8 @@ pipeline {
LANGUAGE = "en_US.UTF-8" LANGUAGE = "en_US.UTF-8"
TARGET_OS = 'macos' TARGET_OS = 'macos'
NIX_CONF_DIR = "${env.WORKSPACE}/nix" NIX_CONF_DIR = "${env.WORKSPACE}/nix"
NIX_IGNORE_SYMLINK_STORE = 1 // https://github.com/NixOS/nix/issues/2925#issuecomment-499544039 /* See nix/README.md */
NIX_IGNORE_SYMLINK_STORE = 1
VERBOSE_LEVEL = '3' VERBOSE_LEVEL = '3'
/* We use EXECUTOR_NUMBER to avoid multiple instances clashing */ /* We use EXECUTOR_NUMBER to avoid multiple instances clashing */
LEIN_HOME = "/var/tmp/lein-${EXECUTOR_NUMBER}" LEIN_HOME = "/var/tmp/lein-${EXECUTOR_NUMBER}"
@ -50,7 +51,7 @@ pipeline {
} }
} }
} }
stage('Parallel') { stage('Parallel Assemble') {
parallel { parallel {
stage('Checks') { stages { stage('Checks') { stages {
stage('Lint') { stage('Lint') {
@ -83,14 +84,18 @@ pipeline {
} } } }
} }
} }
stage('Archive') { stage('Parallel Upload') {
steps { parallel {
archiveArtifacts dmg stage('Archive') {
} steps {
} archiveArtifacts dmg
stage('Upload') { }
steps { }
script { env.PKG_URL = cmn.uploadArtifact(dmg) } stage('Upload') {
steps {
script { env.PKG_URL = cmn.uploadArtifact(dmg) }
}
}
} }
} }
stage('Cleanup') { stage('Cleanup') {

22
ci/Jenkinsfile.windows
View File

@ -56,7 +56,7 @@ pipeline {
} }
} }
} }
stage('Parallel') { stage('Parallel Assemble') {
parallel { parallel {
stage('Checks') { stages { stage('Checks') { stages {
stage('Lint') { stage('Lint') {
@ -89,14 +89,18 @@ pipeline {
} } } }
} }
} }
stage('Archive') { stage('Parallel Upload') {
steps { parallel {
archiveArtifacts app stage('Archive') {
} steps {
} archiveArtifacts app
stage('Upload') { }
steps { }
script { env.PKG_URL = cmn.uploadArtifact(app) } stage('Upload') {
steps {
script { env.PKG_URL = cmn.uploadArtifact(app) }
}
}
} }
} }
stage('Cleanup') { stage('Cleanup') {

12
ci/common.groovy
View File

@ -59,11 +59,11 @@ def prep(type = 'nightly') {
if (env.TARGET_OS == 'macos' || env.TARGET_OS == 'linux' || env.TARGET_OS == 'windows') { if (env.TARGET_OS == 'macos' || env.TARGET_OS == 'linux' || env.TARGET_OS == 'windows') {
/* node deps, pods, and status-go download */ /* node deps, pods, and status-go download */
utils.nix.shell('scripts/prepare-for-desktop-platform.sh', pure: false) nix.shell('scripts/prepare-for-desktop-platform.sh', pure: false)
sh('scripts/copy-translations.sh') sh('scripts/copy-translations.sh')
} else if (env.TARGET_OS != 'android') { } else if (env.TARGET_OS != 'android') {
// run script in the nix shell so that node_modules gets instantiated before attempting the copies // run script in the nix shell so that node_modules gets instantiated before attempting the copies
utils.nix.shell('scripts/copy-translations.sh chmod') nix.shell('scripts/copy-translations.sh chmod')
} }
} }
@ -86,15 +86,15 @@ def uploadArtifact(path) {
usernameVariable: 'DO_ACCESS_KEY', usernameVariable: 'DO_ACCESS_KEY',
passwordVariable: 'DO_SECRET_KEY' passwordVariable: 'DO_SECRET_KEY'
)]) { )]) {
nix.shell(""" shell("""
s3cmd \\ s3cmd ${customOpts} \\
--acl-public ${customOpts} \\ --acl-public \\
--host="${domain}" \\ --host="${domain}" \\
--host-bucket="%(bucket)s.${domain}" \\ --host-bucket="%(bucket)s.${domain}" \\
--access_key=${DO_ACCESS_KEY} \\ --access_key=${DO_ACCESS_KEY} \\
--secret_key=${DO_SECRET_KEY} \\ --secret_key=${DO_SECRET_KEY} \\
put ${path} s3://${bucket}/ put ${path} s3://${bucket}/
""", pure: false) """)
} }
return "https://${bucket}.${domain}/${utils.getFilename(path)}" return "https://${bucket}.${domain}/${utils.getFilename(path)}"
} }

6
nix/build.sh
View File

@ -55,6 +55,12 @@ nixOpts=(
"default.nix" "default.nix"
) )
# This variable allows specifying which env vars to keep for Nix pure shell
# The separator is a semicolon
if [[ -n "${_NIX_KEEP}" ]]; then
nixOpts+=("--keep ${_NIX_KEEP//;/ --keep }")
fi
# Run the actual build # Run the actual build
echo "Running: nix-build ${nixOpts[@]}" echo "Running: nix-build ${nixOpts[@]}"
nixResultPath=$(nix-build ${nixOpts[@]}) nixResultPath=$(nix-build ${nixOpts[@]})

1
shell.nix
View File

@ -25,7 +25,6 @@ let
gnumake gnumake
jq jq
wget wget
s3cmd
]; ];
in mkShell { in mkShell {