diff --git a/Makefile b/Makefile index 4859002e58..ccafc53335 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: nix-add-gcroots clean nix-clean disable-githooks react-native-android react-native-ios react-native-desktop test release _list _fix-node-perms +.PHONY: nix-add-gcroots clean nix-clean disable-githooks react-native-android react-native-ios react-native-desktop test release _list _fix-node-perms _tmpdir-mk _tmpdir-rm help: SHELL := /bin/sh help: ##@other Show this help @@ -27,13 +27,18 @@ HELP_FUN = \ } HOST_OS := $(shell uname | tr '[:upper:]' '[:lower:]') +# This can come from Jenkins +ifndef BUILD_TAG +export BUILD_TAG = $(shell git rev-parse --short HEAD) +endif + # Defines which variables will be kept for Nix pure shell, use semicolon as divider -export _NIX_KEEP ?= BUILD_ENV +export _NIX_KEEP ?= TMPDIR,BUILD_ENV export NIX_CONF_DIR = $(PWD)/nix # We don't want to use /run/user/$UID because it runs out of space too easilly -export TMPDIR = /tmp - -export REACT_SERVER_PORT ?= 5001 # any value different from default 5000 will work; this has to be specified for both the Node.JS server process and the Qt process +export TMPDIR = /tmp/tmp-status-react-$(BUILD_TAG) +# this has to be specified for both the Node.JS server process and the Qt process +export REACT_SERVER_PORT ?= 5001 #---------------- # Nix targets @@ -89,9 +94,19 @@ _fix-node-perms: ##@prepare Fix permissions so that directory can be cleaned $(shell test -d node_modules && chmod -R 744 node_modules) $(shell test -d node_modules.tmp && chmod -R 744 node_modules.tmp) +_tmpdir-mk: SHELL := /bin/sh +_tmpdir-mk: ##@prepare Create a TMPDIR for temporary files + @mkdir -p "$(TMPDIR)" +# Make sure TMPDIR exists every time make is called +-include _tmpdir-mk + +_tmpdir-rm: SHELL := /bin/sh +_tmpdir-rm: ##@prepare Remove TMPDIR + rm -fr "$(TMPDIR)" + clean: SHELL := /bin/sh -clean: _fix-node-perms ##@prepare Remove all output folders - git clean -dxf -f +clean: _fix-node-perms _tmpdir-rm ##@prepare Remove all output folders + git clean -dxf watchman-clean: export _NIX_ATTR := targets.watchman.shell watchman-clean: ##@prepare Delete repo directory from watchman diff --git a/ci/Jenkinsfile.android b/ci/Jenkinsfile.android index 990432fb3b..958cd4d38b 100644 --- a/ci/Jenkinsfile.android +++ b/ci/Jenkinsfile.android @@ -61,7 +61,7 @@ pipeline { script { cmn.nix.shell('lein deps :tree', attr: 'targets.leiningen.shell') } } } - stage('Parallel') { + stage('Parallel Assemble') { parallel { stage('Checks') { stages { stage('Lint') { @@ -74,11 +74,6 @@ pipeline { script { cmn.nix.shell('lein test-cljs', attr: 'targets.leiningen.shell') } } } - /* stage('Coverage') { - steps { - script { android.coverage() } - } - } */ } } stage('Build') { stages { stage('JSBundle') { @@ -96,35 +91,41 @@ pipeline { } } } } - stage('Archive') { - steps { script { - apks.each { archiveArtifacts it } - } } - } - stage('Upload') { - steps { - script { - def urls = apks.collect { cmn.uploadArtifact(it) } - /* return only the universal APK */ - if (urls.size() > 1) { - env.PKG_URL = urls.find { it.contains('universal') } - } else { /* if no universal is available pick first */ - env.PKG_URL = urls.first() - } - /* build type specific */ - switch (btype) { - case 'release': - android.uploadToPlayStore(); break; - case 'nightly': - env.DIAWI_URL = android.uploadToDiawi(); break; - case 'e2e': - env.SAUCE_URL = android.uploadToSauceLabs(); break; + stage('Parallel Upload') { + parallel { + stage('Archive') { + steps { script { + apks.each { archiveArtifacts it } + } } + } + stage('Upload') { + steps { + script { + def urls = apks.collect { cmn.uploadArtifact(it) } + /* return only the universal APK */ + if (urls.size() > 1) { + env.PKG_URL = urls.find { it.contains('universal') } + } else { /* if no universal is available pick first */ + env.PKG_URL = urls.first() + } + /* build type specific */ + switch (btype) { + case 'release': + android.uploadToPlayStore(); break; + case 'nightly': + env.DIAWI_URL = android.uploadToDiawi(); break; + case 'e2e': + env.SAUCE_URL = android.uploadToSauceLabs(); break; + } + } } } } } stage('Cleanup') { - steps { sh 'make clean' } + steps { + sh 'make clean' + } } } post { diff --git a/ci/Jenkinsfile.combined b/ci/Jenkinsfile.combined index dd920a08ae..b076f1b56a 100644 --- a/ci/Jenkinsfile.combined +++ b/ci/Jenkinsfile.combined @@ -31,7 +31,7 @@ pipeline { osx = cmn.ci.Build('status-react/combined/desktop-macos') } } } stage('Linux') { steps { script { - nix = cmn.ci.Build('status-react/combined/desktop-linux') + tux = cmn.ci.Build('status-react/combined/desktop-linux') } } } stage('Windows') { steps { script { win = cmn.ci.Build('status-react/combined/desktop-windows') @@ -55,7 +55,7 @@ pipeline { sh('rm -f pkg/*') if (btype != 'release') { cmn.ci.copyArts(osx) - cmn.ci.copyArts(nix) + cmn.ci.copyArts(tux) cmn.ci.copyArts(win) } cmn.ci.copyArts(ios) @@ -78,7 +78,7 @@ pipeline { iOS: cmn.pkgUrl(ios), /*iOSe2e: cmn.pkgUrl(iose2e),*/ Diawi: cmn.utils.getEnv(ios, 'DIAWI_URL'), /* desktop */ - App: cmn.pkgUrl(nix), Mac: cmn.pkgUrl(osx), Win: cmn.pkgUrl(win), + App: cmn.pkgUrl(tux), Mac: cmn.pkgUrl(osx), Win: cmn.pkgUrl(win), /* upload the sha256 checksums file too */ SHA: cmn.uploadArtifact(cmn.utils.pkgFind('sha256')), ] @@ -95,7 +95,7 @@ pipeline { stage('Publish') { steps { script { switch (btype) { - case 'nightly': build('misc/status.im'); break + //case 'nightly': build('misc/status.im'); break case 'release': gh.publishReleaseMobile(); break } } } diff --git a/ci/Jenkinsfile.fastlane.clean b/ci/Jenkinsfile.fastlane.clean index 0ba8a68b6e..5e777678d1 100644 --- a/ci/Jenkinsfile.fastlane.clean +++ b/ci/Jenkinsfile.fastlane.clean @@ -7,6 +7,10 @@ pipeline { LC_ALL = 'en_US.UTF-8' TARGET_OS = 'ios' FASTLANE_DISABLE_COLORS = 1 + /* See nix/README.md */ + NIX_IGNORE_SYMLINK_STORE = 1 + /* avoid writing to r/o /nix */ + GEM_HOME = '~/.rubygems' } options { @@ -25,7 +29,8 @@ pipeline { nix = load('ci/nix.groovy') nix.shell( 'bundle install --gemfile=fastlane/Gemfile', - attr: 'targets.mobile.fastlane.shell') + attr: 'targets.mobile.fastlane.shell' + ) } } } stage('Clean Users'){ diff --git a/ci/Jenkinsfile.ios b/ci/Jenkinsfile.ios index fb2ba9e591..23f2c79440 100644 --- a/ci/Jenkinsfile.ios +++ b/ci/Jenkinsfile.ios @@ -27,7 +27,8 @@ pipeline { LANGUAGE = "en_US.UTF-8" TARGET_OS = 'ios' NIX_CONF_DIR = "${env.WORKSPACE}/nix" - NIX_IGNORE_SYMLINK_STORE = 1 // https://github.com/NixOS/nix/issues/2925#issuecomment-499544039 + /* See nix/README.md */ + NIX_IGNORE_SYMLINK_STORE = 1 FASTLANE_DISABLE_COLORS = 1 BUNDLE_PATH = "${HOME}/.bundle" /* We use EXECUTOR_NUMBER to avoid multiple instances clashing */ @@ -49,7 +50,7 @@ pipeline { } } } - stage('Parallel') { + stage('Parallel Assemble') { parallel { stage('Checks') { stages { stage('Lint') { @@ -77,20 +78,24 @@ pipeline { } } } } - stage('Archive') { - steps { - archiveArtifacts api - } - } - stage('Upload') { - steps { - script { - env.PKG_URL = cmn.uploadArtifact(api) - /* e2e builds get tested in SauceLabs */ - if (btype == 'e2e') { - env.SAUCE_URL = ios.uploadToSauceLabs() - } else { - env.DIAWI_URL = ios.uploadToDiawi() + stage('Parallel Upload') { + parallel { + stage('Archive') { + steps { + archiveArtifacts api + } + } + stage('Upload') { + steps { + script { + env.PKG_URL = cmn.uploadArtifact(api) + /* e2e builds get tested in SauceLabs */ + if (btype == 'e2e') { + env.SAUCE_URL = ios.uploadToSauceLabs() + } else { + env.DIAWI_URL = ios.uploadToDiawi() + } + } } } } diff --git a/ci/Jenkinsfile.linux b/ci/Jenkinsfile.linux index e1d4a4bdc0..3ecfbd53b0 100644 --- a/ci/Jenkinsfile.linux +++ b/ci/Jenkinsfile.linux @@ -53,7 +53,7 @@ pipeline { } } } - stage('Parallel') { + stage('Parallel Assemble') { parallel { stage('Checks') { stages { stage('Lint') { @@ -86,14 +86,18 @@ pipeline { } } } } - stage('Archive') { - steps { - archiveArtifacts app - } - } - stage('Upload') { - steps { - script { env.PKG_URL = cmn.uploadArtifact(app) } + stage('Parallel Upload') { + parallel { + stage('Archive') { + steps { + archiveArtifacts app + } + } + stage('Upload') { + steps { + script { env.PKG_URL = cmn.uploadArtifact(app) } + } + } } } stage('Cleanup') { diff --git a/ci/Jenkinsfile.macos b/ci/Jenkinsfile.macos index f28a035784..7f5710b419 100644 --- a/ci/Jenkinsfile.macos +++ b/ci/Jenkinsfile.macos @@ -27,7 +27,8 @@ pipeline { LANGUAGE = "en_US.UTF-8" TARGET_OS = 'macos' NIX_CONF_DIR = "${env.WORKSPACE}/nix" - NIX_IGNORE_SYMLINK_STORE = 1 // https://github.com/NixOS/nix/issues/2925#issuecomment-499544039 + /* See nix/README.md */ + NIX_IGNORE_SYMLINK_STORE = 1 VERBOSE_LEVEL = '3' /* We use EXECUTOR_NUMBER to avoid multiple instances clashing */ LEIN_HOME = "/var/tmp/lein-${EXECUTOR_NUMBER}" @@ -50,7 +51,7 @@ pipeline { } } } - stage('Parallel') { + stage('Parallel Assemble') { parallel { stage('Checks') { stages { stage('Lint') { @@ -83,14 +84,18 @@ pipeline { } } } } - stage('Archive') { - steps { - archiveArtifacts dmg - } - } - stage('Upload') { - steps { - script { env.PKG_URL = cmn.uploadArtifact(dmg) } + stage('Parallel Upload') { + parallel { + stage('Archive') { + steps { + archiveArtifacts dmg + } + } + stage('Upload') { + steps { + script { env.PKG_URL = cmn.uploadArtifact(dmg) } + } + } } } stage('Cleanup') { diff --git a/ci/Jenkinsfile.windows b/ci/Jenkinsfile.windows index 4cfa104153..4aac303d5c 100644 --- a/ci/Jenkinsfile.windows +++ b/ci/Jenkinsfile.windows @@ -56,7 +56,7 @@ pipeline { } } } - stage('Parallel') { + stage('Parallel Assemble') { parallel { stage('Checks') { stages { stage('Lint') { @@ -89,14 +89,18 @@ pipeline { } } } } - stage('Archive') { - steps { - archiveArtifacts app - } - } - stage('Upload') { - steps { - script { env.PKG_URL = cmn.uploadArtifact(app) } + stage('Parallel Upload') { + parallel { + stage('Archive') { + steps { + archiveArtifacts app + } + } + stage('Upload') { + steps { + script { env.PKG_URL = cmn.uploadArtifact(app) } + } + } } } stage('Cleanup') { diff --git a/ci/common.groovy b/ci/common.groovy index f222d1d2f3..f88d68f67c 100644 --- a/ci/common.groovy +++ b/ci/common.groovy @@ -59,11 +59,11 @@ def prep(type = 'nightly') { if (env.TARGET_OS == 'macos' || env.TARGET_OS == 'linux' || env.TARGET_OS == 'windows') { /* node deps, pods, and status-go download */ - utils.nix.shell('scripts/prepare-for-desktop-platform.sh', pure: false) + nix.shell('scripts/prepare-for-desktop-platform.sh', pure: false) sh('scripts/copy-translations.sh') } else if (env.TARGET_OS != 'android') { // run script in the nix shell so that node_modules gets instantiated before attempting the copies - utils.nix.shell('scripts/copy-translations.sh chmod') + nix.shell('scripts/copy-translations.sh chmod') } } @@ -86,15 +86,15 @@ def uploadArtifact(path) { usernameVariable: 'DO_ACCESS_KEY', passwordVariable: 'DO_SECRET_KEY' )]) { - nix.shell(""" - s3cmd \\ - --acl-public ${customOpts} \\ + shell(""" + s3cmd ${customOpts} \\ + --acl-public \\ --host="${domain}" \\ --host-bucket="%(bucket)s.${domain}" \\ --access_key=${DO_ACCESS_KEY} \\ --secret_key=${DO_SECRET_KEY} \\ put ${path} s3://${bucket}/ - """, pure: false) + """) } return "https://${bucket}.${domain}/${utils.getFilename(path)}" } diff --git a/nix/build.sh b/nix/build.sh index 7f644c6b36..c4372a3574 100755 --- a/nix/build.sh +++ b/nix/build.sh @@ -55,6 +55,12 @@ nixOpts=( "default.nix" ) +# This variable allows specifying which env vars to keep for Nix pure shell +# The separator is a semicolon +if [[ -n "${_NIX_KEEP}" ]]; then + nixOpts+=("--keep ${_NIX_KEEP//;/ --keep }") +fi + # Run the actual build echo "Running: nix-build ${nixOpts[@]}" nixResultPath=$(nix-build ${nixOpts[@]}) diff --git a/shell.nix b/shell.nix index 36ad75c7b6..64ab4f74c2 100644 --- a/shell.nix +++ b/shell.nix @@ -25,7 +25,6 @@ let gnumake jq wget - s3cmd ]; in mkShell {