From 5fb33cc461a0656c7f97ac4d57487205500620da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Soko=C5=82owski?= Date: Tue, 21 Aug 2018 11:17:25 -0400 Subject: [PATCH] split groovy scripts more, use job parameters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jakub SokoĊ‚owski --- ci/Jenkinsfile.android | 25 +++++------ ci/Jenkinsfile.combined | 85 ++++++++++++++---------------------- ci/Jenkinsfile.ios | 8 ++-- ci/android.groovy | 66 ++++++++++++++++++++++++++++ ci/common.groovy | 23 ++++++++++ ci/desktop.groovy | 8 ++++ ci/ios.groovy | 31 ++++++++++++++ ci/mobile.groovy | 95 ++++++----------------------------------- fastlane/Fastfile | 3 +- scripts/build_no.sh | 52 +++++++++------------- 10 files changed, 211 insertions(+), 185 deletions(-) create mode 100644 ci/android.groovy create mode 100644 ci/ios.groovy diff --git a/ci/Jenkinsfile.android b/ci/Jenkinsfile.android index 3d6a716603..69668cbb9c 100644 --- a/ci/Jenkinsfile.android +++ b/ci/Jenkinsfile.android @@ -5,7 +5,7 @@ pipeline { buildDiscarder(logRotator( numToKeepStr: '10', daysToKeepStr: '30', - artifactNumToKeepStr: '4', + artifactNumToKeepStr: '6', )) } @@ -25,9 +25,10 @@ pipeline { stage('Prep') { steps { script { + print "Running ${params.BUILD_TYPE} build!" /* Necessary to load methods */ mobile = load 'ci/mobile.groovy' - mobile.prepDeps() + mobile.prep(type: params.BUILD_TYPE) } } } @@ -43,25 +44,25 @@ pipeline { } stage('Compile') { steps { - script { apk = mobile.compileAndroid(e2e: params.RUN_E2E) } - } - } - stage('Bundle') { - when { expression { !params.RUN_E2E } } - steps { - script { mobile.bundleAndroid() } + script { apk = mobile.android.compile(type: params.BUILD_TYPE) } } } stage('Archive') { - when { expression { !params.RUN_E2E } } steps { script { archiveArtifacts apk } } } + stage('Upload') { + when { expression { params.BUILD_TYPE == 'release' } } + steps { + script { mobile.android.uploadToPlayStore() } + } + } stage('Run e2e') { - when { expression { params.RUN_E2E } } + when { expression { params.BUILD_TYPE == 'e2e' } } steps { script { - apk = mobile.uploadSauceLabs() + mobile.android.uploadToDiawi() + apk = mobile.android.uploadToSauceLabs() build( job: 'end-to-end-tests/status-app-nightly', wait: false, parameters: [string(name: 'apk', value: "--apk=${apk}")] diff --git a/ci/Jenkinsfile.combined b/ci/Jenkinsfile.combined index 813b285e11..afeed22c9b 100644 --- a/ci/Jenkinsfile.combined +++ b/ci/Jenkinsfile.combined @@ -14,69 +14,49 @@ pipeline { stage('Tag') { steps { script { common = load('ci/common.groovy') - /* to avoid race conditions in parallel builds */ + /* to avoid missing build tag parallel builds */ print "Build Number: ${common.tagBuild()}" } } } stage('Build') { parallel { - stage('MacOS') { - steps { script { - osx = build('status-react/combined/desktop-macos') - } } - } - stage('Linux') { - steps { script { - nix = build('status-react/combined/desktop-linux') - } } - } - stage('iOS') { - steps { script { - ios = build('status-react/combined/mobile-ios') - } } - } - stage('Android') { - steps { script { - dro = build('status-react/combined/mobile-android') - } } - } - stage('Android e2e') { - steps { - build('status-react/combined/mobile-android-e2e') - } - } + stage('MacOS') { steps { script { + osx = common.buildBranch('status-react/combined/desktop-macos') + } } } + stage('Linux') { steps { script { + nix = common.buildBranch('status-react/combined/desktop-linux') + } } } + stage('iOS') { steps { script { + ios = common.buildBranch('status-react/combined/mobile-ios') + } } } + stage('Android') { steps { script { + dro = common.buildBranch('status-react/combined/mobile-android') + } } } + stage('Android e2e') { steps { script { + e2e = common.buildBranch('status-react/combined/mobile-android', 'e2e') + } } } } } stage('Archive') { - steps { + steps { script { sh('rm -f pkg/*') - copyArtifacts( - projectName: 'status-react/combined/mobile-ios', target: 'pkg', - flatten: true, selector: specific("${ios.number}") - ) - copyArtifacts( - projectName: 'status-react/combined/mobile-android', target: 'pkg', - flatten: true, selector: specific("${dro.number}") - ) - copyArtifacts( - projectName: 'status-react/combined/desktop-macos', target: 'pkg', - flatten: true, selector: specific("${osx.number}") - ) - copyArtifacts( - projectName: 'status-react/combined/desktop-linux', target: 'pkg', - flatten: true, selector: specific("${nix.number}") - ) + common.copyArts('status-react/combined/desktop-macos', osx.number) + common.copyArts('status-react/combined/desktop-linux', nix.number) + common.copyArts('status-react/combined/mobile-android', dro.number) + common.copyArts('status-react/combined/mobile-android', e2e.number) archiveArtifacts('pkg/*') - } + } } } stage('Upload') { - when { expression { params.PUBLISH } } + when { expression { params.BUILD_TYPE == 'nightly' } } steps { script { - def pkg = "StatusIm-${GIT_COMMIT.take(6)}" - apkUrl = common.uploadArtifact('pkg', "${pkg}.apk") - ipaUrl = common.uploadArtifact('pkg', "${pkg}.ipa") - dmgUrl = common.uploadArtifact('pkg', "${pkg}.dmg") - appUrl = common.uploadArtifact('pkg', "${pkg}.AppImage") + def pkg = "pkg/StatusIm-${GIT_COMMIT.take(6)}" + e2eUrl = common.uploadArtifact("${pkg}-e2e.apk") + apkUrl = common.uploadArtifact("${pkg}.apk") + dmgUrl = common.uploadArtifact("${pkg}.dmg") + appUrl = common.uploadArtifact("${pkg}.AppImage") + /* special case for iOS Diawi link */ + ipaUrl = ios.getEnvironment().DIAWI_URL } } } stage('Notify') { @@ -86,7 +66,8 @@ pipeline { "Build success! "+ "<${currentBuild.absoluteUrl}|${currentBuild.displayName}> "+ "(${currentBuild.durationString})\n"+ - (params.PUBLISH ? + (params.BUILD_TYPE == 'nightly' ? + "E2E: ${e2eUrl}\n"+ "APK: ${apkUrl}\n"+ "IPA: ${ipaUrl}\n"+ "DMG: ${dmgUrl}\n"+ @@ -98,7 +79,7 @@ pipeline { } } stage('Publish') { - when { expression { params.PUBLISH } } + when { expression { params.BUILD_TYPE == 'nightly' } } steps { build( job: 'misc/status-im.github.io-update_env', diff --git a/ci/Jenkinsfile.ios b/ci/Jenkinsfile.ios index 5f3c91caa5..b42c351c2c 100644 --- a/ci/Jenkinsfile.ios +++ b/ci/Jenkinsfile.ios @@ -29,9 +29,10 @@ pipeline { stage('Prep') { steps { script { + print "Running ${params.BUILD_TYPE} build!" /* Necessary to load methods */ mobile = load 'ci/mobile.groovy' - mobile.prepDeps() + mobile.prep(type: params.BUILD_TYPE) } } } @@ -47,7 +48,7 @@ pipeline { } stage('Compile') { steps { - script { api = mobile.compileiOS() } + script { api = mobile.ios.compile(type: params.BUILD_TYPE) } } } stage('Archive') { @@ -56,9 +57,8 @@ pipeline { } } stage('Upload') { - when { expression { params.RUN_E2E } } steps { - script { env.DIAWI_URL = uploadiOS() } + script { env.DIAWI_URL = mobile.ios.uploadToDiawi() } } } } diff --git a/ci/android.groovy b/ci/android.groovy new file mode 100644 index 0000000000..d0e2430b41 --- /dev/null +++ b/ci/android.groovy @@ -0,0 +1,66 @@ +common = load 'ci/common.groovy' + +def uploadArtifact() { + def artifact_dir = pwd() + '/android/app/build/outputs/apk/release/' + println (artifact_dir + 'app-release.apk') + def artifact = (artifact_dir + 'app-release.apk') + def server = Artifactory.server('artifacts') + def filename = "im.status.ethereum-${GIT_COMMIT.take(6)}-n-fl.apk" + def newArtifact = (artifact_dir + filename) + sh "cp ${artifact} ${newArtifact}" + def uploadSpec = '{ "files": [ { "pattern": "*apk/release/' + filename + '", "target": "nightlies-local" }]}' + def buildInfo = server.upload(uploadSpec) + return 'http://artifacts.status.im:8081/artifactory/nightlies-local/' + filename +} + +def compile(type = 'nightly') { + common.tagBuild() + def gradleOpt = '' + if (type == 'release') { + gradleOpt = "-PreleaseVersion=${common.version()}" + } + dir('android') { + sh './gradlew react-native-android:installArchives' + sh "./gradlew assembleRelease ${gradleOpt}" + } + def pkg = "StatusIm-${GIT_COMMIT.take(6)}${(type = 'e2e' ?: '-e2e')}.apk" + sh "cp android/app/build/outputs/apk/release/app-release.apk ${pkg}" + return pkg +} + +def uploadToPlayStore() { + withCredentials([ + string(credentialsId: "SUPPLY_JSON_KEY_DATA", variable: 'GOOGLE_PLAY_JSON_KEY'), + string(credentialsId: "SLACK_URL", variable: 'SLACK_URL') + ]) { + sh 'bundle exec fastlane android nightly' + } +} + +def uploadToSauceLabs() { + env.SAUCE_LABS_APK = "im.status.ethereum-e2e-${GIT_COMMIT.take(6)}.apk" + withCredentials([ + string(credentialsId: 'SAUCE_ACCESS_KEY', variable: 'SAUCE_ACCESS_KEY'), + string(credentialsId: 'SAUCE_USERNAME', variable: 'SAUCE_USERNAME'), + string(credentialsId: 'GIT_HUB_TOKEN', variable: 'GITHUB_TOKEN'), + string(credentialsId: 'SLACK_JENKINS_WEBHOOK', variable: 'SLACK_URL') + ]) { + sh 'fastlane android saucelabs' + } + return env.SAUCE_LABS_APK +} + +def uploadToDiawi() { + env.SAUCE_LABS_APK = "im.status.ethereum-e2e-${GIT_COMMIT.take(6)}.apk" + withCredentials([ + string(credentialsId: 'SAUCE_ACCESS_KEY', variable: 'SAUCE_ACCESS_KEY'), + string(credentialsId: 'SAUCE_USERNAME', variable: 'SAUCE_USERNAME'), + string(credentialsId: 'GIT_HUB_TOKEN', variable: 'GITHUB_TOKEN'), + string(credentialsId: 'SLACK_JENKINS_WEBHOOK', variable: 'SLACK_URL') + ]) { + sh 'fastlane android saucelabs' + } + return env.SAUCE_LABS_APK +} + +return this diff --git a/ci/common.groovy b/ci/common.groovy index f1fda97bc9..dc354fd6e4 100644 --- a/ci/common.groovy +++ b/ci/common.groovy @@ -1,3 +1,26 @@ +def version() { + return readFile("${env.WORKSPACE}/VERSION").trim() +} + +def buildBranch(name = null, buildType = params.BUILD_TYPE) { + /* always pass the BRANCH and BUILD_TYPE params with current branch */ + return build( + job: name, + parameters: [ + [name: 'BRANCH', value: env.GIT_BRANCH, $class: 'StringParameterValue'], + [name: 'BUILD_TYPE', value: buildType, $class: 'StringParameterValue'], + ]) +} + +def copyArts(projectName, buildNo) { + copyArtifacts( + projectName: projectName, + target: 'pkg', + flatten: true, + selector: specific("${buildNo}") + ) +} + def installJSDeps(platform) { def attempt = 1 def maxAttempts = 10 diff --git a/ci/desktop.groovy b/ci/desktop.groovy index cb7edb063e..4ee1ff3a45 100644 --- a/ci/desktop.groovy +++ b/ci/desktop.groovy @@ -13,6 +13,12 @@ external_modules_dir = [ 'modules/react-native-status/desktop', ] +external_fonts = [ + '../../../../../resources/fonts/SF-Pro-Text-Regular.otf', + '../../../../../resources/fonts/SF-Pro-Text-Medium.otf', + '../../../../../resources/fonts/SF-Pro-Text-Light.otf', +] + def cleanupBuild() { sh """ rm -rf \\ @@ -88,6 +94,7 @@ def compileLinux() { cmake -Wno-dev \\ -DCMAKE_BUILD_TYPE=Release \\ -DEXTERNAL_MODULES_DIR='${external_modules_dir.join(";")}' \\ + -DDESKTOP_FONTS='${external_fonts.join(";")}' \\ -DJS_BUNDLE_PATH='${workspace}/${packageFolder}/StatusIm.jsbundle' \\ -DCMAKE_CXX_FLAGS:='-DBUILD_FOR_BUNDLE=1' """ @@ -154,6 +161,7 @@ def compileMacOS() { cmake -Wno-dev \\ -DCMAKE_BUILD_TYPE=Release \\ -DEXTERNAL_MODULES_DIR='${external_modules_dir.join(";")}' \\ + -DDESKTOP_FONTS='${external_fonts.join(";")}' \\ -DJS_BUNDLE_PATH='${workspace}/${packageFolder}/StatusIm.jsbundle' \\ -DCMAKE_CXX_FLAGS:='-DBUILD_FOR_BUNDLE=1' """ diff --git a/ci/ios.groovy b/ci/ios.groovy new file mode 100644 index 0000000000..2a4254bf29 --- /dev/null +++ b/ci/ios.groovy @@ -0,0 +1,31 @@ +def compile(type = 'nightly') { + def target = (type == 'release' ? 'adhoc' : 'nightly') + withCredentials([ + string(credentialsId: 'SLACK_URL', variable: 'SLACK_URL'), + string(credentialsId: "slave-pass-${env.NODE_NAME}", variable: 'KEYCHAIN_PASSWORD'), + string(credentialsId: 'FASTLANE_PASSWORD', variable: 'FASTLANE_PASSWORD'), + string(credentialsId: 'APPLE_ID', variable: 'APPLE_ID'), + string(credentialsId: 'fastlane-match-password', variable:'MATCH_PASSWORD') + ]) { + sh "plutil -replace CFBundleShortVersionString -string ${common.version()} ios/StatusIm/Info.plist" + sh "plutil -replace CFBundleVersion -string ${common.tagBuild()} ios/StatusIm/Info.plist" + sh "fastlane ios ${target}" + } + def pkg = "StatusIm-${GIT_COMMIT.take(6)}.ipa" + sh "cp status-adhoc/StatusIm.ipa ${pkg}" + return pkg +} + +def uploadToDiawi() { + withCredentials([ + string(credentialsId: 'diawi-token', variable: 'DIAWI_TOKEN'), + string(credentialsId: 'GIT_HUB_TOKEN', variable: 'GITHUB_TOKEN'), + string(credentialsId: 'SLACK_JENKINS_WEBHOOK', variable: 'SLACK_URL') + ]) { + sh 'fastlane ios upload_diawi' + } + diawiUrl = readFile "${env.WORKSPACE}/fastlane/diawi.out" + return diawiUrl +} + +return this diff --git a/ci/mobile.groovy b/ci/mobile.groovy index 2199f2bd6d..de7bf67e84 100644 --- a/ci/mobile.groovy +++ b/ci/mobile.groovy @@ -1,22 +1,19 @@ common = load 'ci/common.groovy' +ios = load 'ci/ios.groovy' +android = load 'ci/android.groovy' -def uploadArtifact() { - def artifact_dir = pwd() + '/android/app/build/outputs/apk/release/' - println (artifact_dir + 'app-release.apk') - def artifact = (artifact_dir + 'app-release.apk') - def server = Artifactory.server('artifacts') - def filename = "im.status.ethereum-${GIT_COMMIT.take(6)}-n-fl.apk" - def newArtifact = (artifact_dir + filename) - sh "cp ${artifact} ${newArtifact}" - def uploadSpec = '{ "files": [ { "pattern": "*apk/release/' + filename + '", "target": "nightlies-local" }]}' - def buildInfo = server.upload(uploadSpec) - return 'http://artifacts.status.im:8081/artifactory/nightlies-local/' + filename -} - -def prepDeps() { - sh 'rm -rf node_modules' - sh 'cp .env.nightly .env' +def prep(type = 'debug') { + /* select type of build */ + switch (type) { + case 'debug': + sh 'cp .env.nightly .env'; break + case 'release': + sh 'cp .env.prod .env'; break + case 'e2e': + sh 'cp .env.e2e .env'; break + } common.installJSDeps('mobile') + /* install Maven dependencies */ sh 'mvn -f modules/react-native-status/ios/RCTStatus dependency:unpack' /* generate ios/StatusIm.xcworkspace */ dir('ios') { @@ -32,70 +29,4 @@ def leinBuild() { sh 'lein prod-build' } -def compileAndroid(e2e = false) { - if (e2e) { - env.ENVFILE=".env.e2e" - } - dir('android') { - sh './gradlew react-native-android:installArchives' - sh './gradlew assembleRelease' - } - def pkg = "StatusIm-${GIT_COMMIT.take(6)}.apk" - sh "cp android/app/build/outputs/apk/release/app-release.apk ${pkg}" - return pkg -} - -def bundleAndroid() { - withCredentials([ - string(credentialsId: "SUPPLY_JSON_KEY_DATA", variable: 'GOOGLE_PLAY_JSON_KEY'), - string(credentialsId: "SLACK_URL", variable: 'SLACK_URL') - ]) { - sh 'bundle exec fastlane android nightly' - } -} - -def uploadSauceLabs() { - env.SAUCE_LABS_APK = "im.status.ethereum-e2e-${GIT_COMMIT.take(6)}.apk" - withCredentials([ - string(credentialsId: 'SAUCE_ACCESS_KEY', variable: 'SAUCE_ACCESS_KEY'), - string(credentialsId: 'SAUCE_USERNAME', variable: 'SAUCE_USERNAME'), - string(credentialsId: 'GIT_HUB_TOKEN', variable: 'GITHUB_TOKEN'), - string(credentialsId: 'SLACK_JENKINS_WEBHOOK', variable: 'SLACK_URL') - ]) { - sh 'fastlane android saucelabs' - } - return env.SAUCE_LABS_APK -} - -def compileiOS() { - version = readFile("${env.WORKSPACE}/VERSION").trim() - build_no = common.tagBuild() - withCredentials([ - string(credentialsId: 'SLACK_URL', variable: 'SLACK_URL'), - string(credentialsId: "slave-pass-${env.NODE_NAME}", variable: 'KEYCHAIN_PASSWORD'), - string(credentialsId: 'FASTLANE_PASSWORD', variable: 'FASTLANE_PASSWORD'), - string(credentialsId: 'APPLE_ID', variable: 'APPLE_ID'), - string(credentialsId: 'fastlane-match-password', variable:'MATCH_PASSWORD') - ]) { - sh "plutil -replace CFBundleShortVersionString -string ${version} ios/StatusIm/Info.plist" - sh "plutil -replace CFBundleVersion -string ${build_no} ios/StatusIm/Info.plist" - sh 'fastlane ios nightly' - } - def pkg = "StatusIm-${GIT_COMMIT.take(6)}.ipa" - sh "cp status-adhoc/StatusIm.ipa ${pkg}" - return pkg -} - -def uploadiOS() { - withCredentials([ - string(credentialsId: 'diawi-token', variable: 'DIAWI_TOKEN'), - string(credentialsId: 'GIT_HUB_TOKEN', variable: 'GITHUB_TOKEN'), - string(credentialsId: 'SLACK_JENKINS_WEBHOOK', variable: 'SLACK_URL') - ]) { - sh 'fastlane android upload_diawi' - } - diawiUrl = readFile "${env.WORKSPACE}/fastlane/diawi.out" - return diawiUrl -} - return this diff --git a/fastlane/Fastfile b/fastlane/Fastfile index 2e1f125634..0cb1eea779 100644 --- a/fastlane/Fastfile +++ b/fastlane/Fastfile @@ -134,8 +134,7 @@ platform :ios do end desc "`fastlane ios nightly` - makes a new nightly in TestFlight" - desc "This lane builds a new nightly and uploads it to TestFlight" - desc "It also leaves an .ipa that is ad-hoc signed (can be uploaded to diawi)" + desc "This lane builds a new nightly and leaves an .ipa that is ad-hoc signed (can be uploaded to diawi)" lane :nightly do unlock_keychain_if_needed diff --git a/scripts/build_no.sh b/scripts/build_no.sh index b94c98b8d0..f5e7cbcb8a 100755 --- a/scripts/build_no.sh +++ b/scripts/build_no.sh @@ -1,4 +1,7 @@ #!/usr/bin/env sh + +set -e + # # This script manages app build numbers. # It returns the next build number to be used. @@ -18,7 +21,7 @@ # getNumber () { - echo "$1" | sed 's/[^0-9]*//g' + echo "$BUILD" | sed 's/[^0-9]*//g' } REGEX='^build-[0-9]\+$' @@ -26,45 +29,28 @@ REGEX='^build-[0-9]\+$' # make sure we have all the tags git fetch --tags --quiet >/dev/null || >&2 echo "Could not fetch tags from remote" -# check if current commit has a build tag -# since we are building in separate jobs we have to check for a tag -BUILD_TAG=$(git tag --points-at HEAD | grep -e "$REGEX") - -# chech for multiple lines -if [ 1 -lt $(echo "$BUILD_TAG" | grep -c -) ]; then - echo "Commit marked with more than one build tag!" >&2 - echo "$BUILD_TAG" >&2 - exit 1 -fi - -# use already existing build number if applicable -if [ -n "$BUILD_TAG" ]; then - echo "Current commit already tagged: $BUILD_TAG" >&2 - getNumber $BUILD_TAG - exit 0 -fi - -# if no tag was found and --increment was not given stop -if [ "$1" != "--increment" ]; then - exit 0 -fi +# even if the current commit has a tag already, it is normal that the same commit +# is built multiple times (with different build configurations, for instance), +# so we increment the build number every time. # find the last used build number BUILD=$(git tag -l --sort=-v:refname | grep -e "$REGEX" | head -n 1) # extract the number BUILD_NO=$(getNumber "$BUILD") -# These need to be provided by Jenkins -if [ -z "${GIT_USER}" ] || [ -z "${GIT_PASS}" ]; then - echo "Git credentials not specified! (GIT_USER, GIT_PASS)" >&2 - exit 1 -fi -# increment -BUILD_NO="$((BUILD_NO+1))" +if [ "$1" = "--increment" ]; then + # These need to be provided by Jenkins + if [ -z "${GIT_USER}" ] || [ -z "${GIT_PASS}" ]; then + echo "Git credentials not specified! (GIT_USER, GIT_PASS)" >&2 + exit 1 + fi + # increment + BUILD_NO="$((BUILD_NO+1))" -echo "Tagging HEAD: build-$BUILD_NO" >&2 -git tag "build-$BUILD_NO" HEAD -git push --tags https://${GIT_USER}:${GIT_PASS}@github.com/status-im/status-react + echo "Tagging HEAD: build-$BUILD_NO" >&2 + git tag "build-$BUILD_NO" HEAD + git push --tags https://${GIT_USER}:${GIT_PASS}@github.com/status-im/status-react +fi # finally print build number echo "$BUILD_NO"