diff --git a/ci/Jenkinsfile.android b/ci/Jenkinsfile.android index 8552e6af34..635e229ad1 100644 --- a/ci/Jenkinsfile.android +++ b/ci/Jenkinsfile.android @@ -3,6 +3,7 @@ pipeline { options { timestamps() + disableConcurrentBuilds() /* Prevent Jenkins jobs from running forever */ timeout(time: 35, unit: 'MINUTES') /* Limit builds retained */ @@ -14,9 +15,9 @@ pipeline { } parameters { - booleanParam( + string( name: 'BUILD_TYPE', - description: 'Specify build type. Values: pr / nightly / release', + description: 'Specify build type. Values: pr / e2e / nightly / release', defaultValue: 'pr', ) } @@ -39,10 +40,10 @@ pipeline { stage('Prep') { steps { script { - print "Running ${params.BUILD_TYPE} build!" /* Necessary to load methods */ mobile = load 'ci/mobile.groovy' cmn = load 'ci/common.groovy' + print "Running ${cmn.getBuildType()} build!" /* Cleanup and Prep */ mobile.prep(cmn.getBuildType()) /* Run at start to void mismatched numbers */ @@ -72,12 +73,14 @@ pipeline { } stage('Archive') { steps { - script { archiveArtifacts apk } + archiveArtifacts apk } } stage('Upload') { steps { script { + env.PKG_URL = cmn.uploadArtifact(apk) + /* build type specific */ switch (cmn.getBuildType()) { case 'release': mobile.android.uploadToPlayStore(); break; @@ -89,5 +92,13 @@ pipeline { } } } + stage('Notify') { + steps { + script { cmn.gitHubNotifyPRSuccess() } + } + } + } + post { + failure { script { load('ci/common.groovy').gitHubNotifyPRFail() } } } } diff --git a/ci/Jenkinsfile.combined b/ci/Jenkinsfile.combined index 240c9cfcce..8732f1a346 100644 --- a/ci/Jenkinsfile.combined +++ b/ci/Jenkinsfile.combined @@ -2,6 +2,7 @@ pipeline { agent { label 'master' } options { + timestamps() disableConcurrentBuilds() /* Prevent Jenkins jobs from running forever */ timeout(time: 45, unit: 'MINUTES') @@ -70,47 +71,27 @@ pipeline { } stage('Upload') { steps { script { - apke2eUrl = cmn.uploadArtifact(cmn.pkgFind('e2e.apk')) - apkUrl = cmn.uploadArtifact(cmn.pkgFind("${btype}.apk")) - if (btype != 'release') { - dmgUrl = cmn.uploadArtifact(cmn.pkgFind('dmg')) - appUrl = cmn.uploadArtifact(cmn.pkgFind('AppImage')) - exeUrl = cmn.uploadArtifact(cmn.pkgFind('exe')) - } else { - dmgUrl = null - appUrl = null - exeUrl = null - } - iose2eUrl = cmn.uploadArtifact(cmn.pkgFind("e2e.app.zip")) - /* special case for iOS Diawi links */ - ipaUrl = ios.getBuildVariables().get('DIAWI_URL') - /* upload the sha256 checksums file too */ - shaUrl = cmn.uploadArtifact(cmn.pkgFind('sha256')) + /* object for easier URLs handling */ + urls = [ + /* mobile */ + Apk: cmn.pkgUrl(apk), Apke2e: cmn.pkgUrl(apke2e), + iOS: cmn.pkgUrl(ios), iOSe2e: cmn.pkgUrl(iose2e), + /* desktop */ + App: cmn.pkgUrl(nix), Mac: cmn.pkgUrl(ios), Win: cmn.pkgUrl(win), + /* upload the sha256 checksums file too */ + SHA: cmn.uploadArtifact(cmn.pkgFind('sha256')), + ] /* add URLs to the build description */ - cmn.setBuildDesc( - Apk: apkUrl, Apke2e: apke2eUrl, - iOS: ipaUrl, iOSe2e: iose2eUrl, - App: appUrl, Mac: dmgUrl, Win: exeUrl, - ) + cmn.setBuildDesc(urls) /* Create latest.json with newest nightly URLs */ if (btype == 'nightly') { - cmn.updateLatestNightlies( - APK: apkUrl, IOS: ipaUrl, - APP: appUrl, MAC: dmgUrl, - WIN: exeUrl, SHA: shaUrl - ) + cmn.updateLatestNightlies(urls) } } } } - stage('Notify') { + stage('Notify') { when { expression { env.CHANGE_ID != null } } steps { script { - if (env.CHANGE_ID != null) { - cmn.githubNotify( - apk: apkUrl, apke2e: apke2eUrl, - ipa: ipaUrl, iose2e: iose2eUrl, - app: appUrl, dmg: dmgUrl, win: exeUrl, - ) - } + cmn.gitHubNotifyFull(urls) } } } stage('Publish') { @@ -123,7 +104,7 @@ pipeline { build( job: 'misc/cn.status.im', parameters: [ - [name: 'APK_URL', value: apkUrl, $class: 'StringParameterValue'], + [name: 'APK_URL', value: urls.Apk, $class: 'StringParameterValue'], ] ) break @@ -133,7 +114,7 @@ pipeline { stage('Run e2e') { when { expression { btype == 'nightly' } } steps { script { - e2eApk = apke2e.getBuildVariables().get('SAUCE_URL') + e2eApk = cmn.getEnv(apke2e, 'SAUCE_URL') build( job: 'end-to-end-tests/status-app-nightly', wait: false, parameters: [string(name: 'apk', value: "--apk=${e2eApk}")] diff --git a/ci/Jenkinsfile.ios b/ci/Jenkinsfile.ios index 9cc1af000d..19eecc7733 100644 --- a/ci/Jenkinsfile.ios +++ b/ci/Jenkinsfile.ios @@ -2,15 +2,16 @@ pipeline { agent { label 'macos' } parameters { - booleanParam( + string( name: 'BUILD_TYPE', - description: 'Specify build type. Values: pr / nightly / release', + description: 'Specify build type. Values: pr / e2e / nightly / release', defaultValue: 'pr', ) } options { timestamps() + disableConcurrentBuilds() /* Prevent Jenkins jobs from running forever */ timeout(time: 35, unit: 'MINUTES') /* Limit builds retained */ @@ -36,10 +37,10 @@ pipeline { stage('Prep') { steps { script { nvm(env.NODE_VERSION) { - print "Running ${params.BUILD_TYPE} build!" /* Necessary to load methods */ mobile = load 'ci/mobile.groovy' cmn = load 'ci/common.groovy' + print "Running ${cmn.getBuildType()} build!" /* Cleanup and Prep */ mobile.prep(cmn.getBuildType()) /* Run at start to void mismatched numbers */ @@ -69,20 +70,28 @@ pipeline { } stage('Archive') { steps { - script { archiveArtifacts api } + archiveArtifacts api } } stage('Upload') { steps { script { /* e2e builds get tested in SauceLabs */ - if (params.BUILD_TYPE == 'e2e') { + if (cmn.getBuildType() == 'e2e') { env.SAUCE_URL = mobile.ios.uploadToSauceLabs() } else { - env.DIAWI_URL = mobile.ios.uploadToDiawi() + env.PKG_URL = mobile.ios.uploadToDiawi() } } } } + stage('Notify') { + steps { + script { cmn.gitHubNotifyPRSuccess() } + } + } + } + post { + failure { script { load('ci/common.groovy').gitHubNotifyPRFail() } } } } diff --git a/ci/Jenkinsfile.linux b/ci/Jenkinsfile.linux index 2ac709df6f..132b5990a8 100644 --- a/ci/Jenkinsfile.linux +++ b/ci/Jenkinsfile.linux @@ -3,7 +3,7 @@ pipeline { /* privileged mode is necessary for fuse */ docker { label 'linux-new' - image 'statusteam/linux-desktop-ubuntu:1.1.0' + image 'statusteam/linux-desktop-ubuntu:1.1.1' args ( "--privileged "+ "-v /dev/fuse:/dev/fuse "+ @@ -15,7 +15,7 @@ pipeline { } parameters { - booleanParam( + string( name: 'BUILD_TYPE', description: 'Specify build type. Values: pr / nightly / release', defaultValue: 'pr', @@ -24,6 +24,7 @@ pipeline { options { timestamps() + disableConcurrentBuilds() /* Prevent Jenkins jobs from running forever */ timeout(time: 35, unit: 'MINUTES') /* Limit builds retained */ @@ -88,7 +89,22 @@ pipeline { } } stage('Archive') { - steps { archiveArtifacts app } + steps { + archiveArtifacts app + } + } + stage('Upload') { + steps { + script { env.PKG_URL = cmn.uploadArtifact(app) } + } + } + stage('Notify') { + steps { + script { cmn.gitHubNotifyPRSuccess() } + } } } + post { + failure { script { load('ci/common.groovy').gitHubNotifyPRFail() } } + } } diff --git a/ci/Jenkinsfile.macos b/ci/Jenkinsfile.macos index 5c3a45d8d8..bebf8177d9 100644 --- a/ci/Jenkinsfile.macos +++ b/ci/Jenkinsfile.macos @@ -2,7 +2,7 @@ pipeline { agent { label 'macos' } parameters { - booleanParam( + string( name: 'BUILD_TYPE', description: 'Specify build type. Values: pr / nightly / release', defaultValue: 'pr', @@ -11,6 +11,7 @@ pipeline { options { timestamps() + disableConcurrentBuilds() /* Prevent Jenkins jobs from running forever */ timeout(time: 25, unit: 'MINUTES') /* Limit builds retained */ @@ -69,7 +70,22 @@ pipeline { } } } stage('Archive') { - steps { archiveArtifacts dmg } + steps { + archiveArtifacts dmg + } + } + stage('Upload') { + steps { + script { env.PKG_URL = cmn.uploadArtifact(dmg) } + } + } + stage('Notify') { + steps { + script { cmn.gitHubNotifyPRSuccess() } + } } } + post { + failure { script { load('ci/common.groovy').gitHubNotifyPRFail() } } + } } diff --git a/ci/Jenkinsfile.windows b/ci/Jenkinsfile.windows index 63c19880e3..af6f95b669 100644 --- a/ci/Jenkinsfile.windows +++ b/ci/Jenkinsfile.windows @@ -3,7 +3,7 @@ pipeline { /* privileged mode is necessary for fuse */ docker { label 'linux-new' - image 'statusteam/windows-desktop-ubuntu:1.1.0' + image 'statusteam/windows-desktop-ubuntu:1.1.1' args ( "--privileged "+ "-v /dev/fuse:/dev/fuse "+ @@ -15,7 +15,7 @@ pipeline { } parameters { - booleanParam( + string( name: 'BUILD_TYPE', description: 'Specify build type. Values: pr / nightly / release', defaultValue: 'pr', @@ -23,6 +23,8 @@ pipeline { } options { + timestamps() + disableConcurrentBuilds() /* Prevent Jenkins jobs from running forever */ timeout(time: 45, unit: 'MINUTES') /* Limit builds retained */ @@ -39,7 +41,7 @@ pipeline { **/ environment { NODE_VERSION = 'v10.14.1' - BUILD_PLATFORM = 'linux' + BUILD_PLATFORM = 'windows' LANG = 'en_US.UTF-8' LANGUAGE = 'en_US.UTF-8' LC_ALL = 'en_US.UTF-8' @@ -90,7 +92,22 @@ pipeline { } } stage('Archive') { - steps { archiveArtifacts app } + steps { + archiveArtifacts app + } + } + stage('Upload') { + steps { + script { env.PKG_URL = cmn.uploadArtifact(app) } + } + } + stage('Notify') { + steps { + script { cmn.gitHubNotifyPRSuccess() } + } } } + post { + failure { script { load('ci/common.groovy').gitHubNotifyPRFail() } } + } } diff --git a/ci/common.groovy b/ci/common.groovy index 63050a3b26..cb80b5c53e 100644 --- a/ci/common.groovy +++ b/ci/common.groovy @@ -6,6 +6,9 @@ def version() { def getBuildType() { def jobName = env.JOB_NAME + if (jobName.contains('e2e')) { + return 'e2e' + } if (jobName.startsWith('status-react/pull requests')) { return 'pr' } @@ -29,8 +32,9 @@ def buildBranch(name = null, buildType = null) { /* this allows us to analize the job even after failure */ propagate: false, parameters: [ - [name: 'BRANCH', value: branchName, $class: 'StringParameterValue'], - [name: 'BUILD_TYPE', value: buildType, $class: 'StringParameterValue'], + [name: 'BRANCH', value: branchName, $class: 'StringParameterValue'], + [name: 'BUILD_TYPE', value: buildType, $class: 'StringParameterValue'], + [name: 'CHANGE_ID', value: env.CHANGE_ID, $class: 'StringParameterValue'], ]) /* BlueOcean seems to not show child-build links */ print "Build: ${b.getAbsoluteUrl()} (${b.result})" @@ -65,8 +69,10 @@ def installJSDeps(platform) { } def doGitRebase() { + sh 'git status' + sh 'git fetch --force origin develop:develop' try { - sh 'git rebase origin/develop' + sh 'git rebase develop' } catch (e) { sh 'git rebase --abort' throw e @@ -136,32 +142,78 @@ def pkgFilename(type, ext) { return "StatusIm-${timestamp()}-${gitCommit()}-${type}.${ext}" } +def buildDuration() { + def duration = currentBuild.durationString + return '~' + duration.take(duration.lastIndexOf(' and counting')) +} -def githubNotify(Map urls) { +def gitHubNotify(message) { def githubIssuesUrl = 'https://api.github.com/repos/status-im/status-react/issues' - withCredentials([string(credentialsId: 'GIT_HUB_TOKEN', variable: 'githubToken')]) { - def message = "#### :white_check_mark: [${currentBuild.displayName}](${currentBuild.absoluteUrl}) " - message += "CI BUILD SUCCESSFUL in ${currentBuild.durationString} (${GIT_COMMIT})\n" - message += '| | | | | |\n' - message += '|-|-|-|-|-|\n' - message += "| [Android](${urls.apk}) ([e2e](${urls.apke2e})) | [iOS](${urls.ipa}) ([e2e](${urls.iose2e})) |" - if (dmgUrl != null) { - message += " [MacOS](${urls.dmg}) | [AppImage](${urls.app}) | [Windows](${urls.win}) |" - } else { - message += " ~~MacOS~~ | ~~AppImage~~ | ~~Windows~~~ |" - } - def msgObj = [body: message] - def msgJson = new JsonBuilder(msgObj).toPrettyString() + /* CHANGE_ID can be provided via the build parameters */ + def changeId = params.CHANGE_ID ? params.CHANGE_ID : env.CHANGE_ID + def msgObj = [body: message] + def msgJson = new JsonBuilder(msgObj).toPrettyString() + withCredentials([usernamePassword( + credentialsId: 'status-im-auto', + usernameVariable: 'GH_USER', + passwordVariable: 'GH_PASS' + )]) { sh """ curl --silent \ - -u status-im:${githubToken} \ - -H "Content-Type: application/json" \ + -u '${GH_USER}:${GH_PASS}' \ --data '${msgJson}' \ - "${githubIssuesUrl}/${env.CHANGE_ID}/comments" + -H "Content-Type: application/json" \ + "${githubIssuesUrl}/${changeId}/comments" """.trim() } } +def gitHubNotifyFull(urls) { + def msg = "#### :white_check_mark: " + msg += "[${env.JOB_NAME}${currentBuild.displayName}](${currentBuild.absoluteUrl}) " + msg += "CI BUILD SUCCESSFUL in ${buildDuration()} (${GIT_COMMIT.take(8)})\n" + msg += '| | | | | |\n' + msg += '|-|-|-|-|-|\n' + msg += "| [Android](${urls.Apk}) ([e2e](${urls.Apke2e})) " + msg += "| [iOS](${urls.iOS}) ([e2e](${urls.iOSe2e})) |" + if (urls.Mac != null) { + msg += " [MacOS](${urls.Mac}) | [AppImage](${urls.App}) | [Windows](${urls.Win}) |" + } else { + msg += " ~~MacOS~~ | ~~AppImage~~ | ~~Windows~~~ |" + } + gitHubNotify(msg) +} + + +def gitHubNotifyPRFail() { + def d = ":small_orange_diamond:" + def msg = "#### :x: " + msg += "[${env.JOB_NAME}${currentBuild.displayName}](${currentBuild.absoluteUrl}) ${d} " + msg += "${buildDuration()} ${d} ${GIT_COMMIT.take(8)} ${d} " + msg += "[:page_facing_up: build log](${currentBuild.absoluteUrl}/consoleText)" + //msg += "Failed in stage: ${env.STAGE_NAME}\n" + //msg += "```${currentBuild.rawBuild.getLog(5)}```" + gitHubNotify(msg) +} + +def gitHubNotifyPRSuccess() { + def d = ":small_blue_diamond:" + def msg = "#### :heavy_check_mark: " + def type = getBuildType() == 'e2e' ? ' e2e' : '' + msg += "[${env.JOB_NAME}${currentBuild.displayName}](${currentBuild.absoluteUrl}) ${d} " + msg += "${buildDuration()} ${d} ${GIT_COMMIT.take(8)} ${d} " + msg += "[:package: ${env.BUILD_PLATFORM}${type} package](${env.PKG_URL})" + gitHubNotify(msg) +} + +def getEnv(build, envvar) { + return build.getBuildVariables().get(envvar) +} + +def pkgUrl(build) { + return getEnv(build, 'PKG_URL') +} + def pkgFind(glob) { def fullGlob = "pkg/*${glob}" def found = findFiles(glob: fullGlob) @@ -182,11 +234,17 @@ def setBuildDesc(Map links) { currentBuild.description = desc } -def updateLatestNightlies(Map links) { +def updateLatestNightlies(urls) { + /* latest.json has slightly different key names */ + def latest = [ + APK: urls.Apk, IOS: urls.iOS, + APP: urls.App, MAC: urls.Mac, + WIN: urls.Win, SHA: urls.SHA + ] def latestFile = pwd() + '/' + 'pkg/latest.json' /* it might not exist */ sh 'mkdir -p pkg' - def latestJson = new JsonBuilder(links).toPrettyString() + def latestJson = new JsonBuilder(latest).toPrettyString() println("latest.json:\n${latestJson}") new File(latestFile).write(latestJson) return uploadArtifact(latestFile) diff --git a/ci/desktop.groovy b/ci/desktop.groovy index 6685de68c1..dd9a007f60 100644 --- a/ci/desktop.groovy +++ b/ci/desktop.groovy @@ -1,4 +1,4 @@ -common = load 'ci/common.groovy' +cmn = load 'ci/common.groovy' packageFolder = './StatusImPackage' @@ -10,7 +10,7 @@ def cleanupAndDeps() { cleanupBuild() sh 'cp .env.jenkins .env' sh 'lein deps' - common.installJSDeps('desktop') + cmn.installJSDeps('desktop') } def slackNotify(message, color = 'good') { @@ -52,7 +52,7 @@ def uploadArtifact(filename) { /* MAIN --------------------------------------------------*/ def prepDeps() { - common.doGitRebase() + cmn.doGitRebase() cleanupAndDeps() } @@ -69,7 +69,7 @@ def bundleWindows(type = 'nightly') { sh './scripts/build-desktop.sh bundle' dir(packageFolder) { - pkg = common.pkgFilename(type, 'exe') + pkg = cmn.pkgFilename(type, 'exe') sh "mv ../Status-x86_64-setup.exe ${pkg}" } return "${packageFolder}/${pkg}".drop(2) @@ -80,14 +80,14 @@ def bundleLinux(type = 'nightly') { sh './scripts/build-desktop.sh bundle' dir(packageFolder) { - pkg = common.pkgFilename(type, 'AppImage') + pkg = cmn.pkgFilename(type, 'AppImage') sh "mv ../Status-x86_64.AppImage ${pkg}" } return "${packageFolder}/${pkg}".drop(2) } def bundleMacOS(type = 'nightly') { - def pkg = common.pkgFilename(type, 'dmg') + def pkg = cmn.pkgFilename(type, 'dmg') sh './scripts/build-desktop.sh bundle' dir(packageFolder) { withCredentials([ diff --git a/ci/mobile.groovy b/ci/mobile.groovy index c7601c9ef8..e7eb5cc590 100644 --- a/ci/mobile.groovy +++ b/ci/mobile.groovy @@ -1,4 +1,4 @@ -common = load 'ci/common.groovy' +cmn = load 'ci/common.groovy' ios = load 'ci/ios.groovy' android = load 'ci/android.groovy' @@ -34,6 +34,7 @@ def podUpdate() { } def prep(type = 'nightly') { + cmn.doGitRebase() /* ensure that we start from a known state */ sh 'make clean' /* select type of build */ @@ -53,7 +54,9 @@ def prep(type = 'nightly') { sh "make prepare-${env.BUILD_PLATFORM}" /* generate ios/StatusIm.xcworkspace */ dir('ios') { - podUpdate() + if (env.BUILD_PLATFORM == 'ios') { + podUpdate() + } sh 'pod install --silent' } } diff --git a/desktop/docker/linux/Dockerfile b/desktop/docker/linux/Dockerfile index b778cb3a39..11b0e6f7ac 100644 --- a/desktop/docker/linux/Dockerfile +++ b/desktop/docker/linux/Dockerfile @@ -48,7 +48,7 @@ RUN apt-get update && apt-get -q -y --no-install-recommends install curl softwar apt-get update && \ DEBIAN_FRONTEND=noninteractive \ apt-get -q -y --no-install-recommends install \ - wget git unzip golang-go nodejs yarn file \ + wget git unzip golang-go nodejs yarn file s3cmd \ python python3-pip python3-setuptools python3-wheel \ apt-transport-https locales openjdk-8-jdk-headless \ extra-cmake-modules build-essential gcc g++ fuse \ diff --git a/desktop/docker/linux/Makefile b/desktop/docker/linux/Makefile index 88ca0aaabf..3ab29d3bd7 100644 --- a/desktop/docker/linux/Makefile +++ b/desktop/docker/linux/Makefile @@ -8,7 +8,7 @@ QT_MD5SUM = 974fda61267cfb6e45984ee5f0a285f8 QT_URL = https://download.qt.io/archive/qt # WARNING: Remember to change the tag when updating the image -IMAGE_TAG = 1.1.0 +IMAGE_TAG = 1.1.1 IMAGE_NAME = statusteam/linux-desktop-ubuntu:$(IMAGE_TAG) build: $(QT_ARCHIVE) diff --git a/desktop/docker/windows/Dockerfile b/desktop/docker/windows/Dockerfile index 4014bc7533..864b1b6410 100644 --- a/desktop/docker/windows/Dockerfile +++ b/desktop/docker/windows/Dockerfile @@ -18,7 +18,7 @@ RUN apt-get update && apt-get -q -y --no-install-recommends install curl softwar apt-get update && \ DEBIAN_FRONTEND=noninteractive \ apt-get -q -y --no-install-recommends install \ - wget git nsis unzip golang-go nodejs yarn file jq \ + wget git nsis unzip golang-go nodejs yarn file jq s3cmd \ python python3-pip python3-setuptools python3-wheel \ apt-transport-https locales openjdk-8-jdk-headless \ extra-cmake-modules build-essential fuse \ diff --git a/desktop/docker/windows/Makefile b/desktop/docker/windows/Makefile index 59ded7f6a2..965847935a 100644 --- a/desktop/docker/windows/Makefile +++ b/desktop/docker/windows/Makefile @@ -1,7 +1,7 @@ GIT_COMMIT = $(shell git rev-parse --short HEAD) # WARNING: Remember to change the tag when updating the image -IMAGE_TAG = 1.1.0 +IMAGE_TAG = 1.1.1 IMAGE_NAME = statusteam/windows-desktop-ubuntu:$(IMAGE_TAG) build: