move upload step top platform specific builds, cleanup

Signed-off-by: Jakub Sokołowski <jakub@status.im>

ci/Jenkinsfile.android

@@ -3,6 +3,7 @@ pipeline {
 
   options {
     timestamps()
+    disableConcurrentBuilds()
     /* Prevent Jenkins jobs from running forever */
     timeout(time: 35, unit: 'MINUTES')
     /* Limit builds retained */
@@ -14,9 +15,9 @@ pipeline {
   }
 
   parameters {
-    booleanParam(
+    string(
       name: 'BUILD_TYPE',
-      description: 'Specify build type. Values: pr / nightly / release',
+      description: 'Specify build type. Values: pr / e2e / nightly / release',
       defaultValue: 'pr',
     )
   }
@@ -39,10 +40,10 @@ pipeline {
     stage('Prep') {
       steps {
         script {
-          print "Running ${params.BUILD_TYPE} build!"
           /* Necessary to load methods */
           mobile = load 'ci/mobile.groovy'
           cmn    = load 'ci/common.groovy'
+          print "Running ${cmn.getBuildType()} build!"
           /* Cleanup and Prep */
           mobile.prep(cmn.getBuildType())
           /* Run at start to void mismatched numbers */
@@ -72,12 +73,14 @@ pipeline {
     }
     stage('Archive') {
       steps {
-        script { archiveArtifacts apk }
+        archiveArtifacts apk
       }
     }
     stage('Upload') {
       steps {
         script {
+          env.PKG_URL = cmn.uploadArtifact(apk)
+          /* build type specific */
           switch (cmn.getBuildType()) {
             case 'release':
               mobile.android.uploadToPlayStore(); break;
@@ -89,5 +92,13 @@ pipeline {
         }
       }
     }
+    stage('Notify') {
+      steps {
+        script { cmn.gitHubNotifyPRSuccess() }
+      }
+    }
+  }
+  post {
+    failure { script { load('ci/common.groovy').gitHubNotifyPRFail() } }
   }
 }

ci/Jenkinsfile.combined

@@ -2,6 +2,7 @@ pipeline {
   agent { label 'master' }
 
   options {
+    timestamps()
     disableConcurrentBuilds()
     /* Prevent Jenkins jobs from running forever */
     timeout(time: 45, unit: 'MINUTES')
@@ -70,47 +71,27 @@ pipeline {
     }
     stage('Upload') {
       steps { script {
-        apke2eUrl = cmn.uploadArtifact(cmn.pkgFind('e2e.apk'))
+        /* object for easier URLs handling */
-        apkUrl = cmn.uploadArtifact(cmn.pkgFind("${btype}.apk"))
+        urls = [
-        if (btype != 'release') {
+          /* mobile */
-          dmgUrl = cmn.uploadArtifact(cmn.pkgFind('dmg'))
+          Apk: cmn.pkgUrl(apk), Apke2e: cmn.pkgUrl(apke2e),
-          appUrl = cmn.uploadArtifact(cmn.pkgFind('AppImage'))
+          iOS: cmn.pkgUrl(ios), iOSe2e: cmn.pkgUrl(iose2e),
-          exeUrl = cmn.uploadArtifact(cmn.pkgFind('exe'))
+          /* desktop */
-        } else {
+          App: cmn.pkgUrl(nix), Mac: cmn.pkgUrl(ios), Win: cmn.pkgUrl(win),
-          dmgUrl = null
+          /* upload the sha256 checksums file too */
-          appUrl = null
+          SHA: cmn.uploadArtifact(cmn.pkgFind('sha256')),
-          exeUrl = null
+        ]
-        }
-        iose2eUrl = cmn.uploadArtifact(cmn.pkgFind("e2e.app.zip"))
-        /* special case for iOS Diawi links */
-        ipaUrl = ios.getBuildVariables().get('DIAWI_URL')
-        /* upload the sha256 checksums file too */
-        shaUrl = cmn.uploadArtifact(cmn.pkgFind('sha256'))
         /* add URLs to the build description */
-        cmn.setBuildDesc(
+        cmn.setBuildDesc(urls)
-          Apk: apkUrl, Apke2e: apke2eUrl,
-          iOS: ipaUrl, iOSe2e: iose2eUrl,
-          App: appUrl, Mac: dmgUrl, Win: exeUrl,
-        )
         /* Create latest.json with newest nightly URLs */
         if (btype == 'nightly') {
-          cmn.updateLatestNightlies(
+          cmn.updateLatestNightlies(urls)
-            APK: apkUrl, IOS: ipaUrl,
-            APP: appUrl, MAC: dmgUrl,
-            WIN: exeUrl, SHA: shaUrl
-          )
         }
       } }
     }
-    stage('Notify') {
+    stage('Notify') { when { expression { env.CHANGE_ID != null } }
       steps { script {
-        if (env.CHANGE_ID != null) {
+        cmn.gitHubNotifyFull(urls)
-          cmn.githubNotify(
-            apk: apkUrl, apke2e: apke2eUrl,
-            ipa: ipaUrl, iose2e: iose2eUrl,
-            app: appUrl, dmg: dmgUrl, win: exeUrl,
-          )
-        }
       } }
     }
     stage('Publish') {
@@ -123,7 +104,7 @@ pipeline {
             build(
               job: 'misc/cn.status.im',
               parameters: [
-                [name: 'APK_URL', value: apkUrl, $class: 'StringParameterValue'],
+                [name: 'APK_URL', value: urls.Apk, $class: 'StringParameterValue'],
               ]
             )
             break
@@ -133,7 +114,7 @@ pipeline {
     stage('Run e2e') {
       when { expression { btype == 'nightly' } }
       steps { script {
-        e2eApk = apke2e.getBuildVariables().get('SAUCE_URL')
+        e2eApk = cmn.getEnv(apke2e, 'SAUCE_URL')
         build(
           job: 'end-to-end-tests/status-app-nightly', wait: false,
           parameters: [string(name: 'apk', value: "--apk=${e2eApk}")]

ci/Jenkinsfile.ios

@@ -2,15 +2,16 @@ pipeline {
   agent { label 'macos' }
 
   parameters {
-    booleanParam(
+    string(
       name: 'BUILD_TYPE',
-      description: 'Specify build type. Values: pr / nightly / release',
+      description: 'Specify build type. Values: pr / e2e / nightly / release',
       defaultValue: 'pr',
     )
   }
 
   options {
     timestamps()
+    disableConcurrentBuilds()
     /* Prevent Jenkins jobs from running forever */
     timeout(time: 35, unit: 'MINUTES')
     /* Limit builds retained */
@@ -36,10 +37,10 @@ pipeline {
     stage('Prep') {
       steps {
         script { nvm(env.NODE_VERSION) {
-          print "Running ${params.BUILD_TYPE} build!"
           /* Necessary to load methods */
           mobile = load 'ci/mobile.groovy'
           cmn    = load 'ci/common.groovy'
+          print "Running ${cmn.getBuildType()} build!"
           /* Cleanup and Prep */
           mobile.prep(cmn.getBuildType())
           /* Run at start to void mismatched numbers */
@@ -69,20 +70,28 @@ pipeline {
     }
     stage('Archive') {
       steps {
-        script { archiveArtifacts api }
+        archiveArtifacts api
       }
     }
     stage('Upload') {
       steps {
         script {
           /* e2e builds get tested in SauceLabs */
-          if (params.BUILD_TYPE == 'e2e') {
+          if (cmn.getBuildType() == 'e2e') {
             env.SAUCE_URL = mobile.ios.uploadToSauceLabs()
           } else {
-            env.DIAWI_URL = mobile.ios.uploadToDiawi()
+            env.PKG_URL = mobile.ios.uploadToDiawi()
           }
         }
       }
     }
+    stage('Notify') {
+      steps {
+        script { cmn.gitHubNotifyPRSuccess() }
+      }
+    }
+  }
+  post {
+    failure { script { load('ci/common.groovy').gitHubNotifyPRFail() } }
   }
 }

ci/Jenkinsfile.linux

@@ -3,7 +3,7 @@ pipeline {
     /* privileged mode is necessary for fuse */
     docker {
       label 'linux-new'
-      image 'statusteam/linux-desktop-ubuntu:1.1.0'
+      image 'statusteam/linux-desktop-ubuntu:1.1.1'
       args (
         "--privileged "+
         "-v /dev/fuse:/dev/fuse "+
@@ -15,7 +15,7 @@ pipeline {
   }
 
   parameters {
-    booleanParam(
+    string(
       name: 'BUILD_TYPE',
       description: 'Specify build type. Values: pr / nightly / release',
       defaultValue: 'pr',
@@ -24,6 +24,7 @@ pipeline {
 
   options {
     timestamps()
+    disableConcurrentBuilds()
     /* Prevent Jenkins jobs from running forever */
     timeout(time: 35, unit: 'MINUTES')
     /* Limit builds retained */
@@ -88,7 +89,22 @@ pipeline {
       }
     }
     stage('Archive') {
-      steps { archiveArtifacts app }
+      steps {
+        archiveArtifacts app
+      }
+    }
+    stage('Upload') {
+      steps {
+        script { env.PKG_URL = cmn.uploadArtifact(app) }
+      }
+    }
+    stage('Notify') {
+      steps {
+        script { cmn.gitHubNotifyPRSuccess() }
+      }
     }
   }
+  post {
+    failure { script { load('ci/common.groovy').gitHubNotifyPRFail() } }
+  }
 }

ci/Jenkinsfile.macos

@@ -2,7 +2,7 @@ pipeline {
   agent { label 'macos' }
 
   parameters {
-    booleanParam(
+    string(
       name: 'BUILD_TYPE',
       description: 'Specify build type. Values: pr / nightly / release',
       defaultValue: 'pr',
@@ -11,6 +11,7 @@ pipeline {
 
   options {
     timestamps()
+    disableConcurrentBuilds()
     /* Prevent Jenkins jobs from running forever */
     timeout(time: 25, unit: 'MINUTES')
     /* Limit builds retained */
@@ -69,7 +70,22 @@ pipeline {
       } }
     }
     stage('Archive') {
-      steps { archiveArtifacts dmg }
+      steps {
+        archiveArtifacts dmg
+      }
+    }
+    stage('Upload') {
+      steps {
+        script { env.PKG_URL = cmn.uploadArtifact(dmg) }
+      }
+    }
+    stage('Notify') {
+      steps {
+        script { cmn.gitHubNotifyPRSuccess() }
+      }
     }
   }
+  post {
+    failure { script { load('ci/common.groovy').gitHubNotifyPRFail() } }
+  }
 }

ci/Jenkinsfile.windows

@@ -3,7 +3,7 @@ pipeline {
     /* privileged mode is necessary for fuse */
     docker {
       label 'linux-new'
-      image 'statusteam/windows-desktop-ubuntu:1.1.0'
+      image 'statusteam/windows-desktop-ubuntu:1.1.1'
       args (
         "--privileged "+
         "-v /dev/fuse:/dev/fuse "+
@@ -15,7 +15,7 @@ pipeline {
   }
 
   parameters {
-    booleanParam(
+    string(
       name: 'BUILD_TYPE',
       description: 'Specify build type. Values: pr / nightly / release',
       defaultValue: 'pr',
@@ -23,6 +23,8 @@ pipeline {
   }
 
   options {
+    timestamps()
+    disableConcurrentBuilds()
     /* Prevent Jenkins jobs from running forever */
     timeout(time: 45, unit: 'MINUTES')
     /* Limit builds retained */
@@ -39,7 +41,7 @@ pipeline {
    **/
   environment {
     NODE_VERSION = 'v10.14.1'
-    BUILD_PLATFORM = 'linux'
+    BUILD_PLATFORM = 'windows'
     LANG = 'en_US.UTF-8'
     LANGUAGE = 'en_US.UTF-8'
     LC_ALL = 'en_US.UTF-8'
@@ -90,7 +92,22 @@ pipeline {
       }
     }
     stage('Archive') {
-      steps { archiveArtifacts app }
+      steps {
+        archiveArtifacts app
+      }
+    }
+    stage('Upload') {
+      steps {
+        script { env.PKG_URL = cmn.uploadArtifact(app) }
+      }
+    }
+    stage('Notify') {
+      steps {
+        script { cmn.gitHubNotifyPRSuccess() }
+      }
     }
   }
+  post {
+    failure { script { load('ci/common.groovy').gitHubNotifyPRFail() } }
+  }
 }

ci/common.groovy

@@ -6,6 +6,9 @@ def version() {
 
 def getBuildType() {
   def jobName = env.JOB_NAME
+  if (jobName.contains('e2e')) {
+      return 'e2e'
+  }
   if (jobName.startsWith('status-react/pull requests')) {
       return 'pr'
   }
@@ -29,8 +32,9 @@ def buildBranch(name = null, buildType = null) {
     /* this allows us to analize the job even after failure */
     propagate: false,
     parameters: [
-      [name: 'BRANCH',     value: branchName, $class: 'StringParameterValue'],
+      [name: 'BRANCH',     value: branchName,    $class: 'StringParameterValue'],
-      [name: 'BUILD_TYPE', value: buildType,  $class: 'StringParameterValue'],
+      [name: 'BUILD_TYPE', value: buildType,     $class: 'StringParameterValue'],
+      [name: 'CHANGE_ID',  value: env.CHANGE_ID, $class: 'StringParameterValue'],
   ])
   /* BlueOcean seems to not show child-build links */
   print "Build: ${b.getAbsoluteUrl()} (${b.result})"
@@ -65,8 +69,10 @@ def installJSDeps(platform) {
 }
 
 def doGitRebase() {
+  sh 'git status'
+  sh 'git fetch --force origin develop:develop'
   try {
-    sh 'git rebase origin/develop'
+    sh 'git rebase develop'
   } catch (e) {
     sh 'git rebase --abort'
     throw e
@@ -136,32 +142,78 @@ def pkgFilename(type, ext) {
   return "StatusIm-${timestamp()}-${gitCommit()}-${type}.${ext}"
 }
 
+def buildDuration() {
+  def duration = currentBuild.durationString
+  return '~' + duration.take(duration.lastIndexOf(' and counting'))
+}
 
-def githubNotify(Map urls) {
+def gitHubNotify(message) {
   def githubIssuesUrl = 'https://api.github.com/repos/status-im/status-react/issues'
-  withCredentials([string(credentialsId: 'GIT_HUB_TOKEN', variable: 'githubToken')]) {
+  /* CHANGE_ID can be provided via the build parameters */
-    def message = "#### :white_check_mark: [${currentBuild.displayName}](${currentBuild.absoluteUrl}) "
+  def changeId = params.CHANGE_ID ? params.CHANGE_ID : env.CHANGE_ID
-    message += "CI BUILD SUCCESSFUL in ${currentBuild.durationString} (${GIT_COMMIT})\n"
+  def msgObj = [body: message]
-    message += '| | | | | |\n'
+  def msgJson = new JsonBuilder(msgObj).toPrettyString()
-    message += '|-|-|-|-|-|\n'
+  withCredentials([usernamePassword(
-    message += "| [Android](${urls.apk}) ([e2e](${urls.apke2e})) | [iOS](${urls.ipa}) ([e2e](${urls.iose2e})) |"
+    credentialsId:  'status-im-auto',
-    if (dmgUrl != null) {
+    usernameVariable: 'GH_USER',
-      message += " [MacOS](${urls.dmg}) | [AppImage](${urls.app}) | [Windows](${urls.win}) |"
+    passwordVariable: 'GH_PASS'
-    } else {
+  )]) {
-      message += " ~~MacOS~~ | ~~AppImage~~ | ~~Windows~~~ |"
-    }
-    def msgObj = [body: message]
-    def msgJson = new JsonBuilder(msgObj).toPrettyString()
     sh """
       curl --silent \
-        -u status-im:${githubToken} \
+        -u '${GH_USER}:${GH_PASS}' \
-        -H "Content-Type: application/json" \
         --data '${msgJson}' \
-        "${githubIssuesUrl}/${env.CHANGE_ID}/comments"
+        -H "Content-Type: application/json" \
+        "${githubIssuesUrl}/${changeId}/comments"
     """.trim()
   }
 }
 
+def gitHubNotifyFull(urls) {
+  def msg = "#### :white_check_mark: "
+  msg += "[${env.JOB_NAME}${currentBuild.displayName}](${currentBuild.absoluteUrl}) "
+  msg += "CI BUILD SUCCESSFUL in ${buildDuration()} (${GIT_COMMIT.take(8)})\n"
+  msg += '| | | | | |\n'
+  msg += '|-|-|-|-|-|\n'
+  msg += "| [Android](${urls.Apk}) ([e2e](${urls.Apke2e})) "
+  msg += "| [iOS](${urls.iOS}) ([e2e](${urls.iOSe2e})) |"
+  if (urls.Mac != null) {
+    msg += " [MacOS](${urls.Mac}) | [AppImage](${urls.App}) | [Windows](${urls.Win}) |"
+  } else {
+    msg += " ~~MacOS~~ | ~~AppImage~~ | ~~Windows~~~ |"
+  }
+  gitHubNotify(msg)
+}
+
+
+def gitHubNotifyPRFail() {
+  def d = ":small_orange_diamond:"
+  def msg = "#### :x: "
+  msg += "[${env.JOB_NAME}${currentBuild.displayName}](${currentBuild.absoluteUrl}) ${d} "
+  msg += "${buildDuration()} ${d} ${GIT_COMMIT.take(8)} ${d} "
+  msg += "[:page_facing_up: build log](${currentBuild.absoluteUrl}/consoleText)"
+  //msg += "Failed in stage: ${env.STAGE_NAME}\n"
+  //msg += "```${currentBuild.rawBuild.getLog(5)}```"
+  gitHubNotify(msg)
+}
+
+def gitHubNotifyPRSuccess() {
+  def d = ":small_blue_diamond:"
+  def msg = "#### :heavy_check_mark: "
+  def type = getBuildType() == 'e2e' ? ' e2e' : ''
+  msg += "[${env.JOB_NAME}${currentBuild.displayName}](${currentBuild.absoluteUrl}) ${d} "
+  msg += "${buildDuration()} ${d} ${GIT_COMMIT.take(8)} ${d} "
+  msg += "[:package: ${env.BUILD_PLATFORM}${type} package](${env.PKG_URL})"
+  gitHubNotify(msg)
+}
+
+def getEnv(build, envvar) {
+  return build.getBuildVariables().get(envvar)
+}
+
+def pkgUrl(build) {
+  return getEnv(build, 'PKG_URL')
+}
+
 def pkgFind(glob) {
   def fullGlob =  "pkg/*${glob}"
   def found = findFiles(glob: fullGlob)
@@ -182,11 +234,17 @@ def setBuildDesc(Map links) {
   currentBuild.description = desc
 }
 
-def updateLatestNightlies(Map links) {
+def updateLatestNightlies(urls) {
+  /* latest.json has slightly different key names */
+  def latest = [
+    APK: urls.Apk, IOS: urls.iOS,
+    APP: urls.App, MAC: urls.Mac,
+    WIN: urls.Win, SHA: urls.SHA
+  ]
   def latestFile = pwd() + '/' + 'pkg/latest.json'
   /* it might not exist */
   sh 'mkdir -p pkg'
-  def latestJson = new JsonBuilder(links).toPrettyString()
+  def latestJson = new JsonBuilder(latest).toPrettyString()
   println("latest.json:\n${latestJson}")
   new File(latestFile).write(latestJson)
   return uploadArtifact(latestFile)

ci/desktop.groovy

@@ -1,4 +1,4 @@
-common = load 'ci/common.groovy'
+cmn = load 'ci/common.groovy'
 
 packageFolder = './StatusImPackage'
 
@@ -10,7 +10,7 @@ def cleanupAndDeps() {
   cleanupBuild()
   sh 'cp .env.jenkins .env'
   sh 'lein deps'
-  common.installJSDeps('desktop')
+  cmn.installJSDeps('desktop')
 }
 
 def slackNotify(message, color = 'good') {
@@ -52,7 +52,7 @@ def uploadArtifact(filename) {
 /* MAIN --------------------------------------------------*/
 
 def prepDeps() {
-  common.doGitRebase()
+  cmn.doGitRebase()
   cleanupAndDeps()
 }
 
@@ -69,7 +69,7 @@ def bundleWindows(type = 'nightly') {
 
   sh './scripts/build-desktop.sh bundle'
   dir(packageFolder) {
-    pkg = common.pkgFilename(type, 'exe')
+    pkg = cmn.pkgFilename(type, 'exe')
     sh "mv ../Status-x86_64-setup.exe ${pkg}"
   }
   return "${packageFolder}/${pkg}".drop(2)
@@ -80,14 +80,14 @@ def bundleLinux(type = 'nightly') {
 
   sh './scripts/build-desktop.sh bundle'
   dir(packageFolder) {
-    pkg = common.pkgFilename(type, 'AppImage')
+    pkg = cmn.pkgFilename(type, 'AppImage')
     sh "mv ../Status-x86_64.AppImage ${pkg}"
   }
   return "${packageFolder}/${pkg}".drop(2)
 }
 
 def bundleMacOS(type = 'nightly') {
-  def pkg = common.pkgFilename(type, 'dmg')
+  def pkg = cmn.pkgFilename(type, 'dmg')
   sh './scripts/build-desktop.sh bundle'
   dir(packageFolder) {
     withCredentials([

ci/mobile.groovy

@@ -1,4 +1,4 @@
-common = load 'ci/common.groovy'
+cmn = load 'ci/common.groovy'
 ios = load 'ci/ios.groovy'
 android = load 'ci/android.groovy'
 
@@ -34,6 +34,7 @@ def podUpdate() {
 }
 
 def prep(type = 'nightly') {
+  cmn.doGitRebase()
   /* ensure that we start from a known state */
   sh 'make clean'
   /* select type of build */
@@ -53,7 +54,9 @@ def prep(type = 'nightly') {
   sh "make prepare-${env.BUILD_PLATFORM}"
   /* generate ios/StatusIm.xcworkspace */
   dir('ios') {
-    podUpdate()
+    if (env.BUILD_PLATFORM == 'ios') {
+      podUpdate()
+    }
     sh 'pod install --silent'
   }
 }

desktop/docker/linux/Dockerfile

@@ -48,7 +48,7 @@ RUN apt-get update && apt-get -q -y --no-install-recommends install curl softwar
     apt-get update && \
     DEBIAN_FRONTEND=noninteractive \
       apt-get -q -y --no-install-recommends install \
-      wget git unzip golang-go nodejs yarn file \
+      wget git unzip golang-go nodejs yarn file s3cmd \
       python python3-pip python3-setuptools python3-wheel \
       apt-transport-https locales openjdk-8-jdk-headless \
       extra-cmake-modules build-essential gcc g++ fuse \

desktop/docker/linux/Makefile

@@ -8,7 +8,7 @@ QT_MD5SUM  = 974fda61267cfb6e45984ee5f0a285f8
 QT_URL = https://download.qt.io/archive/qt
 
 # WARNING: Remember to change the tag when updating the image
-IMAGE_TAG = 1.1.0
+IMAGE_TAG = 1.1.1
 IMAGE_NAME = statusteam/linux-desktop-ubuntu:$(IMAGE_TAG)
 
 build: $(QT_ARCHIVE)

desktop/docker/windows/Dockerfile

@@ -18,7 +18,7 @@ RUN apt-get update && apt-get -q -y --no-install-recommends install curl softwar
     apt-get update && \
     DEBIAN_FRONTEND=noninteractive \
       apt-get -q -y --no-install-recommends install \
-      wget git nsis unzip golang-go nodejs yarn file jq \
+      wget git nsis unzip golang-go nodejs yarn file jq s3cmd \
       python python3-pip python3-setuptools python3-wheel \
       apt-transport-https locales openjdk-8-jdk-headless \
       extra-cmake-modules build-essential fuse \

desktop/docker/windows/Makefile

@@ -1,7 +1,7 @@
 GIT_COMMIT = $(shell git rev-parse --short HEAD)
 
 # WARNING: Remember to change the tag when updating the image
-IMAGE_TAG = 1.1.0
+IMAGE_TAG = 1.1.1
 IMAGE_NAME = statusteam/windows-desktop-ubuntu:$(IMAGE_TAG)
 
 build: