add a Jenkinsfile for nightly end-to-end tests

also add error handling for TestRail

Signed-off-by: Jakub Sokołowski <jakub@status.im>

ci/Jenkinsfile.nightly-end-to-end

@@ -0,0 +1,64 @@
+pipeline {
+
+  agent { label 'linux1' }
+
+  parameters {
+    string(
+      name: 'NETWORK',
+      description: 'Name of test network to use.',
+      defaultValue: 'ropsten',
+    )
+    string(
+      name: 'APK_NAME',
+      description: 'Filename of APK uploaded to SauceLabs.',
+    )
+  }
+
+  options {
+    disableConcurrentBuilds()
+  }
+
+
+  stages {
+    stage('Test') {
+      steps {
+        withCredentials([
+          usernamePassword(
+            credentialsId:  'test-rail-api',
+            usernameVariable: 'TESTRAIL_USER',
+            passwordVariable: 'TESTRAIL_PASS'
+          ),
+          usernamePassword(
+            credentialsId:  'sauce-labs-api',
+            usernameVariable: 'SAUCE_USERNAME',
+            passwordVariable: 'SAUCE_ACCESS_KEY'
+          ),
+        ]) {
+          dir('test/appium/tests') {
+            sh """
+              python3 -m pytest -m testrail_id \
+                -n24 --rerun_count=2 \
+                --testrail_report=True \
+                --network=${params.NETWORK} \
+                --apk=${params.APK_NAME}
+            """
+          }
+        }
+      }
+    }
+  }
+
+  post {
+    always {
+      script {
+        sauce('12e007ad-48cf-4c20-92f3-b923bb5641bd') {
+          saucePublisher()
+        }
+        junit(
+          testDataPublishers: [[$class: 'SauceOnDemandReportPublisher', jobVisibility: 'public']],
+          testResults: 'test/appium/tests/*.xml'
+        )
+      }
+    }
+  }
+}

ci/android.groovy

@@ -86,7 +86,6 @@ def uploadToSauceLabs() {
 }
 
 def uploadToDiawi() {
-  env.SAUCE_LABS_NAME = "im.status.ethereum-e2e-${GIT_COMMIT.take(6)}.apk"
   withCredentials([
     string(credentialsId: 'diawi-token', variable: 'DIAWI_TOKEN'),
   ]) {

test/appium/support/testrail_report.py

@@ -31,7 +31,10 @@ class TestrailReport(BaseTestReport):
         self.api_url = self.url + 'api/v2/'
 
     def get(self, method):
-        return requests.get(self.api_url + method, headers=self.headers).json()
+        rval = requests.get(self.api_url + method, headers=self.headers).json()
+        if 'error' in rval:
+            raise Exception('Failed request: %s' % rval['error'])
+        return rval
 
     def post(self, method, data):
         data = bytes(json.dumps(data), 'utf-8')