diff --git a/ci/tests/Jenkinsfile.e2e-prs b/ci/tests/Jenkinsfile.e2e-prs
index 8c38feb1c3..3caae1507a 100644
--- a/ci/tests/Jenkinsfile.e2e-prs
+++ b/ci/tests/Jenkinsfile.e2e-prs
@@ -1,3 +1,5 @@
+library 'status-jenkins-lib@v1.4.0'
+
pipeline {
agent { label 'linux' }
@@ -101,7 +103,7 @@ pipeline {
-m \"${params.TEST_MARKERS}\" \
-k \"${params.KEYWORD_EXPRESSION}\" \
--apk=${params.APK_NAME} \
- --build=PR-${params.PR_ID} \
+ --build=PR-${params.PR_ID}-${utils.timestamp()} \
--pr_number=${params.PR_ID} \
${extraPytestOpts}
"""
diff --git a/test/appium/support/base_test_report.py b/test/appium/support/base_test_report.py
index 0fe1b535da..9ad2a92d71 100644
--- a/test/appium/support/base_test_report.py
+++ b/test/appium/support/base_test_report.py
@@ -48,7 +48,8 @@ class BaseTestReport:
'testrail_case_id': test.testrail_case_id,
'name': test.name,
'geth_paths': geth_paths,
- 'testruns': list()
+ 'testruns': list(),
+ 'group_name': test.group_name
}
for testrun in test.testruns:
test_dict['testruns'].append(testrun.__dict__)
@@ -70,7 +71,8 @@ class BaseTestReport:
tests.append(SingleTestData(name=test_data['name'],
geth_paths=test_data['geth_paths'],
testruns=testruns,
- testrail_case_id=test_data['testrail_case_id']))
+ testrail_case_id=test_data['testrail_case_id'],
+ grop_name=test_data['group_name']))
return tests
def get_failed_tests(self):
diff --git a/test/appium/support/github_report.py b/test/appium/support/github_report.py
index 538a496574..66172f9514 100644
--- a/test/appium/support/github_report.py
+++ b/test/appium/support/github_report.py
@@ -20,6 +20,7 @@ class GithubHtmlReport(BaseTestReport):
tests = self.get_all_tests()
passed_tests = self.get_passed_tests()
failed_tests = self.get_failed_tests()
+ not_executed_tests = TestrailReport().get_not_executed_tests(run_id)
if len(tests) > 0:
title_html = "## %.0f%% of end-end tests have passed\n" % (len(passed_tests) / len(tests) * 100)
@@ -27,9 +28,18 @@ class GithubHtmlReport(BaseTestReport):
summary_html += "Total executed tests: %d\n" % len(tests)
summary_html += "Failed tests: %d\n" % len(failed_tests)
summary_html += "Passed tests: %d\n" % len(passed_tests)
+ if not_executed_tests:
+ summary_html += "Not executed tests: %d\n" % len(not_executed_tests)
summary_html += "```\n"
+ not_executed_tests_html = str()
failed_tests_html = str()
passed_tests_html = str()
+ if not_executed_tests:
+ not_executed_tests_html = self.build_tests_table_html(not_executed_tests, run_id,
+ not_executed_tests=True)
+ summary_html += "```\n"
+ summary_html += 'IDs of not executed tests: %s \n' % ','.join([str(i) for i in not_executed_tests])
+ summary_html += "```\n"
if failed_tests:
failed_tests_html = self.build_tests_table_html(failed_tests, run_id, failed_tests=True)
summary_html += "```\n"
@@ -37,39 +47,53 @@ class GithubHtmlReport(BaseTestReport):
summary_html += "```\n"
if passed_tests:
passed_tests_html = self.build_tests_table_html(passed_tests, run_id, failed_tests=False)
- return title_html + summary_html + failed_tests_html + passed_tests_html
+ return title_html + summary_html + not_executed_tests_html + failed_tests_html + passed_tests_html
else:
return None
- def build_tests_table_html(self, tests, run_id, failed_tests=False):
- tests_type = "Failed tests" if failed_tests else "Passed tests"
+ def build_tests_table_html(self, tests, run_id, failed_tests=False, not_executed_tests=False):
+ if failed_tests:
+ tests_type = "Failed tests"
+ elif not_executed_tests:
+ tests_type = "Not executed tests"
+ else:
+ tests_type = "Passed tests"
html = "
%s (%d)
" % (tests_type, len(tests))
html += ""
html += "Click to expand
"
html += "
"
- if failed_tests:
- from tests import pytest_config_global
- pr_id = pytest_config_global['pr_number']
- apk_name = pytest_config_global['apk']
- tr_case_ids = self.list_of_failed_testrail_ids(self.get_failed_tests())
- html += "Rerun tests" % self.get_jenkins_link_to_rerun_e2e(pr_id=pr_id,
- apk_name=apk_name,
- tr_case_ids=tr_case_ids)
+ from tests import pytest_config_global
+ pr_id = pytest_config_global['pr_number']
+ apk_name = pytest_config_global['apk']
- html += "
"
- html += ""
- html += ""
- html += ""
- html += ""
- html += ""
- html += ""
- html += ""
- html += "
"
- for i, test in enumerate(tests):
- html += self.build_test_row_html(i, test, run_id)
- html += ""
- html += "
"
+ if not_executed_tests:
+ html += "Rerun not executed tests" % self.get_jenkins_link_to_rerun_e2e(
+ pr_id=pr_id,
+ apk_name=apk_name,
+ tr_case_ids=','.join([str(i) for i in tests]))
+
+ if failed_tests:
+ tr_case_ids = self.list_of_failed_testrail_ids(self.get_failed_tests())
+ html += "Rerun failed tests" % self.get_jenkins_link_to_rerun_e2e(
+ pr_id=pr_id,
+ apk_name=apk_name,
+ tr_case_ids=tr_case_ids)
+
+ if not not_executed_tests:
+ html += "
"
+ html += ""
+ html += ""
+ html += ""
+ html += ""
+ html += ""
+ html += ""
+ html += ""
+ html += "
"
+ for i, test in enumerate(tests):
+ html += self.build_test_row_html(i, test, run_id)
+ html += ""
+ html += "
"
html += " "
return html
@@ -77,7 +101,7 @@ class GithubHtmlReport(BaseTestReport):
test_rail_link = TestrailReport().get_test_result_link(run_id, test.testrail_case_id)
if test_rail_link:
html = "%s. %s, id: %s |
" % (
- index + 1, test_rail_link, test.name, test.testrail_case_id)
+ index + 1, test_rail_link, test.name, test.testrail_case_id)
else:
html = "%d. %s (TestRail link is not found) |
" % (index + 1, test.name)
html += ""
@@ -95,6 +119,8 @@ class GithubHtmlReport(BaseTestReport):
html += ""
html += "%s " % last_testrun.error[:255]
html += "
"
+ if test.group_name:
+ html += "Class: %s " % test.group_name
if last_testrun.jobs:
html += self.build_device_sessions_html(last_testrun)
html += " |
"
@@ -117,4 +143,4 @@ class GithubHtmlReport(BaseTestReport):
html += "Failure screenshot" % self.get_sauce_final_screenshot_url(job_id)
html += ""
html += ""
- return html
\ No newline at end of file
+ return html
diff --git a/test/appium/support/test_data.py b/test/appium/support/test_data.py
index ae56616981..a98e297949 100644
--- a/test/appium/support/test_data.py
+++ b/test/appium/support/test_data.py
@@ -2,11 +2,12 @@ from typing import Dict
class SingleTestData(object):
- def __init__(self, name, testruns, testrail_case_id, geth_paths):
+ def __init__(self, name, testruns, testrail_case_id, geth_paths, grop_name):
self.testrail_case_id = testrail_case_id
self.name = name
self.testruns = testruns
self.geth_paths = geth_paths
+ self.group_name = grop_name
class TestRunData(object):
def __init__(self, steps, jobs, error, first_commands: Dict[str, int]):
@@ -30,6 +31,6 @@ class TestSuiteData(object):
if existing_test:
self.current_test = existing_test
else:
- test = SingleTestData(test_name, list(), testrail_case_id, list())
+ test = SingleTestData(test_name, list(), testrail_case_id, list(), None)
self.tests.append(test)
- self.current_test = test
\ No newline at end of file
+ self.current_test = test
diff --git a/test/appium/support/testrail_report.py b/test/appium/support/testrail_report.py
index f960a371ca..5fb77f5603 100644
--- a/test/appium/support/testrail_report.py
+++ b/test/appium/support/testrail_report.py
@@ -159,14 +159,21 @@ class TestrailReport(BaseTestReport):
i + 1, self.get_sauce_job_url(job_id=device, first_command=last_testrun.first_commands[device]))
else:
devices += "# [Device %d](%s) \n" % (i + 1, self.get_sauce_job_url(job_id=device))
+ comment = str()
+ if test.group_name:
+ comment += "# Class: %s \n" % test.group_name
+ if last_testrun.error:
+ comment += '%s' % ('# Error: \n %s \n' % emoji.demojize(last_testrun.error)) + devices + test_steps
+ else:
+ comment += devices + test_steps
data = {'status_id': self.outcomes['undefined_fail'] if last_testrun.error else self.outcomes['passed'],
- 'comment': '%s' % ('# Error: \n %s \n' % emoji.demojize(
- last_testrun.error)) + devices + test_steps if last_testrun.error
- else devices + test_steps}
+ 'comment': comment}
+ result = self.post(method, data=data)
try:
- result_id = self.post(method, data=data)['id']
+ result_id = result['id']
except KeyError:
result_id = ''
+ print("Got TestRail error when adding results for case %s: \n%s" % (test.testrail_case_id, result))
if last_testrun.error:
try:
for geth in test.geth_paths.keys():
@@ -198,7 +205,7 @@ class TestrailReport(BaseTestReport):
case_title = '\n'
case_title += '-------\n'
case_title += "## %s) ID %s: [%s](%s) \n" % (
- i + 1, test.testrail_case_id, test.name, test_rail_link)
+ i + 1, test.testrail_case_id, test.name, test_rail_link)
error = "```%s```\n" % last_testrun.error[:255]
for job_id, f in last_testrun.jobs.items():
if last_testrun.first_commands:
@@ -216,8 +223,8 @@ class TestrailReport(BaseTestReport):
request_body = {'description': final_description}
return self.post('update_run/%s' % self.run_id, request_body)
- def get_run_results(self):
- return self.get('get_results_for_run/%s' % self.run_id)['results']
+ def get_run_results(self, test_run_id=None):
+ return self.get('get_results_for_run/%s' % (test_run_id if test_run_id else self.run_id))['results']
def is_run_successful(self):
for test in self.get_run_results():
@@ -231,4 +238,8 @@ class TestrailReport(BaseTestReport):
test_id = self.get('get_results_for_case/%s/%s' % (test_run_id, test_case_id))['results'][0]['test_id']
return '%stests/view/%s' % (self.url, test_id)
except KeyError:
- return None
\ No newline at end of file
+ return None
+
+ def get_not_executed_tests(self, test_run_id):
+ results = self.get("get_tests/%s&status_id=3" % test_run_id)
+ return [result['case_id'] for result in results["tests"]]
diff --git a/test/appium/tests/__init__.py b/test/appium/tests/__init__.py
index 123b98b249..b107d03dba 100644
--- a/test/appium/tests/__init__.py
+++ b/test/appium/tests/__init__.py
@@ -12,7 +12,7 @@ import time
async def start_threads(quantity: int, func: type, returns: dict, *args):
loop = asyncio.get_event_loop()
from tests.conftest import sauce
- for _ in range(10):
+ for _ in range(60):
if 16 - len([job for job in sauce.jobs.get_jobs() if job['status'] == 'in progress']) < quantity:
time.sleep(10)
for i in range(quantity):
diff --git a/test/appium/tests/base_test_case.py b/test/appium/tests/base_test_case.py
index db610f272d..e5d2574647 100644
--- a/test/appium/tests/base_test_case.py
+++ b/test/appium/tests/base_test_case.py
@@ -349,6 +349,7 @@ class SauceSharedMultipleDeviceTestCase(AbstractTestCase):
for index, driver in self.drivers.items():
jobs[driver.session_id] = index + 1
self.errors = Errors()
+ test_suite_data.current_test.group_name = self.__class__.__name__
def teardown_method(self, method):
geth_names, geth_contents = [], []
@@ -410,4 +411,4 @@ class NoDeviceTestCase(AbstractTestCase):
pass
def teardown_method(self, method):
- self.github_report.save_test(test_suite_data.current_test)
\ No newline at end of file
+ self.github_report.save_test(test_suite_data.current_test)