Updated GH and TR reports
Signed-off-by: yevh-berdnyk <ie.berdnyk@gmail.com>
This commit is contained in:
parent
60ce2ea5d7
commit
fd6b14fccc
|
@ -1,3 +1,5 @@
|
||||||
|
library 'status-jenkins-lib@v1.4.0'
|
||||||
|
|
||||||
pipeline {
|
pipeline {
|
||||||
|
|
||||||
agent { label 'linux' }
|
agent { label 'linux' }
|
||||||
|
@ -101,7 +103,7 @@ pipeline {
|
||||||
-m \"${params.TEST_MARKERS}\" \
|
-m \"${params.TEST_MARKERS}\" \
|
||||||
-k \"${params.KEYWORD_EXPRESSION}\" \
|
-k \"${params.KEYWORD_EXPRESSION}\" \
|
||||||
--apk=${params.APK_NAME} \
|
--apk=${params.APK_NAME} \
|
||||||
--build=PR-${params.PR_ID} \
|
--build=PR-${params.PR_ID}-${utils.timestamp()} \
|
||||||
--pr_number=${params.PR_ID} \
|
--pr_number=${params.PR_ID} \
|
||||||
${extraPytestOpts}
|
${extraPytestOpts}
|
||||||
"""
|
"""
|
||||||
|
|
|
@ -48,7 +48,8 @@ class BaseTestReport:
|
||||||
'testrail_case_id': test.testrail_case_id,
|
'testrail_case_id': test.testrail_case_id,
|
||||||
'name': test.name,
|
'name': test.name,
|
||||||
'geth_paths': geth_paths,
|
'geth_paths': geth_paths,
|
||||||
'testruns': list()
|
'testruns': list(),
|
||||||
|
'group_name': test.group_name
|
||||||
}
|
}
|
||||||
for testrun in test.testruns:
|
for testrun in test.testruns:
|
||||||
test_dict['testruns'].append(testrun.__dict__)
|
test_dict['testruns'].append(testrun.__dict__)
|
||||||
|
@ -70,7 +71,8 @@ class BaseTestReport:
|
||||||
tests.append(SingleTestData(name=test_data['name'],
|
tests.append(SingleTestData(name=test_data['name'],
|
||||||
geth_paths=test_data['geth_paths'],
|
geth_paths=test_data['geth_paths'],
|
||||||
testruns=testruns,
|
testruns=testruns,
|
||||||
testrail_case_id=test_data['testrail_case_id']))
|
testrail_case_id=test_data['testrail_case_id'],
|
||||||
|
grop_name=test_data['group_name']))
|
||||||
return tests
|
return tests
|
||||||
|
|
||||||
def get_failed_tests(self):
|
def get_failed_tests(self):
|
||||||
|
|
|
@ -20,6 +20,7 @@ class GithubHtmlReport(BaseTestReport):
|
||||||
tests = self.get_all_tests()
|
tests = self.get_all_tests()
|
||||||
passed_tests = self.get_passed_tests()
|
passed_tests = self.get_passed_tests()
|
||||||
failed_tests = self.get_failed_tests()
|
failed_tests = self.get_failed_tests()
|
||||||
|
not_executed_tests = TestrailReport().get_not_executed_tests(run_id)
|
||||||
|
|
||||||
if len(tests) > 0:
|
if len(tests) > 0:
|
||||||
title_html = "## %.0f%% of end-end tests have passed\n" % (len(passed_tests) / len(tests) * 100)
|
title_html = "## %.0f%% of end-end tests have passed\n" % (len(passed_tests) / len(tests) * 100)
|
||||||
|
@ -27,9 +28,18 @@ class GithubHtmlReport(BaseTestReport):
|
||||||
summary_html += "Total executed tests: %d\n" % len(tests)
|
summary_html += "Total executed tests: %d\n" % len(tests)
|
||||||
summary_html += "Failed tests: %d\n" % len(failed_tests)
|
summary_html += "Failed tests: %d\n" % len(failed_tests)
|
||||||
summary_html += "Passed tests: %d\n" % len(passed_tests)
|
summary_html += "Passed tests: %d\n" % len(passed_tests)
|
||||||
|
if not_executed_tests:
|
||||||
|
summary_html += "Not executed tests: %d\n" % len(not_executed_tests)
|
||||||
summary_html += "```\n"
|
summary_html += "```\n"
|
||||||
|
not_executed_tests_html = str()
|
||||||
failed_tests_html = str()
|
failed_tests_html = str()
|
||||||
passed_tests_html = str()
|
passed_tests_html = str()
|
||||||
|
if not_executed_tests:
|
||||||
|
not_executed_tests_html = self.build_tests_table_html(not_executed_tests, run_id,
|
||||||
|
not_executed_tests=True)
|
||||||
|
summary_html += "```\n"
|
||||||
|
summary_html += 'IDs of not executed tests: %s \n' % ','.join([str(i) for i in not_executed_tests])
|
||||||
|
summary_html += "```\n"
|
||||||
if failed_tests:
|
if failed_tests:
|
||||||
failed_tests_html = self.build_tests_table_html(failed_tests, run_id, failed_tests=True)
|
failed_tests_html = self.build_tests_table_html(failed_tests, run_id, failed_tests=True)
|
||||||
summary_html += "```\n"
|
summary_html += "```\n"
|
||||||
|
@ -37,26 +47,40 @@ class GithubHtmlReport(BaseTestReport):
|
||||||
summary_html += "```\n"
|
summary_html += "```\n"
|
||||||
if passed_tests:
|
if passed_tests:
|
||||||
passed_tests_html = self.build_tests_table_html(passed_tests, run_id, failed_tests=False)
|
passed_tests_html = self.build_tests_table_html(passed_tests, run_id, failed_tests=False)
|
||||||
return title_html + summary_html + failed_tests_html + passed_tests_html
|
return title_html + summary_html + not_executed_tests_html + failed_tests_html + passed_tests_html
|
||||||
else:
|
else:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def build_tests_table_html(self, tests, run_id, failed_tests=False):
|
def build_tests_table_html(self, tests, run_id, failed_tests=False, not_executed_tests=False):
|
||||||
tests_type = "Failed tests" if failed_tests else "Passed tests"
|
if failed_tests:
|
||||||
|
tests_type = "Failed tests"
|
||||||
|
elif not_executed_tests:
|
||||||
|
tests_type = "Not executed tests"
|
||||||
|
else:
|
||||||
|
tests_type = "Passed tests"
|
||||||
html = "<h3>%s (%d)</h3>" % (tests_type, len(tests))
|
html = "<h3>%s (%d)</h3>" % (tests_type, len(tests))
|
||||||
html += "<details>"
|
html += "<details>"
|
||||||
html += "<summary>Click to expand</summary>"
|
html += "<summary>Click to expand</summary>"
|
||||||
html += "<br/>"
|
html += "<br/>"
|
||||||
|
|
||||||
if failed_tests:
|
|
||||||
from tests import pytest_config_global
|
from tests import pytest_config_global
|
||||||
pr_id = pytest_config_global['pr_number']
|
pr_id = pytest_config_global['pr_number']
|
||||||
apk_name = pytest_config_global['apk']
|
apk_name = pytest_config_global['apk']
|
||||||
|
|
||||||
|
if not_executed_tests:
|
||||||
|
html += "<li><a href=\"%s\">Rerun not executed tests</a></li>" % self.get_jenkins_link_to_rerun_e2e(
|
||||||
|
pr_id=pr_id,
|
||||||
|
apk_name=apk_name,
|
||||||
|
tr_case_ids=','.join([str(i) for i in tests]))
|
||||||
|
|
||||||
|
if failed_tests:
|
||||||
tr_case_ids = self.list_of_failed_testrail_ids(self.get_failed_tests())
|
tr_case_ids = self.list_of_failed_testrail_ids(self.get_failed_tests())
|
||||||
html += "<li><a href=\"%s\">Rerun tests</a></li>" % self.get_jenkins_link_to_rerun_e2e(pr_id=pr_id,
|
html += "<li><a href=\"%s\">Rerun failed tests</a></li>" % self.get_jenkins_link_to_rerun_e2e(
|
||||||
|
pr_id=pr_id,
|
||||||
apk_name=apk_name,
|
apk_name=apk_name,
|
||||||
tr_case_ids=tr_case_ids)
|
tr_case_ids=tr_case_ids)
|
||||||
|
|
||||||
|
if not not_executed_tests:
|
||||||
html += "<br/>"
|
html += "<br/>"
|
||||||
html += "<table style=\"width: 100%\">"
|
html += "<table style=\"width: 100%\">"
|
||||||
html += "<colgroup>"
|
html += "<colgroup>"
|
||||||
|
@ -95,6 +119,8 @@ class GithubHtmlReport(BaseTestReport):
|
||||||
html += "</p>"
|
html += "</p>"
|
||||||
html += "<code>%s</code>" % last_testrun.error[:255]
|
html += "<code>%s</code>" % last_testrun.error[:255]
|
||||||
html += "<br/><br/>"
|
html += "<br/><br/>"
|
||||||
|
if test.group_name:
|
||||||
|
html += "<p><b>Class: %s</b></p>" % test.group_name
|
||||||
if last_testrun.jobs:
|
if last_testrun.jobs:
|
||||||
html += self.build_device_sessions_html(last_testrun)
|
html += self.build_device_sessions_html(last_testrun)
|
||||||
html += "</td></tr>"
|
html += "</td></tr>"
|
||||||
|
|
|
@ -2,11 +2,12 @@ from typing import Dict
|
||||||
|
|
||||||
|
|
||||||
class SingleTestData(object):
|
class SingleTestData(object):
|
||||||
def __init__(self, name, testruns, testrail_case_id, geth_paths):
|
def __init__(self, name, testruns, testrail_case_id, geth_paths, grop_name):
|
||||||
self.testrail_case_id = testrail_case_id
|
self.testrail_case_id = testrail_case_id
|
||||||
self.name = name
|
self.name = name
|
||||||
self.testruns = testruns
|
self.testruns = testruns
|
||||||
self.geth_paths = geth_paths
|
self.geth_paths = geth_paths
|
||||||
|
self.group_name = grop_name
|
||||||
|
|
||||||
class TestRunData(object):
|
class TestRunData(object):
|
||||||
def __init__(self, steps, jobs, error, first_commands: Dict[str, int]):
|
def __init__(self, steps, jobs, error, first_commands: Dict[str, int]):
|
||||||
|
@ -30,6 +31,6 @@ class TestSuiteData(object):
|
||||||
if existing_test:
|
if existing_test:
|
||||||
self.current_test = existing_test
|
self.current_test = existing_test
|
||||||
else:
|
else:
|
||||||
test = SingleTestData(test_name, list(), testrail_case_id, list())
|
test = SingleTestData(test_name, list(), testrail_case_id, list(), None)
|
||||||
self.tests.append(test)
|
self.tests.append(test)
|
||||||
self.current_test = test
|
self.current_test = test
|
|
@ -159,14 +159,21 @@ class TestrailReport(BaseTestReport):
|
||||||
i + 1, self.get_sauce_job_url(job_id=device, first_command=last_testrun.first_commands[device]))
|
i + 1, self.get_sauce_job_url(job_id=device, first_command=last_testrun.first_commands[device]))
|
||||||
else:
|
else:
|
||||||
devices += "# [Device %d](%s) \n" % (i + 1, self.get_sauce_job_url(job_id=device))
|
devices += "# [Device %d](%s) \n" % (i + 1, self.get_sauce_job_url(job_id=device))
|
||||||
|
comment = str()
|
||||||
|
if test.group_name:
|
||||||
|
comment += "# Class: %s \n" % test.group_name
|
||||||
|
if last_testrun.error:
|
||||||
|
comment += '%s' % ('# Error: \n %s \n' % emoji.demojize(last_testrun.error)) + devices + test_steps
|
||||||
|
else:
|
||||||
|
comment += devices + test_steps
|
||||||
data = {'status_id': self.outcomes['undefined_fail'] if last_testrun.error else self.outcomes['passed'],
|
data = {'status_id': self.outcomes['undefined_fail'] if last_testrun.error else self.outcomes['passed'],
|
||||||
'comment': '%s' % ('# Error: \n %s \n' % emoji.demojize(
|
'comment': comment}
|
||||||
last_testrun.error)) + devices + test_steps if last_testrun.error
|
result = self.post(method, data=data)
|
||||||
else devices + test_steps}
|
|
||||||
try:
|
try:
|
||||||
result_id = self.post(method, data=data)['id']
|
result_id = result['id']
|
||||||
except KeyError:
|
except KeyError:
|
||||||
result_id = ''
|
result_id = ''
|
||||||
|
print("Got TestRail error when adding results for case %s: \n%s" % (test.testrail_case_id, result))
|
||||||
if last_testrun.error:
|
if last_testrun.error:
|
||||||
try:
|
try:
|
||||||
for geth in test.geth_paths.keys():
|
for geth in test.geth_paths.keys():
|
||||||
|
@ -216,8 +223,8 @@ class TestrailReport(BaseTestReport):
|
||||||
request_body = {'description': final_description}
|
request_body = {'description': final_description}
|
||||||
return self.post('update_run/%s' % self.run_id, request_body)
|
return self.post('update_run/%s' % self.run_id, request_body)
|
||||||
|
|
||||||
def get_run_results(self):
|
def get_run_results(self, test_run_id=None):
|
||||||
return self.get('get_results_for_run/%s' % self.run_id)['results']
|
return self.get('get_results_for_run/%s' % (test_run_id if test_run_id else self.run_id))['results']
|
||||||
|
|
||||||
def is_run_successful(self):
|
def is_run_successful(self):
|
||||||
for test in self.get_run_results():
|
for test in self.get_run_results():
|
||||||
|
@ -232,3 +239,7 @@ class TestrailReport(BaseTestReport):
|
||||||
return '%stests/view/%s' % (self.url, test_id)
|
return '%stests/view/%s' % (self.url, test_id)
|
||||||
except KeyError:
|
except KeyError:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
def get_not_executed_tests(self, test_run_id):
|
||||||
|
results = self.get("get_tests/%s&status_id=3" % test_run_id)
|
||||||
|
return [result['case_id'] for result in results["tests"]]
|
||||||
|
|
|
@ -12,7 +12,7 @@ import time
|
||||||
async def start_threads(quantity: int, func: type, returns: dict, *args):
|
async def start_threads(quantity: int, func: type, returns: dict, *args):
|
||||||
loop = asyncio.get_event_loop()
|
loop = asyncio.get_event_loop()
|
||||||
from tests.conftest import sauce
|
from tests.conftest import sauce
|
||||||
for _ in range(10):
|
for _ in range(60):
|
||||||
if 16 - len([job for job in sauce.jobs.get_jobs() if job['status'] == 'in progress']) < quantity:
|
if 16 - len([job for job in sauce.jobs.get_jobs() if job['status'] == 'in progress']) < quantity:
|
||||||
time.sleep(10)
|
time.sleep(10)
|
||||||
for i in range(quantity):
|
for i in range(quantity):
|
||||||
|
|
|
@ -349,6 +349,7 @@ class SauceSharedMultipleDeviceTestCase(AbstractTestCase):
|
||||||
for index, driver in self.drivers.items():
|
for index, driver in self.drivers.items():
|
||||||
jobs[driver.session_id] = index + 1
|
jobs[driver.session_id] = index + 1
|
||||||
self.errors = Errors()
|
self.errors = Errors()
|
||||||
|
test_suite_data.current_test.group_name = self.__class__.__name__
|
||||||
|
|
||||||
def teardown_method(self, method):
|
def teardown_method(self, method):
|
||||||
geth_names, geth_contents = [], []
|
geth_names, geth_contents = [], []
|
||||||
|
|
Loading…
Reference in New Issue