status-im
/
status-mobile
mirror of https://github.com/status-im/status-mobile.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Updated GH and TR reports 

Signed-off-by: yevh-berdnyk <ie.berdnyk@gmail.com>
This commit is contained in:
yevh-berdnyk 2022-04-14 15:59:14 +03:00
parent 60ce2ea5d7
commit fd6b14fccc
No known key found for this signature in database
GPG Key ID: 0642C73C66214825
7 changed files with 85 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
ci/tests/Jenkinsfile.e2e-prs
View File

@ -1,3 +1,5 @@
library 'status-jenkins-lib@v1.4.0'
pipeline {
agent { label 'linux' }
@ -101,7 +103,7 @@ pipeline {
-m \"${params.TEST_MARKERS}\" \
-k \"${params.KEYWORD_EXPRESSION}\" \
--apk=${params.APK_NAME} \
--build=PR-${params.PR_ID} \
--build=PR-${params.PR_ID}-${utils.timestamp()} \
--pr_number=${params.PR_ID} \
${extraPytestOpts}
"""

6
test/appium/support/base_test_report.py
View File

@ -48,7 +48,8 @@ class BaseTestReport:
'testrail_case_id': test.testrail_case_id,
'name': test.name,
'geth_paths': geth_paths,
'testruns': list()
'testruns': list(),
'group_name': test.group_name
}
for testrun in test.testruns:
test_dict['testruns'].append(testrun.__dict__)
@ -70,7 +71,8 @@ class BaseTestReport:
tests.append(SingleTestData(name=test_data['name'],
geth_paths=test_data['geth_paths'],
testruns=testruns,
testrail_case_id=test_data['testrail_case_id']))
testrail_case_id=test_data['testrail_case_id'],
grop_name=test_data['group_name']))
return tests
def get_failed_tests(self):

78
test/appium/support/github_report.py
View File

@ -20,6 +20,7 @@ class GithubHtmlReport(BaseTestReport):
tests = self.get_all_tests()
passed_tests = self.get_passed_tests()
failed_tests = self.get_failed_tests()
not_executed_tests = TestrailReport().get_not_executed_tests(run_id)
if len(tests) > 0:
title_html = "## %.0f%% of end-end tests have passed\n" % (len(passed_tests) / len(tests) * 100)
@ -27,9 +28,18 @@ class GithubHtmlReport(BaseTestReport):
summary_html += "Total executed tests: %d\n" % len(tests)
summary_html += "Failed tests: %d\n" % len(failed_tests)
summary_html += "Passed tests: %d\n" % len(passed_tests)
if not_executed_tests:
summary_html += "Not executed tests: %d\n" % len(not_executed_tests)
summary_html += "```\n"
not_executed_tests_html = str()
failed_tests_html = str()
passed_tests_html = str()
if not_executed_tests:
not_executed_tests_html = self.build_tests_table_html(not_executed_tests, run_id,
not_executed_tests=True)
summary_html += "```\n"
summary_html += 'IDs of not executed tests: %s \n' % ','.join([str(i) for i in not_executed_tests])
summary_html += "```\n"
if failed_tests:
failed_tests_html = self.build_tests_table_html(failed_tests, run_id, failed_tests=True)
summary_html += "```\n"
@ -37,39 +47,53 @@ class GithubHtmlReport(BaseTestReport):
summary_html += "```\n"
if passed_tests:
passed_tests_html = self.build_tests_table_html(passed_tests, run_id, failed_tests=False)
return title_html + summary_html + failed_tests_html + passed_tests_html
return title_html + summary_html + not_executed_tests_html + failed_tests_html + passed_tests_html
else:
return None
def build_tests_table_html(self, tests, run_id, failed_tests=False):
tests_type = "Failed tests" if failed_tests else "Passed tests"
def build_tests_table_html(self, tests, run_id, failed_tests=False, not_executed_tests=False):
if failed_tests:
tests_type = "Failed tests"
elif not_executed_tests:
tests_type = "Not executed tests"
else:
tests_type = "Passed tests"
html = "<h3>%s (%d)</h3>" % (tests_type, len(tests))
html += "<details>"
html += "<summary>Click to expand</summary>"
html += "<br/>"
if failed_tests:
from tests import pytest_config_global
pr_id = pytest_config_global['pr_number']
apk_name = pytest_config_global['apk']
tr_case_ids = self.list_of_failed_testrail_ids(self.get_failed_tests())
html += "<li><a href=\"%s\">Rerun tests</a></li>" % self.get_jenkins_link_to_rerun_e2e(pr_id=pr_id,
apk_name=apk_name,
tr_case_ids=tr_case_ids)
from tests import pytest_config_global
pr_id = pytest_config_global['pr_number']
apk_name = pytest_config_global['apk']
html += "<br/>"
html += "<table style=\"width: 100%\">"
html += "<colgroup>"
html += "<col span=\"1\" style=\"width: 20%;\">"
html += "<col span=\"1\" style=\"width: 80%;\">"
html += "</colgroup>"
html += "<tbody>"
html += "<tr>"
html += "</tr>"
for i, test in enumerate(tests):
html += self.build_test_row_html(i, test, run_id)
html += "</tbody>"
html += "</table>"
if not_executed_tests:
html += "<li><a href=\"%s\">Rerun not executed tests</a></li>" % self.get_jenkins_link_to_rerun_e2e(
pr_id=pr_id,
apk_name=apk_name,
tr_case_ids=','.join([str(i) for i in tests]))
if failed_tests:
tr_case_ids = self.list_of_failed_testrail_ids(self.get_failed_tests())
html += "<li><a href=\"%s\">Rerun failed tests</a></li>" % self.get_jenkins_link_to_rerun_e2e(
pr_id=pr_id,
apk_name=apk_name,
tr_case_ids=tr_case_ids)
if not not_executed_tests:
html += "<br/>"
html += "<table style=\"width: 100%\">"
html += "<colgroup>"
html += "<col span=\"1\" style=\"width: 20%;\">"
html += "<col span=\"1\" style=\"width: 80%;\">"
html += "</colgroup>"
html += "<tbody>"
html += "<tr>"
html += "</tr>"
for i, test in enumerate(tests):
html += self.build_test_row_html(i, test, run_id)
html += "</tbody>"
html += "</table>"
html += "</details>"
return html
@ -77,7 +101,7 @@ class GithubHtmlReport(BaseTestReport):
test_rail_link = TestrailReport().get_test_result_link(run_id, test.testrail_case_id)
if test_rail_link:
html = "<tr><td><b>%s. <a href=\"%s\">%s</a>, id: %s </b></td></tr>" % (
index + 1, test_rail_link, test.name, test.testrail_case_id)
index + 1, test_rail_link, test.name, test.testrail_case_id)
else:
html = "<tr><td><b>%d. %s</b> (TestRail link is not found)</td></tr>" % (index + 1, test.name)
html += "<tr><td>"
@ -95,6 +119,8 @@ class GithubHtmlReport(BaseTestReport):
html += "</p>"
html += "<code>%s</code>" % last_testrun.error[:255]
html += "<br/><br/>"
if test.group_name:
html += "<p><b>Class: %s</b></p>" % test.group_name
if last_testrun.jobs:
html += self.build_device_sessions_html(last_testrun)
html += "</td></tr>"
@ -117,4 +143,4 @@ class GithubHtmlReport(BaseTestReport):
html += "<li><a href=\"%s\">Failure screenshot</a></li>" % self.get_sauce_final_screenshot_url(job_id)
html += "</ul></p>"
html += "</ul></p>"
return html
return html

7
test/appium/support/test_data.py
View File

@ -2,11 +2,12 @@ from typing import Dict
class SingleTestData(object):
def __init__(self, name, testruns, testrail_case_id, geth_paths):
def __init__(self, name, testruns, testrail_case_id, geth_paths, grop_name):
self.testrail_case_id = testrail_case_id
self.name = name
self.testruns = testruns
self.geth_paths = geth_paths
self.group_name = grop_name
class TestRunData(object):
def __init__(self, steps, jobs, error, first_commands: Dict[str, int]):
@ -30,6 +31,6 @@ class TestSuiteData(object):
if existing_test:
self.current_test = existing_test
else:
test = SingleTestData(test_name, list(), testrail_case_id, list())
test = SingleTestData(test_name, list(), testrail_case_id, list(), None)
self.tests.append(test)
self.current_test = test
self.current_test = test

27
test/appium/support/testrail_report.py
View File

@ -159,14 +159,21 @@ class TestrailReport(BaseTestReport):
i + 1, self.get_sauce_job_url(job_id=device, first_command=last_testrun.first_commands[device]))
else:
devices += "# [Device %d](%s) \n" % (i + 1, self.get_sauce_job_url(job_id=device))
comment = str()
if test.group_name:
comment += "# Class: %s \n" % test.group_name
if last_testrun.error:
comment += '%s' % ('# Error: \n %s \n' % emoji.demojize(last_testrun.error)) + devices + test_steps
else:
comment += devices + test_steps
data = {'status_id': self.outcomes['undefined_fail'] if last_testrun.error else self.outcomes['passed'],
'comment': '%s' % ('# Error: \n %s \n' % emoji.demojize(
last_testrun.error)) + devices + test_steps if last_testrun.error
else devices + test_steps}
'comment': comment}
result = self.post(method, data=data)
try:
result_id = self.post(method, data=data)['id']
result_id = result['id']
except KeyError:
result_id = ''
print("Got TestRail error when adding results for case %s: \n%s" % (test.testrail_case_id, result))
if last_testrun.error:
try:
for geth in test.geth_paths.keys():
@ -198,7 +205,7 @@ class TestrailReport(BaseTestReport):
case_title = '\n'
case_title += '-------\n'
case_title += "## %s) ID %s: [%s](%s) \n" % (
i + 1, test.testrail_case_id, test.name, test_rail_link)
i + 1, test.testrail_case_id, test.name, test_rail_link)
error = "```%s```\n" % last_testrun.error[:255]
for job_id, f in last_testrun.jobs.items():
if last_testrun.first_commands:
@ -216,8 +223,8 @@ class TestrailReport(BaseTestReport):
request_body = {'description': final_description}
return self.post('update_run/%s' % self.run_id, request_body)
def get_run_results(self):
return self.get('get_results_for_run/%s' % self.run_id)['results']
def get_run_results(self, test_run_id=None):
return self.get('get_results_for_run/%s' % (test_run_id if test_run_id else self.run_id))['results']
def is_run_successful(self):
for test in self.get_run_results():
@ -231,4 +238,8 @@ class TestrailReport(BaseTestReport):
test_id = self.get('get_results_for_case/%s/%s' % (test_run_id, test_case_id))['results'][0]['test_id']
return '%stests/view/%s' % (self.url, test_id)
except KeyError:
return None
return None
def get_not_executed_tests(self, test_run_id):
results = self.get("get_tests/%s&status_id=3" % test_run_id)
return [result['case_id'] for result in results["tests"]]

2
test/appium/tests/__init__.py
View File

@ -12,7 +12,7 @@ import time
async def start_threads(quantity: int, func: type, returns: dict, *args):
loop = asyncio.get_event_loop()
from tests.conftest import sauce
for _ in range(10):
for _ in range(60):
if 16 - len([job for job in sauce.jobs.get_jobs() if job['status'] == 'in progress']) < quantity:
time.sleep(10)
for i in range(quantity):

3
test/appium/tests/base_test_case.py
View File

@ -349,6 +349,7 @@ class SauceSharedMultipleDeviceTestCase(AbstractTestCase):
for index, driver in self.drivers.items():
jobs[driver.session_id] = index + 1
self.errors = Errors()
test_suite_data.current_test.group_name = self.__class__.__name__
def teardown_method(self, method):
geth_names, geth_contents = [], []
@ -410,4 +411,4 @@ class NoDeviceTestCase(AbstractTestCase):
pass
def teardown_method(self, method):
self.github_report.save_test(test_suite_data.current_test)
self.github_report.save_test(test_suite_data.current_test)