From fd6b14fcccb26ca00360d091a7c2536c42004246 Mon Sep 17 00:00:00 2001 From: yevh-berdnyk Date: Thu, 14 Apr 2022 15:59:14 +0300 Subject: [PATCH] Updated GH and TR reports Signed-off-by: yevh-berdnyk --- ci/tests/Jenkinsfile.e2e-prs | 4 +- test/appium/support/base_test_report.py | 6 +- test/appium/support/github_report.py | 78 ++++++++++++++++--------- test/appium/support/test_data.py | 7 ++- test/appium/support/testrail_report.py | 27 ++++++--- test/appium/tests/__init__.py | 2 +- test/appium/tests/base_test_case.py | 3 +- 7 files changed, 85 insertions(+), 42 deletions(-) diff --git a/ci/tests/Jenkinsfile.e2e-prs b/ci/tests/Jenkinsfile.e2e-prs index 8c38feb1c3..3caae1507a 100644 --- a/ci/tests/Jenkinsfile.e2e-prs +++ b/ci/tests/Jenkinsfile.e2e-prs @@ -1,3 +1,5 @@ +library 'status-jenkins-lib@v1.4.0' + pipeline { agent { label 'linux' } @@ -101,7 +103,7 @@ pipeline { -m \"${params.TEST_MARKERS}\" \ -k \"${params.KEYWORD_EXPRESSION}\" \ --apk=${params.APK_NAME} \ - --build=PR-${params.PR_ID} \ + --build=PR-${params.PR_ID}-${utils.timestamp()} \ --pr_number=${params.PR_ID} \ ${extraPytestOpts} """ diff --git a/test/appium/support/base_test_report.py b/test/appium/support/base_test_report.py index 0fe1b535da..9ad2a92d71 100644 --- a/test/appium/support/base_test_report.py +++ b/test/appium/support/base_test_report.py @@ -48,7 +48,8 @@ class BaseTestReport: 'testrail_case_id': test.testrail_case_id, 'name': test.name, 'geth_paths': geth_paths, - 'testruns': list() + 'testruns': list(), + 'group_name': test.group_name } for testrun in test.testruns: test_dict['testruns'].append(testrun.__dict__) @@ -70,7 +71,8 @@ class BaseTestReport: tests.append(SingleTestData(name=test_data['name'], geth_paths=test_data['geth_paths'], testruns=testruns, - testrail_case_id=test_data['testrail_case_id'])) + testrail_case_id=test_data['testrail_case_id'], + grop_name=test_data['group_name'])) return tests def get_failed_tests(self): diff --git a/test/appium/support/github_report.py b/test/appium/support/github_report.py index 538a496574..66172f9514 100644 --- a/test/appium/support/github_report.py +++ b/test/appium/support/github_report.py @@ -20,6 +20,7 @@ class GithubHtmlReport(BaseTestReport): tests = self.get_all_tests() passed_tests = self.get_passed_tests() failed_tests = self.get_failed_tests() + not_executed_tests = TestrailReport().get_not_executed_tests(run_id) if len(tests) > 0: title_html = "## %.0f%% of end-end tests have passed\n" % (len(passed_tests) / len(tests) * 100) @@ -27,9 +28,18 @@ class GithubHtmlReport(BaseTestReport): summary_html += "Total executed tests: %d\n" % len(tests) summary_html += "Failed tests: %d\n" % len(failed_tests) summary_html += "Passed tests: %d\n" % len(passed_tests) + if not_executed_tests: + summary_html += "Not executed tests: %d\n" % len(not_executed_tests) summary_html += "```\n" + not_executed_tests_html = str() failed_tests_html = str() passed_tests_html = str() + if not_executed_tests: + not_executed_tests_html = self.build_tests_table_html(not_executed_tests, run_id, + not_executed_tests=True) + summary_html += "```\n" + summary_html += 'IDs of not executed tests: %s \n' % ','.join([str(i) for i in not_executed_tests]) + summary_html += "```\n" if failed_tests: failed_tests_html = self.build_tests_table_html(failed_tests, run_id, failed_tests=True) summary_html += "```\n" @@ -37,39 +47,53 @@ class GithubHtmlReport(BaseTestReport): summary_html += "```\n" if passed_tests: passed_tests_html = self.build_tests_table_html(passed_tests, run_id, failed_tests=False) - return title_html + summary_html + failed_tests_html + passed_tests_html + return title_html + summary_html + not_executed_tests_html + failed_tests_html + passed_tests_html else: return None - def build_tests_table_html(self, tests, run_id, failed_tests=False): - tests_type = "Failed tests" if failed_tests else "Passed tests" + def build_tests_table_html(self, tests, run_id, failed_tests=False, not_executed_tests=False): + if failed_tests: + tests_type = "Failed tests" + elif not_executed_tests: + tests_type = "Not executed tests" + else: + tests_type = "Passed tests" html = "

%s (%d)

" % (tests_type, len(tests)) html += "
" html += "Click to expand" html += "
" - if failed_tests: - from tests import pytest_config_global - pr_id = pytest_config_global['pr_number'] - apk_name = pytest_config_global['apk'] - tr_case_ids = self.list_of_failed_testrail_ids(self.get_failed_tests()) - html += "
  • Rerun tests
    • " % self.get_jenkins_link_to_rerun_e2e(pr_id=pr_id, - apk_name=apk_name, - tr_case_ids=tr_case_ids) + from tests import pytest_config_global + pr_id = pytest_config_global['pr_number'] + apk_name = pytest_config_global['apk'] - html += "
    " - html += "" - html += "" - html += "" - html += "" - html += "" - html += "" - html += "" - html += "" - for i, test in enumerate(tests): - html += self.build_test_row_html(i, test, run_id) - html += "" - html += "
    " + if not_executed_tests: + html += "
  • Rerun not executed tests
    • " % self.get_jenkins_link_to_rerun_e2e( + pr_id=pr_id, + apk_name=apk_name, + tr_case_ids=','.join([str(i) for i in tests])) + + if failed_tests: + tr_case_ids = self.list_of_failed_testrail_ids(self.get_failed_tests()) + html += "
  • Rerun failed tests
    • " % self.get_jenkins_link_to_rerun_e2e( + pr_id=pr_id, + apk_name=apk_name, + tr_case_ids=tr_case_ids) + + if not not_executed_tests: + html += "
    " + html += "" + html += "" + html += "" + html += "" + html += "" + html += "" + html += "" + html += "" + for i, test in enumerate(tests): + html += self.build_test_row_html(i, test, run_id) + html += "" + html += "
    " html += "
    " return html @@ -77,7 +101,7 @@ class GithubHtmlReport(BaseTestReport): test_rail_link = TestrailReport().get_test_result_link(run_id, test.testrail_case_id) if test_rail_link: html = "%s. %s, id: %s " % ( - index + 1, test_rail_link, test.name, test.testrail_case_id) + index + 1, test_rail_link, test.name, test.testrail_case_id) else: html = "%d. %s (TestRail link is not found)" % (index + 1, test.name) html += "" @@ -95,6 +119,8 @@ class GithubHtmlReport(BaseTestReport): html += "

    " html += "%s" % last_testrun.error[:255] html += "

    " + if test.group_name: + html += "

    Class: %s

    " % test.group_name if last_testrun.jobs: html += self.build_device_sessions_html(last_testrun) html += "" @@ -117,4 +143,4 @@ class GithubHtmlReport(BaseTestReport): html += "
  • Failure screenshot
    • " % self.get_sauce_final_screenshot_url(job_id) html += "

    " html += "

    " - return html \ No newline at end of file + return html diff --git a/test/appium/support/test_data.py b/test/appium/support/test_data.py index ae56616981..a98e297949 100644 --- a/test/appium/support/test_data.py +++ b/test/appium/support/test_data.py @@ -2,11 +2,12 @@ from typing import Dict class SingleTestData(object): - def __init__(self, name, testruns, testrail_case_id, geth_paths): + def __init__(self, name, testruns, testrail_case_id, geth_paths, grop_name): self.testrail_case_id = testrail_case_id self.name = name self.testruns = testruns self.geth_paths = geth_paths + self.group_name = grop_name class TestRunData(object): def __init__(self, steps, jobs, error, first_commands: Dict[str, int]): @@ -30,6 +31,6 @@ class TestSuiteData(object): if existing_test: self.current_test = existing_test else: - test = SingleTestData(test_name, list(), testrail_case_id, list()) + test = SingleTestData(test_name, list(), testrail_case_id, list(), None) self.tests.append(test) - self.current_test = test \ No newline at end of file + self.current_test = test diff --git a/test/appium/support/testrail_report.py b/test/appium/support/testrail_report.py index f960a371ca..5fb77f5603 100644 --- a/test/appium/support/testrail_report.py +++ b/test/appium/support/testrail_report.py @@ -159,14 +159,21 @@ class TestrailReport(BaseTestReport): i + 1, self.get_sauce_job_url(job_id=device, first_command=last_testrun.first_commands[device])) else: devices += "# [Device %d](%s) \n" % (i + 1, self.get_sauce_job_url(job_id=device)) + comment = str() + if test.group_name: + comment += "# Class: %s \n" % test.group_name + if last_testrun.error: + comment += '%s' % ('# Error: \n %s \n' % emoji.demojize(last_testrun.error)) + devices + test_steps + else: + comment += devices + test_steps data = {'status_id': self.outcomes['undefined_fail'] if last_testrun.error else self.outcomes['passed'], - 'comment': '%s' % ('# Error: \n %s \n' % emoji.demojize( - last_testrun.error)) + devices + test_steps if last_testrun.error - else devices + test_steps} + 'comment': comment} + result = self.post(method, data=data) try: - result_id = self.post(method, data=data)['id'] + result_id = result['id'] except KeyError: result_id = '' + print("Got TestRail error when adding results for case %s: \n%s" % (test.testrail_case_id, result)) if last_testrun.error: try: for geth in test.geth_paths.keys(): @@ -198,7 +205,7 @@ class TestrailReport(BaseTestReport): case_title = '\n' case_title += '-------\n' case_title += "## %s) ID %s: [%s](%s) \n" % ( - i + 1, test.testrail_case_id, test.name, test_rail_link) + i + 1, test.testrail_case_id, test.name, test_rail_link) error = "```%s```\n" % last_testrun.error[:255] for job_id, f in last_testrun.jobs.items(): if last_testrun.first_commands: @@ -216,8 +223,8 @@ class TestrailReport(BaseTestReport): request_body = {'description': final_description} return self.post('update_run/%s' % self.run_id, request_body) - def get_run_results(self): - return self.get('get_results_for_run/%s' % self.run_id)['results'] + def get_run_results(self, test_run_id=None): + return self.get('get_results_for_run/%s' % (test_run_id if test_run_id else self.run_id))['results'] def is_run_successful(self): for test in self.get_run_results(): @@ -231,4 +238,8 @@ class TestrailReport(BaseTestReport): test_id = self.get('get_results_for_case/%s/%s' % (test_run_id, test_case_id))['results'][0]['test_id'] return '%stests/view/%s' % (self.url, test_id) except KeyError: - return None \ No newline at end of file + return None + + def get_not_executed_tests(self, test_run_id): + results = self.get("get_tests/%s&status_id=3" % test_run_id) + return [result['case_id'] for result in results["tests"]] diff --git a/test/appium/tests/__init__.py b/test/appium/tests/__init__.py index 123b98b249..b107d03dba 100644 --- a/test/appium/tests/__init__.py +++ b/test/appium/tests/__init__.py @@ -12,7 +12,7 @@ import time async def start_threads(quantity: int, func: type, returns: dict, *args): loop = asyncio.get_event_loop() from tests.conftest import sauce - for _ in range(10): + for _ in range(60): if 16 - len([job for job in sauce.jobs.get_jobs() if job['status'] == 'in progress']) < quantity: time.sleep(10) for i in range(quantity): diff --git a/test/appium/tests/base_test_case.py b/test/appium/tests/base_test_case.py index db610f272d..e5d2574647 100644 --- a/test/appium/tests/base_test_case.py +++ b/test/appium/tests/base_test_case.py @@ -349,6 +349,7 @@ class SauceSharedMultipleDeviceTestCase(AbstractTestCase): for index, driver in self.drivers.items(): jobs[driver.session_id] = index + 1 self.errors = Errors() + test_suite_data.current_test.group_name = self.__class__.__name__ def teardown_method(self, method): geth_names, geth_contents = [], [] @@ -410,4 +411,4 @@ class NoDeviceTestCase(AbstractTestCase): pass def teardown_method(self, method): - self.github_report.save_test(test_suite_data.current_test) \ No newline at end of file + self.github_report.save_test(test_suite_data.current_test)