diff --git a/test/appium/support/base_test_report.py b/test/appium/support/base_test_report.py index ca5580d0de..2b2e5cd0e4 100644 --- a/test/appium/support/base_test_report.py +++ b/test/appium/support/base_test_report.py @@ -84,18 +84,10 @@ class BaseTestReport: tests = self.get_all_tests() failed = list() for test in tests: - if not self.is_test_successful(test) and not self.is_test_xfailed(test): + if not self.is_test_successful(test): failed.append(test) return failed - def get_xfailed_tests(self): - tests = self.get_all_tests() - xfailed = list() - for test in tests: - if self.is_test_xfailed(test) and not self.is_test_successful(test): - xfailed.append(test) - return xfailed - def get_passed_tests(self): tests = self.get_all_tests() passed = list() @@ -134,11 +126,6 @@ class BaseTestReport: # Test passed if last testrun has passed return test.testruns[-1].error is None - @staticmethod - def is_test_xfailed(test): - # Test failed if xfail is defined - return test.testruns[-1].xfail != "" - @staticmethod def separate_xfail_error(error): issue_id_list = re.findall(r'#\d+', error) diff --git a/test/appium/support/github_report.py b/test/appium/support/github_report.py index 7db7d9e3aa..0e30871ea0 100644 --- a/test/appium/support/github_report.py +++ b/test/appium/support/github_report.py @@ -21,7 +21,6 @@ class GithubHtmlReport(BaseTestReport): tests = self.get_all_tests() passed_tests = self.get_passed_tests() failed_tests = self.get_failed_tests() - xfailed_tests = self.get_xfailed_tests() not_executed_tests = TestrailReport().get_not_executed_tests(run_id) if len(tests) > 0: @@ -33,7 +32,9 @@ class GithubHtmlReport(BaseTestReport): if not_executed_tests: summary_html += "Not executed tests: %d\n" % len(not_executed_tests) summary_html += "```\n" - failed_tests_html, xfailed_tests_html, passed_tests_html, not_executed_tests_html = str(), str(), str(), str() + not_executed_tests_html = str() + failed_tests_html = str() + passed_tests_html = str() if not_executed_tests: not_executed_tests_html = self.build_tests_table_html(not_executed_tests, run_id, not_executed_tests=True) @@ -43,26 +44,19 @@ class GithubHtmlReport(BaseTestReport): if failed_tests: failed_tests_html = self.build_tests_table_html(failed_tests, run_id, failed_tests=True) summary_html += "```\n" - summary_html += 'IDs of failed tests(should be reviewed): %s \n' % self.list_of_failed_testrail_ids(failed_tests) - summary_html += "```\n" - if xfailed_tests: - xfailed_tests_html = self.build_tests_table_html(xfailed_tests, run_id, failed_tests=False, xfailed_tests=True) - summary_html += "```\n" - summary_html += 'IDs of failed tests(existing issues): %s \n' % self.list_of_failed_testrail_ids(xfailed_tests) + summary_html += 'IDs of failed tests: %s \n' % self.list_of_failed_testrail_ids(failed_tests) summary_html += "```\n" if passed_tests: passed_tests_html = self.build_tests_table_html(passed_tests, run_id, failed_tests=False) - return title_html + summary_html + not_executed_tests_html + failed_tests_html + passed_tests_html + xfailed_tests_html + return title_html + summary_html + not_executed_tests_html + failed_tests_html + passed_tests_html else: return None - def build_tests_table_html(self, tests, run_id, failed_tests=False, not_executed_tests=False, xfailed_tests=False): + def build_tests_table_html(self, tests, run_id, failed_tests=False, not_executed_tests=False): if failed_tests: - tests_type = "Failed tests (for review)" + tests_type = "Failed tests" elif not_executed_tests: tests_type = "Not executed tests" - elif xfailed_tests: - tests_type = "Failed tests (known)" else: tests_type = "Passed tests" html = "

%s (%d)

" % (tests_type, len(tests)) diff --git a/test/appium/support/testrail_report.py b/test/appium/support/testrail_report.py index 302ce1192e..347a689e5d 100644 --- a/test/appium/support/testrail_report.py +++ b/test/appium/support/testrail_report.py @@ -28,8 +28,7 @@ class TestrailReport(BaseTestReport): self.outcomes = { 'passed': 1, - 'undefined_fail': 10, - 'fail': 5} + 'undefined_fail': 10} self.headers = dict() self.headers['Authorization'] = 'Basic %s' % str( @@ -182,7 +181,6 @@ class TestrailReport(BaseTestReport): test_steps = "# Steps: \n" devices = str() last_testrun = test.testruns[-1] - status = self.outcomes['passed'] for step in last_testrun.steps: test_steps += step + "\n" for i, device in enumerate(last_testrun.jobs): @@ -194,12 +192,7 @@ class TestrailReport(BaseTestReport): comment = str() if test.group_name: comment += "# Class: %s \n" % test.group_name - if last_testrun.error and not last_testrun.xfail: - status = self.outcomes['fail'] - error = last_testrun.error - comment += '%s' % ('# Error: \n %s \n' % emoji.demojize(error)) + devices + test_steps - elif last_testrun.error and last_testrun.xfail: - status = self.outcomes['undefined_fail'] + if last_testrun.error: full_error = last_testrun.error (code_error, no_code_error_str, issue_id) = self.separate_xfail_error(full_error) if issue_id: @@ -213,7 +206,7 @@ class TestrailReport(BaseTestReport): comment += devices + test_steps data.append( {'case_id': test.testrail_case_id, - 'status_id': status, + 'status_id': self.outcomes['undefined_fail'] if last_testrun.error else self.outcomes['passed'], 'comment': comment}) results = self.post('add_results_for_cases/%s' % self.run_id, data={"results": data}) @@ -246,77 +239,58 @@ class TestrailReport(BaseTestReport): self.change_test_run_description() - def make_single_group_report(self, tests, xfailed=False): - single_devices_block, group_blocks, case_info, test_ids = str(), dict(), str(), [] - for test in tests: - if test.group_name: - group_blocks[test.group_name] = "\n-------\n## Class: %s:\n" % test.group_name - for test in tests: - last_testrun = test.testruns[-1] - test_rail_link = self.get_test_result_link(self.run_id, test.testrail_case_id) - test_ids.append(test.testrail_case_id) - case_title = '\n' - case_title += '-------\n' - case_title += "### ID %s: [%s](%s) \n" % (test.testrail_case_id, test.name, test_rail_link) - if xfailed: - full_error = last_testrun.error[-255:] - (code_error, no_code_error_str, issue_id) = self.separate_xfail_error(full_error) - if issue_id: - test_rail_xfail = self.make_error_with_gh_issue_link(no_code_error_str, issue_id) - error = "```%s```\n **%s** \n" % (code_error, test_rail_xfail) - else: - error = "```%s```\n **%s** \n" % (code_error, no_code_error_str) - else: - error = last_testrun.error[-255:] - for job_id, f in last_testrun.jobs.items(): - if last_testrun.first_commands: - job_url = self.get_sauce_job_url(job_id=job_id, - first_command=last_testrun.first_commands[job_id]) - else: - job_url = self.get_sauce_job_url(job_id=job_id) - case_info = "Logs for device %d: [steps](%s), [failure screenshot](%s)" \ - % (f, job_url, self.get_sauce_final_screenshot_url(job_id)) - - if test.group_name: - group_blocks[test.group_name] += case_title + error + case_info - else: - single_devices_block += case_title + error + case_info - description = single_devices_block + ''.join([i for i in group_blocks.values()]) - return description, test_ids - def change_test_run_description(self): tests = self.get_all_tests() passed_tests = self.get_passed_tests() failed_tests = self.get_failed_tests() - xfailed_tests = self.get_xfailed_tests() not_executed_tests = self.get_not_executed_tests(self.run_id) final_description = "Nothing to report this time..." if len(tests) > 0: - description = "" description_title = "# %.0f%% of end-end tests have passed\n" % (len(passed_tests) / len(tests) * 100) description_title += "\n" description_title += "Total executed tests: %d\n" % len(tests) - description_title += "Undefined failed tests: %d\n" % len(failed_tests) - description_title += "Failed tests: %d\n" % len(xfailed_tests) + description_title += "Failed tests: %d\n" % len(failed_tests) description_title += "Passed tests: %d\n" % len(passed_tests) if not_executed_tests: description_title += "Not executed tests: %d\n" % len(not_executed_tests) description_title += "\n" + ids_failed_test = [] + single_devices_block, group_blocks, case_info = str(), dict(), str() if failed_tests: - description += "\n" - description += "## Undefined failed tests:" - failed_description, ids_failed_test = self.make_single_group_report(failed_tests) - description += failed_description - description_title += "## Undefined failed tests: %s\n" % ','.join([str(i) for i in ids_failed_test]) - if xfailed_tests: - description += "\n" - description += "## Failed tests (due to known issues):" - xfailed_description, id_xfailed_tests = self.make_single_group_report(xfailed_tests, xfailed=True) - description += xfailed_description - description_title += "## Failed tests: %s\n" % ','.join([str(i) for i in id_xfailed_tests]) + for test in failed_tests: + if test.group_name: + group_blocks[test.group_name] = "\n-------\n## Class: %s:\n" % test.group_name + for test in failed_tests: + last_testrun = test.testruns[-1] + test_rail_link = self.get_test_result_link(self.run_id, test.testrail_case_id) + ids_failed_test.append(test.testrail_case_id) + case_title = '\n' + case_title += '-------\n' + case_title += "### ID %s: [%s](%s) \n" % (test.testrail_case_id, test.name, test_rail_link) + full_error = last_testrun.error[-255:] + (code_error, no_code_error_str, issue_id) = self.separate_xfail_error(full_error) + if issue_id: + test_rail_xfail = self.make_error_with_gh_issue_link(no_code_error_str, issue_id) + error = "```%s```\n **%s** \n" % (code_error, test_rail_xfail) + else: + error = "```%s```\n **%s** \n" % (code_error, no_code_error_str) + for job_id, f in last_testrun.jobs.items(): + if last_testrun.first_commands: + job_url = self.get_sauce_job_url(job_id=job_id, + first_command=last_testrun.first_commands[job_id]) + else: + job_url = self.get_sauce_job_url(job_id=job_id) + case_info = "Logs for device %d: [steps](%s), [failure screenshot](%s)" \ + % (f, job_url, self.get_sauce_final_screenshot_url(job_id)) + + if test.group_name: + group_blocks[test.group_name] += case_title + error + case_info + else: + single_devices_block += case_title + error + case_info + description_title += '## Failed tests: %s \n' % ','.join(map(str, ids_failed_test)) if not_executed_tests: description_title += "## Not executed tests: %s\n" % ','.join([str(i) for i in not_executed_tests]) - final_description = description_title + description + final_description = description_title + single_devices_block + ''.join([i for i in group_blocks.values()]) request_body = {'description': final_description} return self.post('update_run/%s' % self.run_id, request_body) diff --git a/test/appium/tests/conftest.py b/test/appium/tests/conftest.py index d667d68845..0e94a8a2c4 100644 --- a/test/appium/tests/conftest.py +++ b/test/appium/tests/conftest.py @@ -230,7 +230,6 @@ def pytest_runtest_makereport(item, call): is_sauce_env = item.config.getoption('env') == 'sauce' case_ids_set = item.config.getoption("run_testrail_ids") - in_run = str([mark.args[0] for mark in item.iter_markers(name='testrail_id')][0]) in str(case_ids_set) def catch_error(): error = report.longreprtext @@ -244,7 +243,7 @@ def pytest_runtest_makereport(item, call): is_group = "xdist_group" in item.keywords._markers or "xdist_group" in item.parent.keywords._markers error_intro, error = 'Test setup failed:', '' final_error = '%s %s' % (error_intro, error) - if (hasattr(report, 'wasxfail') and not case_ids_set) or (hasattr(report, 'wasxfail') and in_run): + if (hasattr(report, 'wasxfail') and not case_ids_set) or (hasattr(report, 'wasxfail') and (str([mark.args[0] for mark in item.iter_markers(name='testrail_id')][0]) in str(case_ids_set))): if '[NOTRUN]' in report.wasxfail: test_suite_data.set_current_test(item.name, testrail_case_id=get_testrail_case_id(item)) test_suite_data.current_test.create_new_testrun() @@ -276,7 +275,7 @@ def pytest_runtest_makereport(item, call): error = catch_error() if report.failed: current_test.testruns[-1].error = error - if (hasattr(report, 'wasxfail') and not case_ids_set) or (hasattr(report, 'wasxfail') and in_run): + if (hasattr(report, 'wasxfail') and not case_ids_set) or (hasattr(report, 'wasxfail') and (str([mark.args[0] for mark in item.iter_markers(name='testrail_id')][0]) in str(case_ids_set))): current_test.testruns[-1].xfail = report.wasxfail if error: current_test.testruns[-1].error = '%s [[%s]]' % (error, report.wasxfail) diff --git a/test/appium/tests/medium/test_activity_center.py b/test/appium/tests/medium/test_activity_center.py index 884fab7d8e..b571a8a1db 100644 --- a/test/appium/tests/medium/test_activity_center.py +++ b/test/appium/tests/medium/test_activity_center.py @@ -238,6 +238,7 @@ class TestActivityCenterMultipleDevicePR(MultipleSharedDeviceTestCase): self.home_2.element_by_translation_id("remove-from-contacts").click() if not self.home_2.element_by_translation_id("no-contacts").is_element_displayed(30): self.errors.append("Contact was not removed from contact list") + self.errors.verify_no_errors() @marks.testrail_id(702851) @@ -316,4 +317,5 @@ class TestActivityCenterMultipleDevicePR(MultipleSharedDeviceTestCase): # self.home_1.driver.fail("No PN for mention in community!") # if not self.channel_1.chat_element_by_text(group_chat_message).is_element_displayed(20): # self.errors.append("No redirect to channel after tap on PN with mention!") + self.errors.verify_no_errors()