e2e: introduced failed due to issues tests in results

test/appium/support/base_test_report.py

@@ -84,10 +84,18 @@ class BaseTestReport:
         tests = self.get_all_tests()
         failed = list()
         for test in tests:
-            if not self.is_test_successful(test):
+            if not self.is_test_successful(test) and not self.is_test_xfailed(test):
                 failed.append(test)
         return failed
 
+    def get_xfailed_tests(self):
+        tests = self.get_all_tests()
+        xfailed = list()
+        for test in tests:
+            if self.is_test_xfailed(test) and not self.is_test_successful(test):
+                xfailed.append(test)
+        return xfailed
+
     def get_passed_tests(self):
         tests = self.get_all_tests()
         passed = list()
@@ -126,6 +134,11 @@ class BaseTestReport:
         # Test passed if last testrun has passed
         return test.testruns[-1].error is None
 
+    @staticmethod
+    def is_test_xfailed(test):
+    # Test failed if xfail is defined
+        return test.testruns[-1].xfail != ""
+
     @staticmethod
     def separate_xfail_error(error):
         issue_id_list = re.findall(r'#\d+', error)

test/appium/support/github_report.py

@@ -21,6 +21,7 @@ class GithubHtmlReport(BaseTestReport):
         tests = self.get_all_tests()
         passed_tests = self.get_passed_tests()
         failed_tests = self.get_failed_tests()
+        xfailed_tests = self.get_xfailed_tests()
         not_executed_tests = TestrailReport().get_not_executed_tests(run_id)
 
         if len(tests) > 0:
@@ -32,9 +33,7 @@ class GithubHtmlReport(BaseTestReport):
             if not_executed_tests:
                 summary_html += "Not executed tests: %d\n" % len(not_executed_tests)
             summary_html += "```\n"
-            not_executed_tests_html = str()
+            failed_tests_html, xfailed_tests_html, passed_tests_html, not_executed_tests_html = str(), str(), str(), str()
-            failed_tests_html = str()
-            passed_tests_html = str()
             if not_executed_tests:
                 not_executed_tests_html = self.build_tests_table_html(not_executed_tests, run_id,
                                                                       not_executed_tests=True)
@@ -44,19 +43,26 @@ class GithubHtmlReport(BaseTestReport):
             if failed_tests:
                 failed_tests_html = self.build_tests_table_html(failed_tests, run_id, failed_tests=True)
                 summary_html += "```\n"
-                summary_html += 'IDs of failed tests: %s \n' % self.list_of_failed_testrail_ids(failed_tests)
+                summary_html += 'IDs of failed tests(should be reviewed): %s \n' % self.list_of_failed_testrail_ids(failed_tests)
+                summary_html += "```\n"
+            if xfailed_tests:
+                xfailed_tests_html = self.build_tests_table_html(xfailed_tests, run_id, failed_tests=False, xfailed_tests=True)
+                summary_html += "```\n"
+                summary_html += 'IDs of failed tests(existing issues): %s \n' % self.list_of_failed_testrail_ids(xfailed_tests)
                 summary_html += "```\n"
             if passed_tests:
                 passed_tests_html = self.build_tests_table_html(passed_tests, run_id, failed_tests=False)
-            return title_html + summary_html + not_executed_tests_html + failed_tests_html + passed_tests_html
+            return title_html + summary_html + not_executed_tests_html + failed_tests_html + passed_tests_html + xfailed_tests_html
         else:
             return None
 
-    def build_tests_table_html(self, tests, run_id, failed_tests=False, not_executed_tests=False):
+    def build_tests_table_html(self, tests, run_id, failed_tests=False, not_executed_tests=False, xfailed_tests=False):
         if failed_tests:
-            tests_type = "Failed tests"
+            tests_type = "Failed tests (for review)"
         elif not_executed_tests:
             tests_type = "Not executed tests"
+        elif xfailed_tests:
+            tests_type = "Failed tests (known)"
         else:
             tests_type = "Passed tests"
         html = "<h3>%s (%d)</h3>" % (tests_type, len(tests))

test/appium/support/testrail_report.py

@@ -28,7 +28,8 @@ class TestrailReport(BaseTestReport):
 
         self.outcomes = {
             'passed': 1,
-            'undefined_fail': 10}
+            'undefined_fail': 10,
+            'fail': 5}
 
         self.headers = dict()
         self.headers['Authorization'] = 'Basic %s' % str(
@@ -181,6 +182,7 @@ class TestrailReport(BaseTestReport):
             test_steps = "# Steps: \n"
             devices = str()
             last_testrun = test.testruns[-1]
+            status = self.outcomes['passed']
             for step in last_testrun.steps:
                 test_steps += step + "\n"
             for i, device in enumerate(last_testrun.jobs):
@@ -192,7 +194,12 @@ class TestrailReport(BaseTestReport):
             comment = str()
             if test.group_name:
                 comment += "# Class: %s \n" % test.group_name
-            if last_testrun.error:
+            if last_testrun.error and not last_testrun.xfail:
+                status = self.outcomes['fail']
+                error = last_testrun.error
+                comment += '%s' % ('# Error: \n %s \n' % emoji.demojize(error)) + devices + test_steps
+            elif last_testrun.error and last_testrun.xfail:
+                status = self.outcomes['undefined_fail']
                 full_error = last_testrun.error
                 (code_error, no_code_error_str, issue_id) = self.separate_xfail_error(full_error)
                 if issue_id:
@@ -206,7 +213,7 @@ class TestrailReport(BaseTestReport):
                 comment += devices + test_steps
             data.append(
                 {'case_id': test.testrail_case_id,
-                 'status_id': self.outcomes['undefined_fail'] if last_testrun.error else self.outcomes['passed'],
+                 'status_id': status,
                  'comment': comment})
 
         results = self.post('add_results_for_cases/%s' % self.run_id, data={"results": data})
@@ -239,58 +246,77 @@ class TestrailReport(BaseTestReport):
 
         self.change_test_run_description()
 
+    def make_single_group_report(self, tests, xfailed=False):
+        single_devices_block, group_blocks, case_info, test_ids = str(), dict(), str(), []
+        for test in tests:
+            if test.group_name:
+                group_blocks[test.group_name] = "\n-------\n## Class: %s:\n" % test.group_name
+        for test in tests:
+            last_testrun = test.testruns[-1]
+            test_rail_link = self.get_test_result_link(self.run_id, test.testrail_case_id)
+            test_ids.append(test.testrail_case_id)
+            case_title = '\n'
+            case_title += '-------\n'
+            case_title += "### ID %s: [%s](%s) \n" % (test.testrail_case_id, test.name, test_rail_link)
+            if xfailed:
+                full_error = last_testrun.error[-255:]
+                (code_error, no_code_error_str, issue_id) = self.separate_xfail_error(full_error)
+                if issue_id:
+                    test_rail_xfail = self.make_error_with_gh_issue_link(no_code_error_str, issue_id)
+                    error = "```%s```\n **%s**  \n" % (code_error, test_rail_xfail)
+                else:
+                    error = "```%s```\n **%s**  \n" % (code_error, no_code_error_str)
+            else:
+                error = last_testrun.error[-255:]
+            for job_id, f in last_testrun.jobs.items():
+                if last_testrun.first_commands:
+                    job_url = self.get_sauce_job_url(job_id=job_id,
+                                                     first_command=last_testrun.first_commands[job_id])
+                else:
+                    job_url = self.get_sauce_job_url(job_id=job_id)
+                case_info = "Logs for device %d: [steps](%s), [failure screenshot](%s)" \
+                            % (f, job_url, self.get_sauce_final_screenshot_url(job_id))
+
+            if test.group_name:
+                group_blocks[test.group_name] += case_title + error + case_info
+            else:
+                single_devices_block += case_title + error + case_info
+        description = single_devices_block + ''.join([i for i in group_blocks.values()])
+        return description, test_ids
+
     def change_test_run_description(self):
         tests = self.get_all_tests()
         passed_tests = self.get_passed_tests()
         failed_tests = self.get_failed_tests()
+        xfailed_tests = self.get_xfailed_tests()
         not_executed_tests = self.get_not_executed_tests(self.run_id)
         final_description = "Nothing to report this time..."
         if len(tests) > 0:
+            description = ""
             description_title = "# %.0f%% of end-end tests have passed\n" % (len(passed_tests) / len(tests) * 100)
             description_title += "\n"
             description_title += "Total executed tests: %d\n" % len(tests)
-            description_title += "Failed tests: %d\n" % len(failed_tests)
+            description_title += "Undefined failed tests: %d\n" % len(failed_tests)
+            description_title += "Failed tests: %d\n" % len(xfailed_tests)
             description_title += "Passed tests: %d\n" % len(passed_tests)
             if not_executed_tests:
                 description_title += "Not executed tests: %d\n" % len(not_executed_tests)
             description_title += "\n"
-            ids_failed_test = []
-            single_devices_block, group_blocks, case_info = str(), dict(), str()
             if failed_tests:
-                for test in failed_tests:
+                description += "\n"
-                    if test.group_name:
+                description += "## Undefined failed tests:"
-                        group_blocks[test.group_name] = "\n-------\n## Class: %s:\n" % test.group_name
+                failed_description, ids_failed_test = self.make_single_group_report(failed_tests)
-                for test in failed_tests:
+                description += failed_description
-                    last_testrun = test.testruns[-1]
+                description_title += "## Undefined failed tests: %s\n" % ','.join([str(i) for i in ids_failed_test])
-                    test_rail_link = self.get_test_result_link(self.run_id, test.testrail_case_id)
+            if xfailed_tests:
-                    ids_failed_test.append(test.testrail_case_id)
+                description += "\n"
-                    case_title = '\n'
+                description += "## Failed tests (due to known issues):"
-                    case_title += '-------\n'
+                xfailed_description, id_xfailed_tests = self.make_single_group_report(xfailed_tests, xfailed=True)
-                    case_title += "### ID %s: [%s](%s) \n" % (test.testrail_case_id, test.name, test_rail_link)
+                description += xfailed_description
-                    full_error = last_testrun.error[-255:]
+                description_title += "## Failed tests: %s\n" % ','.join([str(i) for i in id_xfailed_tests])
-                    (code_error, no_code_error_str, issue_id) = self.separate_xfail_error(full_error)
-                    if issue_id:
-                        test_rail_xfail = self.make_error_with_gh_issue_link(no_code_error_str, issue_id)
-                        error = "```%s```\n **%s**  \n" % (code_error, test_rail_xfail)
-                    else:
-                        error = "```%s```\n **%s**  \n" % (code_error, no_code_error_str)
-                    for job_id, f in last_testrun.jobs.items():
-                        if last_testrun.first_commands:
-                            job_url = self.get_sauce_job_url(job_id=job_id,
-                                                             first_command=last_testrun.first_commands[job_id])
-                        else:
-                            job_url = self.get_sauce_job_url(job_id=job_id)
-                        case_info = "Logs for device %d: [steps](%s), [failure screenshot](%s)" \
-                                    % (f, job_url, self.get_sauce_final_screenshot_url(job_id))
-
-                    if test.group_name:
-                        group_blocks[test.group_name] += case_title + error + case_info
-                    else:
-                        single_devices_block += case_title + error + case_info
-                description_title += '## Failed tests: %s \n' % ','.join(map(str, ids_failed_test))
             if not_executed_tests:
                 description_title += "## Not executed tests: %s\n" % ','.join([str(i) for i in not_executed_tests])
-            final_description = description_title + single_devices_block + ''.join([i for i in group_blocks.values()])
+            final_description = description_title + description
 
         request_body = {'description': final_description}
         return self.post('update_run/%s' % self.run_id, request_body)

test/appium/tests/medium/test_activity_center.py

@@ -238,7 +238,6 @@ class TestActivityCenterMultipleDevicePR(MultipleSharedDeviceTestCase):
         self.home_2.element_by_translation_id("remove-from-contacts").click()
         if not self.home_2.element_by_translation_id("no-contacts").is_element_displayed(30):
             self.errors.append("Contact was not removed from contact list")
-
         self.errors.verify_no_errors()
 
     @marks.testrail_id(702851)
@@ -317,5 +316,4 @@ class TestActivityCenterMultipleDevicePR(MultipleSharedDeviceTestCase):
         #     self.home_1.driver.fail("No PN for mention in community!")
         # if not self.channel_1.chat_element_by_text(group_chat_message).is_element_displayed(20):
         #     self.errors.append("No redirect to channel after tap on PN with mention!")
-
         self.errors.verify_no_errors()