e2e: introduced failed due to issues tests in results

test/appium/support/base_test_report.py

@@ -84,10 +84,18 @@ class BaseTestReport:
         tests = self.get_all_tests()
         failed = list()
         for test in tests:
-            if not self.is_test_successful(test):
+            if not self.is_test_successful(test) and not self.is_test_xfailed(test):
                 failed.append(test)
         return failed
 
+    def get_xfailed_tests(self):
+        tests = self.get_all_tests()
+        xfailed = list()
+        for test in tests:
+            if self.is_test_xfailed(test) and not self.is_test_successful(test):
+                xfailed.append(test)
+        return xfailed
+
     def get_passed_tests(self):
         tests = self.get_all_tests()
         passed = list()
@@ -126,6 +134,11 @@ class BaseTestReport:
         # Test passed if last testrun has passed
         return test.testruns[-1].error is None
 
+    @staticmethod
+    def is_test_xfailed(test):
+    # Test failed if xfail is defined
+        return test.testruns[-1].xfail != ""
+
     @staticmethod
     def separate_xfail_error(error):
         issue_id_list = re.findall(r'#\d+', error)

test/appium/support/github_report.py

@@ -21,6 +21,7 @@ class GithubHtmlReport(BaseTestReport):
         tests = self.get_all_tests()
         passed_tests = self.get_passed_tests()
         failed_tests = self.get_failed_tests()
+        xfailed_tests = self.get_xfailed_tests()
         not_executed_tests = TestrailReport().get_not_executed_tests(run_id)
 
         if len(tests) > 0:
@@ -32,9 +33,7 @@ class GithubHtmlReport(BaseTestReport):
             if not_executed_tests:
                 summary_html += "Not executed tests: %d\n" % len(not_executed_tests)
             summary_html += "```\n"
-            not_executed_tests_html = str()
+            failed_tests_html, xfailed_tests_html, passed_tests_html, not_executed_tests_html = str(), str(), str(), str()
-            failed_tests_html = str()
-            passed_tests_html = str()
             if not_executed_tests:
                 not_executed_tests_html = self.build_tests_table_html(not_executed_tests, run_id,
                                                                       not_executed_tests=True)
@@ -44,19 +43,26 @@ class GithubHtmlReport(BaseTestReport):
             if failed_tests:
                 failed_tests_html = self.build_tests_table_html(failed_tests, run_id, failed_tests=True)
                 summary_html += "```\n"
-                summary_html += 'IDs of failed tests: %s \n' % self.list_of_failed_testrail_ids(failed_tests)
+                summary_html += 'IDs of failed tests(should be reviewed): %s \n' % self.list_of_failed_testrail_ids(failed_tests)
+                summary_html += "```\n"
+            if xfailed_tests:
+                xfailed_tests_html = self.build_tests_table_html(xfailed_tests, run_id, failed_tests=False, xfailed_tests=True)
+                summary_html += "```\n"
+                summary_html += 'IDs of failed tests(existing issues): %s \n' % self.list_of_failed_testrail_ids(xfailed_tests)
                 summary_html += "```\n"
             if passed_tests:
                 passed_tests_html = self.build_tests_table_html(passed_tests, run_id, failed_tests=False)
-            return title_html + summary_html + not_executed_tests_html + failed_tests_html + passed_tests_html
+            return title_html + summary_html + not_executed_tests_html + failed_tests_html + passed_tests_html + xfailed_tests_html
         else:
             return None
 
-    def build_tests_table_html(self, tests, run_id, failed_tests=False, not_executed_tests=False):
+    def build_tests_table_html(self, tests, run_id, failed_tests=False, not_executed_tests=False, xfailed_tests=False):
         if failed_tests:
-            tests_type = "Failed tests"
+            tests_type = "Failed tests (for review)"
         elif not_executed_tests:
             tests_type = "Not executed tests"
+        elif xfailed_tests:
+            tests_type = "Failed tests (known)"
         else:
             tests_type = "Passed tests"
         html = "<h3>%s (%d)</h3>" % (tests_type, len(tests))

test/appium/support/testrail_report.py

@@ -28,7 +28,8 @@ class TestrailReport(BaseTestReport):
 
         self.outcomes = {
             'passed': 1,
-            'undefined_fail': 10}
+            'undefined_fail': 10,
+            'fail': 5}
 
         self.headers = dict()
         self.headers['Authorization'] = 'Basic %s' % str(
@@ -181,6 +182,7 @@ class TestrailReport(BaseTestReport):
             test_steps = "# Steps: \n"
             devices = str()
             last_testrun = test.testruns[-1]
+            status = self.outcomes['passed']
             for step in last_testrun.steps:
                 test_steps += step + "\n"
             for i, device in enumerate(last_testrun.jobs):
@@ -192,7 +194,12 @@ class TestrailReport(BaseTestReport):
             comment = str()
             if test.group_name:
                 comment += "# Class: %s \n" % test.group_name
-            if last_testrun.error:
+            if last_testrun.error and not last_testrun.xfail:
+                status = self.outcomes['fail']
+                error = last_testrun.error
+                comment += '%s' % ('# Error: \n %s \n' % emoji.demojize(error)) + devices + test_steps
+            elif last_testrun.error and last_testrun.xfail:
+                status = self.outcomes['undefined_fail']
                 full_error = last_testrun.error
                 (code_error, no_code_error_str, issue_id) = self.separate_xfail_error(full_error)
                 if issue_id:
@@ -206,7 +213,7 @@ class TestrailReport(BaseTestReport):
                 comment += devices + test_steps
             data.append(
                 {'case_id': test.testrail_case_id,
-                 'status_id': self.outcomes['undefined_fail'] if last_testrun.error else self.outcomes['passed'],
+                 'status_id': status,
                  'comment': comment})
 
         results = self.post('add_results_for_cases/%s' % self.run_id, data={"results": data})
@@ -239,34 +246,19 @@ class TestrailReport(BaseTestReport):
 
         self.change_test_run_description()
 
-    def change_test_run_description(self):
+    def make_single_group_report(self, tests, xfailed=False):
-        tests = self.get_all_tests()
+        single_devices_block, group_blocks, case_info, test_ids = str(), dict(), str(), []
-        passed_tests = self.get_passed_tests()
+        for test in tests:
-        failed_tests = self.get_failed_tests()
-        not_executed_tests = self.get_not_executed_tests(self.run_id)
-        final_description = "Nothing to report this time..."
-        if len(tests) > 0:
-            description_title = "# %.0f%% of end-end tests have passed\n" % (len(passed_tests) / len(tests) * 100)
-            description_title += "\n"
-            description_title += "Total executed tests: %d\n" % len(tests)
-            description_title += "Failed tests: %d\n" % len(failed_tests)
-            description_title += "Passed tests: %d\n" % len(passed_tests)
-            if not_executed_tests:
-                description_title += "Not executed tests: %d\n" % len(not_executed_tests)
-            description_title += "\n"
-            ids_failed_test = []
-            single_devices_block, group_blocks, case_info = str(), dict(), str()
-            if failed_tests:
-                for test in failed_tests:
             if test.group_name:
                 group_blocks[test.group_name] = "\n-------\n## Class: %s:\n" % test.group_name
-                for test in failed_tests:
+        for test in tests:
             last_testrun = test.testruns[-1]
             test_rail_link = self.get_test_result_link(self.run_id, test.testrail_case_id)
-                    ids_failed_test.append(test.testrail_case_id)
+            test_ids.append(test.testrail_case_id)
             case_title = '\n'
             case_title += '-------\n'
             case_title += "### ID %s: [%s](%s) \n" % (test.testrail_case_id, test.name, test_rail_link)
+            if xfailed:
                 full_error = last_testrun.error[-255:]
                 (code_error, no_code_error_str, issue_id) = self.separate_xfail_error(full_error)
                 if issue_id:
@@ -274,6 +266,8 @@ class TestrailReport(BaseTestReport):
                     error = "```%s```\n **%s**  \n" % (code_error, test_rail_xfail)
                 else:
                     error = "```%s```\n **%s**  \n" % (code_error, no_code_error_str)
+            else:
+                error = last_testrun.error[-255:]
             for job_id, f in last_testrun.jobs.items():
                 if last_testrun.first_commands:
                     job_url = self.get_sauce_job_url(job_id=job_id,
@@ -287,10 +281,42 @@ class TestrailReport(BaseTestReport):
                 group_blocks[test.group_name] += case_title + error + case_info
             else:
                 single_devices_block += case_title + error + case_info
-                description_title += '## Failed tests: %s \n' % ','.join(map(str, ids_failed_test))
+        description = single_devices_block + ''.join([i for i in group_blocks.values()])
+        return description, test_ids
+
+    def change_test_run_description(self):
+        tests = self.get_all_tests()
+        passed_tests = self.get_passed_tests()
+        failed_tests = self.get_failed_tests()
+        xfailed_tests = self.get_xfailed_tests()
+        not_executed_tests = self.get_not_executed_tests(self.run_id)
+        final_description = "Nothing to report this time..."
+        if len(tests) > 0:
+            description = ""
+            description_title = "# %.0f%% of end-end tests have passed\n" % (len(passed_tests) / len(tests) * 100)
+            description_title += "\n"
+            description_title += "Total executed tests: %d\n" % len(tests)
+            description_title += "Undefined failed tests: %d\n" % len(failed_tests)
+            description_title += "Failed tests: %d\n" % len(xfailed_tests)
+            description_title += "Passed tests: %d\n" % len(passed_tests)
+            if not_executed_tests:
+                description_title += "Not executed tests: %d\n" % len(not_executed_tests)
+            description_title += "\n"
+            if failed_tests:
+                description += "\n"
+                description += "## Undefined failed tests:"
+                failed_description, ids_failed_test = self.make_single_group_report(failed_tests)
+                description += failed_description
+                description_title += "## Undefined failed tests: %s\n" % ','.join([str(i) for i in ids_failed_test])
+            if xfailed_tests:
+                description += "\n"
+                description += "## Failed tests (due to known issues):"
+                xfailed_description, id_xfailed_tests = self.make_single_group_report(xfailed_tests, xfailed=True)
+                description += xfailed_description
+                description_title += "## Failed tests: %s\n" % ','.join([str(i) for i in id_xfailed_tests])
             if not_executed_tests:
                 description_title += "## Not executed tests: %s\n" % ','.join([str(i) for i in not_executed_tests])
-            final_description = description_title + single_devices_block + ''.join([i for i in group_blocks.values()])
+            final_description = description_title + description
 
         request_body = {'description': final_description}
         return self.post('update_run/%s' % self.run_id, request_body)

test/appium/tests/medium/test_activity_center.py

@@ -238,7 +238,6 @@ class TestActivityCenterMultipleDevicePR(MultipleSharedDeviceTestCase):
         self.home_2.element_by_translation_id("remove-from-contacts").click()
         if not self.home_2.element_by_translation_id("no-contacts").is_element_displayed(30):
             self.errors.append("Contact was not removed from contact list")
-
         self.errors.verify_no_errors()
 
     @marks.testrail_id(702851)
@@ -317,5 +316,4 @@ class TestActivityCenterMultipleDevicePR(MultipleSharedDeviceTestCase):
         #     self.home_1.driver.fail("No PN for mention in community!")
         # if not self.channel_1.chat_element_by_text(group_chat_message).is_element_displayed(20):
         #     self.errors.append("No redirect to channel after tap on PN with mention!")
-
         self.errors.verify_no_errors()