[ISSUE #3433] Update end-end test report for Github

Signed-off-by: Lukasz Fryc <fryc.lukasz@gmail.com>

.gitignore

@@ -32,6 +32,9 @@ project.xcworkspace
 local.properties
 *.iml
 
+# Atom
+.tags*
+
 # node.js
 #
 node_modules/

test/appium/support/github_test_report.py

@@ -0,0 +1,135 @@
+import json
+from hashlib import md5
+import hmac
+import os
+
+from tests import SingleTestData
+
+
+class GithubHtmlReport:
+
+    TEST_REPORT_DIR = "%s/../report" % os.path.dirname(os.path.abspath(__file__))
+
+    def __init__(self, sauce_username, sauce_access_key):
+        self.sauce_username = sauce_username
+        self.sauce_access_key = sauce_access_key
+        self.init_report()
+
+    def init_report(self):
+        if not os.path.exists(self.TEST_REPORT_DIR):
+            os.makedirs(self.TEST_REPORT_DIR)
+        # delete all old files in report dir
+        file_list = [f for f in os.listdir(self.TEST_REPORT_DIR)]
+        for f in file_list:
+            os.remove(os.path.join(self.TEST_REPORT_DIR, f))
+
+    def get_test_report_file_path(self, test_name):
+        file_name = "%s.json" % test_name
+        return os.path.join(self.TEST_REPORT_DIR, file_name)
+
+    def build_html_report(self):
+        tests = self.get_all_tests()
+        passed_tests = self.get_passed_tests()
+        failed_tests = self.get_failed_tests()
+
+        if len(tests) > 0:
+            title_html = "## %.0f%% of end-end tests have passed\n" % (len(passed_tests) / len(tests) * 100)
+            summary_html = "```\n"
+            summary_html += "Total executed tests: %d\n" % len(tests)
+            summary_html += "Failed tests: %d\n" % len(failed_tests)
+            summary_html += "Passed tests: %d\n" % len(passed_tests)
+            summary_html += "```\n"
+            failed_tests_html = str()
+            passed_tests_html = str()
+            if failed_tests:
+                failed_tests_html = self.build_tests_table_html(failed_tests, failed_tests=True)
+            if passed_tests:
+                passed_tests_html = self.build_tests_table_html(passed_tests, failed_tests=False)
+            return title_html + summary_html + failed_tests_html + passed_tests_html
+        else:
+            return None
+
+    def save_test(self, test):
+        file_path = self.get_test_report_file_path(test.name)
+        json.dump(test.__dict__, open(file_path, 'w'))
+
+    def get_all_tests(self):
+        tests = list()
+        file_list = [f for f in os.listdir(self.TEST_REPORT_DIR)]
+        for file_name in file_list:
+            file_path = os.path.join(self.TEST_REPORT_DIR, file_name)
+            test_dict = json.load(open(file_path))
+            tests.append(SingleTestData(name=test_dict['name'], steps=test_dict['steps'],
+                                        jobs=test_dict['jobs'], error=test_dict['error']))
+        return tests
+
+    def get_failed_tests(self):
+        tests = self.get_all_tests()
+        failed = list()
+        for test in tests:
+            if test.error is not None:
+                failed.append(test)
+        return failed
+
+    def get_passed_tests(self):
+        tests = self.get_all_tests()
+        passed = list()
+        for test in tests:
+            if test.error is None:
+                passed.append(test)
+        return passed
+
+    def build_tests_table_html(self, tests, failed_tests=False):
+        tests_type = "Failed tests" if failed_tests else "Passed tests"
+        html = "<h3>%s (%d)</h3>" % (tests_type, len(tests))
+        html += "<details>"
+        html += "<summary>Click to expand</summary>"
+        html += "<br/>"
+        html += "<table style=\"width: 100%\">"
+        html += "<colgroup>"
+        html += "<col span=\"1\" style=\"width: 20%;\">"
+        html += "<col span=\"1\" style=\"width: 80%;\">"
+        html += "</colgroup>"
+        html += "<tbody>"
+        html += "<tr>"
+        html += "</tr>"
+        for i, test in enumerate(tests):
+            html += self.build_test_row_html(i, test)
+        html += "</tbody>"
+        html += "</table>"
+        html += "</details>"
+        return html
+
+    def build_test_row_html(self, index, test):
+        html = "<tr><td><b>%d. %s</b></td></tr>" % (index+1, test.name)
+        html += "<tr><td>"
+        test_steps_html = list()
+        for step in test.steps:
+            test_steps_html.append("<div>%s</div>" % step)
+        if test.error:
+            if test_steps_html:
+                html += "<p>"
+                html += "<blockquote>"
+                # last 2 steps as summary
+                html += "%s" % ''.join(test_steps_html[-2:])
+                html += "</blockquote>"
+                html += "</p>"
+            html += "<code>%s</code>" % test.error
+            html += "<br/><br/>"
+        if test.jobs:
+            html += self.build_device_sessions_html(test.jobs)
+        html += "</td></tr>"
+        return html
+
+    def get_sauce_job_url(self, job_id):
+        token = hmac.new(bytes(self.sauce_username + ":" + self.sauce_access_key, 'latin-1'),
+                         bytes(job_id, 'latin-1'), md5).hexdigest()
+        return "https://saucelabs.com/jobs/%s?auth=%s" % (job_id, token)
+
+    def build_device_sessions_html(self, jobs):
+        html = "<ins>Device sessions:</ins>"
+        html += "<p><ul>"
+        for i, job_id in enumerate(jobs):
+            html += "<li><a href=\"%s\">Device %d</a></li>" % (self.get_sauce_job_url(job_id), i+1)
+        html += "</ul></p>"
+        return html

test/appium/tests/__init__.py

@@ -20,18 +20,29 @@ def get_current_time():
 def info(text: str):
     if "Base" not in text:
         logging.info(text)
-        test_data.test_info[test_data.test_name]['steps'] += text + '\n'
+        test_suite_data.current_test.steps.append(text)
 
 
-class TestData(object):
+class SingleTestData(object):
+    def __init__(self, name, steps=list(), jobs=list(), error=None):
+        self.name = name
+        self.steps = steps
+        self.jobs = jobs
+        self.error = error
 
+
+class TestSuiteData(object):
     def __init__(self):
-        self.test_name = None
         self.apk_name = None
-        self.test_info = dict()
+        self.current_test = None
+        self.tests = list()
+
+    def add_test(self, test):
+        self.tests.append(test)
+        self.current_test = test
 
 
-test_data = TestData()
+test_suite_data = TestSuiteData()
 
 
 basic_user = dict()

test/appium/tests/base_test_case.py

@@ -4,7 +4,7 @@ import re
 import subprocess
 import asyncio
 from selenium.common.exceptions import WebDriverException
-from tests import test_data, start_threads
+from tests import test_suite_data, start_threads
 from os import environ
 from appium import webdriver
 from abc import ABCMeta, abstractmethod
@@ -49,10 +49,10 @@ class AbstractTestCase:
     @property
     def capabilities_sauce_lab(self):
         desired_caps = dict()
-        desired_caps['app'] = 'sauce-storage:' + test_data.apk_name
+        desired_caps['app'] = 'sauce-storage:' + test_suite_data.apk_name
 
         desired_caps['build'] = pytest.config.getoption('build')
-        desired_caps['name'] = test_data.test_name
+        desired_caps['name'] = test_suite_data.current_test.name
         desired_caps['platformName'] = 'Android'
         desired_caps['appiumVersion'] = '1.7.1'
         desired_caps['platformVersion'] = '6.0'
@@ -92,11 +92,6 @@ class AbstractTestCase:
     def implicitly_wait(self):
         return 8
 
-    def update_test_info_dict(self):
-        test_data.test_info[test_data.test_name] = dict()
-        test_data.test_info[test_data.test_name]['jobs'] = list()
-        test_data.test_info[test_data.test_name]['steps'] = str()
-
     errors = []
 
     def verify_no_errors(self):
@@ -107,8 +102,6 @@ class AbstractTestCase:
 class SingleDeviceTestCase(AbstractTestCase):
 
     def setup_method(self, method):
-        self.update_test_info_dict()
-
         capabilities = {'local': {'executor': self.executor_local,
                                   'capabilities': self.capabilities_local},
                         'sauce': {'executor': self.executor_sauce_lab,
@@ -117,7 +110,7 @@ class SingleDeviceTestCase(AbstractTestCase):
         self.driver = webdriver.Remote(capabilities[self.environment]['executor'],
                                        capabilities[self.environment]['capabilities'])
         self.driver.implicitly_wait(self.implicitly_wait)
-        test_data.test_info[test_data.test_name]['jobs'].append(self.driver.session_id)
+        test_suite_data.current_test.jobs.append(self.driver.session_id)
 
     def teardown_method(self, method):
         if self.environment == 'sauce':
@@ -131,7 +124,6 @@ class SingleDeviceTestCase(AbstractTestCase):
 class LocalMultipleDeviceTestCase(AbstractTestCase):
 
     def setup_method(self, method):
-        self.update_test_info_dict()
         self.drivers = dict()
 
     def create_drivers(self, quantity):
@@ -139,7 +131,7 @@ class LocalMultipleDeviceTestCase(AbstractTestCase):
         for driver in range(quantity):
             self.drivers[driver] = webdriver.Remote(self.executor_local, capabilities[driver])
             self.drivers[driver].implicitly_wait(self.implicitly_wait)
-            test_data.test_info[test_data.test_name]['jobs'].append(self.drivers[driver].session_id)
+            test_suite_data.current_test.jobs.append(self.drivers[driver].session_id)
 
     def teardown_method(self, method):
         for driver in self.drivers:
@@ -157,7 +149,6 @@ class SauceMultipleDeviceTestCase(AbstractTestCase):
         asyncio.set_event_loop(cls.loop)
 
     def setup_method(self, method):
-        self.update_test_info_dict()
         self.drivers = dict()
 
     def create_drivers(self, quantity=2):
@@ -167,7 +158,7 @@ class SauceMultipleDeviceTestCase(AbstractTestCase):
                                                     self.capabilities_sauce_lab))
         for driver in range(quantity):
             self.drivers[driver].implicitly_wait(self.implicitly_wait)
-            test_data.test_info[test_data.test_name]['jobs'].append(self.drivers[driver].session_id)
+            test_suite_data.current_test.jobs.append(self.drivers[driver].session_id)
 
     def teardown_method(self, method):
         for driver in self.drivers:

test/appium/tests/conftest.py

@@ -1,4 +1,4 @@
-from tests import test_data
+from tests import test_suite_data, SingleTestData
 import requests
 import re
 import pytest
@@ -6,8 +6,7 @@ from datetime import datetime
 from os import environ
 from io import BytesIO
 from sauceclient import SauceClient
-from hashlib import md5
+from support.github_test_report import GithubHtmlReport
-import hmac
 
 storage = 'http://artifacts.status.im:8081/artifactory/nightlies-local/'
 
@@ -16,6 +15,7 @@ sauce_access_key = environ.get('SAUCE_ACCESS_KEY')
 github_token = environ.get('GIT_HUB_TOKEN')
 
 sauce = SauceClient(sauce_username, sauce_access_key)
+github_report = GithubHtmlReport(sauce_username, sauce_access_key)
 
 
 def get_latest_apk():
@@ -60,7 +60,7 @@ def is_master(config):
 def is_uploaded():
     stored_files = sauce.storage.get_stored_files()
     for i in range(len(stored_files['files'])):
-        if stored_files['files'][i]['name'] == test_data.apk_name:
+        if stored_files['files'][i]['name'] == test_suite_data.apk_name:
             return True
 
 
@@ -68,8 +68,8 @@ def pytest_configure(config):
     if config.getoption('log'):
         import logging
         logging.basicConfig(level=logging.INFO)
-    test_data.apk_name = ([i for i in [i for i in config.getoption('apk').split('/')
+    test_suite_data.apk_name = ([i for i in [i for i in config.getoption('apk').split('/')
-                                       if '.apk' in i]])[0]
+                                             if '.apk' in i]])[0]
     if is_master(config) and config.getoption('env') == 'sauce':
         if config.getoption('pr_number'):
             with open('github_comment.txt', 'w') as _:
@@ -81,7 +81,7 @@ def pytest_configure(config):
                 file = BytesIO(response.content)
                 del response
                 requests.post('http://saucelabs.com/rest/v1/storage/'
-                              + sauce_username + '/' + test_data.apk_name + '?overwrite=true',
+                              + sauce_username + '/' + test_suite_data.apk_name + '?overwrite=true',
                               auth=(sauce_username, sauce_access_key),
                               data=file,
                               headers={'Content-Type': 'application/octet-stream'})
@@ -94,46 +94,27 @@ def pytest_unconfigure(config):
         from github import Github
         repo = Github(github_token).get_user('status-im').get_repo('status-react')
         pull = repo.get_pull(int(config.getoption('pr_number')))
-        with open('github_comment.txt', 'r') as comment:
+        pull.create_issue_comment(github_report.build_html_report())
-            pull.create_issue_comment('# Automated test results: \n' + comment.read())
-
-
-def get_public_url(job_id):
-    token = hmac.new(bytes(sauce_username + ":" + sauce_access_key, 'latin-1'),
-                     bytes(job_id, 'latin-1'), md5).hexdigest()
-    return "https://saucelabs.com/jobs/%s?auth=%s" % (job_id, token)
-
-
-def make_github_report(error=None):
-    if pytest.config.getoption('pr_number'):
-        title = '### %s' % test_data.test_name
-        outcome = '%s' % ':x:' if error else ':white_check_mark:' + ':\n'
-        title += outcome
-        steps = '\n\n <details>\n<summary>Test Steps & Error message:</summary>\n\n ```%s ```%s\n\n</details>\n' % \
-                (test_data.test_info[test_data.test_name]['steps'], '\n```' + error + '```' if error else '')
-        sessions = str()
-
-        for job_id in test_data.test_info[test_data.test_name]['jobs']:
-            sessions += '  - [Android Device Session](%s) \n' % get_public_url(job_id)
-        with open('github_comment.txt', 'a') as comment:
-            comment.write(title + '\n' + steps + '\n' + sessions + '---\n')
 
 
 @pytest.mark.hookwrapper
 def pytest_runtest_makereport(item, call):
     outcome = yield
     report = outcome.get_result()
-    if pytest.config.getoption('env') == 'sauce':
+    is_sauce_env = pytest.config.getoption('env') == 'sauce'
-        if report.when == 'call':
+    current_test = test_suite_data.current_test
-            if report.passed:
+    if report.when == 'call':
-                for job_id in test_data.test_info[test_data.test_name]['jobs']:
+        if report.failed:
-                    sauce.jobs.update_job(job_id, name=test_data.test_name, passed=True)
+            current_test.error = report.longreprtext
-                make_github_report()
+        if is_sauce_env:
-            if report.failed:
+            update_sauce_jobs(current_test.name, current_test.jobs, report.passed)
-                for job_id in test_data.test_info[test_data.test_name]['jobs']:
+        github_report.save_test(current_test)
-                    sauce.jobs.update_job(job_id, name=test_data.test_name, passed=False)
+
-                make_github_report(error=report.longreprtext)
+
+def update_sauce_jobs(test_name, job_ids, passed):
+    for job_id in job_ids:
+        sauce.jobs.update_job(job_id, name=test_name, passed=passed)
 
 
 def pytest_runtest_setup(item):
-    test_data.test_name = item.name
+    test_suite_data.add_test(SingleTestData(item.name))