diff --git a/.gitignore b/.gitignore index 7b3ee17582..675633d7ff 100644 --- a/.gitignore +++ b/.gitignore @@ -32,6 +32,9 @@ project.xcworkspace local.properties *.iml +# Atom +.tags* + # node.js # node_modules/ diff --git a/test/appium/support/github_test_report.py b/test/appium/support/github_test_report.py new file mode 100644 index 0000000000..3426cb056d --- /dev/null +++ b/test/appium/support/github_test_report.py @@ -0,0 +1,135 @@ +import json +from hashlib import md5 +import hmac +import os + +from tests import SingleTestData + + +class GithubHtmlReport: + + TEST_REPORT_DIR = "%s/../report" % os.path.dirname(os.path.abspath(__file__)) + + def __init__(self, sauce_username, sauce_access_key): + self.sauce_username = sauce_username + self.sauce_access_key = sauce_access_key + self.init_report() + + def init_report(self): + if not os.path.exists(self.TEST_REPORT_DIR): + os.makedirs(self.TEST_REPORT_DIR) + # delete all old files in report dir + file_list = [f for f in os.listdir(self.TEST_REPORT_DIR)] + for f in file_list: + os.remove(os.path.join(self.TEST_REPORT_DIR, f)) + + def get_test_report_file_path(self, test_name): + file_name = "%s.json" % test_name + return os.path.join(self.TEST_REPORT_DIR, file_name) + + def build_html_report(self): + tests = self.get_all_tests() + passed_tests = self.get_passed_tests() + failed_tests = self.get_failed_tests() + + if len(tests) > 0: + title_html = "## %.0f%% of end-end tests have passed\n" % (len(passed_tests) / len(tests) * 100) + summary_html = "```\n" + summary_html += "Total executed tests: %d\n" % len(tests) + summary_html += "Failed tests: %d\n" % len(failed_tests) + summary_html += "Passed tests: %d\n" % len(passed_tests) + summary_html += "```\n" + failed_tests_html = str() + passed_tests_html = str() + if failed_tests: + failed_tests_html = self.build_tests_table_html(failed_tests, failed_tests=True) + if passed_tests: + passed_tests_html = self.build_tests_table_html(passed_tests, failed_tests=False) + return title_html + summary_html + failed_tests_html + passed_tests_html + else: + return None + + def save_test(self, test): + file_path = self.get_test_report_file_path(test.name) + json.dump(test.__dict__, open(file_path, 'w')) + + def get_all_tests(self): + tests = list() + file_list = [f for f in os.listdir(self.TEST_REPORT_DIR)] + for file_name in file_list: + file_path = os.path.join(self.TEST_REPORT_DIR, file_name) + test_dict = json.load(open(file_path)) + tests.append(SingleTestData(name=test_dict['name'], steps=test_dict['steps'], + jobs=test_dict['jobs'], error=test_dict['error'])) + return tests + + def get_failed_tests(self): + tests = self.get_all_tests() + failed = list() + for test in tests: + if test.error is not None: + failed.append(test) + return failed + + def get_passed_tests(self): + tests = self.get_all_tests() + passed = list() + for test in tests: + if test.error is None: + passed.append(test) + return passed + + def build_tests_table_html(self, tests, failed_tests=False): + tests_type = "Failed tests" if failed_tests else "Passed tests" + html = "

%s (%d)

" % (tests_type, len(tests)) + html += "
" + html += "Click to expand" + html += "
" + html += "" + html += "" + html += "" + html += "" + html += "" + html += "" + html += "" + html += "" + for i, test in enumerate(tests): + html += self.build_test_row_html(i, test) + html += "" + html += "
" + html += "
" + return html + + def build_test_row_html(self, index, test): + html = "%d. %s" % (index+1, test.name) + html += "" + test_steps_html = list() + for step in test.steps: + test_steps_html.append("
%s
" % step) + if test.error: + if test_steps_html: + html += "

" + html += "

" + # last 2 steps as summary + html += "%s" % ''.join(test_steps_html[-2:]) + html += "
" + html += "

" + html += "%s" % test.error + html += "

" + if test.jobs: + html += self.build_device_sessions_html(test.jobs) + html += "" + return html + + def get_sauce_job_url(self, job_id): + token = hmac.new(bytes(self.sauce_username + ":" + self.sauce_access_key, 'latin-1'), + bytes(job_id, 'latin-1'), md5).hexdigest() + return "https://saucelabs.com/jobs/%s?auth=%s" % (job_id, token) + + def build_device_sessions_html(self, jobs): + html = "Device sessions:" + html += "

" + return html diff --git a/test/appium/tests/__init__.py b/test/appium/tests/__init__.py index 5d906f45b5..dc82541243 100644 --- a/test/appium/tests/__init__.py +++ b/test/appium/tests/__init__.py @@ -20,18 +20,29 @@ def get_current_time(): def info(text: str): if "Base" not in text: logging.info(text) - test_data.test_info[test_data.test_name]['steps'] += text + '\n' + test_suite_data.current_test.steps.append(text) -class TestData(object): +class SingleTestData(object): + def __init__(self, name, steps=list(), jobs=list(), error=None): + self.name = name + self.steps = steps + self.jobs = jobs + self.error = error + +class TestSuiteData(object): def __init__(self): - self.test_name = None self.apk_name = None - self.test_info = dict() + self.current_test = None + self.tests = list() + + def add_test(self, test): + self.tests.append(test) + self.current_test = test -test_data = TestData() +test_suite_data = TestSuiteData() basic_user = dict() diff --git a/test/appium/tests/base_test_case.py b/test/appium/tests/base_test_case.py index ccef86970c..e8712bc02f 100644 --- a/test/appium/tests/base_test_case.py +++ b/test/appium/tests/base_test_case.py @@ -4,7 +4,7 @@ import re import subprocess import asyncio from selenium.common.exceptions import WebDriverException -from tests import test_data, start_threads +from tests import test_suite_data, start_threads from os import environ from appium import webdriver from abc import ABCMeta, abstractmethod @@ -49,10 +49,10 @@ class AbstractTestCase: @property def capabilities_sauce_lab(self): desired_caps = dict() - desired_caps['app'] = 'sauce-storage:' + test_data.apk_name + desired_caps['app'] = 'sauce-storage:' + test_suite_data.apk_name desired_caps['build'] = pytest.config.getoption('build') - desired_caps['name'] = test_data.test_name + desired_caps['name'] = test_suite_data.current_test.name desired_caps['platformName'] = 'Android' desired_caps['appiumVersion'] = '1.7.1' desired_caps['platformVersion'] = '6.0' @@ -92,11 +92,6 @@ class AbstractTestCase: def implicitly_wait(self): return 8 - def update_test_info_dict(self): - test_data.test_info[test_data.test_name] = dict() - test_data.test_info[test_data.test_name]['jobs'] = list() - test_data.test_info[test_data.test_name]['steps'] = str() - errors = [] def verify_no_errors(self): @@ -107,8 +102,6 @@ class AbstractTestCase: class SingleDeviceTestCase(AbstractTestCase): def setup_method(self, method): - self.update_test_info_dict() - capabilities = {'local': {'executor': self.executor_local, 'capabilities': self.capabilities_local}, 'sauce': {'executor': self.executor_sauce_lab, @@ -117,7 +110,7 @@ class SingleDeviceTestCase(AbstractTestCase): self.driver = webdriver.Remote(capabilities[self.environment]['executor'], capabilities[self.environment]['capabilities']) self.driver.implicitly_wait(self.implicitly_wait) - test_data.test_info[test_data.test_name]['jobs'].append(self.driver.session_id) + test_suite_data.current_test.jobs.append(self.driver.session_id) def teardown_method(self, method): if self.environment == 'sauce': @@ -131,7 +124,6 @@ class SingleDeviceTestCase(AbstractTestCase): class LocalMultipleDeviceTestCase(AbstractTestCase): def setup_method(self, method): - self.update_test_info_dict() self.drivers = dict() def create_drivers(self, quantity): @@ -139,7 +131,7 @@ class LocalMultipleDeviceTestCase(AbstractTestCase): for driver in range(quantity): self.drivers[driver] = webdriver.Remote(self.executor_local, capabilities[driver]) self.drivers[driver].implicitly_wait(self.implicitly_wait) - test_data.test_info[test_data.test_name]['jobs'].append(self.drivers[driver].session_id) + test_suite_data.current_test.jobs.append(self.drivers[driver].session_id) def teardown_method(self, method): for driver in self.drivers: @@ -157,7 +149,6 @@ class SauceMultipleDeviceTestCase(AbstractTestCase): asyncio.set_event_loop(cls.loop) def setup_method(self, method): - self.update_test_info_dict() self.drivers = dict() def create_drivers(self, quantity=2): @@ -167,7 +158,7 @@ class SauceMultipleDeviceTestCase(AbstractTestCase): self.capabilities_sauce_lab)) for driver in range(quantity): self.drivers[driver].implicitly_wait(self.implicitly_wait) - test_data.test_info[test_data.test_name]['jobs'].append(self.drivers[driver].session_id) + test_suite_data.current_test.jobs.append(self.drivers[driver].session_id) def teardown_method(self, method): for driver in self.drivers: diff --git a/test/appium/tests/conftest.py b/test/appium/tests/conftest.py index 80e6cbd97f..60995ff9b0 100644 --- a/test/appium/tests/conftest.py +++ b/test/appium/tests/conftest.py @@ -1,4 +1,4 @@ -from tests import test_data +from tests import test_suite_data, SingleTestData import requests import re import pytest @@ -6,8 +6,7 @@ from datetime import datetime from os import environ from io import BytesIO from sauceclient import SauceClient -from hashlib import md5 -import hmac +from support.github_test_report import GithubHtmlReport storage = 'http://artifacts.status.im:8081/artifactory/nightlies-local/' @@ -16,6 +15,7 @@ sauce_access_key = environ.get('SAUCE_ACCESS_KEY') github_token = environ.get('GIT_HUB_TOKEN') sauce = SauceClient(sauce_username, sauce_access_key) +github_report = GithubHtmlReport(sauce_username, sauce_access_key) def get_latest_apk(): @@ -60,7 +60,7 @@ def is_master(config): def is_uploaded(): stored_files = sauce.storage.get_stored_files() for i in range(len(stored_files['files'])): - if stored_files['files'][i]['name'] == test_data.apk_name: + if stored_files['files'][i]['name'] == test_suite_data.apk_name: return True @@ -68,8 +68,8 @@ def pytest_configure(config): if config.getoption('log'): import logging logging.basicConfig(level=logging.INFO) - test_data.apk_name = ([i for i in [i for i in config.getoption('apk').split('/') - if '.apk' in i]])[0] + test_suite_data.apk_name = ([i for i in [i for i in config.getoption('apk').split('/') + if '.apk' in i]])[0] if is_master(config) and config.getoption('env') == 'sauce': if config.getoption('pr_number'): with open('github_comment.txt', 'w') as _: @@ -81,7 +81,7 @@ def pytest_configure(config): file = BytesIO(response.content) del response requests.post('http://saucelabs.com/rest/v1/storage/' - + sauce_username + '/' + test_data.apk_name + '?overwrite=true', + + sauce_username + '/' + test_suite_data.apk_name + '?overwrite=true', auth=(sauce_username, sauce_access_key), data=file, headers={'Content-Type': 'application/octet-stream'}) @@ -94,46 +94,27 @@ def pytest_unconfigure(config): from github import Github repo = Github(github_token).get_user('status-im').get_repo('status-react') pull = repo.get_pull(int(config.getoption('pr_number'))) - with open('github_comment.txt', 'r') as comment: - pull.create_issue_comment('# Automated test results: \n' + comment.read()) - - -def get_public_url(job_id): - token = hmac.new(bytes(sauce_username + ":" + sauce_access_key, 'latin-1'), - bytes(job_id, 'latin-1'), md5).hexdigest() - return "https://saucelabs.com/jobs/%s?auth=%s" % (job_id, token) - - -def make_github_report(error=None): - if pytest.config.getoption('pr_number'): - title = '### %s' % test_data.test_name - outcome = '%s' % ':x:' if error else ':white_check_mark:' + ':\n' - title += outcome - steps = '\n\n
\nTest Steps & Error message:\n\n ```%s ```%s\n\n
\n' % \ - (test_data.test_info[test_data.test_name]['steps'], '\n```' + error + '```' if error else '') - sessions = str() - - for job_id in test_data.test_info[test_data.test_name]['jobs']: - sessions += ' - [Android Device Session](%s) \n' % get_public_url(job_id) - with open('github_comment.txt', 'a') as comment: - comment.write(title + '\n' + steps + '\n' + sessions + '---\n') + pull.create_issue_comment(github_report.build_html_report()) @pytest.mark.hookwrapper def pytest_runtest_makereport(item, call): outcome = yield report = outcome.get_result() - if pytest.config.getoption('env') == 'sauce': - if report.when == 'call': - if report.passed: - for job_id in test_data.test_info[test_data.test_name]['jobs']: - sauce.jobs.update_job(job_id, name=test_data.test_name, passed=True) - make_github_report() - if report.failed: - for job_id in test_data.test_info[test_data.test_name]['jobs']: - sauce.jobs.update_job(job_id, name=test_data.test_name, passed=False) - make_github_report(error=report.longreprtext) + is_sauce_env = pytest.config.getoption('env') == 'sauce' + current_test = test_suite_data.current_test + if report.when == 'call': + if report.failed: + current_test.error = report.longreprtext + if is_sauce_env: + update_sauce_jobs(current_test.name, current_test.jobs, report.passed) + github_report.save_test(current_test) + + +def update_sauce_jobs(test_name, job_ids, passed): + for job_id in job_ids: + sauce.jobs.update_job(job_id, name=test_name, passed=passed) def pytest_runtest_setup(item): - test_data.test_name = item.name + test_suite_data.add_test(SingleTestData(item.name)) \ No newline at end of file