346 lines
15 KiB
Python
346 lines
15 KiB
Python
import base64
|
|
import itertools
|
|
import json
|
|
import logging
|
|
import re
|
|
from json import JSONDecodeError
|
|
from os import environ
|
|
from sys import argv
|
|
|
|
import emoji
|
|
import requests
|
|
|
|
from support.base_test_report import BaseTestReport
|
|
|
|
|
|
class TestrailReport(BaseTestReport):
|
|
|
|
def __init__(self):
|
|
super(TestrailReport, self).__init__()
|
|
|
|
self.password = environ.get('TESTRAIL_PASS')
|
|
self.user = environ.get('TESTRAIL_USER')
|
|
|
|
self.run_id = None
|
|
# self.suite_id = 48
|
|
self.suite_id = 5274
|
|
self.project_id = 14
|
|
|
|
self.outcomes = {
|
|
'passed': 1,
|
|
'undefined_fail': 10}
|
|
|
|
self.headers = dict()
|
|
self.headers['Authorization'] = 'Basic %s' % str(
|
|
base64.b64encode(bytes('%s:%s' % (self.user, self.password), 'utf-8')), 'ascii').strip()
|
|
self.headers['Content-Type'] = 'application/json'
|
|
self.headers['x-api-ident'] = 'beta'
|
|
|
|
self.url = 'https://ethstatus.testrail.net/index.php?/'
|
|
self.api_url = self.url + 'api/v2/'
|
|
|
|
def get(self, method):
|
|
rval = requests.get(self.api_url + method, headers=self.headers).json()
|
|
if 'error' in rval:
|
|
logging.error("Failed TestRail request: %s" % rval['error'])
|
|
return rval
|
|
|
|
def post(self, method, data):
|
|
data = bytes(json.dumps(data), 'utf-8')
|
|
return requests.post(self.api_url + method, data=data, headers=self.headers).json()
|
|
|
|
def add_attachment(self, method, path):
|
|
files = {'attachment': (open(path, 'rb'))}
|
|
result = requests.post(self.api_url + method, headers={'Authorization': self.headers['Authorization']},
|
|
files=files)
|
|
files['attachment'].close()
|
|
try:
|
|
return result.json()
|
|
except JSONDecodeError:
|
|
pass
|
|
|
|
def get_suites(self):
|
|
return self.get('get_suites/%s' % self.project_id)
|
|
|
|
def get_tests(self):
|
|
return self.get('get_tests/%s' % self.run_id)['tests']
|
|
|
|
def get_milestones(self):
|
|
return self.get('get_milestones/%s' % self.project_id)['milestones']
|
|
|
|
def get_runs(self, pr_number):
|
|
return [i for i in self.get('get_runs/%s' % self.project_id)['runs'] if 'PR-%s ' % pr_number in i['name']]
|
|
|
|
def get_run(self, run_id: int):
|
|
return self.get('get_run/%s' % run_id)
|
|
|
|
def get_last_pr_run(self, pr_number):
|
|
run_id = max([run['id'] for run in self.get_runs(pr_number)])
|
|
return self.get_run(run_id=run_id)
|
|
|
|
@property
|
|
def actual_milestone_id(self):
|
|
return self.get_milestones()[-1]['id']
|
|
|
|
def add_run(self, run_name):
|
|
request_body = {'suite_id': self.suite_id,
|
|
'name': run_name,
|
|
'milestone_id': self.actual_milestone_id,
|
|
'case_ids': self.get_regression_cases(),
|
|
'include_all': False}
|
|
run = self.post('add_run/%s' % self.project_id, request_body)
|
|
try:
|
|
self.run_id = run['id']
|
|
except KeyError:
|
|
print("TestRail error when creating a run: %s" % run)
|
|
print("Testrun: %sruns/view/%s" % (self.url, self.run_id))
|
|
|
|
def get_cases(self, section_ids):
|
|
test_cases = list()
|
|
for section_id in section_ids:
|
|
test_cases.append(
|
|
self.get('get_cases/%s&suite_id=%s§ion_id=%s' % (self.project_id, self.suite_id, section_id))[
|
|
'cases'])
|
|
return itertools.chain.from_iterable(test_cases)
|
|
|
|
def get_regression_cases(self):
|
|
test_cases = dict()
|
|
test_cases['pr'] = dict()
|
|
test_cases['nightly'] = dict()
|
|
test_cases['upgrade'] = dict()
|
|
## PR e2e old UI
|
|
# test_cases['pr']['critical'] = 730
|
|
# test_cases['pr']['contacts'] = 50831
|
|
# test_cases['pr']['public_chat'] = 50654
|
|
# test_cases['pr']['one_to_one_chat'] = 50655
|
|
# test_cases['pr']['group_chat'] = 50656
|
|
# test_cases['pr']['onboarding'] = 50659
|
|
# test_cases['pr']['recovery'] = 50660
|
|
# test_cases['pr']['wallet'] = 50661
|
|
# test_cases['pr']['send_tx'] = 50662
|
|
# test_cases['pr']['keycard_tx'] = 50663
|
|
# test_cases['pr']['1_1_chat_commands'] = 50825
|
|
# test_cases['pr']['ens'] = 50827
|
|
# test_cases['pr']['sync'] = 50834
|
|
# test_cases['pr']['browser'] = 50812
|
|
|
|
test_cases['pr']['critical'] = 50955
|
|
test_cases['pr']['one_to_one_chat'] = 50956
|
|
test_cases['pr']['deep_links'] = 51535
|
|
test_cases['pr']['group_chat'] = 50964
|
|
test_cases['pr']['community_single'] = 50983
|
|
test_cases['pr']['community_multiple'] = 50982
|
|
test_cases['pr']['activity_centre_contact_request'] = 50984
|
|
test_cases['pr']['activity_centre_other'] = 51005
|
|
test_cases['pr']['wallet'] = 59443
|
|
|
|
## Nightly e2e
|
|
# test_cases['nightly']['activity_center'] = 736
|
|
# test_cases['nightly']['chat'] = 50811
|
|
# test_cases['nightly']['browser'] = 50826
|
|
# test_cases['nightly']['profile'] = 50828
|
|
# test_cases['nightly']['deep_link'] = 50836
|
|
# test_cases['nightly']['share_profile'] = 50837
|
|
# test_cases['nightly']['chat_2'] = 50838
|
|
# test_cases['nightly']['group_chat'] = 50839
|
|
# test_cases['nightly']['pairing'] = 50840
|
|
# test_cases['nightly']['activity_center'] = 50833
|
|
# test_cases['nightly']['timeline'] = 50842
|
|
# test_cases['nightly']['community'] = 50841
|
|
# test_cases['nightly']['permissions'] = 50843
|
|
# test_cases['nightly']['scan qr'] = 50844
|
|
# test_cases['nightly']['mentions'] = 50845
|
|
# test_cases['nightly']['mutual_contact_requests'] = 50857
|
|
# test_cases['nightly']['keycard'] = 50850
|
|
# test_cases['nightly']['wallet'] = 50851
|
|
|
|
## Upgrade e2e
|
|
# test_cases['upgrade']['general'] = 881
|
|
|
|
case_ids = list()
|
|
for arg in argv:
|
|
if "run_testrail_ids" in arg:
|
|
key, value = arg.split('=')
|
|
case_ids = value.split(',')
|
|
if len(case_ids) == 0:
|
|
# if 'critical' in argv:
|
|
if 'new_ui_critical' in argv:
|
|
for category in test_cases['pr']:
|
|
for case in self.get_cases([test_cases['pr'][category]]):
|
|
case_ids.append(case['id'])
|
|
elif 'upgrade' in argv and 'not upgrade' not in argv:
|
|
for case in self.get_cases([test_cases['upgrade']['general']]):
|
|
case_ids.append(case['id'])
|
|
else:
|
|
for phase in test_cases:
|
|
if phase != 'upgrade':
|
|
for category in test_cases[phase]:
|
|
for case in self.get_cases([test_cases[phase][category]]):
|
|
case_ids.append(case['id'])
|
|
return case_ids
|
|
|
|
def add_results(self):
|
|
data = list()
|
|
all_tests = self.get_all_tests()
|
|
for test in all_tests:
|
|
test_steps = "# Steps: \n"
|
|
devices = str()
|
|
last_testrun = test.testruns[-1]
|
|
for step in last_testrun.steps:
|
|
test_steps += step + "\n"
|
|
for i, device in enumerate(last_testrun.jobs):
|
|
if last_testrun.first_commands:
|
|
try:
|
|
first_command = last_testrun.first_commands[device]
|
|
except KeyError:
|
|
first_command = 0
|
|
else:
|
|
first_command = 0
|
|
try:
|
|
devices += "# [Device %d](%s) \n" % (i + 1, self.get_sauce_job_url(job_id=device,
|
|
first_command=first_command))
|
|
except KeyError:
|
|
devices += "# Device %s: SauceLabs session was not found \n" % (i + 1)
|
|
comment = str()
|
|
if test.group_name:
|
|
comment += "# Class: %s \n" % test.group_name
|
|
if last_testrun.error:
|
|
full_error = last_testrun.error
|
|
(code_error, no_code_error_str, issue_id) = self.separate_xfail_error(full_error)
|
|
if issue_id:
|
|
test_rail_xfail = self.make_error_with_gh_issue_link(no_code_error_str, issue_id)
|
|
error = "%s %s" % (code_error, test_rail_xfail)
|
|
else:
|
|
error = full_error
|
|
error = error.replace("[[", "**").replace("]]", "**")
|
|
comment += '%s' % ('# Error: \n %s \n' % emoji.demojize(error)) + devices + test_steps
|
|
else:
|
|
comment += devices + test_steps
|
|
data.append(
|
|
{'case_id': test.testrail_case_id,
|
|
'status_id': self.outcomes['undefined_fail'] if last_testrun.error else self.outcomes['passed'],
|
|
'comment': comment})
|
|
|
|
results = self.post('add_results_for_cases/%s' % self.run_id, data={"results": data})
|
|
try:
|
|
results[0]
|
|
except (IndexError, KeyError):
|
|
print("Got TestRail error when adding results: \n%s" % results)
|
|
|
|
for test in all_tests:
|
|
last_testrun = test.testruns[-1]
|
|
if last_testrun.error:
|
|
try:
|
|
device = list(last_testrun.jobs.keys())[0]
|
|
except IndexError:
|
|
continue
|
|
for res in results:
|
|
if last_testrun.first_commands:
|
|
try:
|
|
pattern = r"%s\?auth=.*#%s" % (device, str(last_testrun.first_commands[device]))
|
|
except KeyError:
|
|
pattern = device
|
|
else:
|
|
pattern = device
|
|
if re.findall(pattern, res['comment']):
|
|
res_id = res['id']
|
|
try:
|
|
for geth in test.geth_paths.keys():
|
|
self.add_attachment(method='add_attachment_to_result/%s' % str(res_id),
|
|
path=test.geth_paths[geth])
|
|
except (AttributeError, FileNotFoundError):
|
|
pass
|
|
break
|
|
|
|
self.change_test_run_description()
|
|
|
|
def change_test_run_description(self):
|
|
tests = self.get_all_tests()
|
|
passed, failed, xfailed = self.get_tests_by_status()
|
|
not_executed_tests = self.get_not_executed_tests(self.run_id)
|
|
final_description = "Nothing to report this time..."
|
|
if len(tests) > 0:
|
|
description_title = "# %.0f%% of end-end tests have passed\n" % (len(passed) / len(tests) * 100)
|
|
description_title += "\n"
|
|
description_title += "Total executed tests: %d\n" % len(tests)
|
|
description_title += "Failed tests: %d\n" % len(failed)
|
|
description_title += "Expected to fail tests: %d\n" % len(xfailed)
|
|
description_title += "Passed tests: %d\n" % len(passed)
|
|
if not_executed_tests:
|
|
description_title += "Not executed tests: %d\n" % len(not_executed_tests)
|
|
description_title += "\n"
|
|
ids_failed_test = []
|
|
single_devices_block, group_blocks, case_info = str(), dict(), str()
|
|
if failed:
|
|
for test in failed:
|
|
if test.group_name:
|
|
group_blocks[test.group_name] = "\n-------\n## Class: %s:\n" % test.group_name
|
|
for test in failed:
|
|
last_testrun = test.testruns[-1]
|
|
test_rail_link = self.get_test_result_link(self.run_id, test.testrail_case_id)
|
|
ids_failed_test.append(test.testrail_case_id)
|
|
case_title = '\n'
|
|
case_title += '-------\n'
|
|
case_title += "### ID %s: [%s](%s) \n" % (test.testrail_case_id, test.name, test_rail_link)
|
|
full_error = last_testrun.error[-255:]
|
|
(code_error, no_code_error_str, issue_id) = self.separate_xfail_error(full_error)
|
|
if issue_id:
|
|
test_rail_xfail = self.make_error_with_gh_issue_link(no_code_error_str, issue_id)
|
|
error = "```%s```\n **%s** \n" % (code_error, test_rail_xfail)
|
|
else:
|
|
error = "```%s```\n **%s** \n" % (code_error, no_code_error_str)
|
|
for job_id, f in last_testrun.jobs.items():
|
|
if last_testrun.first_commands:
|
|
try:
|
|
first_command = last_testrun.first_commands[job_id]
|
|
except KeyError:
|
|
first_command = 0
|
|
else:
|
|
first_command = 0
|
|
job_url = self.get_sauce_job_url(job_id=job_id, first_command=first_command)
|
|
case_info = "Logs for device %d: [steps](%s), [failure screenshot](%s)" \
|
|
% (f, job_url, self.get_sauce_final_screenshot_url(job_id))
|
|
|
|
if test.group_name:
|
|
group_blocks[test.group_name] += case_title + error + case_info
|
|
else:
|
|
single_devices_block += case_title + error + case_info
|
|
description_title += '## Failed tests: %s \n' % ','.join(map(str, ids_failed_test))
|
|
if not_executed_tests:
|
|
description_title += "## Not executed tests: %s\n" % ','.join([str(i) for i in not_executed_tests])
|
|
final_description = description_title + single_devices_block + ''.join([i for i in group_blocks.values()])
|
|
|
|
request_body = {'description': final_description}
|
|
return self.post('update_run/%s' % self.run_id, request_body)
|
|
|
|
def get_run_results(self, test_run_id=None):
|
|
return self.get('get_results_for_run/%s' % (test_run_id if test_run_id else self.run_id))['results']
|
|
|
|
def is_run_successful(self):
|
|
for test in self.get_run_results():
|
|
if test['status_id'] != 1:
|
|
return False
|
|
else:
|
|
return True
|
|
|
|
def get_test_result_link(self, test_run_id, test_case_id):
|
|
try:
|
|
test_id = self.get('get_results_for_case/%s/%s' % (test_run_id, test_case_id))['results'][0]['test_id']
|
|
return '%stests/view/%s' % (self.url, test_id)
|
|
except (KeyError, JSONDecodeError):
|
|
print('Cannot extract result for %s e2e' % test_case_id)
|
|
return None
|
|
|
|
def get_not_executed_tests(self, test_run_id):
|
|
try:
|
|
results = self.get("get_tests/%s&status_id=3" % test_run_id)
|
|
return [result['case_id'] for result in results["tests"]]
|
|
except KeyError:
|
|
print('Cannot extract result for %s' % test_run_id)
|
|
pass
|
|
|
|
@staticmethod
|
|
def make_error_with_gh_issue_link(error, issue_id):
|
|
return error.replace(issue_id,
|
|
'[%s](https://github.com/status-im/status-mobile/issues/%s)' % (issue_id, issue_id[1:]))
|