e2e: shared steps for session

This commit is contained in:
Churikova Tetiana 2022-01-24 12:40:02 +01:00
parent 012b3e396e
commit 7bbb882ffa
No known key found for this signature in database
GPG Key ID: 0D4EA7B33B47E6D8
5 changed files with 85 additions and 27 deletions

View File

@ -28,8 +28,7 @@ class BaseTestReport:
file_name = "%s.json" % test_name
return os.path.join(self.TEST_REPORT_DIR, file_name)
def save_test(self, test, geth: dict):
file_path = self.get_test_report_file_path(test.name)
def save_geth(self, geth: dict):
geth_paths = {}
for log in geth.keys():
geth_path = os.path.join(self.TEST_REPORT_DIR, log)
@ -37,6 +36,14 @@ class BaseTestReport:
result.write(geth[log])
result.close()
geth_paths[log] = geth_path
return geth_paths
def save_test(self, test, geth: dict = None):
if geth:
geth_paths = self.save_geth(geth)
else:
geth_paths = test.geth_paths
file_path = self.get_test_report_file_path(test.name)
test_dict = {
'testrail_case_id': test.testrail_case_id,
'name': test.name,
@ -58,7 +65,8 @@ class BaseTestReport:
testruns.append(SingleTestData.TestRunData(
steps=testrun_data['steps'],
jobs=testrun_data['jobs'],
error=testrun_data['error']))
error=testrun_data['error'],
first_commands=testrun_data['first_commands']))
tests.append(SingleTestData(name=test_data['name'],
geth_paths=test_data['geth_paths'],
testruns=testruns,
@ -85,9 +93,12 @@ class BaseTestReport:
return hmac.new(bytes(self.sauce_username + ":" + self.sauce_access_key, 'latin-1'),
bytes(job_id, 'latin-1'), md5).hexdigest()
def get_sauce_job_url(self, job_id):
def get_sauce_job_url(self, job_id, first_command=0):
token = self.get_sauce_token(job_id)
return 'https://saucelabs.com/jobs/%s?auth=%s' % (job_id, token)
url = 'https://saucelabs.com/jobs/%s?auth=%s' % (job_id, token)
if first_command > 0:
url += "#%s" % first_command
return url
@staticmethod
def get_jenkins_link_to_rerun_e2e(branch_name="develop", pr_id="", apk_name="", tr_case_ids=""):
@ -107,4 +118,4 @@ class BaseTestReport:
@staticmethod
def is_test_successful(test):
# Test passed if last testrun has passed
return test.testruns[-1].error is None
return test.testruns[-1].error is None

View File

@ -2,6 +2,7 @@ import os
from support.base_test_report import BaseTestReport
from support.testrail_report import TestrailReport
class GithubHtmlReport(BaseTestReport):
TEST_REPORT_DIR = "%s/../report" % os.path.dirname(os.path.abspath(__file__))
@ -52,7 +53,9 @@ class GithubHtmlReport(BaseTestReport):
pr_id = pytest_config_global['pr_number']
apk_name = pytest_config_global['apk']
tr_case_ids = self.list_of_failed_testrail_ids(self.get_failed_tests())
html += "<li><a href=\"%s\">Rerun tests</a></li>" % self.get_jenkins_link_to_rerun_e2e(pr_id=pr_id, apk_name=apk_name, tr_case_ids=tr_case_ids)
html += "<li><a href=\"%s\">Rerun tests</a></li>" % self.get_jenkins_link_to_rerun_e2e(pr_id=pr_id,
apk_name=apk_name,
tr_case_ids=tr_case_ids)
html += "<br/>"
html += "<table style=\"width: 100%\">"
@ -73,7 +76,8 @@ class GithubHtmlReport(BaseTestReport):
def build_test_row_html(self, index, test, run_id):
test_rail_link = TestrailReport().get_test_result_link(run_id, test.testrail_case_id)
if test_rail_link:
html = "<tr><td><b>%s. <a href=\"%s\">%s</a>, id: %s </b></td></tr>" % (index + 1, test_rail_link, test.name, test.testrail_case_id)
html = "<tr><td><b>%s. <a href=\"%s\">%s</a>, id: %s </b></td></tr>" % (
index + 1, test_rail_link, test.name, test.testrail_case_id)
else:
html = "<tr><td><b>%d. %s</b> (TestRail link is not found)</td></tr>" % (index + 1, test.name)
html += "<tr><td>"
@ -92,20 +96,25 @@ class GithubHtmlReport(BaseTestReport):
html += "<code>%s</code>" % last_testrun.error[:255]
html += "<br/><br/>"
if last_testrun.jobs:
html += self.build_device_sessions_html(last_testrun.jobs, last_testrun)
html += self.build_device_sessions_html(last_testrun)
html += "</td></tr>"
return html
def build_device_sessions_html(self, jobs, test_run):
def build_device_sessions_html(self, test_run):
html = "<ins>Device sessions</ins>"
html += "<p><ul>"
for job_id, i in jobs.items():
for job_id, i in test_run.jobs.items():
html += "<p>"
html += "Device %d:" % i
html += "<ul>"
html += "<li><a href=\"%s\">Steps, video, logs</a></li>" % self.get_sauce_job_url(job_id)
if test_run.first_commands:
html += "<li><a href=\"%s\">Steps, video, logs</a></li>" % self.get_sauce_job_url(job_id,
test_run.first_commands[
job_id])
else:
html += "<li><a href=\"%s\">Steps, video, logs</a></li>" % self.get_sauce_job_url(job_id)
if test_run.error:
html += "<li><a href=\"%s\">Failure screenshot</a></li>" % self.get_sauce_final_screenshot_url(job_id)
html += "</ul></p>"
html += "</ul></p>"
return html
return html

View File

@ -1,3 +1,4 @@
from typing import Dict
class SingleTestData(object):
@ -8,13 +9,14 @@ class SingleTestData(object):
self.geth_paths = geth_paths
class TestRunData(object):
def __init__(self, steps, jobs, error):
def __init__(self, steps, jobs, error, first_commands: Dict[str, int]):
self.steps = steps
self.jobs = jobs
self.error = error
self.first_commands = first_commands
def create_new_testrun(self):
self.testruns.append(SingleTestData.TestRunData(list(), dict(), None))
self.testruns.append(SingleTestData.TestRunData(list(), dict(), None, dict()))
class TestSuiteData(object):
@ -30,4 +32,4 @@ class TestSuiteData(object):
else:
test = SingleTestData(test_name, list(), testrail_case_id, list())
self.tests.append(test)
self.current_test = test
self.current_test = test

View File

@ -130,7 +130,11 @@ class TestrailReport(BaseTestReport):
for step in last_testrun.steps:
test_steps += step + "\n"
for i, device in enumerate(last_testrun.jobs):
devices += "# [Device %d](%s) \n" % (i + 1, self.get_sauce_job_url(device))
if last_testrun.first_commands:
devices += "# [Device %d](%s) \n" % (
i + 1, self.get_sauce_job_url(job_id=device, first_command=last_testrun.first_commands[device]))
else:
devices += "# [Device %d](%s) \n" % (i + 1, self.get_sauce_job_url(job_id=device))
data = {'status_id': self.outcomes['undefined_fail'] if last_testrun.error else self.outcomes['passed'],
'comment': '%s' % ('# Error: \n %s \n' % emoji.demojize(
last_testrun.error)) + devices + test_steps if last_testrun.error
@ -166,13 +170,17 @@ class TestrailReport(BaseTestReport):
ids_failed_test.append(test.testrail_case_id)
case_title = '\n'
case_title += '-------\n'
case_title += "## %s) ID %s: [%s](%s) \n" % (i + 1, test.testrail_case_id, test.name, test_rail_link)
case_title += "## %s) ID %s: [%s](%s) \n" % (
i + 1, test.testrail_case_id, test.name, test_rail_link)
error = "```%s```\n" % last_testrun.error[:255]
for job_id, f in last_testrun.jobs.items():
if last_testrun.first_commands:
job_url = self.get_sauce_job_url(job_id=job_id,
first_command=last_testrun.first_commands[job_id])
else:
job_url = self.get_sauce_job_url(job_id=job_id)
case_info = "Logs for device %d: [steps](%s), [failure screenshot](%s)" \
% (f,
self.get_sauce_job_url(job_id),
self.get_sauce_final_screenshot_url(job_id))
% (f, job_url, self.get_sauce_final_screenshot_url(job_id))
description += case_title + error + case_info
description_title += '## Failed tests: %s \n' % ','.join(map(str, ids_failed_test))
@ -196,4 +204,4 @@ class TestrailReport(BaseTestReport):
test_id = self.get('get_results_for_case/%s/%s' % (test_run_id, test_case_id))['results'][0]['test_id']
return '%stests/view/%s' % (self.url, test_id)
except KeyError:
return None
return None

View File

@ -4,13 +4,18 @@ import re
import subprocess
import sys
from abc import ABCMeta, abstractmethod
from http.client import RemoteDisconnected
from os import environ
import pytest
import requests
from appium import webdriver
from appium.webdriver.common.mobileby import MobileBy
from sauceclient import SauceException
from selenium.common.exceptions import NoSuchElementException
from selenium.common.exceptions import WebDriverException
from selenium.webdriver.support.wait import WebDriverWait
from tests import transl
from support.api.network_api import NetworkApi
@ -19,8 +24,9 @@ from tests import test_suite_data, start_threads, appium_container, pytest_confi
import base64
from re import findall
sauce_username = environ.get('SAUCE_USERNAME')
from tests.conftest import sauce
sauce_username = environ.get('SAUCE_USERNAME')
sauce_access_key = environ.get('SAUCE_ACCESS_KEY')
@ -334,6 +340,8 @@ class LocalSharedMultipleDeviceTestCase(AbstractTestCase):
class SauceSharedMultipleDeviceTestCase(AbstractTestCase):
def setup_method(self, method):
for _, driver in self.drivers.items():
driver.execute_script("sauce:context=Started %s" % method.__name__)
jobs = test_suite_data.current_test.testruns[-1].jobs
if not jobs:
for index, driver in self.drivers.items():
@ -354,16 +362,36 @@ class SauceSharedMultipleDeviceTestCase(AbstractTestCase):
pass
finally:
geth = {geth_names[i]: geth_contents[i] for i in range(len(geth_names))}
self.github_report.save_test(test_suite_data.current_test, geth)
test_suite_data.current_test.geth_paths = self.github_report.save_geth(geth)
@classmethod
def teardown_class(cls):
for driver in cls.drivers:
requests_session = requests.Session()
requests_session.auth = (sauce_username, sauce_access_key)
for _, driver in cls.drivers.items():
session_id = driver.session_id
try:
cls.drivers[driver].quit()
sauce.jobs.update_job(job_id=session_id, name=cls.__name__)
except (RemoteDisconnected, SauceException):
pass
try:
driver.quit()
except WebDriverException:
pass
url = sauce.jobs.get_job_asset_url(job_id=session_id, filename="log.json")
WebDriverWait(driver, 60, 2).until(lambda _: requests_session.get(url).status_code == 200)
commands = requests_session.get(url).json()
for command in commands:
try:
if command['message'].startswith("Started "):
for test in test_suite_data.tests:
if command['message'] == "Started %s" % test.name:
test.testruns[-1].first_commands[session_id] = commands.index(command) + 1
except KeyError:
continue
cls.loop.close()
for test in test_suite_data.tests:
cls.github_report.save_test(test)
if pytest_config_global['env'] == 'local':
@ -380,4 +408,4 @@ class NoDeviceTestCase(AbstractTestCase):
pass
def teardown_method(self, method):
self.github_report.save_test(test_suite_data.current_test)
self.github_report.save_test(test_suite_data.current_test)