added runner and testrail report

Signed-off-by: Anton Danchenko <ant.danchenko@gmail.com>

test/desktop.sikuli/pytest.ini

@@ -1,2 +0,0 @@
-[pytest]
-addopts = -s -v --junitxml=result.xml --tb=line

test/desktop.sikuli/start_here.py

@@ -1,33 +0,0 @@
-import requests
-import re
-import docker
-from urllib.request import urlretrieve
-
-raw_data = requests.request('GET', 'https://status-im.github.io/nightly/').text
-app_url = re.findall('href="(.*AppImage)', raw_data)[0]
-urlretrieve(app_url, 'nightly.AppImage')
-
-
-ps = None
-
-
-client = docker.from_env()
-client.images.build(tag='status_desktop', path='.')
-
-try:
-    a = client.containers.run("status_desktop",
-                              detach=True, tty=True, ports={'5900/tcp': 5900})
-
-    it = a.attach(stdout=True, stderr=True)
-
-    print(it)
-
-    ps = a.exec_run(['/jython-2.7.1/bin/jython', '-m', 'pytest', '/home/tests/test_create_account.py'],
-                    stdout=True, stderr=False)
-except Exception:
-    pass
-finally:
-    a.stop()
-
-for line in ps.output.decode("utf-8").split("\n"):
-    print(line)

test/desktop.sikuli/tests/base_test_case.py

@@ -1,13 +0,0 @@
-import org.sikuli.script.SikulixForJython
-from sikuli import *
-
-
-class BaseTestCase:
-
-    Settings.ActionLogs = 0
-
-    def setup_method(self, method):
-        openApp('/home/squashfs-root/AppRun')
-
-    def teardown_method(self, method):
-        pass

test/desktop.sikuli/tests/conftest.py

@@ -1,46 +0,0 @@
-import pytest
-from subprocess import check_output
-
-from tests.report import save_test_result, TEST_REPORT_DIR
-
-
-def pytest_addoption(parser):
-    parser.addoption('--os',
-                     action='store',
-                     default='linux')
-    parser.addoption("--nightly",
-                     action="store",
-                     default=True)
-    parser.addoption('--dmg',
-                     action='store',
-                     default=None,
-                     help='Url or local path to dmg')
-
-
-def pytest_configure(config):
-    import logging
-    logging.basicConfig(level=logging.INFO)
-
-
-@pytest.mark.hookwrapper
-def pytest_runtest_makereport(item, call):
-    import logging
-    # logging.basicConfig(filename='%s/%s.log' % (TEST_REPORT_DIR, item.name), filemode='w', level=logging.INFO)
-    outcome = yield
-    report = outcome.get_result()
-    if report.when == 'call':
-        pass
-        # save_test_result(item, report)
-
-
-def after_all():
-    if pytest.config.getoption('os') == 'linux':
-        pass
-        # check_output(['rm', '-rf', 'nightly.AppImage'])
-    else:
-        check_output(['rm', '-rf', 'nightly.dmg'])
-
-
-@pytest.fixture(scope="session", autouse=True)
-def before_all(request):
-    request.addfinalizer(finalizer=after_all)

test/desktop.sikuli/tests/report.py

@@ -1,33 +0,0 @@
-import json
-import os
-from subprocess import check_output
-
-TEST_REPORT_DIR = "%s/../report" % os.path.dirname(os.path.abspath(__file__))
-
-
-def get_test_report_file_path(test_name):
-    file_name = "%s.json" % test_name
-    if not os.path.exists(TEST_REPORT_DIR):
-        os.makedirs(TEST_REPORT_DIR)
-    return os.path.join(TEST_REPORT_DIR, file_name)
-
-
-def get_testrail_case_id(obj):
-    if 'testrail_id' in obj.keywords._markers:
-        return obj.keywords._markers['testrail_id'].args[0]
-
-
-def save_test_result(test, report):
-    test_name = test.name
-    file_path = get_test_report_file_path(test_name)
-    if report.failed:
-        check_output(['screencapture', '%s/%s.png' % (TEST_REPORT_DIR, test_name)])
-    with open('%s/%s.log' % (TEST_REPORT_DIR, test_name), 'r') as log:
-        steps = [i for i in log]
-    test_dict = {
-        'testrail_case_id': get_testrail_case_id(test),
-        'name': test_name,
-        'steps': steps,
-        'error': str(report.longrepr.reprcrash.message),
-        'screenshot': test_name + '.png'}
-    json.dump(test_dict, open(file_path, 'w'))

test/desktop_sikuli/Dockerfile

@@ -39,6 +39,7 @@ RUN LDFLAGS="-L/usr/local/lib" CFLAGS="-I/usr/local/include" make
 RUN make install
 RUN make install-langs
 RUN ldconfig
+RUN apt-get install -y imagemagick
 
 ADD . /home
 WORKDIR "/home"

test/desktop_sikuli/Jenkinsfile

@@ -0,0 +1,19 @@
+node ('linux1'){
+  stage('Git & Setup') {
+  checkout([$class: 'GitSCM', branches: [[name: '$branch']],
+  doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'CleanBeforeCheckout']],
+  submoduleCfg: [], userRemoteConfigs: [[url: 'https://github.com/status-im/status-react.git']]])
+ }
+  stage('Tests & Report'){
+  try {withCredentials([string(credentialsId: 'TESTRAIL_PASS', variable: 'TESTRAIL_PASS'),
+                        string(credentialsId: 'TESTRAIL_USER', variable: 'TESTRAIL_USER')])
+          {
+            sh "cd test/desktop_sikuli && python3 main.py " +
+                    "--test_results_path=/${env.WORKSPACE}/test/desktop_sikuli/report " +
+                    "--linux_app_url=${linux_app_url} " +
+                    "--testrail_report=True " +
+                    "--jenkins_build_num=${env.BUILD_NUMBER}"}}
+  finally {
+            archiveArtifacts 'test/desktop_sikuli/report/*.png'}
+  }
+}

test/desktop_sikuli/main.py

@@ -0,0 +1,72 @@
+import docker
+import pytest
+import argparse
+import os
+from datetime import datetime
+from urllib.request import urlretrieve
+from tests.report import TEST_REPORT_DIR_CONTAINER
+from tests.report import TestrailReportDesktop
+
+
+def main(**kwargs):
+
+    if not os.path.exists(kwargs['test_results_path']):
+        os.makedirs(kwargs['test_results_path'])
+    app_version = None
+    if kwargs['linux_app_url']:
+        app_version = ([i for i in [i for i in kwargs['linux_app_url'].split('/') if '.AppImage' in i]])[0]
+        urlretrieve(kwargs['linux_app_url'], 'nightly.AppImage')
+    client = docker.from_env()
+    client.images.build(tag='status_desktop', path='.')
+    pytest.main(['--collect-only', '-m %s' % kwargs['mark']])
+    from tests import test_data
+    if kwargs['testrail_report']:
+        testrail_report = TestrailReportDesktop(kwargs['test_results_path'])
+        testrail_report.add_run(app_version if app_version else '%s' % datetime.now().strftime("%Y-%m-%d %H:%M"))
+    for test in test_data.tests_to_run:
+        print(test)
+        try:
+            container = client.containers.run("status_desktop",
+                                              detach=True, tty=True, ports={'5900/tcp': 5900},
+                                              volumes={kwargs['test_results_path']: {'bind': TEST_REPORT_DIR_CONTAINER,
+                                                                                     'mode': 'rw'}})
+            outcome = container.exec_run(['/jython-2.7.1/bin/jython', '-m', 'pytest', '/home/tests/' + test],
+                                         stdout=True, stderr=False, stdin=True)
+        except Exception:
+            print(outcome)
+        finally:
+            for line in outcome.output.decode("utf-8").split("\n"):
+                print(line)
+            if container:
+                container.stop()
+    if kwargs['testrail_report']:
+        testrail_report.add_results(kwargs['jenkins_build_num'])
+    for container in client.containers.list(all=True):
+        container.remove()
+    for image_id in [image.id for image in client.images.list() if not image.attrs["RepoTags"]]:
+        try:
+            client.images.remove(image_id)
+            print("%s is removed" % image_id)
+        except client.errors.APIError:
+            print("%s is not removed, the image is still in use" % image_id)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--test_results_path',
+                        action='store',
+                        default='/Users/adan/Desktop/desktop_e2e_results')
+    parser.add_argument('--linux_app_url',
+                        action='store',
+                        default=None)
+    parser.add_argument('--mark',
+                        action='store',
+                        default='testrail_id')
+    parser.add_argument('--testrail_report',
+                        action='store',
+                        default=False)
+    parser.add_argument('--jenkins_build_num',
+                        action='store',
+                        default=None)
+    args = parser.parse_args()
+    main(**vars(args))

test/desktop_sikuli/pytest.ini

@@ -0,0 +1,2 @@
+[pytest]
+addopts = -s -v --tb=line

test/desktop_sikuli/tests/__init__.py

@@ -1,5 +1,14 @@
+
+class TestData(object):
+
+    def __init__(self):
+        self.tests_to_run = list()
+
+
+test_data = TestData()
+
 base_user = dict()
 base_user['passphrase'] = 'tree weekend ceiling awkward universe pyramid glimpse raven pair lounge grant grief'
 base_user['username'] = 'Little Weighty Iberianmole'
 base_user['public_key'] = '0x040d3400f0ba80b2f6017a9021a66e042abc33cf7051ddf98a24a815c93d6c052ce2b787' \
-                          '3d799f0963259f41c5a1bf08133dd4f3fe63ea1cceaa1e86ebc4bc42c9'
+                          '3d799f0963259f41c5a1bf08133dd4f3fe63ea1cceaa1e86ebc4bc42c9'

test/desktop_sikuli/tests/base_test_case.py

@@ -0,0 +1,20 @@
+try:
+    import org.sikuli.script.SikulixForJython
+    from sikuli import *
+except Exception:
+    pass
+
+
+class BaseTestCase:
+
+    try:
+        Settings.ActionLogs = 0
+        Settings.MinSimilarity = 0.4
+    except NameError:
+        pass
+
+    def setup_method(self, method):
+        openApp('/home/squashfs-root/AppRun')
+
+    def teardown_method(self, method):
+        pass

test/desktop_sikuli/tests/conftest.py

@@ -0,0 +1,26 @@
+import pytest
+import re
+from tests.report import save_test_result, TEST_REPORT_DIR_CONTAINER
+import logging
+from tests import test_data
+
+
+@pytest.mark.hookwrapper
+def pytest_runtest_makereport(item, call):
+    logging.basicConfig(filename='%s/%s.log' % (TEST_REPORT_DIR_CONTAINER, item.name),
+                        filemode='w', level=logging.INFO)
+    outcome = yield
+    report = outcome.get_result()
+    if report.when == 'call':
+        save_test_result(item, report)
+
+
+def pytest_collection(session):
+    items = session.perform_collect()
+    if session.config.getoption('--collect-only'):
+        for i, item in enumerate(items):
+            name = item.__dict__['name']
+            cls = re.findall('\.(Test.*) object', str(item.__dict__['_obj']))[0]
+            module = re.findall('(test_.*.py)', str(item.__dict__['fspath']))[0]
+            test_data.tests_to_run.append('%s::%s::%s' % (module, cls, name))
+    return items

test/desktop_sikuli/tests/report.py

@@ -0,0 +1,150 @@
+import json
+import os
+import requests
+import base64
+from subprocess import check_output
+
+TEST_REPORT_DIR_CONTAINER = '/var/log/takoe'
+
+
+def get_test_report_file_path(test_name):
+    file_name = "%s.json" % test_name
+    if not os.path.exists(TEST_REPORT_DIR_CONTAINER):
+        os.makedirs(TEST_REPORT_DIR_CONTAINER)
+    return os.path.join(TEST_REPORT_DIR_CONTAINER, file_name)
+
+
+def get_testrail_case_id(obj):
+    if 'testrail_id' in obj.keywords._markers:
+        return obj.keywords._markers['testrail_id'].args[0]
+
+
+def save_test_result(test, report):
+    test_name = test.name
+    file_path = get_test_report_file_path(test_name)
+    error = None
+    if report.failed:
+        check_output(['import', '-window', 'root', '%s/%s.png' % (TEST_REPORT_DIR_CONTAINER, test_name)])
+        error = report.longrepr.reprcrash.message
+    with open('%s/%s.log' % (TEST_REPORT_DIR_CONTAINER, test_name), 'r') as log:
+        steps = [i for i in log]
+    test_dict = {
+        'testrail_case_id': get_testrail_case_id(test),
+        'name': test_name,
+        'steps': steps,
+        'error': error,
+        'screenshot': test_name + '.png'}
+    json.dump(test_dict, open(file_path, 'w'))
+
+
+class SingleTestData(object):
+    def __init__(self, name, testruns, testrail_case_id):
+        self.testrail_case_id = testrail_case_id
+        self.name = name
+        self.testruns = testruns
+
+    class TestRunData(object):
+        def __init__(self, steps, error):
+            self.steps = steps
+            self.error = error
+
+
+class BaseReportDesktop:
+
+    def __init__(self, test_report_dir_master):
+        self.test_reports_dir = test_report_dir_master
+
+    def init_report(self):
+        # overridden in order to prevent deletion of all actual report files, do not remove!
+        pass
+
+    def get_all_tests(self):
+        tests = list()
+        file_list = [f for f in os.listdir(self.test_reports_dir)]
+        for file_name in file_list:
+            if file_name.endswith('json'):
+                file_path = os.path.join(self.test_reports_dir, file_name)
+                test_data = json.load(open(file_path))
+                testruns = list()
+                testruns.append(SingleTestData.TestRunData(
+                    steps=test_data['steps'], error=test_data['error']))
+                tests.append(SingleTestData(name=test_data['name'], testruns=testruns,
+                                            testrail_case_id=test_data['testrail_case_id']))
+        return tests
+
+
+class TestrailReportDesktop(BaseReportDesktop):
+
+    def __init__(self, test_report_dir_master):
+        super(TestrailReportDesktop, self).__init__(test_report_dir_master)
+
+        self.password = os.environ.get('TESTRAIL_PASS')
+        self.user = os.environ.get('TESTRAIL_USER')
+
+        self.outcomes = {
+            'passed': 1,
+            'undefined_fail': 10}
+
+        self.headers = dict()
+        self.headers['Authorization'] = 'Basic %s' % str(
+            base64.b64encode(bytes('%s:%s' % (self.user, self.password), 'utf-8')), 'ascii').strip()
+        self.headers['Content-Type'] = 'application/json'
+
+        self.url = 'https://ethstatus.testrail.net/index.php?/'
+        self.api_url = self.url + 'api/v2/'
+
+        self.run_id = 1917
+        self.suite_id = 52
+        self.project_id = 16
+
+    def add_run(self, run_name):
+        request_body = {'suite_id': self.suite_id,
+                        'name': run_name,
+                        'case_ids': self.get_regression_cases(is_pr='PR-' in run_name),
+                        'include_all': False}
+        run = self.post('add_run/%s' % self.project_id, request_body)
+        self.run_id = run['id']
+
+    def get(self, method):
+        return requests.get(self.api_url + method, headers=self.headers).json()
+
+    def post(self, method, data):
+        data = bytes(json.dumps(data), 'utf-8')
+        return requests.post(self.api_url + method, data=data, headers=self.headers).json()
+
+    def get_cases(self, section_id):
+        return self.get('get_cases/%s&suite_id=%s&section_id=%s' % (self.project_id, self.suite_id, section_id))
+
+    def get_regression_cases(self, is_pr=False):
+        test_cases = dict()
+        test_cases['critical'] = 822
+        test_cases['high'] = 799
+        test_cases['medium'] = 823
+        test_cases['low'] = 801
+        case_ids = list()
+        if is_pr:
+            case_ids = [case['id'] for case in self.get_cases(test_cases['critical'])]
+        else:
+            for phase in test_cases:
+                for case in self.get_cases(test_cases[phase]):
+                    case_ids.append(case['id'])
+        return case_ids
+
+    def get_screenshot_url(self, build_num, test_name):
+        jenkins_url = 'https://jenkins.status.im'
+        job_name = 'desktop-sandbox'
+        artifacts = '/job/end-to-end-tests/job/%s/%s/artifact/test/desktop_sikuli/report/' % (job_name, build_num)
+        return jenkins_url + artifacts + test_name + '.png'
+
+    def add_results(self, build_num):
+        for test in self.get_all_tests():
+            test_steps = "\n# Steps: \n"
+            method = 'add_result_for_case/%s/%s' % (self.run_id, test.testrail_case_id)
+            last_testrun = test.testruns[-1]
+            for step in last_testrun.steps:
+                test_steps += step + "\n"
+            data = {'status_id': self.outcomes['undefined_fail'] if last_testrun.error else self.outcomes['passed'],
+                    'comment': '%s' % ('# Error: \n %s \n# Screenshot:\n "![](%s)"' % (
+                        last_testrun.error + test_steps, self.get_screenshot_url(build_num, test.name))
+                    if last_testrun.error else test_steps)}
+            self.post(method, data=data)

test/desktop_sikuli/tests/test_chats.py

@@ -1,10 +1,12 @@
 from tests import base_user
 from tests.base_test_case import BaseTestCase
 from views.sign_in_view import SignInView
+import pytest
 
 
 class TestChats(BaseTestCase):
 
+    @pytest.mark.testrail_id(5618)
     def test_start_new_chat(self):
         sign_in = SignInView()
         home = sign_in.create_account()

test/desktop_sikuli/tests/test_create_account.py

@@ -1,9 +1,11 @@
 from tests.base_test_case import BaseTestCase
 from views.sign_in_view import SignInView
+import pytest
 
 
 class TestCreateAccount(BaseTestCase):
 
+    @pytest.mark.testrail_id(5565)
     def test_create_account(self):
         sign_in = SignInView()
         sign_in.create_account_button.click()
@@ -15,6 +17,7 @@ class TestCreateAccount(BaseTestCase):
         sign_in.next_button.click()
         sign_in.home_button.find_element()
 
+    @pytest.mark.testrail_id(5570)
     def test_create_account_proceed_with_enter(self):
         sign_in = SignInView()
         sign_in.create_account_button.click()
@@ -26,6 +29,7 @@ class TestCreateAccount(BaseTestCase):
         sign_in.press_enter()
         sign_in.home_button.find_element()
 
+    @pytest.mark.testrail_id(5568)
     def test_create_account_go_back(self):
         sign_in = SignInView()
         sign_in.create_account_button.click()
@@ -44,6 +48,7 @@ class TestCreateAccount(BaseTestCase):
         sign_in.username_input.find_element()
         sign_in.back_button.verify_element_is_not_present()
 
+    @pytest.mark.testrail_id(5567)
     def test_switch_accounts(self):
         sign_in = SignInView()
         sign_in.create_account(username='user_1')

test/desktop_sikuli/tests/test_profile.py

@@ -1,8 +1,7 @@
-import pytest
-
 from tests import base_user
 from tests.base_test_case import BaseTestCase
 from views.sign_in_view import SignInView
+import pytest
 
 
 class TestProfile(BaseTestCase):

test/desktop_sikuli/views/base_element.py

@@ -2,9 +2,11 @@ import logging
 import time
 import pytest
 import re
-
-import org.sikuli.script.SikulixForJython
-from sikuli import *
+try:
+    import org.sikuli.script.SikulixForJython
+    from sikuli import *
+except Exception:
+    pass
 
 
 class BaseElement(object):

test/desktop_sikuli/views/base_view.py

@@ -1,6 +1,9 @@
 import logging
-import org.sikuli.script.SikulixForJython
-from sikuli import *
+try:
+    import org.sikuli.script.SikulixForJython
+    from sikuli import *
+except Exception:
+    pass
 import os
 import pytest
 

test/desktop_sikuli/views/profile_view.py

@@ -1,7 +1,11 @@
-import org.sikuli.script.SikulixForJython
-from sikuli import *
 import os
 
+try:
+    import org.sikuli.script.SikulixForJython
+    from sikuli import *
+except Exception:
+    pass
+
 from views.base_element import BaseElement, TextElement
 from views.base_view import BaseView
 

test/desktop_sikuli/views/sign_in_view.py

@@ -1,10 +1,10 @@
-import org.sikuli.script.SikulixForJython
-import pytest
-from sikuli import *
+try:
+    import org.sikuli.script.SikulixForJython
+    from sikuli import *
+except Exception:
+    pass
 import os
-
-from _pytest.runner import Failed
-
+import pytest
 from views.base_element import BaseElement, InputField
 from views.base_view import BaseView
 from views.home_view import HomeView
@@ -17,6 +17,7 @@ class CreateAccountButton(BaseElement):
         super(CreateAccountButton, self).__init__(IMAGES_PATH + '/create_account.png')
 
     def find_element(self, log=False):
+        from _pytest.runner import Failed
         try:
             super(CreateAccountButton, self).find_element(log=log)
         except Failed: