chore: update marks and readMe file

Fixes https://github.com/status-im/desktop-qa-automation/issues/391 1. replaced all the skip marks for tests that are linked to status-desktop tickets to xfail 2. updated readme to explain marks
2023-12-20 16:12:45 +03:00 · 2023-12-20 16:12:45 +03:00 · ba3626e017
parent 767f388b57
commit ba3626e017
17 changed files with 42 additions and 17 deletions
--- a/test/e2e/README.md
+++ b/test/e2e/README.md
@ -21,3 +21,30 @@ for example `AUT_PATH = "/Users/anastasiya/status-desktop/bin/nim_status_client"
    **2.2** Mac **requires entitlements**  for Squish which we don't add by default, so please go here https://ci.status.im/job/status-desktop/job/systems/job/macos/
 and select architecture you need (arm or intel), click Build with parameters and select Squish entitlements. Select a branch if u like (master is default)
    ![img_1.png](img_1.png)
+
+## Pytest marks used
+
+You can run tests by mark, just use it like this in command line:
+
+```bash
+python3 -m pytest -m critical
+```
+
+or directly in pycharm terminal:
+
+```bash
+pytest -m critical
+```
+
+You can obtain the list of all marks we have by running this `pytest --markers`
+
+- `critical`, mark used to select the most important checks we do for PRs in desktop repository 
+(the same for our repo PRs)
+- `xfail`, used to link tests to existing tickets in desktop, so if test fails it will be marked as
+expected to fail in report with a reference to the ticket. At the same time, if such test is passing,
+it will be shown as XPASS (unexpectedly passing) in report, which will indicate the initial bug is gone
+- `skip`, used to just skip tests for various reasons, normally with a ticket linked
+- `flaky`, used to mark the tests that are normally passing but sometimes fail. If such test passes, then
+if will be shown as passed in report normally. If the test fails, then the total run wont be failed, but 
+the corresponding test will be marked as `xfail` in the report. It is done for a few tests that are not super
+stable yet, but passes most of the time. This mark should be used with caution and in case of real need only.
--- a/test/e2e/pytest.ini
+++ b/test/e2e/pytest.ini
@ -8,8 +8,6 @@ log_cli_format = %(asctime)s.%(msecs)03d %(levelname)7s CLI %(name)s  %(message)
 addopts = --disable-warnings -p no:logging

 markers =
-    smoke: Smoke tests
-    self: Framework self-tests
    critical: Critical checks for every PR
    settings_all: All tests related to the settings
    settings_keycard: Tests related to Keycard settings
--- a/test/e2e/tests/messages/test_messaging_group_chat.py
+++ b/test/e2e/tests/messages/test_messaging_group_chat.py
@ -18,7 +18,7 @@ pytestmark = marks
    (configs.testpath.TEST_USER_DATA / 'user_account_one', configs.testpath.TEST_USER_DATA / 'user_account_two',
     configs.testpath.TEST_USER_DATA / 'user_account_two')
 ])
-@pytest.mark.skip(reason="https://github.com/status-im/status-desktop/issues/12440")
+@pytest.mark.xfail(reason="https://github.com/status-im/status-desktop/issues/12440")
 def test_group_chat(multiple_instance, user_data_one, user_data_two, user_data_three):
    user_one: UserAccount = constants.user_account_one
    user_two: UserAccount = constants.user_account_two
--- a/test/e2e/tests/onboarding/test_onboarding_syncing.py
+++ b/test/e2e/tests/onboarding/test_onboarding_syncing.py
@ -35,7 +35,7 @@ def sync_screen(main_window) -> SyncCodeView:
@allure.testcase('https://ethstatus.testrail.net/index.php?/cases/view/703592', 'Sync device during onboarding')
@pytest.mark.case(703592)
@pytest.mark.parametrize('user_data', [configs.testpath.TEST_USER_DATA / 'user_account_one'])
-@pytest.mark.skip(reason='https://github.com/status-im/status-desktop/issues/12972')
+@pytest.mark.xfail(reason='https://github.com/status-im/status-desktop/issues/12972')
 def test_sync_device_during_onboarding(multiple_instance, user_data):
    user: UserAccount = constants.user_account_one
    main_window = MainWindow()
--- a/test/e2e/tests/settings/settings_keycard/test_redirect_to_keycard_settings_from_wallet.py
+++ b/test/e2e/tests/settings/settings_keycard/test_redirect_to_keycard_settings_from_wallet.py
@ -12,7 +12,7 @@ pytestmark = marks
@allure.testcase('https://ethstatus.testrail.net/index.php?/cases/view/703514',
                 'Choosing Use Keycard when adding account')
@pytest.mark.case(703514)
-@pytest.mark.skip(reason="https://github.com/status-im/status-desktop/issues/12914")
+@pytest.mark.xfail(reason="https://github.com/status-im/status-desktop/issues/12914")
 def test_use_keycard_when_adding_account(main_screen: MainWindow):
    with step('Choose continue in keycard settings'):
        wallet = main_screen.left_panel.open_wallet()
--- a/test/e2e/tests/settings/settings_wallet/test_wallet_settings_account_order.py
+++ b/test/e2e/tests/settings/settings_wallet/test_wallet_settings_account_order.py
@ -20,7 +20,7 @@ pytestmark = marks
        pytest.param('0xea123F7beFF45E3C9fdF54B324c29DBdA14a639A', 'Status account',
                     'WatchOnly', '#2a4af5', 'sunglasses', '😎 ', 'Generated', '#216266', 'thumbsup', '👍 ')
    ])
-@pytest.mark.skip(reason="https://github.com/status-im/status-desktop/issues/12777")
+@pytest.mark.xfail(reason="https://github.com/status-im/status-desktop/issues/12777")
 def test_change_account_order_by_drag_and_drop(main_screen: MainWindow, user_account, address: str, default_name,
                                               name: str, color: str, emoji: str, acc_emoji: str, second_name: str,
                                               second_color: str, second_emoji: str, second_acc_emoji: str):
--- a/test/e2e/tests/settings/settings_wallet/test_wallet_settings_watched_addr_include_in_total_balance.py
+++ b/test/e2e/tests/settings/settings_wallet/test_wallet_settings_watched_addr_include_in_total_balance.py
@ -21,7 +21,7 @@ pytestmark = marks
    pytest.param('0x7f1502605A2f2Cc01f9f4E7dd55e549954A8cD0C', ''.join(random.choices(string.ascii_letters +
                                                                                      string.digits, k=20)))
 ])
-@pytest.mark.skip(reason="https://github.com/status-im/status-desktop/issues/12987")
+@pytest.mark.xfail(reason="https://github.com/status-im/status-desktop/issues/12987")
 def test_settings_include_in_total_balance(main_screen: MainWindow, name, watched_address):
    with (step('Open wallet on main screen and check the total balance for new account is 0')):
        wallet_main_screen = main_screen.left_panel.open_wallet()
--- a/test/e2e/tests/settings/test_ens_name_purchase.py
+++ b/test/e2e/tests/settings/test_ens_name_purchase.py
@ -31,7 +31,7 @@ def keys_screen(main_window) -> KeysView:
@pytest.mark.case(704597)
@pytest.mark.parametrize('user_account', [constants.user.user_with_funds])
@pytest.mark.parametrize('ens_name', [pytest.param(constants.user.ens_user_name)])
-@pytest.mark.skip(reason='https://github.com/status-im/status-desktop/issues/12988')
+@pytest.mark.xfail(reason='https://github.com/status-im/status-desktop/issues/12988')
 def test_ens_name_purchase(keys_screen, main_window, user_account, ens_name):
    with step('Open import seed phrase view and enter seed phrase'):
        input_view = keys_screen.open_import_seed_phrase_view().open_seed_phrase_input_view()
--- a/test/e2e/tests/wallet_main_screen/test_plus_button_add_account_after_restart.py
+++ b/test/e2e/tests/wallet_main_screen/test_plus_button_add_account_after_restart.py
@ -25,7 +25,7 @@ pytestmark = marks
                         [
                             pytest.param('GenAcc2', '#2a4af5', 'sunglasses', '1f60e')
                         ])
-@pytest.mark.skip(reason='https://github.com/status-im/status-desktop/issues/12973')
+@pytest.mark.xfail(reason='https://github.com/status-im/status-desktop/issues/12973')
 def test_add_generated_account_restart_add_again(
        aut: AUT, main_screen: MainWindow, user_account,
        color: str, emoji: str, emoji_unicode: str, name: str,
--- a/test/e2e/tests/wallet_main_screen/test_plus_button_add_watched_address.py
+++ b/test/e2e/tests/wallet_main_screen/test_plus_button_add_watched_address.py
@ -18,7 +18,7 @@ pytestmark = marks
    pytest.param('0xea123F7beFF45E3C9fdF54B324c29DBdA14a639A', 'AccWatch1', '#2a4af5',
                 'sunglasses', '1f60e')
 ])
-@pytest.mark.skip(reason="https://github.com/status-im/status-desktop/issues/12914")
+@pytest.mark.xfail(reason="https://github.com/status-im/status-desktop/issues/12914")
 def test_plus_button_add_watched_address(
        main_screen: MainWindow, address: str, color: str, emoji: str, emoji_unicode: str,
        name: str):
--- a/test/e2e/tests/wallet_main_screen/test_plus_button_manage_account_added_for_imported_seed_phrase.py
+++ b/test/e2e/tests/wallet_main_screen/test_plus_button_manage_account_added_for_imported_seed_phrase.py
@ -21,7 +21,7 @@ pytestmark = marks
                         'new_name, new_color, new_emoji, new_emoji_unicode, keypair_name', [
                             pytest.param('SPAcc', '#2a4af5', 'sunglasses', '1f60e',
                                          'SPAccedited', '#216266', 'thumbsup', '1f44d', 'SPKeyPair')])
-@pytest.mark.skip(reason="https://github.com/status-im/status-desktop/issues/12914")
+@pytest.mark.xfail(reason="https://github.com/status-im/status-desktop/issues/12914")
 def test_plus_button_manage_account_added_for_imported_seed_phrase(main_screen: MainWindow, user_account,
                                                                   name: str, color: str, emoji: str, emoji_unicode: str,
                                                                   new_name: str, new_color: str, new_emoji: str, new_emoji_unicode: str,
--- a/test/e2e/tests/wallet_main_screen/test_plus_button_manage_account_from_private_key.py
+++ b/test/e2e/tests/wallet_main_screen/test_plus_button_manage_account_from_private_key.py
@ -22,7 +22,7 @@ pytestmark = marks
                             pytest.param('PrivKeyAcc1', '#2a4af5', 'sunglasses', '1f60e',
                                          'PrivKeyAcc1edited', '#216266', 'thumbsup', '1f44d')
                         ])
-@pytest.mark.skip(reason="https://github.com/status-im/status-desktop/issues/12914")
+@pytest.mark.xfail(reason="https://github.com/status-im/status-desktop/issues/12914")
 def test_plus_button_manage_account_from_private_key(main_screen: MainWindow, user_account, address_pair,
                                                     name: str, color: str, emoji: str, emoji_unicode: str,
                                                     new_name: str, new_color: str, new_emoji: str, new_emoji_unicode: str):
--- a/test/e2e/tests/wallet_main_screen/test_plus_button_manage_account_from_seed_phrase.py
+++ b/test/e2e/tests/wallet_main_screen/test_plus_button_manage_account_from_seed_phrase.py
@ -30,7 +30,7 @@ pytestmark = marks
                                          'SPAcc12edited', '#216266', 'thumbsup', '1f44d',
                                          'pelican chief sudden oval media rare swamp elephant lawsuit wheat knife initial')
                         ])
-@pytest.mark.skip(reason="https://github.com/status-im/status-desktop/issues/12914")
+@pytest.mark.xfail(reason="https://github.com/status-im/status-desktop/issues/12914")
 def test_plus_button_manage_account_from_seed_phrase(main_screen: MainWindow, user_account,
                                                     name: str, color: str, emoji: str, emoji_unicode: str,
                                                     new_name: str, new_color: str, new_emoji: str, new_emoji_unicode: str,
--- a/test/e2e/tests/wallet_main_screen/test_plus_button_manage_generated_account.py
+++ b/test/e2e/tests/wallet_main_screen/test_plus_button_manage_generated_account.py
@ -21,7 +21,7 @@ pytestmark = marks
                             pytest.param('GenAcc1', '#2a4af5', 'sunglasses', '1f60e',
                                          'GenAcc1edited', '#216266', 'thumbsup', '1f44d')
                         ])
-@pytest.mark.skip(reason="https://github.com/status-im/status-desktop/issues/12914")
+@pytest.mark.xfail(reason="https://github.com/status-im/status-desktop/issues/12914")
 def test_plus_button_manage_generated_account(main_screen: MainWindow, user_account,
                                              color: str, emoji: str, emoji_unicode: str,
                                              name: str, new_name: str, new_color: str, new_emoji: str, new_emoji_unicode: str):
--- a/test/e2e/tests/wallet_main_screen/test_plus_button_manage_generated_account_custom_derivation_path.py
+++ b/test/e2e/tests/wallet_main_screen/test_plus_button_manage_generated_account_custom_derivation_path.py
@ -23,7 +23,7 @@ pytestmark = marks
    pytest.param('N/A', '95', 'Custom path', '#216266', 'sunglasses', '1f60e')
 ])
@pytest.mark.skip(reason='https://github.com/status-im/desktop-qa-automation/issues/220')
-@pytest.mark.skip(reason="https://github.com/status-im/status-desktop/issues/12914")
+@pytest.mark.xfail(reason="https://github.com/status-im/status-desktop/issues/12914")
 def test_plus_button_manage_generated_account_custom_derivation_path(main_screen: MainWindow, user_account,
                                                                     derivation_path: str, generated_address_index: int,
                                                                     name: str, color: str, emoji: str, emoji_unicode: str):
--- a/test/e2e/tests/wallet_main_screen/test_saved_addresses.py
+++ b/test/e2e/tests/wallet_main_screen/test_saved_addresses.py
@ -15,7 +15,7 @@ pytestmark = marks
    pytest.param('Saved address name before', '0x8397bc3c5a60a1883174f722403d63a8833312b7', 'Saved address name after'),
    pytest.param('Ens name before', 'nastya.stateofus.eth', 'Ens name after')
 ])
-@pytest.mark.skip(reason="https://github.com/status-im/status-desktop/issues/12914")
+@pytest.mark.xfail(reason="https://github.com/status-im/status-desktop/issues/12914")
 def test_manage_saved_address(main_screen: MainWindow, name: str, address: str, new_name: str):
    with step('Add new address'):
        wallet = main_screen.left_panel.open_wallet()
--- a/test/e2e/tests/wallet_main_screen/test_send.py
+++ b/test/e2e/tests/wallet_main_screen/test_send.py
@ -31,7 +31,7 @@ def keys_screen(main_window) -> KeysView:
@pytest.mark.parametrize('receiver_account_address, amount, asset', [
    pytest.param(constants.user.user_account_one.status_address, 0, 'Ether')
 ])
-@pytest.mark.skip(reason="https://github.com/status-im/status-desktop/issues/12987")
+@pytest.mark.xfail(reason="https://github.com/status-im/status-desktop/issues/12987")
 def test_wallet_send_0_eth(keys_screen, main_window, user_account, receiver_account_address, amount, asset):
    with step('Open import seed phrase view and enter seed phrase'):
        input_view = keys_screen.open_import_seed_phrase_view().open_seed_phrase_input_view()