chore: update marks and readMe file

Fixes https://github.com/status-im/desktop-qa-automation/issues/391

1. replaced all the skip marks for tests that are linked to status-desktop tickets to xfail
2. updated readme to explain marks

test/e2e/README.md

@@ -21,3 +21,30 @@ for example `AUT_PATH = "/Users/anastasiya/status-desktop/bin/nim_status_client"
     **2.2** Mac **requires entitlements**  for Squish which we don't add by default, so please go here https://ci.status.im/job/status-desktop/job/systems/job/macos/
 and select architecture you need (arm or intel), click Build with parameters and select Squish entitlements. Select a branch if u like (master is default)
     ![img_1.png](img_1.png)
+
+## Pytest marks used
+
+You can run tests by mark, just use it like this in command line:
+
+```bash
+python3 -m pytest -m critical
+```
+
+or directly in pycharm terminal:
+
+```bash
+pytest -m critical
+```
+
+You can obtain the list of all marks we have by running this `pytest --markers`
+
+- `critical`, mark used to select the most important checks we do for PRs in desktop repository 
+(the same for our repo PRs)
+- `xfail`, used to link tests to existing tickets in desktop, so if test fails it will be marked as
+expected to fail in report with a reference to the ticket. At the same time, if such test is passing,
+it will be shown as XPASS (unexpectedly passing) in report, which will indicate the initial bug is gone
+- `skip`, used to just skip tests for various reasons, normally with a ticket linked
+- `flaky`, used to mark the tests that are normally passing but sometimes fail. If such test passes, then
+if will be shown as passed in report normally. If the test fails, then the total run wont be failed, but 
+the corresponding test will be marked as `xfail` in the report. It is done for a few tests that are not super
+stable yet, but passes most of the time. This mark should be used with caution and in case of real need only.

test/e2e/pytest.ini

@@ -8,8 +8,6 @@ log_cli_format = %(asctime)s.%(msecs)03d %(levelname)7s CLI %(name)s  %(message)
 addopts = --disable-warnings -p no:logging
 
 markers =
-    smoke: Smoke tests
-    self: Framework self-tests
     critical: Critical checks for every PR
     settings_all: All tests related to the settings
     settings_keycard: Tests related to Keycard settings

test/e2e/tests/messages/test_messaging_group_chat.py

@@ -18,7 +18,7 @@ pytestmark = marks
     (configs.testpath.TEST_USER_DATA / 'user_account_one', configs.testpath.TEST_USER_DATA / 'user_account_two',
      configs.testpath.TEST_USER_DATA / 'user_account_two')
 ])
-@pytest.mark.skip(reason="https://github.com/status-im/status-desktop/issues/12440")
+@pytest.mark.xfail(reason="https://github.com/status-im/status-desktop/issues/12440")
 def test_group_chat(multiple_instance, user_data_one, user_data_two, user_data_three):
     user_one: UserAccount = constants.user_account_one
     user_two: UserAccount = constants.user_account_two

test/e2e/tests/onboarding/test_onboarding_syncing.py

@@ -35,7 +35,7 @@ def sync_screen(main_window) -> SyncCodeView:
 @allure.testcase('https://ethstatus.testrail.net/index.php?/cases/view/703592', 'Sync device during onboarding')
 @pytest.mark.case(703592)
 @pytest.mark.parametrize('user_data', [configs.testpath.TEST_USER_DATA / 'user_account_one'])
-@pytest.mark.skip(reason='https://github.com/status-im/status-desktop/issues/12972')
+@pytest.mark.xfail(reason='https://github.com/status-im/status-desktop/issues/12972')
 def test_sync_device_during_onboarding(multiple_instance, user_data):
     user: UserAccount = constants.user_account_one
     main_window = MainWindow()

test/e2e/tests/settings/settings_keycard/test_redirect_to_keycard_settings_from_wallet.py

@@ -12,7 +12,7 @@ pytestmark = marks
 @allure.testcase('https://ethstatus.testrail.net/index.php?/cases/view/703514',
                  'Choosing Use Keycard when adding account')
 @pytest.mark.case(703514)
-@pytest.mark.skip(reason="https://github.com/status-im/status-desktop/issues/12914")
+@pytest.mark.xfail(reason="https://github.com/status-im/status-desktop/issues/12914")
 def test_use_keycard_when_adding_account(main_screen: MainWindow):
     with step('Choose continue in keycard settings'):
         wallet = main_screen.left_panel.open_wallet()

test/e2e/tests/settings/settings_wallet/test_wallet_settings_account_order.py

@@ -20,7 +20,7 @@ pytestmark = marks
         pytest.param('0xea123F7beFF45E3C9fdF54B324c29DBdA14a639A', 'Status account',
                      'WatchOnly', '#2a4af5', 'sunglasses', '😎 ', 'Generated', '#216266', 'thumbsup', '👍 ')
     ])
-@pytest.mark.skip(reason="https://github.com/status-im/status-desktop/issues/12777")
+@pytest.mark.xfail(reason="https://github.com/status-im/status-desktop/issues/12777")
 def test_change_account_order_by_drag_and_drop(main_screen: MainWindow, user_account, address: str, default_name,
                                                name: str, color: str, emoji: str, acc_emoji: str, second_name: str,
                                                second_color: str, second_emoji: str, second_acc_emoji: str):

test/e2e/tests/settings/settings_wallet/test_wallet_settings_watched_addr_include_in_total_balance.py

@@ -21,7 +21,7 @@ pytestmark = marks
     pytest.param('0x7f1502605A2f2Cc01f9f4E7dd55e549954A8cD0C', ''.join(random.choices(string.ascii_letters +
                                                                                       string.digits, k=20)))
 ])
-@pytest.mark.skip(reason="https://github.com/status-im/status-desktop/issues/12987")
+@pytest.mark.xfail(reason="https://github.com/status-im/status-desktop/issues/12987")
 def test_settings_include_in_total_balance(main_screen: MainWindow, name, watched_address):
     with (step('Open wallet on main screen and check the total balance for new account is 0')):
         wallet_main_screen = main_screen.left_panel.open_wallet()

test/e2e/tests/settings/test_ens_name_purchase.py

@@ -31,7 +31,7 @@ def keys_screen(main_window) -> KeysView:
 @pytest.mark.case(704597)
 @pytest.mark.parametrize('user_account', [constants.user.user_with_funds])
 @pytest.mark.parametrize('ens_name', [pytest.param(constants.user.ens_user_name)])
-@pytest.mark.skip(reason='https://github.com/status-im/status-desktop/issues/12988')
+@pytest.mark.xfail(reason='https://github.com/status-im/status-desktop/issues/12988')
 def test_ens_name_purchase(keys_screen, main_window, user_account, ens_name):
     with step('Open import seed phrase view and enter seed phrase'):
         input_view = keys_screen.open_import_seed_phrase_view().open_seed_phrase_input_view()

test/e2e/tests/wallet_main_screen/test_plus_button_add_account_after_restart.py

@@ -25,7 +25,7 @@ pytestmark = marks
                          [
                              pytest.param('GenAcc2', '#2a4af5', 'sunglasses', '1f60e')
                          ])
-@pytest.mark.skip(reason='https://github.com/status-im/status-desktop/issues/12973')
+@pytest.mark.xfail(reason='https://github.com/status-im/status-desktop/issues/12973')
 def test_add_generated_account_restart_add_again(
         aut: AUT, main_screen: MainWindow, user_account,
         color: str, emoji: str, emoji_unicode: str, name: str,

test/e2e/tests/wallet_main_screen/test_plus_button_add_watched_address.py

@@ -18,7 +18,7 @@ pytestmark = marks
     pytest.param('0xea123F7beFF45E3C9fdF54B324c29DBdA14a639A', 'AccWatch1', '#2a4af5',
                  'sunglasses', '1f60e')
 ])
-@pytest.mark.skip(reason="https://github.com/status-im/status-desktop/issues/12914")
+@pytest.mark.xfail(reason="https://github.com/status-im/status-desktop/issues/12914")
 def test_plus_button_add_watched_address(
         main_screen: MainWindow, address: str, color: str, emoji: str, emoji_unicode: str,
         name: str):

test/e2e/tests/wallet_main_screen/test_plus_button_manage_account_added_for_imported_seed_phrase.py

@@ -21,7 +21,7 @@ pytestmark = marks
                          'new_name, new_color, new_emoji, new_emoji_unicode, keypair_name', [
                              pytest.param('SPAcc', '#2a4af5', 'sunglasses', '1f60e',
                                           'SPAccedited', '#216266', 'thumbsup', '1f44d', 'SPKeyPair')])
-@pytest.mark.skip(reason="https://github.com/status-im/status-desktop/issues/12914")
+@pytest.mark.xfail(reason="https://github.com/status-im/status-desktop/issues/12914")
 def test_plus_button_manage_account_added_for_imported_seed_phrase(main_screen: MainWindow, user_account,
                                                                    name: str, color: str, emoji: str, emoji_unicode: str,
                                                                    new_name: str, new_color: str, new_emoji: str, new_emoji_unicode: str,

test/e2e/tests/wallet_main_screen/test_plus_button_manage_account_from_private_key.py

@@ -22,7 +22,7 @@ pytestmark = marks
                              pytest.param('PrivKeyAcc1', '#2a4af5', 'sunglasses', '1f60e',
                                           'PrivKeyAcc1edited', '#216266', 'thumbsup', '1f44d')
                          ])
-@pytest.mark.skip(reason="https://github.com/status-im/status-desktop/issues/12914")
+@pytest.mark.xfail(reason="https://github.com/status-im/status-desktop/issues/12914")
 def test_plus_button_manage_account_from_private_key(main_screen: MainWindow, user_account, address_pair,
                                                      name: str, color: str, emoji: str, emoji_unicode: str,
                                                      new_name: str, new_color: str, new_emoji: str, new_emoji_unicode: str):

test/e2e/tests/wallet_main_screen/test_plus_button_manage_account_from_seed_phrase.py

@@ -30,7 +30,7 @@ pytestmark = marks
                                           'SPAcc12edited', '#216266', 'thumbsup', '1f44d',
                                           'pelican chief sudden oval media rare swamp elephant lawsuit wheat knife initial')
                          ])
-@pytest.mark.skip(reason="https://github.com/status-im/status-desktop/issues/12914")
+@pytest.mark.xfail(reason="https://github.com/status-im/status-desktop/issues/12914")
 def test_plus_button_manage_account_from_seed_phrase(main_screen: MainWindow, user_account,
                                                      name: str, color: str, emoji: str, emoji_unicode: str,
                                                      new_name: str, new_color: str, new_emoji: str, new_emoji_unicode: str,

test/e2e/tests/wallet_main_screen/test_plus_button_manage_generated_account.py

@@ -21,7 +21,7 @@ pytestmark = marks
                              pytest.param('GenAcc1', '#2a4af5', 'sunglasses', '1f60e',
                                           'GenAcc1edited', '#216266', 'thumbsup', '1f44d')
                          ])
-@pytest.mark.skip(reason="https://github.com/status-im/status-desktop/issues/12914")
+@pytest.mark.xfail(reason="https://github.com/status-im/status-desktop/issues/12914")
 def test_plus_button_manage_generated_account(main_screen: MainWindow, user_account,
                                               color: str, emoji: str, emoji_unicode: str,
                                               name: str, new_name: str, new_color: str, new_emoji: str, new_emoji_unicode: str):

test/e2e/tests/wallet_main_screen/test_plus_button_manage_generated_account_custom_derivation_path.py

@@ -23,7 +23,7 @@ pytestmark = marks
     pytest.param('N/A', '95', 'Custom path', '#216266', 'sunglasses', '1f60e')
 ])
 @pytest.mark.skip(reason='https://github.com/status-im/desktop-qa-automation/issues/220')
-@pytest.mark.skip(reason="https://github.com/status-im/status-desktop/issues/12914")
+@pytest.mark.xfail(reason="https://github.com/status-im/status-desktop/issues/12914")
 def test_plus_button_manage_generated_account_custom_derivation_path(main_screen: MainWindow, user_account,
                                                                      derivation_path: str, generated_address_index: int,
                                                                      name: str, color: str, emoji: str, emoji_unicode: str):

test/e2e/tests/wallet_main_screen/test_saved_addresses.py

@@ -15,7 +15,7 @@ pytestmark = marks
     pytest.param('Saved address name before', '0x8397bc3c5a60a1883174f722403d63a8833312b7', 'Saved address name after'),
     pytest.param('Ens name before', 'nastya.stateofus.eth', 'Ens name after')
 ])
-@pytest.mark.skip(reason="https://github.com/status-im/status-desktop/issues/12914")
+@pytest.mark.xfail(reason="https://github.com/status-im/status-desktop/issues/12914")
 def test_manage_saved_address(main_screen: MainWindow, name: str, address: str, new_name: str):
     with step('Add new address'):
         wallet = main_screen.left_panel.open_wallet()

test/e2e/tests/wallet_main_screen/test_send.py

@@ -31,7 +31,7 @@ def keys_screen(main_window) -> KeysView:
 @pytest.mark.parametrize('receiver_account_address, amount, asset', [
     pytest.param(constants.user.user_account_one.status_address, 0, 'Ether')
 ])
-@pytest.mark.skip(reason="https://github.com/status-im/status-desktop/issues/12987")
+@pytest.mark.xfail(reason="https://github.com/status-im/status-desktop/issues/12987")
 def test_wallet_send_0_eth(keys_screen, main_window, user_account, receiver_account_address, amount, asset):
     with step('Open import seed phrase view and enter seed phrase'):
         input_view = keys_screen.open_import_seed_phrase_view().open_seed_phrase_input_view()