From 21f30f2ee68a05593de0f9645f67f942febec7b6 Mon Sep 17 00:00:00 2001 From: Anastasiya Semenkevich Date: Wed, 20 Dec 2023 16:12:45 +0300 Subject: [PATCH] chore: update marks and readMe file Fixes https://github.com/status-im/desktop-qa-automation/issues/391 1. replaced all the skip marks for tests that are linked to status-desktop tickets to xfail 2. updated readme to explain marks --- README.md | 27 +++++++++++++++++++ pytest.ini | 2 -- tests/messages/test_messaging_group_chat.py | 2 +- tests/onboarding/test_onboarding_syncing.py | 2 +- ...edirect_to_keycard_settings_from_wallet.py | 2 +- .../test_wallet_settings_account_order.py | 2 +- ...s_watched_addr_include_in_total_balance.py | 2 +- tests/settings/test_ens_name_purchase.py | 2 +- ...t_plus_button_add_account_after_restart.py | 2 +- .../test_plus_button_add_watched_address.py | 2 +- ..._account_added_for_imported_seed_phrase.py | 2 +- ..._button_manage_account_from_private_key.py | 2 +- ..._button_manage_account_from_seed_phrase.py | 2 +- ...st_plus_button_manage_generated_account.py | 2 +- ...enerated_account_custom_derivation_path.py | 2 +- .../test_saved_addresses.py | 2 +- tests/wallet_main_screen/test_send.py | 2 +- 17 files changed, 42 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 42cf613..c664cc1 100644 --- a/README.md +++ b/README.md @@ -21,3 +21,30 @@ for example `AUT_PATH = "/Users/anastasiya/status-desktop/bin/nim_status_client" **2.2** Mac **requires entitlements** for Squish which we don't add by default, so please go here https://ci.status.im/job/status-desktop/job/systems/job/macos/ and select architecture you need (arm or intel), click Build with parameters and select Squish entitlements. Select a branch if u like (master is default) ![img_1.png](img_1.png) + +## Pytest marks used + +You can run tests by mark, just use it like this in command line: + +```bash +python3 -m pytest -m critical +``` + +or directly in pycharm terminal: + +```bash +pytest -m critical +``` + +You can obtain the list of all marks we have by running this `pytest --markers` + +- `critical`, mark used to select the most important checks we do for PRs in desktop repository +(the same for our repo PRs) +- `xfail`, used to link tests to existing tickets in desktop, so if test fails it will be marked as +expected to fail in report with a reference to the ticket. At the same time, if such test is passing, +it will be shown as XPASS (unexpectedly passing) in report, which will indicate the initial bug is gone +- `skip`, used to just skip tests for various reasons, normally with a ticket linked +- `flaky`, used to mark the tests that are normally passing but sometimes fail. If such test passes, then +if will be shown as passed in report normally. If the test fails, then the total run wont be failed, but +the corresponding test will be marked as `xfail` in the report. It is done for a few tests that are not super +stable yet, but passes most of the time. This mark should be used with caution and in case of real need only. diff --git a/pytest.ini b/pytest.ini index b6542e4..61a9b15 100644 --- a/pytest.ini +++ b/pytest.ini @@ -8,8 +8,6 @@ log_cli_format = %(asctime)s.%(msecs)03d %(levelname)7s CLI %(name)s %(message) addopts = --disable-warnings -p no:logging markers = - smoke: Smoke tests - self: Framework self-tests critical: Critical checks for every PR settings_all: All tests related to the settings settings_keycard: Tests related to Keycard settings diff --git a/tests/messages/test_messaging_group_chat.py b/tests/messages/test_messaging_group_chat.py index b97cef6..7640238 100644 --- a/tests/messages/test_messaging_group_chat.py +++ b/tests/messages/test_messaging_group_chat.py @@ -18,7 +18,7 @@ pytestmark = marks (configs.testpath.TEST_USER_DATA / 'user_account_one', configs.testpath.TEST_USER_DATA / 'user_account_two', configs.testpath.TEST_USER_DATA / 'user_account_two') ]) -@pytest.mark.skip(reason="https://github.com/status-im/status-desktop/issues/12440") +@pytest.mark.xfail(reason="https://github.com/status-im/status-desktop/issues/12440") def test_group_chat(multiple_instance, user_data_one, user_data_two, user_data_three): user_one: UserAccount = constants.user_account_one user_two: UserAccount = constants.user_account_two diff --git a/tests/onboarding/test_onboarding_syncing.py b/tests/onboarding/test_onboarding_syncing.py index 78b0dc0..38974b3 100644 --- a/tests/onboarding/test_onboarding_syncing.py +++ b/tests/onboarding/test_onboarding_syncing.py @@ -35,7 +35,7 @@ def sync_screen(main_window) -> SyncCodeView: @allure.testcase('https://ethstatus.testrail.net/index.php?/cases/view/703592', 'Sync device during onboarding') @pytest.mark.case(703592) @pytest.mark.parametrize('user_data', [configs.testpath.TEST_USER_DATA / 'user_account_one']) -@pytest.mark.skip(reason='https://github.com/status-im/status-desktop/issues/12972') +@pytest.mark.xfail(reason='https://github.com/status-im/status-desktop/issues/12972') def test_sync_device_during_onboarding(multiple_instance, user_data): user: UserAccount = constants.user_account_one main_window = MainWindow() diff --git a/tests/settings/settings_keycard/test_redirect_to_keycard_settings_from_wallet.py b/tests/settings/settings_keycard/test_redirect_to_keycard_settings_from_wallet.py index bfd2062..14bb673 100644 --- a/tests/settings/settings_keycard/test_redirect_to_keycard_settings_from_wallet.py +++ b/tests/settings/settings_keycard/test_redirect_to_keycard_settings_from_wallet.py @@ -12,7 +12,7 @@ pytestmark = marks @allure.testcase('https://ethstatus.testrail.net/index.php?/cases/view/703514', 'Choosing Use Keycard when adding account') @pytest.mark.case(703514) -@pytest.mark.skip(reason="https://github.com/status-im/status-desktop/issues/12914") +@pytest.mark.xfail(reason="https://github.com/status-im/status-desktop/issues/12914") def test_use_keycard_when_adding_account(main_screen: MainWindow): with step('Choose continue in keycard settings'): wallet = main_screen.left_panel.open_wallet() diff --git a/tests/settings/settings_wallet/test_wallet_settings_account_order.py b/tests/settings/settings_wallet/test_wallet_settings_account_order.py index 84b5cea..4be353f 100644 --- a/tests/settings/settings_wallet/test_wallet_settings_account_order.py +++ b/tests/settings/settings_wallet/test_wallet_settings_account_order.py @@ -20,7 +20,7 @@ pytestmark = marks pytest.param('0xea123F7beFF45E3C9fdF54B324c29DBdA14a639A', 'Status account', 'WatchOnly', '#2a4af5', 'sunglasses', '😎 ', 'Generated', '#216266', 'thumbsup', '👍 ') ]) -@pytest.mark.skip(reason="https://github.com/status-im/status-desktop/issues/12777") +@pytest.mark.xfail(reason="https://github.com/status-im/status-desktop/issues/12777") def test_change_account_order_by_drag_and_drop(main_screen: MainWindow, user_account, address: str, default_name, name: str, color: str, emoji: str, acc_emoji: str, second_name: str, second_color: str, second_emoji: str, second_acc_emoji: str): diff --git a/tests/settings/settings_wallet/test_wallet_settings_watched_addr_include_in_total_balance.py b/tests/settings/settings_wallet/test_wallet_settings_watched_addr_include_in_total_balance.py index e4ad3dd..9adb264 100644 --- a/tests/settings/settings_wallet/test_wallet_settings_watched_addr_include_in_total_balance.py +++ b/tests/settings/settings_wallet/test_wallet_settings_watched_addr_include_in_total_balance.py @@ -21,7 +21,7 @@ pytestmark = marks pytest.param('0x7f1502605A2f2Cc01f9f4E7dd55e549954A8cD0C', ''.join(random.choices(string.ascii_letters + string.digits, k=20))) ]) -@pytest.mark.skip(reason="https://github.com/status-im/status-desktop/issues/12987") +@pytest.mark.xfail(reason="https://github.com/status-im/status-desktop/issues/12987") def test_settings_include_in_total_balance(main_screen: MainWindow, name, watched_address): with (step('Open wallet on main screen and check the total balance for new account is 0')): wallet_main_screen = main_screen.left_panel.open_wallet() diff --git a/tests/settings/test_ens_name_purchase.py b/tests/settings/test_ens_name_purchase.py index 5b975c6..6ee34a9 100644 --- a/tests/settings/test_ens_name_purchase.py +++ b/tests/settings/test_ens_name_purchase.py @@ -31,7 +31,7 @@ def keys_screen(main_window) -> KeysView: @pytest.mark.case(704597) @pytest.mark.parametrize('user_account', [constants.user.user_with_funds]) @pytest.mark.parametrize('ens_name', [pytest.param(constants.user.ens_user_name)]) -@pytest.mark.skip(reason='https://github.com/status-im/status-desktop/issues/12988') +@pytest.mark.xfail(reason='https://github.com/status-im/status-desktop/issues/12988') def test_ens_name_purchase(keys_screen, main_window, user_account, ens_name): with step('Open import seed phrase view and enter seed phrase'): input_view = keys_screen.open_import_seed_phrase_view().open_seed_phrase_input_view() diff --git a/tests/wallet_main_screen/test_plus_button_add_account_after_restart.py b/tests/wallet_main_screen/test_plus_button_add_account_after_restart.py index bba659c..219dc32 100644 --- a/tests/wallet_main_screen/test_plus_button_add_account_after_restart.py +++ b/tests/wallet_main_screen/test_plus_button_add_account_after_restart.py @@ -25,7 +25,7 @@ pytestmark = marks [ pytest.param('GenAcc2', '#2a4af5', 'sunglasses', '1f60e') ]) -@pytest.mark.skip(reason='https://github.com/status-im/status-desktop/issues/12973') +@pytest.mark.xfail(reason='https://github.com/status-im/status-desktop/issues/12973') def test_add_generated_account_restart_add_again( aut: AUT, main_screen: MainWindow, user_account, color: str, emoji: str, emoji_unicode: str, name: str, diff --git a/tests/wallet_main_screen/test_plus_button_add_watched_address.py b/tests/wallet_main_screen/test_plus_button_add_watched_address.py index 90ba8f0..6815cb8 100644 --- a/tests/wallet_main_screen/test_plus_button_add_watched_address.py +++ b/tests/wallet_main_screen/test_plus_button_add_watched_address.py @@ -18,7 +18,7 @@ pytestmark = marks pytest.param('0xea123F7beFF45E3C9fdF54B324c29DBdA14a639A', 'AccWatch1', '#2a4af5', 'sunglasses', '1f60e') ]) -@pytest.mark.skip(reason="https://github.com/status-im/status-desktop/issues/12914") +@pytest.mark.xfail(reason="https://github.com/status-im/status-desktop/issues/12914") def test_plus_button_add_watched_address( main_screen: MainWindow, address: str, color: str, emoji: str, emoji_unicode: str, name: str): diff --git a/tests/wallet_main_screen/test_plus_button_manage_account_added_for_imported_seed_phrase.py b/tests/wallet_main_screen/test_plus_button_manage_account_added_for_imported_seed_phrase.py index d740f3b..0668a9a 100644 --- a/tests/wallet_main_screen/test_plus_button_manage_account_added_for_imported_seed_phrase.py +++ b/tests/wallet_main_screen/test_plus_button_manage_account_added_for_imported_seed_phrase.py @@ -21,7 +21,7 @@ pytestmark = marks 'new_name, new_color, new_emoji, new_emoji_unicode, keypair_name', [ pytest.param('SPAcc', '#2a4af5', 'sunglasses', '1f60e', 'SPAccedited', '#216266', 'thumbsup', '1f44d', 'SPKeyPair')]) -@pytest.mark.skip(reason="https://github.com/status-im/status-desktop/issues/12914") +@pytest.mark.xfail(reason="https://github.com/status-im/status-desktop/issues/12914") def test_plus_button_manage_account_added_for_imported_seed_phrase(main_screen: MainWindow, user_account, name: str, color: str, emoji: str, emoji_unicode: str, new_name: str, new_color: str, new_emoji: str, new_emoji_unicode: str, diff --git a/tests/wallet_main_screen/test_plus_button_manage_account_from_private_key.py b/tests/wallet_main_screen/test_plus_button_manage_account_from_private_key.py index 7b73378..f7d94a2 100644 --- a/tests/wallet_main_screen/test_plus_button_manage_account_from_private_key.py +++ b/tests/wallet_main_screen/test_plus_button_manage_account_from_private_key.py @@ -22,7 +22,7 @@ pytestmark = marks pytest.param('PrivKeyAcc1', '#2a4af5', 'sunglasses', '1f60e', 'PrivKeyAcc1edited', '#216266', 'thumbsup', '1f44d') ]) -@pytest.mark.skip(reason="https://github.com/status-im/status-desktop/issues/12914") +@pytest.mark.xfail(reason="https://github.com/status-im/status-desktop/issues/12914") def test_plus_button_manage_account_from_private_key(main_screen: MainWindow, user_account, address_pair, name: str, color: str, emoji: str, emoji_unicode: str, new_name: str, new_color: str, new_emoji: str, new_emoji_unicode: str): diff --git a/tests/wallet_main_screen/test_plus_button_manage_account_from_seed_phrase.py b/tests/wallet_main_screen/test_plus_button_manage_account_from_seed_phrase.py index 770823b..d1ae100 100644 --- a/tests/wallet_main_screen/test_plus_button_manage_account_from_seed_phrase.py +++ b/tests/wallet_main_screen/test_plus_button_manage_account_from_seed_phrase.py @@ -30,7 +30,7 @@ pytestmark = marks 'SPAcc12edited', '#216266', 'thumbsup', '1f44d', 'pelican chief sudden oval media rare swamp elephant lawsuit wheat knife initial') ]) -@pytest.mark.skip(reason="https://github.com/status-im/status-desktop/issues/12914") +@pytest.mark.xfail(reason="https://github.com/status-im/status-desktop/issues/12914") def test_plus_button_manage_account_from_seed_phrase(main_screen: MainWindow, user_account, name: str, color: str, emoji: str, emoji_unicode: str, new_name: str, new_color: str, new_emoji: str, new_emoji_unicode: str, diff --git a/tests/wallet_main_screen/test_plus_button_manage_generated_account.py b/tests/wallet_main_screen/test_plus_button_manage_generated_account.py index 72bb2c4..d4c949f 100644 --- a/tests/wallet_main_screen/test_plus_button_manage_generated_account.py +++ b/tests/wallet_main_screen/test_plus_button_manage_generated_account.py @@ -21,7 +21,7 @@ pytestmark = marks pytest.param('GenAcc1', '#2a4af5', 'sunglasses', '1f60e', 'GenAcc1edited', '#216266', 'thumbsup', '1f44d') ]) -@pytest.mark.skip(reason="https://github.com/status-im/status-desktop/issues/12914") +@pytest.mark.xfail(reason="https://github.com/status-im/status-desktop/issues/12914") def test_plus_button_manage_generated_account(main_screen: MainWindow, user_account, color: str, emoji: str, emoji_unicode: str, name: str, new_name: str, new_color: str, new_emoji: str, new_emoji_unicode: str): diff --git a/tests/wallet_main_screen/test_plus_button_manage_generated_account_custom_derivation_path.py b/tests/wallet_main_screen/test_plus_button_manage_generated_account_custom_derivation_path.py index 93be40f..3fc2099 100644 --- a/tests/wallet_main_screen/test_plus_button_manage_generated_account_custom_derivation_path.py +++ b/tests/wallet_main_screen/test_plus_button_manage_generated_account_custom_derivation_path.py @@ -23,7 +23,7 @@ pytestmark = marks pytest.param('N/A', '95', 'Custom path', '#216266', 'sunglasses', '1f60e') ]) @pytest.mark.skip(reason='https://github.com/status-im/desktop-qa-automation/issues/220') -@pytest.mark.skip(reason="https://github.com/status-im/status-desktop/issues/12914") +@pytest.mark.xfail(reason="https://github.com/status-im/status-desktop/issues/12914") def test_plus_button_manage_generated_account_custom_derivation_path(main_screen: MainWindow, user_account, derivation_path: str, generated_address_index: int, name: str, color: str, emoji: str, emoji_unicode: str): diff --git a/tests/wallet_main_screen/test_saved_addresses.py b/tests/wallet_main_screen/test_saved_addresses.py index e933114..0bdabed 100644 --- a/tests/wallet_main_screen/test_saved_addresses.py +++ b/tests/wallet_main_screen/test_saved_addresses.py @@ -15,7 +15,7 @@ pytestmark = marks pytest.param('Saved address name before', '0x8397bc3c5a60a1883174f722403d63a8833312b7', 'Saved address name after'), pytest.param('Ens name before', 'nastya.stateofus.eth', 'Ens name after') ]) -@pytest.mark.skip(reason="https://github.com/status-im/status-desktop/issues/12914") +@pytest.mark.xfail(reason="https://github.com/status-im/status-desktop/issues/12914") def test_manage_saved_address(main_screen: MainWindow, name: str, address: str, new_name: str): with step('Add new address'): wallet = main_screen.left_panel.open_wallet() diff --git a/tests/wallet_main_screen/test_send.py b/tests/wallet_main_screen/test_send.py index 4e42145..32b277f 100644 --- a/tests/wallet_main_screen/test_send.py +++ b/tests/wallet_main_screen/test_send.py @@ -31,7 +31,7 @@ def keys_screen(main_window) -> KeysView: @pytest.mark.parametrize('receiver_account_address, amount, asset', [ pytest.param(constants.user.user_account_one.status_address, 0, 'Ether') ]) -@pytest.mark.skip(reason="https://github.com/status-im/status-desktop/issues/12987") +@pytest.mark.xfail(reason="https://github.com/status-im/status-desktop/issues/12987") def test_wallet_send_0_eth(keys_screen, main_window, user_account, receiver_account_address, amount, asset): with step('Open import seed phrase view and enter seed phrase'): input_view = keys_screen.open_import_seed_phrase_view().open_seed_phrase_input_view()