mirror of
https://github.com/status-im/status-app-benchmarks.git
synced 2026-08-27 11:11:13 +00:00
324 lines
14 KiB
TOML
324 lines
14 KiB
TOML
# Benchmark Configuration
|
|
|
|
[defaults]
|
|
slow_threshold_s = 1.0
|
|
fast_threshold_s = 0.5
|
|
ok_near_slow_ratio = 0.10
|
|
regression_pct = 0.15
|
|
regression_consecutive = 3
|
|
rolling_window = 5
|
|
backlog_slow_of_last_n = 5
|
|
backlog_slow_min_count = 3
|
|
baselines = ["5f66de"]
|
|
reference_build = "5f66de"
|
|
|
|
# --- Pages ---
|
|
|
|
[[pages]]
|
|
slug = "wallet-fresh"
|
|
title = "New user profile"
|
|
description = "Newly created user profile (no-preseeded user data)"
|
|
user_data_size = "No pre-seeded data"
|
|
wallet = { accounts = "1", tokens = "0", nfts = "0", transactions = "0" }
|
|
messenger = { direct_chats = "0", group_chats = "0" }
|
|
communities = { joined = "0", spectated = "0" }
|
|
|
|
[[pages]]
|
|
slug = "wallet-wallet-load"
|
|
title = "Returning user (semi-heavy wallet account)"
|
|
description = "Returning user with semi-heavy wallet account (~34 MB user data)."
|
|
user_data_size = "~34 MB"
|
|
wallet = { accounts = "3", tokens = "83", nfts = "166", transactions = "736" }
|
|
messenger = { direct_chats = "0", group_chats = "0" }
|
|
communities = { joined = "0", spectated = "0" }
|
|
|
|
[[pages]]
|
|
slug = "wallet-wallet-load-alex"
|
|
title = "Returning user (heavy account from Alex)"
|
|
description = "Returning user with heavy account from Alex (~35 MB user data)."
|
|
user_data_size = "~35 MB"
|
|
wallet = { accounts = "4", tokens = "144", nfts = "773", transactions = "5239" }
|
|
messenger = { direct_chats = "0", group_chats = "0" }
|
|
communities = { joined = "0", spectated = "0" }
|
|
|
|
[[pages]]
|
|
slug = "community-populated"
|
|
title = "Returning user (Status community member)"
|
|
description = "Returning user with Status community already joined."
|
|
user_data_size = "TBD"
|
|
wallet = { accounts = "1", tokens = "0", nfts = "0", transactions = "0" }
|
|
messenger = { direct_chats = "0", group_chats = "0" }
|
|
communities = { joined = "1", spectated = "0" }
|
|
test_ids = [
|
|
"test_status_community_first_open_loading_time_member",
|
|
"test_status_community_second_open_loading_time_member",
|
|
"test_status_community_first_open_cpu_member",
|
|
"test_status_community_second_open_cpu_member",
|
|
"test_status_community_first_open_ram_member",
|
|
"test_status_community_second_open_ram_member",
|
|
]
|
|
|
|
# --- Wallet profile matrix ---
|
|
|
|
[[wallet_profile_variants]]
|
|
suffix = "fresh"
|
|
param_id = "fresh_user"
|
|
page_slug = "wallet-fresh"
|
|
footnote_prefix = "New user profile"
|
|
|
|
[[wallet_profile_variants]]
|
|
suffix = "wallet_load"
|
|
param_id = "wallet_load_user"
|
|
page_slug = "wallet-wallet-load"
|
|
footnote_prefix = "semi-heavy wallet account"
|
|
|
|
[[wallet_profile_variants]]
|
|
suffix = "wallet_load_alex"
|
|
param_id = "wallet_load_alex_user"
|
|
page_slug = "wallet-wallet-load-alex"
|
|
footnote_prefix = "heavy account (from Alex)"
|
|
|
|
# Each declaration expands to load time, CPU, and RAM charts for all profiles.
|
|
|
|
[[wallet_scenarios]]
|
|
scenario_id = "wallet_first_open"
|
|
display_name = "Time to open Wallet for the first time after login"
|
|
resource_action = "opening Wallet for the first time after login"
|
|
test_pattern = "test_wallet_first_open_loading_time"
|
|
graph_stem = "wallet_first_open"
|
|
footnote = "each point = one first-open sample on that build"
|
|
|
|
[[wallet_scenarios]]
|
|
scenario_id = "wallet_repeat_open"
|
|
display_name = "Time to reopen Wallet in the same session"
|
|
resource_action = "reopening Wallet in the same session"
|
|
test_pattern = "test_wallet_repeat_open_loading_time"
|
|
historical_test_patterns = ["test_wallet_loading_time"]
|
|
graph_stem = "wallet_repeat_open"
|
|
footnote = "each point = average of 5 runs on that build"
|
|
|
|
[[wallet_scenarios]]
|
|
scenario_id = "wallet_account_first_open"
|
|
display_name = "Time to open a Wallet account for the first time in the session"
|
|
resource_action = "opening a Wallet account for the first time in the session"
|
|
test_pattern = "test_wallet_account_open_loading_time"
|
|
graph_stem = "wallet_account_first_open"
|
|
footnote = "each point = one first account-open sample on that build"
|
|
|
|
[[wallet_scenarios]]
|
|
scenario_id = "wallet_add_account_first_open"
|
|
display_name = "Time to open the Add account modal for the first time in the session"
|
|
resource_action = "opening the Add account modal for the first time in the session"
|
|
test_pattern = "test_wallet_add_account_loading_time"
|
|
series_test_pattern = "test_wallet_add_account_first_open_loading_time"
|
|
graph_stem = "wallet_add_account_first_open"
|
|
attachment_subject = "Wallet Add account modal first open"
|
|
footnote = "each point = one first-open sample on that build"
|
|
|
|
[[wallet_scenarios]]
|
|
scenario_id = "wallet_add_account"
|
|
display_name = "Time to reopen the Add account modal in the same session"
|
|
resource_action = "reopening the Add account modal in the same session"
|
|
test_pattern = "test_wallet_add_account_loading_time"
|
|
graph_stem = "wallet_add_account"
|
|
attachment_subject = "Wallet Add account modal reopen"
|
|
footnote = "each point = average of 5 runs on that build"
|
|
|
|
[[wallet_scenarios]]
|
|
scenario_id = "wallet_receive_first_open"
|
|
display_name = "Time to open the Receive modal for the first time in the session"
|
|
resource_action = "opening the Receive modal for the first time in the session"
|
|
test_pattern = "test_wallet_receive_loading_time"
|
|
series_test_pattern = "test_wallet_receive_first_open_loading_time"
|
|
graph_stem = "wallet_receive_first_open"
|
|
attachment_subject = "Wallet Receive modal first open"
|
|
footnote = "each point = one first-open sample on that build"
|
|
|
|
[[wallet_scenarios]]
|
|
scenario_id = "wallet_receive"
|
|
display_name = "Time to reopen the Receive modal in the same session"
|
|
resource_action = "reopening the Receive modal in the same session"
|
|
test_pattern = "test_wallet_receive_loading_time"
|
|
graph_stem = "wallet_receive"
|
|
attachment_subject = "Wallet Receive modal reopen"
|
|
footnote = "each point = average of 5 runs on that build"
|
|
|
|
[[wallet_scenarios]]
|
|
scenario_id = "wallet_send_first_open"
|
|
display_name = "Time to open the Send modal for the first time in the session"
|
|
resource_action = "opening the Send modal for the first time in the session"
|
|
test_pattern = "test_wallet_send_loading_time"
|
|
series_test_pattern = "test_wallet_send_first_open_loading_time"
|
|
graph_stem = "wallet_send_first_open"
|
|
attachment_subject = "Wallet Send modal first open"
|
|
footnote = "each point = one first-open sample on that build"
|
|
|
|
[[wallet_scenarios]]
|
|
scenario_id = "wallet_send"
|
|
display_name = "Time to reopen the Send modal in the same session"
|
|
resource_action = "reopening the Send modal in the same session"
|
|
test_pattern = "test_wallet_send_loading_time"
|
|
graph_stem = "wallet_send"
|
|
attachment_subject = "Wallet Send modal reopen"
|
|
footnote = "each point = average of 5 runs on that build"
|
|
|
|
[[wallet_scenarios]]
|
|
scenario_id = "wallet_swap_first_open"
|
|
display_name = "Time to open the Swap modal for the first time in the session"
|
|
resource_action = "opening the Swap modal for the first time in the session"
|
|
test_pattern = "test_wallet_swap_loading_time"
|
|
series_test_pattern = "test_wallet_swap_first_open_loading_time"
|
|
graph_stem = "wallet_swap_first_open"
|
|
attachment_subject = "Wallet Swap modal first open"
|
|
footnote = "each point = one first-open sample on that build"
|
|
|
|
[[wallet_scenarios]]
|
|
scenario_id = "wallet_swap"
|
|
display_name = "Time to reopen the Swap modal in the same session"
|
|
resource_action = "reopening the Swap modal in the same session"
|
|
test_pattern = "test_wallet_swap_loading_time"
|
|
historical_test_patterns = ["test_swap_loading_time"]
|
|
historical_attachment_subjects = ["Swap modal"]
|
|
graph_stem = "wallet_swap"
|
|
attachment_subject = "Wallet Swap modal reopen"
|
|
footnote = "each point = average of 5 runs on that build"
|
|
|
|
[[wallet_scenarios]]
|
|
scenario_id = "wallet_assets_tab_first_open"
|
|
display_name = "Time to open the Assets tab for the first time in the session"
|
|
resource_action = "opening the Assets tab for the first time in the session"
|
|
test_pattern = "test_wallet_assets_tab_loading_time"
|
|
series_test_pattern = "test_wallet_assets_tab_first_open_loading_time"
|
|
graph_stem = "wallet_assets_tab_first_open"
|
|
attachment_subject = "Wallet Assets tab first open"
|
|
reference_build = false
|
|
footnote = "each point = one first-open sample on that build · 2.38.0 baseline not comparable — test waits for tab content since Jul 2026"
|
|
|
|
[[wallet_scenarios]]
|
|
scenario_id = "wallet_assets_tab"
|
|
display_name = "Time to reopen the Assets tab in the same session"
|
|
resource_action = "reopening the Assets tab in the same session"
|
|
test_pattern = "test_wallet_assets_tab_loading_time"
|
|
graph_stem = "wallet_assets_tab"
|
|
attachment_subject = "Wallet Assets tab reopen"
|
|
reference_build = false
|
|
footnote = "each point = average of 5 runs on that build · 2.38.0 baseline not comparable — test waits for tab content since Jul 2026"
|
|
|
|
[[wallet_scenarios]]
|
|
scenario_id = "wallet_collectibles_tab_first_open"
|
|
display_name = "Time to open the Collectibles tab for the first time in the session"
|
|
resource_action = "opening the Collectibles tab for the first time in the session"
|
|
test_pattern = "test_wallet_collectibles_tab_loading_time"
|
|
series_test_pattern = "test_wallet_collectibles_tab_first_open_loading_time"
|
|
graph_stem = "wallet_collectibles_tab_first_open"
|
|
attachment_subject = "Wallet Collectibles tab first open"
|
|
reference_build = false
|
|
footnote = "each point = one first-open sample on that build · 2.38.0 baseline not comparable — test waits for tab content since Jul 2026"
|
|
|
|
[[wallet_scenarios]]
|
|
scenario_id = "wallet_collectibles_tab"
|
|
display_name = "Time to reopen the Collectibles tab in the same session"
|
|
resource_action = "reopening the Collectibles tab in the same session"
|
|
test_pattern = "test_wallet_collectibles_tab_loading_time"
|
|
graph_stem = "wallet_collectibles_tab"
|
|
attachment_subject = "Wallet Collectibles tab reopen"
|
|
reference_build = false
|
|
footnote = "each point = average of 5 runs on that build · 2.38.0 baseline not comparable — test waits for tab content since Jul 2026"
|
|
|
|
[[wallet_scenarios]]
|
|
scenario_id = "wallet_activity_tab_first_open"
|
|
display_name = "Time to open the History tab for the first time in the session"
|
|
resource_action = "opening the History tab for the first time in the session"
|
|
test_pattern = "test_wallet_activity_tab_loading_time"
|
|
series_test_pattern = "test_wallet_activity_tab_first_open_loading_time"
|
|
graph_stem = "wallet_activity_tab_first_open"
|
|
attachment_subject = "Wallet History tab first open"
|
|
reference_build = false
|
|
footnote = "each point = one first-open sample on that build · 2.38.0 baseline not comparable — test waits for tab content since Jul 2026"
|
|
|
|
[[wallet_scenarios]]
|
|
scenario_id = "wallet_activity_tab"
|
|
display_name = "Time to reopen the History tab in the same session"
|
|
resource_action = "reopening the History tab in the same session"
|
|
test_pattern = "test_wallet_activity_tab_loading_time"
|
|
graph_stem = "wallet_activity_tab"
|
|
attachment_subject = "Wallet History tab reopen"
|
|
reference_build = false
|
|
footnote = "each point = average of 5 runs on that build · 2.38.0 baseline not comparable — test waits for tab content since Jul 2026"
|
|
|
|
# --- Community first open (member only) ---
|
|
|
|
[[tests]]
|
|
test_id = "test_status_community_first_open_loading_time_member"
|
|
area = "communities"
|
|
display_name = "Time to open Status community for the first time after login"
|
|
description = "Lower is better."
|
|
pattern = "test_status_community_first_open_loading_time[user_data0-user_account0]"
|
|
graph_filename = "community_first_open_loading_time_member.png"
|
|
ylabel = "seconds"
|
|
footnote = "status_community_member profile · each point = one first-open sample on that build"
|
|
|
|
[[tests]]
|
|
test_id = "test_status_community_second_open_loading_time_member"
|
|
area = "communities"
|
|
display_name = "Time to reopen Status community in the same session"
|
|
description = "Lower is better."
|
|
pattern = "test_status_community_second_open_loading_time[user_data0-user_account0]"
|
|
graph_filename = "community_second_open_loading_time_member.png"
|
|
ylabel = "seconds"
|
|
footnote = "status_community_member profile · each point = average of 5 runs on that build"
|
|
|
|
[[cpu_tests]]
|
|
test_id = "test_status_community_first_open_cpu_member"
|
|
area = "communities"
|
|
display_name = "CPU usage while opening Status community for the first time after login"
|
|
pattern = "test_status_community_first_open_loading_time[user_data0-user_account0]"
|
|
graph_filename = "community_first_open_cpu_member.png"
|
|
attachment_keyword = "cpu usage"
|
|
ylabel = "CPU Usage (%)"
|
|
footnote = "status_community_member profile · each point = one first-open sample on that build"
|
|
|
|
[[cpu_tests]]
|
|
test_id = "test_status_community_second_open_cpu_member"
|
|
area = "communities"
|
|
display_name = "CPU usage while reopening Status community in the same session"
|
|
pattern = "test_status_community_second_open_loading_time[user_data0-user_account0]"
|
|
graph_filename = "community_second_open_cpu_member.png"
|
|
attachment_keyword = "cpu usage"
|
|
ylabel = "CPU Usage (%)"
|
|
footnote = "status_community_member profile · each point = average of 5 runs on that build"
|
|
|
|
[[ram_tests]]
|
|
test_id = "test_status_community_first_open_ram_member"
|
|
area = "communities"
|
|
display_name = "RAM usage while opening Status community for the first time after login"
|
|
pattern = "test_status_community_first_open_loading_time[user_data0-user_account0]"
|
|
graph_filename = "community_first_open_ram_member.png"
|
|
attachment_keyword = "ram usage"
|
|
ylabel = "RAM Usage (MB)"
|
|
footnote = "status_community_member profile · each point = one first-open sample on that build"
|
|
|
|
[[ram_tests]]
|
|
test_id = "test_status_community_second_open_ram_member"
|
|
area = "communities"
|
|
display_name = "RAM usage while reopening Status community in the same session"
|
|
pattern = "test_status_community_second_open_loading_time[user_data0-user_account0]"
|
|
graph_filename = "community_second_open_ram_member.png"
|
|
attachment_keyword = "ram usage"
|
|
ylabel = "RAM Usage (MB)"
|
|
footnote = "status_community_member profile · each point = average of 5 runs on that build"
|
|
|
|
# --- Flag tickets (shown on Flags report when a test_id is flagged) ---
|
|
|
|
[[flag_tickets]]
|
|
test_id = "test_wallet_collectibles_tab_first_open_time_fresh"
|
|
issue = 21732
|
|
url = "https://github.com/status-im/status-app/issues/21732"
|
|
|
|
[[flag_tickets]]
|
|
test_id = "test_wallet_assets_tab_first_open_time_fresh"
|
|
issue = 21694
|
|
url = "https://github.com/status-im/status-app/issues/21694"
|
|
|
|
# Per-chart baselines/reference_build override [defaults] when set.
|