From 089b8eefce468a039fb1b48e0f1f71a59e95359d Mon Sep 17 00:00:00 2001 From: fbarbu15 Date: Fri, 30 Aug 2024 14:15:45 +0300 Subject: [PATCH] metrics tests (#68) --- src/node/waku_node.py | 5 +- src/steps/metrics.py | 39 +++++++- src/steps/store.py | 3 +- src/test_data.py | 181 +++++++++++++++++++++++++++++++++- tests/metrics/__init__.py | 0 tests/metrics/test_metrics.py | 114 +++++++++++++++++++++ 6 files changed, 337 insertions(+), 5 deletions(-) create mode 100644 tests/metrics/__init__.py create mode 100644 tests/metrics/test_metrics.py diff --git a/src/node/waku_node.py b/src/node/waku_node.py index 551719ce01..6db2033115 100644 --- a/src/node/waku_node.py +++ b/src/node/waku_node.py @@ -302,11 +302,14 @@ class WakuNode: ws_address = next((addr for addr in addresses if "/ws" not in addr), None) if ws_address: identifier = ws_address.split("/p2p/")[-1] - new_address = f"/ip4/{self._ext_ip}/tcp/{self._tcp_port}/p2p/{identifier}" + new_address = f"{self.get_tcp_address()}/p2p/{identifier}" return new_address else: raise AttributeError("No '/ws' address found") + def get_tcp_address(self): + return f"/ip4/{self._ext_ip}/tcp/{self._tcp_port}" + def info(self): return self._api.info() diff --git a/src/steps/metrics.py b/src/steps/metrics.py index d7f73b5d0c..f536869995 100644 --- a/src/steps/metrics.py +++ b/src/steps/metrics.py @@ -1,14 +1,17 @@ +import re from src.libs.custom_logger import get_custom_logger import allure from tenacity import retry, stop_after_delay, wait_fixed +from src.test_data import METRICS_WITH_INITIAL_VALUE_ZERO + logger = get_custom_logger(__name__) class StepsMetrics: @allure.step - def check_metric(self, node, metric_name, expected_value): + def check_metric(self, node, metric_name, expected_value, exact=False): logger.debug(f"Checking metric: {metric_name} has {expected_value}") response = node.get_metrics() lines = response.split("\n") @@ -22,7 +25,10 @@ class StepsMetrics: if actual_value is None: raise AttributeError(f"Metric '{metric_name}' not found") logger.debug(f"Found metric: {metric_name} with value {actual_value}") - assert actual_value == expected_value, f"Expected value for '{metric_name}' is {expected_value}, but got {actual_value}" + if exact: + assert actual_value == expected_value, f"Expected value for '{metric_name}' is {expected_value}, but got {actual_value}" + else: + assert actual_value >= expected_value, f"Expected value for '{metric_name}' is >= {expected_value}, but got {actual_value}" @allure.step def wait_for_metric(self, node, metric_name, expected_value, timeout_duration=90): @@ -31,3 +37,32 @@ class StepsMetrics: self.check_metric(node, metric_name, expected_value) check_metric_with_retry() + + def validate_initial_metrics(self, node): + metrics_data = node.get_metrics() + + # Regular expression to match metric lines, accounting for optional labels + metric_pattern = re.compile(r"^(?P[a-zA-Z0-9_:]+(?:{[^}]+})?)\s+(?P[0-9]+\.?[0-9]*)$", re.MULTILINE) + + # Dictionary to store the metrics and their values + metrics_dict = {} + for match in metric_pattern.finditer(metrics_data): + metric_name = match.group("metric_name") + value = float(match.group("value")) + metrics_dict[metric_name] = value + + errors = [] + # Assert that specific metrics have a value of 0.0 + for metric in METRICS_WITH_INITIAL_VALUE_ZERO: + if metric not in metrics_dict: + errors.append(f"Metric {metric} is missing from the metrics data") + elif metrics_dict[metric] != 0.0: + errors.append(f"Expected {metric} to be 0.0, but got {metrics_dict[metric]}") + + # Assert that all other metrics have a value greater than 0.0 + for metric, value in metrics_dict.items(): + if metric not in METRICS_WITH_INITIAL_VALUE_ZERO and value <= 0.0: + errors.append(f"Expected {metric} to have a positive value, but got {value}") + + assert not errors, f"Metrics validation failed:\n" + "\n".join(errors) + logger.debug(f"All metrics are present and have valid values.") diff --git a/src/steps/store.py b/src/steps/store.py index bebc3b6426..ca18788f0a 100644 --- a/src/steps/store.py +++ b/src/steps/store.py @@ -33,9 +33,10 @@ class StepsStore(StepsCommon): @pytest.fixture(scope="function", autouse=False) def node_setup(self, store_setup): + logger.debug(f"Running fixture setup: {inspect.currentframe().f_code.co_name}") self.setup_first_publishing_node(store="true", relay="true") self.setup_first_store_node(store="true", relay="true") - self.subscribe_to_pubsub_topics_via_relay() + self.subscribe_to_pubsub_topics_via_relay(node=self.main_publishing_nodes) @allure.step def start_publishing_node(self, image, node_index, **kwargs): diff --git a/src/test_data.py b/src/test_data.py index 4536d260d1..745ee7c98f 100644 --- a/src/test_data.py +++ b/src/test_data.py @@ -177,7 +177,6 @@ LOG_ERROR_KEYWORDS = [ "abort", "segfault", "corrupt", - "unreachable", "terminated", "oom", "unhandled", @@ -195,3 +194,183 @@ LOG_ERROR_KEYWORDS = [ "race condition", "double free", ] + +METRICS_WITH_INITIAL_VALUE_ZERO = [ + "libp2p_peers", + "libp2p_failed_upgrades_incoming_total", + "libp2p_failed_upgrades_outgoing_total", + "libp2p_total_dial_attempts_total", + "libp2p_successful_dials_total", + "libp2p_failed_dials_total", + "waku_rln_messages_total_total", + "waku_rln_spam_messages_total_total", + "waku_rln_valid_messages_total_sum", + "waku_rln_valid_messages_total_count", + 'waku_rln_valid_messages_total_bucket{le="1.0"}', + 'waku_rln_valid_messages_total_bucket{le="2.0"}', + 'waku_rln_valid_messages_total_bucket{le="3.0"}', + 'waku_rln_valid_messages_total_bucket{le="4.0"}', + 'waku_rln_valid_messages_total_bucket{le="5.0"}', + 'waku_rln_valid_messages_total_bucket{le="+Inf"}', + "waku_rln_proof_verification_total_total", + "waku_rln_number_registered_memberships", + "waku_rln_proof_verification_duration_seconds", + "waku_rln_proof_generation_duration_seconds", + "waku_rln_instance_creation_duration_seconds", + "waku_rln_membership_insertion_duration_seconds", + "waku_rln_membership_credentials_import_duration_seconds", + "libp2p_pubsub_sig_verify_success_total", + "libp2p_pubsub_sig_verify_failure_total", + "libp2p_pubsub_disconnects_over_non_priority_queue_limit_total", + "libp2p_pubsub_peers", + "libp2p_pubsub_topics", + "libp2p_pubsub_subscriptions_total", + "libp2p_pubsub_unsubscriptions_total", + "libp2p_pubsub_validation_success_total", + "libp2p_pubsub_validation_failure_total", + "libp2p_pubsub_validation_ignore_total", + "libp2p_pubsub_broadcast_iwant_total", + "libp2p_pubsub_received_iwant_total", + "libp2p_gossipsub_cache_window_size", + 'libp2p_gossipsub_peers_per_topic_mesh{topic="other"}', + 'libp2p_gossipsub_peers_per_topic_fanout{topic="other"}', + 'libp2p_gossipsub_peers_per_topic_gossipsub{topic="other"}', + "libp2p_gossipsub_under_dout_topics", + "libp2p_gossipsub_no_peers_topics", + "libp2p_gossipsub_low_peers_topics", + "libp2p_gossipsub_healthy_peers_topics", + "libp2p_gossipsub_failed_publish_total", + "libp2p_gossipsub_invalid_topic_subscription_total", + "libp2p_gossipsub_duplicate_during_validation_total", + "libp2p_gossipsub_idontwant_saved_messages_total", + "libp2p_gossipsub_duplicate_total", + "libp2p_gossipsub_received_total", + "libp2p_rendezvous_register_total", + "libp2p_rendezvous_discover_total", + "libp2p_rendezvous_registered", + "libp2p_rendezvous_namespaces", + "waku_peer_store_size", + "waku_total_unique_peers", + "presto_server_missing_requests_count", + "presto_server_invalid_requests_count", + 'waku_archive_messages{type="stored"}', + "waku_archive_queries", + "waku_archive_insert_duration_seconds_sum", + "waku_archive_insert_duration_seconds_count", + 'waku_archive_insert_duration_seconds_bucket{le="0.005"}', + 'waku_archive_insert_duration_seconds_bucket{le="0.01"}', + 'waku_archive_insert_duration_seconds_bucket{le="0.025"}', + 'waku_archive_insert_duration_seconds_bucket{le="0.05"}', + 'waku_archive_insert_duration_seconds_bucket{le="0.075"}', + 'waku_archive_insert_duration_seconds_bucket{le="0.1"}', + 'waku_archive_insert_duration_seconds_bucket{le="0.25"}', + 'waku_archive_insert_duration_seconds_bucket{le="0.5"}', + 'waku_archive_insert_duration_seconds_bucket{le="0.75"}', + 'waku_archive_insert_duration_seconds_bucket{le="1.0"}', + 'waku_archive_insert_duration_seconds_bucket{le="2.5"}', + 'waku_archive_insert_duration_seconds_bucket{le="5.0"}', + 'waku_archive_insert_duration_seconds_bucket{le="7.5"}', + 'waku_archive_insert_duration_seconds_bucket{le="10.0"}', + 'waku_archive_insert_duration_seconds_bucket{le="+Inf"}', + "waku_archive_query_duration_seconds_sum", + "waku_archive_query_duration_seconds_count", + 'waku_archive_query_duration_seconds_bucket{le="0.005"}', + 'waku_archive_query_duration_seconds_bucket{le="0.01"}', + 'waku_archive_query_duration_seconds_bucket{le="0.025"}', + 'waku_archive_query_duration_seconds_bucket{le="0.05"}', + 'waku_archive_query_duration_seconds_bucket{le="0.075"}', + 'waku_archive_query_duration_seconds_bucket{le="0.1"}', + 'waku_archive_query_duration_seconds_bucket{le="0.25"}', + 'waku_archive_query_duration_seconds_bucket{le="0.5"}', + 'waku_archive_query_duration_seconds_bucket{le="0.75"}', + 'waku_archive_query_duration_seconds_bucket{le="1.0"}', + 'waku_archive_query_duration_seconds_bucket{le="2.5"}', + 'waku_archive_query_duration_seconds_bucket{le="5.0"}', + 'waku_archive_query_duration_seconds_bucket{le="7.5"}', + 'waku_archive_query_duration_seconds_bucket{le="10.0"}', + 'waku_archive_query_duration_seconds_bucket{le="+Inf"}', + "waku_legacy_archive_queries", + "waku_legacy_archive_insert_duration_seconds_sum", + "waku_legacy_archive_insert_duration_seconds_count", + 'waku_legacy_archive_insert_duration_seconds_bucket{le="0.005"}', + 'waku_legacy_archive_insert_duration_seconds_bucket{le="0.01"}', + 'waku_legacy_archive_insert_duration_seconds_bucket{le="0.025"}', + 'waku_legacy_archive_insert_duration_seconds_bucket{le="0.05"}', + 'waku_legacy_archive_insert_duration_seconds_bucket{le="0.075"}', + 'waku_legacy_archive_insert_duration_seconds_bucket{le="0.1"}', + 'waku_legacy_archive_insert_duration_seconds_bucket{le="0.25"}', + 'waku_legacy_archive_insert_duration_seconds_bucket{le="0.5"}', + 'waku_legacy_archive_insert_duration_seconds_bucket{le="0.75"}', + 'waku_legacy_archive_insert_duration_seconds_bucket{le="1.0"}', + 'waku_legacy_archive_insert_duration_seconds_bucket{le="2.5"}', + 'waku_legacy_archive_insert_duration_seconds_bucket{le="5.0"}', + 'waku_legacy_archive_insert_duration_seconds_bucket{le="7.5"}', + 'waku_legacy_archive_insert_duration_seconds_bucket{le="10.0"}', + 'waku_legacy_archive_insert_duration_seconds_bucket{le="+Inf"}', + "waku_legacy_archive_query_duration_seconds_sum", + "waku_legacy_archive_query_duration_seconds_count", + 'waku_legacy_archive_query_duration_seconds_bucket{le="0.005"}', + 'waku_legacy_archive_query_duration_seconds_bucket{le="0.01"}', + 'waku_legacy_archive_query_duration_seconds_bucket{le="0.025"}', + 'waku_legacy_archive_query_duration_seconds_bucket{le="0.05"}', + 'waku_legacy_archive_query_duration_seconds_bucket{le="0.075"}', + 'waku_legacy_archive_query_duration_seconds_bucket{le="0.1"}', + 'waku_legacy_archive_query_duration_seconds_bucket{le="0.25"}', + 'waku_legacy_archive_query_duration_seconds_bucket{le="0.5"}', + 'waku_legacy_archive_query_duration_seconds_bucket{le="0.75"}', + 'waku_legacy_archive_query_duration_seconds_bucket{le="1.0"}', + 'waku_legacy_archive_query_duration_seconds_bucket{le="2.5"}', + 'waku_legacy_archive_query_duration_seconds_bucket{le="5.0"}', + 'waku_legacy_archive_query_duration_seconds_bucket{le="7.5"}', + 'waku_legacy_archive_query_duration_seconds_bucket{le="10.0"}', + 'waku_legacy_archive_query_duration_seconds_bucket{le="+Inf"}', + "waku_legacy_store_queries", + "waku_store_queries", + "waku_filter_subscriptions", + "waku_filter_handle_message_duration_seconds_sum", + "waku_filter_handle_message_duration_seconds_count", + 'waku_filter_handle_message_duration_seconds_bucket{le="0.005"}', + 'waku_filter_handle_message_duration_seconds_bucket{le="0.01"}', + 'waku_filter_handle_message_duration_seconds_bucket{le="0.025"}', + 'waku_filter_handle_message_duration_seconds_bucket{le="0.05"}', + 'waku_filter_handle_message_duration_seconds_bucket{le="0.075"}', + 'waku_filter_handle_message_duration_seconds_bucket{le="0.1"}', + 'waku_filter_handle_message_duration_seconds_bucket{le="0.25"}', + 'waku_filter_handle_message_duration_seconds_bucket{le="0.5"}', + 'waku_filter_handle_message_duration_seconds_bucket{le="0.75"}', + 'waku_filter_handle_message_duration_seconds_bucket{le="1.0"}', + 'waku_filter_handle_message_duration_seconds_bucket{le="2.5"}', + 'waku_filter_handle_message_duration_seconds_bucket{le="5.0"}', + 'waku_filter_handle_message_duration_seconds_bucket{le="7.5"}', + 'waku_filter_handle_message_duration_seconds_bucket{le="10.0"}', + 'waku_filter_handle_message_duration_seconds_bucket{le="+Inf"}', + "discovery_session_lru_cache_hits_total", + "discovery_session_lru_cache_misses_total", + "discovery_session_decrypt_failures_total", + "discovery_unsolicited_messages_total", + "discovery_enr_auto_update_total", + "waku_discv5_discovered", + "waku_px_peers_received_total", + "waku_px_peers_received_unknown", + "waku_px_peers_sent", + "waku_px_peers_cached", + "waku_histogram_message_size_sum", + "waku_histogram_message_size_count", + 'waku_histogram_message_size_bucket{le="0.0"}', + 'waku_histogram_message_size_bucket{le="5.0"}', + 'waku_histogram_message_size_bucket{le="15.0"}', + 'waku_histogram_message_size_bucket{le="50.0"}', + 'waku_histogram_message_size_bucket{le="75.0"}', + 'waku_histogram_message_size_bucket{le="100.0"}', + 'waku_histogram_message_size_bucket{le="125.0"}', + 'waku_histogram_message_size_bucket{le="150.0"}', + 'waku_histogram_message_size_bucket{le="300.0"}', + 'waku_histogram_message_size_bucket{le="700.0"}', + 'waku_histogram_message_size_bucket{le="1000.0"}', + 'waku_histogram_message_size_bucket{le="+Inf"}', + "waku_lightpush_peers", + "waku_filter_peers", + "waku_store_peers", + "waku_px_peers", + "waku_dnsdisc_discovered", +] diff --git a/tests/metrics/__init__.py b/tests/metrics/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/metrics/test_metrics.py b/tests/metrics/test_metrics.py new file mode 100644 index 0000000000..ef4e4b6135 --- /dev/null +++ b/tests/metrics/test_metrics.py @@ -0,0 +1,114 @@ +import pytest +from src.env_vars import DEFAULT_NWAKU +from src.libs.common import delay +from src.node.waku_node import WakuNode +from src.steps.filter import StepsFilter +from src.steps.light_push import StepsLightPush +from src.steps.metrics import StepsMetrics +from src.steps.relay import StepsRelay +from src.steps.store import StepsStore + + +class TestMetrics(StepsRelay, StepsMetrics, StepsFilter, StepsLightPush, StepsStore): + def test_metrics_initial_value(self): + node = WakuNode(DEFAULT_NWAKU, f"node1_{self.test_id}") + node.start(relay="true", filter="true", store="true", lightpush="true") + delay(5) + self.validate_initial_metrics(node) + + @pytest.mark.usefixtures("setup_main_relay_nodes", "subscribe_main_relay_nodes", "relay_warm_up") + def test_metrics_after_relay_publish(self): + self.node1.send_relay_message(self.create_message(), self.test_pubsub_topic) + delay(0.5) + self.node2.get_relay_messages(self.test_pubsub_topic) + delay(5) + for node in self.main_nodes: + if node.is_nwaku(): + self.check_metric(node, "libp2p_peers", 1) + self.check_metric(node, "libp2p_pubsub_peers", 1) + self.check_metric(node, "libp2p_pubsub_topics", 1) + self.check_metric(node, "libp2p_pubsub_subscriptions_total", 1) + self.check_metric(node, 'libp2p_gossipsub_peers_per_topic_mesh{topic="other"}', 1) + self.check_metric(node, "waku_peer_store_size", 1) + self.check_metric(node, "waku_histogram_message_size_count", 1) + self.check_metric(node, 'waku_node_messages_total{type="relay"}', 1) + + @pytest.mark.usefixtures("setup_main_relay_node", "setup_main_filter_node", "subscribe_main_nodes") + def test_metrics_after_filter_get(self): + message = self.create_message() + self.node1.send_relay_message(message, self.test_pubsub_topic) + delay(0.5) + self.get_filter_messages(message["contentTopic"], pubsub_topic=self.test_pubsub_topic, node=self.node2) + delay(5) + self.check_metric(self.node1, "libp2p_peers", 1) + self.check_metric(self.node1, "libp2p_pubsub_peers", 1) + self.check_metric(self.node1, "libp2p_pubsub_topics", 1) + self.check_metric(self.node1, "libp2p_pubsub_subscriptions_total", 1) + self.check_metric(self.node1, "waku_peer_store_size", 1) + self.check_metric(self.node1, "waku_histogram_message_size_count", 1) + self.check_metric(self.node1, 'waku_node_messages_total{type="relay"}', 1) + self.check_metric(self.node1, 'waku_filter_requests{type="SUBSCRIBE"}', 1) + if self.node2.is_nwaku(): + self.check_metric( + self.node2, f'waku_service_peers{{protocol="/vac/waku/filter-subscribe/2.0.0-beta1",peerId="{self.node1.get_tcp_address()}"}}', 1 + ) + self.check_metric(self.node2, "libp2p_peers", 1) + self.check_metric(self.node2, "libp2p_total_dial_attempts_total", 1) + self.check_metric(self.node2, "waku_peer_store_size", 1) + + def test_metrics_after_light_push(self): + self.setup_first_receiving_node() + self.setup_second_receiving_node(lightpush="false", relay="true") + self.setup_first_lightpush_node() + self.subscribe_to_pubsub_topics_via_relay() + payload = self.create_payload(self.test_pubsub_topic) + self.light_push_node1.send_light_push_message(payload) + delay(0.5) + self.receiving_node1.get_relay_messages(self.test_pubsub_topic) + delay(5) + if self.light_push_node1.is_nwaku(): + self.check_metric( + self.light_push_node1, + f'waku_service_peers{{protocol="/vac/waku/lightpush/2.0.0-beta1",peerId="{self.receiving_node1.get_tcp_address()}"}}', + 1, + ) + self.check_metric(self.light_push_node1, "libp2p_peers", 1) + self.check_metric(self.light_push_node1, "waku_peer_store_size", 1) + if self.receiving_node1.is_nwaku(): + self.check_metric(self.receiving_node1, "libp2p_peers", 1) + self.check_metric(self.receiving_node1, "libp2p_pubsub_peers", 1) + self.check_metric(self.receiving_node1, "libp2p_pubsub_topics", 1) + self.check_metric(self.receiving_node1, "libp2p_pubsub_subscriptions_total", 1) + self.check_metric(self.receiving_node1, "waku_peer_store_size", 1) + self.check_metric(self.receiving_node1, "waku_histogram_message_size_count", 1) + self.check_metric(self.receiving_node1, 'waku_node_messages_total{type="relay"}', 1) + + def test_metrics_after_store_get(self, node_setup): + self.publish_message(message=self.create_message()) + self.check_published_message_is_stored(page_size=50, ascending="true") + delay(5) + self.check_metric(self.publishing_node1, "libp2p_peers", 1) + self.check_metric(self.publishing_node1, "libp2p_pubsub_peers", 1) + self.check_metric(self.publishing_node1, "libp2p_pubsub_topics", 1) + self.check_metric(self.publishing_node1, "libp2p_pubsub_subscriptions_total", 1) + self.check_metric(self.publishing_node1, "waku_peer_store_size", 1) + self.check_metric(self.publishing_node1, "waku_histogram_message_size_count", 1) + self.check_metric(self.publishing_node1, 'waku_node_messages_total{type="relay"}', 1) + if self.store_node1.is_nwaku(): + self.check_metric( + self.store_node1, + f'waku_service_peers{{protocol="/vac/waku/store/2.0.0-beta4",peerId="{self.publishing_node1.get_tcp_address()}"}}', + 1, + ) + self.check_metric( + self.store_node1, + f'waku_service_peers{{protocol="/vac/waku/store-query/3.0.0",peerId="{self.publishing_node1.get_tcp_address()}"}}', + 1, + ) + self.check_metric(self.store_node1, "libp2p_peers", 1) + self.check_metric(self.store_node1, "libp2p_pubsub_peers", 1) + self.check_metric(self.store_node1, "libp2p_pubsub_topics", 1) + self.check_metric(self.store_node1, "libp2p_pubsub_subscriptions_total", 1) + self.check_metric(self.store_node1, "waku_peer_store_size", 1) + self.check_metric(self.store_node1, "waku_histogram_message_size_count", 1) + self.check_metric(self.store_node1, 'waku_node_messages_total{type="relay"}', 1)