From bd187ad1181ce051199008ef8ff446754089cefe Mon Sep 17 00:00:00 2001 From: AYAHASSAN287 <49167455+AYAHASSAN287@users.noreply.github.com> Date: Thu, 24 Oct 2024 13:16:37 +0300 Subject: [PATCH] adding test "test_store_not_include_data" (#76) * adding test "test_store_not_include_data" * adding test case for wrong peer address formatting * try restore message with wrong peer id * send peer address with wrong protocol * Fixing review comments & add test for wrong topic * adding test for topic content positive scenario * adding test for different wrong content topic * adding content topic & pubsubtopic tests * adding test cases for wrong encoding 7 no encoding of pubsub topic * adding test case test_time_filter_start_time_after_end_time * adding more time tests * adding tests with wrong start & end time * adding invalid ascending test * adding page_size tests * adding invalid ascending test * fix minor issues and adding comments * fixing review comments * minor change in test name * Adding review comments --- src/node/api_clients/rest.py | 20 ++++- src/node/store_response.py | 10 +++ tests/store/test_api_flags.py | 57 ++++++++++++++ tests/store/test_page_size.py | 31 ++++++++ tests/store/test_sorting.py | 18 +++++ tests/store/test_time_filter.py | 134 ++++++++++++++++++++++++++++++++ tests/store/test_topics.py | 91 ++++++++++++++++++++++ 7 files changed, 359 insertions(+), 2 deletions(-) diff --git a/src/node/api_clients/rest.py b/src/node/api_clients/rest.py index 72963ae103..210cae38b2 100644 --- a/src/node/api_clients/rest.py +++ b/src/node/api_clients/rest.py @@ -93,7 +93,20 @@ class REST(BaseClient): return get_messages_response.json() def get_store_messages( - self, peer_addr, include_data, pubsub_topic, content_topics, start_time, end_time, hashes, cursor, page_size, ascending, store_v, **kwargs + self, + peer_addr, + include_data, + pubsub_topic, + content_topics, + start_time, + end_time, + hashes, + cursor, + page_size, + ascending, + store_v, + encode_pubsubtopic=True, + **kwargs, ): base_url = f"store/{store_v}/messages" params = [] @@ -103,7 +116,10 @@ class REST(BaseClient): if include_data is not None: params.append(f"includeData={include_data}") if pubsub_topic is not None: - params.append(f"pubsubTopic={quote(pubsub_topic, safe='')}") + if encode_pubsubtopic: + params.append(f"pubsubTopic={quote(pubsub_topic, safe='')}") + else: + params.append(f"pubsubTopic={pubsub_topic}") if content_topics is not None: params.append(f"contentTopics={quote(content_topics, safe='')}") if start_time is not None: diff --git a/src/node/store_response.py b/src/node/store_response.py index 602ad007b2..d6274ce787 100644 --- a/src/node/store_response.py +++ b/src/node/store_response.py @@ -59,6 +59,16 @@ class StoreResponse: else: return None + def message_content(self, index): + try: + if self.messages is not None: + payload = self.messages[index]["message"]["contentTopic"] + return payload + else: + return None + except IndexError: + return None + def message_payload(self, index): try: if self.messages is not None: diff --git a/tests/store/test_api_flags.py b/tests/store/test_api_flags.py index 8c0cba1b61..35ce168d88 100644 --- a/tests/store/test_api_flags.py +++ b/tests/store/test_api_flags.py @@ -1,8 +1,14 @@ import pytest +from time import time +from src.libs.custom_logger import get_custom_logger from src.libs.common import to_base64 from src.node.waku_message import WakuMessage +from src.node.store_response import StoreResponse from src.steps.store import StepsStore from src.test_data import SAMPLE_INPUTS +from src.test_data import PUBSUB_TOPICS_STORE + +logger = get_custom_logger(__name__) @pytest.mark.usefixtures("node_setup") @@ -11,6 +17,36 @@ class TestApiFlags(StepsStore): self.publish_message() self.check_published_message_is_stored(store_node=self.store_node1, peer_addr=self.multiaddr_list[0]) + def test_store_with_wrongPeerAddr(self): + self.publish_message() + wrong_peer_addr = self.multiaddr_list[0][1:] + logger.debug(f"Running test with wrong_peer_addr: {wrong_peer_addr}") + try: + self.check_published_message_is_stored(store_node=self.store_node1, peer_addr=wrong_peer_addr) + raise Exception("Message stored with wrong peer address") + except Exception as ex: + logger.debug(f" Response with wrong peer address is { ex.args[0]}") + assert ex.args[0].find("Invalid MultiAddress") != -1 + # try to send wrong peer id + wrong_peer_id = self.multiaddr_list[0][:-1] + logger.debug(f"Running test with wrong_peer_addr {wrong_peer_id}") + try: + self.check_published_message_is_stored(store_node=self.store_node1, peer_addr=wrong_peer_id) + raise Exception("Message restored with wrong peer id") + except Exception as ex: + logger.debug(f" Response with wrong peer id is { ex.args[0]}") + assert ex.args[0].find("Failed parsing remote peer info") != -1 + + # send address without /tcp number + wrong_peer_addr = self.multiaddr_list[0].replace("/tcp", "") + logger.debug(f"logger is {wrong_peer_addr}") + try: + self.check_published_message_is_stored(store_node=self.store_node1, peer_addr=wrong_peer_addr) + raise Exception("Message stored with wrong peer address") + except Exception as ex: + logger.debug(f" Response with wrong peer address is { ex.args[0]}") + assert ex.args[0].find("Unsupported protocol") != -1 + def test_store_include_data(self): message_list = [] for payload in SAMPLE_INPUTS: @@ -25,3 +61,24 @@ class TestApiFlags(StepsStore): assert store_response.message_pubsub_topic(index) == self.test_pubsub_topic waku_message = WakuMessage([store_response.message_at(index)]) waku_message.assert_received_message(message_list[index]) + + def test_store_not_include_data(self): + self.publish_message(message=self.create_message()) + store_response = self.get_messages_from_store(self.store_node1, include_data="false") + logger.debug(f" Message restored with hash only is {store_response.messages} ") + assert "message" not in store_response.messages + + def test_get_store_messages_with_wrong_typo_content_topic(self): + # positive scenario + content_topic = "/myapp/1/latest/protoo" + logger.debug(f"Trying to publish msg with content topic {content_topic}") + msg = self.publish_message(message=self.create_message(contentTopic=content_topic)) + store_response = self.get_messages_from_store(self.store_node1, include_data="true", content_topics=content_topic) + try: + if store_response.messages is not None: + stored_contentTopic = store_response.message_content(0) + logger.debug(f"stored content topic is {stored_contentTopic}") + assert stored_contentTopic == content_topic, "content topics don't match" + + except Exception as e: + raise Exception(f"can't get message with content topic {content_topic}") diff --git a/tests/store/test_page_size.py b/tests/store/test_page_size.py index ee65453e4b..0d7fe7bb14 100644 --- a/tests/store/test_page_size.py +++ b/tests/store/test_page_size.py @@ -1,6 +1,9 @@ import pytest from src.libs.common import to_base64 from src.steps.store import StepsStore +from src.libs.custom_logger import get_custom_logger + +logger = get_custom_logger(__name__) @pytest.mark.usefixtures("node_setup") @@ -33,3 +36,31 @@ class TestPageSize(StepsStore): for node in self.store_nodes: store_response = self.get_messages_from_store(node, page_size=page_size) assert len(store_response.messages) == page_size, "Message count mismatch" + + def test_extreme_number_page_size(self): + for i in range(150): + self.publish_message(message=self.create_message(payload=to_base64(f"Message_{i}"))) + for node in self.store_nodes: + store_response = self.get_messages_from_store(node, page_size=1000000) + assert len(store_response.messages) == 100, "Message count mismatch" + + def test_negative_number_page_size(self): + page_size = -1 + for i in range(10): + self.publish_message(message=self.create_message(payload=to_base64(f"Message_{i}"))) + logger.debug(f"requesting stored message with wrong page_size = {page_size}") + for node in self.store_nodes: + store_response = self.get_messages_from_store(node, page_size=page_size) + assert len(store_response.messages) == 10, "Message count mismatch" + + def test_invalid_page_size(self): + page_size = "$2" + for i in range(10): + self.publish_message(message=self.create_message(payload=to_base64(f"Message_{i}"))) + for node in self.store_nodes: + try: + store_response = self.get_messages_from_store(node, page_size=page_size) + raise Exception(f"invalid page number {page_size} was acepted !!") + except Exception as e: + logger.debug(f"the invalid page_size {page_size} wasn't accepted ") + assert e.args[0].find("400 Client Error: Bad Request for url"), "Error logged isn't for bad url" diff --git a/tests/store/test_sorting.py b/tests/store/test_sorting.py index 512e44b725..fc1990aa60 100644 --- a/tests/store/test_sorting.py +++ b/tests/store/test_sorting.py @@ -1,6 +1,9 @@ import pytest from src.libs.common import to_base64 from src.steps.store import StepsStore +from src.libs.custom_logger import get_custom_logger + +logger = get_custom_logger(__name__) @pytest.mark.usefixtures("node_setup") @@ -21,3 +24,18 @@ class TestSorting(StepsStore): assert response_message_hash_list == expected_message_hash_list[:5], "Message hash mismatch for acending order" else: assert response_message_hash_list == expected_message_hash_list[5:], "Message hash mismatch for descending order" + + def test_store_invalid_ascending(self): + expected_message_hash_list = [] + ascending = "##" + for i in range(4): + message = self.create_message(payload=to_base64(f"Message_{i}")) + self.publish_message(message=message) + expected_message_hash_list.append(self.compute_message_hash(self.test_pubsub_topic, message)) + logger.debug(f"requesting stored messages with invalid ascending ={ascending}") + for node in self.store_nodes: + store_response = self.get_messages_from_store(node, ascending=ascending, page_size=2) + response_message_hash_list = [] + for index in range(len(store_response.messages)): + response_message_hash_list.append(store_response.message_hash(index)) + assert response_message_hash_list == expected_message_hash_list[:2], "pages aren't forward as expected" diff --git a/tests/store/test_time_filter.py b/tests/store/test_time_filter.py index ad0cb0cf15..b50735e1b4 100644 --- a/tests/store/test_time_filter.py +++ b/tests/store/test_time_filter.py @@ -1,3 +1,5 @@ +import time + import pytest from datetime import timedelta, datetime from src.libs.custom_logger import get_custom_logger @@ -112,3 +114,135 @@ class TestTimeFilter(StepsStore): ) assert len(store_response.messages) == 1, "Message count mismatch" assert store_response.message_hash(0) == message_hash_list[0], "Incorrect messaged filtered based on time" + + def test_time_filter_start_time_after_end_time(self): + start_time = self.ts_pass[4]["value"] # 2 sec Future + end_time = self.ts_pass[0]["value"] # 3 sec past + for timestamp in self.ts_pass: + message = self.create_message(timestamp=timestamp["value"]) + self.publish_message(message=message) + logger.debug(f"inquering stored messages with start time {start_time} after end time {end_time}") + for node in self.store_nodes: + store_response = self.get_messages_from_store( + node, + page_size=20, + start_time=start_time, + end_time=end_time, + ) + logger.debug(f"response for wrong time message is {store_response.response}") + assert len(store_response.messages) == 0, "got messages with start time after end time !" + + def test_time_filter_negative_start_time(self): + for timestamp in self.ts_pass: + message = self.create_message(timestamp=timestamp["value"]) + self.publish_message(message=message) + + start_time = -10000 + logger.debug(f"inquering stored messages with start time {start_time}") + for node in self.store_nodes: + store_response = self.get_messages_from_store(node, page_size=20, start_time=start_time, include_data=True) + logger.debug(f"number of messages stored for " f"start time = {start_time} is {len(store_response.messages)}") + + assert len(store_response.messages) == 6, "number of messages retrieved doesn't match time filter " + + def test_time_filter_zero_start_time(self): + for timestamp in self.ts_pass: + message = self.create_message(timestamp=timestamp["value"]) + self.publish_message(message=message) + start_time = 0 + logger.debug(f"inquering stored messages with start time {start_time}") + for node in self.store_nodes: + store_response = self.get_messages_from_store(node, page_size=20, start_time=start_time, include_data=True) + logger.debug(f"number of messages stored for " f"start time = {start_time} is {len(store_response.messages)}") + + assert len(store_response.messages) == 6, "number of messages retrieved doesn't match time filter " + + def test_time_filter_zero_start_end_time(self): + for timestamp in self.ts_pass: + message = self.create_message(timestamp=timestamp["value"]) + self.publish_message(message=message) + start_time = 0 + end_time = 0 + logger.debug(f"inquering stored messages with start time {start_time}") + for node in self.store_nodes: + store_response = self.get_messages_from_store(node, page_size=20, start_time=start_time, end_time=end_time, include_data=True) + logger.debug(f"number of messages stored for " f"start time = {start_time} is {len(store_response.messages)}") + + assert len(store_response.messages) == 6, "number of messages retrieved doesn't match time filter " + + def test_time_filter_invalid_start_time(self): + for timestamp in self.ts_pass: + message = self.create_message(timestamp=timestamp["value"]) + self.publish_message(message=message) + start_time = "abc" + logger.debug(f"inquering stored messages with start time {start_time}") + try: + for node in self.store_nodes: + store_response = self.get_messages_from_store(node, page_size=20, start_time=start_time, include_data=True) + raise Exception(f" request for stored messages with invalid start time {start_time} is successful") + except Exception as e: + logger.debug(f"invalid start_time cause error {e}") + assert e.args[0].find("Bad Request for url"), "url with wrong start_time is accepted" + + def test_time_filter_end_time_now(self): + self.ts_pass[3]["value"] = int((datetime.now() + timedelta(seconds=4)).timestamp() * 1e9) + start_time = self.ts_pass[0]["value"] + i = 0 + for timestamp in self.ts_pass: + message = self.create_message(timestamp=timestamp["value"]) + self.publish_message(message=message) + i += 1 + end_time = int(datetime.now().timestamp() * 1e9) + logger.debug(f"inquering stored messages with start time {start_time} after end time {end_time}") + for node in self.store_nodes: + store_response = self.get_messages_from_store(node, page_size=20, start_time=start_time, end_time=end_time, include_data=True) + logger.debug(f"number of messages stored for start time {start_time} and " f"end time = {end_time} is {len(store_response.messages)}") + assert len(store_response.messages) == 4, "number of messages retrieved doesn't match time filter " + + def test_time_filter_big_timestamp(self): + start_time = self.ts_pass[0]["value"] + for timestamp in self.ts_pass: + message = self.create_message(timestamp=timestamp["value"]) + self.publish_message(message=message) + end_time = int((datetime.now() + timedelta(days=8000)).timestamp() * 1e9) + logger.debug(f"inquering stored messages with start time {start_time} after end time {end_time}") + for node in self.store_nodes: + store_response = self.get_messages_from_store(node, page_size=20, start_time=start_time, end_time=end_time, include_data=True) + logger.debug(f"number of messages stored for start time {start_time} and " f"end time = {end_time} is {len(store_response.messages)}") + assert len(store_response.messages) == 6, "number of messages retrieved doesn't match time filter " + + def test_time_filter_small_timestamp(self): + start_time = self.ts_pass[0]["value"] + for timestamp in self.ts_pass: + message = self.create_message(timestamp=timestamp["value"]) + self.publish_message(message=message) + end_time = self.ts_pass[5]["value"] + 1 + logger.debug(f"inquering stored messages with start time {start_time} after end time {end_time}") + for node in self.store_nodes: + store_response = self.get_messages_from_store(node, page_size=20, start_time=start_time, end_time=end_time, include_data=True) + logger.debug(f"number of messages stored for start time {start_time} and " f"end time = {end_time} is {len(store_response.messages)}") + + assert len(store_response.messages) == 6, "number of messages retrieved doesn't match time filter " + + def test_time_filter_negative_end_time(self): + for timestamp in self.ts_pass: + message = self.create_message(timestamp=timestamp["value"]) + self.publish_message(message=message) + end_time = -10000 + logger.debug(f"inquering stored messages with end time {end_time}") + for node in self.store_nodes: + store_response = self.get_messages_from_store(node, page_size=20, end_time=end_time, include_data=True) + logger.debug(f"number of messages stored for " f"end time = {end_time} is {len(store_response.messages)}") + + assert len(store_response.messages) == 6, "number of messages retrieved doesn't match time filter " + + def test_time_filter_zero_end_time(self): + for timestamp in self.ts_pass: + message = self.create_message(timestamp=timestamp["value"]) + self.publish_message(message=message) + end_time = 0 + logger.debug(f"inquering stored messages with end time {end_time}") + for node in self.store_nodes: + store_response = self.get_messages_from_store(node, page_size=20, end_time=end_time, include_data=True) + logger.debug(f"number of messages stored for " f"end time = {end_time} is {len(store_response.messages)}") + assert len(store_response.messages) == 6, "number of messages retrieved doesn't match time filter " diff --git a/tests/store/test_topics.py b/tests/store/test_topics.py index fc315c0106..521108d53e 100644 --- a/tests/store/test_topics.py +++ b/tests/store/test_topics.py @@ -2,6 +2,10 @@ import pytest from src.env_vars import NODE_2 from src.steps.store import StepsStore from src.test_data import CONTENT_TOPICS_DIFFERENT_SHARDS +from src.libs.custom_logger import get_custom_logger +from src.test_data import PUBSUB_TOPICS_STORE + +logger = get_custom_logger(__name__) @pytest.mark.xfail("go-waku" in NODE_2, reason="Bug reported: https://github.com/waku-org/go-waku/issues/1108") @@ -79,3 +83,90 @@ class TestTopics(StepsStore): for node in self.store_nodes: store_response = node.get_store_messages(page_size=20, ascending="true") assert len(store_response["messages"]) == len(CONTENT_TOPICS_DIFFERENT_SHARDS), "Message count mismatch" + + def test_store_with_not_valid_content_topic(self): + empty_content_topic = "##" + logger.debug(f"trying to find stored messages with wrong content_topic ={empty_content_topic}") + for node in self.store_nodes: + store_response = node.get_store_messages(page_size=20, include_data="true", ascending="true", content_topics=empty_content_topic) + assert len(store_response["messages"]) == 0, "Messages shouldn't ne retrieved with invalid content_topic" + # test with space string content topic + space_content_topic = " " + try: + store_response = self.store_nodes[0].get_store_messages( + page_size=20, include_data="true", ascending="true", content_topics=space_content_topic + ) + logger.debug(f" response for invalid content_topic {store_response}") + assert store_response["messages"] == [], "message stored with wrong topic " + except Exception as e: + raise Exception("couldn't get stored message with invalid content_topic") + + def test_store_with_wrong_url_content_topic(self): + # test with wrong url + wrong_content_topic = "myapp/1/latest/proto" + logger.debug(f"trying to find stored messages with wrong content_topic ={wrong_content_topic}") + try: + store_response = self.store_nodes[0].get_store_messages( + page_size=20, include_data="true", ascending="true", content_topics=wrong_content_topic + ) + logger.debug(f" response for wrong url content topic is {store_response}") + assert store_response["messages"] == [], "message stored with wrong topic " + except Exception as e: + raise Exception(f"couldn't get stored message with wrong url {wrong_content_topic}") + + def test_store_with_empty_pubsub_topics(self): + # empty pubsub topic + empty_topic = "" + index = iter(self.store_nodes) + logger.debug(f"Trying to get stored msg with empty pubsub topic") + store_response = self.store_nodes[0].get_store_messages(pubsub_topic=empty_topic, include_data="true", page_size=20, ascending="true") + logger.debug(f"getting the following response when sending empty pubsub_topic {store_response}") + for msg in store_response["messages"]: + assert msg["pubsubTopic"] == self.test_pubsub_topic, "wrong pubsub topic" + logger.debug(f"messages successfully queried with empty pubsub topic ") + + def test_store_with_wrong_url_pubsub_topic(self): + # wrong url pubsub topic + wrong_url_topic = PUBSUB_TOPICS_STORE[0][1:] + logger.debug(f"Trying to get stored msg with wrong url topic {wrong_url_topic}") + try: + self.publish_message(pubsub_topic=PUBSUB_TOPICS_STORE[0]) + self.check_published_message_is_stored(pubsub_topic=wrong_url_topic) + raise Exception("getting stored message with wrong url pubsub topic") + except Exception as e: + logger.error(f"Topic {wrong_url_topic} is wrong ''n: {str(e)}") + assert e.args[0].find("messages': []") != -1, "Message shall not be stored for wrong topic" + + def test_store_with_long_string_pubsub_topic(self): + # long topic string + long_url_topic = PUBSUB_TOPICS_STORE[0][:-1] + million = 1000000 + for i in range(million): + long_url_topic += str(i) + logger.debug(f"Trying to get stored msg with url topic size million ") + self.publish_message(pubsub_topic=PUBSUB_TOPICS_STORE[0]) + try: + self.check_published_message_is_stored(pubsub_topic=long_url_topic) + raise Exception("request stored message with very long topic string shouldn't be accepted") + except Exception as e: + logger.error(f"store request with very long pubsub topic wasn't accepted ") + assert e.args[0].find("Client Error: Request Header Fields Too Large for url") != -1, "error isn't for large url" + + def test_store_with_wrong_encoding_pubsubtopic(self): + wrong_encoidng_topic = "%23waku%2F2%2Frs%2F3%2F0" + empty_response = [] + logger.debug(f"trying get message with wrong encoded pubsub topic {wrong_encoidng_topic}") + store_response = self.store_nodes[0].get_store_messages( + pubsub_topic=wrong_encoidng_topic, encode_pubsubtopic=False, include_data="true", page_size=20, ascending="true" + ) + logger.debug(f"response for getting message with wrong encoded pubsub topic {store_response}") + assert store_response["messages"] == empty_response, "Error !! message retrieved with wrong encoding pubsub_topic" + + def test_store_without_encoding_pubsubtopic(self): + topic = PUBSUB_TOPICS_STORE[0] + logger.debug(f"trying get message with wrong encoded pubsub topic {topic}") + store_response = self.store_nodes[0].get_store_messages( + pubsub_topic=topic, encode_pubsubtopic=False, include_data="true", page_size=20, ascending="true" + ) + logger.debug(f"response for getting message without encoding pubsub topic {store_response}") + assert store_response["messages"][0]["pubsubTopic"] == PUBSUB_TOPICS_STORE[0], "topics Don't match !!"