From c230f4b4c4062f950bde1778351f297a448ce37a Mon Sep 17 00:00:00 2001 From: fryorcraken <110212804+fryorcraken@users.noreply.github.com> Date: Mon, 30 Jun 2025 18:34:29 +1000 Subject: [PATCH] fix: auto-sharding needs to know number of shards in network (#126) * fix: auto-sharding needs to know number of shards in network * Content topics are setup to split among 8 shards --- src/steps/sharding.py | 1 + tests/sharding/test_relay_auto_sharding.py | 52 ++++++++++++++++------ 2 files changed, 40 insertions(+), 13 deletions(-) diff --git a/src/steps/sharding.py b/src/steps/sharding.py index 9193b7d5..e39f7f3d 100644 --- a/src/steps/sharding.py +++ b/src/steps/sharding.py @@ -23,6 +23,7 @@ class StepsSharding(StepsRelay): test_pubsub_topic = "/waku/2/rs/2/0" test_payload = "Sharding works!!" auto_cluster = 2 + num_shards_in_network = 8 @pytest.fixture(scope="function", autouse=True) def sharding_setup(self): diff --git a/tests/sharding/test_relay_auto_sharding.py b/tests/sharding/test_relay_auto_sharding.py index 5709b915..9553c33a 100644 --- a/tests/sharding/test_relay_auto_sharding.py +++ b/tests/sharding/test_relay_auto_sharding.py @@ -12,12 +12,16 @@ logger = get_custom_logger(__name__) class TestRelayAutosharding(StepsSharding): def test_publish_without_subscribing_via_api_works(self): - self.setup_main_relay_nodes(cluster_id=self.auto_cluster, content_topic=self.test_content_topic) + self.setup_main_relay_nodes( + cluster_id=self.auto_cluster, content_topic=self.test_content_topic, num_shards_in_network=self.num_shards_in_network + ) for node in self.main_nodes: self.relay_message(node, self.create_message(contentTopic=self.test_content_topic)) def test_retrieve_messages_without_subscribing_via_api(self): - self.setup_main_relay_nodes(cluster_id=self.auto_cluster, content_topic=self.test_content_topic) + self.setup_main_relay_nodes( + cluster_id=self.auto_cluster, content_topic=self.test_content_topic, num_shards_in_network=self.num_shards_in_network + ) try: self.check_published_message_reaches_relay_peer(content_topic=self.test_content_topic) if self.node2.is_nwaku(): @@ -28,25 +32,33 @@ class TestRelayAutosharding(StepsSharding): assert "Failed to publish: Node not subscribed to topic" in str(ex) def test_subscribe_and_publish_on_another_content_topic_from_same_shard(self): - self.setup_main_relay_nodes(cluster_id=self.auto_cluster, content_topic=self.test_content_topic) + self.setup_main_relay_nodes( + cluster_id=self.auto_cluster, content_topic=self.test_content_topic, num_shards_in_network=self.num_shards_in_network + ) self.subscribe_main_relay_nodes(content_topics=["/newsService/1.0/weekly/protobuf"]) self.check_published_message_reaches_relay_peer(content_topic="/newsService/1.0/weekly/protobuf") self.check_published_message_reaches_relay_peer(content_topic=self.test_content_topic) def test_subscribe_and_publish_on_another_content_topic_from_another_shard(self): - self.setup_main_relay_nodes(cluster_id=self.auto_cluster, content_topic=self.test_content_topic) + self.setup_main_relay_nodes( + cluster_id=self.auto_cluster, content_topic=self.test_content_topic, num_shards_in_network=self.num_shards_in_network + ) self.subscribe_main_relay_nodes(content_topics=["/toychat/2/huilong/proto"]) self.check_published_message_reaches_relay_peer(content_topic="/toychat/2/huilong/proto") self.check_published_message_reaches_relay_peer(content_topic=self.test_content_topic) def test_publish_on_not_subscribed_content_topic_works(self): - self.setup_main_relay_nodes(cluster_id=self.auto_cluster, content_topic=self.test_content_topic) + self.setup_main_relay_nodes( + cluster_id=self.auto_cluster, content_topic=self.test_content_topic, num_shards_in_network=self.num_shards_in_network + ) self.subscribe_main_relay_nodes(content_topics=[self.test_content_topic]) for node in self.main_nodes: self.relay_message(node, self.create_message(contentTopic="/toychat/2/huilong/proto")) def test_cant_retrieve_messages_on_not_subscribed_content_topic(self): - self.setup_main_relay_nodes(cluster_id=self.auto_cluster, content_topic=self.test_content_topic) + self.setup_main_relay_nodes( + cluster_id=self.auto_cluster, content_topic=self.test_content_topic, num_shards_in_network=self.num_shards_in_network + ) self.subscribe_main_relay_nodes(content_topics=[self.test_content_topic]) self.check_published_message_doesnt_reach_relay_peer(content_topic="/toychat/2/huilong/proto") @@ -58,7 +70,9 @@ class TestRelayAutosharding(StepsSharding): self.check_published_message_reaches_relay_peer(content_topic=content_topic) def test_subscribe_one_by_one_to_different_content_topics_and_send_messages(self): - self.setup_main_relay_nodes(cluster_id=self.auto_cluster, content_topic=self.test_content_topic) + self.setup_main_relay_nodes( + cluster_id=self.auto_cluster, content_topic=self.test_content_topic, num_shards_in_network=self.num_shards_in_network + ) for content_topic in CONTENT_TOPICS_DIFFERENT_SHARDS + CONTENT_TOPICS_SHARD_0: self.subscribe_main_relay_nodes(content_topics=[content_topic]) self.check_published_message_reaches_relay_peer(content_topic=content_topic) @@ -66,7 +80,9 @@ class TestRelayAutosharding(StepsSharding): @pytest.mark.smoke @pytest.mark.parametrize("content_topic_list", [CONTENT_TOPICS_SHARD_0, CONTENT_TOPICS_DIFFERENT_SHARDS]) def test_unsubscribe_from_some_content_topics(self, content_topic_list): - self.setup_main_relay_nodes(cluster_id=self.auto_cluster, content_topic=self.test_content_topic) + self.setup_main_relay_nodes( + cluster_id=self.auto_cluster, content_topic=self.test_content_topic, num_shards_in_network=self.num_shards_in_network + ) self.subscribe_main_relay_nodes(content_topics=content_topic_list) for content_topic in content_topic_list: self.check_published_message_reaches_relay_peer(content_topic=content_topic) @@ -78,7 +94,9 @@ class TestRelayAutosharding(StepsSharding): @pytest.mark.smoke def test_unsubscribe_from_all_content_topics(self): - self.setup_main_relay_nodes(cluster_id=self.auto_cluster, content_topic=self.test_content_topic) + self.setup_main_relay_nodes( + cluster_id=self.auto_cluster, content_topic=self.test_content_topic, num_shards_in_network=self.num_shards_in_network + ) self.subscribe_main_relay_nodes(content_topics=CONTENT_TOPICS_DIFFERENT_SHARDS) for content_topic in CONTENT_TOPICS_DIFFERENT_SHARDS: self.check_published_message_reaches_relay_peer(content_topic=content_topic) @@ -87,7 +105,9 @@ class TestRelayAutosharding(StepsSharding): self.check_published_message_doesnt_reach_relay_peer(content_topic=content_topic) def test_unsubscribe_from_all_content_topics_one_by_one(self): - self.setup_main_relay_nodes(cluster_id=self.auto_cluster, content_topic=self.test_content_topic) + self.setup_main_relay_nodes( + cluster_id=self.auto_cluster, content_topic=self.test_content_topic, num_shards_in_network=self.num_shards_in_network + ) self.subscribe_main_relay_nodes(content_topics=CONTENT_TOPICS_DIFFERENT_SHARDS) for content_topic in CONTENT_TOPICS_DIFFERENT_SHARDS: self.check_published_message_reaches_relay_peer(content_topic=content_topic) @@ -98,7 +118,9 @@ class TestRelayAutosharding(StepsSharding): @pytest.mark.smoke def test_resubscribe_to_unsubscribed_content_topics(self): - self.setup_main_relay_nodes(cluster_id=self.auto_cluster, content_topic=self.test_content_topic) + self.setup_main_relay_nodes( + cluster_id=self.auto_cluster, content_topic=self.test_content_topic, num_shards_in_network=self.num_shards_in_network + ) self.subscribe_main_relay_nodes(content_topics=CONTENT_TOPICS_DIFFERENT_SHARDS) for content_topic in CONTENT_TOPICS_DIFFERENT_SHARDS: self.check_published_message_reaches_relay_peer(content_topic=content_topic) @@ -110,14 +132,18 @@ class TestRelayAutosharding(StepsSharding): self.check_published_message_reaches_relay_peer(content_topic=content_topic) def test_unsubscribe_from_non_subscribed_content_topics(self): - self.setup_main_relay_nodes(cluster_id=self.auto_cluster, content_topic=self.test_content_topic) + self.setup_main_relay_nodes( + cluster_id=self.auto_cluster, content_topic=self.test_content_topic, num_shards_in_network=self.num_shards_in_network + ) self.unsubscribe_main_relay_nodes(content_topics=CONTENT_TOPICS_DIFFERENT_SHARDS) for content_topic in CONTENT_TOPICS_DIFFERENT_SHARDS: self.check_published_message_doesnt_reach_relay_peer(content_topic=content_topic) @pytest.mark.parametrize("content_topic_list", [CONTENT_TOPICS_SHARD_0, CONTENT_TOPICS_DIFFERENT_SHARDS]) def test_publish_on_multiple_content_topics_and_only_after_fetch_them(self, content_topic_list): - self.setup_main_relay_nodes(cluster_id=self.auto_cluster, content_topic=self.test_content_topic) + self.setup_main_relay_nodes( + cluster_id=self.auto_cluster, content_topic=self.test_content_topic, num_shards_in_network=self.num_shards_in_network + ) self.subscribe_main_relay_nodes(content_topics=content_topic_list) for content_topic in content_topic_list: self.relay_message(self.node1, self.create_message(payload=to_base64(content_topic), contentTopic=content_topic))