logos-messaging/logos-messaging-interop-tests
1
0
Fork 0
mirror of https://github.com/logos-messaging/logos-messaging-interop-tests.git synced 2026-01-02 14:03:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: auto-sharding needs to know number of shards in network (#126)

Browse Source 
* fix: auto-sharding needs to know number of shards in network

* Content topics are setup to split among 8 shards
This commit is contained in:
fryorcraken 2025-06-30 18:34:29 +10:00 committed by GitHub
parent 30030ec3fe
commit c230f4b4c4
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 40 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/steps/sharding.py
View File

@ -23,6 +23,7 @@ class StepsSharding(StepsRelay):
test_pubsub_topic = "/waku/2/rs/2/0"
test_payload = "Sharding works!!"
auto_cluster = 2
num_shards_in_network = 8
@pytest.fixture(scope="function", autouse=True)
def sharding_setup(self):

52
tests/sharding/test_relay_auto_sharding.py
View File

@ -12,12 +12,16 @@ logger = get_custom_logger(__name__)
class TestRelayAutosharding(StepsSharding):
def test_publish_without_subscribing_via_api_works(self):
self.setup_main_relay_nodes(cluster_id=self.auto_cluster, content_topic=self.test_content_topic)
self.setup_main_relay_nodes(
cluster_id=self.auto_cluster, content_topic=self.test_content_topic, num_shards_in_network=self.num_shards_in_network
)
for node in self.main_nodes:
self.relay_message(node, self.create_message(contentTopic=self.test_content_topic))
def test_retrieve_messages_without_subscribing_via_api(self):
self.setup_main_relay_nodes(cluster_id=self.auto_cluster, content_topic=self.test_content_topic)
self.setup_main_relay_nodes(
cluster_id=self.auto_cluster, content_topic=self.test_content_topic, num_shards_in_network=self.num_shards_in_network
)
try:
self.check_published_message_reaches_relay_peer(content_topic=self.test_content_topic)
if self.node2.is_nwaku():
@ -28,25 +32,33 @@ class TestRelayAutosharding(StepsSharding):
assert "Failed to publish: Node not subscribed to topic" in str(ex)
def test_subscribe_and_publish_on_another_content_topic_from_same_shard(self):
self.setup_main_relay_nodes(cluster_id=self.auto_cluster, content_topic=self.test_content_topic)
self.setup_main_relay_nodes(
cluster_id=self.auto_cluster, content_topic=self.test_content_topic, num_shards_in_network=self.num_shards_in_network
)
self.subscribe_main_relay_nodes(content_topics=["/newsService/1.0/weekly/protobuf"])
self.check_published_message_reaches_relay_peer(content_topic="/newsService/1.0/weekly/protobuf")
self.check_published_message_reaches_relay_peer(content_topic=self.test_content_topic)
def test_subscribe_and_publish_on_another_content_topic_from_another_shard(self):
self.setup_main_relay_nodes(cluster_id=self.auto_cluster, content_topic=self.test_content_topic)
self.setup_main_relay_nodes(
cluster_id=self.auto_cluster, content_topic=self.test_content_topic, num_shards_in_network=self.num_shards_in_network
)
self.subscribe_main_relay_nodes(content_topics=["/toychat/2/huilong/proto"])
self.check_published_message_reaches_relay_peer(content_topic="/toychat/2/huilong/proto")
self.check_published_message_reaches_relay_peer(content_topic=self.test_content_topic)
def test_publish_on_not_subscribed_content_topic_works(self):
self.setup_main_relay_nodes(cluster_id=self.auto_cluster, content_topic=self.test_content_topic)
self.setup_main_relay_nodes(
cluster_id=self.auto_cluster, content_topic=self.test_content_topic, num_shards_in_network=self.num_shards_in_network
)
self.subscribe_main_relay_nodes(content_topics=[self.test_content_topic])
for node in self.main_nodes:
self.relay_message(node, self.create_message(contentTopic="/toychat/2/huilong/proto"))
def test_cant_retrieve_messages_on_not_subscribed_content_topic(self):
self.setup_main_relay_nodes(cluster_id=self.auto_cluster, content_topic=self.test_content_topic)
self.setup_main_relay_nodes(
cluster_id=self.auto_cluster, content_topic=self.test_content_topic, num_shards_in_network=self.num_shards_in_network
)
self.subscribe_main_relay_nodes(content_topics=[self.test_content_topic])
self.check_published_message_doesnt_reach_relay_peer(content_topic="/toychat/2/huilong/proto")
@ -58,7 +70,9 @@ class TestRelayAutosharding(StepsSharding):
self.check_published_message_reaches_relay_peer(content_topic=content_topic)
def test_subscribe_one_by_one_to_different_content_topics_and_send_messages(self):
self.setup_main_relay_nodes(cluster_id=self.auto_cluster, content_topic=self.test_content_topic)
self.setup_main_relay_nodes(
cluster_id=self.auto_cluster, content_topic=self.test_content_topic, num_shards_in_network=self.num_shards_in_network
)
for content_topic in CONTENT_TOPICS_DIFFERENT_SHARDS + CONTENT_TOPICS_SHARD_0:
self.subscribe_main_relay_nodes(content_topics=[content_topic])
self.check_published_message_reaches_relay_peer(content_topic=content_topic)
@ -66,7 +80,9 @@ class TestRelayAutosharding(StepsSharding):
@pytest.mark.smoke
@pytest.mark.parametrize("content_topic_list", [CONTENT_TOPICS_SHARD_0, CONTENT_TOPICS_DIFFERENT_SHARDS])
def test_unsubscribe_from_some_content_topics(self, content_topic_list):
self.setup_main_relay_nodes(cluster_id=self.auto_cluster, content_topic=self.test_content_topic)
self.setup_main_relay_nodes(
cluster_id=self.auto_cluster, content_topic=self.test_content_topic, num_shards_in_network=self.num_shards_in_network
)
self.subscribe_main_relay_nodes(content_topics=content_topic_list)
for content_topic in content_topic_list:
self.check_published_message_reaches_relay_peer(content_topic=content_topic)
@ -78,7 +94,9 @@ class TestRelayAutosharding(StepsSharding):
@pytest.mark.smoke
def test_unsubscribe_from_all_content_topics(self):
self.setup_main_relay_nodes(cluster_id=self.auto_cluster, content_topic=self.test_content_topic)
self.setup_main_relay_nodes(
cluster_id=self.auto_cluster, content_topic=self.test_content_topic, num_shards_in_network=self.num_shards_in_network
)
self.subscribe_main_relay_nodes(content_topics=CONTENT_TOPICS_DIFFERENT_SHARDS)
for content_topic in CONTENT_TOPICS_DIFFERENT_SHARDS:
self.check_published_message_reaches_relay_peer(content_topic=content_topic)
@ -87,7 +105,9 @@ class TestRelayAutosharding(StepsSharding):
self.check_published_message_doesnt_reach_relay_peer(content_topic=content_topic)
def test_unsubscribe_from_all_content_topics_one_by_one(self):
self.setup_main_relay_nodes(cluster_id=self.auto_cluster, content_topic=self.test_content_topic)
self.setup_main_relay_nodes(
cluster_id=self.auto_cluster, content_topic=self.test_content_topic, num_shards_in_network=self.num_shards_in_network
)
self.subscribe_main_relay_nodes(content_topics=CONTENT_TOPICS_DIFFERENT_SHARDS)
for content_topic in CONTENT_TOPICS_DIFFERENT_SHARDS:
self.check_published_message_reaches_relay_peer(content_topic=content_topic)
@ -98,7 +118,9 @@ class TestRelayAutosharding(StepsSharding):
@pytest.mark.smoke
def test_resubscribe_to_unsubscribed_content_topics(self):
self.setup_main_relay_nodes(cluster_id=self.auto_cluster, content_topic=self.test_content_topic)
self.setup_main_relay_nodes(
cluster_id=self.auto_cluster, content_topic=self.test_content_topic, num_shards_in_network=self.num_shards_in_network
)
self.subscribe_main_relay_nodes(content_topics=CONTENT_TOPICS_DIFFERENT_SHARDS)
for content_topic in CONTENT_TOPICS_DIFFERENT_SHARDS:
self.check_published_message_reaches_relay_peer(content_topic=content_topic)
@ -110,14 +132,18 @@ class TestRelayAutosharding(StepsSharding):
self.check_published_message_reaches_relay_peer(content_topic=content_topic)
def test_unsubscribe_from_non_subscribed_content_topics(self):
self.setup_main_relay_nodes(cluster_id=self.auto_cluster, content_topic=self.test_content_topic)
self.setup_main_relay_nodes(
cluster_id=self.auto_cluster, content_topic=self.test_content_topic, num_shards_in_network=self.num_shards_in_network
)
self.unsubscribe_main_relay_nodes(content_topics=CONTENT_TOPICS_DIFFERENT_SHARDS)
for content_topic in CONTENT_TOPICS_DIFFERENT_SHARDS:
self.check_published_message_doesnt_reach_relay_peer(content_topic=content_topic)
@pytest.mark.parametrize("content_topic_list", [CONTENT_TOPICS_SHARD_0, CONTENT_TOPICS_DIFFERENT_SHARDS])
def test_publish_on_multiple_content_topics_and_only_after_fetch_them(self, content_topic_list):
self.setup_main_relay_nodes(cluster_id=self.auto_cluster, content_topic=self.test_content_topic)
self.setup_main_relay_nodes(
cluster_id=self.auto_cluster, content_topic=self.test_content_topic, num_shards_in_network=self.num_shards_in_network
)
self.subscribe_main_relay_nodes(content_topics=content_topic_list)
for content_topic in content_topic_list:
self.relay_message(self.node1, self.create_message(payload=to_base64(content_topic), contentTopic=content_topic))