diff --git a/src/env_vars.py b/src/env_vars.py index 0b58d1d1e..59674ab42 100644 --- a/src/env_vars.py +++ b/src/env_vars.py @@ -14,7 +14,7 @@ def get_env_var(var_name, default=None): # Configuration constants. Need to be upercase to appear in reports -DEFAULT_NWAKU = "wakuorg/nwaku:wakunode2-v0.37.2-rc.0" +DEFAULT_NWAKU = "wakuorg/nwaku:wakunode2-v0.37.2" STRESS_ENABLED = False NODE_1 = get_env_var("NODE_1", DEFAULT_NWAKU) NODE_2 = get_env_var("NODE_2", DEFAULT_NWAKU) diff --git a/src/node/docker_mananger.py b/src/node/docker_mananger.py index 44329d098..7e19605b6 100644 --- a/src/node/docker_mananger.py +++ b/src/node/docker_mananger.py @@ -1,6 +1,7 @@ import os import re import time +from collections import deque from src.libs.custom_logger import get_custom_logger import random import threading @@ -48,12 +49,24 @@ class DockerManager: cli_args_str_for_log = " ".join(cli_args) logger.debug(f"docker run -i -t {port_bindings_for_log} {image_name} {cli_args_str_for_log}") container = self._client.containers.run( - image_name, command=cli_args, ports=port_bindings, detach=True, remove=remove_container, auto_remove=remove_container, volumes=volumes + image_name, + command=cli_args, + ports=port_bindings, + detach=True, + auto_remove=False, + volumes=volumes, ) network = self._client.networks.get(NETWORK_NAME) logger.debug(f"docker network connect --ip {container_ip} {NETWORK_NAME} {container.id}") - network.connect(container, ipv4_address=container_ip) + try: + network.connect(container, ipv4_address=container_ip) + except APIError as ex: + recent_logs = self.get_container_logs(container, tail=120) + if recent_logs.startswith("" + + def get_log_file_tail(self, log_path, tail=200): + try: + if not os.path.exists(log_path): + return f"" + + with open(log_path, "r", encoding="utf-8", errors="ignore") as log_file: + lines = deque(log_file, maxlen=tail) + + return "".join(lines) if lines else "" + except Exception as ex: + return f"" + def _log_container_output(self, container, log_path): os.makedirs(os.path.dirname(log_path), exist_ok=True) retry_count = 0 @@ -86,6 +120,10 @@ class DockerManager: except (APIError, IOError) as e: retry_count += 1 if retry_count >= 5: + recent_logs = self.get_container_logs(container, tail=120) + if recent_logs.startswith(" 0: logger.debug(111111111111111) self.keystore_prefixes = self.generate_keystore_prefixes(count) + self.rln_membership_indexes = [] for i, prefix in enumerate(self.keystore_prefixes): logger.debug(000000000000000000000) - self.register_rln_single_node(prefix=prefix, rln_creds_source=RLN_CREDENTIALS, rln_creds_id=f"{i+1}") + membership_index = self.register_rln_single_node(prefix=prefix, rln_creds_source=RLN_CREDENTIALS, rln_creds_id=f"{i+1}") + self.rln_membership_indexes.append(membership_index) else: - self.keystore_prefixes = orig_prefixes + # Backward-compatible support for legacy list-based cache values. + if isinstance(orig_prefixes, dict): + self.keystore_prefixes = orig_prefixes.get("keystore_prefixes", []) + self.rln_membership_indexes = orig_prefixes.get("rln_membership_indexes", []) + else: + self.keystore_prefixes = orig_prefixes + self.rln_membership_indexes = [] - return self.keystore_prefixes + return { + "keystore_prefixes": self.keystore_prefixes, + "rln_membership_indexes": self.rln_membership_indexes, + } @allure.step def setup_main_rln_relay_nodes(self, **kwargs): @@ -60,7 +72,7 @@ class StepsRLN(StepsCommon): relay="true", rln_creds_source=RLN_CREDENTIALS, rln_creds_id="1", - rln_relay_membership_index="1", + rln_relay_membership_index=self.resolve_rln_membership_index(0, **kwargs), rln_keystore_prefix=self.keystore_prefixes[0], **kwargs, ) @@ -78,7 +90,7 @@ class StepsRLN(StepsCommon): discv5_bootstrap_node=self.enr_uri, rln_creds_source=RLN_CREDENTIALS, rln_creds_id="2", - rln_relay_membership_index="1", + rln_relay_membership_index=self.resolve_rln_membership_index(1, **kwargs), rln_keystore_prefix=self.keystore_prefixes[1], **kwargs, ) @@ -101,7 +113,7 @@ class StepsRLN(StepsCommon): discv5_bootstrap_node=self.enr_uri, rln_creds_source=RLN_CREDENTIALS, rln_creds_id=f"{index + 3}", - rln_relay_membership_index="1", + rln_relay_membership_index=self.resolve_rln_membership_index(index + 2, **kwargs), rln_keystore_prefix=self.keystore_prefixes[index + 2], **kwargs, ) @@ -118,7 +130,7 @@ class StepsRLN(StepsCommon): lightpushnode=self.multiaddr_list[0], rln_creds_source=RLN_CREDENTIALS, rln_creds_id="2", - rln_relay_membership_index="1", + rln_relay_membership_index=self.resolve_rln_membership_index(1, **kwargs), rln_keystore_prefix=self.keystore_prefixes[1], **kwargs, ) @@ -131,7 +143,23 @@ class StepsRLN(StepsCommon): def register_rln_single_node(self, prefix="", **kwargs): logger.debug("Registering RLN credentials for single node") self.node = WakuNode(DEFAULT_NWAKU, f"node_{gen_step_id()}") - self.node.register_rln(rln_keystore_prefix=prefix, rln_creds_source=kwargs["rln_creds_source"], rln_creds_id=kwargs["rln_creds_id"]) + return self.node.register_rln(rln_keystore_prefix=prefix, rln_creds_source=kwargs["rln_creds_source"], rln_creds_id=kwargs["rln_creds_id"]) + + @allure.step + def resolve_rln_membership_index(self, index, **kwargs): + explicit_index = kwargs.get("rln_relay_membership_index") + if explicit_index is not None: + return str(explicit_index) + + if len(self.rln_membership_indexes) > index and self.rln_membership_indexes[index] is not None: + inferred_index = str(self.rln_membership_indexes[index]) + logger.debug(f"Using inferred RLN membership index for position {index}: {inferred_index}") + return inferred_index + + raise ValueError( + f"RLN membership index for position {index} is not available. " + "Register credentials and persist rln_membership_indexes together with keystore_prefixes before node startup." + ) @allure.step def check_rln_registration(self, prefix, key_id): diff --git a/src/test_data.py b/src/test_data.py index 02ad8c2fc..21510e6fd 100644 --- a/src/test_data.py +++ b/src/test_data.py @@ -168,7 +168,7 @@ SAMPLE_TIMESTAMPS = [ {"description": "Missing", "value": None, "valid_for": []}, ] -PUBSUB_TOPICS_RLN = ["/waku/2/rs/1/0"] +PUBSUB_TOPICS_RLN = [f"/waku/2/rs/{DEFAULT_CLUSTER_ID}/0"] LOG_ERROR_KEYWORDS = [ "crash", diff --git a/tests/relay/test_rln.py b/tests/relay/test_rln.py index 42f479b29..73e42864d 100644 --- a/tests/relay/test_rln.py +++ b/tests/relay/test_rln.py @@ -17,14 +17,18 @@ logger = get_custom_logger(__name__) class TestRelayRLN(StepsRLN, StepsRelay): SAMPLE_INPUTS_RLN = SAMPLE_INPUTS + SAMPLE_INPUTS + SAMPLE_INPUTS - def test_single_node_registration(self, pytestconfig): - pytestconfig.cache.set("keystore-prefixes", self.register_rln_relay_nodes(1, [])) - @pytest.mark.smoke def test_valid_payloads_lightpush_at_spam_rate(self, pytestconfig): message_limit = 1 epoch_sec = 1 - pytestconfig.cache.set("keystore-prefixes", self.register_rln_relay_nodes(2, [])) + rln_state = self.register_rln_relay_nodes(2, []) + pytestconfig.cache.set( + "keystore-prefixes", + { + "keystore_prefixes": rln_state["keystore_prefixes"], + "rln_membership_indexes": rln_state["rln_membership_indexes"], + }, + ) self.setup_first_rln_relay_node(lightpush="true", rln_relay_user_message_limit=message_limit, rln_relay_epoch_sec=epoch_sec) self.setup_second_rln_lightpush_node(rln_relay_user_message_limit=message_limit, rln_relay_epoch_sec=epoch_sec) self.subscribe_main_relay_nodes() @@ -43,7 +47,14 @@ class TestRelayRLN(StepsRLN, StepsRelay): def test_valid_payloads_at_slow_rate(self, pytestconfig): message_limit = 20 - self.register_rln_relay_nodes(0, pytestconfig.cache.get("keystore-prefixes", [])) + rln_state = self.register_rln_relay_nodes(2, []) + pytestconfig.cache.set( + "keystore-prefixes", + { + "keystore_prefixes": rln_state["keystore_prefixes"], + "rln_membership_indexes": rln_state["rln_membership_indexes"], + }, + ) self.setup_main_rln_relay_nodes(rln_relay_user_message_limit=message_limit, rln_relay_epoch_sec=600) self.subscribe_main_relay_nodes() failed_payloads = [] @@ -65,7 +76,10 @@ class TestRelayRLN(StepsRLN, StepsRelay): def test_valid_payloads_at_spam_rate(self, pytestconfig): message_limit = 20 epoch_sec = 600 - self.register_rln_relay_nodes(0, pytestconfig.cache.get("keystore-prefixes", [])) + self.register_rln_relay_nodes( + 0, + pytestconfig.cache.get("keystore-prefixes", {"keystore_prefixes": [], "rln_membership_indexes": []}), + ) self.setup_main_rln_relay_nodes(rln_relay_user_message_limit=message_limit, rln_relay_epoch_sec=epoch_sec) self.subscribe_main_relay_nodes() start = math.trunc(time()) @@ -82,7 +96,10 @@ class TestRelayRLN(StepsRLN, StepsRelay): assert "RLN validation failed" or "NonceLimitReached" in str(e) def test_valid_payload_at_variable_rate(self, pytestconfig): - self.register_rln_relay_nodes(0, pytestconfig.cache.get("keystore-prefixes", [])) + self.register_rln_relay_nodes( + 0, + pytestconfig.cache.get("keystore-prefixes", {"keystore_prefixes": [], "rln_membership_indexes": []}), + ) self.setup_main_rln_relay_nodes(rln_relay_user_message_limit=1, rln_relay_epoch_sec=1) self.subscribe_main_relay_nodes() payload_desc = SAMPLE_INPUTS[0]["description"] @@ -107,7 +124,10 @@ class TestRelayRLN(StepsRLN, StepsRelay): def test_valid_payloads_random_epoch_at_slow_rate(self, pytestconfig): epoch_sec = random.randint(2, 5) - self.register_rln_relay_nodes(0, pytestconfig.cache.get("keystore-prefixes", [])) + self.register_rln_relay_nodes( + 0, + pytestconfig.cache.get("keystore-prefixes", {"keystore_prefixes": [], "rln_membership_indexes": []}), + ) self.setup_main_rln_relay_nodes(rln_relay_user_message_limit=1, rln_relay_epoch_sec=epoch_sec) self.subscribe_main_relay_nodes() failed_payloads = [] @@ -124,7 +144,10 @@ class TestRelayRLN(StepsRLN, StepsRelay): def test_valid_payloads_random_user_message_limit(self, pytestconfig): user_message_limit = random.randint(2, 4) - self.register_rln_relay_nodes(0, pytestconfig.cache.get("keystore-prefixes", [])) + self.register_rln_relay_nodes( + 0, + pytestconfig.cache.get("keystore-prefixes", {"keystore_prefixes": [], "rln_membership_indexes": []}), + ) self.setup_main_rln_relay_nodes(rln_relay_user_message_limit=user_message_limit, rln_relay_epoch_sec=1) self.subscribe_main_relay_nodes() failed_payloads = [] @@ -143,7 +166,14 @@ class TestRelayRLN(StepsRLN, StepsRelay): def test_valid_payloads_dynamic_at_spam_rate(self, pytestconfig): message_limit = 100 epoch_sec = 600 - pytestconfig.cache.set("keystore-prefixes", self.register_rln_relay_nodes(2, [])) + rln_state = self.register_rln_relay_nodes(2, []) + pytestconfig.cache.set( + "keystore-prefixes", + { + "keystore_prefixes": rln_state["keystore_prefixes"], + "rln_membership_indexes": rln_state["rln_membership_indexes"], + }, + ) self.setup_main_rln_relay_nodes( rln_relay_user_message_limit=message_limit, rln_relay_epoch_sec=epoch_sec, @@ -168,7 +198,14 @@ class TestRelayRLN(StepsRLN, StepsRelay): @pytest.mark.timeout(600) def test_valid_payloads_dynamic_at_slow_rate(self, pytestconfig): message_limit = 100 - pytestconfig.cache.set("keystore-prefixes", self.register_rln_relay_nodes(2, [])) + rln_state = self.register_rln_relay_nodes(2, []) + pytestconfig.cache.set( + "keystore-prefixes", + { + "keystore_prefixes": rln_state["keystore_prefixes"], + "rln_membership_indexes": rln_state["rln_membership_indexes"], + }, + ) self.setup_main_rln_relay_nodes( rln_relay_user_message_limit=message_limit, rln_relay_epoch_sec=600, @@ -194,7 +231,10 @@ class TestRelayRLN(StepsRLN, StepsRelay): def test_valid_payloads_n1_with_rln_n2_without_rln_at_spam_rate(self, pytestconfig): message_limit = 1 epoch_sec = 1 - self.register_rln_relay_nodes(0, pytestconfig.cache.get("keystore-prefixes", [])) + self.register_rln_relay_nodes( + 0, + pytestconfig.cache.get("keystore-prefixes", {"keystore_prefixes": [], "rln_membership_indexes": []}), + ) self.setup_first_rln_relay_node(rln_relay_user_message_limit=message_limit, rln_relay_epoch_sec=epoch_sec) self.setup_second_relay_node() self.subscribe_main_relay_nodes() @@ -211,7 +251,14 @@ class TestRelayRLN(StepsRLN, StepsRelay): assert "RLN validation failed" or "NonceLimitReached" in str(e) def test_valid_payloads_with_optional_nodes_at_slow_rate(self, pytestconfig): - pytestconfig.cache.set("keystore-prefixes", self.register_rln_relay_nodes(5, [])) + rln_state = self.register_rln_relay_nodes(5, []) + pytestconfig.cache.set( + "keystore-prefixes", + { + "keystore_prefixes": rln_state["keystore_prefixes"], + "rln_membership_indexes": rln_state["rln_membership_indexes"], + }, + ) self.setup_main_rln_relay_nodes(rln_relay_user_message_limit=1, rln_relay_epoch_sec=1) self.setup_optional_rln_relay_nodes(rln_relay_user_message_limit=1, rln_relay_epoch_sec=1) self.subscribe_main_relay_nodes() @@ -230,7 +277,10 @@ class TestRelayRLN(StepsRLN, StepsRelay): assert not failed_payloads, f"Payloads failed: {failed_payloads}" def test_valid_payloads_with_optional_nodes_at_spam_rate(self, pytestconfig): - self.register_rln_relay_nodes(0, pytestconfig.cache.get("keystore-prefixes", [])) + self.register_rln_relay_nodes( + 0, + pytestconfig.cache.get("keystore-prefixes", {"keystore_prefixes": [], "rln_membership_indexes": []}), + ) self.setup_main_rln_relay_nodes(rln_relay_user_message_limit=1, rln_relay_epoch_sec=1) self.setup_optional_rln_relay_nodes(rln_relay_user_message_limit=1, rln_relay_epoch_sec=1) self.subscribe_main_relay_nodes()