From f108cb9ab382e0c22b49ed42e7f2603e39dcbd7e Mon Sep 17 00:00:00 2001 From: Roman Date: Mon, 6 May 2024 13:33:12 +0800 Subject: [PATCH] fix: run RLN tests in serial - add grouping of tests to runner - use smaller dataset for spam rate tests --- .github/workflows/test_common.yml | 2 +- tests/relay/test_rln.py | 22 +++++++++++++++++----- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test_common.yml b/.github/workflows/test_common.yml index 1774d891..6863cff4 100644 --- a/.github/workflows/test_common.yml +++ b/.github/workflows/test_common.yml @@ -49,7 +49,7 @@ jobs: - run: pip install -r requirements.txt - name: Run tests - run: pytest -n 4 --reruns 2 --alluredir=allure-results + run: pytest -n 4 --dist loadgroup --reruns 2 --alluredir=allure-results - name: Get allure history if: always() diff --git a/tests/relay/test_rln.py b/tests/relay/test_rln.py index 5240b399..aecd637a 100644 --- a/tests/relay/test_rln.py +++ b/tests/relay/test_rln.py @@ -16,6 +16,7 @@ logger = get_custom_logger(__name__) @pytest.mark.usefixtures("register_main_rln_relay_nodes") @pytest.mark.skipif("go-waku" in (NODE_1 + NODE_2), reason="Test works only with nwaku") class TestRelayRLN(StepsRLN, StepsRelay): + @pytest.mark.xdist_group(name="RLN serial tests") def test_valid_payloads_at_slow_rate(self): self.setup_main_rln_relay_nodes() self.subscribe_main_relay_nodes() @@ -31,11 +32,12 @@ class TestRelayRLN(StepsRLN, StepsRelay): delay(1) assert not failed_payloads, f"Payloads failed: {failed_payloads}" + @pytest.mark.xdist_group(name="RLN serial tests") def test_valid_payloads_at_spam_rate(self): self.setup_main_rln_relay_nodes() self.subscribe_main_relay_nodes() previous = math.trunc(time()) - for i, payload in enumerate(SAMPLE_INPUTS): + for i, payload in enumerate(SAMPLE_INPUTS[:5]): logger.debug(f'Running test with payload {payload["description"]}') message = self.create_message(payload=to_base64(payload["value"])) try: @@ -49,6 +51,7 @@ class TestRelayRLN(StepsRLN, StepsRelay): except Exception as e: assert "RLN validation failed" in str(e) + @pytest.mark.xdist_group(name="RLN serial tests") def test_valid_payload_at_variable_rate(self): self.setup_main_rln_relay_nodes() self.subscribe_main_relay_nodes() @@ -71,6 +74,7 @@ class TestRelayRLN(StepsRLN, StepsRelay): except Exception as e: assert "RLN validation failed" in str(e) + @pytest.mark.xdist_group(name="RLN serial tests") def test_valid_payloads_random_epoch_at_slow_rate(self): epoch_sec = random.randint(2, 5) self.setup_main_rln_relay_nodes(rln_relay_epoch_sec=epoch_sec) @@ -87,6 +91,7 @@ class TestRelayRLN(StepsRLN, StepsRelay): delay(epoch_sec) assert not failed_payloads, f"Payloads failed: {failed_payloads}" + @pytest.mark.xdist_group(name="RLN serial tests") @pytest.mark.skip(reason="waiting for RLN v2 implementation") def test_valid_payloads_random_user_message_limit(self): user_message_limit = random.randint(2, 4) @@ -103,6 +108,7 @@ class TestRelayRLN(StepsRLN, StepsRelay): failed_payloads.append(payload["description"]) assert not failed_payloads, f"Payloads failed: {failed_payloads}" + @pytest.mark.xdist_group(name="RLN serial tests") @pytest.mark.skip(reason="exceeding timeout, waiting for https://github.com/waku-org/nwaku/pull/2612 to be part of the release") @pytest.mark.timeout(600) def test_valid_payloads_dynamic_at_slow_rate(self): @@ -120,13 +126,14 @@ class TestRelayRLN(StepsRLN, StepsRelay): delay(1) assert not failed_payloads, f"Payloads failed: {failed_payloads}" + @pytest.mark.xdist_group(name="RLN serial tests") @pytest.mark.skip(reason="exceeding timeout, waiting for https://github.com/waku-org/nwaku/pull/2612 to be part of the release") @pytest.mark.timeout(600) def test_valid_payloads_dynamic_at_spam_rate(self): self.setup_main_rln_relay_nodes(rln_relay_dynamic="true", wait_for_node_sec=600) self.subscribe_main_relay_nodes() previous = math.trunc(time()) - for i, payload in enumerate(SAMPLE_INPUTS): + for i, payload in enumerate(SAMPLE_INPUTS[:5]): logger.debug(f'Running test with payload {payload["description"]}') message = self.create_message(payload=to_base64(payload["value"])) try: @@ -139,12 +146,13 @@ class TestRelayRLN(StepsRLN, StepsRelay): except Exception as e: assert "RLN validation failed" in str(e) + @pytest.mark.xdist_group(name="RLN serial tests") def test_valid_payloads_n1_with_rln_n2_without_rln_at_spam_rate(self): self.setup_first_rln_relay_node() self.setup_second_relay_node() self.subscribe_main_relay_nodes() previous = math.trunc(time()) - for i, payload in enumerate(SAMPLE_INPUTS): + for i, payload in enumerate(SAMPLE_INPUTS[:5]): logger.debug(f'Running test with payload {payload["description"]}') message = self.create_message(payload=to_base64(payload["value"])) try: @@ -157,6 +165,7 @@ class TestRelayRLN(StepsRLN, StepsRelay): except Exception as e: assert "RLN validation failed" in str(e) + @pytest.mark.xdist_group(name="RLN serial tests") @pytest.mark.skip(reason="Epoch settings aren't compatible across nodes") def test_valid_payloads_mixed_epoch_at_slow_rate(self): n1_epoch_sec = 5 @@ -176,13 +185,14 @@ class TestRelayRLN(StepsRLN, StepsRelay): delay(n1_epoch_sec) assert not failed_payloads, f"Payloads failed: {failed_payloads}" + @pytest.mark.xdist_group(name="RLN serial tests") @pytest.mark.skip(reason="waiting for NWAKU lightpush + RLN node implementation") def test_valid_payloads_lightpush_at_spam_rate(self): self.setup_first_rln_relay_node(lightpush="true") self.setup_second_lightpush_node() self.subscribe_main_relay_nodes() previous = math.trunc(time()) - for i, payload in enumerate(SAMPLE_INPUTS): + for i, payload in enumerate(SAMPLE_INPUTS[:5]): logger.debug(f'Running test with payload {payload["description"]}') message = self.create_message(payload=to_base64(payload["value"])) try: @@ -195,6 +205,7 @@ class TestRelayRLN(StepsRLN, StepsRelay): except Exception as e: assert "RLN validation failed" in str(e) + @pytest.mark.xdist_group(name="RLN serial tests") @pytest.mark.usefixtures("register_main_rln_relay_nodes", "register_optional_rln_relay_nodes") def test_valid_payloads_with_optional_nodes_at_slow_rate(self): self.setup_main_rln_relay_nodes() @@ -213,6 +224,7 @@ class TestRelayRLN(StepsRLN, StepsRelay): delay(1) assert not failed_payloads, f"Payloads failed: {failed_payloads}" + @pytest.mark.xdist_group(name="RLN serial tests") @pytest.mark.usefixtures("register_main_rln_relay_nodes", "register_optional_rln_relay_nodes") def test_valid_payloads_with_optional_nodes_at_spam_rate(self): self.setup_main_rln_relay_nodes() @@ -220,7 +232,7 @@ class TestRelayRLN(StepsRLN, StepsRelay): self.subscribe_main_relay_nodes() self.subscribe_optional_relay_nodes() previous = math.trunc(time()) - for i, payload in enumerate(SAMPLE_INPUTS): + for i, payload in enumerate(SAMPLE_INPUTS[:5]): logger.debug(f'Running test with payload {payload["description"]}') message = self.create_message(payload=to_base64(payload["value"])) try: