From c95fff8ac35cbb9a3aab574aa748c330ed91bec9 Mon Sep 17 00:00:00 2001 From: fbarbu15 Date: Wed, 27 Nov 2024 18:05:52 +0200 Subject: [PATCH 1/6] chore: fix go waku lightprotocol tests (#91) * chore: fix go waku lightprotocol tests * chore: fix go waku lightprotocol tests --- src/node/waku_node.py | 5 ++++- src/steps/light_push.py | 2 +- tests/e2e/test_e2e.py | 8 +++++++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/node/waku_node.py b/src/node/waku_node.py index 3669fa93..44886f95 100644 --- a/src/node/waku_node.py +++ b/src/node/waku_node.py @@ -14,7 +14,7 @@ from src.node.api_clients.rest import REST from src.node.docker_mananger import DockerManager from src.env_vars import DOCKER_LOG_DIR from src.data_storage import DS -from src.test_data import DEFAULT_CLUSTER_ID, LOG_ERROR_KEYWORDS +from src.test_data import DEFAULT_CLUSTER_ID, LOG_ERROR_KEYWORDS, VALID_PUBSUB_TOPICS logger = get_custom_logger(__name__) @@ -152,6 +152,9 @@ class WakuNode: default_args.update(sanitize_docker_flags(kwargs)) + if self.is_gowaku() and default_args.get("relay") == "false": + default_args["pubsub-topic"] = VALID_PUBSUB_TOPICS[1] + rln_args, rln_creds_set, keystore_path = self.parse_rln_credentials(default_args, False) default_args.pop("rln-creds-id", None) diff --git a/src/steps/light_push.py b/src/steps/light_push.py index ddd10a05..d4cb7d66 100644 --- a/src/steps/light_push.py +++ b/src/steps/light_push.py @@ -65,7 +65,7 @@ class StepsLightPush(StepsCommon): else: pytest.skip("ADDITIONAL_NODES/node_list is empty, cannot run test") for index, node in enumerate(nodes): - self.start_receiving_node(node, node_index=index + 2, lightpush="true", relay="true", **kwargs) + self.start_receiving_node(node, node_index=index + 2, lightpush="true", relay="true", pubsub_topic=self.test_pubsub_topic, **kwargs) @allure.step def setup_first_lightpush_node(self, lightpush="true", relay="false", **kwargs): diff --git a/tests/e2e/test_e2e.py b/tests/e2e/test_e2e.py index 2f70444a..e40131a3 100644 --- a/tests/e2e/test_e2e.py +++ b/tests/e2e/test_e2e.py @@ -348,7 +348,13 @@ class TestE2E(StepsFilter, StepsStore, StepsRelay, StepsLightPush): logger.debug("Start 3 nodes with their corresponding config") self.node1.start(relay="true", store="true") self.node2.start(relay="true", store="true", filter="true", discv5_bootstrap_node=self.node1.get_enr_uri()) - self.node3.start(relay="true", filternode=self.node2.get_multiaddr_with_id(), store="false", discv5_bootstrap_node=self.node2.get_enr_uri()) + self.node3.start( + relay="true", + filternode=self.node2.get_multiaddr_with_id(), + store="false", + pubsub_topic=self.test_pubsub_topic, + discv5_bootstrap_node=self.node2.get_enr_uri(), + ) logger.debug("Subscribe nodes to relay pubsub topics") self.node1.set_relay_subscriptions([self.test_pubsub_topic]) From 5caedf13c441f6dad220dc455322901276709984 Mon Sep 17 00:00:00 2001 From: fbarbu15 Date: Fri, 6 Dec 2024 10:54:35 +0200 Subject: [PATCH 2/6] chore: metric updates (#93) * chore: fix rln tests * chore: fix rln tests * revert wait_for_node_sec --- src/test_data.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test_data.py b/src/test_data.py index a56cc45f..29f5b6c2 100644 --- a/src/test_data.py +++ b/src/test_data.py @@ -218,6 +218,8 @@ METRICS_WITH_INITIAL_VALUE_ZERO = [ "waku_rln_instance_creation_duration_seconds", "waku_rln_membership_insertion_duration_seconds", "waku_rln_membership_credentials_import_duration_seconds", + "waku_rln_proof_remining", + "waku_rln_proofs_generated_total", "libp2p_pubsub_sig_verify_success_total", "libp2p_pubsub_sig_verify_failure_total", "libp2p_pubsub_disconnects_over_non_priority_queue_limit_total", From 8425c0fed731be0dace586a19b9835bc92e4971d Mon Sep 17 00:00:00 2001 From: fbarbu15 Date: Tue, 10 Dec 2024 16:58:28 +0200 Subject: [PATCH 3/6] chore: fix wrong store node type (#94) * chore: fix wrong store node type * chore: adjust flaky LP test --- src/steps/light_push.py | 4 +++- tests/store/test_reliability.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/steps/light_push.py b/src/steps/light_push.py index d4cb7d66..52eedb62 100644 --- a/src/steps/light_push.py +++ b/src/steps/light_push.py @@ -143,4 +143,6 @@ class StepsLightPush(StepsCommon): @retry(stop=stop_after_delay(120), wait=wait_fixed(1), reraise=True) def subscribe_and_light_push_with_retry(self): self.subscribe_to_pubsub_topics_via_relay() - self.check_light_pushed_message_reaches_receiving_peer() + self.light_push_node1.send_light_push_message(self.create_payload()) + get_messages_response = self.main_receiving_nodes[0].get_relay_messages(self.test_pubsub_topic) + assert len(get_messages_response) >= 1, f"Expected al least 1 message but got {len(get_messages_response)}" diff --git a/tests/store/test_reliability.py b/tests/store/test_reliability.py index ed1a4e5d..7c6ee4ca 100644 --- a/tests/store/test_reliability.py +++ b/tests/store/test_reliability.py @@ -85,7 +85,7 @@ class TestReliability(StepsStore): def test_message_relayed_while_store_node_is_stopped_without_removing(self): self.setup_first_publishing_node(store="true", relay="true") - self.setup_first_store_node(store="false", relay="true", remove_container=False) + self.setup_first_store_node(store="false", relay="false", remove_container=False) self.subscribe_to_pubsub_topics_via_relay() self.publish_message() self.check_published_message_is_stored(page_size=5) From 013c6e8bb00146eca68c67a2e4a5ad34cc7a4b3a Mon Sep 17 00:00:00 2001 From: AYAHASSAN287 <49167455+AYAHASSAN287@users.noreply.github.com> Date: Thu, 12 Dec 2024 12:03:42 +0300 Subject: [PATCH 4/6] smoke_tests (#90) * Selecting initial smoke tests set * adding mark "smoke" in pytest.ini * adding mark smoke to workflow file * Remove allure reporting from yml file * Adding more smoke tests * Add PR image to new nwaku workflow * change nwaku_daily.yml to test the PR job changes * remove dependencies from yml file * Remove secrets check from container yml file * remove secrets from all files * change image tag * revert image tag * Revert nwaku_daily.yml to origin * Adding PR number to choose image * adding docker-build-image to yml file * adding docker-image-build dependencies * Adding quay user & password * Adding quay username & password * Fix yml build * Remove changes causing errors * remove tests part to speedup the job * add flag workflow_call: for reusable workflow * checkout on branch instead of master * trying to fetch from branch * Check out specific branch * make node1 input * Adding node1 as input * Add type to required input * fix node input format * change input node1 format * Delete .github/workflows/container-image.yml * Delete .github/workflows/nim_nwaku_pr.yml * Create 2 files for PR tests * revert original yml files * Fix review points on PR * fix review points * revert test_common file to master * revert nim_waku_daily to master * Fix review points --- .github/workflows/nim_waku_PR.yml | 27 +++++++++ .github/workflows/test_PR_image.yml | 89 +++++++++++++++++++++++++++++ pytest.ini | 2 + tests/discv5/test_discv5.py | 6 ++ tests/e2e/test_e2e.py | 1 + tests/filter/test_get_messages.py | 1 + tests/filter/test_ping.py | 1 + tests/relay/test_multiple_nodes.py | 1 + tests/relay/test_rln.py | 2 + tests/relay/test_subscribe.py | 2 + tests/store/test_running_nodes.py | 2 + tests/store/test_topics.py | 1 + 12 files changed, 135 insertions(+) create mode 100644 .github/workflows/nim_waku_PR.yml create mode 100644 .github/workflows/test_PR_image.yml diff --git a/.github/workflows/nim_waku_PR.yml b/.github/workflows/nim_waku_PR.yml new file mode 100644 index 00000000..6a6d0318 --- /dev/null +++ b/.github/workflows/nim_waku_PR.yml @@ -0,0 +1,27 @@ +name: Nim -> Interop Tests PR + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: false + +on: + workflow_dispatch: + inputs: + node_nwaku: + required: true + type: string + workflow_call: + inputs: + node_nwaku: + required: true + type: string + +jobs: + test-pr: + uses: ./.github/workflows/test_PR_image.yml + secrets: inherit + with: + node1: ${{ inputs.node_nwaku }} + node2: ${{ inputs.node_nwaku }} + additional_nodes: ${{ inputs.node_nwaku }} + caller: "nim" \ No newline at end of file diff --git a/.github/workflows/test_PR_image.yml b/.github/workflows/test_PR_image.yml new file mode 100644 index 00000000..3f88abd5 --- /dev/null +++ b/.github/workflows/test_PR_image.yml @@ -0,0 +1,89 @@ + +name: Interop Tests PR + +on: + workflow_call: + inputs: + node1: + required: true + type: string + description: "Node that usually publishes messages. Used for all tests" + default: "wakuorg/go-waku:latest" + node2: + required: true + description: "Node that usually queries for published messages. Used for all tests" + type: string + default: "wakuorg/nwaku:latest" + additional_nodes: + required: false + description: "Additional optional nodes used in e2e tests, separated by ," + type: string + default: "wakuorg/nwaku:latest,wakuorg/go-waku:latest,wakuorg/nwaku:latest" + caller: + required: false + description: "Workflow caller. Used in reporting" + type: string + +env: + FORCE_COLOR: "1" + NODE_1: ${{ inputs.node1 }} + NODE_2: ${{ inputs.node2 }} + ADDITIONAL_NODES: ${{ inputs.additional_nodes }} + CALLER: ${{ inputs.caller || 'manual' }} + RLN_CREDENTIALS: ${{ secrets.RLN_CREDENTIALS }} + +jobs: + + tests: + name: tests + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + + - uses: actions/checkout@v4 + with: + repository: waku-org/waku-interop-tests + + - uses: actions/setup-python@v4 + with: + + python-version: '3.12' + cache: 'pip' + + - run: pip install -r requirements.txt + + - name: Run tests + timeout-minutes: 30 + run: | + pytest -m 'smoke' -n 4 --dist loadgroup --reruns 1 --junit-xml=pytest_results.xml + + - name: Test Report + if: always() + uses: dorny/test-reporter@95058abb17504553158e70e2c058fe1fda4392c2 + with: + name: Pytest JUnit Test Report + path: pytest_results.xml + reporter: java-junit + use-actions-summary: 'true' + + - name: Create job summary + if: always() + env: + JOB_STATUS: ${{ job.status }} + run: | + echo "## Run Information" >> $GITHUB_STEP_SUMMARY + echo "- **Event**: ${{ github.event_name }}" >> $GITHUB_STEP_SUMMARY + echo "- **Actor**: ${{ github.actor }}" >> $GITHUB_STEP_SUMMARY + echo "- **Node1**: ${{ env.NODE_1 }}" >> $GITHUB_STEP_SUMMARY + echo "- **Node2**: ${{ env.NODE_2 }}" >> $GITHUB_STEP_SUMMARY + echo "- **Additonal Nodes**: ${{ env.ADDITIONAL_NODES }}" >> $GITHUB_STEP_SUMMARY + if [ "$JOB_STATUS" != "success" ]; then + echo "There are failures with nwaku node. cc <@&1111608257824440330>" >> $GITHUB_STEP_SUMMARY + fi + { + echo 'JOB_SUMMARY<> $GITHUB_ENV + + diff --git a/pytest.ini b/pytest.ini index a876cae5..958bc09c 100644 --- a/pytest.ini +++ b/pytest.ini @@ -6,3 +6,5 @@ log_file = log/test.log log_cli_format = %(asctime)s %(name)s %(levelname)s %(message)s log_file_format = %(asctime)s %(name)s %(levelname)s %(message)s timeout = 300 +markers = + smoke: marks tests as smoke test (deselect with '-m "not smoke"') \ No newline at end of file diff --git a/tests/discv5/test_discv5.py b/tests/discv5/test_discv5.py index 2557ea90..b8ab88b3 100644 --- a/tests/discv5/test_discv5.py +++ b/tests/discv5/test_discv5.py @@ -1,3 +1,5 @@ +import pytest + from src.env_vars import NODE_1, NODE_2 from src.libs.custom_logger import get_custom_logger from src.node.waku_node import WakuNode @@ -11,6 +13,7 @@ logger = get_custom_logger(__name__) class TestDiscv5(StepsRelay, StepsFilter, StepsStore, StepsLightPush): + @pytest.mark.smoke def running_a_node(self, image, **kwargs): node = WakuNode(image, f"node{len(self.main_nodes) + 1}_{self.test_id}") node.start(**kwargs) @@ -25,6 +28,7 @@ class TestDiscv5(StepsRelay, StepsFilter, StepsStore, StepsLightPush): def wait_for_lightpushed_message_to_be_stored(self): self.check_light_pushed_message_reaches_receiving_peer(peer_list=[self.receiving_node1, self.receiving_node2]) + @pytest.mark.smoke def test_relay(self): self.node1 = self.running_a_node(NODE_1, relay="true") self.node2 = self.running_a_node(NODE_2, relay="true", discv5_bootstrap_node=self.node1.get_enr_uri()) @@ -32,6 +36,7 @@ class TestDiscv5(StepsRelay, StepsFilter, StepsStore, StepsLightPush): self.ensure_relay_subscriptions_on_nodes(self.main_nodes, [self.test_pubsub_topic]) self.wait_for_published_message_to_reach_relay_peer() + @pytest.mark.smoke def test_filter(self): self.node1 = self.running_a_node(NODE_1, relay="true", filter="true") self.node2 = self.running_a_node( @@ -54,6 +59,7 @@ class TestDiscv5(StepsRelay, StepsFilter, StepsStore, StepsLightPush): self.subscribe_to_pubsub_topics_via_relay(self.main_nodes) self.wait_for_published_message_to_be_stored() + @pytest.mark.smoke def test_lightpush(self): self.receiving_node1 = self.running_a_node(NODE_1, lightpush="true", relay="true") self.receiving_node2 = self.running_a_node(NODE_1, lightpush="false", relay="true", discv5_bootstrap_node=self.receiving_node1.get_enr_uri()) diff --git a/tests/e2e/test_e2e.py b/tests/e2e/test_e2e.py index e40131a3..11bdff7d 100644 --- a/tests/e2e/test_e2e.py +++ b/tests/e2e/test_e2e.py @@ -304,6 +304,7 @@ class TestE2E(StepsFilter, StepsStore, StepsRelay, StepsLightPush): assert len(response_list) == max_subscribed_nodes, "Received message count doesn't match sent " @pytest.mark.skipif("go-waku" in NODE_2, reason="Test works only with nwaku") + @pytest.mark.smoke def test_store_filter_interaction_with_six_nodes(self): logger.debug("Create 6 nodes") self.node4 = WakuNode(NODE_2, f"node4_{self.test_id}") diff --git a/tests/filter/test_get_messages.py b/tests/filter/test_get_messages.py index 1aa8e51a..a2227e55 100644 --- a/tests/filter/test_get_messages.py +++ b/tests/filter/test_get_messages.py @@ -11,6 +11,7 @@ logger = get_custom_logger(__name__) # here we will also implicitly test filter push, see: https://rfc.vac.dev/spec/12/#messagepush @pytest.mark.usefixtures("setup_main_relay_node", "setup_main_filter_node", "subscribe_main_nodes") class TestFilterGetMessages(StepsFilter): + @pytest.mark.smoke def test_filter_get_message_with_valid_payloads(self): failed_payloads = [] for payload in SAMPLE_INPUTS: diff --git a/tests/filter/test_ping.py b/tests/filter/test_ping.py index 497318ca..1328ca7c 100644 --- a/tests/filter/test_ping.py +++ b/tests/filter/test_ping.py @@ -5,6 +5,7 @@ from src.steps.filter import StepsFilter @pytest.mark.usefixtures("setup_main_relay_node", "setup_main_filter_node") class TestFilterPing(StepsFilter): + @pytest.mark.smoke def test_filter_ping_on_subscribed_peer(self, subscribe_main_nodes): self.ping_filter_subscriptions(str(uuid4())) diff --git a/tests/relay/test_multiple_nodes.py b/tests/relay/test_multiple_nodes.py index 3bb007f1..80622028 100644 --- a/tests/relay/test_multiple_nodes.py +++ b/tests/relay/test_multiple_nodes.py @@ -4,6 +4,7 @@ from src.steps.relay import StepsRelay @pytest.mark.usefixtures("setup_main_relay_nodes", "setup_optional_relay_nodes", "subscribe_main_relay_nodes") class TestRelayMultipleNodes(StepsRelay): + @pytest.mark.smoke def test_first_node_to_start_publishes(self, subscribe_optional_relay_nodes, relay_warm_up): self.check_published_message_reaches_relay_peer() diff --git a/tests/relay/test_rln.py b/tests/relay/test_rln.py index 4707635c..921ca6b3 100644 --- a/tests/relay/test_rln.py +++ b/tests/relay/test_rln.py @@ -18,6 +18,7 @@ logger = get_custom_logger(__name__) class TestRelayRLN(StepsRLN, StepsRelay): SAMPLE_INPUTS_RLN = SAMPLE_INPUTS + SAMPLE_INPUTS + SAMPLE_INPUTS + @pytest.mark.smoke def test_valid_payloads_lightpush_at_spam_rate(self, pytestconfig): message_limit = 1 epoch_sec = 1 @@ -58,6 +59,7 @@ class TestRelayRLN(StepsRLN, StepsRelay): if i == message_limit - 1: break + @pytest.mark.smoke def test_valid_payloads_at_spam_rate(self, pytestconfig): message_limit = 20 epoch_sec = 600 diff --git a/tests/relay/test_subscribe.py b/tests/relay/test_subscribe.py index 21018137..d809eada 100644 --- a/tests/relay/test_subscribe.py +++ b/tests/relay/test_subscribe.py @@ -9,9 +9,11 @@ logger = get_custom_logger(__name__) @pytest.mark.usefixtures("setup_main_relay_nodes") class TestRelaySubscribe(StepsRelay): + @pytest.mark.smoke def test_relay_no_subscription(self): self.check_publish_without_relay_subscription(self.test_pubsub_topic) + @pytest.mark.smoke def test_relay_subscribe_to_single_pubsub_topic(self): self.ensure_relay_subscriptions_on_nodes(self.main_nodes, [self.test_pubsub_topic]) self.wait_for_published_message_to_reach_relay_peer() diff --git a/tests/store/test_running_nodes.py b/tests/store/test_running_nodes.py index 51727dec..547bf0c8 100644 --- a/tests/store/test_running_nodes.py +++ b/tests/store/test_running_nodes.py @@ -59,6 +59,7 @@ class TestRunningNodes(StepsStore): except Exception as ex: assert "failed to negotiate protocol: protocols not supported" in str(ex) or "PEER_DIAL_FAILURE" in str(ex) + @pytest.mark.smoke def test_store_lightpushed_message(self): self.setup_first_publishing_node(store="true", relay="true", lightpush="true") self.setup_second_publishing_node(store="false", relay="true") @@ -67,6 +68,7 @@ class TestRunningNodes(StepsStore): self.publish_message(via="lightpush", sender=self.store_node1) self.check_published_message_is_stored(page_size=5, ascending="true") + @pytest.mark.smoke def test_store_with_filter(self): self.setup_first_publishing_node(store="true", relay="true", filter="true") self.setup_first_store_node(store="false", relay="false", filter="true") diff --git a/tests/store/test_topics.py b/tests/store/test_topics.py index 521108d5..6cf48d6a 100644 --- a/tests/store/test_topics.py +++ b/tests/store/test_topics.py @@ -68,6 +68,7 @@ class TestTopics(StepsStore): ) assert len(store_response["messages"]) == 0, "Message count mismatch" + @pytest.mark.smoke def test_store_with_both_pubsub_topic_and_content_topic(self): for node in self.store_nodes: for index, content_topic in enumerate(CONTENT_TOPICS_DIFFERENT_SHARDS): From dfdb7d32752bdf5624e76e501482a47e58864066 Mon Sep 17 00:00:00 2001 From: Roman Zajic Date: Fri, 13 Dec 2024 18:36:19 +0800 Subject: [PATCH 5/6] fix: comment out onchain rln tests (#95) --- tests/relay/test_rln.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/relay/test_rln.py b/tests/relay/test_rln.py index 921ca6b3..1a291157 100644 --- a/tests/relay/test_rln.py +++ b/tests/relay/test_rln.py @@ -136,6 +136,7 @@ class TestRelayRLN(StepsRLN, StepsRelay): failed_payloads.append(payload["description"]) assert not failed_payloads, f"Payloads failed: {failed_payloads}" + @pytest.mark.skip(reason="Waiting for issue resolution https://github.com/waku-org/nwaku/issues/3208") @pytest.mark.timeout(600) def test_valid_payloads_dynamic_at_spam_rate(self, pytestconfig): message_limit = 100 @@ -161,6 +162,7 @@ class TestRelayRLN(StepsRLN, StepsRelay): except Exception as e: assert "RLN validation failed" or "NonceLimitReached" in str(e) + @pytest.mark.skip(reason="Waiting for issue resolution https://github.com/waku-org/nwaku/issues/3208") @pytest.mark.timeout(600) def test_valid_payloads_dynamic_at_slow_rate(self, pytestconfig): message_limit = 100 From 25b995295b3513674438028fd8e7dea4728a331b Mon Sep 17 00:00:00 2001 From: fbarbu15 Date: Fri, 20 Dec 2024 04:22:36 +0200 Subject: [PATCH 6/6] chore: add rendezvousPeerFoundTotal_total metric (#97) --- src/test_data.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test_data.py b/src/test_data.py index 29f5b6c2..6f378a10 100644 --- a/src/test_data.py +++ b/src/test_data.py @@ -372,4 +372,5 @@ METRICS_WITH_INITIAL_VALUE_ZERO = [ "waku_store_peers", "waku_px_peers", "waku_dnsdisc_discovered", + "rendezvousPeerFoundTotal_total", ]