From 013c6e8bb00146eca68c67a2e4a5ad34cc7a4b3a Mon Sep 17 00:00:00 2001 From: AYAHASSAN287 <49167455+AYAHASSAN287@users.noreply.github.com> Date: Thu, 12 Dec 2024 12:03:42 +0300 Subject: [PATCH] smoke_tests (#90) * Selecting initial smoke tests set * adding mark "smoke" in pytest.ini * adding mark smoke to workflow file * Remove allure reporting from yml file * Adding more smoke tests * Add PR image to new nwaku workflow * change nwaku_daily.yml to test the PR job changes * remove dependencies from yml file * Remove secrets check from container yml file * remove secrets from all files * change image tag * revert image tag * Revert nwaku_daily.yml to origin * Adding PR number to choose image * adding docker-build-image to yml file * adding docker-image-build dependencies * Adding quay user & password * Adding quay username & password * Fix yml build * Remove changes causing errors * remove tests part to speedup the job * add flag workflow_call: for reusable workflow * checkout on branch instead of master * trying to fetch from branch * Check out specific branch * make node1 input * Adding node1 as input * Add type to required input * fix node input format * change input node1 format * Delete .github/workflows/container-image.yml * Delete .github/workflows/nim_nwaku_pr.yml * Create 2 files for PR tests * revert original yml files * Fix review points on PR * fix review points * revert test_common file to master * revert nim_waku_daily to master * Fix review points --- .github/workflows/nim_waku_PR.yml | 27 +++++++++ .github/workflows/test_PR_image.yml | 89 +++++++++++++++++++++++++++++ pytest.ini | 2 + tests/discv5/test_discv5.py | 6 ++ tests/e2e/test_e2e.py | 1 + tests/filter/test_get_messages.py | 1 + tests/filter/test_ping.py | 1 + tests/relay/test_multiple_nodes.py | 1 + tests/relay/test_rln.py | 2 + tests/relay/test_subscribe.py | 2 + tests/store/test_running_nodes.py | 2 + tests/store/test_topics.py | 1 + 12 files changed, 135 insertions(+) create mode 100644 .github/workflows/nim_waku_PR.yml create mode 100644 .github/workflows/test_PR_image.yml diff --git a/.github/workflows/nim_waku_PR.yml b/.github/workflows/nim_waku_PR.yml new file mode 100644 index 00000000..6a6d0318 --- /dev/null +++ b/.github/workflows/nim_waku_PR.yml @@ -0,0 +1,27 @@ +name: Nim -> Interop Tests PR + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: false + +on: + workflow_dispatch: + inputs: + node_nwaku: + required: true + type: string + workflow_call: + inputs: + node_nwaku: + required: true + type: string + +jobs: + test-pr: + uses: ./.github/workflows/test_PR_image.yml + secrets: inherit + with: + node1: ${{ inputs.node_nwaku }} + node2: ${{ inputs.node_nwaku }} + additional_nodes: ${{ inputs.node_nwaku }} + caller: "nim" \ No newline at end of file diff --git a/.github/workflows/test_PR_image.yml b/.github/workflows/test_PR_image.yml new file mode 100644 index 00000000..3f88abd5 --- /dev/null +++ b/.github/workflows/test_PR_image.yml @@ -0,0 +1,89 @@ + +name: Interop Tests PR + +on: + workflow_call: + inputs: + node1: + required: true + type: string + description: "Node that usually publishes messages. Used for all tests" + default: "wakuorg/go-waku:latest" + node2: + required: true + description: "Node that usually queries for published messages. Used for all tests" + type: string + default: "wakuorg/nwaku:latest" + additional_nodes: + required: false + description: "Additional optional nodes used in e2e tests, separated by ," + type: string + default: "wakuorg/nwaku:latest,wakuorg/go-waku:latest,wakuorg/nwaku:latest" + caller: + required: false + description: "Workflow caller. Used in reporting" + type: string + +env: + FORCE_COLOR: "1" + NODE_1: ${{ inputs.node1 }} + NODE_2: ${{ inputs.node2 }} + ADDITIONAL_NODES: ${{ inputs.additional_nodes }} + CALLER: ${{ inputs.caller || 'manual' }} + RLN_CREDENTIALS: ${{ secrets.RLN_CREDENTIALS }} + +jobs: + + tests: + name: tests + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + + - uses: actions/checkout@v4 + with: + repository: waku-org/waku-interop-tests + + - uses: actions/setup-python@v4 + with: + + python-version: '3.12' + cache: 'pip' + + - run: pip install -r requirements.txt + + - name: Run tests + timeout-minutes: 30 + run: | + pytest -m 'smoke' -n 4 --dist loadgroup --reruns 1 --junit-xml=pytest_results.xml + + - name: Test Report + if: always() + uses: dorny/test-reporter@95058abb17504553158e70e2c058fe1fda4392c2 + with: + name: Pytest JUnit Test Report + path: pytest_results.xml + reporter: java-junit + use-actions-summary: 'true' + + - name: Create job summary + if: always() + env: + JOB_STATUS: ${{ job.status }} + run: | + echo "## Run Information" >> $GITHUB_STEP_SUMMARY + echo "- **Event**: ${{ github.event_name }}" >> $GITHUB_STEP_SUMMARY + echo "- **Actor**: ${{ github.actor }}" >> $GITHUB_STEP_SUMMARY + echo "- **Node1**: ${{ env.NODE_1 }}" >> $GITHUB_STEP_SUMMARY + echo "- **Node2**: ${{ env.NODE_2 }}" >> $GITHUB_STEP_SUMMARY + echo "- **Additonal Nodes**: ${{ env.ADDITIONAL_NODES }}" >> $GITHUB_STEP_SUMMARY + if [ "$JOB_STATUS" != "success" ]; then + echo "There are failures with nwaku node. cc <@&1111608257824440330>" >> $GITHUB_STEP_SUMMARY + fi + { + echo 'JOB_SUMMARY<> $GITHUB_ENV + + diff --git a/pytest.ini b/pytest.ini index a876cae5..958bc09c 100644 --- a/pytest.ini +++ b/pytest.ini @@ -6,3 +6,5 @@ log_file = log/test.log log_cli_format = %(asctime)s %(name)s %(levelname)s %(message)s log_file_format = %(asctime)s %(name)s %(levelname)s %(message)s timeout = 300 +markers = + smoke: marks tests as smoke test (deselect with '-m "not smoke"') \ No newline at end of file diff --git a/tests/discv5/test_discv5.py b/tests/discv5/test_discv5.py index 2557ea90..b8ab88b3 100644 --- a/tests/discv5/test_discv5.py +++ b/tests/discv5/test_discv5.py @@ -1,3 +1,5 @@ +import pytest + from src.env_vars import NODE_1, NODE_2 from src.libs.custom_logger import get_custom_logger from src.node.waku_node import WakuNode @@ -11,6 +13,7 @@ logger = get_custom_logger(__name__) class TestDiscv5(StepsRelay, StepsFilter, StepsStore, StepsLightPush): + @pytest.mark.smoke def running_a_node(self, image, **kwargs): node = WakuNode(image, f"node{len(self.main_nodes) + 1}_{self.test_id}") node.start(**kwargs) @@ -25,6 +28,7 @@ class TestDiscv5(StepsRelay, StepsFilter, StepsStore, StepsLightPush): def wait_for_lightpushed_message_to_be_stored(self): self.check_light_pushed_message_reaches_receiving_peer(peer_list=[self.receiving_node1, self.receiving_node2]) + @pytest.mark.smoke def test_relay(self): self.node1 = self.running_a_node(NODE_1, relay="true") self.node2 = self.running_a_node(NODE_2, relay="true", discv5_bootstrap_node=self.node1.get_enr_uri()) @@ -32,6 +36,7 @@ class TestDiscv5(StepsRelay, StepsFilter, StepsStore, StepsLightPush): self.ensure_relay_subscriptions_on_nodes(self.main_nodes, [self.test_pubsub_topic]) self.wait_for_published_message_to_reach_relay_peer() + @pytest.mark.smoke def test_filter(self): self.node1 = self.running_a_node(NODE_1, relay="true", filter="true") self.node2 = self.running_a_node( @@ -54,6 +59,7 @@ class TestDiscv5(StepsRelay, StepsFilter, StepsStore, StepsLightPush): self.subscribe_to_pubsub_topics_via_relay(self.main_nodes) self.wait_for_published_message_to_be_stored() + @pytest.mark.smoke def test_lightpush(self): self.receiving_node1 = self.running_a_node(NODE_1, lightpush="true", relay="true") self.receiving_node2 = self.running_a_node(NODE_1, lightpush="false", relay="true", discv5_bootstrap_node=self.receiving_node1.get_enr_uri()) diff --git a/tests/e2e/test_e2e.py b/tests/e2e/test_e2e.py index e40131a3..11bdff7d 100644 --- a/tests/e2e/test_e2e.py +++ b/tests/e2e/test_e2e.py @@ -304,6 +304,7 @@ class TestE2E(StepsFilter, StepsStore, StepsRelay, StepsLightPush): assert len(response_list) == max_subscribed_nodes, "Received message count doesn't match sent " @pytest.mark.skipif("go-waku" in NODE_2, reason="Test works only with nwaku") + @pytest.mark.smoke def test_store_filter_interaction_with_six_nodes(self): logger.debug("Create 6 nodes") self.node4 = WakuNode(NODE_2, f"node4_{self.test_id}") diff --git a/tests/filter/test_get_messages.py b/tests/filter/test_get_messages.py index 1aa8e51a..a2227e55 100644 --- a/tests/filter/test_get_messages.py +++ b/tests/filter/test_get_messages.py @@ -11,6 +11,7 @@ logger = get_custom_logger(__name__) # here we will also implicitly test filter push, see: https://rfc.vac.dev/spec/12/#messagepush @pytest.mark.usefixtures("setup_main_relay_node", "setup_main_filter_node", "subscribe_main_nodes") class TestFilterGetMessages(StepsFilter): + @pytest.mark.smoke def test_filter_get_message_with_valid_payloads(self): failed_payloads = [] for payload in SAMPLE_INPUTS: diff --git a/tests/filter/test_ping.py b/tests/filter/test_ping.py index 497318ca..1328ca7c 100644 --- a/tests/filter/test_ping.py +++ b/tests/filter/test_ping.py @@ -5,6 +5,7 @@ from src.steps.filter import StepsFilter @pytest.mark.usefixtures("setup_main_relay_node", "setup_main_filter_node") class TestFilterPing(StepsFilter): + @pytest.mark.smoke def test_filter_ping_on_subscribed_peer(self, subscribe_main_nodes): self.ping_filter_subscriptions(str(uuid4())) diff --git a/tests/relay/test_multiple_nodes.py b/tests/relay/test_multiple_nodes.py index 3bb007f1..80622028 100644 --- a/tests/relay/test_multiple_nodes.py +++ b/tests/relay/test_multiple_nodes.py @@ -4,6 +4,7 @@ from src.steps.relay import StepsRelay @pytest.mark.usefixtures("setup_main_relay_nodes", "setup_optional_relay_nodes", "subscribe_main_relay_nodes") class TestRelayMultipleNodes(StepsRelay): + @pytest.mark.smoke def test_first_node_to_start_publishes(self, subscribe_optional_relay_nodes, relay_warm_up): self.check_published_message_reaches_relay_peer() diff --git a/tests/relay/test_rln.py b/tests/relay/test_rln.py index 4707635c..921ca6b3 100644 --- a/tests/relay/test_rln.py +++ b/tests/relay/test_rln.py @@ -18,6 +18,7 @@ logger = get_custom_logger(__name__) class TestRelayRLN(StepsRLN, StepsRelay): SAMPLE_INPUTS_RLN = SAMPLE_INPUTS + SAMPLE_INPUTS + SAMPLE_INPUTS + @pytest.mark.smoke def test_valid_payloads_lightpush_at_spam_rate(self, pytestconfig): message_limit = 1 epoch_sec = 1 @@ -58,6 +59,7 @@ class TestRelayRLN(StepsRLN, StepsRelay): if i == message_limit - 1: break + @pytest.mark.smoke def test_valid_payloads_at_spam_rate(self, pytestconfig): message_limit = 20 epoch_sec = 600 diff --git a/tests/relay/test_subscribe.py b/tests/relay/test_subscribe.py index 21018137..d809eada 100644 --- a/tests/relay/test_subscribe.py +++ b/tests/relay/test_subscribe.py @@ -9,9 +9,11 @@ logger = get_custom_logger(__name__) @pytest.mark.usefixtures("setup_main_relay_nodes") class TestRelaySubscribe(StepsRelay): + @pytest.mark.smoke def test_relay_no_subscription(self): self.check_publish_without_relay_subscription(self.test_pubsub_topic) + @pytest.mark.smoke def test_relay_subscribe_to_single_pubsub_topic(self): self.ensure_relay_subscriptions_on_nodes(self.main_nodes, [self.test_pubsub_topic]) self.wait_for_published_message_to_reach_relay_peer() diff --git a/tests/store/test_running_nodes.py b/tests/store/test_running_nodes.py index 51727dec..547bf0c8 100644 --- a/tests/store/test_running_nodes.py +++ b/tests/store/test_running_nodes.py @@ -59,6 +59,7 @@ class TestRunningNodes(StepsStore): except Exception as ex: assert "failed to negotiate protocol: protocols not supported" in str(ex) or "PEER_DIAL_FAILURE" in str(ex) + @pytest.mark.smoke def test_store_lightpushed_message(self): self.setup_first_publishing_node(store="true", relay="true", lightpush="true") self.setup_second_publishing_node(store="false", relay="true") @@ -67,6 +68,7 @@ class TestRunningNodes(StepsStore): self.publish_message(via="lightpush", sender=self.store_node1) self.check_published_message_is_stored(page_size=5, ascending="true") + @pytest.mark.smoke def test_store_with_filter(self): self.setup_first_publishing_node(store="true", relay="true", filter="true") self.setup_first_store_node(store="false", relay="false", filter="true") diff --git a/tests/store/test_topics.py b/tests/store/test_topics.py index 521108d5..6cf48d6a 100644 --- a/tests/store/test_topics.py +++ b/tests/store/test_topics.py @@ -68,6 +68,7 @@ class TestTopics(StepsStore): ) assert len(store_response["messages"]) == 0, "Message count mismatch" + @pytest.mark.smoke def test_store_with_both_pubsub_topic_and_content_topic(self): for node in self.store_nodes: for index, content_topic in enumerate(CONTENT_TOPICS_DIFFERENT_SHARDS):