smoke_tests (#90)

* Selecting initial smoke tests set

* adding mark "smoke" in pytest.ini

* adding mark smoke to workflow file

* Remove allure reporting from yml file

* Adding more smoke tests

* Add PR image to new nwaku workflow

* change nwaku_daily.yml to test the PR job changes

* remove dependencies from yml file

* Remove secrets check from container yml file

* remove secrets from all files

* change image tag

* revert image tag

* Revert nwaku_daily.yml to origin

* Adding PR number to choose image

* adding docker-build-image to yml file

* adding docker-image-build dependencies

* Adding quay user & password

* Adding quay username & password

* Fix yml build

* Remove changes causing errors

* remove tests part to speedup the job

* add flag workflow_call: for reusable workflow

* checkout on branch instead of master

* trying to fetch from branch

* Check out specific branch

* make node1 input

* Adding node1 as input

* Add type to required input

* fix node input format

* change input node1 format

* Delete .github/workflows/container-image.yml

* Delete .github/workflows/nim_nwaku_pr.yml

* Create 2  files for PR tests

* revert original yml files

* Fix review points on PR

* fix review points

* revert test_common file to master

* revert nim_waku_daily to master

* Fix review points

.github/workflows/nim_waku_PR.yml

@@ -0,0 +1,27 @@
+name: Nim -> Interop Tests PR
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: false
+
+on:
+  workflow_dispatch:
+    inputs:
+      node_nwaku:
+        required: true
+        type: string
+  workflow_call:
+    inputs:
+      node_nwaku:
+        required: true
+        type: string
+
+jobs:
+  test-pr:
+    uses: ./.github/workflows/test_PR_image.yml
+    secrets: inherit
+    with:
+      node1: ${{ inputs.node_nwaku }}
+      node2: ${{ inputs.node_nwaku }}
+      additional_nodes:  ${{ inputs.node_nwaku }}
+      caller: "nim"

.github/workflows/test_PR_image.yml

@@ -0,0 +1,89 @@
+
+name: Interop Tests PR
+
+on:
+  workflow_call:
+    inputs:
+      node1:
+        required: true
+        type: string
+        description: "Node that usually publishes messages. Used for all tests"
+        default: "wakuorg/go-waku:latest"
+      node2:
+        required: true
+        description: "Node that usually queries for published messages. Used for all tests"
+        type: string
+        default: "wakuorg/nwaku:latest"
+      additional_nodes:
+        required: false
+        description: "Additional optional nodes used in e2e tests, separated by ,"
+        type: string
+        default: "wakuorg/nwaku:latest,wakuorg/go-waku:latest,wakuorg/nwaku:latest"
+      caller:
+        required: false
+        description: "Workflow caller. Used in reporting"
+        type: string
+
+env:
+  FORCE_COLOR: "1"
+  NODE_1: ${{ inputs.node1 }}
+  NODE_2: ${{ inputs.node2 }}
+  ADDITIONAL_NODES: ${{ inputs.additional_nodes }}
+  CALLER: ${{ inputs.caller || 'manual' }}
+  RLN_CREDENTIALS: ${{ secrets.RLN_CREDENTIALS }}
+
+jobs:
+
+  tests:
+    name: tests
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
+    steps:
+
+      - uses: actions/checkout@v4
+        with:
+          repository: waku-org/waku-interop-tests
+
+      - uses: actions/setup-python@v4
+        with:
+
+          python-version: '3.12'
+          cache: 'pip'
+
+      - run: pip install -r requirements.txt
+
+      - name: Run tests
+        timeout-minutes: 30
+        run: |
+          pytest -m 'smoke' -n 4 --dist loadgroup --reruns 1 --junit-xml=pytest_results.xml
+
+      - name: Test Report
+        if: always()
+        uses: dorny/test-reporter@95058abb17504553158e70e2c058fe1fda4392c2
+        with:
+          name: Pytest JUnit Test Report
+          path: pytest_results.xml
+          reporter: java-junit
+          use-actions-summary: 'true'
+
+      - name: Create job summary
+        if: always()
+        env:
+          JOB_STATUS: ${{ job.status }}
+        run: |
+          echo "## Run Information" >> $GITHUB_STEP_SUMMARY
+          echo "- **Event**: ${{ github.event_name }}" >> $GITHUB_STEP_SUMMARY
+          echo "- **Actor**: ${{ github.actor }}" >> $GITHUB_STEP_SUMMARY
+          echo "- **Node1**: ${{ env.NODE_1 }}" >> $GITHUB_STEP_SUMMARY
+          echo "- **Node2**: ${{ env.NODE_2 }}" >> $GITHUB_STEP_SUMMARY
+          echo "- **Additonal Nodes**: ${{ env.ADDITIONAL_NODES }}" >> $GITHUB_STEP_SUMMARY
+          if [ "$JOB_STATUS" != "success" ]; then
+            echo "There are failures with nwaku node. cc <@&1111608257824440330>" >> $GITHUB_STEP_SUMMARY
+          fi
+          {
+            echo 'JOB_SUMMARY<<EOF'
+            cat $GITHUB_STEP_SUMMARY
+            echo EOF
+          } >> $GITHUB_ENV
+
+    

pytest.ini

@@ -6,3 +6,5 @@ log_file = log/test.log
 log_cli_format = %(asctime)s %(name)s %(levelname)s %(message)s 
 log_file_format = %(asctime)s %(name)s %(levelname)s %(message)s 
 timeout = 300
+markers =
+    smoke: marks tests as smoke test (deselect with '-m "not smoke"')

tests/discv5/test_discv5.py

@@ -1,3 +1,5 @@
+import pytest
+
 from src.env_vars import NODE_1, NODE_2
 from src.libs.custom_logger import get_custom_logger
 from src.node.waku_node import WakuNode
@@ -11,6 +13,7 @@ logger = get_custom_logger(__name__)
 
 
 class TestDiscv5(StepsRelay, StepsFilter, StepsStore, StepsLightPush):
+    @pytest.mark.smoke
     def running_a_node(self, image, **kwargs):
         node = WakuNode(image, f"node{len(self.main_nodes) + 1}_{self.test_id}")
         node.start(**kwargs)
@@ -25,6 +28,7 @@ class TestDiscv5(StepsRelay, StepsFilter, StepsStore, StepsLightPush):
     def wait_for_lightpushed_message_to_be_stored(self):
         self.check_light_pushed_message_reaches_receiving_peer(peer_list=[self.receiving_node1, self.receiving_node2])
 
+    @pytest.mark.smoke
     def test_relay(self):
         self.node1 = self.running_a_node(NODE_1, relay="true")
         self.node2 = self.running_a_node(NODE_2, relay="true", discv5_bootstrap_node=self.node1.get_enr_uri())
@@ -32,6 +36,7 @@ class TestDiscv5(StepsRelay, StepsFilter, StepsStore, StepsLightPush):
         self.ensure_relay_subscriptions_on_nodes(self.main_nodes, [self.test_pubsub_topic])
         self.wait_for_published_message_to_reach_relay_peer()
 
+    @pytest.mark.smoke
     def test_filter(self):
         self.node1 = self.running_a_node(NODE_1, relay="true", filter="true")
         self.node2 = self.running_a_node(
@@ -54,6 +59,7 @@ class TestDiscv5(StepsRelay, StepsFilter, StepsStore, StepsLightPush):
         self.subscribe_to_pubsub_topics_via_relay(self.main_nodes)
         self.wait_for_published_message_to_be_stored()
 
+    @pytest.mark.smoke
     def test_lightpush(self):
         self.receiving_node1 = self.running_a_node(NODE_1, lightpush="true", relay="true")
         self.receiving_node2 = self.running_a_node(NODE_1, lightpush="false", relay="true", discv5_bootstrap_node=self.receiving_node1.get_enr_uri())

tests/e2e/test_e2e.py

@@ -304,6 +304,7 @@ class TestE2E(StepsFilter, StepsStore, StepsRelay, StepsLightPush):
         assert len(response_list) == max_subscribed_nodes, "Received message count doesn't match sent "
 
     @pytest.mark.skipif("go-waku" in NODE_2, reason="Test works only with nwaku")
+    @pytest.mark.smoke
     def test_store_filter_interaction_with_six_nodes(self):
         logger.debug("Create  6 nodes")
         self.node4 = WakuNode(NODE_2, f"node4_{self.test_id}")

tests/filter/test_get_messages.py

@@ -11,6 +11,7 @@ logger = get_custom_logger(__name__)
 # here we will also implicitly test filter push, see: https://rfc.vac.dev/spec/12/#messagepush
 @pytest.mark.usefixtures("setup_main_relay_node", "setup_main_filter_node", "subscribe_main_nodes")
 class TestFilterGetMessages(StepsFilter):
+    @pytest.mark.smoke
     def test_filter_get_message_with_valid_payloads(self):
         failed_payloads = []
         for payload in SAMPLE_INPUTS:

tests/filter/test_ping.py

@@ -5,6 +5,7 @@ from src.steps.filter import StepsFilter
 
 @pytest.mark.usefixtures("setup_main_relay_node", "setup_main_filter_node")
 class TestFilterPing(StepsFilter):
+    @pytest.mark.smoke
     def test_filter_ping_on_subscribed_peer(self, subscribe_main_nodes):
         self.ping_filter_subscriptions(str(uuid4()))
 

tests/relay/test_multiple_nodes.py

@@ -4,6 +4,7 @@ from src.steps.relay import StepsRelay
 
 @pytest.mark.usefixtures("setup_main_relay_nodes", "setup_optional_relay_nodes", "subscribe_main_relay_nodes")
 class TestRelayMultipleNodes(StepsRelay):
+    @pytest.mark.smoke
     def test_first_node_to_start_publishes(self, subscribe_optional_relay_nodes, relay_warm_up):
         self.check_published_message_reaches_relay_peer()
 

tests/relay/test_rln.py

@@ -18,6 +18,7 @@ logger = get_custom_logger(__name__)
 class TestRelayRLN(StepsRLN, StepsRelay):
     SAMPLE_INPUTS_RLN = SAMPLE_INPUTS + SAMPLE_INPUTS + SAMPLE_INPUTS
 
+    @pytest.mark.smoke
     def test_valid_payloads_lightpush_at_spam_rate(self, pytestconfig):
         message_limit = 1
         epoch_sec = 1
@@ -58,6 +59,7 @@ class TestRelayRLN(StepsRLN, StepsRelay):
             if i == message_limit - 1:
                 break
 
+    @pytest.mark.smoke
     def test_valid_payloads_at_spam_rate(self, pytestconfig):
         message_limit = 20
         epoch_sec = 600

tests/relay/test_subscribe.py

@@ -9,9 +9,11 @@ logger = get_custom_logger(__name__)
 
 @pytest.mark.usefixtures("setup_main_relay_nodes")
 class TestRelaySubscribe(StepsRelay):
+    @pytest.mark.smoke
     def test_relay_no_subscription(self):
         self.check_publish_without_relay_subscription(self.test_pubsub_topic)
 
+    @pytest.mark.smoke
     def test_relay_subscribe_to_single_pubsub_topic(self):
         self.ensure_relay_subscriptions_on_nodes(self.main_nodes, [self.test_pubsub_topic])
         self.wait_for_published_message_to_reach_relay_peer()

tests/store/test_running_nodes.py

@@ -59,6 +59,7 @@ class TestRunningNodes(StepsStore):
         except Exception as ex:
             assert "failed to negotiate protocol: protocols not supported" in str(ex) or "PEER_DIAL_FAILURE" in str(ex)
 
+    @pytest.mark.smoke
     def test_store_lightpushed_message(self):
         self.setup_first_publishing_node(store="true", relay="true", lightpush="true")
         self.setup_second_publishing_node(store="false", relay="true")
@@ -67,6 +68,7 @@ class TestRunningNodes(StepsStore):
         self.publish_message(via="lightpush", sender=self.store_node1)
         self.check_published_message_is_stored(page_size=5, ascending="true")
 
+    @pytest.mark.smoke
     def test_store_with_filter(self):
         self.setup_first_publishing_node(store="true", relay="true", filter="true")
         self.setup_first_store_node(store="false", relay="false", filter="true")

tests/store/test_topics.py

@@ -68,6 +68,7 @@ class TestTopics(StepsStore):
             )
             assert len(store_response["messages"]) == 0, "Message count mismatch"
 
+    @pytest.mark.smoke
     def test_store_with_both_pubsub_topic_and_content_topic(self):
         for node in self.store_nodes:
             for index, content_topic in enumerate(CONTENT_TOPICS_DIFFERENT_SHARDS):