Modify test command

fix: add multiple machines

fix: prevent fail fast

checkout on smoke_tests tag (#96)

* checkout on smoke_tests tag

* MOdify pytest command

* Update README.md

Add steps on how to use new tag for PR tests in readme file

Remove multiple skips

Revert changes in test_rln file

.github/workflows/test_PR_image.yml

@@ -43,7 +43,7 @@ jobs:
       - uses: actions/checkout@v4
         with:
           repository: waku-org/waku-interop-tests
-
+          ref: SMOKE_TESTS
       - uses: actions/setup-python@v4
         with:
 
@@ -55,7 +55,7 @@ jobs:
       - name: Run tests
         timeout-minutes: 30
         run: |
-          pytest -m 'smoke' -n 4 --dist loadgroup --reruns 1 --junit-xml=pytest_results.xml
+          pytest -m 'smoke' -n 4 --dist=loadgroup --reruns 1 --junit-xml=pytest_results.xml
 
       - name: Test Report
         if: always()

.github/workflows/test_common.yml

@@ -36,8 +36,9 @@ jobs:
   tests:
     name: tests
     strategy:
+      fail-fast: false
       matrix:
-        shard: [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15]
+        shard: [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17]
 
     runs-on: ubuntu-latest
     timeout-minutes: 120
@@ -57,7 +58,13 @@ jobs:
 
     - name: Run tests
       run: |
-        pytest --ignore=tests/relay/test_rln.py --ignore=tests/store/test_cursor_many_msgs.py --reruns 2 --shard-id=${{ matrix.shard }} --num-shards=16  --alluredir=allure-results-${{ matrix.shard }}
+        if [ "${{ matrix.shard }}" == "16" ]; then
+            pytest tests/relay/test_rln.py --alluredir=allure-results-${{ matrix.shard }}
+        elif [ "${{ matrix.shard }}" == "17" ]; then
+            pytest tests/store/test_cursor_many_msgs.py --alluredir=allure-results-${{ matrix.shard }}
+        elif [ "${{ matrix.shard }}" != "17" ]; then
+            pytest --ignore=tests/relay/test_rln.py --ignore=tests/store/test_cursor_many_msgs.py --reruns 2 --shard-id=${{ matrix.shard }} --num-shards=16 --alluredir=allure-results-${{ matrix.shard }}
+        fi
 
     - name: Upload allure results
       if: always()
@@ -71,6 +78,7 @@ jobs:
   aggregate-reports:
     runs-on: ubuntu-latest
     needs: tests
+    if: always()
     steps:
       - name: Download all allure results
         uses: actions/download-artifact@v4

README.md

@@ -15,11 +15,24 @@ pre-commit install
 pytest
 ```
 
-## CI
+## CI 
 
 - Test runs via github actions
 - [Allure Test Reports](https://waku-org.github.io/waku-interop-tests/3/) are published via github pages
 
+
+## CI NWAKU job 
+
+To update tests in PRs at nwaku repo following steps shall be done 
+
+ - Make a tag at the desired commit on master with these 2 commands
+   - git tag tagname
+   - git push origin tagname
+ - Navigate to test_PR_image.yml file and modify job "tests" to explicilty use the tag in ref section
+  
+ ![Screenshot from 2024-12-24 16-24-51](https://github.com/user-attachments/assets/dd3f95bd-fe79-475b-92b7-891d82346382)
+
+
 ## License
 
 Licensed and distributed under either of

tests/metrics/test_metrics.py

@@ -10,7 +10,6 @@ from src.steps.store import StepsStore
 
 
 class TestMetrics(StepsRelay, StepsMetrics, StepsFilter, StepsLightPush, StepsStore):
-    @pytest.mark.skip
     def test_metrics_initial_value(self):
         node = WakuNode(DEFAULT_NWAKU, f"node1_{self.test_id}")
         node.start(relay="true", filter="true", store="true", lightpush="true")

tests/relay/test_rln.py

@@ -14,7 +14,6 @@ logger = get_custom_logger(__name__)
 
 
 @pytest.mark.xdist_group(name="RLN serial tests")
-@pytest.mark.rln
 @pytest.mark.skipif("go-waku" in (NODE_1 + NODE_2), reason="Test works only with nwaku")
 class TestRelayRLN(StepsRLN, StepsRelay):
     SAMPLE_INPUTS_RLN = SAMPLE_INPUTS + SAMPLE_INPUTS + SAMPLE_INPUTS
@@ -40,7 +39,6 @@ class TestRelayRLN(StepsRLN, StepsRelay):
             except Exception as e:
                 assert "RLN validation failed" or "NonceLimitReached" in str(e)
 
-    @pytest.mark.skipif("nwaku" in (NODE_1 + NODE_2), reason="Test works only with nwaku")
     def test_valid_payloads_at_slow_rate(self, pytestconfig):
         message_limit = 20
         self.register_rln_relay_nodes(0, pytestconfig.cache.get("keystore-prefixes", []))
@@ -61,7 +59,7 @@ class TestRelayRLN(StepsRLN, StepsRelay):
             if i == message_limit - 1:
                 break
 
-    @pytest.mark.skipif("nwaku" in (NODE_1 + NODE_2), reason="Test works only with nwaku")
+    @pytest.mark.smoke
     def test_valid_payloads_at_spam_rate(self, pytestconfig):
         message_limit = 20
         epoch_sec = 600
@@ -81,7 +79,6 @@ class TestRelayRLN(StepsRLN, StepsRelay):
             except Exception as e:
                 assert "RLN validation failed" or "NonceLimitReached" in str(e)
 
-    @pytest.mark.skipif("nwaku" in (NODE_1 + NODE_2), reason="Test works only with nwaku")
     def test_valid_payload_at_variable_rate(self, pytestconfig):
         self.register_rln_relay_nodes(0, pytestconfig.cache.get("keystore-prefixes", []))
         self.setup_main_rln_relay_nodes(rln_relay_user_message_limit=1, rln_relay_epoch_sec=1)