From 1f853b3a1154923f43a0261f3f2acca9d4bf8e68 Mon Sep 17 00:00:00 2001 From: AYAHASSAN287 <49167455+AYAHASSAN287@users.noreply.github.com> Date: Fri, 27 Dec 2024 11:10:12 +0300 Subject: [PATCH 1/6] checkout on smoke_tests tag (#96) * checkout on smoke_tests tag * MOdify pytest command * Update README.md Add steps on how to use new tag for PR tests in readme file --- .github/workflows/test_PR_image.yml | 4 ++-- README.md | 15 ++++++++++++++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test_PR_image.yml b/.github/workflows/test_PR_image.yml index 3f88abd5..89d779ef 100644 --- a/.github/workflows/test_PR_image.yml +++ b/.github/workflows/test_PR_image.yml @@ -43,7 +43,7 @@ jobs: - uses: actions/checkout@v4 with: repository: waku-org/waku-interop-tests - + ref: SMOKE_TESTS - uses: actions/setup-python@v4 with: @@ -55,7 +55,7 @@ jobs: - name: Run tests timeout-minutes: 30 run: | - pytest -m 'smoke' -n 4 --dist loadgroup --reruns 1 --junit-xml=pytest_results.xml + pytest -m 'smoke' -n 4 --dist=loadgroup --reruns 1 --junit-xml=pytest_results.xml - name: Test Report if: always() diff --git a/README.md b/README.md index eaf4d05a..c9150175 100644 --- a/README.md +++ b/README.md @@ -15,11 +15,24 @@ pre-commit install pytest ``` -## CI +## CI - Test runs via github actions - [Allure Test Reports](https://waku-org.github.io/waku-interop-tests/3/) are published via github pages + +## CI NWAKU job + +To update tests in PRs at nwaku repo following steps shall be done + + - Make a tag at the desired commit on master with these 2 commands + - git tag tagname + - git push origin tagname + - Navigate to test_PR_image.yml file and modify job "tests" to explicilty use the tag in ref section + + ![Screenshot from 2024-12-24 16-24-51](https://github.com/user-attachments/assets/dd3f95bd-fe79-475b-92b7-891d82346382) + + ## License Licensed and distributed under either of From e4e92afb3b7027500edf708b413bd0d2c5e96d79 Mon Sep 17 00:00:00 2001 From: aya Date: Fri, 27 Dec 2024 13:52:52 +0200 Subject: [PATCH 2/6] Modify test command --- .github/workflows/test_common.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_common.yml b/.github/workflows/test_common.yml index 02ce6caa..e6c31e31 100644 --- a/.github/workflows/test_common.yml +++ b/.github/workflows/test_common.yml @@ -56,8 +56,12 @@ jobs: - run: pip install -r requirements.txt - name: Run tests - run: | - pytest --ignore=tests/relay/test_rln.py --ignore=tests/store/test_cursor_many_msgs.py --reruns 2 --shard-id=${{ matrix.shard }} --num-shards=16 --alluredir=allure-results-${{ matrix.shard }} + run: if [ "${{ matrix.shard }}" == 16 ]; then + pytest tests/relay/test_rln.py --alluredir=allure-results-${{ matrix.shard }} + elif [ "${{ matrix.shard }}" == 17 ]; then + pytest tests/store/test_cursor_many_msgs.py --alluredir=allure-results-${{ matrix.shard }} + elif [ "${{ matrix.shard }}" != 17 ]; then + pytest --ignore=tests/relay/test_rln.py --ignore=tests/store/test_cursor_many_msgs.py --reruns 2 --shard-id=${{ matrix.shard }} --num-shards=16 --alluredir=allure-results-${{ matrix.shard }} - name: Upload allure results if: always() From e9850143664b2cbd35dfb5868bc3600132d1356b Mon Sep 17 00:00:00 2001 From: Florin Barbu Date: Fri, 27 Dec 2024 15:43:02 +0200 Subject: [PATCH 3/6] fix: add multiple machines --- .github/workflows/test_common.yml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test_common.yml b/.github/workflows/test_common.yml index e6c31e31..66a7377f 100644 --- a/.github/workflows/test_common.yml +++ b/.github/workflows/test_common.yml @@ -37,7 +37,7 @@ jobs: name: tests strategy: matrix: - shard: [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15] + shard: [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17] runs-on: ubuntu-latest timeout-minutes: 120 @@ -56,12 +56,14 @@ jobs: - run: pip install -r requirements.txt - name: Run tests - run: if [ "${{ matrix.shard }}" == 16 ]; then - pytest tests/relay/test_rln.py --alluredir=allure-results-${{ matrix.shard }} - elif [ "${{ matrix.shard }}" == 17 ]; then - pytest tests/store/test_cursor_many_msgs.py --alluredir=allure-results-${{ matrix.shard }} - elif [ "${{ matrix.shard }}" != 17 ]; then - pytest --ignore=tests/relay/test_rln.py --ignore=tests/store/test_cursor_many_msgs.py --reruns 2 --shard-id=${{ matrix.shard }} --num-shards=16 --alluredir=allure-results-${{ matrix.shard }} + run: | + if [ "${{ matrix.shard }}" == "16" ]; then + pytest tests/relay/test_rln.py --alluredir=allure-results-${{ matrix.shard }} + elif [ "${{ matrix.shard }}" == "17" ]; then + pytest tests/store/test_cursor_many_msgs.py --alluredir=allure-results-${{ matrix.shard }} + elif [ "${{ matrix.shard }}" != "17" ]; then + pytest --ignore=tests/relay/test_rln.py --ignore=tests/store/test_cursor_many_msgs.py --reruns 2 --shard-id=${{ matrix.shard }} --num-shards=16 --alluredir=allure-results-${{ matrix.shard }} + fi - name: Upload allure results if: always() From e16c475f0bbcd8784b245287586440f8cf46f2a1 Mon Sep 17 00:00:00 2001 From: Florin Barbu Date: Fri, 27 Dec 2024 15:47:50 +0200 Subject: [PATCH 4/6] fix: prevent fail fast --- .github/workflows/test_common.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test_common.yml b/.github/workflows/test_common.yml index 66a7377f..2aee0bd1 100644 --- a/.github/workflows/test_common.yml +++ b/.github/workflows/test_common.yml @@ -36,6 +36,7 @@ jobs: tests: name: tests strategy: + fail-fast: false matrix: shard: [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17] @@ -77,6 +78,7 @@ jobs: aggregate-reports: runs-on: ubuntu-latest needs: tests + if: always() steps: - name: Download all allure results uses: actions/download-artifact@v4 From 89b39edbf8c398fb40588f60014ba26e8a2c0fd7 Mon Sep 17 00:00:00 2001 From: aya Date: Sun, 5 Jan 2025 15:57:18 +0200 Subject: [PATCH 5/6] Remove multiple skips --- tests/metrics/test_metrics.py | 1 - tests/relay/test_rln.py | 4 ---- 2 files changed, 5 deletions(-) diff --git a/tests/metrics/test_metrics.py b/tests/metrics/test_metrics.py index 730e2d04..ef4e4b61 100644 --- a/tests/metrics/test_metrics.py +++ b/tests/metrics/test_metrics.py @@ -10,7 +10,6 @@ from src.steps.store import StepsStore class TestMetrics(StepsRelay, StepsMetrics, StepsFilter, StepsLightPush, StepsStore): - @pytest.mark.skip def test_metrics_initial_value(self): node = WakuNode(DEFAULT_NWAKU, f"node1_{self.test_id}") node.start(relay="true", filter="true", store="true", lightpush="true") diff --git a/tests/relay/test_rln.py b/tests/relay/test_rln.py index 6942f2aa..5be2487b 100644 --- a/tests/relay/test_rln.py +++ b/tests/relay/test_rln.py @@ -14,7 +14,6 @@ logger = get_custom_logger(__name__) @pytest.mark.xdist_group(name="RLN serial tests") -@pytest.mark.rln @pytest.mark.skipif("go-waku" in (NODE_1 + NODE_2), reason="Test works only with nwaku") class TestRelayRLN(StepsRLN, StepsRelay): SAMPLE_INPUTS_RLN = SAMPLE_INPUTS + SAMPLE_INPUTS + SAMPLE_INPUTS @@ -40,7 +39,6 @@ class TestRelayRLN(StepsRLN, StepsRelay): except Exception as e: assert "RLN validation failed" or "NonceLimitReached" in str(e) - @pytest.mark.skipif("nwaku" in (NODE_1 + NODE_2), reason="Test works only with nwaku") def test_valid_payloads_at_slow_rate(self, pytestconfig): message_limit = 20 self.register_rln_relay_nodes(0, pytestconfig.cache.get("keystore-prefixes", [])) @@ -61,7 +59,6 @@ class TestRelayRLN(StepsRLN, StepsRelay): if i == message_limit - 1: break - @pytest.mark.skipif("nwaku" in (NODE_1 + NODE_2), reason="Test works only with nwaku") def test_valid_payloads_at_spam_rate(self, pytestconfig): message_limit = 20 epoch_sec = 600 @@ -81,7 +78,6 @@ class TestRelayRLN(StepsRLN, StepsRelay): except Exception as e: assert "RLN validation failed" or "NonceLimitReached" in str(e) - @pytest.mark.skipif("nwaku" in (NODE_1 + NODE_2), reason="Test works only with nwaku") def test_valid_payload_at_variable_rate(self, pytestconfig): self.register_rln_relay_nodes(0, pytestconfig.cache.get("keystore-prefixes", [])) self.setup_main_rln_relay_nodes(rln_relay_user_message_limit=1, rln_relay_epoch_sec=1) From 824983cae88040deeb95f34672402361904bd45d Mon Sep 17 00:00:00 2001 From: aya Date: Sun, 5 Jan 2025 16:05:59 +0200 Subject: [PATCH 6/6] Revert changes in test_rln file --- tests/relay/test_rln.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/relay/test_rln.py b/tests/relay/test_rln.py index 5be2487b..1a291157 100644 --- a/tests/relay/test_rln.py +++ b/tests/relay/test_rln.py @@ -59,6 +59,7 @@ class TestRelayRLN(StepsRLN, StepsRelay): if i == message_limit - 1: break + @pytest.mark.smoke def test_valid_payloads_at_spam_rate(self, pytestconfig): message_limit = 20 epoch_sec = 600