From 7d5af004c656ccc6d311e4f9c7be2850d968febe Mon Sep 17 00:00:00 2001 From: Egor Rachkovskii Date: Thu, 6 Aug 2026 19:00:55 +0100 Subject: [PATCH] ci: run the nightly suite against our own node image The nightly full run installed Nim and built liblogosdelivery.so in every shard, only to collect tests/wrappers_tests, which now live in the core repository. Skips that directory and drops the build steps, so the run is python + docker. Points node1/node2/additional_nodes and the self-built nodes in rest_flags and metrics at quay.io/wakuorg/nwaku-pr:nightly, so the suite finally exercises our build. sharding keeps upstream nodes through UPSTREAM_NWAKU: setup_nwaku_relay_nodes mixes them with our main node, which is the only compatibility check left in the suite. Drops the RLN shard, whose class is skipped in full, and stops ignoring test_rln.py in the sharded run so the tests are collected wherever they land once that skip goes away. For the measurement window: reruns is an input pinned to 0 for the nightly so flakes stay visible, every shard writes a JUnit report, and the digest of each image is printed before the tests start. Wrapper coverage moves entirely to pr_tests.yml, which was missing test_channel_lifecycle.py and the docker_required tests; both are added there. --- .github/workflows/nim_waku_daily.yml | 13 ++- .github/workflows/pr_tests.yml | 24 ++++- .github/workflows/test_common.yml | 137 +++++++++------------------ README.md | 10 +- src/env_vars.py | 5 +- src/steps/sharding.py | 4 +- 6 files changed, 86 insertions(+), 107 deletions(-) diff --git a/.github/workflows/nim_waku_daily.yml b/.github/workflows/nim_waku_daily.yml index 44e47d531..21607bd11 100644 --- a/.github/workflows/nim_waku_daily.yml +++ b/.github/workflows/nim_waku_daily.yml @@ -5,8 +5,9 @@ concurrency: cancel-in-progress: false on: + # The nightly image is pushed at 04:34-04:41 UTC; 06:00 keeps this run behind it. schedule: - - cron: '0 4 * * *' + - cron: '0 6 * * *' workflow_dispatch: jobs: @@ -14,7 +15,9 @@ jobs: uses: ./.github/workflows/test_common.yml secrets: inherit with: - node1: "wakuorg/nwaku:latest" - node2: "wakuorg/nwaku:latest" - additional_nodes: "wakuorg/nwaku:latest,wakuorg/nwaku:latest,wakuorg/nwaku:latest" - caller: "nim" \ No newline at end of file + node1: "quay.io/wakuorg/nwaku-pr:nightly" + node2: "quay.io/wakuorg/nwaku-pr:nightly" + additional_nodes: "quay.io/wakuorg/nwaku-pr:nightly,quay.io/wakuorg/nwaku-pr:nightly,quay.io/wakuorg/nwaku-pr:nightly" + default_nwaku: "quay.io/wakuorg/nwaku-pr:nightly" + caller: "nim-core" + reruns: 0 \ No newline at end of file diff --git a/.github/workflows/pr_tests.yml b/.github/workflows/pr_tests.yml index a04f30dec..f7dad40a1 100644 --- a/.github/workflows/pr_tests.yml +++ b/.github/workflows/pr_tests.yml @@ -21,7 +21,7 @@ on: workflow_dispatch: inputs: run_full_suite: - description: "Run the full test suite (18 shards)" + description: "Run the full test suite (17 shards)" required: false default: false type: boolean @@ -127,7 +127,7 @@ jobs: name: Wrapper Tests runs-on: ubuntu-latest needs: [build] - timeout-minutes: 45 + timeout-minutes: 60 if: >- github.event_name != 'push' && (github.event.action != 'labeled' || github.event.label.name == 'full-test') @@ -159,6 +159,26 @@ jobs: --reruns 2 \ --junit-xml=wrapper-results-basic.xml + - name: Run wrapper tests - channel lifecycle + continue-on-error: true + env: + PYTHONPATH: ${{ github.workspace }}/vendor/logos-delivery-python-bindings/waku + run: | + pytest tests/wrappers_tests/test_channel_lifecycle.py \ + -m "not docker_required" \ + --reruns 2 \ + --junit-xml=wrapper-results-channel-lifecycle.xml + + - name: Run wrapper tests - docker required + continue-on-error: true + env: + PYTHONPATH: ${{ github.workspace }}/vendor/logos-delivery-python-bindings/waku + run: | + pytest tests/wrappers_tests \ + -m docker_required \ + --reruns 2 \ + --junit-xml=wrapper-results-docker.xml + - name: Run wrapper tests - send handle and subscription continue-on-error: true env: diff --git a/.github/workflows/test_common.yml b/.github/workflows/test_common.yml index 13657dd07..5469e7b5e 100644 --- a/.github/workflows/test_common.yml +++ b/.github/workflows/test_common.yml @@ -37,14 +37,32 @@ on: description: "Send test results to Discord" type: boolean default: false + reruns: + required: false + description: "pytest --reruns; 0 keeps flaky runs visible in the JUnit report" + type: number + default: 2 + default_nwaku: + required: false + description: "Image for tests that build their own nodes instead of using node1/node2" + type: string + default: "" + upstream_nwaku: + required: false + description: "Image for setup_nwaku_relay_nodes, which stays an interop check against upstream" + type: string + default: "" env: FORCE_COLOR: "1" NODE_1: ${{ inputs.node1 }} NODE_2: ${{ inputs.node2 }} ADDITIONAL_NODES: ${{ inputs.additional_nodes }} + DEFAULT_NWAKU: ${{ inputs.default_nwaku }} + UPSTREAM_NWAKU: ${{ inputs.upstream_nwaku }} CALLER: ${{ inputs.caller || 'manual' }} RLN_CREDENTIALS: ${{ secrets.RLN_CREDENTIALS }} + DISCORD_WEBHOOK: ${{ secrets.DISCORD_TEST_REPORTS_WH }} jobs: tests: @@ -52,11 +70,9 @@ jobs: strategy: fail-fast: false matrix: - shard: [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17] - # total number of shards =18 means tests will split into 18 thread and run in parallel to increase execution speed + shard: [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,17] # command for sharding : # pytest --shard-id= --num-shards= - # shard 16 for test_rln.py file as they shall run sequentially # shard 17 for test_cursor_many_msgs.py as it takes time >7 mins runs-on: ubuntu-latest @@ -78,13 +94,10 @@ jobs: jobResult_13: ${{ steps.set_result.outputs.JOB_RESULT_13 }} jobResult_14: ${{ steps.set_result.outputs.JOB_RESULT_14 }} jobResult_15: ${{ steps.set_result.outputs.JOB_RESULT_15 }} - jobResult_16: ${{ steps.set_result.outputs.JOB_RESULT_16 }} jobResult_17: ${{ steps.set_result.outputs.JOB_RESULT_17 }} steps: - uses: actions/checkout@v4 - with: - submodules: recursive - name: Remove unwanted software uses: ./.github/actions/prune-vm @@ -94,9 +107,6 @@ jobs: python-version: "3.12" cache: "pip" - - name: Add python bindings to PYTHONPATH - run: echo "PYTHONPATH=$(pwd)/vendor/logos-delivery-python-bindings/waku:$PYTHONPATH" >> $GITHUB_ENV - - name: Install system deps for tc / nsenter run: | sudo apt-get update @@ -110,110 +120,43 @@ jobs: gcc \ g++ - - name: Install Nim 2.2.4 - run: | - set -euo pipefail - curl https://nim-lang.org/choosenim/init.sh -sSf | sh -s -- -y - echo "$HOME/.nimble/bin" >> "$GITHUB_PATH" - export PATH="$HOME/.nimble/bin:$PATH" - choosenim 2.2.4 - # Pin the nimble that generated logos-delivery's nimble.lock. A newer - # nimble recomputes the locked Nim package checksum differently, which - # makes `nimble setup --localdeps` abort with a checksum mismatch. - (cd /tmp && nimble install "nimble@0.22.3" -y) - nim --version - nimble --version - - run: pip install -r requirements.txt - - name: Build liblogosdelivery.so for python bindings + - name: Pull and inspect node images run: | set -euo pipefail - - export PATH="$HOME/.nimble/bin:$PATH" - - BINDINGS_DIR="$(pwd)/vendor/logos-delivery-python-bindings" - DELIVERY_DIR="$BINDINGS_DIR/vendor/logos-delivery" - - mkdir -p "$BINDINGS_DIR/lib" - - cd "$DELIVERY_DIR" - - ln -sf waku.nimble waku.nims - - # `make liblogosdelivery` resolves the locked deps via - # `nimble setup --localdeps` (build-deps prerequisite); a bare - # `nimble install -y` here is redundant and pulls Nim from the lock, - # which is what triggered the checksum mismatch. - make liblogosdelivery - - SO_PATH="$(find . -type f -name 'liblogosdelivery.so' | head -n 1)" - - if [ -z "$SO_PATH" ]; then - echo "liblogosdelivery.so was not built" - exit 1 - fi - - cp "$SO_PATH" "$BINDINGS_DIR/lib/liblogosdelivery.so" - - echo "Built library:" - ls -l "$BINDINGS_DIR/lib/liblogosdelivery.so" - - # Cleanup build artifacts - rm -rf .gh-pages/nim - - - - name: Verify wrapper library - run: | - test -f vendor/logos-delivery-python-bindings/lib/liblogosdelivery.so - - - name: Debug Python import paths - run: | - pwd - echo "PYTHONPATH=$PYTHONPATH" - find . -maxdepth 5 | grep wrapper || true - python - <<'PY' - import sys - print("sys.path:") - for p in sys.path: - print(p) - try: - import wrapper - print("wrapper import OK:", wrapper) - except Exception as e: - print("wrapper import failed:", e) - raise - PY + printf '%s\n' "$NODE_1" "$NODE_2" "$DEFAULT_NWAKU" "$UPSTREAM_NWAKU" ${ADDITIONAL_NODES:-} | tr ',' '\n' | sort -u | while read -r image; do + [ -n "$image" ] || continue + docker pull -q "$image" + docker image inspect "$image" \ + --format "- \`$image\` digest={{if .RepoDigests}}{{index .RepoDigests 0}}{{else}}{{end}} created={{.Created}}" \ + | tee -a "$GITHUB_STEP_SUMMARY" + done - name: Run tests + id: run_tests run: | - export PATH="$HOME/.nimble/bin:$PATH" - export PYTHONPATH="$(pwd)/vendor/logos-delivery-python-bindings/waku:$PYTHONPATH" - if [ "${{ inputs.fleet_tests }}" == "true" ]; then if [ "${{ matrix.shard }}" == "0" ]; then pytest --fleet -m waku_test_fleet \ - --ignore=vendor/logos-delivery-python-bindings/tests \ + --ignore=tests/wrappers_tests \ + --junit-xml=junit-${{ matrix.shard }}.xml \ --alluredir=allure-results-${{ matrix.shard }} else echo "Skipping shard ${{ matrix.shard }}: fleet tests run on shard 0 only" fi - elif [ "${{ matrix.shard }}" == "16" ]; then - pytest tests/relay/test_rln.py \ - --ignore=vendor/logos-delivery-python-bindings/tests \ - --alluredir=allure-results-${{ matrix.shard }} elif [ "${{ matrix.shard }}" == "17" ]; then pytest tests/store/test_cursor_many_msgs.py \ - --ignore=vendor/logos-delivery-python-bindings/tests \ + --junit-xml=junit-${{ matrix.shard }}.xml \ --alluredir=allure-results-${{ matrix.shard }} else pytest \ - --ignore=vendor/logos-delivery-python-bindings/tests \ - --ignore=tests/relay/test_rln.py \ + --ignore=tests/wrappers_tests \ --ignore=tests/store/test_cursor_many_msgs.py \ - --reruns 2 \ + --reruns ${{ inputs.reruns }} \ --shard-id=${{ matrix.shard }} \ --num-shards=16 \ + --junit-xml=junit-${{ matrix.shard }}.xml \ --alluredir=allure-results-${{ matrix.shard }} fi @@ -224,6 +167,14 @@ jobs: name: allure-results-${{ matrix.shard }} path: allure-results-${{ matrix.shard }} + - name: Upload junit report + if: always() && steps.run_tests.outcome != 'skipped' + uses: actions/upload-artifact@v4 + with: + name: junit-${{ matrix.shard }} + path: junit-${{ matrix.shard }}.xml + if-no-files-found: ${{ inputs.fleet_tests && 'ignore' || 'error' }} + - name: Set job result id: set_result if: always() @@ -240,6 +191,7 @@ jobs: - name: Download all allure results uses: actions/download-artifact@v4 with: + pattern: allure-results-* path: all-results merge-multiple: true @@ -253,7 +205,6 @@ jobs: fetch-depth: 1 - name: Prepare empty gh-pages fallback - if: steps.checkout_gh_pages.outcome != 'success' run: | mkdir -p gh-pages/${{ env.CALLER }}/last-history @@ -376,7 +327,7 @@ jobs: - name: Send report to Discord uses: rjstone/discord-webhook-notify@v1 - if: always() && env.CALLER != 'manual' && inputs.send_discord + if: always() && env.CALLER != 'manual' && inputs.send_discord && env.DISCORD_WEBHOOK != '' with: severity: ${{ env.TESTS_RESULT == 'success' && 'info' || 'error' }} username: ${{ github.workflow }} diff --git a/README.md b/README.md index 1b6903fbf..b33829226 100644 --- a/README.md +++ b/README.md @@ -42,9 +42,9 @@ Waku logs can be found in `log/docker` folder while test log can be seen either ## Continuous Integration (CI) -### Daily build on *nwaku\:latest* +### Daily build on the logos-delivery nightly image -Every day the workflow **nim\_waku\_daily.yml** triggers against the image `wakuorg/nwaku:latest`. +Every day the workflow **nim\_waku\_daily.yml** triggers against `quay.io/wakuorg/nwaku-pr:nightly`, the image built from logos-delivery master by that repository's pre-release workflow. It is scheduled after the image is published. To launch it manually: @@ -57,10 +57,12 @@ To launch it manually: Every push to a pull request triggers **pr\_tests.yml** which runs: 1. **Build** — compiles `liblogosdelivery.so` (cached by submodule commit hash). -2. **Wrapper tests** — all tests under `tests/wrappers_tests/` that don't require Docker (~5 min). +2. **Wrapper tests** — all tests under `tests/wrappers_tests/`, including the `docker_required` ones (~5 min). 3. **Smoke tests** — `pytest -m smoke` with Docker nodes (~10 min). -To run the **full test suite** (18 shards, same as daily) on a PR, add the label **`full-test`** to the pull request. The full suite will start automatically. +The daily and full-suite runs no longer build `liblogosdelivery.so`: they skip `tests/wrappers_tests/`, which runs in the PR tests above. Most of that suite is also mirrored in the logos-delivery repository (`tests-e2e/`). + +To run the **full test suite** (17 shards) on a PR, add the label **`full-test`** to the pull request. Note it runs against `wakuorg/nwaku:latest`, unlike the daily run, which uses our own image. The full suite will start automatically. ### On‑demand matrix against custom *logos-messaging-nim* versions diff --git a/src/env_vars.py b/src/env_vars.py index 4985a6f17..d7852f196 100644 --- a/src/env_vars.py +++ b/src/env_vars.py @@ -14,7 +14,10 @@ def get_env_var(var_name, default=None): # Configuration constants. Need to be upercase to appear in reports -DEFAULT_NWAKU = "wakuorg/nwaku:latest" +# Tests that build their own nodes read this instead of NODE_1/NODE_2. +DEFAULT_NWAKU = get_env_var("DEFAULT_NWAKU", "wakuorg/nwaku:latest") +# Kept separate so setup_nwaku_relay_nodes stays an interop check against upstream. +UPSTREAM_NWAKU = get_env_var("UPSTREAM_NWAKU", "wakuorg/nwaku:latest") STRESS_ENABLED = False USE_WRAPPERS = True NODE_1 = get_env_var("NODE_1", DEFAULT_NWAKU) diff --git a/src/steps/sharding.py b/src/steps/sharding.py index 7c8aabc25..8bd71efe6 100644 --- a/src/steps/sharding.py +++ b/src/steps/sharding.py @@ -7,7 +7,7 @@ import allure from src.libs.common import to_base64, delay from src.node.waku_message import WakuMessage from src.env_vars import ( - DEFAULT_NWAKU, + UPSTREAM_NWAKU, NODE_2, ADDITIONAL_NODES, ) @@ -64,7 +64,7 @@ class StepsSharding(StepsRelay): @allure.step def setup_nwaku_relay_nodes(self, num_nodes, **kwargs): for index in range(num_nodes): - node = WakuNode(DEFAULT_NWAKU, f"node{index + 3}_{self.test_id}") + node = WakuNode(UPSTREAM_NWAKU, f"node{index + 3}_{self.test_id}") node.start(relay="true", discv5_bootstrap_node=self.enr_uri, **kwargs) self.add_node_peer(node, [self.multiaddr_with_id]) self.optional_nodes.append(node)