diff --git a/.github/workflows/nim_waku_daily.yml b/.github/workflows/nim_waku_daily.yml index 44e47d531..21607bd11 100644 --- a/.github/workflows/nim_waku_daily.yml +++ b/.github/workflows/nim_waku_daily.yml @@ -5,8 +5,9 @@ concurrency: cancel-in-progress: false on: + # The nightly image is pushed at 04:34-04:41 UTC; 06:00 keeps this run behind it. schedule: - - cron: '0 4 * * *' + - cron: '0 6 * * *' workflow_dispatch: jobs: @@ -14,7 +15,9 @@ jobs: uses: ./.github/workflows/test_common.yml secrets: inherit with: - node1: "wakuorg/nwaku:latest" - node2: "wakuorg/nwaku:latest" - additional_nodes: "wakuorg/nwaku:latest,wakuorg/nwaku:latest,wakuorg/nwaku:latest" - caller: "nim" \ No newline at end of file + node1: "quay.io/wakuorg/nwaku-pr:nightly" + node2: "quay.io/wakuorg/nwaku-pr:nightly" + additional_nodes: "quay.io/wakuorg/nwaku-pr:nightly,quay.io/wakuorg/nwaku-pr:nightly,quay.io/wakuorg/nwaku-pr:nightly" + default_nwaku: "quay.io/wakuorg/nwaku-pr:nightly" + caller: "nim-core" + reruns: 0 \ No newline at end of file diff --git a/.github/workflows/pr_tests.yml b/.github/workflows/pr_tests.yml index a04f30dec..f7dad40a1 100644 --- a/.github/workflows/pr_tests.yml +++ b/.github/workflows/pr_tests.yml @@ -21,7 +21,7 @@ on: workflow_dispatch: inputs: run_full_suite: - description: "Run the full test suite (18 shards)" + description: "Run the full test suite (17 shards)" required: false default: false type: boolean @@ -127,7 +127,7 @@ jobs: name: Wrapper Tests runs-on: ubuntu-latest needs: [build] - timeout-minutes: 45 + timeout-minutes: 60 if: >- github.event_name != 'push' && (github.event.action != 'labeled' || github.event.label.name == 'full-test') @@ -159,6 +159,26 @@ jobs: --reruns 2 \ --junit-xml=wrapper-results-basic.xml + - name: Run wrapper tests - channel lifecycle + continue-on-error: true + env: + PYTHONPATH: ${{ github.workspace }}/vendor/logos-delivery-python-bindings/waku + run: | + pytest tests/wrappers_tests/test_channel_lifecycle.py \ + -m "not docker_required" \ + --reruns 2 \ + --junit-xml=wrapper-results-channel-lifecycle.xml + + - name: Run wrapper tests - docker required + continue-on-error: true + env: + PYTHONPATH: ${{ github.workspace }}/vendor/logos-delivery-python-bindings/waku + run: | + pytest tests/wrappers_tests \ + -m docker_required \ + --reruns 2 \ + --junit-xml=wrapper-results-docker.xml + - name: Run wrapper tests - send handle and subscription continue-on-error: true env: diff --git a/.github/workflows/test_common.yml b/.github/workflows/test_common.yml index 13657dd07..5469e7b5e 100644 --- a/.github/workflows/test_common.yml +++ b/.github/workflows/test_common.yml @@ -37,14 +37,32 @@ on: description: "Send test results to Discord" type: boolean default: false + reruns: + required: false + description: "pytest --reruns; 0 keeps flaky runs visible in the JUnit report" + type: number + default: 2 + default_nwaku: + required: false + description: "Image for tests that build their own nodes instead of using node1/node2" + type: string + default: "" + upstream_nwaku: + required: false + description: "Image for setup_nwaku_relay_nodes, which stays an interop check against upstream" + type: string + default: "" env: FORCE_COLOR: "1" NODE_1: ${{ inputs.node1 }} NODE_2: ${{ inputs.node2 }} ADDITIONAL_NODES: ${{ inputs.additional_nodes }} + DEFAULT_NWAKU: ${{ inputs.default_nwaku }} + UPSTREAM_NWAKU: ${{ inputs.upstream_nwaku }} CALLER: ${{ inputs.caller || 'manual' }} RLN_CREDENTIALS: ${{ secrets.RLN_CREDENTIALS }} + DISCORD_WEBHOOK: ${{ secrets.DISCORD_TEST_REPORTS_WH }} jobs: tests: @@ -52,11 +70,9 @@ jobs: strategy: fail-fast: false matrix: - shard: [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17] - # total number of shards =18 means tests will split into 18 thread and run in parallel to increase execution speed + shard: [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,17] # command for sharding : # pytest --shard-id= --num-shards= - # shard 16 for test_rln.py file as they shall run sequentially # shard 17 for test_cursor_many_msgs.py as it takes time >7 mins runs-on: ubuntu-latest @@ -78,13 +94,10 @@ jobs: jobResult_13: ${{ steps.set_result.outputs.JOB_RESULT_13 }} jobResult_14: ${{ steps.set_result.outputs.JOB_RESULT_14 }} jobResult_15: ${{ steps.set_result.outputs.JOB_RESULT_15 }} - jobResult_16: ${{ steps.set_result.outputs.JOB_RESULT_16 }} jobResult_17: ${{ steps.set_result.outputs.JOB_RESULT_17 }} steps: - uses: actions/checkout@v4 - with: - submodules: recursive - name: Remove unwanted software uses: ./.github/actions/prune-vm @@ -94,9 +107,6 @@ jobs: python-version: "3.12" cache: "pip" - - name: Add python bindings to PYTHONPATH - run: echo "PYTHONPATH=$(pwd)/vendor/logos-delivery-python-bindings/waku:$PYTHONPATH" >> $GITHUB_ENV - - name: Install system deps for tc / nsenter run: | sudo apt-get update @@ -110,110 +120,43 @@ jobs: gcc \ g++ - - name: Install Nim 2.2.4 - run: | - set -euo pipefail - curl https://nim-lang.org/choosenim/init.sh -sSf | sh -s -- -y - echo "$HOME/.nimble/bin" >> "$GITHUB_PATH" - export PATH="$HOME/.nimble/bin:$PATH" - choosenim 2.2.4 - # Pin the nimble that generated logos-delivery's nimble.lock. A newer - # nimble recomputes the locked Nim package checksum differently, which - # makes `nimble setup --localdeps` abort with a checksum mismatch. - (cd /tmp && nimble install "nimble@0.22.3" -y) - nim --version - nimble --version - - run: pip install -r requirements.txt - - name: Build liblogosdelivery.so for python bindings + - name: Pull and inspect node images run: | set -euo pipefail - - export PATH="$HOME/.nimble/bin:$PATH" - - BINDINGS_DIR="$(pwd)/vendor/logos-delivery-python-bindings" - DELIVERY_DIR="$BINDINGS_DIR/vendor/logos-delivery" - - mkdir -p "$BINDINGS_DIR/lib" - - cd "$DELIVERY_DIR" - - ln -sf waku.nimble waku.nims - - # `make liblogosdelivery` resolves the locked deps via - # `nimble setup --localdeps` (build-deps prerequisite); a bare - # `nimble install -y` here is redundant and pulls Nim from the lock, - # which is what triggered the checksum mismatch. - make liblogosdelivery - - SO_PATH="$(find . -type f -name 'liblogosdelivery.so' | head -n 1)" - - if [ -z "$SO_PATH" ]; then - echo "liblogosdelivery.so was not built" - exit 1 - fi - - cp "$SO_PATH" "$BINDINGS_DIR/lib/liblogosdelivery.so" - - echo "Built library:" - ls -l "$BINDINGS_DIR/lib/liblogosdelivery.so" - - # Cleanup build artifacts - rm -rf .gh-pages/nim - - - - name: Verify wrapper library - run: | - test -f vendor/logos-delivery-python-bindings/lib/liblogosdelivery.so - - - name: Debug Python import paths - run: | - pwd - echo "PYTHONPATH=$PYTHONPATH" - find . -maxdepth 5 | grep wrapper || true - python - <<'PY' - import sys - print("sys.path:") - for p in sys.path: - print(p) - try: - import wrapper - print("wrapper import OK:", wrapper) - except Exception as e: - print("wrapper import failed:", e) - raise - PY + printf '%s\n' "$NODE_1" "$NODE_2" "$DEFAULT_NWAKU" "$UPSTREAM_NWAKU" ${ADDITIONAL_NODES:-} | tr ',' '\n' | sort -u | while read -r image; do + [ -n "$image" ] || continue + docker pull -q "$image" + docker image inspect "$image" \ + --format "- \`$image\` digest={{if .RepoDigests}}{{index .RepoDigests 0}}{{else}}{{end}} created={{.Created}}" \ + | tee -a "$GITHUB_STEP_SUMMARY" + done - name: Run tests + id: run_tests run: | - export PATH="$HOME/.nimble/bin:$PATH" - export PYTHONPATH="$(pwd)/vendor/logos-delivery-python-bindings/waku:$PYTHONPATH" - if [ "${{ inputs.fleet_tests }}" == "true" ]; then if [ "${{ matrix.shard }}" == "0" ]; then pytest --fleet -m waku_test_fleet \ - --ignore=vendor/logos-delivery-python-bindings/tests \ + --ignore=tests/wrappers_tests \ + --junit-xml=junit-${{ matrix.shard }}.xml \ --alluredir=allure-results-${{ matrix.shard }} else echo "Skipping shard ${{ matrix.shard }}: fleet tests run on shard 0 only" fi - elif [ "${{ matrix.shard }}" == "16" ]; then - pytest tests/relay/test_rln.py \ - --ignore=vendor/logos-delivery-python-bindings/tests \ - --alluredir=allure-results-${{ matrix.shard }} elif [ "${{ matrix.shard }}" == "17" ]; then pytest tests/store/test_cursor_many_msgs.py \ - --ignore=vendor/logos-delivery-python-bindings/tests \ + --junit-xml=junit-${{ matrix.shard }}.xml \ --alluredir=allure-results-${{ matrix.shard }} else pytest \ - --ignore=vendor/logos-delivery-python-bindings/tests \ - --ignore=tests/relay/test_rln.py \ + --ignore=tests/wrappers_tests \ --ignore=tests/store/test_cursor_many_msgs.py \ - --reruns 2 \ + --reruns ${{ inputs.reruns }} \ --shard-id=${{ matrix.shard }} \ --num-shards=16 \ + --junit-xml=junit-${{ matrix.shard }}.xml \ --alluredir=allure-results-${{ matrix.shard }} fi @@ -224,6 +167,14 @@ jobs: name: allure-results-${{ matrix.shard }} path: allure-results-${{ matrix.shard }} + - name: Upload junit report + if: always() && steps.run_tests.outcome != 'skipped' + uses: actions/upload-artifact@v4 + with: + name: junit-${{ matrix.shard }} + path: junit-${{ matrix.shard }}.xml + if-no-files-found: ${{ inputs.fleet_tests && 'ignore' || 'error' }} + - name: Set job result id: set_result if: always() @@ -240,6 +191,7 @@ jobs: - name: Download all allure results uses: actions/download-artifact@v4 with: + pattern: allure-results-* path: all-results merge-multiple: true @@ -253,7 +205,6 @@ jobs: fetch-depth: 1 - name: Prepare empty gh-pages fallback - if: steps.checkout_gh_pages.outcome != 'success' run: | mkdir -p gh-pages/${{ env.CALLER }}/last-history @@ -376,7 +327,7 @@ jobs: - name: Send report to Discord uses: rjstone/discord-webhook-notify@v1 - if: always() && env.CALLER != 'manual' && inputs.send_discord + if: always() && env.CALLER != 'manual' && inputs.send_discord && env.DISCORD_WEBHOOK != '' with: severity: ${{ env.TESTS_RESULT == 'success' && 'info' || 'error' }} username: ${{ github.workflow }} diff --git a/README.md b/README.md index 1b6903fbf..b33829226 100644 --- a/README.md +++ b/README.md @@ -42,9 +42,9 @@ Waku logs can be found in `log/docker` folder while test log can be seen either ## Continuous Integration (CI) -### Daily build on *nwaku\:latest* +### Daily build on the logos-delivery nightly image -Every day the workflow **nim\_waku\_daily.yml** triggers against the image `wakuorg/nwaku:latest`. +Every day the workflow **nim\_waku\_daily.yml** triggers against `quay.io/wakuorg/nwaku-pr:nightly`, the image built from logos-delivery master by that repository's pre-release workflow. It is scheduled after the image is published. To launch it manually: @@ -57,10 +57,12 @@ To launch it manually: Every push to a pull request triggers **pr\_tests.yml** which runs: 1. **Build** — compiles `liblogosdelivery.so` (cached by submodule commit hash). -2. **Wrapper tests** — all tests under `tests/wrappers_tests/` that don't require Docker (~5 min). +2. **Wrapper tests** — all tests under `tests/wrappers_tests/`, including the `docker_required` ones (~5 min). 3. **Smoke tests** — `pytest -m smoke` with Docker nodes (~10 min). -To run the **full test suite** (18 shards, same as daily) on a PR, add the label **`full-test`** to the pull request. The full suite will start automatically. +The daily and full-suite runs no longer build `liblogosdelivery.so`: they skip `tests/wrappers_tests/`, which runs in the PR tests above. Most of that suite is also mirrored in the logos-delivery repository (`tests-e2e/`). + +To run the **full test suite** (17 shards) on a PR, add the label **`full-test`** to the pull request. Note it runs against `wakuorg/nwaku:latest`, unlike the daily run, which uses our own image. The full suite will start automatically. ### On‑demand matrix against custom *logos-messaging-nim* versions diff --git a/src/env_vars.py b/src/env_vars.py index 4985a6f17..d7852f196 100644 --- a/src/env_vars.py +++ b/src/env_vars.py @@ -14,7 +14,10 @@ def get_env_var(var_name, default=None): # Configuration constants. Need to be upercase to appear in reports -DEFAULT_NWAKU = "wakuorg/nwaku:latest" +# Tests that build their own nodes read this instead of NODE_1/NODE_2. +DEFAULT_NWAKU = get_env_var("DEFAULT_NWAKU", "wakuorg/nwaku:latest") +# Kept separate so setup_nwaku_relay_nodes stays an interop check against upstream. +UPSTREAM_NWAKU = get_env_var("UPSTREAM_NWAKU", "wakuorg/nwaku:latest") STRESS_ENABLED = False USE_WRAPPERS = True NODE_1 = get_env_var("NODE_1", DEFAULT_NWAKU) diff --git a/src/steps/sharding.py b/src/steps/sharding.py index 7c8aabc25..8bd71efe6 100644 --- a/src/steps/sharding.py +++ b/src/steps/sharding.py @@ -7,7 +7,7 @@ import allure from src.libs.common import to_base64, delay from src.node.waku_message import WakuMessage from src.env_vars import ( - DEFAULT_NWAKU, + UPSTREAM_NWAKU, NODE_2, ADDITIONAL_NODES, ) @@ -64,7 +64,7 @@ class StepsSharding(StepsRelay): @allure.step def setup_nwaku_relay_nodes(self, num_nodes, **kwargs): for index in range(num_nodes): - node = WakuNode(DEFAULT_NWAKU, f"node{index + 3}_{self.test_id}") + node = WakuNode(UPSTREAM_NWAKU, f"node{index + 3}_{self.test_id}") node.start(relay="true", discv5_bootstrap_node=self.enr_uri, **kwargs) self.add_node_peer(node, [self.multiaddr_with_id]) self.optional_nodes.append(node)