From 6ebb3ec8d89f3302720e3b7165650514cdc02066 Mon Sep 17 00:00:00 2001 From: Aya Hassan Date: Tue, 23 Jun 2026 23:45:38 +0200 Subject: [PATCH] make pr job fetch the nightly built .so lib --- .github/workflows/test_PR_image.yml | 57 +++++++++++ .github/workflows/test_common.yml | 153 +++++++++++++++++++--------- 2 files changed, 160 insertions(+), 50 deletions(-) diff --git a/.github/workflows/test_PR_image.yml b/.github/workflows/test_PR_image.yml index a2d23cb11..30fe013a1 100644 --- a/.github/workflows/test_PR_image.yml +++ b/.github/workflows/test_PR_image.yml @@ -44,6 +44,7 @@ jobs: with: repository: logos-messaging/logos-messaging-interop-tests ref: SMOKE_TEST_STABLE + submodules: recursive - uses: actions/setup-python@v4 with: @@ -52,8 +53,64 @@ jobs: - run: pip install -r requirements.txt + - name: Download nightly liblogosdelivery.so + id: download_lib + continue-on-error: true + uses: dawidd6/action-download-artifact@v6 + with: + # Reuse the portable .so built each night by test_common.yml against the + # latest logos-delivery commit, instead of rebuilding it here. + workflow: nim_waku_daily.yml + workflow_conclusion: success + name: liblogosdelivery + path: vendor/logos-delivery-python-bindings/lib/ + if_no_artifact_found: warn + + - name: Build liblogosdelivery.so (fallback) + if: steps.download_lib.outputs.found_artifact != 'true' + run: | + set -euo pipefail + # No nightly artifact available (nightly failed / never ran). Build a + # portable .so from source so smoke tests stay self-sufficient. + sudo apt-get update + sudo apt-get install -y \ + util-linux iproute2 sudo ca-certificates curl make gcc g++ + + curl https://nim-lang.org/choosenim/init.sh -sSf | sh -s -- -y + export PATH="$HOME/.nimble/bin:$PATH" + choosenim 2.2.4 + # Pin the nimble that generated nimble.lock; newer ones fail the checksum. + (cd /tmp && nimble install "nimble@0.22.3" -y) + + BINDINGS_DIR="$(pwd)/vendor/logos-delivery-python-bindings" + DELIVERY_DIR="$BINDINGS_DIR/vendor/logos-delivery" + mkdir -p "$BINDINGS_DIR/lib" + + cd "$DELIVERY_DIR" + # Match the nightly: build against the latest logos-delivery HEAD. + git fetch origin + DEFAULT_REF=$(git remote show origin | awk '/HEAD branch/ {print $NF}') + git checkout "origin/$DEFAULT_REF" + git submodule sync --recursive + git submodule update --init --recursive + + ln -sf waku.nimble waku.nims + NIM_PARAMS="-d:disableMarchNative -d:marchOptimized" make liblogosdelivery + + SO_PATH="$(find . -type f -name 'liblogosdelivery.so' | head -n 1)" + if [ -z "$SO_PATH" ]; then + echo "liblogosdelivery.so was not built" + exit 1 + fi + cp "$SO_PATH" "$BINDINGS_DIR/lib/liblogosdelivery.so" + + - name: Verify wrapper library + run: test -f vendor/logos-delivery-python-bindings/lib/liblogosdelivery.so + - name: Run tests timeout-minutes: 30 + env: + PYTHONPATH: ${{ github.workspace }}/vendor/logos-delivery-python-bindings/waku run: | pytest -m 'smoke' -n 4 --dist=loadgroup --reruns 1 --junit-xml=pytest_results.xml diff --git a/.github/workflows/test_common.yml b/.github/workflows/test_common.yml index 9c35b562a..f60d9c42c 100644 --- a/.github/workflows/test_common.yml +++ b/.github/workflows/test_common.yml @@ -47,8 +47,106 @@ env: RLN_CREDENTIALS: ${{ secrets.RLN_CREDENTIALS }} jobs: + build: + name: Build liblogosdelivery (latest logos-delivery) + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Remove unwanted software + uses: ./.github/actions/prune-vm + + - name: Install system deps + run: | + sudo apt-get update + sudo apt-get install -y \ + util-linux \ + iproute2 \ + sudo \ + ca-certificates \ + curl \ + make \ + gcc \ + g++ + + - name: Install Nim 2.2.4 and Nimble 0.22.3 + run: | + set -euo pipefail + curl https://nim-lang.org/choosenim/init.sh -sSf | sh -s -- -y + echo "$HOME/.nimble/bin" >> "$GITHUB_PATH" + export PATH="$HOME/.nimble/bin:$PATH" + choosenim 2.2.4 + # Pin the nimble that generated nimble.lock; newer ones fail the checksum. + (cd /tmp && nimble install "nimble@0.22.3" -y) + nim --version + nimble --version + + - name: Update logos-delivery to latest commit + run: | + set -euo pipefail + DELIVERY_DIR="$(pwd)/vendor/logos-delivery-python-bindings/vendor/logos-delivery" + cd "$DELIVERY_DIR" + # Build against the freshest logos-delivery HEAD rather than the pinned + # submodule commit. Relies on the auth context actions/checkout sets up + # for submodules (SSH URL rewritten to token-authenticated HTTPS). + git fetch origin + DEFAULT_REF=$(git remote show origin | awk '/HEAD branch/ {print $NF}') + echo "Building against latest logos-delivery: origin/$DEFAULT_REF" + git checkout "origin/$DEFAULT_REF" + git rev-parse HEAD + # The new commit may move nested submodule pointers needed by the build. + git submodule sync --recursive + git submodule update --init --recursive + + - name: Build liblogosdelivery.so for python bindings + run: | + set -euo pipefail + + export PATH="$HOME/.nimble/bin:$PATH" + + BINDINGS_DIR="$(pwd)/vendor/logos-delivery-python-bindings" + DELIVERY_DIR="$BINDINGS_DIR/vendor/logos-delivery" + + mkdir -p "$BINDINGS_DIR/lib" + + cd "$DELIVERY_DIR" + + ln -sf waku.nimble waku.nims + + # Portable build (no -march=native) so the shared .so doesn't SIGILL on + # test runners with older CPUs; via env so the Makefile's NIM_PARAMS apply. + NIM_PARAMS="-d:disableMarchNative -d:marchOptimized" make liblogosdelivery + + SO_PATH="$(find . -type f -name 'liblogosdelivery.so' | head -n 1)" + + if [ -z "$SO_PATH" ]; then + echo "liblogosdelivery.so was not built" + exit 1 + fi + + cp "$SO_PATH" "$BINDINGS_DIR/lib/liblogosdelivery.so" + + echo "Built library:" + ls -l "$BINDINGS_DIR/lib/liblogosdelivery.so" + + # Cleanup build artifacts + rm -rf .gh-pages/nim + + - name: Upload liblogosdelivery.so artifact + uses: actions/upload-artifact@v4 + with: + name: liblogosdelivery + path: vendor/logos-delivery-python-bindings/lib/liblogosdelivery.so + # Consumed by the next nightly + by test_PR_image.yml smoke runs; a short + # window is enough since a fresh build is uploaded every night. + retention-days: 2 + tests: name: tests + needs: [build] strategy: fail-fast: false matrix: @@ -110,58 +208,13 @@ jobs: gcc \ g++ - - name: Install Nim 2.2.4 - run: | - set -euo pipefail - curl https://nim-lang.org/choosenim/init.sh -sSf | sh -s -- -y - echo "$HOME/.nimble/bin" >> "$GITHUB_PATH" - export PATH="$HOME/.nimble/bin:$PATH" - choosenim 2.2.4 - # Pin the nimble that generated logos-delivery's nimble.lock. A newer - # nimble recomputes the locked Nim package checksum differently, which - # makes `nimble setup --localdeps` abort with a checksum mismatch. - (cd /tmp && nimble install "nimble@0.22.3" -y) - nim --version - nimble --version - - run: pip install -r requirements.txt - - name: Build liblogosdelivery.so for python bindings - run: | - set -euo pipefail - - export PATH="$HOME/.nimble/bin:$PATH" - - BINDINGS_DIR="$(pwd)/vendor/logos-delivery-python-bindings" - DELIVERY_DIR="$BINDINGS_DIR/vendor/logos-delivery" - - mkdir -p "$BINDINGS_DIR/lib" - - cd "$DELIVERY_DIR" - - ln -sf waku.nimble waku.nims - - # `make liblogosdelivery` resolves the locked deps via - # `nimble setup --localdeps` (build-deps prerequisite); a bare - # `nimble install -y` here is redundant and pulls Nim from the lock, - # which is what triggered the checksum mismatch. - make liblogosdelivery - - SO_PATH="$(find . -type f -name 'liblogosdelivery.so' | head -n 1)" - - if [ -z "$SO_PATH" ]; then - echo "liblogosdelivery.so was not built" - exit 1 - fi - - cp "$SO_PATH" "$BINDINGS_DIR/lib/liblogosdelivery.so" - - echo "Built library:" - ls -l "$BINDINGS_DIR/lib/liblogosdelivery.so" - - # Cleanup build artifacts - rm -rf .gh-pages/nim - + - name: Download liblogosdelivery.so + uses: actions/download-artifact@v4 + with: + name: liblogosdelivery + path: vendor/logos-delivery-python-bindings/lib/ - name: Verify wrapper library run: |