From 451df112f8574aea2840d04fffb7e16e76d24f42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lex?= Date: Mon, 26 Jan 2026 11:25:36 +0100 Subject: [PATCH] fix(ci): Cache pruning (#2005) --- .github/actions/prune-cache/action.yml | 142 +++++++++++++++--- .github/actions/setup-cargo-cache/action.yml | 45 ++++-- .github/actions/update-cargo-cache/action.yml | 55 ++++++- .github/workflows/clear-cache-pr.yml | 28 ++++ .github/workflows/clear-cache.yml | 18 --- .github/workflows/code-check.yml | 24 ++- scripts/cargo-hack-check/script.py | 1 + 7 files changed, 245 insertions(+), 68 deletions(-) create mode 100644 .github/workflows/clear-cache-pr.yml delete mode 100644 .github/workflows/clear-cache.yml diff --git a/.github/actions/prune-cache/action.yml b/.github/actions/prune-cache/action.yml index b990a3b82..8b176e3f5 100644 --- a/.github/actions/prune-cache/action.yml +++ b/.github/actions/prune-cache/action.yml @@ -1,34 +1,134 @@ -name: Prune unnecessary cache +name: Prune Cache inputs: - key-prefix: - description: The cache key prefix to prune - required: true github-token: - description: The GH token to use for the API request + description: GH token for API access required: true + key-prefix: + description: Cache key prefix + required: true + ref: + description: Optional ref to scope cache deletion (omit for all) + required: false + limit: + # NOTE: Uses a single page. Add pagination if the cache count grows. + description: Max number of caches to fetch (page size) + required: false + default: "100" + mode: + description: "Prune mode. Available options: 'delete-all' (default), 'keep-latest'" + required: false + default: "delete-all" -description: Prune unnecessary cache for the provided key +description: Prune caches matching a prefix runs: using: composite steps: - - name: Obtain cache entry to delete - id: fetch-outdated-cache + - name: Validate mode shell: bash run: | - CACHE=$(curl \ - -H "Accept: application/vnd.github.v3+json" \ - -H "Authorization: token ${{ inputs.github-token }}" \ - https://api.github.com/repos/logos-blockchain/logos-blockchain/actions/caches | jq '.actions_caches[] | select(.key | startswith("${{ inputs.key-prefix }}")) | .id') - echo "cache=$CACHE" >> "$GITHUB_OUTPUT" - continue-on-error: true - - name: Delete cache entry - if: steps.fetch-outdated-cache.outputs.cache != null + case "${{ inputs.mode }}" in + delete-all|keep-latest) + exit 0 + ;; + *) + echo "Invalid prune mode: '${{ inputs.mode }}'" + echo "Valid modes: delete-all, keep-latest" + exit 1 + ;; + esac + + - name: Select all caches + id: select + shell: bash + env: + GH_TOKEN: ${{ inputs.github-token }} + KEY_PREFIX: ${{ inputs.key-prefix }} + REF: ${{ inputs.ref }} + LIMIT: ${{ inputs.limit }} + run: | + set -euo pipefail + + OWNER="${GITHUB_REPOSITORY%%/*}" + REPOSITORY="${GITHUB_REPOSITORY##*/}" + + URL="https://api.github.com/repos/${OWNER}/${REPOSITORY}/actions/caches?per_page=${LIMIT}" + if [ -n "${REF:-}" ]; then + URL="${URL}&ref=${REF}" + fi + + RESPONSE=$(curl -sS \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: token ${GH_TOKEN}" \ + "$URL") + + IDS=$(echo "$RESPONSE" | jq -r ' + .actions_caches + | sort_by(.created_at) + | reverse + | .[] + | select(.key | startswith("'"$KEY_PREFIX"'")) + | "\(.id) \(.key)" + ') + + echo "ids<> "$GITHUB_OUTPUT" + echo "$IDS" >> "$GITHUB_OUTPUT" + echo "EOF" >> "$GITHUB_OUTPUT" + + - name: Filter for keep-latest + id: keep-latest + if: inputs.mode == 'keep-latest' shell: bash run: | - curl -X DELETE \ - -H "Accept: application/vnd.github.v3+json" \ - -H "Authorization: token ${{ inputs.github-token }}" \ - https://api.github.com/repos/logos-blockchain/logos-blockchain/actions/caches/${{ steps.fetch-outdated-cache.outputs.cache }} - continue-on-error: true + IDS="${{ steps.select.outputs.ids }}" + + if [ -z "$IDS" ]; then + echo "ids=" >> "$GITHUB_OUTPUT" + exit 0 + fi + + OUT=$(echo "$IDS" | tail -n +2) + + echo "ids<> "$GITHUB_OUTPUT" + echo "$OUT" >> "$GITHUB_OUTPUT" + echo "EOF" >> "$GITHUB_OUTPUT" + + - name: Prune selected caches + shell: bash + env: + GH_TOKEN: ${{ inputs.github-token }} + run: | + set -euo pipefail + + if [ "${{ inputs.mode }}" = "keep-latest" ]; then + IDS="${{ steps.keep-latest.outputs.ids }}" + elif [ "${{ inputs.mode }}" = "delete-all" ]; then + IDS="${{ steps.select.outputs.ids }}" + else + # This should never happen due to prior validation + exit 1 + fi + + OWNER="${GITHUB_REPOSITORY%%/*}" + REPOSITORY="${GITHUB_REPOSITORY##*/}" + URL="https://api.github.com/repos/${OWNER}/${REPOSITORY}/actions/caches" + + if [ -z "$IDS" ]; then + echo "No caches to delete" + exit 0 + fi + + echo "Deleting caches:" + set +e + while read -r id key; do + [ -z "$id" ] && continue + echo "- $key ($id)" + curl -sS -X DELETE \ + -H "Accept: application/vnd.github.v3+json" \ + -H "Authorization: token ${GH_TOKEN}" \ + "${URL}/${id}" + done <<< "$IDS" + set -e + + echo "Done" diff --git a/.github/actions/setup-cargo-cache/action.yml b/.github/actions/setup-cargo-cache/action.yml index b364de980..030a6f091 100644 --- a/.github/actions/setup-cargo-cache/action.yml +++ b/.github/actions/setup-cargo-cache/action.yml @@ -1,34 +1,51 @@ -name: Setup Cargo cache +name: Setup Cargo Cache inputs: - key: - description: The key to use for the cache - required: true github-token: description: Valid GitHub token to interact with GitHub REST API required: true + key-prefix: + description: Prefix for the cache key. + required: true -description: Set up cache read operations for Cargo artifacts +description: Set up Cargo Cache for Rust projects runs: using: composite steps: - - name: Set up Cargo cache + - name: Set up Cargo cache (main) + id: setup-main + if: github.event_name != 'pull_request' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master') uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # Version 4.2.3 + continue-on-error: true with: path: | ~/.cargo/registry/index/ ~/.cargo/registry/cache/ ~/.cargo/git/db/ target/ - # Must be in sync with the key used in the update-cargo-cache action - key: ${{ inputs.key }}->#${{ github.run_id }} + + # Must be in sync with the keys used in the `update-cargo-cache` workflow. + key: ${{ inputs.key-prefix }}/main/${{ github.sha }} restore-keys: | - ${{ inputs.key }} + ${{ inputs.key-prefix }}/main + ${{ inputs.key-prefix }} + + - name: Set up Cargo cache (PR) + id: setup-pr + if: github.event_name == 'pull_request' + uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # Version 4.2.3 continue-on-error: true - - name: Delete retrieved cache - uses: ./.github/actions/prune-cache with: - key-prefix: ${{ inputs.key }}-># - github-token: ${{ inputs.github-token }} - continue-on-error: true + path: | + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + + # Must be in sync with the keys used in the `update-cargo-cache` workflow. + key: ${{ inputs.key-prefix }}/pr#${{ github.event.pull_request.number }}/${{ github.sha }} + restore-keys: | + ${{ inputs.key-prefix }}/pr#${{ github.event.pull_request.number }} + ${{ inputs.key-prefix }}/main + ${{ inputs.key-prefix }} diff --git a/.github/actions/update-cargo-cache/action.yml b/.github/actions/update-cargo-cache/action.yml index 18d7faa7e..84927c842 100644 --- a/.github/actions/update-cargo-cache/action.yml +++ b/.github/actions/update-cargo-cache/action.yml @@ -1,23 +1,64 @@ -name: Update Cargo cache +name: Update Cargo Cache inputs: - key: - description: The key to use for the cache + github-token: + description: Valid GitHub token to interact with GitHub REST API + required: true + key-prefix: + description: Prefix for the cache key. required: true -description: Update the Cargo artifacts cache at the provided key +description: Update Cargo Cache for Rust projects runs: using: composite steps: - - name: Set up Cargo cache + - name: Update Cargo cache (main) + id: update-main + if: github.event_name != 'pull_request' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master') uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # Version 4.2.3 + continue-on-error: true with: path: | ~/.cargo/registry/index/ ~/.cargo/registry/cache/ ~/.cargo/git/db/ target/ - # Must be in sync with the key used in the setup-cargo-cache action - key: ${{ inputs.key }}->#${{ github.run_id }} + + # Must be in sync with the keys used in the `setup-cargo-cache` workflow. + key: ${{ inputs.key-prefix }}/main/${{ github.sha }} + + - name: Update Cargo cache (PR) + id: update-pr + if: github.event_name == 'pull_request' + uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # Version 4.2.3 continue-on-error: true + with: + path: | + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + + # Must be in sync with the keys used in the `setup-cargo-cache` workflow. + key: ${{ inputs.key-prefix }}/pr#${{ github.event.pull_request.number }}/${{ github.sha }} + + - name: Delete retrieved cache (main) + if: github.event_name != 'pull_request' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master') + uses: ./.github/actions/prune-cache + continue-on-error: true + with: + github-token: ${{ inputs.github-token }} + # Must be in sync with the keys used in the `setup-cargo-cache` workflow. + key-prefix: ${{ inputs.key-prefix }}/main + mode: keep-latest + + - name: Delete retrieved cache (PR) + if: github.event_name == 'pull_request' + uses: ./.github/actions/prune-cache + continue-on-error: true + with: + github-token: ${{ inputs.github-token }} + # Must be in sync with the keys used in the `setup-cargo-cache` workflow. + key-prefix: ${{ inputs.key-prefix }}/pr#${{ github.event.pull_request.number }} + mode: keep-latest diff --git a/.github/workflows/clear-cache-pr.yml b/.github/workflows/clear-cache-pr.yml new file mode 100644 index 000000000..a591463ea --- /dev/null +++ b/.github/workflows/clear-cache-pr.yml @@ -0,0 +1,28 @@ +name: Clear GitHub Runner caches on PR close + +on: + pull_request: + types: + - closed + +jobs: + clear-caches: + runs-on: ubuntu-latest + permissions: + actions: write + steps: + - name: Prune Cargo Hack Cache + uses: ./.github/actions/prune-cache + continue-on-error: true + with: + github-token: ${{ inputs.github-token }} + key-prefix: "cargo-hack-check/" + ref: ${{ github.ref }} + + - name: Prune Code Check Cache + uses: ./.github/actions/prune-cache + continue-on-error: true + with: + github-token: ${{ inputs.github-token }} + key-prefix: "code-check/" + ref: ${{ github.ref }} diff --git a/.github/workflows/clear-cache.yml b/.github/workflows/clear-cache.yml deleted file mode 100644 index c7a5d57e5..000000000 --- a/.github/workflows/clear-cache.yml +++ /dev/null @@ -1,18 +0,0 @@ -name: Clean up GitHub Runner caches on PR close -on: - pull_request: - types: - - closed - -jobs: - cleanup: - runs-on: ubuntu-latest - permissions: - actions: write - steps: - - name: Delete Cargo Hack Cache - uses: ./.github/actions/prune-cache - continue-on-error: true - with: - key-prefix: "cargo-hack-check/" - github-token: ${{ inputs.github-token }} diff --git a/.github/workflows/code-check.yml b/.github/workflows/code-check.yml index 90717dd97..168ef900d 100644 --- a/.github/workflows/code-check.yml +++ b/.github/workflows/code-check.yml @@ -62,11 +62,13 @@ jobs: strategy: fail-fast: false matrix: - os: + platform: # Using self-hosted runners because we were running out of memory on GitHub-hosted runners - - [self-hosted, Linux] - - [self-hosted, macOS] - runs-on: ${{ matrix.os }} + - os: [self-hosted, Linux] + label: self-hosted-linux + - os: [self-hosted, macOS] + label: self-hosted-macos + runs-on: ${{ matrix.platform.os }} steps: - name: Checkout repository uses: actions/checkout@85e6279cec87321a52edac9c87bce653a07cf6c2 # Version 4.2.2 @@ -83,8 +85,8 @@ jobs: - name: Set up Cargo cache uses: ./.github/actions/setup-cargo-cache with: - key: ${{ github.ref }}->${{ github.workflow }}->${{ github.job }}->${{ matrix.os }} github-token: ${{ secrets.GITHUB_TOKEN }} + key-prefix: code-check/feature-combinations/${{ matrix.platform.label }} - name: Run Cargo Hack Check (Cache-Aware) uses: ./.github/actions/cargo-hack-check-cached @@ -93,7 +95,8 @@ jobs: if: success() || failure() uses: ./.github/actions/update-cargo-cache with: - key: ${{ github.ref }}->${{ github.workflow }}->${{ github.job }}->${{ matrix.os }} + github-token: ${{ secrets.GITHUB_TOKEN }} + key-prefix: code-check/feature-combinations/${{ matrix.platform.label }} unused-deps: name: Check for unused dependencies @@ -125,25 +128,30 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@85e6279cec87321a52edac9c87bce653a07cf6c2 # Version 4.2.2 + - name: Set up Cargo cache uses: ./.github/actions/setup-cargo-cache with: - key: ${{ github.ref }}->${{ github.workflow }}->${{ github.job }}->${{ matrix.os }} github-token: ${{ secrets.GITHUB_TOKEN }} + key-prefix: code-check/lints/${{ matrix.os }} + - name: Setup Circuits uses: ./.github/actions/setup-circuits with: github-token: ${{ secrets.GITHUB_TOKEN }} + - name: Run cargo clippy on workspace uses: actions-rs/cargo@9e120dd99b0fbad1c065f686657e914e76bd7b72 # Version 1.0.1 with: command: clippy args: --locked --all --all-targets --all-features + - name: Update Cargo cache if: success() || failure() uses: ./.github/actions/update-cargo-cache with: - key: ${{ github.ref }}->${{ github.workflow }}->${{ github.job }}->${{ matrix.os }} + github-token: ${{ secrets.GITHUB_TOKEN }} + key-prefix: code-check/lints/${{ matrix.os }} tests: name: Run test suite diff --git a/scripts/cargo-hack-check/script.py b/scripts/cargo-hack-check/script.py index 21df3c2c8..0a7ffe1e1 100755 --- a/scripts/cargo-hack-check/script.py +++ b/scripts/cargo-hack-check/script.py @@ -23,6 +23,7 @@ import re # - WORKSPACE_ROOT pointing to the root of the workspace. # - HASH_SCRIPT being in the same directory as this script. # If these prerequisites are not met, the script will not behave as expected. +# Moving these to parameters would be safer. CURRENT_FILE_DIRECTORY = Path(__file__).parent.resolve() WORKSPACE_ROOT = CURRENT_FILE_DIRECTORY.parent.parent