chore: clean github hosted runners as they run out of disk space (#2276)

.github/actions/prune-disk-space/action.yml

@@ -0,0 +1,114 @@
+name: Prune disk space (cache-aware, self-hosted)
+description: Frees disk space from previous run leftovers on self-hosted runners (Linux + macOS)
+
+runs:
+  using: composite
+  steps:
+    - name: Prune disk space
+      if: always()
+      shell: bash
+      run: |
+        set -uo pipefail
+
+        echo "Disk before:"
+        df -h || true
+
+        avail_kb() { df -Pk . | awk 'NR==2 {print $4}'; }
+
+        # --- 1) macOS: remove big, known-safe Xcode junk (keep it surgical)
+        if [ "${RUNNER_OS}" = "macOS" ]; then
+          rm -rf ~/Library/Developer/Xcode/DerivedData 2>/dev/null || true
+          rm -rf ~/Library/Caches/com.apple.python 2>/dev/null || true
+          rm -rf ~/Library/Caches/gmp-mpfr-sys 2>/dev/null || true
+        fi
+
+        # --- 2) Repo-local reproducible bloat
+        find . -type d -name node_modules -prune -exec rm -rf {} + 2>/dev/null || true
+        find . -type d -name dist         -prune -exec rm -rf {} + 2>/dev/null || true
+        find . -type d -name build        -prune -exec rm -rf {} + 2>/dev/null || true
+        find . -type d -name .cache       -prune -exec rm -rf {} + 2>/dev/null || true
+        find . -type d -name .vite        -prune -exec rm -rf {} + 2>/dev/null || true
+        find . -type d -name __pycache__  -prune -exec rm -rf {} + 2>/dev/null || true
+        find . -not -path './target/*' -name "*.pyc" -delete 2>/dev/null || true
+
+        # npm/pip cache cleanup — safe to run always; no-ops if not installed/used
+        npm cache clean --force 2>/dev/null || true
+        pip cache purge 2>/dev/null || true
+
+        # NOTE: Do NOT prune ~/.cargo here in normal mode.
+        # The workflow restores ~/.cargo via actions/cache; pruning it here defeats caching and can cause refetch storms.
+
+        # --- 3) Target thinning: keep cache value, remove worst disk hogs
+        # (incremental + fingerprints + build-script outputs). Do NOT delete deps in normal mode.
+        if [ -d "target" ]; then
+          echo "Target before:"
+          du -sh target 2>/dev/null || true
+
+          find target -type d -name incremental   -prune -exec rm -rf {} + 2>/dev/null || true
+          find target -type d -name .fingerprint  -prune -exec rm -rf {} + 2>/dev/null || true
+          find target -type d -name build         -prune -exec rm -rf {} + 2>/dev/null || true
+
+          echo "Target after:"
+          du -sh target 2>/dev/null || true
+        fi
+
+        # --- 4) Diagnostics: show biggest offenders (cheap, useful)
+        echo "Top workspace dirs (largest 30):"
+        (
+          du -xh -d 2 . 2>/dev/null | sort -rh 2>/dev/null
+        ) || (
+          du -xh -d 2 . 2>/dev/null | sort -r
+        ) | head -n 30 || true
+
+        echo "npm cache:        $(du -sh ~/.npm 2>/dev/null || echo 'n/a')"
+        echo "cargo registry:   $(du -sh ~/.cargo/registry 2>/dev/null || echo 'n/a')"
+        echo "cargo git:        $(du -sh ~/.cargo/git 2>/dev/null || echo 'n/a')"
+
+        if [ -d target ]; then
+          echo "Top target dirs (largest 30):"
+          (
+            du -xh -d 2 target 2>/dev/null | sort -rh 2>/dev/null
+          ) || (
+            du -xh -d 2 target 2>/dev/null | sort -r
+          ) | head -n 30 || true
+        fi
+
+        # --- 5) Runner diag trimming: robust globbing across Linux + macOS runner layouts
+        # Use nullglob so non-matching globs expand to nothing (no noisy literal patterns).
+        (
+          shopt -s nullglob
+          for d in /github-runner/*/_diag /Users/*/github-runner-node-*/_diag; do
+            [ -d "$d" ] || continue
+            echo "Diag before ($d):"
+            du -sh "$d" 2>/dev/null || true
+            # Keep recent logs; delete older ones
+            find "$d" -type f -mtime +1 -delete 2>/dev/null || true
+            echo "Diag after ($d):"
+            du -sh "$d" 2>/dev/null || true
+          done
+        )
+
+        # --- 6) Emergency mode: if still low disk, apply stronger pruning (rebuildable)
+        # 15 GB threshold
+        free_now=$(avail_kb 2>/dev/null || echo 999999999)
+        if [ "$free_now" -lt $((15*1024*1024)) ]; then
+          echo "Low disk (<15GB). Applying emergency pruning..."
+
+          # In emergency, it's acceptable to drop deps to prevent runner death.
+          if [ -d target ]; then
+            for profile in debug release; do
+              if [ -d "target/$profile" ]; then
+                find "target/$profile" -maxdepth 1 -type d \
+                  \( -name deps -o -name build -o -name .fingerprint -o -name incremental \) \
+                  -exec rm -rf {} + 2>/dev/null || true
+              fi
+            done
+          fi
+
+          # As a last resort, prune old cargo *download* artifacts only (least harmful),
+          # but keep this very conservative and best-effort.
+          find ~/.cargo/registry/cache -type f -name "*.crate" -mtime +14 -delete 2>/dev/null || true
+        fi
+
+        echo "Disk after:"
+        df -h || true

.github/actions/setup-cargo-cache/action.yml

@@ -19,11 +19,18 @@ runs:
       uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # Version 4.2.3
       continue-on-error: true
       with:
+        # This excludes the final linked binaries and test executables (which are large but fast to re-link) while
+        # keeping the compiled dependency objects (which are slow to recompile).
         path: |
           ~/.cargo/registry/index/
           ~/.cargo/registry/cache/
           ~/.cargo/git/db/
-          target/
+          target/debug/.fingerprint/
+          target/debug/build/
+          target/debug/deps/
+          target/release/.fingerprint/
+          target/release/build/
+          target/release/deps/
 
         # Must be in sync with the keys used in the `update-cargo-cache` workflow.
         key: ${{ inputs.key-prefix }}/main/${{ github.sha }}
@@ -37,11 +44,18 @@ runs:
       uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # Version 4.2.3
       continue-on-error: true
       with:
+        # This excludes the final linked binaries and test executables (which are large but fast to re-link) while
+        # keeping the compiled dependency objects (which are slow to recompile).
         path: |
           ~/.cargo/registry/index/
           ~/.cargo/registry/cache/
           ~/.cargo/git/db/
-          target/
+          target/debug/.fingerprint/
+          target/debug/build/
+          target/debug/deps/
+          target/release/.fingerprint/
+          target/release/build/
+          target/release/deps/
 
         # Must be in sync with the keys used in the `update-cargo-cache` workflow.
         key: ${{ inputs.key-prefix }}/pr#${{ github.event.pull_request.number }}/${{ github.sha }}
@@ -49,3 +63,31 @@ runs:
           ${{ inputs.key-prefix }}/pr#${{ github.event.pull_request.number }}
           ${{ inputs.key-prefix }}/main
           ${{ inputs.key-prefix }}
+
+    - name: Log cache directory sizes
+      shell: bash
+      run: |
+        echo "=== Cargo registry ==="
+        du -sh ~/.cargo/registry/index/  2>/dev/null || echo "  index: not found"
+        du -sh ~/.cargo/registry/cache/  2>/dev/null || echo "  cache: not found"
+        du -sh ~/.cargo/git/db/          2>/dev/null || echo "  git/db: not found"
+        echo "=== target/debug ==="
+        du -sh target/debug/.fingerprint/ 2>/dev/null || echo "  .fingerprint: not found"
+        du -sh target/debug/build/        2>/dev/null || echo "  build: not found"
+        du -sh target/debug/deps/         2>/dev/null || echo "  deps: not found"
+        echo "=== target/release ==="
+        du -sh target/release/.fingerprint/ 2>/dev/null || echo "  .fingerprint: not found"
+        du -sh target/release/build/        2>/dev/null || echo "  build: not found"
+        du -sh target/release/deps/         2>/dev/null || echo "  deps: not found"
+        echo "=== Total estimated cache size ==="
+        du -shc \
+          ~/.cargo/registry/index/        \
+          ~/.cargo/registry/cache/        \
+          ~/.cargo/git/db/                \
+          target/debug/.fingerprint/      \
+          target/debug/build/             \
+          target/debug/deps/              \
+          target/release/.fingerprint/    \
+          target/release/build/           \
+          target/release/deps/            \
+          2>/dev/null | tail -1 || true

.github/actions/update-cargo-cache/action.yml

@@ -13,17 +13,52 @@ description: Update Cargo Cache for Rust projects
 runs:
   using: composite
   steps:
+    - name: Log cache directory sizes
+      shell: bash
+      run: |
+        echo "=== Cargo registry ==="
+        du -sh ~/.cargo/registry/index/  2>/dev/null || echo "  index: not found"
+        du -sh ~/.cargo/registry/cache/  2>/dev/null || echo "  cache: not found"
+        du -sh ~/.cargo/git/db/          2>/dev/null || echo "  git/db: not found"
+        echo "=== target/debug ==="
+        du -sh target/debug/.fingerprint/ 2>/dev/null || echo "  .fingerprint: not found"
+        du -sh target/debug/build/        2>/dev/null || echo "  build: not found"
+        du -sh target/debug/deps/         2>/dev/null || echo "  deps: not found"
+        echo "=== target/release ==="
+        du -sh target/release/.fingerprint/ 2>/dev/null || echo "  .fingerprint: not found"
+        du -sh target/release/build/        2>/dev/null || echo "  build: not found"
+        du -sh target/release/deps/         2>/dev/null || echo "  deps: not found"
+        echo "=== Total estimated cache size ==="
+        du -shc \
+          ~/.cargo/registry/index/        \
+          ~/.cargo/registry/cache/        \
+          ~/.cargo/git/db/                \
+          target/debug/.fingerprint/      \
+          target/debug/build/             \
+          target/debug/deps/              \
+          target/release/.fingerprint/    \
+          target/release/build/           \
+          target/release/deps/            \
+          2>/dev/null | tail -1 || true
+
     - name: Update Cargo cache (main)
       id: update-main
       if: github.event_name != 'pull_request' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master')
       uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # Version 4.2.3
       continue-on-error: true
       with:
+        # This excludes the final linked binaries and test executables (which are large but fast to re-link) while
+        # keeping the compiled dependency objects (which are slow to recompile).
         path: |
           ~/.cargo/registry/index/
           ~/.cargo/registry/cache/
           ~/.cargo/git/db/
-          target/
+          target/debug/.fingerprint/
+          target/debug/build/
+          target/debug/deps/
+          target/release/.fingerprint/
+          target/release/build/
+          target/release/deps/
 
         # Must be in sync with the keys used in the `setup-cargo-cache` workflow.
         key: ${{ inputs.key-prefix }}/main/${{ github.sha }}
@@ -34,11 +69,18 @@ runs:
       uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # Version 4.2.3
       continue-on-error: true
       with:
+        # This excludes the final linked binaries and test executables (which are large but fast to re-link) while
+        # keeping the compiled dependency objects (which are slow to recompile).
         path: |
           ~/.cargo/registry/index/
           ~/.cargo/registry/cache/
           ~/.cargo/git/db/
-          target/
+          target/debug/.fingerprint/
+          target/debug/build/
+          target/debug/deps/
+          target/release/.fingerprint/
+          target/release/build/
+          target/release/deps/
 
         # Must be in sync with the keys used in the `setup-cargo-cache` workflow.
         key: ${{ inputs.key-prefix }}/pr#${{ github.event.pull_request.number }}/${{ github.sha }}

.github/workflows/code-check.yml

@@ -70,6 +70,12 @@ jobs:
             label: self-hosted-macos
     runs-on: ${{ matrix.platform.os }}
     steps:
+      # Disable rustup override before checkout, so the toolchain state is clean before any workspace files or
+      # .rust-toolchain.toml are present. Prevents a stale override from affecting the toolchain resolution during
+      # checkout or subsequent steps.
+      - name: Disable rustup override
+        run: rustup override unset || true
+
       - name: Checkout repository
         uses: actions/checkout@85e6279cec87321a52edac9c87bce653a07cf6c2  # Version 4.2.2
       - name: Setup Circuits
@@ -82,6 +88,13 @@ jobs:
         with:
           tool: cargo-hack
 
+      # Prune after Install cargo-hack, before Set up Cargo cache — frees disk space from the previous run's leftover
+      # artifacts (incremental compile dirs, fingerprints, build dirs) before the cache is restored. This means the
+      # restored cache lands onto a disk that has already been cleaned, maximizing the chance the restore succeeds
+      # without hitting disk limits.
+      - name: Prune disk space (cache-aware, self-hosted)
+        uses: ./.github/actions/prune-disk-space
+
       - name: Set up Cargo cache
         uses: ./.github/actions/setup-cargo-cache
         with:
@@ -172,7 +185,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     steps:
       - name: Disable rustup override
-        run: rustup override unset
+        run: rustup override unset || true
 
       - name: Checkout repository
         uses: actions/checkout@85e6279cec87321a52edac9c87bce653a07cf6c2  # Version 4.2.2
@@ -182,6 +195,10 @@ jobs:
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
 
+      # Prune before any builds.
+      - name: Prune disk space (cache-aware, self-hosted)
+        uses: ./.github/actions/prune-disk-space
+
       - name: Run the node binary with the testing feature to check the one-node config files validity
         uses: actions-rs/cargo@9e120dd99b0fbad1c065f686657e914e76bd7b72  # Version 1.0.1
         with:
@@ -196,6 +213,7 @@ jobs:
           tool: cargo-nextest
 
       - name: Cargo test all except end-to-end integration tests
+        timeout-minutes: 90
         uses: actions-rs/cargo@9e120dd99b0fbad1c065f686657e914e76bd7b72  # Version 1.0.1
         with:
           command: nextest

.github/workflows/end-to-end-with-cucumber.yml

@@ -40,7 +40,7 @@ jobs:
     steps:
       # Remove any rustup override to ensure default toolchain is used
       - name: Disable rustup override
-        run: rustup override unset
+        run: rustup override unset || true
 
       # Checkout the repository code
       # Notes:
@@ -65,6 +65,10 @@ jobs:
       - name: Set circuits path environment variable (new)
         run: echo "LOGOS_BLOCKCHAIN_CIRCUITS=$HOME/.logos-blockchain-circuits" >> $GITHUB_ENV
 
+      # Prune before any builds.
+      - name: Prune disk space (cache-aware, self-hosted)
+        uses: ./.github/actions/prune-disk-space
+
       - name: Install cargo-nextest
         uses: taiki-e/install-action@5651179950649c44da31d346537e20c0534f0f25 # Version 2.49.35
         with: