diff --git a/.github/actions/prune-disk-space/action.yml b/.github/actions/prune-disk-space/action.yml new file mode 100644 index 000000000..4dc5dae5f --- /dev/null +++ b/.github/actions/prune-disk-space/action.yml @@ -0,0 +1,114 @@ +name: Prune disk space (cache-aware, self-hosted) +description: Frees disk space from previous run leftovers on self-hosted runners (Linux + macOS) + +runs: + using: composite + steps: + - name: Prune disk space + if: always() + shell: bash + run: | + set -uo pipefail + + echo "Disk before:" + df -h || true + + avail_kb() { df -Pk . | awk 'NR==2 {print $4}'; } + + # --- 1) macOS: remove big, known-safe Xcode junk (keep it surgical) + if [ "${RUNNER_OS}" = "macOS" ]; then + rm -rf ~/Library/Developer/Xcode/DerivedData 2>/dev/null || true + rm -rf ~/Library/Caches/com.apple.python 2>/dev/null || true + rm -rf ~/Library/Caches/gmp-mpfr-sys 2>/dev/null || true + fi + + # --- 2) Repo-local reproducible bloat + find . -type d -name node_modules -prune -exec rm -rf {} + 2>/dev/null || true + find . -type d -name dist -prune -exec rm -rf {} + 2>/dev/null || true + find . -type d -name build -prune -exec rm -rf {} + 2>/dev/null || true + find . -type d -name .cache -prune -exec rm -rf {} + 2>/dev/null || true + find . -type d -name .vite -prune -exec rm -rf {} + 2>/dev/null || true + find . -type d -name __pycache__ -prune -exec rm -rf {} + 2>/dev/null || true + find . -not -path './target/*' -name "*.pyc" -delete 2>/dev/null || true + + # npm/pip cache cleanup — safe to run always; no-ops if not installed/used + npm cache clean --force 2>/dev/null || true + pip cache purge 2>/dev/null || true + + # NOTE: Do NOT prune ~/.cargo here in normal mode. + # The workflow restores ~/.cargo via actions/cache; pruning it here defeats caching and can cause refetch storms. + + # --- 3) Target thinning: keep cache value, remove worst disk hogs + # (incremental + fingerprints + build-script outputs). Do NOT delete deps in normal mode. + if [ -d "target" ]; then + echo "Target before:" + du -sh target 2>/dev/null || true + + find target -type d -name incremental -prune -exec rm -rf {} + 2>/dev/null || true + find target -type d -name .fingerprint -prune -exec rm -rf {} + 2>/dev/null || true + find target -type d -name build -prune -exec rm -rf {} + 2>/dev/null || true + + echo "Target after:" + du -sh target 2>/dev/null || true + fi + + # --- 4) Diagnostics: show biggest offenders (cheap, useful) + echo "Top workspace dirs (largest 30):" + ( + du -xh -d 2 . 2>/dev/null | sort -rh 2>/dev/null + ) || ( + du -xh -d 2 . 2>/dev/null | sort -r + ) | head -n 30 || true + + echo "npm cache: $(du -sh ~/.npm 2>/dev/null || echo 'n/a')" + echo "cargo registry: $(du -sh ~/.cargo/registry 2>/dev/null || echo 'n/a')" + echo "cargo git: $(du -sh ~/.cargo/git 2>/dev/null || echo 'n/a')" + + if [ -d target ]; then + echo "Top target dirs (largest 30):" + ( + du -xh -d 2 target 2>/dev/null | sort -rh 2>/dev/null + ) || ( + du -xh -d 2 target 2>/dev/null | sort -r + ) | head -n 30 || true + fi + + # --- 5) Runner diag trimming: robust globbing across Linux + macOS runner layouts + # Use nullglob so non-matching globs expand to nothing (no noisy literal patterns). + ( + shopt -s nullglob + for d in /github-runner/*/_diag /Users/*/github-runner-node-*/_diag; do + [ -d "$d" ] || continue + echo "Diag before ($d):" + du -sh "$d" 2>/dev/null || true + # Keep recent logs; delete older ones + find "$d" -type f -mtime +1 -delete 2>/dev/null || true + echo "Diag after ($d):" + du -sh "$d" 2>/dev/null || true + done + ) + + # --- 6) Emergency mode: if still low disk, apply stronger pruning (rebuildable) + # 15 GB threshold + free_now=$(avail_kb 2>/dev/null || echo 999999999) + if [ "$free_now" -lt $((15*1024*1024)) ]; then + echo "Low disk (<15GB). Applying emergency pruning..." + + # In emergency, it's acceptable to drop deps to prevent runner death. + if [ -d target ]; then + for profile in debug release; do + if [ -d "target/$profile" ]; then + find "target/$profile" -maxdepth 1 -type d \ + \( -name deps -o -name build -o -name .fingerprint -o -name incremental \) \ + -exec rm -rf {} + 2>/dev/null || true + fi + done + fi + + # As a last resort, prune old cargo *download* artifacts only (least harmful), + # but keep this very conservative and best-effort. + find ~/.cargo/registry/cache -type f -name "*.crate" -mtime +14 -delete 2>/dev/null || true + fi + + echo "Disk after:" + df -h || true diff --git a/.github/actions/setup-cargo-cache/action.yml b/.github/actions/setup-cargo-cache/action.yml index 030a6f091..0c4df953e 100644 --- a/.github/actions/setup-cargo-cache/action.yml +++ b/.github/actions/setup-cargo-cache/action.yml @@ -19,11 +19,18 @@ runs: uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # Version 4.2.3 continue-on-error: true with: + # This excludes the final linked binaries and test executables (which are large but fast to re-link) while + # keeping the compiled dependency objects (which are slow to recompile). path: | ~/.cargo/registry/index/ ~/.cargo/registry/cache/ ~/.cargo/git/db/ - target/ + target/debug/.fingerprint/ + target/debug/build/ + target/debug/deps/ + target/release/.fingerprint/ + target/release/build/ + target/release/deps/ # Must be in sync with the keys used in the `update-cargo-cache` workflow. key: ${{ inputs.key-prefix }}/main/${{ github.sha }} @@ -37,11 +44,18 @@ runs: uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # Version 4.2.3 continue-on-error: true with: + # This excludes the final linked binaries and test executables (which are large but fast to re-link) while + # keeping the compiled dependency objects (which are slow to recompile). path: | ~/.cargo/registry/index/ ~/.cargo/registry/cache/ ~/.cargo/git/db/ - target/ + target/debug/.fingerprint/ + target/debug/build/ + target/debug/deps/ + target/release/.fingerprint/ + target/release/build/ + target/release/deps/ # Must be in sync with the keys used in the `update-cargo-cache` workflow. key: ${{ inputs.key-prefix }}/pr#${{ github.event.pull_request.number }}/${{ github.sha }} @@ -49,3 +63,31 @@ runs: ${{ inputs.key-prefix }}/pr#${{ github.event.pull_request.number }} ${{ inputs.key-prefix }}/main ${{ inputs.key-prefix }} + + - name: Log cache directory sizes + shell: bash + run: | + echo "=== Cargo registry ===" + du -sh ~/.cargo/registry/index/ 2>/dev/null || echo " index: not found" + du -sh ~/.cargo/registry/cache/ 2>/dev/null || echo " cache: not found" + du -sh ~/.cargo/git/db/ 2>/dev/null || echo " git/db: not found" + echo "=== target/debug ===" + du -sh target/debug/.fingerprint/ 2>/dev/null || echo " .fingerprint: not found" + du -sh target/debug/build/ 2>/dev/null || echo " build: not found" + du -sh target/debug/deps/ 2>/dev/null || echo " deps: not found" + echo "=== target/release ===" + du -sh target/release/.fingerprint/ 2>/dev/null || echo " .fingerprint: not found" + du -sh target/release/build/ 2>/dev/null || echo " build: not found" + du -sh target/release/deps/ 2>/dev/null || echo " deps: not found" + echo "=== Total estimated cache size ===" + du -shc \ + ~/.cargo/registry/index/ \ + ~/.cargo/registry/cache/ \ + ~/.cargo/git/db/ \ + target/debug/.fingerprint/ \ + target/debug/build/ \ + target/debug/deps/ \ + target/release/.fingerprint/ \ + target/release/build/ \ + target/release/deps/ \ + 2>/dev/null | tail -1 || true diff --git a/.github/actions/update-cargo-cache/action.yml b/.github/actions/update-cargo-cache/action.yml index 84927c842..c9fe5a4d9 100644 --- a/.github/actions/update-cargo-cache/action.yml +++ b/.github/actions/update-cargo-cache/action.yml @@ -13,17 +13,52 @@ description: Update Cargo Cache for Rust projects runs: using: composite steps: + - name: Log cache directory sizes + shell: bash + run: | + echo "=== Cargo registry ===" + du -sh ~/.cargo/registry/index/ 2>/dev/null || echo " index: not found" + du -sh ~/.cargo/registry/cache/ 2>/dev/null || echo " cache: not found" + du -sh ~/.cargo/git/db/ 2>/dev/null || echo " git/db: not found" + echo "=== target/debug ===" + du -sh target/debug/.fingerprint/ 2>/dev/null || echo " .fingerprint: not found" + du -sh target/debug/build/ 2>/dev/null || echo " build: not found" + du -sh target/debug/deps/ 2>/dev/null || echo " deps: not found" + echo "=== target/release ===" + du -sh target/release/.fingerprint/ 2>/dev/null || echo " .fingerprint: not found" + du -sh target/release/build/ 2>/dev/null || echo " build: not found" + du -sh target/release/deps/ 2>/dev/null || echo " deps: not found" + echo "=== Total estimated cache size ===" + du -shc \ + ~/.cargo/registry/index/ \ + ~/.cargo/registry/cache/ \ + ~/.cargo/git/db/ \ + target/debug/.fingerprint/ \ + target/debug/build/ \ + target/debug/deps/ \ + target/release/.fingerprint/ \ + target/release/build/ \ + target/release/deps/ \ + 2>/dev/null | tail -1 || true + - name: Update Cargo cache (main) id: update-main if: github.event_name != 'pull_request' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master') uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # Version 4.2.3 continue-on-error: true with: + # This excludes the final linked binaries and test executables (which are large but fast to re-link) while + # keeping the compiled dependency objects (which are slow to recompile). path: | ~/.cargo/registry/index/ ~/.cargo/registry/cache/ ~/.cargo/git/db/ - target/ + target/debug/.fingerprint/ + target/debug/build/ + target/debug/deps/ + target/release/.fingerprint/ + target/release/build/ + target/release/deps/ # Must be in sync with the keys used in the `setup-cargo-cache` workflow. key: ${{ inputs.key-prefix }}/main/${{ github.sha }} @@ -34,11 +69,18 @@ runs: uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # Version 4.2.3 continue-on-error: true with: + # This excludes the final linked binaries and test executables (which are large but fast to re-link) while + # keeping the compiled dependency objects (which are slow to recompile). path: | ~/.cargo/registry/index/ ~/.cargo/registry/cache/ ~/.cargo/git/db/ - target/ + target/debug/.fingerprint/ + target/debug/build/ + target/debug/deps/ + target/release/.fingerprint/ + target/release/build/ + target/release/deps/ # Must be in sync with the keys used in the `setup-cargo-cache` workflow. key: ${{ inputs.key-prefix }}/pr#${{ github.event.pull_request.number }}/${{ github.sha }} diff --git a/.github/workflows/code-check.yml b/.github/workflows/code-check.yml index 6cfc74be1..cb0da163b 100644 --- a/.github/workflows/code-check.yml +++ b/.github/workflows/code-check.yml @@ -70,6 +70,12 @@ jobs: label: self-hosted-macos runs-on: ${{ matrix.platform.os }} steps: + # Disable rustup override before checkout, so the toolchain state is clean before any workspace files or + # .rust-toolchain.toml are present. Prevents a stale override from affecting the toolchain resolution during + # checkout or subsequent steps. + - name: Disable rustup override + run: rustup override unset || true + - name: Checkout repository uses: actions/checkout@85e6279cec87321a52edac9c87bce653a07cf6c2 # Version 4.2.2 - name: Setup Circuits @@ -82,6 +88,13 @@ jobs: with: tool: cargo-hack + # Prune after Install cargo-hack, before Set up Cargo cache — frees disk space from the previous run's leftover + # artifacts (incremental compile dirs, fingerprints, build dirs) before the cache is restored. This means the + # restored cache lands onto a disk that has already been cleaned, maximizing the chance the restore succeeds + # without hitting disk limits. + - name: Prune disk space (cache-aware, self-hosted) + uses: ./.github/actions/prune-disk-space + - name: Set up Cargo cache uses: ./.github/actions/setup-cargo-cache with: @@ -172,7 +185,7 @@ jobs: runs-on: ${{ matrix.os }} steps: - name: Disable rustup override - run: rustup override unset + run: rustup override unset || true - name: Checkout repository uses: actions/checkout@85e6279cec87321a52edac9c87bce653a07cf6c2 # Version 4.2.2 @@ -182,6 +195,10 @@ jobs: with: github-token: ${{ secrets.GITHUB_TOKEN }} + # Prune before any builds. + - name: Prune disk space (cache-aware, self-hosted) + uses: ./.github/actions/prune-disk-space + - name: Run the node binary with the testing feature to check the one-node config files validity uses: actions-rs/cargo@9e120dd99b0fbad1c065f686657e914e76bd7b72 # Version 1.0.1 with: @@ -196,6 +213,7 @@ jobs: tool: cargo-nextest - name: Cargo test all except end-to-end integration tests + timeout-minutes: 90 uses: actions-rs/cargo@9e120dd99b0fbad1c065f686657e914e76bd7b72 # Version 1.0.1 with: command: nextest diff --git a/.github/workflows/end-to-end-with-cucumber.yml b/.github/workflows/end-to-end-with-cucumber.yml index e496070e0..3ba21b264 100644 --- a/.github/workflows/end-to-end-with-cucumber.yml +++ b/.github/workflows/end-to-end-with-cucumber.yml @@ -40,7 +40,7 @@ jobs: steps: # Remove any rustup override to ensure default toolchain is used - name: Disable rustup override - run: rustup override unset + run: rustup override unset || true # Checkout the repository code # Notes: @@ -65,6 +65,10 @@ jobs: - name: Set circuits path environment variable (new) run: echo "LOGOS_BLOCKCHAIN_CIRCUITS=$HOME/.logos-blockchain-circuits" >> $GITHUB_ENV + # Prune before any builds. + - name: Prune disk space (cache-aware, self-hosted) + uses: ./.github/actions/prune-disk-space + - name: Install cargo-nextest uses: taiki-e/install-action@5651179950649c44da31d346537e20c0534f0f25 # Version 2.49.35 with: