From e0720cbceb3ae50dda62653ccd5e1b931b103911 Mon Sep 17 00:00:00 2001 From: Roman Date: Thu, 28 May 2026 20:41:11 +0800 Subject: [PATCH] test: initial mutants for props and protocol --- .github/workflows/mutants.yml | 216 +++++++++++++++++++++++++++++++++ .gitignore | 7 ++ Cargo.toml | 16 +++ Justfile | 103 ++++++++++++++++ scripts/mutants-corpus-test.sh | 53 ++++++++ 5 files changed, 395 insertions(+) create mode 100644 .github/workflows/mutants.yml create mode 100644 scripts/mutants-corpus-test.sh diff --git a/.github/workflows/mutants.yml b/.github/workflows/mutants.yml new file mode 100644 index 0000000..337d92e --- /dev/null +++ b/.github/workflows/mutants.yml @@ -0,0 +1,216 @@ +name: Mutation Testing + +# ── When to run ─────────────────────────────────────────────────────────────── +# Plane A (fuzz_props invariants) runs on every PR that touches harness code. +# Plane B (LEZ protocol vs corpus) is slow (minutes per mutant × many mutants) +# so it only runs on a weekly schedule or on manual dispatch. +on: + pull_request: + paths: + - "fuzz_props/**" + - "fuzz/fuzz_targets/**" + - ".github/workflows/mutants.yml" + schedule: + - cron: "0 4 * * 1" # 04:00 UTC every Monday + workflow_dispatch: + +env: + RISC0_DEV_MODE: "1" + CARGO_TERM_COLOR: always + +jobs: + # ── Plane A: mutate fuzz_props (invariant harness) ──────────────────────── + # Oracle: cargo test -p fuzz_props --release + # Fast (~30–120 s total). Blocks PRs if any invariant-check logic is + # under-tested. + mutants-harness: + name: Mutants — fuzz_props invariants + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Checkout logos-execution-zone + uses: ./.github/actions/checkout-lez + + - name: Install stable Rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Install logos-blockchain-circuits + uses: ./logos-execution-zone/.github/actions/install-logos-blockchain-circuits + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Cache cargo registry + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: mutants-harness-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }} + + - name: Install cargo-mutants + run: cargo install cargo-mutants --locked + + # workspace.metadata.cargo-mutants in Cargo.toml sets: + # additional_cargo_args = ["--release"] + # exclude_globs = ["fuzz/fuzz_targets/**"] + # timeout_multiplier = 3.0 + - name: Run mutation tests on fuzz_props + run: | + cargo mutants \ + --package fuzz_props \ + --output mutants-harness.out + + - name: Upload mutants report + if: always() + uses: actions/upload-artifact@v4 + with: + name: mutants-harness-report + path: mutants-harness.out/ + + - name: Write GitHub Step Summary + if: always() + run: | + MISSED=$(wc -l < mutants-harness.out/missed.txt 2>/dev/null | tr -d ' ' || echo 0) + CAUGHT=$(wc -l < mutants-harness.out/caught.txt 2>/dev/null | tr -d ' ' || echo 0) + { + echo "## Mutation Testing — \`fuzz_props\` invariants" + echo "" + echo "| Result | Count |" + echo "|--------|-------|" + echo "| ✅ Caught | ${CAUGHT} |" + echo "| ❌ Survived | ${MISSED} |" + echo "" + if [ "${MISSED}" -gt 0 ]; then + echo "### Surviving mutants (invariant-checker gaps)" + echo '```' + cat mutants-harness.out/missed.txt 2>/dev/null || true + echo '```' + echo "" + echo "> Each surviving mutant represents a mutation in the invariant-checking" + echo "> code that \`cargo test -p fuzz_props\` did not detect." + echo "> Add a property-test that specifically exercises that code path." + else + echo "> All mutants caught — invariant-checking logic is fully covered." + fi + } >> "$GITHUB_STEP_SUMMARY" + + - name: Fail if any mutations survived + run: | + if [ -s mutants-harness.out/missed.txt ]; then + echo "ERROR: surviving mutants found in fuzz_props — see artifact and Step Summary" + cat mutants-harness.out/missed.txt + exit 1 + fi + + # ── Plane B: mutate LEZ protocol code, oracle = corpus regression ───────── + # Each mutant: rebuild nssa/common + replay all 15 fuzz corpora (-runs=0). + # Surviving mutants = protocol bugs the committed corpus has never caught. + # Runs on schedule (weekly Monday) or manual workflow_dispatch only. + mutants-protocol: + name: Mutants — LEZ protocol vs corpus + runs-on: ubuntu-latest + if: github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' + + steps: + - uses: actions/checkout@v4 + + - name: Checkout logos-execution-zone + uses: ./.github/actions/checkout-lez + + # cargo-fuzz requires nightly. + - name: Install Rust nightly toolchain + uses: dtolnay/rust-toolchain@nightly + with: + components: llvm-tools-preview + + - name: Install logos-blockchain-circuits + uses: ./logos-execution-zone/.github/actions/install-logos-blockchain-circuits + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Cache cargo registry + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: mutants-protocol-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }} + + - name: Install cargo-fuzz and cargo-mutants + run: | + cargo install cargo-fuzz --locked + cargo install cargo-mutants --locked + + - name: Make corpus-regression wrapper executable + run: chmod +x scripts/mutants-corpus-test.sh + + # Build all 15 fuzz targets once before the mutation loop so that each + # mutant only needs to rebuild the mutated crate, not the fuzz harness. + - name: Pre-build fuzz targets + run: | + for target in \ + fuzz_transaction_decoding fuzz_stateless_verification \ + fuzz_state_transition fuzz_block_verification \ + fuzz_encoding_roundtrip fuzz_signature_verification \ + fuzz_replay_prevention fuzz_state_diff_computation \ + fuzz_validate_execute_consistency fuzz_state_serialization \ + fuzz_witness_set_verification fuzz_program_deployment_lifecycle \ + fuzz_apply_state_diff_split_path fuzz_multi_block_state_sequence \ + fuzz_sequencer_vs_replayer; do + cargo fuzz build "${target}" + done + + # cargo-mutants is invoked from the LEZ workspace (sibling directory). + # FUZZ_REPO is exported so the wrapper script can locate the corpus and + # fuzz directory without relying on relative paths. + - name: Run mutation tests against LEZ (nssa + common) + env: + FUZZ_REPO: ${{ github.workspace }} + working-directory: logos-execution-zone + run: | + cargo mutants \ + --package nssa \ + --package common \ + --in-place \ + --test-command "${{ github.workspace }}/scripts/mutants-corpus-test.sh" \ + --output "${{ github.workspace }}/mutants-protocol.out" \ + --timeout-multiplier 5.0 + + - name: Upload mutants report + if: always() + uses: actions/upload-artifact@v4 + with: + name: mutants-protocol-report + path: mutants-protocol.out/ + + - name: Write GitHub Step Summary + if: always() + run: | + MISSED=$(wc -l < mutants-protocol.out/missed.txt 2>/dev/null | tr -d ' ' || echo 0) + CAUGHT=$(wc -l < mutants-protocol.out/caught.txt 2>/dev/null | tr -d ' ' || echo 0) + { + echo "## Mutation Testing — LEZ protocol vs committed corpus" + echo "" + echo "| Result | Count |" + echo "|--------|-------|" + echo "| ✅ Caught by corpus | ${CAUGHT} |" + echo "| ❌ Survived (corpus gap) | ${MISSED} |" + echo "" + if [ "${MISSED}" -gt 0 ]; then + echo "### Surviving mutants (corpus gaps — protocol bugs not yet reached)" + echo '```' + cat mutants-protocol.out/missed.txt 2>/dev/null || true + echo '```' + echo "" + echo "> For each surviving mutant:" + echo "> 1. Run \`cargo fuzz run \` targeting the mutated function." + echo "> 2. Save the crashing input to \`corpus/libfuzz//\`." + echo "> 3. Commit the corpus entry — the next run will show \`CAUGHT\`." + else + echo "> All mutants caught — committed corpus covers all tested mutation points." + fi + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.gitignore b/.gitignore index 0a1758b..48506a6 100644 --- a/.gitignore +++ b/.gitignore @@ -27,6 +27,13 @@ fuzz/coverage/ .DS_Store **/.DS_Store +# ── cargo-mutants outputs ───────────────────────────────────────────────────── +# Local mutation-testing reports (caught.txt, missed.txt, etc.) +# Created by `just mutants-harness` and `just mutants-protocol`. +mutants.out/ +mutants-harness.out/ +mutants-protocol.out/ + # ── Misc ────────────────────────────────────────────────────────────────────── # Performance baseline output from `just perf-baseline` or CI perf_baseline.txt diff --git a/Cargo.toml b/Cargo.toml index af287fc..39d3287 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -84,3 +84,19 @@ hmac-sha512 = "1.1.7" itertools = "0.14.0" risc0-build = "3.0.5" logos-blockchain-common-http-client = { git = "https://github.com/logos-blockchain/logos-blockchain.git" } + +# ── cargo-mutants configuration (Plane A: mutate fuzz_props invariants) ─────── +# Run with --release to match CI timing. +# fuzz/fuzz_targets/** entry-points use fuzz_entry!() macros that are not +# reachable via `cargo test`; mutations there produce false survivors. +[workspace.metadata.cargo-mutants] +additional_cargo_args = ["--release"] +exclude_globs = ["fuzz/fuzz_targets/**"] +# RISC0 release builds are slower than typical crates; give each mutant extra time. +timeout_multiplier = 3.0 +# The workspace uses path dependencies outside its own directory +# (../logos-execution-zone/*). cargo-mutants normally copies the workspace to a +# temp directory, but the copy does not include the sibling LEZ directory, so the +# build fails immediately. --in-place mutates the original source files in-place +# and avoids the copy, letting cargo resolve ../logos-execution-zone as usual. +in_place = true diff --git a/Justfile b/Justfile index 7bfb9be..7c6563a 100644 --- a/Justfile +++ b/Justfile @@ -643,6 +643,109 @@ coverage-all ENGINE="all": fi fi +# ── Mutation testing ────────────────────────────────────────────────────────── +# +# Prerequisites (install once): +# cargo install cargo-mutants +# +# Two planes — run them independently: +# +# Plane A (fast, ~1-5 min): mutates fuzz_props invariant logic. +# Oracle: cargo test -p fuzz_props --release +# Run on every PR that touches fuzz_props/ or fuzz/fuzz_targets/. +# +# Plane B (slow, ~hours): mutates LEZ protocol code (nssa, common). +# Oracle: all 15 fuzz targets replayed against their committed corpus. +# Run weekly or manually to find corpus gaps. + +# Plane A — mutation testing of fuzz_props invariant implementations. +# +# Mutates every function in fuzz_props and checks whether `cargo test -p fuzz_props +# --release` catches the mutation. Surviving mutants identify invariant-checker +# logic that the property tests do not fully exercise. +# +# Workspace metadata in Cargo.toml configures --release, exclude_globs, and +# timeout_multiplier automatically; no extra flags are needed here. +# +# Output: mutants.out/ (human-readable report, also printed to stdout) +mutants-harness: + cargo mutants --package fuzz_props + +# Plane B — mutation testing of the LEZ protocol code against the committed corpus. +# +# Mutates nssa and common in the logos-execution-zone sibling workspace and uses +# scripts/mutants-corpus-test.sh as the oracle. The oracle replays all 15 +# committed libFuzzer corpora (cargo fuzz run -runs=0) against each mutant. +# +# A mutant that SURVIVES means there is no corpus input that triggers the +# relevant protocol invariant at that mutation point — a corpus gap worth +# investigating with a longer fuzz run. +# +# Prerequisites: +# - logos-execution-zone cloned at ../logos-execution-zone +# - cargo-fuzz installed (cargo install cargo-fuzz) +# - cargo-mutants installed (cargo install cargo-mutants --locked) +# +# PACKAGES selects which LEZ crates to mutate (space-separated). +# Default covers the two highest-value protocol crates. +# +# Output report: mutants-protocol.out/ in the repository root. +mutants-protocol PACKAGES="nssa common": + #!/bin/bash + set -euo pipefail + REPO_DIR="$(pwd)" + + if [ ! -d "${REPO_DIR}/../logos-execution-zone" ]; then + echo "ERROR: logos-execution-zone not found at ../logos-execution-zone" + exit 1 + fi + LEZ_DIR="$(cd "${REPO_DIR}/../logos-execution-zone" && pwd)" + + # Build --package flags (one per crate name) + PKG_FLAGS=() + for pkg in {{PACKAGES}}; do + PKG_FLAGS+=(--package "$pkg") + done + + echo "=== Plane B: mutating [{{PACKAGES}}] in logos-execution-zone ===" + echo " Oracle: scripts/mutants-corpus-test.sh (corpus regression, -runs=0)" + echo " Report: ${REPO_DIR}/mutants-protocol.out/" + echo "" + + # cargo-mutants must be run from inside the target workspace. + # FUZZ_REPO tells the oracle script where to find the corpus and fuzz/ dir. + # --output puts the report in our repo root so it's easy to browse/commit. + # --in-place is required because LEZ depends on path crates outside its own + # directory (e.g. the Rust standard toolchain); without it cargo-mutants copies + # the workspace to a temp dir where those relative paths would not resolve. + cd "$LEZ_DIR" + FUZZ_REPO="$REPO_DIR" \ + cargo mutants \ + "${PKG_FLAGS[@]}" \ + --in-place \ + --test-command "${REPO_DIR}/scripts/mutants-corpus-test.sh" \ + --output "${REPO_DIR}/mutants-protocol.out" \ + --timeout-multiplier 5.0 + + echo "" + echo "=== Mutation report summary ===" + MISSED_FILE="${REPO_DIR}/mutants-protocol.out/missed.txt" + CAUGHT_FILE="${REPO_DIR}/mutants-protocol.out/caught.txt" + MISSED=$(wc -l < "$MISSED_FILE" 2>/dev/null | tr -d ' ' || echo 0) + CAUGHT=$(wc -l < "$CAUGHT_FILE" 2>/dev/null | tr -d ' ' || echo 0) + echo "Caught: ${CAUGHT} | Survived: ${MISSED}" + echo "" + if [ "${MISSED}" -gt 0 ]; then + echo "Surviving mutants (corpus gaps):" + cat "$MISSED_FILE" || true + echo "" + echo "For each surviving mutant: run 'just fuzz ' targeting the" + echo "mutated function, add the crashing input to corpus/libfuzz//," + echo "then re-run 'just mutants-protocol' to confirm it is now CAUGHT." + else + echo "All mutants caught — corpus covers all tested mutation points." + fi + # ── Housekeeping ────────────────────────────────────────────────────────────── # Remove all Cargo build artefacts (workspace + fuzz sub-crate) diff --git a/scripts/mutants-corpus-test.sh b/scripts/mutants-corpus-test.sh new file mode 100644 index 0000000..1bfdead --- /dev/null +++ b/scripts/mutants-corpus-test.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# Plane-B mutation-testing oracle. +# +# Called by `cargo mutants --test-command` from *inside* the logos-execution-zone +# workspace directory after each source mutation. Replays the committed +# libFuzzer corpus against every fuzz target (cargo fuzz run -runs=0). +# +# Exit behaviour (used by cargo-mutants to classify each mutant): +# exit 0 → all corpus replays passed → mutant SURVIVED (corpus gap) +# exit ≠0 → at least one replay panicked → mutant CAUGHT (corpus covers it) +# +# Environment variables: +# FUZZ_REPO absolute path to the lez-fuzzing repository root. +# Defaults to the directory one level above this script. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +FUZZ_REPO="${FUZZ_REPO:-"$(cd "${SCRIPT_DIR}/.." && pwd)"}" + +CORPUS_ROOT="${FUZZ_REPO}/corpus/libfuzz" +FUZZ_DIR="${FUZZ_REPO}/fuzz" + +targets=( + fuzz_transaction_decoding + fuzz_stateless_verification + fuzz_state_transition + fuzz_block_verification + fuzz_encoding_roundtrip + fuzz_signature_verification + fuzz_replay_prevention + fuzz_state_diff_computation + fuzz_validate_execute_consistency + fuzz_state_serialization + fuzz_witness_set_verification + fuzz_program_deployment_lifecycle + fuzz_apply_state_diff_split_path + fuzz_multi_block_state_sequence + fuzz_sequencer_vs_replayer +) + +for target in "${targets[@]}"; do + corpus="${CORPUS_ROOT}/${target}" + mkdir -p "${corpus}" + + # -runs=0 → replay every file in the corpus directory exactly once, then exit. + # A panic (invariant violation) causes cargo fuzz to exit non-zero, which + # propagates through this script and causes cargo-mutants to mark the mutant + # as CAUGHT. + cargo fuzz run "${target}" \ + --fuzz-dir "${FUZZ_DIR}" \ + "${corpus}" \ + -- -runs=0 +done