logos-blockchain/lez-fuzzing
1
0
Fork 0
mirror of https://github.com/logos-blockchain/lez-fuzzing.git synced 2026-06-07 11:39:30 +00:00
Code Issues Packages Projects Releases Wiki Activity
lez-fuzzing/Justfile
Roman ee7b3b0f69
fix: mutants-protocol invocation
2026-05-28 21:27:48 +08:00

818 lines
37 KiB
Makefile
Raw Blame History

# ── Fuzzing ───────────────────────────────────────────────────────────────────
export RISC0_DEV_MODE := "1"
# ── Directory layout ──────────────────────────────────────────────────────────
#
# corpus/
# libfuzz/<target>/ — inputs generated/discovered by cargo-fuzz (libFuzzer)
# afl/<target>/ — inputs generated by AFL++ (synced from afl-output queue)
#
# coverage/
# libfuzz/<target>/ — per-target libFuzzer coverage report + profdata
# libfuzz/summary/ — merged libFuzzer summary (all targets)
# afl/<target>/ — per-target AFL++ corpus coverage report + profdata
# afl/summary/ — merged AFL++ corpus summary (all targets)
#
# afl-output/<target>/ — AFL++'s raw working directory (queue, crashes, hangs)
# The queue is synced to corpus/afl/<target>/ via
# `just afl-corpus-sync`; crashes/hangs are kept here.
#
# Note: cargo-fuzz (coverage, run, cmin) always writes its profdata to the
# fixed path fuzz/coverage/<TARGET>/coverage.profdata regardless of the
# corpus argument. The coverage recipes copy that file into the organised
# coverage/ tree immediately after it is produced, so AFL passes can never
# overwrite a libFuzzer profdata that is still needed.
# List all registered fuzz targets (reads fuzz/Cargo.toml via cargo-fuzz)
list-targets:
cargo fuzz list
# Run all fuzz targets for TIME seconds each (default: 30).
# Targets are discovered automatically from fuzz/Cargo.toml — no edit needed here
# when a new [[bin]] entry is added.
fuzz TIME="30":
#!/bin/bash
set -euo pipefail
for target in $(cargo fuzz list 2>/dev/null); do
echo "=== fuzzing $target for {{TIME}}s ==="
mkdir -p "corpus/libfuzz/$target"
cargo fuzz run "$target" "corpus/libfuzz/$target" -- -max_total_time={{TIME}}
done
# Re-run the saved corpus for every target (regression mode, no new mutations)
fuzz-regression:
#!/bin/bash
set -euo pipefail
for target in $(cargo fuzz list 2>/dev/null); do
echo "=== regression $target ==="
mkdir -p "corpus/libfuzz/$target"
cargo fuzz run "$target" "corpus/libfuzz/$target" -- -runs=0
done
# Minimise a crash artifact
# Usage: just fuzz-tmin fuzz_state_transition fuzz/artifacts/fuzz_state_transition/crash-XXX
fuzz-tmin TARGET ARTIFACT:
cargo fuzz tmin {{TARGET}} {{ARTIFACT}}
# Run the proptest-based property tests
fuzz-props:
cargo test -p fuzz_props --release
# Pull the latest LEZ changes from the sibling logos-execution-zone directory
update-lez:
git -C ../logos-execution-zone pull --ff-only
# ── Corpus management ─────────────────────────────────────────────────────────
# Minimise the corpus for all targets (removes dominated inputs)
corpus-cmin:
#!/bin/bash
set -euo pipefail
for target in $(cargo fuzz list 2>/dev/null); do
echo "=== cmin $target ==="
mkdir -p "corpus/libfuzz/$target"
cargo fuzz cmin "$target" "corpus/libfuzz/$target"
done
# Minimise the corpus for a single target
# Usage: just corpus-cmin-target fuzz_state_transition
corpus-cmin-target TARGET:
mkdir -p corpus/libfuzz/{{TARGET}}
cargo fuzz cmin {{TARGET}} corpus/libfuzz/{{TARGET}}
# ── Adding a new target ───────────────────────────────────────────────────────
# Scaffold a new fuzz target — fully automated, no manual edits required.
#
# Steps performed automatically:
# 1. Creates corpus/libfuzz/<TARGET>/
# 2. Copies fuzz/fuzz_targets/_template.rs → fuzz/fuzz_targets/<TARGET>.rs
# 3. Appends the [[bin]] entry to fuzz/Cargo.toml
# 4. Inserts <TARGET> into every strategy matrix in .github/workflows/fuzz.yml
#
# Usage: just new-target my_feature
# (the "fuzz_" prefix is added automatically)
new-target NAME:
#!/bin/bash
set -euo pipefail
TARGET="fuzz_{{NAME}}"
TEMPLATE="fuzz/fuzz_targets/_template.rs"
RS_FILE="fuzz/fuzz_targets/${TARGET}.rs"
CORPUS_DIR="corpus/libfuzz/${TARGET}"
# ── 1. Create corpus directory ────────────────────────────────────────────
mkdir -p "$CORPUS_DIR"
echo "[1/4] Created corpus directory: $CORPUS_DIR"
# ── 2. Copy the typed fuzz target template ────────────────────────────────
if [ -f "$RS_FILE" ]; then
echo "SKIP [2/4]: $RS_FILE already exists — not overwriting."
else
cp "$TEMPLATE" "$RS_FILE"
echo "[2/4] Created target from template: $RS_FILE"
fi
# ── 3 & 4. Update Cargo.toml and fuzz.yml automatically ──────────────────
python3 scripts/add_fuzz_target.py "$TARGET"
echo ""
echo "Done! Verify the libFuzzer build with:"
echo " RISC0_DEV_MODE=1 cargo fuzz build ${TARGET}"
echo ""
echo "Verify the AFL++ build with:"
echo " cd fuzz && cargo afl build --no-default-features --features fuzzer-afl --release --bin ${TARGET}"
# ── AFL++ fuzzing ──────────────────────────────────────────────────────────────
# Prerequisites (install once):
# macOS: brew install afl-fuzz && cargo install cargo-afl
# Linux: Build AFL++ from source (recommended — Debian/Ubuntu apt packages are
# several major versions behind; see https://github.com/AFLplusplus/AFLplusplus):
# git clone https://github.com/AFLplusplus/AFLplusplus
# cd AFLplusplus && make distrib && sudo make install
# Then: cargo install cargo-afl
# Build ALL fuzz targets for AFL++ (output: fuzz/target/release/<target>)
afl-build:
cd fuzz && cargo afl build --no-default-features --features fuzzer-afl --release
# Build a SINGLE fuzz target for AFL++
# Usage: just afl-build-target fuzz_state_transition
afl-build-target TARGET:
cd fuzz && cargo afl build --no-default-features --features fuzzer-afl --release --bin {{TARGET}}
# Disable the macOS crash reporter daemon so AFL++ can detect crashes reliably.
# This is a macOS-only requirement; on Linux this is a no-op.
# The `fuzz-afl` recipe calls this automatically; run it manually if you want
# to keep the reporter disabled across multiple just invocations.
#
# Re-enable with: just afl-macos-teardown
afl-macos-setup:
#!/bin/bash
if [ "$(uname)" != "Darwin" ]; then echo "Not macOS — nothing to do."; exit 0; fi
SL=/System/Library; PL=com.apple.ReportCrash
echo "Disabling macOS crash reporter (required by AFL++)…"
launchctl unload -w "${SL}/LaunchAgents/${PL}.plist" 2>/dev/null || true
sudo launchctl unload -w "${SL}/LaunchDaemons/${PL}.Root.plist" 2>/dev/null || true
echo "Done. Re-enable with: just afl-macos-teardown"
# Re-enable the macOS crash reporter after an AFL++ session.
afl-macos-teardown:
#!/bin/bash
if [ "$(uname)" != "Darwin" ]; then echo "Not macOS — nothing to do."; exit 0; fi
SL=/System/Library; PL=com.apple.ReportCrash
echo "Re-enabling macOS crash reporter…"
launchctl load -w "${SL}/LaunchAgents/${PL}.plist" 2>/dev/null || true
sudo launchctl load -w "${SL}/LaunchDaemons/${PL}.Root.plist" 2>/dev/null || true
echo "Done."
# Run AFL++ on one target or ALL targets when no target is supplied.
# Builds binaries as needed; syncs the queue to corpus/afl/<target>/ when done.
#
# AFL++ is seeded from corpus/libfuzz/<target>/ (the libFuzzer corpus).
# After the run, new inputs discovered by AFL++ are synced to corpus/afl/<target>/
# via `just afl-corpus-sync`.
#
# On macOS the crash reporter is disabled automatically for the duration of the
# run and re-enabled when the script exits (via a shell trap).
#
# Requires afl-fuzz and cargo-afl to be installed locally:
# macOS: brew install afl-fuzz && cargo install cargo-afl
# Linux: Build AFL++ from source (apt packages are several major versions
# behind): see https://github.com/AFLplusplus/AFLplusplus
#
# Usage: just fuzz-afl # all targets, 30 s each
# just fuzz-afl "" 60 # all targets, 60 s each
# just fuzz-afl fuzz_state_transition # single target, 30 s
# just fuzz-afl fuzz_state_transition 300 # single target, 300 s
fuzz-afl TARGET="" TIME="30":
#!/bin/bash
set -euo pipefail
TARGET="{{TARGET}}"
TIME="{{TIME}}"
# ── Collect targets to run ────────────────────────────────────────────────
if [ -z "$TARGET" ]; then
TARGETS=($(cargo fuzz list 2>/dev/null))
else
TARGETS=("$TARGET")
fi
# ── Require local AFL++ installation ─────────────────────────────────────
if ! command -v afl-fuzz &>/dev/null; then
echo "ERROR: afl-fuzz not found in PATH."
echo ""
echo "Install AFL++ before running this recipe:"
echo ""
echo " macOS : brew install afl-fuzz"
echo ""
echo " Linux : Build from source (apt packages are several major versions behind):"
echo " git clone https://github.com/AFLplusplus/AFLplusplus"
echo " cd AFLplusplus && make distrib && sudo make install"
echo ""
echo "Also install the cargo-afl build wrapper:"
echo " cargo install cargo-afl"
echo ""
exit 1
fi
if ! command -v cargo-afl &>/dev/null && ! cargo afl --version &>/dev/null 2>&1; then
echo "ERROR: cargo-afl not found."
echo " cargo install cargo-afl"
exit 1
fi
# ── macOS: disable crash reporter for the duration of this run ───────────
if [ "$(uname)" = "Darwin" ]; then
SL=/System/Library; PL=com.apple.ReportCrash
echo "macOS: disabling crash reporter (AFL++ requirement)…"
launchctl unload -w "${SL}/LaunchAgents/${PL}.plist" 2>/dev/null || true
sudo launchctl unload -w "${SL}/LaunchDaemons/${PL}.Root.plist" 2>/dev/null || true
# Re-enable on any exit — normal, error, or Ctrl-C
trap '
echo "Re-enabling macOS crash reporter…"
SL=/System/Library; PL=com.apple.ReportCrash
launchctl load -w "${SL}/LaunchAgents/${PL}.plist" 2>/dev/null || true
sudo launchctl load -w "${SL}/LaunchDaemons/${PL}.Root.plist" 2>/dev/null || true
' EXIT
fi
# ── Run targets ───────────────────────────────────────────────────────────
_run_one() {
local t="$1"
local BINARY="fuzz/target/release/$t"
local CORPUS="corpus/libfuzz/$t" # seed from libFuzzer corpus
local OUTPUT="afl-output/$t"
mkdir -p "$CORPUS" "$OUTPUT"
if [ ! -f "$BINARY" ]; then
echo "Binary not found — building $t first…"
just afl-build-target "$t"
fi
timeout "$TIME" afl-fuzz -i "$CORPUS" -o "$OUTPUT" -- "$BINARY" || true
}
for t in "${TARGETS[@]}"; do
echo "=== afl++ $t for ${TIME}s ==="
_run_one "$t"
done
just afl-corpus-sync
# ── Crash / hang summary ──────────────────────────────────────────────────
echo ""
echo "=== AFL++ crash / hang summary ==="
total_crashes=0
total_hangs=0
for target_dir in afl-output/*/; do
[ -d "$target_dir" ] || continue
for instance_dir in "$target_dir"*/; do
[ -d "$instance_dir" ] || continue
crashes_dir="${instance_dir}crashes"
hangs_dir="${instance_dir}hangs"
n_crashes=0
n_hangs=0
if [ -d "$crashes_dir" ]; then
n_crashes=$(find "$crashes_dir" -maxdepth 1 -type f | wc -l | tr -d ' ')
fi
if [ -d "$hangs_dir" ]; then
n_hangs=$(find "$hangs_dir" -maxdepth 1 -type f | wc -l | tr -d ' ')
fi
if [ "$n_crashes" -gt 0 ] || [ "$n_hangs" -gt 0 ]; then
echo " !! $(basename "$target_dir")/$(basename "$instance_dir") crashes=$n_crashes hangs=$n_hangs"
for f in "$crashes_dir"/id:*; do
[ -f "$f" ] && echo " $f"
done
for f in "$hangs_dir"/id:*; do
[ -f "$f" ] && echo " $f"
done
fi
total_crashes=$((total_crashes + n_crashes))
total_hangs=$((total_hangs + n_hangs))
done
done
echo ""
if [ "$total_crashes" -eq 0 ] && [ "$total_hangs" -eq 0 ]; then
echo " ✓ No crashes or hangs found across all targets."
else
echo " TOTAL crashes=$total_crashes hangs=$total_hangs"
echo ""
echo " Minimise a crash : just afl-tmin <target> <crash-file>"
echo " Format for a report: just afl-fmt <crash-file>"
fi
# Run AFL++ with N parallel instances (1 main + N-1 secondary) for TIME seconds.
# Requires that afl-fuzz is on PATH; all instances share afl-output/{{TARGET}}/.
# On macOS the crash reporter is disabled automatically for the duration of the
# run and re-enabled when the script exits.
#
# Usage: just fuzz-afl-parallel fuzz_state_transition
# just fuzz-afl-parallel fuzz_state_transition 8 600
fuzz-afl-parallel TARGET WORKERS="4" TIME="300":
#!/bin/bash
set -euo pipefail
BINARY="fuzz/target/release/{{TARGET}}"
CORPUS="corpus/libfuzz/{{TARGET}}" # seed from libFuzzer corpus
OUTPUT="afl-output/{{TARGET}}"
mkdir -p "$CORPUS" "$OUTPUT"
if [ ! -f "$BINARY" ]; then
echo "Binary not found — building first…"
just afl-build-target {{TARGET}}
fi
# ── macOS: disable crash reporter for the duration of this run ───────────
if [ "$(uname)" = "Darwin" ]; then
SL=/System/Library; PL=com.apple.ReportCrash
echo "macOS: disabling crash reporter (AFL++ requirement)…"
launchctl unload -w "${SL}/LaunchAgents/${PL}.plist" 2>/dev/null || true
sudo launchctl unload -w "${SL}/LaunchDaemons/${PL}.Root.plist" 2>/dev/null || true
trap '
echo "Re-enabling macOS crash reporter…"
SL=/System/Library; PL=com.apple.ReportCrash
launchctl load -w "${SL}/LaunchAgents/${PL}.plist" 2>/dev/null || true
sudo launchctl load -w "${SL}/LaunchDaemons/${PL}.Root.plist" 2>/dev/null || true
' EXIT
fi
# Main instance
afl-fuzz -M main -i "$CORPUS" -o "$OUTPUT" -- "$BINARY" &
# Secondary instances
for i in $(seq 1 $(( {{WORKERS}} - 1 ))); do
afl-fuzz -S "secondary${i}" -i "$CORPUS" -o "$OUTPUT" -- "$BINARY" &
done
sleep {{TIME}}
kill $(jobs -p) 2>/dev/null || true
wait 2>/dev/null || true
just afl-corpus-sync
# Copy all queue entries from every AFL++ output directory into the matching
# AFL corpus directory (corpus/afl/<target>/). Run after any AFL++ session
# to make new interesting inputs available for coverage measurement and future runs.
afl-corpus-sync:
#!/bin/bash
set -euo pipefail
if [ ! -d afl-output ]; then
echo "afl-output/ does not exist — nothing to sync."
exit 0
fi
for target_dir in afl-output/*/; do
TARGET=$(basename "$target_dir")
DEST="corpus/afl/${TARGET}"
mkdir -p "$DEST"
count=0
for instance_dir in "$target_dir"*/; do
QUEUE="${instance_dir}queue"
[ -d "$QUEUE" ] || continue
for f in "$QUEUE"/id:*; do
[ -f "$f" ] || continue
HASH=$(sha1sum "$f" | cut -d' ' -f1)
DEST_FILE="${DEST}/${HASH}"
if [ ! -f "$DEST_FILE" ]; then
cp "$f" "$DEST_FILE"
count=$((count + 1))
fi
done
done
echo "Synced $count new input(s) → $DEST"
done
# Show AFL++ campaign statistics for a target
# Usage: just afl-status fuzz_state_transition
afl-status TARGET:
afl-whatsup afl-output/{{TARGET}}
# Minimise a crash or hang artifact to the smallest reproducing input.
# Usage: just afl-tmin fuzz_state_transition afl-output/fuzz_state_transition/crashes/id:000000,...
afl-tmin TARGET ARTIFACT:
afl-tmin -i {{ARTIFACT}} -o {{ARTIFACT}}.min -- fuzz/target/release/{{TARGET}}
# Pretty-print an AFL++ artifact as a Rust byte-string literal (for copy-paste
# into a unit test or issue report).
# Usage: just afl-fmt afl-output/fuzz_state_transition/crashes/id:000000,...
afl-fmt ARTIFACT:
python3 -c "import sys; data=open(sys.argv[1],'rb').read(); print('b\"' + ''.join(f'\\\\x{b:02x}' for b in data) + '\"')" {{ARTIFACT}}
# ── Coverage ──────────────────────────────────────────────────────────────────
#
# cargo-fuzz always writes its profdata to the fixed path:
# fuzz/coverage/<TARGET>/coverage.profdata
# Each coverage recipe immediately copies that file into the organised tree
# (coverage/libfuzz/ or coverage/afl/) so that a subsequent run of the other
# engine cannot overwrite the data we need for the summary reports.
# Generate a libFuzzer-only coverage report for a single target.
# Runs `cargo fuzz coverage` against corpus/libfuzz/<TARGET>/, then copies the
# profdata into coverage/libfuzz/<TARGET>/ and renders an HTML report there.
# Output: coverage/libfuzz/{{TARGET}}/html/index.html
# Usage: just coverage-libfuzz fuzz_state_transition
coverage-libfuzz TARGET:
#!/bin/bash
set -euo pipefail
# ── Resolve LLVM tools from the active Rust toolchain ─────────────────────
_SYSROOT=$(rustc --print sysroot)
_HOST=$(rustc -vV | sed -n 's/^host: //p')
_LLVM_BIN="${_SYSROOT}/lib/rustlib/${_HOST}/bin"
LLVM_COV="${_LLVM_BIN}/llvm-cov"
command -v "$LLVM_COV" &>/dev/null || LLVM_COV=$(command -v llvm-cov 2>/dev/null || true)
if [ -z "$LLVM_COV" ] || [ ! -x "$LLVM_COV" ]; then
echo "ERROR: llvm-cov not found in Rust sysroot (${_LLVM_BIN}) or PATH."
echo " Run: rustup component add llvm-tools-preview"
exit 1
fi
CORPUS="corpus/libfuzz/{{TARGET}}"
mkdir -p "$CORPUS"
echo "=== cargo fuzz coverage {{TARGET}} (libFuzzer corpus) ==="
cargo fuzz coverage {{TARGET}} "$CORPUS"
# ── Copy profdata to the organised tree ───────────────────────────────────
# cargo-fuzz always writes here; we copy immediately so a later AFL pass
# cannot clobber this file before the summary reads it.
CARGO_PROFDATA="fuzz/coverage/{{TARGET}}/coverage.profdata"
LF_COV_DIR="coverage/libfuzz/{{TARGET}}"
mkdir -p "$LF_COV_DIR"
if [ ! -f "$CARGO_PROFDATA" ]; then
echo "WARNING: profdata not produced — skipping HTML generation."
exit 0
fi
cp "$CARGO_PROFDATA" "${LF_COV_DIR}/coverage.profdata"
PROFDATA="${LF_COV_DIR}/coverage.profdata"
# ── Render HTML ───────────────────────────────────────────────────────────
BINARY="target/${_HOST}/coverage/${_HOST}/release/{{TARGET}}"
HTML_DIR="${LF_COV_DIR}/html"
if [ -f "$BINARY" ]; then
mkdir -p "$HTML_DIR"
"$LLVM_COV" show \
"$BINARY" \
--instr-profile="$PROFDATA" \
--format=html \
--output-dir="$HTML_DIR" \
--ignore-filename-regex='\.cargo|rustc'
echo "libFuzzer HTML coverage report: ${HTML_DIR}/index.html"
else
echo "WARNING: binary not found — skipping HTML generation."
echo " Binary: $BINARY"
fi
# Measure code coverage exercised by the AFL++ corpus for a single target.
#
# Strategy: replay corpus/afl/<TARGET>/ through the libFuzzer coverage binary
# (built by `cargo fuzz coverage`). Run `just afl-corpus-sync` first to
# populate corpus/afl/<TARGET>/ from the AFL++ queue.
#
# Output: coverage/afl/{{TARGET}}/html/index.html
# Usage: just coverage-afl fuzz_state_transition
coverage-afl TARGET:
#!/bin/bash
set -euo pipefail
# ── Resolve LLVM tools from the active Rust toolchain ─────────────────────
_SYSROOT=$(rustc --print sysroot)
_HOST=$(rustc -vV | sed -n 's/^host: //p')
_LLVM_BIN="${_SYSROOT}/lib/rustlib/${_HOST}/bin"
LLVM_COV="${_LLVM_BIN}/llvm-cov"
command -v "$LLVM_COV" &>/dev/null || LLVM_COV=$(command -v llvm-cov 2>/dev/null || true)
if [ -z "$LLVM_COV" ] || [ ! -x "$LLVM_COV" ]; then
echo "ERROR: llvm-cov not found in Rust sysroot (${_LLVM_BIN}) or PATH."
echo " Run: rustup component add llvm-tools-preview"
exit 1
fi
AFL_CORPUS="corpus/afl/{{TARGET}}"
if [ ! -d "$AFL_CORPUS" ] || [ -z "$(ls -A "$AFL_CORPUS" 2>/dev/null)" ]; then
echo "No AFL++ corpus for {{TARGET}} at $AFL_CORPUS."
echo " Run 'just afl-corpus-sync' after an AFL++ session to populate it."
exit 0
fi
echo "=== AFL++ corpus coverage for {{TARGET}} ==="
# ── Replay AFL++ corpus through the libFuzzer coverage binary ─────────────
# cargo fuzz coverage always writes:
# profdata → fuzz/coverage/{{TARGET}}/coverage.profdata
# binary → target/<host>/coverage/<host>/release/{{TARGET}}
cargo fuzz coverage {{TARGET}} "$AFL_CORPUS"
# ── Copy profdata to the organised tree ───────────────────────────────────
CARGO_PROFDATA="fuzz/coverage/{{TARGET}}/coverage.profdata"
AFL_COV_DIR="coverage/afl/{{TARGET}}"
mkdir -p "$AFL_COV_DIR"
if [ ! -f "$CARGO_PROFDATA" ]; then
echo "WARNING: profdata not produced — skipping HTML generation."
exit 0
fi
cp "$CARGO_PROFDATA" "${AFL_COV_DIR}/coverage.profdata"
# ── Render HTML ───────────────────────────────────────────────────────────
BINARY="target/${_HOST}/coverage/${_HOST}/release/{{TARGET}}"
HTML_DIR="${AFL_COV_DIR}/html"
mkdir -p "$HTML_DIR"
if [ -f "$BINARY" ]; then
"$LLVM_COV" show \
"$BINARY" \
--instr-profile="${AFL_COV_DIR}/coverage.profdata" \
--format=html \
--output-dir="$HTML_DIR" \
--ignore-filename-regex='\.cargo|rustc'
echo "AFL++ corpus HTML coverage report: ${HTML_DIR}/index.html"
else
echo "WARNING: binary not found: $BINARY"
fi
# Generate a combined coverage report for a single target (libFuzzer + AFL++).
# Delegates to coverage-libfuzz then coverage-afl.
# Usage: just coverage fuzz_state_transition
coverage TARGET:
just coverage-libfuzz {{TARGET}} || true
just coverage-afl {{TARGET}}
# Generate coverage for ALL registered fuzz targets.
# ENGINE selects which fuzzer engine to measure:
# "all" — libFuzzer + AFL++ (default)
# "libfuzz" — libFuzzer only (cargo fuzz coverage against corpus/libfuzz/)
# "afl" — AFL++ only (cargo fuzz coverage against corpus/afl/)
#
# After the per-target loop, a merged summary HTML report is written:
# libfuzz → coverage/libfuzz/summary/html/index.html
# afl → coverage/afl/summary/html/index.html
#
# Usage: just coverage-all # both engines
# just coverage-all libfuzz # libFuzzer only
# just coverage-all afl # AFL++ only
coverage-all ENGINE="all":
#!/bin/bash
set -euo pipefail
# ── Resolve LLVM tools from the active Rust toolchain ─────────────────────
_SYSROOT=$(rustc --print sysroot)
_HOST=$(rustc -vV | sed -n 's/^host: //p')
_LLVM_BIN="${_SYSROOT}/lib/rustlib/${_HOST}/bin"
LLVM_COV="${_LLVM_BIN}/llvm-cov"
LLVM_PROFDATA="${_LLVM_BIN}/llvm-profdata"
command -v "$LLVM_COV" &>/dev/null || LLVM_COV=$(command -v llvm-cov 2>/dev/null || true)
command -v "$LLVM_PROFDATA" &>/dev/null || LLVM_PROFDATA=$(command -v llvm-profdata 2>/dev/null || true)
TARGETS=($(cargo fuzz list 2>/dev/null))
# ── Per-target passes ─────────────────────────────────────────────────────
# Each coverage-libfuzz / coverage-afl call copies its profdata into the
# organised tree (coverage/libfuzz/ or coverage/afl/) immediately after
# cargo-fuzz produces it. Because the two engines write to distinct
# directories there is no risk of one overwriting the other's data,
# regardless of the order in which the targets are processed.
if [ "{{ENGINE}}" = "all" ] || [ "{{ENGINE}}" = "libfuzz" ]; then
for target in "${TARGETS[@]}"; do
echo "=== coverage (libfuzz) $target ==="
just coverage-libfuzz "$target" || true
done
fi
if [ "{{ENGINE}}" = "all" ] || [ "{{ENGINE}}" = "afl" ]; then
for target in "${TARGETS[@]}"; do
echo "=== coverage (afl) $target ==="
just coverage-afl "$target"
done
fi
# ── Merged summary report (libfuzz) ───────────────────────────────────────
if [ "{{ENGINE}}" = "libfuzz" ] || [ "{{ENGINE}}" = "all" ]; then
echo ""
echo "=== libFuzzer summary report (all targets merged) ==="
SUMMARY_DIR="coverage/libfuzz/summary"
mkdir -p "$SUMMARY_DIR"
PROFDATA_FILES=()
BINARY_ARGS=()
for t in "${TARGETS[@]}"; do
PD="coverage/libfuzz/$t/coverage.profdata"
BIN="target/${_HOST}/coverage/${_HOST}/release/$t"
[ -f "$PD" ] && PROFDATA_FILES+=("$PD")
[ -f "$BIN" ] && BINARY_ARGS+=("--object" "$BIN")
done
if [ ${#PROFDATA_FILES[@]} -eq 0 ]; then
echo "No libFuzzer profdata found — skipping summary."
else
MERGED="${SUMMARY_DIR}/merged.profdata"
"$LLVM_PROFDATA" merge -sparse "${PROFDATA_FILES[@]}" -o "$MERGED"
HTML_DIR="${SUMMARY_DIR}/html"
mkdir -p "$HTML_DIR"
# First binary is positional; the rest are --object flags.
FIRST_BIN="${BINARY_ARGS[1]}" # index 1 is the path after '--object'
REST_ARGS=("${BINARY_ARGS[@]:2}")
"$LLVM_COV" show \
"$FIRST_BIN" \
"${REST_ARGS[@]}" \
--instr-profile="$MERGED" \
--format=html \
--output-dir="$HTML_DIR" \
--ignore-filename-regex='\.cargo|rustc'
echo "libFuzzer summary HTML report: ${HTML_DIR}/index.html"
fi
fi
# ── Merged summary report (afl) ───────────────────────────────────────────
if [ "{{ENGINE}}" = "afl" ] || [ "{{ENGINE}}" = "all" ]; then
echo ""
echo "=== AFL++ corpus summary report (all targets merged) ==="
SUMMARY_DIR="coverage/afl/summary"
mkdir -p "$SUMMARY_DIR"
PROFDATA_FILES=()
BINARY_ARGS=()
for t in "${TARGETS[@]}"; do
PD="coverage/afl/$t/coverage.profdata"
BIN="target/${_HOST}/coverage/${_HOST}/release/$t"
[ -f "$PD" ] && PROFDATA_FILES+=("$PD")
[ -f "$BIN" ] && BINARY_ARGS+=("--object" "$BIN")
done
if [ ${#PROFDATA_FILES[@]} -eq 0 ]; then
echo "No AFL++ profdata found — skipping summary."
else
MERGED="${SUMMARY_DIR}/merged.profdata"
"$LLVM_PROFDATA" merge -sparse "${PROFDATA_FILES[@]}" -o "$MERGED"
HTML_DIR="${SUMMARY_DIR}/html"
mkdir -p "$HTML_DIR"
FIRST_BIN="${BINARY_ARGS[1]}"
REST_ARGS=("${BINARY_ARGS[@]:2}")
"$LLVM_COV" show \
"$FIRST_BIN" \
"${REST_ARGS[@]}" \
--instr-profile="$MERGED" \
--format=html \
--output-dir="$HTML_DIR" \
--ignore-filename-regex='\.cargo|rustc'
echo "AFL++ corpus summary HTML report: ${HTML_DIR}/index.html"
fi
fi
# ── Mutation testing ──────────────────────────────────────────────────────────
#
# Prerequisites (install once):
# cargo install cargo-mutants
#
# Two planes — run them independently:
#
# Plane A (fast, ~1-5 min): mutates fuzz_props invariant logic.
# Oracle: cargo test -p fuzz_props --release
# Run on every PR that touches fuzz_props/ or fuzz/fuzz_targets/.
#
# Plane B (slow, ~hours): mutates LEZ protocol code (nssa, common).
# Oracle: all 15 fuzz targets replayed against their committed corpus.
# Run weekly or manually to find corpus gaps.
# Plane A — mutation testing of fuzz_props invariant implementations.
#
# Mutates every function in fuzz_props and checks whether `cargo test -p fuzz_props
# --release` catches the mutation. Surviving mutants identify invariant-checker
# logic that the property tests do not fully exercise.
#
# Workspace metadata in Cargo.toml configures --release, exclude_globs, and
# timeout_multiplier automatically.
#
# --in-place is mandatory: fuzz_props depends on LEZ crates via relative path
# (../logos-execution-zone/...) — without it cargo-mutants copies the workspace
# to /tmp and the copy cannot resolve those relative paths.
#
# Output: mutants-harness.out/ (human-readable report also printed to stdout)
mutants-harness:
cargo mutants --package fuzz_props --in-place --output mutants-harness.out
# Plane B — mutation testing of the LEZ protocol code against the committed corpus.
#
# Mutates nssa and common in the logos-execution-zone sibling workspace and uses
# scripts/mutants-corpus-test.sh as the oracle. The oracle replays all 15
# committed libFuzzer corpora (cargo fuzz run -runs=0) against each mutant.
#
# A mutant that SURVIVES means there is no corpus input that triggers the
# relevant protocol invariant at that mutation point — a corpus gap worth
# investigating with a longer fuzz run.
#
# Prerequisites:
# - logos-execution-zone cloned at ../logos-execution-zone
# - cargo-fuzz installed (cargo install cargo-fuzz)
# - cargo-mutants installed (cargo install cargo-mutants --locked)
#
# PACKAGES selects which LEZ crates to mutate (space-separated).
# Default covers the two highest-value protocol crates.
#
# Output report: mutants-protocol.out/ in the repository root.
mutants-protocol PACKAGES="nssa common":
#!/bin/bash
set -euo pipefail
REPO_DIR="$(pwd)"
if [ ! -d "${REPO_DIR}/../logos-execution-zone" ]; then
echo "ERROR: logos-execution-zone not found at ../logos-execution-zone"
exit 1
fi
LEZ_DIR="$(cd "${REPO_DIR}/../logos-execution-zone" && pwd)"
# Build --package flags (one per crate name)
PKG_FLAGS=()
for pkg in {{PACKAGES}}; do
PKG_FLAGS+=(--package "$pkg")
done
echo "=== Plane B: mutating [{{PACKAGES}}] in logos-execution-zone ==="
echo " Oracle: scripts/mutants-corpus-test.sh (corpus regression, -runs=0)"
echo " Report: ${REPO_DIR}/mutants-protocol.out/"
echo ""
# cargo-mutants must be run from inside the target workspace.
# FUZZ_REPO tells the oracle script where to find the corpus and fuzz/ dir.
# --output puts the report in our repo root so it's easy to browse/commit.
# --in-place is required because LEZ depends on path crates outside its own
# directory (e.g. the Rust standard toolchain); without it cargo-mutants copies
# the workspace to a temp dir where those relative paths would not resolve.
#
# cargo-mutants >=24 dropped --test-command and only supports --test-tool cargo|nextest.
# Work around: create a fake `cargo` wrapper that intercepts `cargo test` and
# runs the corpus oracle instead; every other sub-command is delegated to the
# real cargo. We call the cargo-mutants binary directly so that cargo's own
# process launch doesn't override the CARGO env var back to the real binary.
REAL_CARGO="$(command -v cargo)"
FAKE_CARGO=$(mktemp /tmp/fake-cargo-XXXXXX)
FAKE_CARGO_LOG=$(mktemp /tmp/fake-cargo-log-XXXXXX.txt)
trap 'rm -f "$FAKE_CARGO" "$FAKE_CARGO_LOG"' EXIT
# The fake cargo intercepts the test *execution* phase only.
# cargo-mutants drives two kinds of "cargo test" invocations:
# Build phase: cargo test --no-run --verbose --package=... (compile only)
# Test phase: cargo test --verbose --package=... (run tests)
# The oracle must only replace the test execution phase; the build phase
# must be forwarded to the real cargo so mutants are actually compiled.
printf '#!/bin/bash\necho "FAKE_CARGO: $*" >> "%s"\n_has_no_run=false\nfor _a in "$@"; do [ "$_a" = "--no-run" ] && _has_no_run=true && break; done\nif [ "${1:-}" = "test" ] && [ "$_has_no_run" = "false" ]; then\n FUZZ_REPO="%s" exec "%s"\nelse\n exec "%s" "$@"\nfi\n' \
"$FAKE_CARGO_LOG" \
"$REPO_DIR" \
"${REPO_DIR}/scripts/mutants-corpus-test.sh" \
"$REAL_CARGO" > "$FAKE_CARGO"
chmod +x "$FAKE_CARGO"
# Locate the cargo-mutants binary (installed by `cargo install cargo-mutants`).
MUTANTS_BIN="$(command -v cargo-mutants 2>/dev/null || true)"
if [ -z "$MUTANTS_BIN" ]; then
MUTANTS_BIN="$(dirname "$REAL_CARGO")/cargo-mutants"
fi
if [ ! -x "$MUTANTS_BIN" ]; then
echo "ERROR: cargo-mutants not found. Install with: cargo install cargo-mutants --locked"
exit 1
fi
# cargo-mutants is a Cargo plugin. When invoked via `cargo mutants`, Cargo
# automatically prepends "mutants" as argv[1]. When we invoke the binary
# directly (to keep our CARGO env override alive), we must supply it ourselves.
cd "$LEZ_DIR"
CARGO="$FAKE_CARGO" \
"$MUTANTS_BIN" mutants \
"${PKG_FLAGS[@]}" \
--in-place \
--output "${REPO_DIR}/mutants-protocol.out" \
--timeout-multiplier 5.0 \
|| { echo "--- fake-cargo invocations ---"; cat "$FAKE_CARGO_LOG"; exit 1; }
echo ""
echo "=== Mutation report summary ==="
MISSED_FILE="${REPO_DIR}/mutants-protocol.out/missed.txt"
CAUGHT_FILE="${REPO_DIR}/mutants-protocol.out/caught.txt"
MISSED=$(wc -l < "$MISSED_FILE" 2>/dev/null | tr -d ' ' || echo 0)
CAUGHT=$(wc -l < "$CAUGHT_FILE" 2>/dev/null | tr -d ' ' || echo 0)
echo "Caught: ${CAUGHT} | Survived: ${MISSED}"
echo ""
if [ "${MISSED}" -gt 0 ]; then
echo "Surviving mutants (corpus gaps):"
cat "$MISSED_FILE" || true
echo ""
echo "For each surviving mutant: run 'just fuzz <target>' targeting the"
echo "mutated function, add the crashing input to corpus/libfuzz/<target>/,"
echo "then re-run 'just mutants-protocol' to confirm it is now CAUGHT."
else
echo "All mutants caught — corpus covers all tested mutation points."
fi
# ── Housekeeping ──────────────────────────────────────────────────────────────
# Remove all Cargo build artefacts (workspace + fuzz sub-crate)
clean:
cargo clean
cargo clean --manifest-path fuzz/Cargo.toml
# Remove libFuzzer crash/timeout artifacts for all targets (corpus is kept)
clean-artifacts:
rm -rf fuzz/artifacts/
# Remove all coverage reports.
# Also removes the temporary profdata that cargo-fuzz writes to fuzz/coverage/
# and any stray .profraw files left by the instrumented binaries.
clean-coverage:
rm -rf coverage/ fuzz/coverage/
find . -name '*.profraw' -delete
# Remove AFL++ output directories (crashes, hangs, queue).
# Note: the queue is also stored in corpus/afl/ via `just afl-corpus-sync`.
clean-afl:
rm -rf afl-output/
# Remove the corpus directories (libFuzzer and AFL).
clean-corpus:
rm -rf corpus/
# Remove everything: builds, artifacts, coverage, and AFL++ output (preserves corpus/)
clean-all: clean clean-artifacts clean-coverage clean-afl
Reference in New Issue View Git Blame Copy Permalink