mirror of https://github.com/status-im/nim-eth.git
Update fuzzing tests now that the helpers have moved to ntu
This commit is contained in:
kdeme
zah
parent
225a9ad41c
commit
4d0a7a46ba
|
|
@ -15,7 +15,8 @@ requires "nim >= 1.2.0",
|
|||
"nat_traversal",
|
||||
"metrics",
|
||||
"sqlite3_abi",
|
||||
"confutils"
|
||||
"confutils",
|
||||
"testutils"
|
||||
|
||||
proc runTest(path: string) =
|
||||
echo "\nRunning: ", path
|
||||
|
|
|
|||
|
|
@ -1,31 +0,0 @@
|
|||
proc aflSwitches() =
|
||||
switch("define", "afl")
|
||||
switch("define", "noSignalHandler")
|
||||
switch("cc", "gcc")
|
||||
switch("gcc.exe", "afl-gcc")
|
||||
switch("gcc.linkerexe", "afl-gcc")
|
||||
switch("out", "fuzz-afl")
|
||||
|
||||
proc libFuzzerSwitches() =
|
||||
switch("define", "libFuzzer")
|
||||
switch("noMain", "")
|
||||
switch("cc", "clang")
|
||||
switch("passC", "-fsanitize=fuzzer,address")
|
||||
switch("passL", "-fsanitize=fuzzer,address")
|
||||
switch("out", "fuzz-libfuzzer")
|
||||
|
||||
proc generalSwitches() =
|
||||
switch("verbosity", "0")
|
||||
switch("hints", "off")
|
||||
switch("warnings", "off")
|
||||
switch("define", "chronicles_log_level:fatal")
|
||||
|
||||
task build_afl, "Build for afl fuzzing":
|
||||
aflSwitches()
|
||||
generalSwitches()
|
||||
setCommand("c")
|
||||
|
||||
task build_libfuzzer, "Build for libFuzzer fuzzing":
|
||||
libFuzzerSwitches()
|
||||
generalSwitches()
|
||||
setCommand("c")
|
||||
|
|
@ -1,6 +1,7 @@
|
|||
import
|
||||
chronicles, eth/p2p/[discovery, enode], eth/[keys, rlp],
|
||||
../../p2p/p2p_test_helper, ../fuzztest
|
||||
testutils/fuzzing, chronicles,
|
||||
eth/p2p/[discovery, enode], eth/[keys, rlp],
|
||||
../../p2p/p2p_test_helper
|
||||
|
||||
const DefaultListeningPort = 30303
|
||||
var targetNode: DiscoveryProtocol
|
||||
|
|
@ -8,7 +9,7 @@ var targetNode: DiscoveryProtocol
|
|||
init:
|
||||
# Set up a discovery node, this is the node we target when fuzzing
|
||||
var
|
||||
targetNodeKey = PrivateKey.fromRaw("a2b50376a79b1a8c8a3296485572bdfbf54708bb46d3c25d73d2723aaaf6a617")[]
|
||||
targetNodeKey = PrivateKey.fromHex("a2b50376a79b1a8c8a3296485572bdfbf54708bb46d3c25d73d2723aaaf6a617")[]
|
||||
targetNodeAddr = localAddress(DefaultListeningPort)
|
||||
targetNode = newDiscoveryProtocol(targetNodeKey, targetNodeAddr, @[])
|
||||
# Create the transport as else replies on the messages send will fail.
|
||||
|
|
@ -22,7 +23,7 @@ test:
|
|||
# Sending raw payload is possible but won't find us much. We need a hash and
|
||||
# a signature, and without it there is a big chance it will always result in
|
||||
# "Wrong msg mac from" error.
|
||||
let nodeKey = PrivateKey.fromRaw("a2b50376a79b1a8c8a3296485572bdfbf54708bb46d3c25d73d2723aaaf6a618")[]
|
||||
let nodeKey = PrivateKey.fromHex("a2b50376a79b1a8c8a3296485572bdfbf54708bb46d3c25d73d2723aaaf6a618")[]
|
||||
msg = packData(payload, nodeKey)
|
||||
address = localAddress(DefaultListeningPort + 1)
|
||||
|
||||
|
|
@ -30,5 +31,7 @@ test:
|
|||
targetNode.receive(address, msg)
|
||||
# These errors are also catched in `processClient` in discovery.nim
|
||||
# TODO: move them a layer down in discovery so we can do a cleaner test there?
|
||||
except RlpError, DiscProtocolError as e:
|
||||
except RlpError as e:
|
||||
debug "Receive failed", err = e.msg
|
||||
except DiscProtocolError as e:
|
||||
debug "Receive failed", err = e.msg
|
||||
|
|
|
|||
|
|
@ -1,28 +0,0 @@
|
|||
import ./fuzz_helpers
|
||||
|
||||
# TODO: get this some nice cmd line options when confutils works for nimscript
|
||||
# or if we want to put this in a nim application instead of script
|
||||
|
||||
if paramCount() < 3:
|
||||
echo "Usage: nim fuzz.nims FUZZER TARGET"
|
||||
echo "Fuzzer options are afl or libFuzzer"
|
||||
quit 1
|
||||
|
||||
let
|
||||
fuzzer = paramStr(2)
|
||||
targetPath = paramStr(3)
|
||||
|
||||
if not fileExists(targetPath):
|
||||
echo "Target file does not exist"
|
||||
quit 1
|
||||
|
||||
case fuzzer
|
||||
of "afl":
|
||||
runFuzzer(targetPath, afl)
|
||||
of "libFuzzer":
|
||||
runFuzzer(targetPath, libFuzzer)
|
||||
|
||||
else:
|
||||
echo "Invalid fuzzer option: ", fuzzer
|
||||
echo "Fuzzer options are afl or libFuzzer"
|
||||
quit 1
|
||||
|
|
@ -1,94 +0,0 @@
|
|||
import strformat, ospaths
|
||||
|
||||
# Dependencies:
|
||||
# - afl fuzzing: afl and gcc or clang/llvm
|
||||
# - libFuzzer fuzzing: libFuzzer and clang/llvm
|
||||
# - in afl experimental modes clang/llvm is also required
|
||||
|
||||
# TODO:
|
||||
# - switch clang / gcc option for afl
|
||||
# - afl init and persistent modes
|
||||
# - parallel fuzzing options
|
||||
# - custom generate test cases from this script?
|
||||
# - rerun testcases option (or create tests from failed cases)
|
||||
# - currently not cross platform
|
||||
# - ...
|
||||
|
||||
const
|
||||
aflGcc = "--cc=gcc " &
|
||||
"--gcc.exe=afl-gcc " &
|
||||
"--gcc.linkerexe=afl-gcc"
|
||||
aflClang = "--cc=clang " &
|
||||
"--clang.exe=afl-clang " &
|
||||
"--clang.linkerexe=afl-clang"
|
||||
aflClangFast = "--cc=clang " &
|
||||
"--clang.exe=afl-clang-fast " &
|
||||
"--clang.linkerexe=afl-clang-fast " &
|
||||
"-d:clangfast"
|
||||
libFuzzerClang = "--cc=clang " &
|
||||
"--passC='-fsanitize=fuzzer,address' " &
|
||||
"--passL='-fsanitize=fuzzer,address'"
|
||||
# Can also test in debug mode obviously, but might be slower
|
||||
# Can turn on more logging, in case of libFuzzer it will get very verbose though
|
||||
defaultFlags = "-d:release -d:chronicles_log_level=fatal " &
|
||||
"--hints:off --warnings:off --verbosity:0"
|
||||
|
||||
type
|
||||
Fuzzer* = enum
|
||||
afl,
|
||||
libFuzzer
|
||||
|
||||
Compiler* = enum
|
||||
gcc = aflGcc,
|
||||
clang = aflClang,
|
||||
clangFast = aflClangFast
|
||||
|
||||
proc aflCompile*(target: string, c: Compiler) =
|
||||
let aflOptions = &"-d:afl -d:noSignalHandler {$c}"
|
||||
let compileCmd = &"nim c {defaultFlags} {aflOptions} {target.quoteShell()}"
|
||||
exec compileCmd
|
||||
|
||||
proc aflExec*(target: string, inputDir: string, resultsDir: string,
|
||||
cleanStart = false) =
|
||||
let exe = target.addFileExt(ExeExt)
|
||||
if not dirExists(inputDir):
|
||||
# create a input dir with one 0 file for afl
|
||||
mkDir(inputDir)
|
||||
# TODO: improve
|
||||
withDir inputDir: exec "echo '0' > test"
|
||||
|
||||
var fuzzCmd: string
|
||||
# if there is an output dir already, continue fuzzing from previous run
|
||||
if (not dirExists(resultsDir)) or cleanStart:
|
||||
fuzzCmd = &"afl-fuzz -i {inputDir.quoteShell()} -o {resultsDir.quoteShell()} -M fuzzer01 -- {exe.quoteShell()}"
|
||||
else:
|
||||
fuzzCmd = &"afl-fuzz -i - -o {resultsDir.quoteShell()} -M fuzzer01 -- {exe.quoteShell()}"
|
||||
exec fuzzCmd
|
||||
|
||||
proc libFuzzerCompile*(target: string) =
|
||||
let libFuzzerOptions = &"-d:libFuzzer --noMain {libFuzzerClang}"
|
||||
let compileCmd = &"nim c {defaultFlags} {libFuzzerOptions} {target.quoteShell()}"
|
||||
exec compileCmd
|
||||
|
||||
proc libFuzzerExec*(target: string, corpusDir: string) =
|
||||
let exe = target.addFileExt(ExeExt)
|
||||
if not dirExists(corpusDir):
|
||||
# libFuzzer is OK when starting with empty corpus dir
|
||||
mkDir(corpusDir)
|
||||
|
||||
exec &"{exe.quoteShell()} {corpusDir.quoteShell()}"
|
||||
|
||||
proc runFuzzer*(targetPath: string, fuzzer: Fuzzer) =
|
||||
let (path, target, ext) = splitFile(targetPath)
|
||||
|
||||
case fuzzer
|
||||
of afl:
|
||||
aflCompile(targetPath, gcc)
|
||||
aflExec(path / target, path / "input", path / "results")
|
||||
|
||||
of libFuzzer:
|
||||
libFuzzerCompile(targetPath)
|
||||
# Note: Lets not mix afl input with libFuzzer corpus default. This can have
|
||||
# consequences on speed for afl. Better to look into merging afl results &
|
||||
# libFuzzer corpus.
|
||||
libFuzzerExec(path / target, path / "corpus")
|
||||
|
|
@ -1,96 +0,0 @@
|
|||
import streams, posix, strutils, chronicles, macros, stew/ranges/ptr_arith
|
||||
|
||||
template fuzz(body) =
|
||||
# For code we want to fuzz, SIGSEGV is needed on unwanted exceptions.
|
||||
# However, this is only needed when fuzzing with afl.
|
||||
when defined(afl):
|
||||
try:
|
||||
body
|
||||
except Exception as e:
|
||||
error "Fuzzer input created exception", exception=e.name, trace=e.repr,
|
||||
msg=e.msg
|
||||
discard kill(getpid(), SIGSEGV)
|
||||
else:
|
||||
body
|
||||
|
||||
proc readStdin(): seq[byte] =
|
||||
# Read input from stdin (fastest for AFL)
|
||||
let s = newFileStream(stdin)
|
||||
if s.isNil:
|
||||
error "Error opening stdin"
|
||||
discard kill(getpid(), SIGSEGV)
|
||||
# We use binary files as with hex we can get lots of "not hex" failures
|
||||
var input = s.readAll()
|
||||
s.close()
|
||||
# Remove newline if it is there
|
||||
input.removeSuffix
|
||||
result = cast[seq[byte]](input)
|
||||
|
||||
proc NimMain() {.importc: "NimMain".}
|
||||
|
||||
# The default init, gets redefined when init template is used.
|
||||
template initImpl(): untyped =
|
||||
when not defined(libFuzzer):
|
||||
discard
|
||||
else:
|
||||
proc fuzzerInit(): cint {.exportc: "LLVMFuzzerInitialize".} =
|
||||
NimMain()
|
||||
|
||||
return 0
|
||||
|
||||
template init*(body: untyped) =
|
||||
## Init block to do any initialisation for the fuzzing test.
|
||||
##
|
||||
## For AFL this is currently only cosmetic and will be run each time, before
|
||||
## the test block.
|
||||
##
|
||||
## For libFuzzer this will only be run once. So only put data which is
|
||||
## stateless or make sure everything gets properply reset for each new run in
|
||||
## the test block.
|
||||
when not defined(libFuzzer):
|
||||
template initImpl(): untyped = fuzz: `body`
|
||||
else:
|
||||
template initImpl() =
|
||||
proc fuzzerInit(): cint {.exportc: "LLVMFuzzerInitialize".} =
|
||||
NimMain()
|
||||
|
||||
`body`
|
||||
|
||||
return 0
|
||||
|
||||
template test*(body: untyped): untyped =
|
||||
## Test block to do the actual test that will be fuzzed in a loop.
|
||||
##
|
||||
## Within this test block there is access to the payload OpenArray which
|
||||
## contains the payload provided by the fuzzer.
|
||||
mixin initImpl
|
||||
initImpl()
|
||||
when not defined(libFuzzer):
|
||||
var payload {.inject.} = readStdin()
|
||||
|
||||
fuzz: `body`
|
||||
else:
|
||||
proc fuzzerCall(data: ptr byte, len: csize):
|
||||
cint {.exportc: "LLVMFuzzerTestOneInput".} =
|
||||
template payload(): auto =
|
||||
makeOpenArray(data, len)
|
||||
|
||||
`body`
|
||||
|
||||
when defined(clangfast):
|
||||
## Can be used for deferred instrumentation.
|
||||
## Should be placed on a suitable location in the code where the delayed
|
||||
## cloning can take place (e.g. NOT after creation of threads)
|
||||
proc aflInit*() {.importc: "__AFL_INIT", noDecl.}
|
||||
## Can be used for persistent mode.
|
||||
## Should be used as value for controlling a loop around a test case.
|
||||
## Test case should be able to handle repeated inputs. No repeated fork() will
|
||||
## be done.
|
||||
# TODO: Lets use this in the test block when afl-clang-fast is used?
|
||||
proc aflLoopImpl(count: cuint): cint {.importc: "__AFL_LOOP", noDecl.}
|
||||
template aflLoop*(body: untyped): untyped =
|
||||
while aflLoopImpl(1000) != 0:
|
||||
`body`
|
||||
else:
|
||||
proc aflInit*() = discard
|
||||
template aflLoop*(body: untyped): untyped = `body`
|
||||
|
|
@ -1,204 +1,24 @@
|
|||
# Fuzzing
|
||||
## tldr:
|
||||
* [Install afl](#Install-afl).
|
||||
* Create a testcase.
|
||||
* Run: `nim fuzz.nims afl testfolder/testcase.nim`
|
||||
# Fuzzing Tests
|
||||
The fuzzing tests use the fuzzing templates from `nim-testutils`.
|
||||
|
||||
Or
|
||||
For more details see [the fuzzing readme of nim-testutils](https://github.com/status-im/nim-testutils/tree/master/testutils/fuzzing).
|
||||
|
||||
* [Install libFuzzer](#Install-libFuzzer) (comes with LLVM).
|
||||
* Create a testcase.
|
||||
* Run: `nim fuzz.nims libFuzzer testfolder/testcase.nim`
|
||||
## Prerequisites
|
||||
As [explained](https://github.com/status-im/nim-testutils/tree/master/testutils/fuzzing#supported-fuzzers)
|
||||
in `nim-testutils` fuzzing readme, first install the fuzzer you want to run.
|
||||
|
||||
## Fuzzing Helpers
|
||||
There are two convenience templates which will help you set up a quick fuzzing
|
||||
test.
|
||||
Next install `nim-testutils` its `ntu` application.
|
||||
|
||||
These are the mandatory `test` block and the optional `init` block.
|
||||
|
||||
Example usage:
|
||||
```nim
|
||||
test:
|
||||
var rlp = rlpFromBytes(payload)
|
||||
discard rlp.inspect()
|
||||
```
|
||||
|
||||
Any unhandled `Exception` will result in a failure of the testcase. If certain
|
||||
`Exception`s are to be allowed to occur within the test, they should be caught.
|
||||
|
||||
E.g.:
|
||||
```nim
|
||||
test:
|
||||
try:
|
||||
var rlp = rlpFromBytes(payload)
|
||||
discard rlp.inspect()
|
||||
except RlpError as e:
|
||||
debug "Inspect failed", err = e.msg
|
||||
```
|
||||
|
||||
## Supported Fuzzers
|
||||
The two templates can prepare the code for both
|
||||
[afl](http://lcamtuf.coredump.cx/afl/) and
|
||||
[libFuzzer](http://llvm.org/docs/LibFuzzer.html).
|
||||
|
||||
You will need to install first the fuzzer you want to use.
|
||||
### Install afl
|
||||
E.g. by running the `nim-testutils` nimble install:
|
||||
```sh
|
||||
# Ubuntu / Debian
|
||||
sudo apt-get install afl
|
||||
|
||||
# Fedora
|
||||
dnf install american-fuzzy-lop
|
||||
# for usage with clang & clang-fast you will have to install
|
||||
# american-fuzzy-lop-clang or american-fuzzy-lop-clang-fast
|
||||
|
||||
# Arch Linux
|
||||
pacman -S afl
|
||||
|
||||
# NixOS
|
||||
nix-env -i afl
|
||||
|
||||
nimble install nim-testutils
|
||||
```
|
||||
|
||||
### Install libFuzzer
|
||||
|
||||
LibFuzzer is part of llvm and will be installed together with llvm-libs in
|
||||
recent versions. Installing clang should install llvm-libs.
|
||||
## How to run
|
||||
To start fuzzing a testcase run following command:
|
||||
```sh
|
||||
# Ubuntu / Debian
|
||||
sudo apt-get install clang
|
||||
|
||||
# Fedora
|
||||
dnf install clang
|
||||
|
||||
# Arch Linux
|
||||
pacman -S clang
|
||||
|
||||
# NixOS
|
||||
nix-env -iA nixos.clang_7 nixos.llvm_7
|
||||
# For libFuzzer
|
||||
ntu fuzz --fuzzer:libFuzzer rlp/rlp_inspect
|
||||
# For afl
|
||||
ntu fuzz --fuzzer:afl rlp/rlp_inspect
|
||||
```
|
||||
|
||||
## Compiling & Starting the Fuzzer
|
||||
### Scripted helper
|
||||
There is a nimscript helper to compile & start the fuzzer:
|
||||
```sh
|
||||
# for afl
|
||||
nim fuzz.nims afl testcase.nim
|
||||
|
||||
# for libFuzzer
|
||||
nim fuzz.nims libFuzzer testcase.nim
|
||||
```
|
||||
### Manually with afl
|
||||
#### Compiling
|
||||
With gcc:
|
||||
```sh
|
||||
nim c -d:afl -d:release -d:chronicles_log_level=fatal -d:noSignalHandler --cc=gcc --gcc.exe=afl-gcc --gcc.linkerexe=afl-gcc testcase.nim
|
||||
```
|
||||
The `afl` define is specifically required for the `init` and `test`
|
||||
templates.
|
||||
|
||||
You typically want to fuzz in `-d:release` and probably also want to lower down
|
||||
the logging. But this is not strictly necessary.
|
||||
|
||||
There is also a nimscript task in `config.nims` for this:
|
||||
```
|
||||
nim c build_afl testcase.nim
|
||||
```
|
||||
|
||||
With clang:
|
||||
```sh
|
||||
# afl-clang
|
||||
nim c -d:afl -d:noSignalHandler --cc=clang --clang.exe=afl-clang --clang.linkerexe=afl-clang ftestcase.nim
|
||||
# afl-clang-fast
|
||||
nim c -d:afl -d:noSignalHandler --cc=clang --clang.exe=afl-clang-fast --clang.linkerexe=afl-clang-fast testcase.nim
|
||||
```
|
||||
|
||||
#### Starting the Fuzzer
|
||||
|
||||
To start the fuzzer:
|
||||
```sh
|
||||
afl-fuzz -i input -o results -- ./testcase
|
||||
```
|
||||
|
||||
To rerun it without losing previous results/corpus:
|
||||
```sh
|
||||
afl-fuzz -i - -o results -- ./testcase
|
||||
```
|
||||
|
||||
To run several parallel fuzzing sessions:
|
||||
```sh
|
||||
# Start master fuzzer
|
||||
afl-fuzz -i input -o results -M fuzzer01 -- ./testcase
|
||||
# Start slaves (usually 1 per core available)
|
||||
afl-fuzz -i input -o results -S fuzzer02 -- ./testcase
|
||||
afl-fuzz -i input -o results -S fuzzer03 -- ./testcase
|
||||
# add more if needed
|
||||
```
|
||||
|
||||
When compiled with `-d:afl` the resulting application can also be run
|
||||
manually by providing it input data, e.g.:
|
||||
```sh
|
||||
./testcase < testfile
|
||||
```
|
||||
|
||||
During debugging you might not want the testcase to generate a segmentation
|
||||
fault on exceptions. You can do this by rebuilding the test without the `-d:afl`
|
||||
flag. Changing to `-d:debug` will also help but might also change the
|
||||
behaviour.
|
||||
|
||||
### Manually with libFuzzer
|
||||
#### Compiling
|
||||
```sh
|
||||
nim c -d:libFuzzer -d:release -d:chronicles_log_level=fatal --noMain --cc=clang --passC="-fsanitize=fuzzer" --passL="-fsanitize=fuzzer" testcase.nim
|
||||
```
|
||||
|
||||
The `libFuzzer` define is specifically required for the `init` and `test`
|
||||
templates.
|
||||
|
||||
You typically want to fuzz in `-d:release` and probably also want to lower down
|
||||
the logging. But this is not strictly necessary.
|
||||
|
||||
There is also a nimscript task in `config.nims` for compiling:
|
||||
```
|
||||
nim c build_libFuzzer testcase.nim
|
||||
```
|
||||
|
||||
#### Starting the Fuzzer
|
||||
Starting the fuzzer is as simple as running the compiled program:
|
||||
```sh
|
||||
./testcase corpus_dir -runs=1000000
|
||||
```
|
||||
|
||||
To see the available options:
|
||||
```sh
|
||||
./testcase test=1
|
||||
```
|
||||
|
||||
Parallel fuzzing on 8 cores:
|
||||
```sh
|
||||
./fuzz-libfuzzer -jobs=8 -workers=8
|
||||
```
|
||||
|
||||
You can also use the application to verify a specific test case:
|
||||
```sh
|
||||
./testcase input_file
|
||||
```
|
||||
|
||||
## Additional notes
|
||||
The `init` template, when used with **afl**, is only cosmetic. It will be
|
||||
run before each test block, compared to libFuzzer, where it will be run only
|
||||
once.
|
||||
|
||||
In case of using afl with `alf-clang-fast` you can make use of `aflInit()` proc
|
||||
and `aflLoop()` template.
|
||||
|
||||
`aflInit()` will allow using what is called deferred instrumentation. Basically,
|
||||
the forking of the process will only happen after this call, where normally it
|
||||
is done right before `main()`.
|
||||
|
||||
`aflLoop:` will allow for (experimental) persistant mode. It will run the test
|
||||
in loop (1000 iterations) with different payloads. This is more comparable with
|
||||
libFuzzer.
|
||||
|
||||
These calls are enabled with `-d:clangfast`, and have to be manually added.
|
||||
They are currently not part of the `test` or `init` templates.
|
||||
|
|
|
|||
|
|
@ -1,4 +1,6 @@
|
|||
import chronicles, eth/rlp, ../fuzztest
|
||||
import
|
||||
testutils/fuzzing, chronicles,
|
||||
eth/rlp
|
||||
|
||||
test:
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
import
|
||||
chronos, eth/p2p, eth/p2p/rlpx, eth/p2p/private/p2p_types,
|
||||
testutils/fuzzing, chronos,
|
||||
eth/p2p, eth/p2p/rlpx, eth/p2p/private/p2p_types,
|
||||
eth/p2p/rlpx_protocols/[whisper_protocol, eth_protocol],
|
||||
../fuzztest, ../p2p/p2p_test_helper
|
||||
../p2p/p2p_test_helper
|
||||
|
||||
var
|
||||
node1: EthereumNode
|
||||
|
|
|
|||
Loading…
Reference in New Issue