status-im/nim-eth
2
0
Fork 0
mirror of https://github.com/status-im/nim-eth.git synced 2025-01-27 06:46:32 +00:00
Code Issues Projects Releases Wiki Activity

Update fuzzing tests now that the helpers have moved to ntu

Browse Source
This commit is contained in:
kdeme 2020-06-12 16:19:58 +02:00 committed by zah
parent 225a9ad41c
commit 4d0a7a46ba
9 changed files with 31 additions and 453 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
eth.nimble
View File

@ -15,7 +15,8 @@ requires "nim >= 1.2.0",
"nat_traversal", "nat_traversal",
"metrics", "metrics",
"sqlite3_abi", "sqlite3_abi",
"confutils" "confutils",
"testutils"
proc runTest(path: string) = proc runTest(path: string) =
echo "\nRunning: ", path echo "\nRunning: ", path

31
tests/fuzzing/config.nims
View File

@ -1,31 +0,0 @@
proc aflSwitches() =
switch("define", "afl")
switch("define", "noSignalHandler")
switch("cc", "gcc")
switch("gcc.exe", "afl-gcc")
switch("gcc.linkerexe", "afl-gcc")
switch("out", "fuzz-afl")
proc libFuzzerSwitches() =
switch("define", "libFuzzer")
switch("noMain", "")
switch("cc", "clang")
switch("passC", "-fsanitize=fuzzer,address")
switch("passL", "-fsanitize=fuzzer,address")
switch("out", "fuzz-libfuzzer")
proc generalSwitches() =
switch("verbosity", "0")
switch("hints", "off")
switch("warnings", "off")
switch("define", "chronicles_log_level:fatal")
task build_afl, "Build for afl fuzzing":
aflSwitches()
generalSwitches()
setCommand("c")
task build_libfuzzer, "Build for libFuzzer fuzzing":
libFuzzerSwitches()
generalSwitches()
setCommand("c")

13
tests/fuzzing/discovery/fuzz.nim
View File

@ -1,6 +1,7 @@
import import
chronicles, eth/p2p/[discovery, enode], eth/[keys, rlp], testutils/fuzzing, chronicles,
../../p2p/p2p_test_helper, ../fuzztest eth/p2p/[discovery, enode], eth/[keys, rlp],
../../p2p/p2p_test_helper
const DefaultListeningPort = 30303 const DefaultListeningPort = 30303
var targetNode: DiscoveryProtocol var targetNode: DiscoveryProtocol
@ -8,7 +9,7 @@ var targetNode: DiscoveryProtocol
init: init:
# Set up a discovery node, this is the node we target when fuzzing # Set up a discovery node, this is the node we target when fuzzing
var var
targetNodeKey = PrivateKey.fromRaw("a2b50376a79b1a8c8a3296485572bdfbf54708bb46d3c25d73d2723aaaf6a617")[] targetNodeKey = PrivateKey.fromHex("a2b50376a79b1a8c8a3296485572bdfbf54708bb46d3c25d73d2723aaaf6a617")[]
targetNodeAddr = localAddress(DefaultListeningPort) targetNodeAddr = localAddress(DefaultListeningPort)
targetNode = newDiscoveryProtocol(targetNodeKey, targetNodeAddr, @[]) targetNode = newDiscoveryProtocol(targetNodeKey, targetNodeAddr, @[])
# Create the transport as else replies on the messages send will fail. # Create the transport as else replies on the messages send will fail.
@ -22,7 +23,7 @@ test:
# Sending raw payload is possible but won't find us much. We need a hash and # Sending raw payload is possible but won't find us much. We need a hash and
# a signature, and without it there is a big chance it will always result in # a signature, and without it there is a big chance it will always result in
# "Wrong msg mac from" error. # "Wrong msg mac from" error.
let nodeKey = PrivateKey.fromRaw("a2b50376a79b1a8c8a3296485572bdfbf54708bb46d3c25d73d2723aaaf6a618")[] let nodeKey = PrivateKey.fromHex("a2b50376a79b1a8c8a3296485572bdfbf54708bb46d3c25d73d2723aaaf6a618")[]
msg = packData(payload, nodeKey) msg = packData(payload, nodeKey)
address = localAddress(DefaultListeningPort + 1) address = localAddress(DefaultListeningPort + 1)
@ -30,5 +31,7 @@ test:
targetNode.receive(address, msg) targetNode.receive(address, msg)
# These errors are also catched in `processClient` in discovery.nim # These errors are also catched in `processClient` in discovery.nim
# TODO: move them a layer down in discovery so we can do a cleaner test there? # TODO: move them a layer down in discovery so we can do a cleaner test there?
except RlpError, DiscProtocolError as e: except RlpError as e:
debug "Receive failed", err = e.msg
except DiscProtocolError as e:
debug "Receive failed", err = e.msg debug "Receive failed", err = e.msg

28
tests/fuzzing/fuzz.nims
View File

@ -1,28 +0,0 @@
import ./fuzz_helpers
# TODO: get this some nice cmd line options when confutils works for nimscript
# or if we want to put this in a nim application instead of script
if paramCount() < 3:
echo "Usage: nim fuzz.nims FUZZER TARGET"
echo "Fuzzer options are afl or libFuzzer"
quit 1
let
fuzzer = paramStr(2)
targetPath = paramStr(3)
if not fileExists(targetPath):
echo "Target file does not exist"
quit 1
case fuzzer
of "afl":
runFuzzer(targetPath, afl)
of "libFuzzer":
runFuzzer(targetPath, libFuzzer)
else:
echo "Invalid fuzzer option: ", fuzzer
echo "Fuzzer options are afl or libFuzzer"
quit 1

94
tests/fuzzing/fuzz_helpers.nim
View File

@ -1,94 +0,0 @@
import strformat, ospaths
# Dependencies:
# - afl fuzzing: afl and gcc or clang/llvm
# - libFuzzer fuzzing: libFuzzer and clang/llvm
# - in afl experimental modes clang/llvm is also required
# TODO:
# - switch clang / gcc option for afl
# - afl init and persistent modes
# - parallel fuzzing options
# - custom generate test cases from this script?
# - rerun testcases option (or create tests from failed cases)
# - currently not cross platform
# - ...
const
aflGcc = "--cc=gcc " &
"--gcc.exe=afl-gcc " &
"--gcc.linkerexe=afl-gcc"
aflClang = "--cc=clang " &
"--clang.exe=afl-clang " &
"--clang.linkerexe=afl-clang"
aflClangFast = "--cc=clang " &
"--clang.exe=afl-clang-fast " &
"--clang.linkerexe=afl-clang-fast " &
"-d:clangfast"
libFuzzerClang = "--cc=clang " &
"--passC='-fsanitize=fuzzer,address' " &
"--passL='-fsanitize=fuzzer,address'"
# Can also test in debug mode obviously, but might be slower
# Can turn on more logging, in case of libFuzzer it will get very verbose though
defaultFlags = "-d:release -d:chronicles_log_level=fatal " &
"--hints:off --warnings:off --verbosity:0"
type
Fuzzer* = enum
afl,
libFuzzer
Compiler* = enum
gcc = aflGcc,
clang = aflClang,
clangFast = aflClangFast
proc aflCompile*(target: string, c: Compiler) =
let aflOptions = &"-d:afl -d:noSignalHandler {$c}"
let compileCmd = &"nim c {defaultFlags} {aflOptions} {target.quoteShell()}"
exec compileCmd
proc aflExec*(target: string, inputDir: string, resultsDir: string,
cleanStart = false) =
let exe = target.addFileExt(ExeExt)
if not dirExists(inputDir):
# create a input dir with one 0 file for afl
mkDir(inputDir)
# TODO: improve
withDir inputDir: exec "echo '0' > test"
var fuzzCmd: string
# if there is an output dir already, continue fuzzing from previous run
if (not dirExists(resultsDir)) or cleanStart:
fuzzCmd = &"afl-fuzz -i {inputDir.quoteShell()} -o {resultsDir.quoteShell()} -M fuzzer01 -- {exe.quoteShell()}"
else:
fuzzCmd = &"afl-fuzz -i - -o {resultsDir.quoteShell()} -M fuzzer01 -- {exe.quoteShell()}"
exec fuzzCmd
proc libFuzzerCompile*(target: string) =
let libFuzzerOptions = &"-d:libFuzzer --noMain {libFuzzerClang}"
let compileCmd = &"nim c {defaultFlags} {libFuzzerOptions} {target.quoteShell()}"
exec compileCmd
proc libFuzzerExec*(target: string, corpusDir: string) =
let exe = target.addFileExt(ExeExt)
if not dirExists(corpusDir):
# libFuzzer is OK when starting with empty corpus dir
mkDir(corpusDir)
exec &"{exe.quoteShell()} {corpusDir.quoteShell()}"
proc runFuzzer*(targetPath: string, fuzzer: Fuzzer) =
let (path, target, ext) = splitFile(targetPath)
case fuzzer
of afl:
aflCompile(targetPath, gcc)
aflExec(path / target, path / "input", path / "results")
of libFuzzer:
libFuzzerCompile(targetPath)
# Note: Lets not mix afl input with libFuzzer corpus default. This can have
# consequences on speed for afl. Better to look into merging afl results &
# libFuzzer corpus.
libFuzzerExec(path / target, path / "corpus")

96
tests/fuzzing/fuzztest.nim
View File

@ -1,96 +0,0 @@
import streams, posix, strutils, chronicles, macros, stew/ranges/ptr_arith
template fuzz(body) =
# For code we want to fuzz, SIGSEGV is needed on unwanted exceptions.
# However, this is only needed when fuzzing with afl.
when defined(afl):
try:
body
except Exception as e:
error "Fuzzer input created exception", exception=e.name, trace=e.repr,
msg=e.msg
discard kill(getpid(), SIGSEGV)
else:
body
proc readStdin(): seq[byte] =
# Read input from stdin (fastest for AFL)
let s = newFileStream(stdin)
if s.isNil:
error "Error opening stdin"
discard kill(getpid(), SIGSEGV)
# We use binary files as with hex we can get lots of "not hex" failures
var input = s.readAll()
s.close()
# Remove newline if it is there
input.removeSuffix
result = cast[seq[byte]](input)
proc NimMain() {.importc: "NimMain".}
# The default init, gets redefined when init template is used.
template initImpl(): untyped =
when not defined(libFuzzer):
discard
else:
proc fuzzerInit(): cint {.exportc: "LLVMFuzzerInitialize".} =
NimMain()
return 0
template init*(body: untyped) =
## Init block to do any initialisation for the fuzzing test.
##
## For AFL this is currently only cosmetic and will be run each time, before
## the test block.
##
## For libFuzzer this will only be run once. So only put data which is
## stateless or make sure everything gets properply reset for each new run in
## the test block.
when not defined(libFuzzer):
template initImpl(): untyped = fuzz: `body`
else:
template initImpl() =
proc fuzzerInit(): cint {.exportc: "LLVMFuzzerInitialize".} =
NimMain()
`body`
return 0
template test*(body: untyped): untyped =
## Test block to do the actual test that will be fuzzed in a loop.
##
## Within this test block there is access to the payload OpenArray which
## contains the payload provided by the fuzzer.
mixin initImpl
initImpl()
when not defined(libFuzzer):
var payload {.inject.} = readStdin()
fuzz: `body`
else:
proc fuzzerCall(data: ptr byte, len: csize):
cint {.exportc: "LLVMFuzzerTestOneInput".} =
template payload(): auto =
makeOpenArray(data, len)
`body`
when defined(clangfast):
## Can be used for deferred instrumentation.
## Should be placed on a suitable location in the code where the delayed
## cloning can take place (e.g. NOT after creation of threads)
proc aflInit*() {.importc: "__AFL_INIT", noDecl.}
## Can be used for persistent mode.
## Should be used as value for controlling a loop around a test case.
## Test case should be able to handle repeated inputs. No repeated fork() will
## be done.
# TODO: Lets use this in the test block when afl-clang-fast is used?
proc aflLoopImpl(count: cuint): cint {.importc: "__AFL_LOOP", noDecl.}
template aflLoop*(body: untyped): untyped =
while aflLoopImpl(1000) != 0:
`body`
else:
proc aflInit*() = discard
template aflLoop*(body: untyped): untyped = `body`

210
tests/fuzzing/readme.md
View File

@ -1,204 +1,24 @@
# Fuzzing # Fuzzing Tests
## tldr: The fuzzing tests use the fuzzing templates from `nim-testutils`.
* [Install afl](#Install-afl).
* Create a testcase.
* Run: `nim fuzz.nims afl testfolder/testcase.nim`
Or For more details see [the fuzzing readme of nim-testutils](https://github.com/status-im/nim-testutils/tree/master/testutils/fuzzing).
* [Install libFuzzer](#Install-libFuzzer) (comes with LLVM). ## Prerequisites
* Create a testcase. As [explained](https://github.com/status-im/nim-testutils/tree/master/testutils/fuzzing#supported-fuzzers)
* Run: `nim fuzz.nims libFuzzer testfolder/testcase.nim` in `nim-testutils` fuzzing readme, first install the fuzzer you want to run.
## Fuzzing Helpers Next install `nim-testutils` its `ntu` application.
There are two convenience templates which will help you set up a quick fuzzing
test.
These are the mandatory `test` block and the optional `init` block. E.g. by running the `nim-testutils` nimble install:
Example usage:
```nim
test:
var rlp = rlpFromBytes(payload)
discard rlp.inspect()
```
Any unhandled `Exception` will result in a failure of the testcase. If certain
`Exception`s are to be allowed to occur within the test, they should be caught.
E.g.:
```nim
test:
try:
var rlp = rlpFromBytes(payload)
discard rlp.inspect()
except RlpError as e:
debug "Inspect failed", err = e.msg
```
## Supported Fuzzers
The two templates can prepare the code for both
[afl](http://lcamtuf.coredump.cx/afl/) and
[libFuzzer](http://llvm.org/docs/LibFuzzer.html).
You will need to install first the fuzzer you want to use.
### Install afl
```sh ```sh
# Ubuntu / Debian nimble install nim-testutils
sudo apt-get install afl
# Fedora
dnf install american-fuzzy-lop
# for usage with clang & clang-fast you will have to install
# american-fuzzy-lop-clang or american-fuzzy-lop-clang-fast
# Arch Linux
pacman -S afl
# NixOS
nix-env -i afl
``` ```
### Install libFuzzer ## How to run
To start fuzzing a testcase run following command:
LibFuzzer is part of llvm and will be installed together with llvm-libs in
recent versions. Installing clang should install llvm-libs.
```sh ```sh
# Ubuntu / Debian # For libFuzzer
sudo apt-get install clang ntu fuzz --fuzzer:libFuzzer rlp/rlp_inspect
# For afl
# Fedora ntu fuzz --fuzzer:afl rlp/rlp_inspect
dnf install clang
# Arch Linux
pacman -S clang
# NixOS
nix-env -iA nixos.clang_7 nixos.llvm_7
``` ```
## Compiling & Starting the Fuzzer
### Scripted helper
There is a nimscript helper to compile & start the fuzzer:
```sh
# for afl
nim fuzz.nims afl testcase.nim
# for libFuzzer
nim fuzz.nims libFuzzer testcase.nim
```
### Manually with afl
#### Compiling
With gcc:
```sh
nim c -d:afl -d:release -d:chronicles_log_level=fatal -d:noSignalHandler --cc=gcc --gcc.exe=afl-gcc --gcc.linkerexe=afl-gcc testcase.nim
```
The `afl` define is specifically required for the `init` and `test`
templates.
You typically want to fuzz in `-d:release` and probably also want to lower down
the logging. But this is not strictly necessary.
There is also a nimscript task in `config.nims` for this:
```
nim c build_afl testcase.nim
```
With clang:
```sh
# afl-clang
nim c -d:afl -d:noSignalHandler --cc=clang --clang.exe=afl-clang --clang.linkerexe=afl-clang ftestcase.nim
# afl-clang-fast
nim c -d:afl -d:noSignalHandler --cc=clang --clang.exe=afl-clang-fast --clang.linkerexe=afl-clang-fast testcase.nim
```
#### Starting the Fuzzer
To start the fuzzer:
```sh
afl-fuzz -i input -o results -- ./testcase
```
To rerun it without losing previous results/corpus:
```sh
afl-fuzz -i - -o results -- ./testcase
```
To run several parallel fuzzing sessions:
```sh
# Start master fuzzer
afl-fuzz -i input -o results -M fuzzer01 -- ./testcase
# Start slaves (usually 1 per core available)
afl-fuzz -i input -o results -S fuzzer02 -- ./testcase
afl-fuzz -i input -o results -S fuzzer03 -- ./testcase
# add more if needed
```
When compiled with `-d:afl` the resulting application can also be run
manually by providing it input data, e.g.:
```sh
./testcase < testfile
```
During debugging you might not want the testcase to generate a segmentation
fault on exceptions. You can do this by rebuilding the test without the `-d:afl`
flag. Changing to `-d:debug` will also help but might also change the
behaviour.
### Manually with libFuzzer
#### Compiling
```sh
nim c -d:libFuzzer -d:release -d:chronicles_log_level=fatal --noMain --cc=clang --passC="-fsanitize=fuzzer" --passL="-fsanitize=fuzzer" testcase.nim
```
The `libFuzzer` define is specifically required for the `init` and `test`
templates.
You typically want to fuzz in `-d:release` and probably also want to lower down
the logging. But this is not strictly necessary.
There is also a nimscript task in `config.nims` for compiling:
```
nim c build_libFuzzer testcase.nim
```
#### Starting the Fuzzer
Starting the fuzzer is as simple as running the compiled program:
```sh
./testcase corpus_dir -runs=1000000
```
To see the available options:
```sh
./testcase test=1
```
Parallel fuzzing on 8 cores:
```sh
./fuzz-libfuzzer -jobs=8 -workers=8
```
You can also use the application to verify a specific test case:
```sh
./testcase input_file
```
## Additional notes
The `init` template, when used with **afl**, is only cosmetic. It will be
run before each test block, compared to libFuzzer, where it will be run only
once.
In case of using afl with `alf-clang-fast` you can make use of `aflInit()` proc
and `aflLoop()` template.
`aflInit()` will allow using what is called deferred instrumentation. Basically,
the forking of the process will only happen after this call, where normally it
is done right before `main()`.
`aflLoop:` will allow for (experimental) persistant mode. It will run the test
in loop (1000 iterations) with different payloads. This is more comparable with
libFuzzer.
These calls are enabled with `-d:clangfast`, and have to be manually added.
They are currently not part of the `test` or `init` templates.

4
tests/fuzzing/rlp/rlp_inspect.nim
View File

@ -1,4 +1,6 @@
import chronicles, eth/rlp, ../fuzztest import
testutils/fuzzing, chronicles,
eth/rlp
test: test:
try: try:

5
tests/fuzzing/rlpx/thunk.nim
View File

@ -1,7 +1,8 @@
import import
chronos, eth/p2p, eth/p2p/rlpx, eth/p2p/private/p2p_types, testutils/fuzzing, chronos,
eth/p2p, eth/p2p/rlpx, eth/p2p/private/p2p_types,
eth/p2p/rlpx_protocols/[whisper_protocol, eth_protocol], eth/p2p/rlpx_protocols/[whisper_protocol, eth_protocol],
../fuzztest, ../p2p/p2p_test_helper ../p2p/p2p_test_helper
var var
node1: EthereumNode node1: EthereumNode