aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMarcoFalke <falke.marco@gmail.com>2019-01-16 11:49:01 -0500
committerMarcoFalke <falke.marco@gmail.com>2019-02-13 17:12:28 -0500
commitfa7ca8ef58bf3e3b91d1f5a67fa42008e63b1f7b (patch)
tree2bb0b1b63931c12593aafded53b4021fe566ab32
parentad039aa0d3e8a831559434022b1da1de4d72a847 (diff)
qa: Add test/fuzz/test_runner.py
-rw-r--r--.travis.yml15
-rwxr-xr-x.travis/test_04_install.sh5
-rwxr-xr-x.travis/test_06_script_b.sh6
-rw-r--r--Makefile.am6
-rw-r--r--doc/fuzzing.md57
-rw-r--r--src/Makefile.test.include3
-rw-r--r--test/config.ini.in1
-rwxr-xr-xtest/fuzz/test_runner.py133
8 files changed, 199 insertions, 27 deletions
diff --git a/.travis.yml b/.travis.yml
index d5086b084a..d2d462306a 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -15,6 +15,7 @@ env:
- MAKEJOBS=-j3
- RUN_UNIT_TESTS=true
- RUN_FUNCTIONAL_TESTS=true
+ - RUN_FUZZ_TESTS=false
- DOCKER_NAME_TAG=ubuntu:18.04
- BOOST_TEST_RANDOM=1$TRAVIS_BUILD_ID
- CCACHE_SIZE=100M
@@ -100,7 +101,7 @@ jobs:
PACKAGES="python3-zmq qtbase5-dev qttools5-dev-tools protobuf-compiler libdbus-1-dev libharfbuzz-dev libprotobuf-dev"
DEP_OPTS="NO_QT=1 NO_UPNP=1 DEBUG=1 ALLOW_HOST_PACKAGES=1"
GOAL="install"
- BITCOIN_CONFIG="--enable-zmq --with-gui=qt5 --enable-fuzz --enable-glibc-back-compat --enable-reduce-exports --enable-debug CXXFLAGS=\"-g0 -O2\""
+ BITCOIN_CONFIG="--enable-zmq --with-gui=qt5 --enable-glibc-back-compat --enable-reduce-exports --enable-debug CXXFLAGS=\"-g0 -O2\""
- stage: test
name: 'x86_64 Linux [GOAL: install] [trusty] [no functional tests, no depends, only system libs]'
@@ -133,6 +134,18 @@ jobs:
BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=qt5 CPPFLAGS=-DDEBUG_LOCKORDER --with-sanitizers=address,integer,undefined CC=clang CXX=clang++"
- stage: test
+ name: 'x86_64 Linux [GOAL: install] [bionic] [no depends, only system libs, sanitizers: fuzzer,address]'
+ env: >-
+ HOST=x86_64-unknown-linux-gnu
+ PACKAGES="clang llvm python3 libssl1.0-dev libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev"
+ NO_DEPENDS=1
+ RUN_UNIT_TESTS=false
+ RUN_FUNCTIONAL_TESTS=false
+ RUN_FUZZ_TESTS=true
+ GOAL="install"
+ BITCOIN_CONFIG="--disable-wallet --disable-bench --with-utils=no --with-daemon=no --with-libs=no --with-gui=no --enable-fuzz --with-sanitizers=fuzzer,address CC=clang CXX=clang++"
+
+ - stage: test
name: 'x86_64 Linux [GOAL: install] [bionic] [no wallet]'
env: >-
HOST=x86_64-unknown-linux-gnu
diff --git a/.travis/test_04_install.sh b/.travis/test_04_install.sh
index a111387f10..8055bbdd19 100755
--- a/.travis/test_04_install.sh
+++ b/.travis/test_04_install.sh
@@ -7,6 +7,11 @@
export LC_ALL=C.UTF-8
travis_retry docker pull "$DOCKER_NAME_TAG"
+
+export DIR_FUZZ_IN=${TRAVIS_BUILD_DIR}/qa-assets
+git clone https://github.com/bitcoin-core/qa-assets ${DIR_FUZZ_IN}
+export DIR_FUZZ_IN=${DIR_FUZZ_IN}/fuzz_seed_corpus/
+
mkdir -p "${TRAVIS_BUILD_DIR}/sanitizer-output/"
export ASAN_OPTIONS=""
export LSAN_OPTIONS="suppressions=${TRAVIS_BUILD_DIR}/test/sanitizer_suppressions/lsan"
diff --git a/.travis/test_06_script_b.sh b/.travis/test_06_script_b.sh
index b6b5925be8..e13abfd52f 100755
--- a/.travis/test_06_script_b.sh
+++ b/.travis/test_06_script_b.sh
@@ -19,3 +19,9 @@ if [ "$RUN_FUNCTIONAL_TESTS" = "true" ]; then
DOCKER_EXEC test/functional/test_runner.py --ci --combinedlogslen=4000 --coverage --quiet --failfast
END_FOLD
fi
+
+if [ "$RUN_FUZZ_TESTS" = "true" ]; then
+ BEGIN_FOLD fuzz-tests
+ DOCKER_EXEC test/fuzz/test_runner.py -l DEBUG ${DIR_FUZZ_IN}
+ END_FOLD
+fi
diff --git a/Makefile.am b/Makefile.am
index 9a6e15f0dd..f4e4dc98be 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -220,7 +220,11 @@ endif
dist_noinst_SCRIPTS = autogen.sh
-EXTRA_DIST = $(DIST_SHARE) test/functional/test_runner.py test/functional $(DIST_CONTRIB) $(DIST_DOCS) $(WINDOWS_PACKAGING) $(OSX_PACKAGING) $(BIN_CHECKS)
+EXTRA_DIST = $(DIST_SHARE) $(DIST_CONTRIB) $(DIST_DOCS) $(WINDOWS_PACKAGING) $(OSX_PACKAGING) $(BIN_CHECKS)
+
+EXTRA_DIST += \
+ test/functional \
+ test/fuzz
EXTRA_DIST += \
test/util/bitcoin-util-test.py \
diff --git a/doc/fuzzing.md b/doc/fuzzing.md
index 08b73d3b3c..f9221dde5b 100644
--- a/doc/fuzzing.md
+++ b/doc/fuzzing.md
@@ -5,6 +5,29 @@ A special test harness in `src/test/fuzz/` is provided for each fuzz target to
provide an easy entry point for fuzzers and the like. In this document we'll
describe how to use it with AFL and libFuzzer.
+## Preparing fuzzing
+
+AFL needs an input directory with examples, and an output directory where it
+will place examples that it found. These can be anywhere in the file system,
+we'll define environment variables to make it easy to reference them.
+
+libFuzzer will use the input directory as output directory.
+
+Extract the example seeds (or other starting inputs) into the inputs
+directory before starting fuzzing.
+
+```
+git clone https://github.com/bitcoin-core/qa-assets
+export DIR_FUZZ_IN=$PWD/qa-assets/fuzz_seed_corpus
+```
+
+Only for AFL:
+
+```
+mkdir outputs
+export AFLOUT=$PWD/outputs
+```
+
## AFL
### Building AFL
@@ -23,7 +46,7 @@ export AFLPATH=$PWD
To build Bitcoin Core using AFL instrumentation (this assumes that the
`AFLPATH` was set as above):
```
-./configure --disable-ccache --disable-shared --enable-tests --enable-fuzz CC=${AFLPATH}/afl-gcc CXX=${AFLPATH}/afl-g++
+./configure --disable-ccache --disable-shared --enable-tests --enable-fuzz --disable-wallet --disable-bench --with-utils=no --with-daemon=no --with-libs=no --with-gui=no CC=${AFLPATH}/afl-gcc CXX=${AFLPATH}/afl-g++
export AFL_HARDEN=1
cd src/
make
@@ -39,31 +62,14 @@ binary will be instrumented in such a way that the AFL
features "persistent mode" and "deferred forkserver" can be used. See
https://github.com/mcarpenter/afl/tree/master/llvm_mode for details.
-### Preparing fuzzing
-
-AFL needs an input directory with examples, and an output directory where it
-will place examples that it found. These can be anywhere in the file system,
-we'll define environment variables to make it easy to reference them.
-
-```
-mkdir inputs
-AFLIN=$PWD/inputs
-mkdir outputs
-AFLOUT=$PWD/outputs
-```
-
-Example inputs are available from:
-
-- https://download.visucore.com/bitcoin/bitcoin_fuzzy_in.tar.xz
-- http://strateman.ninja/fuzzing.tar.xz
-
-Extract these (or other starting inputs) into the `inputs` directory before starting fuzzing.
-
### Fuzzing
To start the actual fuzzing use:
+
```
-$AFLPATH/afl-fuzz -i ${AFLIN} -o ${AFLOUT} -m52 -- test/fuzz/fuzz_target_foo
+export FUZZ_TARGET=fuzz_target_foo # Pick a fuzz_target
+mkdir ${AFLOUT}/${FUZZ_TARGET}
+$AFLPATH/afl-fuzz -i ${DIR_FUZZ_IN}/${FUZZ_TARGET} -o ${AFLOUT}/${FUZZ_TARGET} -m52 -- test/fuzz/${FUZZ_TARGET}
```
You may have to change a few kernel parameters to test optimally - `afl-fuzz`
@@ -74,10 +80,10 @@ will print an error and suggestion if so.
A recent version of `clang`, the address sanitizer and libFuzzer is needed (all
found in the `compiler-rt` runtime libraries package).
-To build the `test/test_bitcoin_fuzzy` executable run
+To build all fuzz targets with libFuzzer, run
```
-./configure --disable-ccache --enable-fuzz --with-sanitizers=fuzzer,address CC=clang CXX=clang++
+./configure --disable-ccache --disable-wallet --disable-bench --with-utils=no --with-daemon=no --with-libs=no --with-gui=no --enable-fuzz --with-sanitizers=fuzzer,address CC=clang CXX=clang++
make
```
@@ -86,3 +92,6 @@ interchangeably between libFuzzer and AFL.
See https://llvm.org/docs/LibFuzzer.html#running on how to run the libFuzzer
instrumented executable.
+
+Alternatively run the script in `./test/fuzz/test_runner.py` and provide it
+with the `${DIR_FUZZ_IN}` created earlier.
diff --git a/src/Makefile.test.include b/src/Makefile.test.include
index 12112375ea..f40a205a73 100644
--- a/src/Makefile.test.include
+++ b/src/Makefile.test.include
@@ -2,7 +2,6 @@
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
-bin_PROGRAMS += test/test_bitcoin
FUZZ_TARGETS = \
test/fuzz/address_deserialize \
@@ -28,6 +27,8 @@ FUZZ_TARGETS = \
if ENABLE_FUZZ
noinst_PROGRAMS += $(FUZZ_TARGETS:=)
+else
+bin_PROGRAMS += test/test_bitcoin
endif
TEST_SRCDIR = test
diff --git a/test/config.ini.in b/test/config.ini.in
index 28abee2a3d..6b7ef70659 100644
--- a/test/config.ini.in
+++ b/test/config.ini.in
@@ -16,4 +16,5 @@ RPCAUTH=@abs_top_srcdir@/share/rpcauth/rpcauth.py
@ENABLE_WALLET_TRUE@ENABLE_WALLET=true
@BUILD_BITCOIN_CLI_TRUE@ENABLE_CLI=true
@BUILD_BITCOIND_TRUE@ENABLE_BITCOIND=true
+@ENABLE_FUZZ_TRUE@ENABLE_FUZZ=true
@ENABLE_ZMQ_TRUE@ENABLE_ZMQ=true
diff --git a/test/fuzz/test_runner.py b/test/fuzz/test_runner.py
new file mode 100755
index 0000000000..eb4fe78cd4
--- /dev/null
+++ b/test/fuzz/test_runner.py
@@ -0,0 +1,133 @@
+#!/usr/bin/env python3
+# Copyright (c) 2019 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+"""Run fuzz test targets.
+"""
+
+import argparse
+import configparser
+import os
+import sys
+import subprocess
+import logging
+
+
+def main():
+ parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+ parser.add_argument(
+ "-l",
+ "--loglevel",
+ dest="loglevel",
+ default="INFO",
+ help="log events at this level and higher to the console. Can be set to DEBUG, INFO, WARNING, ERROR or CRITICAL. Passing --loglevel DEBUG will output all logs to console.",
+ )
+ parser.add_argument(
+ '--export_coverage',
+ action='store_true',
+ help='If true, export coverage information to files in the seed corpus',
+ )
+ parser.add_argument(
+ 'seed_dir',
+ help='The seed corpus to run on (must contain subfolders for each fuzz target).',
+ )
+ parser.add_argument(
+ 'target',
+ nargs='*',
+ help='The target(s) to run. Default is to run all targets.',
+ )
+
+ args = parser.parse_args()
+
+ # Set up logging
+ logging.basicConfig(
+ format='%(message)s',
+ level=int(args.loglevel) if args.loglevel.isdigit() else args.loglevel.upper(),
+ )
+
+ # Read config generated by configure.
+ config = configparser.ConfigParser()
+ configfile = os.path.abspath(os.path.dirname(__file__)) + "/../config.ini"
+ config.read_file(open(configfile, encoding="utf8"))
+
+ if not config["components"].getboolean("ENABLE_FUZZ"):
+ logging.error("Must have fuzz targets built")
+ sys.exit(1)
+
+ # Build list of tests
+ test_list_all = parse_test_list(makefile=os.path.join(config["environment"]["SRCDIR"], 'src', 'Makefile.test.include'))
+
+ if not test_list_all:
+ logging.error("No fuzz targets found")
+ sys.exit(1)
+
+ logging.info("Fuzz targets found: {}".format(test_list_all))
+
+ args.target = args.target or test_list_all # By default run all
+ test_list_error = list(set(args.target).difference(set(test_list_all)))
+ if test_list_error:
+ logging.error("Unknown fuzz targets selected: {}".format(test_list_error))
+ test_list_selection = list(set(test_list_all).intersection(set(args.target)))
+ if not test_list_selection:
+ logging.error("No fuzz targets selected")
+ logging.info("Fuzz targets selected: {}".format(test_list_selection))
+
+ help_output = subprocess.run(
+ args=[
+ os.path.join(config["environment"]["BUILDDIR"], 'src', 'test', 'fuzz', test_list_selection[0]),
+ '-help=1',
+ ],
+ check=True,
+ stderr=subprocess.PIPE,
+ universal_newlines=True,
+ ).stderr
+ if "libFuzzer" not in help_output:
+ logging.error("Must be built with libFuzzer")
+ sys.exit(1)
+
+ run_once(
+ corpus=args.seed_dir,
+ test_list=test_list_selection,
+ build_dir=config["environment"]["BUILDDIR"],
+ export_coverage=args.export_coverage,
+ )
+
+
+def run_once(*, corpus, test_list, build_dir, export_coverage):
+ for t in test_list:
+ args = [
+ os.path.join(build_dir, 'src', 'test', 'fuzz', t),
+ '-runs=1',
+ os.path.join(corpus, t),
+ ]
+ logging.debug('Run {} with args {}'.format(t, args))
+ output = subprocess.run(args, check=True, stderr=subprocess.PIPE, universal_newlines=True).stderr
+ logging.debug('Output: {}'.format(output))
+ if not export_coverage:
+ continue
+ for l in output.splitlines():
+ if 'INITED' in l:
+ with open(os.path.join(corpus, t + '_coverage'), 'w', encoding='utf-8') as cov_file:
+ cov_file.write(l)
+ break
+
+
+def parse_test_list(makefile):
+ with open(makefile, encoding='utf-8') as makefile_test:
+ test_list_all = []
+ read_targets = False
+ for line in makefile_test.readlines():
+ line = line.strip().replace('test/fuzz/', '').replace(' \\', '')
+ if read_targets:
+ if not line:
+ break
+ test_list_all.append(line)
+ continue
+
+ if line == 'FUZZ_TARGETS =':
+ read_targets = True
+ return test_list_all
+
+
+if __name__ == '__main__':
+ main()