qa: Add test/fuzz/test_runner.py

2 files changed, 134 insertions, 0 deletions

diff --git a/test/config.ini.in b/test/config.ini.in
index 28abee2a3d..6b7ef70659 100644
--- a/test/config.ini.in
+++ b/test/config.ini.in
@@ -16,4 +16,5 @@ RPCAUTH=@abs_top_srcdir@/share/rpcauth/rpcauth.py
 @ENABLE_WALLET_TRUE@ENABLE_WALLET=true
 @BUILD_BITCOIN_CLI_TRUE@ENABLE_CLI=true
 @BUILD_BITCOIND_TRUE@ENABLE_BITCOIND=true
+@ENABLE_FUZZ_TRUE@ENABLE_FUZZ=true
 @ENABLE_ZMQ_TRUE@ENABLE_ZMQ=true
diff --git a/test/fuzz/test_runner.py b/test/fuzz/test_runner.py
new file mode 100755
index 0000000000..eb4fe78cd4
--- /dev/null
+++ b/test/fuzz/test_runner.py
@@ -0,0 +1,133 @@
+#!/usr/bin/env python3
+# Copyright (c) 2019 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+"""Run fuzz test targets.
+"""
+
+import argparse
+import configparser
+import os
+import sys
+import subprocess
+import logging
+
+
+def main():
+    parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    parser.add_argument(
+        "-l",
+        "--loglevel",
+        dest="loglevel",
+        default="INFO",
+        help="log events at this level and higher to the console. Can be set to DEBUG, INFO, WARNING, ERROR or CRITICAL. Passing --loglevel DEBUG will output all logs to console.",
+    )
+    parser.add_argument(
+        '--export_coverage',
+        action='store_true',
+        help='If true, export coverage information to files in the seed corpus',
+    )
+    parser.add_argument(
+        'seed_dir',
+        help='The seed corpus to run on (must contain subfolders for each fuzz target).',
+    )
+    parser.add_argument(
+        'target',
+        nargs='*',
+        help='The target(s) to run. Default is to run all targets.',
+    )
+
+    args = parser.parse_args()
+
+    # Set up logging
+    logging.basicConfig(
+        format='%(message)s',
+        level=int(args.loglevel) if args.loglevel.isdigit() else args.loglevel.upper(),
+    )
+
+    # Read config generated by configure.
+    config = configparser.ConfigParser()
+    configfile = os.path.abspath(os.path.dirname(__file__)) + "/../config.ini"
+    config.read_file(open(configfile, encoding="utf8"))
+
+    if not config["components"].getboolean("ENABLE_FUZZ"):
+        logging.error("Must have fuzz targets built")
+        sys.exit(1)
+
+    # Build list of tests
+    test_list_all = parse_test_list(makefile=os.path.join(config["environment"]["SRCDIR"], 'src', 'Makefile.test.include'))
+
+    if not test_list_all:
+        logging.error("No fuzz targets found")
+        sys.exit(1)
+
+    logging.info("Fuzz targets found: {}".format(test_list_all))
+
+    args.target = args.target or test_list_all  # By default run all
+    test_list_error = list(set(args.target).difference(set(test_list_all)))
+    if test_list_error:
+        logging.error("Unknown fuzz targets selected: {}".format(test_list_error))
+    test_list_selection = list(set(test_list_all).intersection(set(args.target)))
+    if not test_list_selection:
+        logging.error("No fuzz targets selected")
+    logging.info("Fuzz targets selected: {}".format(test_list_selection))
+
+    help_output = subprocess.run(
+        args=[
+            os.path.join(config["environment"]["BUILDDIR"], 'src', 'test', 'fuzz', test_list_selection[0]),
+            '-help=1',
+        ],
+        check=True,
+        stderr=subprocess.PIPE,
+        universal_newlines=True,
+    ).stderr
+    if "libFuzzer" not in help_output:
+        logging.error("Must be built with libFuzzer")
+        sys.exit(1)
+
+    run_once(
+        corpus=args.seed_dir,
+        test_list=test_list_selection,
+        build_dir=config["environment"]["BUILDDIR"],
+        export_coverage=args.export_coverage,
+    )
+
+
+def run_once(*, corpus, test_list, build_dir, export_coverage):
+    for t in test_list:
+        args = [
+            os.path.join(build_dir, 'src', 'test', 'fuzz', t),
+            '-runs=1',
+            os.path.join(corpus, t),
+        ]
+        logging.debug('Run {} with args {}'.format(t, args))
+        output = subprocess.run(args, check=True, stderr=subprocess.PIPE, universal_newlines=True).stderr
+        logging.debug('Output: {}'.format(output))
+        if not export_coverage:
+            continue
+        for l in output.splitlines():
+            if 'INITED' in l:
+                with open(os.path.join(corpus, t + '_coverage'), 'w', encoding='utf-8') as cov_file:
+                    cov_file.write(l)
+                    break
+
+
+def parse_test_list(makefile):
+    with open(makefile, encoding='utf-8') as makefile_test:
+        test_list_all = []
+        read_targets = False
+        for line in makefile_test.readlines():
+            line = line.strip().replace('test/fuzz/', '').replace(' \\', '')
+            if read_targets:
+                if not line:
+                    break
+                test_list_all.append(line)
+                continue
+
+            if line == 'FUZZ_TARGETS =':
+                read_targets = True
+    return test_list_all
+
+
+if __name__ == '__main__':
+    main()