aboutsummaryrefslogtreecommitdiff
path: root/test
diff options
context:
space:
mode:
authorMarcoFalke <falke.marco@gmail.com>2020-12-15 18:59:52 +0100
committerMarcoFalke <falke.marco@gmail.com>2020-12-15 19:00:36 +0100
commit8bb40d5f56c8e46f12786da0b6444cdde3b6f7c8 (patch)
tree6a77bfc259d458c235ad79632c73550b2e2d10ee /test
parenta35a3466efd187a2e443aaa230472c8c22f5cfc3 (diff)
parentfa13e1b0c52738492310b6b421d8e38cb04da5b1 (diff)
Merge #20560: fuzz: Link all targets once
fa13e1b0c52738492310b6b421d8e38cb04da5b1 build: Add option --enable-danger-fuzz-link-all (MarcoFalke) 44444ba759480237172d83f42374c5c29c76eda0 fuzz: Link all targets once (MarcoFalke) Pull request description: Currently the linker is invoked more than 150 times when compiling with `--enable-fuzz`. This is problematic for several reasons: * It wastes disk space north of 20 GB, as all libraries and sanitizers are linked more than 150 times * It wastes CPU time, as the link step can practically not be cached (similar to ccache for object files) * It makes it a blocker to compile the fuzz tests by default for non-fuzz builds #19388, for the aforementioned reasons * The build file is several thousand lines of code, without doing anything meaningful except listing each fuzz target in a highly verbose manner * It makes writing new fuzz tests unnecessarily hard, as build system knowledge is required; Compare that to boost unit tests, which can be added by simply editing an existing cpp file * It encourages fuzz tests that re-use the `buffer` or assume the `buffer` to be concatenations of seeds, which increases complexity of seeds and complexity for the fuzz engine to explore; Thus reducing the effectiveness of the affected fuzz targets Fixes #20088 ACKs for top commit: practicalswift: Tested ACK fa13e1b0c52738492310b6b421d8e38cb04da5b1 sipa: ACK fa13e1b0c52738492310b6b421d8e38cb04da5b1. Reviewed the code changes, and tested the 3 different test_runner.py modes (run once, merge, generate). I also tested building with the new --enable-danger-fuzz-link-all Tree-SHA512: 962ab33269ebd51810924c51266ecc62edd6ddf2fcd9a8c359ed906766f58c3f73c223f8d3cc49f2c60f0053f65e8bdd86ce9c19e673f8c2b3cd676e913f2642
Diffstat (limited to 'test')
-rwxr-xr-xtest/fuzz/test_runner.py65
1 files changed, 39 insertions, 26 deletions
diff --git a/test/fuzz/test_runner.py b/test/fuzz/test_runner.py
index c7895edbcc..3a2cefbe2b 100755
--- a/test/fuzz/test_runner.py
+++ b/test/fuzz/test_runner.py
@@ -83,7 +83,7 @@ def main():
sys.exit(1)
# Build list of tests
- test_list_all = parse_test_list(makefile=os.path.join(config["environment"]["SRCDIR"], 'src', 'Makefile.test.include'))
+ test_list_all = parse_test_list(fuzz_bin=os.path.join(config["environment"]["BUILDDIR"], 'src', 'test', 'fuzz', 'fuzz'))
if not test_list_all:
logging.error("No fuzz targets found")
@@ -126,9 +126,12 @@ def main():
try:
help_output = subprocess.run(
args=[
- os.path.join(config["environment"]["BUILDDIR"], 'src', 'test', 'fuzz', test_list_selection[0]),
+ os.path.join(config["environment"]["BUILDDIR"], 'src', 'test', 'fuzz', 'fuzz'),
'-help=1',
],
+ env={
+ 'FUZZ': test_list_selection[0]
+ },
timeout=20,
check=True,
stderr=subprocess.PIPE,
@@ -177,24 +180,30 @@ def generate_corpus_seeds(*, fuzz_pool, build_dir, seed_dir, targets):
"""
logging.info("Generating corpus seeds to {}".format(seed_dir))
- def job(command):
+ def job(command, t):
logging.debug("Running '{}'\n".format(" ".join(command)))
logging.debug("Command '{}' output:\n'{}'\n".format(
' '.join(command),
- subprocess.run(command, check=True, stderr=subprocess.PIPE,
- universal_newlines=True).stderr
- ))
+ subprocess.run(
+ command,
+ env={
+ 'FUZZ': t
+ },
+ check=True,
+ stderr=subprocess.PIPE,
+ universal_newlines=True,
+ ).stderr))
futures = []
for target in targets:
target_seed_dir = os.path.join(seed_dir, target)
os.makedirs(target_seed_dir, exist_ok=True)
command = [
- os.path.join(build_dir, "src", "test", "fuzz", target),
+ os.path.join(build_dir, 'src', 'test', 'fuzz', 'fuzz'),
"-runs=100000",
target_seed_dir,
]
- futures.append(fuzz_pool.submit(job, command))
+ futures.append(fuzz_pool.submit(job, command, target))
for future in as_completed(futures):
future.result()
@@ -205,7 +214,7 @@ def merge_inputs(*, fuzz_pool, corpus, test_list, build_dir, merge_dir):
jobs = []
for t in test_list:
args = [
- os.path.join(build_dir, 'src', 'test', 'fuzz', t),
+ os.path.join(build_dir, 'src', 'test', 'fuzz', 'fuzz'),
'-merge=1',
'-use_value_profile=1', # Also done by oss-fuzz https://github.com/google/oss-fuzz/issues/1406#issuecomment-387790487
os.path.join(corpus, t),
@@ -216,7 +225,15 @@ def merge_inputs(*, fuzz_pool, corpus, test_list, build_dir, merge_dir):
def job(t, args):
output = 'Run {} with args {}\n'.format(t, " ".join(args))
- output += subprocess.run(args, check=True, stderr=subprocess.PIPE, universal_newlines=True).stderr
+ output += subprocess.run(
+ args,
+ env={
+ 'FUZZ': t
+ },
+ check=True,
+ stderr=subprocess.PIPE,
+ universal_newlines=True,
+ ).stderr
logging.debug(output)
jobs.append(fuzz_pool.submit(job, t, args))
@@ -231,7 +248,7 @@ def run_once(*, fuzz_pool, corpus, test_list, build_dir, use_valgrind):
corpus_path = os.path.join(corpus, t)
os.makedirs(corpus_path, exist_ok=True)
args = [
- os.path.join(build_dir, 'src', 'test', 'fuzz', t),
+ os.path.join(build_dir, 'src', 'test', 'fuzz', 'fuzz'),
'-runs=1',
corpus_path,
]
@@ -240,7 +257,7 @@ def run_once(*, fuzz_pool, corpus, test_list, build_dir, use_valgrind):
def job(t, args):
output = 'Run {} with args {}'.format(t, args)
- result = subprocess.run(args, stderr=subprocess.PIPE, universal_newlines=True)
+ result = subprocess.run(args, env={'FUZZ': t}, stderr=subprocess.PIPE, universal_newlines=True)
output += result.stderr
return output, result
@@ -260,20 +277,16 @@ def run_once(*, fuzz_pool, corpus, test_list, build_dir, use_valgrind):
sys.exit(1)
-def parse_test_list(makefile):
- with open(makefile, encoding='utf-8') as makefile_test:
- test_list_all = []
- read_targets = False
- for line in makefile_test.readlines():
- line = line.strip().replace('test/fuzz/', '').replace(' \\', '')
- if read_targets:
- if not line:
- break
- test_list_all.append(line)
- continue
-
- if line == 'FUZZ_TARGETS =':
- read_targets = True
+def parse_test_list(*, fuzz_bin):
+ test_list_all = subprocess.run(
+ fuzz_bin,
+ env={
+ 'PRINT_ALL_FUZZ_TARGETS_AND_ABORT': ''
+ },
+ stdout=subprocess.PIPE,
+ stderr=subprocess.DEVNULL,
+ universal_newlines=True,
+ ).stdout.splitlines()
return test_list_all