Merge #20560: fuzz: Link all targets once

fa13e1b0c52738492310b6b421d8e38cb04da5b1 build: Add option --enable-danger-fuzz-link-all (MarcoFalke)
44444ba759480237172d83f42374c5c29c76eda0 fuzz: Link all targets once (MarcoFalke)

Pull request description:

  Currently the linker is invoked more than 150 times when compiling with `--enable-fuzz`. This is problematic for several reasons:

  * It wastes disk space north of 20 GB, as all libraries and sanitizers are linked more than 150 times
  * It wastes CPU time, as the link step can practically not be cached (similar to ccache for object files)
  * It makes it a blocker to compile the fuzz tests by default for non-fuzz builds #19388, for the aforementioned reasons
  * The build file is several thousand lines of code, without doing anything meaningful except listing each fuzz target in a highly verbose manner
  * It makes writing new fuzz tests unnecessarily hard, as build system knowledge is required; Compare that to boost unit tests, which can be added by simply editing an existing cpp file
  * It encourages fuzz tests that re-use the `buffer` or assume the `buffer` to be concatenations of seeds, which increases complexity of seeds and complexity for the fuzz engine to explore; Thus reducing the effectiveness of the affected fuzz targets

  Fixes #20088

ACKs for top commit:
  practicalswift:
    Tested ACK fa13e1b0c52738492310b6b421d8e38cb04da5b1
  sipa:
    ACK fa13e1b0c52738492310b6b421d8e38cb04da5b1. Reviewed the code changes, and tested the 3 different test_runner.py modes (run once, merge, generate). I also tested building with the new --enable-danger-fuzz-link-all

Tree-SHA512: 962ab33269ebd51810924c51266ecc62edd6ddf2fcd9a8c359ed906766f58c3f73c223f8d3cc49f2c60f0053f65e8bdd86ce9c19e673f8c2b3cd676e913f2642

1 files changed, 39 insertions, 26 deletions

diff --git a/test/fuzz/test_runner.py b/test/fuzz/test_runner.py
index c7895edbcc..3a2cefbe2b 100755
--- a/test/fuzz/test_runner.py
+++ b/test/fuzz/test_runner.py
@@ -83,7 +83,7 @@ def main():
         sys.exit(1)
 
     # Build list of tests
-    test_list_all = parse_test_list(makefile=os.path.join(config["environment"]["SRCDIR"], 'src', 'Makefile.test.include'))
+    test_list_all = parse_test_list(fuzz_bin=os.path.join(config["environment"]["BUILDDIR"], 'src', 'test', 'fuzz', 'fuzz'))
 
     if not test_list_all:
         logging.error("No fuzz targets found")
@@ -126,9 +126,12 @@ def main():
     try:
         help_output = subprocess.run(
             args=[
-                os.path.join(config["environment"]["BUILDDIR"], 'src', 'test', 'fuzz', test_list_selection[0]),
+                os.path.join(config["environment"]["BUILDDIR"], 'src', 'test', 'fuzz', 'fuzz'),
                 '-help=1',
             ],
+            env={
+                'FUZZ': test_list_selection[0]
+            },
             timeout=20,
             check=True,
             stderr=subprocess.PIPE,
@@ -177,24 +180,30 @@ def generate_corpus_seeds(*, fuzz_pool, build_dir, seed_dir, targets):
     """
     logging.info("Generating corpus seeds to {}".format(seed_dir))
 
-    def job(command):
+    def job(command, t):
         logging.debug("Running '{}'\n".format(" ".join(command)))
         logging.debug("Command '{}' output:\n'{}'\n".format(
             ' '.join(command),
-            subprocess.run(command, check=True, stderr=subprocess.PIPE,
-                           universal_newlines=True).stderr
-        ))
+            subprocess.run(
+                command,
+                env={
+                    'FUZZ': t
+                },
+                check=True,
+                stderr=subprocess.PIPE,
+                universal_newlines=True,
+            ).stderr))
 
     futures = []
     for target in targets:
         target_seed_dir = os.path.join(seed_dir, target)
         os.makedirs(target_seed_dir, exist_ok=True)
         command = [
-            os.path.join(build_dir, "src", "test", "fuzz", target),
+            os.path.join(build_dir, 'src', 'test', 'fuzz', 'fuzz'),
             "-runs=100000",
             target_seed_dir,
         ]
-        futures.append(fuzz_pool.submit(job, command))
+        futures.append(fuzz_pool.submit(job, command, target))
 
     for future in as_completed(futures):
         future.result()
@@ -205,7 +214,7 @@ def merge_inputs(*, fuzz_pool, corpus, test_list, build_dir, merge_dir):
     jobs = []
     for t in test_list:
         args = [
-            os.path.join(build_dir, 'src', 'test', 'fuzz', t),
+            os.path.join(build_dir, 'src', 'test', 'fuzz', 'fuzz'),
             '-merge=1',
             '-use_value_profile=1',  # Also done by oss-fuzz https://github.com/google/oss-fuzz/issues/1406#issuecomment-387790487
             os.path.join(corpus, t),
@@ -216,7 +225,15 @@ def merge_inputs(*, fuzz_pool, corpus, test_list, build_dir, merge_dir):
 
         def job(t, args):
             output = 'Run {} with args {}\n'.format(t, " ".join(args))
-            output += subprocess.run(args, check=True, stderr=subprocess.PIPE, universal_newlines=True).stderr
+            output += subprocess.run(
+                args,
+                env={
+                    'FUZZ': t
+                },
+                check=True,
+                stderr=subprocess.PIPE,
+                universal_newlines=True,
+            ).stderr
             logging.debug(output)
 
         jobs.append(fuzz_pool.submit(job, t, args))
@@ -231,7 +248,7 @@ def run_once(*, fuzz_pool, corpus, test_list, build_dir, use_valgrind):
         corpus_path = os.path.join(corpus, t)
         os.makedirs(corpus_path, exist_ok=True)
         args = [
-            os.path.join(build_dir, 'src', 'test', 'fuzz', t),
+            os.path.join(build_dir, 'src', 'test', 'fuzz', 'fuzz'),
             '-runs=1',
             corpus_path,
         ]
@@ -240,7 +257,7 @@ def run_once(*, fuzz_pool, corpus, test_list, build_dir, use_valgrind):
 
         def job(t, args):
             output = 'Run {} with args {}'.format(t, args)
-            result = subprocess.run(args, stderr=subprocess.PIPE, universal_newlines=True)
+            result = subprocess.run(args, env={'FUZZ': t}, stderr=subprocess.PIPE, universal_newlines=True)
             output += result.stderr
             return output, result
 
@@ -260,20 +277,16 @@ def run_once(*, fuzz_pool, corpus, test_list, build_dir, use_valgrind):
             sys.exit(1)
 
 
-def parse_test_list(makefile):
-    with open(makefile, encoding='utf-8') as makefile_test:
-        test_list_all = []
-        read_targets = False
-        for line in makefile_test.readlines():
-            line = line.strip().replace('test/fuzz/', '').replace(' \\', '')
-            if read_targets:
-                if not line:
-                    break
-                test_list_all.append(line)
-                continue
-
-            if line == 'FUZZ_TARGETS =':
-                read_targets = True
+def parse_test_list(*, fuzz_bin):
+    test_list_all = subprocess.run(
+        fuzz_bin,
+        env={
+            'PRINT_ALL_FUZZ_TARGETS_AND_ABORT': ''
+        },
+        stdout=subprocess.PIPE,
+        stderr=subprocess.DEVNULL,
+        universal_newlines=True,
+    ).stdout.splitlines()
     return test_list_all