Merge bitcoin/bitcoin#31018: test: Treat exclude list warning as failure in CI

fa6d14eacb2a8c1c3243e3075254dfdebaa9290e test: Treat exclude list warning as failure in CI (MarcoFalke)

Pull request description:

  An outdated exclude list or otherwise an error in the exclude list handling is usually a bug.

  So make it fatal in the CI, instead of silently ignoring it.

  Fixes https://github.com/bitcoin/bitcoin/pull/30872/files#r1757015334

  Can be tested with something like (with and without `--ci`):

  ```
  ./bld-cmake/test/functional/test_runner.py wallet_disable -x wallet_disablee

ACKs for top commit:
  tdb3:
    ACK fa6d14eacb2a8c1c3243e3075254dfdebaa9290e
  ismaelsadeeq:
    utACK fa6d14eacb2a8c1c3243e3075254dfdebaa9290e

Tree-SHA512: 03a70dff9d1272d982591d60ab764f9233d4802488bc1bad305a2755e2d7ed86e691ee94767a3bc5f68321b63214aba44e6f9edd1543dfad7a20f9397cf78734

1 files changed, 7 insertions, 4 deletions

diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py
index 14a1e42924..3d8c230066 100755
--- a/test/functional/test_runner.py
+++ b/test/functional/test_runner.py
@@ -447,8 +447,8 @@ def main():
                         help="Leave bitcoinds and test.* datadir on exit or error")
     parser.add_argument('--resultsfile', '-r', help='store test results (as CSV) to the provided file')
 
-
     args, unknown_args = parser.parse_known_args()
+    fail_on_warn = args.ci
     if not args.ansi:
         global DEFAULT, BOLD, GREEN, RED
         DEFAULT = ("", "")
@@ -525,8 +525,12 @@ def main():
     # Remove the test cases that the user has explicitly asked to exclude.
     # The user can specify a test case with or without the .py extension.
     if args.exclude:
+
         def print_warning_missing_test(test_name):
-            print("{}WARNING!{} Test '{}' not found in current test list.".format(BOLD[1], BOLD[0], test_name))
+            print("{}WARNING!{} Test '{}' not found in current test list. Check the --exclude list.".format(BOLD[1], BOLD[0], test_name))
+            if fail_on_warn:
+                sys.exit(1)
+
         def remove_tests(exclude_list):
             if not exclude_list:
                 print_warning_missing_test(exclude_test)
@@ -563,7 +567,7 @@ def main():
                   f"A minimum of {MIN_NO_CLEANUP_SPACE // (1024 * 1024 * 1024)} GB of free space is required.")
         passon_args.append("--nocleanup")
 
-    check_script_list(src_dir=config["environment"]["SRCDIR"], fail_on_warn=args.ci)
+    check_script_list(src_dir=config["environment"]["SRCDIR"], fail_on_warn=fail_on_warn)
     check_script_prefixes()
 
     if not args.keepcache:
@@ -872,7 +876,6 @@ def check_script_list(*, src_dir, fail_on_warn):
     if len(missed_tests) != 0:
         print("%sWARNING!%s The following scripts are not being run: %s. Check the test lists in test_runner.py." % (BOLD[1], BOLD[0], str(missed_tests)))
         if fail_on_warn:
-            # On CI this warning is an error to prevent merging incomplete commits into master
             sys.exit(1)