diff options
Diffstat (limited to 'test/functional/test_runner.py')
| -rwxr-xr-x | test/functional/test_runner.py | 68 |
1 files changed, 42 insertions, 26 deletions
diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index d9960460d9..da55a3a156 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -68,9 +68,6 @@ if os.name != 'nt' or sys.getwindowsversion() >= (10, 0, 14393): TEST_EXIT_PASSED = 0 TEST_EXIT_SKIPPED = 77 -# 20 minutes represented in seconds -TRAVIS_TIMEOUT_DURATION = 20 * 60 - BASE_SCRIPTS = [ # Scripts that are run by the travis build process. # Longest test should go first, to favor running tests in parallel @@ -123,6 +120,7 @@ BASE_SCRIPTS = [ 'wallet_disableprivatekeys.py', 'wallet_disableprivatekeys.py --usecli', 'interface_http.py', + 'interface_rpc.py', 'rpc_psbt.py', 'rpc_users.py', 'feature_proxy.py', @@ -139,6 +137,7 @@ BASE_SCRIPTS = [ 'mining_prioritisetransaction.py', 'p2p_invalid_locator.py', 'p2p_invalid_block.py', + 'p2p_invalid_messages.py', 'p2p_invalid_tx.py', 'feature_assumevalid.py', 'example_test.py', @@ -152,11 +151,12 @@ BASE_SCRIPTS = [ 'feature_versionbits_warning.py', 'rpc_preciousblock.py', 'wallet_importprunedfunds.py', - 'rpc_zmq.py', + 'p2p_leak_tx.py', 'rpc_signmessage.py', 'feature_nulldummy.py', 'mempool_accept.py', 'wallet_import_rescan.py', + 'wallet_import_with_label.py', 'rpc_bind.py --ipv4', 'rpc_bind.py --ipv6', 'rpc_bind.py --nonloopback', @@ -175,6 +175,7 @@ BASE_SCRIPTS = [ 'rpc_getblockstats.py', 'p2p_fingerprint.py', 'feature_uacomment.py', + 'feature_filelock.py', 'p2p_unrequested_blocks.py', 'feature_includeconf.py', 'rpc_scantxoutset.py', @@ -213,15 +214,16 @@ def main(): epilog=''' Help text and arguments for individual test script:''', formatter_class=argparse.RawTextHelpFormatter) - parser.add_argument('--combinedlogslen', '-c', type=int, default=0, help='print a combined log (of length n lines) from all test nodes and test framework to the console on failure.') + parser.add_argument('--combinedlogslen', '-c', type=int, default=0, metavar='n', help='On failure, print a log (of length n lines) to the console, combined from the test framework and all test nodes.') parser.add_argument('--coverage', action='store_true', help='generate a basic coverage report for the RPC interface') + parser.add_argument('--ci', action='store_true', help='Run checks and code that are usually only enabled in a continuous integration environment') parser.add_argument('--exclude', '-x', help='specify a comma-separated-list of scripts to exclude.') parser.add_argument('--extended', action='store_true', help='run the extended test suite in addition to the basic tests') parser.add_argument('--force', '-f', action='store_true', help='run tests even on platforms where they are disabled by default (e.g. windows).') parser.add_argument('--help', '-h', '-?', action='store_true', help='print help text and exit') parser.add_argument('--jobs', '-j', type=int, default=4, help='how many test scripts to run in parallel. Default=4.') parser.add_argument('--keepcache', '-k', action='store_true', help='the default behavior is to flush the cache directory on startup. --keepcache retains the cache from the previous testrun.') - parser.add_argument('--quiet', '-q', action='store_true', help='only print results summary and failure logs') + parser.add_argument('--quiet', '-q', action='store_true', help='only print dots, results summary and failure logs') parser.add_argument('--tmpdirprefix', '-t', default=tempfile.gettempdir(), help="Root directory for datadirs") parser.add_argument('--failfast', action='store_true', help='stop execution after the first test failure') args, unknown_args = parser.parse_known_args() @@ -305,25 +307,26 @@ def main(): subprocess.check_call([sys.executable, os.path.join(config["environment"]["SRCDIR"], 'test', 'functional', test_list[0].split()[0]), '-h']) sys.exit(0) - check_script_list(config["environment"]["SRCDIR"]) + check_script_list(src_dir=config["environment"]["SRCDIR"], fail_on_warn=args.ci) check_script_prefixes() if not args.keepcache: shutil.rmtree("%s/test/cache" % config["environment"]["BUILDDIR"], ignore_errors=True) run_tests( - test_list, - config["environment"]["SRCDIR"], - config["environment"]["BUILDDIR"], - tmpdir, + test_list=test_list, + src_dir=config["environment"]["SRCDIR"], + build_dir=config["environment"]["BUILDDIR"], + tmpdir=tmpdir, jobs=args.jobs, enable_coverage=args.coverage, args=passon_args, combined_logs_len=args.combinedlogslen, failfast=args.failfast, + runs_ci=args.ci, ) -def run_tests(test_list, src_dir, build_dir, tmpdir, jobs=1, enable_coverage=False, args=None, combined_logs_len=0, failfast=False): +def run_tests(*, test_list, src_dir, build_dir, tmpdir, jobs=1, enable_coverage=False, args=None, combined_logs_len=0, failfast=False, runs_ci): args = args or [] # Warn if bitcoind is already running (unix only) @@ -358,22 +361,29 @@ def run_tests(test_list, src_dir, build_dir, tmpdir, jobs=1, enable_coverage=Fal raise #Run Tests - job_queue = TestHandler(jobs, tests_dir, tmpdir, test_list, flags) + job_queue = TestHandler( + num_tests_parallel=jobs, + tests_dir=tests_dir, + tmpdir=tmpdir, + test_list=test_list, + flags=flags, + timeout_duration=40 * 60 if runs_ci else float('inf'), # in seconds + ) start_time = time.time() test_results = [] max_len_name = len(max(test_list, key=len)) - - for _ in range(len(test_list)): + test_count = len(test_list) + for i in range(test_count): test_result, testdir, stdout, stderr = job_queue.get_next() test_results.append(test_result) - + done_str = "{}/{} - {}{}{}".format(i + 1, test_count, BOLD[1], test_result.name, BOLD[0]) if test_result.status == "Passed": - logging.debug("\n%s%s%s passed, Duration: %s s" % (BOLD[1], test_result.name, BOLD[0], test_result.time)) + logging.debug("%s passed, Duration: %s s" % (done_str, test_result.time)) elif test_result.status == "Skipped": - logging.debug("\n%s%s%s skipped" % (BOLD[1], test_result.name, BOLD[0])) + logging.debug("%s skipped" % (done_str)) else: - print("\n%s%s%s failed, Duration: %s s\n" % (BOLD[1], test_result.name, BOLD[0], test_result.time)) + print("%s failed, Duration: %s s\n" % (done_str, test_result.time)) print(BOLD[1] + 'stdout:\n' + BOLD[0] + stdout + '\n') print(BOLD[1] + 'stderr:\n' + BOLD[0] + stderr + '\n') if combined_logs_len and os.path.isdir(testdir): @@ -439,11 +449,12 @@ class TestHandler: Trigger the test scripts passed in via the list. """ - def __init__(self, num_tests_parallel, tests_dir, tmpdir, test_list=None, flags=None): - assert(num_tests_parallel >= 1) + def __init__(self, *, num_tests_parallel, tests_dir, tmpdir, test_list, flags, timeout_duration): + assert num_tests_parallel >= 1 self.num_jobs = num_tests_parallel self.tests_dir = tests_dir self.tmpdir = tmpdir + self.timeout_duration = timeout_duration self.test_list = test_list self.flags = flags self.num_running = 0 @@ -472,12 +483,13 @@ class TestHandler: log_stderr)) if not self.jobs: raise IndexError('pop from empty list') + dot_count = 0 while True: # Return first proc that finishes time.sleep(.5) for job in self.jobs: (name, start_time, proc, testdir, log_out, log_err) = job - if os.getenv('TRAVIS') == 'true' and int(time.time() - start_time) > TRAVIS_TIMEOUT_DURATION: + if int(time.time() - start_time) > self.timeout_duration: # In travis, timeout individual tests (to stop tests hanging and not providing useful output). proc.send_signal(signal.SIGINT) if proc.poll() is not None: @@ -492,9 +504,12 @@ class TestHandler: status = "Failed" self.num_running -= 1 self.jobs.remove(job) - + clearline = '\r' + (' ' * dot_count) + '\r' + print(clearline, end='', flush=True) + dot_count = 0 return TestResult(name, status, int(time.time() - start_time)), testdir, stdout, stderr print('.', end='', flush=True) + dot_count += 1 def kill_and_join(self): """Send SIGKILL to all jobs and block until all have ended.""" @@ -552,7 +567,7 @@ def check_script_prefixes(): raise AssertionError("Some tests are not following naming convention!") -def check_script_list(src_dir): +def check_script_list(*, src_dir, fail_on_warn): """Check scripts directory. Check that there are no scripts in the functional tests directory which are @@ -562,10 +577,11 @@ def check_script_list(src_dir): missed_tests = list(python_files - set(map(lambda x: x.split()[0], ALL_SCRIPTS + NON_SCRIPTS))) if len(missed_tests) != 0: print("%sWARNING!%s The following scripts are not being run: %s. Check the test lists in test_runner.py." % (BOLD[1], BOLD[0], str(missed_tests))) - if os.getenv('TRAVIS') == 'true': + if fail_on_warn: # On travis this warning is an error to prevent merging incomplete commits into master sys.exit(1) + class RPCCoverage(): """ Coverage reporting utilities for test_runner. @@ -621,7 +637,7 @@ class RPCCoverage(): with open(coverage_ref_filename, 'r', encoding="utf8") as coverage_ref_file: all_cmds.update([line.strip() for line in coverage_ref_file.readlines()]) - for root, dirs, files in os.walk(self.dir): + for root, _, files in os.walk(self.dir): for filename in files: if filename.startswith(coverage_file_prefix): coverage_filenames.add(os.path.join(root, filename)) |