aboutsummaryrefslogtreecommitdiff
path: root/test/functional/test_runner.py
diff options
context:
space:
mode:
Diffstat (limited to 'test/functional/test_runner.py')
-rwxr-xr-xtest/functional/test_runner.py269
1 files changed, 144 insertions, 125 deletions
diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py
index bfd1192d3a..09f7f50de0 100755
--- a/test/functional/test_runner.py
+++ b/test/functional/test_runner.py
@@ -1,5 +1,5 @@
#!/usr/bin/env python3
-# Copyright (c) 2014-2016 The Bitcoin Core developers
+# Copyright (c) 2014-2017 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
"""Run regression test suite.
@@ -52,112 +52,121 @@ if os.name == 'posix':
TEST_EXIT_PASSED = 0
TEST_EXIT_SKIPPED = 77
+# 20 minutes represented in seconds
+TRAVIS_TIMEOUT_DURATION = 20 * 60
+
BASE_SCRIPTS= [
# Scripts that are run by the travis build process.
# Longest test should go first, to favor running tests in parallel
- 'wallet-hd.py',
- 'walletbackup.py',
+ 'wallet_hd.py',
+ 'wallet_backup.py',
# vv Tests less than 5m vv
- 'p2p-fullblocktest.py',
- 'fundrawtransaction.py',
- 'p2p-compactblocks.py',
- 'segwit.py',
+ 'feature_block.py',
+ 'rpc_fundrawtransaction.py',
+ 'p2p_compactblocks.py',
+ 'feature_segwit.py',
# vv Tests less than 2m vv
- 'wallet.py',
- 'wallet-accounts.py',
- 'p2p-segwit.py',
- 'wallet-dump.py',
- 'listtransactions.py',
+ 'wallet_basic.py',
+ 'wallet_accounts.py',
+ 'p2p_segwit.py',
+ 'wallet_dump.py',
+ 'rpc_listtransactions.py',
# vv Tests less than 60s vv
- 'sendheaders.py',
- 'zapwallettxes.py',
- 'importmulti.py',
+ 'p2p_sendheaders.py',
+ 'wallet_zapwallettxes.py',
+ 'wallet_importmulti.py',
'mempool_limit.py',
- 'merkle_blocks.py',
- 'receivedby.py',
- 'abandonconflict.py',
- 'bip68-112-113-p2p.py',
- 'rawtransactions.py',
- 'reindex.py',
+ 'rpc_txoutproof.py',
+ 'wallet_listreceivedby.py',
+ 'wallet_abandonconflict.py',
+ 'feature_csv_activation.py',
+ 'rpc_rawtransaction.py',
+ 'wallet_address_types.py',
+ 'feature_reindex.py',
# vv Tests less than 30s vv
- 'keypool-topup.py',
- 'zmq_test.py',
- 'bitcoin_cli.py',
- 'mempool_resurrect_test.py',
- 'txn_doublespend.py --mineblock',
- 'txn_clone.py',
- 'getchaintips.py',
- 'rest.py',
- 'mempool_spendcoinbase.py',
+ 'wallet_keypool_topup.py',
+ 'interface_zmq.py',
+ 'interface_bitcoin_cli.py',
+ 'mempool_resurrect.py',
+ 'wallet_txn_doublespend.py --mineblock',
+ 'wallet_txn_clone.py',
+ 'wallet_txn_clone.py --segwit',
+ 'rpc_getchaintips.py',
+ 'interface_rest.py',
+ 'mempool_spend_coinbase.py',
'mempool_reorg.py',
'mempool_persist.py',
- 'multiwallet.py',
- 'httpbasics.py',
- 'multi_rpc.py',
- 'proxy_test.py',
- 'signrawtransactions.py',
- 'disconnect_ban.py',
- 'decodescript.py',
- 'blockchain.py',
- 'deprecated_rpc.py',
- 'disablewallet.py',
- 'net.py',
- 'keypool.py',
- 'p2p-mempool.py',
- 'prioritise_transaction.py',
- 'invalidblockrequest.py',
- 'invalidtxrequest.py',
- 'p2p-versionbits-warning.py',
- 'preciousblock.py',
- 'importprunedfunds.py',
- 'signmessages.py',
- 'nulldummy.py',
- 'import-rescan.py',
- 'mining.py',
- 'bumpfee.py',
- 'rpcnamedargs.py',
- 'listsinceblock.py',
- 'p2p-leaktests.py',
- 'wallet-encryption.py',
- 'bipdersig-p2p.py',
- 'bip65-cltv-p2p.py',
- 'uptime.py',
- 'resendwallettransactions.py',
- 'minchainwork.py',
- 'p2p-fingerprint.py',
- 'uacomment.py',
- 'p2p-acceptblock.py',
+ 'wallet_multiwallet.py',
+ 'wallet_multiwallet.py --usecli',
+ 'interface_http.py',
+ 'rpc_users.py',
+ 'feature_proxy.py',
+ 'rpc_signrawtransaction.py',
+ 'p2p_disconnect_ban.py',
+ 'rpc_decodescript.py',
+ 'rpc_blockchain.py',
+ 'rpc_deprecated.py',
+ 'wallet_disable.py',
+ 'rpc_net.py',
+ 'wallet_keypool.py',
+ 'p2p_mempool.py',
+ 'mining_prioritisetransaction.py',
+ 'p2p_invalid_block.py',
+ 'p2p_invalid_tx.py',
+ 'feature_versionbits_warning.py',
+ 'rpc_preciousblock.py',
+ 'wallet_importprunedfunds.py',
+ 'rpc_signmessage.py',
+ 'feature_nulldummy.py',
+ 'wallet_import_rescan.py',
+ 'mining_basic.py',
+ 'wallet_bumpfee.py',
+ 'rpc_named_arguments.py',
+ 'wallet_listsinceblock.py',
+ 'p2p_leak.py',
+ 'wallet_encryption.py',
+ 'feature_dersig.py',
+ 'feature_cltv.py',
+ 'rpc_uptime.py',
+ 'wallet_resendwallettransactions.py',
+ 'wallet_fallbackfee.py',
+ 'feature_minchainwork.py',
+ 'p2p_fingerprint.py',
+ 'feature_uacomment.py',
+ 'p2p_unrequested_blocks.py',
'feature_logging.py',
- 'node_network_limited.py',
- 'conf_args.py',
+ 'p2p_node_network_limited.py',
+ 'feature_config_args.py',
+ # Don't append tests at the end to avoid merge conflicts
+ # Put them in a random line within the section that fits their approximate run-time
]
EXTENDED_SCRIPTS = [
# These tests are not run by the travis build process.
# Longest test should go first, to favor running tests in parallel
- 'pruning.py',
+ 'feature_pruning.py',
# vv Tests less than 20m vv
- 'smartfees.py',
+ 'feature_fee_estimation.py',
# vv Tests less than 5m vv
- 'maxuploadtarget.py',
+ 'feature_maxuploadtarget.py',
'mempool_packages.py',
- 'dbcrash.py',
+ 'feature_dbcrash.py',
# vv Tests less than 2m vv
- 'bip68-sequence.py',
- 'getblocktemplate_longpoll.py',
- 'p2p-timeouts.py',
+ 'feature_bip68_sequence.py',
+ 'mining_getblocktemplate_longpoll.py',
+ 'p2p_timeouts.py',
# vv Tests less than 60s vv
- 'bip9-softforks.py',
- 'p2p-feefilter.py',
- 'rpcbind_test.py',
+ 'feature_bip9_softforks.py',
+ 'p2p_feefilter.py',
+ 'rpc_bind.py',
# vv Tests less than 30s vv
- 'assumevalid.py',
+ 'feature_assumevalid.py',
'example_test.py',
- 'txn_doublespend.py',
- 'txn_clone.py --mineblock',
- 'notifications.py',
- 'invalidateblock.py',
- 'replace-by-fee.py',
+ 'wallet_txn_doublespend.py',
+ 'wallet_txn_clone.py --mineblock',
+ 'feature_notifications.py',
+ 'rpc_invalidateblock.py',
+ 'feature_rbf.py',
]
# Place EXTENDED_SCRIPTS first since it has the 3 longest running tests
@@ -227,29 +236,27 @@ def main():
sys.exit(0)
# Build list of tests
+ test_list = []
if tests:
# Individual tests have been specified. Run specified tests that exist
# in the ALL_SCRIPTS list. Accept the name with or without .py extension.
- tests = [re.sub("\.py$", "", t) + ".py" for t in tests]
- test_list = []
- for t in tests:
- if t in ALL_SCRIPTS:
- test_list.append(t)
+ tests = [re.sub("\.py$", "", test) + ".py" for test in tests]
+ for test in tests:
+ if test in ALL_SCRIPTS:
+ test_list.append(test)
else:
- print("{}WARNING!{} Test '{}' not found in full test list.".format(BOLD[1], BOLD[0], t))
+ print("{}WARNING!{} Test '{}' not found in full test list.".format(BOLD[1], BOLD[0], test))
+ elif args.extended:
+ # Include extended tests
+ test_list += ALL_SCRIPTS
else:
- # No individual tests have been specified.
- # Run all base tests, and optionally run extended tests.
- test_list = BASE_SCRIPTS
- if args.extended:
- # place the EXTENDED_SCRIPTS first since the three longest ones
- # are there and the list is shorter
- test_list = EXTENDED_SCRIPTS + test_list
+ # Run base tests only
+ test_list += BASE_SCRIPTS
# Remove the test cases that the user has explicitly asked to exclude.
if args.exclude:
- tests_excl = [re.sub("\.py$", "", t) + ".py" for t in args.exclude.split(',')]
- for exclude_test in tests_excl:
+ exclude_tests = [re.sub("\.py$", "", test) + ".py" for test in args.exclude.split(',')]
+ for exclude_test in exclude_tests:
if exclude_test in test_list:
test_list.remove(exclude_test)
else:
@@ -263,10 +270,11 @@ def main():
if args.help:
# Print help for test_runner.py, then print help of the first script (with args removed) and exit.
parser.print_help()
- subprocess.check_call([(config["environment"]["SRCDIR"] + '/test/functional/' + test_list[0].split()[0])] + ['-h'])
+ subprocess.check_call([sys.executable, os.path.join(config["environment"]["SRCDIR"], 'test', 'functional', test_list[0].split()[0]), '-h'])
sys.exit(0)
check_script_list(config["environment"]["SRCDIR"])
+ check_script_prefixes()
if not args.keepcache:
shutil.rmtree("%s/test/cache" % config["environment"]["BUILDDIR"], ignore_errors=True)
@@ -306,14 +314,14 @@ def run_tests(test_list, src_dir, build_dir, exeext, tmpdir, jobs=1, enable_cove
if len(test_list) > 1 and jobs > 1:
# Populate cache
try:
- subprocess.check_output([tests_dir + 'create_cache.py'] + flags + ["--tmpdir=%s/cache" % tmpdir])
- except Exception as e:
- print(e.output)
- raise e
+ subprocess.check_output([sys.executable, tests_dir + 'create_cache.py'] + flags + ["--tmpdir=%s/cache" % tmpdir])
+ except subprocess.CalledProcessError as e:
+ sys.stdout.buffer.write(e.output)
+ raise
#Run Tests
job_queue = TestHandler(jobs, tests_dir, tmpdir, test_list, flags)
- time0 = time.time()
+ start_time = time.time()
test_results = []
max_len_name = len(max(test_list, key=len))
@@ -336,10 +344,10 @@ def run_tests(test_list, src_dir, build_dir, exeext, tmpdir, jobs=1, enable_cove
print('\n============')
print('{}Combined log for {}:{}'.format(BOLD[1], testdir, BOLD[0]))
print('============\n')
- combined_logs, _ = subprocess.Popen([os.path.join(tests_dir, 'combine_logs.py'), '-c', testdir], universal_newlines=True, stdout=subprocess.PIPE).communicate()
+ combined_logs, _ = subprocess.Popen([sys.executable, os.path.join(tests_dir, 'combine_logs.py'), '-c', testdir], universal_newlines=True, stdout=subprocess.PIPE).communicate()
print("\n".join(deque(combined_logs.splitlines(), combined_logs_len)))
- print_results(test_results, max_len_name, (int(time.time() - time0)))
+ print_results(test_results, max_len_name, (int(time.time() - start_time)))
if coverage:
coverage.report_rpc_coverage()
@@ -396,17 +404,17 @@ class TestHandler:
while self.num_running < self.num_jobs and self.test_list:
# Add tests
self.num_running += 1
- t = self.test_list.pop(0)
+ test = self.test_list.pop(0)
portseed = len(self.test_list) + self.portseed_offset
portseed_arg = ["--portseed={}".format(portseed)]
log_stdout = tempfile.SpooledTemporaryFile(max_size=2**16)
log_stderr = tempfile.SpooledTemporaryFile(max_size=2**16)
- test_argv = t.split()
+ test_argv = test.split()
testdir = "{}/{}_{}".format(self.tmpdir, re.sub(".py$", "", test_argv[0]), portseed)
tmpdir_arg = ["--tmpdir={}".format(testdir)]
- self.jobs.append((t,
+ self.jobs.append((test,
time.time(),
- subprocess.Popen([self.tests_dir + test_argv[0]] + test_argv[1:] + self.flags + portseed_arg + tmpdir_arg,
+ subprocess.Popen([sys.executable, self.tests_dir + test_argv[0]] + test_argv[1:] + self.flags + portseed_arg + tmpdir_arg,
universal_newlines=True,
stdout=log_stdout,
stderr=log_stderr),
@@ -418,15 +426,14 @@ class TestHandler:
while True:
# Return first proc that finishes
time.sleep(.5)
- for j in self.jobs:
- (name, time0, proc, testdir, log_out, log_err) = j
- if os.getenv('TRAVIS') == 'true' and int(time.time() - time0) > 20 * 60:
- # In travis, timeout individual tests after 20 minutes (to stop tests hanging and not
- # providing useful output.
+ for job in self.jobs:
+ (name, start_time, proc, testdir, log_out, log_err) = job
+ if os.getenv('TRAVIS') == 'true' and int(time.time() - start_time) > TRAVIS_TIMEOUT_DURATION:
+ # In travis, timeout individual tests (to stop tests hanging and not providing useful output).
proc.send_signal(signal.SIGINT)
if proc.poll() is not None:
log_out.seek(0), log_err.seek(0)
- [stdout, stderr] = [l.read().decode('utf-8') for l in (log_out, log_err)]
+ [stdout, stderr] = [log_file.read().decode('utf-8') for log_file in (log_out, log_err)]
log_out.close(), log_err.close()
if proc.returncode == TEST_EXIT_PASSED and stderr == "":
status = "Passed"
@@ -435,9 +442,9 @@ class TestHandler:
else:
status = "Failed"
self.num_running -= 1
- self.jobs.remove(j)
+ self.jobs.remove(job)
- return TestResult(name, status, int(time.time() - time0)), testdir, stdout, stderr
+ return TestResult(name, status, int(time.time() - start_time)), testdir, stdout, stderr
print('.', end='', flush=True)
class TestResult():
@@ -465,13 +472,25 @@ class TestResult():
return self.status != "Failed"
+def check_script_prefixes():
+ """Check that test scripts start with one of the allowed name prefixes."""
+
+ good_prefixes_re = re.compile("(example|feature|interface|mempool|mining|p2p|rpc|wallet)_")
+ bad_script_names = [script for script in ALL_SCRIPTS if good_prefixes_re.match(script) is None]
+
+ if bad_script_names:
+ print("%sERROR:%s %d tests not meeting naming conventions:" % (BOLD[1], BOLD[0], len(bad_script_names)))
+ print(" %s" % ("\n ".join(sorted(bad_script_names))))
+ raise AssertionError("Some tests are not following naming convention!")
+
+
def check_script_list(src_dir):
"""Check scripts directory.
Check that there are no scripts in the functional tests directory which are
not being run by pull-tester.py."""
script_dir = src_dir + '/test/functional/'
- python_files = set([t for t in os.listdir(script_dir) if t[-3:] == ".py"])
+ python_files = set([test_file for test_file in os.listdir(script_dir) if test_file.endswith(".py")])
missed_tests = list(python_files - set(map(lambda x: x.split()[0], ALL_SCRIPTS + NON_SCRIPTS)))
if len(missed_tests) != 0:
print("%sWARNING!%s The following scripts are not being run: %s. Check the test lists in test_runner.py." % (BOLD[1], BOLD[0], str(missed_tests)))
@@ -507,7 +526,7 @@ class RPCCoverage():
if uncovered:
print("Uncovered RPC commands:")
- print("".join((" - %s\n" % i) for i in sorted(uncovered)))
+ print("".join((" - %s\n" % command) for command in sorted(uncovered)))
else:
print("All RPC commands covered.")
@@ -531,8 +550,8 @@ class RPCCoverage():
if not os.path.isfile(coverage_ref_filename):
raise RuntimeError("No coverage reference found")
- with open(coverage_ref_filename, 'r') as f:
- all_cmds.update([i.strip() for i in f.readlines()])
+ with open(coverage_ref_filename, 'r') as coverage_ref_file:
+ all_cmds.update([line.strip() for line in coverage_ref_file.readlines()])
for root, dirs, files in os.walk(self.dir):
for filename in files:
@@ -540,8 +559,8 @@ class RPCCoverage():
coverage_filenames.add(os.path.join(root, filename))
for filename in coverage_filenames:
- with open(filename, 'r') as f:
- covered_cmds.update([i.strip() for i in f.readlines()])
+ with open(filename, 'r') as coverage_file:
+ covered_cmds.update([line.strip() for line in coverage_file.readlines()])
return all_cmds - covered_cmds