aboutsummaryrefslogtreecommitdiff
path: root/test/functional/test_runner.py
diff options
context:
space:
mode:
Diffstat (limited to 'test/functional/test_runner.py')
-rwxr-xr-xtest/functional/test_runner.py76
1 files changed, 59 insertions, 17 deletions
diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py
index 12eb92028f..b1a5d2de94 100755
--- a/test/functional/test_runner.py
+++ b/test/functional/test_runner.py
@@ -23,6 +23,10 @@ import sys
import subprocess
import tempfile
import re
+import logging
+
+TEST_EXIT_PASSED = 0
+TEST_EXIT_SKIPPED = 77
BASE_SCRIPTS= [
# Scripts that are run by the travis build process.
@@ -67,6 +71,7 @@ BASE_SCRIPTS= [
'decodescript.py',
'blockchain.py',
'disablewallet.py',
+ 'net.py',
'keypool.py',
'p2p-mempool.py',
'prioritise_transaction.py',
@@ -87,7 +92,7 @@ BASE_SCRIPTS= [
ZMQ_SCRIPTS = [
# ZMQ test can only be run if bitcoin was built with zmq-enabled.
# call test_runner.py with -nozmq to explicitly exclude these tests.
- "zmq_test.py"]
+ 'zmq_test.py']
EXTENDED_SCRIPTS = [
# These tests are not run by the travis build process.
@@ -107,6 +112,7 @@ EXTENDED_SCRIPTS = [
'p2p-feefilter.py',
'rpcbind_test.py',
# vv Tests less than 30s vv
+ 'assumevalid.py',
'bip65-cltv.py',
'bip65-cltv-p2p.py',
'bipdersig-p2p.py',
@@ -123,6 +129,13 @@ EXTENDED_SCRIPTS = [
ALL_SCRIPTS = BASE_SCRIPTS + ZMQ_SCRIPTS + EXTENDED_SCRIPTS
+NON_SCRIPTS = [
+ # These are python files that live in the functional tests directory, but are not test scripts.
+ "combine_logs.py",
+ "create_cache.py",
+ "test_runner.py",
+]
+
def main():
# Parse arguments and pass through unrecognised args
parser = argparse.ArgumentParser(add_help=False,
@@ -137,6 +150,7 @@ def main():
parser.add_argument('--force', '-f', action='store_true', help='run tests even on platforms where they are disabled by default (e.g. windows).')
parser.add_argument('--help', '-h', '-?', action='store_true', help='print help text and exit')
parser.add_argument('--jobs', '-j', type=int, default=4, help='how many test scripts to run in parallel. Default=4.')
+ parser.add_argument('--quiet', '-q', action='store_true', help='only print results summary and failure logs')
parser.add_argument('--nozmq', action='store_true', help='do not run the zmq tests')
args, unknown_args = parser.parse_known_args()
@@ -148,6 +162,10 @@ def main():
config = configparser.ConfigParser()
config.read_file(open(os.path.dirname(__file__) + "/config.ini"))
+ # Set up logging
+ logging_level = logging.INFO if args.quiet else logging.DEBUG
+ logging.basicConfig(format='%(message)s', level=logging_level)
+
enable_wallet = config["components"].getboolean("ENABLE_WALLET")
enable_utils = config["components"].getboolean("ENABLE_UTILS")
enable_bitcoind = config["components"].getboolean("ENABLE_BITCOIND")
@@ -203,11 +221,13 @@ def main():
sys.exit(0)
if args.help:
- # Print help for test_runner.py, then print help of the first script and exit.
+ # Print help for test_runner.py, then print help of the first script (with args removed) and exit.
parser.print_help()
- subprocess.check_call((config["environment"]["SRCDIR"] + '/test/functional/' + test_list[0]).split() + ['-h'])
+ subprocess.check_call([(config["environment"]["SRCDIR"] + '/test/functional/' + test_list[0].split()[0])] + ['-h'])
sys.exit(0)
+ check_script_list(config["environment"]["SRCDIR"])
+
run_tests(test_list, config["environment"]["SRCDIR"], config["environment"]["BUILDDIR"], config["environment"]["EXEEXT"], args.jobs, args.coverage, passon_args)
def run_tests(test_list, src_dir, build_dir, exeext, jobs=1, enable_coverage=False, args=[]):
@@ -229,7 +249,7 @@ def run_tests(test_list, src_dir, build_dir, exeext, jobs=1, enable_coverage=Fal
if enable_coverage:
coverage = RPCCoverage()
flags.append(coverage.flag)
- print("Initializing coverage directory at %s\n" % coverage.dir)
+ logging.debug("Initializing coverage directory at %s" % coverage.dir)
else:
coverage = None
@@ -245,27 +265,31 @@ def run_tests(test_list, src_dir, build_dir, exeext, jobs=1, enable_coverage=Fal
job_queue = TestHandler(jobs, tests_dir, test_list, flags)
max_len_name = len(max(test_list, key=len))
- results = BOLD[1] + "%s | %s | %s\n\n" % ("TEST".ljust(max_len_name), "PASSED", "DURATION") + BOLD[0]
+ results = "\n" + BOLD[1] + "%s | %s | %s\n\n" % ("TEST".ljust(max_len_name), "STATUS ", "DURATION") + BOLD[0]
for _ in range(len(test_list)):
- (name, stdout, stderr, passed, duration) = job_queue.get_next()
- all_passed = all_passed and passed
+ (name, stdout, stderr, status, duration) = job_queue.get_next()
+ all_passed = all_passed and status != "Failed"
time_sum += duration
- print('\n' + BOLD[1] + name + BOLD[0] + ":")
- print('' if passed else stdout + '\n', end='')
- print('' if stderr == '' else 'stderr:\n' + stderr + '\n', end='')
- print("Pass: %s%s%s, Duration: %s s\n" % (BOLD[1], passed, BOLD[0], duration))
+ if status == "Passed":
+ logging.debug("\n%s%s%s passed, Duration: %s s" % (BOLD[1], name, BOLD[0], duration))
+ elif status == "Skipped":
+ logging.debug("\n%s%s%s skipped" % (BOLD[1], name, BOLD[0]))
+ else:
+ print("\n%s%s%s failed, Duration: %s s\n" % (BOLD[1], name, BOLD[0], duration))
+ print(BOLD[1] + 'stdout:\n' + BOLD[0] + stdout + '\n')
+ print(BOLD[1] + 'stderr:\n' + BOLD[0] + stderr + '\n')
- results += "%s | %s | %s s\n" % (name.ljust(max_len_name), str(passed).ljust(6), duration)
+ results += "%s | %s | %s s\n" % (name.ljust(max_len_name), status.ljust(7), duration)
- results += BOLD[1] + "\n%s | %s | %s s (accumulated)" % ("ALL".ljust(max_len_name), str(all_passed).ljust(6), time_sum) + BOLD[0]
+ results += BOLD[1] + "\n%s | %s | %s s (accumulated)" % ("ALL".ljust(max_len_name), str(all_passed).ljust(7), time_sum) + BOLD[0]
print(results)
print("\nRuntime: %s s" % (int(time.time() - time0)))
if coverage:
coverage.report_rpc_coverage()
- print("Cleaning up coverage data")
+ logging.debug("Cleaning up coverage data")
coverage.cleanup()
sys.exit(not all_passed)
@@ -296,9 +320,10 @@ class TestHandler:
port_seed = ["--portseed={}".format(len(self.test_list) + self.portseed_offset)]
log_stdout = tempfile.SpooledTemporaryFile(max_size=2**16)
log_stderr = tempfile.SpooledTemporaryFile(max_size=2**16)
+ test_argv = t.split()
self.jobs.append((t,
time.time(),
- subprocess.Popen((self.tests_dir + t).split() + self.flags + port_seed,
+ subprocess.Popen([self.tests_dir + test_argv[0]] + test_argv[1:] + self.flags + port_seed,
universal_newlines=True,
stdout=log_stdout,
stderr=log_stderr),
@@ -315,12 +340,29 @@ class TestHandler:
log_out.seek(0), log_err.seek(0)
[stdout, stderr] = [l.read().decode('utf-8') for l in (log_out, log_err)]
log_out.close(), log_err.close()
- passed = stderr == "" and proc.returncode == 0
+ if proc.returncode == TEST_EXIT_PASSED and stderr == "":
+ status = "Passed"
+ elif proc.returncode == TEST_EXIT_SKIPPED:
+ status = "Skipped"
+ else:
+ status = "Failed"
self.num_running -= 1
self.jobs.remove(j)
- return name, stdout, stderr, passed, int(time.time() - time0)
+ return name, stdout, stderr, status, int(time.time() - time0)
print('.', end='', flush=True)
+def check_script_list(src_dir):
+ """Check scripts directory.
+
+ Check that there are no scripts in the functional tests directory which are
+ not being run by pull-tester.py."""
+ script_dir = src_dir + '/test/functional/'
+ python_files = set([t for t in os.listdir(script_dir) if t[-3:] == ".py"])
+ missed_tests = list(python_files - set(map(lambda x: x.split()[0], ALL_SCRIPTS + NON_SCRIPTS)))
+ if len(missed_tests) != 0:
+ print("The following scripts are not being run:" + str(missed_tests))
+ print("Check the test lists in test_runner.py")
+ sys.exit(1)
class RPCCoverage(object):
"""