aboutsummaryrefslogtreecommitdiff
path: root/test/functional/test_runner.py
diff options
context:
space:
mode:
Diffstat (limited to 'test/functional/test_runner.py')
-rwxr-xr-xtest/functional/test_runner.py53
1 files changed, 38 insertions, 15 deletions
diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py
index c87010b0f4..b2aee7c739 100755
--- a/test/functional/test_runner.py
+++ b/test/functional/test_runner.py
@@ -16,6 +16,7 @@ For a description of arguments recognized by test scripts, see
import argparse
import configparser
+import datetime
import os
import time
import shutil
@@ -163,17 +164,18 @@ def main():
Help text and arguments for individual test script:''',
formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('--coverage', action='store_true', help='generate a basic coverage report for the RPC interface')
- parser.add_argument('--exclude', '-x', help='specify a comma-seperated-list of scripts to exclude. Do not include the .py extension in the name.')
+ parser.add_argument('--exclude', '-x', help='specify a comma-seperated-list of scripts to exclude.')
parser.add_argument('--extended', action='store_true', help='run the extended test suite in addition to the basic tests')
parser.add_argument('--force', '-f', action='store_true', help='run tests even on platforms where they are disabled by default (e.g. windows).')
parser.add_argument('--help', '-h', '-?', action='store_true', help='print help text and exit')
parser.add_argument('--jobs', '-j', type=int, default=4, help='how many test scripts to run in parallel. Default=4.')
parser.add_argument('--keepcache', '-k', action='store_true', help='the default behavior is to flush the cache directory on startup. --keepcache retains the cache from the previous testrun.')
parser.add_argument('--quiet', '-q', action='store_true', help='only print results summary and failure logs')
+ parser.add_argument('--tmpdirprefix', '-t', default=tempfile.gettempdir(), help="Root directory for datadirs")
args, unknown_args = parser.parse_known_args()
- # Create a set to store arguments and create the passon string
- tests = set(arg for arg in unknown_args if arg[:2] != "--")
+ # args to be passed on always start with two dashes; tests are the remaining unknown args
+ tests = [arg for arg in unknown_args if arg[:2] != "--"]
passon_args = [arg for arg in unknown_args if arg[:2] == "--"]
# Read config generated by configure.
@@ -187,6 +189,12 @@ def main():
logging_level = logging.INFO if args.quiet else logging.DEBUG
logging.basicConfig(format='%(message)s', level=logging_level)
+ # Create base test directory
+ tmpdir = "%s/bitcoin_test_runner_%s" % (args.tmpdirprefix, datetime.datetime.now().strftime("%Y%m%d_%H%M%S"))
+ os.makedirs(tmpdir)
+
+ logging.debug("Temporary test directory at %s" % tmpdir)
+
enable_wallet = config["components"].getboolean("ENABLE_WALLET")
enable_utils = config["components"].getboolean("ENABLE_UTILS")
enable_bitcoind = config["components"].getboolean("ENABLE_BITCOIND")
@@ -206,8 +214,13 @@ def main():
if tests:
# Individual tests have been specified. Run specified tests that exist
# in the ALL_SCRIPTS list. Accept the name with or without .py extension.
- test_list = [t for t in ALL_SCRIPTS if
- (t in tests or re.sub(".py$", "", t) in tests)]
+ tests = [re.sub("\.py$", "", t) + ".py" for t in tests]
+ test_list = []
+ for t in tests:
+ if t in ALL_SCRIPTS:
+ test_list.append(t)
+ else:
+ print("{}WARNING!{} Test '{}' not found in full test list.".format(BOLD[1], BOLD[0], t))
else:
# No individual tests have been specified.
# Run all base tests, and optionally run extended tests.
@@ -219,9 +232,12 @@ def main():
# Remove the test cases that the user has explicitly asked to exclude.
if args.exclude:
- for exclude_test in args.exclude.split(','):
- if exclude_test + ".py" in test_list:
- test_list.remove(exclude_test + ".py")
+ tests_excl = [re.sub("\.py$", "", t) + ".py" for t in args.exclude.split(',')]
+ for exclude_test in tests_excl:
+ if exclude_test in test_list:
+ test_list.remove(exclude_test)
+ else:
+ print("{}WARNING!{} Test '{}' not found in current test list.".format(BOLD[1], BOLD[0], exclude_test))
if not test_list:
print("No valid test scripts specified. Check that your test is in one "
@@ -239,9 +255,9 @@ def main():
if not args.keepcache:
shutil.rmtree("%s/test/cache" % config["environment"]["BUILDDIR"], ignore_errors=True)
- run_tests(test_list, config["environment"]["SRCDIR"], config["environment"]["BUILDDIR"], config["environment"]["EXEEXT"], args.jobs, args.coverage, passon_args)
+ run_tests(test_list, config["environment"]["SRCDIR"], config["environment"]["BUILDDIR"], config["environment"]["EXEEXT"], tmpdir, args.jobs, args.coverage, passon_args)
-def run_tests(test_list, src_dir, build_dir, exeext, jobs=1, enable_coverage=False, args=[]):
+def run_tests(test_list, src_dir, build_dir, exeext, tmpdir, jobs=1, enable_coverage=False, args=[]):
# Warn if bitcoind is already running (unix only)
try:
if subprocess.check_output(["pidof", "bitcoind"]) is not None:
@@ -272,10 +288,10 @@ def run_tests(test_list, src_dir, build_dir, exeext, jobs=1, enable_coverage=Fal
if len(test_list) > 1 and jobs > 1:
# Populate cache
- subprocess.check_output([tests_dir + 'create_cache.py'] + flags)
+ subprocess.check_output([tests_dir + 'create_cache.py'] + flags + ["--tmpdir=%s/cache" % tmpdir])
#Run Tests
- job_queue = TestHandler(jobs, tests_dir, test_list, flags)
+ job_queue = TestHandler(jobs, tests_dir, tmpdir, test_list, flags)
time0 = time.time()
test_results = []
@@ -302,6 +318,10 @@ def run_tests(test_list, src_dir, build_dir, exeext, jobs=1, enable_coverage=Fal
logging.debug("Cleaning up coverage data")
coverage.cleanup()
+ # Clear up the temp directory if all subdirectories are gone
+ if not os.listdir(tmpdir):
+ os.rmdir(tmpdir)
+
all_passed = all(map(lambda test_result: test_result.was_successful, test_results))
sys.exit(not all_passed)
@@ -329,10 +349,11 @@ class TestHandler:
Trigger the testscrips passed in via the list.
"""
- def __init__(self, num_tests_parallel, tests_dir, test_list=None, flags=None):
+ def __init__(self, num_tests_parallel, tests_dir, tmpdir, test_list=None, flags=None):
assert(num_tests_parallel >= 1)
self.num_jobs = num_tests_parallel
self.tests_dir = tests_dir
+ self.tmpdir = tmpdir
self.test_list = test_list
self.flags = flags
self.num_running = 0
@@ -347,13 +368,15 @@ class TestHandler:
# Add tests
self.num_running += 1
t = self.test_list.pop(0)
- port_seed = ["--portseed={}".format(len(self.test_list) + self.portseed_offset)]
+ portseed = len(self.test_list) + self.portseed_offset
+ portseed_arg = ["--portseed={}".format(portseed)]
log_stdout = tempfile.SpooledTemporaryFile(max_size=2**16)
log_stderr = tempfile.SpooledTemporaryFile(max_size=2**16)
test_argv = t.split()
+ tmpdir = ["--tmpdir=%s/%s_%s" % (self.tmpdir, re.sub(".py$", "", test_argv[0]), portseed)]
self.jobs.append((t,
time.time(),
- subprocess.Popen([self.tests_dir + test_argv[0]] + test_argv[1:] + self.flags + port_seed,
+ subprocess.Popen([self.tests_dir + test_argv[0]] + test_argv[1:] + self.flags + portseed_arg + tmpdir,
universal_newlines=True,
stdout=log_stdout,
stderr=log_stderr),