diff options
Diffstat (limited to 'test')
| -rw-r--r-- | test/config.ini.in (renamed from test/functional/config.ini.in) | 2 | ||||
| -rwxr-xr-x | test/functional/test_runner.py | 2 | ||||
| -rw-r--r-- | test/util/bctest.py | 139 | ||||
| -rwxr-xr-x | test/util/bitcoin-util-test.py | 167 | ||||
| -rw-r--r-- | test/util/buildenv.py.in | 4 |
5 files changed, 152 insertions, 162 deletions
diff --git a/test/functional/config.ini.in b/test/config.ini.in index 29586c555d..35ee092be4 100644 --- a/test/functional/config.ini.in +++ b/test/config.ini.in @@ -3,7 +3,7 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. # These environment variables are set by the build process and read by -# test/functional/test_runner.py +# test/functional/test_runner.py and test/util/bitcoin-util-test.py [environment] SRCDIR=@abs_top_srcdir@ diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index b2aee7c739..4702f2d773 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -180,7 +180,7 @@ def main(): # Read config generated by configure. config = configparser.ConfigParser() - configfile = os.path.abspath(os.path.dirname(__file__)) + "/config.ini" + configfile = os.path.abspath(os.path.dirname(__file__)) + "/../config.ini" config.read_file(open(configfile)) passon_args.append("--configfile=%s" % configfile) diff --git a/test/util/bctest.py b/test/util/bctest.py deleted file mode 100644 index b17cf77ae3..0000000000 --- a/test/util/bctest.py +++ /dev/null @@ -1,139 +0,0 @@ -# Copyright 2014 BitPay Inc. -# Copyright 2016 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -from __future__ import division,print_function,unicode_literals -import subprocess -import os -import json -import sys -import binascii -import difflib -import logging -import pprint - -def parse_output(a, fmt): - """Parse the output according to specified format. - - Raise an error if the output can't be parsed.""" - if fmt == 'json': # json: compare parsed data - return json.loads(a) - elif fmt == 'hex': # hex: parse and compare binary data - return binascii.a2b_hex(a.strip()) - else: - raise NotImplementedError("Don't know how to compare %s" % fmt) - -def bctest(testDir, testObj, buildenv): - """Runs a single test, comparing output and RC to expected output and RC. - - Raises an error if input can't be read, executable fails, or output/RC - are not as expected. Error is caught by bctester() and reported. - """ - # Get the exec names and arguments - execprog = buildenv.BUILDDIR + "/src/" + testObj['exec'] + buildenv.exeext - execargs = testObj['args'] - execrun = [execprog] + execargs - - # Read the input data (if there is any) - stdinCfg = None - inputData = None - if "input" in testObj: - filename = testDir + "/" + testObj['input'] - inputData = open(filename).read() - stdinCfg = subprocess.PIPE - - # Read the expected output data (if there is any) - outputFn = None - outputData = None - if "output_cmp" in testObj: - outputFn = testObj['output_cmp'] - outputType = os.path.splitext(outputFn)[1][1:] # output type from file extension (determines how to compare) - try: - outputData = open(testDir + "/" + outputFn).read() - except: - logging.error("Output file " + outputFn + " can not be opened") - raise - if not outputData: - logging.error("Output data missing for " + outputFn) - raise Exception - - # Run the test - proc = subprocess.Popen(execrun, stdin=stdinCfg, stdout=subprocess.PIPE, stderr=subprocess.PIPE,universal_newlines=True) - try: - outs = proc.communicate(input=inputData) - except OSError: - logging.error("OSError, Failed to execute " + execprog) - raise - - if outputData: - data_mismatch, formatting_mismatch = False, False - # Parse command output and expected output - try: - a_parsed = parse_output(outs[0], outputType) - except Exception as e: - logging.error('Error parsing command output as %s: %s' % (outputType,e)) - raise - try: - b_parsed = parse_output(outputData, outputType) - except Exception as e: - logging.error('Error parsing expected output %s as %s: %s' % (outputFn,outputType,e)) - raise - # Compare data - if a_parsed != b_parsed: - logging.error("Output data mismatch for " + outputFn + " (format " + outputType + ")") - data_mismatch = True - # Compare formatting - if outs[0] != outputData: - error_message = "Output formatting mismatch for " + outputFn + ":\n" - error_message += "".join(difflib.context_diff(outputData.splitlines(True), - outs[0].splitlines(True), - fromfile=outputFn, - tofile="returned")) - logging.error(error_message) - formatting_mismatch = True - - assert not data_mismatch and not formatting_mismatch - - # Compare the return code to the expected return code - wantRC = 0 - if "return_code" in testObj: - wantRC = testObj['return_code'] - if proc.returncode != wantRC: - logging.error("Return code mismatch for " + outputFn) - raise Exception - - if "error_txt" in testObj: - want_error = testObj["error_txt"] - # Compare error text - # TODO: ideally, we'd compare the strings exactly and also assert - # That stderr is empty if no errors are expected. However, bitcoin-tx - # emits DISPLAY errors when running as a windows application on - # linux through wine. Just assert that the expected error text appears - # somewhere in stderr. - if want_error not in outs[1]: - logging.error("Error mismatch:\n" + "Expected: " + want_error + "\nReceived: " + outs[1].rstrip()) - raise Exception - -def bctester(testDir, input_basename, buildenv): - """ Loads and parses the input file, runs all tests and reports results""" - input_filename = testDir + "/" + input_basename - raw_data = open(input_filename).read() - input_data = json.loads(raw_data) - - failed_testcases = [] - - for testObj in input_data: - try: - bctest(testDir, testObj, buildenv) - logging.info("PASSED: " + testObj["description"]) - except: - logging.info("FAILED: " + testObj["description"]) - failed_testcases.append(testObj["description"]) - - if failed_testcases: - error_message = "FAILED_TESTCASES:\n" - error_message += pprint.pformat(failed_testcases, width=400) - logging.error(error_message) - sys.exit(1) - else: - sys.exit(0) diff --git a/test/util/bitcoin-util-test.py b/test/util/bitcoin-util-test.py index e09a25159d..d15d6a6011 100755 --- a/test/util/bitcoin-util-test.py +++ b/test/util/bitcoin-util-test.py @@ -1,26 +1,30 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright 2014 BitPay Inc. -# Copyright 2016 The Bitcoin Core developers +# Copyright 2016-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -from __future__ import division,print_function,unicode_literals -import os -import sys -import argparse -import logging +"""Test framework for bitcoin utils. -help_text="""Test framework for bitcoin utils. - -Runs automatically during `make check`. +Runs automatically during `make check`. Can also be run manually.""" -if __name__ == '__main__': - sys.path.append(os.path.dirname(os.path.abspath(__file__))) - import buildenv - import bctest +import argparse +import binascii +import configparser +import difflib +import json +import logging +import os +import pprint +import subprocess +import sys - parser = argparse.ArgumentParser(description=help_text) +def main(): + config = configparser.ConfigParser() + config.read_file(open(os.path.dirname(__file__) + "/../config.ini")) + + parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('-v', '--verbose', action='store_true') args = parser.parse_args() verbose = args.verbose @@ -31,6 +35,135 @@ if __name__ == '__main__': level = logging.ERROR formatter = '%(asctime)s - %(levelname)s - %(message)s' # Add the format/level to the logger - logging.basicConfig(format = formatter, level=level) + logging.basicConfig(format=formatter, level=level) + + bctester(config["environment"]["SRCDIR"] + "/test/util/data", "bitcoin-util-test.json", config["environment"]) + +def bctester(testDir, input_basename, buildenv): + """ Loads and parses the input file, runs all tests and reports results""" + input_filename = testDir + "/" + input_basename + raw_data = open(input_filename).read() + input_data = json.loads(raw_data) + + failed_testcases = [] + + for testObj in input_data: + try: + bctest(testDir, testObj, buildenv) + logging.info("PASSED: " + testObj["description"]) + except: + logging.info("FAILED: " + testObj["description"]) + failed_testcases.append(testObj["description"]) - bctest.bctester(buildenv.SRCDIR + "/test/util/data", "bitcoin-util-test.json", buildenv) + if failed_testcases: + error_message = "FAILED_TESTCASES:\n" + error_message += pprint.pformat(failed_testcases, width=400) + logging.error(error_message) + sys.exit(1) + else: + sys.exit(0) + +def bctest(testDir, testObj, buildenv): + """Runs a single test, comparing output and RC to expected output and RC. + + Raises an error if input can't be read, executable fails, or output/RC + are not as expected. Error is caught by bctester() and reported. + """ + # Get the exec names and arguments + execprog = buildenv["BUILDDIR"] + "/src/" + testObj['exec'] + buildenv["EXEEXT"] + execargs = testObj['args'] + execrun = [execprog] + execargs + + # Read the input data (if there is any) + stdinCfg = None + inputData = None + if "input" in testObj: + filename = testDir + "/" + testObj['input'] + inputData = open(filename).read() + stdinCfg = subprocess.PIPE + + # Read the expected output data (if there is any) + outputFn = None + outputData = None + if "output_cmp" in testObj: + outputFn = testObj['output_cmp'] + outputType = os.path.splitext(outputFn)[1][1:] # output type from file extension (determines how to compare) + try: + outputData = open(testDir + "/" + outputFn).read() + except: + logging.error("Output file " + outputFn + " can not be opened") + raise + if not outputData: + logging.error("Output data missing for " + outputFn) + raise Exception + + # Run the test + proc = subprocess.Popen(execrun, stdin=stdinCfg, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) + try: + outs = proc.communicate(input=inputData) + except OSError: + logging.error("OSError, Failed to execute " + execprog) + raise + + if outputData: + data_mismatch, formatting_mismatch = False, False + # Parse command output and expected output + try: + a_parsed = parse_output(outs[0], outputType) + except Exception as e: + logging.error('Error parsing command output as %s: %s' % (outputType, e)) + raise + try: + b_parsed = parse_output(outputData, outputType) + except Exception as e: + logging.error('Error parsing expected output %s as %s: %s' % (outputFn, outputType, e)) + raise + # Compare data + if a_parsed != b_parsed: + logging.error("Output data mismatch for " + outputFn + " (format " + outputType + ")") + data_mismatch = True + # Compare formatting + if outs[0] != outputData: + error_message = "Output formatting mismatch for " + outputFn + ":\n" + error_message += "".join(difflib.context_diff(outputData.splitlines(True), + outs[0].splitlines(True), + fromfile=outputFn, + tofile="returned")) + logging.error(error_message) + formatting_mismatch = True + + assert not data_mismatch and not formatting_mismatch + + # Compare the return code to the expected return code + wantRC = 0 + if "return_code" in testObj: + wantRC = testObj['return_code'] + if proc.returncode != wantRC: + logging.error("Return code mismatch for " + outputFn) + raise Exception + + if "error_txt" in testObj: + want_error = testObj["error_txt"] + # Compare error text + # TODO: ideally, we'd compare the strings exactly and also assert + # That stderr is empty if no errors are expected. However, bitcoin-tx + # emits DISPLAY errors when running as a windows application on + # linux through wine. Just assert that the expected error text appears + # somewhere in stderr. + if want_error not in outs[1]: + logging.error("Error mismatch:\n" + "Expected: " + want_error + "\nReceived: " + outs[1].rstrip()) + raise Exception + +def parse_output(a, fmt): + """Parse the output according to specified format. + + Raise an error if the output can't be parsed.""" + if fmt == 'json': # json: compare parsed data + return json.loads(a) + elif fmt == 'hex': # hex: parse and compare binary data + return binascii.a2b_hex(a.strip()) + else: + raise NotImplementedError("Don't know how to compare %s" % fmt) + +if __name__ == '__main__': + main() diff --git a/test/util/buildenv.py.in b/test/util/buildenv.py.in deleted file mode 100644 index 33030b0348..0000000000 --- a/test/util/buildenv.py.in +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env python -exeext="@EXEEXT@" -SRCDIR="@abs_top_srcdir@" -BUILDDIR="@abs_top_builddir@" |