diff options
Diffstat (limited to 'qa')
-rwxr-xr-x | qa/pull-tester/rpc-tests.py | 251 | ||||
-rw-r--r-- | qa/pull-tester/tests_config.ini.in | 18 | ||||
-rw-r--r-- | qa/pull-tester/tests_config.py.in | 14 | ||||
-rwxr-xr-x | qa/rpc-tests/bip68-112-113-p2p.py | 2 | ||||
-rwxr-xr-x | qa/rpc-tests/bip68-sequence.py | 2 | ||||
-rwxr-xr-x | qa/rpc-tests/bumpfee.py | 11 | ||||
-rwxr-xr-x | qa/rpc-tests/forknotify.py | 2 | ||||
-rwxr-xr-x | qa/rpc-tests/import-rescan.py | 254 | ||||
-rwxr-xr-x | qa/rpc-tests/importmulti.py | 80 | ||||
-rwxr-xr-x | qa/rpc-tests/p2p-compactblocks.py | 1 | ||||
-rwxr-xr-x | qa/rpc-tests/p2p-fullblocktest.py | 4 | ||||
-rwxr-xr-x | qa/rpc-tests/p2p-leaktests.py | 145 | ||||
-rwxr-xr-x | qa/rpc-tests/p2p-segwit.py | 1 | ||||
-rwxr-xr-x | qa/rpc-tests/p2p-timeouts.py | 103 | ||||
-rwxr-xr-x | qa/rpc-tests/receivedby.py | 3 | ||||
-rwxr-xr-x | qa/rpc-tests/replace-by-fee.py | 1 | ||||
-rwxr-xr-x | qa/rpc-tests/rpcnamedargs.py | 8 | ||||
-rwxr-xr-x | qa/rpc-tests/signrawtransactions.py | 34 | ||||
-rwxr-xr-x | qa/rpc-tests/test_framework/mininode.py | 38 | ||||
-rw-r--r-- | qa/rpc-tests/test_framework/util.py | 23 | ||||
-rwxr-xr-x | qa/rpc-tests/wallet-accounts.py | 2 | ||||
-rwxr-xr-x | qa/rpc-tests/wallet.py | 1 |
22 files changed, 712 insertions, 286 deletions
diff --git a/qa/pull-tester/rpc-tests.py b/qa/pull-tester/rpc-tests.py index 26bc6a73df..973165c4c8 100755 --- a/qa/pull-tester/rpc-tests.py +++ b/qa/pull-tester/rpc-tests.py @@ -2,25 +2,21 @@ # Copyright (c) 2014-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. - """ -Run Regression Test Suite +rpc-tests.py - run regression test suite This module calls down into individual test cases via subprocess. It will -forward all unrecognized arguments onto the individual test scripts, other -than: +forward all unrecognized arguments onto the individual test scripts. - - `-extended`: run the "extended" test suite in addition to the basic one. - - `-win`: signal that this is running in a Windows environment, and we - should run the tests. - - `--coverage`: this generates a basic coverage report for the RPC - interface. +RPC tests are disabled on Windows by default. Use --force to run them anyway. For a description of arguments recognized by test scripts, see `qa/pull-tester/test_framework/test_framework.py:BitcoinTestFramework.main`. """ +import argparse +import configparser import os import time import shutil @@ -29,77 +25,9 @@ import subprocess import tempfile import re -sys.path.append("qa/pull-tester/") -from tests_config import * - -BOLD = ("","") -if os.name == 'posix': - # primitive formatting on supported - # terminal via ANSI escape sequences: - BOLD = ('\033[0m', '\033[1m') - -RPC_TESTS_DIR = SRCDIR + '/qa/rpc-tests/' - -#If imported values are not defined then set to zero (or disabled) -if 'ENABLE_WALLET' not in vars(): - ENABLE_WALLET=0 -if 'ENABLE_BITCOIND' not in vars(): - ENABLE_BITCOIND=0 -if 'ENABLE_UTILS' not in vars(): - ENABLE_UTILS=0 -if 'ENABLE_ZMQ' not in vars(): - ENABLE_ZMQ=0 - -ENABLE_COVERAGE=0 - -#Create a set to store arguments and create the passon string -opts = set() -passon_args = [] -PASSON_REGEX = re.compile("^--") -PARALLEL_REGEX = re.compile('^-parallel=') - -print_help = False -run_parallel = 4 - -for arg in sys.argv[1:]: - if arg == "--help" or arg == "-h" or arg == "-?": - print_help = True - break - if arg == '--coverage': - ENABLE_COVERAGE = 1 - elif PASSON_REGEX.match(arg): - passon_args.append(arg) - elif PARALLEL_REGEX.match(arg): - run_parallel = int(arg.split(sep='=', maxsplit=1)[1]) - else: - opts.add(arg) - -#Set env vars -if "BITCOIND" not in os.environ: - os.environ["BITCOIND"] = BUILDDIR + '/src/bitcoind' + EXEEXT - -if EXEEXT == ".exe" and "-win" not in opts: - # https://github.com/bitcoin/bitcoin/commit/d52802551752140cf41f0d9a225a43e84404d3e9 - # https://github.com/bitcoin/bitcoin/pull/5677#issuecomment-136646964 - print("Win tests currently disabled by default. Use -win option to enable") - sys.exit(0) - -if not (ENABLE_WALLET == 1 and ENABLE_UTILS == 1 and ENABLE_BITCOIND == 1): - print("No rpc tests to run. Wallet, utils, and bitcoind must all be enabled") - sys.exit(0) - -# python3-zmq may not be installed. Handle this gracefully and with some helpful info -if ENABLE_ZMQ: - try: - import zmq - except ImportError: - print("ERROR: \"import zmq\" failed. Set ENABLE_ZMQ=0 or " - "to run zmq tests, see dependency info in /qa/README.md.") - # ENABLE_ZMQ=0 - raise - -testScripts = [ - # longest test should go first, to favor running tests in parallel +BASE_SCRIPTS= [ + # Scripts that are run by the travis build process. + # Longest test should go first, to favor running tests in parallel 'wallet-hd.py', 'walletbackup.py', # vv Tests less than 5m vv @@ -154,11 +82,17 @@ testScripts = [ 'bumpfee.py', 'rpcnamedargs.py', 'listsinceblock.py', + 'p2p-leaktests.py', ] -if ENABLE_ZMQ: - testScripts.append('zmq_test.py') -testScriptsExt = [ +ZMQ_SCRIPTS = [ + # ZMQ test can only be run if bitcoin was built with zmq-enabled. + # call rpc_tests.py with -nozmq to explicitly exclude these tests. + "zmq_test.py"] + +EXTENDED_SCRIPTS = [ + # These tests are not run by the travis build process. + # Longest test should go first, to favor running tests in parallel 'pruning.py', # vv Tests less than 20m vv 'smartfees.py', @@ -168,6 +102,7 @@ testScriptsExt = [ # vv Tests less than 2m vv 'bip68-sequence.py', 'getblocktemplate_longpoll.py', + 'p2p-timeouts.py', # vv Tests less than 60s vv 'bip9-softforks.py', 'p2p-feefilter.py', @@ -187,44 +122,126 @@ testScriptsExt = [ 'replace-by-fee.py', ] +ALL_SCRIPTS = BASE_SCRIPTS + ZMQ_SCRIPTS + EXTENDED_SCRIPTS + +def main(): + # Parse arguments and pass through unrecognised args + parser = argparse.ArgumentParser(add_help=False, + usage='%(prog)s [rpc-test.py options] [script options] [scripts]', + description=__doc__, + epilog=''' + Help text and arguments for individual test script:''', + formatter_class=argparse.RawTextHelpFormatter) + parser.add_argument('--coverage', action='store_true', help='generate a basic coverage report for the RPC interface') + parser.add_argument('--extended', action='store_true', help='run the extended test suite in addition to the basic tests') + parser.add_argument('--force', '-f', action='store_true', help='run tests even on platforms where they are disabled by default (e.g. windows).') + parser.add_argument('--help', '-h', '-?', action='store_true', help='print help text and exit') + parser.add_argument('--jobs', '-j', type=int, default=4, help='how many test scripts to run in parallel. Default=4.') + parser.add_argument('--nozmq', action='store_true', help='do not run the zmq tests') + args, unknown_args = parser.parse_known_args() + + # Create a set to store arguments and create the passon string + tests = set(arg for arg in unknown_args if arg[:2] != "--") + passon_args = [arg for arg in unknown_args if arg[:2] == "--"] + + # Read config generated by configure. + config = configparser.ConfigParser() + config.read_file(open(os.path.dirname(__file__) + "/tests_config.ini")) + + enable_wallet = config["components"].getboolean("ENABLE_WALLET") + enable_utils = config["components"].getboolean("ENABLE_UTILS") + enable_bitcoind = config["components"].getboolean("ENABLE_BITCOIND") + enable_zmq = config["components"].getboolean("ENABLE_ZMQ") and not args.nozmq + + if config["environment"]["EXEEXT"] == ".exe" and not args.force: + # https://github.com/bitcoin/bitcoin/commit/d52802551752140cf41f0d9a225a43e84404d3e9 + # https://github.com/bitcoin/bitcoin/pull/5677#issuecomment-136646964 + print("Tests currently disabled on Windows by default. Use --force option to enable") + sys.exit(0) -def runtests(): - test_list = [] - if '-extended' in opts: - test_list = testScripts + testScriptsExt - elif len(opts) == 0 or (len(opts) == 1 and "-win" in opts): - test_list = testScripts - else: - for t in testScripts + testScriptsExt: - if t in opts or re.sub(".py$", "", t) in opts: - test_list.append(t) + if not (enable_wallet and enable_utils and enable_bitcoind): + print("No rpc tests to run. Wallet, utils, and bitcoind must all be enabled") + print("Rerun `configure` with -enable-wallet, -with-utils and -with-daemon and rerun make") + sys.exit(0) + + # python3-zmq may not be installed. Handle this gracefully and with some helpful info + if enable_zmq: + try: + import zmq + except ImportError: + print("ERROR: \"import zmq\" failed. Use -nozmq to run without the ZMQ tests." + "To run zmq tests, see dependency info in /qa/README.md.") + raise + + # Build list of tests + if tests: + # Individual tests have been specified. Run specified tests that exist + # in the ALL_SCRIPTS list. Accept the name with or without .py extension. + test_list = [t for t in ALL_SCRIPTS if + (t in tests or re.sub(".py$", "", t) in tests)] + if not test_list: + print("No valid test scripts specified. Check that your test is in one " + "of the test lists in rpc-tests.py or run rpc-tests.py with no arguments to run all tests") + print("Scripts not found:") + print(tests) + sys.exit(0) - if print_help: - # Only print help of the first script and exit - subprocess.check_call((RPC_TESTS_DIR + test_list[0]).split() + ['-h']) + else: + # No individual tests have been specified. Run base tests, and + # optionally ZMQ tests and extended tests. + test_list = BASE_SCRIPTS + if enable_zmq: + test_list += ZMQ_SCRIPTS + if args.extended: + test_list += EXTENDED_SCRIPTS + # TODO: BASE_SCRIPTS and EXTENDED_SCRIPTS are sorted by runtime + # (for parallel running efficiency). This combined list will is no + # longer sorted. + + if args.help: + # Print help for rpc-tests.py, then print help of the first script and exit. + parser.print_help() + subprocess.check_call((config["environment"]["SRCDIR"] + '/qa/rpc-tests/' + test_list[0]).split() + ['-h']) sys.exit(0) - coverage = None + run_tests(test_list, config["environment"]["SRCDIR"], config["environment"]["BUILDDIR"], config["environment"]["EXEEXT"], args.jobs, args.coverage, passon_args) + +def run_tests(test_list, src_dir, build_dir, exeext, jobs=1, enable_coverage=False, args=[]): + BOLD = ("","") + if os.name == 'posix': + # primitive formatting on supported + # terminal via ANSI escape sequences: + BOLD = ('\033[0m', '\033[1m') + + #Set env vars + if "BITCOIND" not in os.environ: + os.environ["BITCOIND"] = build_dir + '/src/bitcoind' + exeext + + tests_dir = src_dir + '/qa/rpc-tests/' - if ENABLE_COVERAGE: + flags = ["--srcdir=" + src_dir] + args + flags.append("--cachedir=%s/qa/cache" % build_dir) + + if enable_coverage: coverage = RPCCoverage() - print("Initializing coverage directory at %s\n" % coverage.dir) - flags = ["--srcdir=%s/src" % BUILDDIR] + passon_args - flags.append("--cachedir=%s/qa/cache" % BUILDDIR) - if coverage: flags.append(coverage.flag) + print("Initializing coverage directory at %s\n" % coverage.dir) + else: + coverage = None - if len(test_list) > 1 and run_parallel > 1: + if len(test_list) > 1 and jobs > 1: # Populate cache - subprocess.check_output([RPC_TESTS_DIR + 'create_cache.py'] + flags) + subprocess.check_output([tests_dir + 'create_cache.py'] + flags) #Run Tests - max_len_name = len(max(test_list, key=len)) + all_passed = True time_sum = 0 time0 = time.time() - job_queue = RPCTestHandler(run_parallel, test_list, flags) + + job_queue = RPCTestHandler(jobs, tests_dir, test_list, flags) + + max_len_name = len(max(test_list, key=len)) results = BOLD[1] + "%s | %s | %s\n\n" % ("TEST".ljust(max_len_name), "PASSED", "DURATION") + BOLD[0] - all_passed = True for _ in range(len(test_list)): (name, stdout, stderr, passed, duration) = job_queue.get_next() all_passed = all_passed and passed @@ -233,8 +250,10 @@ def runtests(): print('\n' + BOLD[1] + name + BOLD[0] + ":") print('' if passed else stdout + '\n', end='') print('' if stderr == '' else 'stderr:\n' + stderr + '\n', end='') - results += "%s | %s | %s s\n" % (name.ljust(max_len_name), str(passed).ljust(6), duration) print("Pass: %s%s%s, Duration: %s s\n" % (BOLD[1], passed, BOLD[0], duration)) + + results += "%s | %s | %s s\n" % (name.ljust(max_len_name), str(passed).ljust(6), duration) + results += BOLD[1] + "\n%s | %s | %s s (accumulated)" % ("ALL".ljust(max_len_name), str(all_passed).ljust(6), time_sum) + BOLD[0] print(results) print("\nRuntime: %s s" % (int(time.time() - time0))) @@ -247,15 +266,15 @@ def runtests(): sys.exit(not all_passed) - class RPCTestHandler: """ Trigger the testscrips passed in via the list. """ - def __init__(self, num_tests_parallel, test_list=None, flags=None): + def __init__(self, num_tests_parallel, tests_dir, test_list=None, flags=None): assert(num_tests_parallel >= 1) self.num_jobs = num_tests_parallel + self.tests_dir = tests_dir self.test_list = test_list self.flags = flags self.num_running = 0 @@ -275,7 +294,7 @@ class RPCTestHandler: log_stderr = tempfile.SpooledTemporaryFile(max_size=2**16) self.jobs.append((t, time.time(), - subprocess.Popen((RPC_TESTS_DIR + t).split() + self.flags + port_seed, + subprocess.Popen((self.tests_dir + t).split() + self.flags + port_seed, universal_newlines=True, stdout=log_stdout, stderr=log_stderr), @@ -340,10 +359,10 @@ class RPCCoverage(object): """ # This is shared from `qa/rpc-tests/test-framework/coverage.py` - REFERENCE_FILENAME = 'rpc_interface.txt' - COVERAGE_FILE_PREFIX = 'coverage.' + reference_filename = 'rpc_interface.txt' + coverage_file_prefix = 'coverage.' - coverage_ref_filename = os.path.join(self.dir, REFERENCE_FILENAME) + coverage_ref_filename = os.path.join(self.dir, reference_filename) coverage_filenames = set() all_cmds = set() covered_cmds = set() @@ -356,7 +375,7 @@ class RPCCoverage(object): for root, dirs, files in os.walk(self.dir): for filename in files: - if filename.startswith(COVERAGE_FILE_PREFIX): + if filename.startswith(coverage_file_prefix): coverage_filenames.add(os.path.join(root, filename)) for filename in coverage_filenames: @@ -367,4 +386,4 @@ class RPCCoverage(object): if __name__ == '__main__': - runtests() + main() diff --git a/qa/pull-tester/tests_config.ini.in b/qa/pull-tester/tests_config.ini.in new file mode 100644 index 0000000000..e3e457d0b1 --- /dev/null +++ b/qa/pull-tester/tests_config.ini.in @@ -0,0 +1,18 @@ +# Copyright (c) 2013-2016 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +# These environment variables are set by the build process and read by +# rpc-tests.py + +[environment] +SRCDIR=@abs_top_srcdir@ +BUILDDIR=@abs_top_builddir@ +EXEEXT=@EXEEXT@ + +[components] +# Which components are enabled. These are commented out by `configure` if they were disabled when running config. +@ENABLE_WALLET_TRUE@ENABLE_WALLET=true +@BUILD_BITCOIN_UTILS_TRUE@ENABLE_UTILS=true +@BUILD_BITCOIND_TRUE@ENABLE_BITCOIND=true +@ENABLE_ZMQ_TRUE@ENABLE_ZMQ=true diff --git a/qa/pull-tester/tests_config.py.in b/qa/pull-tester/tests_config.py.in deleted file mode 100644 index a0d0a3d98a..0000000000 --- a/qa/pull-tester/tests_config.py.in +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2013-2016 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -SRCDIR="@abs_top_srcdir@" -BUILDDIR="@abs_top_builddir@" -EXEEXT="@EXEEXT@" - -# These will turn into comments if they were disabled when configuring. -@ENABLE_WALLET_TRUE@ENABLE_WALLET=1 -@BUILD_BITCOIN_UTILS_TRUE@ENABLE_UTILS=1 -@BUILD_BITCOIND_TRUE@ENABLE_BITCOIND=1 -@ENABLE_ZMQ_TRUE@ENABLE_ZMQ=1 diff --git a/qa/rpc-tests/bip68-112-113-p2p.py b/qa/rpc-tests/bip68-112-113-p2p.py index c96b746c9d..e03bc1c37b 100755 --- a/qa/rpc-tests/bip68-112-113-p2p.py +++ b/qa/rpc-tests/bip68-112-113-p2p.py @@ -5,7 +5,7 @@ from test_framework.test_framework import ComparisonTestFramework from test_framework.util import * -from test_framework.mininode import ToHex, CTransaction, NetworkThread +from test_framework.mininode import ToHex, NetworkThread from test_framework.blocktools import create_coinbase, create_block from test_framework.comptool import TestInstance, TestManager from test_framework.script import * diff --git a/qa/rpc-tests/bip68-sequence.py b/qa/rpc-tests/bip68-sequence.py index a12bf10ebd..1b099f9339 100755 --- a/qa/rpc-tests/bip68-sequence.py +++ b/qa/rpc-tests/bip68-sequence.py @@ -9,8 +9,6 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * -from test_framework.script import * -from test_framework.mininode import * from test_framework.blocktools import * SEQUENCE_LOCKTIME_DISABLE_FLAG = (1<<31) diff --git a/qa/rpc-tests/bumpfee.py b/qa/rpc-tests/bumpfee.py index 0ebd79f7f3..e02cb4c300 100755 --- a/qa/rpc-tests/bumpfee.py +++ b/qa/rpc-tests/bumpfee.py @@ -8,10 +8,8 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework import blocktools from test_framework.mininode import CTransaction from test_framework.util import * -from test_framework.util import * import io -import time # Sequence number that is BIP 125 opt-in and BIP 68-compliant BIP125_SEQUENCE_NUMBER = 0xfffffffd @@ -69,6 +67,7 @@ class BumpFeeTest(BitcoinTestFramework): test_rebumping(rbf_node, dest_address) test_rebumping_not_replaceable(rbf_node, dest_address) test_unconfirmed_not_spendable(rbf_node, rbf_node_address) + test_bumpfee_metadata(rbf_node, dest_address) test_locked_wallet_fails(rbf_node, dest_address) print("Success") @@ -257,6 +256,14 @@ def test_unconfirmed_not_spendable(rbf_node, rbf_node_address): if t["txid"] == rbfid and t["address"] == rbf_node_address and t["spendable"]), 1) +def test_bumpfee_metadata(rbf_node, dest_address): + rbfid = rbf_node.sendtoaddress(dest_address, 0.00090000, "comment value", "to value") + bumped_tx = rbf_node.bumpfee(rbfid) + bumped_wtx = rbf_node.gettransaction(bumped_tx["txid"]) + assert_equal(bumped_wtx["comment"], "comment value") + assert_equal(bumped_wtx["to"], "to value") + + def test_locked_wallet_fails(rbf_node, dest_address): rbfid = create_fund_sign_send(rbf_node, {dest_address: 0.00090000}) rbf_node.walletlock() diff --git a/qa/rpc-tests/forknotify.py b/qa/rpc-tests/forknotify.py index a1901aedab..8b1403083f 100755 --- a/qa/rpc-tests/forknotify.py +++ b/qa/rpc-tests/forknotify.py @@ -22,7 +22,7 @@ class ForkNotifyTest(BitcoinTestFramework): def setup_network(self): self.nodes = [] self.alert_filename = os.path.join(self.options.tmpdir, "alert.txt") - with open(self.alert_filename, 'w', encoding='utf8') as f: + with open(self.alert_filename, 'w', encoding='utf8'): pass # Just open then close to create zero-length file self.nodes.append(start_node(0, self.options.tmpdir, ["-blockversion=2", "-alertnotify=echo %s >> \"" + self.alert_filename + "\""])) diff --git a/qa/rpc-tests/import-rescan.py b/qa/rpc-tests/import-rescan.py index e683df26db..54cc6d2642 100755 --- a/qa/rpc-tests/import-rescan.py +++ b/qa/rpc-tests/import-rescan.py @@ -2,54 +2,105 @@ # Copyright (c) 2014-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. - +"""Test rescan behavior of importaddress, importpubkey, importprivkey, and +importmulti RPCs with different types of keys and rescan options. + +In the first part of the test, node 0 creates an address for each type of +import RPC call and sends BTC to it. Then other nodes import the addresses, +and the test makes listtransactions and getbalance calls to confirm that the +importing node either did or did not execute rescans picking up the send +transactions. + +In the second part of the test, node 0 sends more BTC to each address, and the +test makes more listtransactions and getbalance calls to confirm that the +importing nodes pick up the new transactions regardless of whether rescans +happened previously. +""" + +from test_framework.authproxy import JSONRPCException from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import (start_nodes, connect_nodes, sync_blocks, assert_equal) +from test_framework.util import (start_nodes, connect_nodes, sync_blocks, assert_equal, set_node_times) from decimal import Decimal import collections import enum import itertools -import functools Call = enum.Enum("Call", "single multi") Data = enum.Enum("Data", "address pub priv") -ImportNode = collections.namedtuple("ImportNode", "rescan") - - -def call_import_rpc(call, data, address, scriptPubKey, pubkey, key, label, node, rescan): - """Helper that calls a wallet import RPC on a bitcoin node.""" - watchonly = data != Data.priv - if call == Call.single: - if data == Data.address: - response = node.importaddress(address, label, rescan) - elif data == Data.pub: - response = node.importpubkey(pubkey, label, rescan) - elif data == Data.priv: - response = node.importprivkey(key, label, rescan) - assert_equal(response, None) - elif call == Call.multi: - response = node.importmulti([{ - "scriptPubKey": { - "address": address - }, - "pubkeys": [pubkey] if data == Data.pub else [], - "keys": [key] if data == Data.priv else [], - "label": label, - "watchonly": watchonly - }], {"rescan": rescan}) - assert_equal(response, [{"success": True}]) - return watchonly - - -# List of RPCs that import a wallet key or address in various ways. -IMPORT_RPCS = [functools.partial(call_import_rpc, call, data) for call, data in itertools.product(Call, Data)] - -# List of bitcoind nodes that will import keys. -IMPORT_NODES = [ - ImportNode(rescan=True), - ImportNode(rescan=False), -] +Rescan = enum.Enum("Rescan", "no yes late_timestamp") + + +class Variant(collections.namedtuple("Variant", "call data rescan prune")): + """Helper for importing one key and verifying scanned transactions.""" + + def do_import(self, timestamp): + """Call one key import RPC.""" + + if self.call == Call.single: + if self.data == Data.address: + response, error = try_rpc(self.node.importaddress, self.address["address"], self.label, + self.rescan == Rescan.yes) + elif self.data == Data.pub: + response, error = try_rpc(self.node.importpubkey, self.address["pubkey"], self.label, + self.rescan == Rescan.yes) + elif self.data == Data.priv: + response, error = try_rpc(self.node.importprivkey, self.key, self.label, self.rescan == Rescan.yes) + assert_equal(response, None) + assert_equal(error, {'message': 'Rescan is disabled in pruned mode', + 'code': -4} if self.expect_disabled else None) + elif self.call == Call.multi: + response = self.node.importmulti([{ + "scriptPubKey": { + "address": self.address["address"] + }, + "timestamp": timestamp + RESCAN_WINDOW + (1 if self.rescan == Rescan.late_timestamp else 0), + "pubkeys": [self.address["pubkey"]] if self.data == Data.pub else [], + "keys": [self.key] if self.data == Data.priv else [], + "label": self.label, + "watchonly": self.data != Data.priv + }], {"rescan": self.rescan in (Rescan.yes, Rescan.late_timestamp)}) + assert_equal(response, [{"success": True}]) + + def check(self, txid=None, amount=None, confirmations=None): + """Verify that getbalance/listtransactions return expected values.""" + + balance = self.node.getbalance(self.label, 0, True) + assert_equal(balance, self.expected_balance) + + txs = self.node.listtransactions(self.label, 10000, 0, True) + assert_equal(len(txs), self.expected_txs) + + if txid is not None: + tx, = [tx for tx in txs if tx["txid"] == txid] + assert_equal(tx["account"], self.label) + assert_equal(tx["address"], self.address["address"]) + assert_equal(tx["amount"], amount) + assert_equal(tx["category"], "receive") + assert_equal(tx["label"], self.label) + assert_equal(tx["txid"], txid) + assert_equal(tx["confirmations"], confirmations) + assert_equal("trusted" not in tx, True) + if self.data != Data.priv: + assert_equal(tx["involvesWatchonly"], True) + else: + assert_equal("involvesWatchonly" not in tx, True) + + +# List of Variants for each way a key or address could be imported. +IMPORT_VARIANTS = [Variant(*variants) for variants in itertools.product(Call, Data, Rescan, (False, True))] + +# List of nodes to import keys to. Half the nodes will have pruning disabled, +# half will have it enabled. Different nodes will be used for imports that are +# expected to cause rescans, and imports that are not expected to cause +# rescans, in order to prevent rescans during later imports picking up +# transactions associated with earlier imports. This makes it easier to keep +# track of expected balances and transactions. +ImportNode = collections.namedtuple("ImportNode", "prune rescan") +IMPORT_NODES = [ImportNode(*fields) for fields in itertools.product((False, True), repeat=2)] + +# Rescans start at the earliest block up to 2 hours before the key timestamp. +RESCAN_WINDOW = 2 * 60 * 60 class ImportRescanTest(BitcoinTestFramework): @@ -59,6 +110,10 @@ class ImportRescanTest(BitcoinTestFramework): def setup_network(self): extra_args = [["-debug=1"] for _ in range(self.num_nodes)] + for i, import_node in enumerate(IMPORT_NODES, 1): + if import_node.prune: + extra_args[i] += ["-prune=1"] + self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, extra_args) for i in range(1, self.num_nodes): connect_nodes(self.nodes[i], 0) @@ -66,89 +121,64 @@ class ImportRescanTest(BitcoinTestFramework): def run_test(self): # Create one transaction on node 0 with a unique amount and label for # each possible type of wallet import RPC. - import_rpc_variants = [] - for i, import_rpc in enumerate(IMPORT_RPCS): - label = "label{}".format(i) - addr = self.nodes[0].validateaddress(self.nodes[0].getnewaddress(label)) - key = self.nodes[0].dumpprivkey(addr["address"]) - amount = 24.9375 - i * .0625 - txid = self.nodes[0].sendtoaddress(addr["address"], amount) - import_rpc = functools.partial(import_rpc, addr["address"], addr["scriptPubKey"], addr["pubkey"], key, - label) - import_rpc_variants.append((import_rpc, label, amount, txid, addr)) - + for i, variant in enumerate(IMPORT_VARIANTS): + variant.label = "label {} {}".format(i, variant) + variant.address = self.nodes[0].validateaddress(self.nodes[0].getnewaddress(variant.label)) + variant.key = self.nodes[0].dumpprivkey(variant.address["address"]) + variant.initial_amount = 25 - (i + 1) / 4.0 + variant.initial_txid = self.nodes[0].sendtoaddress(variant.address["address"], variant.initial_amount) + + # Generate a block containing the initial transactions, then another + # block further in the future (past the rescan window). self.nodes[0].generate(1) assert_equal(self.nodes[0].getrawmempool(), []) + timestamp = self.nodes[0].getblockheader(self.nodes[0].getbestblockhash())["time"] + set_node_times(self.nodes, timestamp + RESCAN_WINDOW + 1) + self.nodes[0].generate(1) sync_blocks(self.nodes) - # For each importing node and variation of wallet import RPC, invoke - # the RPC and check the results from getbalance and listtransactions. - for node, import_node in zip(self.nodes[1:], IMPORT_NODES): - for import_rpc, label, amount, txid, addr in import_rpc_variants: - watchonly = import_rpc(node, import_node.rescan) - - balance = node.getbalance(label, 0, True) - if import_node.rescan: - assert_equal(balance, amount) - else: - assert_equal(balance, 0) - - txs = node.listtransactions(label, 10000, 0, True) - if import_node.rescan: - assert_equal(len(txs), 1) - assert_equal(txs[0]["account"], label) - assert_equal(txs[0]["address"], addr["address"]) - assert_equal(txs[0]["amount"], amount) - assert_equal(txs[0]["category"], "receive") - assert_equal(txs[0]["label"], label) - assert_equal(txs[0]["txid"], txid) - assert_equal(txs[0]["confirmations"], 1) - assert_equal("trusted" not in txs[0], True) - if watchonly: - assert_equal(txs[0]["involvesWatchonly"], True) - else: - assert_equal("involvesWatchonly" not in txs[0], True) - else: - assert_equal(len(txs), 0) - - # Create spends for all the imported addresses. - spend_txids = [] + # For each variation of wallet key import, invoke the import RPC and + # check the results from getbalance and listtransactions. + for variant in IMPORT_VARIANTS: + variant.expect_disabled = variant.rescan == Rescan.yes and variant.prune and variant.call == Call.single + expect_rescan = variant.rescan == Rescan.yes and not variant.expect_disabled + variant.node = self.nodes[1 + IMPORT_NODES.index(ImportNode(variant.prune, expect_rescan))] + variant.do_import(timestamp) + if expect_rescan: + variant.expected_balance = variant.initial_amount + variant.expected_txs = 1 + variant.check(variant.initial_txid, variant.initial_amount, 2) + else: + variant.expected_balance = 0 + variant.expected_txs = 0 + variant.check() + + # Create new transactions sending to each address. fee = self.nodes[0].getnetworkinfo()["relayfee"] - for import_rpc, label, amount, txid, addr in import_rpc_variants: - raw_tx = self.nodes[0].getrawtransaction(txid) - decoded_tx = self.nodes[0].decoderawtransaction(raw_tx) - input_vout = next(out["n"] for out in decoded_tx["vout"] - if out["scriptPubKey"]["addresses"] == [addr["address"]]) - inputs = [{"txid": txid, "vout": input_vout}] - outputs = {self.nodes[0].getnewaddress(): Decimal(amount) - fee} - raw_spend_tx = self.nodes[0].createrawtransaction(inputs, outputs) - signed_spend_tx = self.nodes[0].signrawtransaction(raw_spend_tx) - spend_txid = self.nodes[0].sendrawtransaction(signed_spend_tx["hex"]) - spend_txids.append(spend_txid) + for i, variant in enumerate(IMPORT_VARIANTS): + variant.sent_amount = 25 - (2 * i + 1) / 8.0 + variant.sent_txid = self.nodes[0].sendtoaddress(variant.address["address"], variant.sent_amount) + # Generate a block containing the new transactions. self.nodes[0].generate(1) assert_equal(self.nodes[0].getrawmempool(), []) sync_blocks(self.nodes) - # Check the results from getbalance and listtransactions after the spends. - for node, import_node in zip(self.nodes[1:], IMPORT_NODES): - txs = node.listtransactions("*", 10000, 0, True) - for (import_rpc, label, amount, txid, addr), spend_txid in zip(import_rpc_variants, spend_txids): - balance = node.getbalance(label, 0, True) - spend_tx = [tx for tx in txs if tx["txid"] == spend_txid] - if import_node.rescan: - assert_equal(balance, amount) - assert_equal(len(spend_tx), 1) - assert_equal(spend_tx[0]["account"], "") - assert_equal(spend_tx[0]["amount"] + spend_tx[0]["fee"], -amount) - assert_equal(spend_tx[0]["category"], "send") - assert_equal("label" not in spend_tx[0], True) - assert_equal(spend_tx[0]["confirmations"], 1) - assert_equal("trusted" not in spend_tx[0], True) - assert_equal("involvesWatchonly" not in txs[0], True) - else: - assert_equal(balance, 0) - assert_equal(spend_tx, []) + # Check the latest results from getbalance and listtransactions. + for variant in IMPORT_VARIANTS: + if not variant.expect_disabled: + variant.expected_balance += variant.sent_amount + variant.expected_txs += 1 + variant.check(variant.sent_txid, variant.sent_amount, 1) + else: + variant.check() + + +def try_rpc(func, *args, **kwargs): + try: + return func(*args, **kwargs), None + except JSONRPCException as e: + return None, e.error if __name__ == "__main__": diff --git a/qa/rpc-tests/importmulti.py b/qa/rpc-tests/importmulti.py index e100a3af9d..1aa4ba2e18 100755 --- a/qa/rpc-tests/importmulti.py +++ b/qa/rpc-tests/importmulti.py @@ -20,6 +20,7 @@ class ImportMultiTest (BitcoinTestFramework): print ("Mining blocks...") self.nodes[0].generate(1) self.nodes[1].generate(1) + timestamp = self.nodes[1].getblock(self.nodes[1].getbestblockhash())['mediantime'] # keyword definition PRIV_KEY = 'privkey' @@ -52,31 +53,48 @@ class ImportMultiTest (BitcoinTestFramework): result = self.nodes[1].importmulti([{ "scriptPubKey": { "address": address['address'] - } + }, + "timestamp": "now", }]) assert_equal(result[0]['success'], True) address_assert = self.nodes[1].validateaddress(address['address']) assert_equal(address_assert['iswatchonly'], True) assert_equal(address_assert['ismine'], False) + assert_equal(address_assert['timestamp'], timestamp) + watchonly_address = address['address'] + watchonly_timestamp = timestamp + print("Should not import an invalid address") + result = self.nodes[1].importmulti([{ + "scriptPubKey": { + "address": "not valid address", + }, + "timestamp": "now", + }]) + assert_equal(result[0]['success'], False) + assert_equal(result[0]['error']['code'], -5) + assert_equal(result[0]['error']['message'], 'Invalid address') # ScriptPubKey + internal print("Should import a scriptPubKey with internal flag") address = self.nodes[0].validateaddress(self.nodes[0].getnewaddress()) result = self.nodes[1].importmulti([{ "scriptPubKey": address['scriptPubKey'], + "timestamp": "now", "internal": True }]) assert_equal(result[0]['success'], True) address_assert = self.nodes[1].validateaddress(address['address']) assert_equal(address_assert['iswatchonly'], True) assert_equal(address_assert['ismine'], False) + assert_equal(address_assert['timestamp'], timestamp) # ScriptPubKey + !internal print("Should not import a scriptPubKey without internal flag") address = self.nodes[0].validateaddress(self.nodes[0].getnewaddress()) result = self.nodes[1].importmulti([{ - "scriptPubKey": address['scriptPubKey'] + "scriptPubKey": address['scriptPubKey'], + "timestamp": "now", }]) assert_equal(result[0]['success'], False) assert_equal(result[0]['error']['code'], -8) @@ -84,6 +102,7 @@ class ImportMultiTest (BitcoinTestFramework): address_assert = self.nodes[1].validateaddress(address['address']) assert_equal(address_assert['iswatchonly'], False) assert_equal(address_assert['ismine'], False) + assert_equal('timestamp' in address_assert, False) # Address + Public key + !Internal @@ -93,12 +112,14 @@ class ImportMultiTest (BitcoinTestFramework): "scriptPubKey": { "address": address['address'] }, + "timestamp": "now", "pubkeys": [ address['pubkey'] ] }]) assert_equal(result[0]['success'], True) address_assert = self.nodes[1].validateaddress(address['address']) assert_equal(address_assert['iswatchonly'], True) assert_equal(address_assert['ismine'], False) + assert_equal(address_assert['timestamp'], timestamp) # ScriptPubKey + Public key + internal @@ -106,6 +127,7 @@ class ImportMultiTest (BitcoinTestFramework): address = self.nodes[0].validateaddress(self.nodes[0].getnewaddress()) request = [{ "scriptPubKey": address['scriptPubKey'], + "timestamp": "now", "pubkeys": [ address['pubkey'] ], "internal": True }] @@ -114,12 +136,14 @@ class ImportMultiTest (BitcoinTestFramework): address_assert = self.nodes[1].validateaddress(address['address']) assert_equal(address_assert['iswatchonly'], True) assert_equal(address_assert['ismine'], False) + assert_equal(address_assert['timestamp'], timestamp) # ScriptPubKey + Public key + !internal print("Should not import a scriptPubKey without internal and with public key") address = self.nodes[0].validateaddress(self.nodes[0].getnewaddress()) request = [{ "scriptPubKey": address['scriptPubKey'], + "timestamp": "now", "pubkeys": [ address['pubkey'] ] }] result = self.nodes[1].importmulti(request) @@ -129,6 +153,7 @@ class ImportMultiTest (BitcoinTestFramework): address_assert = self.nodes[1].validateaddress(address['address']) assert_equal(address_assert['iswatchonly'], False) assert_equal(address_assert['ismine'], False) + assert_equal('timestamp' in address_assert, False) # Address + Private key + !watchonly print("Should import an address with private key") @@ -137,12 +162,14 @@ class ImportMultiTest (BitcoinTestFramework): "scriptPubKey": { "address": address['address'] }, + "timestamp": "now", "keys": [ self.nodes[0].dumpprivkey(address['address']) ] }]) assert_equal(result[0]['success'], True) address_assert = self.nodes[1].validateaddress(address['address']) assert_equal(address_assert['iswatchonly'], False) assert_equal(address_assert['ismine'], True) + assert_equal(address_assert['timestamp'], timestamp) # Address + Private key + watchonly print("Should not import an address with private key and with watchonly") @@ -151,6 +178,7 @@ class ImportMultiTest (BitcoinTestFramework): "scriptPubKey": { "address": address['address'] }, + "timestamp": "now", "keys": [ self.nodes[0].dumpprivkey(address['address']) ], "watchonly": True }]) @@ -160,12 +188,14 @@ class ImportMultiTest (BitcoinTestFramework): address_assert = self.nodes[1].validateaddress(address['address']) assert_equal(address_assert['iswatchonly'], False) assert_equal(address_assert['ismine'], False) + assert_equal('timestamp' in address_assert, False) # ScriptPubKey + Private key + internal print("Should import a scriptPubKey with internal and with private key") address = self.nodes[0].validateaddress(self.nodes[0].getnewaddress()) result = self.nodes[1].importmulti([{ "scriptPubKey": address['scriptPubKey'], + "timestamp": "now", "keys": [ self.nodes[0].dumpprivkey(address['address']) ], "internal": True }]) @@ -173,12 +203,14 @@ class ImportMultiTest (BitcoinTestFramework): address_assert = self.nodes[1].validateaddress(address['address']) assert_equal(address_assert['iswatchonly'], False) assert_equal(address_assert['ismine'], True) + assert_equal(address_assert['timestamp'], timestamp) # ScriptPubKey + Private key + !internal print("Should not import a scriptPubKey without internal and with private key") address = self.nodes[0].validateaddress(self.nodes[0].getnewaddress()) result = self.nodes[1].importmulti([{ "scriptPubKey": address['scriptPubKey'], + "timestamp": "now", "keys": [ self.nodes[0].dumpprivkey(address['address']) ] }]) assert_equal(result[0]['success'], False) @@ -187,6 +219,7 @@ class ImportMultiTest (BitcoinTestFramework): address_assert = self.nodes[1].validateaddress(address['address']) assert_equal(address_assert['iswatchonly'], False) assert_equal(address_assert['ismine'], False) + assert_equal('timestamp' in address_assert, False) # P2SH address @@ -197,18 +230,21 @@ class ImportMultiTest (BitcoinTestFramework): self.nodes[1].generate(100) transactionid = self.nodes[1].sendtoaddress(multi_sig_script['address'], 10.00) self.nodes[1].generate(1) + timestamp = self.nodes[1].getblock(self.nodes[1].getbestblockhash())['mediantime'] transaction = self.nodes[1].gettransaction(transactionid) print("Should import a p2sh") result = self.nodes[1].importmulti([{ "scriptPubKey": { "address": multi_sig_script['address'] - } + }, + "timestamp": "now", }]) assert_equal(result[0]['success'], True) address_assert = self.nodes[1].validateaddress(multi_sig_script['address']) assert_equal(address_assert['isscript'], True) assert_equal(address_assert['iswatchonly'], True) + assert_equal(address_assert['timestamp'], timestamp) p2shunspent = self.nodes[1].listunspent(0,999999, [multi_sig_script['address']])[0] assert_equal(p2shunspent['spendable'], False) assert_equal(p2shunspent['solvable'], False) @@ -222,6 +258,7 @@ class ImportMultiTest (BitcoinTestFramework): self.nodes[1].generate(100) transactionid = self.nodes[1].sendtoaddress(multi_sig_script['address'], 10.00) self.nodes[1].generate(1) + timestamp = self.nodes[1].getblock(self.nodes[1].getbestblockhash())['mediantime'] transaction = self.nodes[1].gettransaction(transactionid) print("Should import a p2sh with respective redeem script") @@ -229,9 +266,12 @@ class ImportMultiTest (BitcoinTestFramework): "scriptPubKey": { "address": multi_sig_script['address'] }, + "timestamp": "now", "redeemscript": multi_sig_script['redeemScript'] }]) assert_equal(result[0]['success'], True) + address_assert = self.nodes[1].validateaddress(multi_sig_script['address']) + assert_equal(address_assert['timestamp'], timestamp) p2shunspent = self.nodes[1].listunspent(0,999999, [multi_sig_script['address']])[0] assert_equal(p2shunspent['spendable'], False) @@ -246,6 +286,7 @@ class ImportMultiTest (BitcoinTestFramework): self.nodes[1].generate(100) transactionid = self.nodes[1].sendtoaddress(multi_sig_script['address'], 10.00) self.nodes[1].generate(1) + timestamp = self.nodes[1].getblock(self.nodes[1].getbestblockhash())['mediantime'] transaction = self.nodes[1].gettransaction(transactionid) print("Should import a p2sh with respective redeem script and private keys") @@ -253,10 +294,13 @@ class ImportMultiTest (BitcoinTestFramework): "scriptPubKey": { "address": multi_sig_script['address'] }, + "timestamp": "now", "redeemscript": multi_sig_script['redeemScript'], "keys": [ self.nodes[0].dumpprivkey(sig_address_1['address']), self.nodes[0].dumpprivkey(sig_address_2['address'])] }]) assert_equal(result[0]['success'], True) + address_assert = self.nodes[1].validateaddress(multi_sig_script['address']) + assert_equal(address_assert['timestamp'], timestamp) p2shunspent = self.nodes[1].listunspent(0,999999, [multi_sig_script['address']])[0] assert_equal(p2shunspent['spendable'], False) @@ -277,6 +321,7 @@ class ImportMultiTest (BitcoinTestFramework): "scriptPubKey": { "address": multi_sig_script['address'] }, + "timestamp": "now", "redeemscript": multi_sig_script['redeemScript'], "keys": [ self.nodes[0].dumpprivkey(sig_address_1['address']), self.nodes[0].dumpprivkey(sig_address_2['address'])], "watchonly": True @@ -294,6 +339,7 @@ class ImportMultiTest (BitcoinTestFramework): "scriptPubKey": { "address": address['address'] }, + "timestamp": "now", "pubkeys": [ address2['pubkey'] ] }]) assert_equal(result[0]['success'], False) @@ -302,6 +348,7 @@ class ImportMultiTest (BitcoinTestFramework): address_assert = self.nodes[1].validateaddress(address['address']) assert_equal(address_assert['iswatchonly'], False) assert_equal(address_assert['ismine'], False) + assert_equal('timestamp' in address_assert, False) # ScriptPubKey + Public key + internal + Wrong pubkey @@ -310,6 +357,7 @@ class ImportMultiTest (BitcoinTestFramework): address2 = self.nodes[0].validateaddress(self.nodes[0].getnewaddress()) request = [{ "scriptPubKey": address['scriptPubKey'], + "timestamp": "now", "pubkeys": [ address2['pubkey'] ], "internal": True }] @@ -320,6 +368,7 @@ class ImportMultiTest (BitcoinTestFramework): address_assert = self.nodes[1].validateaddress(address['address']) assert_equal(address_assert['iswatchonly'], False) assert_equal(address_assert['ismine'], False) + assert_equal('timestamp' in address_assert, False) # Address + Private key + !watchonly + Wrong private key @@ -330,6 +379,7 @@ class ImportMultiTest (BitcoinTestFramework): "scriptPubKey": { "address": address['address'] }, + "timestamp": "now", "keys": [ self.nodes[0].dumpprivkey(address2['address']) ] }]) assert_equal(result[0]['success'], False) @@ -338,6 +388,7 @@ class ImportMultiTest (BitcoinTestFramework): address_assert = self.nodes[1].validateaddress(address['address']) assert_equal(address_assert['iswatchonly'], False) assert_equal(address_assert['ismine'], False) + assert_equal('timestamp' in address_assert, False) # ScriptPubKey + Private key + internal + Wrong private key @@ -346,6 +397,7 @@ class ImportMultiTest (BitcoinTestFramework): address2 = self.nodes[0].validateaddress(self.nodes[0].getnewaddress()) result = self.nodes[1].importmulti([{ "scriptPubKey": address['scriptPubKey'], + "timestamp": "now", "keys": [ self.nodes[0].dumpprivkey(address2['address']) ], "internal": True }]) @@ -355,6 +407,28 @@ class ImportMultiTest (BitcoinTestFramework): address_assert = self.nodes[1].validateaddress(address['address']) assert_equal(address_assert['iswatchonly'], False) assert_equal(address_assert['ismine'], False) + assert_equal('timestamp' in address_assert, False) + + # restart nodes to check for proper serialization/deserialization of watch only address + stop_nodes(self.nodes) + self.nodes = start_nodes(2, self.options.tmpdir) + address_assert = self.nodes[1].validateaddress(watchonly_address) + assert_equal(address_assert['iswatchonly'], True) + assert_equal(address_assert['ismine'], False) + assert_equal(address_assert['timestamp'], watchonly_timestamp); + + # Bad or missing timestamps + print("Should throw on invalid or missing timestamp values") + assert_raises_message(JSONRPCException, 'Missing required timestamp field for key', + self.nodes[1].importmulti, [{ + "scriptPubKey": address['scriptPubKey'], + }]) + assert_raises_message(JSONRPCException, 'Expected number or "now" timestamp value for key. got type string', + self.nodes[1].importmulti, [{ + "scriptPubKey": address['scriptPubKey'], + "timestamp": "", + }]) + if __name__ == '__main__': ImportMultiTest ().main () diff --git a/qa/rpc-tests/p2p-compactblocks.py b/qa/rpc-tests/p2p-compactblocks.py index 9151ecf5de..47dfe4f5fd 100755 --- a/qa/rpc-tests/p2p-compactblocks.py +++ b/qa/rpc-tests/p2p-compactblocks.py @@ -7,7 +7,6 @@ from test_framework.mininode import * from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * from test_framework.blocktools import create_block, create_coinbase, add_witness_commitment -from test_framework.siphash import siphash256 from test_framework.script import CScript, OP_TRUE ''' diff --git a/qa/rpc-tests/p2p-fullblocktest.py b/qa/rpc-tests/p2p-fullblocktest.py index e4b889d761..44e5f10322 100755 --- a/qa/rpc-tests/p2p-fullblocktest.py +++ b/qa/rpc-tests/p2p-fullblocktest.py @@ -398,7 +398,7 @@ class FullBlockTest(ComparisonTestFramework): # Extend the b26 chain to make sure bitcoind isn't accepting b26 b27 = block(27, spend=out[7]) - yield rejected(RejectResult(16, b'bad-prevblk')) + yield rejected(RejectResult(0, b'bad-prevblk')) # Now try a too-large-coinbase script tip(15) @@ -410,7 +410,7 @@ class FullBlockTest(ComparisonTestFramework): # Extend the b28 chain to make sure bitcoind isn't accepting b28 b29 = block(29, spend=out[7]) - yield rejected(RejectResult(16, b'bad-prevblk')) + yield rejected(RejectResult(0, b'bad-prevblk')) # b30 has a max-sized coinbase scriptSig. tip(23) diff --git a/qa/rpc-tests/p2p-leaktests.py b/qa/rpc-tests/p2p-leaktests.py new file mode 100755 index 0000000000..41ca84d779 --- /dev/null +++ b/qa/rpc-tests/p2p-leaktests.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +# Copyright (c) 2017 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +from test_framework.mininode import * +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import * + +''' +Test for message sending before handshake completion + +A node should never send anything other than VERSION/VERACK/REJECT until it's +received a VERACK. + +This test connects to a node and sends it a few messages, trying to intice it +into sending us something it shouldn't. +''' + +banscore = 10 + +class CLazyNode(NodeConnCB): + def __init__(self): + self.connection = None + self.unexpected_msg = False + self.connected = False + super().__init__() + + def add_connection(self, conn): + self.connection = conn + + def send_message(self, message): + self.connection.send_message(message) + + def bad_message(self, message): + self.unexpected_msg = True + print("should not have received message: %s" % message.command) + + def on_open(self, conn): + self.connected = True + + def on_version(self, conn, message): self.bad_message(message) + def on_verack(self, conn, message): self.bad_message(message) + def on_reject(self, conn, message): self.bad_message(message) + def on_inv(self, conn, message): self.bad_message(message) + def on_addr(self, conn, message): self.bad_message(message) + def on_alert(self, conn, message): self.bad_message(message) + def on_getdata(self, conn, message): self.bad_message(message) + def on_getblocks(self, conn, message): self.bad_message(message) + def on_tx(self, conn, message): self.bad_message(message) + def on_block(self, conn, message): self.bad_message(message) + def on_getaddr(self, conn, message): self.bad_message(message) + def on_headers(self, conn, message): self.bad_message(message) + def on_getheaders(self, conn, message): self.bad_message(message) + def on_ping(self, conn, message): self.bad_message(message) + def on_mempool(self, conn): self.bad_message(message) + def on_pong(self, conn, message): self.bad_message(message) + def on_feefilter(self, conn, message): self.bad_message(message) + def on_sendheaders(self, conn, message): self.bad_message(message) + def on_sendcmpct(self, conn, message): self.bad_message(message) + def on_cmpctblock(self, conn, message): self.bad_message(message) + def on_getblocktxn(self, conn, message): self.bad_message(message) + def on_blocktxn(self, conn, message): self.bad_message(message) + +# Node that never sends a version. We'll use this to send a bunch of messages +# anyway, and eventually get disconnected. +class CNodeNoVersionBan(CLazyNode): + def __init__(self): + super().__init__() + + # send a bunch of veracks without sending a message. This should get us disconnected. + # NOTE: implementation-specific check here. Remove if bitcoind ban behavior changes + def on_open(self, conn): + super().on_open(conn) + for i in range(banscore): + self.send_message(msg_verack()) + + def on_reject(self, conn, message): pass + +# Node that never sends a version. This one just sits idle and hopes to receive +# any message (it shouldn't!) +class CNodeNoVersionIdle(CLazyNode): + def __init__(self): + super().__init__() + +# Node that sends a version but not a verack. +class CNodeNoVerackIdle(CLazyNode): + def __init__(self): + self.version_received = False + super().__init__() + + def on_reject(self, conn, message): pass + def on_verack(self, conn, message): pass + # When version is received, don't reply with a verack. Instead, see if the + # node will give us a message that it shouldn't. This is not an exhaustive + # list! + def on_version(self, conn, message): + self.version_received = True + conn.send_message(msg_ping()) + conn.send_message(msg_getaddr()) + +class P2PLeakTest(BitcoinTestFramework): + def __init__(self): + super().__init__() + self.num_nodes = 1 + def setup_network(self): + extra_args = [['-debug', '-banscore='+str(banscore)] + for i in range(self.num_nodes)] + self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, extra_args) + + def run_test(self): + no_version_bannode = CNodeNoVersionBan() + no_version_idlenode = CNodeNoVersionIdle() + no_verack_idlenode = CNodeNoVerackIdle() + + connections = [] + connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], no_version_bannode, send_version=False)) + connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], no_version_idlenode, send_version=False)) + connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], no_verack_idlenode)) + no_version_bannode.add_connection(connections[0]) + no_version_idlenode.add_connection(connections[1]) + no_verack_idlenode.add_connection(connections[2]) + + NetworkThread().start() # Start up network handling in another thread + + assert(wait_until(lambda: no_version_bannode.connected and no_version_idlenode.connected and no_verack_idlenode.version_received, timeout=10)) + + # Mine a block and make sure that it's not sent to the connected nodes + self.nodes[0].generate(1) + + #Give the node enough time to possibly leak out a message + time.sleep(5) + + #This node should have been banned + assert(no_version_bannode.connection.state == "closed") + + [conn.disconnect_node() for conn in connections] + + # Make sure no unexpected messages came in + assert(no_version_bannode.unexpected_msg == False) + assert(no_version_idlenode.unexpected_msg == False) + assert(no_verack_idlenode.unexpected_msg == False) + +if __name__ == '__main__': + P2PLeakTest().main() diff --git a/qa/rpc-tests/p2p-segwit.py b/qa/rpc-tests/p2p-segwit.py index a7858ad3d8..2f339bb54f 100755 --- a/qa/rpc-tests/p2p-segwit.py +++ b/qa/rpc-tests/p2p-segwit.py @@ -951,7 +951,6 @@ class SegWitTest(BitcoinTestFramework): tx.rehash() tx_hash = tx.sha256 - tx_value = tx.vout[0].nValue # Verify that unnecessary witnesses are rejected. self.test_node.announce_tx_and_wait_for_getdata(tx) diff --git a/qa/rpc-tests/p2p-timeouts.py b/qa/rpc-tests/p2p-timeouts.py new file mode 100755 index 0000000000..f1b190587d --- /dev/null +++ b/qa/rpc-tests/p2p-timeouts.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +# Copyright (c) 2016 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +""" TimeoutsTest -- test various net timeouts (only in extended tests) + +- Create three bitcoind nodes: + + no_verack_node - we never send a verack in response to their version + no_version_node - we never send a version (only a ping) + no_send_node - we never send any P2P message. + +- Start all three nodes +- Wait 1 second +- Assert that we're connected +- Send a ping to no_verack_node and no_version_node +- Wait 30 seconds +- Assert that we're still connected +- Send a ping to no_verack_node and no_version_node +- Wait 31 seconds +- Assert that we're no longer connected (timeout to receive version/verack is 60 seconds) +""" + +from time import sleep + +from test_framework.mininode import * +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import * + +class TestNode(SingleNodeConnCB): + def __init__(self): + SingleNodeConnCB.__init__(self) + self.connected = False + self.received_version = False + + def on_open(self, conn): + self.connected = True + + def on_close(self, conn): + self.connected = False + + def on_version(self, conn, message): + # Don't send a verack in response + self.received_version = True + +class TimeoutsTest(BitcoinTestFramework): + def __init__(self): + super().__init__() + self.setup_clean_chain = True + self.num_nodes = 1 + + def setup_network(self): + self.nodes = [] + + # Start up node0 to be a version 1, pre-segwit node. + self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, + [["-debug", "-logtimemicros=1"]]) + + def run_test(self): + # Setup the p2p connections and start up the network thread. + self.no_verack_node = TestNode() # never send verack + self.no_version_node = TestNode() # never send version (just ping) + self.no_send_node = TestNode() # never send anything + + connections = [] + connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], self.no_verack_node)) + connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], self.no_version_node, send_version=False)) + connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], self.no_send_node, send_version=False)) + self.no_verack_node.add_connection(connections[0]) + self.no_version_node.add_connection(connections[1]) + self.no_send_node.add_connection(connections[2]) + + NetworkThread().start() # Start up network handling in another thread + + sleep(1) + + assert(self.no_verack_node.connected) + assert(self.no_version_node.connected) + assert(self.no_send_node.connected) + + ping_msg = msg_ping() + connections[0].send_message(ping_msg) + connections[1].send_message(ping_msg) + + sleep(30) + + assert(self.no_verack_node.received_version) + + assert(self.no_verack_node.connected) + assert(self.no_version_node.connected) + assert(self.no_send_node.connected) + + connections[0].send_message(ping_msg) + connections[1].send_message(ping_msg) + + sleep(31) + + assert(not self.no_verack_node.connected) + assert(not self.no_version_node.connected) + assert(not self.no_send_node.connected) + +if __name__ == '__main__': + TimeoutsTest().main() diff --git a/qa/rpc-tests/receivedby.py b/qa/rpc-tests/receivedby.py index 4f17b661cb..1b90b23330 100755 --- a/qa/rpc-tests/receivedby.py +++ b/qa/rpc-tests/receivedby.py @@ -14,7 +14,6 @@ def get_sub_array_from_array(object_array, to_match): Finds and returns a sub array from an array of arrays. to_match should be a unique idetifier of a sub array ''' - num_matched = 0 for item in object_array: all_match = True for key,value in to_match.items(): @@ -104,7 +103,7 @@ class ReceivedByTest(BitcoinTestFramework): received_by_account_json = get_sub_array_from_array(self.nodes[1].listreceivedbyaccount(),{"account":account}) if len(received_by_account_json) == 0: raise AssertionError("No accounts found in node") - balance_by_account = rec_by_accountArr = self.nodes[1].getreceivedbyaccount(account) + balance_by_account = self.nodes[1].getreceivedbyaccount(account) txid = self.nodes[0].sendtoaddress(addr, 0.1) self.sync_all() diff --git a/qa/rpc-tests/replace-by-fee.py b/qa/rpc-tests/replace-by-fee.py index 8aba06c60c..2b29dfdd2b 100755 --- a/qa/rpc-tests/replace-by-fee.py +++ b/qa/rpc-tests/replace-by-fee.py @@ -393,7 +393,6 @@ class ReplaceByFeeTest(BitcoinTestFramework): utxo = make_utxo(self.nodes[0], initial_nValue) fee = int(0.0001*COIN) split_value = int((initial_nValue-fee)/(MAX_REPLACEMENT_LIMIT+1)) - actual_fee = initial_nValue - split_value*(MAX_REPLACEMENT_LIMIT+1) outputs = [] for i in range(MAX_REPLACEMENT_LIMIT+1): diff --git a/qa/rpc-tests/rpcnamedargs.py b/qa/rpc-tests/rpcnamedargs.py index 0484204668..f9a40955c0 100755 --- a/qa/rpc-tests/rpcnamedargs.py +++ b/qa/rpc-tests/rpcnamedargs.py @@ -3,17 +3,11 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -from decimal import Decimal - from test_framework.test_framework import BitcoinTestFramework -from test_framework.authproxy import JSONRPCException from test_framework.util import ( assert_equal, assert_raises_jsonrpc, - assert_is_hex_string, - assert_is_hash_string, start_nodes, - connect_nodes_bi, ) @@ -37,7 +31,7 @@ class NamedArgumentTest(BitcoinTestFramework): h = node.help(command='getinfo') assert(h.startswith('getinfo\n')) - assert_raises_jsonrpc(-8, node.help, random='getinfo') + assert_raises_jsonrpc(-8, 'Unknown named parameter', node.help, random='getinfo') h = node.getblockhash(height=0) node.getblock(blockhash=h) diff --git a/qa/rpc-tests/signrawtransactions.py b/qa/rpc-tests/signrawtransactions.py index c61a280616..109312bd5f 100755 --- a/qa/rpc-tests/signrawtransactions.py +++ b/qa/rpc-tests/signrawtransactions.py @@ -26,12 +26,14 @@ class SignRawTransactionsTest(BitcoinTestFramework): 1) The transaction has a complete set of signatures 2) No script verification error occurred""" - privKeys = ['cUeKHd5orzT3mz8P9pxyREHfsWtVfgsfDjiZZBcjUBAaGk1BTj7N'] + privKeys = ['cUeKHd5orzT3mz8P9pxyREHfsWtVfgsfDjiZZBcjUBAaGk1BTj7N', 'cVKpPfVKSJxKqVpE9awvXNWuLHCa5j5tiE7K6zbUSptFpTEtiFrA'] inputs = [ - # Valid pay-to-pubkey script + # Valid pay-to-pubkey scripts {'txid': '9b907ef1e3c26fc71fe4a4b3580bc75264112f95050014157059c736f0202e71', 'vout': 0, - 'scriptPubKey': '76a91460baa0f494b38ce3c940dea67f3804dc52d1fb9488ac'} + 'scriptPubKey': '76a91460baa0f494b38ce3c940dea67f3804dc52d1fb9488ac'}, + {'txid': '83a4f6a6b73660e13ee6cb3c6063fa3759c50c9b7521d0536022961898f4fb02', 'vout': 0, + 'scriptPubKey': '76a914669b857c03a5ed269d5d85a1ffac9ed5d663072788ac'}, ] outputs = {'mpLQjfK79b7CCV4VMJWEWAj5Mpx8Up5zxB': 0.1} @@ -46,6 +48,22 @@ class SignRawTransactionsTest(BitcoinTestFramework): # 2) No script verification error occurred assert 'errors' not in rawTxSigned + # Check that signrawtransaction doesn't blow up on garbage merge attempts + dummyTxInconsistent = self.nodes[0].createrawtransaction([inputs[0]], outputs) + rawTxUnsigned = self.nodes[0].signrawtransaction(rawTx + dummyTxInconsistent, inputs) + + assert 'complete' in rawTxUnsigned + assert_equal(rawTxUnsigned['complete'], False) + + # Check that signrawtransaction properly merges unsigned and signed txn, even with garbage in the middle + rawTxSigned2 = self.nodes[0].signrawtransaction(rawTxUnsigned["hex"] + dummyTxInconsistent + rawTxSigned["hex"], inputs) + + assert 'complete' in rawTxSigned2 + assert_equal(rawTxSigned2['complete'], True) + + assert 'errors' not in rawTxSigned2 + + def script_verification_error_test(self): """Creates and signs a raw transaction with valid (vin 0), invalid (vin 1) and one missing (vin 2) input script. @@ -78,6 +96,16 @@ class SignRawTransactionsTest(BitcoinTestFramework): outputs = {'mpLQjfK79b7CCV4VMJWEWAj5Mpx8Up5zxB': 0.1} rawTx = self.nodes[0].createrawtransaction(inputs, outputs) + + # Make sure decoderawtransaction is at least marginally sane + decodedRawTx = self.nodes[0].decoderawtransaction(rawTx) + for i, inp in enumerate(inputs): + assert_equal(decodedRawTx["vin"][i]["txid"], inp["txid"]) + assert_equal(decodedRawTx["vin"][i]["vout"], inp["vout"]) + + # Make sure decoderawtransaction throws if there is extra data + assert_raises(JSONRPCException, self.nodes[0].decoderawtransaction, rawTx + "00") + rawTxSigned = self.nodes[0].signrawtransaction(rawTx, scripts, privKeys) # 3) The transaction has no complete set of signatures diff --git a/qa/rpc-tests/test_framework/mininode.py b/qa/rpc-tests/test_framework/mininode.py index 91daa4ab1f..5b563c58ae 100755 --- a/qa/rpc-tests/test_framework/mininode.py +++ b/qa/rpc-tests/test_framework/mininode.py @@ -1540,6 +1540,7 @@ class NodeConnCB(object): if conn.ver_send > BIP0031_VERSION: conn.send_message(msg_pong(message.nonce)) def on_reject(self, conn, message): pass + def on_open(self, conn): pass def on_close(self, conn): pass def on_mempool(self, conn): pass def on_pong(self, conn, message): pass @@ -1614,7 +1615,7 @@ class NodeConn(asyncore.dispatcher): "regtest": b"\xfa\xbf\xb5\xda", # regtest } - def __init__(self, dstaddr, dstport, rpc, callback, net="regtest", services=NODE_NETWORK): + def __init__(self, dstaddr, dstport, rpc, callback, net="regtest", services=NODE_NETWORK, send_version=True): asyncore.dispatcher.__init__(self, map=mininode_socket_map) self.log = logging.getLogger("NodeConn(%s:%d)" % (dstaddr, dstport)) self.dstaddr = dstaddr @@ -1631,14 +1632,16 @@ class NodeConn(asyncore.dispatcher): self.disconnect = False self.nServices = 0 - # stuff version msg into sendbuf - vt = msg_version() - vt.nServices = services - vt.addrTo.ip = self.dstaddr - vt.addrTo.port = self.dstport - vt.addrFrom.ip = "0.0.0.0" - vt.addrFrom.port = 0 - self.send_message(vt, True) + if send_version: + # stuff version msg into sendbuf + vt = msg_version() + vt.nServices = services + vt.addrTo.ip = self.dstaddr + vt.addrTo.port = self.dstport + vt.addrFrom.ip = "0.0.0.0" + vt.addrFrom.port = 0 + self.send_message(vt, True) + print('MiniNode: Connecting to Bitcoin Node IP # ' + dstaddr + ':' \ + str(dstport)) @@ -1652,8 +1655,10 @@ class NodeConn(asyncore.dispatcher): self.log.debug(msg) def handle_connect(self): - self.show_debug_msg("MiniNode: Connected & Listening: \n") - self.state = "connected" + if self.state != "connected": + self.show_debug_msg("MiniNode: Connected & Listening: \n") + self.state = "connected" + self.cb.on_open(self) def handle_close(self): self.show_debug_msg("MiniNode: Closing Connection to %s:%d... " @@ -1681,11 +1686,20 @@ class NodeConn(asyncore.dispatcher): def writable(self): with mininode_lock: + pre_connection = self.state == "connecting" length = len(self.sendbuf) - return (length > 0) + return (length > 0 or pre_connection) def handle_write(self): with mininode_lock: + # asyncore does not expose socket connection, only the first read/write + # event, thus we must check connection manually here to know when we + # actually connect + if self.state == "connecting": + self.handle_connect() + if not self.writable(): + return + try: sent = self.send(self.sendbuf) except: diff --git a/qa/rpc-tests/test_framework/util.py b/qa/rpc-tests/test_framework/util.py index aca82c8b6f..dc8555c44c 100644 --- a/qa/rpc-tests/test_framework/util.py +++ b/qa/rpc-tests/test_framework/util.py @@ -550,13 +550,30 @@ def assert_raises_message(exc, message, fun, *args, **kwds): else: raise AssertionError("No exception raised") -def assert_raises_jsonrpc(code, fun, *args, **kwds): - '''Check for specific JSONRPC exception code''' +def assert_raises_jsonrpc(code, message, fun, *args, **kwds): + """Run an RPC and verify that a specific JSONRPC exception code and message is raised. + + Calls function `fun` with arguments `args` and `kwds`. Catches a JSONRPCException + and verifies that the error code and message are as expected. Throws AssertionError if + no JSONRPCException was returned or if the error code/message are not as expected. + + Args: + code (int), optional: the error code returned by the RPC call (defined + in src/rpc/protocol.h). Set to None if checking the error code is not required. + message (string), optional: [a substring of] the error string returned by the + RPC call. Set to None if checking the error string is not required + fun (function): the function to call. This should be the name of an RPC. + args*: positional arguments for the function. + kwds**: named arguments for the function. + """ try: fun(*args, **kwds) except JSONRPCException as e: - if e.error["code"] != code: + # JSONRPCException was thrown as expected. Check the code and message values are correct. + if (code is not None) and (code != e.error["code"]): raise AssertionError("Unexpected JSONRPC error code %i" % e.error["code"]) + if (message is not None) and (message not in e.error['message']): + raise AssertionError("Expected substring not found:"+e.error['message']) except Exception as e: raise AssertionError("Unexpected exception raised: "+type(e).__name__) else: diff --git a/qa/rpc-tests/wallet-accounts.py b/qa/rpc-tests/wallet-accounts.py index c51181e4f8..45a9db0571 100755 --- a/qa/rpc-tests/wallet-accounts.py +++ b/qa/rpc-tests/wallet-accounts.py @@ -6,9 +6,7 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( start_nodes, - start_node, assert_equal, - connect_nodes_bi, ) diff --git a/qa/rpc-tests/wallet.py b/qa/rpc-tests/wallet.py index f325ecb4a3..c82e0adffa 100755 --- a/qa/rpc-tests/wallet.py +++ b/qa/rpc-tests/wallet.py @@ -359,7 +359,6 @@ class WalletTest (BitcoinTestFramework): rawtx = self.nodes[0].createrawtransaction([{"txid":singletxid, "vout":0}], {chain_addrs[0]:node0_balance/2-Decimal('0.01'), chain_addrs[1]:node0_balance/2-Decimal('0.01')}) signedtx = self.nodes[0].signrawtransaction(rawtx) singletxid = self.nodes[0].sendrawtransaction(signedtx["hex"]) - txids = [singletxid, singletxid] self.nodes[0].generate(1) # Make a long chain of unconfirmed payments without hitting mempool limit |