diff options
Diffstat (limited to 'test')
-rw-r--r-- | test/config.ini.in | 1 | ||||
-rwxr-xr-x | test/functional/feature_blockfilterindex_prune.py | 54 | ||||
-rwxr-xr-x | test/functional/interface_zmq.py | 76 | ||||
-rwxr-xr-x | test/functional/mocks/signer.py | 102 | ||||
-rwxr-xr-x | test/functional/p2p_filter.py | 15 | ||||
-rwxr-xr-x | test/functional/p2p_leak.py | 12 | ||||
-rwxr-xr-x | test/functional/rpc_blockchain.py | 6 | ||||
-rwxr-xr-x | test/functional/rpc_help.py | 5 | ||||
-rwxr-xr-x | test/functional/test_framework/messages.py | 39 | ||||
-rwxr-xr-x | test/functional/test_framework/p2p.py | 20 | ||||
-rwxr-xr-x | test/functional/test_framework/test_framework.py | 9 | ||||
-rwxr-xr-x | test/functional/test_framework/test_node.py | 16 | ||||
-rwxr-xr-x | test/functional/test_runner.py | 2 | ||||
-rwxr-xr-x | test/functional/wallet_descriptor.py | 57 | ||||
-rwxr-xr-x | test/functional/wallet_signer.py | 217 | ||||
-rwxr-xr-x | test/functional/wallet_watchonly.py | 7 | ||||
-rwxr-xr-x | test/lint/lint-circular-dependencies.sh | 2 | ||||
-rw-r--r-- | test/sanitizer_suppressions/tsan | 1 |
18 files changed, 582 insertions, 59 deletions
diff --git a/test/config.ini.in b/test/config.ini.in index 77c9a720c3..e3872181cd 100644 --- a/test/config.ini.in +++ b/test/config.ini.in @@ -23,3 +23,4 @@ RPCAUTH=@abs_top_srcdir@/share/rpcauth/rpcauth.py @BUILD_BITCOIND_TRUE@ENABLE_BITCOIND=true @ENABLE_FUZZ_TRUE@ENABLE_FUZZ=true @ENABLE_ZMQ_TRUE@ENABLE_ZMQ=true +@ENABLE_EXTERNAL_SIGNER_TRUE@ENABLE_EXTERNAL_SIGNER=true diff --git a/test/functional/feature_blockfilterindex_prune.py b/test/functional/feature_blockfilterindex_prune.py new file mode 100755 index 0000000000..455073ef9c --- /dev/null +++ b/test/functional/feature_blockfilterindex_prune.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test blockfilterindex in conjunction with prune.""" +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_raises_rpc_error, + assert_greater_than, +) + + +class FeatureBlockfilterindexPruneTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 2 + self.extra_args = [["-fastprune", "-prune=1"], ["-fastprune", "-prune=1", "-blockfilterindex=1"]] + + def run_test(self): + # test basic pruning compatibility & filter access of pruned blocks + self.log.info("check if we can access a blockfilter when pruning is enabled but no blocks are actually pruned") + assert len(self.nodes[1].getblockfilter(self.nodes[1].getbestblockhash())['filter']) > 0 + # Mine two batches of blocks to avoid hitting NODE_NETWORK_LIMITED_MIN_BLOCKS disconnection + self.nodes[1].generate(250) + self.sync_all() + self.nodes[1].generate(250) + self.sync_all() + self.log.info("prune some blocks") + pruneheight = self.nodes[1].pruneblockchain(400) + assert pruneheight != 0 + self.log.info("check if we can access the tips blockfilter when we have pruned some blocks") + assert len(self.nodes[1].getblockfilter(self.nodes[1].getbestblockhash())['filter']) > 0 + self.log.info("check if we can access the blockfilter of a pruned block") + assert len(self.nodes[1].getblockfilter(self.nodes[1].getblockhash(2))['filter']) > 0 + self.log.info("start node without blockfilterindex") + self.stop_node(1) + self.start_node(1, extra_args=self.extra_args[0]) + self.log.info("make sure accessing the blockfilters throws an error") + assert_raises_rpc_error(-1, "Index is not enabled for filtertype basic", self.nodes[1].getblockfilter, self.nodes[1].getblockhash(2)) + self.nodes[1].generate(1000) + self.log.info("prune below the blockfilterindexes best block while blockfilters are disabled") + pruneheight_new = self.nodes[1].pruneblockchain(1000) + assert_greater_than(pruneheight_new, pruneheight) + self.stop_node(1) + self.log.info("make sure we get an init error when starting the node again with block filters") + with self.nodes[1].assert_debug_log(["basic block filter index best block of the index goes beyond pruned data. Please disable the index or reindex (which will download the whole blockchain again)"]): + self.nodes[1].assert_start_raises_init_error(extra_args=self.extra_args[1]) + self.log.info("make sure the node starts again with the -reindex arg") + reindex_args = self.extra_args[1] + reindex_args.append("-reindex") + self.start_node(1, extra_args=reindex_args) + + +if __name__ == '__main__': + FeatureBlockfilterindexPruneTest().main() diff --git a/test/functional/interface_zmq.py b/test/functional/interface_zmq.py index e9f61be4d4..d0967a9340 100755 --- a/test/functional/interface_zmq.py +++ b/test/functional/interface_zmq.py @@ -27,28 +27,31 @@ def hash256_reversed(byte_str): class ZMQSubscriber: def __init__(self, socket, topic): - self.sequence = 0 + self.sequence = None # no sequence number received yet self.socket = socket self.topic = topic self.socket.setsockopt(zmq.SUBSCRIBE, self.topic) - def receive(self): + # Receive message from publisher and verify that topic and sequence match + def _receive_from_publisher_and_check(self): topic, body, seq = self.socket.recv_multipart() # Topic should match the subscriber topic. assert_equal(topic, self.topic) # Sequence should be incremental. - assert_equal(struct.unpack('<I', seq)[-1], self.sequence) + received_seq = struct.unpack('<I', seq)[-1] + if self.sequence is None: + self.sequence = received_seq + else: + assert_equal(received_seq, self.sequence) self.sequence += 1 return body + def receive(self): + return self._receive_from_publisher_and_check() + def receive_sequence(self): - topic, body, seq = self.socket.recv_multipart() - # Topic should match the subscriber topic. - assert_equal(topic, self.topic) - # Sequence should be incremental. - assert_equal(struct.unpack('<I', seq)[-1], self.sequence) - self.sequence += 1 + body = self._receive_from_publisher_and_check() hash = body[:32].hex() label = chr(body[32]) mempool_sequence = None if len(body) != 32+1+8 else struct.unpack("<Q", body[32+1:])[0] @@ -64,6 +67,9 @@ class ZMQTest (BitcoinTestFramework): self.num_nodes = 2 if self.is_wallet_compiled(): self.requires_wallet = True + # This test isn't testing txn relay/timing, so set whitelist on the + # peers for instant txn relay. This speeds up the test run time 2-3x. + self.extra_args = [["-whitelist=noban@127.0.0.1"]] * self.num_nodes def skip_test_if_missing_module(self): self.skip_if_no_py3_zmq() @@ -84,23 +90,46 @@ class ZMQTest (BitcoinTestFramework): # Restart node with the specified zmq notifications enabled, subscribe to # all of them and return the corresponding ZMQSubscriber objects. - def setup_zmq_test(self, services, recv_timeout=60, connect_nodes=False): + def setup_zmq_test(self, services, *, recv_timeout=60, sync_blocks=True): subscribers = [] for topic, address in services: socket = self.ctx.socket(zmq.SUB) - socket.set(zmq.RCVTIMEO, recv_timeout*1000) subscribers.append(ZMQSubscriber(socket, topic.encode())) - self.restart_node(0, ["-zmqpub%s=%s" % (topic, address) for topic, address in services]) - - if connect_nodes: - self.connect_nodes(0, 1) + self.restart_node(0, ["-zmqpub%s=%s" % (topic, address) for topic, address in services] + + self.extra_args[0]) for i, sub in enumerate(subscribers): sub.socket.connect(services[i][1]) - # Relax so that the subscribers are ready before publishing zmq messages - sleep(0.2) + # Ensure that all zmq publisher notification interfaces are ready by + # running the following "sync up" procedure: + # 1. Generate a block on the node + # 2. Try to receive a notification on all subscribers + # 3. If all subscribers get a message within the timeout (1 second), + # we are done, otherwise repeat starting from step 1 + for sub in subscribers: + sub.socket.set(zmq.RCVTIMEO, 1000) + while True: + self.nodes[0].generate(1) + recv_failed = False + for sub in subscribers: + try: + sub.receive() + except zmq.error.Again: + self.log.debug("Didn't receive sync-up notification, trying again.") + recv_failed = True + if not recv_failed: + self.log.debug("ZMQ sync-up completed, all subscribers are ready.") + break + + # set subscriber's desired timeout for the test + for sub in subscribers: + sub.socket.set(zmq.RCVTIMEO, recv_timeout*1000) + + self.connect_nodes(0, 1) + if sync_blocks: + self.sync_blocks() return subscribers @@ -110,9 +139,7 @@ class ZMQTest (BitcoinTestFramework): self.restart_node(0, ["-zmqpubrawtx=foo", "-zmqpubhashtx=bar"]) address = 'tcp://127.0.0.1:28332' - subs = self.setup_zmq_test( - [(topic, address) for topic in ["hashblock", "hashtx", "rawblock", "rawtx"]], - connect_nodes=True) + subs = self.setup_zmq_test([(topic, address) for topic in ["hashblock", "hashtx", "rawblock", "rawtx"]]) hashblock = subs[0] hashtx = subs[1] @@ -189,6 +216,7 @@ class ZMQTest (BitcoinTestFramework): hashblock, hashtx = self.setup_zmq_test( [(topic, address) for topic in ["hashblock", "hashtx"]], recv_timeout=2) # 2 second timeout to check end of notifications + self.disconnect_nodes(0, 1) # Generate 1 block in nodes[0] with 1 mempool tx and receive all notifications payment_txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 1.0) @@ -237,6 +265,7 @@ class ZMQTest (BitcoinTestFramework): """ self.log.info("Testing 'sequence' publisher") [seq] = self.setup_zmq_test([("sequence", "tcp://127.0.0.1:28333")]) + self.disconnect_nodes(0, 1) # Mempool sequence number starts at 1 seq_num = 1 @@ -387,7 +416,7 @@ class ZMQTest (BitcoinTestFramework): return self.log.info("Testing 'mempool sync' usage of sequence notifier") - [seq] = self.setup_zmq_test([("sequence", "tcp://127.0.0.1:28333")], connect_nodes=True) + [seq] = self.setup_zmq_test([("sequence", "tcp://127.0.0.1:28333")]) # In-memory counter, should always start at 1 next_mempool_seq = self.nodes[0].getrawmempool(mempool_sequence=True)["mempool_sequence"] @@ -487,10 +516,13 @@ class ZMQTest (BitcoinTestFramework): def test_multiple_interfaces(self): # Set up two subscribers with different addresses + # (note that after the reorg test, syncing would fail due to different + # chain lengths on node0 and node1; for this test we only need node0, so + # we can disable syncing blocks on the setup) subscribers = self.setup_zmq_test([ ("hashblock", "tcp://127.0.0.1:28334"), ("hashblock", "tcp://127.0.0.1:28335"), - ]) + ], sync_blocks=False) # Generate 1 block in nodes[0] and receive all notifications self.nodes[0].generatetoaddress(1, ADDRESS_BCRT1_UNSPENDABLE) diff --git a/test/functional/mocks/signer.py b/test/functional/mocks/signer.py new file mode 100755 index 0000000000..676d0a0a4d --- /dev/null +++ b/test/functional/mocks/signer.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +# Copyright (c) 2018 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +import os +import sys +import argparse +import json + +def perform_pre_checks(): + mock_result_path = os.path.join(os.getcwd(), "mock_result") + if(os.path.isfile(mock_result_path)): + with open(mock_result_path, "r", encoding="utf8") as f: + mock_result = f.read() + if mock_result[0]: + sys.stdout.write(mock_result[2:]) + sys.exit(int(mock_result[0])) + +def enumerate(args): + sys.stdout.write(json.dumps([{"fingerprint": "00000001", "type": "trezor", "model": "trezor_t"}, {"fingerprint": "00000002"}])) + +def getdescriptors(args): + xpub = "tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B" + + sys.stdout.write(json.dumps({ + "receive": [ + "pkh([00000001/44'/1'/" + args.account + "']" + xpub + "/0/*)#vt6w3l3j", + "sh(wpkh([00000001/49'/1'/" + args.account + "']" + xpub + "/0/*))#r0grqw5x", + "wpkh([00000001/84'/1'/" + args.account + "']" + xpub + "/0/*)#x30uthjs" + ], + "internal": [ + "pkh([00000001/44'/1'/" + args.account + "']" + xpub + "/1/*)#all0v2p2", + "sh(wpkh([00000001/49'/1'/" + args.account + "']" + xpub + "/1/*))#kwx4c3pe", + "wpkh([00000001/84'/1'/" + args.account + "']" + xpub + "/1/*)#h92akzzg" + ] + })) + + +def displayaddress(args): + # Several descriptor formats are acceptable, so allowing for potential + # changes to InferDescriptor: + if args.fingerprint != "00000001": + return sys.stdout.write(json.dumps({"error": "Unexpected fingerprint", "fingerprint": args.fingerprint})) + + expected_desc = [ + "wpkh([00000001/84'/1'/0'/0/0]02c97dc3f4420402e01a113984311bf4a1b8de376cac0bdcfaf1b3ac81f13433c7)#0yneg42r" + ] + if args.desc not in expected_desc: + return sys.stdout.write(json.dumps({"error": "Unexpected descriptor", "desc": args.desc})) + + return sys.stdout.write(json.dumps({"address": "bcrt1qm90ugl4d48jv8n6e5t9ln6t9zlpm5th68x4f8g"})) + +def signtx(args): + if args.fingerprint != "00000001": + return sys.stdout.write(json.dumps({"error": "Unexpected fingerprint", "fingerprint": args.fingerprint})) + + with open(os.path.join(os.getcwd(), "mock_psbt"), "r", encoding="utf8") as f: + mock_psbt = f.read() + + if args.fingerprint == "00000001" : + sys.stdout.write(json.dumps({ + "psbt": mock_psbt, + "complete": True + })) + else: + sys.stdout.write(json.dumps({"psbt": args.psbt})) + +parser = argparse.ArgumentParser(prog='./signer.py', description='External signer mock') +parser.add_argument('--fingerprint') +parser.add_argument('--chain', default='main') +parser.add_argument('--stdin', action='store_true') + +subparsers = parser.add_subparsers(description='Commands', dest='command') +subparsers.required = True + +parser_enumerate = subparsers.add_parser('enumerate', help='list available signers') +parser_enumerate.set_defaults(func=enumerate) + +parser_getdescriptors = subparsers.add_parser('getdescriptors') +parser_getdescriptors.set_defaults(func=getdescriptors) +parser_getdescriptors.add_argument('--account', metavar='account') + +parser_displayaddress = subparsers.add_parser('displayaddress', help='display address on signer') +parser_displayaddress.add_argument('--desc', metavar='desc') +parser_displayaddress.set_defaults(func=displayaddress) + +parser_signtx = subparsers.add_parser('signtx') +parser_signtx.add_argument('psbt', metavar='psbt') + +parser_signtx.set_defaults(func=signtx) + +if not sys.stdin.isatty(): + buffer = sys.stdin.read() + if buffer and buffer.rstrip() != "": + sys.argv.extend(buffer.rstrip().split(" ")) + +args = parser.parse_args() + +perform_pre_checks() + +args.func(args) diff --git a/test/functional/p2p_filter.py b/test/functional/p2p_filter.py index 458e5235b6..8f64419138 100755 --- a/test/functional/p2p_filter.py +++ b/test/functional/p2p_filter.py @@ -19,7 +19,13 @@ from test_framework.messages import ( msg_mempool, msg_version, ) -from test_framework.p2p import P2PInterface, p2p_lock +from test_framework.p2p import ( + P2PInterface, + P2P_SERVICES, + P2P_SUBVERSION, + P2P_VERSION, + p2p_lock, +) from test_framework.script import MAX_SCRIPT_ELEMENT_SIZE from test_framework.test_framework import BitcoinTestFramework @@ -216,9 +222,12 @@ class FilterTest(BitcoinTestFramework): self.log.info('Test BIP 37 for a node with fRelay = False') # Add peer but do not send version yet filter_peer_without_nrelay = self.nodes[0].add_p2p_connection(P2PBloomFilter(), send_version=False, wait_for_verack=False) - # Send version with fRelay=False + # Send version with relay=False version_without_fRelay = msg_version() - version_without_fRelay.nRelay = 0 + version_without_fRelay.nVersion = P2P_VERSION + version_without_fRelay.strSubVer = P2P_SUBVERSION + version_without_fRelay.nServices = P2P_SERVICES + version_without_fRelay.relay = 0 filter_peer_without_nrelay.send_message(version_without_fRelay) filter_peer_without_nrelay.wait_for_verack() assert not self.nodes[0].getpeerinfo()[0]['relaytxes'] diff --git a/test/functional/p2p_leak.py b/test/functional/p2p_leak.py index ca8bf908a9..12b8b7baff 100755 --- a/test/functional/p2p_leak.py +++ b/test/functional/p2p_leak.py @@ -17,7 +17,12 @@ from test_framework.messages import ( msg_ping, msg_version, ) -from test_framework.p2p import P2PInterface +from test_framework.p2p import ( + P2PInterface, + P2P_SUBVERSION, + P2P_SERVICES, + P2P_VERSION_RELAY, +) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, @@ -125,12 +130,15 @@ class P2PLeakTest(BitcoinTestFramework): assert_equal(ver.addrFrom.port, 0) assert_equal(ver.addrFrom.ip, '0.0.0.0') assert_equal(ver.nStartingHeight, 201) - assert_equal(ver.nRelay, 1) + assert_equal(ver.relay, 1) self.log.info('Check that old peers are disconnected') p2p_old_peer = self.nodes[0].add_p2p_connection(P2PInterface(), send_version=False, wait_for_verack=False) old_version_msg = msg_version() old_version_msg.nVersion = 31799 + old_version_msg.strSubVer = P2P_SUBVERSION + old_version_msg.nServices = P2P_SERVICES + old_version_msg.relay = P2P_VERSION_RELAY with self.nodes[0].assert_debug_log(['peer=3 using obsolete version 31799; disconnecting']): p2p_old_peer.send_message(old_version_msg) p2p_old_peer.wait_for_disconnect() diff --git a/test/functional/rpc_blockchain.py b/test/functional/rpc_blockchain.py index 727698cb67..e090030205 100755 --- a/test/functional/rpc_blockchain.py +++ b/test/functional/rpc_blockchain.py @@ -316,6 +316,9 @@ class BlockchainTest(BitcoinTestFramework): header.calc_sha256() assert_equal(header.hash, besthash) + assert 'previousblockhash' not in node.getblockheader(node.getblockhash(0)) + assert 'nextblockhash' not in node.getblockheader(node.getbestblockhash()) + def _test_getdifficulty(self): difficulty = self.nodes[0].getdifficulty() # 1 hash in 2 should be valid, so difficulty should be 1/2**31 @@ -419,6 +422,9 @@ class BlockchainTest(BitcoinTestFramework): # Restore chain state move_block_file('rev_wrong', 'rev00000.dat') + assert 'previousblockhash' not in node.getblock(node.getblockhash(0)) + assert 'nextblockhash' not in node.getblock(node.getbestblockhash()) + if __name__ == '__main__': BlockchainTest().main() diff --git a/test/functional/rpc_help.py b/test/functional/rpc_help.py index 1eefd109f8..de21f43747 100755 --- a/test/functional/rpc_help.py +++ b/test/functional/rpc_help.py @@ -105,10 +105,13 @@ class HelpRpcTest(BitcoinTestFramework): if self.is_wallet_compiled(): components.append('Wallet') + if self.is_external_signer_compiled(): + components.append('Signer') + if self.is_zmq_compiled(): components.append('Zmq') - assert_equal(titles, components) + assert_equal(titles, sorted(components)) def dump_help(self): dump_dir = os.path.join(self.options.tmpdir, 'rpc_help_dump') diff --git a/test/functional/test_framework/messages.py b/test/functional/test_framework/messages.py index 27a09ef86c..a18a9ec109 100755 --- a/test/functional/test_framework/messages.py +++ b/test/functional/test_framework/messages.py @@ -31,11 +31,6 @@ import time from test_framework.siphash import siphash256 from test_framework.util import hex_str_to_bytes, assert_equal -MIN_VERSION_SUPPORTED = 60001 -MY_VERSION = 70016 # past wtxid relay -MY_SUBVERSION = b"/python-p2p-tester:0.0.3/" -MY_RELAY = 1 # from version 70001 onwards, fRelay should be appended to version messages (BIP37) - MAX_LOCATOR_SZ = 101 MAX_BLOCK_BASE_SIZE = 1000000 MAX_BLOOM_FILTER_SIZE = 36000 @@ -326,22 +321,20 @@ class CBlockLocator: __slots__ = ("nVersion", "vHave") def __init__(self): - self.nVersion = MY_VERSION self.vHave = [] def deserialize(self, f): - self.nVersion = struct.unpack("<i", f.read(4))[0] + struct.unpack("<i", f.read(4))[0] # Ignore version field. self.vHave = deser_uint256_vector(f) def serialize(self): r = b"" - r += struct.pack("<i", self.nVersion) + r += struct.pack("<i", 0) # Bitcoin Core ignores version field. Set it to 0. r += ser_uint256_vector(self.vHave) return r def __repr__(self): - return "CBlockLocator(nVersion=%i vHave=%s)" \ - % (self.nVersion, repr(self.vHave)) + return "CBlockLocator(vHave=%s)" % (repr(self.vHave)) class COutPoint: @@ -1023,20 +1016,20 @@ class CMerkleBlock: # Objects that correspond to messages on the wire class msg_version: - __slots__ = ("addrFrom", "addrTo", "nNonce", "nRelay", "nServices", + __slots__ = ("addrFrom", "addrTo", "nNonce", "relay", "nServices", "nStartingHeight", "nTime", "nVersion", "strSubVer") msgtype = b"version" def __init__(self): - self.nVersion = MY_VERSION - self.nServices = NODE_NETWORK | NODE_WITNESS + self.nVersion = 0 + self.nServices = 0 self.nTime = int(time.time()) self.addrTo = CAddress() self.addrFrom = CAddress() self.nNonce = random.getrandbits(64) - self.strSubVer = MY_SUBVERSION + self.strSubVer = '' self.nStartingHeight = -1 - self.nRelay = MY_RELAY + self.relay = 0 def deserialize(self, f): self.nVersion = struct.unpack("<i", f.read(4))[0] @@ -1048,18 +1041,18 @@ class msg_version: self.addrFrom = CAddress() self.addrFrom.deserialize(f, with_time=False) self.nNonce = struct.unpack("<Q", f.read(8))[0] - self.strSubVer = deser_string(f) + self.strSubVer = deser_string(f).decode('utf-8') self.nStartingHeight = struct.unpack("<i", f.read(4))[0] if self.nVersion >= 70001: # Relay field is optional for version 70001 onwards try: - self.nRelay = struct.unpack("<b", f.read(1))[0] + self.relay = struct.unpack("<b", f.read(1))[0] except: - self.nRelay = 0 + self.relay = 0 else: - self.nRelay = 0 + self.relay = 0 def serialize(self): r = b"" @@ -1069,16 +1062,16 @@ class msg_version: r += self.addrTo.serialize(with_time=False) r += self.addrFrom.serialize(with_time=False) r += struct.pack("<Q", self.nNonce) - r += ser_string(self.strSubVer) + r += ser_string(self.strSubVer.encode('utf-8')) r += struct.pack("<i", self.nStartingHeight) - r += struct.pack("<b", self.nRelay) + r += struct.pack("<b", self.relay) return r def __repr__(self): - return 'msg_version(nVersion=%i nServices=%i nTime=%s addrTo=%s addrFrom=%s nNonce=0x%016X strSubVer=%s nStartingHeight=%i nRelay=%i)' \ + return 'msg_version(nVersion=%i nServices=%i nTime=%s addrTo=%s addrFrom=%s nNonce=0x%016X strSubVer=%s nStartingHeight=%i relay=%i)' \ % (self.nVersion, self.nServices, time.ctime(self.nTime), repr(self.addrTo), repr(self.addrFrom), self.nNonce, - self.strSubVer, self.nStartingHeight, self.nRelay) + self.strSubVer, self.nStartingHeight, self.relay) class msg_verack: diff --git a/test/functional/test_framework/p2p.py b/test/functional/test_framework/p2p.py index fa4a567aac..05099f3339 100755 --- a/test/functional/test_framework/p2p.py +++ b/test/functional/test_framework/p2p.py @@ -31,7 +31,6 @@ import threading from test_framework.messages import ( CBlockHeader, MAX_HEADERS_RESULTS, - MIN_VERSION_SUPPORTED, msg_addr, msg_addrv2, msg_block, @@ -79,6 +78,18 @@ from test_framework.util import ( logger = logging.getLogger("TestFramework.p2p") +# The minimum P2P version that this test framework supports +MIN_P2P_VERSION_SUPPORTED = 60001 +# The P2P version that this test framework implements and sends in its `version` message +# Version 70016 supports wtxid relay +P2P_VERSION = 70016 +# The services that this test framework offers in its `version` message +P2P_SERVICES = NODE_NETWORK | NODE_WITNESS +# The P2P user agent string that this test framework sends in its `version` message +P2P_SUBVERSION = "/python-p2p-tester:0.0.3/" +# Value for relay that this test framework sends in its `version` message +P2P_VERSION_RELAY = 1 + MESSAGEMAP = { b"addr": msg_addr, b"addrv2": msg_addrv2, @@ -327,6 +338,9 @@ class P2PInterface(P2PConnection): def peer_connect_send_version(self, services): # Send a version msg vt = msg_version() + vt.nVersion = P2P_VERSION + vt.strSubVer = P2P_SUBVERSION + vt.relay = P2P_VERSION_RELAY vt.nServices = services vt.addrTo.ip = self.dstaddr vt.addrTo.port = self.dstport @@ -334,7 +348,7 @@ class P2PInterface(P2PConnection): vt.addrFrom.port = 0 self.on_connection_send_msg = vt # Will be sent in connection_made callback - def peer_connect(self, *args, services=NODE_NETWORK | NODE_WITNESS, send_version=True, **kwargs): + def peer_connect(self, *args, services=P2P_SERVICES, send_version=True, **kwargs): create_conn = super().peer_connect(*args, **kwargs) if send_version: @@ -417,7 +431,7 @@ class P2PInterface(P2PConnection): pass def on_version(self, message): - assert message.nVersion >= MIN_VERSION_SUPPORTED, "Version {} received. Test framework only supports versions greater than {}".format(message.nVersion, MIN_VERSION_SUPPORTED) + assert message.nVersion >= MIN_P2P_VERSION_SUPPORTED, "Version {} received. Test framework only supports versions greater than {}".format(message.nVersion, MIN_P2P_VERSION_SUPPORTED) if message.nVersion >= 70016 and self.wtxidrelay: self.send_message(msg_wtxidrelay()) if self.support_addrv2: diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index 6f4a9ebae2..02eb10b5a4 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -829,10 +829,19 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): self.options.previous_releases_path)) return self.options.prev_releases + def skip_if_no_external_signer(self): + """Skip the running test if external signer support has not been compiled.""" + if not self.is_external_signer_compiled(): + raise SkipTest("external signer support has not been compiled.") + def is_cli_compiled(self): """Checks whether bitcoin-cli was compiled.""" return self.config["components"].getboolean("ENABLE_CLI") + def is_external_signer_compiled(self): + """Checks whether external signer support was compiled.""" + return self.config["components"].getboolean("ENABLE_EXTERNAL_SIGNER") + def is_wallet_compiled(self): """Checks whether the wallet module was compiled.""" return self.config["components"].getboolean("ENABLE_WALLET") diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index b61d433652..ce9c1bc024 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -23,9 +23,10 @@ import sys from .authproxy import JSONRPCException from .descriptors import descsum_create -from .messages import MY_SUBVERSION +from .p2p import P2P_SUBVERSION from .util import ( MAX_NODES, + assert_equal, append_config, delete_cookie_file, get_auth_cookie, @@ -114,6 +115,8 @@ class TestNode(): if self.version_is_at_least(190000): self.args.append("-logthreadnames") + if self.version_is_at_least(219900): + self.args.append("-logsourcelocations") self.cli = TestNodeCLI(bitcoin_cli, self.datadir) self.use_cli = use_cli @@ -545,6 +548,11 @@ class TestNode(): # in comparison to the upside of making tests less fragile and unexpected intermittent errors less likely. p2p_conn.sync_with_ping() + # Consistency check that the Bitcoin Core has received our user agent string. This checks the + # node's newest peer. It could be racy if another Bitcoin Core node has connected since we opened + # our connection, but we don't expect that to happen. + assert_equal(self.getpeerinfo()[-1]['subver'], P2P_SUBVERSION) + return p2p_conn def add_outbound_p2p_connection(self, p2p_conn, *, p2p_idx, connection_type="outbound-full-relay", **kwargs): @@ -572,7 +580,7 @@ class TestNode(): def num_test_p2p_connections(self): """Return number of test framework p2p connections to the node.""" - return len([peer for peer in self.getpeerinfo() if peer['subver'] == MY_SUBVERSION.decode("utf-8")]) + return len([peer for peer in self.getpeerinfo() if peer['subver'] == P2P_SUBVERSION]) def disconnect_p2ps(self): """Close all p2p connections to the node.""" @@ -670,10 +678,10 @@ class RPCOverloadWrapper(): def __getattr__(self, name): return getattr(self.rpc, name) - def createwallet(self, wallet_name, disable_private_keys=None, blank=None, passphrase='', avoid_reuse=None, descriptors=None, load_on_startup=None): + def createwallet(self, wallet_name, disable_private_keys=None, blank=None, passphrase='', avoid_reuse=None, descriptors=None, load_on_startup=None, external_signer=None): if descriptors is None: descriptors = self.descriptors - return self.__getattr__('createwallet')(wallet_name, disable_private_keys, blank, passphrase, avoid_reuse, descriptors, load_on_startup) + return self.__getattr__('createwallet')(wallet_name, disable_private_keys, blank, passphrase, avoid_reuse, descriptors, load_on_startup, external_signer) def importprivkey(self, privkey, label=None, rescan=None): wallet_info = self.getwalletinfo() diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 354cacf95a..79ad2cf161 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -111,6 +111,7 @@ BASE_SCRIPTS = [ 'wallet_listtransactions.py --legacy-wallet', 'wallet_listtransactions.py --descriptors', 'feature_taproot.py', + 'wallet_signer.py --descriptors', # vv Tests less than 60s vv 'p2p_sendheaders.py', 'wallet_importmulti.py --legacy-wallet', @@ -289,6 +290,7 @@ BASE_SCRIPTS = [ 'feature_help.py', 'feature_shutdown.py', 'p2p_ibd_txrelay.py', + 'feature_blockfilterindex_prune.py' # Don't append tests at the end to avoid merge conflicts # Put them in a random line within the section that fits their approximate run-time ] diff --git a/test/functional/wallet_descriptor.py b/test/functional/wallet_descriptor.py index c04a15a67c..1e032bdd6c 100755 --- a/test/functional/wallet_descriptor.py +++ b/test/functional/wallet_descriptor.py @@ -151,5 +151,62 @@ class WalletDescriptorTest(BitcoinTestFramework): nopriv_rpc = self.nodes[0].get_wallet_rpc('desc_no_priv') assert_raises_rpc_error(-4, 'This wallet has no available keys', nopriv_rpc.getnewaddress) + self.log.info("Test descriptor exports") + self.nodes[0].createwallet(wallet_name='desc_export', descriptors=True) + exp_rpc = self.nodes[0].get_wallet_rpc('desc_export') + self.nodes[0].createwallet(wallet_name='desc_import', disable_private_keys=True, descriptors=True) + imp_rpc = self.nodes[0].get_wallet_rpc('desc_import') + + addr_types = [('legacy', False, 'pkh(', '44\'/1\'/0\'', -13), + ('p2sh-segwit', False, 'sh(wpkh(', '49\'/1\'/0\'', -14), + ('bech32', False, 'wpkh(', '84\'/1\'/0\'', -13), + ('legacy', True, 'pkh(', '44\'/1\'/0\'', -13), + ('p2sh-segwit', True, 'sh(wpkh(', '49\'/1\'/0\'', -14), + ('bech32', True, 'wpkh(', '84\'/1\'/0\'', -13)] + + for addr_type, internal, desc_prefix, deriv_path, int_idx in addr_types: + int_str = 'internal' if internal else 'external' + + self.log.info("Testing descriptor address type for {} {}".format(addr_type, int_str)) + if internal: + addr = exp_rpc.getrawchangeaddress(address_type=addr_type) + else: + addr = exp_rpc.getnewaddress(address_type=addr_type) + desc = exp_rpc.getaddressinfo(addr)['parent_desc'] + assert_equal(desc_prefix, desc[0:len(desc_prefix)]) + idx = desc.index('/') + 1 + assert_equal(deriv_path, desc[idx:idx + 9]) + if internal: + assert_equal('1', desc[int_idx]) + else: + assert_equal('0', desc[int_idx]) + + self.log.info("Testing the same descriptor is returned for address type {} {}".format(addr_type, int_str)) + for i in range(0, 10): + if internal: + addr = exp_rpc.getrawchangeaddress(address_type=addr_type) + else: + addr = exp_rpc.getnewaddress(address_type=addr_type) + test_desc = exp_rpc.getaddressinfo(addr)['parent_desc'] + assert_equal(desc, test_desc) + + self.log.info("Testing import of exported {} descriptor".format(addr_type)) + imp_rpc.importdescriptors([{ + 'desc': desc, + 'active': True, + 'next_index': 11, + 'timestamp': 'now', + 'internal': internal + }]) + + for i in range(0, 10): + if internal: + exp_addr = exp_rpc.getrawchangeaddress(address_type=addr_type) + imp_addr = imp_rpc.getrawchangeaddress(address_type=addr_type) + else: + exp_addr = exp_rpc.getnewaddress(address_type=addr_type) + imp_addr = imp_rpc.getnewaddress(address_type=addr_type) + assert_equal(exp_addr, imp_addr) + if __name__ == '__main__': WalletDescriptorTest().main () diff --git a/test/functional/wallet_signer.py b/test/functional/wallet_signer.py new file mode 100755 index 0000000000..9dd080dca9 --- /dev/null +++ b/test/functional/wallet_signer.py @@ -0,0 +1,217 @@ +#!/usr/bin/env python3 +# Copyright (c) 2017-2018 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test external signer. + +Verify that a bitcoind node can use an external signer command +""" +import os +import platform + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + assert_raises_rpc_error, +) + + +class SignerTest(BitcoinTestFramework): + def mock_signer_path(self): + path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'mocks', 'signer.py') + if platform.system() == "Windows": + return "py " + path + else: + return path + + def set_test_params(self): + self.num_nodes = 4 + + self.extra_args = [ + [], + [f"-signer={self.mock_signer_path()}", '-keypool=10'], + [f"-signer={self.mock_signer_path()}", '-keypool=10'], + ["-signer=fake.py"], + ] + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + self.skip_if_no_external_signer() + + def set_mock_result(self, node, res): + with open(os.path.join(node.cwd, "mock_result"), "w", encoding="utf8") as f: + f.write(res) + + def clear_mock_result(self, node): + os.remove(os.path.join(node.cwd, "mock_result")) + + def run_test(self): + self.log.debug(f"-signer={self.mock_signer_path()}") + + assert_raises_rpc_error(-4, 'Error: restart bitcoind with -signer=<cmd>', + self.nodes[0].enumeratesigners + ) + + # Handle script missing: + assert_raises_rpc_error(-1, 'execve failed: No such file or directory', + self.nodes[3].enumeratesigners + ) + + # Handle error thrown by script + self.set_mock_result(self.nodes[1], "2") + assert_raises_rpc_error(-1, 'RunCommandParseJSON error', + self.nodes[1].enumeratesigners + ) + self.clear_mock_result(self.nodes[1]) + + self.set_mock_result(self.nodes[1], '0 [{"type": "trezor", "model": "trezor_t", "error": "fingerprint not found"}]') + assert_raises_rpc_error(-4, 'fingerprint not found', + self.nodes[1].enumeratesigners + ) + self.clear_mock_result(self.nodes[1]) + + # Create new wallets for an external signer. + # disable_private_keys and descriptors must be true: + assert_raises_rpc_error(-4, "Private keys must be disabled when using an external signer", self.nodes[1].createwallet, wallet_name='not_hww', disable_private_keys=False, descriptors=True, external_signer=True) + if self.is_bdb_compiled(): + assert_raises_rpc_error(-4, "Descriptor support must be enabled when using an external signer", self.nodes[1].createwallet, wallet_name='not_hww', disable_private_keys=True, descriptors=False, external_signer=True) + else: + assert_raises_rpc_error(-4, "Compiled without bdb support (required for legacy wallets)", self.nodes[1].createwallet, wallet_name='not_hww', disable_private_keys=True, descriptors=False, external_signer=True) + + self.nodes[1].createwallet(wallet_name='hww', disable_private_keys=True, descriptors=True, external_signer=True) + hww = self.nodes[1].get_wallet_rpc('hww') + + result = hww.enumeratesigners() + assert_equal(len(result['signers']), 2) + assert_equal(result['signers'][0]["fingerprint"], "00000001") + assert_equal(result['signers'][0]["name"], "trezor_t") + + # Flag can't be set afterwards (could be added later for non-blank descriptor based watch-only wallets) + self.nodes[1].createwallet(wallet_name='not_hww', disable_private_keys=True, descriptors=True, external_signer=False) + not_hww = self.nodes[1].get_wallet_rpc('not_hww') + assert_raises_rpc_error(-8, "Wallet flag is immutable: external_signer", not_hww.setwalletflag, "external_signer", True) + + # assert_raises_rpc_error(-4, "Multiple signers found, please specify which to use", wallet_name='not_hww', disable_private_keys=True, descriptors=True, external_signer=True) + + # TODO: Handle error thrown by script + # self.set_mock_result(self.nodes[1], "2") + # assert_raises_rpc_error(-1, 'Unable to parse JSON', + # self.nodes[1].createwallet, wallet_name='not_hww2', disable_private_keys=True, descriptors=True, external_signer=False + # ) + # self.clear_mock_result(self.nodes[1]) + + assert_equal(hww.getwalletinfo()["keypoolsize"], 30) + + address1 = hww.getnewaddress(address_type="bech32") + assert_equal(address1, "bcrt1qm90ugl4d48jv8n6e5t9ln6t9zlpm5th68x4f8g") + address_info = hww.getaddressinfo(address1) + assert_equal(address_info['solvable'], True) + assert_equal(address_info['ismine'], True) + assert_equal(address_info['hdkeypath'], "m/84'/1'/0'/0/0") + + address2 = hww.getnewaddress(address_type="p2sh-segwit") + assert_equal(address2, "2N2gQKzjUe47gM8p1JZxaAkTcoHPXV6YyVp") + address_info = hww.getaddressinfo(address2) + assert_equal(address_info['solvable'], True) + assert_equal(address_info['ismine'], True) + assert_equal(address_info['hdkeypath'], "m/49'/1'/0'/0/0") + + address3 = hww.getnewaddress(address_type="legacy") + assert_equal(address3, "n1LKejAadN6hg2FrBXoU1KrwX4uK16mco9") + address_info = hww.getaddressinfo(address3) + assert_equal(address_info['solvable'], True) + assert_equal(address_info['ismine'], True) + assert_equal(address_info['hdkeypath'], "m/44'/1'/0'/0/0") + + self.log.info('Test signerdisplayaddress') + result = hww.signerdisplayaddress(address1) + assert_equal(result, {"address": address1}) + + # Handle error thrown by script + self.set_mock_result(self.nodes[1], "2") + assert_raises_rpc_error(-1, 'RunCommandParseJSON error', + hww.signerdisplayaddress, address1 + ) + self.clear_mock_result(self.nodes[1]) + + self.log.info('Prepare mock PSBT') + self.nodes[0].sendtoaddress(address1, 1) + self.nodes[0].generate(1) + self.sync_all() + + # Load private key into wallet to generate a signed PSBT for the mock + self.nodes[1].createwallet(wallet_name="mock", disable_private_keys=False, blank=True, descriptors=True) + mock_wallet = self.nodes[1].get_wallet_rpc("mock") + assert mock_wallet.getwalletinfo()['private_keys_enabled'] + + result = mock_wallet.importdescriptors([{ + "desc": "wpkh([00000001/84'/1'/0']tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/0/*)#rweraev0", + "timestamp": 0, + "range": [0,1], + "internal": False, + "active": True + }, + { + "desc": "wpkh([00000001/84'/1'/0']tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/*)#j6uzqvuh", + "timestamp": 0, + "range": [0, 0], + "internal": True, + "active": True + }]) + assert_equal(result[0], {'success': True}) + assert_equal(result[1], {'success': True}) + assert_equal(mock_wallet.getwalletinfo()["txcount"], 1) + dest = self.nodes[0].getnewaddress(address_type='bech32') + mock_psbt = mock_wallet.walletcreatefundedpsbt([], {dest:0.5}, 0, {}, True)['psbt'] + mock_psbt_signed = mock_wallet.walletprocesspsbt(psbt=mock_psbt, sign=True, sighashtype="ALL", bip32derivs=True) + mock_psbt_final = mock_wallet.finalizepsbt(mock_psbt_signed["psbt"]) + mock_tx = mock_psbt_final["hex"] + assert(mock_wallet.testmempoolaccept([mock_tx])[0]["allowed"]) + + # # Create a new wallet and populate with specific public keys, in order + # # to work with the mock signed PSBT. + # self.nodes[1].createwallet(wallet_name="hww4", disable_private_keys=True, descriptors=True, external_signer=True) + # hww4 = self.nodes[1].get_wallet_rpc("hww4") + # + # descriptors = [{ + # "desc": "wpkh([00000001/84'/1'/0']tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/0/*)#x30uthjs", + # "timestamp": "now", + # "range": [0, 1], + # "internal": False, + # "watchonly": True, + # "active": True + # }, + # { + # "desc": "wpkh([00000001/84'/1'/0']tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/1/*)#h92akzzg", + # "timestamp": "now", + # "range": [0, 0], + # "internal": True, + # "watchonly": True, + # "active": True + # }] + + # result = hww4.importdescriptors(descriptors) + # assert_equal(result[0], {'success': True}) + # assert_equal(result[1], {'success': True}) + assert_equal(hww.getwalletinfo()["txcount"], 1) + + assert(hww.testmempoolaccept([mock_tx])[0]["allowed"]) + + with open(os.path.join(self.nodes[1].cwd, "mock_psbt"), "w", encoding="utf8") as f: + f.write(mock_psbt_signed["psbt"]) + + self.log.info('Test send using hww1') + + res = hww.send(outputs={dest:0.5},options={"add_to_wallet": False}) + assert(res["complete"]) + assert_equal(res["hex"], mock_tx) + + # # Handle error thrown by script + # self.set_mock_result(self.nodes[4], "2") + # assert_raises_rpc_error(-1, 'Unable to parse JSON', + # hww4.signerprocesspsbt, psbt_orig, "00000001" + # ) + # self.clear_mock_result(self.nodes[4]) + +if __name__ == '__main__': + SignerTest().main() diff --git a/test/functional/wallet_watchonly.py b/test/functional/wallet_watchonly.py index 24799fe5f2..c345c382d0 100755 --- a/test/functional/wallet_watchonly.py +++ b/test/functional/wallet_watchonly.py @@ -2,7 +2,7 @@ # Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test createwallet arguments. +"""Test createwallet watchonly arguments. """ from test_framework.test_framework import BitcoinTestFramework @@ -49,6 +49,11 @@ class CreateWalletWatchonlyTest(BitcoinTestFramework): assert_equal(len(wo_wallet.listtransactions()), 1) assert_equal(wo_wallet.getbalance(include_watchonly=False), 0) + self.log.info('Test sending from a watch-only wallet raises RPC error') + msg = "Error: Private keys are disabled for this wallet" + assert_raises_rpc_error(-4, msg, wo_wallet.sendtoaddress, a1, 0.1) + assert_raises_rpc_error(-4, msg, wo_wallet.sendmany, amounts={a1: 0.1}) + self.log.info('Testing listreceivedbyaddress watch-only defaults') result = wo_wallet.listreceivedbyaddress() assert_equal(len(result), 1) diff --git a/test/lint/lint-circular-dependencies.sh b/test/lint/lint-circular-dependencies.sh index c4ad00e954..0b15f99448 100755 --- a/test/lint/lint-circular-dependencies.sh +++ b/test/lint/lint-circular-dependencies.sh @@ -11,6 +11,7 @@ export LC_ALL=C EXPECTED_CIRCULAR_DEPENDENCIES=( "chainparamsbase -> util/system -> chainparamsbase" "index/txindex -> validation -> index/txindex" + "index/blockfilterindex -> validation -> index/blockfilterindex" "policy/fees -> txmempool -> policy/fees" "qt/addresstablemodel -> qt/walletmodel -> qt/addresstablemodel" "qt/bitcoingui -> qt/walletframe -> qt/bitcoingui" @@ -20,6 +21,7 @@ EXPECTED_CIRCULAR_DEPENDENCIES=( "txmempool -> validation -> txmempool" "wallet/fees -> wallet/wallet -> wallet/fees" "wallet/wallet -> wallet/walletdb -> wallet/wallet" + "node/coinstats -> validation -> node/coinstats" ) EXIT_CODE=0 diff --git a/test/sanitizer_suppressions/tsan b/test/sanitizer_suppressions/tsan index 3a04418e8b..3fc9fac25c 100644 --- a/test/sanitizer_suppressions/tsan +++ b/test/sanitizer_suppressions/tsan @@ -28,6 +28,7 @@ race:BerkeleyBatch race:BerkeleyDatabase race:DatabaseBatch race:leveldb::DBImpl::DeleteObsoleteFiles +race:validation_chainstatemanager_tests race:zmq::* race:bitcoin-qt |