diff options
Diffstat (limited to 'test')
20 files changed, 642 insertions, 155 deletions
diff --git a/test/functional/feature_backwards_compatibility.py b/test/functional/feature_backwards_compatibility.py index 166c28d376..9cff79a42c 100755 --- a/test/functional/feature_backwards_compatibility.py +++ b/test/functional/feature_backwards_compatibility.py @@ -18,15 +18,16 @@ needs an older patch version. import os import shutil -from test_framework.test_framework import BitcoinTestFramework, SkipTest +from test_framework.test_framework import BitcoinTestFramework from test_framework.descriptors import descsum_create from test_framework.util import ( assert_equal, sync_blocks, - sync_mempools + sync_mempools, ) + class BackwardsCompatibilityTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True @@ -42,35 +43,15 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): def skip_test_if_missing_module(self): self.skip_if_no_wallet() + self.skip_if_no_previous_releases() def setup_nodes(self): - if os.getenv("TEST_PREVIOUS_RELEASES") == "false": - raise SkipTest("backwards compatibility tests") - - releases_path = os.getenv("PREVIOUS_RELEASES_DIR") or os.getcwd() + "/releases" - if not os.path.isdir(releases_path): - if os.getenv("TEST_PREVIOUS_RELEASES") == "true": - raise AssertionError("TEST_PREVIOUS_RELEASES=1 but releases missing: " + releases_path) - raise SkipTest("This test requires binaries for previous releases") - self.add_nodes(self.num_nodes, extra_args=self.extra_args, versions=[ None, None, - 190000, + 190001, 180100, - 170100 - ], binary=[ - self.options.bitcoind, - self.options.bitcoind, - releases_path + "/v0.19.0.1/bin/bitcoind", - releases_path + "/v0.18.1/bin/bitcoind", - releases_path + "/v0.17.1/bin/bitcoind" - ], binary_cli=[ - self.options.bitcoincli, - self.options.bitcoincli, - releases_path + "/v0.19.0.1/bin/bitcoin-cli", - releases_path + "/v0.18.1/bin/bitcoin-cli", - releases_path + "/v0.17.1/bin/bitcoin-cli" + 170100, ]) self.start_nodes() @@ -321,9 +302,9 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): node_v17.rpc.createwallet(wallet_name="u1_v17") wallet = node_v17.get_wallet_rpc("u1_v17") address = wallet.getnewaddress("bech32") - info = wallet.getaddressinfo(address) - hdkeypath = info["hdkeypath"] - pubkey = info["pubkey"] + v17_info = wallet.getaddressinfo(address) + hdkeypath = v17_info["hdkeypath"] + pubkey = v17_info["pubkey"] # Copy the 0.17 wallet to the last Bitcoin Core version and open it: node_v17.unloadwallet("u1_v17") @@ -337,6 +318,18 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): descriptor = "wpkh([" + info["hdmasterfingerprint"] + hdkeypath[1:] + "]" + pubkey + ")" assert_equal(info["desc"], descsum_create(descriptor)) + # Now copy that same wallet back to 0.17 to make sure no automatic upgrade breaks it + node_master.unloadwallet("u1_v17") + shutil.rmtree(os.path.join(node_v17_wallets_dir, "u1_v17")) + shutil.copytree( + os.path.join(node_master_wallets_dir, "u1_v17"), + os.path.join(node_v17_wallets_dir, "u1_v17") + ) + node_v17.loadwallet("u1_v17") + wallet = node_v17.get_wallet_rpc("u1_v17") + info = wallet.getaddressinfo(address) + assert_equal(info, v17_info) + # Copy the 0.19 wallet to the last Bitcoin Core version and open it: shutil.copytree( os.path.join(node_v19_wallets_dir, "w1_v19"), @@ -346,5 +339,16 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): wallet = node_master.get_wallet_rpc("w1_v19") assert wallet.getaddressinfo(address_18075)["solvable"] + # Now copy that same wallet back to 0.19 to make sure no automatic upgrade breaks it + node_master.unloadwallet("w1_v19") + shutil.rmtree(os.path.join(node_v19_wallets_dir, "w1_v19")) + shutil.copytree( + os.path.join(node_master_wallets_dir, "w1_v19"), + os.path.join(node_v19_wallets_dir, "w1_v19") + ) + node_v19.loadwallet("w1_v19") + wallet = node_v19.get_wallet_rpc("w1_v19") + assert wallet.getaddressinfo(address_18075)["solvable"] + if __name__ == '__main__': BackwardsCompatibilityTest().main() diff --git a/test/functional/framework_test_script.py b/test/functional/framework_test_script.py deleted file mode 100755 index 9d916c0022..0000000000 --- a/test/functional/framework_test_script.py +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Tests for test_framework.script.""" - -from test_framework.test_framework import BitcoinTestFramework -from test_framework.script import bn2vch -from test_framework.util import assert_equal - -def test_bn2vch(): - assert_equal(bn2vch(0), bytes([])) - assert_equal(bn2vch(1), bytes([0x01])) - assert_equal(bn2vch(-1), bytes([0x81])) - assert_equal(bn2vch(0x7F), bytes([0x7F])) - assert_equal(bn2vch(-0x7F), bytes([0xFF])) - assert_equal(bn2vch(0x80), bytes([0x80, 0x00])) - assert_equal(bn2vch(-0x80), bytes([0x80, 0x80])) - assert_equal(bn2vch(0xFF), bytes([0xFF, 0x00])) - assert_equal(bn2vch(-0xFF), bytes([0xFF, 0x80])) - assert_equal(bn2vch(0x100), bytes([0x00, 0x01])) - assert_equal(bn2vch(-0x100), bytes([0x00, 0x81])) - assert_equal(bn2vch(0x7FFF), bytes([0xFF, 0x7F])) - assert_equal(bn2vch(-0x8000), bytes([0x00, 0x80, 0x80])) - assert_equal(bn2vch(-0x7FFFFF), bytes([0xFF, 0xFF, 0xFF])) - assert_equal(bn2vch(0x80000000), bytes([0x00, 0x00, 0x00, 0x80, 0x00])) - assert_equal(bn2vch(-0x80000000), bytes([0x00, 0x00, 0x00, 0x80, 0x80])) - assert_equal(bn2vch(0xFFFFFFFF), bytes([0xFF, 0xFF, 0xFF, 0xFF, 0x00])) - - assert_equal(bn2vch(123456789), bytes([0x15, 0xCD, 0x5B, 0x07])) - assert_equal(bn2vch(-54321), bytes([0x31, 0xD4, 0x80])) - -class FrameworkTestScript(BitcoinTestFramework): - def setup_network(self): - pass - - def set_test_params(self): - self.num_nodes = 0 - - def run_test(self): - test_bn2vch() - -if __name__ == '__main__': - FrameworkTestScript().main() diff --git a/test/functional/mempool_persist.py b/test/functional/mempool_persist.py index e1671624a8..99003d2d1f 100755 --- a/test/functional/mempool_persist.py +++ b/test/functional/mempool_persist.py @@ -40,10 +40,13 @@ import os import time from test_framework.test_framework import BitcoinTestFramework +from test_framework.mininode import P2PTxInvStore from test_framework.util import ( assert_equal, assert_greater_than_or_equal, assert_raises_rpc_error, + connect_nodes, + disconnect_nodes, wait_until, ) @@ -80,6 +83,11 @@ class MempoolPersistTest(BitcoinTestFramework): assert_greater_than_or_equal(tx_creation_time, tx_creation_time_lower) assert_greater_than_or_equal(tx_creation_time_higher, tx_creation_time) + # disconnect nodes & make a txn that remains in the unbroadcast set. + disconnect_nodes(self.nodes[0], 2) + self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), Decimal("12")) + connect_nodes(self.nodes[0], 2) + self.log.debug("Stop-start the nodes. Verify that node0 has the transactions in its mempool and node1 does not. Verify that node2 calculates its balance correctly after loading wallet transactions.") self.stop_nodes() # Give this node a head-start, so we can be "extra-sure" that it didn't load anything later @@ -89,7 +97,7 @@ class MempoolPersistTest(BitcoinTestFramework): self.start_node(2) wait_until(lambda: self.nodes[0].getmempoolinfo()["loaded"], timeout=1) wait_until(lambda: self.nodes[2].getmempoolinfo()["loaded"], timeout=1) - assert_equal(len(self.nodes[0].getrawmempool()), 5) + assert_equal(len(self.nodes[0].getrawmempool()), 6) assert_equal(len(self.nodes[2].getrawmempool()), 5) # The others have loaded their mempool. If node_1 loaded anything, we'd probably notice by now: assert_equal(len(self.nodes[1].getrawmempool()), 0) @@ -105,9 +113,10 @@ class MempoolPersistTest(BitcoinTestFramework): self.nodes[2].syncwithvalidationinterfacequeue() # Flush mempool to wallet assert_equal(node2_balance, self.nodes[2].getbalance()) + # start node0 with wallet disabled so wallet transactions don't get resubmitted self.log.debug("Stop-start node0 with -persistmempool=0. Verify that it doesn't load its mempool.dat file.") self.stop_nodes() - self.start_node(0, extra_args=["-persistmempool=0"]) + self.start_node(0, extra_args=["-persistmempool=0", "-disablewallet"]) wait_until(lambda: self.nodes[0].getmempoolinfo()["loaded"]) assert_equal(len(self.nodes[0].getrawmempool()), 0) @@ -115,7 +124,7 @@ class MempoolPersistTest(BitcoinTestFramework): self.stop_nodes() self.start_node(0) wait_until(lambda: self.nodes[0].getmempoolinfo()["loaded"]) - assert_equal(len(self.nodes[0].getrawmempool()), 5) + assert_equal(len(self.nodes[0].getrawmempool()), 6) mempooldat0 = os.path.join(self.nodes[0].datadir, self.chain, 'mempool.dat') mempooldat1 = os.path.join(self.nodes[1].datadir, self.chain, 'mempool.dat') @@ -124,12 +133,12 @@ class MempoolPersistTest(BitcoinTestFramework): self.nodes[0].savemempool() assert os.path.isfile(mempooldat0) - self.log.debug("Stop nodes, make node1 use mempool.dat from node0. Verify it has 5 transactions") + self.log.debug("Stop nodes, make node1 use mempool.dat from node0. Verify it has 6 transactions") os.rename(mempooldat0, mempooldat1) self.stop_nodes() self.start_node(1, extra_args=[]) wait_until(lambda: self.nodes[1].getmempoolinfo()["loaded"]) - assert_equal(len(self.nodes[1].getrawmempool()), 5) + assert_equal(len(self.nodes[1].getrawmempool()), 6) self.log.debug("Prevent bitcoind from writing mempool.dat to disk. Verify that `savemempool` fails") # to test the exception we are creating a tmp folder called mempool.dat.new @@ -139,6 +148,27 @@ class MempoolPersistTest(BitcoinTestFramework): assert_raises_rpc_error(-1, "Unable to dump mempool to disk", self.nodes[1].savemempool) os.rmdir(mempooldotnew1) + self.test_persist_unbroadcast() + + def test_persist_unbroadcast(self): + node0 = self.nodes[0] + self.start_node(0) + + # clear out mempool + node0.generate(1) + + # disconnect nodes to make a txn that remains in the unbroadcast set. + disconnect_nodes(node0, 1) + node0.sendtoaddress(self.nodes[1].getnewaddress(), Decimal("12")) + + # shutdown, then startup with wallet disabled + self.stop_nodes() + self.start_node(0, extra_args=["-disablewallet"]) + + # check that txn gets broadcast due to unbroadcast logic + conn = node0.add_p2p_connection(P2PTxInvStore()) + node0.mockscheduler(16*60) # 15 min + 1 for buffer + wait_until(lambda: len(conn.get_invs()) == 1) if __name__ == '__main__': MempoolPersistTest().main() diff --git a/test/functional/mempool_unbroadcast.py b/test/functional/mempool_unbroadcast.py new file mode 100755 index 0000000000..a561f28b91 --- /dev/null +++ b/test/functional/mempool_unbroadcast.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +# Copyright (c) 2017-2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test that the mempool ensures transaction delivery by periodically sending +to peers until a GETDATA is received.""" + +import time + +from test_framework.mininode import P2PTxInvStore +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + connect_nodes, + create_confirmed_utxos, + disconnect_nodes, +) + + +class MempoolUnbroadcastTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 2 + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + + def run_test(self): + self.test_broadcast() + self.test_txn_removal() + + def test_broadcast(self): + self.log.info("Test that mempool reattempts delivery of locally submitted transaction") + node = self.nodes[0] + + min_relay_fee = node.getnetworkinfo()["relayfee"] + utxos = create_confirmed_utxos(min_relay_fee, node, 10) + + disconnect_nodes(node, 1) + + self.log.info("Generate transactions that only node 0 knows about") + + # generate a wallet txn + addr = node.getnewaddress() + wallet_tx_hsh = node.sendtoaddress(addr, 0.0001) + + # generate a txn using sendrawtransaction + us0 = utxos.pop() + inputs = [{"txid": us0["txid"], "vout": us0["vout"]}] + outputs = {addr: 0.0001} + tx = node.createrawtransaction(inputs, outputs) + node.settxfee(min_relay_fee) + txF = node.fundrawtransaction(tx) + txFS = node.signrawtransactionwithwallet(txF["hex"]) + rpc_tx_hsh = node.sendrawtransaction(txFS["hex"]) + + # check that second node doesn't have these two txns + mempool = self.nodes[1].getrawmempool() + assert rpc_tx_hsh not in mempool + assert wallet_tx_hsh not in mempool + + # ensure that unbroadcast txs are persisted to mempool.dat + self.restart_node(0) + + self.log.info("Reconnect nodes & check if they are sent to node 1") + connect_nodes(node, 1) + + # fast forward into the future & ensure that the second node has the txns + node.mockscheduler(15 * 60) # 15 min in seconds + self.sync_mempools(timeout=30) + mempool = self.nodes[1].getrawmempool() + assert rpc_tx_hsh in mempool + assert wallet_tx_hsh in mempool + + self.log.info("Add another connection & ensure transactions aren't broadcast again") + + conn = node.add_p2p_connection(P2PTxInvStore()) + node.mockscheduler(15 * 60) + time.sleep(5) + assert_equal(len(conn.get_invs()), 0) + + def test_txn_removal(self): + self.log.info("Test that transactions removed from mempool are removed from unbroadcast set") + node = self.nodes[0] + disconnect_nodes(node, 1) + node.disconnect_p2ps + + # since the node doesn't have any connections, it will not receive + # any GETDATAs & thus the transaction will remain in the unbroadcast set. + addr = node.getnewaddress() + txhsh = node.sendtoaddress(addr, 0.0001) + + # check transaction was removed from unbroadcast set due to presence in + # a block + removal_reason = "Removed {} from set of unbroadcast txns before confirmation that txn was sent out".format(txhsh) + with node.assert_debug_log([removal_reason]): + node.generate(1) + +if __name__ == "__main__": + MempoolUnbroadcastTest().main() diff --git a/test/functional/mempool_updatefromblock.py b/test/functional/mempool_updatefromblock.py new file mode 100755 index 0000000000..8a703ef009 --- /dev/null +++ b/test/functional/mempool_updatefromblock.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test mempool descendants/ancestors information update. + +Test mempool update of transaction descendants/ancestors information (count, size) +when transactions have been re-added from a disconnected block to the mempool. +""" +import time + +from decimal import Decimal +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal + + +class MempoolUpdateFromBlockTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + self.extra_args = [['-limitdescendantsize=1000', '-limitancestorsize=1000']] + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + + def transaction_graph_test(self, size, n_tx_to_mine=None, start_input_txid='', end_address='', fee=Decimal(0.00100000)): + """Create an acyclic tournament (a type of directed graph) of transactions and use it for testing. + + Keyword arguments: + size -- the order N of the tournament which is equal to the number of the created transactions + n_tx_to_mine -- the number of transaction that should be mined into a block + + If all of the N created transactions tx[0]..tx[N-1] reside in the mempool, + the following holds: + the tx[K] transaction: + - has N-K descendants (including this one), and + - has K+1 ancestors (including this one) + + More details: https://en.wikipedia.org/wiki/Tournament_(graph_theory) + """ + + if not start_input_txid: + start_input_txid = self.nodes[0].getblock(self.nodes[0].getblockhash(1))['tx'][0] + + if not end_address: + end_address = self.nodes[0].getnewaddress() + + first_block_hash = '' + tx_id = [] + tx_size = [] + self.log.info('Creating {} transactions...'.format(size)) + for i in range(0, size): + self.log.debug('Preparing transaction #{}...'.format(i)) + # Prepare inputs. + if i == 0: + inputs = [{'txid': start_input_txid, 'vout': 0}] + inputs_value = self.nodes[0].gettxout(start_input_txid, 0)['value'] + else: + inputs = [] + inputs_value = 0 + for j, tx in enumerate(tx_id[0:i]): + # Transaction tx[K] is a child of each of previous transactions tx[0]..tx[K-1] at their output K-1. + vout = i - j - 1 + inputs.append({'txid': tx_id[j], 'vout': vout}) + inputs_value += self.nodes[0].gettxout(tx, vout)['value'] + + self.log.debug('inputs={}'.format(inputs)) + self.log.debug('inputs_value={}'.format(inputs_value)) + + # Prepare outputs. + tx_count = i + 1 + if tx_count < size: + # Transaction tx[K] is an ancestor of each of subsequent transactions tx[K+1]..tx[N-1]. + n_outputs = size - tx_count + output_value = ((inputs_value - fee) / Decimal(n_outputs)).quantize(Decimal('0.00000001')) + outputs = {} + for n in range(0, n_outputs): + outputs[self.nodes[0].getnewaddress()] = output_value + else: + output_value = (inputs_value - fee).quantize(Decimal('0.00000001')) + outputs = {end_address: output_value} + + self.log.debug('output_value={}'.format(output_value)) + self.log.debug('outputs={}'.format(outputs)) + + # Create a new transaction. + unsigned_raw_tx = self.nodes[0].createrawtransaction(inputs, outputs) + signed_raw_tx = self.nodes[0].signrawtransactionwithwallet(unsigned_raw_tx) + tx_id.append(self.nodes[0].sendrawtransaction(signed_raw_tx['hex'])) + tx_size.append(self.nodes[0].getrawmempool(True)[tx_id[-1]]['vsize']) + + if tx_count in n_tx_to_mine: + # The created transactions are mined into blocks by batches. + self.log.info('The batch of {} transactions has been accepted into the mempool.'.format(len(self.nodes[0].getrawmempool()))) + block_hash = self.nodes[0].generate(1)[0] + if not first_block_hash: + first_block_hash = block_hash + assert_equal(len(self.nodes[0].getrawmempool()), 0) + self.log.info('All of the transactions from the current batch have been mined into a block.') + elif tx_count == size: + # At the end all of the mined blocks are invalidated, and all of the created + # transactions should be re-added from disconnected blocks to the mempool. + self.log.info('The last batch of {} transactions has been accepted into the mempool.'.format(len(self.nodes[0].getrawmempool()))) + start = time.time() + self.nodes[0].invalidateblock(first_block_hash) + end = time.time() + assert_equal(len(self.nodes[0].getrawmempool()), size) + self.log.info('All of the recently mined transactions have been re-added into the mempool in {} seconds.'.format(end - start)) + + self.log.info('Checking descendants/ancestors properties of all of the in-mempool transactions...') + for k, tx in enumerate(tx_id): + self.log.debug('Check transaction #{}.'.format(k)) + assert_equal(self.nodes[0].getrawmempool(True)[tx]['descendantcount'], size - k) + assert_equal(self.nodes[0].getrawmempool(True)[tx]['descendantsize'], sum(tx_size[k:size])) + assert_equal(self.nodes[0].getrawmempool(True)[tx]['ancestorcount'], k + 1) + assert_equal(self.nodes[0].getrawmempool(True)[tx]['ancestorsize'], sum(tx_size[0:(k + 1)])) + + def run_test(self): + # Use batch size limited by DEFAULT_ANCESTOR_LIMIT = 25 to not fire "too many unconfirmed parents" error. + self.transaction_graph_test(size=100, n_tx_to_mine=[25, 50, 75]) + + +if __name__ == '__main__': + MempoolUpdateFromBlockTest().main() diff --git a/test/functional/p2p_filter.py b/test/functional/p2p_filter.py index a8b768c144..15955a938c 100755 --- a/test/functional/p2p_filter.py +++ b/test/functional/p2p_filter.py @@ -64,19 +64,40 @@ class FilterTest(BitcoinTestFramework): def skip_test_if_missing_module(self): self.skip_if_no_wallet() - def run_test(self): - filter_node = self.nodes[0].add_p2p_connection(FilterNode()) - + def test_size_limits(self, filter_node): self.log.info('Check that too large filter is rejected') with self.nodes[0].assert_debug_log(['Misbehaving']): - filter_node.send_and_ping(msg_filterload(data=b'\xaa', nHashFuncs=MAX_BLOOM_HASH_FUNCS+1)) - with self.nodes[0].assert_debug_log(['Misbehaving']): filter_node.send_and_ping(msg_filterload(data=b'\xbb'*(MAX_BLOOM_FILTER_SIZE+1))) + self.log.info('Check that max size filter is accepted') + with self.nodes[0].assert_debug_log([], unexpected_msgs=['Misbehaving']): + filter_node.send_and_ping(msg_filterload(data=b'\xbb'*(MAX_BLOOM_FILTER_SIZE))) + filter_node.send_and_ping(msg_filterclear()) + + self.log.info('Check that filter with too many hash functions is rejected') + with self.nodes[0].assert_debug_log(['Misbehaving']): + filter_node.send_and_ping(msg_filterload(data=b'\xaa', nHashFuncs=MAX_BLOOM_HASH_FUNCS+1)) + + self.log.info('Check that filter with max hash functions is accepted') + with self.nodes[0].assert_debug_log([], unexpected_msgs=['Misbehaving']): + filter_node.send_and_ping(msg_filterload(data=b'\xaa', nHashFuncs=MAX_BLOOM_HASH_FUNCS)) + # Don't send filterclear until next two filteradd checks are done + + self.log.info('Check that max size data element to add to the filter is accepted') + with self.nodes[0].assert_debug_log([], unexpected_msgs=['Misbehaving']): + filter_node.send_and_ping(msg_filteradd(data=b'\xcc'*(MAX_SCRIPT_ELEMENT_SIZE))) + self.log.info('Check that too large data element to add to the filter is rejected') with self.nodes[0].assert_debug_log(['Misbehaving']): filter_node.send_and_ping(msg_filteradd(data=b'\xcc'*(MAX_SCRIPT_ELEMENT_SIZE+1))) + filter_node.send_and_ping(msg_filterclear()) + + def run_test(self): + filter_node = self.nodes[0].add_p2p_connection(FilterNode()) + + self.test_size_limits(filter_node) + self.log.info('Add filtered P2P connection to the node') filter_node.send_and_ping(filter_node.watch_filter_init) filter_address = self.nodes[0].decodescript(filter_node.watch_script_pubkey)['addresses'][0] diff --git a/test/functional/p2p_leak.py b/test/functional/p2p_leak.py index 7f7430d04e..157af68203 100755 --- a/test/functional/p2p_leak.py +++ b/test/functional/p2p_leak.py @@ -141,12 +141,11 @@ class P2PLeakTest(BitcoinTestFramework): assert no_verack_idlenode.unexpected_msg == False self.log.info('Check that the version message does not leak the local address of the node') - time_begin = int(time.time()) p2p_version_store = self.nodes[0].add_p2p_connection(P2PVersionStore()) - time_end = time.time() ver = p2p_version_store.version_received - assert_greater_than_or_equal(ver.nTime, time_begin) - assert_greater_than_or_equal(time_end, ver.nTime) + # Check that received time is within one hour of now + assert_greater_than_or_equal(ver.nTime, time.time() - 3600) + assert_greater_than_or_equal(time.time() + 3600, ver.nTime) assert_equal(ver.addrFrom.port, 0) assert_equal(ver.addrFrom.ip, '0.0.0.0') assert_equal(ver.nStartingHeight, 201) diff --git a/test/functional/p2p_segwit.py b/test/functional/p2p_segwit.py index dbdce6552a..6fb0fec32b 100755 --- a/test/functional/p2p_segwit.py +++ b/test/functional/p2p_segwit.py @@ -1896,12 +1896,12 @@ class SegWitTest(BitcoinTestFramework): def test_upgrade_after_activation(self): """Test the behavior of starting up a segwit-aware node after the softfork has activated.""" - # Restart with the new binary self.stop_node(2) self.start_node(2, extra_args=["-segwitheight={}".format(SEGWIT_HEIGHT)]) connect_nodes(self.nodes[0], 2) - self.sync_blocks() + # We reconnect more than 100 blocks, give it plenty of time + self.sync_blocks(timeout=240) # Make sure that this peer thinks segwit has activated. assert softfork_active(self.nodes[2], 'segwit') diff --git a/test/functional/test_framework/mininode.py b/test/functional/test_framework/mininode.py index 6aa73623e6..31cec66ee7 100755 --- a/test/functional/test_framework/mininode.py +++ b/test/functional/test_framework/mininode.py @@ -12,7 +12,10 @@ found in the mini-node branch of http://github.com/jgarzik/pynode. P2PConnection: A low-level connection object to a node's P2P interface P2PInterface: A high-level interface object for communicating to a node over P2P P2PDataStore: A p2p interface class that keeps a store of transactions and blocks - and can respond correctly to getdata and getheaders messages""" + and can respond correctly to getdata and getheaders messages +P2PTxInvStore: A p2p interface class that inherits from P2PDataStore, and keeps + a count of how many times each txid has been announced.""" + import asyncio from collections import defaultdict from io import BytesIO @@ -117,8 +120,9 @@ class P2PConnection(asyncio.Protocol): def is_connected(self): return self._transport is not None - def peer_connect(self, dstaddr, dstport, *, net): + def peer_connect(self, dstaddr, dstport, *, net, factor): assert not self.is_connected + self.factor = factor self.dstaddr = dstaddr self.dstport = dstport # The initial message to send after the connection was made: @@ -364,9 +368,12 @@ class P2PInterface(P2PConnection): # Connection helper methods + def wait_until(self, test_function, timeout): + wait_until(test_function, timeout=timeout, lock=mininode_lock, factor=self.factor) + def wait_for_disconnect(self, timeout=60): test_function = lambda: not self.is_connected - wait_until(test_function, timeout=timeout, lock=mininode_lock) + self.wait_until(test_function, timeout=timeout) # Message receiving helper methods @@ -377,14 +384,14 @@ class P2PInterface(P2PConnection): return False return self.last_message['tx'].tx.rehash() == txid - wait_until(test_function, timeout=timeout, lock=mininode_lock) + self.wait_until(test_function, timeout=timeout) def wait_for_block(self, blockhash, timeout=60): def test_function(): assert self.is_connected return self.last_message.get("block") and self.last_message["block"].block.rehash() == blockhash - wait_until(test_function, timeout=timeout, lock=mininode_lock) + self.wait_until(test_function, timeout=timeout) def wait_for_header(self, blockhash, timeout=60): def test_function(): @@ -394,7 +401,7 @@ class P2PInterface(P2PConnection): return False return last_headers.headers[0].rehash() == int(blockhash, 16) - wait_until(test_function, timeout=timeout, lock=mininode_lock) + self.wait_until(test_function, timeout=timeout) def wait_for_merkleblock(self, blockhash, timeout=60): def test_function(): @@ -404,7 +411,7 @@ class P2PInterface(P2PConnection): return False return last_filtered_block.merkleblock.header.rehash() == int(blockhash, 16) - wait_until(test_function, timeout=timeout, lock=mininode_lock) + self.wait_until(test_function, timeout=timeout) def wait_for_getdata(self, hash_list, timeout=60): """Waits for a getdata message. @@ -418,7 +425,7 @@ class P2PInterface(P2PConnection): return False return [x.hash for x in last_data.inv] == hash_list - wait_until(test_function, timeout=timeout, lock=mininode_lock) + self.wait_until(test_function, timeout=timeout) def wait_for_getheaders(self, timeout=60): """Waits for a getheaders message. @@ -432,7 +439,7 @@ class P2PInterface(P2PConnection): assert self.is_connected return self.last_message.get("getheaders") - wait_until(test_function, timeout=timeout, lock=mininode_lock) + self.wait_until(test_function, timeout=timeout) def wait_for_inv(self, expected_inv, timeout=60): """Waits for an INV message and checks that the first inv object in the message was as expected.""" @@ -445,13 +452,13 @@ class P2PInterface(P2PConnection): self.last_message["inv"].inv[0].type == expected_inv[0].type and \ self.last_message["inv"].inv[0].hash == expected_inv[0].hash - wait_until(test_function, timeout=timeout, lock=mininode_lock) + self.wait_until(test_function, timeout=timeout) def wait_for_verack(self, timeout=60): def test_function(): return self.message_count["verack"] - wait_until(test_function, timeout=timeout, lock=mininode_lock) + self.wait_until(test_function, timeout=timeout) # Message sending helper functions @@ -467,7 +474,7 @@ class P2PInterface(P2PConnection): assert self.is_connected return self.last_message.get("pong") and self.last_message["pong"].nonce == self.ping_counter - wait_until(test_function, timeout=timeout, lock=mininode_lock) + self.wait_until(test_function, timeout=timeout) self.ping_counter += 1 @@ -583,7 +590,7 @@ class P2PDataStore(P2PInterface): self.send_message(msg_block(block=b)) else: self.send_message(msg_headers([CBlockHeader(block) for block in blocks])) - wait_until(lambda: blocks[-1].sha256 in self.getdata_requests, timeout=timeout, lock=mininode_lock) + self.wait_until(lambda: blocks[-1].sha256 in self.getdata_requests, timeout=timeout) if expect_disconnect: self.wait_for_disconnect(timeout=timeout) @@ -591,7 +598,7 @@ class P2PDataStore(P2PInterface): self.sync_with_ping(timeout=timeout) if success: - wait_until(lambda: node.getbestblockhash() == blocks[-1].hash, timeout=timeout) + self.wait_until(lambda: node.getbestblockhash() == blocks[-1].hash, timeout=timeout) else: assert node.getbestblockhash() != blocks[-1].hash @@ -627,3 +634,20 @@ class P2PDataStore(P2PInterface): # Check that none of the txs are now in the mempool for tx in txs: assert tx.hash not in raw_mempool, "{} tx found in mempool".format(tx.hash) + +class P2PTxInvStore(P2PInterface): + """A P2PInterface which stores a count of how many times each txid has been announced.""" + def __init__(self): + super().__init__() + self.tx_invs_received = defaultdict(int) + + def on_inv(self, message): + # Store how many times invs have been received for each tx. + for i in message.inv: + if i.type == MSG_TX: + # save txid + self.tx_invs_received[i.hash] += 1 + + def get_invs(self): + with mininode_lock: + return list(self.tx_invs_received.keys()) diff --git a/test/functional/test_framework/script.py b/test/functional/test_framework/script.py index e475ed8596..9102266456 100644 --- a/test/functional/test_framework/script.py +++ b/test/functional/test_framework/script.py @@ -8,6 +8,7 @@ This file is modified from python-bitcoinlib. """ import hashlib import struct +import unittest from .messages import ( CTransaction, @@ -708,3 +709,25 @@ def SegwitV0SignatureHash(script, txTo, inIdx, hashtype, amount): ss += struct.pack("<I", hashtype) return hash256(ss) + +class TestFrameworkScript(unittest.TestCase): + def test_bn2vch(self): + self.assertEqual(bn2vch(0), bytes([])) + self.assertEqual(bn2vch(1), bytes([0x01])) + self.assertEqual(bn2vch(-1), bytes([0x81])) + self.assertEqual(bn2vch(0x7F), bytes([0x7F])) + self.assertEqual(bn2vch(-0x7F), bytes([0xFF])) + self.assertEqual(bn2vch(0x80), bytes([0x80, 0x00])) + self.assertEqual(bn2vch(-0x80), bytes([0x80, 0x80])) + self.assertEqual(bn2vch(0xFF), bytes([0xFF, 0x00])) + self.assertEqual(bn2vch(-0xFF), bytes([0xFF, 0x80])) + self.assertEqual(bn2vch(0x100), bytes([0x00, 0x01])) + self.assertEqual(bn2vch(-0x100), bytes([0x00, 0x81])) + self.assertEqual(bn2vch(0x7FFF), bytes([0xFF, 0x7F])) + self.assertEqual(bn2vch(-0x8000), bytes([0x00, 0x80, 0x80])) + self.assertEqual(bn2vch(-0x7FFFFF), bytes([0xFF, 0xFF, 0xFF])) + self.assertEqual(bn2vch(0x80000000), bytes([0x00, 0x00, 0x00, 0x80, 0x00])) + self.assertEqual(bn2vch(-0x80000000), bytes([0x00, 0x00, 0x00, 0x80, 0x80])) + self.assertEqual(bn2vch(0xFFFFFFFF), bytes([0xFF, 0xFF, 0xFF, 0xFF, 0x00])) + self.assertEqual(bn2vch(123456789), bytes([0x15, 0xCD, 0x5B, 0x07])) + self.assertEqual(bn2vch(-54321), bytes([0x31, 0xD4, 0x80])) diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index 8719bd0d39..11c96deefb 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -6,11 +6,12 @@ import configparser from enum import Enum -import logging import argparse +import logging import os import pdb import random +import re import shutil import subprocess import sys @@ -101,6 +102,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): self.bind_to_localhost_only = True self.set_test_params() self.parse_args() + self.rpc_timeout = int(self.rpc_timeout * self.options.factor) # optionally, increase timeout by a factor def main(self): """Main function. This should not be overridden by the subclass test scripts.""" @@ -167,6 +169,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): help="set a random seed for deterministically reproducing a previous test run") parser.add_argument("--descriptors", default=False, action="store_true", help="Run test using a descriptor wallet") + parser.add_argument('--factor', type=float, default=1.0, help='adjust test timeouts by a factor') self.add_options(parser) self.options = parser.parse_args() @@ -185,10 +188,11 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): self.options.bitcoind = os.getenv("BITCOIND", default=config["environment"]["BUILDDIR"] + '/src/bitcoind' + config["environment"]["EXEEXT"]) self.options.bitcoincli = os.getenv("BITCOINCLI", default=config["environment"]["BUILDDIR"] + '/src/bitcoin-cli' + config["environment"]["EXEEXT"]) + self.options.previous_releases_path = os.getenv("PREVIOUS_RELEASES_DIR") or os.getcwd() + "/releases" + os.environ['PATH'] = os.pathsep.join([ os.path.join(config['environment']['BUILDDIR'], 'src'), - os.path.join(config['environment']['BUILDDIR'], 'src', 'qt'), - os.environ['PATH'] + os.path.join(config['environment']['BUILDDIR'], 'src', 'qt'), os.environ['PATH'] ]) # Set up temp directory and start logging @@ -388,6 +392,25 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): Should only be called once after the nodes have been specified in set_test_params().""" + def get_bin_from_version(version, bin_name, bin_default): + if not version: + return bin_default + return os.path.join( + self.options.previous_releases_path, + re.sub( + r'\.0$', + '', # remove trailing .0 for point releases + 'v{}.{}.{}.{}'.format( + (version % 100000000) // 1000000, + (version % 1000000) // 10000, + (version % 10000) // 100, + (version % 100) // 1, + ), + ), + 'bin', + bin_name, + ) + if self.bind_to_localhost_only: extra_confs = [["bind=127.0.0.1"]] * num_nodes else: @@ -397,9 +420,9 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): if versions is None: versions = [None] * num_nodes if binary is None: - binary = [self.options.bitcoind] * num_nodes + binary = [get_bin_from_version(v, 'bitcoind', self.options.bitcoind) for v in versions] if binary_cli is None: - binary_cli = [self.options.bitcoincli] * num_nodes + binary_cli = [get_bin_from_version(v, 'bitcoin-cli', self.options.bitcoincli) for v in versions] assert_equal(len(extra_confs), num_nodes) assert_equal(len(extra_args), num_nodes) assert_equal(len(versions), num_nodes) @@ -412,6 +435,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): chain=self.chain, rpchost=rpchost, timewait=self.rpc_timeout, + factor=self.options.factor, bitcoind=binary[i], bitcoin_cli=binary_cli[i], version=versions[i], @@ -558,6 +582,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): extra_args=['-disablewallet'], rpchost=None, timewait=self.rpc_timeout, + factor=self.options.factor, bitcoind=self.options.bitcoind, bitcoin_cli=self.options.bitcoincli, coverage_dir=None, @@ -640,6 +665,25 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): if not self.is_cli_compiled(): raise SkipTest("bitcoin-cli has not been compiled.") + def skip_if_no_previous_releases(self): + """Skip the running test if previous releases are not available.""" + if not self.has_previous_releases(): + raise SkipTest("previous releases not available or disabled") + + def has_previous_releases(self): + """Checks whether previous releases are present and enabled.""" + if os.getenv("TEST_PREVIOUS_RELEASES") == "false": + # disabled + return False + + if not os.path.isdir(self.options.previous_releases_path): + if os.getenv("TEST_PREVIOUS_RELEASES") == "true": + raise AssertionError("TEST_PREVIOUS_RELEASES=true but releases missing: {}".format( + self.options.previous_releases_path)) + # missing + return False + return True + def is_cli_compiled(self): """Checks whether bitcoin-cli was compiled.""" return self.config["components"].getboolean("ENABLE_CLI") diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index c0075fd8ec..e6ec3c1b2d 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -62,7 +62,7 @@ class TestNode(): To make things easier for the test writer, any unrecognised messages will be dispatched to the RPC connection.""" - def __init__(self, i, datadir, *, chain, rpchost, timewait, bitcoind, bitcoin_cli, coverage_dir, cwd, extra_conf=None, extra_args=None, use_cli=False, start_perf=False, use_valgrind=False, version=None, descriptors=False): + def __init__(self, i, datadir, *, chain, rpchost, timewait, factor, bitcoind, bitcoin_cli, coverage_dir, cwd, extra_conf=None, extra_args=None, use_cli=False, start_perf=False, use_valgrind=False, version=None, descriptors=False): """ Kwargs: start_perf (bool): If True, begin profiling the node with `perf` as soon as @@ -128,6 +128,7 @@ class TestNode(): self.perf_subprocesses = {} self.p2ps = [] + self.factor = factor AddressKeyPair = collections.namedtuple('AddressKeyPair', ['address', 'key']) PRIV_KEYS = [ @@ -324,13 +325,13 @@ class TestNode(): return True def wait_until_stopped(self, timeout=BITCOIND_PROC_WAIT_TIMEOUT): - wait_until(self.is_node_stopped, timeout=timeout) + wait_until(self.is_node_stopped, timeout=timeout, factor=self.factor) @contextlib.contextmanager def assert_debug_log(self, expected_msgs, unexpected_msgs=None, timeout=2): if unexpected_msgs is None: unexpected_msgs = [] - time_end = time.time() + timeout + time_end = time.time() + timeout * self.factor debug_log = os.path.join(self.datadir, self.chain, 'debug.log') with open(debug_log, encoding='utf-8') as dl: dl.seek(0, 2) @@ -487,7 +488,7 @@ class TestNode(): if 'dstaddr' not in kwargs: kwargs['dstaddr'] = '127.0.0.1' - p2p_conn.peer_connect(**kwargs, net=self.chain)() + p2p_conn.peer_connect(**kwargs, net=self.chain, factor=self.factor)() self.p2ps.append(p2p_conn) if wait_for_verack: # Wait for the node to send us the version and verack diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index 64e1aa3bbc..20ab9ee464 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -208,9 +208,10 @@ def str_to_b64str(string): def satoshi_round(amount): return Decimal(amount).quantize(Decimal('0.00000001'), rounding=ROUND_DOWN) -def wait_until(predicate, *, attempts=float('inf'), timeout=float('inf'), lock=None): +def wait_until(predicate, *, attempts=float('inf'), timeout=float('inf'), lock=None, factor=1.0): if attempts == float('inf') and timeout == float('inf'): timeout = 60 + timeout = timeout * factor attempt = 0 time_end = time.time() + timeout @@ -265,7 +266,7 @@ def get_rpc_proxy(url, node_number, *, timeout=None, coveragedir=None): """ proxy_kwargs = {} if timeout is not None: - proxy_kwargs['timeout'] = timeout + proxy_kwargs['timeout'] = int(timeout) proxy = AuthServiceProxy(url, **proxy_kwargs) proxy.url = url # store URL on proxy for info @@ -326,6 +327,13 @@ def initialize_datadir(dirname, n, chain): os.makedirs(os.path.join(datadir, 'stdout'), exist_ok=True) return datadir +def adjust_bitcoin_conf_for_pre_17(conf_file): + with open(conf_file,'r', encoding='utf8') as conf: + conf_data = conf.read() + with open(conf_file, 'w', encoding='utf8') as conf: + conf_data_changed = conf_data.replace('[regtest]', '') + conf.write(conf_data_changed) + def get_datadir_path(dirname, n): return os.path.join(dirname, "node" + str(n)) diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index b8523e16b7..45f591ba4a 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -24,6 +24,7 @@ import sys import tempfile import re import logging +import unittest # Formatting. Default colors to empty strings. BOLD, GREEN, RED, GREY = ("", ""), ("", ""), ("", ""), ("", "") @@ -65,6 +66,10 @@ if os.name != 'nt' or sys.getwindowsversion() >= (10, 0, 14393): TEST_EXIT_PASSED = 0 TEST_EXIT_SKIPPED = 77 +TEST_FRAMEWORK_MODULES = [ + "script", +] + EXTENDED_SCRIPTS = [ # These tests are not run by default. # Longest test should go first, to favor running tests in parallel @@ -93,6 +98,7 @@ BASE_SCRIPTS = [ 'p2p_segwit.py', 'p2p_timeouts.py', 'p2p_tx_download.py', + 'mempool_updatefromblock.py', 'wallet_dump.py', 'wallet_listtransactions.py', # vv Tests less than 60s vv @@ -190,6 +196,7 @@ BASE_SCRIPTS = [ 'wallet_import_rescan.py', 'wallet_import_with_label.py', 'wallet_importdescriptors.py', + 'wallet_upgradewallet.py', 'rpc_bind.py --ipv4', 'rpc_bind.py --ipv6', 'rpc_bind.py --nonloopback', @@ -220,6 +227,7 @@ BASE_SCRIPTS = [ 'p2p_unrequested_blocks.py', 'feature_includeconf.py', 'feature_asmap.py', + 'mempool_unbroadcast.py', 'rpc_deriveaddresses.py', 'rpc_deriveaddresses.py --usecli', 'rpc_scantxoutset.py', @@ -234,7 +242,6 @@ BASE_SCRIPTS = [ 'rpc_help.py', 'feature_help.py', 'feature_shutdown.py', - 'framework_test_script.py', # Don't append tests at the end to avoid merge conflicts # Put them in a random line within the section that fits their approximate run-time ] @@ -397,6 +404,16 @@ def run_tests(*, test_list, src_dir, build_dir, tmpdir, jobs=1, enable_coverage= if os.path.isdir(cache_dir): print("%sWARNING!%s There is a cache directory here: %s. If tests fail unexpectedly, try deleting the cache directory." % (BOLD[1], BOLD[0], cache_dir)) + # Test Framework Tests + print("Running Unit Tests for Test Framework Modules") + test_framework_tests = unittest.TestSuite() + for module in TEST_FRAMEWORK_MODULES: + test_framework_tests.addTest(unittest.TestLoader().loadTestsFromName("test_framework.{}".format(module))) + result = unittest.TextTestRunner(verbosity=1, failfast=True).run(test_framework_tests) + if not result.wasSuccessful(): + logging.debug("Early exiting after failure in TestFramework unit tests") + sys.exit(False) + tests_dir = src_dir + '/test/functional/' flags = ['--cachedir={}'.format(cache_dir)] + args @@ -620,7 +637,7 @@ class TestResult(): def check_script_prefixes(): """Check that test scripts start with one of the allowed name prefixes.""" - good_prefixes_re = re.compile("^(example|feature|interface|mempool|mining|p2p|rpc|wallet|tool|framework_test)_") + good_prefixes_re = re.compile("^(example|feature|interface|mempool|mining|p2p|rpc|wallet|tool)_") bad_script_names = [script for script in ALL_SCRIPTS if good_prefixes_re.match(script) is None] if bad_script_names: diff --git a/test/functional/tool_wallet.py b/test/functional/tool_wallet.py index b3d496dd51..039ce7daee 100755 --- a/test/functional/tool_wallet.py +++ b/test/functional/tool_wallet.py @@ -15,6 +15,7 @@ from test_framework.util import assert_equal BUFFER_SIZE = 16 * 1024 + class ToolWalletTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 @@ -48,7 +49,7 @@ class ToolWalletTest(BitcoinTestFramework): h = hashlib.sha1() mv = memoryview(bytearray(BUFFER_SIZE)) with open(self.wallet_path, 'rb', buffering=0) as f: - for n in iter(lambda : f.readinto(mv), 0): + for n in iter(lambda: f.readinto(mv), 0): h.update(mv[:n]) return h.hexdigest() @@ -69,7 +70,12 @@ class ToolWalletTest(BitcoinTestFramework): self.assert_raises_tool_error('Invalid command: help', 'help') self.assert_raises_tool_error('Error: two methods provided (info and create). Only one method should be provided.', 'info', 'create') self.assert_raises_tool_error('Error parsing command line arguments: Invalid parameter -foo', '-foo') - self.assert_raises_tool_error('Error loading wallet.dat. Is wallet being used by other process?', '-wallet=wallet.dat', 'info') + self.assert_raises_tool_error( + 'Error initializing wallet database environment "{}"!\nError loading wallet.dat. Is wallet being used by other process?' + .format(os.path.join(self.nodes[0].datadir, self.chain, 'wallets')), + '-wallet=wallet.dat', + 'info', + ) self.assert_raises_tool_error('Error: no wallet file at nonexistent.dat', '-wallet=nonexistent.dat', 'info') def test_tool_wallet_info(self): @@ -84,7 +90,7 @@ class ToolWalletTest(BitcoinTestFramework): # # self.log.debug('Setting wallet file permissions to 400 (read-only)') # os.chmod(self.wallet_path, stat.S_IRUSR) - # assert(self.wallet_permissions() in ['400', '666']) # Sanity check. 666 because Appveyor. + # assert self.wallet_permissions() in ['400', '666'] # Sanity check. 666 because Appveyor. # shasum_before = self.wallet_shasum() timestamp_before = self.wallet_timestamp() self.log.debug('Wallet file timestamp before calling info: {}'.format(timestamp_before)) @@ -103,7 +109,7 @@ class ToolWalletTest(BitcoinTestFramework): self.log_wallet_timestamp_comparison(timestamp_before, timestamp_after) self.log.debug('Setting wallet file permissions back to 600 (read/write)') os.chmod(self.wallet_path, stat.S_IRUSR | stat.S_IWUSR) - assert(self.wallet_permissions() in ['600', '666']) # Sanity check. 666 because Appveyor. + assert self.wallet_permissions() in ['600', '666'] # Sanity check. 666 because Appveyor. # # TODO: Wallet tool info should not write to the wallet file. # The following lines should be uncommented and the tests still succeed: diff --git a/test/functional/wallet_bumpfee.py b/test/functional/wallet_bumpfee.py index c09ca8854f..27197e3b6d 100755 --- a/test/functional/wallet_bumpfee.py +++ b/test/functional/wallet_bumpfee.py @@ -23,7 +23,6 @@ from test_framework.util import ( assert_equal, assert_greater_than, assert_raises_rpc_error, - connect_nodes, hex_str_to_bytes, ) @@ -37,6 +36,7 @@ NORMAL = 0.00100000 HIGH = 0.00500000 TOO_HIGH = 1.00000000 + class BumpFeeTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 @@ -55,9 +55,6 @@ class BumpFeeTest(BitcoinTestFramework): self.nodes[1].encryptwallet(WALLET_PASSPHRASE) self.nodes[1].walletpassphrase(WALLET_PASSPHRASE, WALLET_PASSPHRASE_TIMEOUT) - connect_nodes(self.nodes[0], 1) - self.sync_all() - peer_node, rbf_node = self.nodes rbf_node_address = rbf_node.getnewaddress() @@ -94,7 +91,6 @@ class BumpFeeTest(BitcoinTestFramework): # These tests wipe out a number of utxos that are expected in other tests test_small_output_with_feerate_succeeds(self, rbf_node, dest_address) test_no_more_inputs_fails(self, rbf_node, dest_address) - self.log.info("Success") def test_simple_bumpfee_succeeds(self, mode, rbf_node, peer_node, dest_address): @@ -124,6 +120,7 @@ def test_simple_bumpfee_succeeds(self, mode, rbf_node, peer_node, dest_address): assert_equal(oldwtx["replaced_by_txid"], bumped_tx["txid"]) assert_equal(bumpedwtx["replaces_txid"], rbfid) + def test_feerate_args(self, rbf_node, peer_node, dest_address): self.log.info('Test fee_rate args') rbfid = spend_one_input(rbf_node, dest_address) @@ -137,7 +134,7 @@ def test_feerate_args(self, rbf_node, peer_node, dest_address): # Bumping to just above minrelay should fail to increase total fee enough, at least assert_raises_rpc_error(-8, "Insufficient total fee", rbf_node.bumpfee, rbfid, {"fee_rate": INSUFFICIENT}) - assert_raises_rpc_error(-3, "Amount out of range", rbf_node.bumpfee, rbfid, {"fee_rate":-1}) + assert_raises_rpc_error(-3, "Amount out of range", rbf_node.bumpfee, rbfid, {"fee_rate": -1}) assert_raises_rpc_error(-4, "is too high (cannot be higher than", rbf_node.bumpfee, rbfid, {"fee_rate": TOO_HIGH}) @@ -209,6 +206,7 @@ def test_bumpfee_with_descendant_fails(self, rbf_node, rbf_node_address, dest_ad rbf_node.sendrawtransaction(tx["hex"]) assert_raises_rpc_error(-8, "Transaction has descendants in the wallet", rbf_node.bumpfee, parent_id) + def test_small_output_with_feerate_succeeds(self, rbf_node, dest_address): self.log.info('Testing small output with feerate bump succeeds') @@ -249,6 +247,7 @@ def test_small_output_with_feerate_succeeds(self, rbf_node, dest_address): rbf_node.generatetoaddress(1, rbf_node.getnewaddress()) assert_equal(rbf_node.gettransaction(rbfid)["confirmations"], 1) + def test_dust_to_fee(self, rbf_node, dest_address): self.log.info('Test that bumped output that is dust is dropped to fee') rbfid = spend_one_input(rbf_node, dest_address) @@ -301,10 +300,11 @@ def test_maxtxfee_fails(self, rbf_node, dest_address): self.restart_node(1, ['-maxtxfee=0.000025'] + self.extra_args[1]) rbf_node.walletpassphrase(WALLET_PASSPHRASE, WALLET_PASSPHRASE_TIMEOUT) rbfid = spend_one_input(rbf_node, dest_address) - assert_raises_rpc_error(-4, "Unable to create transaction: Fee exceeds maximum configured by -maxtxfee", rbf_node.bumpfee, rbfid) + assert_raises_rpc_error(-4, "Unable to create transaction. Fee exceeds maximum configured by -maxtxfee", rbf_node.bumpfee, rbfid) self.restart_node(1, self.extra_args[1]) rbf_node.walletpassphrase(WALLET_PASSPHRASE, WALLET_PASSPHRASE_TIMEOUT) + def test_watchonly_psbt(self, peer_node, rbf_node, dest_address): self.log.info('Test that PSBT is returned for bumpfee in watchonly wallets') priv_rec_desc = "wpkh([00000001/84'/1'/0']tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/0/*)#rweraev0" @@ -339,12 +339,11 @@ def test_watchonly_psbt(self, peer_node, rbf_node, dest_address): result = watcher.importmulti([{ "desc": pub_rec_desc, "timestamp": 0, - "range": [0,10], + "range": [0, 10], "internal": False, "keypool": True, "watchonly": True - }, - { + }, { "desc": pub_change_desc, "timestamp": 0, "range": [0, 10], @@ -361,7 +360,7 @@ def test_watchonly_psbt(self, peer_node, rbf_node, dest_address): self.sync_all() # Create single-input PSBT for transaction to be bumped - psbt = watcher.walletcreatefundedpsbt([], {dest_address:0.0005}, 0, {"feeRate": 0.00001}, True)['psbt'] + psbt = watcher.walletcreatefundedpsbt([], {dest_address: 0.0005}, 0, {"feeRate": 0.00001}, True)['psbt'] psbt_signed = signer.walletprocesspsbt(psbt=psbt, sign=True, sighashtype="ALL", bip32derivs=True) psbt_final = watcher.finalizepsbt(psbt_signed["psbt"]) original_txid = watcher.sendrawtransaction(psbt_final["hex"]) @@ -387,6 +386,7 @@ def test_watchonly_psbt(self, peer_node, rbf_node, dest_address): rbf_node.unloadwallet("watcher") rbf_node.unloadwallet("signer") + def test_rebumping(self, rbf_node, dest_address): self.log.info('Test that re-bumping the original tx fails, but bumping successor works') rbfid = spend_one_input(rbf_node, dest_address) @@ -461,6 +461,7 @@ def test_locked_wallet_fails(self, rbf_node, dest_address): rbf_node.bumpfee, rbfid) rbf_node.walletpassphrase(WALLET_PASSPHRASE, WALLET_PASSPHRASE_TIMEOUT) + def test_change_script_match(self, rbf_node, dest_address): self.log.info('Test that the same change addresses is used for the replacement transaction when possible') @@ -480,6 +481,7 @@ def test_change_script_match(self, rbf_node, dest_address): bumped_rate_tx = rbf_node.bumpfee(bumped_total_tx["txid"]) assert_equal(change_addresses, get_change_address(bumped_rate_tx['txid'])) + def spend_one_input(node, dest_address, change_size=Decimal("0.00049000")): tx_input = dict( sequence=BIP125_SEQUENCE_NUMBER, **next(u for u in node.listunspent() if u["amount"] == Decimal("0.00100000"))) @@ -491,6 +493,7 @@ def spend_one_input(node, dest_address, change_size=Decimal("0.00049000")): txid = node.sendrawtransaction(signedtx["hex"]) return txid + def submit_block_with_tx(node, tx): ctx = CTransaction() ctx.deserialize(io.BytesIO(hex_str_to_bytes(tx))) @@ -507,13 +510,14 @@ def submit_block_with_tx(node, tx): node.submitblock(block.serialize().hex()) return block + def test_no_more_inputs_fails(self, rbf_node, dest_address): self.log.info('Test that bumpfee fails when there are no available confirmed outputs') # feerate rbf requires confirmed outputs when change output doesn't exist or is insufficient rbf_node.generatetoaddress(1, dest_address) # spend all funds, no change output rbfid = rbf_node.sendtoaddress(rbf_node.getnewaddress(), rbf_node.getbalance(), "", "", True) - assert_raises_rpc_error(-4, "Unable to create transaction: Insufficient funds", rbf_node.bumpfee, rbfid) + assert_raises_rpc_error(-4, "Unable to create transaction. Insufficient funds", rbf_node.bumpfee, rbfid) if __name__ == "__main__": diff --git a/test/functional/wallet_importdescriptors.py b/test/functional/wallet_importdescriptors.py index 55995152aa..fc5d653a91 100755 --- a/test/functional/wallet_importdescriptors.py +++ b/test/functional/wallet_importdescriptors.py @@ -331,6 +331,7 @@ class ImportDescriptorsTest(BitcoinTestFramework): assert_equal(wmulti_priv.getwalletinfo()['keypoolsize'], 1000) txid = w0.sendtoaddress(addr, 10) self.nodes[0].generate(6) + self.sync_all() send_txid = wmulti_priv.sendtoaddress(w0.getnewaddress(), 8) decoded = wmulti_priv.decoderawtransaction(wmulti_priv.gettransaction(send_txid)['hex']) assert_equal(len(decoded['vin'][0]['txinwitness']), 4) diff --git a/test/functional/wallet_multiwallet.py b/test/functional/wallet_multiwallet.py index c569416292..580a61f9f3 100755 --- a/test/functional/wallet_multiwallet.py +++ b/test/functional/wallet_multiwallet.py @@ -227,10 +227,10 @@ class MultiWalletTest(BitcoinTestFramework): assert_raises_rpc_error(-18, 'Wallet wallets not found.', self.nodes[0].loadwallet, 'wallets') # Fail to load duplicate wallets - assert_raises_rpc_error(-4, 'Wallet file verification failed: Error loading wallet w1. Duplicate -wallet filename specified.', self.nodes[0].loadwallet, wallet_names[0]) + assert_raises_rpc_error(-4, 'Wallet file verification failed. Error loading wallet w1. Duplicate -wallet filename specified.', self.nodes[0].loadwallet, wallet_names[0]) # Fail to load duplicate wallets by different ways (directory and filepath) - assert_raises_rpc_error(-4, "Wallet file verification failed: Error loading wallet wallet.dat. Duplicate -wallet filename specified.", self.nodes[0].loadwallet, 'wallet.dat') + assert_raises_rpc_error(-4, "Wallet file verification failed. Error loading wallet wallet.dat. Duplicate -wallet filename specified.", self.nodes[0].loadwallet, 'wallet.dat') # Fail to load if one wallet is a copy of another assert_raises_rpc_error(-4, "BerkeleyBatch: Can't open database w8_copy (duplicates fileid", self.nodes[0].loadwallet, 'w8_copy') @@ -240,7 +240,7 @@ class MultiWalletTest(BitcoinTestFramework): # Fail to load if wallet file is a symlink - assert_raises_rpc_error(-4, "Wallet file verification failed: Invalid -wallet path 'w8_symlink'", self.nodes[0].loadwallet, 'w8_symlink') + assert_raises_rpc_error(-4, "Wallet file verification failed. Invalid -wallet path 'w8_symlink'", self.nodes[0].loadwallet, 'w8_symlink') # Fail to load if a directory is specified that doesn't contain a wallet os.mkdir(wallet_dir('empty_wallet_dir')) diff --git a/test/functional/wallet_resendwallettransactions.py b/test/functional/wallet_resendwallettransactions.py index db5902f820..b384998d56 100755 --- a/test/functional/wallet_resendwallettransactions.py +++ b/test/functional/wallet_resendwallettransactions.py @@ -3,29 +3,14 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test that the wallet resends transactions periodically.""" -from collections import defaultdict import time from test_framework.blocktools import create_block, create_coinbase from test_framework.messages import ToHex -from test_framework.mininode import P2PInterface, mininode_lock +from test_framework.mininode import P2PTxInvStore, mininode_lock from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal, wait_until - -class P2PStoreTxInvs(P2PInterface): - def __init__(self): - super().__init__() - self.tx_invs_received = defaultdict(int) - - def on_inv(self, message): - # Store how many times invs have been received for each tx. - for i in message.inv: - if i.type == 1: - # save txid - self.tx_invs_received[i.hash] += 1 - - class ResendWalletTransactionsTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 @@ -36,7 +21,7 @@ class ResendWalletTransactionsTest(BitcoinTestFramework): def run_test(self): node = self.nodes[0] # alias - node.add_p2p_connection(P2PStoreTxInvs()) + node.add_p2p_connection(P2PTxInvStore()) self.log.info("Create a new transaction and wait until it's broadcast") txid = int(node.sendtoaddress(node.getnewaddress(), 1), 16) @@ -51,7 +36,7 @@ class ResendWalletTransactionsTest(BitcoinTestFramework): wait_until(lambda: node.p2p.tx_invs_received[txid] >= 1, lock=mininode_lock) # Add a second peer since txs aren't rebroadcast to the same peer (see filterInventoryKnown) - node.add_p2p_connection(P2PStoreTxInvs()) + node.add_p2p_connection(P2PTxInvStore()) self.log.info("Create a block") # Create and submit a block without the transaction. @@ -69,9 +54,10 @@ class ResendWalletTransactionsTest(BitcoinTestFramework): node.p2ps[1].sync_with_ping() assert_equal(node.p2ps[1].tx_invs_received[txid], 0) - self.log.info("Transaction should be rebroadcast after 30 minutes") - # Use mocktime and give an extra 5 minutes to be sure. - rebroadcast_time = int(time.time()) + 41 * 60 + self.log.info("Bump time & check that transaction is rebroadcast") + # Transaction should be rebroadcast approximately 24 hours in the future, + # but can range from 12-36. So bump 36 hours to be sure. + rebroadcast_time = int(time.time()) + 36 * 60 * 60 node.setmocktime(rebroadcast_time) wait_until(lambda: node.p2ps[1].tx_invs_received[txid] >= 1, lock=mininode_lock) diff --git a/test/functional/wallet_upgradewallet.py b/test/functional/wallet_upgradewallet.py new file mode 100755 index 0000000000..e7e71bf3f6 --- /dev/null +++ b/test/functional/wallet_upgradewallet.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +# Copyright (c) 2018-2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""upgradewallet RPC functional test + +Test upgradewallet RPC. Download v0.15.2 v0.16.3 node binaries: + +contrib/devtools/previous_release.sh -b v0.15.2 v0.16.3 +""" + +import os +import shutil + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + adjust_bitcoin_conf_for_pre_17, + assert_equal, + assert_greater_than, + assert_is_hex_string, +) + + +class UpgradeWalletTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 3 + self.extra_args = [ + ["-addresstype=bech32"], # current wallet version + ["-usehd=1"], # v0.16.3 wallet + ["-usehd=0"] # v0.15.2 wallet + ] + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + self.skip_if_no_previous_releases() + + def setup_network(self): + self.setup_nodes() + + def setup_nodes(self): + self.add_nodes(self.num_nodes, extra_args=self.extra_args, versions=[ + None, + 160300, + 150200, + ]) + # adapt bitcoin.conf, because older bitcoind's don't recognize config sections + adjust_bitcoin_conf_for_pre_17(self.nodes[1].bitcoinconf) + adjust_bitcoin_conf_for_pre_17(self.nodes[2].bitcoinconf) + self.start_nodes() + + def dumb_sync_blocks(self): + """ + Little helper to sync older wallets. + Notice that v0.15.2's regtest is hardforked, so there is + no sync for it. + v0.15.2 is only being used to test for version upgrade + and master hash key presence. + v0.16.3 is being used to test for version upgrade and balances. + Further info: https://github.com/bitcoin/bitcoin/pull/18774#discussion_r416967844 + """ + node_from = self.nodes[0] + v16_3_node = self.nodes[1] + to_height = node_from.getblockcount() + height = self.nodes[1].getblockcount() + for i in range(height, to_height+1): + b = node_from.getblock(blockhash=node_from.getblockhash(i), verbose=0) + v16_3_node.submitblock(b) + assert_equal(v16_3_node.getblockcount(), to_height) + + def run_test(self): + self.nodes[0].generatetoaddress(101, self.nodes[0].getnewaddress()) + self.dumb_sync_blocks() + # # Sanity check the test framework: + res = self.nodes[0].getblockchaininfo() + assert_equal(res['blocks'], 101) + node_master = self.nodes[0] + v16_3_node = self.nodes[1] + v15_2_node = self.nodes[2] + + # Send coins to old wallets for later conversion checks. + v16_3_wallet = v16_3_node.get_wallet_rpc('wallet.dat') + v16_3_address = v16_3_wallet.getnewaddress() + node_master.generatetoaddress(101, v16_3_address) + self.dumb_sync_blocks() + v16_3_balance = v16_3_wallet.getbalance() + + self.log.info("Test upgradewallet RPC...") + # Prepare for copying of the older wallet + node_master_wallet_dir = os.path.join(node_master.datadir, "regtest/wallets") + v16_3_wallet = os.path.join(v16_3_node.datadir, "regtest/wallets/wallet.dat") + v15_2_wallet = os.path.join(v15_2_node.datadir, "regtest/wallet.dat") + self.stop_nodes() + + # Copy the 0.16.3 wallet to the last Bitcoin Core version and open it: + shutil.rmtree(node_master_wallet_dir) + os.mkdir(node_master_wallet_dir) + shutil.copy( + v16_3_wallet, + node_master_wallet_dir + ) + self.restart_node(0, ['-nowallet']) + node_master.loadwallet('') + + wallet = node_master.get_wallet_rpc('') + old_version = wallet.getwalletinfo()["walletversion"] + + # calling upgradewallet without version arguments + # should return nothing if successful + assert_equal(wallet.upgradewallet(), "") + new_version = wallet.getwalletinfo()["walletversion"] + # upgraded wallet version should be greater than older one + assert_greater_than(new_version, old_version) + # wallet should still contain the same balance + assert_equal(wallet.getbalance(), v16_3_balance) + + self.stop_node(0) + # Copy the 0.15.2 wallet to the last Bitcoin Core version and open it: + shutil.rmtree(node_master_wallet_dir) + os.mkdir(node_master_wallet_dir) + shutil.copy( + v15_2_wallet, + node_master_wallet_dir + ) + self.restart_node(0, ['-nowallet']) + node_master.loadwallet('') + + wallet = node_master.get_wallet_rpc('') + # should have no master key hash before conversion + assert_equal('hdseedid' in wallet.getwalletinfo(), False) + # calling upgradewallet with explicit version number + # should return nothing if successful + assert_equal(wallet.upgradewallet(169900), "") + new_version = wallet.getwalletinfo()["walletversion"] + # upgraded wallet should have version 169900 + assert_equal(new_version, 169900) + # after conversion master key hash should be present + assert_is_hex_string(wallet.getwalletinfo()['hdseedid']) + +if __name__ == '__main__': + UpgradeWalletTest().main() |