diff options
Diffstat (limited to 'test')
| -rwxr-xr-x | test/functional/mempool_persist.py | 40 | ||||
| -rwxr-xr-x | test/functional/mempool_unbroadcast.py | 99 | ||||
| -rwxr-xr-x | test/functional/test_framework/mininode.py | 22 | ||||
| -rwxr-xr-x | test/functional/test_runner.py | 1 | ||||
| -rwxr-xr-x | test/functional/wallet_bumpfee.py | 24 | ||||
| -rwxr-xr-x | test/functional/wallet_importdescriptors.py | 1 | ||||
| -rwxr-xr-x | test/functional/wallet_resendwallettransactions.py | 28 | ||||
| -rwxr-xr-x | test/lint/lint-shell.sh | 10 |
8 files changed, 184 insertions, 41 deletions
diff --git a/test/functional/mempool_persist.py b/test/functional/mempool_persist.py index e1671624a8..99003d2d1f 100755 --- a/test/functional/mempool_persist.py +++ b/test/functional/mempool_persist.py @@ -40,10 +40,13 @@ import os import time from test_framework.test_framework import BitcoinTestFramework +from test_framework.mininode import P2PTxInvStore from test_framework.util import ( assert_equal, assert_greater_than_or_equal, assert_raises_rpc_error, + connect_nodes, + disconnect_nodes, wait_until, ) @@ -80,6 +83,11 @@ class MempoolPersistTest(BitcoinTestFramework): assert_greater_than_or_equal(tx_creation_time, tx_creation_time_lower) assert_greater_than_or_equal(tx_creation_time_higher, tx_creation_time) + # disconnect nodes & make a txn that remains in the unbroadcast set. + disconnect_nodes(self.nodes[0], 2) + self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), Decimal("12")) + connect_nodes(self.nodes[0], 2) + self.log.debug("Stop-start the nodes. Verify that node0 has the transactions in its mempool and node1 does not. Verify that node2 calculates its balance correctly after loading wallet transactions.") self.stop_nodes() # Give this node a head-start, so we can be "extra-sure" that it didn't load anything later @@ -89,7 +97,7 @@ class MempoolPersistTest(BitcoinTestFramework): self.start_node(2) wait_until(lambda: self.nodes[0].getmempoolinfo()["loaded"], timeout=1) wait_until(lambda: self.nodes[2].getmempoolinfo()["loaded"], timeout=1) - assert_equal(len(self.nodes[0].getrawmempool()), 5) + assert_equal(len(self.nodes[0].getrawmempool()), 6) assert_equal(len(self.nodes[2].getrawmempool()), 5) # The others have loaded their mempool. If node_1 loaded anything, we'd probably notice by now: assert_equal(len(self.nodes[1].getrawmempool()), 0) @@ -105,9 +113,10 @@ class MempoolPersistTest(BitcoinTestFramework): self.nodes[2].syncwithvalidationinterfacequeue() # Flush mempool to wallet assert_equal(node2_balance, self.nodes[2].getbalance()) + # start node0 with wallet disabled so wallet transactions don't get resubmitted self.log.debug("Stop-start node0 with -persistmempool=0. Verify that it doesn't load its mempool.dat file.") self.stop_nodes() - self.start_node(0, extra_args=["-persistmempool=0"]) + self.start_node(0, extra_args=["-persistmempool=0", "-disablewallet"]) wait_until(lambda: self.nodes[0].getmempoolinfo()["loaded"]) assert_equal(len(self.nodes[0].getrawmempool()), 0) @@ -115,7 +124,7 @@ class MempoolPersistTest(BitcoinTestFramework): self.stop_nodes() self.start_node(0) wait_until(lambda: self.nodes[0].getmempoolinfo()["loaded"]) - assert_equal(len(self.nodes[0].getrawmempool()), 5) + assert_equal(len(self.nodes[0].getrawmempool()), 6) mempooldat0 = os.path.join(self.nodes[0].datadir, self.chain, 'mempool.dat') mempooldat1 = os.path.join(self.nodes[1].datadir, self.chain, 'mempool.dat') @@ -124,12 +133,12 @@ class MempoolPersistTest(BitcoinTestFramework): self.nodes[0].savemempool() assert os.path.isfile(mempooldat0) - self.log.debug("Stop nodes, make node1 use mempool.dat from node0. Verify it has 5 transactions") + self.log.debug("Stop nodes, make node1 use mempool.dat from node0. Verify it has 6 transactions") os.rename(mempooldat0, mempooldat1) self.stop_nodes() self.start_node(1, extra_args=[]) wait_until(lambda: self.nodes[1].getmempoolinfo()["loaded"]) - assert_equal(len(self.nodes[1].getrawmempool()), 5) + assert_equal(len(self.nodes[1].getrawmempool()), 6) self.log.debug("Prevent bitcoind from writing mempool.dat to disk. Verify that `savemempool` fails") # to test the exception we are creating a tmp folder called mempool.dat.new @@ -139,6 +148,27 @@ class MempoolPersistTest(BitcoinTestFramework): assert_raises_rpc_error(-1, "Unable to dump mempool to disk", self.nodes[1].savemempool) os.rmdir(mempooldotnew1) + self.test_persist_unbroadcast() + + def test_persist_unbroadcast(self): + node0 = self.nodes[0] + self.start_node(0) + + # clear out mempool + node0.generate(1) + + # disconnect nodes to make a txn that remains in the unbroadcast set. + disconnect_nodes(node0, 1) + node0.sendtoaddress(self.nodes[1].getnewaddress(), Decimal("12")) + + # shutdown, then startup with wallet disabled + self.stop_nodes() + self.start_node(0, extra_args=["-disablewallet"]) + + # check that txn gets broadcast due to unbroadcast logic + conn = node0.add_p2p_connection(P2PTxInvStore()) + node0.mockscheduler(16*60) # 15 min + 1 for buffer + wait_until(lambda: len(conn.get_invs()) == 1) if __name__ == '__main__': MempoolPersistTest().main() diff --git a/test/functional/mempool_unbroadcast.py b/test/functional/mempool_unbroadcast.py new file mode 100755 index 0000000000..a561f28b91 --- /dev/null +++ b/test/functional/mempool_unbroadcast.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +# Copyright (c) 2017-2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test that the mempool ensures transaction delivery by periodically sending +to peers until a GETDATA is received.""" + +import time + +from test_framework.mininode import P2PTxInvStore +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + connect_nodes, + create_confirmed_utxos, + disconnect_nodes, +) + + +class MempoolUnbroadcastTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 2 + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + + def run_test(self): + self.test_broadcast() + self.test_txn_removal() + + def test_broadcast(self): + self.log.info("Test that mempool reattempts delivery of locally submitted transaction") + node = self.nodes[0] + + min_relay_fee = node.getnetworkinfo()["relayfee"] + utxos = create_confirmed_utxos(min_relay_fee, node, 10) + + disconnect_nodes(node, 1) + + self.log.info("Generate transactions that only node 0 knows about") + + # generate a wallet txn + addr = node.getnewaddress() + wallet_tx_hsh = node.sendtoaddress(addr, 0.0001) + + # generate a txn using sendrawtransaction + us0 = utxos.pop() + inputs = [{"txid": us0["txid"], "vout": us0["vout"]}] + outputs = {addr: 0.0001} + tx = node.createrawtransaction(inputs, outputs) + node.settxfee(min_relay_fee) + txF = node.fundrawtransaction(tx) + txFS = node.signrawtransactionwithwallet(txF["hex"]) + rpc_tx_hsh = node.sendrawtransaction(txFS["hex"]) + + # check that second node doesn't have these two txns + mempool = self.nodes[1].getrawmempool() + assert rpc_tx_hsh not in mempool + assert wallet_tx_hsh not in mempool + + # ensure that unbroadcast txs are persisted to mempool.dat + self.restart_node(0) + + self.log.info("Reconnect nodes & check if they are sent to node 1") + connect_nodes(node, 1) + + # fast forward into the future & ensure that the second node has the txns + node.mockscheduler(15 * 60) # 15 min in seconds + self.sync_mempools(timeout=30) + mempool = self.nodes[1].getrawmempool() + assert rpc_tx_hsh in mempool + assert wallet_tx_hsh in mempool + + self.log.info("Add another connection & ensure transactions aren't broadcast again") + + conn = node.add_p2p_connection(P2PTxInvStore()) + node.mockscheduler(15 * 60) + time.sleep(5) + assert_equal(len(conn.get_invs()), 0) + + def test_txn_removal(self): + self.log.info("Test that transactions removed from mempool are removed from unbroadcast set") + node = self.nodes[0] + disconnect_nodes(node, 1) + node.disconnect_p2ps + + # since the node doesn't have any connections, it will not receive + # any GETDATAs & thus the transaction will remain in the unbroadcast set. + addr = node.getnewaddress() + txhsh = node.sendtoaddress(addr, 0.0001) + + # check transaction was removed from unbroadcast set due to presence in + # a block + removal_reason = "Removed {} from set of unbroadcast txns before confirmation that txn was sent out".format(txhsh) + with node.assert_debug_log([removal_reason]): + node.generate(1) + +if __name__ == "__main__": + MempoolUnbroadcastTest().main() diff --git a/test/functional/test_framework/mininode.py b/test/functional/test_framework/mininode.py index 6aa73623e6..257499fcb9 100755 --- a/test/functional/test_framework/mininode.py +++ b/test/functional/test_framework/mininode.py @@ -12,7 +12,10 @@ found in the mini-node branch of http://github.com/jgarzik/pynode. P2PConnection: A low-level connection object to a node's P2P interface P2PInterface: A high-level interface object for communicating to a node over P2P P2PDataStore: A p2p interface class that keeps a store of transactions and blocks - and can respond correctly to getdata and getheaders messages""" + and can respond correctly to getdata and getheaders messages +P2PTxInvStore: A p2p interface class that inherits from P2PDataStore, and keeps + a count of how many times each txid has been announced.""" + import asyncio from collections import defaultdict from io import BytesIO @@ -627,3 +630,20 @@ class P2PDataStore(P2PInterface): # Check that none of the txs are now in the mempool for tx in txs: assert tx.hash not in raw_mempool, "{} tx found in mempool".format(tx.hash) + +class P2PTxInvStore(P2PInterface): + """A P2PInterface which stores a count of how many times each txid has been announced.""" + def __init__(self): + super().__init__() + self.tx_invs_received = defaultdict(int) + + def on_inv(self, message): + # Store how many times invs have been received for each tx. + for i in message.inv: + if i.type == MSG_TX: + # save txid + self.tx_invs_received[i.hash] += 1 + + def get_invs(self): + with mininode_lock: + return list(self.tx_invs_received.keys()) diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index b0fbdbbdb8..344df368a8 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -221,6 +221,7 @@ BASE_SCRIPTS = [ 'p2p_unrequested_blocks.py', 'feature_includeconf.py', 'feature_asmap.py', + 'mempool_unbroadcast.py', 'rpc_deriveaddresses.py', 'rpc_deriveaddresses.py --usecli', 'rpc_scantxoutset.py', diff --git a/test/functional/wallet_bumpfee.py b/test/functional/wallet_bumpfee.py index c09ca8854f..9ba23db42d 100755 --- a/test/functional/wallet_bumpfee.py +++ b/test/functional/wallet_bumpfee.py @@ -23,7 +23,6 @@ from test_framework.util import ( assert_equal, assert_greater_than, assert_raises_rpc_error, - connect_nodes, hex_str_to_bytes, ) @@ -37,6 +36,7 @@ NORMAL = 0.00100000 HIGH = 0.00500000 TOO_HIGH = 1.00000000 + class BumpFeeTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 @@ -55,9 +55,6 @@ class BumpFeeTest(BitcoinTestFramework): self.nodes[1].encryptwallet(WALLET_PASSPHRASE) self.nodes[1].walletpassphrase(WALLET_PASSPHRASE, WALLET_PASSPHRASE_TIMEOUT) - connect_nodes(self.nodes[0], 1) - self.sync_all() - peer_node, rbf_node = self.nodes rbf_node_address = rbf_node.getnewaddress() @@ -94,7 +91,6 @@ class BumpFeeTest(BitcoinTestFramework): # These tests wipe out a number of utxos that are expected in other tests test_small_output_with_feerate_succeeds(self, rbf_node, dest_address) test_no_more_inputs_fails(self, rbf_node, dest_address) - self.log.info("Success") def test_simple_bumpfee_succeeds(self, mode, rbf_node, peer_node, dest_address): @@ -124,6 +120,7 @@ def test_simple_bumpfee_succeeds(self, mode, rbf_node, peer_node, dest_address): assert_equal(oldwtx["replaced_by_txid"], bumped_tx["txid"]) assert_equal(bumpedwtx["replaces_txid"], rbfid) + def test_feerate_args(self, rbf_node, peer_node, dest_address): self.log.info('Test fee_rate args') rbfid = spend_one_input(rbf_node, dest_address) @@ -137,7 +134,7 @@ def test_feerate_args(self, rbf_node, peer_node, dest_address): # Bumping to just above minrelay should fail to increase total fee enough, at least assert_raises_rpc_error(-8, "Insufficient total fee", rbf_node.bumpfee, rbfid, {"fee_rate": INSUFFICIENT}) - assert_raises_rpc_error(-3, "Amount out of range", rbf_node.bumpfee, rbfid, {"fee_rate":-1}) + assert_raises_rpc_error(-3, "Amount out of range", rbf_node.bumpfee, rbfid, {"fee_rate": -1}) assert_raises_rpc_error(-4, "is too high (cannot be higher than", rbf_node.bumpfee, rbfid, {"fee_rate": TOO_HIGH}) @@ -209,6 +206,7 @@ def test_bumpfee_with_descendant_fails(self, rbf_node, rbf_node_address, dest_ad rbf_node.sendrawtransaction(tx["hex"]) assert_raises_rpc_error(-8, "Transaction has descendants in the wallet", rbf_node.bumpfee, parent_id) + def test_small_output_with_feerate_succeeds(self, rbf_node, dest_address): self.log.info('Testing small output with feerate bump succeeds') @@ -249,6 +247,7 @@ def test_small_output_with_feerate_succeeds(self, rbf_node, dest_address): rbf_node.generatetoaddress(1, rbf_node.getnewaddress()) assert_equal(rbf_node.gettransaction(rbfid)["confirmations"], 1) + def test_dust_to_fee(self, rbf_node, dest_address): self.log.info('Test that bumped output that is dust is dropped to fee') rbfid = spend_one_input(rbf_node, dest_address) @@ -305,6 +304,7 @@ def test_maxtxfee_fails(self, rbf_node, dest_address): self.restart_node(1, self.extra_args[1]) rbf_node.walletpassphrase(WALLET_PASSPHRASE, WALLET_PASSPHRASE_TIMEOUT) + def test_watchonly_psbt(self, peer_node, rbf_node, dest_address): self.log.info('Test that PSBT is returned for bumpfee in watchonly wallets') priv_rec_desc = "wpkh([00000001/84'/1'/0']tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/0/*)#rweraev0" @@ -339,12 +339,11 @@ def test_watchonly_psbt(self, peer_node, rbf_node, dest_address): result = watcher.importmulti([{ "desc": pub_rec_desc, "timestamp": 0, - "range": [0,10], + "range": [0, 10], "internal": False, "keypool": True, "watchonly": True - }, - { + }, { "desc": pub_change_desc, "timestamp": 0, "range": [0, 10], @@ -361,7 +360,7 @@ def test_watchonly_psbt(self, peer_node, rbf_node, dest_address): self.sync_all() # Create single-input PSBT for transaction to be bumped - psbt = watcher.walletcreatefundedpsbt([], {dest_address:0.0005}, 0, {"feeRate": 0.00001}, True)['psbt'] + psbt = watcher.walletcreatefundedpsbt([], {dest_address: 0.0005}, 0, {"feeRate": 0.00001}, True)['psbt'] psbt_signed = signer.walletprocesspsbt(psbt=psbt, sign=True, sighashtype="ALL", bip32derivs=True) psbt_final = watcher.finalizepsbt(psbt_signed["psbt"]) original_txid = watcher.sendrawtransaction(psbt_final["hex"]) @@ -387,6 +386,7 @@ def test_watchonly_psbt(self, peer_node, rbf_node, dest_address): rbf_node.unloadwallet("watcher") rbf_node.unloadwallet("signer") + def test_rebumping(self, rbf_node, dest_address): self.log.info('Test that re-bumping the original tx fails, but bumping successor works') rbfid = spend_one_input(rbf_node, dest_address) @@ -461,6 +461,7 @@ def test_locked_wallet_fails(self, rbf_node, dest_address): rbf_node.bumpfee, rbfid) rbf_node.walletpassphrase(WALLET_PASSPHRASE, WALLET_PASSPHRASE_TIMEOUT) + def test_change_script_match(self, rbf_node, dest_address): self.log.info('Test that the same change addresses is used for the replacement transaction when possible') @@ -480,6 +481,7 @@ def test_change_script_match(self, rbf_node, dest_address): bumped_rate_tx = rbf_node.bumpfee(bumped_total_tx["txid"]) assert_equal(change_addresses, get_change_address(bumped_rate_tx['txid'])) + def spend_one_input(node, dest_address, change_size=Decimal("0.00049000")): tx_input = dict( sequence=BIP125_SEQUENCE_NUMBER, **next(u for u in node.listunspent() if u["amount"] == Decimal("0.00100000"))) @@ -491,6 +493,7 @@ def spend_one_input(node, dest_address, change_size=Decimal("0.00049000")): txid = node.sendrawtransaction(signedtx["hex"]) return txid + def submit_block_with_tx(node, tx): ctx = CTransaction() ctx.deserialize(io.BytesIO(hex_str_to_bytes(tx))) @@ -507,6 +510,7 @@ def submit_block_with_tx(node, tx): node.submitblock(block.serialize().hex()) return block + def test_no_more_inputs_fails(self, rbf_node, dest_address): self.log.info('Test that bumpfee fails when there are no available confirmed outputs') # feerate rbf requires confirmed outputs when change output doesn't exist or is insufficient diff --git a/test/functional/wallet_importdescriptors.py b/test/functional/wallet_importdescriptors.py index 55995152aa..fc5d653a91 100755 --- a/test/functional/wallet_importdescriptors.py +++ b/test/functional/wallet_importdescriptors.py @@ -331,6 +331,7 @@ class ImportDescriptorsTest(BitcoinTestFramework): assert_equal(wmulti_priv.getwalletinfo()['keypoolsize'], 1000) txid = w0.sendtoaddress(addr, 10) self.nodes[0].generate(6) + self.sync_all() send_txid = wmulti_priv.sendtoaddress(w0.getnewaddress(), 8) decoded = wmulti_priv.decoderawtransaction(wmulti_priv.gettransaction(send_txid)['hex']) assert_equal(len(decoded['vin'][0]['txinwitness']), 4) diff --git a/test/functional/wallet_resendwallettransactions.py b/test/functional/wallet_resendwallettransactions.py index db5902f820..b384998d56 100755 --- a/test/functional/wallet_resendwallettransactions.py +++ b/test/functional/wallet_resendwallettransactions.py @@ -3,29 +3,14 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test that the wallet resends transactions periodically.""" -from collections import defaultdict import time from test_framework.blocktools import create_block, create_coinbase from test_framework.messages import ToHex -from test_framework.mininode import P2PInterface, mininode_lock +from test_framework.mininode import P2PTxInvStore, mininode_lock from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal, wait_until - -class P2PStoreTxInvs(P2PInterface): - def __init__(self): - super().__init__() - self.tx_invs_received = defaultdict(int) - - def on_inv(self, message): - # Store how many times invs have been received for each tx. - for i in message.inv: - if i.type == 1: - # save txid - self.tx_invs_received[i.hash] += 1 - - class ResendWalletTransactionsTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 @@ -36,7 +21,7 @@ class ResendWalletTransactionsTest(BitcoinTestFramework): def run_test(self): node = self.nodes[0] # alias - node.add_p2p_connection(P2PStoreTxInvs()) + node.add_p2p_connection(P2PTxInvStore()) self.log.info("Create a new transaction and wait until it's broadcast") txid = int(node.sendtoaddress(node.getnewaddress(), 1), 16) @@ -51,7 +36,7 @@ class ResendWalletTransactionsTest(BitcoinTestFramework): wait_until(lambda: node.p2p.tx_invs_received[txid] >= 1, lock=mininode_lock) # Add a second peer since txs aren't rebroadcast to the same peer (see filterInventoryKnown) - node.add_p2p_connection(P2PStoreTxInvs()) + node.add_p2p_connection(P2PTxInvStore()) self.log.info("Create a block") # Create and submit a block without the transaction. @@ -69,9 +54,10 @@ class ResendWalletTransactionsTest(BitcoinTestFramework): node.p2ps[1].sync_with_ping() assert_equal(node.p2ps[1].tx_invs_received[txid], 0) - self.log.info("Transaction should be rebroadcast after 30 minutes") - # Use mocktime and give an extra 5 minutes to be sure. - rebroadcast_time = int(time.time()) + 41 * 60 + self.log.info("Bump time & check that transaction is rebroadcast") + # Transaction should be rebroadcast approximately 24 hours in the future, + # but can range from 12-36. So bump 36 hours to be sure. + rebroadcast_time = int(time.time()) + 36 * 60 * 60 node.setmocktime(rebroadcast_time) wait_until(lambda: node.p2ps[1].tx_invs_received[txid] >= 1, lock=mininode_lock) diff --git a/test/lint/lint-shell.sh b/test/lint/lint-shell.sh index 5540a0f74f..2bb76ec286 100755 --- a/test/lint/lint-shell.sh +++ b/test/lint/lint-shell.sh @@ -46,15 +46,17 @@ if ! command -v yq > /dev/null; then fi EXCLUDE_GITIAN=${EXCLUDE}",$(IFS=','; echo "${disabled_gitian[*]}")" +SHELLCHECK_CMD="shellcheck --external-sources --check-sourced $EXCLUDE_GITIAN" for descriptor in $(git ls-files -- 'contrib/gitian-descriptors/*.yml') do - echo - echo "$descriptor" + script=$(basename "$descriptor") # Use #!/bin/bash as gitian-builder/bin/gbuild does to complete a script. - SCRIPT=$'#!/bin/bash\n'$(yq -r .script "$descriptor") - if ! echo "$SCRIPT" | shellcheck "$EXCLUDE_GITIAN" -; then + echo "#!/bin/bash" > $script + yq -r .script "$descriptor" >> $script + if ! $SHELLCHECK_CMD $script; then EXIT_CODE=1 fi + rm $script done exit $EXIT_CODE |