diff options
Diffstat (limited to 'test')
41 files changed, 584 insertions, 192 deletions
diff --git a/test/config.ini.in b/test/config.ini.in index 291599da45..3bf79ef25d 100644 --- a/test/config.ini.in +++ b/test/config.ini.in @@ -6,8 +6,8 @@ # test/*/test_runner.py and test/util/rpcauth-test.py [environment] -PACKAGE_NAME=@PACKAGE_NAME@ -PACKAGE_BUGREPORT=@PACKAGE_BUGREPORT@ +CLIENT_NAME=@CLIENT_NAME@ +CLIENT_BUGREPORT=@CLIENT_BUGREPORT@ SRCDIR=@abs_top_srcdir@ BUILDDIR=@abs_top_builddir@ EXEEXT=@EXEEXT@ diff --git a/test/functional/README.md b/test/functional/README.md index a4994f2e7c..a34bf1827c 100644 --- a/test/functional/README.md +++ b/test/functional/README.md @@ -10,7 +10,8 @@ that file and modify to fit your needs. #### Coverage -Running `test/functional/test_runner.py` with the `--coverage` argument tracks which RPCs are +Assuming the build directory is `build`, +running `build/test/functional/test_runner.py` with the `--coverage` argument tracks which RPCs are called by the tests and prints a report of uncovered RPCs in the summary. This can be used (along with the `--extended` argument) to find out which RPCs we don't have test cases for. diff --git a/test/functional/data/invalid_txs.py b/test/functional/data/invalid_txs.py index 2e4ca83bf0..d2d7202d86 100644 --- a/test/functional/data/invalid_txs.py +++ b/test/functional/data/invalid_txs.py @@ -263,6 +263,17 @@ def getDisabledOpcodeTemplate(opcode): 'valid_in_block' : True }) +class NonStandardAndInvalid(BadTxTemplate): + """A non-standard transaction which is also consensus-invalid should return the consensus error.""" + reject_reason = "mandatory-script-verify-flag-failed (OP_RETURN was encountered)" + expect_disconnect = True + valid_in_block = False + + def get_tx(self): + return create_tx_with_script( + self.spend_tx, 0, script_sig=b'\x00' * 3 + b'\xab\x6a', + amount=(self.spend_avail // 2)) + # Disabled opcode tx templates (CVE-2010-5137) DisabledOpcodeTemplates = [getDisabledOpcodeTemplate(opcode) for opcode in [ OP_CAT, diff --git a/test/functional/feature_addrman.py b/test/functional/feature_addrman.py index c4e60fba12..62f9cee1df 100755 --- a/test/functional/feature_addrman.py +++ b/test/functional/feature_addrman.py @@ -54,7 +54,7 @@ class AddrmanTest(BitcoinTestFramework): peers_dat = os.path.join(self.nodes[0].chain_path, "peers.dat") init_error = lambda reason: ( f"Error: Invalid or corrupt peers.dat \\({reason}\\). If you believe this " - f"is a bug, please report it to {self.config['environment']['PACKAGE_BUGREPORT']}. " + f"is a bug, please report it to {self.config['environment']['CLIENT_BUGREPORT']}. " f'As a workaround, you can move the file \\("{re.escape(peers_dat)}"\\) out of the way \\(rename, ' "move, or delete\\) to have a new one created on the next start." ) diff --git a/test/functional/feature_block.py b/test/functional/feature_block.py index 384ca311c7..43bf61c174 100755 --- a/test/functional/feature_block.py +++ b/test/functional/feature_block.py @@ -88,6 +88,7 @@ class FullBlockTest(BitcoinTestFramework): self.extra_args = [[ '-acceptnonstdtxn=1', # This is a consensus block test, we don't care about tx policy '-testactivationheight=bip34@2', + '-par=1', # Until https://github.com/bitcoin/bitcoin/issues/30960 is fixed ]] def run_test(self): diff --git a/test/functional/feature_blocksxor.py b/test/functional/feature_blocksxor.py index 7698a66ec4..9824bf9715 100755 --- a/test/functional/feature_blocksxor.py +++ b/test/functional/feature_blocksxor.py @@ -31,7 +31,7 @@ class BlocksXORTest(BitcoinTestFramework): node = self.nodes[0] wallet = MiniWallet(node) for _ in range(5): - wallet.send_self_transfer(from_node=node, target_weight=80000) + wallet.send_self_transfer(from_node=node, target_vsize=20000) self.generate(wallet, 1) block_files = list(node.blocks_path.glob('blk[0-9][0-9][0-9][0-9][0-9].dat')) diff --git a/test/functional/feature_fee_estimation.py b/test/functional/feature_fee_estimation.py index 83627ff5c2..974d8268a2 100755 --- a/test/functional/feature_fee_estimation.py +++ b/test/functional/feature_fee_estimation.py @@ -398,6 +398,7 @@ class EstimateFeeTest(BitcoinTestFramework): self.start_node(0) self.connect_nodes(0, 1) self.connect_nodes(0, 2) + self.sync_blocks() assert_equal(self.nodes[0].estimatesmartfee(1)["errors"], ["Insufficient data or no feerate found"]) def broadcast_and_mine(self, broadcaster, miner, feerate, count): diff --git a/test/functional/feature_filelock.py b/test/functional/feature_filelock.py index e71871114d..f56643c62e 100755 --- a/test/functional/feature_filelock.py +++ b/test/functional/feature_filelock.py @@ -7,7 +7,10 @@ import random import string from test_framework.test_framework import BitcoinTestFramework -from test_framework.test_node import ErrorMatch +from test_framework.test_node import ( + BITCOIN_PID_FILENAME_DEFAULT, + ErrorMatch, +) class FilelockTest(BitcoinTestFramework): def add_options(self, parser): @@ -27,13 +30,13 @@ class FilelockTest(BitcoinTestFramework): self.log.info(f"Using datadir {datadir}") self.log.info("Check that we can't start a second bitcoind instance using the same datadir") - expected_msg = f"Error: Cannot obtain a lock on data directory {datadir}. {self.config['environment']['PACKAGE_NAME']} is probably already running." + expected_msg = f"Error: Cannot obtain a lock on data directory {datadir}. {self.config['environment']['CLIENT_NAME']} is probably already running." self.nodes[1].assert_start_raises_init_error(extra_args=[f'-datadir={self.nodes[0].datadir_path}', '-noserver'], expected_msg=expected_msg) self.log.info("Check that cookie and PID file are not deleted when attempting to start a second bitcoind using the same datadir") cookie_file = datadir / ".cookie" assert cookie_file.exists() # should not be deleted during the second bitcoind instance shutdown - pid_file = datadir / "bitcoind.pid" + pid_file = datadir / BITCOIN_PID_FILENAME_DEFAULT assert pid_file.exists() if self.is_wallet_compiled(): @@ -43,7 +46,7 @@ class FilelockTest(BitcoinTestFramework): wallet_dir = self.nodes[0].wallets_path self.log.info("Check that we can't start a second bitcoind instance using the same wallet") if descriptors: - expected_msg = f"Error: SQLiteDatabase: Unable to obtain an exclusive lock on the database, is it being used by another instance of {self.config['environment']['PACKAGE_NAME']}?" + expected_msg = f"Error: SQLiteDatabase: Unable to obtain an exclusive lock on the database, is it being used by another instance of {self.config['environment']['CLIENT_NAME']}?" else: expected_msg = "Error: Error initializing wallet database environment" self.nodes[1].assert_start_raises_init_error(extra_args=[f'-walletdir={wallet_dir}', f'-wallet={wallet_name}', '-noserver'], expected_msg=expected_msg, match=ErrorMatch.PARTIAL_REGEX) diff --git a/test/functional/feature_framework_miniwallet.py b/test/functional/feature_framework_miniwallet.py index d1aa24e7cd..f723f7f31e 100755 --- a/test/functional/feature_framework_miniwallet.py +++ b/test/functional/feature_framework_miniwallet.py @@ -9,7 +9,7 @@ import string from test_framework.blocktools import COINBASE_MATURITY from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( - assert_greater_than_or_equal, + assert_equal, ) from test_framework.wallet import ( MiniWallet, @@ -22,17 +22,15 @@ class FeatureFrameworkMiniWalletTest(BitcoinTestFramework): self.num_nodes = 1 def test_tx_padding(self): - """Verify that MiniWallet's transaction padding (`target_weight` parameter) - works accurately enough (i.e. at most 3 WUs higher) with all modes.""" + """Verify that MiniWallet's transaction padding (`target_vsize` parameter) + works accurately with all modes.""" for mode_name, wallet in self.wallets: self.log.info(f"Test tx padding with MiniWallet mode {mode_name}...") utxo = wallet.get_utxo(mark_as_spent=False) - for target_weight in [1000, 2000, 5000, 10000, 20000, 50000, 100000, 200000, 4000000, - 989, 2001, 4337, 13371, 23219, 49153, 102035, 223419, 3999989]: - tx = wallet.create_self_transfer(utxo_to_spend=utxo, target_weight=target_weight)["tx"] - self.log.debug(f"-> target weight: {target_weight}, actual weight: {tx.get_weight()}") - assert_greater_than_or_equal(tx.get_weight(), target_weight) - assert_greater_than_or_equal(target_weight + 3, tx.get_weight()) + for target_vsize in [250, 500, 1250, 2500, 5000, 12500, 25000, 50000, 1000000, + 248, 501, 1085, 3343, 5805, 12289, 25509, 55855, 999998]: + tx = wallet.create_self_transfer(utxo_to_spend=utxo, target_vsize=target_vsize)["tx"] + assert_equal(tx.get_vsize(), target_vsize) def test_wallet_tagging(self): """Verify that tagged wallet instances are able to send funds.""" diff --git a/test/functional/feature_init.py b/test/functional/feature_init.py index 659d33684e..4a2f7ecf42 100755 --- a/test/functional/feature_init.py +++ b/test/functional/feature_init.py @@ -2,17 +2,20 @@ # Copyright (c) 2021-present The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Stress tests related to node initialization.""" +"""Tests related to node initialization.""" from pathlib import Path import platform import shutil from test_framework.test_framework import BitcoinTestFramework, SkipTest -from test_framework.test_node import ErrorMatch +from test_framework.test_node import ( + BITCOIN_PID_FILENAME_DEFAULT, + ErrorMatch, +) from test_framework.util import assert_equal -class InitStressTest(BitcoinTestFramework): +class InitTest(BitcoinTestFramework): """ Ensure that initialization can be interrupted at a number of points and not impair subsequent starts. @@ -25,7 +28,7 @@ class InitStressTest(BitcoinTestFramework): self.setup_clean_chain = False self.num_nodes = 1 - def run_test(self): + def init_stress_test(self): """ - test terminating initialization after seeing a certain log line. - test removing certain essential files to test startup error paths. @@ -97,13 +100,13 @@ class InitStressTest(BitcoinTestFramework): files_to_delete = { 'blocks/index/*.ldb': 'Error opening block database.', - 'chainstate/*.ldb': 'Error opening block database.', + 'chainstate/*.ldb': 'Error opening coins database.', 'blocks/blk*.dat': 'Error loading block database.', } files_to_perturb = { 'blocks/index/*.ldb': 'Error loading block database.', - 'chainstate/*.ldb': 'Error opening block database.', + 'chainstate/*.ldb': 'Error opening coins database.', 'blocks/blk*.dat': 'Corrupted block database detected.', } @@ -147,6 +150,31 @@ class InitStressTest(BitcoinTestFramework): shutil.move(node.chain_path / "blocks_bak", node.chain_path / "blocks") shutil.move(node.chain_path / "chainstate_bak", node.chain_path / "chainstate") + def init_pid_test(self): + BITCOIN_PID_FILENAME_CUSTOM = "my_fancy_bitcoin_pid_file.foobar" + + self.log.info("Test specifying custom pid file via -pid command line option") + custom_pidfile_relative = BITCOIN_PID_FILENAME_CUSTOM + self.log.info(f"-> path relative to datadir ({custom_pidfile_relative})") + self.restart_node(0, [f"-pid={custom_pidfile_relative}"]) + datadir = self.nodes[0].chain_path + assert not (datadir / BITCOIN_PID_FILENAME_DEFAULT).exists() + assert (datadir / custom_pidfile_relative).exists() + self.stop_node(0) + assert not (datadir / custom_pidfile_relative).exists() + + custom_pidfile_absolute = Path(self.options.tmpdir) / BITCOIN_PID_FILENAME_CUSTOM + self.log.info(f"-> absolute path ({custom_pidfile_absolute})") + self.restart_node(0, [f"-pid={custom_pidfile_absolute}"]) + assert not (datadir / BITCOIN_PID_FILENAME_DEFAULT).exists() + assert custom_pidfile_absolute.exists() + self.stop_node(0) + assert not custom_pidfile_absolute.exists() + + def run_test(self): + self.init_pid_test() + self.init_stress_test() + if __name__ == '__main__': - InitStressTest(__file__).main() + InitTest(__file__).main() diff --git a/test/functional/feature_settings.py b/test/functional/feature_settings.py index a7294944bf..e9148b7923 100755 --- a/test/functional/feature_settings.py +++ b/test/functional/feature_settings.py @@ -46,7 +46,7 @@ class SettingsTest(BitcoinTestFramework): # Assert default settings file was created self.stop_node(0) - default_settings = {"_warning_": f"This file is automatically generated and updated by {self.config['environment']['PACKAGE_NAME']}. Please do not edit this file while the node is running, as any changes might be ignored or overwritten."} + default_settings = {"_warning_": f"This file is automatically generated and updated by {self.config['environment']['CLIENT_NAME']}. Please do not edit this file while the node is running, as any changes might be ignored or overwritten."} with settings.open() as fp: assert_equal(json.load(fp), default_settings) diff --git a/test/functional/interface_bitcoin_cli.py b/test/functional/interface_bitcoin_cli.py index 7194c8ece4..3fe6570dd1 100755 --- a/test/functional/interface_bitcoin_cli.py +++ b/test/functional/interface_bitcoin_cli.py @@ -375,7 +375,7 @@ class TestBitcoinCli(BitcoinTestFramework): self.log.info("Test -version with node stopped") self.stop_node(0) cli_response = self.nodes[0].cli('-version').send_cli() - assert f"{self.config['environment']['PACKAGE_NAME']} RPC client version" in cli_response + assert f"{self.config['environment']['CLIENT_NAME']} RPC client version" in cli_response self.log.info("Test -rpcwait option successfully waits for RPC connection") self.nodes[0].start() # start node without RPC connection diff --git a/test/functional/interface_usdt_validation.py b/test/functional/interface_usdt_validation.py index 9a37b96ada..8a98a452de 100755 --- a/test/functional/interface_usdt_validation.py +++ b/test/functional/interface_usdt_validation.py @@ -8,6 +8,7 @@ """ import ctypes +import time # Test will be skipped if we don't have bcc installed try: @@ -105,10 +106,12 @@ class ValidationTracepointTest(BitcoinTestFramework): handle_blockconnected) self.log.info(f"mine {BLOCKS_EXPECTED} blocks") - block_hashes = self.generatetoaddress( - self.nodes[0], BLOCKS_EXPECTED, ADDRESS_BCRT1_UNSPENDABLE) - for block_hash in block_hashes: - expected_blocks[block_hash] = self.nodes[0].getblock(block_hash, 2) + generatetoaddress_duration = dict() + for _ in range(BLOCKS_EXPECTED): + start = time.time() + hash = self.generatetoaddress(self.nodes[0], 1, ADDRESS_BCRT1_UNSPENDABLE)[0] + generatetoaddress_duration[hash] = (time.time() - start) * 1e9 # in nanoseconds + expected_blocks[hash] = self.nodes[0].getblock(hash, 2) bpf.perf_buffer_poll(timeout=200) @@ -123,6 +126,10 @@ class ValidationTracepointTest(BitcoinTestFramework): assert_equal(0, event.sigops) # no sigops in coinbase tx # only plausibility checks assert event.duration > 0 + # generatetoaddress (mining and connecting) takes longer than + # connecting the block. In case the duration unit is off, we'll + # detect it with this assert. + assert event.duration < generatetoaddress_duration[block_hash] del expected_blocks[block_hash] assert_equal(BLOCKS_EXPECTED, len(events)) assert_equal(0, len(expected_blocks)) diff --git a/test/functional/mempool_limit.py b/test/functional/mempool_limit.py index 626928a49a..85d158d611 100755 --- a/test/functional/mempool_limit.py +++ b/test/functional/mempool_limit.py @@ -55,12 +55,12 @@ class MempoolLimitTest(BitcoinTestFramework): self.generate(node, 1) # tx_A needs to be RBF'd, set minfee at set size - A_weight = 1000 + A_vsize = 250 mempoolmin_feerate = node.getmempoolinfo()["mempoolminfee"] tx_A = self.wallet.send_self_transfer( from_node=node, fee_rate=mempoolmin_feerate, - target_weight=A_weight, + target_vsize=A_vsize, utxo_to_spend=rbf_utxo, confirmed_only=True ) @@ -68,15 +68,15 @@ class MempoolLimitTest(BitcoinTestFramework): # RBF's tx_A, is not yet submitted tx_B = self.wallet.create_self_transfer( fee=tx_A["fee"] * 4, - target_weight=A_weight, + target_vsize=A_vsize, utxo_to_spend=rbf_utxo, confirmed_only=True ) # Spends tx_B's output, too big for cpfp carveout (because that would also increase the descendant limit by 1) - non_cpfp_carveout_weight = 40001 # EXTRA_DESCENDANT_TX_SIZE_LIMIT + 1 + non_cpfp_carveout_vsize = 10001 # EXTRA_DESCENDANT_TX_SIZE_LIMIT + 1 tx_C = self.wallet.create_self_transfer( - target_weight=non_cpfp_carveout_weight, + target_vsize=non_cpfp_carveout_vsize, fee_rate=mempoolmin_feerate, utxo_to_spend=tx_B["new_utxo"], confirmed_only=True @@ -85,6 +85,100 @@ class MempoolLimitTest(BitcoinTestFramework): assert_equal(res["package_msg"], "transaction failed") assert "too-long-mempool-chain" in res["tx-results"][tx_C["wtxid"]]["error"] + def test_mid_package_eviction_success(self): + node = self.nodes[0] + self.log.info("Check a package where each parent passes the current mempoolminfee but a parent could be evicted before getting child's descendant feerate") + + # Clear mempool so it can be filled with minrelay txns + self.restart_node(0, extra_args=self.extra_args[0] + ["-persistmempool=0"]) + assert_equal(node.getrawmempool(), []) + + # Restarting the node resets mempool minimum feerate + assert_equal(node.getmempoolinfo()['minrelaytxfee'], Decimal('0.00001000')) + assert_equal(node.getmempoolinfo()['mempoolminfee'], Decimal('0.00001000')) + + fill_mempool(self, node) + current_info = node.getmempoolinfo() + mempoolmin_feerate = current_info["mempoolminfee"] + + mempool_txids = node.getrawmempool() + mempool_entries = [node.getmempoolentry(entry) for entry in mempool_txids] + fees_btc_per_kvb = [entry["fees"]["base"] / (Decimal(entry["vsize"]) / 1000) for entry in mempool_entries] + mempool_entry_minrate = min(fees_btc_per_kvb) + mempool_entry_minrate = mempool_entry_minrate.quantize(Decimal("0.00000000")) + + # There is a gap, our parents will be minrate, with child bringing up descendant fee sufficiently to avoid + # eviction even though parents cause eviction on their own + assert_greater_than(mempool_entry_minrate, mempoolmin_feerate) + + package_hex = [] + # UTXOs to be spent by the ultimate child transaction + parent_utxos = [] + + # Series of parents that don't need CPFP and are submitted individually. Each one is large + # which means in aggregate they could trigger eviction, but child submission should result + # in them not being evicted + parent_vsize = 25000 + num_big_parents = 3 + # Need to be large enough to trigger eviction + # (note that the mempool usage of a tx is about three times its vsize) + assert_greater_than(parent_vsize * num_big_parents * 3, current_info["maxmempool"] - current_info["bytes"]) + + big_parent_txids = [] + big_parent_wtxids = [] + for i in range(num_big_parents): + # Last parent is higher feerate causing other parents to possibly + # be evicted if trimming was allowed, which would cause the package to end up failing + parent_feerate = mempoolmin_feerate + Decimal("0.00000001") if i == num_big_parents - 1 else mempoolmin_feerate + parent = self.wallet.create_self_transfer(fee_rate=parent_feerate, target_vsize=parent_vsize, confirmed_only=True) + parent_utxos.append(parent["new_utxo"]) + package_hex.append(parent["hex"]) + big_parent_txids.append(parent["txid"]) + big_parent_wtxids.append(parent["wtxid"]) + # There is room for each of these transactions independently + assert node.testmempoolaccept([parent["hex"]])[0]["allowed"] + + # Create a child spending everything with an insane fee, bumping the package above mempool_entry_minrate + child = self.wallet.create_self_transfer_multi(utxos_to_spend=parent_utxos, fee_per_output=10000000) + package_hex.append(child["hex"]) + + # Package should be submitted, temporarily exceeding maxmempool, but not evicted. + package_res = None + with node.assert_debug_log(expected_msgs=["rolling minimum fee bumped"]): + package_res = node.submitpackage(package=package_hex, maxfeerate=0) + + assert_equal(package_res["package_msg"], "success") + + # Ensure that intra-package trimming is not happening. + # Each transaction separately satisfies the current + # minfee and shouldn't need package evaluation to + # be included. If trimming of a parent were to happen, + # package evaluation would happen to reintrodce the evicted + # parent. + assert_equal(len(package_res["tx-results"]), len(big_parent_wtxids) + 1) + for wtxid in big_parent_wtxids + [child["wtxid"]]: + assert_equal(len(package_res["tx-results"][wtxid]["fees"]["effective-includes"]), 1) + + # Maximum size must never be exceeded. + assert_greater_than(node.getmempoolinfo()["maxmempool"], node.getmempoolinfo()["bytes"]) + + # Package found in mempool still + resulting_mempool_txids = node.getrawmempool() + assert child["txid"] in resulting_mempool_txids + for txid in big_parent_txids: + assert txid in resulting_mempool_txids + + # Check every evicted tx was higher feerate than parents which evicted it + eviction_set = set(mempool_txids) - set(resulting_mempool_txids) - set(big_parent_txids) + parent_entries = [node.getmempoolentry(entry) for entry in big_parent_txids] + max_parent_feerate = max([entry["fees"]["modified"] / (Decimal(entry["vsize"]) / 1000) for entry in parent_entries]) + for eviction in eviction_set: + assert eviction in mempool_txids + for txid, entry in zip(mempool_txids, mempool_entries): + if txid == eviction: + evicted_feerate_btc_per_kvb = entry["fees"]["modified"] / (Decimal(entry["vsize"]) / 1000) + assert_greater_than(evicted_feerate_btc_per_kvb, max_parent_feerate) + def test_mid_package_eviction(self): node = self.nodes[0] self.log.info("Check a package where each parent passes the current mempoolminfee but would cause eviction before package submission terminates") @@ -103,14 +197,14 @@ class MempoolLimitTest(BitcoinTestFramework): # UTXOs to be spent by the ultimate child transaction parent_utxos = [] - evicted_weight = 8000 + evicted_vsize = 2000 # Mempool transaction which is evicted due to being at the "bottom" of the mempool when the # mempool overflows and evicts by descendant score. It's important that the eviction doesn't # happen in the middle of package evaluation, as it can invalidate the coins cache. mempool_evicted_tx = self.wallet.send_self_transfer( from_node=node, fee_rate=mempoolmin_feerate, - target_weight=evicted_weight, + target_vsize=evicted_vsize, confirmed_only=True ) # Already in mempool when package is submitted. @@ -132,14 +226,16 @@ class MempoolLimitTest(BitcoinTestFramework): # Series of parents that don't need CPFP and are submitted individually. Each one is large and # high feerate, which means they should trigger eviction but not be evicted. - parent_weight = 100000 + parent_vsize = 25000 num_big_parents = 3 - assert_greater_than(parent_weight * num_big_parents, current_info["maxmempool"] - current_info["bytes"]) + # Need to be large enough to trigger eviction + # (note that the mempool usage of a tx is about three times its vsize) + assert_greater_than(parent_vsize * num_big_parents * 3, current_info["maxmempool"] - current_info["bytes"]) parent_feerate = 100 * mempoolmin_feerate big_parent_txids = [] for i in range(num_big_parents): - parent = self.wallet.create_self_transfer(fee_rate=parent_feerate, target_weight=parent_weight, confirmed_only=True) + parent = self.wallet.create_self_transfer(fee_rate=parent_feerate, target_vsize=parent_vsize, confirmed_only=True) parent_utxos.append(parent["new_utxo"]) package_hex.append(parent["hex"]) big_parent_txids.append(parent["txid"]) @@ -311,8 +407,9 @@ class MempoolLimitTest(BitcoinTestFramework): entry = node.getmempoolentry(txid) worst_feerate_btcvb = min(worst_feerate_btcvb, entry["fees"]["descendant"] / entry["descendantsize"]) # Needs to be large enough to trigger eviction - target_weight_each = 200000 - assert_greater_than(target_weight_each * 2, node.getmempoolinfo()["maxmempool"] - node.getmempoolinfo()["bytes"]) + # (note that the mempool usage of a tx is about three times its vsize) + target_vsize_each = 50000 + assert_greater_than(target_vsize_each * 2 * 3, node.getmempoolinfo()["maxmempool"] - node.getmempoolinfo()["bytes"]) # Should be a true CPFP: parent's feerate is just below mempool min feerate parent_feerate = mempoolmin_feerate - Decimal("0.000001") # 0.1 sats/vbyte below min feerate # Parent + child is above mempool minimum feerate @@ -320,8 +417,8 @@ class MempoolLimitTest(BitcoinTestFramework): # However, when eviction is triggered, these transactions should be at the bottom. # This assertion assumes parent and child are the same size. miniwallet.rescan_utxos() - tx_parent_just_below = miniwallet.create_self_transfer(fee_rate=parent_feerate, target_weight=target_weight_each) - tx_child_just_above = miniwallet.create_self_transfer(utxo_to_spend=tx_parent_just_below["new_utxo"], fee_rate=child_feerate, target_weight=target_weight_each) + tx_parent_just_below = miniwallet.create_self_transfer(fee_rate=parent_feerate, target_vsize=target_vsize_each) + tx_child_just_above = miniwallet.create_self_transfer(utxo_to_spend=tx_parent_just_below["new_utxo"], fee_rate=child_feerate, target_vsize=target_vsize_each) # This package ranks below the lowest descendant package in the mempool package_fee = tx_parent_just_below["fee"] + tx_child_just_above["fee"] package_vsize = tx_parent_just_below["tx"].get_vsize() + tx_child_just_above["tx"].get_vsize() @@ -336,6 +433,7 @@ class MempoolLimitTest(BitcoinTestFramework): self.stop_node(0) self.nodes[0].assert_start_raises_init_error(["-maxmempool=4"], "Error: -maxmempool must be at least 5 MB") + self.test_mid_package_eviction_success() self.test_mid_package_replacement() self.test_mid_package_eviction() self.test_rbf_carveout_disallowed() diff --git a/test/functional/mempool_package_limits.py b/test/functional/mempool_package_limits.py index 6e26a684e2..3290ff43c4 100755 --- a/test/functional/mempool_package_limits.py +++ b/test/functional/mempool_package_limits.py @@ -4,9 +4,6 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test logic for limiting mempool and package ancestors/descendants.""" from test_framework.blocktools import COINBASE_MATURITY -from test_framework.messages import ( - WITNESS_SCALE_FACTOR, -) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, @@ -290,19 +287,18 @@ class MempoolPackageLimitsTest(BitcoinTestFramework): parent_utxos = [] target_vsize = 30_000 high_fee = 10 * target_vsize # 10 sats/vB - target_weight = target_vsize * WITNESS_SCALE_FACTOR self.log.info("Check that in-mempool and in-package ancestor size limits are calculated properly in packages") # Mempool transactions A and B for _ in range(2): - bulked_tx = self.wallet.create_self_transfer(target_weight=target_weight) + bulked_tx = self.wallet.create_self_transfer(target_vsize=target_vsize) self.wallet.sendrawtransaction(from_node=node, tx_hex=bulked_tx["hex"]) parent_utxos.append(bulked_tx["new_utxo"]) # Package transaction C - pc_tx = self.wallet.create_self_transfer_multi(utxos_to_spend=parent_utxos, fee_per_output=high_fee, target_weight=target_weight) + pc_tx = self.wallet.create_self_transfer_multi(utxos_to_spend=parent_utxos, fee_per_output=high_fee, target_vsize=target_vsize) # Package transaction D - pd_tx = self.wallet.create_self_transfer(utxo_to_spend=pc_tx["new_utxos"][0], target_weight=target_weight) + pd_tx = self.wallet.create_self_transfer(utxo_to_spend=pc_tx["new_utxos"][0], target_vsize=target_vsize) assert_equal(2, node.getmempoolinfo()["size"]) return [pc_tx["hex"], pd_tx["hex"]] @@ -321,20 +317,19 @@ class MempoolPackageLimitsTest(BitcoinTestFramework): node = self.nodes[0] target_vsize = 21_000 high_fee = 10 * target_vsize # 10 sats/vB - target_weight = target_vsize * WITNESS_SCALE_FACTOR self.log.info("Check that in-mempool and in-package descendant sizes are calculated properly in packages") # Top parent in mempool, Ma - ma_tx = self.wallet.create_self_transfer_multi(num_outputs=2, fee_per_output=high_fee // 2, target_weight=target_weight) + ma_tx = self.wallet.create_self_transfer_multi(num_outputs=2, fee_per_output=high_fee // 2, target_vsize=target_vsize) self.wallet.sendrawtransaction(from_node=node, tx_hex=ma_tx["hex"]) package_hex = [] for j in range(2): # Two legs (left and right) # Mempool transaction (Mb and Mc) - mempool_tx = self.wallet.create_self_transfer(utxo_to_spend=ma_tx["new_utxos"][j], target_weight=target_weight) + mempool_tx = self.wallet.create_self_transfer(utxo_to_spend=ma_tx["new_utxos"][j], target_vsize=target_vsize) self.wallet.sendrawtransaction(from_node=node, tx_hex=mempool_tx["hex"]) # Package transaction (Pd and Pe) - package_tx = self.wallet.create_self_transfer(utxo_to_spend=mempool_tx["new_utxo"], target_weight=target_weight) + package_tx = self.wallet.create_self_transfer(utxo_to_spend=mempool_tx["new_utxo"], target_vsize=target_vsize) package_hex.append(package_tx["hex"]) assert_equal(3, node.getmempoolinfo()["size"]) diff --git a/test/functional/mempool_package_rbf.py b/test/functional/mempool_package_rbf.py index f4d57262f2..a5b8fa5f87 100755 --- a/test/functional/mempool_package_rbf.py +++ b/test/functional/mempool_package_rbf.py @@ -554,7 +554,7 @@ class PackageRBFTest(BitcoinTestFramework): self.generate(node, 1) def test_child_conflicts_parent_mempool_ancestor(self): - fill_mempool(self, self.nodes[0]) + fill_mempool(self, self.nodes[0], tx_sync_fun=self.no_op) # Reset coins since we filled the mempool with current coins self.coins = self.wallet.get_utxos(mark_as_spent=False, confirmed_only=True) diff --git a/test/functional/mempool_truc.py b/test/functional/mempool_truc.py index 28f3256ef1..54a258215d 100755 --- a/test/functional/mempool_truc.py +++ b/test/functional/mempool_truc.py @@ -6,7 +6,6 @@ from decimal import Decimal from test_framework.messages import ( MAX_BIP125_RBF_SEQUENCE, - WITNESS_SCALE_FACTOR, ) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( @@ -23,6 +22,7 @@ from test_framework.wallet import ( MAX_REPLACEMENT_CANDIDATES = 100 TRUC_MAX_VSIZE = 10000 +TRUC_CHILD_MAX_VSIZE = 1000 def cleanup(extra_args=None): def decorator(func): @@ -55,14 +55,14 @@ class MempoolTRUC(BitcoinTestFramework): def test_truc_max_vsize(self): node = self.nodes[0] self.log.info("Test TRUC-specific maximum transaction vsize") - tx_v3_heavy = self.wallet.create_self_transfer(target_weight=(TRUC_MAX_VSIZE + 1) * WITNESS_SCALE_FACTOR, version=3) + tx_v3_heavy = self.wallet.create_self_transfer(target_vsize=TRUC_MAX_VSIZE + 1, version=3) assert_greater_than_or_equal(tx_v3_heavy["tx"].get_vsize(), TRUC_MAX_VSIZE) expected_error_heavy = f"TRUC-violation, version=3 tx {tx_v3_heavy['txid']} (wtxid={tx_v3_heavy['wtxid']}) is too big" assert_raises_rpc_error(-26, expected_error_heavy, node.sendrawtransaction, tx_v3_heavy["hex"]) self.check_mempool([]) # Ensure we are hitting the TRUC-specific limit and not something else - tx_v2_heavy = self.wallet.send_self_transfer(from_node=node, target_weight=(TRUC_MAX_VSIZE + 1) * WITNESS_SCALE_FACTOR, version=2) + tx_v2_heavy = self.wallet.send_self_transfer(from_node=node, target_vsize=TRUC_MAX_VSIZE + 1, version=2) self.check_mempool([tx_v2_heavy["txid"]]) @cleanup(extra_args=["-datacarriersize=1000"]) @@ -73,10 +73,10 @@ class MempoolTRUC(BitcoinTestFramework): self.check_mempool([tx_v3_parent_normal["txid"]]) tx_v3_child_heavy = self.wallet.create_self_transfer( utxo_to_spend=tx_v3_parent_normal["new_utxo"], - target_weight=4004, + target_vsize=TRUC_CHILD_MAX_VSIZE + 1, version=3 ) - assert_greater_than_or_equal(tx_v3_child_heavy["tx"].get_vsize(), 1000) + assert_greater_than_or_equal(tx_v3_child_heavy["tx"].get_vsize(), TRUC_CHILD_MAX_VSIZE) expected_error_child_heavy = f"TRUC-violation, version=3 child tx {tx_v3_child_heavy['txid']} (wtxid={tx_v3_child_heavy['wtxid']}) is too big" assert_raises_rpc_error(-26, expected_error_child_heavy, node.sendrawtransaction, tx_v3_child_heavy["hex"]) self.check_mempool([tx_v3_parent_normal["txid"]]) @@ -88,20 +88,21 @@ class MempoolTRUC(BitcoinTestFramework): from_node=node, fee_rate=DEFAULT_FEE, utxo_to_spend=tx_v3_parent_normal["new_utxo"], - target_weight=3987, + target_vsize=TRUC_CHILD_MAX_VSIZE - 3, version=3 ) - assert_greater_than_or_equal(1000, tx_v3_child_almost_heavy["tx"].get_vsize()) + assert_greater_than_or_equal(TRUC_CHILD_MAX_VSIZE, tx_v3_child_almost_heavy["tx"].get_vsize()) self.check_mempool([tx_v3_parent_normal["txid"], tx_v3_child_almost_heavy["txid"]]) assert_equal(node.getmempoolentry(tx_v3_parent_normal["txid"])["descendantcount"], 2) tx_v3_child_almost_heavy_rbf = self.wallet.send_self_transfer( from_node=node, fee_rate=DEFAULT_FEE * 2, utxo_to_spend=tx_v3_parent_normal["new_utxo"], - target_weight=3500, + target_vsize=875, version=3 ) - assert_greater_than_or_equal(tx_v3_child_almost_heavy["tx"].get_vsize() + tx_v3_child_almost_heavy_rbf["tx"].get_vsize(), 1000) + assert_greater_than_or_equal(tx_v3_child_almost_heavy["tx"].get_vsize() + tx_v3_child_almost_heavy_rbf["tx"].get_vsize(), + TRUC_CHILD_MAX_VSIZE) self.check_mempool([tx_v3_parent_normal["txid"], tx_v3_child_almost_heavy_rbf["txid"]]) assert_equal(node.getmempoolentry(tx_v3_parent_normal["txid"])["descendantcount"], 2) @@ -199,8 +200,8 @@ class MempoolTRUC(BitcoinTestFramework): self.check_mempool([]) tx_v2_from_v3 = self.wallet.send_self_transfer(from_node=node, utxo_to_spend=tx_v3_block["new_utxo"], version=2) tx_v3_from_v2 = self.wallet.send_self_transfer(from_node=node, utxo_to_spend=tx_v2_block["new_utxo"], version=3) - tx_v3_child_large = self.wallet.send_self_transfer(from_node=node, utxo_to_spend=tx_v3_block2["new_utxo"], target_weight=5000, version=3) - assert_greater_than(node.getmempoolentry(tx_v3_child_large["txid"])["vsize"], 1000) + tx_v3_child_large = self.wallet.send_self_transfer(from_node=node, utxo_to_spend=tx_v3_block2["new_utxo"], target_vsize=1250, version=3) + assert_greater_than(node.getmempoolentry(tx_v3_child_large["txid"])["vsize"], TRUC_CHILD_MAX_VSIZE) self.check_mempool([tx_v2_from_v3["txid"], tx_v3_from_v2["txid"], tx_v3_child_large["txid"]]) node.invalidateblock(block[0]) self.check_mempool([tx_v3_block["txid"], tx_v2_block["txid"], tx_v3_block2["txid"], tx_v2_from_v3["txid"], tx_v3_from_v2["txid"], tx_v3_child_large["txid"]]) @@ -217,22 +218,22 @@ class MempoolTRUC(BitcoinTestFramework): """ node = self.nodes[0] self.log.info("Test that a decreased limitdescendantsize also applies to TRUC child") - parent_target_weight = 9990 * WITNESS_SCALE_FACTOR - child_target_weight = 500 * WITNESS_SCALE_FACTOR + parent_target_vsize = 9990 + child_target_vsize = 500 tx_v3_parent_large1 = self.wallet.send_self_transfer( from_node=node, - target_weight=parent_target_weight, + target_vsize=parent_target_vsize, version=3 ) tx_v3_child_large1 = self.wallet.create_self_transfer( utxo_to_spend=tx_v3_parent_large1["new_utxo"], - target_weight=child_target_weight, + target_vsize=child_target_vsize, version=3 ) # Parent and child are within v3 limits, but parent's 10kvB descendant limit is exceeded assert_greater_than_or_equal(TRUC_MAX_VSIZE, tx_v3_parent_large1["tx"].get_vsize()) - assert_greater_than_or_equal(1000, tx_v3_child_large1["tx"].get_vsize()) + assert_greater_than_or_equal(TRUC_CHILD_MAX_VSIZE, tx_v3_child_large1["tx"].get_vsize()) assert_greater_than(tx_v3_parent_large1["tx"].get_vsize() + tx_v3_child_large1["tx"].get_vsize(), 10000) assert_raises_rpc_error(-26, f"too-long-mempool-chain, exceeds descendant size limit for tx {tx_v3_parent_large1['txid']}", node.sendrawtransaction, tx_v3_child_large1["hex"]) @@ -244,18 +245,18 @@ class MempoolTRUC(BitcoinTestFramework): self.restart_node(0, extra_args=["-limitancestorsize=10", "-datacarriersize=40000"]) tx_v3_parent_large2 = self.wallet.send_self_transfer( from_node=node, - target_weight=parent_target_weight, + target_vsize=parent_target_vsize, version=3 ) tx_v3_child_large2 = self.wallet.create_self_transfer( utxo_to_spend=tx_v3_parent_large2["new_utxo"], - target_weight=child_target_weight, + target_vsize=child_target_vsize, version=3 ) # Parent and child are within TRUC limits assert_greater_than_or_equal(TRUC_MAX_VSIZE, tx_v3_parent_large2["tx"].get_vsize()) - assert_greater_than_or_equal(1000, tx_v3_child_large2["tx"].get_vsize()) + assert_greater_than_or_equal(TRUC_CHILD_MAX_VSIZE, tx_v3_child_large2["tx"].get_vsize()) assert_greater_than(tx_v3_parent_large2["tx"].get_vsize() + tx_v3_child_large2["tx"].get_vsize(), 10000) assert_raises_rpc_error(-26, f"too-long-mempool-chain, exceeds ancestor size limit", node.sendrawtransaction, tx_v3_child_large2["hex"]) @@ -267,12 +268,12 @@ class MempoolTRUC(BitcoinTestFramework): node = self.nodes[0] tx_v3_parent_normal = self.wallet.create_self_transfer( fee_rate=0, - target_weight=4004, + target_vsize=1001, version=3 ) tx_v3_parent_2_normal = self.wallet.create_self_transfer( fee_rate=0, - target_weight=4004, + target_vsize=1001, version=3 ) tx_v3_child_multiparent = self.wallet.create_self_transfer_multi( @@ -282,7 +283,7 @@ class MempoolTRUC(BitcoinTestFramework): ) tx_v3_child_heavy = self.wallet.create_self_transfer_multi( utxos_to_spend=[tx_v3_parent_normal["new_utxo"]], - target_weight=4004, + target_vsize=TRUC_CHILD_MAX_VSIZE + 1, fee_per_output=10000, version=3 ) @@ -294,7 +295,7 @@ class MempoolTRUC(BitcoinTestFramework): self.check_mempool([]) result = node.submitpackage([tx_v3_parent_normal["hex"], tx_v3_child_heavy["hex"]]) - # tx_v3_child_heavy is heavy based on weight, not sigops. + # tx_v3_child_heavy is heavy based on vsize, not sigops. assert_equal(result['package_msg'], f"TRUC-violation, version=3 child tx {tx_v3_child_heavy['txid']} (wtxid={tx_v3_child_heavy['wtxid']}) is too big: {tx_v3_child_heavy['tx'].get_vsize()} > 1000 virtual bytes") self.check_mempool([]) @@ -416,7 +417,7 @@ class MempoolTRUC(BitcoinTestFramework): node = self.nodes[0] tx_v3_parent = self.wallet.create_self_transfer( fee_rate=0, - target_weight=4004, + target_vsize=1001, version=3 ) tx_v2_child = self.wallet.create_self_transfer_multi( diff --git a/test/functional/p2p_1p1c_network.py b/test/functional/p2p_1p1c_network.py index f9e782f524..cdc4e1691d 100755 --- a/test/functional/p2p_1p1c_network.py +++ b/test/functional/p2p_1p1c_network.py @@ -49,9 +49,6 @@ class PackageRelayTest(BitcoinTestFramework): def raise_network_minfee(self): fill_mempool(self, self.nodes[0]) - self.log.debug("Wait for the network to sync mempools") - self.sync_mempools() - self.log.debug("Check that all nodes' mempool minimum feerates are above min relay feerate") for node in self.nodes: assert_equal(node.getmempoolinfo()['minrelaytxfee'], FEERATE_1SAT_VB) diff --git a/test/functional/p2p_invalid_tx.py b/test/functional/p2p_invalid_tx.py index 241aefab24..ee8c6c16ca 100755 --- a/test/functional/p2p_invalid_tx.py +++ b/test/functional/p2p_invalid_tx.py @@ -165,7 +165,7 @@ class InvalidTxRequestTest(BitcoinTestFramework): node.p2ps[0].send_txs_and_test([rejected_parent], node, success=False) self.log.info('Test that a peer disconnection causes erase its transactions from the orphan pool') - with node.assert_debug_log(['Erased 100 orphan transaction(s) from peer=25']): + with node.assert_debug_log(['Erased 100 orphan transaction(s) from peer=26']): self.reconnect_p2p(num_connections=1) self.log.info('Test that a transaction in the orphan pool is included in a new tip block causes erase this transaction from the orphan pool') diff --git a/test/functional/p2p_orphan_handling.py b/test/functional/p2p_orphan_handling.py index 22600bf8a4..9ec23e9d6e 100755 --- a/test/functional/p2p_orphan_handling.py +++ b/test/functional/p2p_orphan_handling.py @@ -5,6 +5,7 @@ import time +from test_framework.mempool_util import tx_in_orphanage from test_framework.messages import ( CInv, CTxInWitness, @@ -41,6 +42,8 @@ from test_framework.wallet import ( # for one peer and y seconds for another, use specific values instead. TXREQUEST_TIME_SKIP = NONPREF_PEER_TX_DELAY + TXID_RELAY_DELAY + OVERLOADED_PEER_TX_DELAY + 1 +DEFAULT_MAX_ORPHAN_TRANSACTIONS = 100 + def cleanup(func): # Time to fastfoward (using setmocktime) in between subtests to ensure they do not interfere with # one another, in seconds. Equal to 12 hours, which is enough to expire anything that may exist @@ -566,6 +569,47 @@ class OrphanHandlingTest(BitcoinTestFramework): assert tx_child["txid"] in node_mempool assert_equal(node.getmempoolentry(tx_child["txid"])["wtxid"], tx_child["wtxid"]) + @cleanup + def test_max_orphan_amount(self): + self.log.info("Check that we never exceed our storage limits for orphans") + + node = self.nodes[0] + self.generate(self.wallet, 1) + peer_1 = node.add_p2p_connection(P2PInterface()) + + self.log.info("Check that orphanage is empty on start of test") + assert len(node.getorphantxs()) == 0 + + self.log.info("Filling up orphanage with " + str(DEFAULT_MAX_ORPHAN_TRANSACTIONS) + "(DEFAULT_MAX_ORPHAN_TRANSACTIONS) orphans") + orphans = [] + parent_orphans = [] + for _ in range(DEFAULT_MAX_ORPHAN_TRANSACTIONS): + tx_parent_1 = self.wallet.create_self_transfer() + tx_child_1 = self.wallet.create_self_transfer(utxo_to_spend=tx_parent_1["new_utxo"]) + parent_orphans.append(tx_parent_1["tx"]) + orphans.append(tx_child_1["tx"]) + peer_1.send_message(msg_tx(tx_child_1["tx"])) + + peer_1.sync_with_ping() + orphanage = node.getorphantxs() + assert_equal(len(orphanage), DEFAULT_MAX_ORPHAN_TRANSACTIONS) + + for orphan in orphans: + assert tx_in_orphanage(node, orphan) + + self.log.info("Check that we do not add more than the max orphan amount") + tx_parent_1 = self.wallet.create_self_transfer() + tx_child_1 = self.wallet.create_self_transfer(utxo_to_spend=tx_parent_1["new_utxo"]) + peer_1.send_and_ping(msg_tx(tx_child_1["tx"])) + parent_orphans.append(tx_parent_1["tx"]) + orphanage = node.getorphantxs() + assert_equal(len(orphanage), DEFAULT_MAX_ORPHAN_TRANSACTIONS) + + self.log.info("Clearing the orphanage") + for index, parent_orphan in enumerate(parent_orphans): + peer_1.send_and_ping(msg_tx(parent_orphan)) + assert_equal(len(node.getorphantxs()),0) + def run_test(self): self.nodes[0].setmocktime(int(time.time())) @@ -582,6 +626,7 @@ class OrphanHandlingTest(BitcoinTestFramework): self.test_same_txid_orphan() self.test_same_txid_orphan_of_orphan() self.test_orphan_txid_inv() + self.test_max_orphan_amount() if __name__ == '__main__': diff --git a/test/functional/p2p_seednode.py b/test/functional/p2p_seednode.py index 6c510a6a0b..5a9fca3278 100755 --- a/test/functional/p2p_seednode.py +++ b/test/functional/p2p_seednode.py @@ -20,17 +20,18 @@ class P2PSeedNodes(BitcoinTestFramework): self.disable_autoconnect = False def test_no_seednode(self): - # Check that if no seednode is provided, the node proceeds as usual (without waiting) + self.log.info("Check that if no seednode is provided, the node proceeds as usual (without waiting)") with self.nodes[0].assert_debug_log(expected_msgs=[], unexpected_msgs=["Empty addrman, adding seednode", f"Couldn't connect to peers from addrman after {ADD_NEXT_SEEDNODE} seconds. Adding seednode"], timeout=ADD_NEXT_SEEDNODE): self.restart_node(0) def test_seednode_empty_addrman(self): seed_node = "0.0.0.1" - # Check that the seednode is added to m_addr_fetches on bootstrap on an empty addrman + self.log.info("Check that the seednode is immediately added on bootstrap on an empty addrman") with self.nodes[0].assert_debug_log(expected_msgs=[f"Empty addrman, adding seednode ({seed_node}) to addrfetch"], timeout=ADD_NEXT_SEEDNODE): self.restart_node(0, extra_args=[f'-seednode={seed_node}']) - def test_seednode_addrman_unreachable_peers(self): + def test_seednode_non_empty_addrman(self): + self.log.info("Check that if addrman is non-empty, seednodes are queried with a delay") seed_node = "0.0.0.2" node = self.nodes[0] # Fill the addrman with unreachable nodes @@ -39,17 +40,18 @@ class P2PSeedNodes(BitcoinTestFramework): port = 8333 + i node.addpeeraddress(ip, port) - # Restart the node so seednode is processed again + # Restart the node so seednode is processed again. Specify a non-working proxy to make sure no actual connections to random IPs are attempted. + with node.assert_debug_log(expected_msgs=["trying v1 connection"], timeout=ADD_NEXT_SEEDNODE): + self.restart_node(0, extra_args=[f'-seednode={seed_node}', '-proxy=127.0.0.1:1']) + with node.assert_debug_log(expected_msgs=[f"Couldn't connect to peers from addrman after {ADD_NEXT_SEEDNODE} seconds. Adding seednode ({seed_node}) to addrfetch"], unexpected_msgs=["Empty addrman, adding seednode"], timeout=ADD_NEXT_SEEDNODE * 1.5): - self.restart_node(0, extra_args=[f'-seednode={seed_node}']) node.setmocktime(int(time.time()) + ADD_NEXT_SEEDNODE + 1) def run_test(self): self.test_no_seednode() self.test_seednode_empty_addrman() - self.test_seednode_addrman_unreachable_peers() + self.test_seednode_non_empty_addrman() if __name__ == '__main__': P2PSeedNodes(__file__).main() - diff --git a/test/functional/p2p_tx_download.py b/test/functional/p2p_tx_download.py index efad4e7c0f..c69d6ff405 100755 --- a/test/functional/p2p_tx_download.py +++ b/test/functional/p2p_tx_download.py @@ -250,7 +250,7 @@ class TxDownloadTest(BitcoinTestFramework): def test_rejects_filter_reset(self): self.log.info('Check that rejected tx is not requested again') node = self.nodes[0] - fill_mempool(self, node) + fill_mempool(self, node, tx_sync_fun=self.no_op) self.wallet.rescan_utxos() mempoolminfee = node.getmempoolinfo()['mempoolminfee'] peer = node.add_p2p_connection(TestP2PConn()) diff --git a/test/functional/rpc_blockchain.py b/test/functional/rpc_blockchain.py index 91e9975ad5..f02e6914ef 100755 --- a/test/functional/rpc_blockchain.py +++ b/test/functional/rpc_blockchain.py @@ -90,6 +90,7 @@ class BlockchainTest(BitcoinTestFramework): self._test_getdifficulty() self._test_getnetworkhashps() self._test_stopatheight() + self._test_waitforblock() # also tests waitfornewblock self._test_waitforblockheight() self._test_getblock() self._test_getdeploymentinfo() @@ -507,6 +508,38 @@ class BlockchainTest(BitcoinTestFramework): self.start_node(0) assert_equal(self.nodes[0].getblockcount(), HEIGHT + 7) + def _test_waitforblock(self): + self.log.info("Test waitforblock and waitfornewblock") + node = self.nodes[0] + + current_height = node.getblock(node.getbestblockhash())['height'] + current_hash = node.getblock(node.getbestblockhash())['hash'] + + self.log.debug("Roll the chain back a few blocks and then reconsider it") + rollback_height = current_height - 100 + rollback_hash = node.getblockhash(rollback_height) + rollback_header = node.getblockheader(rollback_hash) + + node.invalidateblock(rollback_hash) + assert_equal(node.getblockcount(), rollback_height - 1) + + self.log.debug("waitforblock should return the same block after its timeout") + assert_equal(node.waitforblock(blockhash=current_hash, timeout=1)['hash'], rollback_header['previousblockhash']) + + node.reconsiderblock(rollback_hash) + # The chain has probably already been restored by the time reconsiderblock returns, + # but poll anyway. + self.wait_until(lambda: node.waitforblock(blockhash=current_hash, timeout=100)['hash'] == current_hash) + + # roll back again + node.invalidateblock(rollback_hash) + assert_equal(node.getblockcount(), rollback_height - 1) + + node.reconsiderblock(rollback_hash) + # The chain has probably already been restored by the time reconsiderblock returns, + # but poll anyway. + self.wait_until(lambda: node.waitfornewblock(timeout=100)['hash'] == current_hash) + def _test_waitforblockheight(self): self.log.info("Test waitforblockheight") node = self.nodes[0] diff --git a/test/functional/rpc_getorphantxs.py b/test/functional/rpc_getorphantxs.py new file mode 100755 index 0000000000..dfa1168b2e --- /dev/null +++ b/test/functional/rpc_getorphantxs.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +# Copyright (c) 2014-2024 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test the getorphantxs RPC.""" + +from test_framework.mempool_util import tx_in_orphanage +from test_framework.messages import msg_tx +from test_framework.p2p import P2PInterface +from test_framework.util import assert_equal +from test_framework.test_framework import BitcoinTestFramework +from test_framework.wallet import MiniWallet + + +class GetOrphanTxsTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + + def run_test(self): + self.wallet = MiniWallet(self.nodes[0]) + self.test_orphan_activity() + self.test_orphan_details() + + def test_orphan_activity(self): + self.log.info("Check that orphaned transactions are returned with getorphantxs") + node = self.nodes[0] + + self.log.info("Create two 1P1C packages, but only broadcast the children") + tx_parent_1 = self.wallet.create_self_transfer() + tx_child_1 = self.wallet.create_self_transfer(utxo_to_spend=tx_parent_1["new_utxo"]) + tx_parent_2 = self.wallet.create_self_transfer() + tx_child_2 = self.wallet.create_self_transfer(utxo_to_spend=tx_parent_2["new_utxo"]) + peer = node.add_p2p_connection(P2PInterface()) + peer.send_and_ping(msg_tx(tx_child_1["tx"])) + peer.send_and_ping(msg_tx(tx_child_2["tx"])) + + self.log.info("Check that neither parent is in the mempool") + assert_equal(node.getmempoolinfo()["size"], 0) + + self.log.info("Check that both children are in the orphanage") + + orphanage = node.getorphantxs(verbosity=0) + self.log.info("Check the size of the orphanage") + assert_equal(len(orphanage), 2) + self.log.info("Check that negative verbosity is treated as 0") + assert_equal(orphanage, node.getorphantxs(verbosity=-1)) + assert tx_in_orphanage(node, tx_child_1["tx"]) + assert tx_in_orphanage(node, tx_child_2["tx"]) + + self.log.info("Broadcast parent 1") + peer.send_and_ping(msg_tx(tx_parent_1["tx"])) + self.log.info("Check that parent 1 and child 1 are in the mempool") + raw_mempool = node.getrawmempool() + assert_equal(len(raw_mempool), 2) + assert tx_parent_1["txid"] in raw_mempool + assert tx_child_1["txid"] in raw_mempool + + self.log.info("Check that orphanage only contains child 2") + orphanage = node.getorphantxs() + assert_equal(len(orphanage), 1) + assert tx_in_orphanage(node, tx_child_2["tx"]) + + peer.send_and_ping(msg_tx(tx_parent_2["tx"])) + self.log.info("Check that all parents and children are now in the mempool") + raw_mempool = node.getrawmempool() + assert_equal(len(raw_mempool), 4) + assert tx_parent_1["txid"] in raw_mempool + assert tx_child_1["txid"] in raw_mempool + assert tx_parent_2["txid"] in raw_mempool + assert tx_child_2["txid"] in raw_mempool + self.log.info("Check that the orphanage is empty") + assert_equal(len(node.getorphantxs()), 0) + + self.log.info("Confirm the transactions (clears mempool)") + self.generate(node, 1) + assert_equal(node.getmempoolinfo()["size"], 0) + + def test_orphan_details(self): + self.log.info("Check the transaction details returned from getorphantxs") + node = self.nodes[0] + + self.log.info("Create two orphans, from different peers") + tx_parent_1 = self.wallet.create_self_transfer() + tx_child_1 = self.wallet.create_self_transfer(utxo_to_spend=tx_parent_1["new_utxo"]) + tx_parent_2 = self.wallet.create_self_transfer() + tx_child_2 = self.wallet.create_self_transfer(utxo_to_spend=tx_parent_2["new_utxo"]) + peer_1 = node.add_p2p_connection(P2PInterface()) + peer_2 = node.add_p2p_connection(P2PInterface()) + peer_1.send_and_ping(msg_tx(tx_child_1["tx"])) + peer_2.send_and_ping(msg_tx(tx_child_2["tx"])) + + orphanage = node.getorphantxs(verbosity=2) + assert tx_in_orphanage(node, tx_child_1["tx"]) + assert tx_in_orphanage(node, tx_child_2["tx"]) + + self.log.info("Check that orphan 1 and 2 were from different peers") + assert orphanage[0]["from"][0] != orphanage[1]["from"][0] + + self.log.info("Unorphan child 2") + peer_2.send_and_ping(msg_tx(tx_parent_2["tx"])) + assert not tx_in_orphanage(node, tx_child_2["tx"]) + + self.log.info("Checking orphan details") + orphanage = node.getorphantxs(verbosity=1) + assert_equal(len(node.getorphantxs()), 1) + orphan_1 = orphanage[0] + self.orphan_details_match(orphan_1, tx_child_1, verbosity=1) + + self.log.info("Checking orphan details (verbosity 2)") + orphanage = node.getorphantxs(verbosity=2) + orphan_1 = orphanage[0] + self.orphan_details_match(orphan_1, tx_child_1, verbosity=2) + + def orphan_details_match(self, orphan, tx, verbosity): + self.log.info("Check txid/wtxid of orphan") + assert_equal(orphan["txid"], tx["txid"]) + assert_equal(orphan["wtxid"], tx["wtxid"]) + + self.log.info("Check the sizes of orphan") + assert_equal(orphan["bytes"], len(tx["tx"].serialize())) + assert_equal(orphan["vsize"], tx["tx"].get_vsize()) + assert_equal(orphan["weight"], tx["tx"].get_weight()) + + if verbosity == 2: + self.log.info("Check the transaction hex of orphan") + assert_equal(orphan["hex"], tx["hex"]) + +if __name__ == '__main__': + GetOrphanTxsTest(__file__).main() diff --git a/test/functional/test_framework/mempool_util.py b/test/functional/test_framework/mempool_util.py index 148cc935ed..a6a7940c60 100644 --- a/test/functional/test_framework/mempool_util.py +++ b/test/functional/test_framework/mempool_util.py @@ -8,6 +8,7 @@ from decimal import Decimal from .blocktools import ( COINBASE_MATURITY, ) +from .messages import CTransaction from .util import ( assert_equal, assert_greater_than, @@ -19,14 +20,11 @@ from .wallet import ( ) -def fill_mempool(test_framework, node): +def fill_mempool(test_framework, node, *, tx_sync_fun=None): """Fill mempool until eviction. Allows for simpler testing of scenarios with floating mempoolminfee > minrelay - Requires -datacarriersize=100000 and - -maxmempool=5. - It will not ensure mempools become synced as it - is based on a single node and assumes -minrelaytxfee + Requires -datacarriersize=100000 and -maxmempool=5 and assumes -minrelaytxfee is 1 sat/vbyte. To avoid unintentional tx dependencies, the mempool filling txs are created with a tagged ephemeral miniwallet instance. @@ -57,18 +55,25 @@ def fill_mempool(test_framework, node): tx_to_be_evicted_id = ephemeral_miniwallet.send_self_transfer( from_node=node, utxo_to_spend=confirmed_utxos.pop(0), fee_rate=relayfee)["txid"] + def send_batch(fee): + utxos = confirmed_utxos[:tx_batch_size] + create_lots_of_big_transactions(ephemeral_miniwallet, node, fee, tx_batch_size, txouts, utxos) + del confirmed_utxos[:tx_batch_size] + # Increase the tx fee rate to give the subsequent transactions a higher priority in the mempool # The tx has an approx. vsize of 65k, i.e. multiplying the previous fee rate (in sats/kvB) # by 130 should result in a fee that corresponds to 2x of that fee rate base_fee = relayfee * 130 + batch_fees = [(i + 1) * base_fee for i in range(num_of_batches)] test_framework.log.debug("Fill up the mempool with txs with higher fee rate") - with node.assert_debug_log(["rolling minimum fee bumped"]): - for batch_of_txid in range(num_of_batches): - fee = (batch_of_txid + 1) * base_fee - utxos = confirmed_utxos[:tx_batch_size] - create_lots_of_big_transactions(ephemeral_miniwallet, node, fee, tx_batch_size, txouts, utxos) - del confirmed_utxos[:tx_batch_size] + for fee in batch_fees[:-3]: + send_batch(fee) + tx_sync_fun() if tx_sync_fun else test_framework.sync_mempools() # sync before any eviction + assert_equal(node.getmempoolinfo()["mempoolminfee"], Decimal("0.00001000")) + for fee in batch_fees[-3:]: + send_batch(fee) + tx_sync_fun() if tx_sync_fun else test_framework.sync_mempools() # sync after all evictions test_framework.log.debug("The tx should be evicted by now") # The number of transactions created should be greater than the ones present in the mempool @@ -79,3 +84,8 @@ def fill_mempool(test_framework, node): test_framework.log.debug("Check that mempoolminfee is larger than minrelaytxfee") assert_equal(node.getmempoolinfo()['minrelaytxfee'], Decimal('0.00001000')) assert_greater_than(node.getmempoolinfo()['mempoolminfee'], Decimal('0.00001000')) + +def tx_in_orphanage(node, tx: CTransaction) -> bool: + """Returns true if the transaction is in the orphanage.""" + found = [o for o in node.getorphantxs(verbosity=1) if o["txid"] == tx.rehash() and o["wtxid"] == tx.getwtxid()] + return len(found) == 1 diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index 49212eb019..70ec8f0988 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -349,7 +349,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): self.log.error("Hint: Call {} '{}' to consolidate all logs".format(os.path.normpath(os.path.dirname(os.path.realpath(__file__)) + "/../combine_logs.py"), self.options.tmpdir)) self.log.error("") self.log.error("If this failure happened unexpectedly or intermittently, please file a bug and provide a link or upload of the combined log.") - self.log.error(self.config['environment']['PACKAGE_BUGREPORT']) + self.log.error(self.config['environment']['CLIENT_BUGREPORT']) self.log.error("") exit_code = TEST_EXIT_FAILED # Logging.shutdown will not remove stream- and filehandlers, so we must diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index 60ca9269a5..e59ff4e392 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -48,6 +48,7 @@ BITCOIND_PROC_WAIT_TIMEOUT = 60 NUM_XOR_BYTES = 8 # The null blocks key (all 0s) NULL_BLK_XOR_KEY = bytes([0] * NUM_XOR_BYTES) +BITCOIN_PID_FILENAME_DEFAULT = "bitcoind.pid" class FailedToStartError(Exception): diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index ce68de7eaa..f9b19f218e 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -433,7 +433,6 @@ def write_config(config_path, *, n, chain, extra_config="", disable_autoconnect= # in tests. f.write("peertimeout=999999999\n") f.write("printtoconsole=0\n") - f.write("upnp=0\n") f.write("natpmp=0\n") f.write("shrinkdebugfile=0\n") f.write("deprecatedrpc=create_bdb\n") # Required to run the tests diff --git a/test/functional/test_framework/wallet.py b/test/functional/test_framework/wallet.py index f3713f297e..1cef714705 100644 --- a/test/functional/test_framework/wallet.py +++ b/test/functional/test_framework/wallet.py @@ -7,7 +7,6 @@ from copy import deepcopy from decimal import Decimal from enum import Enum -import math from typing import ( Any, Optional, @@ -35,7 +34,6 @@ from test_framework.messages import ( CTxOut, hash256, ser_compact_size, - WITNESS_SCALE_FACTOR, ) from test_framework.script import ( CScript, @@ -119,20 +117,18 @@ class MiniWallet: def _create_utxo(self, *, txid, vout, value, height, coinbase, confirmations): return {"txid": txid, "vout": vout, "value": value, "height": height, "coinbase": coinbase, "confirmations": confirmations} - def _bulk_tx(self, tx, target_weight): - """Pad a transaction with extra outputs until it reaches a target weight (or higher). + def _bulk_tx(self, tx, target_vsize): + """Pad a transaction with extra outputs until it reaches a target vsize. returns the tx """ tx.vout.append(CTxOut(nValue=0, scriptPubKey=CScript([OP_RETURN]))) - # determine number of needed padding bytes by converting weight difference to vbytes - dummy_vbytes = (target_weight - tx.get_weight() + 3) // 4 + # determine number of needed padding bytes + dummy_vbytes = target_vsize - tx.get_vsize() # compensate for the increase of the compact-size encoded script length # (note that the length encoding of the unpadded output script needs one byte) dummy_vbytes -= len(ser_compact_size(dummy_vbytes)) - 1 tx.vout[-1].scriptPubKey = CScript([OP_RETURN] + [OP_1] * dummy_vbytes) - # Actual weight should be at most 3 higher than target weight - assert_greater_than_or_equal(tx.get_weight(), target_weight) - assert_greater_than_or_equal(target_weight + 3, tx.get_weight()) + assert_equal(tx.get_vsize(), target_vsize) def get_balance(self): return sum(u['value'] for u in self._utxos) @@ -309,7 +305,7 @@ class MiniWallet: locktime=0, sequence=0, fee_per_output=1000, - target_weight=0, + target_vsize=0, confirmed_only=False, ): """ @@ -338,8 +334,8 @@ class MiniWallet: self.sign_tx(tx) - if target_weight: - self._bulk_tx(tx, target_weight) + if target_vsize: + self._bulk_tx(tx, target_vsize) txid = tx.rehash() return { @@ -364,7 +360,7 @@ class MiniWallet: fee_rate=Decimal("0.003"), fee=Decimal("0"), utxo_to_spend=None, - target_weight=0, + target_vsize=0, confirmed_only=False, **kwargs, ): @@ -379,20 +375,18 @@ class MiniWallet: vsize = Decimal(168) # P2PK (73 bytes scriptSig + 35 bytes scriptPubKey + 60 bytes other) else: assert False - if target_weight and not fee: # respect fee_rate if target weight is passed - # the actual weight might be off by 3 WUs, so calculate based on that (see self._bulk_tx) - max_actual_weight = target_weight + 3 - fee = get_fee(math.ceil(max_actual_weight / WITNESS_SCALE_FACTOR), fee_rate) + if target_vsize and not fee: # respect fee_rate if target vsize is passed + fee = get_fee(target_vsize, fee_rate) send_value = utxo_to_spend["value"] - (fee or (fee_rate * vsize / 1000)) # create tx tx = self.create_self_transfer_multi( utxos_to_spend=[utxo_to_spend], amount_per_output=int(COIN * send_value), - target_weight=target_weight, + target_vsize=target_vsize, **kwargs, ) - if not target_weight: + if not target_vsize: assert_equal(tx["tx"].get_vsize(), vsize) tx["new_utxo"] = tx.pop("new_utxos")[0] diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 41390cf26a..3d8c230066 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -96,10 +96,13 @@ BASE_SCRIPTS = [ 'feature_fee_estimation.py', 'feature_taproot.py', 'feature_block.py', + 'p2p_node_network_limited.py --v1transport', + 'p2p_node_network_limited.py --v2transport', # vv Tests less than 2m vv 'mining_getblocktemplate_longpoll.py', 'p2p_segwit.py', 'feature_maxuploadtarget.py', + 'feature_assumeutxo.py', 'mempool_updatefromblock.py', 'mempool_persist.py --descriptors', # vv Tests less than 60s vv @@ -157,6 +160,7 @@ BASE_SCRIPTS = [ 'wallet_importmulti.py --legacy-wallet', 'mempool_limit.py', 'rpc_txoutproof.py', + 'rpc_getorphantxs.py', 'wallet_listreceivedby.py --legacy-wallet', 'wallet_listreceivedby.py --descriptors', 'wallet_abandonconflict.py --legacy-wallet', @@ -354,7 +358,6 @@ BASE_SCRIPTS = [ 'wallet_coinbase_category.py --descriptors', 'feature_filelock.py', 'feature_loadblock.py', - 'feature_assumeutxo.py', 'wallet_assumeutxo.py --descriptors', 'p2p_dos_header_tree.py', 'p2p_add_connections.py', @@ -385,8 +388,6 @@ BASE_SCRIPTS = [ 'feature_coinstatsindex.py', 'wallet_orphanedreward.py', 'wallet_timelock.py', - 'p2p_node_network_limited.py --v1transport', - 'p2p_node_network_limited.py --v2transport', 'p2p_permissions.py', 'feature_blocksdir.py', 'wallet_startup.py', @@ -446,8 +447,8 @@ def main(): help="Leave bitcoinds and test.* datadir on exit or error") parser.add_argument('--resultsfile', '-r', help='store test results (as CSV) to the provided file') - args, unknown_args = parser.parse_known_args() + fail_on_warn = args.ci if not args.ansi: global DEFAULT, BOLD, GREEN, RED DEFAULT = ("", "") @@ -524,8 +525,12 @@ def main(): # Remove the test cases that the user has explicitly asked to exclude. # The user can specify a test case with or without the .py extension. if args.exclude: + def print_warning_missing_test(test_name): - print("{}WARNING!{} Test '{}' not found in current test list.".format(BOLD[1], BOLD[0], test_name)) + print("{}WARNING!{} Test '{}' not found in current test list. Check the --exclude list.".format(BOLD[1], BOLD[0], test_name)) + if fail_on_warn: + sys.exit(1) + def remove_tests(exclude_list): if not exclude_list: print_warning_missing_test(exclude_test) @@ -562,7 +567,7 @@ def main(): f"A minimum of {MIN_NO_CLEANUP_SPACE // (1024 * 1024 * 1024)} GB of free space is required.") passon_args.append("--nocleanup") - check_script_list(src_dir=config["environment"]["SRCDIR"], fail_on_warn=args.ci) + check_script_list(src_dir=config["environment"]["SRCDIR"], fail_on_warn=fail_on_warn) check_script_prefixes() if not args.keepcache: @@ -871,7 +876,6 @@ def check_script_list(*, src_dir, fail_on_warn): if len(missed_tests) != 0: print("%sWARNING!%s The following scripts are not being run: %s. Check the test lists in test_runner.py." % (BOLD[1], BOLD[0], str(missed_tests))) if fail_on_warn: - # On CI this warning is an error to prevent merging incomplete commits into master sys.exit(1) diff --git a/test/functional/tool_wallet.py b/test/functional/tool_wallet.py index 784a769882..86292c0f0b 100755 --- a/test/functional/tool_wallet.py +++ b/test/functional/tool_wallet.py @@ -203,7 +203,7 @@ class ToolWalletTest(BitcoinTestFramework): locked_dir = self.nodes[0].wallets_path error = 'Error initializing wallet database environment "{}"!'.format(locked_dir) if self.options.descriptors: - error = f"SQLiteDatabase: Unable to obtain an exclusive lock on the database, is it being used by another instance of {self.config['environment']['PACKAGE_NAME']}?" + error = f"SQLiteDatabase: Unable to obtain an exclusive lock on the database, is it being used by another instance of {self.config['environment']['CLIENT_NAME']}?" self.assert_raises_tool_error( error, '-wallet=' + self.default_wallet_name, diff --git a/test/functional/wallet_assumeutxo.py b/test/functional/wallet_assumeutxo.py index a5025a64e7..76cd2097a3 100755 --- a/test/functional/wallet_assumeutxo.py +++ b/test/functional/wallet_assumeutxo.py @@ -11,7 +11,9 @@ See feature_assumeutxo.py for background. - TODO: test loading a wallet (backup) on a pruned node """ +from test_framework.address import address_to_scriptpubkey from test_framework.test_framework import BitcoinTestFramework +from test_framework.messages import COIN from test_framework.util import ( assert_equal, assert_raises_rpc_error, @@ -62,8 +64,16 @@ class AssumeutxoTest(BitcoinTestFramework): for n in self.nodes: n.setmocktime(n.getblockheader(n.getbestblockhash())['time']) + # Create a wallet that we will create a backup for later (at snapshot height) n0.createwallet('w') w = n0.get_wallet_rpc("w") + w_address = w.getnewaddress() + + # Create another wallet and backup now (before snapshot height) + n0.createwallet('w2') + w2 = n0.get_wallet_rpc("w2") + w2_address = w2.getnewaddress() + w2.backupwallet("backup_w2.dat") # Generate a series of blocks that `n0` will have in the snapshot, # but that n1 doesn't yet see. In order for the snapshot to activate, @@ -84,6 +94,8 @@ class AssumeutxoTest(BitcoinTestFramework): assert_equal(n.getblockchaininfo()[ "headers"], SNAPSHOT_BASE_HEIGHT) + # This backup is created at the snapshot height, so it's + # not part of the background sync anymore w.backupwallet("backup_w.dat") self.log.info("-- Testing assumeutxo") @@ -103,7 +115,13 @@ class AssumeutxoTest(BitcoinTestFramework): # Mine more blocks on top of the snapshot that n1 hasn't yet seen. This # will allow us to test n1's sync-to-tip on top of a snapshot. - self.generate(n0, nblocks=100, sync_fun=self.no_op) + w_skp = address_to_scriptpubkey(w_address) + w2_skp = address_to_scriptpubkey(w2_address) + for i in range(100): + if i % 3 == 0: + self.mini_wallet.send_to(from_node=n0, scriptPubKey=w_skp, amount=1 * COIN) + self.mini_wallet.send_to(from_node=n0, scriptPubKey=w2_skp, amount=10 * COIN) + self.generate(n0, nblocks=1, sync_fun=self.no_op) assert_equal(n0.getblockcount(), FINAL_HEIGHT) assert_equal(n1.getblockcount(), START_HEIGHT) @@ -126,8 +144,13 @@ class AssumeutxoTest(BitcoinTestFramework): assert_equal(n1.getblockchaininfo()["blocks"], SNAPSHOT_BASE_HEIGHT) - self.log.info("Backup can't be loaded during background sync") - assert_raises_rpc_error(-4, "Wallet loading failed. Error loading wallet. Wallet requires blocks to be downloaded, and software does not currently support loading wallets while blocks are being downloaded out of order when using assumeutxo snapshots. Wallet should be able to load successfully after node sync reaches height 299", n1.restorewallet, "w", "backup_w.dat") + self.log.info("Backup from the snapshot height can be loaded during background sync") + n1.restorewallet("w", "backup_w.dat") + # Balance of w wallet is still still 0 because n1 has not synced yet + assert_equal(n1.getbalance(), 0) + + self.log.info("Backup from before the snapshot height can't be loaded during background sync") + assert_raises_rpc_error(-4, "Wallet loading failed. Error loading wallet. Wallet requires blocks to be downloaded, and software does not currently support loading wallets while blocks are being downloaded out of order when using assumeutxo snapshots. Wallet should be able to load successfully after node sync reaches height 299", n1.restorewallet, "w2", "backup_w2.dat") PAUSE_HEIGHT = FINAL_HEIGHT - 40 @@ -159,8 +182,15 @@ class AssumeutxoTest(BitcoinTestFramework): self.log.info("Ensuring background validation completes") self.wait_until(lambda: len(n1.getchainstates()['chainstates']) == 1) - self.log.info("Ensuring wallet can be restored from backup") - n1.restorewallet("w", "backup_w.dat") + self.log.info("Ensuring wallet can be restored from a backup that was created before the snapshot height") + n1.restorewallet("w2", "backup_w2.dat") + # Check balance of w2 wallet + assert_equal(n1.getbalance(), 340) + + # Check balance of w wallet after node is synced + n1.loadwallet("w") + w = n1.get_wallet_rpc("w") + assert_equal(w.getbalance(), 34) if __name__ == '__main__': diff --git a/test/functional/wallet_backup.py b/test/functional/wallet_backup.py index a639c34377..83267f77e1 100755 --- a/test/functional/wallet_backup.py +++ b/test/functional/wallet_backup.py @@ -140,6 +140,25 @@ class WalletBackupTest(BitcoinTestFramework): assert_raises_rpc_error(-36, error_message, node.restorewallet, wallet_name, backup_file) assert wallet_file.exists() + def test_pruned_wallet_backup(self): + self.log.info("Test loading backup on a pruned node when the backup was created close to the prune height of the restoring node") + node = self.nodes[3] + self.restart_node(3, ["-prune=1", "-fastprune=1"]) + # Ensure the chain tip is at height 214, because this test assume it is. + assert_equal(node.getchaintips()[0]["height"], 214) + # We need a few more blocks so we can actually get above an realistic + # minimal prune height + self.generate(node, 50, sync_fun=self.no_op) + # Backup created at block height 264 + node.backupwallet(node.datadir_path / 'wallet_pruned.bak') + # Generate more blocks so we can actually prune the older blocks + self.generate(node, 300, sync_fun=self.no_op) + # This gives us an actual prune height roughly in the range of 220 - 240 + node.pruneblockchain(250) + # The backup should be updated with the latest height (locator) for + # the backup to load successfully this close to the prune height + node.restorewallet(f'pruned', node.datadir_path / 'wallet_pruned.bak') + def run_test(self): self.log.info("Generating initial blockchain") self.generate(self.nodes[0], 1) @@ -242,6 +261,8 @@ class WalletBackupTest(BitcoinTestFramework): for sourcePath in sourcePaths: assert_raises_rpc_error(-4, "backup failed", self.nodes[0].backupwallet, sourcePath) + self.test_pruned_wallet_backup() + if __name__ == '__main__': WalletBackupTest(__file__).main() diff --git a/test/functional/wallet_backwards_compatibility.py b/test/functional/wallet_backwards_compatibility.py index e71283b928..775786fbb1 100755 --- a/test/functional/wallet_backwards_compatibility.py +++ b/test/functional/wallet_backwards_compatibility.py @@ -33,7 +33,7 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True - self.num_nodes = 12 + self.num_nodes = 11 # Add new version after each release: self.extra_args = [ ["-addresstype=bech32", "-whitelist=noban@127.0.0.1"], # Pre-release: use to mine blocks. noban for immediate tx relay @@ -47,7 +47,6 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): ["-nowallet", "-walletrbf=1", "-addresstype=bech32", "-whitelist=noban@127.0.0.1"], # v0.19.1 ["-nowallet", "-walletrbf=1", "-addresstype=bech32", "-whitelist=127.0.0.1"], # v0.18.1 ["-nowallet", "-walletrbf=1", "-addresstype=bech32", "-whitelist=127.0.0.1"], # v0.17.2 - ["-nowallet", "-walletrbf=1", "-addresstype=bech32", "-whitelist=127.0.0.1", "-wallet=wallet.dat"], # v0.16.3 ] self.wallet_names = [self.default_wallet_name] @@ -68,7 +67,6 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): 190100, 180100, 170200, - 160300, ]) self.start_nodes() @@ -133,18 +131,17 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): def run_test(self): node_miner = self.nodes[0] node_master = self.nodes[1] - node_v21 = self.nodes[self.num_nodes - 6] - node_v17 = self.nodes[self.num_nodes - 2] - node_v16 = self.nodes[self.num_nodes - 1] + node_v21 = self.nodes[self.num_nodes - 5] + node_v17 = self.nodes[self.num_nodes - 1] legacy_nodes = self.nodes[2:] # Nodes that support legacy wallets - legacy_only_nodes = self.nodes[-5:] # Nodes that only support legacy wallets - descriptors_nodes = self.nodes[2:-5] # Nodes that support descriptor wallets + legacy_only_nodes = self.nodes[-4:] # Nodes that only support legacy wallets + descriptors_nodes = self.nodes[2:-4] # Nodes that support descriptor wallets self.generatetoaddress(node_miner, COINBASE_MATURITY + 1, node_miner.getnewaddress()) # Sanity check the test framework: - res = node_v16.getblockchaininfo() + res = node_v17.getblockchaininfo() assert_equal(res['blocks'], COINBASE_MATURITY + 1) self.log.info("Test wallet backwards compatibility...") @@ -215,9 +212,6 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): # In descriptors wallet mode, run this test on the nodes that support descriptor wallets # In legacy wallets mode, run this test on the nodes that support legacy wallets for node in descriptors_nodes if self.options.descriptors else legacy_nodes: - if self.major_version_less_than(node, 17): - # loadwallet was introduced in v0.17.0 - continue self.log.info(f"- {node.version}") for wallet_name in ["w1", "w2", "w3"]: if self.major_version_less_than(node, 18) and wallet_name == "w3": @@ -290,15 +284,6 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): node_v17.assert_start_raises_init_error(["-wallet=w3"], "Error: Error loading w3: Wallet requires newer version of Bitcoin Core") self.start_node(node_v17.index) - # No wallet created in master can be opened in 0.16 - self.log.info("Test that wallets created in master are too new for 0.16") - self.stop_node(node_v16.index) - for wallet_name in ["w1", "w2", "w3"]: - if self.options.descriptors: - node_v16.assert_start_raises_init_error([f"-wallet={wallet_name}"], f"Error: {wallet_name} corrupt, salvage failed") - else: - node_v16.assert_start_raises_init_error([f"-wallet={wallet_name}"], f"Error: Error loading {wallet_name}: Wallet requires newer version of Bitcoin Core") - # When descriptors are enabled, w1 cannot be opened by 0.21 since it contains a taproot descriptor if self.options.descriptors: self.log.info("Test that 0.21 cannot open wallet containing tr() descriptors") diff --git a/test/functional/wallet_multiwallet.py b/test/functional/wallet_multiwallet.py index 149b1246d8..9a339088fc 100755 --- a/test/functional/wallet_multiwallet.py +++ b/test/functional/wallet_multiwallet.py @@ -204,7 +204,7 @@ class MultiWalletTest(BitcoinTestFramework): self.restart_node(0, ['-nowallet', '-walletdir=' + competing_wallet_dir]) self.nodes[0].createwallet(self.default_wallet_name) if self.options.descriptors: - exp_stderr = f"Error: SQLiteDatabase: Unable to obtain an exclusive lock on the database, is it being used by another instance of {self.config['environment']['PACKAGE_NAME']}?" + exp_stderr = f"Error: SQLiteDatabase: Unable to obtain an exclusive lock on the database, is it being used by another instance of {self.config['environment']['CLIENT_NAME']}?" else: exp_stderr = r"Error: Error initializing wallet database environment \"\S+competing_walletdir\S*\"!" self.nodes[1].assert_start_raises_init_error(['-walletdir=' + competing_wallet_dir], exp_stderr, match=ErrorMatch.PARTIAL_REGEX) diff --git a/test/functional/wallet_upgradewallet.py b/test/functional/wallet_upgradewallet.py index ef3f925ee8..c909336a25 100755 --- a/test/functional/wallet_upgradewallet.py +++ b/test/functional/wallet_upgradewallet.py @@ -185,6 +185,7 @@ class UpgradeWalletTest(BitcoinTestFramework): self.restart_node(0) copy_v16() wallet = node_master.get_wallet_rpc(self.default_wallet_name) + assert_equal(wallet.getbalance(), v16_3_balance) self.log.info("Test upgradewallet without a version argument") self.test_upgradewallet(wallet, previous_version=159900, expected_version=169900) # wallet should still contain the same balance diff --git a/test/lint/check-doc.py b/test/lint/check-doc.py index f55d0f8cb7..3e9e5ba230 100755 --- a/test/lint/check-doc.py +++ b/test/lint/check-doc.py @@ -23,7 +23,7 @@ CMD_GREP_WALLET_ARGS = r"git grep --function-context 'void WalletInit::AddWallet CMD_GREP_WALLET_HIDDEN_ARGS = r"git grep --function-context 'void DummyWalletInit::AddWalletOptions' -- {}".format(CMD_ROOT_DIR) CMD_GREP_DOCS = r"git grep --perl-regexp '{}' {}".format(REGEX_DOC, CMD_ROOT_DIR) # list unsupported, deprecated and duplicate args as they need no documentation -SET_DOC_OPTIONAL = set(['-h', '-help', '-dbcrashratio', '-forcecompactdb']) +SET_DOC_OPTIONAL = set(['-h', '-?', '-dbcrashratio', '-forcecompactdb']) def lint_missing_argument_documentation(): diff --git a/test/lint/commit-script-check.sh b/test/lint/commit-script-check.sh index fe845ed19e..52ae95fbb6 100755 --- a/test/lint/commit-script-check.sh +++ b/test/lint/commit-script-check.sh @@ -35,20 +35,20 @@ for commit in $(git rev-list --reverse "$1"); do git checkout --quiet "$commit"^ || exit SCRIPT="$(git rev-list --format=%b -n1 "$commit" | sed '/^-BEGIN VERIFY SCRIPT-$/,/^-END VERIFY SCRIPT-$/{//!b};d')" if test -z "$SCRIPT"; then - echo "Error: missing script for: $commit" - echo "Failed" + echo "Error: missing script for: $commit" >&2 + echo "Failed" >&2 RET=1 else - echo "Running script for: $commit" - echo "$SCRIPT" + echo "Running script for: $commit" >&2 + echo "$SCRIPT" >&2 (eval "$SCRIPT") - git --no-pager diff --exit-code "$commit" && echo "OK" || (echo "Failed"; false) || RET=1 + git --no-pager diff --exit-code "$commit" && echo "OK" >&2 || (echo "Failed" >&2; false) || RET=1 fi git reset --quiet --hard HEAD else if git rev-list "--format=%b" -n1 "$commit" | grep -q '^-\(BEGIN\|END\)[ a-zA-Z]*-$'; then - echo "Error: script block marker but no scripted-diff in title of commit $commit" - echo "Failed" + echo "Error: script block marker but no scripted-diff in title of commit $commit" >&2 + echo "Failed" >&2 RET=1 fi fi diff --git a/test/lint/lint-format-strings.py b/test/lint/lint-format-strings.py index a809851ec6..86a17fb0f8 100755 --- a/test/lint/lint-format-strings.py +++ b/test/lint/lint-format-strings.py @@ -62,7 +62,7 @@ def main(): matching_files_filtered = [] for matching_file in matching_files: - if not re.search('^src/(leveldb|secp256k1|minisketch|tinyformat|test/fuzz/strprintf.cpp)|contrib/devtools/bitcoin-tidy/example_logprintf.cpp', matching_file): + if not re.search('^src/(leveldb|secp256k1|minisketch|tinyformat|test/fuzz/strprintf.cpp)', matching_file): matching_files_filtered.append(matching_file) matching_files_filtered.sort() diff --git a/test/lint/test_runner/src/main.rs b/test/lint/test_runner/src/main.rs index 64a1486a4c..42c880052e 100644 --- a/test/lint/test_runner/src/main.rs +++ b/test/lint/test_runner/src/main.rs @@ -29,7 +29,7 @@ fn get_linter_list() -> Vec<&'static Linter> { lint_fn: lint_doc }, &Linter { - description: "Check that no symbol from bitcoin-config.h is used without the header being included", + description: "Check that no symbol from bitcoin-build-config.h is used without the header being included", name: "includes_build_config", lint_fn: lint_includes_build_config }, @@ -395,7 +395,7 @@ Please add any false positives, such as subtrees, or externally sourced files to } fn lint_includes_build_config() -> LintResult { - let config_path = "./cmake/bitcoin-config.h.in"; + let config_path = "./cmake/bitcoin-build-config.h.in"; let defines_regex = format!( r"^\s*(?!//).*({})", check_output(Command::new("grep").args(["define", "--", config_path])) @@ -429,7 +429,7 @@ fn lint_includes_build_config() -> LintResult { ]) .args(get_pathspecs_exclude_subtrees()) .args([ - // These are exceptions which don't use bitcoin-config.h, rather the Makefile.am adds + // These are exceptions which don't use bitcoin-build-config.h, rather CMakeLists.txt adds // these cppflags manually. ":(exclude)src/crypto/sha256_arm_shani.cpp", ":(exclude)src/crypto/sha256_avx2.cpp", @@ -447,9 +447,9 @@ fn lint_includes_build_config() -> LintResult { "--files-with-matches" }, if mode { - "^#include <config/bitcoin-config.h> // IWYU pragma: keep$" + "^#include <bitcoin-build-config.h> // IWYU pragma: keep$" } else { - "#include <config/bitcoin-config.h>" // Catch redundant includes with and without the IWYU pragma + "#include <bitcoin-build-config.h>" // Catch redundant includes with and without the IWYU pragma }, "--", ]) @@ -463,11 +463,11 @@ fn lint_includes_build_config() -> LintResult { return Err(format!( r#" ^^^ -One or more files use a symbol declared in the bitcoin-config.h header. However, they are not +One or more files use a symbol declared in the bitcoin-build-config.h header. However, they are not including the header. This is problematic, because the header may or may not be indirectly included. If the indirect include were to be intentionally or accidentally removed, the build could still succeed, but silently be buggy. For example, a slower fallback algorithm could be picked, -even though bitcoin-config.h indicates that a faster feature is available and should be used. +even though bitcoin-build-config.h indicates that a faster feature is available and should be used. If you are unsure which symbol is used, you can find it with this command: git grep --perl-regexp '{}' -- file_name @@ -475,7 +475,7 @@ git grep --perl-regexp '{}' -- file_name Make sure to include it with the IWYU pragma. Otherwise, IWYU may falsely instruct to remove the include again. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep "#, defines_regex )); @@ -484,7 +484,7 @@ include again. if redundant { return Err(r#" ^^^ -None of the files use a symbol declared in the bitcoin-config.h header. However, they are including +None of the files use a symbol declared in the bitcoin-build-config.h header. However, they are including the header. Consider removing the unused include. "# .to_string()); diff --git a/test/util/test_runner.py b/test/util/test_runner.py index e4a77d53d6..cac184ca30 100755 --- a/test/util/test_runner.py +++ b/test/util/test_runner.py @@ -83,13 +83,11 @@ def bctest(testDir, testObj, buildenv): execrun = [execprog] + execargs # Read the input data (if there is any) - stdinCfg = None inputData = None if "input" in testObj: filename = os.path.join(testDir, testObj["input"]) with open(filename, encoding="utf8") as f: inputData = f.read() - stdinCfg = subprocess.PIPE # Read the expected output data (if there is any) outputFn = None @@ -112,9 +110,8 @@ def bctest(testDir, testObj, buildenv): raise Exception # Run the test - proc = subprocess.Popen(execrun, stdin=stdinCfg, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) try: - outs = proc.communicate(input=inputData) + res = subprocess.run(execrun, capture_output=True, text=True, input=inputData) except OSError: logging.error("OSError, Failed to execute " + execprog) raise @@ -123,9 +120,9 @@ def bctest(testDir, testObj, buildenv): data_mismatch, formatting_mismatch = False, False # Parse command output and expected output try: - a_parsed = parse_output(outs[0], outputType) + a_parsed = parse_output(res.stdout, outputType) except Exception as e: - logging.error('Error parsing command output as %s: %s' % (outputType, e)) + logging.error(f"Error parsing command output as {outputType}: '{str(e)}'; res: {str(res)}") raise try: b_parsed = parse_output(outputData, outputType) @@ -134,13 +131,13 @@ def bctest(testDir, testObj, buildenv): raise # Compare data if a_parsed != b_parsed: - logging.error("Output data mismatch for " + outputFn + " (format " + outputType + ")") + logging.error(f"Output data mismatch for {outputFn} (format {outputType}); res: {str(res)}") data_mismatch = True # Compare formatting - if outs[0] != outputData: - error_message = "Output formatting mismatch for " + outputFn + ":\n" + if res.stdout != outputData: + error_message = f"Output formatting mismatch for {outputFn}:\nres: {str(res)}\n" error_message += "".join(difflib.context_diff(outputData.splitlines(True), - outs[0].splitlines(True), + res.stdout.splitlines(True), fromfile=outputFn, tofile="returned")) logging.error(error_message) @@ -152,8 +149,8 @@ def bctest(testDir, testObj, buildenv): wantRC = 0 if "return_code" in testObj: wantRC = testObj['return_code'] - if proc.returncode != wantRC: - logging.error("Return code mismatch for " + outputFn) + if res.returncode != wantRC: + logging.error(f"Return code mismatch for {outputFn}; res: {str(res)}") raise Exception if "error_txt" in testObj: @@ -164,8 +161,8 @@ def bctest(testDir, testObj, buildenv): # emits DISPLAY errors when running as a windows application on # linux through wine. Just assert that the expected error text appears # somewhere in stderr. - if want_error not in outs[1]: - logging.error("Error mismatch:\n" + "Expected: " + want_error + "\nReceived: " + outs[1].rstrip()) + if want_error not in res.stderr: + logging.error(f"Error mismatch:\nExpected: {want_error}\nReceived: {res.stderr.rstrip()}\nres: {str(res)}") raise Exception def parse_output(a, fmt): |