diff options
Diffstat (limited to 'test')
53 files changed, 929 insertions, 390 deletions
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 9fd4e6e84e..3a5998697d 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -27,7 +27,7 @@ function(create_test_config) set_configure_variable(ENABLE_EXTERNAL_SIGNER ENABLE_EXTERNAL_SIGNER) set_configure_variable(WITH_USDT ENABLE_USDT_TRACEPOINTS) - configure_file(config.ini.in config.ini @ONLY) + configure_file(config.ini.in config.ini USE_SOURCE_PERMISSIONS @ONLY) endfunction() create_test_config() diff --git a/test/functional/README.md b/test/functional/README.md index a4994f2e7c..a34bf1827c 100644 --- a/test/functional/README.md +++ b/test/functional/README.md @@ -10,7 +10,8 @@ that file and modify to fit your needs. #### Coverage -Running `test/functional/test_runner.py` with the `--coverage` argument tracks which RPCs are +Assuming the build directory is `build`, +running `build/test/functional/test_runner.py` with the `--coverage` argument tracks which RPCs are called by the tests and prints a report of uncovered RPCs in the summary. This can be used (along with the `--extended` argument) to find out which RPCs we don't have test cases for. diff --git a/test/functional/data/invalid_txs.py b/test/functional/data/invalid_txs.py index 2e4ca83bf0..d2d7202d86 100644 --- a/test/functional/data/invalid_txs.py +++ b/test/functional/data/invalid_txs.py @@ -263,6 +263,17 @@ def getDisabledOpcodeTemplate(opcode): 'valid_in_block' : True }) +class NonStandardAndInvalid(BadTxTemplate): + """A non-standard transaction which is also consensus-invalid should return the consensus error.""" + reject_reason = "mandatory-script-verify-flag-failed (OP_RETURN was encountered)" + expect_disconnect = True + valid_in_block = False + + def get_tx(self): + return create_tx_with_script( + self.spend_tx, 0, script_sig=b'\x00' * 3 + b'\xab\x6a', + amount=(self.spend_avail // 2)) + # Disabled opcode tx templates (CVE-2010-5137) DisabledOpcodeTemplates = [getDisabledOpcodeTemplate(opcode) for opcode in [ OP_CAT, diff --git a/test/functional/feature_assumeutxo.py b/test/functional/feature_assumeutxo.py index a212704311..2995ece42f 100755 --- a/test/functional/feature_assumeutxo.py +++ b/test/functional/feature_assumeutxo.py @@ -9,6 +9,7 @@ to a hash that has been compiled into bitcoind. The assumeutxo value generated and used here is committed to in `CRegTestParams::m_assumeutxo_data` in `src/kernel/chainparams.cpp`. """ +import time from shutil import rmtree from dataclasses import dataclass @@ -16,12 +17,22 @@ from test_framework.blocktools import ( create_block, create_coinbase ) -from test_framework.messages import tx_from_hex +from test_framework.messages import ( + CBlockHeader, + from_hex, + msg_headers, + tx_from_hex +) +from test_framework.p2p import ( + P2PInterface, +) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_approx, assert_equal, assert_raises_rpc_error, + sha256sum_file, + try_rpc, ) from test_framework.wallet import ( getnewdestination, @@ -247,6 +258,74 @@ class AssumeutxoTest(BitcoinTestFramework): node1.submitheader(main_block1) node1.submitheader(main_block2) + def test_sync_from_assumeutxo_node(self, snapshot): + """ + This test verifies that: + 1. An IBD node can sync headers from an AssumeUTXO node at any time. + 2. IBD nodes do not request historical blocks from AssumeUTXO nodes while they are syncing the background-chain. + 3. The assumeUTXO node dynamically adjusts the network services it offers according to its state. + 4. IBD nodes can fully sync from AssumeUTXO nodes after they finish the background-chain sync. + """ + self.log.info("Testing IBD-sync from assumeUTXO node") + # Node2 starts clean and loads the snapshot. + # Node3 starts clean and seeks to sync-up from snapshot_node. + miner = self.nodes[0] + snapshot_node = self.nodes[2] + ibd_node = self.nodes[3] + + # Start test fresh by cleaning up node directories + for node in (snapshot_node, ibd_node): + self.stop_node(node.index) + rmtree(node.chain_path) + self.start_node(node.index, extra_args=self.extra_args[node.index]) + + # Sync-up headers chain on snapshot_node to load snapshot + headers_provider_conn = snapshot_node.add_p2p_connection(P2PInterface()) + headers_provider_conn.wait_for_getheaders() + msg = msg_headers() + for block_num in range(1, miner.getblockcount()+1): + msg.headers.append(from_hex(CBlockHeader(), miner.getblockheader(miner.getblockhash(block_num), verbose=False))) + headers_provider_conn.send_message(msg) + + # Ensure headers arrived + default_value = {'status': ''} # No status + headers_tip_hash = miner.getbestblockhash() + self.wait_until(lambda: next(filter(lambda x: x['hash'] == headers_tip_hash, snapshot_node.getchaintips()), default_value)['status'] == "headers-only") + snapshot_node.disconnect_p2ps() + + # Load snapshot + snapshot_node.loadtxoutset(snapshot['path']) + + # Connect nodes and verify the ibd_node can sync-up the headers-chain from the snapshot_node + self.connect_nodes(ibd_node.index, snapshot_node.index) + snapshot_block_hash = snapshot['base_hash'] + self.wait_until(lambda: next(filter(lambda x: x['hash'] == snapshot_block_hash, ibd_node.getchaintips()), default_value)['status'] == "headers-only") + + # Once the headers-chain is synced, the ibd_node must avoid requesting historical blocks from the snapshot_node. + # If it does request such blocks, the snapshot_node will ignore requests it cannot fulfill, causing the ibd_node + # to stall. This stall could last for up to 10 min, ultimately resulting in an abrupt disconnection due to the + # ibd_node's perceived unresponsiveness. + time.sleep(3) # Sleep here because we can't detect when a node avoids requesting blocks from other peer. + assert_equal(len(ibd_node.getpeerinfo()[0]['inflight']), 0) + + # Now disconnect nodes and finish background chain sync + self.disconnect_nodes(ibd_node.index, snapshot_node.index) + self.connect_nodes(snapshot_node.index, miner.index) + self.sync_blocks(nodes=(miner, snapshot_node)) + # Check the base snapshot block was stored and ensure node signals full-node service support + self.wait_until(lambda: not try_rpc(-1, "Block not available (not fully downloaded)", snapshot_node.getblock, snapshot_block_hash)) + self.wait_until(lambda: 'NETWORK' in snapshot_node.getnetworkinfo()['localservicesnames']) + + # Now that the snapshot_node is synced, verify the ibd_node can sync from it + self.connect_nodes(snapshot_node.index, ibd_node.index) + assert 'NETWORK' in ibd_node.getpeerinfo()[0]['servicesnames'] + self.sync_blocks(nodes=(ibd_node, snapshot_node)) + + def assert_only_network_limited_service(self, node): + node_services = node.getnetworkinfo()['localservicesnames'] + assert 'NETWORK' not in node_services + assert 'NETWORK_LIMITED' in node_services + def run_test(self): """ Bring up two (disconnected) nodes, mine some new blocks on the first, @@ -295,7 +374,7 @@ class AssumeutxoTest(BitcoinTestFramework): assert_equal(n1.getblockcount(), START_HEIGHT) self.log.info(f"Creating a UTXO snapshot at height {SNAPSHOT_BASE_HEIGHT}") - dump_output = n0.dumptxoutset('utxos.dat') + dump_output = n0.dumptxoutset('utxos.dat', "latest") self.log.info("Test loading snapshot when the node tip is on the same block as the snapshot") assert_equal(n0.getblockcount(), SNAPSHOT_BASE_HEIGHT) @@ -320,12 +399,16 @@ class AssumeutxoTest(BitcoinTestFramework): for n in self.nodes: assert_equal(n.getblockchaininfo()["headers"], SNAPSHOT_BASE_HEIGHT) - assert_equal( - dump_output['txoutset_hash'], - "a4bf3407ccb2cc0145c49ebba8fa91199f8a3903daf0883875941497d2493c27") - assert_equal(dump_output["nchaintx"], blocks[SNAPSHOT_BASE_HEIGHT].chain_tx) assert_equal(n0.getblockchaininfo()["blocks"], SNAPSHOT_BASE_HEIGHT) + def check_dump_output(output): + assert_equal( + output['txoutset_hash'], + "a4bf3407ccb2cc0145c49ebba8fa91199f8a3903daf0883875941497d2493c27") + assert_equal(output["nchaintx"], blocks[SNAPSHOT_BASE_HEIGHT].chain_tx) + + check_dump_output(dump_output) + # Mine more blocks on top of the snapshot that n1 hasn't yet seen. This # will allow us to test n1's sync-to-tip on top of a snapshot. self.generate(n0, nblocks=100, sync_fun=self.no_op) @@ -335,6 +418,39 @@ class AssumeutxoTest(BitcoinTestFramework): assert_equal(n0.getblockchaininfo()["blocks"], FINAL_HEIGHT) + self.log.info(f"Check that dumptxoutset works for past block heights") + # rollback defaults to the snapshot base height + dump_output2 = n0.dumptxoutset('utxos2.dat', "rollback") + check_dump_output(dump_output2) + assert_equal(sha256sum_file(dump_output['path']), sha256sum_file(dump_output2['path'])) + + # Rollback with specific height + dump_output3 = n0.dumptxoutset('utxos3.dat', rollback=SNAPSHOT_BASE_HEIGHT) + check_dump_output(dump_output3) + assert_equal(sha256sum_file(dump_output['path']), sha256sum_file(dump_output3['path'])) + + # Specified height that is not a snapshot height + prev_snap_height = SNAPSHOT_BASE_HEIGHT - 1 + dump_output4 = n0.dumptxoutset(path='utxos4.dat', rollback=prev_snap_height) + assert_equal( + dump_output4['txoutset_hash'], + "8a1db0d6e958ce0d7c963bc6fc91ead596c027129bacec68acc40351037b09d7") + assert sha256sum_file(dump_output['path']) != sha256sum_file(dump_output4['path']) + + # Use a hash instead of a height + prev_snap_hash = n0.getblockhash(prev_snap_height) + dump_output5 = n0.dumptxoutset('utxos5.dat', rollback=prev_snap_hash) + assert_equal(sha256sum_file(dump_output4['path']), sha256sum_file(dump_output5['path'])) + + # TODO: This is a hack to set m_best_header to the correct value after + # dumptxoutset/reconsiderblock. Otherwise the wrong error messages are + # returned in following tests. It can be removed once this bug is + # fixed. See also https://github.com/bitcoin/bitcoin/issues/26245 + self.restart_node(0, ["-reindex"]) + + # Ensure n0 is back at the tip + assert_equal(n0.getblockchaininfo()["blocks"], FINAL_HEIGHT) + self.test_snapshot_with_less_work(dump_output['path']) self.test_invalid_mempool_state(dump_output['path']) self.test_invalid_snapshot_scenarios(dump_output['path']) @@ -343,6 +459,9 @@ class AssumeutxoTest(BitcoinTestFramework): self.test_snapshot_block_invalidated(dump_output['path']) self.test_snapshot_not_on_most_work_chain(dump_output['path']) + # Prune-node sanity check + assert 'NETWORK' not in n1.getnetworkinfo()['localservicesnames'] + self.log.info(f"Loading snapshot into second node from {dump_output['path']}") # This node's tip is on an ancestor block of the snapshot, which should # be the normal case @@ -350,6 +469,10 @@ class AssumeutxoTest(BitcoinTestFramework): assert_equal(loaded['coins_loaded'], SNAPSHOT_BASE_HEIGHT) assert_equal(loaded['base_height'], SNAPSHOT_BASE_HEIGHT) + self.log.info("Confirm that local services remain unchanged") + # Since n1 is a pruned node, the 'NETWORK' service flag must always be unset. + self.assert_only_network_limited_service(n1) + self.log.info("Check that UTXO-querying RPCs operate on snapshot chainstate") snapshot_hash = loaded['tip_hash'] snapshot_num_coins = loaded['coins_loaded'] @@ -362,7 +485,7 @@ class AssumeutxoTest(BitcoinTestFramework): # find coinbase output at snapshot height on node0 and scan for it on node1, # where the block is not available, but the snapshot was loaded successfully coinbase_tx = n0.getblock(snapshot_hash, verbosity=2)['tx'][0] - assert_raises_rpc_error(-1, "Block not found on disk", n1.getblock, snapshot_hash) + assert_raises_rpc_error(-1, "Block not available (not fully downloaded)", n1.getblock, snapshot_hash) coinbase_output_descriptor = coinbase_tx['vout'][0]['scriptPubKey']['desc'] scan_result = n1.scantxoutset('start', [coinbase_output_descriptor]) assert_equal(scan_result['success'], True) @@ -434,7 +557,7 @@ class AssumeutxoTest(BitcoinTestFramework): self.log.info("Submit a spending transaction for a snapshot chainstate coin to the mempool") # spend the coinbase output of the first block that is not available on node1 spend_coin_blockhash = n1.getblockhash(START_HEIGHT + 1) - assert_raises_rpc_error(-1, "Block not found on disk", n1.getblock, spend_coin_blockhash) + assert_raises_rpc_error(-1, "Block not available (not fully downloaded)", n1.getblock, spend_coin_blockhash) prev_tx = n0.getblock(spend_coin_blockhash, 3)['tx'][0] prevout = {"txid": prev_tx['txid'], "vout": 0, "scriptPubKey": prev_tx['vout'][0]['scriptPubKey']['hex']} privkey = n0.get_deterministic_priv_key().key @@ -453,6 +576,9 @@ class AssumeutxoTest(BitcoinTestFramework): self.restart_node(1, extra_args=[ f"-stopatheight={PAUSE_HEIGHT}", *self.extra_args[1]]) + # Upon restart during snapshot tip sync, the node must remain in 'limited' mode. + self.assert_only_network_limited_service(n1) + # Finally connect the nodes and let them sync. # # Set `wait_for_connect=False` to avoid a race between performing connection @@ -469,6 +595,9 @@ class AssumeutxoTest(BitcoinTestFramework): self.log.info("Restarted node before snapshot validation completed, reloading...") self.restart_node(1, extra_args=self.extra_args[1]) + # Upon restart, the node must remain in 'limited' mode + self.assert_only_network_limited_service(n1) + # Send snapshot block to n1 out of order. This makes the test less # realistic because normally the snapshot block is one of the last # blocks downloaded, but its useful to test because it triggers more @@ -487,6 +616,10 @@ class AssumeutxoTest(BitcoinTestFramework): self.log.info("Ensuring background validation completes") self.wait_until(lambda: len(n1.getchainstates()['chainstates']) == 1) + # Since n1 is a pruned node, it will not signal NODE_NETWORK after + # completing the background sync. + self.assert_only_network_limited_service(n1) + # Ensure indexes have synced. completed_idx_state = { 'basic block filter index': COMPLETE_IDX, @@ -517,12 +650,18 @@ class AssumeutxoTest(BitcoinTestFramework): self.log.info("-- Testing all indexes + reindex") assert_equal(n2.getblockcount(), START_HEIGHT) + assert 'NETWORK' in n2.getnetworkinfo()['localservicesnames'] # sanity check self.log.info(f"Loading snapshot into third node from {dump_output['path']}") loaded = n2.loadtxoutset(dump_output['path']) assert_equal(loaded['coins_loaded'], SNAPSHOT_BASE_HEIGHT) assert_equal(loaded['base_height'], SNAPSHOT_BASE_HEIGHT) + # Even though n2 is a full node, it will unset the 'NETWORK' service flag during snapshot loading. + # This indicates other peers that the node will temporarily not provide historical blocks. + self.log.info("Check node2 updated the local services during snapshot load") + self.assert_only_network_limited_service(n2) + for reindex_arg in ['-reindex=1', '-reindex-chainstate=1']: self.log.info(f"Check that restarting with {reindex_arg} will delete the snapshot chainstate") self.restart_node(2, extra_args=[reindex_arg, *self.extra_args[2]]) @@ -546,6 +685,11 @@ class AssumeutxoTest(BitcoinTestFramework): msg = "Unable to load UTXO snapshot: Can't activate a snapshot-based chainstate more than once" assert_raises_rpc_error(-32603, msg, n2.loadtxoutset, dump_output['path']) + # Upon restart, the node must stay in 'limited' mode until the background + # chain sync completes. + self.restart_node(2, extra_args=self.extra_args[2]) + self.assert_only_network_limited_service(n2) + self.connect_nodes(0, 2) self.wait_until(lambda: n2.getchainstates()['chainstates'][-1]['blocks'] == FINAL_HEIGHT) self.sync_blocks(nodes=(n0, n2)) @@ -553,6 +697,9 @@ class AssumeutxoTest(BitcoinTestFramework): self.log.info("Ensuring background validation completes") self.wait_until(lambda: len(n2.getchainstates()['chainstates']) == 1) + # Once background chain sync completes, the full node must start offering historical blocks again. + self.wait_until(lambda: {'NETWORK', 'NETWORK_LIMITED'}.issubset(n2.getnetworkinfo()['localservicesnames'])) + completed_idx_state = { 'basic block filter index': COMPLETE_IDX, 'coinstatsindex': COMPLETE_IDX, @@ -587,6 +734,9 @@ class AssumeutxoTest(BitcoinTestFramework): self.test_snapshot_in_a_divergent_chain(dump_output['path']) + # The following test cleans node2 and node3 chain directories. + self.test_sync_from_assumeutxo_node(snapshot=dump_output) + @dataclass class Block: hash: str diff --git a/test/functional/feature_block.py b/test/functional/feature_block.py index 384ca311c7..43bf61c174 100755 --- a/test/functional/feature_block.py +++ b/test/functional/feature_block.py @@ -88,6 +88,7 @@ class FullBlockTest(BitcoinTestFramework): self.extra_args = [[ '-acceptnonstdtxn=1', # This is a consensus block test, we don't care about tx policy '-testactivationheight=bip34@2', + '-par=1', # Until https://github.com/bitcoin/bitcoin/issues/30960 is fixed ]] def run_test(self): diff --git a/test/functional/feature_blocksxor.py b/test/functional/feature_blocksxor.py index 7698a66ec4..9824bf9715 100755 --- a/test/functional/feature_blocksxor.py +++ b/test/functional/feature_blocksxor.py @@ -31,7 +31,7 @@ class BlocksXORTest(BitcoinTestFramework): node = self.nodes[0] wallet = MiniWallet(node) for _ in range(5): - wallet.send_self_transfer(from_node=node, target_weight=80000) + wallet.send_self_transfer(from_node=node, target_vsize=20000) self.generate(wallet, 1) block_files = list(node.blocks_path.glob('blk[0-9][0-9][0-9][0-9][0-9].dat')) diff --git a/test/functional/feature_config_args.py b/test/functional/feature_config_args.py index bb20e2baa8..44c7edf962 100755 --- a/test/functional/feature_config_args.py +++ b/test/functional/feature_config_args.py @@ -153,6 +153,13 @@ class ConfArgsTest(BitcoinTestFramework): expected_msg='Error: Error parsing command line arguments: Can not set -proxy with no value. Please specify value with -proxy=value.', extra_args=['-proxy'], ) + # Provide a value different from 1 to the -wallet negated option + if self.is_wallet_compiled(): + for value in [0, 'not_a_boolean']: + self.nodes[0].assert_start_raises_init_error( + expected_msg="Error: Invalid value detected for '-wallet' or '-nowallet'. '-wallet' requires a string value, while '-nowallet' accepts only '1' to disable all wallets", + extra_args=[f'-nowallet={value}'], + ) def test_log_buffer(self): self.stop_node(0) diff --git a/test/functional/feature_fee_estimation.py b/test/functional/feature_fee_estimation.py index a3dcb7afda..974d8268a2 100755 --- a/test/functional/feature_fee_estimation.py +++ b/test/functional/feature_fee_estimation.py @@ -4,7 +4,7 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test fee estimation code.""" from copy import deepcopy -from decimal import Decimal +from decimal import Decimal, ROUND_DOWN import os import random import time @@ -40,7 +40,7 @@ def small_txpuzzle_randfee( # Exponentially distributed from 1-128 * fee_increment rand_fee = float(fee_increment) * (1.1892 ** random.randint(0, 28)) # Total fee ranges from min_fee to min_fee + 127*fee_increment - fee = min_fee - fee_increment + satoshi_round(rand_fee) + fee = min_fee - fee_increment + satoshi_round(rand_fee, rounding=ROUND_DOWN) utxos_to_spend = [] total_in = Decimal("0.00000000") while total_in <= (amount + fee) and len(conflist) > 0: @@ -398,6 +398,7 @@ class EstimateFeeTest(BitcoinTestFramework): self.start_node(0) self.connect_nodes(0, 1) self.connect_nodes(0, 2) + self.sync_blocks() assert_equal(self.nodes[0].estimatesmartfee(1)["errors"], ["Insufficient data or no feerate found"]) def broadcast_and_mine(self, broadcaster, miner, feerate, count): diff --git a/test/functional/feature_framework_miniwallet.py b/test/functional/feature_framework_miniwallet.py index d1aa24e7cd..f723f7f31e 100755 --- a/test/functional/feature_framework_miniwallet.py +++ b/test/functional/feature_framework_miniwallet.py @@ -9,7 +9,7 @@ import string from test_framework.blocktools import COINBASE_MATURITY from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( - assert_greater_than_or_equal, + assert_equal, ) from test_framework.wallet import ( MiniWallet, @@ -22,17 +22,15 @@ class FeatureFrameworkMiniWalletTest(BitcoinTestFramework): self.num_nodes = 1 def test_tx_padding(self): - """Verify that MiniWallet's transaction padding (`target_weight` parameter) - works accurately enough (i.e. at most 3 WUs higher) with all modes.""" + """Verify that MiniWallet's transaction padding (`target_vsize` parameter) + works accurately with all modes.""" for mode_name, wallet in self.wallets: self.log.info(f"Test tx padding with MiniWallet mode {mode_name}...") utxo = wallet.get_utxo(mark_as_spent=False) - for target_weight in [1000, 2000, 5000, 10000, 20000, 50000, 100000, 200000, 4000000, - 989, 2001, 4337, 13371, 23219, 49153, 102035, 223419, 3999989]: - tx = wallet.create_self_transfer(utxo_to_spend=utxo, target_weight=target_weight)["tx"] - self.log.debug(f"-> target weight: {target_weight}, actual weight: {tx.get_weight()}") - assert_greater_than_or_equal(tx.get_weight(), target_weight) - assert_greater_than_or_equal(target_weight + 3, tx.get_weight()) + for target_vsize in [250, 500, 1250, 2500, 5000, 12500, 25000, 50000, 1000000, + 248, 501, 1085, 3343, 5805, 12289, 25509, 55855, 999998]: + tx = wallet.create_self_transfer(utxo_to_spend=utxo, target_vsize=target_vsize)["tx"] + assert_equal(tx.get_vsize(), target_vsize) def test_wallet_tagging(self): """Verify that tagged wallet instances are able to send funds.""" diff --git a/test/functional/feature_settings.py b/test/functional/feature_settings.py index 2189eac7dd..a7294944bf 100755 --- a/test/functional/feature_settings.py +++ b/test/functional/feature_settings.py @@ -13,11 +13,32 @@ from test_framework.util import assert_equal class SettingsTest(BitcoinTestFramework): + def add_options(self, parser): + self.add_wallet_options(parser) + def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 1 self.wallet_names = [] + def test_wallet_settings(self, settings_path): + if not self.is_wallet_compiled(): + return + + self.log.info("Testing wallet settings..") + node = self.nodes[0] + # Create wallet to use it during tests + self.start_node(0) + node.createwallet(wallet_name='w1') + self.stop_node(0) + + # Verify wallet settings can only be strings. Either names or paths. Not booleans, nums nor anything else. + for wallets_data in [[10], [True], [[]], [{}], ["w1", 10], ["w1", False]]: + with settings_path.open("w") as fp: + json.dump({"wallet": wallets_data}, fp) + node.assert_start_raises_init_error(expected_msg="Error: Invalid value detected for '-wallet' or '-nowallet'. '-wallet' requires a string value, while '-nowallet' accepts only '1' to disable all wallets", + extra_args=[f'-settings={settings_path}']) + def run_test(self): node, = self.nodes settings = node.chain_path / "settings.json" @@ -86,6 +107,8 @@ class SettingsTest(BitcoinTestFramework): self.start_node(0, extra_args=[f"-settings={altsettings}"]) self.stop_node(0) + self.test_wallet_settings(settings) + if __name__ == '__main__': SettingsTest(__file__).main() diff --git a/test/functional/interface_bitcoin_cli.py b/test/functional/interface_bitcoin_cli.py index e7113f8335..7194c8ece4 100755 --- a/test/functional/interface_bitcoin_cli.py +++ b/test/functional/interface_bitcoin_cli.py @@ -30,7 +30,12 @@ JSON_PARSING_ERROR = 'error: Error parsing JSON: foo' BLOCKS_VALUE_OF_ZERO = 'error: the first argument (number of blocks to generate, default: 1) must be an integer value greater than zero' TOO_MANY_ARGS = 'error: too many arguments (maximum 2 for nblocks and maxtries)' WALLET_NOT_LOADED = 'Requested wallet does not exist or is not loaded' -WALLET_NOT_SPECIFIED = 'Wallet file not specified' +WALLET_NOT_SPECIFIED = ( + "Multiple wallets are loaded. Please select which wallet to use by requesting the RPC " + "through the /wallet/<walletname> URI path. Or for the CLI, specify the \"-rpcwallet=<walletname>\" " + "option before the command (run \"bitcoin-cli -h\" for help or \"bitcoin-cli listwallets\" to see " + "which wallets are currently loaded)." +) def cli_get_info_string_to_dict(cli_get_info_string): @@ -331,6 +336,10 @@ class TestBitcoinCli(BitcoinTestFramework): n4 = 10 blocks = self.nodes[0].getblockcount() + self.log.info('Test -generate -rpcwallet=<filename> raise RPC error') + wallet2_path = f'-rpcwallet={self.nodes[0].wallets_path / wallets[2] / self.wallet_data_filename}' + assert_raises_rpc_error(-18, WALLET_NOT_LOADED, self.nodes[0].cli(wallet2_path, '-generate').echo) + self.log.info('Test -generate -rpcwallet with no args') generate = self.nodes[0].cli(rpcwallet2, '-generate').send_cli() assert_equal(set(generate.keys()), {'address', 'blocks'}) @@ -381,6 +390,9 @@ class TestBitcoinCli(BitcoinTestFramework): assert_raises_process_error(1, "Could not connect to the server", self.nodes[0].cli('-rpcwait', '-rpcwaittimeout=5').echo) assert_greater_than_or_equal(time.time(), start_time + 5) + self.log.info("Test that only one of -addrinfo, -generate, -getinfo, -netinfo may be specified at a time") + assert_raises_process_error(1, "Only one of -getinfo, -netinfo may be specified", self.nodes[0].cli('-getinfo', '-netinfo').send_cli) + if __name__ == '__main__': TestBitcoinCli(__file__).main() diff --git a/test/functional/interface_usdt_coinselection.py b/test/functional/interface_usdt_coinselection.py index dc40986a75..f684848aed 100755 --- a/test/functional/interface_usdt_coinselection.py +++ b/test/functional/interface_usdt_coinselection.py @@ -181,7 +181,7 @@ class CoinSelectionTracepointTest(BitcoinTestFramework): # 5. aps_create_tx_internal (type 4) wallet.sendtoaddress(wallet.getnewaddress(), 10) events = self.get_tracepoints([1, 2, 3, 1, 4]) - success, use_aps, algo, waste, change_pos = self.determine_selection_from_usdt(events) + success, use_aps, _algo, _waste, change_pos = self.determine_selection_from_usdt(events) assert_equal(success, True) assert_greater_than(change_pos, -1) @@ -190,7 +190,7 @@ class CoinSelectionTracepointTest(BitcoinTestFramework): # 1. normal_create_tx_internal (type 2) assert_raises_rpc_error(-6, "Insufficient funds", wallet.sendtoaddress, wallet.getnewaddress(), 102 * 50) events = self.get_tracepoints([2]) - success, use_aps, algo, waste, change_pos = self.determine_selection_from_usdt(events) + success, use_aps, _algo, _waste, change_pos = self.determine_selection_from_usdt(events) assert_equal(success, False) self.log.info("Explicitly enabling APS results in 2 tracepoints") @@ -200,7 +200,7 @@ class CoinSelectionTracepointTest(BitcoinTestFramework): wallet.setwalletflag("avoid_reuse") wallet.sendtoaddress(address=wallet.getnewaddress(), amount=10, avoid_reuse=True) events = self.get_tracepoints([1, 2]) - success, use_aps, algo, waste, change_pos = self.determine_selection_from_usdt(events) + success, use_aps, _algo, _waste, change_pos = self.determine_selection_from_usdt(events) assert_equal(success, True) assert_equal(use_aps, None) @@ -213,7 +213,7 @@ class CoinSelectionTracepointTest(BitcoinTestFramework): # 5. aps_create_tx_internal (type 4) wallet.sendtoaddress(address=wallet.getnewaddress(), amount=wallet.getbalance(), subtractfeefromamount=True, avoid_reuse=False) events = self.get_tracepoints([1, 2, 3, 1, 4]) - success, use_aps, algo, waste, change_pos = self.determine_selection_from_usdt(events) + success, use_aps, _algo, _waste, change_pos = self.determine_selection_from_usdt(events) assert_equal(success, True) assert_equal(change_pos, -1) @@ -223,7 +223,7 @@ class CoinSelectionTracepointTest(BitcoinTestFramework): # 2. normal_create_tx_internal (type 2) wallet.sendtoaddress(address=wallet.getnewaddress(), amount=wallet.getbalance(), subtractfeefromamount=True) events = self.get_tracepoints([1, 2]) - success, use_aps, algo, waste, change_pos = self.determine_selection_from_usdt(events) + success, use_aps, _algo, _waste, change_pos = self.determine_selection_from_usdt(events) assert_equal(success, True) assert_equal(change_pos, -1) diff --git a/test/functional/interface_usdt_validation.py b/test/functional/interface_usdt_validation.py index 9a37b96ada..8a98a452de 100755 --- a/test/functional/interface_usdt_validation.py +++ b/test/functional/interface_usdt_validation.py @@ -8,6 +8,7 @@ """ import ctypes +import time # Test will be skipped if we don't have bcc installed try: @@ -105,10 +106,12 @@ class ValidationTracepointTest(BitcoinTestFramework): handle_blockconnected) self.log.info(f"mine {BLOCKS_EXPECTED} blocks") - block_hashes = self.generatetoaddress( - self.nodes[0], BLOCKS_EXPECTED, ADDRESS_BCRT1_UNSPENDABLE) - for block_hash in block_hashes: - expected_blocks[block_hash] = self.nodes[0].getblock(block_hash, 2) + generatetoaddress_duration = dict() + for _ in range(BLOCKS_EXPECTED): + start = time.time() + hash = self.generatetoaddress(self.nodes[0], 1, ADDRESS_BCRT1_UNSPENDABLE)[0] + generatetoaddress_duration[hash] = (time.time() - start) * 1e9 # in nanoseconds + expected_blocks[hash] = self.nodes[0].getblock(hash, 2) bpf.perf_buffer_poll(timeout=200) @@ -123,6 +126,10 @@ class ValidationTracepointTest(BitcoinTestFramework): assert_equal(0, event.sigops) # no sigops in coinbase tx # only plausibility checks assert event.duration > 0 + # generatetoaddress (mining and connecting) takes longer than + # connecting the block. In case the duration unit is off, we'll + # detect it with this assert. + assert event.duration < generatetoaddress_duration[block_hash] del expected_blocks[block_hash] assert_equal(BLOCKS_EXPECTED, len(events)) assert_equal(0, len(expected_blocks)) diff --git a/test/functional/mempool_limit.py b/test/functional/mempool_limit.py index 626928a49a..a29c103c3f 100755 --- a/test/functional/mempool_limit.py +++ b/test/functional/mempool_limit.py @@ -55,12 +55,12 @@ class MempoolLimitTest(BitcoinTestFramework): self.generate(node, 1) # tx_A needs to be RBF'd, set minfee at set size - A_weight = 1000 + A_vsize = 250 mempoolmin_feerate = node.getmempoolinfo()["mempoolminfee"] tx_A = self.wallet.send_self_transfer( from_node=node, fee_rate=mempoolmin_feerate, - target_weight=A_weight, + target_vsize=A_vsize, utxo_to_spend=rbf_utxo, confirmed_only=True ) @@ -68,15 +68,15 @@ class MempoolLimitTest(BitcoinTestFramework): # RBF's tx_A, is not yet submitted tx_B = self.wallet.create_self_transfer( fee=tx_A["fee"] * 4, - target_weight=A_weight, + target_vsize=A_vsize, utxo_to_spend=rbf_utxo, confirmed_only=True ) # Spends tx_B's output, too big for cpfp carveout (because that would also increase the descendant limit by 1) - non_cpfp_carveout_weight = 40001 # EXTRA_DESCENDANT_TX_SIZE_LIMIT + 1 + non_cpfp_carveout_vsize = 10001 # EXTRA_DESCENDANT_TX_SIZE_LIMIT + 1 tx_C = self.wallet.create_self_transfer( - target_weight=non_cpfp_carveout_weight, + target_vsize=non_cpfp_carveout_vsize, fee_rate=mempoolmin_feerate, utxo_to_spend=tx_B["new_utxo"], confirmed_only=True @@ -103,14 +103,14 @@ class MempoolLimitTest(BitcoinTestFramework): # UTXOs to be spent by the ultimate child transaction parent_utxos = [] - evicted_weight = 8000 + evicted_vsize = 2000 # Mempool transaction which is evicted due to being at the "bottom" of the mempool when the # mempool overflows and evicts by descendant score. It's important that the eviction doesn't # happen in the middle of package evaluation, as it can invalidate the coins cache. mempool_evicted_tx = self.wallet.send_self_transfer( from_node=node, fee_rate=mempoolmin_feerate, - target_weight=evicted_weight, + target_vsize=evicted_vsize, confirmed_only=True ) # Already in mempool when package is submitted. @@ -132,14 +132,16 @@ class MempoolLimitTest(BitcoinTestFramework): # Series of parents that don't need CPFP and are submitted individually. Each one is large and # high feerate, which means they should trigger eviction but not be evicted. - parent_weight = 100000 + parent_vsize = 25000 num_big_parents = 3 - assert_greater_than(parent_weight * num_big_parents, current_info["maxmempool"] - current_info["bytes"]) + # Need to be large enough to trigger eviction + # (note that the mempool usage of a tx is about three times its vsize) + assert_greater_than(parent_vsize * num_big_parents * 3, current_info["maxmempool"] - current_info["bytes"]) parent_feerate = 100 * mempoolmin_feerate big_parent_txids = [] for i in range(num_big_parents): - parent = self.wallet.create_self_transfer(fee_rate=parent_feerate, target_weight=parent_weight, confirmed_only=True) + parent = self.wallet.create_self_transfer(fee_rate=parent_feerate, target_vsize=parent_vsize, confirmed_only=True) parent_utxos.append(parent["new_utxo"]) package_hex.append(parent["hex"]) big_parent_txids.append(parent["txid"]) @@ -311,8 +313,9 @@ class MempoolLimitTest(BitcoinTestFramework): entry = node.getmempoolentry(txid) worst_feerate_btcvb = min(worst_feerate_btcvb, entry["fees"]["descendant"] / entry["descendantsize"]) # Needs to be large enough to trigger eviction - target_weight_each = 200000 - assert_greater_than(target_weight_each * 2, node.getmempoolinfo()["maxmempool"] - node.getmempoolinfo()["bytes"]) + # (note that the mempool usage of a tx is about three times its vsize) + target_vsize_each = 50000 + assert_greater_than(target_vsize_each * 2 * 3, node.getmempoolinfo()["maxmempool"] - node.getmempoolinfo()["bytes"]) # Should be a true CPFP: parent's feerate is just below mempool min feerate parent_feerate = mempoolmin_feerate - Decimal("0.000001") # 0.1 sats/vbyte below min feerate # Parent + child is above mempool minimum feerate @@ -320,8 +323,8 @@ class MempoolLimitTest(BitcoinTestFramework): # However, when eviction is triggered, these transactions should be at the bottom. # This assertion assumes parent and child are the same size. miniwallet.rescan_utxos() - tx_parent_just_below = miniwallet.create_self_transfer(fee_rate=parent_feerate, target_weight=target_weight_each) - tx_child_just_above = miniwallet.create_self_transfer(utxo_to_spend=tx_parent_just_below["new_utxo"], fee_rate=child_feerate, target_weight=target_weight_each) + tx_parent_just_below = miniwallet.create_self_transfer(fee_rate=parent_feerate, target_vsize=target_vsize_each) + tx_child_just_above = miniwallet.create_self_transfer(utxo_to_spend=tx_parent_just_below["new_utxo"], fee_rate=child_feerate, target_vsize=target_vsize_each) # This package ranks below the lowest descendant package in the mempool package_fee = tx_parent_just_below["fee"] + tx_child_just_above["fee"] package_vsize = tx_parent_just_below["tx"].get_vsize() + tx_child_just_above["tx"].get_vsize() diff --git a/test/functional/mempool_package_limits.py b/test/functional/mempool_package_limits.py index 6e26a684e2..3290ff43c4 100755 --- a/test/functional/mempool_package_limits.py +++ b/test/functional/mempool_package_limits.py @@ -4,9 +4,6 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test logic for limiting mempool and package ancestors/descendants.""" from test_framework.blocktools import COINBASE_MATURITY -from test_framework.messages import ( - WITNESS_SCALE_FACTOR, -) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, @@ -290,19 +287,18 @@ class MempoolPackageLimitsTest(BitcoinTestFramework): parent_utxos = [] target_vsize = 30_000 high_fee = 10 * target_vsize # 10 sats/vB - target_weight = target_vsize * WITNESS_SCALE_FACTOR self.log.info("Check that in-mempool and in-package ancestor size limits are calculated properly in packages") # Mempool transactions A and B for _ in range(2): - bulked_tx = self.wallet.create_self_transfer(target_weight=target_weight) + bulked_tx = self.wallet.create_self_transfer(target_vsize=target_vsize) self.wallet.sendrawtransaction(from_node=node, tx_hex=bulked_tx["hex"]) parent_utxos.append(bulked_tx["new_utxo"]) # Package transaction C - pc_tx = self.wallet.create_self_transfer_multi(utxos_to_spend=parent_utxos, fee_per_output=high_fee, target_weight=target_weight) + pc_tx = self.wallet.create_self_transfer_multi(utxos_to_spend=parent_utxos, fee_per_output=high_fee, target_vsize=target_vsize) # Package transaction D - pd_tx = self.wallet.create_self_transfer(utxo_to_spend=pc_tx["new_utxos"][0], target_weight=target_weight) + pd_tx = self.wallet.create_self_transfer(utxo_to_spend=pc_tx["new_utxos"][0], target_vsize=target_vsize) assert_equal(2, node.getmempoolinfo()["size"]) return [pc_tx["hex"], pd_tx["hex"]] @@ -321,20 +317,19 @@ class MempoolPackageLimitsTest(BitcoinTestFramework): node = self.nodes[0] target_vsize = 21_000 high_fee = 10 * target_vsize # 10 sats/vB - target_weight = target_vsize * WITNESS_SCALE_FACTOR self.log.info("Check that in-mempool and in-package descendant sizes are calculated properly in packages") # Top parent in mempool, Ma - ma_tx = self.wallet.create_self_transfer_multi(num_outputs=2, fee_per_output=high_fee // 2, target_weight=target_weight) + ma_tx = self.wallet.create_self_transfer_multi(num_outputs=2, fee_per_output=high_fee // 2, target_vsize=target_vsize) self.wallet.sendrawtransaction(from_node=node, tx_hex=ma_tx["hex"]) package_hex = [] for j in range(2): # Two legs (left and right) # Mempool transaction (Mb and Mc) - mempool_tx = self.wallet.create_self_transfer(utxo_to_spend=ma_tx["new_utxos"][j], target_weight=target_weight) + mempool_tx = self.wallet.create_self_transfer(utxo_to_spend=ma_tx["new_utxos"][j], target_vsize=target_vsize) self.wallet.sendrawtransaction(from_node=node, tx_hex=mempool_tx["hex"]) # Package transaction (Pd and Pe) - package_tx = self.wallet.create_self_transfer(utxo_to_spend=mempool_tx["new_utxo"], target_weight=target_weight) + package_tx = self.wallet.create_self_transfer(utxo_to_spend=mempool_tx["new_utxo"], target_vsize=target_vsize) package_hex.append(package_tx["hex"]) assert_equal(3, node.getmempoolinfo()["size"]) diff --git a/test/functional/mempool_package_rbf.py b/test/functional/mempool_package_rbf.py index 9b4269f0a0..a5b8fa5f87 100755 --- a/test/functional/mempool_package_rbf.py +++ b/test/functional/mempool_package_rbf.py @@ -189,7 +189,7 @@ class PackageRBFTest(BitcoinTestFramework): package_hex4, package_txns4 = self.create_simple_package(coin, parent_fee=DEFAULT_FEE, child_fee=DEFAULT_CHILD_FEE) node.submitpackage(package_hex4) self.assert_mempool_contents(expected=package_txns4) - package_hex5, package_txns5 = self.create_simple_package(coin, parent_fee=DEFAULT_CHILD_FEE, child_fee=DEFAULT_CHILD_FEE) + package_hex5, _package_txns5 = self.create_simple_package(coin, parent_fee=DEFAULT_CHILD_FEE, child_fee=DEFAULT_CHILD_FEE) pkg_results5 = node.submitpackage(package_hex5) assert 'package RBF failed: package feerate is less than or equal to parent feerate' in pkg_results5["package_msg"] self.assert_mempool_contents(expected=package_txns4) @@ -336,16 +336,16 @@ class PackageRBFTest(BitcoinTestFramework): self.assert_mempool_contents(expected=expected_txns) # Now make conflicting packages for each coin - package_hex1, package_txns1 = self.create_simple_package(coin1, DEFAULT_FEE, DEFAULT_CHILD_FEE) + package_hex1, _package_txns1 = self.create_simple_package(coin1, DEFAULT_FEE, DEFAULT_CHILD_FEE) package_result = node.submitpackage(package_hex1) assert_equal(f"package RBF failed: {parent_result['tx'].rehash()} has 2 descendants, max 1 allowed", package_result["package_msg"]) - package_hex2, package_txns2 = self.create_simple_package(coin2, DEFAULT_FEE, DEFAULT_CHILD_FEE) + package_hex2, _package_txns2 = self.create_simple_package(coin2, DEFAULT_FEE, DEFAULT_CHILD_FEE) package_result = node.submitpackage(package_hex2) assert_equal(f"package RBF failed: {child_result['tx'].rehash()} has both ancestor and descendant, exceeding cluster limit of 2", package_result["package_msg"]) - package_hex3, package_txns3 = self.create_simple_package(coin3, DEFAULT_FEE, DEFAULT_CHILD_FEE) + package_hex3, _package_txns3 = self.create_simple_package(coin3, DEFAULT_FEE, DEFAULT_CHILD_FEE) package_result = node.submitpackage(package_hex3) assert_equal(f"package RBF failed: {grandchild_result['tx'].rehash()} has 2 ancestors, max 1 allowed", package_result["package_msg"]) @@ -389,15 +389,15 @@ class PackageRBFTest(BitcoinTestFramework): self.assert_mempool_contents(expected=expected_txns) # Now make conflicting packages for each coin - package_hex1, package_txns1 = self.create_simple_package(coin1, DEFAULT_FEE, DEFAULT_CHILD_FEE) + package_hex1, _package_txns1 = self.create_simple_package(coin1, DEFAULT_FEE, DEFAULT_CHILD_FEE) package_result = node.submitpackage(package_hex1) assert_equal(f"package RBF failed: {parent1_result['tx'].rehash()} is not the only parent of child {child_result['tx'].rehash()}", package_result["package_msg"]) - package_hex2, package_txns2 = self.create_simple_package(coin2, DEFAULT_FEE, DEFAULT_CHILD_FEE) + package_hex2, _package_txns2 = self.create_simple_package(coin2, DEFAULT_FEE, DEFAULT_CHILD_FEE) package_result = node.submitpackage(package_hex2) assert_equal(f"package RBF failed: {parent2_result['tx'].rehash()} is not the only parent of child {child_result['tx'].rehash()}", package_result["package_msg"]) - package_hex3, package_txns3 = self.create_simple_package(coin3, DEFAULT_FEE, DEFAULT_CHILD_FEE) + package_hex3, _package_txns3 = self.create_simple_package(coin3, DEFAULT_FEE, DEFAULT_CHILD_FEE) package_result = node.submitpackage(package_hex3) assert_equal(f"package RBF failed: {child_result['tx'].rehash()} has 2 ancestors, max 1 allowed", package_result["package_msg"]) @@ -443,15 +443,15 @@ class PackageRBFTest(BitcoinTestFramework): self.assert_mempool_contents(expected=expected_txns) # Now make conflicting packages for each coin - package_hex1, package_txns1 = self.create_simple_package(coin1, DEFAULT_FEE, DEFAULT_CHILD_FEE) + package_hex1, _package_txns1 = self.create_simple_package(coin1, DEFAULT_FEE, DEFAULT_CHILD_FEE) package_result = node.submitpackage(package_hex1) assert_equal(f"package RBF failed: {parent_result['tx'].rehash()} has 2 descendants, max 1 allowed", package_result["package_msg"]) - package_hex2, package_txns2 = self.create_simple_package(coin2, DEFAULT_FEE, DEFAULT_CHILD_FEE) + package_hex2, _package_txns2 = self.create_simple_package(coin2, DEFAULT_FEE, DEFAULT_CHILD_FEE) package_result = node.submitpackage(package_hex2) assert_equal(f"package RBF failed: {child1_result['tx'].rehash()} is not the only child of parent {parent_result['tx'].rehash()}", package_result["package_msg"]) - package_hex3, package_txns3 = self.create_simple_package(coin3, DEFAULT_FEE, DEFAULT_CHILD_FEE) + package_hex3, _package_txns3 = self.create_simple_package(coin3, DEFAULT_FEE, DEFAULT_CHILD_FEE) package_result = node.submitpackage(package_hex3) assert_equal(f"package RBF failed: {child2_result['tx'].rehash()} is not the only child of parent {parent_result['tx'].rehash()}", package_result["package_msg"]) @@ -519,7 +519,7 @@ class PackageRBFTest(BitcoinTestFramework): # Package 2 feerate is below the feerate of directly conflicted parent, so it fails even though # total fees are higher than the original package - package_hex2, package_txns2 = self.create_simple_package(coin, parent_fee=DEFAULT_CHILD_FEE - Decimal("0.00000001"), child_fee=DEFAULT_CHILD_FEE) + package_hex2, _package_txns2 = self.create_simple_package(coin, parent_fee=DEFAULT_CHILD_FEE - Decimal("0.00000001"), child_fee=DEFAULT_CHILD_FEE) pkg_results2 = node.submitpackage(package_hex2) assert_equal(pkg_results2["package_msg"], 'package RBF failed: insufficient feerate: does not improve feerate diagram') self.assert_mempool_contents(expected=package_txns1) @@ -554,7 +554,7 @@ class PackageRBFTest(BitcoinTestFramework): self.generate(node, 1) def test_child_conflicts_parent_mempool_ancestor(self): - fill_mempool(self, self.nodes[0]) + fill_mempool(self, self.nodes[0], tx_sync_fun=self.no_op) # Reset coins since we filled the mempool with current coins self.coins = self.wallet.get_utxos(mark_as_spent=False, confirmed_only=True) diff --git a/test/functional/mempool_sigoplimit.py b/test/functional/mempool_sigoplimit.py index 4656176a75..47df0c614a 100755 --- a/test/functional/mempool_sigoplimit.py +++ b/test/functional/mempool_sigoplimit.py @@ -154,7 +154,7 @@ class BytesPerSigOpTest(BitcoinTestFramework): return (tx_utxo, tx) tx_parent_utxo, tx_parent = create_bare_multisig_tx() - tx_child_utxo, tx_child = create_bare_multisig_tx(tx_parent_utxo) + _tx_child_utxo, tx_child = create_bare_multisig_tx(tx_parent_utxo) # Separately, the parent tx is ok parent_individual_testres = self.nodes[0].testmempoolaccept([tx_parent.serialize().hex()])[0] diff --git a/test/functional/mempool_truc.py b/test/functional/mempool_truc.py index 28f3256ef1..54a258215d 100755 --- a/test/functional/mempool_truc.py +++ b/test/functional/mempool_truc.py @@ -6,7 +6,6 @@ from decimal import Decimal from test_framework.messages import ( MAX_BIP125_RBF_SEQUENCE, - WITNESS_SCALE_FACTOR, ) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( @@ -23,6 +22,7 @@ from test_framework.wallet import ( MAX_REPLACEMENT_CANDIDATES = 100 TRUC_MAX_VSIZE = 10000 +TRUC_CHILD_MAX_VSIZE = 1000 def cleanup(extra_args=None): def decorator(func): @@ -55,14 +55,14 @@ class MempoolTRUC(BitcoinTestFramework): def test_truc_max_vsize(self): node = self.nodes[0] self.log.info("Test TRUC-specific maximum transaction vsize") - tx_v3_heavy = self.wallet.create_self_transfer(target_weight=(TRUC_MAX_VSIZE + 1) * WITNESS_SCALE_FACTOR, version=3) + tx_v3_heavy = self.wallet.create_self_transfer(target_vsize=TRUC_MAX_VSIZE + 1, version=3) assert_greater_than_or_equal(tx_v3_heavy["tx"].get_vsize(), TRUC_MAX_VSIZE) expected_error_heavy = f"TRUC-violation, version=3 tx {tx_v3_heavy['txid']} (wtxid={tx_v3_heavy['wtxid']}) is too big" assert_raises_rpc_error(-26, expected_error_heavy, node.sendrawtransaction, tx_v3_heavy["hex"]) self.check_mempool([]) # Ensure we are hitting the TRUC-specific limit and not something else - tx_v2_heavy = self.wallet.send_self_transfer(from_node=node, target_weight=(TRUC_MAX_VSIZE + 1) * WITNESS_SCALE_FACTOR, version=2) + tx_v2_heavy = self.wallet.send_self_transfer(from_node=node, target_vsize=TRUC_MAX_VSIZE + 1, version=2) self.check_mempool([tx_v2_heavy["txid"]]) @cleanup(extra_args=["-datacarriersize=1000"]) @@ -73,10 +73,10 @@ class MempoolTRUC(BitcoinTestFramework): self.check_mempool([tx_v3_parent_normal["txid"]]) tx_v3_child_heavy = self.wallet.create_self_transfer( utxo_to_spend=tx_v3_parent_normal["new_utxo"], - target_weight=4004, + target_vsize=TRUC_CHILD_MAX_VSIZE + 1, version=3 ) - assert_greater_than_or_equal(tx_v3_child_heavy["tx"].get_vsize(), 1000) + assert_greater_than_or_equal(tx_v3_child_heavy["tx"].get_vsize(), TRUC_CHILD_MAX_VSIZE) expected_error_child_heavy = f"TRUC-violation, version=3 child tx {tx_v3_child_heavy['txid']} (wtxid={tx_v3_child_heavy['wtxid']}) is too big" assert_raises_rpc_error(-26, expected_error_child_heavy, node.sendrawtransaction, tx_v3_child_heavy["hex"]) self.check_mempool([tx_v3_parent_normal["txid"]]) @@ -88,20 +88,21 @@ class MempoolTRUC(BitcoinTestFramework): from_node=node, fee_rate=DEFAULT_FEE, utxo_to_spend=tx_v3_parent_normal["new_utxo"], - target_weight=3987, + target_vsize=TRUC_CHILD_MAX_VSIZE - 3, version=3 ) - assert_greater_than_or_equal(1000, tx_v3_child_almost_heavy["tx"].get_vsize()) + assert_greater_than_or_equal(TRUC_CHILD_MAX_VSIZE, tx_v3_child_almost_heavy["tx"].get_vsize()) self.check_mempool([tx_v3_parent_normal["txid"], tx_v3_child_almost_heavy["txid"]]) assert_equal(node.getmempoolentry(tx_v3_parent_normal["txid"])["descendantcount"], 2) tx_v3_child_almost_heavy_rbf = self.wallet.send_self_transfer( from_node=node, fee_rate=DEFAULT_FEE * 2, utxo_to_spend=tx_v3_parent_normal["new_utxo"], - target_weight=3500, + target_vsize=875, version=3 ) - assert_greater_than_or_equal(tx_v3_child_almost_heavy["tx"].get_vsize() + tx_v3_child_almost_heavy_rbf["tx"].get_vsize(), 1000) + assert_greater_than_or_equal(tx_v3_child_almost_heavy["tx"].get_vsize() + tx_v3_child_almost_heavy_rbf["tx"].get_vsize(), + TRUC_CHILD_MAX_VSIZE) self.check_mempool([tx_v3_parent_normal["txid"], tx_v3_child_almost_heavy_rbf["txid"]]) assert_equal(node.getmempoolentry(tx_v3_parent_normal["txid"])["descendantcount"], 2) @@ -199,8 +200,8 @@ class MempoolTRUC(BitcoinTestFramework): self.check_mempool([]) tx_v2_from_v3 = self.wallet.send_self_transfer(from_node=node, utxo_to_spend=tx_v3_block["new_utxo"], version=2) tx_v3_from_v2 = self.wallet.send_self_transfer(from_node=node, utxo_to_spend=tx_v2_block["new_utxo"], version=3) - tx_v3_child_large = self.wallet.send_self_transfer(from_node=node, utxo_to_spend=tx_v3_block2["new_utxo"], target_weight=5000, version=3) - assert_greater_than(node.getmempoolentry(tx_v3_child_large["txid"])["vsize"], 1000) + tx_v3_child_large = self.wallet.send_self_transfer(from_node=node, utxo_to_spend=tx_v3_block2["new_utxo"], target_vsize=1250, version=3) + assert_greater_than(node.getmempoolentry(tx_v3_child_large["txid"])["vsize"], TRUC_CHILD_MAX_VSIZE) self.check_mempool([tx_v2_from_v3["txid"], tx_v3_from_v2["txid"], tx_v3_child_large["txid"]]) node.invalidateblock(block[0]) self.check_mempool([tx_v3_block["txid"], tx_v2_block["txid"], tx_v3_block2["txid"], tx_v2_from_v3["txid"], tx_v3_from_v2["txid"], tx_v3_child_large["txid"]]) @@ -217,22 +218,22 @@ class MempoolTRUC(BitcoinTestFramework): """ node = self.nodes[0] self.log.info("Test that a decreased limitdescendantsize also applies to TRUC child") - parent_target_weight = 9990 * WITNESS_SCALE_FACTOR - child_target_weight = 500 * WITNESS_SCALE_FACTOR + parent_target_vsize = 9990 + child_target_vsize = 500 tx_v3_parent_large1 = self.wallet.send_self_transfer( from_node=node, - target_weight=parent_target_weight, + target_vsize=parent_target_vsize, version=3 ) tx_v3_child_large1 = self.wallet.create_self_transfer( utxo_to_spend=tx_v3_parent_large1["new_utxo"], - target_weight=child_target_weight, + target_vsize=child_target_vsize, version=3 ) # Parent and child are within v3 limits, but parent's 10kvB descendant limit is exceeded assert_greater_than_or_equal(TRUC_MAX_VSIZE, tx_v3_parent_large1["tx"].get_vsize()) - assert_greater_than_or_equal(1000, tx_v3_child_large1["tx"].get_vsize()) + assert_greater_than_or_equal(TRUC_CHILD_MAX_VSIZE, tx_v3_child_large1["tx"].get_vsize()) assert_greater_than(tx_v3_parent_large1["tx"].get_vsize() + tx_v3_child_large1["tx"].get_vsize(), 10000) assert_raises_rpc_error(-26, f"too-long-mempool-chain, exceeds descendant size limit for tx {tx_v3_parent_large1['txid']}", node.sendrawtransaction, tx_v3_child_large1["hex"]) @@ -244,18 +245,18 @@ class MempoolTRUC(BitcoinTestFramework): self.restart_node(0, extra_args=["-limitancestorsize=10", "-datacarriersize=40000"]) tx_v3_parent_large2 = self.wallet.send_self_transfer( from_node=node, - target_weight=parent_target_weight, + target_vsize=parent_target_vsize, version=3 ) tx_v3_child_large2 = self.wallet.create_self_transfer( utxo_to_spend=tx_v3_parent_large2["new_utxo"], - target_weight=child_target_weight, + target_vsize=child_target_vsize, version=3 ) # Parent and child are within TRUC limits assert_greater_than_or_equal(TRUC_MAX_VSIZE, tx_v3_parent_large2["tx"].get_vsize()) - assert_greater_than_or_equal(1000, tx_v3_child_large2["tx"].get_vsize()) + assert_greater_than_or_equal(TRUC_CHILD_MAX_VSIZE, tx_v3_child_large2["tx"].get_vsize()) assert_greater_than(tx_v3_parent_large2["tx"].get_vsize() + tx_v3_child_large2["tx"].get_vsize(), 10000) assert_raises_rpc_error(-26, f"too-long-mempool-chain, exceeds ancestor size limit", node.sendrawtransaction, tx_v3_child_large2["hex"]) @@ -267,12 +268,12 @@ class MempoolTRUC(BitcoinTestFramework): node = self.nodes[0] tx_v3_parent_normal = self.wallet.create_self_transfer( fee_rate=0, - target_weight=4004, + target_vsize=1001, version=3 ) tx_v3_parent_2_normal = self.wallet.create_self_transfer( fee_rate=0, - target_weight=4004, + target_vsize=1001, version=3 ) tx_v3_child_multiparent = self.wallet.create_self_transfer_multi( @@ -282,7 +283,7 @@ class MempoolTRUC(BitcoinTestFramework): ) tx_v3_child_heavy = self.wallet.create_self_transfer_multi( utxos_to_spend=[tx_v3_parent_normal["new_utxo"]], - target_weight=4004, + target_vsize=TRUC_CHILD_MAX_VSIZE + 1, fee_per_output=10000, version=3 ) @@ -294,7 +295,7 @@ class MempoolTRUC(BitcoinTestFramework): self.check_mempool([]) result = node.submitpackage([tx_v3_parent_normal["hex"], tx_v3_child_heavy["hex"]]) - # tx_v3_child_heavy is heavy based on weight, not sigops. + # tx_v3_child_heavy is heavy based on vsize, not sigops. assert_equal(result['package_msg'], f"TRUC-violation, version=3 child tx {tx_v3_child_heavy['txid']} (wtxid={tx_v3_child_heavy['wtxid']}) is too big: {tx_v3_child_heavy['tx'].get_vsize()} > 1000 virtual bytes") self.check_mempool([]) @@ -416,7 +417,7 @@ class MempoolTRUC(BitcoinTestFramework): node = self.nodes[0] tx_v3_parent = self.wallet.create_self_transfer( fee_rate=0, - target_weight=4004, + target_vsize=1001, version=3 ) tx_v2_child = self.wallet.create_self_transfer_multi( diff --git a/test/functional/p2p_1p1c_network.py b/test/functional/p2p_1p1c_network.py index c3cdb3e0b3..cdc4e1691d 100755 --- a/test/functional/p2p_1p1c_network.py +++ b/test/functional/p2p_1p1c_network.py @@ -49,9 +49,6 @@ class PackageRelayTest(BitcoinTestFramework): def raise_network_minfee(self): fill_mempool(self, self.nodes[0]) - self.log.debug("Wait for the network to sync mempools") - self.sync_mempools() - self.log.debug("Check that all nodes' mempool minimum feerates are above min relay feerate") for node in self.nodes: assert_equal(node.getmempoolinfo()['minrelaytxfee'], FEERATE_1SAT_VB) @@ -107,7 +104,7 @@ class PackageRelayTest(BitcoinTestFramework): # 3: 2-parent-1-child package. Both parents are above mempool min feerate. No package submission happens. # We require packages to be child-with-unconfirmed-parents and only allow 1-parent-1-child packages. - package_hex_3, parent_31, parent_32, child_3 = self.create_package_2p1c(self.wallet) + package_hex_3, parent_31, _parent_32, child_3 = self.create_package_2p1c(self.wallet) # 4: parent + child package where the child spends 2 different outputs from the parent. package_hex_4, parent_4, child_4 = self.create_package_2outs(self.wallet) diff --git a/test/functional/p2p_invalid_tx.py b/test/functional/p2p_invalid_tx.py index 241aefab24..ee8c6c16ca 100755 --- a/test/functional/p2p_invalid_tx.py +++ b/test/functional/p2p_invalid_tx.py @@ -165,7 +165,7 @@ class InvalidTxRequestTest(BitcoinTestFramework): node.p2ps[0].send_txs_and_test([rejected_parent], node, success=False) self.log.info('Test that a peer disconnection causes erase its transactions from the orphan pool') - with node.assert_debug_log(['Erased 100 orphan transaction(s) from peer=25']): + with node.assert_debug_log(['Erased 100 orphan transaction(s) from peer=26']): self.reconnect_p2p(num_connections=1) self.log.info('Test that a transaction in the orphan pool is included in a new tip block causes erase this transaction from the orphan pool') diff --git a/test/functional/p2p_node_network_limited.py b/test/functional/p2p_node_network_limited.py index df6e6a2e28..7788be6adb 100755 --- a/test/functional/p2p_node_network_limited.py +++ b/test/functional/p2p_node_network_limited.py @@ -102,10 +102,10 @@ class NodeNetworkLimitedTest(BitcoinTestFramework): tip_height = pruned_node.getblockcount() limit_buffer = 2 # Prevent races by waiting for the tip to arrive first - self.wait_until(lambda: not try_rpc(-1, "Block not found", full_node.getblock, pruned_node.getbestblockhash())) + self.wait_until(lambda: not try_rpc(-1, "Block not available (not fully downloaded)", full_node.getblock, pruned_node.getbestblockhash())) for height in range(start_height_full_node + 1, tip_height + 1): if height <= tip_height - (NODE_NETWORK_LIMITED_MIN_BLOCKS - limit_buffer): - assert_raises_rpc_error(-1, "Block not found on disk", full_node.getblock, pruned_node.getblockhash(height)) + assert_raises_rpc_error(-1, "Block not available (not fully downloaded)", full_node.getblock, pruned_node.getblockhash(height)) else: full_node.getblock(pruned_node.getblockhash(height)) # just assert it does not throw an exception diff --git a/test/functional/p2p_permissions.py b/test/functional/p2p_permissions.py index c881dd6ff4..c37061c307 100755 --- a/test/functional/p2p_permissions.py +++ b/test/functional/p2p_permissions.py @@ -14,8 +14,10 @@ from test_framework.p2p import P2PDataStore from test_framework.test_node import ErrorMatch from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( + append_config, assert_equal, p2p_port, + tor_port, ) from test_framework.wallet import MiniWallet @@ -57,11 +59,14 @@ class P2PPermissionsTests(BitcoinTestFramework): # by modifying the configuration file. ip_port = "127.0.0.1:{}".format(p2p_port(1)) self.nodes[1].replace_in_config([("bind=127.0.0.1", "whitebind=bloomfilter,forcerelay@" + ip_port)]) + # Explicitly bind the tor port to prevent collisions with the default tor port + append_config(self.nodes[1].datadir_path, [f"bind=127.0.0.1:{tor_port(self.nodes[1].index)}=onion"]) self.checkpermission( ["-whitelist=noban@127.0.0.1"], # Check parameter interaction forcerelay should activate relay ["noban", "bloomfilter", "forcerelay", "relay", "download"]) self.nodes[1].replace_in_config([("whitebind=bloomfilter,forcerelay@" + ip_port, "bind=127.0.0.1")]) + self.nodes[1].replace_in_config([(f"bind=127.0.0.1:{tor_port(self.nodes[1].index)}=onion", "")]) self.checkpermission( # legacy whitelistrelay should be ignored diff --git a/test/functional/p2p_seednode.py b/test/functional/p2p_seednode.py new file mode 100755 index 0000000000..6c510a6a0b --- /dev/null +++ b/test/functional/p2p_seednode.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +# Copyright (c) 2019-2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" +Test seednode interaction with the AddrMan +""" +import random +import time + +from test_framework.test_framework import BitcoinTestFramework + +ADD_NEXT_SEEDNODE = 10 + + +class P2PSeedNodes(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + self.disable_autoconnect = False + + def test_no_seednode(self): + # Check that if no seednode is provided, the node proceeds as usual (without waiting) + with self.nodes[0].assert_debug_log(expected_msgs=[], unexpected_msgs=["Empty addrman, adding seednode", f"Couldn't connect to peers from addrman after {ADD_NEXT_SEEDNODE} seconds. Adding seednode"], timeout=ADD_NEXT_SEEDNODE): + self.restart_node(0) + + def test_seednode_empty_addrman(self): + seed_node = "0.0.0.1" + # Check that the seednode is added to m_addr_fetches on bootstrap on an empty addrman + with self.nodes[0].assert_debug_log(expected_msgs=[f"Empty addrman, adding seednode ({seed_node}) to addrfetch"], timeout=ADD_NEXT_SEEDNODE): + self.restart_node(0, extra_args=[f'-seednode={seed_node}']) + + def test_seednode_addrman_unreachable_peers(self): + seed_node = "0.0.0.2" + node = self.nodes[0] + # Fill the addrman with unreachable nodes + for i in range(10): + ip = f"{random.randrange(128,169)}.{random.randrange(1,255)}.{random.randrange(1,255)}.{random.randrange(1,255)}" + port = 8333 + i + node.addpeeraddress(ip, port) + + # Restart the node so seednode is processed again + with node.assert_debug_log(expected_msgs=[f"Couldn't connect to peers from addrman after {ADD_NEXT_SEEDNODE} seconds. Adding seednode ({seed_node}) to addrfetch"], unexpected_msgs=["Empty addrman, adding seednode"], timeout=ADD_NEXT_SEEDNODE * 1.5): + self.restart_node(0, extra_args=[f'-seednode={seed_node}']) + node.setmocktime(int(time.time()) + ADD_NEXT_SEEDNODE + 1) + + def run_test(self): + self.test_no_seednode() + self.test_seednode_empty_addrman() + self.test_seednode_addrman_unreachable_peers() + + +if __name__ == '__main__': + P2PSeedNodes(__file__).main() + diff --git a/test/functional/p2p_tx_download.py b/test/functional/p2p_tx_download.py index 11b4d9cc3b..c69d6ff405 100755 --- a/test/functional/p2p_tx_download.py +++ b/test/functional/p2p_tx_download.py @@ -156,9 +156,9 @@ class TxDownloadTest(BitcoinTestFramework): # One of the peers is asked for the tx peer2.wait_until(lambda: sum(p.tx_getdata_count for p in [peer1, peer2]) == 1) with p2p_lock: - peer_expiry, peer_fallback = (peer1, peer2) if peer1.tx_getdata_count == 1 else (peer2, peer1) + _peer_expiry, peer_fallback = (peer1, peer2) if peer1.tx_getdata_count == 1 else (peer2, peer1) assert_equal(peer_fallback.tx_getdata_count, 0) - self.nodes[0].setmocktime(int(time.time()) + GETDATA_TX_INTERVAL + 1) # Wait for request to peer_expiry to expire + self.nodes[0].setmocktime(int(time.time()) + GETDATA_TX_INTERVAL + 1) # Wait for request to _peer_expiry to expire peer_fallback.wait_until(lambda: peer_fallback.tx_getdata_count >= 1, timeout=1) self.restart_node(0) # reset mocktime @@ -250,7 +250,7 @@ class TxDownloadTest(BitcoinTestFramework): def test_rejects_filter_reset(self): self.log.info('Check that rejected tx is not requested again') node = self.nodes[0] - fill_mempool(self, node) + fill_mempool(self, node, tx_sync_fun=self.no_op) self.wallet.rescan_utxos() mempoolminfee = node.getmempoolinfo()['mempoolminfee'] peer = node.add_p2p_connection(TestP2PConn()) diff --git a/test/functional/p2p_unrequested_blocks.py b/test/functional/p2p_unrequested_blocks.py index 835ecbf184..1430131a97 100755 --- a/test/functional/p2p_unrequested_blocks.py +++ b/test/functional/p2p_unrequested_blocks.py @@ -119,7 +119,7 @@ class AcceptBlockTest(BitcoinTestFramework): assert_equal(x['status'], "headers-only") tip_entry_found = True assert tip_entry_found - assert_raises_rpc_error(-1, "Block not found on disk", self.nodes[0].getblock, block_h1f.hash) + assert_raises_rpc_error(-1, "Block not available (not fully downloaded)", self.nodes[0].getblock, block_h1f.hash) # 4. Send another two block that build on the fork. block_h2f = create_block(block_h1f.sha256, create_coinbase(2), block_time) @@ -191,7 +191,7 @@ class AcceptBlockTest(BitcoinTestFramework): # Blocks 1-287 should be accepted, block 288 should be ignored because it's too far ahead for x in all_blocks[:-1]: self.nodes[0].getblock(x.hash) - assert_raises_rpc_error(-1, "Block not found on disk", self.nodes[0].getblock, all_blocks[-1].hash) + assert_raises_rpc_error(-1, "Block not available (not fully downloaded)", self.nodes[0].getblock, all_blocks[-1].hash) # 5. Test handling of unrequested block on the node that didn't process # Should still not be processed (even though it has a child that has more @@ -230,7 +230,7 @@ class AcceptBlockTest(BitcoinTestFramework): assert_equal(self.nodes[0].getblockcount(), 290) self.nodes[0].getblock(all_blocks[286].hash) assert_equal(self.nodes[0].getbestblockhash(), all_blocks[286].hash) - assert_raises_rpc_error(-1, "Block not found on disk", self.nodes[0].getblock, all_blocks[287].hash) + assert_raises_rpc_error(-1, "Block not available (not fully downloaded)", self.nodes[0].getblock, all_blocks[287].hash) self.log.info("Successfully reorged to longer chain") # 8. Create a chain which is invalid at a height longer than the @@ -260,7 +260,7 @@ class AcceptBlockTest(BitcoinTestFramework): assert_equal(x['status'], "headers-only") tip_entry_found = True assert tip_entry_found - assert_raises_rpc_error(-1, "Block not found on disk", self.nodes[0].getblock, block_292.hash) + assert_raises_rpc_error(-1, "Block not available (not fully downloaded)", self.nodes[0].getblock, block_292.hash) test_node.send_message(msg_block(block_289f)) test_node.send_and_ping(msg_block(block_290f)) diff --git a/test/functional/rpc_bind.py b/test/functional/rpc_bind.py index 8c76c1f5f5..69afd45b9a 100755 --- a/test/functional/rpc_bind.py +++ b/test/functional/rpc_bind.py @@ -45,6 +45,19 @@ class RPCBindTest(BitcoinTestFramework): assert_equal(set(get_bind_addrs(pid)), set(expected)) self.stop_nodes() + def run_invalid_bind_test(self, allow_ips, addresses): + ''' + Attempt to start a node with requested rpcallowip and rpcbind + parameters, expecting that the node will fail. + ''' + self.log.info(f'Invalid bind test for {addresses}') + base_args = ['-disablewallet', '-nolisten'] + if allow_ips: + base_args += ['-rpcallowip=' + x for x in allow_ips] + init_error = 'Error: Invalid port specified in -rpcbind: ' + for addr in addresses: + self.nodes[0].assert_start_raises_init_error(base_args + [f'-rpcbind={addr}'], init_error + f"'{addr}'") + def run_allowip_test(self, allow_ips, rpchost, rpcport): ''' Start a node with rpcallow IP, and request getnetworkinfo @@ -84,6 +97,10 @@ class RPCBindTest(BitcoinTestFramework): if not self.options.run_nonloopback: self._run_loopback_tests() + if self.options.run_ipv4: + self.run_invalid_bind_test(['127.0.0.1'], ['127.0.0.1:notaport', '127.0.0.1:-18443', '127.0.0.1:0', '127.0.0.1:65536']) + if self.options.run_ipv6: + self.run_invalid_bind_test(['[::1]'], ['[::1]:notaport', '[::1]:-18443', '[::1]:0', '[::1]:65536']) if not self.options.run_ipv4 and not self.options.run_ipv6: self._run_nonloopback_tests() diff --git a/test/functional/rpc_blockchain.py b/test/functional/rpc_blockchain.py index 98147237b1..f02e6914ef 100755 --- a/test/functional/rpc_blockchain.py +++ b/test/functional/rpc_blockchain.py @@ -32,14 +32,16 @@ from test_framework.blocktools import ( TIME_GENESIS_BLOCK, create_block, create_coinbase, + create_tx_with_script, ) from test_framework.messages import ( CBlockHeader, + COIN, from_hex, msg_block, ) from test_framework.p2p import P2PInterface -from test_framework.script import hash256 +from test_framework.script import hash256, OP_TRUE from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, @@ -88,6 +90,7 @@ class BlockchainTest(BitcoinTestFramework): self._test_getdifficulty() self._test_getnetworkhashps() self._test_stopatheight() + self._test_waitforblock() # also tests waitfornewblock self._test_waitforblockheight() self._test_getblock() self._test_getdeploymentinfo() @@ -505,6 +508,38 @@ class BlockchainTest(BitcoinTestFramework): self.start_node(0) assert_equal(self.nodes[0].getblockcount(), HEIGHT + 7) + def _test_waitforblock(self): + self.log.info("Test waitforblock and waitfornewblock") + node = self.nodes[0] + + current_height = node.getblock(node.getbestblockhash())['height'] + current_hash = node.getblock(node.getbestblockhash())['hash'] + + self.log.debug("Roll the chain back a few blocks and then reconsider it") + rollback_height = current_height - 100 + rollback_hash = node.getblockhash(rollback_height) + rollback_header = node.getblockheader(rollback_hash) + + node.invalidateblock(rollback_hash) + assert_equal(node.getblockcount(), rollback_height - 1) + + self.log.debug("waitforblock should return the same block after its timeout") + assert_equal(node.waitforblock(blockhash=current_hash, timeout=1)['hash'], rollback_header['previousblockhash']) + + node.reconsiderblock(rollback_hash) + # The chain has probably already been restored by the time reconsiderblock returns, + # but poll anyway. + self.wait_until(lambda: node.waitforblock(blockhash=current_hash, timeout=100)['hash'] == current_hash) + + # roll back again + node.invalidateblock(rollback_hash) + assert_equal(node.getblockcount(), rollback_height - 1) + + node.reconsiderblock(rollback_hash) + # The chain has probably already been restored by the time reconsiderblock returns, + # but poll anyway. + self.wait_until(lambda: node.waitfornewblock(timeout=100)['hash'] == current_hash) + def _test_waitforblockheight(self): self.log.info("Test waitforblockheight") node = self.nodes[0] @@ -556,12 +591,12 @@ class BlockchainTest(BitcoinTestFramework): block = node.getblock(blockhash, verbosity) assert_equal(blockhash, hash256(bytes.fromhex(block[:160]))[::-1].hex()) - def assert_fee_not_in_block(verbosity): - block = node.getblock(blockhash, verbosity) + def assert_fee_not_in_block(hash, verbosity): + block = node.getblock(hash, verbosity) assert 'fee' not in block['tx'][1] - def assert_fee_in_block(verbosity): - block = node.getblock(blockhash, verbosity) + def assert_fee_in_block(hash, verbosity): + block = node.getblock(hash, verbosity) tx = block['tx'][1] assert 'fee' in tx assert_equal(tx['fee'], tx['vsize'] * fee_per_byte) @@ -580,8 +615,8 @@ class BlockchainTest(BitcoinTestFramework): total_vout += vout["value"] assert_equal(total_vin, total_vout + tx["fee"]) - def assert_vin_does_not_contain_prevout(verbosity): - block = node.getblock(blockhash, verbosity) + def assert_vin_does_not_contain_prevout(hash, verbosity): + block = node.getblock(hash, verbosity) tx = block["tx"][1] if isinstance(tx, str): # In verbosity level 1, only the transaction hashes are written @@ -595,16 +630,16 @@ class BlockchainTest(BitcoinTestFramework): assert_hexblock_hashes(False) self.log.info("Test that getblock with verbosity 1 doesn't include fee") - assert_fee_not_in_block(1) - assert_fee_not_in_block(True) + assert_fee_not_in_block(blockhash, 1) + assert_fee_not_in_block(blockhash, True) self.log.info('Test that getblock with verbosity 2 and 3 includes expected fee') - assert_fee_in_block(2) - assert_fee_in_block(3) + assert_fee_in_block(blockhash, 2) + assert_fee_in_block(blockhash, 3) self.log.info("Test that getblock with verbosity 1 and 2 does not include prevout") - assert_vin_does_not_contain_prevout(1) - assert_vin_does_not_contain_prevout(2) + assert_vin_does_not_contain_prevout(blockhash, 1) + assert_vin_does_not_contain_prevout(blockhash, 2) self.log.info("Test that getblock with verbosity 3 includes prevout") assert_vin_contains_prevout(3) @@ -612,7 +647,7 @@ class BlockchainTest(BitcoinTestFramework): self.log.info("Test getblock with invalid verbosity type returns proper error message") assert_raises_rpc_error(-3, "JSON value of type string is not of expected type number", node.getblock, blockhash, "2") - self.log.info("Test that getblock with verbosity 2 and 3 still works with pruned Undo data") + self.log.info("Test that getblock doesn't work with deleted Undo data") def move_block_file(old, new): old_path = self.nodes[0].blocks_path / old @@ -622,10 +657,8 @@ class BlockchainTest(BitcoinTestFramework): # Move instead of deleting so we can restore chain state afterwards move_block_file('rev00000.dat', 'rev_wrong') - assert_fee_not_in_block(2) - assert_fee_not_in_block(3) - assert_vin_does_not_contain_prevout(2) - assert_vin_does_not_contain_prevout(3) + assert_raises_rpc_error(-32603, "Undo data expected but can't be read. This could be due to disk corruption or a conflict with a pruning event.", lambda: node.getblock(blockhash, 2)) + assert_raises_rpc_error(-32603, "Undo data expected but can't be read. This could be due to disk corruption or a conflict with a pruning event.", lambda: node.getblock(blockhash, 3)) # Restore chain state move_block_file('rev_wrong', 'rev00000.dat') @@ -633,6 +666,31 @@ class BlockchainTest(BitcoinTestFramework): assert 'previousblockhash' not in node.getblock(node.getblockhash(0)) assert 'nextblockhash' not in node.getblock(node.getbestblockhash()) + self.log.info("Test getblock when only header is known") + current_height = node.getblock(node.getbestblockhash())['height'] + block_time = node.getblock(node.getbestblockhash())['time'] + 1 + block = create_block(int(blockhash, 16), create_coinbase(current_height + 1, nValue=100), block_time) + block.solve() + node.submitheader(block.serialize().hex()) + assert_raises_rpc_error(-1, "Block not available (not fully downloaded)", lambda: node.getblock(block.hash)) + + self.log.info("Test getblock when block data is available but undo data isn't") + # Submits a block building on the header-only block, so it can't be connected and has no undo data + tx = create_tx_with_script(block.vtx[0], 0, script_sig=bytes([OP_TRUE]), amount=50 * COIN) + block_noundo = create_block(block.sha256, create_coinbase(current_height + 2, nValue=100), block_time + 1, txlist=[tx]) + block_noundo.solve() + node.submitblock(block_noundo.serialize().hex()) + + assert_fee_not_in_block(block_noundo.hash, 2) + assert_fee_not_in_block(block_noundo.hash, 3) + assert_vin_does_not_contain_prevout(block_noundo.hash, 2) + assert_vin_does_not_contain_prevout(block_noundo.hash, 3) + + self.log.info("Test getblock when block is missing") + move_block_file('blk00000.dat', 'blk00000.dat.bak') + assert_raises_rpc_error(-1, "Block not found on disk", node.getblock, blockhash) + move_block_file('blk00000.dat.bak', 'blk00000.dat') + if __name__ == '__main__': BlockchainTest(__file__).main() diff --git a/test/functional/rpc_createmultisig.py b/test/functional/rpc_createmultisig.py index 9f4e17a328..d95820bbf8 100755 --- a/test/functional/rpc_createmultisig.py +++ b/test/functional/rpc_createmultisig.py @@ -47,7 +47,7 @@ class RpcCreateMultiSigTest(BitcoinTestFramework): return node.get_wallet_rpc(wallet_name) def run_test(self): - node0, node1, node2 = self.nodes + node0, node1, _node2 = self.nodes self.wallet = MiniWallet(test_node=node0) if self.is_wallet_compiled(): @@ -122,7 +122,7 @@ class RpcCreateMultiSigTest(BitcoinTestFramework): assert_raises_rpc_error(-4, "Unsupported multisig script size for legacy wallet. Upgrade to descriptors to overcome this limitation for p2sh-segwit or bech32 scripts", wallet_multi.addmultisigaddress, 16, pubkeys, '', 'bech32') def do_multisig(self, nkeys, nsigs, output_type, wallet_multi): - node0, node1, node2 = self.nodes + node0, _node1, node2 = self.nodes pub_keys = self.pub[0: nkeys] priv_keys = self.priv[0: nkeys] diff --git a/test/functional/rpc_dumptxoutset.py b/test/functional/rpc_dumptxoutset.py index aa12da6ceb..ad05060210 100755 --- a/test/functional/rpc_dumptxoutset.py +++ b/test/functional/rpc_dumptxoutset.py @@ -19,6 +19,17 @@ class DumptxoutsetTest(BitcoinTestFramework): self.setup_clean_chain = True self.num_nodes = 1 + def check_expected_network(self, node, active): + rev_file = node.blocks_path / "rev00000.dat" + bogus_file = node.blocks_path / "bogus.dat" + rev_file.rename(bogus_file) + assert_raises_rpc_error( + -1, 'Could not roll back to requested height.', node.dumptxoutset, 'utxos.dat', rollback=99) + assert_equal(node.getnetworkinfo()['networkactive'], active) + + # Cleanup + bogus_file.rename(rev_file) + def run_test(self): """Test a trivial usage of the dumptxoutset RPC command.""" node = self.nodes[0] @@ -27,8 +38,8 @@ class DumptxoutsetTest(BitcoinTestFramework): self.generate(node, COINBASE_MATURITY) FILENAME = 'txoutset.dat' - out = node.dumptxoutset(FILENAME) - expected_path = node.datadir_path / self.chain / FILENAME + out = node.dumptxoutset(FILENAME, "latest") + expected_path = node.chain_path / FILENAME assert expected_path.is_file() @@ -51,10 +62,22 @@ class DumptxoutsetTest(BitcoinTestFramework): # Specifying a path to an existing or invalid file will fail. assert_raises_rpc_error( - -8, '{} already exists'.format(FILENAME), node.dumptxoutset, FILENAME) + -8, '{} already exists'.format(FILENAME), node.dumptxoutset, FILENAME, "latest") invalid_path = node.datadir_path / "invalid" / "path" assert_raises_rpc_error( - -8, "Couldn't open file {}.incomplete for writing".format(invalid_path), node.dumptxoutset, invalid_path) + -8, "Couldn't open file {}.incomplete for writing".format(invalid_path), node.dumptxoutset, invalid_path, "latest") + + self.log.info(f"Test that dumptxoutset with unknown dump type fails") + assert_raises_rpc_error( + -8, 'Invalid snapshot type "bogus" specified. Please specify "rollback" or "latest"', node.dumptxoutset, 'utxos.dat', "bogus") + + self.log.info(f"Test that dumptxoutset failure does not leave the network activity suspended when it was on previously") + self.check_expected_network(node, True) + + self.log.info(f"Test that dumptxoutset failure leaves the network activity suspended when it was off") + node.setnetworkactive(False) + self.check_expected_network(node, False) + node.setnetworkactive(True) if __name__ == '__main__': diff --git a/test/functional/rpc_getblockfrompeer.py b/test/functional/rpc_getblockfrompeer.py index e309018516..62b3d664e0 100755 --- a/test/functional/rpc_getblockfrompeer.py +++ b/test/functional/rpc_getblockfrompeer.py @@ -58,7 +58,7 @@ class GetBlockFromPeerTest(BitcoinTestFramework): self.log.info("Node 0 should only have the header for node 1's block 3") x = next(filter(lambda x: x['hash'] == short_tip, self.nodes[0].getchaintips())) assert_equal(x['status'], "headers-only") - assert_raises_rpc_error(-1, "Block not found on disk", self.nodes[0].getblock, short_tip) + assert_raises_rpc_error(-1, "Block not available (not fully downloaded)", self.nodes[0].getblock, short_tip) self.log.info("Fetch block from node 1") peers = self.nodes[0].getpeerinfo() diff --git a/test/functional/rpc_getblockstats.py b/test/functional/rpc_getblockstats.py index d1e4895eb6..002763201a 100755 --- a/test/functional/rpc_getblockstats.py +++ b/test/functional/rpc_getblockstats.py @@ -114,7 +114,7 @@ class GetblockstatsTest(BitcoinTestFramework): assert_equal(stats[self.max_stat_pos]['height'], self.start_height + self.max_stat_pos) for i in range(self.max_stat_pos+1): - self.log.info('Checking block %d\n' % (i)) + self.log.info('Checking block %d' % (i)) assert_equal(stats[i], self.expected_stats[i]) # Check selecting block by hash too @@ -182,5 +182,16 @@ class GetblockstatsTest(BitcoinTestFramework): assert_equal(tip_stats["utxo_increase_actual"], 4) assert_equal(tip_stats["utxo_size_inc_actual"], 300) + self.log.info("Test when only header is known") + block = self.generateblock(self.nodes[0], output="raw(55)", transactions=[], submit=False) + self.nodes[0].submitheader(block["hex"]) + assert_raises_rpc_error(-1, "Block not available (not fully downloaded)", lambda: self.nodes[0].getblockstats(block['hash'])) + + self.log.info('Test when block is missing') + (self.nodes[0].blocks_path / 'blk00000.dat').rename(self.nodes[0].blocks_path / 'blk00000.dat.backup') + assert_raises_rpc_error(-1, 'Block not found on disk', self.nodes[0].getblockstats, hash_or_height=1) + (self.nodes[0].blocks_path / 'blk00000.dat.backup').rename(self.nodes[0].blocks_path / 'blk00000.dat') + + if __name__ == '__main__': GetblockstatsTest(__file__).main() diff --git a/test/functional/rpc_getorphantxs.py b/test/functional/rpc_getorphantxs.py new file mode 100755 index 0000000000..8d32ce1638 --- /dev/null +++ b/test/functional/rpc_getorphantxs.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +# Copyright (c) 2014-2024 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test the getorphantxs RPC.""" + +from test_framework.mempool_util import tx_in_orphanage +from test_framework.messages import msg_tx +from test_framework.p2p import P2PInterface +from test_framework.util import assert_equal +from test_framework.test_framework import BitcoinTestFramework +from test_framework.wallet import MiniWallet + + +class GetOrphanTxsTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + + def run_test(self): + self.wallet = MiniWallet(self.nodes[0]) + self.test_orphan_activity() + self.test_orphan_details() + + def test_orphan_activity(self): + self.log.info("Check that orphaned transactions are returned with getorphantxs") + node = self.nodes[0] + + self.log.info("Create two 1P1C packages, but only broadcast the children") + tx_parent_1 = self.wallet.create_self_transfer() + tx_child_1 = self.wallet.create_self_transfer(utxo_to_spend=tx_parent_1["new_utxo"]) + tx_parent_2 = self.wallet.create_self_transfer() + tx_child_2 = self.wallet.create_self_transfer(utxo_to_spend=tx_parent_2["new_utxo"]) + peer = node.add_p2p_connection(P2PInterface()) + peer.send_and_ping(msg_tx(tx_child_1["tx"])) + peer.send_and_ping(msg_tx(tx_child_2["tx"])) + + self.log.info("Check that neither parent is in the mempool") + assert_equal(node.getmempoolinfo()["size"], 0) + + self.log.info("Check that both children are in the orphanage") + + orphanage = node.getorphantxs(verbosity=0) + self.log.info("Check the size of the orphanage") + assert_equal(len(orphanage), 2) + self.log.info("Check that negative verbosity is treated as 0") + assert_equal(orphanage, node.getorphantxs(verbosity=-1)) + assert tx_in_orphanage(node, tx_child_1["tx"]) + assert tx_in_orphanage(node, tx_child_2["tx"]) + + self.log.info("Broadcast parent 1") + peer.send_and_ping(msg_tx(tx_parent_1["tx"])) + self.log.info("Check that parent 1 and child 1 are in the mempool") + raw_mempool = node.getrawmempool() + assert_equal(len(raw_mempool), 2) + assert tx_parent_1["txid"] in raw_mempool + assert tx_child_1["txid"] in raw_mempool + + self.log.info("Check that orphanage only contains child 2") + orphanage = node.getorphantxs() + assert_equal(len(orphanage), 1) + assert tx_in_orphanage(node, tx_child_2["tx"]) + + peer.send_and_ping(msg_tx(tx_parent_2["tx"])) + self.log.info("Check that all parents and children are now in the mempool") + raw_mempool = node.getrawmempool() + assert_equal(len(raw_mempool), 4) + assert tx_parent_1["txid"] in raw_mempool + assert tx_child_1["txid"] in raw_mempool + assert tx_parent_2["txid"] in raw_mempool + assert tx_child_2["txid"] in raw_mempool + self.log.info("Check that the orphanage is empty") + assert_equal(len(node.getorphantxs()), 0) + + self.log.info("Confirm the transactions (clears mempool)") + self.generate(node, 1) + assert_equal(node.getmempoolinfo()["size"], 0) + + def test_orphan_details(self): + self.log.info("Check the transaction details returned from getorphantxs") + node = self.nodes[0] + + self.log.info("Create two orphans, from different peers") + tx_parent_1 = self.wallet.create_self_transfer() + tx_child_1 = self.wallet.create_self_transfer(utxo_to_spend=tx_parent_1["new_utxo"]) + tx_parent_2 = self.wallet.create_self_transfer() + tx_child_2 = self.wallet.create_self_transfer(utxo_to_spend=tx_parent_2["new_utxo"]) + peer_1 = node.add_p2p_connection(P2PInterface()) + peer_2 = node.add_p2p_connection(P2PInterface()) + peer_1.send_and_ping(msg_tx(tx_child_1["tx"])) + peer_2.send_and_ping(msg_tx(tx_child_2["tx"])) + + orphanage = node.getorphantxs(verbosity=2) + assert tx_in_orphanage(node, tx_child_1["tx"]) + assert tx_in_orphanage(node, tx_child_2["tx"]) + + self.log.info("Check that orphan 1 and 2 were from different peers") + assert orphanage[0]["from"][0] != orphanage[1]["from"][0] + + self.log.info("Unorphan child 2") + peer_2.send_and_ping(msg_tx(tx_parent_2["tx"])) + assert not tx_in_orphanage(node, tx_child_2["tx"]) + + self.log.info("Checking orphan details") + orphanage = node.getorphantxs(verbosity=1) + assert_equal(len(node.getorphantxs()), 1) + orphan_1 = orphanage[0] + self.orphan_details_match(orphan_1, tx_child_1, verbosity=1) + + self.log.info("Checking orphan details (verbosity 2)") + orphanage = node.getorphantxs(verbosity=2) + orphan_1 = orphanage[0] + self.orphan_details_match(orphan_1, tx_child_1, verbosity=2) + + def orphan_details_match(self, orphan, tx, verbosity): + self.log.info("Check txid/wtxid of orphan") + assert_equal(orphan["txid"], tx["txid"]) + assert_equal(orphan["wtxid"], tx["wtxid"]) + + self.log.info("Check the sizes of orphan") + assert_equal(orphan["bytes"], len(tx["tx"].serialize())) + assert_equal(orphan["vsize"], tx["tx"].get_vsize()) + assert_equal(orphan["weight"], tx["tx"].get_weight()) + + if verbosity == 2: + self.log.info("Check the transaction hex of orphan") + assert_equal(orphan["hex"], tx["hex"]) + + +if __name__ == '__main__': + GetOrphanTxsTest(__file__).main() diff --git a/test/functional/rpc_txoutproof.py b/test/functional/rpc_txoutproof.py index 387132b680..90572245d6 100755 --- a/test/functional/rpc_txoutproof.py +++ b/test/functional/rpc_txoutproof.py @@ -67,6 +67,10 @@ class MerkleBlockTest(BitcoinTestFramework): assert_equal(self.nodes[0].verifytxoutproof(self.nodes[0].gettxoutproof([txid_spent], blockhash)), [txid_spent]) # We can't get the proof if we specify a non-existent block assert_raises_rpc_error(-5, "Block not found", self.nodes[0].gettxoutproof, [txid_spent], "0000000000000000000000000000000000000000000000000000000000000000") + # We can't get the proof if we only have the header of the specified block + block = self.generateblock(self.nodes[0], output="raw(55)", transactions=[], submit=False) + self.nodes[0].submitheader(block["hex"]) + assert_raises_rpc_error(-1, "Block not available (not fully downloaded)", self.nodes[0].gettxoutproof, [txid_spent], block['hash']) # We can get the proof if the transaction is unspent assert_equal(self.nodes[0].verifytxoutproof(self.nodes[0].gettxoutproof([txid_unspent])), [txid_unspent]) # We can get the proof if we provide a list of transactions and one of them is unspent. The ordering of the list should not matter. diff --git a/test/functional/rpc_users.py b/test/functional/rpc_users.py index 44187ce790..49eb64abad 100755 --- a/test/functional/rpc_users.py +++ b/test/functional/rpc_users.py @@ -139,15 +139,32 @@ class HTTPBasicsTest(BitcoinTestFramework): init_error = 'Error: Unable to start HTTP server. See debug log for details.' self.log.info('Check -rpcauth are validated') - # Empty -rpcauth= are ignored - self.restart_node(0, extra_args=['-rpcauth=']) + self.log.info('Empty -rpcauth are treated as error') self.stop_node(0) + self.nodes[0].assert_start_raises_init_error(expected_msg=init_error, extra_args=['-rpcauth']) + self.nodes[0].assert_start_raises_init_error(expected_msg=init_error, extra_args=['-rpcauth=']) + self.nodes[0].assert_start_raises_init_error(expected_msg=init_error, extra_args=['-rpcauth=""']) + self.log.info('Check malformed -rpcauth') self.nodes[0].assert_start_raises_init_error(expected_msg=init_error, extra_args=['-rpcauth=foo']) self.nodes[0].assert_start_raises_init_error(expected_msg=init_error, extra_args=['-rpcauth=foo:bar']) self.nodes[0].assert_start_raises_init_error(expected_msg=init_error, extra_args=['-rpcauth=foo:bar:baz']) self.nodes[0].assert_start_raises_init_error(expected_msg=init_error, extra_args=['-rpcauth=foo$bar:baz']) self.nodes[0].assert_start_raises_init_error(expected_msg=init_error, extra_args=['-rpcauth=foo$bar$baz']) + self.log.info('Check interactions between blank and non-blank rpcauth') + # pw = bitcoin + rpcauth_user1 = '-rpcauth=user1:6dd184e5e69271fdd69103464630014f$eb3d7ce67c4d1ff3564270519b03b636c0291012692a5fa3dd1d2075daedd07b' + rpcauth_user2 = '-rpcauth=user2:57b2f77c919eece63cfa46c2f06e46ae$266b63902f99f97eeaab882d4a87f8667ab84435c3799f2ce042ef5a994d620b' + self.nodes[0].assert_start_raises_init_error(expected_msg=init_error, extra_args=[rpcauth_user1, rpcauth_user2, '-rpcauth=']) + self.nodes[0].assert_start_raises_init_error(expected_msg=init_error, extra_args=[rpcauth_user1, '-rpcauth=', rpcauth_user2]) + self.nodes[0].assert_start_raises_init_error(expected_msg=init_error, extra_args=['-rpcauth=', rpcauth_user1, rpcauth_user2]) + + self.log.info('Check -norpcauth disables previous -rpcauth params') + self.restart_node(0, extra_args=[rpcauth_user1, rpcauth_user2, '-norpcauth']) + assert_equal(401, call_with_auth(self.nodes[0], 'user1', 'bitcoin').status) + assert_equal(401, call_with_auth(self.nodes[0], 'rt', self.rtpassword).status) + self.stop_node(0) + self.log.info('Check that failure to write cookie file will abort the node gracefully') (self.nodes[0].chain_path / ".cookie.tmp").mkdir() self.nodes[0].assert_start_raises_init_error(expected_msg=init_error) diff --git a/test/functional/test_framework/blocktools.py b/test/functional/test_framework/blocktools.py index 5c2fa28a31..705b8e8fe5 100644 --- a/test/functional/test_framework/blocktools.py +++ b/test/functional/test_framework/blocktools.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2022 The Bitcoin Core developers +# Copyright (c) 2015-present The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Utilities for manipulating blocks and transactions.""" @@ -74,7 +74,7 @@ def create_block(hashprev=None, coinbase=None, ntime=None, *, version=None, tmpl block.nVersion = version or tmpl.get('version') or VERSIONBITS_LAST_OLD_BLOCK_VERSION block.nTime = ntime or tmpl.get('curtime') or int(time.time() + 600) block.hashPrevBlock = hashprev or int(tmpl['previousblockhash'], 0x10) - if tmpl and not tmpl.get('bits') is None: + if tmpl and tmpl.get('bits') is not None: block.nBits = struct.unpack('>I', bytes.fromhex(tmpl['bits']))[0] else: block.nBits = 0x207fffff # difficulty retargeting is disabled in REGTEST chainparams diff --git a/test/functional/test_framework/mempool_util.py b/test/functional/test_framework/mempool_util.py index 148cc935ed..a6a7940c60 100644 --- a/test/functional/test_framework/mempool_util.py +++ b/test/functional/test_framework/mempool_util.py @@ -8,6 +8,7 @@ from decimal import Decimal from .blocktools import ( COINBASE_MATURITY, ) +from .messages import CTransaction from .util import ( assert_equal, assert_greater_than, @@ -19,14 +20,11 @@ from .wallet import ( ) -def fill_mempool(test_framework, node): +def fill_mempool(test_framework, node, *, tx_sync_fun=None): """Fill mempool until eviction. Allows for simpler testing of scenarios with floating mempoolminfee > minrelay - Requires -datacarriersize=100000 and - -maxmempool=5. - It will not ensure mempools become synced as it - is based on a single node and assumes -minrelaytxfee + Requires -datacarriersize=100000 and -maxmempool=5 and assumes -minrelaytxfee is 1 sat/vbyte. To avoid unintentional tx dependencies, the mempool filling txs are created with a tagged ephemeral miniwallet instance. @@ -57,18 +55,25 @@ def fill_mempool(test_framework, node): tx_to_be_evicted_id = ephemeral_miniwallet.send_self_transfer( from_node=node, utxo_to_spend=confirmed_utxos.pop(0), fee_rate=relayfee)["txid"] + def send_batch(fee): + utxos = confirmed_utxos[:tx_batch_size] + create_lots_of_big_transactions(ephemeral_miniwallet, node, fee, tx_batch_size, txouts, utxos) + del confirmed_utxos[:tx_batch_size] + # Increase the tx fee rate to give the subsequent transactions a higher priority in the mempool # The tx has an approx. vsize of 65k, i.e. multiplying the previous fee rate (in sats/kvB) # by 130 should result in a fee that corresponds to 2x of that fee rate base_fee = relayfee * 130 + batch_fees = [(i + 1) * base_fee for i in range(num_of_batches)] test_framework.log.debug("Fill up the mempool with txs with higher fee rate") - with node.assert_debug_log(["rolling minimum fee bumped"]): - for batch_of_txid in range(num_of_batches): - fee = (batch_of_txid + 1) * base_fee - utxos = confirmed_utxos[:tx_batch_size] - create_lots_of_big_transactions(ephemeral_miniwallet, node, fee, tx_batch_size, txouts, utxos) - del confirmed_utxos[:tx_batch_size] + for fee in batch_fees[:-3]: + send_batch(fee) + tx_sync_fun() if tx_sync_fun else test_framework.sync_mempools() # sync before any eviction + assert_equal(node.getmempoolinfo()["mempoolminfee"], Decimal("0.00001000")) + for fee in batch_fees[-3:]: + send_batch(fee) + tx_sync_fun() if tx_sync_fun else test_framework.sync_mempools() # sync after all evictions test_framework.log.debug("The tx should be evicted by now") # The number of transactions created should be greater than the ones present in the mempool @@ -79,3 +84,8 @@ def fill_mempool(test_framework, node): test_framework.log.debug("Check that mempoolminfee is larger than minrelaytxfee") assert_equal(node.getmempoolinfo()['minrelaytxfee'], Decimal('0.00001000')) assert_greater_than(node.getmempoolinfo()['mempoolminfee'], Decimal('0.00001000')) + +def tx_in_orphanage(node, tx: CTransaction) -> bool: + """Returns true if the transaction is in the orphanage.""" + found = [o for o in node.getorphantxs(verbosity=1) if o["txid"] == tx.rehash() and o["wtxid"] == tx.getwtxid()] + return len(found) == 1 diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index 00fe5b08e4..ce68de7eaa 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -5,7 +5,7 @@ """Helpful routines for regression testing.""" from base64 import b64encode -from decimal import Decimal, ROUND_DOWN +from decimal import Decimal from subprocess import CalledProcessError import hashlib import inspect @@ -21,7 +21,9 @@ import time from . import coverage from .authproxy import AuthServiceProxy, JSONRPCException from collections.abc import Callable -from typing import Optional +from typing import Optional, Union + +SATOSHI_PRECISION = Decimal('0.00000001') logger = logging.getLogger("TestFramework.utils") @@ -261,8 +263,9 @@ def get_fee(tx_size, feerate_btc_kvb): return target_fee_sat / Decimal(1e8) # Return result in BTC -def satoshi_round(amount): - return Decimal(amount).quantize(Decimal('0.00000001'), rounding=ROUND_DOWN) +def satoshi_round(amount: Union[int, float, str], *, rounding: str) -> Decimal: + """Rounds a Decimal amount to the nearest satoshi using the specified rounding mode.""" + return Decimal(amount).quantize(SATOSHI_PRECISION, rounding=rounding) def wait_until_helper_internal(predicate, *, attempts=float('inf'), timeout=float('inf'), lock=None, timeout_factor=1.0): diff --git a/test/functional/test_framework/wallet.py b/test/functional/test_framework/wallet.py index f3713f297e..1cef714705 100644 --- a/test/functional/test_framework/wallet.py +++ b/test/functional/test_framework/wallet.py @@ -7,7 +7,6 @@ from copy import deepcopy from decimal import Decimal from enum import Enum -import math from typing import ( Any, Optional, @@ -35,7 +34,6 @@ from test_framework.messages import ( CTxOut, hash256, ser_compact_size, - WITNESS_SCALE_FACTOR, ) from test_framework.script import ( CScript, @@ -119,20 +117,18 @@ class MiniWallet: def _create_utxo(self, *, txid, vout, value, height, coinbase, confirmations): return {"txid": txid, "vout": vout, "value": value, "height": height, "coinbase": coinbase, "confirmations": confirmations} - def _bulk_tx(self, tx, target_weight): - """Pad a transaction with extra outputs until it reaches a target weight (or higher). + def _bulk_tx(self, tx, target_vsize): + """Pad a transaction with extra outputs until it reaches a target vsize. returns the tx """ tx.vout.append(CTxOut(nValue=0, scriptPubKey=CScript([OP_RETURN]))) - # determine number of needed padding bytes by converting weight difference to vbytes - dummy_vbytes = (target_weight - tx.get_weight() + 3) // 4 + # determine number of needed padding bytes + dummy_vbytes = target_vsize - tx.get_vsize() # compensate for the increase of the compact-size encoded script length # (note that the length encoding of the unpadded output script needs one byte) dummy_vbytes -= len(ser_compact_size(dummy_vbytes)) - 1 tx.vout[-1].scriptPubKey = CScript([OP_RETURN] + [OP_1] * dummy_vbytes) - # Actual weight should be at most 3 higher than target weight - assert_greater_than_or_equal(tx.get_weight(), target_weight) - assert_greater_than_or_equal(target_weight + 3, tx.get_weight()) + assert_equal(tx.get_vsize(), target_vsize) def get_balance(self): return sum(u['value'] for u in self._utxos) @@ -309,7 +305,7 @@ class MiniWallet: locktime=0, sequence=0, fee_per_output=1000, - target_weight=0, + target_vsize=0, confirmed_only=False, ): """ @@ -338,8 +334,8 @@ class MiniWallet: self.sign_tx(tx) - if target_weight: - self._bulk_tx(tx, target_weight) + if target_vsize: + self._bulk_tx(tx, target_vsize) txid = tx.rehash() return { @@ -364,7 +360,7 @@ class MiniWallet: fee_rate=Decimal("0.003"), fee=Decimal("0"), utxo_to_spend=None, - target_weight=0, + target_vsize=0, confirmed_only=False, **kwargs, ): @@ -379,20 +375,18 @@ class MiniWallet: vsize = Decimal(168) # P2PK (73 bytes scriptSig + 35 bytes scriptPubKey + 60 bytes other) else: assert False - if target_weight and not fee: # respect fee_rate if target weight is passed - # the actual weight might be off by 3 WUs, so calculate based on that (see self._bulk_tx) - max_actual_weight = target_weight + 3 - fee = get_fee(math.ceil(max_actual_weight / WITNESS_SCALE_FACTOR), fee_rate) + if target_vsize and not fee: # respect fee_rate if target vsize is passed + fee = get_fee(target_vsize, fee_rate) send_value = utxo_to_spend["value"] - (fee or (fee_rate * vsize / 1000)) # create tx tx = self.create_self_transfer_multi( utxos_to_spend=[utxo_to_spend], amount_per_output=int(COIN * send_value), - target_weight=target_weight, + target_vsize=target_vsize, **kwargs, ) - if not target_weight: + if not target_vsize: assert_equal(tx["tx"].get_vsize(), vsize) tx["new_utxo"] = tx.pop("new_utxos")[0] diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 59c37aa18f..3d8c230066 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -96,10 +96,13 @@ BASE_SCRIPTS = [ 'feature_fee_estimation.py', 'feature_taproot.py', 'feature_block.py', + 'p2p_node_network_limited.py --v1transport', + 'p2p_node_network_limited.py --v2transport', # vv Tests less than 2m vv 'mining_getblocktemplate_longpoll.py', 'p2p_segwit.py', 'feature_maxuploadtarget.py', + 'feature_assumeutxo.py', 'mempool_updatefromblock.py', 'mempool_persist.py --descriptors', # vv Tests less than 60s vv @@ -157,6 +160,7 @@ BASE_SCRIPTS = [ 'wallet_importmulti.py --legacy-wallet', 'mempool_limit.py', 'rpc_txoutproof.py', + 'rpc_getorphantxs.py', 'wallet_listreceivedby.py --legacy-wallet', 'wallet_listreceivedby.py --descriptors', 'wallet_abandonconflict.py --legacy-wallet', @@ -354,7 +358,6 @@ BASE_SCRIPTS = [ 'wallet_coinbase_category.py --descriptors', 'feature_filelock.py', 'feature_loadblock.py', - 'feature_assumeutxo.py', 'wallet_assumeutxo.py --descriptors', 'p2p_dos_header_tree.py', 'p2p_add_connections.py', @@ -385,8 +388,6 @@ BASE_SCRIPTS = [ 'feature_coinstatsindex.py', 'wallet_orphanedreward.py', 'wallet_timelock.py', - 'p2p_node_network_limited.py --v1transport', - 'p2p_node_network_limited.py --v2transport', 'p2p_permissions.py', 'feature_blocksdir.py', 'wallet_startup.py', @@ -406,6 +407,7 @@ BASE_SCRIPTS = [ 'feature_shutdown.py', 'wallet_migration.py', 'p2p_ibd_txrelay.py', + 'p2p_seednode.py', # Don't append tests at the end to avoid merge conflicts # Put them in a random line within the section that fits their approximate run-time ] @@ -445,8 +447,8 @@ def main(): help="Leave bitcoinds and test.* datadir on exit or error") parser.add_argument('--resultsfile', '-r', help='store test results (as CSV) to the provided file') - args, unknown_args = parser.parse_known_args() + fail_on_warn = args.ci if not args.ansi: global DEFAULT, BOLD, GREEN, RED DEFAULT = ("", "") @@ -487,7 +489,7 @@ def main(): if not enable_bitcoind: print("No functional tests to run.") - print("Rerun ./configure with --with-daemon and then make") + print("Re-compile with the -DBUILD_DAEMON=ON build option") sys.exit(1) # Build list of tests @@ -521,15 +523,28 @@ def main(): test_list += BASE_SCRIPTS # Remove the test cases that the user has explicitly asked to exclude. + # The user can specify a test case with or without the .py extension. if args.exclude: - exclude_tests = [test.split('.py')[0] for test in args.exclude.split(',')] - for exclude_test in exclude_tests: - # Remove <test_name>.py and <test_name>.py --arg from the test list - exclude_list = [test for test in test_list if test.split('.py')[0] == exclude_test] + + def print_warning_missing_test(test_name): + print("{}WARNING!{} Test '{}' not found in current test list. Check the --exclude list.".format(BOLD[1], BOLD[0], test_name)) + if fail_on_warn: + sys.exit(1) + + def remove_tests(exclude_list): + if not exclude_list: + print_warning_missing_test(exclude_test) for exclude_item in exclude_list: test_list.remove(exclude_item) - if not exclude_list: - print("{}WARNING!{} Test '{}' not found in current test list.".format(BOLD[1], BOLD[0], exclude_test)) + + exclude_tests = [test.strip() for test in args.exclude.split(",")] + for exclude_test in exclude_tests: + # A space in the name indicates it has arguments such as "wallet_basic.py --descriptors" + if ' ' in exclude_test: + remove_tests([test for test in test_list if test.replace('.py', '') == exclude_test.replace('.py', '')]) + else: + # Exclude all variants of a test + remove_tests([test for test in test_list if test.split('.py')[0] == exclude_test.split('.py')[0]]) if args.filter: test_list = list(filter(re.compile(args.filter).search, test_list)) @@ -552,7 +567,7 @@ def main(): f"A minimum of {MIN_NO_CLEANUP_SPACE // (1024 * 1024 * 1024)} GB of free space is required.") passon_args.append("--nocleanup") - check_script_list(src_dir=config["environment"]["SRCDIR"], fail_on_warn=args.ci) + check_script_list(src_dir=config["environment"]["SRCDIR"], fail_on_warn=fail_on_warn) check_script_prefixes() if not args.keepcache: @@ -560,7 +575,6 @@ def main(): run_tests( test_list=test_list, - src_dir=config["environment"]["SRCDIR"], build_dir=config["environment"]["BUILDDIR"], tmpdir=tmpdir, jobs=args.jobs, @@ -572,7 +586,7 @@ def main(): results_filepath=results_filepath, ) -def run_tests(*, test_list, src_dir, build_dir, tmpdir, jobs=1, enable_coverage=False, args=None, combined_logs_len=0, failfast=False, use_term_control, results_filepath=None): +def run_tests(*, test_list, build_dir, tmpdir, jobs=1, enable_coverage=False, args=None, combined_logs_len=0, failfast=False, use_term_control, results_filepath=None): args = args or [] # Warn if bitcoind is already running @@ -595,7 +609,7 @@ def run_tests(*, test_list, src_dir, build_dir, tmpdir, jobs=1, enable_coverage= print(f"{BOLD[1]}WARNING!{BOLD[0]} There may be insufficient free space in {tmpdir} to run the Bitcoin functional test suite. " f"Running the test suite with fewer than {min_space // (1024 * 1024)} MB of free space might cause tests to fail.") - tests_dir = build_dir + '/test/functional/' + tests_dir = f"{build_dir}/test/functional/" # This allows `test_runner.py` to work from an out-of-source build directory using a symlink, # a hard link or a copy on any platform. See https://github.com/bitcoin/bitcoin/pull/27561. sys.path.append(tests_dir) @@ -862,7 +876,6 @@ def check_script_list(*, src_dir, fail_on_warn): if len(missed_tests) != 0: print("%sWARNING!%s The following scripts are not being run: %s. Check the test lists in test_runner.py." % (BOLD[1], BOLD[0], str(missed_tests))) if fail_on_warn: - # On CI this warning is an error to prevent merging incomplete commits into master sys.exit(1) diff --git a/test/functional/tool_signet_miner.py b/test/functional/tool_signet_miner.py index bdefb92ae6..67fb5c9f94 100755 --- a/test/functional/tool_signet_miner.py +++ b/test/functional/tool_signet_miner.py @@ -57,6 +57,7 @@ class SignetMinerTest(BitcoinTestFramework): f'--grind-cmd={self.options.bitcoinutil} grind', '--nbits=1d00ffff', f'--set-block-time={int(time.time())}', + '--poolnum=99', ], check=True, stderr=subprocess.STDOUT) assert_equal(node.getblockcount(), 1) diff --git a/test/functional/wallet_assumeutxo.py b/test/functional/wallet_assumeutxo.py index 0bce2f137c..76cd2097a3 100755 --- a/test/functional/wallet_assumeutxo.py +++ b/test/functional/wallet_assumeutxo.py @@ -11,7 +11,9 @@ See feature_assumeutxo.py for background. - TODO: test loading a wallet (backup) on a pruned node """ +from test_framework.address import address_to_scriptpubkey from test_framework.test_framework import BitcoinTestFramework +from test_framework.messages import COIN from test_framework.util import ( assert_equal, assert_raises_rpc_error, @@ -62,8 +64,16 @@ class AssumeutxoTest(BitcoinTestFramework): for n in self.nodes: n.setmocktime(n.getblockheader(n.getbestblockhash())['time']) + # Create a wallet that we will create a backup for later (at snapshot height) n0.createwallet('w') w = n0.get_wallet_rpc("w") + w_address = w.getnewaddress() + + # Create another wallet and backup now (before snapshot height) + n0.createwallet('w2') + w2 = n0.get_wallet_rpc("w2") + w2_address = w2.getnewaddress() + w2.backupwallet("backup_w2.dat") # Generate a series of blocks that `n0` will have in the snapshot, # but that n1 doesn't yet see. In order for the snapshot to activate, @@ -84,6 +94,8 @@ class AssumeutxoTest(BitcoinTestFramework): assert_equal(n.getblockchaininfo()[ "headers"], SNAPSHOT_BASE_HEIGHT) + # This backup is created at the snapshot height, so it's + # not part of the background sync anymore w.backupwallet("backup_w.dat") self.log.info("-- Testing assumeutxo") @@ -93,7 +105,7 @@ class AssumeutxoTest(BitcoinTestFramework): self.log.info( f"Creating a UTXO snapshot at height {SNAPSHOT_BASE_HEIGHT}") - dump_output = n0.dumptxoutset('utxos.dat') + dump_output = n0.dumptxoutset('utxos.dat', "latest") assert_equal( dump_output['txoutset_hash'], @@ -103,7 +115,13 @@ class AssumeutxoTest(BitcoinTestFramework): # Mine more blocks on top of the snapshot that n1 hasn't yet seen. This # will allow us to test n1's sync-to-tip on top of a snapshot. - self.generate(n0, nblocks=100, sync_fun=self.no_op) + w_skp = address_to_scriptpubkey(w_address) + w2_skp = address_to_scriptpubkey(w2_address) + for i in range(100): + if i % 3 == 0: + self.mini_wallet.send_to(from_node=n0, scriptPubKey=w_skp, amount=1 * COIN) + self.mini_wallet.send_to(from_node=n0, scriptPubKey=w2_skp, amount=10 * COIN) + self.generate(n0, nblocks=1, sync_fun=self.no_op) assert_equal(n0.getblockcount(), FINAL_HEIGHT) assert_equal(n1.getblockcount(), START_HEIGHT) @@ -126,8 +144,13 @@ class AssumeutxoTest(BitcoinTestFramework): assert_equal(n1.getblockchaininfo()["blocks"], SNAPSHOT_BASE_HEIGHT) - self.log.info("Backup can't be loaded during background sync") - assert_raises_rpc_error(-4, "Wallet loading failed. Error loading wallet. Wallet requires blocks to be downloaded, and software does not currently support loading wallets while blocks are being downloaded out of order when using assumeutxo snapshots. Wallet should be able to load successfully after node sync reaches height 299", n1.restorewallet, "w", "backup_w.dat") + self.log.info("Backup from the snapshot height can be loaded during background sync") + n1.restorewallet("w", "backup_w.dat") + # Balance of w wallet is still still 0 because n1 has not synced yet + assert_equal(n1.getbalance(), 0) + + self.log.info("Backup from before the snapshot height can't be loaded during background sync") + assert_raises_rpc_error(-4, "Wallet loading failed. Error loading wallet. Wallet requires blocks to be downloaded, and software does not currently support loading wallets while blocks are being downloaded out of order when using assumeutxo snapshots. Wallet should be able to load successfully after node sync reaches height 299", n1.restorewallet, "w2", "backup_w2.dat") PAUSE_HEIGHT = FINAL_HEIGHT - 40 @@ -159,8 +182,15 @@ class AssumeutxoTest(BitcoinTestFramework): self.log.info("Ensuring background validation completes") self.wait_until(lambda: len(n1.getchainstates()['chainstates']) == 1) - self.log.info("Ensuring wallet can be restored from backup") - n1.restorewallet("w", "backup_w.dat") + self.log.info("Ensuring wallet can be restored from a backup that was created before the snapshot height") + n1.restorewallet("w2", "backup_w2.dat") + # Check balance of w2 wallet + assert_equal(n1.getbalance(), 340) + + # Check balance of w wallet after node is synced + n1.loadwallet("w") + w = n1.get_wallet_rpc("w") + assert_equal(w.getbalance(), 34) if __name__ == '__main__': diff --git a/test/functional/wallet_backup.py b/test/functional/wallet_backup.py index a639c34377..83267f77e1 100755 --- a/test/functional/wallet_backup.py +++ b/test/functional/wallet_backup.py @@ -140,6 +140,25 @@ class WalletBackupTest(BitcoinTestFramework): assert_raises_rpc_error(-36, error_message, node.restorewallet, wallet_name, backup_file) assert wallet_file.exists() + def test_pruned_wallet_backup(self): + self.log.info("Test loading backup on a pruned node when the backup was created close to the prune height of the restoring node") + node = self.nodes[3] + self.restart_node(3, ["-prune=1", "-fastprune=1"]) + # Ensure the chain tip is at height 214, because this test assume it is. + assert_equal(node.getchaintips()[0]["height"], 214) + # We need a few more blocks so we can actually get above an realistic + # minimal prune height + self.generate(node, 50, sync_fun=self.no_op) + # Backup created at block height 264 + node.backupwallet(node.datadir_path / 'wallet_pruned.bak') + # Generate more blocks so we can actually prune the older blocks + self.generate(node, 300, sync_fun=self.no_op) + # This gives us an actual prune height roughly in the range of 220 - 240 + node.pruneblockchain(250) + # The backup should be updated with the latest height (locator) for + # the backup to load successfully this close to the prune height + node.restorewallet(f'pruned', node.datadir_path / 'wallet_pruned.bak') + def run_test(self): self.log.info("Generating initial blockchain") self.generate(self.nodes[0], 1) @@ -242,6 +261,8 @@ class WalletBackupTest(BitcoinTestFramework): for sourcePath in sourcePaths: assert_raises_rpc_error(-4, "backup failed", self.nodes[0].backupwallet, sourcePath) + self.test_pruned_wallet_backup() + if __name__ == '__main__': WalletBackupTest(__file__).main() diff --git a/test/functional/wallet_backwards_compatibility.py b/test/functional/wallet_backwards_compatibility.py index e71283b928..775786fbb1 100755 --- a/test/functional/wallet_backwards_compatibility.py +++ b/test/functional/wallet_backwards_compatibility.py @@ -33,7 +33,7 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True - self.num_nodes = 12 + self.num_nodes = 11 # Add new version after each release: self.extra_args = [ ["-addresstype=bech32", "-whitelist=noban@127.0.0.1"], # Pre-release: use to mine blocks. noban for immediate tx relay @@ -47,7 +47,6 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): ["-nowallet", "-walletrbf=1", "-addresstype=bech32", "-whitelist=noban@127.0.0.1"], # v0.19.1 ["-nowallet", "-walletrbf=1", "-addresstype=bech32", "-whitelist=127.0.0.1"], # v0.18.1 ["-nowallet", "-walletrbf=1", "-addresstype=bech32", "-whitelist=127.0.0.1"], # v0.17.2 - ["-nowallet", "-walletrbf=1", "-addresstype=bech32", "-whitelist=127.0.0.1", "-wallet=wallet.dat"], # v0.16.3 ] self.wallet_names = [self.default_wallet_name] @@ -68,7 +67,6 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): 190100, 180100, 170200, - 160300, ]) self.start_nodes() @@ -133,18 +131,17 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): def run_test(self): node_miner = self.nodes[0] node_master = self.nodes[1] - node_v21 = self.nodes[self.num_nodes - 6] - node_v17 = self.nodes[self.num_nodes - 2] - node_v16 = self.nodes[self.num_nodes - 1] + node_v21 = self.nodes[self.num_nodes - 5] + node_v17 = self.nodes[self.num_nodes - 1] legacy_nodes = self.nodes[2:] # Nodes that support legacy wallets - legacy_only_nodes = self.nodes[-5:] # Nodes that only support legacy wallets - descriptors_nodes = self.nodes[2:-5] # Nodes that support descriptor wallets + legacy_only_nodes = self.nodes[-4:] # Nodes that only support legacy wallets + descriptors_nodes = self.nodes[2:-4] # Nodes that support descriptor wallets self.generatetoaddress(node_miner, COINBASE_MATURITY + 1, node_miner.getnewaddress()) # Sanity check the test framework: - res = node_v16.getblockchaininfo() + res = node_v17.getblockchaininfo() assert_equal(res['blocks'], COINBASE_MATURITY + 1) self.log.info("Test wallet backwards compatibility...") @@ -215,9 +212,6 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): # In descriptors wallet mode, run this test on the nodes that support descriptor wallets # In legacy wallets mode, run this test on the nodes that support legacy wallets for node in descriptors_nodes if self.options.descriptors else legacy_nodes: - if self.major_version_less_than(node, 17): - # loadwallet was introduced in v0.17.0 - continue self.log.info(f"- {node.version}") for wallet_name in ["w1", "w2", "w3"]: if self.major_version_less_than(node, 18) and wallet_name == "w3": @@ -290,15 +284,6 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): node_v17.assert_start_raises_init_error(["-wallet=w3"], "Error: Error loading w3: Wallet requires newer version of Bitcoin Core") self.start_node(node_v17.index) - # No wallet created in master can be opened in 0.16 - self.log.info("Test that wallets created in master are too new for 0.16") - self.stop_node(node_v16.index) - for wallet_name in ["w1", "w2", "w3"]: - if self.options.descriptors: - node_v16.assert_start_raises_init_error([f"-wallet={wallet_name}"], f"Error: {wallet_name} corrupt, salvage failed") - else: - node_v16.assert_start_raises_init_error([f"-wallet={wallet_name}"], f"Error: Error loading {wallet_name}: Wallet requires newer version of Bitcoin Core") - # When descriptors are enabled, w1 cannot be opened by 0.21 since it contains a taproot descriptor if self.options.descriptors: self.log.info("Test that 0.21 cannot open wallet containing tr() descriptors") diff --git a/test/functional/wallet_multiwallet.py b/test/functional/wallet_multiwallet.py index 156f4279b4..149b1246d8 100755 --- a/test/functional/wallet_multiwallet.py +++ b/test/functional/wallet_multiwallet.py @@ -229,7 +229,7 @@ class MultiWalletTest(BitcoinTestFramework): assert_raises_rpc_error(-18, "Requested wallet does not exist or is not loaded", wallet_bad.getwalletinfo) # accessing wallet RPC without using wallet endpoint fails - assert_raises_rpc_error(-19, "Wallet file not specified", node.getwalletinfo) + assert_raises_rpc_error(-19, "Multiple wallets are loaded. Please select which wallet", node.getwalletinfo) w1, w2, w3, w4, *_ = wallets self.generatetoaddress(node, nblocks=COINBASE_MATURITY + 1, address=w1.getnewaddress(), sync_fun=self.no_op) @@ -275,7 +275,7 @@ class MultiWalletTest(BitcoinTestFramework): loadwallet_name = node.loadwallet(wallet_names[1]) assert_equal(loadwallet_name['name'], wallet_names[1]) assert_equal(node.listwallets(), wallet_names[0:2]) - assert_raises_rpc_error(-19, "Wallet file not specified", node.getwalletinfo) + assert_raises_rpc_error(-19, "Multiple wallets are loaded. Please select which wallet", node.getwalletinfo) w2 = node.get_wallet_rpc(wallet_names[1]) w2.getwalletinfo() diff --git a/test/functional/wallet_upgradewallet.py b/test/functional/wallet_upgradewallet.py index 7d1d244dff..c909336a25 100755 --- a/test/functional/wallet_upgradewallet.py +++ b/test/functional/wallet_upgradewallet.py @@ -185,6 +185,7 @@ class UpgradeWalletTest(BitcoinTestFramework): self.restart_node(0) copy_v16() wallet = node_master.get_wallet_rpc(self.default_wallet_name) + assert_equal(wallet.getbalance(), v16_3_balance) self.log.info("Test upgradewallet without a version argument") self.test_upgradewallet(wallet, previous_version=159900, expected_version=169900) # wallet should still contain the same balance @@ -231,7 +232,7 @@ class UpgradeWalletTest(BitcoinTestFramework): assert b'\x07hdchain' in new_kvs hd_chain = new_kvs[b'\x07hdchain'] assert_equal(28, len(hd_chain)) - hd_chain_version, external_counter, seed_id = struct.unpack('<iI20s', hd_chain) + hd_chain_version, _external_counter, seed_id = struct.unpack('<iI20s', hd_chain) assert_equal(1, hd_chain_version) seed_id = bytearray(seed_id) seed_id.reverse() @@ -258,7 +259,7 @@ class UpgradeWalletTest(BitcoinTestFramework): new_kvs = dump_bdb_kv(node_master_wallet) hd_chain = new_kvs[b'\x07hdchain'] assert_equal(32, len(hd_chain)) - hd_chain_version, external_counter, seed_id, internal_counter = struct.unpack('<iI20sI', hd_chain) + hd_chain_version, _external_counter, seed_id, internal_counter = struct.unpack('<iI20sI', hd_chain) assert_equal(2, hd_chain_version) assert_equal(0, internal_counter) seed_id = bytearray(seed_id) @@ -284,7 +285,7 @@ class UpgradeWalletTest(BitcoinTestFramework): new_kvs = dump_bdb_kv(node_master_wallet) hd_chain = new_kvs[b'\x07hdchain'] assert_equal(32, len(hd_chain)) - hd_chain_version, external_counter, seed_id, internal_counter = struct.unpack('<iI20sI', hd_chain) + hd_chain_version, _external_counter, seed_id, internal_counter = struct.unpack('<iI20sI', hd_chain) assert_equal(2, hd_chain_version) assert_equal(2, internal_counter) # The next addresses are HD and should be on different HD chains (the one remaining key in each pool should have been flushed) @@ -301,8 +302,8 @@ class UpgradeWalletTest(BitcoinTestFramework): new_kvs = dump_bdb_kv(node_master_wallet) for k, old_v in old_kvs.items(): if k.startswith(b'\x07keymeta'): - new_ver, new_create_time, new_kp_str, new_seed_id, new_fpr, new_path_len, new_path, new_has_key_orig = deser_keymeta(BytesIO(new_kvs[k])) - old_ver, old_create_time, old_kp_str, old_seed_id, old_fpr, old_path_len, old_path, old_has_key_orig = deser_keymeta(BytesIO(old_v)) + new_ver, new_create_time, new_kp_str, new_seed_id, _new_fpr, new_path_len, new_path, new_has_key_orig = deser_keymeta(BytesIO(new_kvs[k])) + old_ver, old_create_time, old_kp_str, old_seed_id, _old_fpr, old_path_len, old_path, old_has_key_orig = deser_keymeta(BytesIO(old_v)) assert_equal(10, old_ver) if old_kp_str == b"": # imported things that don't have keymeta (i.e. imported coinbase privkeys) won't be upgraded assert_equal(new_kvs[k], old_v) diff --git a/test/lint/README.md b/test/lint/README.md index 04a836c4d2..8c1f0fedf0 100644 --- a/test/lint/README.md +++ b/test/lint/README.md @@ -45,13 +45,13 @@ or `--help`: | Lint test | Dependency | |-----------|:----------:| -| [`lint-python.py`](/test/lint/lint-python.py) | [flake8](https://github.com/PyCQA/flake8) | [`lint-python.py`](/test/lint/lint-python.py) | [lief](https://github.com/lief-project/LIEF) | [`lint-python.py`](/test/lint/lint-python.py) | [mypy](https://github.com/python/mypy) | [`lint-python.py`](/test/lint/lint-python.py) | [pyzmq](https://github.com/zeromq/pyzmq) | [`lint-python-dead-code.py`](/test/lint/lint-python-dead-code.py) | [vulture](https://github.com/jendrikseipp/vulture) | [`lint-shell.py`](/test/lint/lint-shell.py) | [ShellCheck](https://github.com/koalaman/shellcheck) | [`lint-spelling.py`](/test/lint/lint-spelling.py) | [codespell](https://github.com/codespell-project/codespell) +| `py_lint` | [ruff](https://github.com/astral-sh/ruff) | markdown link check | [mlc](https://github.com/becheran/mlc) In use versions and install instructions are available in the [CI setup](../../ci/lint/04_install.sh). diff --git a/test/lint/commit-script-check.sh b/test/lint/commit-script-check.sh index fe845ed19e..52ae95fbb6 100755 --- a/test/lint/commit-script-check.sh +++ b/test/lint/commit-script-check.sh @@ -35,20 +35,20 @@ for commit in $(git rev-list --reverse "$1"); do git checkout --quiet "$commit"^ || exit SCRIPT="$(git rev-list --format=%b -n1 "$commit" | sed '/^-BEGIN VERIFY SCRIPT-$/,/^-END VERIFY SCRIPT-$/{//!b};d')" if test -z "$SCRIPT"; then - echo "Error: missing script for: $commit" - echo "Failed" + echo "Error: missing script for: $commit" >&2 + echo "Failed" >&2 RET=1 else - echo "Running script for: $commit" - echo "$SCRIPT" + echo "Running script for: $commit" >&2 + echo "$SCRIPT" >&2 (eval "$SCRIPT") - git --no-pager diff --exit-code "$commit" && echo "OK" || (echo "Failed"; false) || RET=1 + git --no-pager diff --exit-code "$commit" && echo "OK" >&2 || (echo "Failed" >&2; false) || RET=1 fi git reset --quiet --hard HEAD else if git rev-list "--format=%b" -n1 "$commit" | grep -q '^-\(BEGIN\|END\)[ a-zA-Z]*-$'; then - echo "Error: script block marker but no scripted-diff in title of commit $commit" - echo "Failed" + echo "Error: script block marker but no scripted-diff in title of commit $commit" >&2 + echo "Failed" >&2 RET=1 fi fi diff --git a/test/lint/lint-format-strings.py b/test/lint/lint-format-strings.py index 002c59e9a3..86a17fb0f8 100755 --- a/test/lint/lint-format-strings.py +++ b/test/lint/lint-format-strings.py @@ -16,27 +16,8 @@ import re import sys FUNCTION_NAMES_AND_NUMBER_OF_LEADING_ARGUMENTS = [ - 'FatalErrorf,0', - 'fprintf,1', 'tfm::format,1', # Assuming tfm::::format(std::ostream&, ... - 'LogConnectFailure,1', - 'LogError,0', - 'LogWarning,0', - 'LogInfo,0', - 'LogDebug,1', - 'LogTrace,1', - 'LogPrintf,0', - 'LogPrintfCategory,1', - 'LogPrintLevel,2', - 'printf,0', - 'snprintf,2', - 'sprintf,1', 'strprintf,0', - 'vfprintf,1', - 'vprintf,1', - 'vsnprintf,1', - 'vsprintf,1', - 'WalletLogPrintf,0', ] RUN_LINT_FILE = 'test/lint/run-lint-format-strings.py' @@ -81,7 +62,7 @@ def main(): matching_files_filtered = [] for matching_file in matching_files: - if not re.search('^src/(leveldb|secp256k1|minisketch|tinyformat|test/fuzz/strprintf.cpp)|contrib/devtools/bitcoin-tidy/example_logprintf.cpp', matching_file): + if not re.search('^src/(leveldb|secp256k1|minisketch|tinyformat|test/fuzz/strprintf.cpp)', matching_file): matching_files_filtered.append(matching_file) matching_files_filtered.sort() diff --git a/test/lint/lint-python.py b/test/lint/lint-python.py index eabd13322e..e2dbe25b88 100755 --- a/test/lint/lint-python.py +++ b/test/lint/lint-python.py @@ -5,13 +5,12 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """ -Check for specified flake8 and mypy warnings in python files. +Check for specified mypy warnings in python files. """ import os from pathlib import Path import subprocess -import sys from importlib.metadata import metadata, PackageNotFoundError @@ -19,89 +18,12 @@ from importlib.metadata import metadata, PackageNotFoundError cache_dir = Path(__file__).parent.parent / ".mypy_cache" os.environ["MYPY_CACHE_DIR"] = str(cache_dir) -DEPS = ['flake8', 'lief', 'mypy', 'pyzmq'] - -# All .py files, except those in src/ (to exclude subtrees there) -FLAKE_FILES_ARGS = ['git', 'ls-files', '*.py', ':!:src/*.py'] +DEPS = ['lief', 'mypy', 'pyzmq'] # Only .py files in test/functional and contrib/devtools have type annotations # enforced. MYPY_FILES_ARGS = ['git', 'ls-files', 'test/functional/*.py', 'contrib/devtools/*.py'] -ENABLED = ( - 'E101,' # indentation contains mixed spaces and tabs - 'E112,' # expected an indented block - 'E113,' # unexpected indentation - 'E115,' # expected an indented block (comment) - 'E116,' # unexpected indentation (comment) - 'E125,' # continuation line with same indent as next logical line - 'E129,' # visually indented line with same indent as next logical line - 'E131,' # continuation line unaligned for hanging indent - 'E133,' # closing bracket is missing indentation - 'E223,' # tab before operator - 'E224,' # tab after operator - 'E242,' # tab after ',' - 'E266,' # too many leading '#' for block comment - 'E271,' # multiple spaces after keyword - 'E272,' # multiple spaces before keyword - 'E273,' # tab after keyword - 'E274,' # tab before keyword - 'E275,' # missing whitespace after keyword - 'E304,' # blank lines found after function decorator - 'E306,' # expected 1 blank line before a nested definition - 'E401,' # multiple imports on one line - 'E402,' # module level import not at top of file - 'E502,' # the backslash is redundant between brackets - 'E701,' # multiple statements on one line (colon) - 'E702,' # multiple statements on one line (semicolon) - 'E703,' # statement ends with a semicolon - 'E711,' # comparison to None should be 'if cond is None:' - 'E714,' # test for object identity should be "is not" - 'E721,' # do not compare types, use "isinstance()" - 'E722,' # do not use bare 'except' - 'E742,' # do not define classes named "l", "O", or "I" - 'E743,' # do not define functions named "l", "O", or "I" - 'E901,' # SyntaxError: invalid syntax - 'E902,' # TokenError: EOF in multi-line string - 'F401,' # module imported but unused - 'F402,' # import module from line N shadowed by loop variable - 'F403,' # 'from foo_module import *' used; unable to detect undefined names - 'F404,' # future import(s) name after other statements - 'F405,' # foo_function may be undefined, or defined from star imports: bar_module - 'F406,' # "from module import *" only allowed at module level - 'F407,' # an undefined __future__ feature name was imported - 'F601,' # dictionary key name repeated with different values - 'F602,' # dictionary key variable name repeated with different values - 'F621,' # too many expressions in an assignment with star-unpacking - 'F622,' # two or more starred expressions in an assignment (a, *b, *c = d) - 'F631,' # assertion test is a tuple, which are always True - 'F632,' # use ==/!= to compare str, bytes, and int literals - 'F701,' # a break statement outside of a while or for loop - 'F702,' # a continue statement outside of a while or for loop - 'F703,' # a continue statement in a finally block in a loop - 'F704,' # a yield or yield from statement outside of a function - 'F705,' # a return statement with arguments inside a generator - 'F706,' # a return statement outside of a function/method - 'F707,' # an except: block as not the last exception handler - 'F811,' # redefinition of unused name from line N - 'F812,' # list comprehension redefines 'foo' from line N - 'F821,' # undefined name 'Foo' - 'F822,' # undefined name name in __all__ - 'F823,' # local variable name … referenced before assignment - 'F831,' # duplicate argument name in function definition - 'F841,' # local variable 'foo' is assigned to but never used - 'W191,' # indentation contains tabs - 'W291,' # trailing whitespace - 'W292,' # no newline at end of file - 'W293,' # blank line contains whitespace - 'W601,' # .has_key() is deprecated, use "in" - 'W602,' # deprecated form of raising exception - 'W603,' # "<>" is deprecated, use "!=" - 'W604,' # backticks are deprecated, use "repr()" - 'W605,' # invalid escape sequence "x" - 'W606,' # 'async' and 'await' are reserved keywords starting with Python 3.7 -) - def check_dependencies(): for dep in DEPS: @@ -115,20 +37,6 @@ def check_dependencies(): def main(): check_dependencies() - if len(sys.argv) > 1: - flake8_files = sys.argv[1:] - else: - flake8_files = subprocess.check_output(FLAKE_FILES_ARGS).decode("utf-8").splitlines() - - flake8_args = ['flake8', '--ignore=B,C,E,F,I,N,W', f'--select={ENABLED}'] + flake8_files - flake8_env = os.environ.copy() - flake8_env["PYTHONWARNINGS"] = "ignore" - - try: - subprocess.check_call(flake8_args, env=flake8_env) - except subprocess.CalledProcessError: - exit(1) - mypy_files = subprocess.check_output(MYPY_FILES_ARGS).decode("utf-8").splitlines() mypy_args = ['mypy', '--show-error-codes'] + mypy_files diff --git a/test/lint/lint-spelling.py b/test/lint/lint-spelling.py index 3e578b218f..945288a3dd 100755 --- a/test/lint/lint-spelling.py +++ b/test/lint/lint-spelling.py @@ -14,7 +14,7 @@ from subprocess import check_output, STDOUT, CalledProcessError from lint_ignore_dirs import SHARED_EXCLUDED_SUBTREES IGNORE_WORDS_FILE = 'test/lint/spelling.ignore-words.txt' -FILES_ARGS = ['git', 'ls-files', '--', ":(exclude)build-aux/m4/", ":(exclude)contrib/seeds/*.txt", ":(exclude)depends/", ":(exclude)doc/release-notes/", ":(exclude)src/qt/locale/", ":(exclude)src/qt/*.qrc", ":(exclude)contrib/guix/patches"] +FILES_ARGS = ['git', 'ls-files', '--', ":(exclude)contrib/seeds/*.txt", ":(exclude)depends/", ":(exclude)doc/release-notes/", ":(exclude)src/qt/locale/", ":(exclude)src/qt/*.qrc", ":(exclude)contrib/guix/patches"] FILES_ARGS += [f":(exclude){dir}" for dir in SHARED_EXCLUDED_SUBTREES] diff --git a/test/lint/run-lint-format-strings.py b/test/lint/run-lint-format-strings.py index 09a2503452..d3c0ac92e5 100755 --- a/test/lint/run-lint-format-strings.py +++ b/test/lint/run-lint-format-strings.py @@ -13,17 +13,8 @@ import re import sys FALSE_POSITIVES = [ - ("src/dbwrapper.cpp", "vsnprintf(p, limit - p, format, backup_ap)"), - ("src/index/base.cpp", "FatalErrorf(const char* fmt, const Args&... args)"), - ("src/index/base.h", "FatalErrorf(const char* fmt, const Args&... args)"), - ("src/netbase.cpp", "LogConnectFailure(bool manual_connection, const char* fmt, const Args&... args)"), ("src/clientversion.cpp", "strprintf(_(COPYRIGHT_HOLDERS).translated, COPYRIGHT_HOLDERS_SUBSTITUTION)"), ("src/test/translation_tests.cpp", "strprintf(format, arg)"), - ("src/validationinterface.cpp", "LogDebug(BCLog::VALIDATION, fmt \"\\n\", __VA_ARGS__)"), - ("src/wallet/wallet.h", "WalletLogPrintf(const char* fmt, Params... parameters)"), - ("src/wallet/wallet.h", "LogPrintf((\"%s \" + std::string{fmt}).c_str(), GetDisplayName(), parameters...)"), - ("src/wallet/scriptpubkeyman.h", "WalletLogPrintf(const char* fmt, Params... parameters)"), - ("src/wallet/scriptpubkeyman.h", "LogPrintf((\"%s \" + std::string{fmt}).c_str(), m_storage.GetDisplayName(), parameters...)"), ] diff --git a/test/lint/test_runner/src/main.rs b/test/lint/test_runner/src/main.rs index 1a8c11dd42..42c880052e 100644 --- a/test/lint/test_runner/src/main.rs +++ b/test/lint/test_runner/src/main.rs @@ -5,9 +5,12 @@ use std::env; use std::fs; use std::io::ErrorKind; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use std::process::{Command, ExitCode, Stdio}; +/// A possible error returned by any of the linters. +/// +/// The error string should explain the failure type and list all violations. type LintError = String; type LintResult = Result<(), LintError>; type LintFn = fn() -> LintResult; @@ -26,7 +29,7 @@ fn get_linter_list() -> Vec<&'static Linter> { lint_fn: lint_doc }, &Linter { - description: "Check that no symbol from bitcoin-config.h is used without the header being included", + description: "Check that no symbol from bitcoin-build-config.h is used without the header being included", name: "includes_build_config", lint_fn: lint_includes_build_config }, @@ -36,9 +39,9 @@ fn get_linter_list() -> Vec<&'static Linter> { lint_fn: lint_markdown }, &Linter { - description: "Check the default arguments in python", - name: "py_mut_arg_default", - lint_fn: lint_py_mut_arg_default, + description: "Lint Python code", + name: "py_lint", + lint_fn: lint_py_lint, }, &Linter { description: "Check that std::filesystem is not used directly", @@ -46,6 +49,11 @@ fn get_linter_list() -> Vec<&'static Linter> { lint_fn: lint_std_filesystem }, &Linter { + description: "Check that release note snippets are in the right folder", + name: "doc_release_note_snippets", + lint_fn: lint_doc_release_note_snippets + }, + &Linter { description: "Check that subtrees are pure subtrees", name: "subtree", lint_fn: lint_subtree @@ -125,20 +133,27 @@ fn parse_lint_args(args: &[String]) -> Vec<&'static Linter> { } /// Return the git command +/// +/// Lint functions should use this command, so that only files tracked by git are considered and +/// temporary and untracked files are ignored. For example, instead of 'grep', 'git grep' should be +/// used. fn git() -> Command { let mut git = Command::new("git"); git.arg("--no-pager"); git } -/// Return stdout +/// Return stdout on success and a LintError on failure, when invalid UTF8 was detected or the +/// command did not succeed. fn check_output(cmd: &mut std::process::Command) -> Result<String, LintError> { let out = cmd.output().expect("command error"); if !out.status.success() { return Err(String::from_utf8_lossy(&out.stderr).to_string()); } Ok(String::from_utf8(out.stdout) - .map_err(|e| format!("{e}"))? + .map_err(|e| { + format!("All path names, source code, messages, and output must be valid UTF8!\n{e}") + })? .trim() .to_string()) } @@ -185,12 +200,50 @@ fn lint_subtree() -> LintResult { } } -fn lint_py_mut_arg_default() -> LintResult { +fn lint_py_lint() -> LintResult { let bin_name = "ruff"; - let checks = ["B006", "B008"] - .iter() - .map(|c| format!("--select={}", c)) - .collect::<Vec<_>>(); + let checks = format!( + "--select={}", + [ + "B006", // mutable-argument-default + "B008", // function-call-in-default-argument + "E101", // indentation contains mixed spaces and tabs + "E401", // multiple imports on one line + "E402", // module level import not at top of file + "E701", // multiple statements on one line (colon) + "E702", // multiple statements on one line (semicolon) + "E703", // statement ends with a semicolon + "E711", // comparison to None should be 'if cond is None:' + "E714", // test for object identity should be "is not" + "E721", // do not compare types, use "isinstance()" + "E722", // do not use bare 'except' + "E742", // do not define classes named "l", "O", or "I" + "E743", // do not define functions named "l", "O", or "I" + "F401", // module imported but unused + "F402", // import module from line N shadowed by loop variable + "F403", // 'from foo_module import *' used; unable to detect undefined names + "F404", // future import(s) name after other statements + "F405", // foo_function may be undefined, or defined from star imports: bar_module + "F406", // "from module import *" only allowed at module level + "F407", // an undefined __future__ feature name was imported + "F601", // dictionary key name repeated with different values + "F602", // dictionary key variable name repeated with different values + "F621", // too many expressions in an assignment with star-unpacking + "F631", // assertion test is a tuple, which are always True + "F632", // use ==/!= to compare str, bytes, and int literals + "F811", // redefinition of unused name from line N + "F821", // undefined name 'Foo' + "F822", // undefined name name in __all__ + "F823", // local variable name … referenced before assignment + "F841", // local variable 'foo' is assigned to but never used + "W191", // indentation contains tabs + "W291", // trailing whitespace + "W292", // no newline at end of file + "W293", // blank line contains whitespace + "W605", // invalid escape sequence "x" + ] + .join(",") + ); let files = check_output( git() .args(["ls-files", "--", "*.py"]) @@ -198,7 +251,7 @@ fn lint_py_mut_arg_default() -> LintResult { )?; let mut cmd = Command::new(bin_name); - cmd.arg("check").args(checks).args(files.lines()); + cmd.args(["check", &checks]).args(files.lines()); match cmd.status() { Ok(status) if status.success() => Ok(()), @@ -238,6 +291,30 @@ fs:: namespace, which has unsafe filesystem functions marked as deleted. } } +fn lint_doc_release_note_snippets() -> LintResult { + let non_release_notes = check_output(git().args([ + "ls-files", + "--", + "doc/release-notes/", + ":(exclude)doc/release-notes/*.*.md", // Assume that at least one dot implies a proper release note + ]))?; + if non_release_notes.is_empty() { + Ok(()) + } else { + Err(format!( + r#" +{} +^^^ +Release note snippets and other docs must be put into the doc/ folder directly. + +The doc/release-notes/ folder is for archived release notes of previous releases only. Snippets are +expected to follow the naming "/doc/release-notes-<PR number>.md". + "#, + non_release_notes + )) + } +} + /// Return the pathspecs for whitespace related excludes fn get_pathspecs_exclude_whitespace() -> Vec<String> { let mut list = get_pathspecs_exclude_subtrees(); @@ -318,7 +395,7 @@ Please add any false positives, such as subtrees, or externally sourced files to } fn lint_includes_build_config() -> LintResult { - let config_path = "./cmake/bitcoin-config.h.in"; + let config_path = "./cmake/bitcoin-build-config.h.in"; let defines_regex = format!( r"^\s*(?!//).*({})", check_output(Command::new("grep").args(["define", "--", config_path])) @@ -352,7 +429,7 @@ fn lint_includes_build_config() -> LintResult { ]) .args(get_pathspecs_exclude_subtrees()) .args([ - // These are exceptions which don't use bitcoin-config.h, rather the Makefile.am adds + // These are exceptions which don't use bitcoin-build-config.h, rather CMakeLists.txt adds // these cppflags manually. ":(exclude)src/crypto/sha256_arm_shani.cpp", ":(exclude)src/crypto/sha256_avx2.cpp", @@ -370,9 +447,9 @@ fn lint_includes_build_config() -> LintResult { "--files-with-matches" }, if mode { - "^#include <config/bitcoin-config.h> // IWYU pragma: keep$" + "^#include <bitcoin-build-config.h> // IWYU pragma: keep$" } else { - "#include <config/bitcoin-config.h>" // Catch redundant includes with and without the IWYU pragma + "#include <bitcoin-build-config.h>" // Catch redundant includes with and without the IWYU pragma }, "--", ]) @@ -386,11 +463,11 @@ fn lint_includes_build_config() -> LintResult { return Err(format!( r#" ^^^ -One or more files use a symbol declared in the bitcoin-config.h header. However, they are not +One or more files use a symbol declared in the bitcoin-build-config.h header. However, they are not including the header. This is problematic, because the header may or may not be indirectly included. If the indirect include were to be intentionally or accidentally removed, the build could still succeed, but silently be buggy. For example, a slower fallback algorithm could be picked, -even though bitcoin-config.h indicates that a faster feature is available and should be used. +even though bitcoin-build-config.h indicates that a faster feature is available and should be used. If you are unsure which symbol is used, you can find it with this command: git grep --perl-regexp '{}' -- file_name @@ -398,7 +475,7 @@ git grep --perl-regexp '{}' -- file_name Make sure to include it with the IWYU pragma. Otherwise, IWYU may falsely instruct to remove the include again. -#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <bitcoin-build-config.h> // IWYU pragma: keep "#, defines_regex )); @@ -407,7 +484,7 @@ include again. if redundant { return Err(r#" ^^^ -None of the files use a symbol declared in the bitcoin-config.h header. However, they are including +None of the files use a symbol declared in the bitcoin-build-config.h header. However, they are including the header. Consider removing the unused include. "# .to_string()); diff --git a/test/util/test_runner.py b/test/util/test_runner.py index 1cd368f6f4..cac184ca30 100755 --- a/test/util/test_runner.py +++ b/test/util/test_runner.py @@ -5,7 +5,7 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test framework for bitcoin utils. -Runs automatically during `make check`. +Runs automatically during `ctest --test-dir build/`. Can also be run manually.""" @@ -83,13 +83,11 @@ def bctest(testDir, testObj, buildenv): execrun = [execprog] + execargs # Read the input data (if there is any) - stdinCfg = None inputData = None if "input" in testObj: filename = os.path.join(testDir, testObj["input"]) with open(filename, encoding="utf8") as f: inputData = f.read() - stdinCfg = subprocess.PIPE # Read the expected output data (if there is any) outputFn = None @@ -112,9 +110,8 @@ def bctest(testDir, testObj, buildenv): raise Exception # Run the test - proc = subprocess.Popen(execrun, stdin=stdinCfg, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) try: - outs = proc.communicate(input=inputData) + res = subprocess.run(execrun, capture_output=True, text=True, input=inputData) except OSError: logging.error("OSError, Failed to execute " + execprog) raise @@ -123,9 +120,9 @@ def bctest(testDir, testObj, buildenv): data_mismatch, formatting_mismatch = False, False # Parse command output and expected output try: - a_parsed = parse_output(outs[0], outputType) + a_parsed = parse_output(res.stdout, outputType) except Exception as e: - logging.error('Error parsing command output as %s: %s' % (outputType, e)) + logging.error(f"Error parsing command output as {outputType}: '{str(e)}'; res: {str(res)}") raise try: b_parsed = parse_output(outputData, outputType) @@ -134,13 +131,13 @@ def bctest(testDir, testObj, buildenv): raise # Compare data if a_parsed != b_parsed: - logging.error("Output data mismatch for " + outputFn + " (format " + outputType + ")") + logging.error(f"Output data mismatch for {outputFn} (format {outputType}); res: {str(res)}") data_mismatch = True # Compare formatting - if outs[0] != outputData: - error_message = "Output formatting mismatch for " + outputFn + ":\n" + if res.stdout != outputData: + error_message = f"Output formatting mismatch for {outputFn}:\nres: {str(res)}\n" error_message += "".join(difflib.context_diff(outputData.splitlines(True), - outs[0].splitlines(True), + res.stdout.splitlines(True), fromfile=outputFn, tofile="returned")) logging.error(error_message) @@ -152,8 +149,8 @@ def bctest(testDir, testObj, buildenv): wantRC = 0 if "return_code" in testObj: wantRC = testObj['return_code'] - if proc.returncode != wantRC: - logging.error("Return code mismatch for " + outputFn) + if res.returncode != wantRC: + logging.error(f"Return code mismatch for {outputFn}; res: {str(res)}") raise Exception if "error_txt" in testObj: @@ -164,8 +161,8 @@ def bctest(testDir, testObj, buildenv): # emits DISPLAY errors when running as a windows application on # linux through wine. Just assert that the expected error text appears # somewhere in stderr. - if want_error not in outs[1]: - logging.error("Error mismatch:\n" + "Expected: " + want_error + "\nReceived: " + outs[1].rstrip()) + if want_error not in res.stderr: + logging.error(f"Error mismatch:\nExpected: {want_error}\nReceived: {res.stderr.rstrip()}\nres: {str(res)}") raise Exception def parse_output(a, fmt): |