diff options
Diffstat (limited to 'test')
95 files changed, 2905 insertions, 1837 deletions
diff --git a/test/README.md b/test/README.md index 7ff2d6d9f2..6ca7cc0016 100644 --- a/test/README.md +++ b/test/README.md @@ -98,7 +98,7 @@ test/functional/test_runner.py --extended In order to run backwards compatibility tests, download the previous node binaries: ``` -test/get_previous_releases.py -b v22.0 v0.21.0 v0.20.1 v0.19.1 v0.18.1 v0.17.2 v0.16.3 v0.15.2 v0.14.3 +test/get_previous_releases.py -b v23.0 v22.0 v0.21.0 v0.20.1 v0.19.1 v0.18.1 v0.17.2 v0.16.3 v0.15.2 v0.14.3 ``` By default, up to 4 tests will be run in parallel by test_runner. To specify @@ -305,11 +305,11 @@ Use the `-v` option for verbose output. | Lint test | Dependency | |-----------|:----------:| -| [`lint-python.sh`](lint/lint-python.sh) | [flake8](https://gitlab.com/pycqa/flake8) -| [`lint-python.sh`](lint/lint-python.sh) | [mypy](https://github.com/python/mypy) -| [`lint-python.sh`](lint/lint-python.sh) | [pyzmq](https://github.com/zeromq/pyzmq) +| [`lint-python.py`](lint/lint-python.py) | [flake8](https://gitlab.com/pycqa/flake8) +| [`lint-python.py`](lint/lint-python.py) | [mypy](https://github.com/python/mypy) +| [`lint-python.py`](lint/lint-python.py) | [pyzmq](https://github.com/zeromq/pyzmq) | [`lint-python-dead-code.py`](lint/lint-python-dead-code.py) | [vulture](https://github.com/jendrikseipp/vulture) -| [`lint-shell.sh`](lint/lint-shell.sh) | [ShellCheck](https://github.com/koalaman/shellcheck) +| [`lint-shell.py`](lint/lint-shell.py) | [ShellCheck](https://github.com/koalaman/shellcheck) | [`lint-spelling.py`](lint/lint-spelling.py) | [codespell](https://github.com/codespell-project/codespell) In use versions and install instructions are available in the [CI setup](../ci/lint/04_install.sh). @@ -327,7 +327,7 @@ test/lint/lint-files.py You can run all the shell-based lint tests by running: ``` -test/lint/lint-all.sh +test/lint/all-lint.py ``` # Writing functional tests diff --git a/test/config.ini.in b/test/config.ini.in index d7105c419b..5888ef443b 100644 --- a/test/config.ini.in +++ b/test/config.ini.in @@ -19,6 +19,7 @@ RPCAUTH=@abs_top_srcdir@/share/rpcauth/rpcauth.py @USE_SQLITE_TRUE@USE_SQLITE=true @USE_BDB_TRUE@USE_BDB=true @BUILD_BITCOIN_CLI_TRUE@ENABLE_CLI=true +@BUILD_BITCOIN_UTIL_TRUE@ENABLE_BITCOIN_UTIL=true @BUILD_BITCOIN_WALLET_TRUE@ENABLE_WALLET_TOOL=true @BUILD_BITCOIND_TRUE@ENABLE_BITCOIND=true @ENABLE_FUZZ_TRUE@ENABLE_FUZZ=true diff --git a/test/functional/README.md b/test/functional/README.md index 926810cf03..914dbfd977 100644 --- a/test/functional/README.md +++ b/test/functional/README.md @@ -24,7 +24,7 @@ don't have test cases for. Consider using [pyenv](https://github.com/pyenv/pyenv), which checks [.python-version](/.python-version), to prevent accidentally introducing modern syntax from an unsupported Python version. The CI linter job also checks this, but [possibly not in all cases](https://github.com/bitcoin/bitcoin/pull/14884#discussion_r239585126). -- See [the python lint script](/test/lint/lint-python.sh) that checks for violations that +- See [the python lint script](/test/lint/lint-python.py) that checks for violations that could lead to bugs and issues in the test code. - Use [type hints](https://docs.python.org/3/library/typing.html) in your code to improve code readability and to detect possible bugs earlier. diff --git a/test/functional/feature_backwards_compatibility.py b/test/functional/feature_backwards_compatibility.py index a7fb3184a6..59a12193fd 100755 --- a/test/functional/feature_backwards_compatibility.py +++ b/test/functional/feature_backwards_compatibility.py @@ -34,11 +34,12 @@ from test_framework.util import ( class BackwardsCompatibilityTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True - self.num_nodes = 9 + self.num_nodes = 10 # Add new version after each release: self.extra_args = [ ["-addresstype=bech32", "-whitelist=noban@127.0.0.1"], # Pre-release: use to mine blocks. noban for immediate tx relay ["-nowallet", "-walletrbf=1", "-addresstype=bech32", "-whitelist=noban@127.0.0.1"], # Pre-release: use to receive coins, swap wallets, etc + ["-nowallet", "-walletrbf=1", "-addresstype=bech32", "-whitelist=noban@127.0.0.1"], # v23.0 ["-nowallet", "-walletrbf=1", "-addresstype=bech32", "-whitelist=noban@127.0.0.1"], # v22.0 ["-nowallet", "-walletrbf=1", "-addresstype=bech32", "-whitelist=noban@127.0.0.1"], # v0.21.0 ["-nowallet", "-walletrbf=1", "-addresstype=bech32", "-whitelist=noban@127.0.0.1"], # v0.20.1 @@ -57,6 +58,7 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): self.add_nodes(self.num_nodes, extra_args=self.extra_args, versions=[ None, None, + 230000, 220000, 210000, 200100, diff --git a/test/functional/feature_bind_extra.py b/test/functional/feature_bind_extra.py index 6802da8d48..5de9ff203c 100755 --- a/test/functional/feature_bind_extra.py +++ b/test/functional/feature_bind_extra.py @@ -18,12 +18,12 @@ from test_framework.test_framework import ( SkipTest, ) from test_framework.util import ( - PORT_MIN, - PORT_RANGE, assert_equal, + p2p_port, rpc_port, ) + class BindExtraTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True @@ -33,11 +33,6 @@ class BindExtraTest(BitcoinTestFramework): self.num_nodes = 2 def setup_network(self): - # Override setup_network() because we want to put the result of - # p2p_port() in self.extra_args[], before the nodes are started. - # p2p_port() is not usable in set_test_params() because PortSeed.n is - # not set at that time. - # Due to OS-specific network stats queries, we only run on Linux. self.log.info("Checking for Linux") if not sys.platform.startswith('linux'): @@ -45,8 +40,8 @@ class BindExtraTest(BitcoinTestFramework): loopback_ipv4 = addr_to_hex("127.0.0.1") - # Start custom ports after p2p and rpc ports. - port = PORT_MIN + 2 * PORT_RANGE + # Start custom ports by reusing unused p2p ports + port = p2p_port(self.num_nodes) # Array of tuples [command line arguments, expected bind addresses]. self.expected = [] diff --git a/test/functional/feature_blockfilterindex_prune.py b/test/functional/feature_blockfilterindex_prune.py deleted file mode 100755 index c983ceda6f..0000000000 --- a/test/functional/feature_blockfilterindex_prune.py +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2020-2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test blockfilterindex in conjunction with prune.""" -from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import ( - assert_equal, - assert_greater_than, - assert_raises_rpc_error, -) - - -class FeatureBlockfilterindexPruneTest(BitcoinTestFramework): - def set_test_params(self): - self.num_nodes = 1 - self.extra_args = [["-fastprune", "-prune=1", "-blockfilterindex=1"]] - - def sync_index(self, height): - expected = {'basic block filter index': {'synced': True, 'best_block_height': height}} - self.wait_until(lambda: self.nodes[0].getindexinfo() == expected) - - def run_test(self): - self.log.info("check if we can access a blockfilter when pruning is enabled but no blocks are actually pruned") - self.sync_index(height=200) - assert_greater_than(len(self.nodes[0].getblockfilter(self.nodes[0].getbestblockhash())['filter']), 0) - self.generate(self.nodes[0], 500) - self.sync_index(height=700) - - self.log.info("prune some blocks") - pruneheight = self.nodes[0].pruneblockchain(400) - # the prune heights used here and below are magic numbers that are determined by the - # thresholds at which block files wrap, so they depend on disk serialization and default block file size. - assert_equal(pruneheight, 249) - - self.log.info("check if we can access the tips blockfilter when we have pruned some blocks") - assert_greater_than(len(self.nodes[0].getblockfilter(self.nodes[0].getbestblockhash())['filter']), 0) - - self.log.info("check if we can access the blockfilter of a pruned block") - assert_greater_than(len(self.nodes[0].getblockfilter(self.nodes[0].getblockhash(2))['filter']), 0) - - # mine and sync index up to a height that will later be the pruneheight - self.generate(self.nodes[0], 51) - self.sync_index(height=751) - - self.log.info("start node without blockfilterindex") - self.restart_node(0, extra_args=["-fastprune", "-prune=1"]) - - self.log.info("make sure accessing the blockfilters throws an error") - assert_raises_rpc_error(-1, "Index is not enabled for filtertype basic", self.nodes[0].getblockfilter, self.nodes[0].getblockhash(2)) - self.generate(self.nodes[0], 749) - - self.log.info("prune exactly up to the blockfilterindexes best block while blockfilters are disabled") - pruneheight_2 = self.nodes[0].pruneblockchain(1000) - assert_equal(pruneheight_2, 751) - self.restart_node(0, extra_args=["-fastprune", "-prune=1", "-blockfilterindex=1"]) - self.log.info("make sure that we can continue with the partially synced index after having pruned up to the index height") - self.sync_index(height=1500) - - self.log.info("prune below the blockfilterindexes best block while blockfilters are disabled") - self.restart_node(0, extra_args=["-fastprune", "-prune=1"]) - self.generate(self.nodes[0], 1000) - pruneheight_3 = self.nodes[0].pruneblockchain(2000) - assert_greater_than(pruneheight_3, pruneheight_2) - self.stop_node(0) - - self.log.info("make sure we get an init error when starting the node again with block filters") - self.nodes[0].assert_start_raises_init_error( - extra_args=["-fastprune", "-prune=1", "-blockfilterindex=1"], - expected_msg="Error: basic block filter index best block of the index goes beyond pruned data. Please disable the index or reindex (which will download the whole blockchain again)", - ) - - self.log.info("make sure the node starts again with the -reindex arg") - self.start_node(0, extra_args=["-fastprune", "-prune=1", "-blockfilterindex", "-reindex"]) - - -if __name__ == '__main__': - FeatureBlockfilterindexPruneTest().main() diff --git a/test/functional/feature_coinstatsindex.py b/test/functional/feature_coinstatsindex.py index f865661894..2e21638f80 100755 --- a/test/functional/feature_coinstatsindex.py +++ b/test/functional/feature_coinstatsindex.py @@ -223,6 +223,22 @@ class CoinStatsIndexTest(BitcoinTestFramework): res10 = index_node.gettxoutsetinfo('muhash') assert(res8['txouts'] < res10['txouts']) + self.log.info("Test that the index works with -reindex") + + self.restart_node(1, extra_args=["-coinstatsindex", "-reindex"]) + res11 = index_node.gettxoutsetinfo('muhash') + assert_equal(res11, res10) + + self.log.info("Test that -reindex-chainstate is disallowed with coinstatsindex") + + self.stop_node(1) + self.nodes[1].assert_start_raises_init_error( + expected_msg='Error: -reindex-chainstate option is not compatible with -coinstatsindex. ' + 'Please temporarily disable coinstatsindex while using -reindex-chainstate, or replace -reindex-chainstate with -reindex to fully rebuild all indexes.', + extra_args=['-coinstatsindex', '-reindex-chainstate'], + ) + self.restart_node(1, extra_args=["-coinstatsindex"]) + def _test_use_index_option(self): self.log.info("Test use_index option for nodes running the index") diff --git a/test/functional/feature_config_args.py b/test/functional/feature_config_args.py index eea5fa24ee..6c51a5ac31 100755 --- a/test/functional/feature_config_args.py +++ b/test/functional/feature_config_args.py @@ -85,7 +85,7 @@ class ConfArgsTest(BitcoinTestFramework): def test_invalid_command_line_options(self): self.nodes[0].assert_start_raises_init_error( - expected_msg='Error: No proxy server specified. Use -proxy=<ip> or -proxy=<ip:port>.', + expected_msg='Error: Error parsing command line arguments: Can not set -proxy with no value. Please specify value with -proxy=value.', extra_args=['-proxy'], ) @@ -247,7 +247,8 @@ class ConfArgsTest(BitcoinTestFramework): conf_file = os.path.join(default_data_dir, "bitcoin.conf") # datadir needs to be set before [chain] section - conf_file_contents = open(conf_file, encoding='utf8').read() + with open(conf_file, encoding='utf8') as f: + conf_file_contents = f.read() with open(conf_file, 'w', encoding='utf8') as f: f.write(f"datadir={new_data_dir}\n") f.write(conf_file_contents) diff --git a/test/functional/feature_csv_activation.py b/test/functional/feature_csv_activation.py index 6470c1c5eb..bff95c3b94 100755 --- a/test/functional/feature_csv_activation.py +++ b/test/functional/feature_csv_activation.py @@ -112,6 +112,7 @@ class BIP68_112_113Test(BitcoinTestFramework): tx.nVersion = txversion self.miniwallet.sign_tx(tx) tx.vin[0].scriptSig = CScript([-1, OP_CHECKSEQUENCEVERIFY, OP_DROP] + list(CScript(tx.vin[0].scriptSig))) + tx.rehash() return tx def create_bip112emptystack(self, input, txversion): @@ -119,6 +120,7 @@ class BIP68_112_113Test(BitcoinTestFramework): tx.nVersion = txversion self.miniwallet.sign_tx(tx) tx.vin[0].scriptSig = CScript([OP_CHECKSEQUENCEVERIFY] + list(CScript(tx.vin[0].scriptSig))) + tx.rehash() return tx def send_generic_input_tx(self, coinbases): @@ -136,7 +138,6 @@ class BIP68_112_113Test(BitcoinTestFramework): tx.nVersion = txversion tx.vin[0].nSequence = locktime + locktime_delta self.miniwallet.sign_tx(tx) - tx.rehash() txs.append({'tx': tx, 'sdf': sdf, 'stf': stf}) return txs @@ -339,20 +340,16 @@ class BIP68_112_113Test(BitcoinTestFramework): # BIP 113 tests should now fail regardless of version number if nLockTime isn't satisfied by new rules bip113tx_v1.nLockTime = self.last_block_time - 600 * 5 # = MTP of prior block (not <) but < time put on current block self.miniwallet.sign_tx(bip113tx_v1) - bip113tx_v1.rehash() bip113tx_v2.nLockTime = self.last_block_time - 600 * 5 # = MTP of prior block (not <) but < time put on current block self.miniwallet.sign_tx(bip113tx_v2) - bip113tx_v2.rehash() for bip113tx in [bip113tx_v1, bip113tx_v2]: self.send_blocks([self.create_test_block([bip113tx])], success=False, reject_reason='bad-txns-nonfinal') # BIP 113 tests should now pass if the locktime is < MTP bip113tx_v1.nLockTime = self.last_block_time - 600 * 5 - 1 # < MTP of prior block self.miniwallet.sign_tx(bip113tx_v1) - bip113tx_v1.rehash() bip113tx_v2.nLockTime = self.last_block_time - 600 * 5 - 1 # < MTP of prior block self.miniwallet.sign_tx(bip113tx_v2) - bip113tx_v2.rehash() for bip113tx in [bip113tx_v1, bip113tx_v2]: self.send_blocks([self.create_test_block([bip113tx])]) self.nodes[0].invalidateblock(self.nodes[0].getbestblockhash()) @@ -477,7 +474,6 @@ class BIP68_112_113Test(BitcoinTestFramework): for tx in [tx['tx'] for tx in bip112txs_vary_OP_CSV_v2 if not tx['sdf'] and tx['stf']]: tx.vin[0].nSequence = BASE_RELATIVE_LOCKTIME | SEQ_TYPE_FLAG self.miniwallet.sign_tx(tx) - tx.rehash() time_txs.append(tx) self.send_blocks([self.create_test_block(time_txs)]) diff --git a/test/functional/feature_dbcrash.py b/test/functional/feature_dbcrash.py index 3e60efbb3c..a3a5e9e27d 100755 --- a/test/functional/feature_dbcrash.py +++ b/test/functional/feature_dbcrash.py @@ -30,17 +30,17 @@ import http.client import random import time +from test_framework.blocktools import COINBASE_MATURITY from test_framework.messages import ( COIN, - COutPoint, - CTransaction, - CTxIn, - CTxOut, ) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, - create_confirmed_utxos, +) +from test_framework.wallet import ( + MiniWallet, + getnewdestination, ) @@ -66,13 +66,9 @@ class ChainstateWriteCrashTest(BitcoinTestFramework): self.node3_args = ["-blockmaxweight=4000000", "-acceptnonstdtxn"] self.extra_args = [self.node0_args, self.node1_args, self.node2_args, self.node3_args] - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() - def setup_network(self): self.add_nodes(self.num_nodes, extra_args=self.extra_args) self.start_nodes() - self.import_deterministic_coinbase_privkeys() # Leave them unconnected, we'll use submitblock directly in this test def restart_node(self, node_index, expected_tip): @@ -190,34 +186,36 @@ class ChainstateWriteCrashTest(BitcoinTestFramework): num_transactions = 0 random.shuffle(utxo_list) while len(utxo_list) >= 2 and num_transactions < count: - tx = CTransaction() - input_amount = 0 - for _ in range(2): - utxo = utxo_list.pop() - tx.vin.append(CTxIn(COutPoint(int(utxo['txid'], 16), utxo['vout']))) - input_amount += int(utxo['amount'] * COIN) - output_amount = (input_amount - FEE) // 3 - - if output_amount <= 0: + utxos_to_spend = [utxo_list.pop() for _ in range(2)] + input_amount = int(sum([utxo['value'] for utxo in utxos_to_spend]) * COIN) + if input_amount < FEE: # Sanity check -- if we chose inputs that are too small, skip continue - for _ in range(3): - tx.vout.append(CTxOut(output_amount, bytes.fromhex(utxo['scriptPubKey']))) + tx = self.wallet.create_self_transfer_multi( + from_node=node, + utxos_to_spend=utxos_to_spend, + num_outputs=3, + fee_per_output=FEE // 3) - # Sign and send the transaction to get into the mempool - tx_signed_hex = node.signrawtransactionwithwallet(tx.serialize().hex())['hex'] - node.sendrawtransaction(tx_signed_hex) + # Send the transaction to get into the mempool (skip fee-checks to run faster) + node.sendrawtransaction(hexstring=tx.serialize().hex(), maxfeerate=0) num_transactions += 1 def run_test(self): + self.wallet = MiniWallet(self.nodes[3]) + self.wallet.rescan_utxos() + initial_height = self.nodes[3].getblockcount() + self.generate(self.nodes[3], COINBASE_MATURITY, sync_fun=self.no_op) + # Track test coverage statistics self.restart_counts = [0, 0, 0] # Track the restarts for nodes 0-2 self.crashed_on_restart = 0 # Track count of crashes during recovery # Start by creating a lot of utxos on node3 - initial_height = self.nodes[3].getblockcount() - utxo_list = create_confirmed_utxos(self, self.nodes[3].getnetworkinfo()['relayfee'], self.nodes[3], 5000, sync_fun=self.no_op) + utxo_list = self.wallet.send_self_transfer_multi(from_node=self.nodes[3], num_outputs=5000)['new_utxos'] + self.generate(self.nodes[3], 1, sync_fun=self.no_op) + assert_equal(len(self.nodes[3].getrawmempool()), 0) self.log.info(f"Prepped {len(utxo_list)} utxo entries") # Sync these blocks with the other nodes @@ -257,13 +255,14 @@ class ChainstateWriteCrashTest(BitcoinTestFramework): self.nodes[3], nblocks=min(10, current_height + 1 - self.nodes[3].getblockcount()), # new address to avoid mining a block that has just been invalidated - address=self.nodes[3].getnewaddress(), + address=getnewdestination()[2], sync_fun=self.no_op, )) self.log.debug(f"Syncing {len(block_hashes)} new blocks...") self.sync_node3blocks(block_hashes) - utxo_list = self.nodes[3].listunspent() - self.log.debug(f"Node3 utxo count: {len(utxo_list)}") + self.wallet.rescan_utxos() + utxo_list = self.wallet.get_utxos() + self.log.debug(f"MiniWallet utxo count: {len(utxo_list)}") # Check that the utxo hashes agree with node3 # Useful side effect: each utxo cache gets flushed here, so that we diff --git a/test/functional/feature_index_prune.py b/test/functional/feature_index_prune.py new file mode 100755 index 0000000000..bc85e43a57 --- /dev/null +++ b/test/functional/feature_index_prune.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020-2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test indices in conjunction with prune.""" +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + assert_greater_than, + assert_raises_rpc_error, +) + + +class FeatureIndexPruneTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 4 + self.extra_args = [ + ["-fastprune", "-prune=1", "-blockfilterindex=1"], + ["-fastprune", "-prune=1", "-coinstatsindex=1"], + ["-fastprune", "-prune=1", "-blockfilterindex=1", "-coinstatsindex=1"], + [] + ] + + def sync_index(self, height): + expected_filter = { + 'basic block filter index': {'synced': True, 'best_block_height': height}, + } + self.wait_until(lambda: self.nodes[0].getindexinfo() == expected_filter) + + expected_stats = { + 'coinstatsindex': {'synced': True, 'best_block_height': height} + } + self.wait_until(lambda: self.nodes[1].getindexinfo() == expected_stats) + + expected = {**expected_filter, **expected_stats} + self.wait_until(lambda: self.nodes[2].getindexinfo() == expected) + + def reconnect_nodes(self): + self.connect_nodes(0,1) + self.connect_nodes(0,2) + self.connect_nodes(0,3) + + def mine_batches(self, blocks): + n = blocks // 250 + for _ in range(n): + self.generate(self.nodes[0], 250) + self.generate(self.nodes[0], blocks % 250) + self.sync_blocks() + + def restart_without_indices(self): + for i in range(3): + self.restart_node(i, extra_args=["-fastprune", "-prune=1"]) + self.reconnect_nodes() + + def run_test(self): + filter_nodes = [self.nodes[0], self.nodes[2]] + stats_nodes = [self.nodes[1], self.nodes[2]] + + self.log.info("check if we can access blockfilters and coinstats when pruning is enabled but no blocks are actually pruned") + self.sync_index(height=200) + tip = self.nodes[0].getbestblockhash() + for node in filter_nodes: + assert_greater_than(len(node.getblockfilter(tip)['filter']), 0) + for node in stats_nodes: + assert(node.gettxoutsetinfo(hash_type="muhash", hash_or_height=tip)['muhash']) + + self.mine_batches(500) + self.sync_index(height=700) + + self.log.info("prune some blocks") + for node in self.nodes[:2]: + with node.assert_debug_log(['limited pruning to height 689']): + pruneheight_new = node.pruneblockchain(400) + # the prune heights used here and below are magic numbers that are determined by the + # thresholds at which block files wrap, so they depend on disk serialization and default block file size. + assert_equal(pruneheight_new, 248) + + self.log.info("check if we can access the tips blockfilter and coinstats when we have pruned some blocks") + tip = self.nodes[0].getbestblockhash() + for node in filter_nodes: + assert_greater_than(len(node.getblockfilter(tip)['filter']), 0) + for node in stats_nodes: + assert(node.gettxoutsetinfo(hash_type="muhash", hash_or_height=tip)['muhash']) + + self.log.info("check if we can access the blockfilter and coinstats of a pruned block") + height_hash = self.nodes[0].getblockhash(2) + for node in filter_nodes: + assert_greater_than(len(node.getblockfilter(height_hash)['filter']), 0) + for node in stats_nodes: + assert(node.gettxoutsetinfo(hash_type="muhash", hash_or_height=height_hash)['muhash']) + + # mine and sync index up to a height that will later be the pruneheight + self.generate(self.nodes[0], 51) + self.sync_index(height=751) + + self.restart_without_indices() + + self.log.info("make sure trying to access the indices throws errors") + for node in filter_nodes: + msg = "Index is not enabled for filtertype basic" + assert_raises_rpc_error(-1, msg, node.getblockfilter, height_hash) + for node in stats_nodes: + msg = "Querying specific block heights requires coinstatsindex" + assert_raises_rpc_error(-8, msg, node.gettxoutsetinfo, "muhash", height_hash) + + self.mine_batches(749) + + self.log.info("prune exactly up to the indices best blocks while the indices are disabled") + for i in range(3): + pruneheight_2 = self.nodes[i].pruneblockchain(1000) + assert_equal(pruneheight_2, 750) + # Restart the nodes again with the indices activated + self.restart_node(i, extra_args=self.extra_args[i]) + + self.log.info("make sure that we can continue with the partially synced indices after having pruned up to the index height") + self.sync_index(height=1500) + + self.log.info("prune further than the indices best blocks while the indices are disabled") + self.restart_without_indices() + self.mine_batches(1000) + + for i in range(3): + pruneheight_3 = self.nodes[i].pruneblockchain(2000) + assert_greater_than(pruneheight_3, pruneheight_2) + self.stop_node(i) + + self.log.info("make sure we get an init error when starting the nodes again with the indices") + filter_msg = "Error: basic block filter index best block of the index goes beyond pruned data. Please disable the index or reindex (which will download the whole blockchain again)" + stats_msg = "Error: coinstatsindex best block of the index goes beyond pruned data. Please disable the index or reindex (which will download the whole blockchain again)" + for i, msg in enumerate([filter_msg, stats_msg, filter_msg]): + self.nodes[i].assert_start_raises_init_error(extra_args=self.extra_args[i], expected_msg=msg) + + self.log.info("make sure the nodes start again with the indices and an additional -reindex arg") + for i in range(3): + restart_args = self.extra_args[i]+["-reindex"] + self.restart_node(i, extra_args=restart_args) + # The nodes need to be reconnected to the non-pruning node upon restart, otherwise they will be stuck + self.connect_nodes(i, 3) + + self.sync_blocks(timeout=300) + + for node in self.nodes[:2]: + with node.assert_debug_log(['limited pruning to height 2489']): + pruneheight_new = node.pruneblockchain(2500) + assert_equal(pruneheight_new, 2005) + + self.log.info("ensure that prune locks don't prevent indices from failing in a reorg scenario") + with self.nodes[0].assert_debug_log(['basic block filter index prune lock moved back to 2480']): + self.nodes[3].invalidateblock(self.nodes[0].getblockhash(2480)) + self.generate(self.nodes[3], 30) + self.sync_blocks() + + +if __name__ == '__main__': + FeatureIndexPruneTest().main() diff --git a/test/functional/feature_init.py b/test/functional/feature_init.py index d0cb1e10e2..13c7326519 100755 --- a/test/functional/feature_init.py +++ b/test/functional/feature_init.py @@ -49,33 +49,33 @@ class InitStressTest(BitcoinTestFramework): assert_equal(200, node.getblockcount()) lines_to_terminate_after = [ - 'Validating signatures for all blocks', - 'scheduler thread start', - 'Starting HTTP server', - 'Loading P2P addresses', - 'Loading banlist', - 'Loading block index', - 'Switching active chainstate', - 'Checking all blk files are present', - 'Loaded best chain:', - 'init message: Verifying blocks', - 'init message: Starting network threads', - 'net thread start', - 'addcon thread start', - 'loadblk thread start', - 'txindex thread start', - 'block filter index thread start', - 'coinstatsindex thread start', - 'msghand thread start', - 'net thread start', - 'addcon thread start', + b'Validating signatures for all blocks', + b'scheduler thread start', + b'Starting HTTP server', + b'Loading P2P addresses', + b'Loading banlist', + b'Loading block index', + b'Switching active chainstate', + b'Checking all blk files are present', + b'Loaded best chain:', + b'init message: Verifying blocks', + b'init message: Starting network threads', + b'net thread start', + b'addcon thread start', + b'loadblk thread start', + b'txindex thread start', + b'block filter index thread start', + b'coinstatsindex thread start', + b'msghand thread start', + b'net thread start', + b'addcon thread start', ] if self.is_wallet_compiled(): - lines_to_terminate_after.append('Verifying wallet') + lines_to_terminate_after.append(b'Verifying wallet') for terminate_line in lines_to_terminate_after: - self.log.info(f"Starting node and will exit after line '{terminate_line}'") - with node.wait_for_debug_log([terminate_line], ignore_case=True): + self.log.info(f"Starting node and will exit after line {terminate_line}") + with node.wait_for_debug_log([terminate_line]): node.start(extra_args=['-txindex=1', '-blockfilterindex=1', '-coinstatsindex=1']) self.log.debug("Terminating node after terminate line was found") sigterm_node() diff --git a/test/functional/feature_minchainwork.py b/test/functional/feature_minchainwork.py index 489a729cfc..fa10855a98 100755 --- a/test/functional/feature_minchainwork.py +++ b/test/functional/feature_minchainwork.py @@ -17,6 +17,7 @@ only succeeds past a given node once its nMinimumChainWork has been exceeded. import time +from test_framework.p2p import P2PInterface, msg_getheaders from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal @@ -41,6 +42,9 @@ class MinimumChainWorkTest(BitcoinTestFramework): for i in range(self.num_nodes-1): self.connect_nodes(i+1, i) + # Set clock of node2 2 days ahead, to keep it in IBD during this test. + self.nodes[2].setmocktime(int(time.time()) + 48*60*60) + def run_test(self): # Start building a chain on node0. node2 shouldn't be able to sync until node1's # minchainwork is exceeded @@ -71,6 +75,15 @@ class MinimumChainWorkTest(BitcoinTestFramework): assert self.nodes[1].getbestblockhash() != self.nodes[0].getbestblockhash() assert_equal(self.nodes[2].getblockcount(), starting_blockcount) + self.log.info("Check that getheaders requests to node2 are ignored") + peer = self.nodes[2].add_p2p_connection(P2PInterface()) + msg = msg_getheaders() + msg.locator.vHave = [int(self.nodes[2].getbestblockhash(), 16)] + msg.hashstop = 0 + peer.send_and_ping(msg) + time.sleep(5) + assert "headers" not in peer.last_message + self.log.info("Generating one more block") self.generate(self.nodes[0], 1) @@ -85,5 +98,21 @@ class MinimumChainWorkTest(BitcoinTestFramework): self.sync_all() self.log.info(f"Blockcounts: {[n.getblockcount() for n in self.nodes]}") + self.log.info("Test that getheaders requests to node2 are not ignored") + peer.send_and_ping(msg) + assert "headers" in peer.last_message + + # Verify that node2 is in fact still in IBD (otherwise this test may + # not be exercising the logic we want!) + assert_equal(self.nodes[2].getblockchaininfo()['initialblockdownload'], True) + + self.log.info("Test -minimumchainwork with a non-hex value") + self.stop_node(0) + self.nodes[0].assert_start_raises_init_error( + ["-minimumchainwork=test"], + expected_msg='Error: Invalid non-hex (test) minimum chain work value specified', + ) + + if __name__ == '__main__': MinimumChainWorkTest().main() diff --git a/test/functional/feature_proxy.py b/test/functional/feature_proxy.py index 8541c3ed88..dd3cdc96ca 100755 --- a/test/functional/feature_proxy.py +++ b/test/functional/feature_proxy.py @@ -36,22 +36,19 @@ addnode connect to a CJDNS address - Test passing invalid -i2psam - Test passing -onlynet=onion without -proxy or -onion - Test passing -onlynet=onion with -onion=0 and with -noonion +- Test passing unknown -onlynet """ import socket -import os from test_framework.socks5 import Socks5Configuration, Socks5Command, Socks5Server, AddressType from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( - PORT_MIN, - PORT_RANGE, assert_equal, + p2p_port, ) from test_framework.netutil import test_ipv6_local -RANGE_BEGIN = PORT_MIN + 2 * PORT_RANGE # Start after p2p and rpc ports - # Networks returned by RPC getpeerinfo. NET_UNROUTABLE = "not_publicly_routable" NET_IPV4 = "ipv4" @@ -74,19 +71,19 @@ class ProxyTest(BitcoinTestFramework): # Create two proxies on different ports # ... one unauthenticated self.conf1 = Socks5Configuration() - self.conf1.addr = ('127.0.0.1', RANGE_BEGIN + (os.getpid() % 1000)) + self.conf1.addr = ('127.0.0.1', p2p_port(self.num_nodes)) self.conf1.unauth = True self.conf1.auth = False # ... one supporting authenticated and unauthenticated (Tor) self.conf2 = Socks5Configuration() - self.conf2.addr = ('127.0.0.1', RANGE_BEGIN + 1000 + (os.getpid() % 1000)) + self.conf2.addr = ('127.0.0.1', p2p_port(self.num_nodes + 1)) self.conf2.unauth = True self.conf2.auth = True if self.have_ipv6: # ... one on IPv6 with similar configuration self.conf3 = Socks5Configuration() self.conf3.af = socket.AF_INET6 - self.conf3.addr = ('::1', RANGE_BEGIN + 2000 + (os.getpid() % 1000)) + self.conf3.addr = ('::1', p2p_port(self.num_nodes + 2)) self.conf3.unauth = True self.conf3.auth = True else: @@ -349,6 +346,11 @@ class ProxyTest(BitcoinTestFramework): self.nodes[1].extra_args = ["-onlynet=onion", arg] self.nodes[1].assert_start_raises_init_error(expected_msg=msg) + self.log.info("Test passing unknown network to -onlynet raises expected init error") + self.nodes[1].extra_args = ["-onlynet=abc"] + msg = "Error: Unknown network specified in -onlynet: 'abc'" + self.nodes[1].assert_start_raises_init_error(expected_msg=msg) + if __name__ == '__main__': ProxyTest().main() diff --git a/test/functional/feature_pruning.py b/test/functional/feature_pruning.py index bf19384279..7dbeccbc09 100755 --- a/test/functional/feature_pruning.py +++ b/test/functional/feature_pruning.py @@ -125,6 +125,7 @@ class PruneTest(BitcoinTestFramework): self.sync_blocks(self.nodes[0:5]) def test_invalid_command_line_options(self): + self.stop_node(0) self.nodes[0].assert_start_raises_init_error( expected_msg='Error: Prune cannot be configured with a negative value.', extra_args=['-prune=-1'], @@ -138,10 +139,6 @@ class PruneTest(BitcoinTestFramework): extra_args=['-prune=550', '-txindex'], ) self.nodes[0].assert_start_raises_init_error( - expected_msg='Error: Prune mode is incompatible with -coinstatsindex.', - extra_args=['-prune=550', '-coinstatsindex'], - ) - self.nodes[0].assert_start_raises_init_error( expected_msg='Error: Prune mode is incompatible with -reindex-chainstate. Use full -reindex instead.', extra_args=['-prune=550', '-reindex-chainstate'], ) @@ -294,7 +291,7 @@ class PruneTest(BitcoinTestFramework): def prune(index): ret = node.pruneblockchain(height=height(index)) - assert_equal(ret, node.getblockchaininfo()['pruneheight']) + assert_equal(ret + 1, node.getblockchaininfo()['pruneheight']) def has_block(index): return os.path.isfile(os.path.join(self.nodes[node_number].datadir, self.chain, "blocks", f"blk{index:05}.dat")) diff --git a/test/functional/feature_rbf.py b/test/functional/feature_rbf.py index f0ed914461..a8492bd6eb 100755 --- a/test/functional/feature_rbf.py +++ b/test/functional/feature_rbf.py @@ -32,7 +32,7 @@ from test_framework.address import ADDRESS_BCRT1_UNSPENDABLE MAX_REPLACEMENT_LIMIT = 100 class ReplaceByFeeTest(BitcoinTestFramework): def set_test_params(self): - self.num_nodes = 1 + self.num_nodes = 2 self.extra_args = [ [ "-acceptnonstdtxn=1", @@ -42,6 +42,9 @@ class ReplaceByFeeTest(BitcoinTestFramework): "-limitdescendantcount=200", "-limitdescendantsize=101", ], + # second node has default mempool parameters + [ + ], ] self.supports_cli = False @@ -73,6 +76,9 @@ class ReplaceByFeeTest(BitcoinTestFramework): self.log.info("Running test too many replacements...") self.test_too_many_replacements() + self.log.info("Running test too many replacements using default mempool params...") + self.test_too_many_replacements_with_default_mempool_params() + self.log.info("Running test opt-in...") self.test_opt_in() @@ -397,6 +403,94 @@ class ReplaceByFeeTest(BitcoinTestFramework): double_tx_hex = double_tx.serialize().hex() self.nodes[0].sendrawtransaction(double_tx_hex, 0) + def test_too_many_replacements_with_default_mempool_params(self): + """ + Test rule 5 of BIP125 (do not allow replacements that cause more than 100 + evictions) without having to rely on non-default mempool parameters. + + In order to do this, create a number of "root" UTXOs, and then hang + enough transactions off of each root UTXO to exceed the MAX_REPLACEMENT_LIMIT. + Then create a conflicting RBF replacement transaction. + """ + normal_node = self.nodes[1] + wallet = MiniWallet(normal_node) + wallet.rescan_utxos() + # Clear mempools to avoid cross-node sync failure. + for node in self.nodes: + self.generate(node, 1) + + # This has to be chosen so that the total number of transactions can exceed + # MAX_REPLACEMENT_LIMIT without having any one tx graph run into the descendant + # limit; 10 works. + num_tx_graphs = 10 + + # (Number of transactions per graph, BIP125 rule 5 failure expected) + cases = [ + # Test the base case of evicting fewer than MAX_REPLACEMENT_LIMIT + # transactions. + ((MAX_REPLACEMENT_LIMIT // num_tx_graphs) - 1, False), + + # Test hitting the rule 5 eviction limit. + (MAX_REPLACEMENT_LIMIT // num_tx_graphs, True), + ] + + for (txs_per_graph, failure_expected) in cases: + self.log.debug(f"txs_per_graph: {txs_per_graph}, failure: {failure_expected}") + # "Root" utxos of each txn graph that we will attempt to double-spend with + # an RBF replacement. + root_utxos = [] + + # For each root UTXO, create a package that contains the spend of that + # UTXO and `txs_per_graph` children tx. + for graph_num in range(num_tx_graphs): + root_utxos.append(wallet.get_utxo()) + + optin_parent_tx = wallet.send_self_transfer_multi( + from_node=normal_node, + sequence=BIP125_SEQUENCE_NUMBER, + utxos_to_spend=[root_utxos[graph_num]], + num_outputs=txs_per_graph, + ) + assert_equal(True, normal_node.getmempoolentry(optin_parent_tx['txid'])['bip125-replaceable']) + new_utxos = optin_parent_tx['new_utxos'] + + for utxo in new_utxos: + # Create spends for each output from the "root" of this graph. + child_tx = wallet.send_self_transfer( + from_node=normal_node, + utxo_to_spend=utxo, + ) + + assert normal_node.getmempoolentry(child_tx['txid']) + + num_txs_invalidated = len(root_utxos) + (num_tx_graphs * txs_per_graph) + + if failure_expected: + assert num_txs_invalidated > MAX_REPLACEMENT_LIMIT + else: + assert num_txs_invalidated <= MAX_REPLACEMENT_LIMIT + + # Now attempt to submit a tx that double-spends all the root tx inputs, which + # would invalidate `num_txs_invalidated` transactions. + double_tx = wallet.create_self_transfer_multi( + from_node=normal_node, + utxos_to_spend=root_utxos, + fee_per_output=10_000_000, # absurdly high feerate + ) + tx_hex = double_tx.serialize().hex() + + if failure_expected: + assert_raises_rpc_error( + -26, "too many potential replacements", normal_node.sendrawtransaction, tx_hex, 0) + else: + txid = normal_node.sendrawtransaction(tx_hex, 0) + assert normal_node.getmempoolentry(txid) + + # Clear the mempool once finished, and rescan the other nodes' wallet + # to account for the spends we've made on `normal_node`. + self.generate(normal_node, 1) + self.wallet.rescan_utxos() + def test_opt_in(self): """Replacing should only work if orig tx opted in""" tx0_outpoint = self.make_utxo(self.nodes[0], int(1.1 * COIN)) diff --git a/test/functional/feature_taproot.py b/test/functional/feature_taproot.py index c3925dbb00..0e44038196 100755 --- a/test/functional/feature_taproot.py +++ b/test/functional/feature_taproot.py @@ -10,7 +10,6 @@ from test_framework.blocktools import ( create_block, add_witness_commitment, MAX_BLOCK_SIGOPS_WEIGHT, - NORMAL_GBT_REQUEST_PARAMS, WITNESS_SCALE_FACTOR, ) from test_framework.messages import ( @@ -96,10 +95,9 @@ from test_framework.util import assert_raises_rpc_error, assert_equal from test_framework.key import generate_privkey, compute_xonly_pubkey, sign_schnorr, tweak_add_privkey, ECKey from test_framework.address import ( hash160, - program_to_witness + program_to_witness, ) from collections import OrderedDict, namedtuple -from enum import Enum from io import BytesIO import json import hashlib @@ -458,7 +456,7 @@ def spend(tx, idx, utxos, **kwargs): # Each spender is a tuple of: # - A scriptPubKey which is to be spent from (CScript) # - A comment describing the test (string) -# - Whether the spending (on itself) is expected to be standard (Enum.Standard) +# - Whether the spending (on itself) is expected to be standard (bool) # - A tx-signing lambda returning (scriptsig, witness_stack), taking as inputs: # - A transaction to sign (CTransaction) # - An input position (int) @@ -470,14 +468,9 @@ def spend(tx, idx, utxos, **kwargs): # - Whether this test demands being placed in a txin with no corresponding txout (for testing SIGHASH_SINGLE behavior) Spender = namedtuple("Spender", "script,comment,is_standard,sat_function,err_msg,sigops_weight,no_fail,need_vin_vout_mismatch") -# The full node versions that treat the tx standard. -# ALL means any version -# V23 means the major version 23.0 and any later version -# NONE means no version -Standard = Enum('Standard', 'ALL V23 NONE') -def make_spender(comment, *, tap=None, witv0=False, script=None, pkh=None, p2sh=False, spk_mutate_pre_p2sh=None, failure=None, standard=Standard.ALL, err_msg=None, sigops_weight=0, need_vin_vout_mismatch=False, **kwargs): +def make_spender(comment, *, tap=None, witv0=False, script=None, pkh=None, p2sh=False, spk_mutate_pre_p2sh=None, failure=None, standard=True, err_msg=None, sigops_weight=0, need_vin_vout_mismatch=False, **kwargs): """Helper for constructing Spender objects using the context signing framework. * tap: a TaprootInfo object (see taproot_construct), for Taproot spends (cannot be combined with pkh, witv0, or script) @@ -487,18 +480,13 @@ def make_spender(comment, *, tap=None, witv0=False, script=None, pkh=None, p2sh= * p2sh: whether the output is P2SH wrapper (this is supported even for Taproot, where it makes the output unencumbered) * spk_mutate_pre_psh: a callable to be applied to the script (before potentially P2SH-wrapping it) * failure: a dict of entries to override in the context when intentionally failing to spend (if None, no_fail will be set) - * standard: whether the (valid version of) spending is expected to be standard (True is mapped to Standard.ALL, False is mapped to Standard.NONE) + * standard: whether the (valid version of) spending is expected to be standard * err_msg: a string with an expected error message for failure (or None, if not cared about) * sigops_weight: the pre-taproot sigops weight consumed by a successful spend * need_vin_vout_mismatch: whether this test requires being tested in a transaction input that has no corresponding transaction output. """ - if standard == True: - standard = Standard.ALL - elif standard == False: - standard = Standard.NONE - conf = dict() # Compute scriptPubKey and set useful defaults based on the inputs. @@ -1168,24 +1156,20 @@ def spenders_taproot_active(): return spenders -def spenders_taproot_inactive(): - """Spenders for testing that pre-activation Taproot rules don't apply.""" + +def spenders_taproot_nonstandard(): + """Spenders for testing that post-activation Taproot rules may be nonstandard.""" spenders = [] sec = generate_privkey() pub, _ = compute_xonly_pubkey(sec) scripts = [ - ("pk", CScript([pub, OP_CHECKSIG])), ("future_leaf", CScript([pub, OP_CHECKSIG]), 0xc2), ("op_success", CScript([pub, OP_CHECKSIG, OP_0, OP_IF, CScriptOp(0x50), OP_ENDIF])), ] tap = taproot_construct(pub, scripts) - # Test that valid spending is standard. - add_spender(spenders, "inactive/keypath_valid", key=sec, tap=tap, standard=Standard.V23) - add_spender(spenders, "inactive/scriptpath_valid", key=sec, tap=tap, leaf="pk", standard=Standard.V23, inputs=[getter("sign")]) - # Test that features like annex, leaf versions, or OP_SUCCESS are valid but non-standard add_spender(spenders, "inactive/scriptpath_valid_unkleaf", key=sec, tap=tap, leaf="future_leaf", standard=False, inputs=[getter("sign")]) add_spender(spenders, "inactive/scriptpath_invalid_unkleaf", key=sec, tap=tap, leaf="future_leaf", standard=False, inputs=[getter("sign")], sighash=bitflipper(default_sighash)) @@ -1214,7 +1198,7 @@ def dump_json_test(tx, input_utxos, idx, success, failure): # The "final" field indicates that a spend should be always valid, even with more validation flags enabled # than the listed ones. Use standardness as a proxy for this (which gives a conservative underestimate). - if spender.is_standard == Standard.ALL: + if spender.is_standard: fields.append(("final", True)) def dump_witness(wit): @@ -1241,31 +1225,14 @@ class TaprootTest(BitcoinTestFramework): def add_options(self, parser): parser.add_argument("--dumptests", dest="dump_tests", default=False, action="store_true", help="Dump generated test cases to directory set by TEST_DUMP_DIR environment variable") - parser.add_argument("--previous_release", dest="previous_release", default=False, action="store_true", - help="Use a previous release as taproot-inactive node") def skip_test_if_missing_module(self): self.skip_if_no_wallet() - if self.options.previous_release: - self.skip_if_no_previous_releases() def set_test_params(self): - self.num_nodes = 2 + self.num_nodes = 1 self.setup_clean_chain = True - # Node 0 has Taproot inactive, Node 1 active. - self.extra_args = [["-par=1"], ["-par=1"]] - if self.options.previous_release: - self.wallet_names = [None, self.default_wallet_name] - else: - self.extra_args[0].append("-vbparams=taproot:1:1") - - def setup_nodes(self): - self.add_nodes(self.num_nodes, self.extra_args, versions=[ - 200100 if self.options.previous_release else None, - None, - ]) - self.start_nodes() - self.import_deterministic_coinbase_privkeys() + self.extra_args = [["-par=1"]] def block_submit(self, node, txs, msg, err_msg, cb_pubkey=None, fees=0, sigops_weight=0, witness=False, accept=False): @@ -1479,11 +1446,10 @@ class TaprootTest(BitcoinTestFramework): for i in range(len(input_utxos)): tx.vin[i].scriptSig = input_data[i][i != fail_input][0] tx.wit.vtxinwit[i].scriptWitness.stack = input_data[i][i != fail_input][1] - taproot_spend_policy = Standard.V23 if node.version is None else Standard.ALL # Submit to mempool to check standardness is_standard_tx = ( fail_input is None # Must be valid to be standard - and (all(utxo.spender.is_standard == Standard.ALL or utxo.spender.is_standard == taproot_spend_policy for utxo in input_utxos)) # All inputs must be standard + and (all(utxo.spender.is_standard for utxo in input_utxos)) # All inputs must be standard and tx.nVersion >= 1 # The tx version must be standard and tx.nVersion <= 2) tx.rehash() @@ -1510,7 +1476,7 @@ class TaprootTest(BitcoinTestFramework): self.log.info("Unit test scenario...") # Deterministically mine coins to OP_TRUE in block 1 - assert self.nodes[1].getblockcount() == 0 + assert_equal(self.nodes[0].getblockcount(), 0) coinbase = CTransaction() coinbase.nVersion = 1 coinbase.vin = [CTxIn(COutPoint(0, 0xffffffff), CScript([OP_1, OP_1]), SEQUENCE_FINAL)] @@ -1519,12 +1485,12 @@ class TaprootTest(BitcoinTestFramework): coinbase.rehash() assert coinbase.hash == "f60c73405d499a956d3162e3483c395526ef78286458a4cb17b125aa92e49b20" # Mine it - block = create_block(hashprev=int(self.nodes[1].getbestblockhash(), 16), coinbase=coinbase) + block = create_block(hashprev=int(self.nodes[0].getbestblockhash(), 16), coinbase=coinbase) block.rehash() block.solve() - self.nodes[1].submitblock(block.serialize().hex()) - assert self.nodes[1].getblockcount() == 1 - self.generate(self.nodes[1], COINBASE_MATURITY) + self.nodes[0].submitblock(block.serialize().hex()) + assert_equal(self.nodes[0].getblockcount(), 1) + self.generate(self.nodes[0], COINBASE_MATURITY) SEED = 317 VALID_LEAF_VERS = list(range(0xc0, 0x100, 2)) + [0x66, 0x7e, 0x80, 0x84, 0x96, 0x98, 0xba, 0xbc, 0xbe] @@ -1613,8 +1579,8 @@ class TaprootTest(BitcoinTestFramework): spend_info[spk]['prevout'] = COutPoint(tx.sha256, i & 1) spend_info[spk]['utxo'] = CTxOut(val, spk) # Mine those transactions - self.init_blockinfo(self.nodes[1]) - self.block_submit(self.nodes[1], txn, "Crediting txn", None, sigops_weight=10, accept=True) + self.init_blockinfo(self.nodes[0]) + self.block_submit(self.nodes[0], txn, "Crediting txn", None, sigops_weight=10, accept=True) # scriptPubKey computation tests = {"version": 1} @@ -1726,53 +1692,21 @@ class TaprootTest(BitcoinTestFramework): keypath_tests.append(tx_test) assert_equal(hashlib.sha256(tx.serialize()).hexdigest(), "24bab662cb55a7f3bae29b559f651674c62bcc1cd442d44715c0133939107b38") # Mine the spending transaction - self.block_submit(self.nodes[1], [tx], "Spending txn", None, sigops_weight=10000, accept=True, witness=True) + self.block_submit(self.nodes[0], [tx], "Spending txn", None, sigops_weight=10000, accept=True, witness=True) if GEN_TEST_VECTORS: print(json.dumps(tests, indent=4, sort_keys=False)) - def run_test(self): self.gen_test_vectors() - # Post-taproot activation tests go first (pre-taproot tests' blocks are invalid post-taproot). self.log.info("Post-activation tests...") - self.test_spenders(self.nodes[1], spenders_taproot_active(), input_counts=[1, 2, 2, 2, 2, 3]) - - # Re-connect nodes in case they have been disconnected - self.disconnect_nodes(0, 1) - self.connect_nodes(0, 1) - - # Transfer value of the largest 500 coins to pre-taproot node. - addr = self.nodes[0].getnewaddress() - - unsp = self.nodes[1].listunspent() - unsp = sorted(unsp, key=lambda i: i['amount'], reverse=True) - unsp = unsp[:500] - - rawtx = self.nodes[1].createrawtransaction( - inputs=[{ - 'txid': i['txid'], - 'vout': i['vout'] - } for i in unsp], - outputs={addr: sum(i['amount'] for i in unsp)} - ) - rawtx = self.nodes[1].signrawtransactionwithwallet(rawtx)['hex'] - - # Mine a block with the transaction - block = create_block(tmpl=self.nodes[1].getblocktemplate(NORMAL_GBT_REQUEST_PARAMS), txlist=[rawtx]) - add_witness_commitment(block) - block.solve() - assert_equal(None, self.nodes[1].submitblock(block.serialize().hex())) - self.sync_blocks() - - # Pre-taproot activation tests. - self.log.info("Pre-activation tests...") + self.test_spenders(self.nodes[0], spenders_taproot_active(), input_counts=[1, 2, 2, 2, 2, 3]) # Run each test twice; once in isolation, and once combined with others. Testing in isolation # means that the standardness is verified in every test (as combined transactions are only standard # when all their inputs are standard). - self.test_spenders(self.nodes[0], spenders_taproot_inactive(), input_counts=[1]) - self.test_spenders(self.nodes[0], spenders_taproot_inactive(), input_counts=[2, 3]) + self.test_spenders(self.nodes[0], spenders_taproot_nonstandard(), input_counts=[1]) + self.test_spenders(self.nodes[0], spenders_taproot_nonstandard(), input_counts=[2, 3]) if __name__ == '__main__': diff --git a/test/functional/feature_versionbits_warning.py b/test/functional/feature_versionbits_warning.py index e83dd7f446..1572463308 100755 --- a/test/functional/feature_versionbits_warning.py +++ b/test/functional/feature_versionbits_warning.py @@ -58,7 +58,8 @@ class VersionBitsWarningTest(BitcoinTestFramework): def versionbits_in_alert_file(self): """Test that the versionbits warning has been written to the alert file.""" - alert_text = open(self.alert_filename, 'r', encoding='utf8').read() + with open(self.alert_filename, 'r', encoding='utf8') as f: + alert_text = f.read() return VB_PATTERN.search(alert_text) is not None def run_test(self): diff --git a/test/functional/interface_rest.py b/test/functional/interface_rest.py index 30c9e0c9cd..f36bbda3af 100755 --- a/test/functional/interface_rest.py +++ b/test/functional/interface_rest.py @@ -219,7 +219,7 @@ class RESTTest (BitcoinTestFramework): self.generate(self.nodes[0], 1) # generate block to not affect upcoming tests - self.log.info("Test the /block, /blockhashbyheight and /headers URIs") + self.log.info("Test the /block, /blockhashbyheight, /headers, and /blockfilterheaders URIs") bb_hash = self.nodes[0].getbestblockhash() # Check result if block does not exists @@ -300,6 +300,12 @@ class RESTTest (BitcoinTestFramework): assert_equal(first_filter_header, rpc_blockfilter['header']) assert_equal(json_obj['filter'], rpc_blockfilter['filter']) + # Test blockfilterheaders with an invalid hash and filtertype + resp = self.test_rest_request(f"/blockfilterheaders/{INVALID_PARAM}/{bb_hash}", ret_type=RetType.OBJ, status=400) + assert_equal(resp.read().decode('utf-8').rstrip(), f"Unknown filtertype {INVALID_PARAM}") + resp = self.test_rest_request(f"/blockfilterheaders/basic/{INVALID_PARAM}", ret_type=RetType.OBJ, status=400) + assert_equal(resp.read().decode('utf-8').rstrip(), f"Invalid hash: {INVALID_PARAM}") + # Test number parsing for num in ['5a', '-5', '0', '2001', '99999999999999999999999999999999999']: assert_equal( @@ -324,8 +330,15 @@ class RESTTest (BitcoinTestFramework): # the size of the memory pool should be greater than 3x ~100 bytes assert_greater_than(json_obj['bytes'], 300) + mempool_info = self.nodes[0].getmempoolinfo() + assert_equal(json_obj, mempool_info) + # Check that there are our submitted transactions in the TX memory pool json_obj = self.test_rest_request("/mempool/contents") + raw_mempool_verbose = self.nodes[0].getrawmempool(verbose=True) + + assert_equal(json_obj, raw_mempool_verbose) + for i, tx in enumerate(txs): assert tx in json_obj assert_equal(json_obj[tx]['spentby'], txs[i + 1:i + 2]) diff --git a/test/functional/interface_usdt_coinselection.py b/test/functional/interface_usdt_coinselection.py new file mode 100755 index 0000000000..ef32feda99 --- /dev/null +++ b/test/functional/interface_usdt_coinselection.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" Tests the coin_selection:* tracepoint API interface. + See https://github.com/bitcoin/bitcoin/blob/master/doc/tracing.md#context-coin_selection +""" + +# Test will be skipped if we don't have bcc installed +try: + from bcc import BPF, USDT # type: ignore[import] +except ImportError: + pass +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + assert_greater_than, + assert_raises_rpc_error, +) + +coinselection_tracepoints_program = """ +#include <uapi/linux/ptrace.h> + +#define WALLET_NAME_LENGTH 16 +#define ALGO_NAME_LENGTH 16 + +struct event_data +{ + u8 type; + char wallet_name[WALLET_NAME_LENGTH]; + + // selected coins event + char algo[ALGO_NAME_LENGTH]; + s64 target; + s64 waste; + s64 selected_value; + + // create tx event + bool success; + s64 fee; + s32 change_pos; + + // aps create tx event + bool use_aps; +}; + +BPF_QUEUE(coin_selection_events, struct event_data, 1024); + +int trace_selected_coins(struct pt_regs *ctx) { + struct event_data data; + __builtin_memset(&data, 0, sizeof(data)); + data.type = 1; + bpf_usdt_readarg_p(1, ctx, &data.wallet_name, WALLET_NAME_LENGTH); + bpf_usdt_readarg_p(2, ctx, &data.algo, ALGO_NAME_LENGTH); + bpf_usdt_readarg(3, ctx, &data.target); + bpf_usdt_readarg(4, ctx, &data.waste); + bpf_usdt_readarg(5, ctx, &data.selected_value); + coin_selection_events.push(&data, 0); + return 0; +} + +int trace_normal_create_tx(struct pt_regs *ctx) { + struct event_data data; + __builtin_memset(&data, 0, sizeof(data)); + data.type = 2; + bpf_usdt_readarg_p(1, ctx, &data.wallet_name, WALLET_NAME_LENGTH); + bpf_usdt_readarg(2, ctx, &data.success); + bpf_usdt_readarg(3, ctx, &data.fee); + bpf_usdt_readarg(4, ctx, &data.change_pos); + coin_selection_events.push(&data, 0); + return 0; +} + +int trace_attempt_aps(struct pt_regs *ctx) { + struct event_data data; + __builtin_memset(&data, 0, sizeof(data)); + data.type = 3; + bpf_usdt_readarg_p(1, ctx, &data.wallet_name, WALLET_NAME_LENGTH); + coin_selection_events.push(&data, 0); + return 0; +} + +int trace_aps_create_tx(struct pt_regs *ctx) { + struct event_data data; + __builtin_memset(&data, 0, sizeof(data)); + data.type = 4; + bpf_usdt_readarg_p(1, ctx, &data.wallet_name, WALLET_NAME_LENGTH); + bpf_usdt_readarg(2, ctx, &data.use_aps); + bpf_usdt_readarg(3, ctx, &data.success); + bpf_usdt_readarg(4, ctx, &data.fee); + bpf_usdt_readarg(5, ctx, &data.change_pos); + coin_selection_events.push(&data, 0); + return 0; +} +""" + + +class CoinSelectionTracepointTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + self.setup_clean_chain = True + + def skip_test_if_missing_module(self): + self.skip_if_platform_not_linux() + self.skip_if_no_bitcoind_tracepoints() + self.skip_if_no_python_bcc() + self.skip_if_no_bpf_permissions() + self.skip_if_no_wallet() + + def get_tracepoints(self, expected_types): + events = [] + try: + for i in range(0, len(expected_types) + 1): + event = self.bpf["coin_selection_events"].pop() + assert_equal(event.wallet_name.decode(), self.default_wallet_name) + assert_equal(event.type, expected_types[i]) + events.append(event) + else: + # If the loop exits successfully instead of throwing a KeyError, then we have had + # more events than expected. There should be no more than len(expected_types) events. + assert False + except KeyError: + assert_equal(len(events), len(expected_types)) + return events + + + def determine_selection_from_usdt(self, events): + success = None + use_aps = None + algo = None + waste = None + change_pos = None + + is_aps = False + sc_events = [] + for event in events: + if event.type == 1: + if not is_aps: + algo = event.algo.decode() + waste = event.waste + sc_events.append(event) + elif event.type == 2: + success = event.success + if not is_aps: + change_pos = event.change_pos + elif event.type == 3: + is_aps = True + elif event.type == 4: + assert is_aps + if event.use_aps: + use_aps = True + assert_equal(len(sc_events), 2) + algo = sc_events[1].algo.decode() + waste = sc_events[1].waste + change_pos = event.change_pos + return success, use_aps, algo, waste, change_pos + + def run_test(self): + self.log.info("hook into the coin_selection tracepoints") + ctx = USDT(pid=self.nodes[0].process.pid) + ctx.enable_probe(probe="coin_selection:selected_coins", fn_name="trace_selected_coins") + ctx.enable_probe(probe="coin_selection:normal_create_tx_internal", fn_name="trace_normal_create_tx") + ctx.enable_probe(probe="coin_selection:attempting_aps_create_tx", fn_name="trace_attempt_aps") + ctx.enable_probe(probe="coin_selection:aps_create_tx_internal", fn_name="trace_aps_create_tx") + self.bpf = BPF(text=coinselection_tracepoints_program, usdt_contexts=[ctx], debug=0) + + self.log.info("Prepare wallets") + self.generate(self.nodes[0], 101) + wallet = self.nodes[0].get_wallet_rpc(self.default_wallet_name) + + self.log.info("Sending a transaction should result in all tracepoints") + # We should have 5 tracepoints in the order: + # 1. selected_coins (type 1) + # 2. normal_create_tx_internal (type 2) + # 3. attempting_aps_create_tx (type 3) + # 4. selected_coins (type 1) + # 5. aps_create_tx_internal (type 4) + wallet.sendtoaddress(wallet.getnewaddress(), 10) + events = self.get_tracepoints([1, 2, 3, 1, 4]) + success, use_aps, algo, waste, change_pos = self.determine_selection_from_usdt(events) + assert_equal(success, True) + assert_greater_than(change_pos, -1) + + self.log.info("Failing to fund results in 1 tracepoint") + # We should have 1 tracepoints in the order + # 1. normal_create_tx_internal (type 2) + assert_raises_rpc_error(-6, "Insufficient funds", wallet.sendtoaddress, wallet.getnewaddress(), 102 * 50) + events = self.get_tracepoints([2]) + success, use_aps, algo, waste, change_pos = self.determine_selection_from_usdt(events) + assert_equal(success, False) + + self.log.info("Explicitly enabling APS results in 2 tracepoints") + # We should have 2 tracepoints in the order + # 1. selected_coins (type 1) + # 2. normal_create_tx_internal (type 2) + wallet.setwalletflag("avoid_reuse") + wallet.sendtoaddress(address=wallet.getnewaddress(), amount=10, avoid_reuse=True) + events = self.get_tracepoints([1, 2]) + success, use_aps, algo, waste, change_pos = self.determine_selection_from_usdt(events) + assert_equal(success, True) + assert_equal(use_aps, None) + + self.bpf.cleanup() + + +if __name__ == '__main__': + CoinSelectionTracepointTest().main() diff --git a/test/functional/mempool_accept.py b/test/functional/mempool_accept.py index d3961e753d..85464b8d0d 100755 --- a/test/functional/mempool_accept.py +++ b/test/functional/mempool_accept.py @@ -76,7 +76,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): tx.vout[0].nValue = int(0.3 * COIN) tx.vout[1].nValue = int(49 * COIN) raw_tx_in_block = tx.serialize().hex() - txid_in_block = self.wallet.sendrawtransaction(from_node=node, tx_hex=raw_tx_in_block, maxfeerate=0) + txid_in_block = self.wallet.sendrawtransaction(from_node=node, tx_hex=raw_tx_in_block) self.generate(node, 1) self.mempool_size = 0 self.check_mempool_result( @@ -166,7 +166,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): tx.vin[1].prevout = COutPoint(hash=int(txid_1, 16), n=0) tx.vout[0].nValue = int(0.1 * COIN) raw_tx_spend_both = tx.serialize().hex() - txid_spend_both = self.wallet.sendrawtransaction(from_node=node, tx_hex=raw_tx_spend_both, maxfeerate=0) + txid_spend_both = self.wallet.sendrawtransaction(from_node=node, tx_hex=raw_tx_spend_both) self.generate(node, 1) self.mempool_size = 0 # Now see if we can add the coins back to the utxo set by sending the exact txs again diff --git a/test/functional/mempool_packages.py b/test/functional/mempool_packages.py index 068fdc0b65..a2a2caf324 100755 --- a/test/functional/mempool_packages.py +++ b/test/functional/mempool_packages.py @@ -100,6 +100,12 @@ class MempoolPackagesTest(BitcoinTestFramework): entry = self.nodes[0].getmempoolentry(x) assert_equal(entry, mempool[x]) + # Check that gettxspendingprevout is consistent with getrawmempool + witnesstx = self.nodes[0].gettransaction(txid=x, verbose=True)['decoded'] + for tx_in in witnesstx["vin"]: + spending_result = self.nodes[0].gettxspendingprevout([ {'txid' : tx_in["txid"], 'vout' : tx_in["vout"]} ]) + assert_equal(spending_result, [ {'txid' : tx_in["txid"], 'vout' : tx_in["vout"], 'spendingtxid' : x} ]) + # Check that the descendant calculations are correct assert_equal(entry['descendantcount'], descendant_count) descendant_fees += entry['fees']['base'] diff --git a/test/functional/p2p_addr_relay.py b/test/functional/p2p_addr_relay.py index 3218a9b14a..e2e9b6dcb2 100755 --- a/test/functional/p2p_addr_relay.py +++ b/test/functional/p2p_addr_relay.py @@ -21,8 +21,19 @@ from test_framework.p2p import ( P2P_SERVICES, ) from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_greater_than +from test_framework.util import ( + assert_equal, + assert_greater_than, + assert_greater_than_or_equal +) + +ONE_MINUTE = 60 +TEN_MINUTES = 10 * ONE_MINUTE +ONE_HOUR = 60 * ONE_MINUTE +TWO_HOURS = 2 * ONE_HOUR +ONE_DAY = 24 * ONE_HOUR +ADDR_DESTINATIONS_THRESHOLD = 4 class AddrReceiver(P2PInterface): num_ipv4_received = 0 @@ -85,6 +96,9 @@ class AddrTest(BitcoinTestFramework): self.relay_tests() self.inbound_blackhole_tests() + self.destination_rotates_once_in_24_hours_test() + self.destination_rotates_more_than_once_over_several_days_test() + # This test populates the addrman, which can impact the node's behavior # in subsequent tests self.getaddr_tests() @@ -362,6 +376,56 @@ class AddrTest(BitcoinTestFramework): self.nodes[0].disconnect_p2ps() + def get_nodes_that_received_addr(self, peer, receiver_peer, addr_receivers, + time_interval_1, time_interval_2): + + # Clean addr response related to the initial getaddr. There is no way to avoid initial + # getaddr because the peer won't self-announce then. + for addr_receiver in addr_receivers: + addr_receiver.num_ipv4_received = 0 + + for _ in range(10): + self.mocktime += time_interval_1 + self.msg.addrs[0].time = self.mocktime + TEN_MINUTES + self.nodes[0].setmocktime(self.mocktime) + with self.nodes[0].assert_debug_log(['received: addr (31 bytes) peer=0']): + peer.send_and_ping(self.msg) + self.mocktime += time_interval_2 + self.nodes[0].setmocktime(self.mocktime) + receiver_peer.sync_with_ping() + return [node for node in addr_receivers if node.addr_received()] + + def destination_rotates_once_in_24_hours_test(self): + self.restart_node(0, []) + + self.log.info('Test within 24 hours an addr relay destination is rotated at most once') + self.mocktime = int(time.time()) + self.msg = self.setup_addr_msg(1) + self.addr_receivers = [] + peer = self.nodes[0].add_p2p_connection(P2PInterface()) + receiver_peer = self.nodes[0].add_p2p_connection(AddrReceiver()) + addr_receivers = [self.nodes[0].add_p2p_connection(AddrReceiver()) for _ in range(20)] + nodes_received_addr = self.get_nodes_that_received_addr(peer, receiver_peer, addr_receivers, 0, TWO_HOURS) # 10 intervals of 2 hours + # Per RelayAddress, we would announce these addrs to 2 destinations per day. + # Since it's at most one rotation, at most 4 nodes can receive ADDR. + assert_greater_than_or_equal(ADDR_DESTINATIONS_THRESHOLD, len(nodes_received_addr)) + self.nodes[0].disconnect_p2ps() + + def destination_rotates_more_than_once_over_several_days_test(self): + self.restart_node(0, []) + + self.log.info('Test after several days an addr relay destination is rotated more than once') + self.msg = self.setup_addr_msg(1) + peer = self.nodes[0].add_p2p_connection(P2PInterface()) + receiver_peer = self.nodes[0].add_p2p_connection(AddrReceiver()) + addr_receivers = [self.nodes[0].add_p2p_connection(AddrReceiver()) for _ in range(20)] + # 10 intervals of 1 day (+ 1 hour, which should be enough to cover 30-min Poisson in most cases) + nodes_received_addr = self.get_nodes_that_received_addr(peer, receiver_peer, addr_receivers, ONE_DAY, ONE_HOUR) + # Now that there should have been more than one rotation, more than + # ADDR_DESTINATIONS_THRESHOLD nodes should have received ADDR. + assert_greater_than(len(nodes_received_addr), ADDR_DESTINATIONS_THRESHOLD) + self.nodes[0].disconnect_p2ps() + if __name__ == '__main__': AddrTest().main() diff --git a/test/functional/p2p_block_sync.py b/test/functional/p2p_block_sync.py new file mode 100755 index 0000000000..d821edc1b1 --- /dev/null +++ b/test/functional/p2p_block_sync.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test block download + +Ensure that even in IBD, we'll eventually sync chain from inbound peers +(whether we have only inbound peers or both inbound and outbound peers). +""" + +from test_framework.test_framework import BitcoinTestFramework + +class BlockSyncTest(BitcoinTestFramework): + + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 3 + + def setup_network(self): + self.setup_nodes() + # Construct a network: + # node0 -> node1 -> node2 + # So node1 has both an inbound and outbound peer. + # In our test, we will mine a block on node0, and ensure that it makes + # to to both node1 and node2. + self.connect_nodes(0, 1) + self.connect_nodes(1, 2) + + def run_test(self): + self.log.info("Setup network: node0->node1->node2") + self.log.info("Mining one block on node0 and verify all nodes sync") + self.generate(self.nodes[0], 1) + self.log.info("Success!") + + +if __name__ == '__main__': + BlockSyncTest().main() diff --git a/test/functional/p2p_blockfilters.py b/test/functional/p2p_blockfilters.py index e45cef65bd..ef12b5f6b7 100755 --- a/test/functional/p2p_blockfilters.py +++ b/test/functional/p2p_blockfilters.py @@ -250,6 +250,17 @@ class CompactFiltersTest(BitcoinTestFramework): msg = "Error: Cannot set -peerblockfilters without -blockfilterindex." self.nodes[0].assert_start_raises_init_error(expected_msg=msg) + self.log.info("Test unknown value to -blockfilterindex raises an error") + self.nodes[0].extra_args = ["-blockfilterindex=abc"] + msg = "Error: Unknown -blockfilterindex value abc." + self.nodes[0].assert_start_raises_init_error(expected_msg=msg) + + self.log.info("Test -blockfilterindex with -reindex-chainstate raises an error") + self.nodes[0].assert_start_raises_init_error( + expected_msg='Error: -reindex-chainstate option is not compatible with -blockfilterindex. ' + 'Please temporarily disable blockfilterindex while using -reindex-chainstate, or replace -reindex-chainstate with -reindex to fully rebuild all indexes.', + extra_args=['-blockfilterindex', '-reindex-chainstate'], + ) def compute_last_header(prev_header, hashes): """Compute the last filter header from a starting header and a sequence of filter hashes.""" diff --git a/test/functional/p2p_compactblocks.py b/test/functional/p2p_compactblocks.py index 364e806e18..b9ac3c32c5 100755 --- a/test/functional/p2p_compactblocks.py +++ b/test/functional/p2p_compactblocks.py @@ -2,11 +2,7 @@ # Copyright (c) 2016-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test compact blocks (BIP 152). - -Version 1 compact blocks are pre-segwit (txids) -Version 2 compact blocks are post-segwit (wtxids) -""" +"""Test compact blocks (BIP 152).""" import random from test_framework.blocktools import ( @@ -31,7 +27,6 @@ from test_framework.messages import ( MSG_BLOCK, MSG_CMPCT_BLOCK, MSG_WITNESS_FLAG, - NODE_NETWORK, P2PHeaderAndShortIDs, PrefilledTransaction, calculate_shortid, @@ -70,7 +65,7 @@ from test_framework.wallet import MiniWallet # TestP2PConn: A peer we use to send messages to bitcoind, and store responses. class TestP2PConn(P2PInterface): - def __init__(self, cmpct_version): + def __init__(self): super().__init__() self.last_sendcmpct = [] self.block_announced = False @@ -78,7 +73,6 @@ class TestP2PConn(P2PInterface): # This is for synchronizing the p2p message traffic, # so we can eg wait until a particular block is announced. self.announced_blockhashes = set() - self.cmpct_version = cmpct_version def on_sendcmpct(self, message): self.last_sendcmpct.append(message) @@ -152,10 +146,8 @@ class CompactBlocksTest(BitcoinTestFramework): ]] self.utxos = [] - def build_block_on_tip(self, node, segwit=False): + def build_block_on_tip(self, node): block = create_block(tmpl=node.getblocktemplate(NORMAL_GBT_REQUEST_PARAMS)) - if segwit: - add_witness_commitment(block) block.solve() return block @@ -185,15 +177,13 @@ class CompactBlocksTest(BitcoinTestFramework): # Test "sendcmpct" (between peers preferring the same version): # - No compact block announcements unless sendcmpct is sent. - # - If sendcmpct is sent with version > preferred_version, the message is ignored. + # - If sendcmpct is sent with version = 1, the message is ignored. + # - If sendcmpct is sent with version > 2, the message is ignored. # - If sendcmpct is sent with boolean 0, then block announcements are not # made with compact blocks. # - If sendcmpct is then sent with boolean 1, then new block announcements # are made with compact blocks. - # If old_node is passed in, request compact blocks with version=preferred-1 - # and verify that it receives block announcements via compact block. - def test_sendcmpct(self, test_node, old_node=None): - preferred_version = test_node.cmpct_version + def test_sendcmpct(self, test_node): node = self.nodes[0] # Make sure we get a SENDCMPCT message from our peer @@ -201,10 +191,8 @@ class CompactBlocksTest(BitcoinTestFramework): return (len(test_node.last_sendcmpct) > 0) test_node.wait_until(received_sendcmpct, timeout=30) with p2p_lock: - # Check that the first version received is the preferred one - assert_equal(test_node.last_sendcmpct[0].version, preferred_version) - # And that we receive versions down to 1. - assert_equal(test_node.last_sendcmpct[-1].version, 1) + # Check that version 2 is received. + assert_equal(test_node.last_sendcmpct[0].version, 2) test_node.last_sendcmpct = [] tip = int(node.getbestblockhash(), 16) @@ -232,22 +220,29 @@ class CompactBlocksTest(BitcoinTestFramework): # Before each test, sync the headers chain. test_node.request_headers_and_sync(locator=[tip]) + # Now try a SENDCMPCT message with too-low version + test_node.send_and_ping(msg_sendcmpct(announce=True, version=1)) + check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message) + + # Headers sync before next test. + test_node.request_headers_and_sync(locator=[tip]) + # Now try a SENDCMPCT message with too-high version - test_node.send_and_ping(msg_sendcmpct(announce=True, version=preferred_version+1)) + test_node.send_and_ping(msg_sendcmpct(announce=True, version=3)) check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message) # Headers sync before next test. test_node.request_headers_and_sync(locator=[tip]) # Now try a SENDCMPCT message with valid version, but announce=False - test_node.send_and_ping(msg_sendcmpct(announce=False, version=preferred_version)) + test_node.send_and_ping(msg_sendcmpct(announce=False, version=2)) check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message) # Headers sync before next test. test_node.request_headers_and_sync(locator=[tip]) # Finally, try a SENDCMPCT message with announce=True - test_node.send_and_ping(msg_sendcmpct(announce=True, version=preferred_version)) + test_node.send_and_ping(msg_sendcmpct(announce=True, version=2)) check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" in p.last_message) # Try one more time (no headers sync should be needed!) @@ -257,22 +252,14 @@ class CompactBlocksTest(BitcoinTestFramework): test_node.send_and_ping(msg_sendheaders()) check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" in p.last_message) - # Try one more time, after sending a version-1, announce=false message. - test_node.send_and_ping(msg_sendcmpct(announce=False, version=preferred_version-1)) + # Try one more time, after sending a version=1, announce=false message. + test_node.send_and_ping(msg_sendcmpct(announce=False, version=1)) check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" in p.last_message) # Now turn off announcements - test_node.send_and_ping(msg_sendcmpct(announce=False, version=preferred_version)) + test_node.send_and_ping(msg_sendcmpct(announce=False, version=2)) check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message and "headers" in p.last_message) - if old_node is not None: - # Verify that a peer using an older protocol version can receive - # announcements from this node. - old_node.send_and_ping(msg_sendcmpct(announce=True, version=preferred_version-1)) - # Header sync - old_node.request_headers_and_sync(locator=[tip]) - check_announcement_of_new_block(node, old_node, lambda p: "cmpctblock" in p.last_message) - # This test actually causes bitcoind to (reasonably!) disconnect us, so do this last. def test_invalid_cmpctblock_message(self): self.generate(self.nodes[0], COINBASE_MATURITY + 1) @@ -289,8 +276,7 @@ class CompactBlocksTest(BitcoinTestFramework): # Compare the generated shortids to what we expect based on BIP 152, given # bitcoind's choice of nonce. - def test_compactblock_construction(self, test_node, use_witness_address=True): - version = test_node.cmpct_version + def test_compactblock_construction(self, test_node): node = self.nodes[0] # Generate a bunch of transactions. self.generate(node, COINBASE_MATURITY + 1) @@ -303,8 +289,7 @@ class CompactBlocksTest(BitcoinTestFramework): if not tx.wit.is_null(): segwit_tx_generated = True - if use_witness_address: - assert segwit_tx_generated # check that our test is not broken + assert segwit_tx_generated # check that our test is not broken # Wait until we've seen the block announcement for the resulting tip tip = int(node.getbestblockhash(), 16) @@ -331,7 +316,7 @@ class CompactBlocksTest(BitcoinTestFramework): with p2p_lock: # Convert the on-the-wire representation to absolute indexes header_and_shortids = HeaderAndShortIDs(test_node.last_message["cmpctblock"].header_and_shortids) - self.check_compactblock_construction_from_block(version, header_and_shortids, block_hash, block) + self.check_compactblock_construction_from_block(header_and_shortids, block_hash, block) # Now fetch the compact block using a normal non-announce getdata test_node.clear_block_announcement() @@ -345,9 +330,9 @@ class CompactBlocksTest(BitcoinTestFramework): with p2p_lock: # Convert the on-the-wire representation to absolute indexes header_and_shortids = HeaderAndShortIDs(test_node.last_message["cmpctblock"].header_and_shortids) - self.check_compactblock_construction_from_block(version, header_and_shortids, block_hash, block) + self.check_compactblock_construction_from_block(header_and_shortids, block_hash, block) - def check_compactblock_construction_from_block(self, version, header_and_shortids, block_hash, block): + def check_compactblock_construction_from_block(self, header_and_shortids, block_hash, block): # Check that we got the right block! header_and_shortids.header.calc_sha256() assert_equal(header_and_shortids.header.sha256, block_hash) @@ -364,11 +349,7 @@ class CompactBlocksTest(BitcoinTestFramework): # And this checks the witness wtxid = entry.tx.calc_sha256(True) - if version == 2: - assert_equal(wtxid, block.vtx[entry.index].calc_sha256(True)) - else: - # Shouldn't have received a witness - assert entry.tx.wit.is_null() + assert_equal(wtxid, block.vtx[entry.index].calc_sha256(True)) # Check that the cmpctblock message announced all the transactions. assert_equal(len(header_and_shortids.prefilled_txn) + len(header_and_shortids.shortids), len(block.vtx)) @@ -384,9 +365,7 @@ class CompactBlocksTest(BitcoinTestFramework): # Already checked prefilled transactions above header_and_shortids.prefilled_txn.pop(0) else: - tx_hash = block.vtx[index].sha256 - if version == 2: - tx_hash = block.vtx[index].calc_sha256(True) + tx_hash = block.vtx[index].calc_sha256(True) shortid = calculate_shortid(k0, k1, tx_hash) assert_equal(shortid, header_and_shortids.shortids[0]) header_and_shortids.shortids.pop(0) @@ -395,16 +374,12 @@ class CompactBlocksTest(BitcoinTestFramework): # Test that bitcoind requests compact blocks when we announce new blocks # via header or inv, and that responding to getblocktxn causes the block # to be successfully reconstructed. - # Post-segwit: upgraded nodes would only make this request of cb-version-2, - # NODE_WITNESS peers. Unupgraded nodes would still make this request of - # any cb-version-1-supporting peer. - def test_compactblock_requests(self, test_node, segwit=True): - version = test_node.cmpct_version + def test_compactblock_requests(self, test_node): node = self.nodes[0] # Try announcing a block with an inv or header, expect a compactblock # request for announce in ["inv", "header"]: - block = self.build_block_on_tip(node, segwit=segwit) + block = self.build_block_on_tip(node) if announce == "inv": test_node.send_message(msg_inv([CInv(MSG_BLOCK, block.sha256)])) @@ -420,9 +395,7 @@ class CompactBlocksTest(BitcoinTestFramework): comp_block.header = CBlockHeader(block) comp_block.nonce = 0 [k0, k1] = comp_block.get_siphash_keys() - coinbase_hash = block.vtx[0].sha256 - if version == 2: - coinbase_hash = block.vtx[0].calc_sha256(True) + coinbase_hash = block.vtx[0].calc_sha256(True) comp_block.shortids = [calculate_shortid(k0, k1, coinbase_hash)] test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) assert_equal(int(node.getbestblockhash(), 16), block.hashPrevBlock) @@ -433,10 +406,7 @@ class CompactBlocksTest(BitcoinTestFramework): assert_equal(absolute_indexes, [0]) # should be a coinbase request # Send the coinbase, and verify that the tip advances. - if version == 2: - msg = msg_blocktxn() - else: - msg = msg_no_witness_blocktxn() + msg = msg_blocktxn() msg.block_transactions.blockhash = block.sha256 msg.block_transactions.transactions = [block.vtx[0]] test_node.send_and_ping(msg) @@ -462,9 +432,7 @@ class CompactBlocksTest(BitcoinTestFramework): # node needs, and that responding to them causes the block to be # reconstructed. def test_getblocktxn_requests(self, test_node): - version = test_node.cmpct_version node = self.nodes[0] - with_witness = (version == 2) def test_getblocktxn_response(compact_block, peer, expected_result): msg = msg_cmpctblock(compact_block.to_p2p()) @@ -485,13 +453,12 @@ class CompactBlocksTest(BitcoinTestFramework): block = self.build_block_with_transactions(node, utxo, 5) self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue]) comp_block = HeaderAndShortIDs() - comp_block.initialize_from_block(block, use_witness=with_witness) + comp_block.initialize_from_block(block, use_witness=True) test_getblocktxn_response(comp_block, test_node, [1, 2, 3, 4, 5]) msg_bt = msg_no_witness_blocktxn() - if with_witness: - msg_bt = msg_blocktxn() # serialize with witnesses + msg_bt = msg_blocktxn() # serialize with witnesses msg_bt.block_transactions = BlockTransactions(block.sha256, block.vtx[1:]) test_tip_after_message(node, test_node, msg_bt, block.sha256) @@ -500,7 +467,7 @@ class CompactBlocksTest(BitcoinTestFramework): self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue]) # Now try interspersing the prefilled transactions - comp_block.initialize_from_block(block, prefill_list=[0, 1, 5], use_witness=with_witness) + comp_block.initialize_from_block(block, prefill_list=[0, 1, 5], use_witness=True) test_getblocktxn_response(comp_block, test_node, [2, 3, 4]) msg_bt.block_transactions = BlockTransactions(block.sha256, block.vtx[2:5]) test_tip_after_message(node, test_node, msg_bt, block.sha256) @@ -514,7 +481,7 @@ class CompactBlocksTest(BitcoinTestFramework): # Prefill 4 out of the 6 transactions, and verify that only the one # that was not in the mempool is requested. - comp_block.initialize_from_block(block, prefill_list=[0, 2, 3, 4], use_witness=with_witness) + comp_block.initialize_from_block(block, prefill_list=[0, 2, 3, 4], use_witness=True) test_getblocktxn_response(comp_block, test_node, [5]) msg_bt.block_transactions = BlockTransactions(block.sha256, [block.vtx[5]]) @@ -538,7 +505,7 @@ class CompactBlocksTest(BitcoinTestFramework): test_node.last_message.pop("getblocktxn", None) # Send compact block - comp_block.initialize_from_block(block, prefill_list=[0], use_witness=with_witness) + comp_block.initialize_from_block(block, prefill_list=[0], use_witness=True) test_tip_after_message(node, test_node, msg_cmpctblock(comp_block.to_p2p()), block.sha256) with p2p_lock: # Shouldn't have gotten a request for any transaction @@ -547,7 +514,6 @@ class CompactBlocksTest(BitcoinTestFramework): # Incorrectly responding to a getblocktxn shouldn't cause the block to be # permanently failed. def test_incorrect_blocktxn_response(self, test_node): - version = test_node.cmpct_version node = self.nodes[0] utxo = self.utxos.pop(0) @@ -564,7 +530,7 @@ class CompactBlocksTest(BitcoinTestFramework): # Send compact block comp_block = HeaderAndShortIDs() - comp_block.initialize_from_block(block, prefill_list=[0], use_witness=(version == 2)) + comp_block.initialize_from_block(block, prefill_list=[0], use_witness=True) test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) absolute_indexes = [] with p2p_lock: @@ -580,9 +546,7 @@ class CompactBlocksTest(BitcoinTestFramework): # different peer provide the block further down, so that we're still # verifying that the block isn't marked bad permanently. This is good # enough for now. - msg = msg_no_witness_blocktxn() - if version == 2: - msg = msg_blocktxn() + msg = msg_blocktxn() msg.block_transactions = BlockTransactions(block.sha256, [block.vtx[5]] + block.vtx[7:]) test_node.send_and_ping(msg) @@ -595,14 +559,10 @@ class CompactBlocksTest(BitcoinTestFramework): test_node.last_message["getdata"].inv[0].type == MSG_BLOCK | MSG_WITNESS_FLAG # Deliver the block - if version == 2: - test_node.send_and_ping(msg_block(block)) - else: - test_node.send_and_ping(msg_no_witness_block(block)) + test_node.send_and_ping(msg_block(block)) assert_equal(int(node.getbestblockhash(), 16), block.sha256) def test_getblocktxn_handler(self, test_node): - version = test_node.cmpct_version node = self.nodes[0] # bitcoind will not send blocktxn responses for blocks whose height is # more than 10 blocks deep. @@ -628,12 +588,8 @@ class CompactBlocksTest(BitcoinTestFramework): tx = test_node.last_message["blocktxn"].block_transactions.transactions.pop(0) tx.calc_sha256() assert_equal(tx.sha256, block.vtx[index].sha256) - if version == 1: - # Witnesses should have been stripped - assert tx.wit.is_null() - else: - # Check that the witness matches - assert_equal(tx.calc_sha256(True), block.vtx[index].calc_sha256(True)) + # Check that the witness matches + assert_equal(tx.calc_sha256(True), block.vtx[index].calc_sha256(True)) test_node.last_message.pop("blocktxn", None) current_height -= 1 @@ -727,7 +683,7 @@ class CompactBlocksTest(BitcoinTestFramework): # Test that we don't get disconnected if we relay a compact block with valid header, # but invalid transactions. - def test_invalid_tx_in_compactblock(self, test_node, use_segwit=True): + def test_invalid_tx_in_compactblock(self, test_node): node = self.nodes[0] assert len(self.utxos) utxo = self.utxos[0] @@ -735,17 +691,15 @@ class CompactBlocksTest(BitcoinTestFramework): block = self.build_block_with_transactions(node, utxo, 5) del block.vtx[3] block.hashMerkleRoot = block.calc_merkle_root() - if use_segwit: - # If we're testing with segwit, also drop the coinbase witness, - # but include the witness commitment. - add_witness_commitment(block) - block.vtx[0].wit.vtxinwit = [] + # Drop the coinbase witness but include the witness commitment. + add_witness_commitment(block) + block.vtx[0].wit.vtxinwit = [] block.solve() # Now send the compact block with all transactions prefilled, and # verify that we don't get disconnected. comp_block = HeaderAndShortIDs() - comp_block.initialize_from_block(block, prefill_list=[0, 1, 2, 3, 4], use_witness=use_segwit) + comp_block.initialize_from_block(block, prefill_list=[0, 1, 2, 3, 4], use_witness=True) msg = msg_cmpctblock(comp_block.to_p2p()) test_node.send_and_ping(msg) @@ -759,7 +713,7 @@ class CompactBlocksTest(BitcoinTestFramework): node = self.nodes[0] tip = node.getbestblockhash() peer.get_headers(locator=[int(tip, 16)], hashstop=0) - peer.send_and_ping(msg_sendcmpct(announce=True, version=peer.cmpct_version)) + peer.send_and_ping(msg_sendcmpct(announce=True, version=2)) def test_compactblock_reconstruction_multiple_peers(self, stalling_peer, delivery_peer): node = self.nodes[0] @@ -813,7 +767,7 @@ class CompactBlocksTest(BitcoinTestFramework): def test_highbandwidth_mode_states_via_getpeerinfo(self): # create new p2p connection for a fresh state w/o any prior sendcmpct messages sent - hb_test_node = self.nodes[0].add_p2p_connection(TestP2PConn(cmpct_version=2)) + hb_test_node = self.nodes[0].add_p2p_connection(TestP2PConn()) # assert the RPC getpeerinfo boolean fields `bip152_hb_{to, from}` # match the given parameters for the last peer of a given node @@ -843,9 +797,8 @@ class CompactBlocksTest(BitcoinTestFramework): self.wallet = MiniWallet(self.nodes[0]) # Setup the p2p connections - self.segwit_node = self.nodes[0].add_p2p_connection(TestP2PConn(cmpct_version=2)) - self.old_node = self.nodes[0].add_p2p_connection(TestP2PConn(cmpct_version=1), services=NODE_NETWORK) - self.additional_segwit_node = self.nodes[0].add_p2p_connection(TestP2PConn(cmpct_version=2)) + self.segwit_node = self.nodes[0].add_p2p_connection(TestP2PConn()) + self.additional_segwit_node = self.nodes[0].add_p2p_connection(TestP2PConn()) # We will need UTXOs to construct transactions in later tests. self.make_utxos() @@ -853,11 +806,10 @@ class CompactBlocksTest(BitcoinTestFramework): assert softfork_active(self.nodes[0], "segwit") self.log.info("Testing SENDCMPCT p2p message... ") - self.test_sendcmpct(self.segwit_node, old_node=self.old_node) + self.test_sendcmpct(self.segwit_node) self.test_sendcmpct(self.additional_segwit_node) self.log.info("Testing compactblock construction...") - self.test_compactblock_construction(self.old_node) self.test_compactblock_construction(self.segwit_node) self.log.info("Testing compactblock requests (segwit node)... ") @@ -868,11 +820,9 @@ class CompactBlocksTest(BitcoinTestFramework): self.log.info("Testing getblocktxn handler (segwit node should return witnesses)...") self.test_getblocktxn_handler(self.segwit_node) - self.test_getblocktxn_handler(self.old_node) self.log.info("Testing compactblock requests/announcements not at chain tip...") self.test_compactblocks_not_at_tip(self.segwit_node) - self.test_compactblocks_not_at_tip(self.old_node) self.log.info("Testing handling of incorrect blocktxn responses...") self.test_incorrect_blocktxn_response(self.segwit_node) @@ -885,13 +835,12 @@ class CompactBlocksTest(BitcoinTestFramework): # (Post-segwit activation, blocks won't propagate from node0 to node1 # automatically, so don't bother testing a block announced to node0.) self.log.info("Testing end-to-end block relay...") - self.request_cb_announcements(self.old_node) self.request_cb_announcements(self.segwit_node) - self.test_end_to_end_block_relay([self.segwit_node, self.old_node]) + self.request_cb_announcements(self.additional_segwit_node) + self.test_end_to_end_block_relay([self.segwit_node, self.additional_segwit_node]) self.log.info("Testing handling of invalid compact blocks...") self.test_invalid_tx_in_compactblock(self.segwit_node) - self.test_invalid_tx_in_compactblock(self.old_node) self.log.info("Testing invalid index in cmpctblock message...") self.test_invalid_cmpctblock_message() diff --git a/test/functional/p2p_compactblocks_blocksonly.py b/test/functional/p2p_compactblocks_blocksonly.py index 6367eb26a3..3d0c421a93 100755 --- a/test/functional/p2p_compactblocks_blocksonly.py +++ b/test/functional/p2p_compactblocks_blocksonly.py @@ -48,7 +48,7 @@ class P2PCompactBlocksBlocksOnly(BitcoinTestFramework): p2p_conn_high_bw = self.nodes[1].add_p2p_connection(P2PInterface()) p2p_conn_low_bw = self.nodes[3].add_p2p_connection(P2PInterface()) for conn in [p2p_conn_blocksonly, p2p_conn_high_bw, p2p_conn_low_bw]: - assert_equal(conn.message_count['sendcmpct'], 2) + assert_equal(conn.message_count['sendcmpct'], 1) conn.send_and_ping(msg_sendcmpct(announce=False, version=2)) # Nodes: @@ -74,14 +74,14 @@ class P2PCompactBlocksBlocksOnly(BitcoinTestFramework): # receiving a new valid block at the tip. p2p_conn_blocksonly.send_and_ping(msg_block(block0)) assert_equal(int(self.nodes[0].getbestblockhash(), 16), block0.sha256) - assert_equal(p2p_conn_blocksonly.message_count['sendcmpct'], 2) + assert_equal(p2p_conn_blocksonly.message_count['sendcmpct'], 1) assert_equal(p2p_conn_blocksonly.last_message['sendcmpct'].announce, False) # A normal node participating in transaction relay should request BIP152 # high bandwidth mode upon receiving a new valid block at the tip. p2p_conn_high_bw.send_and_ping(msg_block(block0)) assert_equal(int(self.nodes[1].getbestblockhash(), 16), block0.sha256) - p2p_conn_high_bw.wait_until(lambda: p2p_conn_high_bw.message_count['sendcmpct'] == 3) + p2p_conn_high_bw.wait_until(lambda: p2p_conn_high_bw.message_count['sendcmpct'] == 2) assert_equal(p2p_conn_high_bw.last_message['sendcmpct'].announce, True) # Don't send a block from the p2p_conn_low_bw so the low bandwidth node diff --git a/test/functional/p2p_getaddr_caching.py b/test/functional/p2p_getaddr_caching.py index d375af6fe1..8907c34a89 100755 --- a/test/functional/p2p_getaddr_caching.py +++ b/test/functional/p2p_getaddr_caching.py @@ -14,6 +14,7 @@ from test_framework.p2p import ( from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, + p2p_port, ) # As defined in net_processing. @@ -42,6 +43,12 @@ class AddrReceiver(P2PInterface): class AddrTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 + # Use some of the remaining p2p ports for the onion binds. + self.onion_port1 = p2p_port(self.num_nodes) + self.onion_port2 = p2p_port(self.num_nodes + 1) + self.extra_args = [ + [f"-bind=127.0.0.1:{self.onion_port1}=onion", f"-bind=127.0.0.1:{self.onion_port2}=onion"], + ] def run_test(self): self.log.info('Fill peer AddrMan with a lot of records') @@ -55,35 +62,66 @@ class AddrTest(BitcoinTestFramework): # only a fraction of all known addresses can be cached and returned. assert(len(self.nodes[0].getnodeaddresses(0)) > int(MAX_ADDR_TO_SEND / (MAX_PCT_ADDR_TO_SEND / 100))) - responses = [] + last_response_on_local_bind = None + last_response_on_onion_bind1 = None + last_response_on_onion_bind2 = None self.log.info('Send many addr requests within short time to receive same response') N = 5 cur_mock_time = int(time.time()) for i in range(N): - addr_receiver = self.nodes[0].add_p2p_connection(AddrReceiver()) - addr_receiver.send_and_ping(msg_getaddr()) + addr_receiver_local = self.nodes[0].add_p2p_connection(AddrReceiver()) + addr_receiver_local.send_and_ping(msg_getaddr()) + addr_receiver_onion1 = self.nodes[0].add_p2p_connection(AddrReceiver(), dstport=self.onion_port1) + addr_receiver_onion1.send_and_ping(msg_getaddr()) + addr_receiver_onion2 = self.nodes[0].add_p2p_connection(AddrReceiver(), dstport=self.onion_port2) + addr_receiver_onion2.send_and_ping(msg_getaddr()) + # Trigger response cur_mock_time += 5 * 60 self.nodes[0].setmocktime(cur_mock_time) - addr_receiver.wait_until(addr_receiver.addr_received) - responses.append(addr_receiver.get_received_addrs()) - for response in responses[1:]: - assert_equal(response, responses[0]) - assert(len(response) == MAX_ADDR_TO_SEND) + addr_receiver_local.wait_until(addr_receiver_local.addr_received) + addr_receiver_onion1.wait_until(addr_receiver_onion1.addr_received) + addr_receiver_onion2.wait_until(addr_receiver_onion2.addr_received) + + if i > 0: + # Responses from different binds should be unique + assert(last_response_on_local_bind != addr_receiver_onion1.get_received_addrs()) + assert(last_response_on_local_bind != addr_receiver_onion2.get_received_addrs()) + assert(last_response_on_onion_bind1 != addr_receiver_onion2.get_received_addrs()) + # Responses on from the same bind should be the same + assert_equal(last_response_on_local_bind, addr_receiver_local.get_received_addrs()) + assert_equal(last_response_on_onion_bind1, addr_receiver_onion1.get_received_addrs()) + assert_equal(last_response_on_onion_bind2, addr_receiver_onion2.get_received_addrs()) + + last_response_on_local_bind = addr_receiver_local.get_received_addrs() + last_response_on_onion_bind1 = addr_receiver_onion1.get_received_addrs() + last_response_on_onion_bind2 = addr_receiver_onion2.get_received_addrs() + + for response in [last_response_on_local_bind, last_response_on_onion_bind1, last_response_on_onion_bind2]: + assert_equal(len(response), MAX_ADDR_TO_SEND) cur_mock_time += 3 * 24 * 60 * 60 self.nodes[0].setmocktime(cur_mock_time) self.log.info('After time passed, see a new response to addr request') - last_addr_receiver = self.nodes[0].add_p2p_connection(AddrReceiver()) - last_addr_receiver.send_and_ping(msg_getaddr()) + addr_receiver_local = self.nodes[0].add_p2p_connection(AddrReceiver()) + addr_receiver_local.send_and_ping(msg_getaddr()) + addr_receiver_onion1 = self.nodes[0].add_p2p_connection(AddrReceiver(), dstport=self.onion_port1) + addr_receiver_onion1.send_and_ping(msg_getaddr()) + addr_receiver_onion2 = self.nodes[0].add_p2p_connection(AddrReceiver(), dstport=self.onion_port2) + addr_receiver_onion2.send_and_ping(msg_getaddr()) + # Trigger response cur_mock_time += 5 * 60 self.nodes[0].setmocktime(cur_mock_time) - last_addr_receiver.wait_until(last_addr_receiver.addr_received) - # new response is different - assert(set(responses[0]) != set(last_addr_receiver.get_received_addrs())) + addr_receiver_local.wait_until(addr_receiver_local.addr_received) + addr_receiver_onion1.wait_until(addr_receiver_onion1.addr_received) + addr_receiver_onion2.wait_until(addr_receiver_onion2.addr_received) + # new response is different + assert(set(last_response_on_local_bind) != set(addr_receiver_local.get_received_addrs())) + assert(set(last_response_on_onion_bind1) != set(addr_receiver_onion1.get_received_addrs())) + assert(set(last_response_on_onion_bind2) != set(addr_receiver_onion2.get_received_addrs())) if __name__ == '__main__': AddrTest().main() diff --git a/test/functional/p2p_message_capture.py b/test/functional/p2p_message_capture.py index 0a7ae44de4..87c77f4540 100755 --- a/test/functional/p2p_message_capture.py +++ b/test/functional/p2p_message_capture.py @@ -43,12 +43,8 @@ def mini_parser(dat_file): break tmp_header = BytesIO(tmp_header_raw) tmp_header.read(TIME_SIZE) # skip the timestamp field - raw_msgtype = tmp_header.read(MSGTYPE_SIZE) - msgtype: bytes = raw_msgtype.split(b'\x00', 1)[0] - remainder = raw_msgtype.split(b'\x00', 1)[1] - assert(len(msgtype) > 0) + msgtype = tmp_header.read(MSGTYPE_SIZE).rstrip(b'\x00') assert(msgtype in MESSAGEMAP) - assert(len(remainder) == 0 or not remainder.decode().isprintable()) length: int = int.from_bytes(tmp_header.read(LENGTH_SIZE), "little") data = f_in.read(length) assert_equal(len(data), length) diff --git a/test/functional/p2p_segwit.py b/test/functional/p2p_segwit.py index f377fbaaa6..89ddfd3bcf 100755 --- a/test/functional/p2p_segwit.py +++ b/test/functional/p2p_segwit.py @@ -43,7 +43,6 @@ from test_framework.messages import ( ser_uint256, ser_vector, sha256, - tx_from_hex, ) from test_framework.p2p import ( P2PInterface, @@ -89,6 +88,8 @@ from test_framework.util import ( softfork_active, assert_raises_rpc_error, ) +from test_framework.wallet import MiniWallet + MAX_SIGOP_COST = 80000 @@ -221,9 +222,6 @@ class SegWitTest(BitcoinTestFramework): ] self.supports_cli = False - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() - # Helper functions def build_next_block(self): @@ -259,6 +257,7 @@ class SegWitTest(BitcoinTestFramework): self.log.info("Starting tests before segwit activation") self.segwit_active = False + self.wallet = MiniWallet(self.nodes[0]) self.test_non_witness_transaction() self.test_v0_outputs_arent_spendable() @@ -307,7 +306,7 @@ class SegWitTest(BitcoinTestFramework): self.test_node.send_and_ping(msg_no_witness_block(block)) # make sure the block was processed txid = block.vtx[0].sha256 - self.generate(self.nodes[0], 99) # let the block mature + self.generate(self.wallet, 99) # let the block mature # Create a transaction that spends the coinbase tx = CTransaction() @@ -1999,21 +1998,13 @@ class SegWitTest(BitcoinTestFramework): def serialize(self): return serialize_with_bogus_witness(self.tx) - self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(address_type='bech32'), 5) - self.generate(self.nodes[0], 1) - unspent = next(u for u in self.nodes[0].listunspent() if u['spendable'] and u['address'].startswith('bcrt')) - - raw = self.nodes[0].createrawtransaction([{"txid": unspent['txid'], "vout": unspent['vout']}], {self.nodes[0].getnewaddress(): 1}) - tx = tx_from_hex(raw) + tx = self.wallet.create_self_transfer(from_node=self.nodes[0])['tx'] assert_raises_rpc_error(-22, "TX decode failed", self.nodes[0].decoderawtransaction, hexstring=serialize_with_bogus_witness(tx).hex(), iswitness=True) - with self.nodes[0].assert_debug_log(['Superfluous witness record']): + with self.nodes[0].assert_debug_log(['Unknown transaction optional data']): self.test_node.send_and_ping(msg_bogus_tx(tx)) - raw = self.nodes[0].signrawtransactionwithwallet(raw) - assert raw['complete'] - raw = raw['hex'] - tx = tx_from_hex(raw) + tx.wit.vtxinwit = [] # drop witness assert_raises_rpc_error(-22, "TX decode failed", self.nodes[0].decoderawtransaction, hexstring=serialize_with_bogus_witness(tx).hex(), iswitness=True) - with self.nodes[0].assert_debug_log(['Unknown transaction optional data']): + with self.nodes[0].assert_debug_log(['Superfluous witness record']): self.test_node.send_and_ping(msg_bogus_tx(tx)) @subtest diff --git a/test/functional/p2p_unrequested_blocks.py b/test/functional/p2p_unrequested_blocks.py index 9c4e1dd1b1..76d9b045ce 100755 --- a/test/functional/p2p_unrequested_blocks.py +++ b/test/functional/p2p_unrequested_blocks.py @@ -257,16 +257,11 @@ class AcceptBlockTest(BitcoinTestFramework): test_node.send_message(msg_block(block_291)) # At this point we've sent an obviously-bogus block, wait for full processing - # without assuming whether we will be disconnected or not - try: - # Only wait a short while so the test doesn't take forever if we do get - # disconnected - test_node.sync_with_ping(timeout=1) - except AssertionError: - test_node.wait_for_disconnect() - - self.nodes[0].disconnect_p2ps() - test_node = self.nodes[0].add_p2p_connection(P2PInterface()) + # and assume disconnection + test_node.wait_for_disconnect() + + self.nodes[0].disconnect_p2ps() + test_node = self.nodes[0].add_p2p_connection(P2PInterface()) # We should have failed reorg and switched back to 290 (but have block 291) assert_equal(self.nodes[0].getblockcount(), 290) diff --git a/test/functional/rpc_createmultisig.py b/test/functional/rpc_createmultisig.py index 1695acaaa8..716ee8f7ef 100755 --- a/test/functional/rpc_createmultisig.py +++ b/test/functional/rpc_createmultisig.py @@ -91,15 +91,17 @@ class RpcCreateMultiSigTest(BitcoinTestFramework): assert 'warnings' not in result # Generate addresses with the segwit types. These should all make legacy addresses + err_msg = ["Unable to make chosen address type, please ensure no uncompressed public keys are present."] + for addr_type in ['bech32', 'p2sh-segwit']: - result = self.nodes[0].createmultisig(2, keys, addr_type) + result = self.nodes[0].createmultisig(nrequired=2, keys=keys, address_type=addr_type) assert_equal(legacy_addr, result['address']) - assert_equal(result['warnings'], ["Unable to make chosen address type, please ensure no uncompressed public keys are present."]) + assert_equal(result['warnings'], err_msg) if self.is_bdb_compiled(): - result = wmulti0.addmultisigaddress(2, keys, '', addr_type) + result = wmulti0.addmultisigaddress(nrequired=2, keys=keys, address_type=addr_type) assert_equal(legacy_addr, result['address']) - assert_equal(result['warnings'], ["Unable to make chosen address type, please ensure no uncompressed public keys are present."]) + assert_equal(result['warnings'], err_msg) self.log.info('Testing sortedmulti descriptors with BIP 67 test vectors') with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'data/rpc_bip67.json'), encoding='utf-8') as f: @@ -173,6 +175,7 @@ class RpcCreateMultiSigTest(BitcoinTestFramework): desc = descsum_create(desc) msig = node2.createmultisig(self.nsigs, self.pub, self.output_type) + assert 'warnings' not in msig madd = msig["address"] mredeem = msig["redeemScript"] assert_equal(desc, msig['descriptor']) diff --git a/test/functional/rpc_dumptxoutset.py b/test/functional/rpc_dumptxoutset.py index 4ca84748b2..672c9a53dc 100755 --- a/test/functional/rpc_dumptxoutset.py +++ b/test/functional/rpc_dumptxoutset.py @@ -49,9 +49,13 @@ class DumptxoutsetTest(BitcoinTestFramework): out['txoutset_hash'], '1f7e3befd45dc13ae198dfbb22869a9c5c4196f8e9ef9735831af1288033f890') assert_equal(out['nchaintx'], 101) - # Specifying a path to an existing file will fail. + # Specifying a path to an existing or invalid file will fail. assert_raises_rpc_error( -8, '{} already exists'.format(FILENAME), node.dumptxoutset, FILENAME) + invalid_path = str(Path(node.datadir) / "invalid" / "path") + assert_raises_rpc_error( + -8, "Couldn't open file {}.incomplete for writing".format(invalid_path), node.dumptxoutset, invalid_path) + if __name__ == '__main__': DumptxoutsetTest().main() diff --git a/test/functional/rpc_getblockfrompeer.py b/test/functional/rpc_getblockfrompeer.py index b65322d920..a7628b5591 100755 --- a/test/functional/rpc_getblockfrompeer.py +++ b/test/functional/rpc_getblockfrompeer.py @@ -5,6 +5,11 @@ """Test the getblockfrompeer RPC.""" from test_framework.authproxy import JSONRPCException +from test_framework.messages import NODE_WITNESS +from test_framework.p2p import ( + P2P_SERVICES, + P2PInterface, +) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, @@ -58,6 +63,13 @@ class GetBlockFromPeerTest(BitcoinTestFramework): self.log.info("Non-existent peer generates error") assert_raises_rpc_error(-1, "Peer does not exist", self.nodes[0].getblockfrompeer, short_tip, peer_0_peer_1_id + 1) + self.log.info("Fetching from pre-segwit peer generates error") + self.nodes[0].add_p2p_connection(P2PInterface(), services=P2P_SERVICES & ~NODE_WITNESS) + peers = self.nodes[0].getpeerinfo() + assert_equal(len(peers), 2) + presegwit_peer_id = peers[1]["id"] + assert_raises_rpc_error(-1, "Pre-SegWit peer", self.nodes[0].getblockfrompeer, short_tip, presegwit_peer_id) + self.log.info("Successful fetch") result = self.nodes[0].getblockfrompeer(short_tip, peer_0_peer_1_id) self.wait_until(lambda: self.check_for_block(short_tip), timeout=1) @@ -66,5 +78,6 @@ class GetBlockFromPeerTest(BitcoinTestFramework): self.log.info("Don't fetch blocks we already have") assert_raises_rpc_error(-1, "Block already downloaded", self.nodes[0].getblockfrompeer, short_tip, peer_0_peer_1_id) + if __name__ == '__main__': GetBlockFromPeerTest().main() diff --git a/test/functional/rpc_mempool_entry_fee_fields_deprecation.py b/test/functional/rpc_mempool_entry_fee_fields_deprecation.py deleted file mode 100755 index 82761ff7c8..0000000000 --- a/test/functional/rpc_mempool_entry_fee_fields_deprecation.py +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test deprecation of fee fields from top level mempool entry object""" - -from test_framework.blocktools import COIN -from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal -from test_framework.wallet import MiniWallet - - -def assertions_helper(new_object, deprecated_object, deprecated_fields): - for field in deprecated_fields: - assert field in deprecated_object - assert field not in new_object - - -class MempoolFeeFieldsDeprecationTest(BitcoinTestFramework): - def set_test_params(self): - self.num_nodes = 2 - self.extra_args = [[], ["-deprecatedrpc=fees"]] - - def run_test(self): - # we get spendable outputs from the premined chain starting - # at block 76. see BitcoinTestFramework._initialize_chain() for details - self.wallet = MiniWallet(self.nodes[0]) - self.wallet.rescan_utxos() - - # we create the tx on the first node and wait until it syncs to node_deprecated - # thus, any differences must be coming from getmempoolentry or getrawmempool - tx = self.wallet.send_self_transfer(from_node=self.nodes[0]) - self.nodes[1].sendrawtransaction(tx["hex"]) - - deprecated_fields = ["ancestorfees", "descendantfees", "modifiedfee", "fee"] - self.test_getmempoolentry(tx["txid"], deprecated_fields) - self.test_getrawmempool(tx["txid"], deprecated_fields) - self.test_deprecated_fields_match(tx["txid"]) - - def test_getmempoolentry(self, txid, deprecated_fields): - - self.log.info("Test getmempoolentry rpc") - entry = self.nodes[0].getmempoolentry(txid) - deprecated_entry = self.nodes[1].getmempoolentry(txid) - assertions_helper(entry, deprecated_entry, deprecated_fields) - - def test_getrawmempool(self, txid, deprecated_fields): - - self.log.info("Test getrawmempool rpc") - entry = self.nodes[0].getrawmempool(verbose=True)[txid] - deprecated_entry = self.nodes[1].getrawmempool(verbose=True)[txid] - assertions_helper(entry, deprecated_entry, deprecated_fields) - - def test_deprecated_fields_match(self, txid): - - self.log.info("Test deprecated fee fields match new fees object") - entry = self.nodes[0].getmempoolentry(txid) - deprecated_entry = self.nodes[1].getmempoolentry(txid) - - assert_equal(deprecated_entry["fee"], entry["fees"]["base"]) - assert_equal(deprecated_entry["modifiedfee"], entry["fees"]["modified"]) - assert_equal(deprecated_entry["descendantfees"], entry["fees"]["descendant"] * COIN) - assert_equal(deprecated_entry["ancestorfees"], entry["fees"]["ancestor"] * COIN) - - -if __name__ == "__main__": - MempoolFeeFieldsDeprecationTest().main() diff --git a/test/functional/rpc_mempool_info.py b/test/functional/rpc_mempool_info.py new file mode 100755 index 0000000000..cd7a48d387 --- /dev/null +++ b/test/functional/rpc_mempool_info.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +# Copyright (c) 2014-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test RPCs that retrieve information from the mempool.""" + +from test_framework.blocktools import COINBASE_MATURITY +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + assert_raises_rpc_error, +) +from test_framework.wallet import MiniWallet + + +class RPCMempoolInfoTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + + def run_test(self): + self.wallet = MiniWallet(self.nodes[0]) + self.generate(self.wallet, COINBASE_MATURITY + 1) + self.wallet.rescan_utxos() + confirmed_utxo = self.wallet.get_utxo() + + # Create a tree of unconfirmed transactions in the mempool: + # txA + # / \ + # / \ + # / \ + # / \ + # / \ + # txB txC + # / \ / \ + # / \ / \ + # txD txE txF txG + # \ / + # \ / + # txH + + def create_tx(**kwargs): + return self.wallet.send_self_transfer_multi( + from_node=self.nodes[0], + **kwargs, + ) + + txA = create_tx(utxos_to_spend=[confirmed_utxo], num_outputs=2) + txB = create_tx(utxos_to_spend=[txA["new_utxos"][0]], num_outputs=2) + txC = create_tx(utxos_to_spend=[txA["new_utxos"][1]], num_outputs=2) + txD = create_tx(utxos_to_spend=[txB["new_utxos"][0]], num_outputs=1) + txE = create_tx(utxos_to_spend=[txB["new_utxos"][1]], num_outputs=1) + txF = create_tx(utxos_to_spend=[txC["new_utxos"][0]], num_outputs=2) + txG = create_tx(utxos_to_spend=[txC["new_utxos"][1]], num_outputs=1) + txH = create_tx(utxos_to_spend=[txE["new_utxos"][0],txF["new_utxos"][0]], num_outputs=1) + txidA, txidB, txidC, txidD, txidE, txidF, txidG, txidH = [ + tx["txid"] for tx in [txA, txB, txC, txD, txE, txF, txG, txH] + ] + + mempool = self.nodes[0].getrawmempool() + assert_equal(len(mempool), 8) + for txid in [txidA, txidB, txidC, txidD, txidE, txidF, txidG, txidH]: + assert_equal(txid in mempool, True) + + self.log.info("Find transactions spending outputs") + result = self.nodes[0].gettxspendingprevout([ {'txid' : confirmed_utxo['txid'], 'vout' : 0}, {'txid' : txidA, 'vout' : 1} ]) + assert_equal(result, [ {'txid' : confirmed_utxo['txid'], 'vout' : 0, 'spendingtxid' : txidA}, {'txid' : txidA, 'vout' : 1, 'spendingtxid' : txidC} ]) + + self.log.info("Find transaction spending multiple outputs") + result = self.nodes[0].gettxspendingprevout([ {'txid' : txidE, 'vout' : 0}, {'txid' : txidF, 'vout' : 0} ]) + assert_equal(result, [ {'txid' : txidE, 'vout' : 0, 'spendingtxid' : txidH}, {'txid' : txidF, 'vout' : 0, 'spendingtxid' : txidH} ]) + + self.log.info("Find no transaction when output is unspent") + result = self.nodes[0].gettxspendingprevout([ {'txid' : txidH, 'vout' : 0} ]) + assert_equal(result, [ {'txid' : txidH, 'vout' : 0} ]) + result = self.nodes[0].gettxspendingprevout([ {'txid' : txidA, 'vout' : 5} ]) + assert_equal(result, [ {'txid' : txidA, 'vout' : 5} ]) + + self.log.info("Mixed spent and unspent outputs") + result = self.nodes[0].gettxspendingprevout([ {'txid' : txidB, 'vout' : 0}, {'txid' : txidG, 'vout' : 3} ]) + assert_equal(result, [ {'txid' : txidB, 'vout' : 0, 'spendingtxid' : txidD}, {'txid' : txidG, 'vout' : 3} ]) + + self.log.info("Unknown input fields") + assert_raises_rpc_error(-3, "Unexpected key unknown", self.nodes[0].gettxspendingprevout, [{'txid' : txidC, 'vout' : 1, 'unknown' : 42}]) + + self.log.info("Invalid vout provided") + assert_raises_rpc_error(-8, "Invalid parameter, vout cannot be negative", self.nodes[0].gettxspendingprevout, [{'txid' : txidA, 'vout' : -1}]) + + self.log.info("Invalid txid provided") + assert_raises_rpc_error(-3, "Expected type string for txid, got number", self.nodes[0].gettxspendingprevout, [{'txid' : 42, 'vout' : 0}]) + + self.log.info("Missing outputs") + assert_raises_rpc_error(-8, "Invalid parameter, outputs are missing", self.nodes[0].gettxspendingprevout, []) + + self.log.info("Missing vout") + assert_raises_rpc_error(-3, "Missing vout", self.nodes[0].gettxspendingprevout, [{'txid' : txidA}]) + + self.log.info("Missing txid") + assert_raises_rpc_error(-3, "Missing txid", self.nodes[0].gettxspendingprevout, [{'vout' : 3}]) + + +if __name__ == '__main__': + RPCMempoolInfoTest().main() diff --git a/test/functional/rpc_misc.py b/test/functional/rpc_misc.py index f64aae7223..f6ee6a5215 100755 --- a/test/functional/rpc_misc.py +++ b/test/functional/rpc_misc.py @@ -27,7 +27,7 @@ class RpcMiscTest(BitcoinTestFramework): self.log.info("test CHECK_NONFATAL") assert_raises_rpc_error( -1, - 'Internal bug detected: \'request.params[9].get_str() != "trigger_internal_bug"\'', + 'Internal bug detected: "request.params[9].get_str() != "trigger_internal_bug""', lambda: node.echo(arg9='trigger_internal_bug'), ) diff --git a/test/functional/rpc_net.py b/test/functional/rpc_net.py index 81a3cfee97..ad8ba06824 100755 --- a/test/functional/rpc_net.py +++ b/test/functional/rpc_net.py @@ -257,6 +257,10 @@ class NetTest(BitcoinTestFramework): assert_equal(node.addpeeraddress(address="", port=8333), {"success": False}) assert_equal(node.getnodeaddresses(count=0), []) + self.log.debug("Test that adding an address with invalid port fails") + assert_raises_rpc_error(-1, "JSON integer out of range", self.nodes[0].addpeeraddress, address="1.2.3.4", port=-1) + assert_raises_rpc_error(-1, "JSON integer out of range", self.nodes[0].addpeeraddress,address="1.2.3.4", port=65536) + self.log.debug("Test that adding a valid address to the tried table succeeds") assert_equal(node.addpeeraddress(address="1.2.3.4", tried=True, port=8333), {"success": True}) with node.assert_debug_log(expected_msgs=["CheckAddrman: new 0, tried 1, total 1 started"]): diff --git a/test/functional/rpc_psbt.py b/test/functional/rpc_psbt.py index b037807b53..444e56610e 100755 --- a/test/functional/rpc_psbt.py +++ b/test/functional/rpc_psbt.py @@ -10,6 +10,10 @@ from itertools import product from test_framework.descriptors import descsum_create from test_framework.key import ECKey +from test_framework.messages import ( + ser_compact_size, + WITNESS_SCALE_FACTOR, +) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_approx, @@ -615,8 +619,8 @@ class PSBTTest(BitcoinTestFramework): self.nodes[1].createwallet("extfund") wallet = self.nodes[1].get_wallet_rpc("extfund") - # Make a weird but signable script. sh(pkh()) descriptor accomplishes this - desc = descsum_create("sh(pkh({}))".format(privkey)) + # Make a weird but signable script. sh(wsh(pkh())) descriptor accomplishes this + desc = descsum_create("sh(wsh(pkh({})))".format(privkey)) if self.options.descriptors: res = self.nodes[0].importdescriptors([{"desc": desc, "timestamp": "now"}]) else: @@ -634,7 +638,7 @@ class PSBTTest(BitcoinTestFramework): assert_raises_rpc_error(-4, "Insufficient funds", wallet.walletcreatefundedpsbt, [ext_utxo], {self.nodes[0].getnewaddress(): 15}) # But funding should work when the solving data is provided - psbt = wallet.walletcreatefundedpsbt([ext_utxo], {self.nodes[0].getnewaddress(): 15}, 0, {"add_inputs": True, "solving_data": {"pubkeys": [addr_info['pubkey']], "scripts": [addr_info["embedded"]["scriptPubKey"]]}}) + psbt = wallet.walletcreatefundedpsbt([ext_utxo], {self.nodes[0].getnewaddress(): 15}, 0, {"add_inputs": True, "solving_data": {"pubkeys": [addr_info['pubkey']], "scripts": [addr_info["embedded"]["scriptPubKey"], addr_info["embedded"]["embedded"]["scriptPubKey"]]}}) signed = wallet.walletprocesspsbt(psbt['psbt']) assert not signed['complete'] signed = self.nodes[0].walletprocesspsbt(signed['psbt']) @@ -655,10 +659,11 @@ class PSBTTest(BitcoinTestFramework): break psbt_in = dec["inputs"][input_idx] # Calculate the input weight - # (prevout + sequence + length of scriptSig + 2 bytes buffer) * 4 + len of scriptwitness + # (prevout + sequence + length of scriptSig + scriptsig + 1 byte buffer) * WITNESS_SCALE_FACTOR + num scriptWitness stack items + (length of stack item + stack item) * N stack items + 1 byte buffer len_scriptsig = len(psbt_in["final_scriptSig"]["hex"]) // 2 if "final_scriptSig" in psbt_in else 0 - len_scriptwitness = len(psbt_in["final_scriptwitness"]["hex"]) // 2 if "final_scriptwitness" in psbt_in else 0 - input_weight = ((41 + len_scriptsig + 2) * 4) + len_scriptwitness + len_scriptsig += len(ser_compact_size(len_scriptsig)) + 1 + len_scriptwitness = (sum([(len(x) // 2) + len(ser_compact_size(len(x) // 2)) for x in psbt_in["final_scriptwitness"]]) + len(psbt_in["final_scriptwitness"]) + 1) if "final_scriptwitness" in psbt_in else 0 + input_weight = ((40 + len_scriptsig) * WITNESS_SCALE_FACTOR) + len_scriptwitness low_input_weight = input_weight // 2 high_input_weight = input_weight * 2 diff --git a/test/functional/rpc_rawtransaction.py b/test/functional/rpc_rawtransaction.py index a839af0288..1f95814e18 100755 --- a/test/functional/rpc_rawtransaction.py +++ b/test/functional/rpc_rawtransaction.py @@ -17,6 +17,7 @@ from decimal import Decimal from test_framework.blocktools import COINBASE_MATURITY from test_framework.messages import ( + BIP125_SEQUENCE_NUMBER, CTransaction, tx_from_hex, ) @@ -24,7 +25,10 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, assert_raises_rpc_error, - find_vout_for_address, +) +from test_framework.wallet import ( + getnewdestination, + MiniWallet, ) @@ -52,79 +56,67 @@ class multidict(dict): class RawTransactionsTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True - self.num_nodes = 4 + self.num_nodes = 3 self.extra_args = [ ["-txindex"], ["-txindex"], - ["-txindex"], [], ] # whitelist all peers to speed up tx relay / mempool sync for args in self.extra_args: args.append("-whitelist=noban@127.0.0.1") + self.requires_wallet = self.is_specified_wallet_compiled() self.supports_cli = False - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() - def setup_network(self): super().setup_network() self.connect_nodes(0, 2) def run_test(self): + self.wallet = MiniWallet(self.nodes[0]) self.log.info("Prepare some coins for multiple *rawtransaction commands") - self.generate(self.nodes[2], 1) + self.generate(self.wallet, 10) self.generate(self.nodes[0], COINBASE_MATURITY + 1) - for amount in [1.5, 1.0, 5.0]: - self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), amount) - self.sync_all() - self.generate(self.nodes[0], 5) self.getrawtransaction_tests() self.createrawtransaction_tests() - self.signrawtransactionwithwallet_tests() self.sendrawtransaction_tests() self.sendrawtransaction_testmempoolaccept_tests() self.decoderawtransaction_tests() self.transaction_version_number_tests() - if not self.options.descriptors: + if self.requires_wallet and not self.options.descriptors: self.raw_multisig_transaction_legacy_tests() def getrawtransaction_tests(self): - addr = self.nodes[1].getnewaddress() - txid = self.nodes[0].sendtoaddress(addr, 10) + tx = self.wallet.send_self_transfer(from_node=self.nodes[0]) self.generate(self.nodes[0], 1) - vout = find_vout_for_address(self.nodes[1], txid, addr) - rawTx = self.nodes[1].createrawtransaction([{'txid': txid, 'vout': vout}], {self.nodes[1].getnewaddress(): 9.999}) - rawTxSigned = self.nodes[1].signrawtransactionwithwallet(rawTx) - txId = self.nodes[1].sendrawtransaction(rawTxSigned['hex']) - self.generateblock(self.nodes[0], output=self.nodes[0].getnewaddress(), transactions=[rawTxSigned['hex']]) + txId = tx['txid'] err_msg = ( "No such mempool transaction. Use -txindex or provide a block hash to enable" " blockchain transaction queries. Use gettransaction for wallet transactions." ) - for n in [0, 3]: + for n in [0, 2]: self.log.info(f"Test getrawtransaction {'with' if n == 0 else 'without'} -txindex") if n == 0: # With -txindex. # 1. valid parameters - only supply txid - assert_equal(self.nodes[n].getrawtransaction(txId), rawTxSigned['hex']) + assert_equal(self.nodes[n].getrawtransaction(txId), tx['hex']) # 2. valid parameters - supply txid and 0 for non-verbose - assert_equal(self.nodes[n].getrawtransaction(txId, 0), rawTxSigned['hex']) + assert_equal(self.nodes[n].getrawtransaction(txId, 0), tx['hex']) # 3. valid parameters - supply txid and False for non-verbose - assert_equal(self.nodes[n].getrawtransaction(txId, False), rawTxSigned['hex']) + assert_equal(self.nodes[n].getrawtransaction(txId, False), tx['hex']) # 4. valid parameters - supply txid and 1 for verbose. # We only check the "hex" field of the output so we don't need to update this test every time the output format changes. - assert_equal(self.nodes[n].getrawtransaction(txId, 1)["hex"], rawTxSigned['hex']) + assert_equal(self.nodes[n].getrawtransaction(txId, 1)["hex"], tx['hex']) # 5. valid parameters - supply txid and True for non-verbose - assert_equal(self.nodes[n].getrawtransaction(txId, True)["hex"], rawTxSigned['hex']) + assert_equal(self.nodes[n].getrawtransaction(txId, True)["hex"], tx['hex']) else: # Without -txindex, expect to raise. for verbose in [None, 0, False, 1, True]: @@ -141,9 +133,9 @@ class RawTransactionsTest(BitcoinTestFramework): assert_raises_rpc_error(-1, "not a boolean", self.nodes[n].getrawtransaction, txId, {}) # Make a tx by sending, then generate 2 blocks; block1 has the tx in it - tx = self.nodes[2].sendtoaddress(self.nodes[1].getnewaddress(), 1) + tx = self.wallet.send_self_transfer(from_node=self.nodes[2])['txid'] block1, block2 = self.generate(self.nodes[2], 2) - for n in [0, 3]: + for n in [0, 2]: self.log.info(f"Test getrawtransaction {'with' if n == 0 else 'without'} -txindex, with blockhash") # We should be able to get the raw transaction by providing the correct block gottx = self.nodes[n].getrawtransaction(txid=tx, verbose=True, blockhash=block1) @@ -200,20 +192,21 @@ class RawTransactionsTest(BitcoinTestFramework): # sequence number out of range for invalid_seq in [-1, 4294967296]: inputs = [{'txid': TXID, 'vout': 1, 'sequence': invalid_seq}] - outputs = {self.nodes[0].getnewaddress(): 1} + address = getnewdestination()[2] + outputs = {address: 1} assert_raises_rpc_error(-8, 'Invalid parameter, sequence number is out of range', self.nodes[0].createrawtransaction, inputs, outputs) # with valid sequence number for valid_seq in [1000, 4294967294]: inputs = [{'txid': TXID, 'vout': 1, 'sequence': valid_seq}] - outputs = {self.nodes[0].getnewaddress(): 1} + address = getnewdestination()[2] + outputs = {address: 1} rawtx = self.nodes[0].createrawtransaction(inputs, outputs) decrawtx = self.nodes[0].decoderawtransaction(rawtx) assert_equal(decrawtx['vin'][0]['sequence'], valid_seq) # Test `createrawtransaction` invalid `outputs` - address = self.nodes[0].getnewaddress() - address2 = self.nodes[0].getnewaddress() + address = getnewdestination()[2] assert_raises_rpc_error(-1, "JSON value is not an array as expected", self.nodes[0].createrawtransaction, [], 'foo') self.nodes[0].createrawtransaction(inputs=[], outputs={}) # Should not throw for backwards compatibility self.nodes[0].createrawtransaction(inputs=[], outputs=[]) @@ -228,6 +221,10 @@ class RawTransactionsTest(BitcoinTestFramework): assert_raises_rpc_error(-8, "Invalid parameter, key-value pair must contain exactly one key", self.nodes[0].createrawtransaction, [], [{'a': 1, 'b': 2}]) assert_raises_rpc_error(-8, "Invalid parameter, key-value pair not an object as expected", self.nodes[0].createrawtransaction, [], [['key-value pair1'], ['2']]) + # Test `createrawtransaction` mismatch between sequence number(s) and `replaceable` option + assert_raises_rpc_error(-8, "Invalid parameter combination: Sequence number(s) contradict replaceable option", + self.nodes[0].createrawtransaction, [{'txid': TXID, 'vout': 0, 'sequence': BIP125_SEQUENCE_NUMBER+1}], {}, 0, True) + # Test `createrawtransaction` invalid `locktime` assert_raises_rpc_error(-3, "Expected type number", self.nodes[0].createrawtransaction, [], {}, 'foo') assert_raises_rpc_error(-8, "Invalid parameter, locktime out of range", self.nodes[0].createrawtransaction, [], {}, -1) @@ -245,6 +242,7 @@ class RawTransactionsTest(BitcoinTestFramework): self.nodes[2].createrawtransaction(inputs=[{'txid': TXID, 'vout': 9}], outputs=[{address: 99}]), ) # Two outputs + address2 = getnewdestination()[2] tx = tx_from_hex(self.nodes[2].createrawtransaction(inputs=[{'txid': TXID, 'vout': 9}], outputs=OrderedDict([(address, 99), (address2, 99)]))) assert_equal(len(tx.vout), 2) assert_equal( @@ -259,122 +257,53 @@ class RawTransactionsTest(BitcoinTestFramework): self.nodes[2].createrawtransaction(inputs=[{'txid': TXID, 'vout': 9}], outputs=[{address: 99}, {address2: 99}, {'data': '99'}]), ) - def signrawtransactionwithwallet_tests(self): - for type in ["bech32", "p2sh-segwit", "legacy"]: - self.log.info(f"Test signrawtransactionwithwallet with missing prevtx info ({type})") - addr = self.nodes[0].getnewaddress("", type) - addrinfo = self.nodes[0].getaddressinfo(addr) - pubkey = addrinfo["scriptPubKey"] - inputs = [{'txid': TXID, 'vout': 3, 'sequence': 1000}] - outputs = {self.nodes[0].getnewaddress(): 1} - rawtx = self.nodes[0].createrawtransaction(inputs, outputs) - - prevtx = dict(txid=TXID, scriptPubKey=pubkey, vout=3, amount=1) - succ = self.nodes[0].signrawtransactionwithwallet(rawtx, [prevtx]) - assert succ["complete"] - - if type == "legacy": - del prevtx["amount"] - succ = self.nodes[0].signrawtransactionwithwallet(rawtx, [prevtx]) - assert succ["complete"] - else: - assert_raises_rpc_error(-3, "Missing amount", self.nodes[0].signrawtransactionwithwallet, rawtx, [ - { - "txid": TXID, - "scriptPubKey": pubkey, - "vout": 3, - } - ]) - - assert_raises_rpc_error(-3, "Missing vout", self.nodes[0].signrawtransactionwithwallet, rawtx, [ - { - "txid": TXID, - "scriptPubKey": pubkey, - "amount": 1, - } - ]) - assert_raises_rpc_error(-3, "Missing txid", self.nodes[0].signrawtransactionwithwallet, rawtx, [ - { - "scriptPubKey": pubkey, - "vout": 3, - "amount": 1, - } - ]) - assert_raises_rpc_error(-3, "Missing scriptPubKey", self.nodes[0].signrawtransactionwithwallet, rawtx, [ - { - "txid": TXID, - "vout": 3, - "amount": 1 - } - ]) - def sendrawtransaction_tests(self): self.log.info("Test sendrawtransaction with missing input") inputs = [{'txid': TXID, 'vout': 1}] # won't exist - outputs = {self.nodes[0].getnewaddress(): 4.998} + address = getnewdestination()[2] + outputs = {address: 4.998} rawtx = self.nodes[2].createrawtransaction(inputs, outputs) - rawtx = self.nodes[2].signrawtransactionwithwallet(rawtx) - assert_raises_rpc_error(-25, "bad-txns-inputs-missingorspent", self.nodes[2].sendrawtransaction, rawtx['hex']) + assert_raises_rpc_error(-25, "bad-txns-inputs-missingorspent", self.nodes[2].sendrawtransaction, rawtx) def sendrawtransaction_testmempoolaccept_tests(self): self.log.info("Test sendrawtransaction/testmempoolaccept with maxfeerate") fee_exceeds_max = "Fee exceeds maximum configured by user (e.g. -maxtxfee, maxfeerate)" # Test a transaction with a small fee. - txId = self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), 1.0) - rawTx = self.nodes[0].getrawtransaction(txId, True) - vout = next(o for o in rawTx['vout'] if o['value'] == Decimal('1.00000000')) - - self.sync_all() - inputs = [{"txid": txId, "vout": vout['n']}] - # Fee 10,000 satoshis, (1 - (10000 sat * 0.00000001 BTC/sat)) = 0.9999 - outputs = {self.nodes[0].getnewaddress(): Decimal("0.99990000")} - rawTx = self.nodes[2].createrawtransaction(inputs, outputs) - rawTxSigned = self.nodes[2].signrawtransactionwithwallet(rawTx) - assert_equal(rawTxSigned['complete'], True) - # Fee 10,000 satoshis, ~100 b transaction, fee rate should land around 100 sat/byte = 0.00100000 BTC/kB + # Fee rate is 0.00100000 BTC/kvB + tx = self.wallet.create_self_transfer(fee_rate=Decimal('0.00100000')) # Thus, testmempoolaccept should reject - testres = self.nodes[2].testmempoolaccept([rawTxSigned['hex']], 0.00001000)[0] + testres = self.nodes[2].testmempoolaccept([tx['hex']], 0.00001000)[0] assert_equal(testres['allowed'], False) assert_equal(testres['reject-reason'], 'max-fee-exceeded') # and sendrawtransaction should throw - assert_raises_rpc_error(-25, fee_exceeds_max, self.nodes[2].sendrawtransaction, rawTxSigned['hex'], 0.00001000) + assert_raises_rpc_error(-25, fee_exceeds_max, self.nodes[2].sendrawtransaction, tx['hex'], 0.00001000) # and the following calls should both succeed - testres = self.nodes[2].testmempoolaccept(rawtxs=[rawTxSigned['hex']])[0] + testres = self.nodes[2].testmempoolaccept(rawtxs=[tx['hex']])[0] assert_equal(testres['allowed'], True) - self.nodes[2].sendrawtransaction(hexstring=rawTxSigned['hex']) + self.nodes[2].sendrawtransaction(hexstring=tx['hex']) # Test a transaction with a large fee. - txId = self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), 1.0) - rawTx = self.nodes[0].getrawtransaction(txId, True) - vout = next(o for o in rawTx['vout'] if o['value'] == Decimal('1.00000000')) - - self.sync_all() - inputs = [{"txid": txId, "vout": vout['n']}] - # Fee 2,000,000 satoshis, (1 - (2000000 sat * 0.00000001 BTC/sat)) = 0.98 - outputs = {self.nodes[0].getnewaddress() : Decimal("0.98000000")} - rawTx = self.nodes[2].createrawtransaction(inputs, outputs) - rawTxSigned = self.nodes[2].signrawtransactionwithwallet(rawTx) - assert_equal(rawTxSigned['complete'], True) - # Fee 2,000,000 satoshis, ~100 b transaction, fee rate should land around 20,000 sat/byte = 0.20000000 BTC/kB + # Fee rate is 0.20000000 BTC/kvB + tx = self.wallet.create_self_transfer(mempool_valid=False, from_node=self.nodes[0], fee_rate=Decimal('0.20000000')) # Thus, testmempoolaccept should reject - testres = self.nodes[2].testmempoolaccept([rawTxSigned['hex']])[0] + testres = self.nodes[2].testmempoolaccept([tx['hex']])[0] assert_equal(testres['allowed'], False) assert_equal(testres['reject-reason'], 'max-fee-exceeded') # and sendrawtransaction should throw - assert_raises_rpc_error(-25, fee_exceeds_max, self.nodes[2].sendrawtransaction, rawTxSigned['hex']) + assert_raises_rpc_error(-25, fee_exceeds_max, self.nodes[2].sendrawtransaction, tx['hex']) # and the following calls should both succeed - testres = self.nodes[2].testmempoolaccept(rawtxs=[rawTxSigned['hex']], maxfeerate='0.20000000')[0] + testres = self.nodes[2].testmempoolaccept(rawtxs=[tx['hex']], maxfeerate='0.20000000')[0] assert_equal(testres['allowed'], True) - self.nodes[2].sendrawtransaction(hexstring=rawTxSigned['hex'], maxfeerate='0.20000000') + self.nodes[2].sendrawtransaction(hexstring=tx['hex'], maxfeerate='0.20000000') self.log.info("Test sendrawtransaction/testmempoolaccept with tx already in the chain") self.generate(self.nodes[2], 1) for node in self.nodes: - testres = node.testmempoolaccept([rawTxSigned['hex']])[0] + testres = node.testmempoolaccept([tx['hex']])[0] assert_equal(testres['allowed'], False) assert_equal(testres['reject-reason'], 'txn-already-known') - assert_raises_rpc_error(-27, 'Transaction already in block chain', node.sendrawtransaction, rawTxSigned['hex']) + assert_raises_rpc_error(-27, 'Transaction already in block chain', node.sendrawtransaction, tx['hex']) def decoderawtransaction_tests(self): self.log.info("Test decoderawtransaction") diff --git a/test/functional/rpc_signrawtransaction.py b/test/functional/rpc_signrawtransaction.py index a2091b4ece..8da2cfa72b 100755 --- a/test/functional/rpc_signrawtransaction.py +++ b/test/functional/rpc_signrawtransaction.py @@ -334,6 +334,56 @@ class SignRawTransactionsTest(BitcoinTestFramework): assert_equal(signed["complete"], True) self.nodes[0].sendrawtransaction(signed["hex"]) + def test_signing_with_missing_prevtx_info(self): + txid = "1d1d4e24ed99057e84c3f80fd8fbec79ed9e1acee37da269356ecea000000000" + for type in ["bech32", "p2sh-segwit", "legacy"]: + self.log.info(f"Test signing with missing prevtx info ({type})") + addr = self.nodes[0].getnewaddress("", type) + addrinfo = self.nodes[0].getaddressinfo(addr) + pubkey = addrinfo["scriptPubKey"] + inputs = [{'txid': txid, 'vout': 3, 'sequence': 1000}] + outputs = {self.nodes[0].getnewaddress(): 1} + rawtx = self.nodes[0].createrawtransaction(inputs, outputs) + + prevtx = dict(txid=txid, scriptPubKey=pubkey, vout=3, amount=1) + succ = self.nodes[0].signrawtransactionwithwallet(rawtx, [prevtx]) + assert succ["complete"] + + if type == "legacy": + del prevtx["amount"] + succ = self.nodes[0].signrawtransactionwithwallet(rawtx, [prevtx]) + assert succ["complete"] + else: + assert_raises_rpc_error(-3, "Missing amount", self.nodes[0].signrawtransactionwithwallet, rawtx, [ + { + "txid": txid, + "scriptPubKey": pubkey, + "vout": 3, + } + ]) + + assert_raises_rpc_error(-3, "Missing vout", self.nodes[0].signrawtransactionwithwallet, rawtx, [ + { + "txid": txid, + "scriptPubKey": pubkey, + "amount": 1, + } + ]) + assert_raises_rpc_error(-3, "Missing txid", self.nodes[0].signrawtransactionwithwallet, rawtx, [ + { + "scriptPubKey": pubkey, + "vout": 3, + "amount": 1, + } + ]) + assert_raises_rpc_error(-3, "Missing scriptPubKey", self.nodes[0].signrawtransactionwithwallet, rawtx, [ + { + "txid": txid, + "vout": 3, + "amount": 1 + } + ]) + def run_test(self): self.successful_signing_test() self.script_verification_error_test() @@ -343,6 +393,7 @@ class SignRawTransactionsTest(BitcoinTestFramework): self.test_fully_signed_tx() self.test_signing_with_csv() self.test_signing_with_cltv() + self.test_signing_with_missing_prevtx_info() if __name__ == '__main__': diff --git a/test/functional/rpc_users.py b/test/functional/rpc_users.py index 7cedb4336b..1a35a57802 100755 --- a/test/functional/rpc_users.py +++ b/test/functional/rpc_users.py @@ -107,6 +107,9 @@ class HTTPBasicsTest(BitcoinTestFramework): self.stop_node(0) self.nodes[0].assert_start_raises_init_error(expected_msg=init_error, extra_args=['-rpcauth=foo']) self.nodes[0].assert_start_raises_init_error(expected_msg=init_error, extra_args=['-rpcauth=foo:bar']) + self.nodes[0].assert_start_raises_init_error(expected_msg=init_error, extra_args=['-rpcauth=foo:bar:baz']) + self.nodes[0].assert_start_raises_init_error(expected_msg=init_error, extra_args=['-rpcauth=foo$bar:baz']) + self.nodes[0].assert_start_raises_init_error(expected_msg=init_error, extra_args=['-rpcauth=foo$bar$baz']) self.log.info('Check that failure to write cookie file will abort the node gracefully') cookie_file = os.path.join(get_datadir_path(self.options.tmpdir, 0), self.chain, '.cookie.tmp') diff --git a/test/functional/test_framework/messages.py b/test/functional/test_framework/messages.py index f57b6e7494..aae44c0ac0 100755 --- a/test/functional/test_framework/messages.py +++ b/test/functional/test_framework/messages.py @@ -1672,7 +1672,7 @@ class msg_getcfilters: __slots__ = ("filter_type", "start_height", "stop_hash") msgtype = b"getcfilters" - def __init__(self, filter_type, start_height, stop_hash): + def __init__(self, filter_type=None, start_height=None, stop_hash=None): self.filter_type = filter_type self.start_height = start_height self.stop_hash = stop_hash @@ -1722,7 +1722,7 @@ class msg_getcfheaders: __slots__ = ("filter_type", "start_height", "stop_hash") msgtype = b"getcfheaders" - def __init__(self, filter_type, start_height, stop_hash): + def __init__(self, filter_type=None, start_height=None, stop_hash=None): self.filter_type = filter_type self.start_height = start_height self.stop_hash = stop_hash @@ -1775,7 +1775,7 @@ class msg_getcfcheckpt: __slots__ = ("filter_type", "stop_hash") msgtype = b"getcfcheckpt" - def __init__(self, filter_type, stop_hash): + def __init__(self, filter_type=None, stop_hash=None): self.filter_type = filter_type self.stop_hash = stop_hash diff --git a/test/functional/test_framework/p2p.py b/test/functional/test_framework/p2p.py index 251d3d5eae..fc72a9ab73 100755 --- a/test/functional/test_framework/p2p.py +++ b/test/functional/test_framework/p2p.py @@ -47,6 +47,9 @@ from test_framework.messages import ( msg_getaddr, msg_getblocks, msg_getblocktxn, + msg_getcfcheckpt, + msg_getcfheaders, + msg_getcfilters, msg_getdata, msg_getheaders, msg_headers, @@ -108,6 +111,9 @@ MESSAGEMAP = { b"getaddr": msg_getaddr, b"getblocks": msg_getblocks, b"getblocktxn": msg_getblocktxn, + b"getcfcheckpt": msg_getcfcheckpt, + b"getcfheaders": msg_getcfheaders, + b"getcfilters": msg_getcfilters, b"getdata": msg_getdata, b"getheaders": msg_getheaders, b"headers": msg_headers, diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index 2fb9ec0942..3f02d21d42 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -223,11 +223,11 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): # It still needs to exist and be None in order for tests to work however. self.options.descriptors = None + PortSeed.n = self.options.port_seed + def setup(self): """Call this method to start up the test framework object with options set.""" - PortSeed.n = self.options.port_seed - check_json_precision() self.options.cachedir = os.path.abspath(self.options.cachedir) @@ -244,8 +244,14 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): "src", "bitcoin-cli" + config["environment"]["EXEEXT"], ) + fname_bitcoinutil = os.path.join( + config["environment"]["BUILDDIR"], + "src", + "bitcoin-util" + config["environment"]["EXEEXT"], + ) self.options.bitcoind = os.getenv("BITCOIND", default=fname_bitcoind) self.options.bitcoincli = os.getenv("BITCOINCLI", default=fname_bitcoincli) + self.options.bitcoinutil = os.getenv("BITCOINUTIL", default=fname_bitcoinutil) os.environ['PATH'] = os.pathsep.join([ os.path.join(config['environment']['BUILDDIR'], 'src'), @@ -880,6 +886,11 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): if not self.is_wallet_tool_compiled(): raise SkipTest("bitcoin-wallet has not been compiled") + def skip_if_no_bitcoin_util(self): + """Skip the running test if bitcoin-util has not been compiled.""" + if not self.is_bitcoin_util_compiled(): + raise SkipTest("bitcoin-util has not been compiled") + def skip_if_no_cli(self): """Skip the running test if bitcoin-cli has not been compiled.""" if not self.is_cli_compiled(): @@ -927,6 +938,10 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): """Checks whether bitcoin-wallet was compiled.""" return self.config["components"].getboolean("ENABLE_WALLET_TOOL") + def is_bitcoin_util_compiled(self): + """Checks whether bitcoin-util was compiled.""" + return self.config["components"].getboolean("ENABLE_BITCOIN_UTIL") + def is_zmq_compiled(self): """Checks whether the zmq module was compiled.""" return self.config["components"].getboolean("ENABLE_ZMQ") diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index e56d4aa492..03f6c8adea 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -423,7 +423,7 @@ class TestNode(): self._raise_assertion_error('Expected messages "{}" does not partially match log:\n\n{}\n\n'.format(str(expected_msgs), print_log)) @contextlib.contextmanager - def wait_for_debug_log(self, expected_msgs, timeout=60, ignore_case=False): + def wait_for_debug_log(self, expected_msgs, timeout=60): """ Block until we see a particular debug log message fragment or until we exceed the timeout. Return: @@ -431,18 +431,17 @@ class TestNode(): """ time_end = time.time() + timeout * self.timeout_factor prev_size = self.debug_log_bytes() - re_flags = re.MULTILINE | (re.IGNORECASE if ignore_case else 0) yield while True: found = True - with open(self.debug_log_path, encoding='utf-8') as dl: + with open(self.debug_log_path, "rb") as dl: dl.seek(prev_size) log = dl.read() for expected_msg in expected_msgs: - if re.search(re.escape(expected_msg), log, flags=re_flags) is None: + if expected_msg not in log: found = False if found: @@ -545,6 +544,7 @@ class TestNode(): Will throw if bitcoind starts without an error. Will throw if an expected_msg is provided and it does not match bitcoind's stdout.""" + assert not self.running with tempfile.NamedTemporaryFile(dir=self.stderr_dir, delete=False) as log_stderr, \ tempfile.NamedTemporaryFile(dir=self.stdout_dir, delete=False) as log_stdout: try: diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index 6c2a02bcf6..b942642074 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -378,6 +378,7 @@ def write_config(config_path, *, n, chain, extra_config="", disable_autoconnect= f.write("[{}]\n".format(chain_name_conf_section)) f.write("port=" + str(p2p_port(n)) + "\n") f.write("rpcport=" + str(rpc_port(n)) + "\n") + f.write("rpcdoccheck=1\n") f.write("fallbackfee=0.0002\n") f.write("server=1\n") f.write("keypool=1\n") diff --git a/test/functional/test_framework/wallet.py b/test/functional/test_framework/wallet.py index e86f365f11..e43dd9f61a 100644 --- a/test/functional/test_framework/wallet.py +++ b/test/functional/test_framework/wallet.py @@ -10,6 +10,7 @@ from enum import Enum from random import choice from typing import ( Any, + List, Optional, ) from test_framework.address import ( @@ -127,6 +128,7 @@ class MiniWallet: if not fixed_length: break tx.vin[0].scriptSig = CScript([der_sig + bytes(bytearray([SIGHASH_ALL]))]) + tx.rehash() def generate(self, num_blocks, **kwargs): """Generate blocks with coinbase outputs to the internal address, and append the outputs to the internal list""" @@ -146,7 +148,7 @@ class MiniWallet: def get_address(self): return self._address - def get_utxo(self, *, txid: str = '', vout: Optional[int] = None, mark_as_spent=True): + def get_utxo(self, *, txid: str = '', vout: Optional[int] = None, mark_as_spent=True) -> dict: """ Returns a utxo and marks it as spent (pops it from the internal list) @@ -166,6 +168,13 @@ class MiniWallet: else: return self._utxos[index] + def get_utxos(self, *, mark_as_spent=True): + """Returns the list of all utxos and optionally mark them as spent""" + utxos = deepcopy(self._utxos) + if mark_as_spent: + self._utxos = [] + return utxos + def send_self_transfer(self, **kwargs): """Create and send a tx with the specified fee_rate. Fee may be exact or at most one satoshi higher than needed.""" tx = self.create_self_transfer(**kwargs) @@ -207,14 +216,21 @@ class MiniWallet: return {'new_utxos': [self.get_utxo(txid=txid, vout=vout) for vout in range(len(tx.vout))], 'txid': txid, 'hex': tx.serialize().hex(), 'tx': tx} - def create_self_transfer_multi(self, *, from_node, utxos_to_spend=None, num_outputs=1, fee_per_output=1000): + def create_self_transfer_multi( + self, *, from_node, + utxos_to_spend: Optional[List[dict]] = None, + num_outputs=1, + sequence=0, + fee_per_output=1000): """ Create and return a transaction that spends the given UTXOs and creates a certain number of outputs with equal amounts. """ utxos_to_spend = utxos_to_spend or [self.get_utxo()] # create simple tx template (1 input, 1 output) - tx = self.create_self_transfer(fee_rate=0, from_node=from_node, utxo_to_spend=utxos_to_spend[0], mempool_valid=False)['tx'] + tx = self.create_self_transfer( + fee_rate=0, from_node=from_node, + utxo_to_spend=utxos_to_spend[0], sequence=sequence, mempool_valid=False)['tx'] # duplicate inputs, witnesses and outputs tx.vin = [deepcopy(tx.vin[0]) for _ in range(len(utxos_to_spend))] @@ -233,7 +249,8 @@ class MiniWallet: return tx def create_self_transfer(self, *, fee_rate=Decimal("0.003"), from_node=None, utxo_to_spend=None, mempool_valid=True, locktime=0, sequence=0): - """Create and return a tx with the specified fee_rate. Fee may be exact or at most one satoshi higher than needed.""" + """Create and return a tx with the specified fee_rate. Fee may be exact or at most one satoshi higher than needed. + Checking mempool validity via the testmempoolaccept RPC can be skipped by setting mempool_valid to False.""" from_node = from_node or self._test_node utxo_to_spend = utxo_to_spend or self.get_utxo() if self._priv_key is None: @@ -260,15 +277,16 @@ class MiniWallet: tx.wit.vtxinwit[0].scriptWitness.stack = [CScript([OP_TRUE]), bytes([LEAF_VERSION_TAPSCRIPT]) + self._internal_key] tx_hex = tx.serialize().hex() - tx_info = from_node.testmempoolaccept([tx_hex])[0] - assert_equal(mempool_valid, tx_info['allowed']) if mempool_valid: + tx_info = from_node.testmempoolaccept([tx_hex])[0] + assert_equal(tx_info['allowed'], True) assert_equal(tx_info['vsize'], vsize) assert_equal(tx_info['fees']['base'], utxo_to_spend['value'] - Decimal(send_value) / COIN) - return {'txid': tx_info['txid'], 'wtxid': tx_info['wtxid'], 'hex': tx_hex, 'tx': tx} - def sendrawtransaction(self, *, from_node, tx_hex, **kwargs): - txid = from_node.sendrawtransaction(hexstring=tx_hex, **kwargs) + return {'txid': tx.rehash(), 'wtxid': tx.getwtxid(), 'hex': tx_hex, 'tx': tx} + + def sendrawtransaction(self, *, from_node, tx_hex, maxfeerate=0, **kwargs): + txid = from_node.sendrawtransaction(hexstring=tx_hex, maxfeerate=maxfeerate, **kwargs) self.scan_tx(from_node.decoderawtransaction(tx_hex)) return txid diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 1f0f806d91..6a44f9d21d 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -28,7 +28,7 @@ import logging import unittest # Formatting. Default colors to empty strings. -BOLD, GREEN, RED, GREY = ("", ""), ("", ""), ("", ""), ("", "") +DEFAULT, BOLD, GREEN, RED = ("", ""), ("", ""), ("", ""), ("", "") try: # Make sure python thinks it can write unicode to its stdout "\u2713".encode("utf_8").decode(sys.stdout.encoding) @@ -59,10 +59,10 @@ if os.name != 'nt' or sys.getwindowsversion() >= (10, 0, 14393): #type:ignore kernel32.SetConsoleMode(stderr, stderr_mode.value | ENABLE_VIRTUAL_TERMINAL_PROCESSING) # primitive formatting on supported # terminal via ANSI escape sequences: + DEFAULT = ('\033[0m', '\033[0m') BOLD = ('\033[0m', '\033[1m') GREEN = ('\033[0m', '\033[0;32m') RED = ('\033[0m', '\033[0;31m') - GREY = ('\033[0m', '\033[1;30m') TEST_EXIT_PASSED = 0 TEST_EXIT_SKIPPED = 77 @@ -82,6 +82,7 @@ EXTENDED_SCRIPTS = [ # Longest test should go first, to favor running tests in parallel 'feature_pruning.py', 'feature_dbcrash.py', + 'feature_index_prune.py', ] BASE_SCRIPTS = [ @@ -111,7 +112,6 @@ BASE_SCRIPTS = [ 'p2p_tx_download.py', 'mempool_updatefromblock.py', 'wallet_dump.py --legacy-wallet', - 'feature_taproot.py --previous_release', 'feature_taproot.py', 'rpc_signer.py', 'wallet_signer.py --descriptors', @@ -145,6 +145,8 @@ BASE_SCRIPTS = [ 'wallet_txn_doublespend.py --mineblock', 'tool_wallet.py --legacy-wallet', 'tool_wallet.py --descriptors', + 'tool_signet_miner.py --legacy-wallet', + 'tool_signet_miner.py --descriptors', 'wallet_txn_clone.py', 'wallet_txn_clone.py --segwit', 'rpc_getchaintips.py', @@ -155,6 +157,7 @@ BASE_SCRIPTS = [ 'wallet_avoidreuse.py --descriptors', 'mempool_reorg.py', 'mempool_persist.py', + 'p2p_block_sync.py', 'wallet_multiwallet.py --legacy-wallet', 'wallet_multiwallet.py --descriptors', 'wallet_multiwallet.py --usecli', @@ -168,6 +171,7 @@ BASE_SCRIPTS = [ 'wallet_reorgsrestore.py', 'interface_http.py', 'interface_rpc.py', + 'interface_usdt_coinselection.py', 'interface_usdt_net.py', 'interface_usdt_utxocache.py', 'interface_usdt_validation.py', @@ -180,7 +184,6 @@ BASE_SCRIPTS = [ 'rpc_signrawtransaction.py --legacy-wallet', 'rpc_signrawtransaction.py --descriptors', 'rpc_rawtransaction.py --legacy-wallet', - 'rpc_rawtransaction.py --descriptors', 'wallet_groups.py --legacy-wallet', 'wallet_transactiontime_rescan.py --descriptors', 'wallet_transactiontime_rescan.py --legacy-wallet', @@ -252,6 +255,7 @@ BASE_SCRIPTS = [ 'rpc_bind.py --ipv4', 'rpc_bind.py --ipv6', 'rpc_bind.py --nonloopback', + 'wallet_crosschain.py', 'mining_basic.py', 'feature_signet.py', 'wallet_bumpfee.py --legacy-wallet', @@ -324,13 +328,12 @@ BASE_SCRIPTS = [ 'feature_presegwit_node_upgrade.py', 'feature_settings.py', 'rpc_getdescriptorinfo.py', - 'rpc_mempool_entry_fee_fields_deprecation.py', + 'rpc_mempool_info.py', 'rpc_help.py', 'feature_dirsymlinks.py', 'feature_help.py', 'feature_shutdown.py', 'p2p_ibd_txrelay.py', - 'feature_blockfilterindex_prune.py' # Don't append tests at the end to avoid merge conflicts # Put them in a random line within the section that fits their approximate run-time ] @@ -369,11 +372,11 @@ def main(): args, unknown_args = parser.parse_known_args() if not args.ansi: - global BOLD, GREEN, RED, GREY + global DEFAULT, BOLD, GREEN, RED + DEFAULT = ("", "") BOLD = ("", "") GREEN = ("", "") RED = ("", "") - GREY = ("", "") # args to be passed on always start with two dashes; tests are the remaining unknown args tests = [arg for arg in unknown_args if arg[:2] != "--"] @@ -717,7 +720,7 @@ class TestResult(): color = RED glyph = CROSS elif self.status == "Skipped": - color = GREY + color = DEFAULT glyph = CIRCLE return color[1] + "%s | %s%s | %s s\n" % (self.name.ljust(self.padding), glyph, self.status.ljust(7), self.time) + color[0] diff --git a/test/functional/tool_signet_miner.py b/test/functional/tool_signet_miner.py new file mode 100755 index 0000000000..e6fc9072ab --- /dev/null +++ b/test/functional/tool_signet_miner.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test signet miner tool""" + +import os.path +import subprocess +import sys +import time + +from test_framework.key import ECKey +from test_framework.script_util import key_to_p2wpkh_script +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal +from test_framework.wallet_util import bytes_to_wif + + +CHALLENGE_PRIVATE_KEY = (42).to_bytes(32, 'big') + + +class SignetMinerTest(BitcoinTestFramework): + def set_test_params(self): + self.chain = "signet" + self.setup_clean_chain = True + self.num_nodes = 1 + + # generate and specify signet challenge (simple p2wpkh script) + privkey = ECKey() + privkey.set(CHALLENGE_PRIVATE_KEY, True) + pubkey = privkey.get_pubkey().get_bytes() + challenge = key_to_p2wpkh_script(pubkey) + self.extra_args = [[f'-signetchallenge={challenge.hex()}']] + + def skip_test_if_missing_module(self): + self.skip_if_no_cli() + self.skip_if_no_wallet() + self.skip_if_no_bitcoin_util() + + def run_test(self): + node = self.nodes[0] + # import private key needed for signing block + node.importprivkey(bytes_to_wif(CHALLENGE_PRIVATE_KEY)) + + # generate block with signet miner tool + base_dir = self.config["environment"]["SRCDIR"] + signet_miner_path = os.path.join(base_dir, "contrib", "signet", "miner") + subprocess.run([ + sys.executable, + signet_miner_path, + f'--cli={node.cli.binary} -datadir={node.cli.datadir}', + 'generate', + f'--address={node.getnewaddress()}', + f'--grind-cmd={self.options.bitcoinutil} grind', + '--nbits=1d00ffff', + f'--set-block-time={int(time.time())}', + ], check=True, stderr=subprocess.STDOUT) + assert_equal(node.getblockcount(), 1) + + +if __name__ == "__main__": + SignetMinerTest().main() diff --git a/test/functional/wallet_avoidreuse.py b/test/functional/wallet_avoidreuse.py index dc823c2c60..f663666f57 100755 --- a/test/functional/wallet_avoidreuse.py +++ b/test/functional/wallet_avoidreuse.py @@ -118,6 +118,17 @@ class AvoidReuseTest(BitcoinTestFramework): assert_raises_rpc_error(-8, "Wallet flag is already set to false", self.nodes[0].setwalletflag, 'avoid_reuse', False) assert_raises_rpc_error(-8, "Wallet flag is already set to true", self.nodes[1].setwalletflag, 'avoid_reuse', True) + # Create a wallet with avoid reuse, and test that disabling it afterwards persists + self.nodes[1].createwallet(wallet_name="avoid_reuse_persist", avoid_reuse=True) + w = self.nodes[1].get_wallet_rpc("avoid_reuse_persist") + assert_equal(w.getwalletinfo()["avoid_reuse"], True) + w.setwalletflag("avoid_reuse", False) + assert_equal(w.getwalletinfo()["avoid_reuse"], False) + w.unloadwallet() + self.nodes[1].loadwallet("avoid_reuse_persist") + assert_equal(w.getwalletinfo()["avoid_reuse"], False) + w.unloadwallet() + def test_immutable(self): '''Test immutable wallet flags''' self.log.info("Test immutable wallet flags") diff --git a/test/functional/wallet_createwallet.py b/test/functional/wallet_createwallet.py index e8234de032..12480d4d1e 100755 --- a/test/functional/wallet_createwallet.py +++ b/test/functional/wallet_createwallet.py @@ -26,6 +26,11 @@ class CreateWalletTest(BitcoinTestFramework): node = self.nodes[0] self.generate(node, 1) # Leave IBD for sethdseed + self.log.info("Run createwallet with invalid parameters.") + # Run createwallet with invalid parameters. This must not prevent a new wallet with the same name from being created with the correct parameters. + assert_raises_rpc_error(-4, "Passphrase provided but private keys are disabled. A passphrase is only used to encrypt private keys, so cannot be used for wallets with private keys disabled.", + self.nodes[0].createwallet, wallet_name='w0', disable_private_keys=True, passphrase="passphrase") + self.nodes[0].createwallet(wallet_name='w0') w0 = node.get_wallet_rpc('w0') address1 = w0.getnewaddress() diff --git a/test/functional/wallet_crosschain.py b/test/functional/wallet_crosschain.py new file mode 100755 index 0000000000..b6d0c87985 --- /dev/null +++ b/test/functional/wallet_crosschain.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +import os + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_raises_rpc_error + +class WalletCrossChain(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 2 + self.setup_clean_chain = True + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + + def setup_network(self): + self.add_nodes(self.num_nodes) + + # Switch node 1 to testnet before starting it. + self.nodes[1].chain = 'testnet3' + self.nodes[1].extra_args = ['-maxconnections=0'] # disable testnet sync + with open(self.nodes[1].bitcoinconf, 'r', encoding='utf8') as conf: + conf_data = conf.read() + with open (self.nodes[1].bitcoinconf, 'w', encoding='utf8') as conf: + conf.write(conf_data.replace('regtest=', 'testnet=').replace('[regtest]', '[test]')) + + self.start_nodes() + + def run_test(self): + self.log.info("Creating wallets") + + node0_wallet = os.path.join(self.nodes[0].datadir, 'node0_wallet') + self.nodes[0].createwallet(node0_wallet) + self.nodes[0].unloadwallet(node0_wallet) + node1_wallet = os.path.join(self.nodes[1].datadir, 'node1_wallet') + self.nodes[1].createwallet(node1_wallet) + self.nodes[1].unloadwallet(node1_wallet) + + self.log.info("Loading wallets into nodes with a different genesis blocks") + + if self.options.descriptors: + assert_raises_rpc_error(-18, 'Wallet file verification failed.', self.nodes[0].loadwallet, node1_wallet) + assert_raises_rpc_error(-18, 'Wallet file verification failed.', self.nodes[1].loadwallet, node0_wallet) + else: + assert_raises_rpc_error(-4, 'Wallet files should not be reused across chains.', self.nodes[0].loadwallet, node1_wallet) + assert_raises_rpc_error(-4, 'Wallet files should not be reused across chains.', self.nodes[1].loadwallet, node0_wallet) + + if not self.options.descriptors: + self.log.info("Override cross-chain wallet load protection") + self.stop_nodes() + self.start_nodes([['-walletcrosschain']] * self.num_nodes) + self.nodes[0].loadwallet(node1_wallet) + self.nodes[1].loadwallet(node0_wallet) + + +if __name__ == '__main__': + WalletCrossChain().main() diff --git a/test/functional/wallet_listreceivedby.py b/test/functional/wallet_listreceivedby.py index 48b92796fc..db1d8eb54a 100755 --- a/test/functional/wallet_listreceivedby.py +++ b/test/functional/wallet_listreceivedby.py @@ -18,17 +18,12 @@ from test_framework.wallet_util import test_address class ReceivedByTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 - # Test deprecated exclude coinbase on second node - self.extra_args = [[], ["-deprecatedrpc=exclude_coinbase"]] def skip_test_if_missing_module(self): self.skip_if_no_wallet() self.skip_if_no_cli() def run_test(self): - # Generate block to get out of IBD - self.generate(self.nodes[0], 1) - # save the number of coinbase reward addresses so far num_cb_reward_addresses = len(self.nodes[1].listreceivedbyaddress(minconf=0, include_empty=True, include_watchonly=True)) @@ -134,6 +129,9 @@ class ReceivedByTest(BitcoinTestFramework): txid = self.nodes[0].sendtoaddress(addr, 0.1) self.sync_all() + # getreceivedbylabel returns an error if the wallet doesn't own the label + assert_raises_rpc_error(-4, "Label not found in wallet", self.nodes[0].getreceivedbylabel, "dummy") + # listreceivedbylabel should return received_by_label_json because of 0 confirmations assert_array_result(self.nodes[1].listreceivedbylabel(), {"label": label}, @@ -172,7 +170,7 @@ class ReceivedByTest(BitcoinTestFramework): address = self.nodes[0].getnewaddress(label) reward = Decimal("25") - self.generatetoaddress(self.nodes[0], 1, address, sync_fun=self.no_op) + self.generatetoaddress(self.nodes[0], 1, address) hash = self.nodes[0].getbestblockhash() self.log.info("getreceivedbyaddress returns nothing with defaults") @@ -212,7 +210,7 @@ class ReceivedByTest(BitcoinTestFramework): {"label": label, "amount": reward}) self.log.info("Generate 100 more blocks") - self.generate(self.nodes[0], COINBASE_MATURITY, sync_fun=self.no_op) + self.generate(self.nodes[0], COINBASE_MATURITY) self.log.info("getreceivedbyaddress returns reward with defaults") balance = self.nodes[0].getreceivedbyaddress(address) @@ -253,35 +251,6 @@ class ReceivedByTest(BitcoinTestFramework): {"label": label}, {}, True) - # Test exclude_coinbase - address2 = self.nodes[1].getnewaddress(label) - self.generatetoaddress(self.nodes[1], COINBASE_MATURITY + 1, address2, sync_fun=self.no_op) - - self.log.info("getreceivedbyaddress returns nothing when excluding coinbase") - balance = self.nodes[1].getreceivedbyaddress(address2) - assert_equal(balance, 0) - - self.log.info("getreceivedbylabel returns nothing when excluding coinbase") - balance = self.nodes[1].getreceivedbylabel("label") - assert_equal(balance, 0) - - self.log.info("listreceivedbyaddress does not include address when excluding coinbase") - assert_array_result(self.nodes[1].listreceivedbyaddress(), - {"address": address2}, - {}, True) - - self.log.info("listreceivedbylabel does not include label when excluding coinbase") - assert_array_result(self.nodes[1].listreceivedbylabel(), - {"label": label}, - {}, True) - - self.log.info("getreceivedbyaddress throws when setting include_immature_coinbase with deprecated exclude_coinbase") - assert_raises_rpc_error(-8, 'include_immature_coinbase is incompatible with deprecated exclude_coinbase', self.nodes[1].getreceivedbyaddress, address2, 1, True) - - - self.log.info("listreceivedbyaddress throws when setting include_immature_coinbase with deprecated exclude_coinbase") - assert_raises_rpc_error(-8, 'include_immature_coinbase is incompatible with deprecated exclude_coinbase', self.nodes[1].listreceivedbyaddress, 1, False, False, "", True) - if __name__ == '__main__': ReceivedByTest().main() diff --git a/test/functional/wallet_send.py b/test/functional/wallet_send.py index 86e36be8f7..07baa0595e 100755 --- a/test/functional/wallet_send.py +++ b/test/functional/wallet_send.py @@ -10,6 +10,10 @@ from itertools import product from test_framework.authproxy import JSONRPCException from test_framework.descriptors import descsum_create from test_framework.key import ECKey +from test_framework.messages import ( + ser_compact_size, + WITNESS_SCALE_FACTOR, +) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, @@ -488,8 +492,8 @@ class WalletSendTest(BitcoinTestFramework): self.nodes[1].createwallet("extfund") ext_fund = self.nodes[1].get_wallet_rpc("extfund") - # Make a weird but signable script. sh(pkh()) descriptor accomplishes this - desc = descsum_create("sh(pkh({}))".format(privkey)) + # Make a weird but signable script. sh(wsh(pkh())) descriptor accomplishes this + desc = descsum_create("sh(wsh(pkh({})))".format(privkey)) if self.options.descriptors: res = ext_fund.importdescriptors([{"desc": desc, "timestamp": "now"}]) else: @@ -507,7 +511,7 @@ class WalletSendTest(BitcoinTestFramework): self.test_send(from_wallet=ext_wallet, to_wallet=self.nodes[0], amount=15, inputs=[ext_utxo], add_inputs=True, psbt=True, include_watching=True, expect_error=(-4, "Insufficient funds")) # But funding should work when the solving data is provided - res = self.test_send(from_wallet=ext_wallet, to_wallet=self.nodes[0], amount=15, inputs=[ext_utxo], add_inputs=True, psbt=True, include_watching=True, solving_data={"pubkeys": [addr_info['pubkey']], "scripts": [addr_info["embedded"]["scriptPubKey"]]}) + res = self.test_send(from_wallet=ext_wallet, to_wallet=self.nodes[0], amount=15, inputs=[ext_utxo], add_inputs=True, psbt=True, include_watching=True, solving_data={"pubkeys": [addr_info['pubkey']], "scripts": [addr_info["embedded"]["scriptPubKey"], addr_info["embedded"]["embedded"]["scriptPubKey"]]}) signed = ext_wallet.walletprocesspsbt(res["psbt"]) signed = ext_fund.walletprocesspsbt(res["psbt"]) assert signed["complete"] @@ -526,10 +530,11 @@ class WalletSendTest(BitcoinTestFramework): break psbt_in = dec["inputs"][input_idx] # Calculate the input weight - # (prevout + sequence + length of scriptSig + 2 bytes buffer) * 4 + len of scriptwitness + # (prevout + sequence + length of scriptSig + scriptsig + 1 byte buffer) * WITNESS_SCALE_FACTOR + num scriptWitness stack items + (length of stack item + stack item) * N stack items + 1 byte buffer len_scriptsig = len(psbt_in["final_scriptSig"]["hex"]) // 2 if "final_scriptSig" in psbt_in else 0 - len_scriptwitness = len(psbt_in["final_scriptwitness"]["hex"]) // 2 if "final_scriptwitness" in psbt_in else 0 - input_weight = ((41 + len_scriptsig + 2) * 4) + len_scriptwitness + len_scriptsig += len(ser_compact_size(len_scriptsig)) + 1 + len_scriptwitness = (sum([(len(x) // 2) + len(ser_compact_size(len(x) // 2)) for x in psbt_in["final_scriptwitness"]]) + len(psbt_in["final_scriptwitness"]) + 1) if "final_scriptwitness" in psbt_in else 0 + input_weight = ((40 + len_scriptsig) * WITNESS_SCALE_FACTOR) + len_scriptwitness # Input weight error conditions assert_raises_rpc_error( diff --git a/test/functional/wallet_taproot.py b/test/functional/wallet_taproot.py index d3731b135a..a4d836c8fe 100755 --- a/test/functional/wallet_taproot.py +++ b/test/functional/wallet_taproot.py @@ -192,9 +192,9 @@ class WalletTaprootTest(BitcoinTestFramework): """Test generation and spending of P2TR address outputs.""" def set_test_params(self): - self.num_nodes = 3 + self.num_nodes = 2 self.setup_clean_chain = True - self.extra_args = [['-keypool=100'], ['-keypool=100'], ["-vbparams=taproot:1:1"]] + self.extra_args = [['-keypool=100'], ['-keypool=100']] self.supports_cli = False def skip_test_if_missing_module(self): @@ -243,15 +243,11 @@ class WalletTaprootTest(BitcoinTestFramework): assert_equal(len(rederive), 1) assert_equal(rederive[0], addr_g) - # tr descriptors can be imported regardless of Taproot status + # tr descriptors can be imported result = self.privs_tr_enabled.importdescriptors([{"desc": desc, "timestamp": "now"}]) assert(result[0]["success"]) result = self.pubs_tr_enabled.importdescriptors([{"desc": desc_pub, "timestamp": "now"}]) assert(result[0]["success"]) - result = self.privs_tr_disabled.importdescriptors([{"desc": desc, "timestamp": "now"}]) - assert result[0]["success"] - result = self.pubs_tr_disabled.importdescriptors([{"desc": desc_pub, "timestamp": "now"}]) - assert result[0]["success"] def do_test_sendtoaddress(self, comment, pattern, privmap, treefn, keys_pay, keys_change): self.log.info("Testing %s through sendtoaddress" % comment) @@ -328,12 +324,8 @@ class WalletTaprootTest(BitcoinTestFramework): self.log.info("Creating wallets...") self.nodes[0].createwallet(wallet_name="privs_tr_enabled", descriptors=True, blank=True) self.privs_tr_enabled = self.nodes[0].get_wallet_rpc("privs_tr_enabled") - self.nodes[2].createwallet(wallet_name="privs_tr_disabled", descriptors=True, blank=True) - self.privs_tr_disabled=self.nodes[2].get_wallet_rpc("privs_tr_disabled") self.nodes[0].createwallet(wallet_name="pubs_tr_enabled", descriptors=True, blank=True, disable_private_keys=True) self.pubs_tr_enabled = self.nodes[0].get_wallet_rpc("pubs_tr_enabled") - self.nodes[2].createwallet(wallet_name="pubs_tr_disabled", descriptors=True, blank=True, disable_private_keys=True) - self.pubs_tr_disabled=self.nodes[2].get_wallet_rpc("pubs_tr_disabled") self.nodes[0].createwallet(wallet_name="boring") self.nodes[0].createwallet(wallet_name="addr_gen", descriptors=True, disable_private_keys=True, blank=True) self.nodes[0].createwallet(wallet_name="rpc_online", descriptors=True, blank=True) @@ -449,11 +441,11 @@ class WalletTaprootTest(BitcoinTestFramework): self.log.info("Sending everything back...") - txid = self.rpc_online.sendtoaddress(address=self.boring.getnewaddress(), amount=self.rpc_online.getbalance(), subtractfeefromamount=True) + txid = self.rpc_online.sendall(recipients=[self.boring.getnewaddress()])["txid"] self.generatetoaddress(self.nodes[0], 1, self.boring.getnewaddress(), sync_fun=self.no_op) assert(self.rpc_online.gettransaction(txid)["confirmations"] > 0) - psbt = self.psbt_online.walletcreatefundedpsbt([], [{self.boring.getnewaddress(): self.psbt_online.getbalance()}], None, {"subtractFeeFromOutputs": [0]})['psbt'] + psbt = self.psbt_online.sendall(recipients=[self.boring.getnewaddress()], options={"psbt": True})["psbt"] res = self.psbt_offline.walletprocesspsbt(psbt) assert(res['complete']) rawtx = self.nodes[0].finalizepsbt(res['psbt'])['hex'] diff --git a/test/get_previous_releases.py b/test/get_previous_releases.py index 688ca58d7f..7b7cfbfef5 100755 --- a/test/get_previous_releases.py +++ b/test/get_previous_releases.py @@ -71,6 +71,15 @@ SHA256_SUMS = { "91b1e012975c5a363b5b5fcc81b5b7495e86ff703ec8262d4b9afcfec633c30d": "bitcoin-22.0-powerpc64le-linux-gnu.tar.gz", "9cc3a62c469fe57e11485fdd32c916f10ce7a2899299855a2e479256ff49ff3c": "bitcoin-22.0-riscv64-linux-gnu.tar.gz", "59ebd25dd82a51638b7a6bb914586201e67db67b919b2a1ff08925a7936d1b16": "bitcoin-22.0-x86_64-linux-gnu.tar.gz", + + "06f4c78271a77752ba5990d60d81b1751507f77efda1e5981b4e92fd4d9969fb": "bitcoin-23.0-aarch64-linux-gnu.tar.gz", + "952c574366aff76f6d6ad1c9ee45a361d64fa04155e973e926dfe7e26f9703a3": "bitcoin-23.0-arm-linux-gnueabihf.tar.gz", + "7c8bc63731aa872b7b334a8a7d96e33536ad77d49029bad179b09dca32cd77ac": "bitcoin-23.0-arm64-apple-darwin.tar.gz", + "2caa5898399e415f61d9af80a366a3008e5856efa15aaff74b88acf429674c99": "bitcoin-23.0-powerpc64-linux-gnu.tar.gz", + "217dd0469d0f4962d22818c368358575f6a0abcba8804807bb75325eb2f28b19": "bitcoin-23.0-powerpc64le-linux-gnu.tar.gz", + "078f96b1e92895009c798ab827fb3fde5f6719eee886bd0c0e93acab18ea4865": "bitcoin-23.0-riscv64-linux-gnu.tar.gz", + "c816780583009a9dad426dc0c183c89be9da98906e1e2c7ebae91041c1aaaaf3": "bitcoin-23.0-x86_64-apple-darwin.tar.gz", + "2cca490c1f2842884a3c5b0606f179f9f937177da4eadd628e3f7fd7e25d26d0": "bitcoin-23.0-x86_64-linux-gnu.tar.gz", } @@ -96,8 +105,11 @@ def download_binary(tag, args) -> int: if match: bin_path = 'bin/bitcoin-core-{}/test.{}'.format( match.group(1), match.group(2)) + platform = args.platform + if tag < "v23" and platform in ["x86_64-apple-darwin", "arm64-apple-darwin"]: + platform = "osx64" tarball = 'bitcoin-{tag}-{platform}.tar.gz'.format( - tag=tag[1:], platform=args.platform) + tag=tag[1:], platform=platform) tarballUrl = 'https://bitcoincore.org/{bin_path}/{tarball}'.format( bin_path=bin_path, tarball=tarball) @@ -201,8 +213,8 @@ def check_host(args) -> int: platforms = { 'aarch64-*-linux*': 'aarch64-linux-gnu', 'x86_64-*-linux*': 'x86_64-linux-gnu', - 'x86_64-apple-darwin*': 'osx64', - 'aarch64-apple-darwin*': 'osx64', + 'x86_64-apple-darwin*': 'x86_64-apple-darwin', + 'aarch64-apple-darwin*': 'arm64-apple-darwin', } args.platform = '' for pattern, target in platforms.items(): diff --git a/test/lint/README.md b/test/lint/README.md index f4165f908e..a23211a72b 100644 --- a/test/lint/README.md +++ b/test/lint/README.md @@ -39,6 +39,6 @@ To do so, add the upstream repository as remote: git remote add --fetch secp256k1 https://github.com/bitcoin-core/secp256k1.git ``` -lint-all.sh +all-lint.py =========== Calls other scripts with the `lint-` prefix. diff --git a/test/lint/all-lint.py b/test/lint/all-lint.py new file mode 100755 index 0000000000..34a7b9742a --- /dev/null +++ b/test/lint/all-lint.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2017-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# This script runs all test/lint/lint-* files, and fails if any exit +# with a non-zero status code. + +from glob import glob +from pathlib import Path +from subprocess import run + +exit_code = 0 +mod_path = Path(__file__).parent +for lint in glob(f"{mod_path}/lint-*.py"): + result = run([lint]) + if result.returncode != 0: + print(f"^---- failure generated from {lint.split('/')[-1]}") + exit_code |= result.returncode + +exit(exit_code) diff --git a/test/lint/extended-lint-all.sh b/test/lint/extended-lint-all.sh deleted file mode 100755 index be5d9db4a9..0000000000 --- a/test/lint/extended-lint-all.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2019-2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# This script runs all contrib/devtools/extended-lint-*.sh files, and fails if -# any exit with a non-zero status code. - -# This script is intentionally locale dependent by not setting "export LC_ALL=C" -# in order to allow for the executed lint scripts to opt in or opt out of locale -# dependence themselves. - -set -u - -SCRIPTDIR=$(dirname "${BASH_SOURCE[0]}") -LINTALL=$(basename "${BASH_SOURCE[0]}") - -for f in "${SCRIPTDIR}"/extended-lint-*.sh; do - if [ "$(basename "$f")" != "$LINTALL" ]; then - if ! "$f"; then - echo "^---- failure generated from $f" - exit 1 - fi - fi -done diff --git a/test/lint/extended-lint-cppcheck.sh b/test/lint/extended-lint-cppcheck.sh deleted file mode 100755 index 2af39ed60a..0000000000 --- a/test/lint/extended-lint-cppcheck.sh +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2019-2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# - -export LC_ALL=C - -ENABLED_CHECKS=( - "Class '.*' has a constructor with 1 argument that is not explicit." - "Struct '.*' has a constructor with 1 argument that is not explicit." -) - -IGNORED_WARNINGS=( - "src/arith_uint256.h:.* Class 'arith_uint256' has a constructor with 1 argument that is not explicit." - "src/arith_uint256.h:.* Class 'base_uint < 256 >' has a constructor with 1 argument that is not explicit." - "src/arith_uint256.h:.* Class 'base_uint' has a constructor with 1 argument that is not explicit." - "src/coins.h:.* Class 'CCoinsViewBacked' has a constructor with 1 argument that is not explicit." - "src/coins.h:.* Class 'CCoinsViewCache' has a constructor with 1 argument that is not explicit." - "src/coins.h:.* Class 'CCoinsViewCursor' has a constructor with 1 argument that is not explicit." - "src/net.h:.* Class 'CNetMessage' has a constructor with 1 argument that is not explicit." - "src/policy/feerate.h:.* Class 'CFeeRate' has a constructor with 1 argument that is not explicit." - "src/prevector.h:.* Class 'const_iterator' has a constructor with 1 argument that is not explicit." - "src/prevector.h:.* Class 'const_reverse_iterator' has a constructor with 1 argument that is not explicit." - "src/prevector.h:.* Class 'iterator' has a constructor with 1 argument that is not explicit." - "src/prevector.h:.* Class 'reverse_iterator' has a constructor with 1 argument that is not explicit." - "src/primitives/block.h:.* Class 'CBlock' has a constructor with 1 argument that is not explicit." - "src/primitives/transaction.h:.* Class 'CTransaction' has a constructor with 1 argument that is not explicit." - "src/protocol.h:.* Class 'CMessageHeader' has a constructor with 1 argument that is not explicit." - "src/qt/guiutil.h:.* Class 'ItemDelegate' has a constructor with 1 argument that is not explicit." - "src/rpc/util.h:.* Struct 'RPCResults' has a constructor with 1 argument that is not explicit." - "src/rpc/util.h:.* Struct 'UniValueType' has a constructor with 1 argument that is not explicit." - "src/rpc/util.h:.* style: Struct 'UniValueType' has a constructor with 1 argument that is not explicit." - "src/script/descriptor.cpp:.* Class 'AddressDescriptor' has a constructor with 1 argument that is not explicit." - "src/script/descriptor.cpp:.* Class 'ComboDescriptor' has a constructor with 1 argument that is not explicit." - "src/script/descriptor.cpp:.* Class 'ConstPubkeyProvider' has a constructor with 1 argument that is not explicit." - "src/script/descriptor.cpp:.* Class 'PKDescriptor' has a constructor with 1 argument that is not explicit." - "src/script/descriptor.cpp:.* Class 'PKHDescriptor' has a constructor with 1 argument that is not explicit." - "src/script/descriptor.cpp:.* Class 'RawDescriptor' has a constructor with 1 argument that is not explicit." - "src/script/descriptor.cpp:.* Class 'SHDescriptor' has a constructor with 1 argument that is not explicit." - "src/script/descriptor.cpp:.* Class 'WPKHDescriptor' has a constructor with 1 argument that is not explicit." - "src/script/descriptor.cpp:.* Class 'WSHDescriptor' has a constructor with 1 argument that is not explicit." - "src/script/script.h:.* Class 'CScript' has a constructor with 1 argument that is not explicit." - "src/script/standard.h:.* Class 'CScriptID' has a constructor with 1 argument that is not explicit." - "src/span.h:.* Class 'Span < const CRPCCommand >' has a constructor with 1 argument that is not explicit." - "src/span.h:.* Class 'Span < const char >' has a constructor with 1 argument that is not explicit." - "src/span.h:.* Class 'Span < const std :: vector <unsigned char > >' has a constructor with 1 argument that is not explicit." - "src/span.h:.* Class 'Span < const uint8_t >' has a constructor with 1 argument that is not explicit." - "src/span.h:.* Class 'Span' has a constructor with 1 argument that is not explicit." - "src/support/allocators/secure.h:.* Struct 'secure_allocator < char >' has a constructor with 1 argument that is not explicit." - "src/support/allocators/secure.h:.* Struct 'secure_allocator < RNGState >' has a constructor with 1 argument that is not explicit." - "src/support/allocators/secure.h:.* Struct 'secure_allocator < unsigned char >' has a constructor with 1 argument that is not explicit." - "src/support/allocators/zeroafterfree.h:.* Struct 'zero_after_free_allocator < char >' has a constructor with 1 argument that is not explicit." - "src/test/checkqueue_tests.cpp:.* Struct 'FailingCheck' has a constructor with 1 argument that is not explicit." - "src/test/checkqueue_tests.cpp:.* Struct 'MemoryCheck' has a constructor with 1 argument that is not explicit." - "src/test/checkqueue_tests.cpp:.* Struct 'UniqueCheck' has a constructor with 1 argument that is not explicit." - "src/test/fuzz/util.h:.* Class 'FuzzedFileProvider' has a constructor with 1 argument that is not explicit." - "src/test/fuzz/util.h:.* Class 'FuzzedAutoFileProvider' has a constructor with 1 argument that is not explicit." - "src/wallet/db.h:.* Class 'BerkeleyEnvironment' has a constructor with 1 argument that is not explicit." -) - -if ! command -v cppcheck > /dev/null; then - echo "Skipping cppcheck linting since cppcheck is not installed. Install by running \"apt install cppcheck\"" - exit 0 -fi - -function join_array { - local IFS="$1" - shift - echo "$*" -} - -ENABLED_CHECKS_REGEXP=$(join_array "|" "${ENABLED_CHECKS[@]}") -IGNORED_WARNINGS_REGEXP=$(join_array "|" "${IGNORED_WARNINGS[@]}") -WARNINGS=$(git ls-files -- "*.cpp" "*.h" ":(exclude)src/leveldb/" ":(exclude)src/crc32c/" ":(exclude)src/secp256k1/" ":(exclude)src/minisketch/" ":(exclude)src/univalue/" | \ - xargs cppcheck --enable=all -j "$(getconf _NPROCESSORS_ONLN)" --language=c++ --std=c++17 --template=gcc -D__cplusplus -DCLIENT_VERSION_BUILD -DCLIENT_VERSION_IS_RELEASE -DCLIENT_VERSION_MAJOR -DCLIENT_VERSION_MINOR -DCOPYRIGHT_YEAR -DDEBUG -I src/ -q 2>&1 | sort -u | \ - grep -E "${ENABLED_CHECKS_REGEXP}" | \ - grep -vE "${IGNORED_WARNINGS_REGEXP}") -if [[ ${WARNINGS} != "" ]]; then - echo "${WARNINGS}" - echo - echo "Advice not applicable in this specific case? Add an exception by updating" - echo "IGNORED_WARNINGS in $0" - # Uncomment to enforce the developer note policy "By default, declare single-argument constructors `explicit`" - # exit 1 -fi -exit 0 diff --git a/test/lint/lint-all.sh b/test/lint/lint-all.sh deleted file mode 100755 index fa37fa51c6..0000000000 --- a/test/lint/lint-all.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2017-2019 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# This script runs all contrib/devtools/lint-* files, and fails if any exit -# with a non-zero status code. - -# This script is intentionally locale dependent by not setting "export LC_ALL=C" -# in order to allow for the executed lint scripts to opt in or opt out of locale -# dependence themselves. - -set -u - -SCRIPTDIR=$(dirname "${BASH_SOURCE[0]}") -LINTALL=$(basename "${BASH_SOURCE[0]}") - -EXIT_CODE=0 - -for f in "${SCRIPTDIR}"/lint-*; do - if [ "$(basename "$f")" != "$LINTALL" ]; then - if ! "$f"; then - echo "^---- failure generated from $f" - EXIT_CODE=1 - fi - fi -done - -exit ${EXIT_CODE} diff --git a/test/lint/lint-assertions.py b/test/lint/lint-assertions.py new file mode 100755 index 0000000000..195ff33d11 --- /dev/null +++ b/test/lint/lint-assertions.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2018-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# Check for assertions with obvious side effects. + +import sys +import subprocess + + +def git_grep(params: [], error_msg: ""): + try: + output = subprocess.check_output(["git", "grep", *params], universal_newlines=True, encoding="utf8") + print(error_msg) + print(output) + return 1 + except subprocess.CalledProcessError as ex1: + if ex1.returncode > 1: + raise ex1 + return 0 + + +def main(): + # PRE31-C (SEI CERT C Coding Standard): + # "Assertions should not contain assignments, increment, or decrement operators." + exit_code = git_grep([ + "-E", + r"[^_]assert\(.*(\+\+|\-\-|[^=!<>]=[^=!<>]).*\);", + "--", + "*.cpp", + "*.h", + ], "Assertions should not have side effects:") + + # Aborting the whole process is undesirable for RPC code. So nonfatal + # checks should be used over assert. See: src/util/check.h + # src/rpc/server.cpp is excluded from this check since it's mostly meta-code. + exit_code |= git_grep([ + "-nE", + r"\<(A|a)ss(ume|ert) *\(.*\);", + "--", + "src/rpc/", + "src/wallet/rpc*", + ":(exclude)src/rpc/server.cpp", + ], "CHECK_NONFATAL(condition) or NONFATAL_UNREACHABLE should be used instead of assert for RPC code.") + + sys.exit(exit_code) + + +if __name__ == "__main__": + main() diff --git a/test/lint/lint-assertions.sh b/test/lint/lint-assertions.sh deleted file mode 100755 index 2860f5621b..0000000000 --- a/test/lint/lint-assertions.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Check for assertions with obvious side effects. - -export LC_ALL=C - -EXIT_CODE=0 - -# PRE31-C (SEI CERT C Coding Standard): -# "Assertions should not contain assignments, increment, or decrement operators." -OUTPUT=$(git grep -E '[^_]assert\(.*(\+\+|\-\-|[^=!<>]=[^=!<>]).*\);' -- "*.cpp" "*.h") -if [[ ${OUTPUT} != "" ]]; then - echo "Assertions should not have side effects:" - echo - echo "${OUTPUT}" - EXIT_CODE=1 -fi - -# Macro CHECK_NONFATAL(condition) should be used instead of assert for RPC code, where it -# is undesirable to crash the whole program. See: src/util/check.h -# src/rpc/server.cpp is excluded from this check since it's mostly meta-code. -OUTPUT=$(git grep -nE '\<(A|a)ssert *\(.*\);' -- "src/rpc/" "src/wallet/rpc*" ":(exclude)src/rpc/server.cpp") -if [[ ${OUTPUT} != "" ]]; then - echo "CHECK_NONFATAL(condition) should be used instead of assert for RPC code." - echo - echo "${OUTPUT}" - EXIT_CODE=1 -fi - -exit ${EXIT_CODE} diff --git a/test/lint/lint-circular-dependencies.py b/test/lint/lint-circular-dependencies.py new file mode 100755 index 0000000000..5d157eb4b1 --- /dev/null +++ b/test/lint/lint-circular-dependencies.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2020-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# Check for circular dependencies + +import os +import re +import subprocess +import sys + +EXPECTED_CIRCULAR_DEPENDENCIES = ( + "chainparamsbase -> util/system -> chainparamsbase", + "node/blockstorage -> validation -> node/blockstorage", + "policy/fees -> txmempool -> policy/fees", + "qt/addresstablemodel -> qt/walletmodel -> qt/addresstablemodel", + "qt/recentrequeststablemodel -> qt/walletmodel -> qt/recentrequeststablemodel", + "qt/sendcoinsdialog -> qt/walletmodel -> qt/sendcoinsdialog", + "qt/transactiontablemodel -> qt/walletmodel -> qt/transactiontablemodel", + "wallet/fees -> wallet/wallet -> wallet/fees", + "wallet/wallet -> wallet/walletdb -> wallet/wallet", + "kernel/coinstats -> validation -> kernel/coinstats", +) + +CODE_DIR = "src" + + +def main(): + circular_dependencies = [] + exit_code = 0 + + os.chdir(CODE_DIR) + files = subprocess.check_output( + ['git', 'ls-files', '--', '*.h', '*.cpp'], + universal_newlines=True, + ).splitlines() + + command = [sys.executable, "../contrib/devtools/circular-dependencies.py", *files] + dependencies_output = subprocess.run( + command, + stdout=subprocess.PIPE, + universal_newlines=True, + ) + + for dependency_str in dependencies_output.stdout.rstrip().split("\n"): + circular_dependencies.append( + re.sub("^Circular dependency: ", "", dependency_str) + ) + + # Check for an unexpected dependencies + for dependency in circular_dependencies: + if dependency not in EXPECTED_CIRCULAR_DEPENDENCIES: + exit_code = 1 + print( + f'A new circular dependency in the form of "{dependency}" appears to have been introduced.\n', + file=sys.stderr, + ) + + # Check for missing expected dependencies + for expected_dependency in EXPECTED_CIRCULAR_DEPENDENCIES: + if expected_dependency not in circular_dependencies: + exit_code = 1 + print( + f'Good job! The circular dependency "{expected_dependency}" is no longer present.', + ) + print( + f"Please remove it from EXPECTED_CIRCULAR_DEPENDENCIES in {__file__}", + ) + print( + "to make sure this circular dependency is not accidentally reintroduced.\n", + ) + + sys.exit(exit_code) + + +if __name__ == "__main__": + main() diff --git a/test/lint/lint-circular-dependencies.sh b/test/lint/lint-circular-dependencies.sh deleted file mode 100755 index 69185090d1..0000000000 --- a/test/lint/lint-circular-dependencies.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Check for circular dependencies - -export LC_ALL=C - -EXPECTED_CIRCULAR_DEPENDENCIES=( - "chainparamsbase -> util/system -> chainparamsbase" - "node/blockstorage -> validation -> node/blockstorage" - "index/blockfilterindex -> node/blockstorage -> validation -> index/blockfilterindex" - "index/base -> validation -> index/blockfilterindex -> index/base" - "index/coinstatsindex -> node/coinstats -> index/coinstatsindex" - "policy/fees -> txmempool -> policy/fees" - "qt/addresstablemodel -> qt/walletmodel -> qt/addresstablemodel" - "qt/recentrequeststablemodel -> qt/walletmodel -> qt/recentrequeststablemodel" - "qt/sendcoinsdialog -> qt/walletmodel -> qt/sendcoinsdialog" - "qt/transactiontablemodel -> qt/walletmodel -> qt/transactiontablemodel" - "wallet/fees -> wallet/wallet -> wallet/fees" - "wallet/wallet -> wallet/walletdb -> wallet/wallet" - "node/coinstats -> validation -> node/coinstats" -) - -EXIT_CODE=0 - -CIRCULAR_DEPENDENCIES=() - -IFS=$'\n' -for CIRC in $(cd src && ../contrib/devtools/circular-dependencies.py {*,*/*,*/*/*}.{h,cpp} | sed -e 's/^Circular dependency: //'); do - CIRCULAR_DEPENDENCIES+=( "$CIRC" ) - IS_EXPECTED_CIRC=0 - for EXPECTED_CIRC in "${EXPECTED_CIRCULAR_DEPENDENCIES[@]}"; do - if [[ "${CIRC}" == "${EXPECTED_CIRC}" ]]; then - IS_EXPECTED_CIRC=1 - break - fi - done - if [[ ${IS_EXPECTED_CIRC} == 0 ]]; then - echo "A new circular dependency in the form of \"${CIRC}\" appears to have been introduced." - echo - EXIT_CODE=1 - fi -done - -for EXPECTED_CIRC in "${EXPECTED_CIRCULAR_DEPENDENCIES[@]}"; do - IS_PRESENT_EXPECTED_CIRC=0 - for CIRC in "${CIRCULAR_DEPENDENCIES[@]}"; do - if [[ "${CIRC}" == "${EXPECTED_CIRC}" ]]; then - IS_PRESENT_EXPECTED_CIRC=1 - break - fi - done - if [[ ${IS_PRESENT_EXPECTED_CIRC} == 0 ]]; then - echo "Good job! The circular dependency \"${EXPECTED_CIRC}\" is no longer present." - echo "Please remove it from EXPECTED_CIRCULAR_DEPENDENCIES in $0" - echo "to make sure this circular dependency is not accidentally reintroduced." - echo - EXIT_CODE=1 - fi -done - -exit ${EXIT_CODE} diff --git a/test/lint/lint-files.py b/test/lint/lint-files.py index 3723e0ee6a..dbb51ce54e 100755 --- a/test/lint/lint-files.py +++ b/test/lint/lint-files.py @@ -11,19 +11,20 @@ import os import re import sys from subprocess import check_output -from typing import Optional, NoReturn +from typing import Dict, Optional, NoReturn CMD_TOP_LEVEL = ["git", "rev-parse", "--show-toplevel"] -CMD_ALL_FILES = "git ls-files -z --full-name" -CMD_SOURCE_FILES = 'git ls-files -z --full-name -- "*.[cC][pP][pP]" "*.[hH]" "*.[pP][yY]" "*.[sS][hH]"' -CMD_SHEBANG_FILES = "git grep --full-name --line-number -I '^#!'" +CMD_ALL_FILES = ["git", "ls-files", "-z", "--full-name", "--stage"] +CMD_SHEBANG_FILES = ["git", "grep", "--full-name", "--line-number", "-I", "^#!"] + +ALL_SOURCE_FILENAMES_REGEXP = r"^.*\.(cpp|h|py|sh)$" ALLOWED_FILENAME_REGEXP = "^[a-zA-Z0-9/_.@][a-zA-Z0-9/_.@-]*$" ALLOWED_SOURCE_FILENAME_REGEXP = "^[a-z0-9_./-]+$" ALLOWED_SOURCE_FILENAME_EXCEPTION_REGEXP = ( "^src/(secp256k1/|minisketch/|univalue/|test/fuzz/FuzzedDataProvider.h)" ) -ALLOWED_PERMISSION_NON_EXECUTABLES = 644 -ALLOWED_PERMISSION_EXECUTABLES = 755 +ALLOWED_PERMISSION_NON_EXECUTABLES = 0o644 +ALLOWED_PERMISSION_EXECUTABLES = 0o755 ALLOWED_EXECUTABLE_SHEBANG = { "py": [b"#!/usr/bin/env python3"], "sh": [b"#!/usr/bin/env bash", b"#!/bin/sh"], @@ -31,8 +32,15 @@ ALLOWED_EXECUTABLE_SHEBANG = { class FileMeta(object): - def __init__(self, file_path: str): - self.file_path = file_path + def __init__(self, file_spec: str): + '''Parse a `git ls files --stage` output line.''' + # 100755 5a150d5f8031fcd75e80a4dd9843afa33655f579 0 ci/test/00_setup_env.sh + meta, self.file_path = file_spec.split('\t', 2) + meta = meta.split() + # The octal file permission of the file. Internally, git only + # keeps an 'executable' bit, so this will always be 0o644 or 0o755. + self.permissions = int(meta[0], 8) & 0o7777 + # We don't currently care about the other fields @property def extension(self) -> Optional[str]: @@ -60,20 +68,24 @@ class FileMeta(object): except IndexError: return None - @property - def permissions(self) -> int: - """ - Returns the octal file permission of the file - """ - return int(oct(os.stat(self.file_path).st_mode)[-3:]) +def get_git_file_metadata() -> Dict[str, FileMeta]: + ''' + Return a dictionary mapping the name of all files in the repository to git tree metadata. + ''' + files_raw = check_output(CMD_ALL_FILES).decode("utf8").rstrip("\0").split("\0") + files = {} + for file_spec in files_raw: + meta = FileMeta(file_spec) + files[meta.file_path] = meta + return files -def check_all_filenames() -> int: +def check_all_filenames(files) -> int: """ Checks every file in the repository against an allowed regexp to make sure only lowercase or uppercase alphanumerics (a-zA-Z0-9), underscores (_), hyphens (-), at (@) and dots (.) are used in repository filenames. """ - filenames = check_output(CMD_ALL_FILES, shell=True).decode("utf8").rstrip("\0").split("\0") + filenames = files.keys() filename_regex = re.compile(ALLOWED_FILENAME_REGEXP) failed_tests = 0 for filename in filenames: @@ -85,14 +97,14 @@ def check_all_filenames() -> int: return failed_tests -def check_source_filenames() -> int: +def check_source_filenames(files) -> int: """ Checks only source files (*.cpp, *.h, *.py, *.sh) against a stricter allowed regexp to make sure only lowercase alphanumerics (a-z0-9), underscores (_), hyphens (-) and dots (.) are used in source code filenames. Additionally there is an exception regexp for directories or files which are excepted from matching this regexp. """ - filenames = check_output(CMD_SOURCE_FILES, shell=True).decode("utf8").rstrip("\0").split("\0") + filenames = [filename for filename in files.keys() if re.match(ALL_SOURCE_FILENAMES_REGEXP, filename, re.IGNORECASE)] filename_regex = re.compile(ALLOWED_SOURCE_FILENAME_REGEXP) filename_exception_regex = re.compile(ALLOWED_SOURCE_FILENAME_EXCEPTION_REGEXP) failed_tests = 0 @@ -105,16 +117,14 @@ def check_source_filenames() -> int: return failed_tests -def check_all_file_permissions() -> int: +def check_all_file_permissions(files) -> int: """ Checks all files in the repository match an allowed executable or non-executable file permission octal. Additionally checks that for executable files, the file contains a shebang line """ - filenames = check_output(CMD_ALL_FILES, shell=True).decode("utf8").rstrip("\0").split("\0") failed_tests = 0 - for filename in filenames: - file_meta = FileMeta(filename) + for filename, file_meta in files.items(): if file_meta.permissions == ALLOWED_PERMISSION_EXECUTABLES: with open(filename, "rb") as f: shebang = f.readline().rstrip(b"\n") @@ -122,7 +132,7 @@ def check_all_file_permissions() -> int: # For any file with executable permissions the first line must contain a shebang if not shebang.startswith(b"#!"): print( - f"""File "{filename}" has permission {ALLOWED_PERMISSION_EXECUTABLES} (executable) and is thus expected to contain a shebang '#!'. Add shebang or do "chmod {ALLOWED_PERMISSION_NON_EXECUTABLES} {filename}" to make it non-executable.""" + f"""File "{filename}" has permission {ALLOWED_PERMISSION_EXECUTABLES:03o} (executable) and is thus expected to contain a shebang '#!'. Add shebang or do "chmod {ALLOWED_PERMISSION_NON_EXECUTABLES:03o} {filename}" to make it non-executable.""" ) failed_tests += 1 @@ -145,18 +155,18 @@ def check_all_file_permissions() -> int: continue else: print( - f"""File "{filename}" has unexpected permission {file_meta.permissions}. Do "chmod {ALLOWED_PERMISSION_NON_EXECUTABLES} {filename}" (if non-executable) or "chmod {ALLOWED_PERMISSION_EXECUTABLES} {filename}" (if executable).""" + f"""File "{filename}" has unexpected permission {file_meta.permissions:03o}. Do "chmod {ALLOWED_PERMISSION_NON_EXECUTABLES:03o} {filename}" (if non-executable) or "chmod {ALLOWED_PERMISSION_EXECUTABLES:03o} {filename}" (if executable).""" ) failed_tests += 1 return failed_tests -def check_shebang_file_permissions() -> int: +def check_shebang_file_permissions(files_meta) -> int: """ Checks every file that contains a shebang line to ensure it has an executable permission """ - filenames = check_output(CMD_SHEBANG_FILES, shell=True).decode("utf8").strip().split("\n") + filenames = check_output(CMD_SHEBANG_FILES).decode("utf8").strip().split("\n") # The git grep command we use returns files which contain a shebang on any line within the file # so we need to filter the list to only files with the shebang on the first line @@ -164,7 +174,7 @@ def check_shebang_file_permissions() -> int: failed_tests = 0 for filename in filenames: - file_meta = FileMeta(filename) + file_meta = files_meta[filename] if file_meta.permissions != ALLOWED_PERMISSION_EXECUTABLES: # These file types are typically expected to be sourced and not executed directly if file_meta.full_extension in ["bash", "init", "openrc", "sh.in"]: @@ -178,7 +188,7 @@ def check_shebang_file_permissions() -> int: continue print( - f"""File "{filename}" contains a shebang line, but has the file permission {file_meta.permissions} instead of the expected executable permission {ALLOWED_PERMISSION_EXECUTABLES}. Do "chmod {ALLOWED_PERMISSION_EXECUTABLES} {filename}" (or remove the shebang line).""" + f"""File "{filename}" contains a shebang line, but has the file permission {file_meta.permissions:03o} instead of the expected executable permission {ALLOWED_PERMISSION_EXECUTABLES:03o}. Do "chmod {ALLOWED_PERMISSION_EXECUTABLES:03o} {filename}" (or remove the shebang line).""" ) failed_tests += 1 return failed_tests @@ -187,11 +197,14 @@ def check_shebang_file_permissions() -> int: def main() -> NoReturn: root_dir = check_output(CMD_TOP_LEVEL).decode("utf8").strip() os.chdir(root_dir) + + files = get_git_file_metadata() + failed_tests = 0 - failed_tests += check_all_filenames() - failed_tests += check_source_filenames() - failed_tests += check_all_file_permissions() - failed_tests += check_shebang_file_permissions() + failed_tests += check_all_filenames(files) + failed_tests += check_source_filenames(files) + failed_tests += check_all_file_permissions(files) + failed_tests += check_shebang_file_permissions(files) if failed_tests: print( diff --git a/test/lint/lint-format-strings.py b/test/lint/lint-format-strings.py new file mode 100755 index 0000000000..412cf86791 --- /dev/null +++ b/test/lint/lint-format-strings.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2018-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# + +""" +Lint format strings: This program checks that the number of arguments passed +to a variadic format string function matches the number of format specifiers +in the format string. +""" + +import subprocess +import re +import sys + +FUNCTION_NAMES_AND_NUMBER_OF_LEADING_ARGUMENTS = [ + 'FatalError,0', + 'fprintf,1', + 'tfm::format,1', # Assuming tfm::::format(std::ostream&, ... + 'LogConnectFailure,1', + 'LogPrint,1', + 'LogPrintf,0', + 'LogPrintLevel,2', + 'printf,0', + 'snprintf,2', + 'sprintf,1', + 'strprintf,0', + 'vfprintf,1', + 'vprintf,1', + 'vsnprintf,1', + 'vsprintf,1', + 'WalletLogPrintf,0', +] +RUN_LINT_FILE = 'test/lint/run-lint-format-strings.py' + +def check_doctest(): + command = [ + sys.executable, + '-m', + 'doctest', + RUN_LINT_FILE, + ] + try: + subprocess.run(command, check = True) + except subprocess.CalledProcessError: + sys.exit(1) + +def get_matching_files(function_name): + command = [ + 'git', + 'grep', + '--full-name', + '-l', + function_name, + '--', + '*.c', + '*.cpp', + '*.h', + ] + try: + return subprocess.check_output(command, stderr = subprocess.STDOUT).decode('utf-8').splitlines() + except subprocess.CalledProcessError as e: + if e.returncode > 1: # return code is 1 when match is empty + print(e.output.decode('utf-8'), end='') + sys.exit(1) + return [] + +def main(): + exit_code = 0 + check_doctest() + for s in FUNCTION_NAMES_AND_NUMBER_OF_LEADING_ARGUMENTS: + function_name, skip_arguments = s.split(',') + matching_files = get_matching_files(function_name) + + matching_files_filtered = [] + for matching_file in matching_files: + if not re.search('^src/(leveldb|secp256k1|minisketch|tinyformat|univalue|test/fuzz/strprintf.cpp)', matching_file): + matching_files_filtered.append(matching_file) + matching_files_filtered.sort() + + run_lint_args = [ + RUN_LINT_FILE, + '--skip-arguments', + skip_arguments, + function_name, + ] + run_lint_args.extend(matching_files_filtered) + + try: + subprocess.run(run_lint_args, check = True) + except subprocess.CalledProcessError: + exit_code = 1 + + sys.exit(exit_code) + +if __name__ == '__main__': + main() diff --git a/test/lint/lint-format-strings.sh b/test/lint/lint-format-strings.sh deleted file mode 100755 index 73730f16b3..0000000000 --- a/test/lint/lint-format-strings.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Lint format strings: This program checks that the number of arguments passed -# to a variadic format string function matches the number of format specifiers -# in the format string. - -export LC_ALL=C - -FUNCTION_NAMES_AND_NUMBER_OF_LEADING_ARGUMENTS=( - "FatalError,0" - "fprintf,1" - "tfm::format,1" # Assuming tfm::::format(std::ostream&, ... - "LogConnectFailure,1" - "LogPrint,1" - "LogPrintf,0" - "printf,0" - "snprintf,2" - "sprintf,1" - "strprintf,0" - "vfprintf,1" - "vprintf,1" - "vsnprintf,1" - "vsprintf,1" - "WalletLogPrintf,0" -) - -EXIT_CODE=0 -if ! python3 -m doctest "test/lint/run-lint-format-strings.py"; then - EXIT_CODE=1 -fi -for S in "${FUNCTION_NAMES_AND_NUMBER_OF_LEADING_ARGUMENTS[@]}"; do - IFS="," read -r FUNCTION_NAME SKIP_ARGUMENTS <<< "${S}" - for MATCHING_FILE in $(git grep --full-name -l "${FUNCTION_NAME}" -- "*.c" "*.cpp" "*.h" | sort | grep -vE "^src/(leveldb|secp256k1|minisketch|tinyformat|univalue|test/fuzz/strprintf.cpp)"); do - MATCHING_FILES+=("${MATCHING_FILE}") - done - if ! "test/lint/run-lint-format-strings.py" --skip-arguments "${SKIP_ARGUMENTS}" "${FUNCTION_NAME}" "${MATCHING_FILES[@]}"; then - EXIT_CODE=1 - fi -done -exit ${EXIT_CODE} diff --git a/test/lint/lint-git-commit-check.py b/test/lint/lint-git-commit-check.py new file mode 100755 index 0000000000..a1d03370e8 --- /dev/null +++ b/test/lint/lint-git-commit-check.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2020-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# Linter to check that commit messages have a new line before the body +# or no body at all + +import argparse +import os +import sys + +from subprocess import check_output + + +def parse_args(): + """Parse command line arguments.""" + parser = argparse.ArgumentParser( + description=""" + Linter to check that commit messages have a new line before + the body or no body at all. + """, + epilog=f""" + You can manually set the commit-range with the COMMIT_RANGE + environment variable (e.g. "COMMIT_RANGE='47ba2c3...ee50c9e' + {sys.argv[0]}"). Defaults to current merge base when neither + prev-commits nor the environment variable is set. + """) + + parser.add_argument("--prev-commits", "-p", required=False, help="The previous n commits to check") + + return parser.parse_args() + + +def main(): + args = parse_args() + exit_code = 0 + + if not os.getenv("COMMIT_RANGE"): + if args.prev_commits: + commit_range = "HEAD~" + args.prev_commits + "...HEAD" + else: + # This assumes that the target branch of the pull request will be master. + merge_base = check_output(["git", "merge-base", "HEAD", "master"], universal_newlines=True, encoding="utf8").rstrip("\n") + commit_range = merge_base + "..HEAD" + else: + commit_range = os.getenv("COMMIT_RANGE") + + commit_hashes = check_output(["git", "log", commit_range, "--format=%H"], universal_newlines=True, encoding="utf8").splitlines() + + for hash in commit_hashes: + commit_info = check_output(["git", "log", "--format=%B", "-n", "1", hash], universal_newlines=True, encoding="utf8").splitlines() + if len(commit_info) >= 2: + if commit_info[1]: + print(f"The subject line of commit hash {hash} is followed by a non-empty line. Subject lines should always be followed by a blank line.") + exit_code = 1 + + sys.exit(exit_code) + + +if __name__ == "__main__": + main() diff --git a/test/lint/lint-git-commit-check.sh b/test/lint/lint-git-commit-check.sh deleted file mode 100755 index f77373ed00..0000000000 --- a/test/lint/lint-git-commit-check.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2020-2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Linter to check that commit messages have a new line before the body -# or no body at all - -export LC_ALL=C - -EXIT_CODE=0 - -while getopts "?" opt; do - case $opt in - ?) - echo "Usage: $0 [N]" - echo " COMMIT_RANGE='<commit range>' $0" - echo " $0 -?" - echo "Checks unmerged commits, the previous N commits, or a commit range." - echo "COMMIT_RANGE='47ba2c3...ee50c9e' $0" - exit ${EXIT_CODE} - ;; - esac -done - -if [ -z "${COMMIT_RANGE}" ]; then - if [ -n "$1" ]; then - COMMIT_RANGE="HEAD~$1...HEAD" - else - # This assumes that the target branch of the pull request will be master. - MERGE_BASE=$(git merge-base HEAD master) - COMMIT_RANGE="$MERGE_BASE..HEAD" - fi -fi - -while IFS= read -r commit_hash || [[ -n "$commit_hash" ]]; do - n_line=0 - while IFS= read -r line || [[ -n "$line" ]]; do - n_line=$((n_line+1)) - length=${#line} - if [ $n_line -eq 2 ] && [ "$length" -ne 0 ]; then - echo "The subject line of commit hash ${commit_hash} is followed by a non-empty line. Subject lines should always be followed by a blank line." - EXIT_CODE=1 - fi - done < <(git log --format=%B -n 1 "$commit_hash") -done < <(git log "${COMMIT_RANGE}" --format=%H) - -exit ${EXIT_CODE} diff --git a/test/lint/lint-include-guards.py b/test/lint/lint-include-guards.py new file mode 100755 index 0000000000..86284517d5 --- /dev/null +++ b/test/lint/lint-include-guards.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2018-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" +Check include guards. +""" + +import re +import sys +from subprocess import check_output +from typing import List + + +HEADER_ID_PREFIX = 'BITCOIN_' +HEADER_ID_SUFFIX = '_H' + +EXCLUDE_FILES_WITH_PREFIX = ['src/crypto/ctaes', + 'src/leveldb', + 'src/crc32c', + 'src/secp256k1', + 'src/minisketch', + 'src/univalue', + 'src/tinyformat.h', + 'src/bench/nanobench.h', + 'src/test/fuzz/FuzzedDataProvider.h'] + + +def _get_header_file_lst() -> List[str]: + """ Helper function to get a list of header filepaths to be + checked for include guards. + """ + git_cmd_lst = ['git', 'ls-files', '--', '*.h'] + header_file_lst = check_output( + git_cmd_lst).decode('utf-8').splitlines() + + header_file_lst = [hf for hf in header_file_lst + if not any(ef in hf for ef + in EXCLUDE_FILES_WITH_PREFIX)] + + return header_file_lst + + +def _get_header_id(header_file: str) -> str: + """ Helper function to get the header id from a header file + string. + + eg: 'src/wallet/walletdb.h' -> 'BITCOIN_WALLET_WALLETDB_H' + + Args: + header_file: Filepath to header file. + + Returns: + The header id. + """ + header_id_base = header_file.split('/')[1:] + header_id_base = '_'.join(header_id_base) + header_id_base = header_id_base.replace('.h', '').replace('-', '_') + header_id_base = header_id_base.upper() + + header_id = f'{HEADER_ID_PREFIX}{header_id_base}{HEADER_ID_SUFFIX}' + + return header_id + + +def main(): + exit_code = 0 + + header_file_lst = _get_header_file_lst() + for header_file in header_file_lst: + header_id = _get_header_id(header_file) + + regex_pattern = f'^#(ifndef|define|endif //) {header_id}' + + with open(header_file, 'r', encoding='utf-8') as f: + header_file_contents = f.readlines() + + count = 0 + for header_file_contents_string in header_file_contents: + include_guard_lst = re.findall( + regex_pattern, header_file_contents_string) + + count += len(include_guard_lst) + + if count != 3: + print(f'{header_file} seems to be missing the expected ' + 'include guard:') + print(f' #ifndef {header_id}') + print(f' #define {header_id}') + print(' ...') + print(f' #endif // {header_id}\n') + exit_code = 1 + + sys.exit(exit_code) + + +if __name__ == '__main__': + main() diff --git a/test/lint/lint-include-guards.sh b/test/lint/lint-include-guards.sh deleted file mode 100755 index f14218aa74..0000000000 --- a/test/lint/lint-include-guards.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Check include guards. - -export LC_ALL=C -HEADER_ID_PREFIX="BITCOIN_" -HEADER_ID_SUFFIX="_H" - -REGEXP_EXCLUDE_FILES_WITH_PREFIX="src/(crypto/ctaes/|leveldb/|crc32c/|secp256k1/|minisketch/|test/fuzz/FuzzedDataProvider.h|tinyformat.h|bench/nanobench.h|univalue/)" - -EXIT_CODE=0 -for HEADER_FILE in $(git ls-files -- "*.h" | grep -vE "^${REGEXP_EXCLUDE_FILES_WITH_PREFIX}") -do - HEADER_ID_BASE=$(cut -f2- -d/ <<< "${HEADER_FILE}" | sed "s/\.h$//g" | tr / _ | tr - _ | tr "[:lower:]" "[:upper:]") - HEADER_ID="${HEADER_ID_PREFIX}${HEADER_ID_BASE}${HEADER_ID_SUFFIX}" - if [[ $(grep --count --extended-regexp "^#(ifndef|define|endif //) ${HEADER_ID}" "${HEADER_FILE}") != 3 ]]; then - echo "${HEADER_FILE} seems to be missing the expected include guard:" - echo " #ifndef ${HEADER_ID}" - echo " #define ${HEADER_ID}" - echo " ..." - echo " #endif // ${HEADER_ID}" - echo - EXIT_CODE=1 - fi -done -exit ${EXIT_CODE} diff --git a/test/lint/lint-includes.py b/test/lint/lint-includes.py new file mode 100755 index 0000000000..ae62994642 --- /dev/null +++ b/test/lint/lint-includes.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2018-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# Check for duplicate includes. +# Guard against accidental introduction of new Boost dependencies. +# Check includes: Check for duplicate includes. Enforce bracket syntax includes. + +import os +import re +import sys + +from subprocess import check_output, CalledProcessError + + +EXCLUDED_DIRS = ["src/leveldb/", + "src/crc32c/", + "src/secp256k1/", + "src/minisketch/", + "src/univalue/"] + +EXPECTED_BOOST_INCLUDES = ["boost/algorithm/string/replace.hpp", + "boost/date_time/posix_time/posix_time.hpp", + "boost/multi_index/hashed_index.hpp", + "boost/multi_index/ordered_index.hpp", + "boost/multi_index/sequenced_index.hpp", + "boost/multi_index_container.hpp", + "boost/process.hpp", + "boost/signals2/connection.hpp", + "boost/signals2/optional_last_value.hpp", + "boost/signals2/signal.hpp", + "boost/test/included/unit_test.hpp", + "boost/test/unit_test.hpp"] + + +def get_toplevel(): + return check_output(["git", "rev-parse", "--show-toplevel"], universal_newlines=True, encoding="utf8").rstrip("\n") + + +def list_files_by_suffix(suffixes): + exclude_args = [":(exclude)" + dir for dir in EXCLUDED_DIRS] + + files_list = check_output(["git", "ls-files", "src"] + exclude_args, universal_newlines=True, encoding="utf8").splitlines() + + return [file for file in files_list if file.endswith(suffixes)] + + +def find_duplicate_includes(include_list): + tempset = set() + duplicates = set() + + for inclusion in include_list: + if inclusion in tempset: + duplicates.add(inclusion) + else: + tempset.add(inclusion) + + return duplicates + + +def find_included_cpps(): + included_cpps = list() + + try: + included_cpps = check_output(["git", "grep", "-E", r"^#include [<\"][^>\"]+\.cpp[>\"]", "--", "*.cpp", "*.h"], universal_newlines=True, encoding="utf8").splitlines() + except CalledProcessError as e: + if e.returncode > 1: + raise e + + return included_cpps + + +def find_extra_boosts(): + included_boosts = list() + filtered_included_boost_set = set() + exclusion_set = set() + + try: + included_boosts = check_output(["git", "grep", "-E", r"^#include <boost/", "--", "*.cpp", "*.h"], universal_newlines=True, encoding="utf8").splitlines() + except CalledProcessError as e: + if e.returncode > 1: + raise e + + for boost in included_boosts: + filtered_included_boost_set.add(re.findall(r'(?<=\<).+?(?=\>)', boost)[0]) + + for expected_boost in EXPECTED_BOOST_INCLUDES: + for boost in filtered_included_boost_set: + if expected_boost in boost: + exclusion_set.add(boost) + + extra_boosts = set(filtered_included_boost_set.difference(exclusion_set)) + + return extra_boosts + + +def find_quote_syntax_inclusions(): + exclude_args = [":(exclude)" + dir for dir in EXCLUDED_DIRS] + quote_syntax_inclusions = list() + + try: + quote_syntax_inclusions = check_output(["git", "grep", r"^#include \"", "--", "*.cpp", "*.h"] + exclude_args, universal_newlines=True, encoding="utf8").splitlines() + except CalledProcessError as e: + if e.returncode > 1: + raise e + + return quote_syntax_inclusions + + +def main(): + exit_code = 0 + + os.chdir(get_toplevel()) + + # Check for duplicate includes + for filename in list_files_by_suffix((".cpp", ".h")): + with open(filename, "r", encoding="utf8") as file: + include_list = [line.rstrip("\n") for line in file if re.match(r"^#include", line)] + + duplicates = find_duplicate_includes(include_list) + + if duplicates: + print(f"Duplicate include(s) in {filename}:") + for duplicate in duplicates: + print(duplicate) + print("") + exit_code = 1 + + # Check if code includes .cpp-files + included_cpps = find_included_cpps() + + if included_cpps: + print("The following files #include .cpp files:") + for included_cpp in included_cpps: + print(included_cpp) + print("") + exit_code = 1 + + # Guard against accidental introduction of new Boost dependencies + extra_boosts = find_extra_boosts() + + if extra_boosts: + for boost in extra_boosts: + print(f"A new Boost dependency in the form of \"{boost}\" appears to have been introduced:") + print(check_output(["git", "grep", boost, "--", "*.cpp", "*.h"], universal_newlines=True, encoding="utf8")) + exit_code = 1 + + # Check if Boost dependencies are no longer used + for expected_boost in EXPECTED_BOOST_INCLUDES: + try: + check_output(["git", "grep", "-q", r"^#include <%s>" % expected_boost, "--", "*.cpp", "*.h"], universal_newlines=True, encoding="utf8") + except CalledProcessError as e: + if e.returncode > 1: + raise e + else: + print(f"Good job! The Boost dependency \"{expected_boost}\" is no longer used. " + "Please remove it from EXPECTED_BOOST_INCLUDES in test/lint/lint-includes.py " + "to make sure this dependency is not accidentally reintroduced.\n") + exit_code = 1 + + # Enforce bracket syntax includes + quote_syntax_inclusions = find_quote_syntax_inclusions() + + if quote_syntax_inclusions: + print("Please use bracket syntax includes (\"#include <foo.h>\") instead of quote syntax includes:") + for quote_syntax_inclusion in quote_syntax_inclusions: + print(quote_syntax_inclusion) + exit_code = 1 + + sys.exit(exit_code) + + +if __name__ == "__main__": + main() diff --git a/test/lint/lint-includes.sh b/test/lint/lint-includes.sh deleted file mode 100755 index 9e72831ee9..0000000000 --- a/test/lint/lint-includes.sh +++ /dev/null @@ -1,103 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Check for duplicate includes. -# Guard against accidental introduction of new Boost dependencies. -# Check includes: Check for duplicate includes. Enforce bracket syntax includes. - -export LC_ALL=C -IGNORE_REGEXP="/(leveldb|secp256k1|minisketch|univalue|crc32c)/" - -# cd to root folder of git repo for git ls-files to work properly -cd "$(dirname "$0")/../.." || exit 1 - -filter_suffix() { - git ls-files | grep -E "^src/.*\.${1}"'$' | grep -Ev "${IGNORE_REGEXP}" -} - -EXIT_CODE=0 - -for HEADER_FILE in $(filter_suffix h); do - DUPLICATE_INCLUDES_IN_HEADER_FILE=$(grep -E "^#include " < "${HEADER_FILE}" | sort | uniq -d) - if [[ ${DUPLICATE_INCLUDES_IN_HEADER_FILE} != "" ]]; then - echo "Duplicate include(s) in ${HEADER_FILE}:" - echo "${DUPLICATE_INCLUDES_IN_HEADER_FILE}" - echo - EXIT_CODE=1 - fi -done - -for CPP_FILE in $(filter_suffix cpp); do - DUPLICATE_INCLUDES_IN_CPP_FILE=$(grep -E "^#include " < "${CPP_FILE}" | sort | uniq -d) - if [[ ${DUPLICATE_INCLUDES_IN_CPP_FILE} != "" ]]; then - echo "Duplicate include(s) in ${CPP_FILE}:" - echo "${DUPLICATE_INCLUDES_IN_CPP_FILE}" - echo - EXIT_CODE=1 - fi -done - -INCLUDED_CPP_FILES=$(git grep -E "^#include [<\"][^>\"]+\.cpp[>\"]" -- "*.cpp" "*.h") -if [[ ${INCLUDED_CPP_FILES} != "" ]]; then - echo "The following files #include .cpp files:" - echo "${INCLUDED_CPP_FILES}" - echo - EXIT_CODE=1 -fi - -EXPECTED_BOOST_INCLUDES=( - boost/algorithm/string.hpp - boost/algorithm/string/classification.hpp - boost/algorithm/string/replace.hpp - boost/algorithm/string/split.hpp - boost/date_time/posix_time/posix_time.hpp - boost/multi_index/hashed_index.hpp - boost/multi_index/ordered_index.hpp - boost/multi_index/sequenced_index.hpp - boost/multi_index_container.hpp - boost/process.hpp - boost/signals2/connection.hpp - boost/signals2/optional_last_value.hpp - boost/signals2/signal.hpp - boost/test/included/unit_test.hpp - boost/test/unit_test.hpp -) - -for BOOST_INCLUDE in $(git grep '^#include <boost/' -- "*.cpp" "*.h" | cut -f2 -d: | cut -f2 -d'<' | cut -f1 -d'>' | sort -u); do - IS_EXPECTED_INCLUDE=0 - for EXPECTED_BOOST_INCLUDE in "${EXPECTED_BOOST_INCLUDES[@]}"; do - if [[ "${BOOST_INCLUDE}" == "${EXPECTED_BOOST_INCLUDE}" ]]; then - IS_EXPECTED_INCLUDE=1 - break - fi - done - if [[ ${IS_EXPECTED_INCLUDE} == 0 ]]; then - EXIT_CODE=1 - echo "A new Boost dependency in the form of \"${BOOST_INCLUDE}\" appears to have been introduced:" - git grep "${BOOST_INCLUDE}" -- "*.cpp" "*.h" - echo - fi -done - -for EXPECTED_BOOST_INCLUDE in "${EXPECTED_BOOST_INCLUDES[@]}"; do - if ! git grep -q "^#include <${EXPECTED_BOOST_INCLUDE}>" -- "*.cpp" "*.h"; then - echo "Good job! The Boost dependency \"${EXPECTED_BOOST_INCLUDE}\" is no longer used." - echo "Please remove it from EXPECTED_BOOST_INCLUDES in $0" - echo "to make sure this dependency is not accidentally reintroduced." - echo - EXIT_CODE=1 - fi -done - -QUOTE_SYNTAX_INCLUDES=$(git grep '^#include "' -- "*.cpp" "*.h" | grep -Ev "${IGNORE_REGEXP}") -if [[ ${QUOTE_SYNTAX_INCLUDES} != "" ]]; then - echo "Please use bracket syntax includes (\"#include <foo.h>\") instead of quote syntax includes:" - echo "${QUOTE_SYNTAX_INCLUDES}" - echo - EXIT_CODE=1 -fi - -exit ${EXIT_CODE} diff --git a/test/lint/lint-locale-dependence.py b/test/lint/lint-locale-dependence.py new file mode 100755 index 0000000000..9b2cf4587a --- /dev/null +++ b/test/lint/lint-locale-dependence.py @@ -0,0 +1,261 @@ +#!/usr/bin/env python3 +# Copyright (c) 2018-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# Be aware that bitcoind and bitcoin-qt differ in terms of localization: Qt +# opts in to POSIX localization by running setlocale(LC_ALL, "") on startup, +# whereas no such call is made in bitcoind. +# +# Qt runs setlocale(LC_ALL, "") on initialization. This installs the locale +# specified by the user's LC_ALL (or LC_*) environment variable as the new +# C locale. +# +# In contrast, bitcoind does not opt in to localization -- no call to +# setlocale(LC_ALL, "") is made and the environment variables LC_* are +# thus ignored. +# +# This results in situations where bitcoind is guaranteed to be running +# with the classic locale ("C") whereas the locale of bitcoin-qt will vary +# depending on the user's environment variables. +# +# An example: Assuming the environment variable LC_ALL=de_DE then the +# call std::to_string(1.23) will return "1.230000" in bitcoind but +# "1,230000" in bitcoin-qt. +# +# From the Qt documentation: +# "On Unix/Linux Qt is configured to use the system locale settings by default. +# This can cause a conflict when using POSIX functions, for instance, when +# converting between data types such as floats and strings, since the notation +# may differ between locales. To get around this problem, call the POSIX function +# setlocale(LC_NUMERIC,"C") right after initializing QApplication, QGuiApplication +# or QCoreApplication to reset the locale that is used for number formatting to +# "C"-locale." +# +# See https://doc.qt.io/qt-5/qcoreapplication.html#locale-settings and +# https://stackoverflow.com/a/34878283 for more details. +# +# TODO: Reduce KNOWN_VIOLATIONS by replacing uses of locale dependent snprintf with strprintf. + +import re +import sys + +from subprocess import check_output, CalledProcessError + + +KNOWN_VIOLATIONS = [ + "src/dbwrapper.cpp:.*vsnprintf", + "src/test/dbwrapper_tests.cpp:.*snprintf", + "src/test/fuzz/locale.cpp:.*setlocale", + "src/test/fuzz/string.cpp:.*strtol", + "src/test/fuzz/string.cpp:.*strtoul", + "src/test/util_tests.cpp:.*strtoll", + "src/wallet/bdb.cpp:.*DbEnv::strerror", # False positive + "src/util/syserror.cpp:.*strerror", # Outside this function use `SysErrorString` +] + +REGEXP_EXTERNAL_DEPENDENCIES_EXCLUSIONS = [ + "src/crypto/ctaes/", + "src/leveldb/", + "src/secp256k1/", + "src/minisketch/", + "src/tinyformat.h", + "src/univalue/" +] + +LOCALE_DEPENDENT_FUNCTIONS = [ + "alphasort", # LC_COLLATE (via strcoll) + "asctime", # LC_TIME (directly) + "asprintf", # (via vasprintf) + "atof", # LC_NUMERIC (via strtod) + "atoi", # LC_NUMERIC (via strtol) + "atol", # LC_NUMERIC (via strtol) + "atoll", # (via strtoll) + "atoq", + "btowc", # LC_CTYPE (directly) + "ctime", # (via asctime or localtime) + "dprintf", # (via vdprintf) + "fgetwc", + "fgetws", + "fold_case", # boost::locale::fold_case + "fprintf", # (via vfprintf) + "fputwc", + "fputws", + "fscanf", # (via __vfscanf) + "fwprintf", # (via __vfwprintf) + "getdate", # via __getdate_r => isspace // __localtime_r + "getwc", + "getwchar", + "is_digit", # boost::algorithm::is_digit + "is_space", # boost::algorithm::is_space + "isalnum", # LC_CTYPE + "isalpha", # LC_CTYPE + "isblank", # LC_CTYPE + "iscntrl", # LC_CTYPE + "isctype", # LC_CTYPE + "isdigit", # LC_CTYPE + "isgraph", # LC_CTYPE + "islower", # LC_CTYPE + "isprint", # LC_CTYPE + "ispunct", # LC_CTYPE + "isspace", # LC_CTYPE + "isupper", # LC_CTYPE + "iswalnum", # LC_CTYPE + "iswalpha", # LC_CTYPE + "iswblank", # LC_CTYPE + "iswcntrl", # LC_CTYPE + "iswctype", # LC_CTYPE + "iswdigit", # LC_CTYPE + "iswgraph", # LC_CTYPE + "iswlower", # LC_CTYPE + "iswprint", # LC_CTYPE + "iswpunct", # LC_CTYPE + "iswspace", # LC_CTYPE + "iswupper", # LC_CTYPE + "iswxdigit", # LC_CTYPE + "isxdigit", # LC_CTYPE + "localeconv", # LC_NUMERIC + LC_MONETARY + "mblen", # LC_CTYPE + "mbrlen", + "mbrtowc", + "mbsinit", + "mbsnrtowcs", + "mbsrtowcs", + "mbstowcs", # LC_CTYPE + "mbtowc", # LC_CTYPE + "mktime", + "normalize", # boost::locale::normalize + "printf", # LC_NUMERIC + "putwc", + "putwchar", + "scanf", # LC_NUMERIC + "setlocale", + "snprintf", + "sprintf", + "sscanf", + "std::locale::global", + "std::to_string", + "stod", + "stof", + "stoi", + "stol", + "stold", + "stoll", + "stoul", + "stoull", + "strcasecmp", + "strcasestr", + "strcoll", # LC_COLLATE + "strerror", + "strfmon", + "strftime", # LC_TIME + "strncasecmp", + "strptime", + "strtod", # LC_NUMERIC + "strtof", + "strtoimax", + "strtol", # LC_NUMERIC + "strtold", + "strtoll", + "strtoq", + "strtoul", # LC_NUMERIC + "strtoull", + "strtoumax", + "strtouq", + "strxfrm", # LC_COLLATE + "swprintf", + "to_lower", # boost::locale::to_lower + "to_title", # boost::locale::to_title + "to_upper", # boost::locale::to_upper + "tolower", # LC_CTYPE + "toupper", # LC_CTYPE + "towctrans", + "towlower", # LC_CTYPE + "towupper", # LC_CTYPE + "trim", # boost::algorithm::trim + "trim_left", # boost::algorithm::trim_left + "trim_right", # boost::algorithm::trim_right + "ungetwc", + "vasprintf", + "vdprintf", + "versionsort", + "vfprintf", + "vfscanf", + "vfwprintf", + "vprintf", + "vscanf", + "vsnprintf", + "vsprintf", + "vsscanf", + "vswprintf", + "vwprintf", + "wcrtomb", + "wcscasecmp", + "wcscoll", # LC_COLLATE + "wcsftime", # LC_TIME + "wcsncasecmp", + "wcsnrtombs", + "wcsrtombs", + "wcstod", # LC_NUMERIC + "wcstof", + "wcstoimax", + "wcstol", # LC_NUMERIC + "wcstold", + "wcstoll", + "wcstombs", # LC_CTYPE + "wcstoul", # LC_NUMERIC + "wcstoull", + "wcstoumax", + "wcswidth", + "wcsxfrm", # LC_COLLATE + "wctob", + "wctomb", # LC_CTYPE + "wctrans", + "wctype", + "wcwidth", + "wprintf" +] + + +def find_locale_dependent_function_uses(): + regexp_locale_dependent_functions = "|".join(LOCALE_DEPENDENT_FUNCTIONS) + exclude_args = [":(exclude)" + excl for excl in REGEXP_EXTERNAL_DEPENDENCIES_EXCLUSIONS] + git_grep_command = ["git", "grep", "-E", "[^a-zA-Z0-9_\\`'\"<>](" + regexp_locale_dependent_functions + ")(_r|_s)?[^a-zA-Z0-9_\\`'\"<>]", "--", "*.cpp", "*.h"] + exclude_args + git_grep_output = list() + + try: + git_grep_output = check_output(git_grep_command, universal_newlines=True, encoding="utf8").splitlines() + except CalledProcessError as e: + if e.returncode > 1: + raise e + + return git_grep_output + + +def main(): + exit_code = 0 + + regexp_ignore_known_violations = "|".join(KNOWN_VIOLATIONS) + git_grep_output = find_locale_dependent_function_uses() + + for locale_dependent_function in LOCALE_DEPENDENT_FUNCTIONS: + matches = [line for line in git_grep_output + if re.search("[^a-zA-Z0-9_\\`'\"<>]" + locale_dependent_function + "(_r|_s)?[^a-zA-Z0-9_\\`'\"<>]", line) + and not re.search("\\.(c|cpp|h):\\s*(//|\\*|/\\*|\").*" + locale_dependent_function, line) + and not re.search(regexp_ignore_known_violations, line)] + if matches: + print(f"The locale dependent function {locale_dependent_function}(...) appears to be used:") + for match in matches: + print(match) + print("") + exit_code = 1 + + if exit_code == 1: + print("Unnecessary locale depedence can cause bugs that are very tricky to isolate and fix. Please avoid using locale dependent functions if possible.\n") + print(f"Advice not applicable in this specific case? Add an exception by updating the ignore list in {sys.argv[0]}") + + sys.exit(exit_code) + + +if __name__ == "__main__": + main() diff --git a/test/lint/lint-locale-dependence.sh b/test/lint/lint-locale-dependence.sh deleted file mode 100755 index 7d608eed6a..0000000000 --- a/test/lint/lint-locale-dependence.sh +++ /dev/null @@ -1,241 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2018-2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C - -# Be aware that bitcoind and bitcoin-qt differ in terms of localization: Qt -# opts in to POSIX localization by running setlocale(LC_ALL, "") on startup, -# whereas no such call is made in bitcoind. -# -# Qt runs setlocale(LC_ALL, "") on initialization. This installs the locale -# specified by the user's LC_ALL (or LC_*) environment variable as the new -# C locale. -# -# In contrast, bitcoind does not opt in to localization -- no call to -# setlocale(LC_ALL, "") is made and the environment variables LC_* are -# thus ignored. -# -# This results in situations where bitcoind is guaranteed to be running -# with the classic locale ("C") whereas the locale of bitcoin-qt will vary -# depending on the user's environment variables. -# -# An example: Assuming the environment variable LC_ALL=de_DE then the -# call std::to_string(1.23) will return "1.230000" in bitcoind but -# "1,230000" in bitcoin-qt. -# -# From the Qt documentation: -# "On Unix/Linux Qt is configured to use the system locale settings by default. -# This can cause a conflict when using POSIX functions, for instance, when -# converting between data types such as floats and strings, since the notation -# may differ between locales. To get around this problem, call the POSIX function -# setlocale(LC_NUMERIC,"C") right after initializing QApplication, QGuiApplication -# or QCoreApplication to reset the locale that is used for number formatting to -# "C"-locale." -# -# See https://doc.qt.io/qt-5/qcoreapplication.html#locale-settings and -# https://stackoverflow.com/a/34878283 for more details. - -# TODO: Reduce KNOWN_VIOLATIONS by replacing uses of locale dependent snprintf with strprintf. -KNOWN_VIOLATIONS=( - "src/dbwrapper.cpp:.*vsnprintf" - "src/test/dbwrapper_tests.cpp:.*snprintf" - "src/test/fuzz/locale.cpp" - "src/test/fuzz/string.cpp" - "src/test/util_tests.cpp" -) - -REGEXP_IGNORE_EXTERNAL_DEPENDENCIES="^src/(crypto/ctaes/|leveldb/|secp256k1/|minisketch/|tinyformat.h|univalue/)" - -LOCALE_DEPENDENT_FUNCTIONS=( - alphasort # LC_COLLATE (via strcoll) - asctime # LC_TIME (directly) - asprintf # (via vasprintf) - atof # LC_NUMERIC (via strtod) - atoi # LC_NUMERIC (via strtol) - atol # LC_NUMERIC (via strtol) - atoll # (via strtoll) - atoq - btowc # LC_CTYPE (directly) - ctime # (via asctime or localtime) - dprintf # (via vdprintf) - fgetwc - fgetws - fold_case # boost::locale::fold_case - fprintf # (via vfprintf) - fputwc - fputws - fscanf # (via __vfscanf) - fwprintf # (via __vfwprintf) - getdate # via __getdate_r => isspace // __localtime_r - getwc - getwchar - is_digit # boost::algorithm::is_digit - is_space # boost::algorithm::is_space - isalnum # LC_CTYPE - isalpha # LC_CTYPE - isblank # LC_CTYPE - iscntrl # LC_CTYPE - isctype # LC_CTYPE - isdigit # LC_CTYPE - isgraph # LC_CTYPE - islower # LC_CTYPE - isprint # LC_CTYPE - ispunct # LC_CTYPE - isspace # LC_CTYPE - isupper # LC_CTYPE - iswalnum # LC_CTYPE - iswalpha # LC_CTYPE - iswblank # LC_CTYPE - iswcntrl # LC_CTYPE - iswctype # LC_CTYPE - iswdigit # LC_CTYPE - iswgraph # LC_CTYPE - iswlower # LC_CTYPE - iswprint # LC_CTYPE - iswpunct # LC_CTYPE - iswspace # LC_CTYPE - iswupper # LC_CTYPE - iswxdigit # LC_CTYPE - isxdigit # LC_CTYPE - localeconv # LC_NUMERIC + LC_MONETARY - mblen # LC_CTYPE - mbrlen - mbrtowc - mbsinit - mbsnrtowcs - mbsrtowcs - mbstowcs # LC_CTYPE - mbtowc # LC_CTYPE - mktime - normalize # boost::locale::normalize - printf # LC_NUMERIC - putwc - putwchar - scanf # LC_NUMERIC - setlocale - snprintf - sprintf - sscanf - std::locale::global - std::to_string - stod - stof - stoi - stol - stold - stoll - stoul - stoull - strcasecmp - strcasestr - strcoll # LC_COLLATE -# strerror - strfmon - strftime # LC_TIME - strncasecmp - strptime - strtod # LC_NUMERIC - strtof - strtoimax - strtol # LC_NUMERIC - strtold - strtoll - strtoq - strtoul # LC_NUMERIC - strtoull - strtoumax - strtouq - strxfrm # LC_COLLATE - swprintf - to_lower # boost::locale::to_lower - to_title # boost::locale::to_title - to_upper # boost::locale::to_upper - tolower # LC_CTYPE - toupper # LC_CTYPE - towctrans - towlower # LC_CTYPE - towupper # LC_CTYPE - trim # boost::algorithm::trim - trim_left # boost::algorithm::trim_left - trim_right # boost::algorithm::trim_right - ungetwc - vasprintf - vdprintf - versionsort - vfprintf - vfscanf - vfwprintf - vprintf - vscanf - vsnprintf - vsprintf - vsscanf - vswprintf - vwprintf - wcrtomb - wcscasecmp - wcscoll # LC_COLLATE - wcsftime # LC_TIME - wcsncasecmp - wcsnrtombs - wcsrtombs - wcstod # LC_NUMERIC - wcstof - wcstoimax - wcstol # LC_NUMERIC - wcstold - wcstoll - wcstombs # LC_CTYPE - wcstoul # LC_NUMERIC - wcstoull - wcstoumax - wcswidth - wcsxfrm # LC_COLLATE - wctob - wctomb # LC_CTYPE - wctrans - wctype - wcwidth - wprintf -) - -function join_array { - local IFS="$1" - shift - echo "$*" -} - -REGEXP_IGNORE_KNOWN_VIOLATIONS=$(join_array "|" "${KNOWN_VIOLATIONS[@]}") - -# Invoke "git grep" only once in order to minimize run-time -REGEXP_LOCALE_DEPENDENT_FUNCTIONS=$(join_array "|" "${LOCALE_DEPENDENT_FUNCTIONS[@]}") -GIT_GREP_OUTPUT=$(git grep -E "[^a-zA-Z0-9_\`'\"<>](${REGEXP_LOCALE_DEPENDENT_FUNCTIONS}(_r|_s)?)[^a-zA-Z0-9_\`'\"<>]" -- "*.cpp" "*.h") - -EXIT_CODE=0 -for LOCALE_DEPENDENT_FUNCTION in "${LOCALE_DEPENDENT_FUNCTIONS[@]}"; do - MATCHES=$(grep -E "[^a-zA-Z0-9_\`'\"<>]${LOCALE_DEPENDENT_FUNCTION}(_r|_s)?[^a-zA-Z0-9_\`'\"<>]" <<< "${GIT_GREP_OUTPUT}" | \ - grep -vE "\.(c|cpp|h):\s*(//|\*|/\*|\").*${LOCALE_DEPENDENT_FUNCTION}") - if [[ ${REGEXP_IGNORE_EXTERNAL_DEPENDENCIES} != "" ]]; then - MATCHES=$(grep -vE "${REGEXP_IGNORE_EXTERNAL_DEPENDENCIES}" <<< "${MATCHES}") - fi - if [[ ${REGEXP_IGNORE_KNOWN_VIOLATIONS} != "" ]]; then - MATCHES=$(grep -vE "${REGEXP_IGNORE_KNOWN_VIOLATIONS}" <<< "${MATCHES}") - fi - if [[ ${MATCHES} != "" ]]; then - echo "The locale dependent function ${LOCALE_DEPENDENT_FUNCTION}(...) appears to be used:" - echo "${MATCHES}" - echo - EXIT_CODE=1 - fi -done -if [[ ${EXIT_CODE} != 0 ]]; then - echo "Unnecessary locale dependence can cause bugs that are very" - echo "tricky to isolate and fix. Please avoid using locale dependent" - echo "functions if possible." - echo - echo "Advice not applicable in this specific case? Add an exception" - echo "by updating the ignore list in $0" -fi -exit ${EXIT_CODE} diff --git a/test/lint/lint-logs.py b/test/lint/lint-logs.py new file mode 100755 index 0000000000..de53729b4e --- /dev/null +++ b/test/lint/lint-logs.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2018-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# Check that all logs are terminated with '\n' +# +# Some logs are continued over multiple lines. They should be explicitly +# commented with /* Continued */ + +import re +import sys + +from subprocess import check_output + + +def main(): + logs_list = check_output(["git", "grep", "--extended-regexp", r"(LogPrintLevel|LogPrintf?)\(", "--", "*.cpp"], universal_newlines=True, encoding="utf8").splitlines() + + unterminated_logs = [line for line in logs_list if not re.search(r'(\\n"|/\* Continued \*/)', line)] + + if unterminated_logs != []: + print("All calls to LogPrintf(), LogPrint(), LogPrintLevel(), and WalletLogPrintf() should be terminated with \"\\n\".") + print("") + + for line in unterminated_logs: + print(line) + + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/test/lint/lint-logs.sh b/test/lint/lint-logs.sh deleted file mode 100755 index 6d5165f649..0000000000 --- a/test/lint/lint-logs.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Check that all logs are terminated with '\n' -# -# Some logs are continued over multiple lines. They should be explicitly -# commented with /* Continued */ -# -# There are some instances of LogPrintf() in comments. Those can be -# ignored - -export LC_ALL=C -UNTERMINATED_LOGS=$(git grep --extended-regexp "LogPrintf?\(" -- "*.cpp" | \ - grep -v '\\n"' | \ - grep -v '\.\.\.' | \ - grep -v "/\* Continued \*/" | \ - grep -v "LogPrint()" | \ - grep -v "LogPrintf()") -if [[ ${UNTERMINATED_LOGS} != "" ]]; then - # shellcheck disable=SC2028 - echo "All calls to LogPrintf() and LogPrint() should be terminated with \\n" - echo - echo "${UNTERMINATED_LOGS}" - exit 1 -fi diff --git a/test/lint/lint-python-utf8-encoding.py b/test/lint/lint-python-utf8-encoding.py new file mode 100755 index 0000000000..62fdc34d50 --- /dev/null +++ b/test/lint/lint-python-utf8-encoding.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2018-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# Make sure we explicitly open all text files using UTF-8 (or ASCII) encoding to +# avoid potential issues on the BSDs where the locale is not always set. + +import sys +import re + +from subprocess import check_output, CalledProcessError + +EXCLUDED_DIRS = ["src/crc32c/"] + + +def get_exclude_args(): + return [":(exclude)" + dir for dir in EXCLUDED_DIRS] + + +def check_fileopens(): + fileopens = list() + + try: + fileopens = check_output(["git", "grep", r" open(", "--", "*.py"] + get_exclude_args(), universal_newlines=True, encoding="utf8").splitlines() + except CalledProcessError as e: + if e.returncode > 1: + raise e + + filtered_fileopens = [fileopen for fileopen in fileopens if not re.search(r"encoding=.(ascii|utf8|utf-8).|open\([^,]*, ['\"][^'\"]*b[^'\"]*['\"]", fileopen)] + + return filtered_fileopens + + +def check_checked_outputs(): + checked_outputs = list() + + try: + checked_outputs = check_output(["git", "grep", "check_output(", "--", "*.py"] + get_exclude_args(), universal_newlines=True, encoding="utf8").splitlines() + except CalledProcessError as e: + if e.returncode > 1: + raise e + + filtered_checked_outputs = [checked_output for checked_output in checked_outputs if re.search(r"universal_newlines=True", checked_output) and not re.search(r"encoding=.(ascii|utf8|utf-8).", checked_output)] + + return filtered_checked_outputs + + +def main(): + exit_code = 0 + + nonexplicit_utf8_fileopens = check_fileopens() + if nonexplicit_utf8_fileopens: + print("Python's open(...) seems to be used to open text files without explicitly specifying encoding='utf8':\n") + for fileopen in nonexplicit_utf8_fileopens: + print(fileopen) + exit_code = 1 + + nonexplicit_utf8_checked_outputs = check_checked_outputs() + if nonexplicit_utf8_checked_outputs: + if nonexplicit_utf8_fileopens: + print("\n") + print("Python's check_output(...) seems to be used to get program outputs without explicitly specifying encoding='utf8':\n") + for checked_output in nonexplicit_utf8_checked_outputs: + print(checked_output) + exit_code = 1 + + sys.exit(exit_code) + + +if __name__ == "__main__": + main() diff --git a/test/lint/lint-python-utf8-encoding.sh b/test/lint/lint-python-utf8-encoding.sh deleted file mode 100755 index 6e5b18fc23..0000000000 --- a/test/lint/lint-python-utf8-encoding.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Make sure we explicitly open all text files using UTF-8 (or ASCII) encoding to -# avoid potential issues on the BSDs where the locale is not always set. - -export LC_ALL=C -EXIT_CODE=0 -OUTPUT=$(git grep " open(" -- "*.py" ":(exclude)src/crc32c/" | grep -vE "encoding=.(ascii|utf8|utf-8)." | grep -vE "open\([^,]*, ['\"][^'\"]*b[^'\"]*['\"]") -if [[ ${OUTPUT} != "" ]]; then - echo "Python's open(...) seems to be used to open text files without explicitly" - echo "specifying encoding=\"utf8\":" - echo - echo "${OUTPUT}" - EXIT_CODE=1 -fi -OUTPUT=$(git grep "check_output(" -- "*.py" ":(exclude)src/crc32c/"| grep "universal_newlines=True" | grep -vE "encoding=.(ascii|utf8|utf-8).") -if [[ ${OUTPUT} != "" ]]; then - echo "Python's check_output(...) seems to be used to get program outputs without explicitly" - echo "specifying encoding=\"utf8\":" - echo - echo "${OUTPUT}" - EXIT_CODE=1 -fi -exit ${EXIT_CODE} diff --git a/test/lint/lint-python.py b/test/lint/lint-python.py new file mode 100755 index 0000000000..4d16facfea --- /dev/null +++ b/test/lint/lint-python.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" +Check for specified flake8 and mypy warnings in python files. +""" + +import os +import pkg_resources +import subprocess +import sys + +DEPS = ['flake8', 'mypy', 'pyzmq'] +MYPY_CACHE_DIR = f"{os.getenv('BASE_ROOT_DIR', '')}/test/.mypy_cache" +FILES_ARGS = ['git', 'ls-files', 'test/functional/*.py', 'contrib/devtools/*.py'] + +ENABLED = ( + 'E101,' # indentation contains mixed spaces and tabs + 'E112,' # expected an indented block + 'E113,' # unexpected indentation + 'E115,' # expected an indented block (comment) + 'E116,' # unexpected indentation (comment) + 'E125,' # continuation line with same indent as next logical line + 'E129,' # visually indented line with same indent as next logical line + 'E131,' # continuation line unaligned for hanging indent + 'E133,' # closing bracket is missing indentation + 'E223,' # tab before operator + 'E224,' # tab after operator + 'E242,' # tab after ',' + 'E266,' # too many leading '#' for block comment + 'E271,' # multiple spaces after keyword + 'E272,' # multiple spaces before keyword + 'E273,' # tab after keyword + 'E274,' # tab before keyword + 'E275,' # missing whitespace after keyword + 'E304,' # blank lines found after function decorator + 'E306,' # expected 1 blank line before a nested definition + 'E401,' # multiple imports on one line + 'E402,' # module level import not at top of file + 'E502,' # the backslash is redundant between brackets + 'E701,' # multiple statements on one line (colon) + 'E702,' # multiple statements on one line (semicolon) + 'E703,' # statement ends with a semicolon + 'E711,' # comparison to None should be 'if cond is None:' + 'E714,' # test for object identity should be "is not" + 'E721,' # do not compare types, use "isinstance()" + 'E742,' # do not define classes named "l", "O", or "I" + 'E743,' # do not define functions named "l", "O", or "I" + 'E901,' # SyntaxError: invalid syntax + 'E902,' # TokenError: EOF in multi-line string + 'F401,' # module imported but unused + 'F402,' # import module from line N shadowed by loop variable + 'F403,' # 'from foo_module import *' used; unable to detect undefined names + 'F404,' # future import(s) name after other statements + 'F405,' # foo_function may be undefined, or defined from star imports: bar_module + 'F406,' # "from module import *" only allowed at module level + 'F407,' # an undefined __future__ feature name was imported + 'F601,' # dictionary key name repeated with different values + 'F602,' # dictionary key variable name repeated with different values + 'F621,' # too many expressions in an assignment with star-unpacking + 'F622,' # two or more starred expressions in an assignment (a, *b, *c = d) + 'F631,' # assertion test is a tuple, which are always True + 'F632,' # use ==/!= to compare str, bytes, and int literals + 'F701,' # a break statement outside of a while or for loop + 'F702,' # a continue statement outside of a while or for loop + 'F703,' # a continue statement in a finally block in a loop + 'F704,' # a yield or yield from statement outside of a function + 'F705,' # a return statement with arguments inside a generator + 'F706,' # a return statement outside of a function/method + 'F707,' # an except: block as not the last exception handler + 'F811,' # redefinition of unused name from line N + 'F812,' # list comprehension redefines 'foo' from line N + 'F821,' # undefined name 'Foo' + 'F822,' # undefined name name in __all__ + 'F823,' # local variable name … referenced before assignment + 'F831,' # duplicate argument name in function definition + 'F841,' # local variable 'foo' is assigned to but never used + 'W191,' # indentation contains tabs + 'W291,' # trailing whitespace + 'W292,' # no newline at end of file + 'W293,' # blank line contains whitespace + 'W601,' # .has_key() is deprecated, use "in" + 'W602,' # deprecated form of raising exception + 'W603,' # "<>" is deprecated, use "!=" + 'W604,' # backticks are deprecated, use "repr()" + 'W605,' # invalid escape sequence "x" + 'W606,' # 'async' and 'await' are reserved keywords starting with Python 3.7 +) + + +def check_dependencies(): + working_set = {pkg.key for pkg in pkg_resources.working_set} + + for dep in DEPS: + if dep not in working_set: + print(f"Skipping Python linting since {dep} is not installed.") + exit(0) + + +def main(): + check_dependencies() + + if len(sys.argv) > 1: + flake8_files = sys.argv[1:] + else: + files_args = ['git', 'ls-files', '*.py'] + flake8_files = subprocess.check_output(files_args).decode("utf-8").splitlines() + + flake8_args = ['flake8', '--ignore=B,C,E,F,I,N,W', f'--select={ENABLED}'] + flake8_files + flake8_env = os.environ.copy() + flake8_env["PYTHONWARNINGS"] = "ignore" + + try: + subprocess.check_call(flake8_args, env=flake8_env) + except subprocess.CalledProcessError: + exit(1) + + mypy_files = subprocess.check_output(FILES_ARGS).decode("utf-8").splitlines() + mypy_args = ['mypy', '--show-error-codes'] + mypy_files + + try: + subprocess.check_call(mypy_args) + except subprocess.CalledProcessError: + exit(1) + + +if __name__ == "__main__": + main() diff --git a/test/lint/lint-python.sh b/test/lint/lint-python.sh deleted file mode 100755 index 7d7857d325..0000000000 --- a/test/lint/lint-python.sh +++ /dev/null @@ -1,111 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2017-2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Check for specified flake8 warnings in python files. - -export LC_ALL=C -export MYPY_CACHE_DIR="${BASE_ROOT_DIR}/test/.mypy_cache" - -enabled=( - E101 # indentation contains mixed spaces and tabs - E112 # expected an indented block - E113 # unexpected indentation - E115 # expected an indented block (comment) - E116 # unexpected indentation (comment) - E125 # continuation line with same indent as next logical line - E129 # visually indented line with same indent as next logical line - E131 # continuation line unaligned for hanging indent - E133 # closing bracket is missing indentation - E223 # tab before operator - E224 # tab after operator - E242 # tab after ',' - E266 # too many leading '#' for block comment - E271 # multiple spaces after keyword - E272 # multiple spaces before keyword - E273 # tab after keyword - E274 # tab before keyword - E275 # missing whitespace after keyword - E304 # blank lines found after function decorator - E306 # expected 1 blank line before a nested definition - E401 # multiple imports on one line - E402 # module level import not at top of file - E502 # the backslash is redundant between brackets - E701 # multiple statements on one line (colon) - E702 # multiple statements on one line (semicolon) - E703 # statement ends with a semicolon - E711 # comparison to None should be 'if cond is None:' - E714 # test for object identity should be "is not" - E721 # do not compare types, use "isinstance()" - E742 # do not define classes named "l", "O", or "I" - E743 # do not define functions named "l", "O", or "I" - E901 # SyntaxError: invalid syntax - E902 # TokenError: EOF in multi-line string - F401 # module imported but unused - F402 # import module from line N shadowed by loop variable - F403 # 'from foo_module import *' used; unable to detect undefined names - F404 # future import(s) name after other statements - F405 # foo_function may be undefined, or defined from star imports: bar_module - F406 # "from module import *" only allowed at module level - F407 # an undefined __future__ feature name was imported - F601 # dictionary key name repeated with different values - F602 # dictionary key variable name repeated with different values - F621 # too many expressions in an assignment with star-unpacking - F622 # two or more starred expressions in an assignment (a, *b, *c = d) - F631 # assertion test is a tuple, which are always True - F632 # use ==/!= to compare str, bytes, and int literals - F701 # a break statement outside of a while or for loop - F702 # a continue statement outside of a while or for loop - F703 # a continue statement in a finally block in a loop - F704 # a yield or yield from statement outside of a function - F705 # a return statement with arguments inside a generator - F706 # a return statement outside of a function/method - F707 # an except: block as not the last exception handler - F811 # redefinition of unused name from line N - F812 # list comprehension redefines 'foo' from line N - F821 # undefined name 'Foo' - F822 # undefined name name in __all__ - F823 # local variable name … referenced before assignment - F831 # duplicate argument name in function definition - F841 # local variable 'foo' is assigned to but never used - W191 # indentation contains tabs - W291 # trailing whitespace - W292 # no newline at end of file - W293 # blank line contains whitespace - W601 # .has_key() is deprecated, use "in" - W602 # deprecated form of raising exception - W603 # "<>" is deprecated, use "!=" - W604 # backticks are deprecated, use "repr()" - W605 # invalid escape sequence "x" - W606 # 'async' and 'await' are reserved keywords starting with Python 3.7 -) - -if ! command -v flake8 > /dev/null; then - echo "Skipping Python linting since flake8 is not installed." - exit 0 -elif PYTHONWARNINGS="ignore" flake8 --version | grep -q "Python 2"; then - echo "Skipping Python linting since flake8 is running under Python 2. Install the Python 3 version of flake8." - exit 0 -fi - -EXIT_CODE=0 - -# shellcheck disable=SC2046 -if ! PYTHONWARNINGS="ignore" flake8 --ignore=B,C,E,F,I,N,W --select=$(IFS=","; echo "${enabled[*]}") $( - if [[ $# == 0 ]]; then - git ls-files "*.py" - else - echo "$@" - fi -); then - EXIT_CODE=1 -fi - -mapfile -t FILES < <(git ls-files "test/functional/*.py" "contrib/devtools/*.py") -if ! mypy --show-error-codes "${FILES[@]}"; then - EXIT_CODE=1 -fi - -exit $EXIT_CODE diff --git a/test/lint/lint-shell-locale.py b/test/lint/lint-shell-locale.py new file mode 100755 index 0000000000..f3dfe18a95 --- /dev/null +++ b/test/lint/lint-shell-locale.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2018-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" +Make sure all shell scripts are: +a.) explicitly opt out of locale dependence using + "export LC_ALL=C" or "export LC_ALL=C.UTF-8", or +b.) explicitly opt in to locale dependence using the annotation below. +""" + +import subprocess +import sys +import re + +OPT_IN_LINE = '# This script is intentionally locale dependent by not setting \"export LC_ALL=C\"' + +OPT_OUT_LINES = [ + 'export LC_ALL=C', + 'export LC_ALL=C.UTF-8', +] + +def get_shell_files_list(): + command = [ + 'git', + 'ls-files', + '--', + '*.sh', + ] + try: + return subprocess.check_output(command, stderr = subprocess.STDOUT).decode('utf-8').splitlines() + except subprocess.CalledProcessError as e: + if e.returncode > 1: # return code is 1 when match is empty + print(e.output.decode('utf-8'), end='') + sys.exit(1) + return [] + +def main(): + exit_code = 0 + shell_files = get_shell_files_list() + for file_path in shell_files: + if re.search('src/(secp256k1|minisketch|univalue)/', file_path): + continue + + with open(file_path, 'r', encoding='utf-8') as file_obj: + contents = file_obj.read() + + if OPT_IN_LINE in contents: + continue + + non_comment_pattern = re.compile(r'^\s*((?!#).+)$', re.MULTILINE) + non_comment_lines = re.findall(non_comment_pattern, contents) + if not non_comment_lines: + continue + + first_non_comment_line = non_comment_lines[0] + if first_non_comment_line not in OPT_OUT_LINES: + print(f'Missing "export LC_ALL=C" (to avoid locale dependence) as first non-comment non-empty line in {file_path}') + exit_code = 1 + + return sys.exit(exit_code) + +if __name__ == '__main__': + main() + diff --git a/test/lint/lint-shell-locale.sh b/test/lint/lint-shell-locale.sh deleted file mode 100755 index 4c6b8a57e6..0000000000 --- a/test/lint/lint-shell-locale.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Make sure all shell scripts: -# a.) explicitly opt out of locale dependence using -# "export LC_ALL=C" or "export LC_ALL=C.UTF-8", or -# b.) explicitly opt in to locale dependence using the annotation below. - -export LC_ALL=C - -EXIT_CODE=0 -for SHELL_SCRIPT in $(git ls-files -- "*.sh" | grep -vE "src/(secp256k1|minisketch|univalue)/"); do - if grep -q "# This script is intentionally locale dependent by not setting \"export LC_ALL=C\"" "${SHELL_SCRIPT}"; then - continue - fi - FIRST_NON_COMMENT_LINE=$(grep -vE '^(#.*)?$' "${SHELL_SCRIPT}" | head -1) - if [[ ${FIRST_NON_COMMENT_LINE} != "export LC_ALL=C" && ${FIRST_NON_COMMENT_LINE} != "export LC_ALL=C.UTF-8" ]]; then - echo "Missing \"export LC_ALL=C\" (to avoid locale dependence) as first non-comment non-empty line in ${SHELL_SCRIPT}" - EXIT_CODE=1 - fi -done -exit ${EXIT_CODE} diff --git a/test/lint/lint-shell.py b/test/lint/lint-shell.py new file mode 100755 index 0000000000..f1e4494350 --- /dev/null +++ b/test/lint/lint-shell.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2018-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" +Check for shellcheck warnings in shell scripts. +""" + +import subprocess +import re +import sys + +# Disabled warnings: +DISABLED = [ + 'SC2162', # read without -r will mangle backslashes. +] + +def check_shellcheck_install(): + try: + subprocess.run(['shellcheck', '--version'], stdout=subprocess.DEVNULL, check=True) + except FileNotFoundError: + print('Skipping shell linting since shellcheck is not installed.') + sys.exit(0) + +def get_files(command): + output = subprocess.run(command, stdout=subprocess.PIPE, universal_newlines=True) + files = output.stdout.split('\n') + + # remove whitespace element + files = list(filter(None, files)) + return files + +def main(): + check_shellcheck_install() + + # build the `exclude` flag + exclude = '--exclude=' + ','.join(DISABLED) + + # build the `sourced files` list + sourced_files_cmd = [ + 'git', + 'grep', + '-El', + r'^# shellcheck shell=', + ] + sourced_files = get_files(sourced_files_cmd) + + # build the `guix files` list + guix_files_cmd = [ + 'git', + 'grep', + '-El', + r'^#!\/usr\/bin\/env bash', + '--', + 'contrib/guix', + 'contrib/shell', + ] + guix_files = get_files(guix_files_cmd) + + # build the other script files list + files_cmd = [ + 'git', + 'ls-files', + '--', + '*.sh', + ] + files = get_files(files_cmd) + # remove everything that doesn't match this regex + reg = re.compile(r'src/[leveldb,secp256k1,minisketch,univalue]') + files[:] = [file for file in files if not reg.match(file)] + + # build the `shellcheck` command + shellcheck_cmd = [ + 'shellcheck', + '--external-sources', + '--check-sourced', + '--source-path=SCRIPTDIR', + ] + shellcheck_cmd.append(exclude) + shellcheck_cmd.extend(sourced_files) + shellcheck_cmd.extend(guix_files) + shellcheck_cmd.extend(files) + + # run the `shellcheck` command + try: + subprocess.check_call(shellcheck_cmd) + except subprocess.CalledProcessError: + sys.exit(1) + +if __name__ == '__main__': + main() diff --git a/test/lint/lint-shell.sh b/test/lint/lint-shell.sh deleted file mode 100755 index 5fa104fce6..0000000000 --- a/test/lint/lint-shell.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Check for shellcheck warnings in shell scripts. - -export LC_ALL=C - -# Disabled warnings: -disabled=( - SC2162 # read without -r will mangle backslashes. -) - -EXIT_CODE=0 - -if ! command -v shellcheck > /dev/null; then - echo "Skipping shell linting since shellcheck is not installed." - exit $EXIT_CODE -fi - -SHELLCHECK_CMD=(shellcheck --external-sources --check-sourced --source-path=SCRIPTDIR) -EXCLUDE="--exclude=$(IFS=','; echo "${disabled[*]}")" -# Check shellcheck directive used for sourced files -mapfile -t SOURCED_FILES < <(git ls-files | xargs gawk '/^# shellcheck shell=/ {print FILENAME} {nextfile}') -mapfile -t GUIX_FILES < <(git ls-files contrib/guix contrib/shell | xargs gawk '/^#!\/usr\/bin\/env bash/ {print FILENAME} {nextfile}') -mapfile -t FILES < <(git ls-files -- '*.sh' | grep -vE 'src/(leveldb|secp256k1|minisketch|univalue)/') -if ! "${SHELLCHECK_CMD[@]}" "$EXCLUDE" "${SOURCED_FILES[@]}" "${GUIX_FILES[@]}" "${FILES[@]}"; then - EXIT_CODE=1 -fi - -exit $EXIT_CODE diff --git a/test/lint/lint-submodule.py b/test/lint/lint-submodule.py new file mode 100755 index 0000000000..89d4c80f55 --- /dev/null +++ b/test/lint/lint-submodule.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" +This script checks for git modules +""" + +import subprocess +import sys + +def main(): + submodules_list = subprocess.check_output(['git', 'submodule', 'status', '--recursive'], + universal_newlines = True, encoding = 'utf8').rstrip('\n') + if submodules_list: + print("These submodules were found, delete them:\n", submodules_list) + sys.exit(1) + sys.exit(0) + +if __name__ == '__main__': + main() diff --git a/test/lint/lint-submodule.sh b/test/lint/lint-submodule.sh deleted file mode 100755 index d9aa021df7..0000000000 --- a/test/lint/lint-submodule.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# This script checks for git modules -export LC_ALL=C -EXIT_CODE=0 - -CMD=$(git submodule status --recursive) -if test -n "$CMD"; -then - echo These submodules were found, delete them: - echo "$CMD" - EXIT_CODE=1 -fi - -exit $EXIT_CODE - diff --git a/test/lint/lint-tests.py b/test/lint/lint-tests.py new file mode 100755 index 0000000000..849ddcb961 --- /dev/null +++ b/test/lint/lint-tests.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2018-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" +Check the test suite naming conventions +""" + +import re +import subprocess +import sys + + +def grep_boost_fixture_test_suite(): + command = [ + "git", + "grep", + "-E", + r"^BOOST_FIXTURE_TEST_SUITE\(", + "--", + "src/test/**.cpp", + "src/wallet/test/**.cpp", + ] + return subprocess.check_output(command, universal_newlines=True, encoding="utf8") + + +def check_matching_test_names(test_suite_list): + not_matching = [ + x + for x in test_suite_list + if re.search(r"/(.*?)\.cpp:BOOST_FIXTURE_TEST_SUITE\(\1, .*\)", x) is None + ] + if len(not_matching) > 0: + not_matching = "\n".join(not_matching) + error_msg = ( + "The test suite in file src/test/foo_tests.cpp should be named\n" + '"foo_tests". Please make sure the following test suites follow\n' + "that convention:\n\n" + f"{not_matching}\n" + ) + print(error_msg) + return 1 + return 0 + + +def get_duplicates(input_list): + """ + From https://stackoverflow.com/a/9835819 + """ + seen = set() + dupes = set() + for x in input_list: + if x in seen: + dupes.add(x) + else: + seen.add(x) + return dupes + + +def check_unique_test_names(test_suite_list): + output = [re.search(r"\((.*?),", x) for x in test_suite_list] + output = [x.group(1) for x in output if x is not None] + output = get_duplicates(output) + output = sorted(list(output)) + + if len(output) > 0: + output = "\n".join(output) + error_msg = ( + "Test suite names must be unique. The following test suite names\n" + f"appear to be used more than once:\n\n{output}" + ) + print(error_msg) + return 1 + return 0 + + +def main(): + test_suite_list = grep_boost_fixture_test_suite().splitlines() + exit_code = check_matching_test_names(test_suite_list) + exit_code |= check_unique_test_names(test_suite_list) + sys.exit(exit_code) + + +if __name__ == "__main__": + main() diff --git a/test/lint/lint-tests.sh b/test/lint/lint-tests.sh deleted file mode 100755 index 35d11023eb..0000000000 --- a/test/lint/lint-tests.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Check the test suite naming conventions - -export LC_ALL=C -EXIT_CODE=0 - -NAMING_INCONSISTENCIES=$(git grep -E '^BOOST_FIXTURE_TEST_SUITE\(' -- \ - "src/test/**.cpp" "src/wallet/test/**.cpp" | \ - grep -vE '/(.*?)\.cpp:BOOST_FIXTURE_TEST_SUITE\(\1, .*\)$') -if [[ ${NAMING_INCONSISTENCIES} != "" ]]; then - echo "The test suite in file src/test/foo_tests.cpp should be named" - echo "\"foo_tests\". Please make sure the following test suites follow" - echo "that convention:" - echo - echo "${NAMING_INCONSISTENCIES}" - EXIT_CODE=1 -fi - -TEST_SUITE_NAME_COLLISSIONS=$(git grep -E '^BOOST_FIXTURE_TEST_SUITE\(' -- \ - "src/test/**.cpp" "src/wallet/test/**.cpp" | cut -f2 -d'(' | cut -f1 -d, | \ - sort | uniq -d) -if [[ ${TEST_SUITE_NAME_COLLISSIONS} != "" ]]; then - echo "Test suite names must be unique. The following test suite names" - echo "appear to be used more than once:" - echo - echo "${TEST_SUITE_NAME_COLLISSIONS}" - EXIT_CODE=1 -fi - -exit ${EXIT_CODE} diff --git a/test/lint/lint-whitespace.py b/test/lint/lint-whitespace.py new file mode 100755 index 0000000000..d98fc8d9a2 --- /dev/null +++ b/test/lint/lint-whitespace.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2017-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# Check for new lines in diff that introduce trailing whitespace or +# tab characters instead of spaces. + +# We can't run this check unless we know the commit range for the PR. + +import argparse +import os +import re +import sys + +from subprocess import check_output + +EXCLUDED_DIRS = ["depends/patches/", + "contrib/guix/patches/", + "src/leveldb/", + "src/crc32c/", + "src/secp256k1/", + "src/minisketch/", + "src/univalue/", + "doc/release-notes/", + "src/qt/locale"] + +def parse_args(): + """Parse command line arguments.""" + parser = argparse.ArgumentParser( + description=""" + Check for new lines in diff that introduce trailing whitespace + or tab characters instead of spaces in unstaged changes, the + previous n commits, or a commit-range. + """, + epilog=f""" + You can manually set the commit-range with the COMMIT_RANGE + environment variable (e.g. "COMMIT_RANGE='47ba2c3...ee50c9e' + {sys.argv[0]}"). Defaults to current merge base when neither + prev-commits nor the environment variable is set. + """) + + parser.add_argument("--prev-commits", "-p", required=False, help="The previous n commits to check") + + return parser.parse_args() + + +def report_diff(selection): + filename = "" + seen = False + seenln = False + + print("The following changes were suspected:") + + for line in selection: + if re.match(r"^diff", line): + filename = line + seen = False + elif re.match(r"^@@", line): + linenumber = line + seenln = False + else: + if not seen: + # The first time a file is seen with trailing whitespace or a tab character, we print the + # filename (preceded by a newline). + print("") + print(filename) + seen = True + if not seenln: + print(linenumber) + seenln = True + print(line) + + +def get_diff(commit_range, check_only_code): + exclude_args = [":(exclude)" + dir for dir in EXCLUDED_DIRS] + + if check_only_code: + what_files = ["*.cpp", "*.h", "*.md", "*.py", "*.sh"] + else: + what_files = ["."] + + diff = check_output(["git", "diff", "-U0", commit_range, "--"] + what_files + exclude_args, universal_newlines=True, encoding="utf8") + + return diff + + +def main(): + args = parse_args() + + if not os.getenv("COMMIT_RANGE"): + if args.prev_commits: + commit_range = "HEAD~" + args.prev_commits + "...HEAD" + else: + # This assumes that the target branch of the pull request will be master. + merge_base = check_output(["git", "merge-base", "HEAD", "master"], universal_newlines=True, encoding="utf8").rstrip("\n") + commit_range = merge_base + "..HEAD" + else: + commit_range = os.getenv("COMMIT_RANGE") + + whitespace_selection = [] + tab_selection = [] + + # Check if trailing whitespace was found in the diff. + for line in get_diff(commit_range, check_only_code=False).splitlines(): + if re.match(r"^(diff --git|\@@|^\+.*\s+$)", line): + whitespace_selection.append(line) + + whitespace_additions = [i for i in whitespace_selection if i.startswith("+")] + + # Check if tab characters were found in the diff. + for line in get_diff(commit_range, check_only_code=True).splitlines(): + if re.match(r"^(diff --git|\@@|^\+.*\t)", line): + tab_selection.append(line) + + tab_additions = [i for i in tab_selection if i.startswith("+")] + + ret = 0 + + if len(whitespace_additions) > 0: + print("This diff appears to have added new lines with trailing whitespace.") + report_diff(whitespace_selection) + ret = 1 + + if len(tab_additions) > 0: + print("This diff appears to have added new lines with tab characters instead of spaces.") + report_diff(tab_selection) + ret = 1 + + sys.exit(ret) + + +if __name__ == "__main__": + main() diff --git a/test/lint/lint-whitespace.sh b/test/lint/lint-whitespace.sh deleted file mode 100755 index 9d55c71eb5..0000000000 --- a/test/lint/lint-whitespace.sh +++ /dev/null @@ -1,115 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2017-2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Check for new lines in diff that introduce trailing whitespace. - -# We can't run this check unless we know the commit range for the PR. - -export LC_ALL=C -while getopts "?" opt; do - case $opt in - ?) - echo "Usage: $0 [N]" - echo " COMMIT_RANGE='<commit range>' $0" - echo " $0 -?" - echo "Checks unstaged changes, the previous N commits, or a commit range." - echo "COMMIT_RANGE='47ba2c3...ee50c9e' $0" - exit 0 - ;; - esac -done - -if [ -z "${COMMIT_RANGE}" ]; then - if [ -n "$1" ]; then - COMMIT_RANGE="HEAD~$1...HEAD" - else - # This assumes that the target branch of the pull request will be master. - MERGE_BASE=$(git merge-base HEAD master) - COMMIT_RANGE="$MERGE_BASE..HEAD" - fi -fi - -showdiff() { - if ! git diff -U0 "${COMMIT_RANGE}" -- "." ":(exclude)depends/patches/" ":(exclude)contrib/guix/patches/" ":(exclude)src/leveldb/" ":(exclude)src/crc32c/" ":(exclude)src/secp256k1/" ":(exclude)src/minisketch/" ":(exclude)src/univalue/" ":(exclude)doc/release-notes/" ":(exclude)src/qt/locale/"; then - echo "Failed to get a diff" - exit 1 - fi -} - -showcodediff() { - if ! git diff -U0 "${COMMIT_RANGE}" -- *.cpp *.h *.md *.py *.sh ":(exclude)src/leveldb/" ":(exclude)src/crc32c/" ":(exclude)src/secp256k1/" ":(exclude)src/minisketch/" ":(exclude)src/univalue/" ":(exclude)doc/release-notes/" ":(exclude)src/qt/locale/"; then - echo "Failed to get a diff" - exit 1 - fi -} - -RET=0 - -# Check if trailing whitespace was found in the diff. -if showdiff | grep -E -q '^\+.*\s+$'; then - echo "This diff appears to have added new lines with trailing whitespace." - echo "The following changes were suspected:" - FILENAME="" - SEEN=0 - SEENLN=0 - while read -r line; do - if [[ "$line" =~ ^diff ]]; then - FILENAME="$line" - SEEN=0 - elif [[ "$line" =~ ^@@ ]]; then - LINENUMBER="$line" - SEENLN=0 - else - if [ "$SEEN" -eq 0 ]; then - # The first time a file is seen with trailing whitespace, we print the - # filename (preceded by a newline). - echo - echo "$FILENAME" - SEEN=1 - fi - if [ "$SEENLN" -eq 0 ]; then - echo "$LINENUMBER" - SEENLN=1 - fi - echo "$line" - fi - done < <(showdiff | grep -E '^(diff --git |@@|\+.*\s+$)') - RET=1 -fi - -# Check if tab characters were found in the diff. -if showcodediff | perl -nle '$MATCH++ if m{^\+.*\t}; END{exit 1 unless $MATCH>0}' > /dev/null; then - echo "This diff appears to have added new lines with tab characters instead of spaces." - echo "The following changes were suspected:" - FILENAME="" - SEEN=0 - SEENLN=0 - while read -r line; do - if [[ "$line" =~ ^diff ]]; then - FILENAME="$line" - SEEN=0 - elif [[ "$line" =~ ^@@ ]]; then - LINENUMBER="$line" - SEENLN=0 - else - if [ "$SEEN" -eq 0 ]; then - # The first time a file is seen with a tab character, we print the - # filename (preceded by a newline). - echo - echo "$FILENAME" - SEEN=1 - fi - if [ "$SEENLN" -eq 0 ]; then - echo "$LINENUMBER" - SEENLN=1 - fi - echo "$line" - fi - done < <(showcodediff | perl -nle 'print if m{^(diff --git |@@|\+.*\t)}') - RET=1 -fi - -exit $RET diff --git a/test/lint/spelling.ignore-words.txt b/test/lint/spelling.ignore-words.txt index afdb0692d8..0efd298408 100644 --- a/test/lint/spelling.ignore-words.txt +++ b/test/lint/spelling.ignore-words.txt @@ -3,6 +3,7 @@ ba blockin cachable creat +desig fo fpr hights @@ -11,6 +12,7 @@ inout invokable keypair mor +nd nin ser unparseable diff --git a/test/util/test_runner.py b/test/util/test_runner.py index a7fc3b1dc1..03db05c563 100755 --- a/test/util/test_runner.py +++ b/test/util/test_runner.py @@ -22,7 +22,8 @@ import sys def main(): config = configparser.ConfigParser() config.optionxform = str - config.read_file(open(os.path.join(os.path.dirname(__file__), "../config.ini"), encoding="utf8")) + with open(os.path.join(os.path.dirname(__file__), "../config.ini"), encoding="utf8") as f: + config.read_file(f) env_conf = dict(config.items('environment')) parser = argparse.ArgumentParser(description=__doc__) @@ -43,7 +44,8 @@ def main(): def bctester(testDir, input_basename, buildenv): """ Loads and parses the input file, runs all tests and reports results""" input_filename = os.path.join(testDir, input_basename) - raw_data = open(input_filename, encoding="utf8").read() + with open(input_filename, encoding="utf8") as f: + raw_data = f.read() input_data = json.loads(raw_data) failed_testcases = [] @@ -80,7 +82,8 @@ def bctest(testDir, testObj, buildenv): inputData = None if "input" in testObj: filename = os.path.join(testDir, testObj["input"]) - inputData = open(filename, encoding="utf8").read() + with open(filename, encoding="utf8") as f: + inputData = f.read() stdinCfg = subprocess.PIPE # Read the expected output data (if there is any) @@ -91,7 +94,8 @@ def bctest(testDir, testObj, buildenv): outputFn = testObj['output_cmp'] outputType = os.path.splitext(outputFn)[1][1:] # output type from file extension (determines how to compare) try: - outputData = open(os.path.join(testDir, outputFn), encoding="utf8").read() + with open(os.path.join(testDir, outputFn), encoding="utf8") as f: + outputData = f.read() except: logging.error("Output file " + outputFn + " cannot be opened") raise |