diff options
Diffstat (limited to 'test')
39 files changed, 1411 insertions, 963 deletions
diff --git a/test/README.md b/test/README.md index 7ff2d6d9f2..e5a184d23c 100644 --- a/test/README.md +++ b/test/README.md @@ -305,9 +305,9 @@ Use the `-v` option for verbose output. | Lint test | Dependency | |-----------|:----------:| -| [`lint-python.sh`](lint/lint-python.sh) | [flake8](https://gitlab.com/pycqa/flake8) -| [`lint-python.sh`](lint/lint-python.sh) | [mypy](https://github.com/python/mypy) -| [`lint-python.sh`](lint/lint-python.sh) | [pyzmq](https://github.com/zeromq/pyzmq) +| [`lint-python.py`](lint/lint-python.py) | [flake8](https://gitlab.com/pycqa/flake8) +| [`lint-python.py`](lint/lint-python.py) | [mypy](https://github.com/python/mypy) +| [`lint-python.py`](lint/lint-python.py) | [pyzmq](https://github.com/zeromq/pyzmq) | [`lint-python-dead-code.py`](lint/lint-python-dead-code.py) | [vulture](https://github.com/jendrikseipp/vulture) | [`lint-shell.sh`](lint/lint-shell.sh) | [ShellCheck](https://github.com/koalaman/shellcheck) | [`lint-spelling.py`](lint/lint-spelling.py) | [codespell](https://github.com/codespell-project/codespell) diff --git a/test/config.ini.in b/test/config.ini.in index d7105c419b..5888ef443b 100644 --- a/test/config.ini.in +++ b/test/config.ini.in @@ -19,6 +19,7 @@ RPCAUTH=@abs_top_srcdir@/share/rpcauth/rpcauth.py @USE_SQLITE_TRUE@USE_SQLITE=true @USE_BDB_TRUE@USE_BDB=true @BUILD_BITCOIN_CLI_TRUE@ENABLE_CLI=true +@BUILD_BITCOIN_UTIL_TRUE@ENABLE_BITCOIN_UTIL=true @BUILD_BITCOIN_WALLET_TRUE@ENABLE_WALLET_TOOL=true @BUILD_BITCOIND_TRUE@ENABLE_BITCOIND=true @ENABLE_FUZZ_TRUE@ENABLE_FUZZ=true diff --git a/test/functional/README.md b/test/functional/README.md index 926810cf03..914dbfd977 100644 --- a/test/functional/README.md +++ b/test/functional/README.md @@ -24,7 +24,7 @@ don't have test cases for. Consider using [pyenv](https://github.com/pyenv/pyenv), which checks [.python-version](/.python-version), to prevent accidentally introducing modern syntax from an unsupported Python version. The CI linter job also checks this, but [possibly not in all cases](https://github.com/bitcoin/bitcoin/pull/14884#discussion_r239585126). -- See [the python lint script](/test/lint/lint-python.sh) that checks for violations that +- See [the python lint script](/test/lint/lint-python.py) that checks for violations that could lead to bugs and issues in the test code. - Use [type hints](https://docs.python.org/3/library/typing.html) in your code to improve code readability and to detect possible bugs earlier. diff --git a/test/functional/feature_fee_estimation.py b/test/functional/feature_fee_estimation.py index 233ffd60da..422612a78e 100755 --- a/test/functional/feature_fee_estimation.py +++ b/test/functional/feature_fee_estimation.py @@ -3,25 +3,13 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test fee estimation code.""" +from copy import deepcopy from decimal import Decimal import os import random from test_framework.messages import ( COIN, - COutPoint, - CTransaction, - CTxIn, - CTxOut, -) -from test_framework.script import ( - CScript, - OP_1, - OP_DROP, - OP_TRUE, -) -from test_framework.script_util import ( - script_to_p2sh_script, ) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( @@ -31,22 +19,14 @@ from test_framework.util import ( assert_raises_rpc_error, satoshi_round, ) - -# Construct 2 trivial P2SH's and the ScriptSigs that spend them -# So we can create many transactions without needing to spend -# time signing. -SCRIPT = CScript([OP_1, OP_DROP]) -P2SH = script_to_p2sh_script(SCRIPT) -REDEEM_SCRIPT = CScript([OP_TRUE, SCRIPT]) +from test_framework.wallet import MiniWallet def small_txpuzzle_randfee( - from_node, conflist, unconflist, amount, min_fee, fee_increment + wallet, from_node, conflist, unconflist, amount, min_fee, fee_increment ): - """Create and send a transaction with a random fee. + """Create and send a transaction with a random fee using MiniWallet. - The transaction pays to a trivial P2SH script, and assumes that its inputs - are of the same form. The function takes a list of confirmed outputs and unconfirmed outputs and attempts to use the confirmed list first for its inputs. It adds the newly created outputs to the unconfirmed list. @@ -58,23 +38,29 @@ def small_txpuzzle_randfee( rand_fee = float(fee_increment) * (1.1892 ** random.randint(0, 28)) # Total fee ranges from min_fee to min_fee + 127*fee_increment fee = min_fee - fee_increment + satoshi_round(rand_fee) - tx = CTransaction() + utxos_to_spend = [] total_in = Decimal("0.00000000") while total_in <= (amount + fee) and len(conflist) > 0: t = conflist.pop(0) - total_in += t["amount"] - tx.vin.append(CTxIn(COutPoint(int(t["txid"], 16), t["vout"]), REDEEM_SCRIPT)) + total_in += t["value"] + utxos_to_spend.append(t) while total_in <= (amount + fee) and len(unconflist) > 0: t = unconflist.pop(0) - total_in += t["amount"] - tx.vin.append(CTxIn(COutPoint(int(t["txid"], 16), t["vout"]), REDEEM_SCRIPT)) + total_in += t["value"] + utxos_to_spend.append(t) if total_in <= amount + fee: raise RuntimeError(f"Insufficient funds: need {amount + fee}, have {total_in}") - tx.vout.append(CTxOut(int((total_in - amount - fee) * COIN), P2SH)) - tx.vout.append(CTxOut(int(amount * COIN), P2SH)) + tx = wallet.create_self_transfer_multi( + from_node=from_node, + utxos_to_spend=utxos_to_spend, + fee_per_output=0) + tx.vout[0].nValue = int((total_in - amount - fee) * COIN) + tx.vout.append(deepcopy(tx.vout[0])) + tx.vout[1].nValue = int(amount * COIN) + txid = from_node.sendrawtransaction(hexstring=tx.serialize().hex(), maxfeerate=0) - unconflist.append({"txid": txid, "vout": 0, "amount": total_in - amount - fee}) - unconflist.append({"txid": txid, "vout": 1, "amount": amount}) + unconflist.append({"txid": txid, "vout": 0, "value": total_in - amount - fee}) + unconflist.append({"txid": txid, "vout": 1, "value": amount}) return (tx.serialize().hex(), fee) @@ -129,17 +115,13 @@ def check_estimates(node, fees_seen): check_smart_estimates(node, fees_seen) -def send_tx(node, utxo, feerate): +def send_tx(wallet, node, utxo, feerate): """Broadcast a 1in-1out transaction with a specific input and feerate (sat/vb).""" - tx = CTransaction() - tx.vin = [CTxIn(COutPoint(int(utxo["txid"], 16), utxo["vout"]), REDEEM_SCRIPT)] - tx.vout = [CTxOut(int(utxo["amount"] * COIN), P2SH)] - - # vbytes == bytes as we are using legacy transactions - fee = tx.get_vsize() * feerate - tx.vout[0].nValue -= fee - - return node.sendrawtransaction(tx.serialize().hex()) + return wallet.send_self_transfer( + from_node=node, + utxo_to_spend=utxo, + fee_rate=Decimal(feerate * 1000) / COIN, + )['txid'] class EstimateFeeTest(BitcoinTestFramework): @@ -152,9 +134,6 @@ class EstimateFeeTest(BitcoinTestFramework): ["-whitelist=noban@127.0.0.1", "-blockmaxweight=32000"], ] - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() - def setup_network(self): """ We'll setup the network to have 3 nodes that all mine with different parameters. @@ -168,9 +147,6 @@ class EstimateFeeTest(BitcoinTestFramework): # (68k weight is room enough for 120 or so transactions) # Node2 is a stingy miner, that # produces too small blocks (room for only 55 or so transactions) - self.start_nodes() - self.import_deterministic_coinbase_privkeys() - self.stop_nodes() def transact_and_mine(self, numblocks, mining_node): min_fee = Decimal("0.00001") @@ -183,6 +159,7 @@ class EstimateFeeTest(BitcoinTestFramework): for _ in range(random.randrange(100 - 50, 100 + 50)): from_index = random.randint(1, 2) (txhex, fee) = small_txpuzzle_randfee( + self.wallet, self.nodes[from_index], self.confutxo, self.memutxo, @@ -205,24 +182,10 @@ class EstimateFeeTest(BitcoinTestFramework): def initial_split(self, node): """Split two coinbase UTxOs into many small coins""" - utxo_count = 2048 - self.confutxo = [] - splitted_amount = Decimal("0.04") - fee = Decimal("0.1") - change = Decimal("100") - splitted_amount * utxo_count - fee - tx = CTransaction() - tx.vin = [ - CTxIn(COutPoint(int(cb["txid"], 16), cb["vout"])) - for cb in node.listunspent()[:2] - ] - tx.vout = [CTxOut(int(splitted_amount * COIN), P2SH) for _ in range(utxo_count)] - tx.vout.append(CTxOut(int(change * COIN), P2SH)) - txhex = node.signrawtransactionwithwallet(tx.serialize().hex())["hex"] - txid = node.sendrawtransaction(txhex) - self.confutxo = [ - {"txid": txid, "vout": i, "amount": splitted_amount} - for i in range(utxo_count) - ] + self.confutxo = self.wallet.send_self_transfer_multi( + from_node=node, + utxos_to_spend=[self.wallet.get_utxo() for _ in range(2)], + num_outputs=2048)['new_utxos'] while len(node.getrawmempool()) > 0: self.generate(node, 1, sync_fun=self.no_op) @@ -284,12 +247,12 @@ class EstimateFeeTest(BitcoinTestFramework): # Broadcast 45 low fee transactions that will need to be RBF'd for _ in range(45): u = utxos.pop(0) - txid = send_tx(node, u, low_feerate) + txid = send_tx(self.wallet, node, u, low_feerate) utxos_to_respend.append(u) txids_to_replace.append(txid) # Broadcast 5 low fee transaction which don't need to for _ in range(5): - send_tx(node, utxos.pop(0), low_feerate) + send_tx(self.wallet, node, utxos.pop(0), low_feerate) # Mine the transactions on another node self.sync_mempools(wait=0.1, nodes=[node, miner]) for txid in txids_to_replace: @@ -298,7 +261,7 @@ class EstimateFeeTest(BitcoinTestFramework): # RBF the low-fee transactions while len(utxos_to_respend) > 0: u = utxos_to_respend.pop(0) - send_tx(node, u, high_feerate) + send_tx(self.wallet, node, u, high_feerate) # Mine the last replacement txs self.sync_mempools(wait=0.1, nodes=[node, miner]) @@ -316,6 +279,8 @@ class EstimateFeeTest(BitcoinTestFramework): # Split two coinbases into many small utxos self.start_node(0) + self.wallet = MiniWallet(self.nodes[0]) + self.wallet.rescan_utxos() self.initial_split(self.nodes[0]) self.log.info("Finished splitting") diff --git a/test/functional/interface_rest.py b/test/functional/interface_rest.py index 4f8676ec53..95dc40cb52 100755 --- a/test/functional/interface_rest.py +++ b/test/functional/interface_rest.py @@ -326,6 +326,10 @@ class RESTTest (BitcoinTestFramework): # Check that there are our submitted transactions in the TX memory pool json_obj = self.test_rest_request("/mempool/contents") + raw_mempool_verbose = self.nodes[0].getrawmempool(verbose=True) + + assert_equal(json_obj, raw_mempool_verbose) + for i, tx in enumerate(txs): assert tx in json_obj assert_equal(json_obj[tx]['spentby'], txs[i + 1:i + 2]) @@ -361,6 +365,10 @@ class RESTTest (BitcoinTestFramework): json_obj = self.test_rest_request("/chaininfo") assert_equal(json_obj['bestblockhash'], bb_hash) + # Compare with normal RPC getblockchaininfo response + blockchain_info = self.nodes[0].getblockchaininfo() + assert_equal(blockchain_info, json_obj) + # Test compatibility of deprecated and newer endpoints self.log.info("Test compatibility of deprecated and newer endpoints") assert_equal(self.test_rest_request(f"/headers/{bb_hash}", query_params={"count": 1}), self.test_rest_request(f"/headers/1/{bb_hash}")) diff --git a/test/functional/p2p_segwit.py b/test/functional/p2p_segwit.py index f377fbaaa6..89ddfd3bcf 100755 --- a/test/functional/p2p_segwit.py +++ b/test/functional/p2p_segwit.py @@ -43,7 +43,6 @@ from test_framework.messages import ( ser_uint256, ser_vector, sha256, - tx_from_hex, ) from test_framework.p2p import ( P2PInterface, @@ -89,6 +88,8 @@ from test_framework.util import ( softfork_active, assert_raises_rpc_error, ) +from test_framework.wallet import MiniWallet + MAX_SIGOP_COST = 80000 @@ -221,9 +222,6 @@ class SegWitTest(BitcoinTestFramework): ] self.supports_cli = False - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() - # Helper functions def build_next_block(self): @@ -259,6 +257,7 @@ class SegWitTest(BitcoinTestFramework): self.log.info("Starting tests before segwit activation") self.segwit_active = False + self.wallet = MiniWallet(self.nodes[0]) self.test_non_witness_transaction() self.test_v0_outputs_arent_spendable() @@ -307,7 +306,7 @@ class SegWitTest(BitcoinTestFramework): self.test_node.send_and_ping(msg_no_witness_block(block)) # make sure the block was processed txid = block.vtx[0].sha256 - self.generate(self.nodes[0], 99) # let the block mature + self.generate(self.wallet, 99) # let the block mature # Create a transaction that spends the coinbase tx = CTransaction() @@ -1999,21 +1998,13 @@ class SegWitTest(BitcoinTestFramework): def serialize(self): return serialize_with_bogus_witness(self.tx) - self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(address_type='bech32'), 5) - self.generate(self.nodes[0], 1) - unspent = next(u for u in self.nodes[0].listunspent() if u['spendable'] and u['address'].startswith('bcrt')) - - raw = self.nodes[0].createrawtransaction([{"txid": unspent['txid'], "vout": unspent['vout']}], {self.nodes[0].getnewaddress(): 1}) - tx = tx_from_hex(raw) + tx = self.wallet.create_self_transfer(from_node=self.nodes[0])['tx'] assert_raises_rpc_error(-22, "TX decode failed", self.nodes[0].decoderawtransaction, hexstring=serialize_with_bogus_witness(tx).hex(), iswitness=True) - with self.nodes[0].assert_debug_log(['Superfluous witness record']): + with self.nodes[0].assert_debug_log(['Unknown transaction optional data']): self.test_node.send_and_ping(msg_bogus_tx(tx)) - raw = self.nodes[0].signrawtransactionwithwallet(raw) - assert raw['complete'] - raw = raw['hex'] - tx = tx_from_hex(raw) + tx.wit.vtxinwit = [] # drop witness assert_raises_rpc_error(-22, "TX decode failed", self.nodes[0].decoderawtransaction, hexstring=serialize_with_bogus_witness(tx).hex(), iswitness=True) - with self.nodes[0].assert_debug_log(['Unknown transaction optional data']): + with self.nodes[0].assert_debug_log(['Superfluous witness record']): self.test_node.send_and_ping(msg_bogus_tx(tx)) @subtest diff --git a/test/functional/rpc_misc.py b/test/functional/rpc_misc.py index 2f1796d7cc..f6ee6a5215 100755 --- a/test/functional/rpc_misc.py +++ b/test/functional/rpc_misc.py @@ -27,7 +27,7 @@ class RpcMiscTest(BitcoinTestFramework): self.log.info("test CHECK_NONFATAL") assert_raises_rpc_error( -1, - 'Internal bug detected: \'request.params[9].get_str() != "trigger_internal_bug"\'', + 'Internal bug detected: "request.params[9].get_str() != "trigger_internal_bug""', lambda: node.echo(arg9='trigger_internal_bug'), ) @@ -56,9 +56,6 @@ class RpcMiscTest(BitcoinTestFramework): self.log.info("test logging rpc and help") - # Test logging RPC returns the expected number of logging categories. - assert_equal(len(node.logging()), 27) - # Test toggling a logging category on/off/on with the logging RPC. assert_equal(node.logging()['qt'], True) node.logging(exclude=['qt']) diff --git a/test/functional/rpc_psbt.py b/test/functional/rpc_psbt.py index b037807b53..444e56610e 100755 --- a/test/functional/rpc_psbt.py +++ b/test/functional/rpc_psbt.py @@ -10,6 +10,10 @@ from itertools import product from test_framework.descriptors import descsum_create from test_framework.key import ECKey +from test_framework.messages import ( + ser_compact_size, + WITNESS_SCALE_FACTOR, +) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_approx, @@ -615,8 +619,8 @@ class PSBTTest(BitcoinTestFramework): self.nodes[1].createwallet("extfund") wallet = self.nodes[1].get_wallet_rpc("extfund") - # Make a weird but signable script. sh(pkh()) descriptor accomplishes this - desc = descsum_create("sh(pkh({}))".format(privkey)) + # Make a weird but signable script. sh(wsh(pkh())) descriptor accomplishes this + desc = descsum_create("sh(wsh(pkh({})))".format(privkey)) if self.options.descriptors: res = self.nodes[0].importdescriptors([{"desc": desc, "timestamp": "now"}]) else: @@ -634,7 +638,7 @@ class PSBTTest(BitcoinTestFramework): assert_raises_rpc_error(-4, "Insufficient funds", wallet.walletcreatefundedpsbt, [ext_utxo], {self.nodes[0].getnewaddress(): 15}) # But funding should work when the solving data is provided - psbt = wallet.walletcreatefundedpsbt([ext_utxo], {self.nodes[0].getnewaddress(): 15}, 0, {"add_inputs": True, "solving_data": {"pubkeys": [addr_info['pubkey']], "scripts": [addr_info["embedded"]["scriptPubKey"]]}}) + psbt = wallet.walletcreatefundedpsbt([ext_utxo], {self.nodes[0].getnewaddress(): 15}, 0, {"add_inputs": True, "solving_data": {"pubkeys": [addr_info['pubkey']], "scripts": [addr_info["embedded"]["scriptPubKey"], addr_info["embedded"]["embedded"]["scriptPubKey"]]}}) signed = wallet.walletprocesspsbt(psbt['psbt']) assert not signed['complete'] signed = self.nodes[0].walletprocesspsbt(signed['psbt']) @@ -655,10 +659,11 @@ class PSBTTest(BitcoinTestFramework): break psbt_in = dec["inputs"][input_idx] # Calculate the input weight - # (prevout + sequence + length of scriptSig + 2 bytes buffer) * 4 + len of scriptwitness + # (prevout + sequence + length of scriptSig + scriptsig + 1 byte buffer) * WITNESS_SCALE_FACTOR + num scriptWitness stack items + (length of stack item + stack item) * N stack items + 1 byte buffer len_scriptsig = len(psbt_in["final_scriptSig"]["hex"]) // 2 if "final_scriptSig" in psbt_in else 0 - len_scriptwitness = len(psbt_in["final_scriptwitness"]["hex"]) // 2 if "final_scriptwitness" in psbt_in else 0 - input_weight = ((41 + len_scriptsig + 2) * 4) + len_scriptwitness + len_scriptsig += len(ser_compact_size(len_scriptsig)) + 1 + len_scriptwitness = (sum([(len(x) // 2) + len(ser_compact_size(len(x) // 2)) for x in psbt_in["final_scriptwitness"]]) + len(psbt_in["final_scriptwitness"]) + 1) if "final_scriptwitness" in psbt_in else 0 + input_weight = ((40 + len_scriptsig) * WITNESS_SCALE_FACTOR) + len_scriptwitness low_input_weight = input_weight // 2 high_input_weight = input_weight * 2 diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index 2fb9ec0942..a39ee003ef 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -244,8 +244,14 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): "src", "bitcoin-cli" + config["environment"]["EXEEXT"], ) + fname_bitcoinutil = os.path.join( + config["environment"]["BUILDDIR"], + "src", + "bitcoin-util" + config["environment"]["EXEEXT"], + ) self.options.bitcoind = os.getenv("BITCOIND", default=fname_bitcoind) self.options.bitcoincli = os.getenv("BITCOINCLI", default=fname_bitcoincli) + self.options.bitcoinutil = os.getenv("BITCOINUTIL", default=fname_bitcoinutil) os.environ['PATH'] = os.pathsep.join([ os.path.join(config['environment']['BUILDDIR'], 'src'), @@ -880,6 +886,11 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): if not self.is_wallet_tool_compiled(): raise SkipTest("bitcoin-wallet has not been compiled") + def skip_if_no_bitcoin_util(self): + """Skip the running test if bitcoin-util has not been compiled.""" + if not self.is_bitcoin_util_compiled(): + raise SkipTest("bitcoin-util has not been compiled") + def skip_if_no_cli(self): """Skip the running test if bitcoin-cli has not been compiled.""" if not self.is_cli_compiled(): @@ -927,6 +938,10 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): """Checks whether bitcoin-wallet was compiled.""" return self.config["components"].getboolean("ENABLE_WALLET_TOOL") + def is_bitcoin_util_compiled(self): + """Checks whether bitcoin-util was compiled.""" + return self.config["components"].getboolean("ENABLE_BITCOIN_UTIL") + def is_zmq_compiled(self): """Checks whether the zmq module was compiled.""" return self.config["components"].getboolean("ENABLE_ZMQ") diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 1f0f806d91..a3c938ae26 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -145,6 +145,8 @@ BASE_SCRIPTS = [ 'wallet_txn_doublespend.py --mineblock', 'tool_wallet.py --legacy-wallet', 'tool_wallet.py --descriptors', + 'tool_signet_miner.py --legacy-wallet', + 'tool_signet_miner.py --descriptors', 'wallet_txn_clone.py', 'wallet_txn_clone.py --segwit', 'rpc_getchaintips.py', diff --git a/test/functional/tool_signet_miner.py b/test/functional/tool_signet_miner.py new file mode 100755 index 0000000000..e6fc9072ab --- /dev/null +++ b/test/functional/tool_signet_miner.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test signet miner tool""" + +import os.path +import subprocess +import sys +import time + +from test_framework.key import ECKey +from test_framework.script_util import key_to_p2wpkh_script +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal +from test_framework.wallet_util import bytes_to_wif + + +CHALLENGE_PRIVATE_KEY = (42).to_bytes(32, 'big') + + +class SignetMinerTest(BitcoinTestFramework): + def set_test_params(self): + self.chain = "signet" + self.setup_clean_chain = True + self.num_nodes = 1 + + # generate and specify signet challenge (simple p2wpkh script) + privkey = ECKey() + privkey.set(CHALLENGE_PRIVATE_KEY, True) + pubkey = privkey.get_pubkey().get_bytes() + challenge = key_to_p2wpkh_script(pubkey) + self.extra_args = [[f'-signetchallenge={challenge.hex()}']] + + def skip_test_if_missing_module(self): + self.skip_if_no_cli() + self.skip_if_no_wallet() + self.skip_if_no_bitcoin_util() + + def run_test(self): + node = self.nodes[0] + # import private key needed for signing block + node.importprivkey(bytes_to_wif(CHALLENGE_PRIVATE_KEY)) + + # generate block with signet miner tool + base_dir = self.config["environment"]["SRCDIR"] + signet_miner_path = os.path.join(base_dir, "contrib", "signet", "miner") + subprocess.run([ + sys.executable, + signet_miner_path, + f'--cli={node.cli.binary} -datadir={node.cli.datadir}', + 'generate', + f'--address={node.getnewaddress()}', + f'--grind-cmd={self.options.bitcoinutil} grind', + '--nbits=1d00ffff', + f'--set-block-time={int(time.time())}', + ], check=True, stderr=subprocess.STDOUT) + assert_equal(node.getblockcount(), 1) + + +if __name__ == "__main__": + SignetMinerTest().main() diff --git a/test/functional/wallet_avoidreuse.py b/test/functional/wallet_avoidreuse.py index dc823c2c60..f663666f57 100755 --- a/test/functional/wallet_avoidreuse.py +++ b/test/functional/wallet_avoidreuse.py @@ -118,6 +118,17 @@ class AvoidReuseTest(BitcoinTestFramework): assert_raises_rpc_error(-8, "Wallet flag is already set to false", self.nodes[0].setwalletflag, 'avoid_reuse', False) assert_raises_rpc_error(-8, "Wallet flag is already set to true", self.nodes[1].setwalletflag, 'avoid_reuse', True) + # Create a wallet with avoid reuse, and test that disabling it afterwards persists + self.nodes[1].createwallet(wallet_name="avoid_reuse_persist", avoid_reuse=True) + w = self.nodes[1].get_wallet_rpc("avoid_reuse_persist") + assert_equal(w.getwalletinfo()["avoid_reuse"], True) + w.setwalletflag("avoid_reuse", False) + assert_equal(w.getwalletinfo()["avoid_reuse"], False) + w.unloadwallet() + self.nodes[1].loadwallet("avoid_reuse_persist") + assert_equal(w.getwalletinfo()["avoid_reuse"], False) + w.unloadwallet() + def test_immutable(self): '''Test immutable wallet flags''' self.log.info("Test immutable wallet flags") diff --git a/test/functional/wallet_createwallet.py b/test/functional/wallet_createwallet.py index e8234de032..dcf2e98638 100755 --- a/test/functional/wallet_createwallet.py +++ b/test/functional/wallet_createwallet.py @@ -26,6 +26,11 @@ class CreateWalletTest(BitcoinTestFramework): node = self.nodes[0] self.generate(node, 1) # Leave IBD for sethdseed + self.log.info("Run createwallet with invalid parameters.") + # Run createwallet with invalid parameters. This must not prevent a new wallet with the same name from being created with the correct parameters. + assert_raises_rpc_error(-4, "Passphrase provided but private keys are disabled. A passphrase is only used to encrypt private keys, so cannot be used for wallets with private keys disabled.", + self.nodes[0].createwallet, wallet_name='w0', descriptors=True, disable_private_keys=True, passphrase="passphrase") + self.nodes[0].createwallet(wallet_name='w0') w0 = node.get_wallet_rpc('w0') address1 = w0.getnewaddress() diff --git a/test/functional/wallet_send.py b/test/functional/wallet_send.py index 86e36be8f7..07baa0595e 100755 --- a/test/functional/wallet_send.py +++ b/test/functional/wallet_send.py @@ -10,6 +10,10 @@ from itertools import product from test_framework.authproxy import JSONRPCException from test_framework.descriptors import descsum_create from test_framework.key import ECKey +from test_framework.messages import ( + ser_compact_size, + WITNESS_SCALE_FACTOR, +) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, @@ -488,8 +492,8 @@ class WalletSendTest(BitcoinTestFramework): self.nodes[1].createwallet("extfund") ext_fund = self.nodes[1].get_wallet_rpc("extfund") - # Make a weird but signable script. sh(pkh()) descriptor accomplishes this - desc = descsum_create("sh(pkh({}))".format(privkey)) + # Make a weird but signable script. sh(wsh(pkh())) descriptor accomplishes this + desc = descsum_create("sh(wsh(pkh({})))".format(privkey)) if self.options.descriptors: res = ext_fund.importdescriptors([{"desc": desc, "timestamp": "now"}]) else: @@ -507,7 +511,7 @@ class WalletSendTest(BitcoinTestFramework): self.test_send(from_wallet=ext_wallet, to_wallet=self.nodes[0], amount=15, inputs=[ext_utxo], add_inputs=True, psbt=True, include_watching=True, expect_error=(-4, "Insufficient funds")) # But funding should work when the solving data is provided - res = self.test_send(from_wallet=ext_wallet, to_wallet=self.nodes[0], amount=15, inputs=[ext_utxo], add_inputs=True, psbt=True, include_watching=True, solving_data={"pubkeys": [addr_info['pubkey']], "scripts": [addr_info["embedded"]["scriptPubKey"]]}) + res = self.test_send(from_wallet=ext_wallet, to_wallet=self.nodes[0], amount=15, inputs=[ext_utxo], add_inputs=True, psbt=True, include_watching=True, solving_data={"pubkeys": [addr_info['pubkey']], "scripts": [addr_info["embedded"]["scriptPubKey"], addr_info["embedded"]["embedded"]["scriptPubKey"]]}) signed = ext_wallet.walletprocesspsbt(res["psbt"]) signed = ext_fund.walletprocesspsbt(res["psbt"]) assert signed["complete"] @@ -526,10 +530,11 @@ class WalletSendTest(BitcoinTestFramework): break psbt_in = dec["inputs"][input_idx] # Calculate the input weight - # (prevout + sequence + length of scriptSig + 2 bytes buffer) * 4 + len of scriptwitness + # (prevout + sequence + length of scriptSig + scriptsig + 1 byte buffer) * WITNESS_SCALE_FACTOR + num scriptWitness stack items + (length of stack item + stack item) * N stack items + 1 byte buffer len_scriptsig = len(psbt_in["final_scriptSig"]["hex"]) // 2 if "final_scriptSig" in psbt_in else 0 - len_scriptwitness = len(psbt_in["final_scriptwitness"]["hex"]) // 2 if "final_scriptwitness" in psbt_in else 0 - input_weight = ((41 + len_scriptsig + 2) * 4) + len_scriptwitness + len_scriptsig += len(ser_compact_size(len_scriptsig)) + 1 + len_scriptwitness = (sum([(len(x) // 2) + len(ser_compact_size(len(x) // 2)) for x in psbt_in["final_scriptwitness"]]) + len(psbt_in["final_scriptwitness"]) + 1) if "final_scriptwitness" in psbt_in else 0 + input_weight = ((40 + len_scriptsig) * WITNESS_SCALE_FACTOR) + len_scriptwitness # Input weight error conditions assert_raises_rpc_error( diff --git a/test/functional/wallet_taproot.py b/test/functional/wallet_taproot.py index d3731b135a..41bb86f962 100755 --- a/test/functional/wallet_taproot.py +++ b/test/functional/wallet_taproot.py @@ -192,9 +192,9 @@ class WalletTaprootTest(BitcoinTestFramework): """Test generation and spending of P2TR address outputs.""" def set_test_params(self): - self.num_nodes = 3 + self.num_nodes = 2 self.setup_clean_chain = True - self.extra_args = [['-keypool=100'], ['-keypool=100'], ["-vbparams=taproot:1:1"]] + self.extra_args = [['-keypool=100'], ['-keypool=100']] self.supports_cli = False def skip_test_if_missing_module(self): @@ -243,15 +243,11 @@ class WalletTaprootTest(BitcoinTestFramework): assert_equal(len(rederive), 1) assert_equal(rederive[0], addr_g) - # tr descriptors can be imported regardless of Taproot status + # tr descriptors can be imported result = self.privs_tr_enabled.importdescriptors([{"desc": desc, "timestamp": "now"}]) assert(result[0]["success"]) result = self.pubs_tr_enabled.importdescriptors([{"desc": desc_pub, "timestamp": "now"}]) assert(result[0]["success"]) - result = self.privs_tr_disabled.importdescriptors([{"desc": desc, "timestamp": "now"}]) - assert result[0]["success"] - result = self.pubs_tr_disabled.importdescriptors([{"desc": desc_pub, "timestamp": "now"}]) - assert result[0]["success"] def do_test_sendtoaddress(self, comment, pattern, privmap, treefn, keys_pay, keys_change): self.log.info("Testing %s through sendtoaddress" % comment) @@ -328,12 +324,8 @@ class WalletTaprootTest(BitcoinTestFramework): self.log.info("Creating wallets...") self.nodes[0].createwallet(wallet_name="privs_tr_enabled", descriptors=True, blank=True) self.privs_tr_enabled = self.nodes[0].get_wallet_rpc("privs_tr_enabled") - self.nodes[2].createwallet(wallet_name="privs_tr_disabled", descriptors=True, blank=True) - self.privs_tr_disabled=self.nodes[2].get_wallet_rpc("privs_tr_disabled") self.nodes[0].createwallet(wallet_name="pubs_tr_enabled", descriptors=True, blank=True, disable_private_keys=True) self.pubs_tr_enabled = self.nodes[0].get_wallet_rpc("pubs_tr_enabled") - self.nodes[2].createwallet(wallet_name="pubs_tr_disabled", descriptors=True, blank=True, disable_private_keys=True) - self.pubs_tr_disabled=self.nodes[2].get_wallet_rpc("pubs_tr_disabled") self.nodes[0].createwallet(wallet_name="boring") self.nodes[0].createwallet(wallet_name="addr_gen", descriptors=True, disable_private_keys=True, blank=True) self.nodes[0].createwallet(wallet_name="rpc_online", descriptors=True, blank=True) diff --git a/test/lint/lint-format-strings.py b/test/lint/lint-format-strings.py new file mode 100755 index 0000000000..5a36da11fd --- /dev/null +++ b/test/lint/lint-format-strings.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2018-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# + +""" +Lint format strings: This program checks that the number of arguments passed +to a variadic format string function matches the number of format specifiers +in the format string. +""" + +import subprocess +import re +import sys + +FUNCTION_NAMES_AND_NUMBER_OF_LEADING_ARGUMENTS = [ + 'FatalError,0', + 'fprintf,1', + 'tfm::format,1', # Assuming tfm::::format(std::ostream&, ... + 'LogConnectFailure,1', + 'LogPrint,1', + 'LogPrintf,0', + 'printf,0', + 'snprintf,2', + 'sprintf,1', + 'strprintf,0', + 'vfprintf,1', + 'vprintf,1', + 'vsnprintf,1', + 'vsprintf,1', + 'WalletLogPrintf,0', +] +RUN_LINT_FILE = 'test/lint/run-lint-format-strings.py' + +def check_doctest(): + command = [ + 'python3', + '-m', + 'doctest', + RUN_LINT_FILE, + ] + try: + subprocess.run(command, check = True) + except subprocess.CalledProcessError: + sys.exit(1) + +def get_matching_files(function_name): + command = [ + 'git', + 'grep', + '--full-name', + '-l', + function_name, + '--', + '*.c', + '*.cpp', + '*.h', + ] + try: + return subprocess.check_output(command, stderr = subprocess.STDOUT).decode('utf-8').splitlines() + except subprocess.CalledProcessError as e: + if e.returncode > 1: # return code is 1 when match is empty + print(e.output.decode('utf-8'), end='') + sys.exit(1) + return [] + +def main(): + exit_code = 0 + check_doctest() + for s in FUNCTION_NAMES_AND_NUMBER_OF_LEADING_ARGUMENTS: + function_name, skip_arguments = s.split(',') + matching_files = get_matching_files(function_name) + + matching_files_filtered = [] + for matching_file in matching_files: + if not re.search('^src/(leveldb|secp256k1|minisketch|tinyformat|univalue|test/fuzz/strprintf.cpp)', matching_file): + matching_files_filtered.append(matching_file) + matching_files_filtered.sort() + + run_lint_args = [ + RUN_LINT_FILE, + '--skip-arguments', + skip_arguments, + function_name, + ] + run_lint_args.extend(matching_files_filtered) + + try: + subprocess.run(run_lint_args, check = True) + except subprocess.CalledProcessError: + exit_code = 1 + + sys.exit(exit_code) + +if __name__ == '__main__': + main() diff --git a/test/lint/lint-format-strings.sh b/test/lint/lint-format-strings.sh deleted file mode 100755 index 73730f16b3..0000000000 --- a/test/lint/lint-format-strings.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Lint format strings: This program checks that the number of arguments passed -# to a variadic format string function matches the number of format specifiers -# in the format string. - -export LC_ALL=C - -FUNCTION_NAMES_AND_NUMBER_OF_LEADING_ARGUMENTS=( - "FatalError,0" - "fprintf,1" - "tfm::format,1" # Assuming tfm::::format(std::ostream&, ... - "LogConnectFailure,1" - "LogPrint,1" - "LogPrintf,0" - "printf,0" - "snprintf,2" - "sprintf,1" - "strprintf,0" - "vfprintf,1" - "vprintf,1" - "vsnprintf,1" - "vsprintf,1" - "WalletLogPrintf,0" -) - -EXIT_CODE=0 -if ! python3 -m doctest "test/lint/run-lint-format-strings.py"; then - EXIT_CODE=1 -fi -for S in "${FUNCTION_NAMES_AND_NUMBER_OF_LEADING_ARGUMENTS[@]}"; do - IFS="," read -r FUNCTION_NAME SKIP_ARGUMENTS <<< "${S}" - for MATCHING_FILE in $(git grep --full-name -l "${FUNCTION_NAME}" -- "*.c" "*.cpp" "*.h" | sort | grep -vE "^src/(leveldb|secp256k1|minisketch|tinyformat|univalue|test/fuzz/strprintf.cpp)"); do - MATCHING_FILES+=("${MATCHING_FILE}") - done - if ! "test/lint/run-lint-format-strings.py" --skip-arguments "${SKIP_ARGUMENTS}" "${FUNCTION_NAME}" "${MATCHING_FILES[@]}"; then - EXIT_CODE=1 - fi -done -exit ${EXIT_CODE} diff --git a/test/lint/lint-git-commit-check.py b/test/lint/lint-git-commit-check.py new file mode 100755 index 0000000000..a1d03370e8 --- /dev/null +++ b/test/lint/lint-git-commit-check.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2020-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# Linter to check that commit messages have a new line before the body +# or no body at all + +import argparse +import os +import sys + +from subprocess import check_output + + +def parse_args(): + """Parse command line arguments.""" + parser = argparse.ArgumentParser( + description=""" + Linter to check that commit messages have a new line before + the body or no body at all. + """, + epilog=f""" + You can manually set the commit-range with the COMMIT_RANGE + environment variable (e.g. "COMMIT_RANGE='47ba2c3...ee50c9e' + {sys.argv[0]}"). Defaults to current merge base when neither + prev-commits nor the environment variable is set. + """) + + parser.add_argument("--prev-commits", "-p", required=False, help="The previous n commits to check") + + return parser.parse_args() + + +def main(): + args = parse_args() + exit_code = 0 + + if not os.getenv("COMMIT_RANGE"): + if args.prev_commits: + commit_range = "HEAD~" + args.prev_commits + "...HEAD" + else: + # This assumes that the target branch of the pull request will be master. + merge_base = check_output(["git", "merge-base", "HEAD", "master"], universal_newlines=True, encoding="utf8").rstrip("\n") + commit_range = merge_base + "..HEAD" + else: + commit_range = os.getenv("COMMIT_RANGE") + + commit_hashes = check_output(["git", "log", commit_range, "--format=%H"], universal_newlines=True, encoding="utf8").splitlines() + + for hash in commit_hashes: + commit_info = check_output(["git", "log", "--format=%B", "-n", "1", hash], universal_newlines=True, encoding="utf8").splitlines() + if len(commit_info) >= 2: + if commit_info[1]: + print(f"The subject line of commit hash {hash} is followed by a non-empty line. Subject lines should always be followed by a blank line.") + exit_code = 1 + + sys.exit(exit_code) + + +if __name__ == "__main__": + main() diff --git a/test/lint/lint-git-commit-check.sh b/test/lint/lint-git-commit-check.sh deleted file mode 100755 index f77373ed00..0000000000 --- a/test/lint/lint-git-commit-check.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2020-2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Linter to check that commit messages have a new line before the body -# or no body at all - -export LC_ALL=C - -EXIT_CODE=0 - -while getopts "?" opt; do - case $opt in - ?) - echo "Usage: $0 [N]" - echo " COMMIT_RANGE='<commit range>' $0" - echo " $0 -?" - echo "Checks unmerged commits, the previous N commits, or a commit range." - echo "COMMIT_RANGE='47ba2c3...ee50c9e' $0" - exit ${EXIT_CODE} - ;; - esac -done - -if [ -z "${COMMIT_RANGE}" ]; then - if [ -n "$1" ]; then - COMMIT_RANGE="HEAD~$1...HEAD" - else - # This assumes that the target branch of the pull request will be master. - MERGE_BASE=$(git merge-base HEAD master) - COMMIT_RANGE="$MERGE_BASE..HEAD" - fi -fi - -while IFS= read -r commit_hash || [[ -n "$commit_hash" ]]; do - n_line=0 - while IFS= read -r line || [[ -n "$line" ]]; do - n_line=$((n_line+1)) - length=${#line} - if [ $n_line -eq 2 ] && [ "$length" -ne 0 ]; then - echo "The subject line of commit hash ${commit_hash} is followed by a non-empty line. Subject lines should always be followed by a blank line." - EXIT_CODE=1 - fi - done < <(git log --format=%B -n 1 "$commit_hash") -done < <(git log "${COMMIT_RANGE}" --format=%H) - -exit ${EXIT_CODE} diff --git a/test/lint/lint-include-guards.py b/test/lint/lint-include-guards.py new file mode 100755 index 0000000000..86284517d5 --- /dev/null +++ b/test/lint/lint-include-guards.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2018-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" +Check include guards. +""" + +import re +import sys +from subprocess import check_output +from typing import List + + +HEADER_ID_PREFIX = 'BITCOIN_' +HEADER_ID_SUFFIX = '_H' + +EXCLUDE_FILES_WITH_PREFIX = ['src/crypto/ctaes', + 'src/leveldb', + 'src/crc32c', + 'src/secp256k1', + 'src/minisketch', + 'src/univalue', + 'src/tinyformat.h', + 'src/bench/nanobench.h', + 'src/test/fuzz/FuzzedDataProvider.h'] + + +def _get_header_file_lst() -> List[str]: + """ Helper function to get a list of header filepaths to be + checked for include guards. + """ + git_cmd_lst = ['git', 'ls-files', '--', '*.h'] + header_file_lst = check_output( + git_cmd_lst).decode('utf-8').splitlines() + + header_file_lst = [hf for hf in header_file_lst + if not any(ef in hf for ef + in EXCLUDE_FILES_WITH_PREFIX)] + + return header_file_lst + + +def _get_header_id(header_file: str) -> str: + """ Helper function to get the header id from a header file + string. + + eg: 'src/wallet/walletdb.h' -> 'BITCOIN_WALLET_WALLETDB_H' + + Args: + header_file: Filepath to header file. + + Returns: + The header id. + """ + header_id_base = header_file.split('/')[1:] + header_id_base = '_'.join(header_id_base) + header_id_base = header_id_base.replace('.h', '').replace('-', '_') + header_id_base = header_id_base.upper() + + header_id = f'{HEADER_ID_PREFIX}{header_id_base}{HEADER_ID_SUFFIX}' + + return header_id + + +def main(): + exit_code = 0 + + header_file_lst = _get_header_file_lst() + for header_file in header_file_lst: + header_id = _get_header_id(header_file) + + regex_pattern = f'^#(ifndef|define|endif //) {header_id}' + + with open(header_file, 'r', encoding='utf-8') as f: + header_file_contents = f.readlines() + + count = 0 + for header_file_contents_string in header_file_contents: + include_guard_lst = re.findall( + regex_pattern, header_file_contents_string) + + count += len(include_guard_lst) + + if count != 3: + print(f'{header_file} seems to be missing the expected ' + 'include guard:') + print(f' #ifndef {header_id}') + print(f' #define {header_id}') + print(' ...') + print(f' #endif // {header_id}\n') + exit_code = 1 + + sys.exit(exit_code) + + +if __name__ == '__main__': + main() diff --git a/test/lint/lint-include-guards.sh b/test/lint/lint-include-guards.sh deleted file mode 100755 index f14218aa74..0000000000 --- a/test/lint/lint-include-guards.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Check include guards. - -export LC_ALL=C -HEADER_ID_PREFIX="BITCOIN_" -HEADER_ID_SUFFIX="_H" - -REGEXP_EXCLUDE_FILES_WITH_PREFIX="src/(crypto/ctaes/|leveldb/|crc32c/|secp256k1/|minisketch/|test/fuzz/FuzzedDataProvider.h|tinyformat.h|bench/nanobench.h|univalue/)" - -EXIT_CODE=0 -for HEADER_FILE in $(git ls-files -- "*.h" | grep -vE "^${REGEXP_EXCLUDE_FILES_WITH_PREFIX}") -do - HEADER_ID_BASE=$(cut -f2- -d/ <<< "${HEADER_FILE}" | sed "s/\.h$//g" | tr / _ | tr - _ | tr "[:lower:]" "[:upper:]") - HEADER_ID="${HEADER_ID_PREFIX}${HEADER_ID_BASE}${HEADER_ID_SUFFIX}" - if [[ $(grep --count --extended-regexp "^#(ifndef|define|endif //) ${HEADER_ID}" "${HEADER_FILE}") != 3 ]]; then - echo "${HEADER_FILE} seems to be missing the expected include guard:" - echo " #ifndef ${HEADER_ID}" - echo " #define ${HEADER_ID}" - echo " ..." - echo " #endif // ${HEADER_ID}" - echo - EXIT_CODE=1 - fi -done -exit ${EXIT_CODE} diff --git a/test/lint/lint-includes.py b/test/lint/lint-includes.py new file mode 100755 index 0000000000..b29c7f8b4d --- /dev/null +++ b/test/lint/lint-includes.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2018-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# Check for duplicate includes. +# Guard against accidental introduction of new Boost dependencies. +# Check includes: Check for duplicate includes. Enforce bracket syntax includes. + +import os +import re +import sys + +from subprocess import check_output, CalledProcessError + + +EXCLUDED_DIRS = ["src/leveldb/", + "src/crc32c/", + "src/secp256k1/", + "src/minisketch/", + "src/univalue/"] + +EXPECTED_BOOST_INCLUDES = ["boost/algorithm/string.hpp", + "boost/algorithm/string/classification.hpp", + "boost/algorithm/string/replace.hpp", + "boost/algorithm/string/split.hpp", + "boost/date_time/posix_time/posix_time.hpp", + "boost/multi_index/hashed_index.hpp", + "boost/multi_index/ordered_index.hpp", + "boost/multi_index/sequenced_index.hpp", + "boost/multi_index_container.hpp", + "boost/process.hpp", + "boost/signals2/connection.hpp", + "boost/signals2/optional_last_value.hpp", + "boost/signals2/signal.hpp", + "boost/test/included/unit_test.hpp", + "boost/test/unit_test.hpp"] + + +def get_toplevel(): + return check_output(["git", "rev-parse", "--show-toplevel"], universal_newlines=True, encoding="utf8").rstrip("\n") + + +def list_files_by_suffix(suffixes): + exclude_args = [":(exclude)" + dir for dir in EXCLUDED_DIRS] + + files_list = check_output(["git", "ls-files", "src"] + exclude_args, universal_newlines=True, encoding="utf8").splitlines() + + return [file for file in files_list if file.endswith(suffixes)] + + +def find_duplicate_includes(include_list): + tempset = set() + duplicates = set() + + for inclusion in include_list: + if inclusion in tempset: + duplicates.add(inclusion) + else: + tempset.add(inclusion) + + return duplicates + + +def find_included_cpps(): + included_cpps = list() + + try: + included_cpps = check_output(["git", "grep", "-E", r"^#include [<\"][^>\"]+\.cpp[>\"]", "--", "*.cpp", "*.h"], universal_newlines=True, encoding="utf8").splitlines() + except CalledProcessError as e: + if e.returncode > 1: + raise e + + return included_cpps + + +def find_extra_boosts(): + included_boosts = list() + filtered_included_boost_set = set() + exclusion_set = set() + + try: + included_boosts = check_output(["git", "grep", "-E", r"^#include <boost/", "--", "*.cpp", "*.h"], universal_newlines=True, encoding="utf8").splitlines() + except CalledProcessError as e: + if e.returncode > 1: + raise e + + for boost in included_boosts: + filtered_included_boost_set.add(re.findall(r'(?<=\<).+?(?=\>)', boost)[0]) + + for expected_boost in EXPECTED_BOOST_INCLUDES: + for boost in filtered_included_boost_set: + if expected_boost in boost: + exclusion_set.add(boost) + + extra_boosts = set(filtered_included_boost_set.difference(exclusion_set)) + + return extra_boosts + + +def find_quote_syntax_inclusions(): + exclude_args = [":(exclude)" + dir for dir in EXCLUDED_DIRS] + quote_syntax_inclusions = list() + + try: + quote_syntax_inclusions = check_output(["git", "grep", r"^#include \"", "--", "*.cpp", "*.h"] + exclude_args, universal_newlines=True, encoding="utf8").splitlines() + except CalledProcessError as e: + if e.returncode > 1: + raise e + + return quote_syntax_inclusions + + +def main(): + exit_code = 0 + + os.chdir(get_toplevel()) + + # Check for duplicate includes + for filename in list_files_by_suffix((".cpp", ".h")): + with open(filename, "r", encoding="utf8") as file: + include_list = [line.rstrip("\n") for line in file if re.match(r"^#include", line)] + + duplicates = find_duplicate_includes(include_list) + + if duplicates: + print(f"Duplicate include(s) in {filename}:") + for duplicate in duplicates: + print(duplicate) + print("") + exit_code = 1 + + # Check if code includes .cpp-files + included_cpps = find_included_cpps() + + if included_cpps: + print("The following files #include .cpp files:") + for included_cpp in included_cpps: + print(included_cpp) + print("") + exit_code = 1 + + # Guard against accidental introduction of new Boost dependencies + extra_boosts = find_extra_boosts() + + if extra_boosts: + for boost in extra_boosts: + print(f"A new Boost dependency in the form of \"{boost}\" appears to have been introduced:") + print(check_output(["git", "grep", boost, "--", "*.cpp", "*.h"], universal_newlines=True, encoding="utf8")) + exit_code = 1 + + # Check if Boost dependencies are no longer used + for expected_boost in EXPECTED_BOOST_INCLUDES: + try: + check_output(["git", "grep", "-q", r"^#include <%s>" % expected_boost, "--", "*.cpp", "*.h"], universal_newlines=True, encoding="utf8") + except CalledProcessError as e: + if e.returncode > 1: + raise e + else: + print(f"Good job! The Boost dependency \"{expected_boost}\" is no longer used. " + "Please remove it from EXPECTED_BOOST_INCLUDES in test/lint/lint-includes.py " + "to make sure this dependency is not accidentally reintroduced.\n") + exit_code = 1 + + # Enforce bracket syntax includes + quote_syntax_inclusions = find_quote_syntax_inclusions() + + if quote_syntax_inclusions: + print("Please use bracket syntax includes (\"#include <foo.h>\") instead of quote syntax includes:") + for quote_syntax_inclusion in quote_syntax_inclusions: + print(quote_syntax_inclusion) + exit_code = 1 + + sys.exit(exit_code) + + +if __name__ == "__main__": + main() diff --git a/test/lint/lint-includes.sh b/test/lint/lint-includes.sh deleted file mode 100755 index 9e72831ee9..0000000000 --- a/test/lint/lint-includes.sh +++ /dev/null @@ -1,103 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Check for duplicate includes. -# Guard against accidental introduction of new Boost dependencies. -# Check includes: Check for duplicate includes. Enforce bracket syntax includes. - -export LC_ALL=C -IGNORE_REGEXP="/(leveldb|secp256k1|minisketch|univalue|crc32c)/" - -# cd to root folder of git repo for git ls-files to work properly -cd "$(dirname "$0")/../.." || exit 1 - -filter_suffix() { - git ls-files | grep -E "^src/.*\.${1}"'$' | grep -Ev "${IGNORE_REGEXP}" -} - -EXIT_CODE=0 - -for HEADER_FILE in $(filter_suffix h); do - DUPLICATE_INCLUDES_IN_HEADER_FILE=$(grep -E "^#include " < "${HEADER_FILE}" | sort | uniq -d) - if [[ ${DUPLICATE_INCLUDES_IN_HEADER_FILE} != "" ]]; then - echo "Duplicate include(s) in ${HEADER_FILE}:" - echo "${DUPLICATE_INCLUDES_IN_HEADER_FILE}" - echo - EXIT_CODE=1 - fi -done - -for CPP_FILE in $(filter_suffix cpp); do - DUPLICATE_INCLUDES_IN_CPP_FILE=$(grep -E "^#include " < "${CPP_FILE}" | sort | uniq -d) - if [[ ${DUPLICATE_INCLUDES_IN_CPP_FILE} != "" ]]; then - echo "Duplicate include(s) in ${CPP_FILE}:" - echo "${DUPLICATE_INCLUDES_IN_CPP_FILE}" - echo - EXIT_CODE=1 - fi -done - -INCLUDED_CPP_FILES=$(git grep -E "^#include [<\"][^>\"]+\.cpp[>\"]" -- "*.cpp" "*.h") -if [[ ${INCLUDED_CPP_FILES} != "" ]]; then - echo "The following files #include .cpp files:" - echo "${INCLUDED_CPP_FILES}" - echo - EXIT_CODE=1 -fi - -EXPECTED_BOOST_INCLUDES=( - boost/algorithm/string.hpp - boost/algorithm/string/classification.hpp - boost/algorithm/string/replace.hpp - boost/algorithm/string/split.hpp - boost/date_time/posix_time/posix_time.hpp - boost/multi_index/hashed_index.hpp - boost/multi_index/ordered_index.hpp - boost/multi_index/sequenced_index.hpp - boost/multi_index_container.hpp - boost/process.hpp - boost/signals2/connection.hpp - boost/signals2/optional_last_value.hpp - boost/signals2/signal.hpp - boost/test/included/unit_test.hpp - boost/test/unit_test.hpp -) - -for BOOST_INCLUDE in $(git grep '^#include <boost/' -- "*.cpp" "*.h" | cut -f2 -d: | cut -f2 -d'<' | cut -f1 -d'>' | sort -u); do - IS_EXPECTED_INCLUDE=0 - for EXPECTED_BOOST_INCLUDE in "${EXPECTED_BOOST_INCLUDES[@]}"; do - if [[ "${BOOST_INCLUDE}" == "${EXPECTED_BOOST_INCLUDE}" ]]; then - IS_EXPECTED_INCLUDE=1 - break - fi - done - if [[ ${IS_EXPECTED_INCLUDE} == 0 ]]; then - EXIT_CODE=1 - echo "A new Boost dependency in the form of \"${BOOST_INCLUDE}\" appears to have been introduced:" - git grep "${BOOST_INCLUDE}" -- "*.cpp" "*.h" - echo - fi -done - -for EXPECTED_BOOST_INCLUDE in "${EXPECTED_BOOST_INCLUDES[@]}"; do - if ! git grep -q "^#include <${EXPECTED_BOOST_INCLUDE}>" -- "*.cpp" "*.h"; then - echo "Good job! The Boost dependency \"${EXPECTED_BOOST_INCLUDE}\" is no longer used." - echo "Please remove it from EXPECTED_BOOST_INCLUDES in $0" - echo "to make sure this dependency is not accidentally reintroduced." - echo - EXIT_CODE=1 - fi -done - -QUOTE_SYNTAX_INCLUDES=$(git grep '^#include "' -- "*.cpp" "*.h" | grep -Ev "${IGNORE_REGEXP}") -if [[ ${QUOTE_SYNTAX_INCLUDES} != "" ]]; then - echo "Please use bracket syntax includes (\"#include <foo.h>\") instead of quote syntax includes:" - echo "${QUOTE_SYNTAX_INCLUDES}" - echo - EXIT_CODE=1 -fi - -exit ${EXIT_CODE} diff --git a/test/lint/lint-locale-dependence.py b/test/lint/lint-locale-dependence.py new file mode 100755 index 0000000000..2abf1be6b3 --- /dev/null +++ b/test/lint/lint-locale-dependence.py @@ -0,0 +1,259 @@ +#!/usr/bin/env python3 +# Copyright (c) 2018-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# Be aware that bitcoind and bitcoin-qt differ in terms of localization: Qt +# opts in to POSIX localization by running setlocale(LC_ALL, "") on startup, +# whereas no such call is made in bitcoind. +# +# Qt runs setlocale(LC_ALL, "") on initialization. This installs the locale +# specified by the user's LC_ALL (or LC_*) environment variable as the new +# C locale. +# +# In contrast, bitcoind does not opt in to localization -- no call to +# setlocale(LC_ALL, "") is made and the environment variables LC_* are +# thus ignored. +# +# This results in situations where bitcoind is guaranteed to be running +# with the classic locale ("C") whereas the locale of bitcoin-qt will vary +# depending on the user's environment variables. +# +# An example: Assuming the environment variable LC_ALL=de_DE then the +# call std::to_string(1.23) will return "1.230000" in bitcoind but +# "1,230000" in bitcoin-qt. +# +# From the Qt documentation: +# "On Unix/Linux Qt is configured to use the system locale settings by default. +# This can cause a conflict when using POSIX functions, for instance, when +# converting between data types such as floats and strings, since the notation +# may differ between locales. To get around this problem, call the POSIX function +# setlocale(LC_NUMERIC,"C") right after initializing QApplication, QGuiApplication +# or QCoreApplication to reset the locale that is used for number formatting to +# "C"-locale." +# +# See https://doc.qt.io/qt-5/qcoreapplication.html#locale-settings and +# https://stackoverflow.com/a/34878283 for more details. +# +# TODO: Reduce KNOWN_VIOLATIONS by replacing uses of locale dependent snprintf with strprintf. + +import re +import sys + +from subprocess import check_output, CalledProcessError + + +KNOWN_VIOLATIONS = [ + "src/dbwrapper.cpp:.*vsnprintf", + "src/test/dbwrapper_tests.cpp:.*snprintf", + "src/test/fuzz/locale.cpp:.*setlocale", + "src/test/fuzz/string.cpp:.*strtol", + "src/test/fuzz/string.cpp:.*strtoul", + "src/test/util_tests.cpp:.*strtoll" +] + +REGEXP_EXTERNAL_DEPENDENCIES_EXCLUSIONS = [ + "src/crypto/ctaes/", + "src/leveldb/", + "src/secp256k1/", + "src/minisketch/", + "src/tinyformat.h", + "src/univalue/" +] + +LOCALE_DEPENDENT_FUNCTIONS = [ + "alphasort", # LC_COLLATE (via strcoll) + "asctime", # LC_TIME (directly) + "asprintf", # (via vasprintf) + "atof", # LC_NUMERIC (via strtod) + "atoi", # LC_NUMERIC (via strtol) + "atol", # LC_NUMERIC (via strtol) + "atoll", # (via strtoll) + "atoq", + "btowc", # LC_CTYPE (directly) + "ctime", # (via asctime or localtime) + "dprintf", # (via vdprintf) + "fgetwc", + "fgetws", + "fold_case", # boost::locale::fold_case + "fprintf", # (via vfprintf) + "fputwc", + "fputws", + "fscanf", # (via __vfscanf) + "fwprintf", # (via __vfwprintf) + "getdate", # via __getdate_r => isspace // __localtime_r + "getwc", + "getwchar", + "is_digit", # boost::algorithm::is_digit + "is_space", # boost::algorithm::is_space + "isalnum", # LC_CTYPE + "isalpha", # LC_CTYPE + "isblank", # LC_CTYPE + "iscntrl", # LC_CTYPE + "isctype", # LC_CTYPE + "isdigit", # LC_CTYPE + "isgraph", # LC_CTYPE + "islower", # LC_CTYPE + "isprint", # LC_CTYPE + "ispunct", # LC_CTYPE + "isspace", # LC_CTYPE + "isupper", # LC_CTYPE + "iswalnum", # LC_CTYPE + "iswalpha", # LC_CTYPE + "iswblank", # LC_CTYPE + "iswcntrl", # LC_CTYPE + "iswctype", # LC_CTYPE + "iswdigit", # LC_CTYPE + "iswgraph", # LC_CTYPE + "iswlower", # LC_CTYPE + "iswprint", # LC_CTYPE + "iswpunct", # LC_CTYPE + "iswspace", # LC_CTYPE + "iswupper", # LC_CTYPE + "iswxdigit", # LC_CTYPE + "isxdigit", # LC_CTYPE + "localeconv", # LC_NUMERIC + LC_MONETARY + "mblen", # LC_CTYPE + "mbrlen", + "mbrtowc", + "mbsinit", + "mbsnrtowcs", + "mbsrtowcs", + "mbstowcs", # LC_CTYPE + "mbtowc", # LC_CTYPE + "mktime", + "normalize", # boost::locale::normalize + "printf", # LC_NUMERIC + "putwc", + "putwchar", + "scanf", # LC_NUMERIC + "setlocale", + "snprintf", + "sprintf", + "sscanf", + "std::locale::global", + "std::to_string", + "stod", + "stof", + "stoi", + "stol", + "stold", + "stoll", + "stoul", + "stoull", + "strcasecmp", + "strcasestr", + "strcoll", # LC_COLLATE + #"strerror", + "strfmon", + "strftime", # LC_TIME + "strncasecmp", + "strptime", + "strtod", # LC_NUMERIC + "strtof", + "strtoimax", + "strtol", # LC_NUMERIC + "strtold", + "strtoll", + "strtoq", + "strtoul", # LC_NUMERIC + "strtoull", + "strtoumax", + "strtouq", + "strxfrm", # LC_COLLATE + "swprintf", + "to_lower", # boost::locale::to_lower + "to_title", # boost::locale::to_title + "to_upper", # boost::locale::to_upper + "tolower", # LC_CTYPE + "toupper", # LC_CTYPE + "towctrans", + "towlower", # LC_CTYPE + "towupper", # LC_CTYPE + "trim", # boost::algorithm::trim + "trim_left", # boost::algorithm::trim_left + "trim_right", # boost::algorithm::trim_right + "ungetwc", + "vasprintf", + "vdprintf", + "versionsort", + "vfprintf", + "vfscanf", + "vfwprintf", + "vprintf", + "vscanf", + "vsnprintf", + "vsprintf", + "vsscanf", + "vswprintf", + "vwprintf", + "wcrtomb", + "wcscasecmp", + "wcscoll", # LC_COLLATE + "wcsftime", # LC_TIME + "wcsncasecmp", + "wcsnrtombs", + "wcsrtombs", + "wcstod", # LC_NUMERIC + "wcstof", + "wcstoimax", + "wcstol", # LC_NUMERIC + "wcstold", + "wcstoll", + "wcstombs", # LC_CTYPE + "wcstoul", # LC_NUMERIC + "wcstoull", + "wcstoumax", + "wcswidth", + "wcsxfrm", # LC_COLLATE + "wctob", + "wctomb", # LC_CTYPE + "wctrans", + "wctype", + "wcwidth", + "wprintf" +] + + +def find_locale_dependent_function_uses(): + regexp_locale_dependent_functions = "|".join(LOCALE_DEPENDENT_FUNCTIONS) + exclude_args = [":(exclude)" + excl for excl in REGEXP_EXTERNAL_DEPENDENCIES_EXCLUSIONS] + git_grep_command = ["git", "grep", "-E", "[^a-zA-Z0-9_\\`'\"<>](" + regexp_locale_dependent_functions + "(_r|_s)?)[^a-zA-Z0-9_\\`'\"<>]", "--", "*.cpp", "*.h"] + exclude_args + git_grep_output = list() + + try: + git_grep_output = check_output(git_grep_command, universal_newlines=True, encoding="utf8").splitlines() + except CalledProcessError as e: + if e.returncode > 1: + raise e + + return git_grep_output + + +def main(): + exit_code = 0 + + regexp_ignore_known_violations = "|".join(KNOWN_VIOLATIONS) + git_grep_output = find_locale_dependent_function_uses() + + for locale_dependent_function in LOCALE_DEPENDENT_FUNCTIONS: + matches = [line for line in git_grep_output + if re.search("[^a-zA-Z0-9_\\`'\"<>]" + locale_dependent_function + "(_r|_s)?[^a-zA-Z0-9_\\`'\"<>]", line) + and not re.search("\\.(c|cpp|h):\\s*(//|\\*|/\\*|\").*" + locale_dependent_function, line) + and not re.search(regexp_ignore_known_violations, line)] + if matches: + print(f"The locale dependent function {locale_dependent_function}(...) appears to be used:") + for match in matches: + print(match) + print("") + exit_code = 1 + + if exit_code == 1: + print("Unnecessary locale depedence can cause bugs that are very tricky to isolate and fix. Please avoid using locale dependent functions if possible.\n") + print(f"Advice not applicable in this specific case? Add an exception by updating the ignore list in {sys.argv[0]}") + + sys.exit(exit_code) + + +if __name__ == "__main__": + main() diff --git a/test/lint/lint-locale-dependence.sh b/test/lint/lint-locale-dependence.sh deleted file mode 100755 index 7d608eed6a..0000000000 --- a/test/lint/lint-locale-dependence.sh +++ /dev/null @@ -1,241 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2018-2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C - -# Be aware that bitcoind and bitcoin-qt differ in terms of localization: Qt -# opts in to POSIX localization by running setlocale(LC_ALL, "") on startup, -# whereas no such call is made in bitcoind. -# -# Qt runs setlocale(LC_ALL, "") on initialization. This installs the locale -# specified by the user's LC_ALL (or LC_*) environment variable as the new -# C locale. -# -# In contrast, bitcoind does not opt in to localization -- no call to -# setlocale(LC_ALL, "") is made and the environment variables LC_* are -# thus ignored. -# -# This results in situations where bitcoind is guaranteed to be running -# with the classic locale ("C") whereas the locale of bitcoin-qt will vary -# depending on the user's environment variables. -# -# An example: Assuming the environment variable LC_ALL=de_DE then the -# call std::to_string(1.23) will return "1.230000" in bitcoind but -# "1,230000" in bitcoin-qt. -# -# From the Qt documentation: -# "On Unix/Linux Qt is configured to use the system locale settings by default. -# This can cause a conflict when using POSIX functions, for instance, when -# converting between data types such as floats and strings, since the notation -# may differ between locales. To get around this problem, call the POSIX function -# setlocale(LC_NUMERIC,"C") right after initializing QApplication, QGuiApplication -# or QCoreApplication to reset the locale that is used for number formatting to -# "C"-locale." -# -# See https://doc.qt.io/qt-5/qcoreapplication.html#locale-settings and -# https://stackoverflow.com/a/34878283 for more details. - -# TODO: Reduce KNOWN_VIOLATIONS by replacing uses of locale dependent snprintf with strprintf. -KNOWN_VIOLATIONS=( - "src/dbwrapper.cpp:.*vsnprintf" - "src/test/dbwrapper_tests.cpp:.*snprintf" - "src/test/fuzz/locale.cpp" - "src/test/fuzz/string.cpp" - "src/test/util_tests.cpp" -) - -REGEXP_IGNORE_EXTERNAL_DEPENDENCIES="^src/(crypto/ctaes/|leveldb/|secp256k1/|minisketch/|tinyformat.h|univalue/)" - -LOCALE_DEPENDENT_FUNCTIONS=( - alphasort # LC_COLLATE (via strcoll) - asctime # LC_TIME (directly) - asprintf # (via vasprintf) - atof # LC_NUMERIC (via strtod) - atoi # LC_NUMERIC (via strtol) - atol # LC_NUMERIC (via strtol) - atoll # (via strtoll) - atoq - btowc # LC_CTYPE (directly) - ctime # (via asctime or localtime) - dprintf # (via vdprintf) - fgetwc - fgetws - fold_case # boost::locale::fold_case - fprintf # (via vfprintf) - fputwc - fputws - fscanf # (via __vfscanf) - fwprintf # (via __vfwprintf) - getdate # via __getdate_r => isspace // __localtime_r - getwc - getwchar - is_digit # boost::algorithm::is_digit - is_space # boost::algorithm::is_space - isalnum # LC_CTYPE - isalpha # LC_CTYPE - isblank # LC_CTYPE - iscntrl # LC_CTYPE - isctype # LC_CTYPE - isdigit # LC_CTYPE - isgraph # LC_CTYPE - islower # LC_CTYPE - isprint # LC_CTYPE - ispunct # LC_CTYPE - isspace # LC_CTYPE - isupper # LC_CTYPE - iswalnum # LC_CTYPE - iswalpha # LC_CTYPE - iswblank # LC_CTYPE - iswcntrl # LC_CTYPE - iswctype # LC_CTYPE - iswdigit # LC_CTYPE - iswgraph # LC_CTYPE - iswlower # LC_CTYPE - iswprint # LC_CTYPE - iswpunct # LC_CTYPE - iswspace # LC_CTYPE - iswupper # LC_CTYPE - iswxdigit # LC_CTYPE - isxdigit # LC_CTYPE - localeconv # LC_NUMERIC + LC_MONETARY - mblen # LC_CTYPE - mbrlen - mbrtowc - mbsinit - mbsnrtowcs - mbsrtowcs - mbstowcs # LC_CTYPE - mbtowc # LC_CTYPE - mktime - normalize # boost::locale::normalize - printf # LC_NUMERIC - putwc - putwchar - scanf # LC_NUMERIC - setlocale - snprintf - sprintf - sscanf - std::locale::global - std::to_string - stod - stof - stoi - stol - stold - stoll - stoul - stoull - strcasecmp - strcasestr - strcoll # LC_COLLATE -# strerror - strfmon - strftime # LC_TIME - strncasecmp - strptime - strtod # LC_NUMERIC - strtof - strtoimax - strtol # LC_NUMERIC - strtold - strtoll - strtoq - strtoul # LC_NUMERIC - strtoull - strtoumax - strtouq - strxfrm # LC_COLLATE - swprintf - to_lower # boost::locale::to_lower - to_title # boost::locale::to_title - to_upper # boost::locale::to_upper - tolower # LC_CTYPE - toupper # LC_CTYPE - towctrans - towlower # LC_CTYPE - towupper # LC_CTYPE - trim # boost::algorithm::trim - trim_left # boost::algorithm::trim_left - trim_right # boost::algorithm::trim_right - ungetwc - vasprintf - vdprintf - versionsort - vfprintf - vfscanf - vfwprintf - vprintf - vscanf - vsnprintf - vsprintf - vsscanf - vswprintf - vwprintf - wcrtomb - wcscasecmp - wcscoll # LC_COLLATE - wcsftime # LC_TIME - wcsncasecmp - wcsnrtombs - wcsrtombs - wcstod # LC_NUMERIC - wcstof - wcstoimax - wcstol # LC_NUMERIC - wcstold - wcstoll - wcstombs # LC_CTYPE - wcstoul # LC_NUMERIC - wcstoull - wcstoumax - wcswidth - wcsxfrm # LC_COLLATE - wctob - wctomb # LC_CTYPE - wctrans - wctype - wcwidth - wprintf -) - -function join_array { - local IFS="$1" - shift - echo "$*" -} - -REGEXP_IGNORE_KNOWN_VIOLATIONS=$(join_array "|" "${KNOWN_VIOLATIONS[@]}") - -# Invoke "git grep" only once in order to minimize run-time -REGEXP_LOCALE_DEPENDENT_FUNCTIONS=$(join_array "|" "${LOCALE_DEPENDENT_FUNCTIONS[@]}") -GIT_GREP_OUTPUT=$(git grep -E "[^a-zA-Z0-9_\`'\"<>](${REGEXP_LOCALE_DEPENDENT_FUNCTIONS}(_r|_s)?)[^a-zA-Z0-9_\`'\"<>]" -- "*.cpp" "*.h") - -EXIT_CODE=0 -for LOCALE_DEPENDENT_FUNCTION in "${LOCALE_DEPENDENT_FUNCTIONS[@]}"; do - MATCHES=$(grep -E "[^a-zA-Z0-9_\`'\"<>]${LOCALE_DEPENDENT_FUNCTION}(_r|_s)?[^a-zA-Z0-9_\`'\"<>]" <<< "${GIT_GREP_OUTPUT}" | \ - grep -vE "\.(c|cpp|h):\s*(//|\*|/\*|\").*${LOCALE_DEPENDENT_FUNCTION}") - if [[ ${REGEXP_IGNORE_EXTERNAL_DEPENDENCIES} != "" ]]; then - MATCHES=$(grep -vE "${REGEXP_IGNORE_EXTERNAL_DEPENDENCIES}" <<< "${MATCHES}") - fi - if [[ ${REGEXP_IGNORE_KNOWN_VIOLATIONS} != "" ]]; then - MATCHES=$(grep -vE "${REGEXP_IGNORE_KNOWN_VIOLATIONS}" <<< "${MATCHES}") - fi - if [[ ${MATCHES} != "" ]]; then - echo "The locale dependent function ${LOCALE_DEPENDENT_FUNCTION}(...) appears to be used:" - echo "${MATCHES}" - echo - EXIT_CODE=1 - fi -done -if [[ ${EXIT_CODE} != 0 ]]; then - echo "Unnecessary locale dependence can cause bugs that are very" - echo "tricky to isolate and fix. Please avoid using locale dependent" - echo "functions if possible." - echo - echo "Advice not applicable in this specific case? Add an exception" - echo "by updating the ignore list in $0" -fi -exit ${EXIT_CODE} diff --git a/test/lint/lint-logs.py b/test/lint/lint-logs.py new file mode 100755 index 0000000000..e6c4c068fb --- /dev/null +++ b/test/lint/lint-logs.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2018-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# Check that all logs are terminated with '\n' +# +# Some logs are continued over multiple lines. They should be explicitly +# commented with /* Continued */ + +import re +import sys + +from subprocess import check_output + + +def main(): + logs_list = check_output(["git", "grep", "--extended-regexp", r"LogPrintf?\(", "--", "*.cpp"], universal_newlines=True, encoding="utf8").splitlines() + + unterminated_logs = [line for line in logs_list if not re.search(r'(\\n"|/\* Continued \*/)', line)] + + if unterminated_logs != []: + print("All calls to LogPrintf() and LogPrint() should be terminated with \\n") + print("") + + for line in unterminated_logs: + print(line) + + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/test/lint/lint-logs.sh b/test/lint/lint-logs.sh deleted file mode 100755 index 6d5165f649..0000000000 --- a/test/lint/lint-logs.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Check that all logs are terminated with '\n' -# -# Some logs are continued over multiple lines. They should be explicitly -# commented with /* Continued */ -# -# There are some instances of LogPrintf() in comments. Those can be -# ignored - -export LC_ALL=C -UNTERMINATED_LOGS=$(git grep --extended-regexp "LogPrintf?\(" -- "*.cpp" | \ - grep -v '\\n"' | \ - grep -v '\.\.\.' | \ - grep -v "/\* Continued \*/" | \ - grep -v "LogPrint()" | \ - grep -v "LogPrintf()") -if [[ ${UNTERMINATED_LOGS} != "" ]]; then - # shellcheck disable=SC2028 - echo "All calls to LogPrintf() and LogPrint() should be terminated with \\n" - echo - echo "${UNTERMINATED_LOGS}" - exit 1 -fi diff --git a/test/lint/lint-python-mutable-default-parameters.py b/test/lint/lint-python-mutable-default-parameters.py new file mode 100755 index 0000000000..7991e3630b --- /dev/null +++ b/test/lint/lint-python-mutable-default-parameters.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" +Detect when a mutable list or dict is used as a default parameter value in a Python function. +""" + +import subprocess +import sys + + +def main(): + command = [ + "git", + "grep", + "-E", + r"^\s*def [a-zA-Z0-9_]+\(.*=\s*(\[|\{)", + "--", + "*.py", + ] + output = subprocess.run(command, stdout=subprocess.PIPE, universal_newlines=True) + if len(output.stdout) > 0: + error_msg = ( + "A mutable list or dict seems to be used as default parameter value:\n\n" + f"{output.stdout}\n" + f"{example()}" + ) + print(error_msg) + sys.exit(1) + else: + sys.exit(0) + + +def example(): + return """This is how mutable list and dict default parameter values behave: + +>>> def f(i, j=[], k={}): +... j.append(i) +... k[i] = True +... return j, k +... +>>> f(1) +([1], {1: True}) +>>> f(1) +([1, 1], {1: True}) +>>> f(2) +([1, 1, 2], {1: True, 2: True}) + +The intended behaviour was likely: + +>>> def f(i, j=None, k=None): +... if j is None: +... j = [] +... if k is None: +... k = {} +... j.append(i) +... k[i] = True +... return j, k +... +>>> f(1) +([1], {1: True}) +>>> f(1) +([1], {1: True}) +>>> f(2) +([2], {2: True})""" + + +if __name__ == "__main__": + main() diff --git a/test/lint/lint-python-mutable-default-parameters.sh b/test/lint/lint-python-mutable-default-parameters.sh deleted file mode 100755 index 1f9f035d30..0000000000 --- a/test/lint/lint-python-mutable-default-parameters.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2019 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Detect when a mutable list or dict is used as a default parameter value in a Python function. - -export LC_ALL=C -EXIT_CODE=0 -OUTPUT=$(git grep -E '^\s*def [a-zA-Z0-9_]+\(.*=\s*(\[|\{)' -- "*.py") -if [[ ${OUTPUT} != "" ]]; then - echo "A mutable list or dict seems to be used as default parameter value:" - echo - echo "${OUTPUT}" - echo - cat << EXAMPLE -This is how mutable list and dict default parameter values behave: - ->>> def f(i, j=[], k={}): -... j.append(i) -... k[i] = True -... return j, k -... ->>> f(1) -([1], {1: True}) ->>> f(1) -([1, 1], {1: True}) ->>> f(2) -([1, 1, 2], {1: True, 2: True}) - -The intended behaviour was likely: - ->>> def f(i, j=None, k=None): -... if j is None: -... j = [] -... if k is None: -... k = {} -... j.append(i) -... k[i] = True -... return j, k -... ->>> f(1) -([1], {1: True}) ->>> f(1) -([1], {1: True}) ->>> f(2) -([2], {2: True}) -EXAMPLE - EXIT_CODE=1 -fi -exit ${EXIT_CODE} diff --git a/test/lint/lint-python-utf8-encoding.py b/test/lint/lint-python-utf8-encoding.py new file mode 100755 index 0000000000..62fdc34d50 --- /dev/null +++ b/test/lint/lint-python-utf8-encoding.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2018-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# Make sure we explicitly open all text files using UTF-8 (or ASCII) encoding to +# avoid potential issues on the BSDs where the locale is not always set. + +import sys +import re + +from subprocess import check_output, CalledProcessError + +EXCLUDED_DIRS = ["src/crc32c/"] + + +def get_exclude_args(): + return [":(exclude)" + dir for dir in EXCLUDED_DIRS] + + +def check_fileopens(): + fileopens = list() + + try: + fileopens = check_output(["git", "grep", r" open(", "--", "*.py"] + get_exclude_args(), universal_newlines=True, encoding="utf8").splitlines() + except CalledProcessError as e: + if e.returncode > 1: + raise e + + filtered_fileopens = [fileopen for fileopen in fileopens if not re.search(r"encoding=.(ascii|utf8|utf-8).|open\([^,]*, ['\"][^'\"]*b[^'\"]*['\"]", fileopen)] + + return filtered_fileopens + + +def check_checked_outputs(): + checked_outputs = list() + + try: + checked_outputs = check_output(["git", "grep", "check_output(", "--", "*.py"] + get_exclude_args(), universal_newlines=True, encoding="utf8").splitlines() + except CalledProcessError as e: + if e.returncode > 1: + raise e + + filtered_checked_outputs = [checked_output for checked_output in checked_outputs if re.search(r"universal_newlines=True", checked_output) and not re.search(r"encoding=.(ascii|utf8|utf-8).", checked_output)] + + return filtered_checked_outputs + + +def main(): + exit_code = 0 + + nonexplicit_utf8_fileopens = check_fileopens() + if nonexplicit_utf8_fileopens: + print("Python's open(...) seems to be used to open text files without explicitly specifying encoding='utf8':\n") + for fileopen in nonexplicit_utf8_fileopens: + print(fileopen) + exit_code = 1 + + nonexplicit_utf8_checked_outputs = check_checked_outputs() + if nonexplicit_utf8_checked_outputs: + if nonexplicit_utf8_fileopens: + print("\n") + print("Python's check_output(...) seems to be used to get program outputs without explicitly specifying encoding='utf8':\n") + for checked_output in nonexplicit_utf8_checked_outputs: + print(checked_output) + exit_code = 1 + + sys.exit(exit_code) + + +if __name__ == "__main__": + main() diff --git a/test/lint/lint-python-utf8-encoding.sh b/test/lint/lint-python-utf8-encoding.sh deleted file mode 100755 index 6e5b18fc23..0000000000 --- a/test/lint/lint-python-utf8-encoding.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Make sure we explicitly open all text files using UTF-8 (or ASCII) encoding to -# avoid potential issues on the BSDs where the locale is not always set. - -export LC_ALL=C -EXIT_CODE=0 -OUTPUT=$(git grep " open(" -- "*.py" ":(exclude)src/crc32c/" | grep -vE "encoding=.(ascii|utf8|utf-8)." | grep -vE "open\([^,]*, ['\"][^'\"]*b[^'\"]*['\"]") -if [[ ${OUTPUT} != "" ]]; then - echo "Python's open(...) seems to be used to open text files without explicitly" - echo "specifying encoding=\"utf8\":" - echo - echo "${OUTPUT}" - EXIT_CODE=1 -fi -OUTPUT=$(git grep "check_output(" -- "*.py" ":(exclude)src/crc32c/"| grep "universal_newlines=True" | grep -vE "encoding=.(ascii|utf8|utf-8).") -if [[ ${OUTPUT} != "" ]]; then - echo "Python's check_output(...) seems to be used to get program outputs without explicitly" - echo "specifying encoding=\"utf8\":" - echo - echo "${OUTPUT}" - EXIT_CODE=1 -fi -exit ${EXIT_CODE} diff --git a/test/lint/lint-python.py b/test/lint/lint-python.py new file mode 100755 index 0000000000..4d16facfea --- /dev/null +++ b/test/lint/lint-python.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" +Check for specified flake8 and mypy warnings in python files. +""" + +import os +import pkg_resources +import subprocess +import sys + +DEPS = ['flake8', 'mypy', 'pyzmq'] +MYPY_CACHE_DIR = f"{os.getenv('BASE_ROOT_DIR', '')}/test/.mypy_cache" +FILES_ARGS = ['git', 'ls-files', 'test/functional/*.py', 'contrib/devtools/*.py'] + +ENABLED = ( + 'E101,' # indentation contains mixed spaces and tabs + 'E112,' # expected an indented block + 'E113,' # unexpected indentation + 'E115,' # expected an indented block (comment) + 'E116,' # unexpected indentation (comment) + 'E125,' # continuation line with same indent as next logical line + 'E129,' # visually indented line with same indent as next logical line + 'E131,' # continuation line unaligned for hanging indent + 'E133,' # closing bracket is missing indentation + 'E223,' # tab before operator + 'E224,' # tab after operator + 'E242,' # tab after ',' + 'E266,' # too many leading '#' for block comment + 'E271,' # multiple spaces after keyword + 'E272,' # multiple spaces before keyword + 'E273,' # tab after keyword + 'E274,' # tab before keyword + 'E275,' # missing whitespace after keyword + 'E304,' # blank lines found after function decorator + 'E306,' # expected 1 blank line before a nested definition + 'E401,' # multiple imports on one line + 'E402,' # module level import not at top of file + 'E502,' # the backslash is redundant between brackets + 'E701,' # multiple statements on one line (colon) + 'E702,' # multiple statements on one line (semicolon) + 'E703,' # statement ends with a semicolon + 'E711,' # comparison to None should be 'if cond is None:' + 'E714,' # test for object identity should be "is not" + 'E721,' # do not compare types, use "isinstance()" + 'E742,' # do not define classes named "l", "O", or "I" + 'E743,' # do not define functions named "l", "O", or "I" + 'E901,' # SyntaxError: invalid syntax + 'E902,' # TokenError: EOF in multi-line string + 'F401,' # module imported but unused + 'F402,' # import module from line N shadowed by loop variable + 'F403,' # 'from foo_module import *' used; unable to detect undefined names + 'F404,' # future import(s) name after other statements + 'F405,' # foo_function may be undefined, or defined from star imports: bar_module + 'F406,' # "from module import *" only allowed at module level + 'F407,' # an undefined __future__ feature name was imported + 'F601,' # dictionary key name repeated with different values + 'F602,' # dictionary key variable name repeated with different values + 'F621,' # too many expressions in an assignment with star-unpacking + 'F622,' # two or more starred expressions in an assignment (a, *b, *c = d) + 'F631,' # assertion test is a tuple, which are always True + 'F632,' # use ==/!= to compare str, bytes, and int literals + 'F701,' # a break statement outside of a while or for loop + 'F702,' # a continue statement outside of a while or for loop + 'F703,' # a continue statement in a finally block in a loop + 'F704,' # a yield or yield from statement outside of a function + 'F705,' # a return statement with arguments inside a generator + 'F706,' # a return statement outside of a function/method + 'F707,' # an except: block as not the last exception handler + 'F811,' # redefinition of unused name from line N + 'F812,' # list comprehension redefines 'foo' from line N + 'F821,' # undefined name 'Foo' + 'F822,' # undefined name name in __all__ + 'F823,' # local variable name … referenced before assignment + 'F831,' # duplicate argument name in function definition + 'F841,' # local variable 'foo' is assigned to but never used + 'W191,' # indentation contains tabs + 'W291,' # trailing whitespace + 'W292,' # no newline at end of file + 'W293,' # blank line contains whitespace + 'W601,' # .has_key() is deprecated, use "in" + 'W602,' # deprecated form of raising exception + 'W603,' # "<>" is deprecated, use "!=" + 'W604,' # backticks are deprecated, use "repr()" + 'W605,' # invalid escape sequence "x" + 'W606,' # 'async' and 'await' are reserved keywords starting with Python 3.7 +) + + +def check_dependencies(): + working_set = {pkg.key for pkg in pkg_resources.working_set} + + for dep in DEPS: + if dep not in working_set: + print(f"Skipping Python linting since {dep} is not installed.") + exit(0) + + +def main(): + check_dependencies() + + if len(sys.argv) > 1: + flake8_files = sys.argv[1:] + else: + files_args = ['git', 'ls-files', '*.py'] + flake8_files = subprocess.check_output(files_args).decode("utf-8").splitlines() + + flake8_args = ['flake8', '--ignore=B,C,E,F,I,N,W', f'--select={ENABLED}'] + flake8_files + flake8_env = os.environ.copy() + flake8_env["PYTHONWARNINGS"] = "ignore" + + try: + subprocess.check_call(flake8_args, env=flake8_env) + except subprocess.CalledProcessError: + exit(1) + + mypy_files = subprocess.check_output(FILES_ARGS).decode("utf-8").splitlines() + mypy_args = ['mypy', '--show-error-codes'] + mypy_files + + try: + subprocess.check_call(mypy_args) + except subprocess.CalledProcessError: + exit(1) + + +if __name__ == "__main__": + main() diff --git a/test/lint/lint-python.sh b/test/lint/lint-python.sh deleted file mode 100755 index 7d7857d325..0000000000 --- a/test/lint/lint-python.sh +++ /dev/null @@ -1,111 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2017-2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Check for specified flake8 warnings in python files. - -export LC_ALL=C -export MYPY_CACHE_DIR="${BASE_ROOT_DIR}/test/.mypy_cache" - -enabled=( - E101 # indentation contains mixed spaces and tabs - E112 # expected an indented block - E113 # unexpected indentation - E115 # expected an indented block (comment) - E116 # unexpected indentation (comment) - E125 # continuation line with same indent as next logical line - E129 # visually indented line with same indent as next logical line - E131 # continuation line unaligned for hanging indent - E133 # closing bracket is missing indentation - E223 # tab before operator - E224 # tab after operator - E242 # tab after ',' - E266 # too many leading '#' for block comment - E271 # multiple spaces after keyword - E272 # multiple spaces before keyword - E273 # tab after keyword - E274 # tab before keyword - E275 # missing whitespace after keyword - E304 # blank lines found after function decorator - E306 # expected 1 blank line before a nested definition - E401 # multiple imports on one line - E402 # module level import not at top of file - E502 # the backslash is redundant between brackets - E701 # multiple statements on one line (colon) - E702 # multiple statements on one line (semicolon) - E703 # statement ends with a semicolon - E711 # comparison to None should be 'if cond is None:' - E714 # test for object identity should be "is not" - E721 # do not compare types, use "isinstance()" - E742 # do not define classes named "l", "O", or "I" - E743 # do not define functions named "l", "O", or "I" - E901 # SyntaxError: invalid syntax - E902 # TokenError: EOF in multi-line string - F401 # module imported but unused - F402 # import module from line N shadowed by loop variable - F403 # 'from foo_module import *' used; unable to detect undefined names - F404 # future import(s) name after other statements - F405 # foo_function may be undefined, or defined from star imports: bar_module - F406 # "from module import *" only allowed at module level - F407 # an undefined __future__ feature name was imported - F601 # dictionary key name repeated with different values - F602 # dictionary key variable name repeated with different values - F621 # too many expressions in an assignment with star-unpacking - F622 # two or more starred expressions in an assignment (a, *b, *c = d) - F631 # assertion test is a tuple, which are always True - F632 # use ==/!= to compare str, bytes, and int literals - F701 # a break statement outside of a while or for loop - F702 # a continue statement outside of a while or for loop - F703 # a continue statement in a finally block in a loop - F704 # a yield or yield from statement outside of a function - F705 # a return statement with arguments inside a generator - F706 # a return statement outside of a function/method - F707 # an except: block as not the last exception handler - F811 # redefinition of unused name from line N - F812 # list comprehension redefines 'foo' from line N - F821 # undefined name 'Foo' - F822 # undefined name name in __all__ - F823 # local variable name … referenced before assignment - F831 # duplicate argument name in function definition - F841 # local variable 'foo' is assigned to but never used - W191 # indentation contains tabs - W291 # trailing whitespace - W292 # no newline at end of file - W293 # blank line contains whitespace - W601 # .has_key() is deprecated, use "in" - W602 # deprecated form of raising exception - W603 # "<>" is deprecated, use "!=" - W604 # backticks are deprecated, use "repr()" - W605 # invalid escape sequence "x" - W606 # 'async' and 'await' are reserved keywords starting with Python 3.7 -) - -if ! command -v flake8 > /dev/null; then - echo "Skipping Python linting since flake8 is not installed." - exit 0 -elif PYTHONWARNINGS="ignore" flake8 --version | grep -q "Python 2"; then - echo "Skipping Python linting since flake8 is running under Python 2. Install the Python 3 version of flake8." - exit 0 -fi - -EXIT_CODE=0 - -# shellcheck disable=SC2046 -if ! PYTHONWARNINGS="ignore" flake8 --ignore=B,C,E,F,I,N,W --select=$(IFS=","; echo "${enabled[*]}") $( - if [[ $# == 0 ]]; then - git ls-files "*.py" - else - echo "$@" - fi -); then - EXIT_CODE=1 -fi - -mapfile -t FILES < <(git ls-files "test/functional/*.py" "contrib/devtools/*.py") -if ! mypy --show-error-codes "${FILES[@]}"; then - EXIT_CODE=1 -fi - -exit $EXIT_CODE diff --git a/test/lint/lint-shell-locale.py b/test/lint/lint-shell-locale.py new file mode 100755 index 0000000000..f3dfe18a95 --- /dev/null +++ b/test/lint/lint-shell-locale.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2018-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" +Make sure all shell scripts are: +a.) explicitly opt out of locale dependence using + "export LC_ALL=C" or "export LC_ALL=C.UTF-8", or +b.) explicitly opt in to locale dependence using the annotation below. +""" + +import subprocess +import sys +import re + +OPT_IN_LINE = '# This script is intentionally locale dependent by not setting \"export LC_ALL=C\"' + +OPT_OUT_LINES = [ + 'export LC_ALL=C', + 'export LC_ALL=C.UTF-8', +] + +def get_shell_files_list(): + command = [ + 'git', + 'ls-files', + '--', + '*.sh', + ] + try: + return subprocess.check_output(command, stderr = subprocess.STDOUT).decode('utf-8').splitlines() + except subprocess.CalledProcessError as e: + if e.returncode > 1: # return code is 1 when match is empty + print(e.output.decode('utf-8'), end='') + sys.exit(1) + return [] + +def main(): + exit_code = 0 + shell_files = get_shell_files_list() + for file_path in shell_files: + if re.search('src/(secp256k1|minisketch|univalue)/', file_path): + continue + + with open(file_path, 'r', encoding='utf-8') as file_obj: + contents = file_obj.read() + + if OPT_IN_LINE in contents: + continue + + non_comment_pattern = re.compile(r'^\s*((?!#).+)$', re.MULTILINE) + non_comment_lines = re.findall(non_comment_pattern, contents) + if not non_comment_lines: + continue + + first_non_comment_line = non_comment_lines[0] + if first_non_comment_line not in OPT_OUT_LINES: + print(f'Missing "export LC_ALL=C" (to avoid locale dependence) as first non-comment non-empty line in {file_path}') + exit_code = 1 + + return sys.exit(exit_code) + +if __name__ == '__main__': + main() + diff --git a/test/lint/lint-shell-locale.sh b/test/lint/lint-shell-locale.sh deleted file mode 100755 index 4c6b8a57e6..0000000000 --- a/test/lint/lint-shell-locale.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Make sure all shell scripts: -# a.) explicitly opt out of locale dependence using -# "export LC_ALL=C" or "export LC_ALL=C.UTF-8", or -# b.) explicitly opt in to locale dependence using the annotation below. - -export LC_ALL=C - -EXIT_CODE=0 -for SHELL_SCRIPT in $(git ls-files -- "*.sh" | grep -vE "src/(secp256k1|minisketch|univalue)/"); do - if grep -q "# This script is intentionally locale dependent by not setting \"export LC_ALL=C\"" "${SHELL_SCRIPT}"; then - continue - fi - FIRST_NON_COMMENT_LINE=$(grep -vE '^(#.*)?$' "${SHELL_SCRIPT}" | head -1) - if [[ ${FIRST_NON_COMMENT_LINE} != "export LC_ALL=C" && ${FIRST_NON_COMMENT_LINE} != "export LC_ALL=C.UTF-8" ]]; then - echo "Missing \"export LC_ALL=C\" (to avoid locale dependence) as first non-comment non-empty line in ${SHELL_SCRIPT}" - EXIT_CODE=1 - fi -done -exit ${EXIT_CODE} diff --git a/test/lint/lint-submodule.py b/test/lint/lint-submodule.py new file mode 100755 index 0000000000..89d4c80f55 --- /dev/null +++ b/test/lint/lint-submodule.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" +This script checks for git modules +""" + +import subprocess +import sys + +def main(): + submodules_list = subprocess.check_output(['git', 'submodule', 'status', '--recursive'], + universal_newlines = True, encoding = 'utf8').rstrip('\n') + if submodules_list: + print("These submodules were found, delete them:\n", submodules_list) + sys.exit(1) + sys.exit(0) + +if __name__ == '__main__': + main() diff --git a/test/lint/lint-submodule.sh b/test/lint/lint-submodule.sh deleted file mode 100755 index d9aa021df7..0000000000 --- a/test/lint/lint-submodule.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# This script checks for git modules -export LC_ALL=C -EXIT_CODE=0 - -CMD=$(git submodule status --recursive) -if test -n "$CMD"; -then - echo These submodules were found, delete them: - echo "$CMD" - EXIT_CODE=1 -fi - -exit $EXIT_CODE - diff --git a/test/lint/lint-whitespace.py b/test/lint/lint-whitespace.py new file mode 100755 index 0000000000..d98fc8d9a2 --- /dev/null +++ b/test/lint/lint-whitespace.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2017-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# Check for new lines in diff that introduce trailing whitespace or +# tab characters instead of spaces. + +# We can't run this check unless we know the commit range for the PR. + +import argparse +import os +import re +import sys + +from subprocess import check_output + +EXCLUDED_DIRS = ["depends/patches/", + "contrib/guix/patches/", + "src/leveldb/", + "src/crc32c/", + "src/secp256k1/", + "src/minisketch/", + "src/univalue/", + "doc/release-notes/", + "src/qt/locale"] + +def parse_args(): + """Parse command line arguments.""" + parser = argparse.ArgumentParser( + description=""" + Check for new lines in diff that introduce trailing whitespace + or tab characters instead of spaces in unstaged changes, the + previous n commits, or a commit-range. + """, + epilog=f""" + You can manually set the commit-range with the COMMIT_RANGE + environment variable (e.g. "COMMIT_RANGE='47ba2c3...ee50c9e' + {sys.argv[0]}"). Defaults to current merge base when neither + prev-commits nor the environment variable is set. + """) + + parser.add_argument("--prev-commits", "-p", required=False, help="The previous n commits to check") + + return parser.parse_args() + + +def report_diff(selection): + filename = "" + seen = False + seenln = False + + print("The following changes were suspected:") + + for line in selection: + if re.match(r"^diff", line): + filename = line + seen = False + elif re.match(r"^@@", line): + linenumber = line + seenln = False + else: + if not seen: + # The first time a file is seen with trailing whitespace or a tab character, we print the + # filename (preceded by a newline). + print("") + print(filename) + seen = True + if not seenln: + print(linenumber) + seenln = True + print(line) + + +def get_diff(commit_range, check_only_code): + exclude_args = [":(exclude)" + dir for dir in EXCLUDED_DIRS] + + if check_only_code: + what_files = ["*.cpp", "*.h", "*.md", "*.py", "*.sh"] + else: + what_files = ["."] + + diff = check_output(["git", "diff", "-U0", commit_range, "--"] + what_files + exclude_args, universal_newlines=True, encoding="utf8") + + return diff + + +def main(): + args = parse_args() + + if not os.getenv("COMMIT_RANGE"): + if args.prev_commits: + commit_range = "HEAD~" + args.prev_commits + "...HEAD" + else: + # This assumes that the target branch of the pull request will be master. + merge_base = check_output(["git", "merge-base", "HEAD", "master"], universal_newlines=True, encoding="utf8").rstrip("\n") + commit_range = merge_base + "..HEAD" + else: + commit_range = os.getenv("COMMIT_RANGE") + + whitespace_selection = [] + tab_selection = [] + + # Check if trailing whitespace was found in the diff. + for line in get_diff(commit_range, check_only_code=False).splitlines(): + if re.match(r"^(diff --git|\@@|^\+.*\s+$)", line): + whitespace_selection.append(line) + + whitespace_additions = [i for i in whitespace_selection if i.startswith("+")] + + # Check if tab characters were found in the diff. + for line in get_diff(commit_range, check_only_code=True).splitlines(): + if re.match(r"^(diff --git|\@@|^\+.*\t)", line): + tab_selection.append(line) + + tab_additions = [i for i in tab_selection if i.startswith("+")] + + ret = 0 + + if len(whitespace_additions) > 0: + print("This diff appears to have added new lines with trailing whitespace.") + report_diff(whitespace_selection) + ret = 1 + + if len(tab_additions) > 0: + print("This diff appears to have added new lines with tab characters instead of spaces.") + report_diff(tab_selection) + ret = 1 + + sys.exit(ret) + + +if __name__ == "__main__": + main() diff --git a/test/lint/lint-whitespace.sh b/test/lint/lint-whitespace.sh deleted file mode 100755 index 9d55c71eb5..0000000000 --- a/test/lint/lint-whitespace.sh +++ /dev/null @@ -1,115 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2017-2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Check for new lines in diff that introduce trailing whitespace. - -# We can't run this check unless we know the commit range for the PR. - -export LC_ALL=C -while getopts "?" opt; do - case $opt in - ?) - echo "Usage: $0 [N]" - echo " COMMIT_RANGE='<commit range>' $0" - echo " $0 -?" - echo "Checks unstaged changes, the previous N commits, or a commit range." - echo "COMMIT_RANGE='47ba2c3...ee50c9e' $0" - exit 0 - ;; - esac -done - -if [ -z "${COMMIT_RANGE}" ]; then - if [ -n "$1" ]; then - COMMIT_RANGE="HEAD~$1...HEAD" - else - # This assumes that the target branch of the pull request will be master. - MERGE_BASE=$(git merge-base HEAD master) - COMMIT_RANGE="$MERGE_BASE..HEAD" - fi -fi - -showdiff() { - if ! git diff -U0 "${COMMIT_RANGE}" -- "." ":(exclude)depends/patches/" ":(exclude)contrib/guix/patches/" ":(exclude)src/leveldb/" ":(exclude)src/crc32c/" ":(exclude)src/secp256k1/" ":(exclude)src/minisketch/" ":(exclude)src/univalue/" ":(exclude)doc/release-notes/" ":(exclude)src/qt/locale/"; then - echo "Failed to get a diff" - exit 1 - fi -} - -showcodediff() { - if ! git diff -U0 "${COMMIT_RANGE}" -- *.cpp *.h *.md *.py *.sh ":(exclude)src/leveldb/" ":(exclude)src/crc32c/" ":(exclude)src/secp256k1/" ":(exclude)src/minisketch/" ":(exclude)src/univalue/" ":(exclude)doc/release-notes/" ":(exclude)src/qt/locale/"; then - echo "Failed to get a diff" - exit 1 - fi -} - -RET=0 - -# Check if trailing whitespace was found in the diff. -if showdiff | grep -E -q '^\+.*\s+$'; then - echo "This diff appears to have added new lines with trailing whitespace." - echo "The following changes were suspected:" - FILENAME="" - SEEN=0 - SEENLN=0 - while read -r line; do - if [[ "$line" =~ ^diff ]]; then - FILENAME="$line" - SEEN=0 - elif [[ "$line" =~ ^@@ ]]; then - LINENUMBER="$line" - SEENLN=0 - else - if [ "$SEEN" -eq 0 ]; then - # The first time a file is seen with trailing whitespace, we print the - # filename (preceded by a newline). - echo - echo "$FILENAME" - SEEN=1 - fi - if [ "$SEENLN" -eq 0 ]; then - echo "$LINENUMBER" - SEENLN=1 - fi - echo "$line" - fi - done < <(showdiff | grep -E '^(diff --git |@@|\+.*\s+$)') - RET=1 -fi - -# Check if tab characters were found in the diff. -if showcodediff | perl -nle '$MATCH++ if m{^\+.*\t}; END{exit 1 unless $MATCH>0}' > /dev/null; then - echo "This diff appears to have added new lines with tab characters instead of spaces." - echo "The following changes were suspected:" - FILENAME="" - SEEN=0 - SEENLN=0 - while read -r line; do - if [[ "$line" =~ ^diff ]]; then - FILENAME="$line" - SEEN=0 - elif [[ "$line" =~ ^@@ ]]; then - LINENUMBER="$line" - SEENLN=0 - else - if [ "$SEEN" -eq 0 ]; then - # The first time a file is seen with a tab character, we print the - # filename (preceded by a newline). - echo - echo "$FILENAME" - SEEN=1 - fi - if [ "$SEENLN" -eq 0 ]; then - echo "$LINENUMBER" - SEENLN=1 - fi - echo "$line" - fi - done < <(showcodediff | perl -nle 'print if m{^(diff --git |@@|\+.*\t)}') - RET=1 -fi - -exit $RET |