diff options
Diffstat (limited to 'test')
-rw-r--r-- | test/functional/README.md | 2 | ||||
-rwxr-xr-x | test/functional/interface_rest.py | 4 | ||||
-rwxr-xr-x | test/functional/p2p_segwit.py | 25 | ||||
-rwxr-xr-x | test/functional/rpc_misc.py | 2 | ||||
-rwxr-xr-x | test/functional/rpc_psbt.py | 17 | ||||
-rwxr-xr-x | test/functional/wallet_send.py | 17 | ||||
-rwxr-xr-x | test/functional/wallet_taproot.py | 14 | ||||
-rwxr-xr-x | test/lint/lint-format-strings.py | 98 | ||||
-rwxr-xr-x | test/lint/lint-format-strings.sh | 44 | ||||
-rwxr-xr-x | test/lint/lint-include-guards.py | 100 | ||||
-rwxr-xr-x | test/lint/lint-include-guards.sh | 30 | ||||
-rwxr-xr-x | test/lint/lint-includes.py | 179 | ||||
-rwxr-xr-x | test/lint/lint-includes.sh | 103 | ||||
-rwxr-xr-x | test/lint/lint-locale-dependence.py | 259 | ||||
-rwxr-xr-x | test/lint/lint-locale-dependence.sh | 241 | ||||
-rwxr-xr-x | test/lint/lint-python-utf8-encoding.py | 73 | ||||
-rwxr-xr-x | test/lint/lint-python-utf8-encoding.sh | 28 | ||||
-rwxr-xr-x | test/lint/lint-shell-locale.py | 67 | ||||
-rwxr-xr-x | test/lint/lint-shell-locale.sh | 25 | ||||
-rwxr-xr-x | test/lint/lint-submodule.py | 23 | ||||
-rwxr-xr-x | test/lint/lint-submodule.sh | 20 | ||||
-rwxr-xr-x | test/lint/lint-tests.py | 87 | ||||
-rwxr-xr-x | test/lint/lint-tests.sh | 35 |
23 files changed, 925 insertions, 568 deletions
diff --git a/test/functional/README.md b/test/functional/README.md index 926810cf03..914dbfd977 100644 --- a/test/functional/README.md +++ b/test/functional/README.md @@ -24,7 +24,7 @@ don't have test cases for. Consider using [pyenv](https://github.com/pyenv/pyenv), which checks [.python-version](/.python-version), to prevent accidentally introducing modern syntax from an unsupported Python version. The CI linter job also checks this, but [possibly not in all cases](https://github.com/bitcoin/bitcoin/pull/14884#discussion_r239585126). -- See [the python lint script](/test/lint/lint-python.sh) that checks for violations that +- See [the python lint script](/test/lint/lint-python.py) that checks for violations that could lead to bugs and issues in the test code. - Use [type hints](https://docs.python.org/3/library/typing.html) in your code to improve code readability and to detect possible bugs earlier. diff --git a/test/functional/interface_rest.py b/test/functional/interface_rest.py index 30c9e0c9cd..95dc40cb52 100755 --- a/test/functional/interface_rest.py +++ b/test/functional/interface_rest.py @@ -326,6 +326,10 @@ class RESTTest (BitcoinTestFramework): # Check that there are our submitted transactions in the TX memory pool json_obj = self.test_rest_request("/mempool/contents") + raw_mempool_verbose = self.nodes[0].getrawmempool(verbose=True) + + assert_equal(json_obj, raw_mempool_verbose) + for i, tx in enumerate(txs): assert tx in json_obj assert_equal(json_obj[tx]['spentby'], txs[i + 1:i + 2]) diff --git a/test/functional/p2p_segwit.py b/test/functional/p2p_segwit.py index f377fbaaa6..89ddfd3bcf 100755 --- a/test/functional/p2p_segwit.py +++ b/test/functional/p2p_segwit.py @@ -43,7 +43,6 @@ from test_framework.messages import ( ser_uint256, ser_vector, sha256, - tx_from_hex, ) from test_framework.p2p import ( P2PInterface, @@ -89,6 +88,8 @@ from test_framework.util import ( softfork_active, assert_raises_rpc_error, ) +from test_framework.wallet import MiniWallet + MAX_SIGOP_COST = 80000 @@ -221,9 +222,6 @@ class SegWitTest(BitcoinTestFramework): ] self.supports_cli = False - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() - # Helper functions def build_next_block(self): @@ -259,6 +257,7 @@ class SegWitTest(BitcoinTestFramework): self.log.info("Starting tests before segwit activation") self.segwit_active = False + self.wallet = MiniWallet(self.nodes[0]) self.test_non_witness_transaction() self.test_v0_outputs_arent_spendable() @@ -307,7 +306,7 @@ class SegWitTest(BitcoinTestFramework): self.test_node.send_and_ping(msg_no_witness_block(block)) # make sure the block was processed txid = block.vtx[0].sha256 - self.generate(self.nodes[0], 99) # let the block mature + self.generate(self.wallet, 99) # let the block mature # Create a transaction that spends the coinbase tx = CTransaction() @@ -1999,21 +1998,13 @@ class SegWitTest(BitcoinTestFramework): def serialize(self): return serialize_with_bogus_witness(self.tx) - self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(address_type='bech32'), 5) - self.generate(self.nodes[0], 1) - unspent = next(u for u in self.nodes[0].listunspent() if u['spendable'] and u['address'].startswith('bcrt')) - - raw = self.nodes[0].createrawtransaction([{"txid": unspent['txid'], "vout": unspent['vout']}], {self.nodes[0].getnewaddress(): 1}) - tx = tx_from_hex(raw) + tx = self.wallet.create_self_transfer(from_node=self.nodes[0])['tx'] assert_raises_rpc_error(-22, "TX decode failed", self.nodes[0].decoderawtransaction, hexstring=serialize_with_bogus_witness(tx).hex(), iswitness=True) - with self.nodes[0].assert_debug_log(['Superfluous witness record']): + with self.nodes[0].assert_debug_log(['Unknown transaction optional data']): self.test_node.send_and_ping(msg_bogus_tx(tx)) - raw = self.nodes[0].signrawtransactionwithwallet(raw) - assert raw['complete'] - raw = raw['hex'] - tx = tx_from_hex(raw) + tx.wit.vtxinwit = [] # drop witness assert_raises_rpc_error(-22, "TX decode failed", self.nodes[0].decoderawtransaction, hexstring=serialize_with_bogus_witness(tx).hex(), iswitness=True) - with self.nodes[0].assert_debug_log(['Unknown transaction optional data']): + with self.nodes[0].assert_debug_log(['Superfluous witness record']): self.test_node.send_and_ping(msg_bogus_tx(tx)) @subtest diff --git a/test/functional/rpc_misc.py b/test/functional/rpc_misc.py index f64aae7223..f6ee6a5215 100755 --- a/test/functional/rpc_misc.py +++ b/test/functional/rpc_misc.py @@ -27,7 +27,7 @@ class RpcMiscTest(BitcoinTestFramework): self.log.info("test CHECK_NONFATAL") assert_raises_rpc_error( -1, - 'Internal bug detected: \'request.params[9].get_str() != "trigger_internal_bug"\'', + 'Internal bug detected: "request.params[9].get_str() != "trigger_internal_bug""', lambda: node.echo(arg9='trigger_internal_bug'), ) diff --git a/test/functional/rpc_psbt.py b/test/functional/rpc_psbt.py index b037807b53..444e56610e 100755 --- a/test/functional/rpc_psbt.py +++ b/test/functional/rpc_psbt.py @@ -10,6 +10,10 @@ from itertools import product from test_framework.descriptors import descsum_create from test_framework.key import ECKey +from test_framework.messages import ( + ser_compact_size, + WITNESS_SCALE_FACTOR, +) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_approx, @@ -615,8 +619,8 @@ class PSBTTest(BitcoinTestFramework): self.nodes[1].createwallet("extfund") wallet = self.nodes[1].get_wallet_rpc("extfund") - # Make a weird but signable script. sh(pkh()) descriptor accomplishes this - desc = descsum_create("sh(pkh({}))".format(privkey)) + # Make a weird but signable script. sh(wsh(pkh())) descriptor accomplishes this + desc = descsum_create("sh(wsh(pkh({})))".format(privkey)) if self.options.descriptors: res = self.nodes[0].importdescriptors([{"desc": desc, "timestamp": "now"}]) else: @@ -634,7 +638,7 @@ class PSBTTest(BitcoinTestFramework): assert_raises_rpc_error(-4, "Insufficient funds", wallet.walletcreatefundedpsbt, [ext_utxo], {self.nodes[0].getnewaddress(): 15}) # But funding should work when the solving data is provided - psbt = wallet.walletcreatefundedpsbt([ext_utxo], {self.nodes[0].getnewaddress(): 15}, 0, {"add_inputs": True, "solving_data": {"pubkeys": [addr_info['pubkey']], "scripts": [addr_info["embedded"]["scriptPubKey"]]}}) + psbt = wallet.walletcreatefundedpsbt([ext_utxo], {self.nodes[0].getnewaddress(): 15}, 0, {"add_inputs": True, "solving_data": {"pubkeys": [addr_info['pubkey']], "scripts": [addr_info["embedded"]["scriptPubKey"], addr_info["embedded"]["embedded"]["scriptPubKey"]]}}) signed = wallet.walletprocesspsbt(psbt['psbt']) assert not signed['complete'] signed = self.nodes[0].walletprocesspsbt(signed['psbt']) @@ -655,10 +659,11 @@ class PSBTTest(BitcoinTestFramework): break psbt_in = dec["inputs"][input_idx] # Calculate the input weight - # (prevout + sequence + length of scriptSig + 2 bytes buffer) * 4 + len of scriptwitness + # (prevout + sequence + length of scriptSig + scriptsig + 1 byte buffer) * WITNESS_SCALE_FACTOR + num scriptWitness stack items + (length of stack item + stack item) * N stack items + 1 byte buffer len_scriptsig = len(psbt_in["final_scriptSig"]["hex"]) // 2 if "final_scriptSig" in psbt_in else 0 - len_scriptwitness = len(psbt_in["final_scriptwitness"]["hex"]) // 2 if "final_scriptwitness" in psbt_in else 0 - input_weight = ((41 + len_scriptsig + 2) * 4) + len_scriptwitness + len_scriptsig += len(ser_compact_size(len_scriptsig)) + 1 + len_scriptwitness = (sum([(len(x) // 2) + len(ser_compact_size(len(x) // 2)) for x in psbt_in["final_scriptwitness"]]) + len(psbt_in["final_scriptwitness"]) + 1) if "final_scriptwitness" in psbt_in else 0 + input_weight = ((40 + len_scriptsig) * WITNESS_SCALE_FACTOR) + len_scriptwitness low_input_weight = input_weight // 2 high_input_weight = input_weight * 2 diff --git a/test/functional/wallet_send.py b/test/functional/wallet_send.py index 86e36be8f7..07baa0595e 100755 --- a/test/functional/wallet_send.py +++ b/test/functional/wallet_send.py @@ -10,6 +10,10 @@ from itertools import product from test_framework.authproxy import JSONRPCException from test_framework.descriptors import descsum_create from test_framework.key import ECKey +from test_framework.messages import ( + ser_compact_size, + WITNESS_SCALE_FACTOR, +) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, @@ -488,8 +492,8 @@ class WalletSendTest(BitcoinTestFramework): self.nodes[1].createwallet("extfund") ext_fund = self.nodes[1].get_wallet_rpc("extfund") - # Make a weird but signable script. sh(pkh()) descriptor accomplishes this - desc = descsum_create("sh(pkh({}))".format(privkey)) + # Make a weird but signable script. sh(wsh(pkh())) descriptor accomplishes this + desc = descsum_create("sh(wsh(pkh({})))".format(privkey)) if self.options.descriptors: res = ext_fund.importdescriptors([{"desc": desc, "timestamp": "now"}]) else: @@ -507,7 +511,7 @@ class WalletSendTest(BitcoinTestFramework): self.test_send(from_wallet=ext_wallet, to_wallet=self.nodes[0], amount=15, inputs=[ext_utxo], add_inputs=True, psbt=True, include_watching=True, expect_error=(-4, "Insufficient funds")) # But funding should work when the solving data is provided - res = self.test_send(from_wallet=ext_wallet, to_wallet=self.nodes[0], amount=15, inputs=[ext_utxo], add_inputs=True, psbt=True, include_watching=True, solving_data={"pubkeys": [addr_info['pubkey']], "scripts": [addr_info["embedded"]["scriptPubKey"]]}) + res = self.test_send(from_wallet=ext_wallet, to_wallet=self.nodes[0], amount=15, inputs=[ext_utxo], add_inputs=True, psbt=True, include_watching=True, solving_data={"pubkeys": [addr_info['pubkey']], "scripts": [addr_info["embedded"]["scriptPubKey"], addr_info["embedded"]["embedded"]["scriptPubKey"]]}) signed = ext_wallet.walletprocesspsbt(res["psbt"]) signed = ext_fund.walletprocesspsbt(res["psbt"]) assert signed["complete"] @@ -526,10 +530,11 @@ class WalletSendTest(BitcoinTestFramework): break psbt_in = dec["inputs"][input_idx] # Calculate the input weight - # (prevout + sequence + length of scriptSig + 2 bytes buffer) * 4 + len of scriptwitness + # (prevout + sequence + length of scriptSig + scriptsig + 1 byte buffer) * WITNESS_SCALE_FACTOR + num scriptWitness stack items + (length of stack item + stack item) * N stack items + 1 byte buffer len_scriptsig = len(psbt_in["final_scriptSig"]["hex"]) // 2 if "final_scriptSig" in psbt_in else 0 - len_scriptwitness = len(psbt_in["final_scriptwitness"]["hex"]) // 2 if "final_scriptwitness" in psbt_in else 0 - input_weight = ((41 + len_scriptsig + 2) * 4) + len_scriptwitness + len_scriptsig += len(ser_compact_size(len_scriptsig)) + 1 + len_scriptwitness = (sum([(len(x) // 2) + len(ser_compact_size(len(x) // 2)) for x in psbt_in["final_scriptwitness"]]) + len(psbt_in["final_scriptwitness"]) + 1) if "final_scriptwitness" in psbt_in else 0 + input_weight = ((40 + len_scriptsig) * WITNESS_SCALE_FACTOR) + len_scriptwitness # Input weight error conditions assert_raises_rpc_error( diff --git a/test/functional/wallet_taproot.py b/test/functional/wallet_taproot.py index d3731b135a..41bb86f962 100755 --- a/test/functional/wallet_taproot.py +++ b/test/functional/wallet_taproot.py @@ -192,9 +192,9 @@ class WalletTaprootTest(BitcoinTestFramework): """Test generation and spending of P2TR address outputs.""" def set_test_params(self): - self.num_nodes = 3 + self.num_nodes = 2 self.setup_clean_chain = True - self.extra_args = [['-keypool=100'], ['-keypool=100'], ["-vbparams=taproot:1:1"]] + self.extra_args = [['-keypool=100'], ['-keypool=100']] self.supports_cli = False def skip_test_if_missing_module(self): @@ -243,15 +243,11 @@ class WalletTaprootTest(BitcoinTestFramework): assert_equal(len(rederive), 1) assert_equal(rederive[0], addr_g) - # tr descriptors can be imported regardless of Taproot status + # tr descriptors can be imported result = self.privs_tr_enabled.importdescriptors([{"desc": desc, "timestamp": "now"}]) assert(result[0]["success"]) result = self.pubs_tr_enabled.importdescriptors([{"desc": desc_pub, "timestamp": "now"}]) assert(result[0]["success"]) - result = self.privs_tr_disabled.importdescriptors([{"desc": desc, "timestamp": "now"}]) - assert result[0]["success"] - result = self.pubs_tr_disabled.importdescriptors([{"desc": desc_pub, "timestamp": "now"}]) - assert result[0]["success"] def do_test_sendtoaddress(self, comment, pattern, privmap, treefn, keys_pay, keys_change): self.log.info("Testing %s through sendtoaddress" % comment) @@ -328,12 +324,8 @@ class WalletTaprootTest(BitcoinTestFramework): self.log.info("Creating wallets...") self.nodes[0].createwallet(wallet_name="privs_tr_enabled", descriptors=True, blank=True) self.privs_tr_enabled = self.nodes[0].get_wallet_rpc("privs_tr_enabled") - self.nodes[2].createwallet(wallet_name="privs_tr_disabled", descriptors=True, blank=True) - self.privs_tr_disabled=self.nodes[2].get_wallet_rpc("privs_tr_disabled") self.nodes[0].createwallet(wallet_name="pubs_tr_enabled", descriptors=True, blank=True, disable_private_keys=True) self.pubs_tr_enabled = self.nodes[0].get_wallet_rpc("pubs_tr_enabled") - self.nodes[2].createwallet(wallet_name="pubs_tr_disabled", descriptors=True, blank=True, disable_private_keys=True) - self.pubs_tr_disabled=self.nodes[2].get_wallet_rpc("pubs_tr_disabled") self.nodes[0].createwallet(wallet_name="boring") self.nodes[0].createwallet(wallet_name="addr_gen", descriptors=True, disable_private_keys=True, blank=True) self.nodes[0].createwallet(wallet_name="rpc_online", descriptors=True, blank=True) diff --git a/test/lint/lint-format-strings.py b/test/lint/lint-format-strings.py new file mode 100755 index 0000000000..5a36da11fd --- /dev/null +++ b/test/lint/lint-format-strings.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2018-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# + +""" +Lint format strings: This program checks that the number of arguments passed +to a variadic format string function matches the number of format specifiers +in the format string. +""" + +import subprocess +import re +import sys + +FUNCTION_NAMES_AND_NUMBER_OF_LEADING_ARGUMENTS = [ + 'FatalError,0', + 'fprintf,1', + 'tfm::format,1', # Assuming tfm::::format(std::ostream&, ... + 'LogConnectFailure,1', + 'LogPrint,1', + 'LogPrintf,0', + 'printf,0', + 'snprintf,2', + 'sprintf,1', + 'strprintf,0', + 'vfprintf,1', + 'vprintf,1', + 'vsnprintf,1', + 'vsprintf,1', + 'WalletLogPrintf,0', +] +RUN_LINT_FILE = 'test/lint/run-lint-format-strings.py' + +def check_doctest(): + command = [ + 'python3', + '-m', + 'doctest', + RUN_LINT_FILE, + ] + try: + subprocess.run(command, check = True) + except subprocess.CalledProcessError: + sys.exit(1) + +def get_matching_files(function_name): + command = [ + 'git', + 'grep', + '--full-name', + '-l', + function_name, + '--', + '*.c', + '*.cpp', + '*.h', + ] + try: + return subprocess.check_output(command, stderr = subprocess.STDOUT).decode('utf-8').splitlines() + except subprocess.CalledProcessError as e: + if e.returncode > 1: # return code is 1 when match is empty + print(e.output.decode('utf-8'), end='') + sys.exit(1) + return [] + +def main(): + exit_code = 0 + check_doctest() + for s in FUNCTION_NAMES_AND_NUMBER_OF_LEADING_ARGUMENTS: + function_name, skip_arguments = s.split(',') + matching_files = get_matching_files(function_name) + + matching_files_filtered = [] + for matching_file in matching_files: + if not re.search('^src/(leveldb|secp256k1|minisketch|tinyformat|univalue|test/fuzz/strprintf.cpp)', matching_file): + matching_files_filtered.append(matching_file) + matching_files_filtered.sort() + + run_lint_args = [ + RUN_LINT_FILE, + '--skip-arguments', + skip_arguments, + function_name, + ] + run_lint_args.extend(matching_files_filtered) + + try: + subprocess.run(run_lint_args, check = True) + except subprocess.CalledProcessError: + exit_code = 1 + + sys.exit(exit_code) + +if __name__ == '__main__': + main() diff --git a/test/lint/lint-format-strings.sh b/test/lint/lint-format-strings.sh deleted file mode 100755 index 73730f16b3..0000000000 --- a/test/lint/lint-format-strings.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Lint format strings: This program checks that the number of arguments passed -# to a variadic format string function matches the number of format specifiers -# in the format string. - -export LC_ALL=C - -FUNCTION_NAMES_AND_NUMBER_OF_LEADING_ARGUMENTS=( - "FatalError,0" - "fprintf,1" - "tfm::format,1" # Assuming tfm::::format(std::ostream&, ... - "LogConnectFailure,1" - "LogPrint,1" - "LogPrintf,0" - "printf,0" - "snprintf,2" - "sprintf,1" - "strprintf,0" - "vfprintf,1" - "vprintf,1" - "vsnprintf,1" - "vsprintf,1" - "WalletLogPrintf,0" -) - -EXIT_CODE=0 -if ! python3 -m doctest "test/lint/run-lint-format-strings.py"; then - EXIT_CODE=1 -fi -for S in "${FUNCTION_NAMES_AND_NUMBER_OF_LEADING_ARGUMENTS[@]}"; do - IFS="," read -r FUNCTION_NAME SKIP_ARGUMENTS <<< "${S}" - for MATCHING_FILE in $(git grep --full-name -l "${FUNCTION_NAME}" -- "*.c" "*.cpp" "*.h" | sort | grep -vE "^src/(leveldb|secp256k1|minisketch|tinyformat|univalue|test/fuzz/strprintf.cpp)"); do - MATCHING_FILES+=("${MATCHING_FILE}") - done - if ! "test/lint/run-lint-format-strings.py" --skip-arguments "${SKIP_ARGUMENTS}" "${FUNCTION_NAME}" "${MATCHING_FILES[@]}"; then - EXIT_CODE=1 - fi -done -exit ${EXIT_CODE} diff --git a/test/lint/lint-include-guards.py b/test/lint/lint-include-guards.py new file mode 100755 index 0000000000..86284517d5 --- /dev/null +++ b/test/lint/lint-include-guards.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2018-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" +Check include guards. +""" + +import re +import sys +from subprocess import check_output +from typing import List + + +HEADER_ID_PREFIX = 'BITCOIN_' +HEADER_ID_SUFFIX = '_H' + +EXCLUDE_FILES_WITH_PREFIX = ['src/crypto/ctaes', + 'src/leveldb', + 'src/crc32c', + 'src/secp256k1', + 'src/minisketch', + 'src/univalue', + 'src/tinyformat.h', + 'src/bench/nanobench.h', + 'src/test/fuzz/FuzzedDataProvider.h'] + + +def _get_header_file_lst() -> List[str]: + """ Helper function to get a list of header filepaths to be + checked for include guards. + """ + git_cmd_lst = ['git', 'ls-files', '--', '*.h'] + header_file_lst = check_output( + git_cmd_lst).decode('utf-8').splitlines() + + header_file_lst = [hf for hf in header_file_lst + if not any(ef in hf for ef + in EXCLUDE_FILES_WITH_PREFIX)] + + return header_file_lst + + +def _get_header_id(header_file: str) -> str: + """ Helper function to get the header id from a header file + string. + + eg: 'src/wallet/walletdb.h' -> 'BITCOIN_WALLET_WALLETDB_H' + + Args: + header_file: Filepath to header file. + + Returns: + The header id. + """ + header_id_base = header_file.split('/')[1:] + header_id_base = '_'.join(header_id_base) + header_id_base = header_id_base.replace('.h', '').replace('-', '_') + header_id_base = header_id_base.upper() + + header_id = f'{HEADER_ID_PREFIX}{header_id_base}{HEADER_ID_SUFFIX}' + + return header_id + + +def main(): + exit_code = 0 + + header_file_lst = _get_header_file_lst() + for header_file in header_file_lst: + header_id = _get_header_id(header_file) + + regex_pattern = f'^#(ifndef|define|endif //) {header_id}' + + with open(header_file, 'r', encoding='utf-8') as f: + header_file_contents = f.readlines() + + count = 0 + for header_file_contents_string in header_file_contents: + include_guard_lst = re.findall( + regex_pattern, header_file_contents_string) + + count += len(include_guard_lst) + + if count != 3: + print(f'{header_file} seems to be missing the expected ' + 'include guard:') + print(f' #ifndef {header_id}') + print(f' #define {header_id}') + print(' ...') + print(f' #endif // {header_id}\n') + exit_code = 1 + + sys.exit(exit_code) + + +if __name__ == '__main__': + main() diff --git a/test/lint/lint-include-guards.sh b/test/lint/lint-include-guards.sh deleted file mode 100755 index f14218aa74..0000000000 --- a/test/lint/lint-include-guards.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Check include guards. - -export LC_ALL=C -HEADER_ID_PREFIX="BITCOIN_" -HEADER_ID_SUFFIX="_H" - -REGEXP_EXCLUDE_FILES_WITH_PREFIX="src/(crypto/ctaes/|leveldb/|crc32c/|secp256k1/|minisketch/|test/fuzz/FuzzedDataProvider.h|tinyformat.h|bench/nanobench.h|univalue/)" - -EXIT_CODE=0 -for HEADER_FILE in $(git ls-files -- "*.h" | grep -vE "^${REGEXP_EXCLUDE_FILES_WITH_PREFIX}") -do - HEADER_ID_BASE=$(cut -f2- -d/ <<< "${HEADER_FILE}" | sed "s/\.h$//g" | tr / _ | tr - _ | tr "[:lower:]" "[:upper:]") - HEADER_ID="${HEADER_ID_PREFIX}${HEADER_ID_BASE}${HEADER_ID_SUFFIX}" - if [[ $(grep --count --extended-regexp "^#(ifndef|define|endif //) ${HEADER_ID}" "${HEADER_FILE}") != 3 ]]; then - echo "${HEADER_FILE} seems to be missing the expected include guard:" - echo " #ifndef ${HEADER_ID}" - echo " #define ${HEADER_ID}" - echo " ..." - echo " #endif // ${HEADER_ID}" - echo - EXIT_CODE=1 - fi -done -exit ${EXIT_CODE} diff --git a/test/lint/lint-includes.py b/test/lint/lint-includes.py new file mode 100755 index 0000000000..b29c7f8b4d --- /dev/null +++ b/test/lint/lint-includes.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2018-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# Check for duplicate includes. +# Guard against accidental introduction of new Boost dependencies. +# Check includes: Check for duplicate includes. Enforce bracket syntax includes. + +import os +import re +import sys + +from subprocess import check_output, CalledProcessError + + +EXCLUDED_DIRS = ["src/leveldb/", + "src/crc32c/", + "src/secp256k1/", + "src/minisketch/", + "src/univalue/"] + +EXPECTED_BOOST_INCLUDES = ["boost/algorithm/string.hpp", + "boost/algorithm/string/classification.hpp", + "boost/algorithm/string/replace.hpp", + "boost/algorithm/string/split.hpp", + "boost/date_time/posix_time/posix_time.hpp", + "boost/multi_index/hashed_index.hpp", + "boost/multi_index/ordered_index.hpp", + "boost/multi_index/sequenced_index.hpp", + "boost/multi_index_container.hpp", + "boost/process.hpp", + "boost/signals2/connection.hpp", + "boost/signals2/optional_last_value.hpp", + "boost/signals2/signal.hpp", + "boost/test/included/unit_test.hpp", + "boost/test/unit_test.hpp"] + + +def get_toplevel(): + return check_output(["git", "rev-parse", "--show-toplevel"], universal_newlines=True, encoding="utf8").rstrip("\n") + + +def list_files_by_suffix(suffixes): + exclude_args = [":(exclude)" + dir for dir in EXCLUDED_DIRS] + + files_list = check_output(["git", "ls-files", "src"] + exclude_args, universal_newlines=True, encoding="utf8").splitlines() + + return [file for file in files_list if file.endswith(suffixes)] + + +def find_duplicate_includes(include_list): + tempset = set() + duplicates = set() + + for inclusion in include_list: + if inclusion in tempset: + duplicates.add(inclusion) + else: + tempset.add(inclusion) + + return duplicates + + +def find_included_cpps(): + included_cpps = list() + + try: + included_cpps = check_output(["git", "grep", "-E", r"^#include [<\"][^>\"]+\.cpp[>\"]", "--", "*.cpp", "*.h"], universal_newlines=True, encoding="utf8").splitlines() + except CalledProcessError as e: + if e.returncode > 1: + raise e + + return included_cpps + + +def find_extra_boosts(): + included_boosts = list() + filtered_included_boost_set = set() + exclusion_set = set() + + try: + included_boosts = check_output(["git", "grep", "-E", r"^#include <boost/", "--", "*.cpp", "*.h"], universal_newlines=True, encoding="utf8").splitlines() + except CalledProcessError as e: + if e.returncode > 1: + raise e + + for boost in included_boosts: + filtered_included_boost_set.add(re.findall(r'(?<=\<).+?(?=\>)', boost)[0]) + + for expected_boost in EXPECTED_BOOST_INCLUDES: + for boost in filtered_included_boost_set: + if expected_boost in boost: + exclusion_set.add(boost) + + extra_boosts = set(filtered_included_boost_set.difference(exclusion_set)) + + return extra_boosts + + +def find_quote_syntax_inclusions(): + exclude_args = [":(exclude)" + dir for dir in EXCLUDED_DIRS] + quote_syntax_inclusions = list() + + try: + quote_syntax_inclusions = check_output(["git", "grep", r"^#include \"", "--", "*.cpp", "*.h"] + exclude_args, universal_newlines=True, encoding="utf8").splitlines() + except CalledProcessError as e: + if e.returncode > 1: + raise e + + return quote_syntax_inclusions + + +def main(): + exit_code = 0 + + os.chdir(get_toplevel()) + + # Check for duplicate includes + for filename in list_files_by_suffix((".cpp", ".h")): + with open(filename, "r", encoding="utf8") as file: + include_list = [line.rstrip("\n") for line in file if re.match(r"^#include", line)] + + duplicates = find_duplicate_includes(include_list) + + if duplicates: + print(f"Duplicate include(s) in {filename}:") + for duplicate in duplicates: + print(duplicate) + print("") + exit_code = 1 + + # Check if code includes .cpp-files + included_cpps = find_included_cpps() + + if included_cpps: + print("The following files #include .cpp files:") + for included_cpp in included_cpps: + print(included_cpp) + print("") + exit_code = 1 + + # Guard against accidental introduction of new Boost dependencies + extra_boosts = find_extra_boosts() + + if extra_boosts: + for boost in extra_boosts: + print(f"A new Boost dependency in the form of \"{boost}\" appears to have been introduced:") + print(check_output(["git", "grep", boost, "--", "*.cpp", "*.h"], universal_newlines=True, encoding="utf8")) + exit_code = 1 + + # Check if Boost dependencies are no longer used + for expected_boost in EXPECTED_BOOST_INCLUDES: + try: + check_output(["git", "grep", "-q", r"^#include <%s>" % expected_boost, "--", "*.cpp", "*.h"], universal_newlines=True, encoding="utf8") + except CalledProcessError as e: + if e.returncode > 1: + raise e + else: + print(f"Good job! The Boost dependency \"{expected_boost}\" is no longer used. " + "Please remove it from EXPECTED_BOOST_INCLUDES in test/lint/lint-includes.py " + "to make sure this dependency is not accidentally reintroduced.\n") + exit_code = 1 + + # Enforce bracket syntax includes + quote_syntax_inclusions = find_quote_syntax_inclusions() + + if quote_syntax_inclusions: + print("Please use bracket syntax includes (\"#include <foo.h>\") instead of quote syntax includes:") + for quote_syntax_inclusion in quote_syntax_inclusions: + print(quote_syntax_inclusion) + exit_code = 1 + + sys.exit(exit_code) + + +if __name__ == "__main__": + main() diff --git a/test/lint/lint-includes.sh b/test/lint/lint-includes.sh deleted file mode 100755 index 9e72831ee9..0000000000 --- a/test/lint/lint-includes.sh +++ /dev/null @@ -1,103 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Check for duplicate includes. -# Guard against accidental introduction of new Boost dependencies. -# Check includes: Check for duplicate includes. Enforce bracket syntax includes. - -export LC_ALL=C -IGNORE_REGEXP="/(leveldb|secp256k1|minisketch|univalue|crc32c)/" - -# cd to root folder of git repo for git ls-files to work properly -cd "$(dirname "$0")/../.." || exit 1 - -filter_suffix() { - git ls-files | grep -E "^src/.*\.${1}"'$' | grep -Ev "${IGNORE_REGEXP}" -} - -EXIT_CODE=0 - -for HEADER_FILE in $(filter_suffix h); do - DUPLICATE_INCLUDES_IN_HEADER_FILE=$(grep -E "^#include " < "${HEADER_FILE}" | sort | uniq -d) - if [[ ${DUPLICATE_INCLUDES_IN_HEADER_FILE} != "" ]]; then - echo "Duplicate include(s) in ${HEADER_FILE}:" - echo "${DUPLICATE_INCLUDES_IN_HEADER_FILE}" - echo - EXIT_CODE=1 - fi -done - -for CPP_FILE in $(filter_suffix cpp); do - DUPLICATE_INCLUDES_IN_CPP_FILE=$(grep -E "^#include " < "${CPP_FILE}" | sort | uniq -d) - if [[ ${DUPLICATE_INCLUDES_IN_CPP_FILE} != "" ]]; then - echo "Duplicate include(s) in ${CPP_FILE}:" - echo "${DUPLICATE_INCLUDES_IN_CPP_FILE}" - echo - EXIT_CODE=1 - fi -done - -INCLUDED_CPP_FILES=$(git grep -E "^#include [<\"][^>\"]+\.cpp[>\"]" -- "*.cpp" "*.h") -if [[ ${INCLUDED_CPP_FILES} != "" ]]; then - echo "The following files #include .cpp files:" - echo "${INCLUDED_CPP_FILES}" - echo - EXIT_CODE=1 -fi - -EXPECTED_BOOST_INCLUDES=( - boost/algorithm/string.hpp - boost/algorithm/string/classification.hpp - boost/algorithm/string/replace.hpp - boost/algorithm/string/split.hpp - boost/date_time/posix_time/posix_time.hpp - boost/multi_index/hashed_index.hpp - boost/multi_index/ordered_index.hpp - boost/multi_index/sequenced_index.hpp - boost/multi_index_container.hpp - boost/process.hpp - boost/signals2/connection.hpp - boost/signals2/optional_last_value.hpp - boost/signals2/signal.hpp - boost/test/included/unit_test.hpp - boost/test/unit_test.hpp -) - -for BOOST_INCLUDE in $(git grep '^#include <boost/' -- "*.cpp" "*.h" | cut -f2 -d: | cut -f2 -d'<' | cut -f1 -d'>' | sort -u); do - IS_EXPECTED_INCLUDE=0 - for EXPECTED_BOOST_INCLUDE in "${EXPECTED_BOOST_INCLUDES[@]}"; do - if [[ "${BOOST_INCLUDE}" == "${EXPECTED_BOOST_INCLUDE}" ]]; then - IS_EXPECTED_INCLUDE=1 - break - fi - done - if [[ ${IS_EXPECTED_INCLUDE} == 0 ]]; then - EXIT_CODE=1 - echo "A new Boost dependency in the form of \"${BOOST_INCLUDE}\" appears to have been introduced:" - git grep "${BOOST_INCLUDE}" -- "*.cpp" "*.h" - echo - fi -done - -for EXPECTED_BOOST_INCLUDE in "${EXPECTED_BOOST_INCLUDES[@]}"; do - if ! git grep -q "^#include <${EXPECTED_BOOST_INCLUDE}>" -- "*.cpp" "*.h"; then - echo "Good job! The Boost dependency \"${EXPECTED_BOOST_INCLUDE}\" is no longer used." - echo "Please remove it from EXPECTED_BOOST_INCLUDES in $0" - echo "to make sure this dependency is not accidentally reintroduced." - echo - EXIT_CODE=1 - fi -done - -QUOTE_SYNTAX_INCLUDES=$(git grep '^#include "' -- "*.cpp" "*.h" | grep -Ev "${IGNORE_REGEXP}") -if [[ ${QUOTE_SYNTAX_INCLUDES} != "" ]]; then - echo "Please use bracket syntax includes (\"#include <foo.h>\") instead of quote syntax includes:" - echo "${QUOTE_SYNTAX_INCLUDES}" - echo - EXIT_CODE=1 -fi - -exit ${EXIT_CODE} diff --git a/test/lint/lint-locale-dependence.py b/test/lint/lint-locale-dependence.py new file mode 100755 index 0000000000..2abf1be6b3 --- /dev/null +++ b/test/lint/lint-locale-dependence.py @@ -0,0 +1,259 @@ +#!/usr/bin/env python3 +# Copyright (c) 2018-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# Be aware that bitcoind and bitcoin-qt differ in terms of localization: Qt +# opts in to POSIX localization by running setlocale(LC_ALL, "") on startup, +# whereas no such call is made in bitcoind. +# +# Qt runs setlocale(LC_ALL, "") on initialization. This installs the locale +# specified by the user's LC_ALL (or LC_*) environment variable as the new +# C locale. +# +# In contrast, bitcoind does not opt in to localization -- no call to +# setlocale(LC_ALL, "") is made and the environment variables LC_* are +# thus ignored. +# +# This results in situations where bitcoind is guaranteed to be running +# with the classic locale ("C") whereas the locale of bitcoin-qt will vary +# depending on the user's environment variables. +# +# An example: Assuming the environment variable LC_ALL=de_DE then the +# call std::to_string(1.23) will return "1.230000" in bitcoind but +# "1,230000" in bitcoin-qt. +# +# From the Qt documentation: +# "On Unix/Linux Qt is configured to use the system locale settings by default. +# This can cause a conflict when using POSIX functions, for instance, when +# converting between data types such as floats and strings, since the notation +# may differ between locales. To get around this problem, call the POSIX function +# setlocale(LC_NUMERIC,"C") right after initializing QApplication, QGuiApplication +# or QCoreApplication to reset the locale that is used for number formatting to +# "C"-locale." +# +# See https://doc.qt.io/qt-5/qcoreapplication.html#locale-settings and +# https://stackoverflow.com/a/34878283 for more details. +# +# TODO: Reduce KNOWN_VIOLATIONS by replacing uses of locale dependent snprintf with strprintf. + +import re +import sys + +from subprocess import check_output, CalledProcessError + + +KNOWN_VIOLATIONS = [ + "src/dbwrapper.cpp:.*vsnprintf", + "src/test/dbwrapper_tests.cpp:.*snprintf", + "src/test/fuzz/locale.cpp:.*setlocale", + "src/test/fuzz/string.cpp:.*strtol", + "src/test/fuzz/string.cpp:.*strtoul", + "src/test/util_tests.cpp:.*strtoll" +] + +REGEXP_EXTERNAL_DEPENDENCIES_EXCLUSIONS = [ + "src/crypto/ctaes/", + "src/leveldb/", + "src/secp256k1/", + "src/minisketch/", + "src/tinyformat.h", + "src/univalue/" +] + +LOCALE_DEPENDENT_FUNCTIONS = [ + "alphasort", # LC_COLLATE (via strcoll) + "asctime", # LC_TIME (directly) + "asprintf", # (via vasprintf) + "atof", # LC_NUMERIC (via strtod) + "atoi", # LC_NUMERIC (via strtol) + "atol", # LC_NUMERIC (via strtol) + "atoll", # (via strtoll) + "atoq", + "btowc", # LC_CTYPE (directly) + "ctime", # (via asctime or localtime) + "dprintf", # (via vdprintf) + "fgetwc", + "fgetws", + "fold_case", # boost::locale::fold_case + "fprintf", # (via vfprintf) + "fputwc", + "fputws", + "fscanf", # (via __vfscanf) + "fwprintf", # (via __vfwprintf) + "getdate", # via __getdate_r => isspace // __localtime_r + "getwc", + "getwchar", + "is_digit", # boost::algorithm::is_digit + "is_space", # boost::algorithm::is_space + "isalnum", # LC_CTYPE + "isalpha", # LC_CTYPE + "isblank", # LC_CTYPE + "iscntrl", # LC_CTYPE + "isctype", # LC_CTYPE + "isdigit", # LC_CTYPE + "isgraph", # LC_CTYPE + "islower", # LC_CTYPE + "isprint", # LC_CTYPE + "ispunct", # LC_CTYPE + "isspace", # LC_CTYPE + "isupper", # LC_CTYPE + "iswalnum", # LC_CTYPE + "iswalpha", # LC_CTYPE + "iswblank", # LC_CTYPE + "iswcntrl", # LC_CTYPE + "iswctype", # LC_CTYPE + "iswdigit", # LC_CTYPE + "iswgraph", # LC_CTYPE + "iswlower", # LC_CTYPE + "iswprint", # LC_CTYPE + "iswpunct", # LC_CTYPE + "iswspace", # LC_CTYPE + "iswupper", # LC_CTYPE + "iswxdigit", # LC_CTYPE + "isxdigit", # LC_CTYPE + "localeconv", # LC_NUMERIC + LC_MONETARY + "mblen", # LC_CTYPE + "mbrlen", + "mbrtowc", + "mbsinit", + "mbsnrtowcs", + "mbsrtowcs", + "mbstowcs", # LC_CTYPE + "mbtowc", # LC_CTYPE + "mktime", + "normalize", # boost::locale::normalize + "printf", # LC_NUMERIC + "putwc", + "putwchar", + "scanf", # LC_NUMERIC + "setlocale", + "snprintf", + "sprintf", + "sscanf", + "std::locale::global", + "std::to_string", + "stod", + "stof", + "stoi", + "stol", + "stold", + "stoll", + "stoul", + "stoull", + "strcasecmp", + "strcasestr", + "strcoll", # LC_COLLATE + #"strerror", + "strfmon", + "strftime", # LC_TIME + "strncasecmp", + "strptime", + "strtod", # LC_NUMERIC + "strtof", + "strtoimax", + "strtol", # LC_NUMERIC + "strtold", + "strtoll", + "strtoq", + "strtoul", # LC_NUMERIC + "strtoull", + "strtoumax", + "strtouq", + "strxfrm", # LC_COLLATE + "swprintf", + "to_lower", # boost::locale::to_lower + "to_title", # boost::locale::to_title + "to_upper", # boost::locale::to_upper + "tolower", # LC_CTYPE + "toupper", # LC_CTYPE + "towctrans", + "towlower", # LC_CTYPE + "towupper", # LC_CTYPE + "trim", # boost::algorithm::trim + "trim_left", # boost::algorithm::trim_left + "trim_right", # boost::algorithm::trim_right + "ungetwc", + "vasprintf", + "vdprintf", + "versionsort", + "vfprintf", + "vfscanf", + "vfwprintf", + "vprintf", + "vscanf", + "vsnprintf", + "vsprintf", + "vsscanf", + "vswprintf", + "vwprintf", + "wcrtomb", + "wcscasecmp", + "wcscoll", # LC_COLLATE + "wcsftime", # LC_TIME + "wcsncasecmp", + "wcsnrtombs", + "wcsrtombs", + "wcstod", # LC_NUMERIC + "wcstof", + "wcstoimax", + "wcstol", # LC_NUMERIC + "wcstold", + "wcstoll", + "wcstombs", # LC_CTYPE + "wcstoul", # LC_NUMERIC + "wcstoull", + "wcstoumax", + "wcswidth", + "wcsxfrm", # LC_COLLATE + "wctob", + "wctomb", # LC_CTYPE + "wctrans", + "wctype", + "wcwidth", + "wprintf" +] + + +def find_locale_dependent_function_uses(): + regexp_locale_dependent_functions = "|".join(LOCALE_DEPENDENT_FUNCTIONS) + exclude_args = [":(exclude)" + excl for excl in REGEXP_EXTERNAL_DEPENDENCIES_EXCLUSIONS] + git_grep_command = ["git", "grep", "-E", "[^a-zA-Z0-9_\\`'\"<>](" + regexp_locale_dependent_functions + "(_r|_s)?)[^a-zA-Z0-9_\\`'\"<>]", "--", "*.cpp", "*.h"] + exclude_args + git_grep_output = list() + + try: + git_grep_output = check_output(git_grep_command, universal_newlines=True, encoding="utf8").splitlines() + except CalledProcessError as e: + if e.returncode > 1: + raise e + + return git_grep_output + + +def main(): + exit_code = 0 + + regexp_ignore_known_violations = "|".join(KNOWN_VIOLATIONS) + git_grep_output = find_locale_dependent_function_uses() + + for locale_dependent_function in LOCALE_DEPENDENT_FUNCTIONS: + matches = [line for line in git_grep_output + if re.search("[^a-zA-Z0-9_\\`'\"<>]" + locale_dependent_function + "(_r|_s)?[^a-zA-Z0-9_\\`'\"<>]", line) + and not re.search("\\.(c|cpp|h):\\s*(//|\\*|/\\*|\").*" + locale_dependent_function, line) + and not re.search(regexp_ignore_known_violations, line)] + if matches: + print(f"The locale dependent function {locale_dependent_function}(...) appears to be used:") + for match in matches: + print(match) + print("") + exit_code = 1 + + if exit_code == 1: + print("Unnecessary locale depedence can cause bugs that are very tricky to isolate and fix. Please avoid using locale dependent functions if possible.\n") + print(f"Advice not applicable in this specific case? Add an exception by updating the ignore list in {sys.argv[0]}") + + sys.exit(exit_code) + + +if __name__ == "__main__": + main() diff --git a/test/lint/lint-locale-dependence.sh b/test/lint/lint-locale-dependence.sh deleted file mode 100755 index 7d608eed6a..0000000000 --- a/test/lint/lint-locale-dependence.sh +++ /dev/null @@ -1,241 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2018-2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C - -# Be aware that bitcoind and bitcoin-qt differ in terms of localization: Qt -# opts in to POSIX localization by running setlocale(LC_ALL, "") on startup, -# whereas no such call is made in bitcoind. -# -# Qt runs setlocale(LC_ALL, "") on initialization. This installs the locale -# specified by the user's LC_ALL (or LC_*) environment variable as the new -# C locale. -# -# In contrast, bitcoind does not opt in to localization -- no call to -# setlocale(LC_ALL, "") is made and the environment variables LC_* are -# thus ignored. -# -# This results in situations where bitcoind is guaranteed to be running -# with the classic locale ("C") whereas the locale of bitcoin-qt will vary -# depending on the user's environment variables. -# -# An example: Assuming the environment variable LC_ALL=de_DE then the -# call std::to_string(1.23) will return "1.230000" in bitcoind but -# "1,230000" in bitcoin-qt. -# -# From the Qt documentation: -# "On Unix/Linux Qt is configured to use the system locale settings by default. -# This can cause a conflict when using POSIX functions, for instance, when -# converting between data types such as floats and strings, since the notation -# may differ between locales. To get around this problem, call the POSIX function -# setlocale(LC_NUMERIC,"C") right after initializing QApplication, QGuiApplication -# or QCoreApplication to reset the locale that is used for number formatting to -# "C"-locale." -# -# See https://doc.qt.io/qt-5/qcoreapplication.html#locale-settings and -# https://stackoverflow.com/a/34878283 for more details. - -# TODO: Reduce KNOWN_VIOLATIONS by replacing uses of locale dependent snprintf with strprintf. -KNOWN_VIOLATIONS=( - "src/dbwrapper.cpp:.*vsnprintf" - "src/test/dbwrapper_tests.cpp:.*snprintf" - "src/test/fuzz/locale.cpp" - "src/test/fuzz/string.cpp" - "src/test/util_tests.cpp" -) - -REGEXP_IGNORE_EXTERNAL_DEPENDENCIES="^src/(crypto/ctaes/|leveldb/|secp256k1/|minisketch/|tinyformat.h|univalue/)" - -LOCALE_DEPENDENT_FUNCTIONS=( - alphasort # LC_COLLATE (via strcoll) - asctime # LC_TIME (directly) - asprintf # (via vasprintf) - atof # LC_NUMERIC (via strtod) - atoi # LC_NUMERIC (via strtol) - atol # LC_NUMERIC (via strtol) - atoll # (via strtoll) - atoq - btowc # LC_CTYPE (directly) - ctime # (via asctime or localtime) - dprintf # (via vdprintf) - fgetwc - fgetws - fold_case # boost::locale::fold_case - fprintf # (via vfprintf) - fputwc - fputws - fscanf # (via __vfscanf) - fwprintf # (via __vfwprintf) - getdate # via __getdate_r => isspace // __localtime_r - getwc - getwchar - is_digit # boost::algorithm::is_digit - is_space # boost::algorithm::is_space - isalnum # LC_CTYPE - isalpha # LC_CTYPE - isblank # LC_CTYPE - iscntrl # LC_CTYPE - isctype # LC_CTYPE - isdigit # LC_CTYPE - isgraph # LC_CTYPE - islower # LC_CTYPE - isprint # LC_CTYPE - ispunct # LC_CTYPE - isspace # LC_CTYPE - isupper # LC_CTYPE - iswalnum # LC_CTYPE - iswalpha # LC_CTYPE - iswblank # LC_CTYPE - iswcntrl # LC_CTYPE - iswctype # LC_CTYPE - iswdigit # LC_CTYPE - iswgraph # LC_CTYPE - iswlower # LC_CTYPE - iswprint # LC_CTYPE - iswpunct # LC_CTYPE - iswspace # LC_CTYPE - iswupper # LC_CTYPE - iswxdigit # LC_CTYPE - isxdigit # LC_CTYPE - localeconv # LC_NUMERIC + LC_MONETARY - mblen # LC_CTYPE - mbrlen - mbrtowc - mbsinit - mbsnrtowcs - mbsrtowcs - mbstowcs # LC_CTYPE - mbtowc # LC_CTYPE - mktime - normalize # boost::locale::normalize - printf # LC_NUMERIC - putwc - putwchar - scanf # LC_NUMERIC - setlocale - snprintf - sprintf - sscanf - std::locale::global - std::to_string - stod - stof - stoi - stol - stold - stoll - stoul - stoull - strcasecmp - strcasestr - strcoll # LC_COLLATE -# strerror - strfmon - strftime # LC_TIME - strncasecmp - strptime - strtod # LC_NUMERIC - strtof - strtoimax - strtol # LC_NUMERIC - strtold - strtoll - strtoq - strtoul # LC_NUMERIC - strtoull - strtoumax - strtouq - strxfrm # LC_COLLATE - swprintf - to_lower # boost::locale::to_lower - to_title # boost::locale::to_title - to_upper # boost::locale::to_upper - tolower # LC_CTYPE - toupper # LC_CTYPE - towctrans - towlower # LC_CTYPE - towupper # LC_CTYPE - trim # boost::algorithm::trim - trim_left # boost::algorithm::trim_left - trim_right # boost::algorithm::trim_right - ungetwc - vasprintf - vdprintf - versionsort - vfprintf - vfscanf - vfwprintf - vprintf - vscanf - vsnprintf - vsprintf - vsscanf - vswprintf - vwprintf - wcrtomb - wcscasecmp - wcscoll # LC_COLLATE - wcsftime # LC_TIME - wcsncasecmp - wcsnrtombs - wcsrtombs - wcstod # LC_NUMERIC - wcstof - wcstoimax - wcstol # LC_NUMERIC - wcstold - wcstoll - wcstombs # LC_CTYPE - wcstoul # LC_NUMERIC - wcstoull - wcstoumax - wcswidth - wcsxfrm # LC_COLLATE - wctob - wctomb # LC_CTYPE - wctrans - wctype - wcwidth - wprintf -) - -function join_array { - local IFS="$1" - shift - echo "$*" -} - -REGEXP_IGNORE_KNOWN_VIOLATIONS=$(join_array "|" "${KNOWN_VIOLATIONS[@]}") - -# Invoke "git grep" only once in order to minimize run-time -REGEXP_LOCALE_DEPENDENT_FUNCTIONS=$(join_array "|" "${LOCALE_DEPENDENT_FUNCTIONS[@]}") -GIT_GREP_OUTPUT=$(git grep -E "[^a-zA-Z0-9_\`'\"<>](${REGEXP_LOCALE_DEPENDENT_FUNCTIONS}(_r|_s)?)[^a-zA-Z0-9_\`'\"<>]" -- "*.cpp" "*.h") - -EXIT_CODE=0 -for LOCALE_DEPENDENT_FUNCTION in "${LOCALE_DEPENDENT_FUNCTIONS[@]}"; do - MATCHES=$(grep -E "[^a-zA-Z0-9_\`'\"<>]${LOCALE_DEPENDENT_FUNCTION}(_r|_s)?[^a-zA-Z0-9_\`'\"<>]" <<< "${GIT_GREP_OUTPUT}" | \ - grep -vE "\.(c|cpp|h):\s*(//|\*|/\*|\").*${LOCALE_DEPENDENT_FUNCTION}") - if [[ ${REGEXP_IGNORE_EXTERNAL_DEPENDENCIES} != "" ]]; then - MATCHES=$(grep -vE "${REGEXP_IGNORE_EXTERNAL_DEPENDENCIES}" <<< "${MATCHES}") - fi - if [[ ${REGEXP_IGNORE_KNOWN_VIOLATIONS} != "" ]]; then - MATCHES=$(grep -vE "${REGEXP_IGNORE_KNOWN_VIOLATIONS}" <<< "${MATCHES}") - fi - if [[ ${MATCHES} != "" ]]; then - echo "The locale dependent function ${LOCALE_DEPENDENT_FUNCTION}(...) appears to be used:" - echo "${MATCHES}" - echo - EXIT_CODE=1 - fi -done -if [[ ${EXIT_CODE} != 0 ]]; then - echo "Unnecessary locale dependence can cause bugs that are very" - echo "tricky to isolate and fix. Please avoid using locale dependent" - echo "functions if possible." - echo - echo "Advice not applicable in this specific case? Add an exception" - echo "by updating the ignore list in $0" -fi -exit ${EXIT_CODE} diff --git a/test/lint/lint-python-utf8-encoding.py b/test/lint/lint-python-utf8-encoding.py new file mode 100755 index 0000000000..62fdc34d50 --- /dev/null +++ b/test/lint/lint-python-utf8-encoding.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2018-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# Make sure we explicitly open all text files using UTF-8 (or ASCII) encoding to +# avoid potential issues on the BSDs where the locale is not always set. + +import sys +import re + +from subprocess import check_output, CalledProcessError + +EXCLUDED_DIRS = ["src/crc32c/"] + + +def get_exclude_args(): + return [":(exclude)" + dir for dir in EXCLUDED_DIRS] + + +def check_fileopens(): + fileopens = list() + + try: + fileopens = check_output(["git", "grep", r" open(", "--", "*.py"] + get_exclude_args(), universal_newlines=True, encoding="utf8").splitlines() + except CalledProcessError as e: + if e.returncode > 1: + raise e + + filtered_fileopens = [fileopen for fileopen in fileopens if not re.search(r"encoding=.(ascii|utf8|utf-8).|open\([^,]*, ['\"][^'\"]*b[^'\"]*['\"]", fileopen)] + + return filtered_fileopens + + +def check_checked_outputs(): + checked_outputs = list() + + try: + checked_outputs = check_output(["git", "grep", "check_output(", "--", "*.py"] + get_exclude_args(), universal_newlines=True, encoding="utf8").splitlines() + except CalledProcessError as e: + if e.returncode > 1: + raise e + + filtered_checked_outputs = [checked_output for checked_output in checked_outputs if re.search(r"universal_newlines=True", checked_output) and not re.search(r"encoding=.(ascii|utf8|utf-8).", checked_output)] + + return filtered_checked_outputs + + +def main(): + exit_code = 0 + + nonexplicit_utf8_fileopens = check_fileopens() + if nonexplicit_utf8_fileopens: + print("Python's open(...) seems to be used to open text files without explicitly specifying encoding='utf8':\n") + for fileopen in nonexplicit_utf8_fileopens: + print(fileopen) + exit_code = 1 + + nonexplicit_utf8_checked_outputs = check_checked_outputs() + if nonexplicit_utf8_checked_outputs: + if nonexplicit_utf8_fileopens: + print("\n") + print("Python's check_output(...) seems to be used to get program outputs without explicitly specifying encoding='utf8':\n") + for checked_output in nonexplicit_utf8_checked_outputs: + print(checked_output) + exit_code = 1 + + sys.exit(exit_code) + + +if __name__ == "__main__": + main() diff --git a/test/lint/lint-python-utf8-encoding.sh b/test/lint/lint-python-utf8-encoding.sh deleted file mode 100755 index 6e5b18fc23..0000000000 --- a/test/lint/lint-python-utf8-encoding.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Make sure we explicitly open all text files using UTF-8 (or ASCII) encoding to -# avoid potential issues on the BSDs where the locale is not always set. - -export LC_ALL=C -EXIT_CODE=0 -OUTPUT=$(git grep " open(" -- "*.py" ":(exclude)src/crc32c/" | grep -vE "encoding=.(ascii|utf8|utf-8)." | grep -vE "open\([^,]*, ['\"][^'\"]*b[^'\"]*['\"]") -if [[ ${OUTPUT} != "" ]]; then - echo "Python's open(...) seems to be used to open text files without explicitly" - echo "specifying encoding=\"utf8\":" - echo - echo "${OUTPUT}" - EXIT_CODE=1 -fi -OUTPUT=$(git grep "check_output(" -- "*.py" ":(exclude)src/crc32c/"| grep "universal_newlines=True" | grep -vE "encoding=.(ascii|utf8|utf-8).") -if [[ ${OUTPUT} != "" ]]; then - echo "Python's check_output(...) seems to be used to get program outputs without explicitly" - echo "specifying encoding=\"utf8\":" - echo - echo "${OUTPUT}" - EXIT_CODE=1 -fi -exit ${EXIT_CODE} diff --git a/test/lint/lint-shell-locale.py b/test/lint/lint-shell-locale.py new file mode 100755 index 0000000000..f3dfe18a95 --- /dev/null +++ b/test/lint/lint-shell-locale.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2018-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" +Make sure all shell scripts are: +a.) explicitly opt out of locale dependence using + "export LC_ALL=C" or "export LC_ALL=C.UTF-8", or +b.) explicitly opt in to locale dependence using the annotation below. +""" + +import subprocess +import sys +import re + +OPT_IN_LINE = '# This script is intentionally locale dependent by not setting \"export LC_ALL=C\"' + +OPT_OUT_LINES = [ + 'export LC_ALL=C', + 'export LC_ALL=C.UTF-8', +] + +def get_shell_files_list(): + command = [ + 'git', + 'ls-files', + '--', + '*.sh', + ] + try: + return subprocess.check_output(command, stderr = subprocess.STDOUT).decode('utf-8').splitlines() + except subprocess.CalledProcessError as e: + if e.returncode > 1: # return code is 1 when match is empty + print(e.output.decode('utf-8'), end='') + sys.exit(1) + return [] + +def main(): + exit_code = 0 + shell_files = get_shell_files_list() + for file_path in shell_files: + if re.search('src/(secp256k1|minisketch|univalue)/', file_path): + continue + + with open(file_path, 'r', encoding='utf-8') as file_obj: + contents = file_obj.read() + + if OPT_IN_LINE in contents: + continue + + non_comment_pattern = re.compile(r'^\s*((?!#).+)$', re.MULTILINE) + non_comment_lines = re.findall(non_comment_pattern, contents) + if not non_comment_lines: + continue + + first_non_comment_line = non_comment_lines[0] + if first_non_comment_line not in OPT_OUT_LINES: + print(f'Missing "export LC_ALL=C" (to avoid locale dependence) as first non-comment non-empty line in {file_path}') + exit_code = 1 + + return sys.exit(exit_code) + +if __name__ == '__main__': + main() + diff --git a/test/lint/lint-shell-locale.sh b/test/lint/lint-shell-locale.sh deleted file mode 100755 index 4c6b8a57e6..0000000000 --- a/test/lint/lint-shell-locale.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Make sure all shell scripts: -# a.) explicitly opt out of locale dependence using -# "export LC_ALL=C" or "export LC_ALL=C.UTF-8", or -# b.) explicitly opt in to locale dependence using the annotation below. - -export LC_ALL=C - -EXIT_CODE=0 -for SHELL_SCRIPT in $(git ls-files -- "*.sh" | grep -vE "src/(secp256k1|minisketch|univalue)/"); do - if grep -q "# This script is intentionally locale dependent by not setting \"export LC_ALL=C\"" "${SHELL_SCRIPT}"; then - continue - fi - FIRST_NON_COMMENT_LINE=$(grep -vE '^(#.*)?$' "${SHELL_SCRIPT}" | head -1) - if [[ ${FIRST_NON_COMMENT_LINE} != "export LC_ALL=C" && ${FIRST_NON_COMMENT_LINE} != "export LC_ALL=C.UTF-8" ]]; then - echo "Missing \"export LC_ALL=C\" (to avoid locale dependence) as first non-comment non-empty line in ${SHELL_SCRIPT}" - EXIT_CODE=1 - fi -done -exit ${EXIT_CODE} diff --git a/test/lint/lint-submodule.py b/test/lint/lint-submodule.py new file mode 100755 index 0000000000..89d4c80f55 --- /dev/null +++ b/test/lint/lint-submodule.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" +This script checks for git modules +""" + +import subprocess +import sys + +def main(): + submodules_list = subprocess.check_output(['git', 'submodule', 'status', '--recursive'], + universal_newlines = True, encoding = 'utf8').rstrip('\n') + if submodules_list: + print("These submodules were found, delete them:\n", submodules_list) + sys.exit(1) + sys.exit(0) + +if __name__ == '__main__': + main() diff --git a/test/lint/lint-submodule.sh b/test/lint/lint-submodule.sh deleted file mode 100755 index d9aa021df7..0000000000 --- a/test/lint/lint-submodule.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# This script checks for git modules -export LC_ALL=C -EXIT_CODE=0 - -CMD=$(git submodule status --recursive) -if test -n "$CMD"; -then - echo These submodules were found, delete them: - echo "$CMD" - EXIT_CODE=1 -fi - -exit $EXIT_CODE - diff --git a/test/lint/lint-tests.py b/test/lint/lint-tests.py new file mode 100755 index 0000000000..849ddcb961 --- /dev/null +++ b/test/lint/lint-tests.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2018-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" +Check the test suite naming conventions +""" + +import re +import subprocess +import sys + + +def grep_boost_fixture_test_suite(): + command = [ + "git", + "grep", + "-E", + r"^BOOST_FIXTURE_TEST_SUITE\(", + "--", + "src/test/**.cpp", + "src/wallet/test/**.cpp", + ] + return subprocess.check_output(command, universal_newlines=True, encoding="utf8") + + +def check_matching_test_names(test_suite_list): + not_matching = [ + x + for x in test_suite_list + if re.search(r"/(.*?)\.cpp:BOOST_FIXTURE_TEST_SUITE\(\1, .*\)", x) is None + ] + if len(not_matching) > 0: + not_matching = "\n".join(not_matching) + error_msg = ( + "The test suite in file src/test/foo_tests.cpp should be named\n" + '"foo_tests". Please make sure the following test suites follow\n' + "that convention:\n\n" + f"{not_matching}\n" + ) + print(error_msg) + return 1 + return 0 + + +def get_duplicates(input_list): + """ + From https://stackoverflow.com/a/9835819 + """ + seen = set() + dupes = set() + for x in input_list: + if x in seen: + dupes.add(x) + else: + seen.add(x) + return dupes + + +def check_unique_test_names(test_suite_list): + output = [re.search(r"\((.*?),", x) for x in test_suite_list] + output = [x.group(1) for x in output if x is not None] + output = get_duplicates(output) + output = sorted(list(output)) + + if len(output) > 0: + output = "\n".join(output) + error_msg = ( + "Test suite names must be unique. The following test suite names\n" + f"appear to be used more than once:\n\n{output}" + ) + print(error_msg) + return 1 + return 0 + + +def main(): + test_suite_list = grep_boost_fixture_test_suite().splitlines() + exit_code = check_matching_test_names(test_suite_list) + exit_code |= check_unique_test_names(test_suite_list) + sys.exit(exit_code) + + +if __name__ == "__main__": + main() diff --git a/test/lint/lint-tests.sh b/test/lint/lint-tests.sh deleted file mode 100755 index 35d11023eb..0000000000 --- a/test/lint/lint-tests.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Check the test suite naming conventions - -export LC_ALL=C -EXIT_CODE=0 - -NAMING_INCONSISTENCIES=$(git grep -E '^BOOST_FIXTURE_TEST_SUITE\(' -- \ - "src/test/**.cpp" "src/wallet/test/**.cpp" | \ - grep -vE '/(.*?)\.cpp:BOOST_FIXTURE_TEST_SUITE\(\1, .*\)$') -if [[ ${NAMING_INCONSISTENCIES} != "" ]]; then - echo "The test suite in file src/test/foo_tests.cpp should be named" - echo "\"foo_tests\". Please make sure the following test suites follow" - echo "that convention:" - echo - echo "${NAMING_INCONSISTENCIES}" - EXIT_CODE=1 -fi - -TEST_SUITE_NAME_COLLISSIONS=$(git grep -E '^BOOST_FIXTURE_TEST_SUITE\(' -- \ - "src/test/**.cpp" "src/wallet/test/**.cpp" | cut -f2 -d'(' | cut -f1 -d, | \ - sort | uniq -d) -if [[ ${TEST_SUITE_NAME_COLLISSIONS} != "" ]]; then - echo "Test suite names must be unique. The following test suite names" - echo "appear to be used more than once:" - echo - echo "${TEST_SUITE_NAME_COLLISSIONS}" - EXIT_CODE=1 -fi - -exit ${EXIT_CODE} |