diff options
Diffstat (limited to 'test')
171 files changed, 6013 insertions, 3187 deletions
diff --git a/test/README.md b/test/README.md index c9e15c4968..0d9b9fb89b 100644 --- a/test/README.md +++ b/test/README.md @@ -98,7 +98,7 @@ test/functional/test_runner.py --extended In order to run backwards compatibility tests, download the previous node binaries: ``` -test/get_previous_releases.py -b v0.20.1 v0.19.1 v0.18.1 v0.17.2 v0.16.3 v0.15.2 +test/get_previous_releases.py -b v23.0 v22.0 v0.21.0 v0.20.1 v0.19.1 v0.18.1 v0.17.2 v0.16.3 v0.15.2 v0.14.3 ``` By default, up to 4 tests will be run in parallel by test_runner. To specify @@ -107,6 +107,34 @@ how many jobs to run, append `--jobs=n` The individual tests and the test_runner harness have many command-line options. Run `test/functional/test_runner.py -h` to see them all. +#### Speed up test runs with a ramdisk + +If you have available RAM on your system you can create a ramdisk to use as the `cache` and `tmp` directories for the functional tests in order to speed them up. +Speed-up amount varies on each system (and according to your ram speed and other variables), but a 2-3x speed-up is not uncommon. + +To create a 4GB ramdisk on Linux at `/mnt/tmp/`: + +```bash +sudo mkdir -p /mnt/tmp +sudo mount -t tmpfs -o size=4g tmpfs /mnt/tmp/ +``` + +Configure the size of the ramdisk using the `size=` option. +The size of the ramdisk needed is relative to the number of concurrent jobs the test suite runs. +For example running the test suite with `--jobs=100` might need a 4GB ramdisk, but running with `--jobs=32` will only need a 2.5GB ramdisk. + +To use, run the test suite specifying the ramdisk as the `cachedir` and `tmpdir`: + +```bash +test/functional/test_runner.py --cachedir=/mnt/tmp/cache --tmpdir=/mnt/tmp +``` + +Once finished with the tests and the disk, and to free the ram, simply unmount the disk: + +```bash +sudo umount /mnt/tmp +``` + #### Troubleshooting and debugging test failures ##### Resource contention @@ -277,11 +305,12 @@ Use the `-v` option for verbose output. | Lint test | Dependency | |-----------|:----------:| -| [`lint-python.sh`](lint/lint-python.sh) | [flake8](https://gitlab.com/pycqa/flake8) -| [`lint-python.sh`](lint/lint-python.sh) | [mypy](https://github.com/python/mypy) -| [`lint-python.sh`](lint/lint-python.sh) | [pyzmq](https://github.com/zeromq/pyzmq) -| [`lint-shell.sh`](lint/lint-shell.sh) | [ShellCheck](https://github.com/koalaman/shellcheck) -| [`lint-spelling.sh`](lint/lint-spelling.sh) | [codespell](https://github.com/codespell-project/codespell) +| [`lint-python.py`](lint/lint-python.py) | [flake8](https://gitlab.com/pycqa/flake8) +| [`lint-python.py`](lint/lint-python.py) | [mypy](https://github.com/python/mypy) +| [`lint-python.py`](lint/lint-python.py) | [pyzmq](https://github.com/zeromq/pyzmq) +| [`lint-python-dead-code.py`](lint/lint-python-dead-code.py) | [vulture](https://github.com/jendrikseipp/vulture) +| [`lint-shell.py`](lint/lint-shell.py) | [ShellCheck](https://github.com/koalaman/shellcheck) +| [`lint-spelling.py`](lint/lint-spelling.py) | [codespell](https://github.com/codespell-project/codespell) In use versions and install instructions are available in the [CI setup](../ci/lint/04_install.sh). @@ -292,13 +321,13 @@ Please be aware that on Linux distributions all dependencies are usually availab Individual tests can be run by directly calling the test script, e.g.: ``` -test/lint/lint-files.sh +test/lint/lint-files.py ``` You can run all the shell-based lint tests by running: ``` -test/lint/lint-all.sh +test/lint/lint-all.py ``` # Writing functional tests diff --git a/test/config.ini.in b/test/config.ini.in index 8bcba1b39c..5888ef443b 100644 --- a/test/config.ini.in +++ b/test/config.ini.in @@ -19,9 +19,11 @@ RPCAUTH=@abs_top_srcdir@/share/rpcauth/rpcauth.py @USE_SQLITE_TRUE@USE_SQLITE=true @USE_BDB_TRUE@USE_BDB=true @BUILD_BITCOIN_CLI_TRUE@ENABLE_CLI=true +@BUILD_BITCOIN_UTIL_TRUE@ENABLE_BITCOIN_UTIL=true @BUILD_BITCOIN_WALLET_TRUE@ENABLE_WALLET_TOOL=true @BUILD_BITCOIND_TRUE@ENABLE_BITCOIND=true @ENABLE_FUZZ_TRUE@ENABLE_FUZZ=true @ENABLE_ZMQ_TRUE@ENABLE_ZMQ=true @ENABLE_EXTERNAL_SIGNER_TRUE@ENABLE_EXTERNAL_SIGNER=true @ENABLE_SYSCALL_SANDBOX_TRUE@ENABLE_SYSCALL_SANDBOX=true +@ENABLE_USDT_TRACEPOINTS_TRUE@ENABLE_USDT_TRACEPOINTS=true diff --git a/test/functional/README.md b/test/functional/README.md index 926810cf03..914dbfd977 100644 --- a/test/functional/README.md +++ b/test/functional/README.md @@ -24,7 +24,7 @@ don't have test cases for. Consider using [pyenv](https://github.com/pyenv/pyenv), which checks [.python-version](/.python-version), to prevent accidentally introducing modern syntax from an unsupported Python version. The CI linter job also checks this, but [possibly not in all cases](https://github.com/bitcoin/bitcoin/pull/14884#discussion_r239585126). -- See [the python lint script](/test/lint/lint-python.sh) that checks for violations that +- See [the python lint script](/test/lint/lint-python.py) that checks for violations that could lead to bugs and issues in the test code. - Use [type hints](https://docs.python.org/3/library/typing.html) in your code to improve code readability and to detect possible bugs earlier. diff --git a/test/functional/data/rpc_decodescript.json b/test/functional/data/rpc_decodescript.json index d1aa9ab00d..8903f5efac 100644 --- a/test/functional/data/rpc_decodescript.json +++ b/test/functional/data/rpc_decodescript.json @@ -4,6 +4,7 @@ { "asm": "1 eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", "address": "bcrt1pamhwamhwamhwamhwamhwamhwamhwamhwamhwamhwamhwamhwamhqz6nvlh", + "desc": "addr(bcrt1pamhwamhwamhwamhwamhwamhwamhwamhwamhwamhwamhwamhwamhqz6nvlh)#v52jnujz", "type": "witness_v1_taproot" } ], @@ -12,6 +13,7 @@ { "asm": "1 -28398", "address": "bcrt1pamhqk96edn", + "desc": "addr(bcrt1pamhqk96edn)#vkh8uj5a", "type": "witness_unknown" } ], @@ -20,6 +22,7 @@ { "asm": "0 eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", "address": "bcrt1qamhwamhwamhwamhwamhwamhwamhwamhwamhwamhwamhwamhwamhqgdn98t", + "desc": "addr(bcrt1qamhwamhwamhwamhwamhwamhwamhwamhwamhwamhwamhwamhwamhqgdn98t)#afaecevx", "type": "witness_v0_scripthash", "p2sh": "2MwGk8mw1GBP6U9D5X8gTvgvXpuknmAK3fo" } @@ -29,6 +32,7 @@ { "asm": "OP_HASH160 eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee OP_EQUAL", "address": "2NF2b3KS8xXb9XHvbRMXdZh8s5g92rUZHtp", + "desc": "addr(2NF2b3KS8xXb9XHvbRMXdZh8s5g92rUZHtp)#ywfcpmh9", "type": "scripthash" } ], @@ -36,6 +40,7 @@ "6a00", { "asm": "OP_RETURN 0", + "desc": "raw(6a00)#ncfmkl43", "type": "nulldata" } ], @@ -43,6 +48,7 @@ "6aee", { "asm": "OP_RETURN OP_UNKNOWN", + "desc": "raw(6aee)#vsyzgqdt", "type": "nonstandard" } ], @@ -50,6 +56,7 @@ "6a02ee", { "asm": "OP_RETURN [error]", + "desc": "raw(6a02ee)#gvdwnlzl", "type": "nonstandard" } ], @@ -57,10 +64,12 @@ "02eeee", { "asm": "-28398", + "desc": "raw(02eeee)#5xzck7pr", "type": "nonstandard", "p2sh": "2N34iiGoUUkVSPiaaTFpJjB1FR9TXQu3PGM", "segwit": { "asm": "0 96c2368fc30514a438a8bd909f93c49a1549d77198ccbdb792043b666cb24f42", + "desc": "addr(bcrt1qjmprdr7rq522gw9ghkgfly7yng25n4m3nrxtmdujqsakvm9jfapqk795l5)#5akkdska", "hex": "002096c2368fc30514a438a8bd909f93c49a1549d77198ccbdb792043b666cb24f42", "address": "bcrt1qjmprdr7rq522gw9ghkgfly7yng25n4m3nrxtmdujqsakvm9jfapqk795l5", "type": "witness_v0_scripthash", @@ -72,6 +81,7 @@ "ba", { "asm": "OP_CHECKSIGADD", + "desc": "raw(ba)#yy0eg44l", "type": "nonstandard" } ], @@ -79,6 +89,7 @@ "50", { "asm": "OP_RESERVED", + "desc": "raw(50)#a7tu03xf", "type": "nonstandard" } ] diff --git a/test/functional/feature_addrman.py b/test/functional/feature_addrman.py index 14a4f8abb7..5e49d0214a 100755 --- a/test/functional/feature_addrman.py +++ b/test/functional/feature_addrman.py @@ -68,18 +68,28 @@ class AddrmanTest(BitcoinTestFramework): self.start_node(0, extra_args=["-checkaddrman=1"]) assert_equal(self.nodes[0].getnodeaddresses(), []) - self.log.info("Check that addrman from future cannot be read") + self.log.info("Check that addrman with negative lowest_compatible cannot be read") self.stop_node(0) - write_addrman(peers_dat, lowest_compatible=111) + write_addrman(peers_dat, lowest_compatible=-32) self.nodes[0].assert_start_raises_init_error( expected_msg=init_error( - "Unsupported format of addrman database: 1. It is compatible with " - "formats >=111, but the maximum supported by this version of " - f"{self.config['environment']['PACKAGE_NAME']} is 4.: (.+)" + "Corrupted addrman database: The compat value \\(0\\) is lower " + "than the expected minimum value 32.: (.+)" ), match=ErrorMatch.FULL_REGEX, ) + self.log.info("Check that addrman from future is overwritten with new addrman") + self.stop_node(0) + write_addrman(peers_dat, lowest_compatible=111) + assert_equal(os.path.exists(peers_dat + ".bak"), False) + with self.nodes[0].assert_debug_log([ + f'Creating new peers.dat because the file version was not compatible ("{peers_dat}"). Original backed up to peers.dat.bak', + ]): + self.start_node(0) + assert_equal(self.nodes[0].getnodeaddresses(), []) + assert_equal(os.path.exists(peers_dat + ".bak"), True) + self.log.info("Check that corrupt addrman cannot be read (EOF)") self.stop_node(0) with open(peers_dat, "wb") as f: diff --git a/test/functional/feature_backwards_compatibility.py b/test/functional/feature_backwards_compatibility.py index 476a6a0c14..59a12193fd 100755 --- a/test/functional/feature_backwards_compatibility.py +++ b/test/functional/feature_backwards_compatibility.py @@ -34,15 +34,19 @@ from test_framework.util import ( class BackwardsCompatibilityTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True - self.num_nodes = 6 + self.num_nodes = 10 # Add new version after each release: self.extra_args = [ - ["-addresstype=bech32"], # Pre-release: use to mine blocks - ["-nowallet", "-walletrbf=1", "-addresstype=bech32"], # Pre-release: use to receive coins, swap wallets, etc - ["-nowallet", "-walletrbf=1", "-addresstype=bech32"], # v0.19.1 - ["-nowallet", "-walletrbf=1", "-addresstype=bech32"], # v0.18.1 - ["-nowallet", "-walletrbf=1", "-addresstype=bech32"], # v0.17.2 - ["-nowallet", "-walletrbf=1", "-addresstype=bech32", "-wallet=wallet.dat"], # v0.16.3 + ["-addresstype=bech32", "-whitelist=noban@127.0.0.1"], # Pre-release: use to mine blocks. noban for immediate tx relay + ["-nowallet", "-walletrbf=1", "-addresstype=bech32", "-whitelist=noban@127.0.0.1"], # Pre-release: use to receive coins, swap wallets, etc + ["-nowallet", "-walletrbf=1", "-addresstype=bech32", "-whitelist=noban@127.0.0.1"], # v23.0 + ["-nowallet", "-walletrbf=1", "-addresstype=bech32", "-whitelist=noban@127.0.0.1"], # v22.0 + ["-nowallet", "-walletrbf=1", "-addresstype=bech32", "-whitelist=noban@127.0.0.1"], # v0.21.0 + ["-nowallet", "-walletrbf=1", "-addresstype=bech32", "-whitelist=noban@127.0.0.1"], # v0.20.1 + ["-nowallet", "-walletrbf=1", "-addresstype=bech32", "-whitelist=noban@127.0.0.1"], # v0.19.1 + ["-nowallet", "-walletrbf=1", "-addresstype=bech32", "-whitelist=127.0.0.1"], # v0.18.1 + ["-nowallet", "-walletrbf=1", "-addresstype=bech32", "-whitelist=127.0.0.1"], # v0.17.2 + ["-nowallet", "-walletrbf=1", "-addresstype=bech32", "-whitelist=127.0.0.1", "-wallet=wallet.dat"], # v0.16.3 ] self.wallet_names = [self.default_wallet_name] @@ -54,6 +58,10 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): self.add_nodes(self.num_nodes, extra_args=self.extra_args, versions=[ None, None, + 230000, + 220000, + 210000, + 200100, 190100, 180100, 170200, @@ -63,19 +71,27 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): self.start_nodes() self.import_deterministic_coinbase_privkeys() - def run_test(self): - self.generatetoaddress(self.nodes[0], COINBASE_MATURITY + 1, self.nodes[0].getnewaddress()) - - # Sanity check the test framework: - res = self.nodes[self.num_nodes - 1].getblockchaininfo() - assert_equal(res['blocks'], COINBASE_MATURITY + 1) + def nodes_wallet_dir(self, node): + if node.version < 170000: + return os.path.join(node.datadir, "regtest") + return os.path.join(node.datadir, "regtest/wallets") - node_master = self.nodes[self.num_nodes - 5] + def run_test(self): + node_miner = self.nodes[0] + node_master = self.nodes[1] node_v19 = self.nodes[self.num_nodes - 4] node_v18 = self.nodes[self.num_nodes - 3] node_v17 = self.nodes[self.num_nodes - 2] node_v16 = self.nodes[self.num_nodes - 1] + legacy_nodes = self.nodes[2:] + + self.generatetoaddress(node_miner, COINBASE_MATURITY + 1, node_miner.getnewaddress()) + + # Sanity check the test framework: + res = node_v16.getblockchaininfo() + assert_equal(res['blocks'], COINBASE_MATURITY + 1) + self.log.info("Test wallet backwards compatibility...") # Create a number of wallets and open them in older versions: @@ -88,21 +104,21 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): assert info['keypoolsize'] > 0 # Create a confirmed transaction, receiving coins address = wallet.getnewaddress() - self.nodes[0].sendtoaddress(address, 10) + node_miner.sendtoaddress(address, 10) self.sync_mempools() - self.generate(self.nodes[0], 1) + self.generate(node_miner, 1) # Create a conflicting transaction using RBF - return_address = self.nodes[0].getnewaddress() - tx1_id = self.nodes[1].sendtoaddress(return_address, 1) - tx2_id = self.nodes[1].bumpfee(tx1_id)["txid"] + return_address = node_miner.getnewaddress() + tx1_id = node_master.sendtoaddress(return_address, 1) + tx2_id = node_master.bumpfee(tx1_id)["txid"] # Confirm the transaction self.sync_mempools() - self.generate(self.nodes[0], 1) + self.generate(node_miner, 1) # Create another conflicting transaction using RBF - tx3_id = self.nodes[1].sendtoaddress(return_address, 1) - tx4_id = self.nodes[1].bumpfee(tx3_id)["txid"] + tx3_id = node_master.sendtoaddress(return_address, 1) + tx4_id = node_master.bumpfee(tx3_id)["txid"] # Abandon transaction, but don't confirm - self.nodes[1].abandontransaction(tx3_id) + node_master.abandontransaction(tx3_id) # w1_v19: regular wallet, created with v0.19 node_v19.rpc.createwallet(wallet_name="w1_v19") @@ -113,6 +129,7 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): # Use addmultisigaddress (see #18075) address_18075 = wallet.rpc.addmultisigaddress(1, ["0296b538e853519c726a2c91e61ec11600ae1390813a627c66fb8be7947be63c52", "037211a824f55b505228e4c3d5194c1fcfaa15a456abdf37f9b9d97a4040afc073"], "", "legacy")["address"] assert wallet.getaddressinfo(address_18075)["solvable"] + node_v19.unloadwallet("w1_v19") # w1_v18: regular wallet, created with v0.18 node_v18.rpc.createwallet(wallet_name="w1_v18") @@ -130,20 +147,6 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): assert info['private_keys_enabled'] == False assert info['keypoolsize'] == 0 - # w2_v19: wallet with private keys disabled, created with v0.19 - node_v19.rpc.createwallet(wallet_name="w2_v19", disable_private_keys=True) - wallet = node_v19.get_wallet_rpc("w2_v19") - info = wallet.getwalletinfo() - assert info['private_keys_enabled'] == False - assert info['keypoolsize'] == 0 - - # w2_v18: wallet with private keys disabled, created with v0.18 - node_v18.rpc.createwallet(wallet_name="w2_v18", disable_private_keys=True) - wallet = node_v18.get_wallet_rpc("w2_v18") - info = wallet.getwalletinfo() - assert info['private_keys_enabled'] == False - assert info['keypoolsize'] == 0 - # w3: blank wallet, created on master: update this # test when default blank wallets can no longer be opened by older versions. node_master.createwallet(wallet_name="w3", blank=True) @@ -152,170 +155,72 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): assert info['private_keys_enabled'] assert info['keypoolsize'] == 0 - # w3_v19: blank wallet, created with v0.19 - node_v19.rpc.createwallet(wallet_name="w3_v19", blank=True) - wallet = node_v19.get_wallet_rpc("w3_v19") - info = wallet.getwalletinfo() - assert info['private_keys_enabled'] - assert info['keypoolsize'] == 0 - - # w3_v18: blank wallet, created with v0.18 - node_v18.rpc.createwallet(wallet_name="w3_v18", blank=True) - wallet = node_v18.get_wallet_rpc("w3_v18") - info = wallet.getwalletinfo() - assert info['private_keys_enabled'] - assert info['keypoolsize'] == 0 - - # Copy the wallets to older nodes: + # Unload wallets and copy to older nodes: node_master_wallets_dir = os.path.join(node_master.datadir, "regtest/wallets") node_v19_wallets_dir = os.path.join(node_v19.datadir, "regtest/wallets") - node_v18_wallets_dir = os.path.join(node_v18.datadir, "regtest/wallets") node_v17_wallets_dir = os.path.join(node_v17.datadir, "regtest/wallets") node_v16_wallets_dir = os.path.join(node_v16.datadir, "regtest") node_master.unloadwallet("w1") node_master.unloadwallet("w2") - node_v19.unloadwallet("w1_v19") - node_v19.unloadwallet("w2_v19") - node_v18.unloadwallet("w1_v18") - node_v18.unloadwallet("w2_v18") - - # Copy wallets to v0.16 - for wallet in os.listdir(node_master_wallets_dir): - shutil.copytree( - os.path.join(node_master_wallets_dir, wallet), - os.path.join(node_v16_wallets_dir, wallet) - ) + node_master.unloadwallet("w3") - # Copy wallets to v0.17 - for wallet in os.listdir(node_master_wallets_dir): - shutil.copytree( - os.path.join(node_master_wallets_dir, wallet), - os.path.join(node_v17_wallets_dir, wallet) - ) - for wallet in os.listdir(node_v18_wallets_dir): - shutil.copytree( - os.path.join(node_v18_wallets_dir, wallet), - os.path.join(node_v17_wallets_dir, wallet) - ) - - # Copy wallets to v0.18 - for wallet in os.listdir(node_master_wallets_dir): - shutil.copytree( - os.path.join(node_master_wallets_dir, wallet), - os.path.join(node_v18_wallets_dir, wallet) - ) - - # Copy wallets to v0.19 - for wallet in os.listdir(node_master_wallets_dir): - shutil.copytree( - os.path.join(node_master_wallets_dir, wallet), - os.path.join(node_v19_wallets_dir, wallet) - ) + for node in legacy_nodes: + # Copy wallets to previous version + for wallet in os.listdir(node_master_wallets_dir): + shutil.copytree( + os.path.join(node_master_wallets_dir, wallet), + os.path.join(self.nodes_wallet_dir(node), wallet) + ) if not self.options.descriptors: # Descriptor wallets break compatibility, only run this test for legacy wallet - # Open the wallets in v0.19 - node_v19.loadwallet("w1") - wallet = node_v19.get_wallet_rpc("w1") - info = wallet.getwalletinfo() - assert info['private_keys_enabled'] - assert info['keypoolsize'] > 0 - txs = wallet.listtransactions() - assert_equal(len(txs), 5) - assert_equal(txs[1]["txid"], tx1_id) - assert_equal(txs[2]["walletconflicts"], [tx1_id]) - assert_equal(txs[1]["replaced_by_txid"], tx2_id) - assert not(txs[1]["abandoned"]) - assert_equal(txs[1]["confirmations"], -1) - assert_equal(txs[2]["blockindex"], 1) - assert txs[3]["abandoned"] - assert_equal(txs[4]["walletconflicts"], [tx3_id]) - assert_equal(txs[3]["replaced_by_txid"], tx4_id) - assert not(hasattr(txs[3], "blockindex")) - - node_v19.loadwallet("w2") - wallet = node_v19.get_wallet_rpc("w2") - info = wallet.getwalletinfo() - assert info['private_keys_enabled'] == False - assert info['keypoolsize'] == 0 - - node_v19.loadwallet("w3") - wallet = node_v19.get_wallet_rpc("w3") - info = wallet.getwalletinfo() - assert info['private_keys_enabled'] - assert info['keypoolsize'] == 0 - - # Open the wallets in v0.18 - node_v18.loadwallet("w1") - wallet = node_v18.get_wallet_rpc("w1") - info = wallet.getwalletinfo() - assert info['private_keys_enabled'] - assert info['keypoolsize'] > 0 - txs = wallet.listtransactions() - assert_equal(len(txs), 5) - assert_equal(txs[1]["txid"], tx1_id) - assert_equal(txs[2]["walletconflicts"], [tx1_id]) - assert_equal(txs[1]["replaced_by_txid"], tx2_id) - assert not(txs[1]["abandoned"]) - assert_equal(txs[1]["confirmations"], -1) - assert_equal(txs[2]["blockindex"], 1) - assert txs[3]["abandoned"] - assert_equal(txs[4]["walletconflicts"], [tx3_id]) - assert_equal(txs[3]["replaced_by_txid"], tx4_id) - assert not(hasattr(txs[3], "blockindex")) - - node_v18.loadwallet("w2") - wallet = node_v18.get_wallet_rpc("w2") - info = wallet.getwalletinfo() - assert info['private_keys_enabled'] == False - assert info['keypoolsize'] == 0 - - node_v18.loadwallet("w3") - wallet = node_v18.get_wallet_rpc("w3") - info = wallet.getwalletinfo() - assert info['private_keys_enabled'] - assert info['keypoolsize'] == 0 - - node_v17.loadwallet("w1") - wallet = node_v17.get_wallet_rpc("w1") - info = wallet.getwalletinfo() - assert info['private_keys_enabled'] - assert info['keypoolsize'] > 0 - - node_v17.loadwallet("w2") - wallet = node_v17.get_wallet_rpc("w2") - info = wallet.getwalletinfo() - assert info['private_keys_enabled'] == False - assert info['keypoolsize'] == 0 + # Load modern wallet with older nodes + for node in legacy_nodes: + for wallet_name in ["w1", "w2", "w3"]: + if node.version < 170000: + # loadwallet was introduced in v0.17.0 + continue + if node.version < 180000 and wallet_name == "w3": + # Blank wallets were introduced in v0.18.0. We test the loading error below. + continue + node.loadwallet(wallet_name) + wallet = node.get_wallet_rpc(wallet_name) + info = wallet.getwalletinfo() + if wallet_name == "w1": + assert info['private_keys_enabled'] == True + assert info['keypoolsize'] > 0 + txs = wallet.listtransactions() + assert_equal(len(txs), 5) + assert_equal(txs[1]["txid"], tx1_id) + assert_equal(txs[2]["walletconflicts"], [tx1_id]) + assert_equal(txs[1]["replaced_by_txid"], tx2_id) + assert not(txs[1]["abandoned"]) + assert_equal(txs[1]["confirmations"], -1) + assert_equal(txs[2]["blockindex"], 1) + assert txs[3]["abandoned"] + assert_equal(txs[4]["walletconflicts"], [tx3_id]) + assert_equal(txs[3]["replaced_by_txid"], tx4_id) + assert not(hasattr(txs[3], "blockindex")) + elif wallet_name == "w2": + assert(info['private_keys_enabled'] == False) + assert info['keypoolsize'] == 0 + else: + assert(info['private_keys_enabled'] == True) + assert info['keypoolsize'] == 0 else: - # Descriptor wallets appear to be corrupted wallets to old software - assert_raises_rpc_error(-4, "Wallet file verification failed: wallet.dat corrupt, salvage failed", node_v19.loadwallet, "w1") - assert_raises_rpc_error(-4, "Wallet file verification failed: wallet.dat corrupt, salvage failed", node_v19.loadwallet, "w2") - assert_raises_rpc_error(-4, "Wallet file verification failed: wallet.dat corrupt, salvage failed", node_v19.loadwallet, "w3") - assert_raises_rpc_error(-4, "Wallet file verification failed: wallet.dat corrupt, salvage failed", node_v18.loadwallet, "w1") - assert_raises_rpc_error(-4, "Wallet file verification failed: wallet.dat corrupt, salvage failed", node_v18.loadwallet, "w2") - assert_raises_rpc_error(-4, "Wallet file verification failed: wallet.dat corrupt, salvage failed", node_v18.loadwallet, "w3") - - # Open the wallets in v0.17 - node_v17.loadwallet("w1_v18") - wallet = node_v17.get_wallet_rpc("w1_v18") - info = wallet.getwalletinfo() - assert info['private_keys_enabled'] - assert info['keypoolsize'] > 0 - - node_v17.loadwallet("w2_v18") - wallet = node_v17.get_wallet_rpc("w2_v18") - info = wallet.getwalletinfo() - assert info['private_keys_enabled'] == False - assert info['keypoolsize'] == 0 + for node in legacy_nodes: + # Descriptor wallets appear to be corrupted wallets to old software + # and loadwallet is introduced in v0.17.0 + if node.version >= 170000 and node.version < 210000: + for wallet_name in ["w1", "w2", "w3"]: + assert_raises_rpc_error(-4, "Wallet file verification failed: wallet.dat corrupt, salvage failed", node.loadwallet, wallet_name) # RPC loadwallet failure causes bitcoind to exit, in addition to the RPC # call failure, so the following test won't work: - # assert_raises_rpc_error(-4, "Wallet loading failed.", node_v17.loadwallet, 'w3_v18') + # assert_raises_rpc_error(-4, "Wallet loading failed.", node_v17.loadwallet, 'w3') # Instead, we stop node and try to launch it with the wallet: - self.stop_node(4) - node_v17.assert_start_raises_init_error(["-wallet=w3_v18"], "Error: Error loading w3_v18: Wallet requires newer version of Bitcoin Core") + self.stop_node(node_v17.index) if self.options.descriptors: # Descriptor wallets appear to be corrupted wallets to old software node_v17.assert_start_raises_init_error(["-wallet=w1"], "Error: wallet.dat corrupt, salvage failed") @@ -323,23 +228,23 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): node_v17.assert_start_raises_init_error(["-wallet=w3"], "Error: wallet.dat corrupt, salvage failed") else: node_v17.assert_start_raises_init_error(["-wallet=w3"], "Error: Error loading w3: Wallet requires newer version of Bitcoin Core") - self.start_node(4) + self.start_node(node_v17.index) if not self.options.descriptors: # Descriptor wallets break compatibility, only run this test for legacy wallets # Open most recent wallet in v0.16 (no loadwallet RPC) - self.restart_node(5, extra_args=["-wallet=w2"]) + self.restart_node(node_v16.index, extra_args=["-wallet=w2"]) wallet = node_v16.get_wallet_rpc("w2") info = wallet.getwalletinfo() assert info['keypoolsize'] == 1 # Create upgrade wallet in v0.16 - self.restart_node(-1, extra_args=["-wallet=u1_v16"]) + self.restart_node(node_v16.index, extra_args=["-wallet=u1_v16"]) wallet = node_v16.get_wallet_rpc("u1_v16") v16_addr = wallet.getnewaddress('', "bech32") v16_info = wallet.validateaddress(v16_addr) v16_pubkey = v16_info['pubkey'] - self.stop_node(-1) + self.stop_node(node_v16.index) self.log.info("Test wallet upgrade path...") # u1: regular wallet, created with v0.17 @@ -371,7 +276,7 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): os.path.join(node_master_wallets_dir, "u1_v16"), os.path.join(node_v16_wallets_dir, "wallets/u1_v16") ) - self.start_node(-1, extra_args=["-wallet=u1_v16"]) + self.start_node(node_v16.index, extra_args=["-wallet=u1_v16"]) wallet = node_v16.get_wallet_rpc("u1_v16") info = wallet.validateaddress(v16_addr) assert_equal(info, v16_info) diff --git a/test/functional/feature_bind_port_discover.py b/test/functional/feature_bind_port_discover.py new file mode 100755 index 0000000000..6e07f2f16c --- /dev/null +++ b/test/functional/feature_bind_port_discover.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020-2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +""" +Test that -discover does not add all interfaces' addresses if we listen on only some of them +""" + +from test_framework.test_framework import BitcoinTestFramework, SkipTest +from test_framework.util import assert_equal + +# We need to bind to a routable address for this test to exercise the relevant code +# and also must have another routable address on another interface which must not +# be named "lo" or "lo0". +# To set these routable addresses on the machine, use: +# Linux: +# ifconfig lo:0 1.1.1.1/32 up && ifconfig lo:1 2.2.2.2/32 up # to set up +# ifconfig lo:0 down && ifconfig lo:1 down # to remove it, after the test +# FreeBSD: +# ifconfig em0 1.1.1.1/32 alias && ifconfig wlan0 2.2.2.2/32 alias # to set up +# ifconfig em0 1.1.1.1 -alias && ifconfig wlan0 2.2.2.2 -alias # to remove it, after the test +ADDR1 = '1.1.1.1' +ADDR2 = '2.2.2.2' + +BIND_PORT = 31001 + +class BindPortDiscoverTest(BitcoinTestFramework): + def set_test_params(self): + # Avoid any -bind= on the command line. Force the framework to avoid adding -bind=127.0.0.1. + self.setup_clean_chain = True + self.bind_to_localhost_only = False + self.extra_args = [ + ['-discover', f'-port={BIND_PORT}'], # bind on any + ['-discover', f'-bind={ADDR1}:{BIND_PORT}'], + ] + self.num_nodes = len(self.extra_args) + + def add_options(self, parser): + parser.add_argument( + "--ihave1111and2222", action='store_true', dest="ihave1111and2222", + help=f"Run the test, assuming {ADDR1} and {ADDR2} are configured on the machine", + default=False) + + def skip_test_if_missing_module(self): + if not self.options.ihave1111and2222: + raise SkipTest( + f"To run this test make sure that {ADDR1} and {ADDR2} (routable addresses) are " + "assigned to the interfaces on this machine and rerun with --ihave1111and2222") + + def run_test(self): + self.log.info( + "Test that if -bind= is not passed then all addresses are " + "added to localaddresses") + found_addr1 = False + found_addr2 = False + for local in self.nodes[0].getnetworkinfo()['localaddresses']: + if local['address'] == ADDR1: + found_addr1 = True + assert_equal(local['port'], BIND_PORT) + if local['address'] == ADDR2: + found_addr2 = True + assert_equal(local['port'], BIND_PORT) + assert found_addr1 + assert found_addr2 + + self.log.info( + "Test that if -bind= is passed then only that address is " + "added to localaddresses") + found_addr1 = False + for local in self.nodes[1].getnetworkinfo()['localaddresses']: + if local['address'] == ADDR1: + found_addr1 = True + assert_equal(local['port'], BIND_PORT) + assert local['address'] != ADDR2 + assert found_addr1 + +if __name__ == '__main__': + BindPortDiscoverTest().main() diff --git a/test/functional/feature_bind_port_externalip.py b/test/functional/feature_bind_port_externalip.py new file mode 100755 index 0000000000..6a74ce5738 --- /dev/null +++ b/test/functional/feature_bind_port_externalip.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020-2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +""" +Test that the proper port is used for -externalip= +""" + +from test_framework.test_framework import BitcoinTestFramework, SkipTest +from test_framework.util import assert_equal, p2p_port + +# We need to bind to a routable address for this test to exercise the relevant code. +# To set a routable address on the machine use: +# Linux: +# ifconfig lo:0 1.1.1.1/32 up # to set up +# ifconfig lo:0 down # to remove it, after the test +# FreeBSD: +# ifconfig lo0 1.1.1.1/32 alias # to set up +# ifconfig lo0 1.1.1.1 -alias # to remove it, after the test +ADDR = '1.1.1.1' + +# array of tuples [arguments, expected port in localaddresses] +EXPECTED = [ + [['-externalip=2.2.2.2', '-port=30001'], 30001], + [['-externalip=2.2.2.2', '-port=30002', f'-bind={ADDR}'], 30002], + [['-externalip=2.2.2.2', f'-bind={ADDR}'], 'default_p2p_port'], + [['-externalip=2.2.2.2', '-port=30003', f'-bind={ADDR}:30004'], 30004], + [['-externalip=2.2.2.2', f'-bind={ADDR}:30005'], 30005], + [['-externalip=2.2.2.2:30006', '-port=30007'], 30006], + [['-externalip=2.2.2.2:30008', '-port=30009', f'-bind={ADDR}'], 30008], + [['-externalip=2.2.2.2:30010', f'-bind={ADDR}'], 30010], + [['-externalip=2.2.2.2:30011', '-port=30012', f'-bind={ADDR}:30013'], 30011], + [['-externalip=2.2.2.2:30014', f'-bind={ADDR}:30015'], 30014], + [['-externalip=2.2.2.2', '-port=30016', f'-bind={ADDR}:30017', + f'-whitebind={ADDR}:30018'], 30017], + [['-externalip=2.2.2.2', '-port=30019', + f'-whitebind={ADDR}:30020'], 30020], +] + +class BindPortExternalIPTest(BitcoinTestFramework): + def set_test_params(self): + # Avoid any -bind= on the command line. Force the framework to avoid adding -bind=127.0.0.1. + self.setup_clean_chain = True + self.bind_to_localhost_only = False + self.num_nodes = len(EXPECTED) + self.extra_args = list(map(lambda e: e[0], EXPECTED)) + + def add_options(self, parser): + parser.add_argument( + "--ihave1111", action='store_true', dest="ihave1111", + help=f"Run the test, assuming {ADDR} is configured on the machine", + default=False) + + def skip_test_if_missing_module(self): + if not self.options.ihave1111: + raise SkipTest( + f"To run this test make sure that {ADDR} (a routable address) is assigned " + "to one of the interfaces on this machine and rerun with --ihave1111") + + def run_test(self): + self.log.info("Test the proper port is used for -externalip=") + for i in range(len(EXPECTED)): + expected_port = EXPECTED[i][1] + if expected_port == 'default_p2p_port': + expected_port = p2p_port(i) + found = False + for local in self.nodes[i].getnetworkinfo()['localaddresses']: + if local['address'] == '2.2.2.2': + assert_equal(local['port'], expected_port) + found = True + break + assert found + +if __name__ == '__main__': + BindPortExternalIPTest().main() diff --git a/test/functional/feature_blockfilterindex_prune.py b/test/functional/feature_blockfilterindex_prune.py deleted file mode 100755 index 2451988135..0000000000 --- a/test/functional/feature_blockfilterindex_prune.py +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2020-2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test blockfilterindex in conjunction with prune.""" -from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import ( - assert_equal, - assert_greater_than, - assert_raises_rpc_error, -) - - -class FeatureBlockfilterindexPruneTest(BitcoinTestFramework): - def set_test_params(self): - self.num_nodes = 1 - self.extra_args = [["-fastprune", "-prune=1", "-blockfilterindex=1"]] - - def sync_index(self, height): - expected = {'basic block filter index': {'synced': True, 'best_block_height': height}} - self.wait_until(lambda: self.nodes[0].getindexinfo() == expected) - - def run_test(self): - self.log.info("check if we can access a blockfilter when pruning is enabled but no blocks are actually pruned") - self.sync_index(height=200) - assert_greater_than(len(self.nodes[0].getblockfilter(self.nodes[0].getbestblockhash())['filter']), 0) - self.generate(self.nodes[0], 500) - self.sync_index(height=700) - - self.log.info("prune some blocks") - pruneheight = self.nodes[0].pruneblockchain(400) - # the prune heights used here and below are magic numbers that are determined by the - # thresholds at which block files wrap, so they depend on disk serialization and default block file size. - assert_equal(pruneheight, 248) - - self.log.info("check if we can access the tips blockfilter when we have pruned some blocks") - assert_greater_than(len(self.nodes[0].getblockfilter(self.nodes[0].getbestblockhash())['filter']), 0) - - self.log.info("check if we can access the blockfilter of a pruned block") - assert_greater_than(len(self.nodes[0].getblockfilter(self.nodes[0].getblockhash(2))['filter']), 0) - - # mine and sync index up to a height that will later be the pruneheight - self.generate(self.nodes[0], 298) - self.sync_index(height=998) - - self.log.info("start node without blockfilterindex") - self.restart_node(0, extra_args=["-fastprune", "-prune=1"]) - - self.log.info("make sure accessing the blockfilters throws an error") - assert_raises_rpc_error(-1, "Index is not enabled for filtertype basic", self.nodes[0].getblockfilter, self.nodes[0].getblockhash(2)) - self.generate(self.nodes[0], 502) - - self.log.info("prune exactly up to the blockfilterindexes best block while blockfilters are disabled") - pruneheight_2 = self.nodes[0].pruneblockchain(1000) - assert_equal(pruneheight_2, 998) - self.restart_node(0, extra_args=["-fastprune", "-prune=1", "-blockfilterindex=1"]) - self.log.info("make sure that we can continue with the partially synced index after having pruned up to the index height") - self.sync_index(height=1500) - - self.log.info("prune below the blockfilterindexes best block while blockfilters are disabled") - self.restart_node(0, extra_args=["-fastprune", "-prune=1"]) - self.generate(self.nodes[0], 1000) - pruneheight_3 = self.nodes[0].pruneblockchain(2000) - assert_greater_than(pruneheight_3, pruneheight_2) - self.stop_node(0) - - self.log.info("make sure we get an init error when starting the node again with block filters") - self.nodes[0].assert_start_raises_init_error( - extra_args=["-fastprune", "-prune=1", "-blockfilterindex=1"], - expected_msg="Error: basic block filter index best block of the index goes beyond pruned data. Please disable the index or reindex (which will download the whole blockchain again)", - ) - - self.log.info("make sure the node starts again with the -reindex arg") - self.start_node(0, extra_args=["-fastprune", "-prune=1", "-blockfilterindex", "-reindex"]) - - -if __name__ == '__main__': - FeatureBlockfilterindexPruneTest().main() diff --git a/test/functional/feature_cltv.py b/test/functional/feature_cltv.py index 7fd0d0140b..9d32749a08 100755 --- a/test/functional/feature_cltv.py +++ b/test/functional/feature_cltv.py @@ -92,7 +92,7 @@ class BIP65Test(BitcoinTestFramework): self.rpc_timeout = 480 def test_cltv_info(self, *, is_active): - assert_equal(self.nodes[0].getblockchaininfo()['softforks']['bip65'], { + assert_equal(self.nodes[0].getdeploymentinfo()['deployments']['bip65'], { "active": is_active, "height": CLTV_HEIGHT, "type": "buried", diff --git a/test/functional/feature_coinstatsindex.py b/test/functional/feature_coinstatsindex.py index c70f8a83db..2e21638f80 100755 --- a/test/functional/feature_coinstatsindex.py +++ b/test/functional/feature_coinstatsindex.py @@ -18,9 +18,6 @@ from test_framework.blocktools import ( ) from test_framework.messages import ( COIN, - COutPoint, - CTransaction, - CTxIn, CTxOut, ) from test_framework.script import ( @@ -33,6 +30,11 @@ from test_framework.util import ( assert_equal, assert_raises_rpc_error, ) +from test_framework.wallet import ( + MiniWallet, + getnewdestination, +) + class CoinStatsIndexTest(BitcoinTestFramework): def set_test_params(self): @@ -40,16 +42,12 @@ class CoinStatsIndexTest(BitcoinTestFramework): self.num_nodes = 2 self.supports_cli = False self.extra_args = [ - # Explicitly set the output type in order to have consistent tx vsize / fees - # for both legacy and descriptor wallets (disables the change address type detection algorithm) - ["-addresstype=bech32", "-changetype=bech32"], + [], ["-coinstatsindex"] ] - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() - def run_test(self): + self.wallet = MiniWallet(self.nodes[0]) self._test_coin_stats_index() self._test_use_index_option() self._test_reorg_index() @@ -69,9 +67,8 @@ class CoinStatsIndexTest(BitcoinTestFramework): index_hash_options = ['none', 'muhash'] # Generate a normal transaction and mine it - self.generate(node, COINBASE_MATURITY + 1) - address = self.nodes[0].get_deterministic_priv_key().address - node.sendtoaddress(address=address, amount=10, subtractfeefromamount=True) + self.generate(self.wallet, COINBASE_MATURITY + 1) + self.wallet.send_self_transfer(from_node=node) self.generate(node, 1) self.log.info("Test that gettxoutsetinfo() output is consistent with or without coinstatsindex option") @@ -136,36 +133,31 @@ class CoinStatsIndexTest(BitcoinTestFramework): assert_equal(res5['block_info'], { 'unspendable': 0, 'prevout_spent': 50, - 'new_outputs_ex_coinbase': Decimal('49.99995560'), - 'coinbase': Decimal('50.00004440'), + 'new_outputs_ex_coinbase': Decimal('49.99968800'), + 'coinbase': Decimal('50.00031200'), 'unspendables': { 'genesis_block': 0, 'bip30': 0, 'scripts': 0, - 'unclaimed_rewards': 0 + 'unclaimed_rewards': 0, } }) self.block_sanity_check(res5['block_info']) # Generate and send a normal tx with two outputs - tx1_inputs = [] - tx1_outputs = {self.nodes[0].getnewaddress(): 21, self.nodes[0].getnewaddress(): 42} - raw_tx1 = self.nodes[0].createrawtransaction(tx1_inputs, tx1_outputs) - funded_tx1 = self.nodes[0].fundrawtransaction(raw_tx1) - signed_tx1 = self.nodes[0].signrawtransactionwithwallet(funded_tx1['hex']) - tx1_txid = self.nodes[0].sendrawtransaction(signed_tx1['hex']) + tx1_txid, tx1_vout = self.wallet.send_to( + from_node=node, + scriptPubKey=self.wallet.get_scriptPubKey(), + amount=21 * COIN, + ) # Find the right position of the 21 BTC output - tx1_final = self.nodes[0].gettransaction(tx1_txid) - for output in tx1_final['details']: - if output['amount'] == Decimal('21.00000000') and output['category'] == 'receive': - n = output['vout'] + tx1_out_21 = self.wallet.get_utxo(txid=tx1_txid, vout=tx1_vout) # Generate and send another tx with an OP_RETURN output (which is unspendable) - tx2 = CTransaction() - tx2.vin.append(CTxIn(COutPoint(int(tx1_txid, 16), n), b'')) - tx2.vout.append(CTxOut(int(Decimal('20.99') * COIN), CScript([OP_RETURN] + [OP_FALSE]*30))) - tx2_hex = self.nodes[0].signrawtransactionwithwallet(tx2.serialize().hex())['hex'] + tx2 = self.wallet.create_self_transfer(utxo_to_spend=tx1_out_21)['tx'] + tx2.vout = [CTxOut(int(Decimal('20.99') * COIN), CScript([OP_RETURN] + [OP_FALSE] * 30))] + tx2_hex = tx2.serialize().hex() self.nodes[0].sendrawtransaction(tx2_hex) # Include both txs in a block @@ -177,14 +169,14 @@ class CoinStatsIndexTest(BitcoinTestFramework): assert_equal(res6['total_unspendable_amount'], Decimal('70.99000000')) assert_equal(res6['block_info'], { 'unspendable': Decimal('20.99000000'), - 'prevout_spent': 111, - 'new_outputs_ex_coinbase': Decimal('89.99993620'), - 'coinbase': Decimal('50.01006380'), + 'prevout_spent': 71, + 'new_outputs_ex_coinbase': Decimal('49.99999000'), + 'coinbase': Decimal('50.01001000'), 'unspendables': { 'genesis_block': 0, 'bip30': 0, 'scripts': Decimal('20.99000000'), - 'unclaimed_rewards': 0 + 'unclaimed_rewards': 0, } }) self.block_sanity_check(res6['block_info']) @@ -231,6 +223,22 @@ class CoinStatsIndexTest(BitcoinTestFramework): res10 = index_node.gettxoutsetinfo('muhash') assert(res8['txouts'] < res10['txouts']) + self.log.info("Test that the index works with -reindex") + + self.restart_node(1, extra_args=["-coinstatsindex", "-reindex"]) + res11 = index_node.gettxoutsetinfo('muhash') + assert_equal(res11, res10) + + self.log.info("Test that -reindex-chainstate is disallowed with coinstatsindex") + + self.stop_node(1) + self.nodes[1].assert_start_raises_init_error( + expected_msg='Error: -reindex-chainstate option is not compatible with -coinstatsindex. ' + 'Please temporarily disable coinstatsindex while using -reindex-chainstate, or replace -reindex-chainstate with -reindex to fully rebuild all indexes.', + extra_args=['-coinstatsindex', '-reindex-chainstate'], + ) + self.restart_node(1, extra_args=["-coinstatsindex"]) + def _test_use_index_option(self): self.log.info("Test use_index option for nodes running the index") @@ -246,7 +254,7 @@ class CoinStatsIndexTest(BitcoinTestFramework): # Generate two block, let the index catch up, then invalidate the blocks index_node = self.nodes[1] - reorg_blocks = self.generatetoaddress(index_node, 2, index_node.getnewaddress()) + reorg_blocks = self.generatetoaddress(index_node, 2, getnewdestination()[2]) reorg_block = reorg_blocks[1] res_invalid = index_node.gettxoutsetinfo('muhash') index_node.invalidateblock(reorg_blocks[0]) diff --git a/test/functional/feature_config_args.py b/test/functional/feature_config_args.py index eea5fa24ee..6c51a5ac31 100755 --- a/test/functional/feature_config_args.py +++ b/test/functional/feature_config_args.py @@ -85,7 +85,7 @@ class ConfArgsTest(BitcoinTestFramework): def test_invalid_command_line_options(self): self.nodes[0].assert_start_raises_init_error( - expected_msg='Error: No proxy server specified. Use -proxy=<ip> or -proxy=<ip:port>.', + expected_msg='Error: Error parsing command line arguments: Can not set -proxy with no value. Please specify value with -proxy=value.', extra_args=['-proxy'], ) @@ -247,7 +247,8 @@ class ConfArgsTest(BitcoinTestFramework): conf_file = os.path.join(default_data_dir, "bitcoin.conf") # datadir needs to be set before [chain] section - conf_file_contents = open(conf_file, encoding='utf8').read() + with open(conf_file, encoding='utf8') as f: + conf_file_contents = f.read() with open(conf_file, 'w', encoding='utf8') as f: f.write(f"datadir={new_data_dir}\n") f.write(conf_file_contents) diff --git a/test/functional/feature_csv_activation.py b/test/functional/feature_csv_activation.py index 6470c1c5eb..bff95c3b94 100755 --- a/test/functional/feature_csv_activation.py +++ b/test/functional/feature_csv_activation.py @@ -112,6 +112,7 @@ class BIP68_112_113Test(BitcoinTestFramework): tx.nVersion = txversion self.miniwallet.sign_tx(tx) tx.vin[0].scriptSig = CScript([-1, OP_CHECKSEQUENCEVERIFY, OP_DROP] + list(CScript(tx.vin[0].scriptSig))) + tx.rehash() return tx def create_bip112emptystack(self, input, txversion): @@ -119,6 +120,7 @@ class BIP68_112_113Test(BitcoinTestFramework): tx.nVersion = txversion self.miniwallet.sign_tx(tx) tx.vin[0].scriptSig = CScript([OP_CHECKSEQUENCEVERIFY] + list(CScript(tx.vin[0].scriptSig))) + tx.rehash() return tx def send_generic_input_tx(self, coinbases): @@ -136,7 +138,6 @@ class BIP68_112_113Test(BitcoinTestFramework): tx.nVersion = txversion tx.vin[0].nSequence = locktime + locktime_delta self.miniwallet.sign_tx(tx) - tx.rehash() txs.append({'tx': tx, 'sdf': sdf, 'stf': stf}) return txs @@ -339,20 +340,16 @@ class BIP68_112_113Test(BitcoinTestFramework): # BIP 113 tests should now fail regardless of version number if nLockTime isn't satisfied by new rules bip113tx_v1.nLockTime = self.last_block_time - 600 * 5 # = MTP of prior block (not <) but < time put on current block self.miniwallet.sign_tx(bip113tx_v1) - bip113tx_v1.rehash() bip113tx_v2.nLockTime = self.last_block_time - 600 * 5 # = MTP of prior block (not <) but < time put on current block self.miniwallet.sign_tx(bip113tx_v2) - bip113tx_v2.rehash() for bip113tx in [bip113tx_v1, bip113tx_v2]: self.send_blocks([self.create_test_block([bip113tx])], success=False, reject_reason='bad-txns-nonfinal') # BIP 113 tests should now pass if the locktime is < MTP bip113tx_v1.nLockTime = self.last_block_time - 600 * 5 - 1 # < MTP of prior block self.miniwallet.sign_tx(bip113tx_v1) - bip113tx_v1.rehash() bip113tx_v2.nLockTime = self.last_block_time - 600 * 5 - 1 # < MTP of prior block self.miniwallet.sign_tx(bip113tx_v2) - bip113tx_v2.rehash() for bip113tx in [bip113tx_v1, bip113tx_v2]: self.send_blocks([self.create_test_block([bip113tx])]) self.nodes[0].invalidateblock(self.nodes[0].getbestblockhash()) @@ -477,7 +474,6 @@ class BIP68_112_113Test(BitcoinTestFramework): for tx in [tx['tx'] for tx in bip112txs_vary_OP_CSV_v2 if not tx['sdf'] and tx['stf']]: tx.vin[0].nSequence = BASE_RELATIVE_LOCKTIME | SEQ_TYPE_FLAG self.miniwallet.sign_tx(tx) - tx.rehash() time_txs.append(tx) self.send_blocks([self.create_test_block(time_txs)]) diff --git a/test/functional/feature_dersig.py b/test/functional/feature_dersig.py index f35ce7e0c9..9a46839969 100755 --- a/test/functional/feature_dersig.py +++ b/test/functional/feature_dersig.py @@ -60,7 +60,7 @@ class BIP66Test(BitcoinTestFramework): return self.miniwallet.create_self_transfer(utxo_to_spend=utxo_to_spend)['tx'] def test_dersig_info(self, *, is_active): - assert_equal(self.nodes[0].getblockchaininfo()['softforks']['bip66'], + assert_equal(self.nodes[0].getdeploymentinfo()['deployments']['bip66'], { "active": is_active, "height": DERSIG_HEIGHT, diff --git a/test/functional/feature_dirsymlinks.py b/test/functional/feature_dirsymlinks.py new file mode 100755 index 0000000000..288754c04c --- /dev/null +++ b/test/functional/feature_dirsymlinks.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test successful startup with symlinked directories. +""" + +import os + +from test_framework.test_framework import BitcoinTestFramework + + +def rename_and_link(*, from_name, to_name): + os.rename(from_name, to_name) + os.symlink(to_name, from_name) + assert os.path.islink(from_name) and os.path.isdir(from_name) + + +class SymlinkTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + + def run_test(self): + dir_new_blocks = self.nodes[0].chain_path / "new_blocks" + dir_new_chainstate = self.nodes[0].chain_path / "new_chainstate" + self.stop_node(0) + + rename_and_link( + from_name=self.nodes[0].chain_path / "blocks", + to_name=dir_new_blocks, + ) + rename_and_link( + from_name=self.nodes[0].chain_path / "chainstate", + to_name=dir_new_chainstate, + ) + + self.start_node(0) + + +if __name__ == "__main__": + SymlinkTest().main() diff --git a/test/functional/feature_fee_estimation.py b/test/functional/feature_fee_estimation.py index 233ffd60da..422612a78e 100755 --- a/test/functional/feature_fee_estimation.py +++ b/test/functional/feature_fee_estimation.py @@ -3,25 +3,13 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test fee estimation code.""" +from copy import deepcopy from decimal import Decimal import os import random from test_framework.messages import ( COIN, - COutPoint, - CTransaction, - CTxIn, - CTxOut, -) -from test_framework.script import ( - CScript, - OP_1, - OP_DROP, - OP_TRUE, -) -from test_framework.script_util import ( - script_to_p2sh_script, ) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( @@ -31,22 +19,14 @@ from test_framework.util import ( assert_raises_rpc_error, satoshi_round, ) - -# Construct 2 trivial P2SH's and the ScriptSigs that spend them -# So we can create many transactions without needing to spend -# time signing. -SCRIPT = CScript([OP_1, OP_DROP]) -P2SH = script_to_p2sh_script(SCRIPT) -REDEEM_SCRIPT = CScript([OP_TRUE, SCRIPT]) +from test_framework.wallet import MiniWallet def small_txpuzzle_randfee( - from_node, conflist, unconflist, amount, min_fee, fee_increment + wallet, from_node, conflist, unconflist, amount, min_fee, fee_increment ): - """Create and send a transaction with a random fee. + """Create and send a transaction with a random fee using MiniWallet. - The transaction pays to a trivial P2SH script, and assumes that its inputs - are of the same form. The function takes a list of confirmed outputs and unconfirmed outputs and attempts to use the confirmed list first for its inputs. It adds the newly created outputs to the unconfirmed list. @@ -58,23 +38,29 @@ def small_txpuzzle_randfee( rand_fee = float(fee_increment) * (1.1892 ** random.randint(0, 28)) # Total fee ranges from min_fee to min_fee + 127*fee_increment fee = min_fee - fee_increment + satoshi_round(rand_fee) - tx = CTransaction() + utxos_to_spend = [] total_in = Decimal("0.00000000") while total_in <= (amount + fee) and len(conflist) > 0: t = conflist.pop(0) - total_in += t["amount"] - tx.vin.append(CTxIn(COutPoint(int(t["txid"], 16), t["vout"]), REDEEM_SCRIPT)) + total_in += t["value"] + utxos_to_spend.append(t) while total_in <= (amount + fee) and len(unconflist) > 0: t = unconflist.pop(0) - total_in += t["amount"] - tx.vin.append(CTxIn(COutPoint(int(t["txid"], 16), t["vout"]), REDEEM_SCRIPT)) + total_in += t["value"] + utxos_to_spend.append(t) if total_in <= amount + fee: raise RuntimeError(f"Insufficient funds: need {amount + fee}, have {total_in}") - tx.vout.append(CTxOut(int((total_in - amount - fee) * COIN), P2SH)) - tx.vout.append(CTxOut(int(amount * COIN), P2SH)) + tx = wallet.create_self_transfer_multi( + from_node=from_node, + utxos_to_spend=utxos_to_spend, + fee_per_output=0) + tx.vout[0].nValue = int((total_in - amount - fee) * COIN) + tx.vout.append(deepcopy(tx.vout[0])) + tx.vout[1].nValue = int(amount * COIN) + txid = from_node.sendrawtransaction(hexstring=tx.serialize().hex(), maxfeerate=0) - unconflist.append({"txid": txid, "vout": 0, "amount": total_in - amount - fee}) - unconflist.append({"txid": txid, "vout": 1, "amount": amount}) + unconflist.append({"txid": txid, "vout": 0, "value": total_in - amount - fee}) + unconflist.append({"txid": txid, "vout": 1, "value": amount}) return (tx.serialize().hex(), fee) @@ -129,17 +115,13 @@ def check_estimates(node, fees_seen): check_smart_estimates(node, fees_seen) -def send_tx(node, utxo, feerate): +def send_tx(wallet, node, utxo, feerate): """Broadcast a 1in-1out transaction with a specific input and feerate (sat/vb).""" - tx = CTransaction() - tx.vin = [CTxIn(COutPoint(int(utxo["txid"], 16), utxo["vout"]), REDEEM_SCRIPT)] - tx.vout = [CTxOut(int(utxo["amount"] * COIN), P2SH)] - - # vbytes == bytes as we are using legacy transactions - fee = tx.get_vsize() * feerate - tx.vout[0].nValue -= fee - - return node.sendrawtransaction(tx.serialize().hex()) + return wallet.send_self_transfer( + from_node=node, + utxo_to_spend=utxo, + fee_rate=Decimal(feerate * 1000) / COIN, + )['txid'] class EstimateFeeTest(BitcoinTestFramework): @@ -152,9 +134,6 @@ class EstimateFeeTest(BitcoinTestFramework): ["-whitelist=noban@127.0.0.1", "-blockmaxweight=32000"], ] - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() - def setup_network(self): """ We'll setup the network to have 3 nodes that all mine with different parameters. @@ -168,9 +147,6 @@ class EstimateFeeTest(BitcoinTestFramework): # (68k weight is room enough for 120 or so transactions) # Node2 is a stingy miner, that # produces too small blocks (room for only 55 or so transactions) - self.start_nodes() - self.import_deterministic_coinbase_privkeys() - self.stop_nodes() def transact_and_mine(self, numblocks, mining_node): min_fee = Decimal("0.00001") @@ -183,6 +159,7 @@ class EstimateFeeTest(BitcoinTestFramework): for _ in range(random.randrange(100 - 50, 100 + 50)): from_index = random.randint(1, 2) (txhex, fee) = small_txpuzzle_randfee( + self.wallet, self.nodes[from_index], self.confutxo, self.memutxo, @@ -205,24 +182,10 @@ class EstimateFeeTest(BitcoinTestFramework): def initial_split(self, node): """Split two coinbase UTxOs into many small coins""" - utxo_count = 2048 - self.confutxo = [] - splitted_amount = Decimal("0.04") - fee = Decimal("0.1") - change = Decimal("100") - splitted_amount * utxo_count - fee - tx = CTransaction() - tx.vin = [ - CTxIn(COutPoint(int(cb["txid"], 16), cb["vout"])) - for cb in node.listunspent()[:2] - ] - tx.vout = [CTxOut(int(splitted_amount * COIN), P2SH) for _ in range(utxo_count)] - tx.vout.append(CTxOut(int(change * COIN), P2SH)) - txhex = node.signrawtransactionwithwallet(tx.serialize().hex())["hex"] - txid = node.sendrawtransaction(txhex) - self.confutxo = [ - {"txid": txid, "vout": i, "amount": splitted_amount} - for i in range(utxo_count) - ] + self.confutxo = self.wallet.send_self_transfer_multi( + from_node=node, + utxos_to_spend=[self.wallet.get_utxo() for _ in range(2)], + num_outputs=2048)['new_utxos'] while len(node.getrawmempool()) > 0: self.generate(node, 1, sync_fun=self.no_op) @@ -284,12 +247,12 @@ class EstimateFeeTest(BitcoinTestFramework): # Broadcast 45 low fee transactions that will need to be RBF'd for _ in range(45): u = utxos.pop(0) - txid = send_tx(node, u, low_feerate) + txid = send_tx(self.wallet, node, u, low_feerate) utxos_to_respend.append(u) txids_to_replace.append(txid) # Broadcast 5 low fee transaction which don't need to for _ in range(5): - send_tx(node, utxos.pop(0), low_feerate) + send_tx(self.wallet, node, utxos.pop(0), low_feerate) # Mine the transactions on another node self.sync_mempools(wait=0.1, nodes=[node, miner]) for txid in txids_to_replace: @@ -298,7 +261,7 @@ class EstimateFeeTest(BitcoinTestFramework): # RBF the low-fee transactions while len(utxos_to_respend) > 0: u = utxos_to_respend.pop(0) - send_tx(node, u, high_feerate) + send_tx(self.wallet, node, u, high_feerate) # Mine the last replacement txs self.sync_mempools(wait=0.1, nodes=[node, miner]) @@ -316,6 +279,8 @@ class EstimateFeeTest(BitcoinTestFramework): # Split two coinbases into many small utxos self.start_node(0) + self.wallet = MiniWallet(self.nodes[0]) + self.wallet.rescan_utxos() self.initial_split(self.nodes[0]) self.log.info("Finished splitting") diff --git a/test/functional/feature_index_prune.py b/test/functional/feature_index_prune.py new file mode 100755 index 0000000000..3ee6a8036c --- /dev/null +++ b/test/functional/feature_index_prune.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020-2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test indices in conjunction with prune.""" +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + assert_greater_than, + assert_raises_rpc_error, +) + + +class FeatureIndexPruneTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 4 + self.extra_args = [ + ["-fastprune", "-prune=1", "-blockfilterindex=1"], + ["-fastprune", "-prune=1", "-coinstatsindex=1"], + ["-fastprune", "-prune=1", "-blockfilterindex=1", "-coinstatsindex=1"], + [] + ] + + def sync_index(self, height): + expected_filter = { + 'basic block filter index': {'synced': True, 'best_block_height': height}, + } + self.wait_until(lambda: self.nodes[0].getindexinfo() == expected_filter) + + expected_stats = { + 'coinstatsindex': {'synced': True, 'best_block_height': height} + } + self.wait_until(lambda: self.nodes[1].getindexinfo() == expected_stats) + + expected = {**expected_filter, **expected_stats} + self.wait_until(lambda: self.nodes[2].getindexinfo() == expected) + + def reconnect_nodes(self): + self.connect_nodes(0,1) + self.connect_nodes(0,2) + self.connect_nodes(0,3) + + def mine_batches(self, blocks): + n = blocks // 250 + for _ in range(n): + self.generate(self.nodes[0], 250) + self.generate(self.nodes[0], blocks % 250) + self.sync_blocks() + + def restart_without_indices(self): + for i in range(3): + self.restart_node(i, extra_args=["-fastprune", "-prune=1"]) + self.reconnect_nodes() + + def run_test(self): + filter_nodes = [self.nodes[0], self.nodes[2]] + stats_nodes = [self.nodes[1], self.nodes[2]] + + self.log.info("check if we can access blockfilters and coinstats when pruning is enabled but no blocks are actually pruned") + self.sync_index(height=200) + tip = self.nodes[0].getbestblockhash() + for node in filter_nodes: + assert_greater_than(len(node.getblockfilter(tip)['filter']), 0) + for node in stats_nodes: + assert(node.gettxoutsetinfo(hash_type="muhash", hash_or_height=tip)['muhash']) + + self.mine_batches(500) + self.sync_index(height=700) + + self.log.info("prune some blocks") + for node in self.nodes[:2]: + with node.assert_debug_log(['limited pruning to height 689']): + pruneheight_new = node.pruneblockchain(400) + # the prune heights used here and below are magic numbers that are determined by the + # thresholds at which block files wrap, so they depend on disk serialization and default block file size. + assert_equal(pruneheight_new, 249) + + self.log.info("check if we can access the tips blockfilter and coinstats when we have pruned some blocks") + tip = self.nodes[0].getbestblockhash() + for node in filter_nodes: + assert_greater_than(len(node.getblockfilter(tip)['filter']), 0) + for node in stats_nodes: + assert(node.gettxoutsetinfo(hash_type="muhash", hash_or_height=tip)['muhash']) + + self.log.info("check if we can access the blockfilter and coinstats of a pruned block") + height_hash = self.nodes[0].getblockhash(2) + for node in filter_nodes: + assert_greater_than(len(node.getblockfilter(height_hash)['filter']), 0) + for node in stats_nodes: + assert(node.gettxoutsetinfo(hash_type="muhash", hash_or_height=height_hash)['muhash']) + + # mine and sync index up to a height that will later be the pruneheight + self.generate(self.nodes[0], 51) + self.sync_index(height=751) + + self.restart_without_indices() + + self.log.info("make sure trying to access the indices throws errors") + for node in filter_nodes: + msg = "Index is not enabled for filtertype basic" + assert_raises_rpc_error(-1, msg, node.getblockfilter, height_hash) + for node in stats_nodes: + msg = "Querying specific block heights requires coinstatsindex" + assert_raises_rpc_error(-8, msg, node.gettxoutsetinfo, "muhash", height_hash) + + self.mine_batches(749) + + self.log.info("prune exactly up to the indices best blocks while the indices are disabled") + for i in range(3): + pruneheight_2 = self.nodes[i].pruneblockchain(1000) + assert_equal(pruneheight_2, 751) + # Restart the nodes again with the indices activated + self.restart_node(i, extra_args=self.extra_args[i]) + + self.log.info("make sure that we can continue with the partially synced indices after having pruned up to the index height") + self.sync_index(height=1500) + + self.log.info("prune further than the indices best blocks while the indices are disabled") + self.restart_without_indices() + self.mine_batches(1000) + + for i in range(3): + pruneheight_3 = self.nodes[i].pruneblockchain(2000) + assert_greater_than(pruneheight_3, pruneheight_2) + self.stop_node(i) + + self.log.info("make sure we get an init error when starting the nodes again with the indices") + filter_msg = "Error: basic block filter index best block of the index goes beyond pruned data. Please disable the index or reindex (which will download the whole blockchain again)" + stats_msg = "Error: coinstatsindex best block of the index goes beyond pruned data. Please disable the index or reindex (which will download the whole blockchain again)" + for i, msg in enumerate([filter_msg, stats_msg, filter_msg]): + self.nodes[i].assert_start_raises_init_error(extra_args=self.extra_args[i], expected_msg=msg) + + self.log.info("make sure the nodes start again with the indices and an additional -reindex arg") + for i in range(3): + restart_args = self.extra_args[i]+["-reindex"] + self.restart_node(i, extra_args=restart_args) + # The nodes need to be reconnected to the non-pruning node upon restart, otherwise they will be stuck + self.connect_nodes(i, 3) + + self.sync_blocks(timeout=300) + + for node in self.nodes[:2]: + with node.assert_debug_log(['limited pruning to height 2489']): + pruneheight_new = node.pruneblockchain(2500) + assert_equal(pruneheight_new, 2006) + + self.log.info("ensure that prune locks don't prevent indices from failing in a reorg scenario") + with self.nodes[0].assert_debug_log(['basic block filter index prune lock moved back to 2480']): + self.nodes[3].invalidateblock(self.nodes[0].getblockhash(2480)) + self.generate(self.nodes[3], 30) + self.sync_blocks() + + +if __name__ == '__main__': + FeatureIndexPruneTest().main() diff --git a/test/functional/feature_init.py b/test/functional/feature_init.py index 4b56b0c26b..d0cb1e10e2 100755 --- a/test/functional/feature_init.py +++ b/test/functional/feature_init.py @@ -3,8 +3,6 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Stress tests related to node initialization.""" -import random -import time import os from pathlib import Path @@ -26,7 +24,6 @@ class InitStressTest(BitcoinTestFramework): def run_test(self): """ - test terminating initialization after seeing a certain log line. - - test terminating init after seeing a random number of log lines. - test removing certain essential files to test startup error paths. """ # TODO: skip Windows for now since it isn't clear how to SIGTERM. @@ -67,6 +64,8 @@ class InitStressTest(BitcoinTestFramework): 'addcon thread start', 'loadblk thread start', 'txindex thread start', + 'block filter index thread start', + 'coinstatsindex thread start', 'msghand thread start', 'net thread start', 'addcon thread start', @@ -76,46 +75,14 @@ class InitStressTest(BitcoinTestFramework): for terminate_line in lines_to_terminate_after: self.log.info(f"Starting node and will exit after line '{terminate_line}'") - node.start(extra_args=['-txindex=1']) - - num_total_logs = node.wait_for_debug_log([terminate_line], ignore_case=True) - self.log.debug(f"Terminating node after {num_total_logs} log lines seen") + with node.wait_for_debug_log([terminate_line], ignore_case=True): + node.start(extra_args=['-txindex=1', '-blockfilterindex=1', '-coinstatsindex=1']) + self.log.debug("Terminating node after terminate line was found") sigterm_node() check_clean_start() self.stop_node(0) - self.log.info( - f"Terminate at some random point in the init process (max logs: {num_total_logs})") - - for _ in range(40): - num_logs = len(Path(node.debug_log_path).read_text().splitlines()) - additional_lines = random.randint(1, num_total_logs) - self.log.debug(f"Starting node and will exit after {additional_lines} lines") - node.start(extra_args=['-txindex=1']) - logfile = open(node.debug_log_path, 'rb') - - MAX_SECS_TO_WAIT = 10 - start = time.time() - num_lines = 0 - - while True: - line = logfile.readline() - if line: - num_lines += 1 - - if num_lines >= (num_logs + additional_lines) or \ - (time.time() - start) > MAX_SECS_TO_WAIT: - self.log.debug(f"Terminating node after {num_lines} log lines seen") - sigterm_node() - break - - if node.process.poll() is not None: - raise AssertionError("node failed to start") - - check_clean_start() - self.stop_node(0) - self.log.info("Test startup errors after removing certain essential files") files_to_disturb = { @@ -144,7 +111,7 @@ class InitStressTest(BitcoinTestFramework): # investigate doing this later. node.assert_start_raises_init_error( - extra_args=['-txindex=1'], + extra_args=['-txindex=1', '-blockfilterindex=1', '-coinstatsindex=1'], expected_msg=err_fragment, match=ErrorMatch.PARTIAL_REGEX, ) diff --git a/test/functional/feature_maxtipage.py b/test/functional/feature_maxtipage.py new file mode 100755 index 0000000000..87f9d6962d --- /dev/null +++ b/test/functional/feature_maxtipage.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test logic for setting nMaxTipAge on command line. + +Nodes don't consider themselves out of "initial block download" as long as +their best known block header time is more than nMaxTipAge in the past. +""" + +import time + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal + + +DEFAULT_MAX_TIP_AGE = 24 * 60 * 60 + + +class MaxTipAgeTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 2 + + def test_maxtipage(self, maxtipage, set_parameter=True): + node_miner = self.nodes[0] + node_ibd = self.nodes[1] + + self.restart_node(1, [f'-maxtipage={maxtipage}'] if set_parameter else None) + self.connect_nodes(0, 1) + + # tips older than maximum age -> stay in IBD + cur_time = int(time.time()) + node_ibd.setmocktime(cur_time) + for delta in [5, 4, 3, 2, 1]: + node_miner.setmocktime(cur_time - maxtipage - delta) + self.generate(node_miner, 1) + assert_equal(node_ibd.getblockchaininfo()['initialblockdownload'], True) + + # tip within maximum age -> leave IBD + node_miner.setmocktime(cur_time - maxtipage) + self.generate(node_miner, 1) + assert_equal(node_ibd.getblockchaininfo()['initialblockdownload'], False) + + def run_test(self): + self.log.info("Test IBD with maximum tip age of 24 hours (default).") + self.test_maxtipage(DEFAULT_MAX_TIP_AGE, set_parameter=False) + + for hours in [20, 10, 5, 2, 1]: + maxtipage = hours * 60 * 60 + self.log.info(f"Test IBD with maximum tip age of {hours} hours (-maxtipage={maxtipage}).") + self.test_maxtipage(maxtipage) + + +if __name__ == '__main__': + MaxTipAgeTest().main() diff --git a/test/functional/feature_maxuploadtarget.py b/test/functional/feature_maxuploadtarget.py index 24f79dda67..0b9d651226 100755 --- a/test/functional/feature_maxuploadtarget.py +++ b/test/functional/feature_maxuploadtarget.py @@ -13,10 +13,19 @@ if uploadtarget has been reached. from collections import defaultdict import time -from test_framework.messages import CInv, MSG_BLOCK, msg_getdata +from test_framework.messages import ( + CInv, + MSG_BLOCK, + msg_getdata, +) from test_framework.p2p import P2PInterface from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, mine_large_block +from test_framework.util import ( + assert_equal, + mine_large_block, +) +from test_framework.wallet import MiniWallet + class TestP2PConn(P2PInterface): def __init__(self): @@ -41,12 +50,6 @@ class MaxUploadTest(BitcoinTestFramework): ]] self.supports_cli = False - # Cache for utxos, as the listunspent may take a long time later in the test - self.utxo_cache = [] - - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() - def run_test(self): # Before we connect anything, we first set the time on the node # to be in the past, otherwise things break because the CNode @@ -55,7 +58,8 @@ class MaxUploadTest(BitcoinTestFramework): self.nodes[0].setmocktime(old_time) # Generate some old blocks - self.generate(self.nodes[0], 130) + self.wallet = MiniWallet(self.nodes[0]) + self.generate(self.wallet, 130) # p2p_conns[0] will only request old blocks # p2p_conns[1] will only request new blocks @@ -66,7 +70,7 @@ class MaxUploadTest(BitcoinTestFramework): p2p_conns.append(self.nodes[0].add_p2p_connection(TestP2PConn())) # Now mine a big block - mine_large_block(self, self.nodes[0], self.utxo_cache) + mine_large_block(self, self.wallet, self.nodes[0]) # Store the hash; we'll request this later big_old_block = self.nodes[0].getbestblockhash() @@ -77,7 +81,7 @@ class MaxUploadTest(BitcoinTestFramework): self.nodes[0].setmocktime(int(time.time()) - 2*60*60*24) # Mine one more block, so that the prior block looks old - mine_large_block(self, self.nodes[0], self.utxo_cache) + mine_large_block(self, self.wallet, self.nodes[0]) # We'll be requesting this new block too big_new_block = self.nodes[0].getbestblockhash() diff --git a/test/functional/feature_proxy.py b/test/functional/feature_proxy.py index 7d9e5b70fc..50e0e2c4cc 100755 --- a/test/functional/feature_proxy.py +++ b/test/functional/feature_proxy.py @@ -30,6 +30,13 @@ addnode connect to generic DNS name addnode connect to a CJDNS address - Test getnetworkinfo for each node + +- Test passing invalid -proxy +- Test passing invalid -onion +- Test passing invalid -i2psam +- Test passing -onlynet=onion without -proxy or -onion +- Test passing -onlynet=onion with -onion=0 and with -noonion +- Test passing unknown -onlynet """ import socket @@ -234,7 +241,15 @@ class ProxyTest(BitcoinTestFramework): return r self.log.info("Test RPC getnetworkinfo") - n0 = networks_dict(self.nodes[0].getnetworkinfo()) + nodes_network_info = [] + + self.log.debug("Test that setting -proxy disables local address discovery, i.e. -discover=0") + for node in self.nodes: + network_info = node.getnetworkinfo() + assert_equal(network_info["localaddresses"], []) + nodes_network_info.append(network_info) + + n0 = networks_dict(nodes_network_info[0]) assert_equal(NETWORKS, n0.keys()) for net in NETWORKS: if net == NET_I2P: @@ -249,7 +264,7 @@ class ProxyTest(BitcoinTestFramework): assert_equal(n0['i2p']['reachable'], False) assert_equal(n0['cjdns']['reachable'], False) - n1 = networks_dict(self.nodes[1].getnetworkinfo()) + n1 = networks_dict(nodes_network_info[1]) assert_equal(NETWORKS, n1.keys()) for net in ['ipv4', 'ipv6']: assert_equal(n1[net]['proxy'], f'{self.conf1.addr[0]}:{self.conf1.addr[1]}') @@ -261,14 +276,15 @@ class ProxyTest(BitcoinTestFramework): assert_equal(n1['i2p']['proxy_randomize_credentials'], False) assert_equal(n1['i2p']['reachable'], True) - n2 = networks_dict(self.nodes[2].getnetworkinfo()) + n2 = networks_dict(nodes_network_info[2]) assert_equal(NETWORKS, n2.keys()) + proxy = f'{self.conf2.addr[0]}:{self.conf2.addr[1]}' for net in NETWORKS: if net == NET_I2P: expected_proxy = '' expected_randomize = False else: - expected_proxy = f'{self.conf2.addr[0]}:{self.conf2.addr[1]}' + expected_proxy = proxy expected_randomize = True assert_equal(n2[net]['proxy'], expected_proxy) assert_equal(n2[net]['proxy_randomize_credentials'], expected_randomize) @@ -277,20 +293,18 @@ class ProxyTest(BitcoinTestFramework): assert_equal(n2['cjdns']['reachable'], False) if self.have_ipv6: - n3 = networks_dict(self.nodes[3].getnetworkinfo()) + n3 = networks_dict(nodes_network_info[3]) assert_equal(NETWORKS, n3.keys()) + proxy = f'[{self.conf3.addr[0]}]:{self.conf3.addr[1]}' for net in NETWORKS: - if net == NET_I2P: - expected_proxy = '' - else: - expected_proxy = f'[{self.conf3.addr[0]}]:{self.conf3.addr[1]}' + expected_proxy = '' if net == NET_I2P or net == NET_ONION else proxy assert_equal(n3[net]['proxy'], expected_proxy) assert_equal(n3[net]['proxy_randomize_credentials'], False) assert_equal(n3['onion']['reachable'], False) assert_equal(n3['i2p']['reachable'], False) assert_equal(n3['cjdns']['reachable'], False) - n4 = networks_dict(self.nodes[4].getnetworkinfo()) + n4 = networks_dict(nodes_network_info[4]) assert_equal(NETWORKS, n4.keys()) for net in NETWORKS: if net == NET_I2P: @@ -305,6 +319,42 @@ class ProxyTest(BitcoinTestFramework): assert_equal(n4['i2p']['reachable'], False) assert_equal(n4['cjdns']['reachable'], True) + self.stop_node(1) + + self.log.info("Test passing invalid -proxy raises expected init error") + self.nodes[1].extra_args = ["-proxy=abc:def"] + msg = "Error: Invalid -proxy address or hostname: 'abc:def'" + self.nodes[1].assert_start_raises_init_error(expected_msg=msg) + + self.log.info("Test passing invalid -onion raises expected init error") + self.nodes[1].extra_args = ["-onion=xyz:abc"] + msg = "Error: Invalid -onion address or hostname: 'xyz:abc'" + self.nodes[1].assert_start_raises_init_error(expected_msg=msg) + + self.log.info("Test passing invalid -i2psam raises expected init error") + self.nodes[1].extra_args = ["-i2psam=def:xyz"] + msg = "Error: Invalid -i2psam address or hostname: 'def:xyz'" + self.nodes[1].assert_start_raises_init_error(expected_msg=msg) + + msg = ( + "Error: Outbound connections restricted to Tor (-onlynet=onion) but " + "the proxy for reaching the Tor network is not provided (no -proxy= " + "and no -onion= given) or it is explicitly forbidden (-onion=0)" + ) + self.log.info("Test passing -onlynet=onion without -proxy or -onion raises expected init error") + self.nodes[1].extra_args = ["-onlynet=onion"] + self.nodes[1].assert_start_raises_init_error(expected_msg=msg) + + self.log.info("Test passing -onlynet=onion with -onion=0/-noonion raises expected init error") + for arg in ["-onion=0", "-noonion"]: + self.nodes[1].extra_args = ["-onlynet=onion", arg] + self.nodes[1].assert_start_raises_init_error(expected_msg=msg) + + self.log.info("Test passing unknown network to -onlynet raises expected init error") + self.nodes[1].extra_args = ["-onlynet=abc"] + msg = "Error: Unknown network specified in -onlynet: 'abc'" + self.nodes[1].assert_start_raises_init_error(expected_msg=msg) + if __name__ == '__main__': ProxyTest().main() diff --git a/test/functional/feature_pruning.py b/test/functional/feature_pruning.py index ba3c5053cb..77524e85a3 100755 --- a/test/functional/feature_pruning.py +++ b/test/functional/feature_pruning.py @@ -125,6 +125,7 @@ class PruneTest(BitcoinTestFramework): self.sync_blocks(self.nodes[0:5]) def test_invalid_command_line_options(self): + self.stop_node(0) self.nodes[0].assert_start_raises_init_error( expected_msg='Error: Prune cannot be configured with a negative value.', extra_args=['-prune=-1'], @@ -138,8 +139,8 @@ class PruneTest(BitcoinTestFramework): extra_args=['-prune=550', '-txindex'], ) self.nodes[0].assert_start_raises_init_error( - expected_msg='Error: Prune mode is incompatible with -coinstatsindex.', - extra_args=['-prune=550', '-coinstatsindex'], + expected_msg='Error: Prune mode is incompatible with -reindex-chainstate. Use full -reindex instead.', + extra_args=['-prune=550', '-reindex-chainstate'], ) def test_height_min(self): diff --git a/test/functional/feature_segwit.py b/test/functional/feature_segwit.py index 6d7f1def88..f0faf1421b 100755 --- a/test/functional/feature_segwit.py +++ b/test/functional/feature_segwit.py @@ -86,18 +86,18 @@ class SegWitTest(BitcoinTestFramework): [ "-acceptnonstdtxn=1", "-rpcserialversion=0", - "-testactivationheight=segwit@432", + "-testactivationheight=segwit@165", "-addresstype=legacy", ], [ "-acceptnonstdtxn=1", "-rpcserialversion=1", - "-testactivationheight=segwit@432", + "-testactivationheight=segwit@165", "-addresstype=legacy", ], [ "-acceptnonstdtxn=1", - "-testactivationheight=segwit@432", + "-testactivationheight=segwit@165", "-addresstype=legacy", ], ] @@ -117,12 +117,6 @@ class SegWitTest(BitcoinTestFramework): assert_equal(len(node.getblock(block[0])["tx"]), 2) self.sync_blocks() - def skip_mine(self, node, txid, sign, redeem_script=""): - send_to_witness(1, node, getutxo(txid), self.pubkey[0], False, Decimal("49.998"), sign, redeem_script) - block = self.generate(node, 1) - assert_equal(len(node.getblock(block[0])["tx"]), 1) - self.sync_blocks() - def fail_accept(self, node, error_msg, txid, sign, redeem_script=""): assert_raises_rpc_error(-26, error_msg, send_to_witness, use_p2wsh=1, node=node, utxo=getutxo(txid), pubkey=self.pubkey[0], encode_p2sh=False, amount=Decimal("49.998"), sign=sign, insert_redeem_script=redeem_script) @@ -197,23 +191,21 @@ class SegWitTest(BitcoinTestFramework): assert_equal(self.nodes[1].getbalance(), 20 * Decimal("49.999")) assert_equal(self.nodes[2].getbalance(), 20 * Decimal("49.999")) - self.generate(self.nodes[0], 260) # block 423 - - self.log.info("Verify witness txs are skipped for mining before the fork") - self.skip_mine(self.nodes[2], wit_ids[NODE_2][P2WPKH][0], True) # block 424 - self.skip_mine(self.nodes[2], wit_ids[NODE_2][P2WSH][0], True) # block 425 - self.skip_mine(self.nodes[2], p2sh_ids[NODE_2][P2WPKH][0], True) # block 426 - self.skip_mine(self.nodes[2], p2sh_ids[NODE_2][P2WSH][0], True) # block 427 - self.log.info("Verify unsigned p2sh witness txs without a redeem script are invalid") self.fail_accept(self.nodes[2], "mandatory-script-verify-flag-failed (Operation not valid with the current stack size)", p2sh_ids[NODE_2][P2WPKH][1], sign=False) self.fail_accept(self.nodes[2], "mandatory-script-verify-flag-failed (Operation not valid with the current stack size)", p2sh_ids[NODE_2][P2WSH][1], sign=False) - self.generate(self.nodes[2], 4) # blocks 428-431 + self.generate(self.nodes[0], 1) # block 164 + + self.log.info("Verify witness txs are mined as soon as segwit activates") + + send_to_witness(1, self.nodes[2], getutxo(wit_ids[NODE_2][P2WPKH][0]), self.pubkey[0], encode_p2sh=False, amount=Decimal("49.998"), sign=True) + send_to_witness(1, self.nodes[2], getutxo(wit_ids[NODE_2][P2WSH][0]), self.pubkey[0], encode_p2sh=False, amount=Decimal("49.998"), sign=True) + send_to_witness(1, self.nodes[2], getutxo(p2sh_ids[NODE_2][P2WPKH][0]), self.pubkey[0], encode_p2sh=False, amount=Decimal("49.998"), sign=True) + send_to_witness(1, self.nodes[2], getutxo(p2sh_ids[NODE_2][P2WSH][0]), self.pubkey[0], encode_p2sh=False, amount=Decimal("49.998"), sign=True) - self.log.info("Verify previous witness txs skipped for mining can now be mined") assert_equal(len(self.nodes[2].getrawmempool()), 4) - blockhash = self.generate(self.nodes[2], 1)[0] # block 432 (first block with new rules; 432 = 144 * 3) + blockhash = self.generate(self.nodes[2], 1)[0] # block 165 (first block with new rules) assert_equal(len(self.nodes[2].getrawmempool()), 0) segwit_tx_list = self.nodes[2].getblock(blockhash)["tx"] assert_equal(len(segwit_tx_list), 5) @@ -255,10 +247,10 @@ class SegWitTest(BitcoinTestFramework): self.fail_accept(self.nodes[2], 'non-mandatory-script-verify-flag (Witness program was passed an empty witness)', p2sh_ids[NODE_2][P2WSH][2], sign=False, redeem_script=witness_script(True, self.pubkey[2])) self.log.info("Verify default node can now use witness txs") - self.success_mine(self.nodes[0], wit_ids[NODE_0][P2WPKH][0], True) # block 432 - self.success_mine(self.nodes[0], wit_ids[NODE_0][P2WSH][0], True) # block 433 - self.success_mine(self.nodes[0], p2sh_ids[NODE_0][P2WPKH][0], True) # block 434 - self.success_mine(self.nodes[0], p2sh_ids[NODE_0][P2WSH][0], True) # block 435 + self.success_mine(self.nodes[0], wit_ids[NODE_0][P2WPKH][0], True) + self.success_mine(self.nodes[0], wit_ids[NODE_0][P2WSH][0], True) + self.success_mine(self.nodes[0], p2sh_ids[NODE_0][P2WPKH][0], True) + self.success_mine(self.nodes[0], p2sh_ids[NODE_0][P2WSH][0], True) self.log.info("Verify sigops are counted in GBT with BIP141 rules after the fork") txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 1) diff --git a/test/functional/feature_syscall_sandbox.py b/test/functional/feature_syscall_sandbox.py index caf7f1e7fc..e430542845 100755 --- a/test/functional/feature_syscall_sandbox.py +++ b/test/functional/feature_syscall_sandbox.py @@ -14,7 +14,7 @@ class SyscallSandboxTest(BitcoinTestFramework): def skip_test_if_missing_module(self): if not self.is_syscall_sandbox_compiled(): raise SkipTest("bitcoind has not been built with syscall sandbox enabled.") - if self.options.nosandbox: + if self.disable_syscall_sandbox: raise SkipTest("--nosandbox passed to test runner.") def run_test(self): diff --git a/test/functional/feature_taproot.py b/test/functional/feature_taproot.py index 3e3d4b3c77..0e44038196 100755 --- a/test/functional/feature_taproot.py +++ b/test/functional/feature_taproot.py @@ -10,7 +10,6 @@ from test_framework.blocktools import ( create_block, add_witness_commitment, MAX_BLOCK_SIGOPS_WEIGHT, - NORMAL_GBT_REQUEST_PARAMS, WITNESS_SCALE_FACTOR, ) from test_framework.messages import ( @@ -96,10 +95,9 @@ from test_framework.util import assert_raises_rpc_error, assert_equal from test_framework.key import generate_privkey, compute_xonly_pubkey, sign_schnorr, tweak_add_privkey, ECKey from test_framework.address import ( hash160, - program_to_witness + program_to_witness, ) from collections import OrderedDict, namedtuple -from enum import Enum from io import BytesIO import json import hashlib @@ -458,7 +456,7 @@ def spend(tx, idx, utxos, **kwargs): # Each spender is a tuple of: # - A scriptPubKey which is to be spent from (CScript) # - A comment describing the test (string) -# - Whether the spending (on itself) is expected to be standard (Enum.Standard) +# - Whether the spending (on itself) is expected to be standard (bool) # - A tx-signing lambda returning (scriptsig, witness_stack), taking as inputs: # - A transaction to sign (CTransaction) # - An input position (int) @@ -470,14 +468,9 @@ def spend(tx, idx, utxos, **kwargs): # - Whether this test demands being placed in a txin with no corresponding txout (for testing SIGHASH_SINGLE behavior) Spender = namedtuple("Spender", "script,comment,is_standard,sat_function,err_msg,sigops_weight,no_fail,need_vin_vout_mismatch") -# The full node versions that treat the tx standard. -# ALL means any version -# V23 means the major version 23.0 and any later version -# NONE means no version -Standard = Enum('Standard', 'ALL V23 NONE') -def make_spender(comment, *, tap=None, witv0=False, script=None, pkh=None, p2sh=False, spk_mutate_pre_p2sh=None, failure=None, standard=Standard.ALL, err_msg=None, sigops_weight=0, need_vin_vout_mismatch=False, **kwargs): +def make_spender(comment, *, tap=None, witv0=False, script=None, pkh=None, p2sh=False, spk_mutate_pre_p2sh=None, failure=None, standard=True, err_msg=None, sigops_weight=0, need_vin_vout_mismatch=False, **kwargs): """Helper for constructing Spender objects using the context signing framework. * tap: a TaprootInfo object (see taproot_construct), for Taproot spends (cannot be combined with pkh, witv0, or script) @@ -487,18 +480,13 @@ def make_spender(comment, *, tap=None, witv0=False, script=None, pkh=None, p2sh= * p2sh: whether the output is P2SH wrapper (this is supported even for Taproot, where it makes the output unencumbered) * spk_mutate_pre_psh: a callable to be applied to the script (before potentially P2SH-wrapping it) * failure: a dict of entries to override in the context when intentionally failing to spend (if None, no_fail will be set) - * standard: whether the (valid version of) spending is expected to be standard (True is mapped to Standard.ALL, False is mapped to Standard.NONE) + * standard: whether the (valid version of) spending is expected to be standard * err_msg: a string with an expected error message for failure (or None, if not cared about) * sigops_weight: the pre-taproot sigops weight consumed by a successful spend * need_vin_vout_mismatch: whether this test requires being tested in a transaction input that has no corresponding transaction output. """ - if standard == True: - standard = Standard.ALL - elif standard == False: - standard = Standard.NONE - conf = dict() # Compute scriptPubKey and set useful defaults based on the inputs. @@ -1168,29 +1156,21 @@ def spenders_taproot_active(): return spenders -def spenders_taproot_inactive(): - """Spenders for testing that pre-activation Taproot rules don't apply.""" + +def spenders_taproot_nonstandard(): + """Spenders for testing that post-activation Taproot rules may be nonstandard.""" spenders = [] sec = generate_privkey() pub, _ = compute_xonly_pubkey(sec) scripts = [ - ("pk", CScript([pub, OP_CHECKSIG])), ("future_leaf", CScript([pub, OP_CHECKSIG]), 0xc2), ("op_success", CScript([pub, OP_CHECKSIG, OP_0, OP_IF, CScriptOp(0x50), OP_ENDIF])), ] tap = taproot_construct(pub, scripts) - # Test that keypath spending is valid & non-standard, regardless of validity. - add_spender(spenders, "inactive/keypath_valid", key=sec, tap=tap, standard=Standard.V23) - add_spender(spenders, "inactive/keypath_invalidsig", key=sec, tap=tap, standard=False, sighash=bitflipper(default_sighash)) - add_spender(spenders, "inactive/keypath_empty", key=sec, tap=tap, standard=False, witness=[]) - - # Same for scriptpath spending (and features like annex, leaf versions, or OP_SUCCESS don't change this) - add_spender(spenders, "inactive/scriptpath_valid", key=sec, tap=tap, leaf="pk", standard=Standard.V23, inputs=[getter("sign")]) - add_spender(spenders, "inactive/scriptpath_invalidsig", key=sec, tap=tap, leaf="pk", standard=False, inputs=[getter("sign")], sighash=bitflipper(default_sighash)) - add_spender(spenders, "inactive/scriptpath_invalidcb", key=sec, tap=tap, leaf="pk", standard=False, inputs=[getter("sign")], controlblock=bitflipper(default_controlblock)) + # Test that features like annex, leaf versions, or OP_SUCCESS are valid but non-standard add_spender(spenders, "inactive/scriptpath_valid_unkleaf", key=sec, tap=tap, leaf="future_leaf", standard=False, inputs=[getter("sign")]) add_spender(spenders, "inactive/scriptpath_invalid_unkleaf", key=sec, tap=tap, leaf="future_leaf", standard=False, inputs=[getter("sign")], sighash=bitflipper(default_sighash)) add_spender(spenders, "inactive/scriptpath_valid_opsuccess", key=sec, tap=tap, leaf="op_success", standard=False, inputs=[getter("sign")]) @@ -1218,7 +1198,7 @@ def dump_json_test(tx, input_utxos, idx, success, failure): # The "final" field indicates that a spend should be always valid, even with more validation flags enabled # than the listed ones. Use standardness as a proxy for this (which gives a conservative underestimate). - if spender.is_standard == Standard.ALL: + if spender.is_standard: fields.append(("final", True)) def dump_witness(wit): @@ -1245,31 +1225,14 @@ class TaprootTest(BitcoinTestFramework): def add_options(self, parser): parser.add_argument("--dumptests", dest="dump_tests", default=False, action="store_true", help="Dump generated test cases to directory set by TEST_DUMP_DIR environment variable") - parser.add_argument("--previous_release", dest="previous_release", default=False, action="store_true", - help="Use a previous release as taproot-inactive node") def skip_test_if_missing_module(self): self.skip_if_no_wallet() - if self.options.previous_release: - self.skip_if_no_previous_releases() def set_test_params(self): - self.num_nodes = 2 + self.num_nodes = 1 self.setup_clean_chain = True - # Node 0 has Taproot inactive, Node 1 active. - self.extra_args = [["-par=1"], ["-par=1"]] - if self.options.previous_release: - self.wallet_names = [None, self.default_wallet_name] - else: - self.extra_args[0].append("-vbparams=taproot:1:1") - - def setup_nodes(self): - self.add_nodes(self.num_nodes, self.extra_args, versions=[ - 200100 if self.options.previous_release else None, - None, - ]) - self.start_nodes() - self.import_deterministic_coinbase_privkeys() + self.extra_args = [["-par=1"]] def block_submit(self, node, txs, msg, err_msg, cb_pubkey=None, fees=0, sigops_weight=0, witness=False, accept=False): @@ -1483,11 +1446,10 @@ class TaprootTest(BitcoinTestFramework): for i in range(len(input_utxos)): tx.vin[i].scriptSig = input_data[i][i != fail_input][0] tx.wit.vtxinwit[i].scriptWitness.stack = input_data[i][i != fail_input][1] - taproot_spend_policy = Standard.V23 if node.version is None else Standard.ALL # Submit to mempool to check standardness is_standard_tx = ( fail_input is None # Must be valid to be standard - and (all(utxo.spender.is_standard == Standard.ALL or utxo.spender.is_standard == taproot_spend_policy for utxo in input_utxos)) # All inputs must be standard + and (all(utxo.spender.is_standard for utxo in input_utxos)) # All inputs must be standard and tx.nVersion >= 1 # The tx version must be standard and tx.nVersion <= 2) tx.rehash() @@ -1514,7 +1476,7 @@ class TaprootTest(BitcoinTestFramework): self.log.info("Unit test scenario...") # Deterministically mine coins to OP_TRUE in block 1 - assert self.nodes[1].getblockcount() == 0 + assert_equal(self.nodes[0].getblockcount(), 0) coinbase = CTransaction() coinbase.nVersion = 1 coinbase.vin = [CTxIn(COutPoint(0, 0xffffffff), CScript([OP_1, OP_1]), SEQUENCE_FINAL)] @@ -1523,12 +1485,12 @@ class TaprootTest(BitcoinTestFramework): coinbase.rehash() assert coinbase.hash == "f60c73405d499a956d3162e3483c395526ef78286458a4cb17b125aa92e49b20" # Mine it - block = create_block(hashprev=int(self.nodes[1].getbestblockhash(), 16), coinbase=coinbase) + block = create_block(hashprev=int(self.nodes[0].getbestblockhash(), 16), coinbase=coinbase) block.rehash() block.solve() - self.nodes[1].submitblock(block.serialize().hex()) - assert self.nodes[1].getblockcount() == 1 - self.generate(self.nodes[1], COINBASE_MATURITY) + self.nodes[0].submitblock(block.serialize().hex()) + assert_equal(self.nodes[0].getblockcount(), 1) + self.generate(self.nodes[0], COINBASE_MATURITY) SEED = 317 VALID_LEAF_VERS = list(range(0xc0, 0x100, 2)) + [0x66, 0x7e, 0x80, 0x84, 0x96, 0x98, 0xba, 0xbc, 0xbe] @@ -1617,8 +1579,8 @@ class TaprootTest(BitcoinTestFramework): spend_info[spk]['prevout'] = COutPoint(tx.sha256, i & 1) spend_info[spk]['utxo'] = CTxOut(val, spk) # Mine those transactions - self.init_blockinfo(self.nodes[1]) - self.block_submit(self.nodes[1], txn, "Crediting txn", None, sigops_weight=10, accept=True) + self.init_blockinfo(self.nodes[0]) + self.block_submit(self.nodes[0], txn, "Crediting txn", None, sigops_weight=10, accept=True) # scriptPubKey computation tests = {"version": 1} @@ -1730,53 +1692,21 @@ class TaprootTest(BitcoinTestFramework): keypath_tests.append(tx_test) assert_equal(hashlib.sha256(tx.serialize()).hexdigest(), "24bab662cb55a7f3bae29b559f651674c62bcc1cd442d44715c0133939107b38") # Mine the spending transaction - self.block_submit(self.nodes[1], [tx], "Spending txn", None, sigops_weight=10000, accept=True, witness=True) + self.block_submit(self.nodes[0], [tx], "Spending txn", None, sigops_weight=10000, accept=True, witness=True) if GEN_TEST_VECTORS: print(json.dumps(tests, indent=4, sort_keys=False)) - def run_test(self): self.gen_test_vectors() - # Post-taproot activation tests go first (pre-taproot tests' blocks are invalid post-taproot). self.log.info("Post-activation tests...") - self.test_spenders(self.nodes[1], spenders_taproot_active(), input_counts=[1, 2, 2, 2, 2, 3]) - - # Re-connect nodes in case they have been disconnected - self.disconnect_nodes(0, 1) - self.connect_nodes(0, 1) - - # Transfer value of the largest 500 coins to pre-taproot node. - addr = self.nodes[0].getnewaddress() - - unsp = self.nodes[1].listunspent() - unsp = sorted(unsp, key=lambda i: i['amount'], reverse=True) - unsp = unsp[:500] - - rawtx = self.nodes[1].createrawtransaction( - inputs=[{ - 'txid': i['txid'], - 'vout': i['vout'] - } for i in unsp], - outputs={addr: sum(i['amount'] for i in unsp)} - ) - rawtx = self.nodes[1].signrawtransactionwithwallet(rawtx)['hex'] - - # Mine a block with the transaction - block = create_block(tmpl=self.nodes[1].getblocktemplate(NORMAL_GBT_REQUEST_PARAMS), txlist=[rawtx]) - add_witness_commitment(block) - block.solve() - assert_equal(None, self.nodes[1].submitblock(block.serialize().hex())) - self.sync_blocks() - - # Pre-taproot activation tests. - self.log.info("Pre-activation tests...") + self.test_spenders(self.nodes[0], spenders_taproot_active(), input_counts=[1, 2, 2, 2, 2, 3]) # Run each test twice; once in isolation, and once combined with others. Testing in isolation # means that the standardness is verified in every test (as combined transactions are only standard # when all their inputs are standard). - self.test_spenders(self.nodes[0], spenders_taproot_inactive(), input_counts=[1]) - self.test_spenders(self.nodes[0], spenders_taproot_inactive(), input_counts=[2, 3]) + self.test_spenders(self.nodes[0], spenders_taproot_nonstandard(), input_counts=[1]) + self.test_spenders(self.nodes[0], spenders_taproot_nonstandard(), input_counts=[2, 3]) if __name__ == '__main__': diff --git a/test/functional/feature_txindex_compatibility.py b/test/functional/feature_txindex_compatibility.py index bbe1d1b537..20b023d82c 100755 --- a/test/functional/feature_txindex_compatibility.py +++ b/test/functional/feature_txindex_compatibility.py @@ -78,7 +78,7 @@ class MempoolCompatibilityTest(BitcoinTestFramework): self.stop_nodes() - self.log.info("Check migrated txindex can not be read by legacy node") + self.log.info("Check migrated txindex cannot be read by legacy node") err_msg = f": You need to rebuild the database using -reindex to change -txindex.{os.linesep}Please restart with -reindex or -reindex-chainstate to recover." shutil.rmtree(legacy_chain_dir) shutil.copytree(migrate_chain_dir, legacy_chain_dir) diff --git a/test/functional/feature_unsupported_utxo_db.py b/test/functional/feature_unsupported_utxo_db.py new file mode 100755 index 0000000000..1c8c08d1d8 --- /dev/null +++ b/test/functional/feature_unsupported_utxo_db.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test that unsupported utxo db causes an init error. + +Previous releases are required by this test, see test/README.md. +""" + +import shutil + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal + + +class UnsupportedUtxoDbTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 2 + + def skip_test_if_missing_module(self): + self.skip_if_no_previous_releases() + + def setup_network(self): + self.add_nodes( + self.num_nodes, + versions=[ + 140300, # Last release with previous utxo db format + None, # For MiniWallet, without migration code + ], + ) + + def run_test(self): + self.log.info("Create previous version (v0.14.3) utxo db") + self.start_node(0) + block = self.generate(self.nodes[0], 1, sync_fun=self.no_op)[-1] + assert_equal(self.nodes[0].getbestblockhash(), block) + assert_equal(self.nodes[0].gettxoutsetinfo()["total_amount"], 50) + self.stop_nodes() + + self.log.info("Check init error") + legacy_utxos_dir = self.nodes[0].chain_path / "chainstate" + legacy_blocks_dir = self.nodes[0].chain_path / "blocks" + recent_utxos_dir = self.nodes[1].chain_path / "chainstate" + recent_blocks_dir = self.nodes[1].chain_path / "blocks" + shutil.copytree(legacy_utxos_dir, recent_utxos_dir) + shutil.copytree(legacy_blocks_dir, recent_blocks_dir) + self.nodes[1].assert_start_raises_init_error( + expected_msg="Error: Unsupported chainstate database format found. " + "Please restart with -reindex-chainstate. " + "This will rebuild the chainstate database.", + ) + + self.log.info("Drop legacy utxo db") + self.start_node(1, extra_args=["-reindex-chainstate"]) + assert_equal(self.nodes[1].getbestblockhash(), block) + assert_equal(self.nodes[1].gettxoutsetinfo()["total_amount"], 50) + + +if __name__ == "__main__": + UnsupportedUtxoDbTest().main() diff --git a/test/functional/feature_utxo_set_hash.py b/test/functional/feature_utxo_set_hash.py index 75180e62a2..4d486bc6f4 100755 --- a/test/functional/feature_utxo_set_hash.py +++ b/test/functional/feature_utxo_set_hash.py @@ -69,8 +69,8 @@ class UTXOSetHashTest(BitcoinTestFramework): assert_equal(finalized[::-1].hex(), node_muhash) self.log.info("Test deterministic UTXO set hash results") - assert_equal(node.gettxoutsetinfo()['hash_serialized_2'], "3a570529b4c32e77268de1f81b903c75cc2da53c48df0d125c1e697ba7c8c7b7") - assert_equal(node.gettxoutsetinfo("muhash")['muhash'], "a13e0e70eb8acc786549596e3bc154623f1a5a622ba2f70715f6773ec745f435") + assert_equal(node.gettxoutsetinfo()['hash_serialized_2'], "f9aa4fb5ffd10489b9a6994e70ccf1de8a8bfa2d5f201d9857332e9954b0855d") + assert_equal(node.gettxoutsetinfo("muhash")['muhash'], "d1725b2fe3ef43e55aa4907480aea98d406fc9e0bf8f60169e2305f1fbf5961b") def run_test(self): self.test_muhash_implementation() diff --git a/test/functional/feature_versionbits_warning.py b/test/functional/feature_versionbits_warning.py index e83dd7f446..1572463308 100755 --- a/test/functional/feature_versionbits_warning.py +++ b/test/functional/feature_versionbits_warning.py @@ -58,7 +58,8 @@ class VersionBitsWarningTest(BitcoinTestFramework): def versionbits_in_alert_file(self): """Test that the versionbits warning has been written to the alert file.""" - alert_text = open(self.alert_filename, 'r', encoding='utf8').read() + with open(self.alert_filename, 'r', encoding='utf8') as f: + alert_text = f.read() return VB_PATTERN.search(alert_text) is not None def run_test(self): diff --git a/test/functional/interface_rest.py b/test/functional/interface_rest.py index 2842b2534d..f36bbda3af 100755 --- a/test/functional/interface_rest.py +++ b/test/functional/interface_rest.py @@ -6,21 +6,33 @@ from decimal import Decimal from enum import Enum +import http.client from io import BytesIO import json from struct import pack, unpack - -import http.client +import typing import urllib.parse + +from test_framework.messages import ( + BLOCK_HEADER_SIZE, + COIN, +) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, assert_greater_than, assert_greater_than_or_equal, ) +from test_framework.wallet import ( + MiniWallet, + getnewdestination, +) + + +INVALID_PARAM = "abc" +UNKNOWN_PARAM = "0000000000000000000000000000000000000000000000000000000000000000" -from test_framework.messages import BLOCK_HEADER_SIZE class ReqType(Enum): JSON = 1 @@ -39,22 +51,28 @@ def filter_output_indices_by_value(vouts, value): class RESTTest (BitcoinTestFramework): def set_test_params(self): - self.setup_clean_chain = True self.num_nodes = 2 self.extra_args = [["-rest", "-blockfilterindex=1"], []] + # whitelist peers to speed up tx relay / mempool sync + for args in self.extra_args: + args.append("-whitelist=noban@127.0.0.1") self.supports_cli = False - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() - - def test_rest_request(self, uri, http_method='GET', req_type=ReqType.JSON, body='', status=200, ret_type=RetType.JSON): + def test_rest_request( + self, + uri: str, + http_method: str = 'GET', + req_type: ReqType = ReqType.JSON, + body: str = '', + status: int = 200, + ret_type: RetType = RetType.JSON, + query_params: typing.Dict[str, typing.Any] = None, + ) -> typing.Union[http.client.HTTPResponse, bytes, str, None]: rest_uri = '/rest' + uri - if req_type == ReqType.JSON: - rest_uri += '.json' - elif req_type == ReqType.BIN: - rest_uri += '.bin' - elif req_type == ReqType.HEX: - rest_uri += '.hex' + if req_type in ReqType: + rest_uri += f'.{req_type.name.lower()}' + if query_params: + rest_uri += f'?{urllib.parse.urlencode(query_params)}' conn = http.client.HTTPConnection(self.url.hostname, self.url.port) self.log.debug(f'{http_method} {rest_uri} {body}') @@ -73,19 +91,15 @@ class RESTTest (BitcoinTestFramework): elif ret_type == RetType.JSON: return json.loads(resp.read().decode('utf-8'), parse_float=Decimal) + return None + def run_test(self): self.url = urllib.parse.urlparse(self.nodes[0].url) - self.log.info("Mine blocks and send Bitcoin to node 1") - - # Random address so node1's balance doesn't increase - not_related_address = "2MxqoHEdNQTyYeX1mHcbrrpzgojbosTpCvJ" + self.wallet = MiniWallet(self.nodes[0]) + self.wallet.rescan_utxos() - self.generate(self.nodes[0], 1) - self.generatetoaddress(self.nodes[1], 100, not_related_address) - - assert_equal(self.nodes[0].getbalance(), 50) - - txid = self.nodes[0].sendtoaddress(self.nodes[1].getnewaddress(), 0.1) + self.log.info("Broadcast test transaction and sync nodes") + txid, _ = self.wallet.send_to(from_node=self.nodes[0], scriptPubKey=getnewdestination()[1], amount=int(0.1 * COIN)) self.sync_all() self.log.info("Test the /tx URI") @@ -103,13 +117,17 @@ class RESTTest (BitcoinTestFramework): n, = filter_output_indices_by_value(json_obj['vout'], Decimal('0.1')) spending = (txid, n) + # Test /tx with an invalid and an unknown txid + resp = self.test_rest_request(uri=f"/tx/{INVALID_PARAM}", ret_type=RetType.OBJ, status=400) + assert_equal(resp.read().decode('utf-8').rstrip(), f"Invalid hash: {INVALID_PARAM}") + resp = self.test_rest_request(uri=f"/tx/{UNKNOWN_PARAM}", ret_type=RetType.OBJ, status=404) + assert_equal(resp.read().decode('utf-8').rstrip(), f"{UNKNOWN_PARAM} not found") + self.log.info("Query an unspent TXO using the /getutxos URI") - self.generatetoaddress(self.nodes[1], 1, not_related_address) + self.generate(self.wallet, 1) bb_hash = self.nodes[0].getbestblockhash() - assert_equal(self.nodes[1].getbalance(), Decimal("0.1")) - # Check chainTip response json_obj = self.test_rest_request(f"/getutxos/{spending[0]}-{spending[1]}") assert_equal(json_obj['chaintipHash'], bb_hash) @@ -151,7 +169,7 @@ class RESTTest (BitcoinTestFramework): response_hash = output.read(32)[::-1].hex() assert_equal(bb_hash, response_hash) # check if getutxo's chaintip during calculation was fine - assert_equal(chain_height, 102) # chain height must be 102 + assert_equal(chain_height, 201) # chain height must be 201 (pre-mined chain [200] + generated block [1]) self.log.info("Test the /getutxos URI with and without /checkmempool") # Create a transaction, check that it's found with /checkmempool, but @@ -159,7 +177,7 @@ class RESTTest (BitcoinTestFramework): # found with or without /checkmempool. # do a tx and don't sync - txid = self.nodes[0].sendtoaddress(self.nodes[1].getnewaddress(), 0.1) + txid, _ = self.wallet.send_to(from_node=self.nodes[0], scriptPubKey=getnewdestination()[1], amount=int(0.1 * COIN)) json_obj = self.test_rest_request(f"/tx/{txid}") # get the spent output to later check for utxo (should be spent by then) spent = (json_obj['vin'][0]['txid'], json_obj['vin'][0]['vout']) @@ -201,16 +219,16 @@ class RESTTest (BitcoinTestFramework): self.generate(self.nodes[0], 1) # generate block to not affect upcoming tests - self.log.info("Test the /block, /blockhashbyheight and /headers URIs") + self.log.info("Test the /block, /blockhashbyheight, /headers, and /blockfilterheaders URIs") bb_hash = self.nodes[0].getbestblockhash() # Check result if block does not exists - assert_equal(self.test_rest_request('/headers/1/0000000000000000000000000000000000000000000000000000000000000000'), []) - self.test_rest_request('/block/0000000000000000000000000000000000000000000000000000000000000000', status=404, ret_type=RetType.OBJ) + assert_equal(self.test_rest_request(f"/headers/{UNKNOWN_PARAM}", query_params={"count": 1}), []) + self.test_rest_request(f"/block/{UNKNOWN_PARAM}", status=404, ret_type=RetType.OBJ) # Check result if block is not in the active chain self.nodes[0].invalidateblock(bb_hash) - assert_equal(self.test_rest_request(f'/headers/1/{bb_hash}'), []) + assert_equal(self.test_rest_request(f'/headers/{bb_hash}', query_params={'count': 1}), []) self.test_rest_request(f'/block/{bb_hash}') self.nodes[0].reconsiderblock(bb_hash) @@ -220,7 +238,7 @@ class RESTTest (BitcoinTestFramework): response_bytes = response.read() # Compare with block header - response_header = self.test_rest_request(f"/headers/1/{bb_hash}", req_type=ReqType.BIN, ret_type=RetType.OBJ) + response_header = self.test_rest_request(f"/headers/{bb_hash}", req_type=ReqType.BIN, ret_type=RetType.OBJ, query_params={"count": 1}) assert_equal(int(response_header.getheader('content-length')), BLOCK_HEADER_SIZE) response_header_bytes = response_header.read() assert_equal(response_bytes[:BLOCK_HEADER_SIZE], response_header_bytes) @@ -232,7 +250,7 @@ class RESTTest (BitcoinTestFramework): assert_equal(response_bytes.hex().encode(), response_hex_bytes) # Compare with hex block header - response_header_hex = self.test_rest_request(f"/headers/1/{bb_hash}", req_type=ReqType.HEX, ret_type=RetType.OBJ) + response_header_hex = self.test_rest_request(f"/headers/{bb_hash}", req_type=ReqType.HEX, ret_type=RetType.OBJ, query_params={"count": 1}) assert_greater_than(int(response_header_hex.getheader('content-length')), BLOCK_HEADER_SIZE*2) response_header_hex_bytes = response_header_hex.read(BLOCK_HEADER_SIZE*2) assert_equal(response_bytes[:BLOCK_HEADER_SIZE].hex().encode(), response_header_hex_bytes) @@ -250,8 +268,8 @@ class RESTTest (BitcoinTestFramework): assert_equal(blockhash, bb_hash) # Check invalid blockhashbyheight requests - resp = self.test_rest_request("/blockhashbyheight/abc", ret_type=RetType.OBJ, status=400) - assert_equal(resp.read().decode('utf-8').rstrip(), "Invalid height: abc") + resp = self.test_rest_request(f"/blockhashbyheight/{INVALID_PARAM}", ret_type=RetType.OBJ, status=400) + assert_equal(resp.read().decode('utf-8').rstrip(), f"Invalid height: {INVALID_PARAM}") resp = self.test_rest_request("/blockhashbyheight/1000000", ret_type=RetType.OBJ, status=404) assert_equal(resp.read().decode('utf-8').rstrip(), "Block height out of range") resp = self.test_rest_request("/blockhashbyheight/-1", ret_type=RetType.OBJ, status=400) @@ -259,7 +277,7 @@ class RESTTest (BitcoinTestFramework): self.test_rest_request("/blockhashbyheight/", ret_type=RetType.OBJ, status=400) # Compare with json block header - json_obj = self.test_rest_request(f"/headers/1/{bb_hash}") + json_obj = self.test_rest_request(f"/headers/{bb_hash}", query_params={"count": 1}) assert_equal(len(json_obj), 1) # ensure that there is one header in the json response assert_equal(json_obj[0]['hash'], bb_hash) # request/response hash should be the same @@ -270,9 +288,9 @@ class RESTTest (BitcoinTestFramework): # See if we can get 5 headers in one response self.generate(self.nodes[1], 5) - json_obj = self.test_rest_request(f"/headers/5/{bb_hash}") + json_obj = self.test_rest_request(f"/headers/{bb_hash}", query_params={"count": 5}) assert_equal(len(json_obj), 5) # now we should have 5 header objects - json_obj = self.test_rest_request(f"/blockfilterheaders/basic/5/{bb_hash}") + json_obj = self.test_rest_request(f"/blockfilterheaders/basic/{bb_hash}", query_params={"count": 5}) first_filter_header = json_obj[0] assert_equal(len(json_obj), 5) # now we should have 5 filter header objects json_obj = self.test_rest_request(f"/blockfilter/basic/{bb_hash}") @@ -282,20 +300,28 @@ class RESTTest (BitcoinTestFramework): assert_equal(first_filter_header, rpc_blockfilter['header']) assert_equal(json_obj['filter'], rpc_blockfilter['filter']) + # Test blockfilterheaders with an invalid hash and filtertype + resp = self.test_rest_request(f"/blockfilterheaders/{INVALID_PARAM}/{bb_hash}", ret_type=RetType.OBJ, status=400) + assert_equal(resp.read().decode('utf-8').rstrip(), f"Unknown filtertype {INVALID_PARAM}") + resp = self.test_rest_request(f"/blockfilterheaders/basic/{INVALID_PARAM}", ret_type=RetType.OBJ, status=400) + assert_equal(resp.read().decode('utf-8').rstrip(), f"Invalid hash: {INVALID_PARAM}") + # Test number parsing for num in ['5a', '-5', '0', '2001', '99999999999999999999999999999999999']: assert_equal( bytes(f'Header count is invalid or out of acceptable range (1-2000): {num}\r\n', 'ascii'), - self.test_rest_request(f"/headers/{num}/{bb_hash}", ret_type=RetType.BYTES, status=400), + self.test_rest_request(f"/headers/{bb_hash}", ret_type=RetType.BYTES, status=400, query_params={"count": num}), ) self.log.info("Test tx inclusion in the /mempool and /block URIs") - # Make 3 tx and mine them on node 1 + # Make 3 chained txs and mine them on node 1 txs = [] - txs.append(self.nodes[0].sendtoaddress(not_related_address, 11)) - txs.append(self.nodes[0].sendtoaddress(not_related_address, 11)) - txs.append(self.nodes[0].sendtoaddress(not_related_address, 11)) + input_txid = txid + for _ in range(3): + utxo_to_spend = self.wallet.get_utxo(txid=input_txid) + txs.append(self.wallet.send_self_transfer(from_node=self.nodes[0], utxo_to_spend=utxo_to_spend)['txid']) + input_txid = txs[-1] self.sync_all() # Check that there are exactly 3 transactions in the TX memory pool before generating the block @@ -304,8 +330,15 @@ class RESTTest (BitcoinTestFramework): # the size of the memory pool should be greater than 3x ~100 bytes assert_greater_than(json_obj['bytes'], 300) + mempool_info = self.nodes[0].getmempoolinfo() + assert_equal(json_obj, mempool_info) + # Check that there are our submitted transactions in the TX memory pool json_obj = self.test_rest_request("/mempool/contents") + raw_mempool_verbose = self.nodes[0].getrawmempool(verbose=True) + + assert_equal(json_obj, raw_mempool_verbose) + for i, tx in enumerate(txs): assert tx in json_obj assert_equal(json_obj[tx]['spentby'], txs[i + 1:i + 2]) @@ -341,5 +374,15 @@ class RESTTest (BitcoinTestFramework): json_obj = self.test_rest_request("/chaininfo") assert_equal(json_obj['bestblockhash'], bb_hash) + # Compare with normal RPC getblockchaininfo response + blockchain_info = self.nodes[0].getblockchaininfo() + assert_equal(blockchain_info, json_obj) + + # Test compatibility of deprecated and newer endpoints + self.log.info("Test compatibility of deprecated and newer endpoints") + assert_equal(self.test_rest_request(f"/headers/{bb_hash}", query_params={"count": 1}), self.test_rest_request(f"/headers/1/{bb_hash}")) + assert_equal(self.test_rest_request(f"/blockfilterheaders/basic/{bb_hash}", query_params={"count": 1}), self.test_rest_request(f"/blockfilterheaders/basic/5/{bb_hash}")) + + if __name__ == '__main__': RESTTest().main() diff --git a/test/functional/interface_usdt_coinselection.py b/test/functional/interface_usdt_coinselection.py new file mode 100755 index 0000000000..ef32feda99 --- /dev/null +++ b/test/functional/interface_usdt_coinselection.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" Tests the coin_selection:* tracepoint API interface. + See https://github.com/bitcoin/bitcoin/blob/master/doc/tracing.md#context-coin_selection +""" + +# Test will be skipped if we don't have bcc installed +try: + from bcc import BPF, USDT # type: ignore[import] +except ImportError: + pass +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + assert_greater_than, + assert_raises_rpc_error, +) + +coinselection_tracepoints_program = """ +#include <uapi/linux/ptrace.h> + +#define WALLET_NAME_LENGTH 16 +#define ALGO_NAME_LENGTH 16 + +struct event_data +{ + u8 type; + char wallet_name[WALLET_NAME_LENGTH]; + + // selected coins event + char algo[ALGO_NAME_LENGTH]; + s64 target; + s64 waste; + s64 selected_value; + + // create tx event + bool success; + s64 fee; + s32 change_pos; + + // aps create tx event + bool use_aps; +}; + +BPF_QUEUE(coin_selection_events, struct event_data, 1024); + +int trace_selected_coins(struct pt_regs *ctx) { + struct event_data data; + __builtin_memset(&data, 0, sizeof(data)); + data.type = 1; + bpf_usdt_readarg_p(1, ctx, &data.wallet_name, WALLET_NAME_LENGTH); + bpf_usdt_readarg_p(2, ctx, &data.algo, ALGO_NAME_LENGTH); + bpf_usdt_readarg(3, ctx, &data.target); + bpf_usdt_readarg(4, ctx, &data.waste); + bpf_usdt_readarg(5, ctx, &data.selected_value); + coin_selection_events.push(&data, 0); + return 0; +} + +int trace_normal_create_tx(struct pt_regs *ctx) { + struct event_data data; + __builtin_memset(&data, 0, sizeof(data)); + data.type = 2; + bpf_usdt_readarg_p(1, ctx, &data.wallet_name, WALLET_NAME_LENGTH); + bpf_usdt_readarg(2, ctx, &data.success); + bpf_usdt_readarg(3, ctx, &data.fee); + bpf_usdt_readarg(4, ctx, &data.change_pos); + coin_selection_events.push(&data, 0); + return 0; +} + +int trace_attempt_aps(struct pt_regs *ctx) { + struct event_data data; + __builtin_memset(&data, 0, sizeof(data)); + data.type = 3; + bpf_usdt_readarg_p(1, ctx, &data.wallet_name, WALLET_NAME_LENGTH); + coin_selection_events.push(&data, 0); + return 0; +} + +int trace_aps_create_tx(struct pt_regs *ctx) { + struct event_data data; + __builtin_memset(&data, 0, sizeof(data)); + data.type = 4; + bpf_usdt_readarg_p(1, ctx, &data.wallet_name, WALLET_NAME_LENGTH); + bpf_usdt_readarg(2, ctx, &data.use_aps); + bpf_usdt_readarg(3, ctx, &data.success); + bpf_usdt_readarg(4, ctx, &data.fee); + bpf_usdt_readarg(5, ctx, &data.change_pos); + coin_selection_events.push(&data, 0); + return 0; +} +""" + + +class CoinSelectionTracepointTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + self.setup_clean_chain = True + + def skip_test_if_missing_module(self): + self.skip_if_platform_not_linux() + self.skip_if_no_bitcoind_tracepoints() + self.skip_if_no_python_bcc() + self.skip_if_no_bpf_permissions() + self.skip_if_no_wallet() + + def get_tracepoints(self, expected_types): + events = [] + try: + for i in range(0, len(expected_types) + 1): + event = self.bpf["coin_selection_events"].pop() + assert_equal(event.wallet_name.decode(), self.default_wallet_name) + assert_equal(event.type, expected_types[i]) + events.append(event) + else: + # If the loop exits successfully instead of throwing a KeyError, then we have had + # more events than expected. There should be no more than len(expected_types) events. + assert False + except KeyError: + assert_equal(len(events), len(expected_types)) + return events + + + def determine_selection_from_usdt(self, events): + success = None + use_aps = None + algo = None + waste = None + change_pos = None + + is_aps = False + sc_events = [] + for event in events: + if event.type == 1: + if not is_aps: + algo = event.algo.decode() + waste = event.waste + sc_events.append(event) + elif event.type == 2: + success = event.success + if not is_aps: + change_pos = event.change_pos + elif event.type == 3: + is_aps = True + elif event.type == 4: + assert is_aps + if event.use_aps: + use_aps = True + assert_equal(len(sc_events), 2) + algo = sc_events[1].algo.decode() + waste = sc_events[1].waste + change_pos = event.change_pos + return success, use_aps, algo, waste, change_pos + + def run_test(self): + self.log.info("hook into the coin_selection tracepoints") + ctx = USDT(pid=self.nodes[0].process.pid) + ctx.enable_probe(probe="coin_selection:selected_coins", fn_name="trace_selected_coins") + ctx.enable_probe(probe="coin_selection:normal_create_tx_internal", fn_name="trace_normal_create_tx") + ctx.enable_probe(probe="coin_selection:attempting_aps_create_tx", fn_name="trace_attempt_aps") + ctx.enable_probe(probe="coin_selection:aps_create_tx_internal", fn_name="trace_aps_create_tx") + self.bpf = BPF(text=coinselection_tracepoints_program, usdt_contexts=[ctx], debug=0) + + self.log.info("Prepare wallets") + self.generate(self.nodes[0], 101) + wallet = self.nodes[0].get_wallet_rpc(self.default_wallet_name) + + self.log.info("Sending a transaction should result in all tracepoints") + # We should have 5 tracepoints in the order: + # 1. selected_coins (type 1) + # 2. normal_create_tx_internal (type 2) + # 3. attempting_aps_create_tx (type 3) + # 4. selected_coins (type 1) + # 5. aps_create_tx_internal (type 4) + wallet.sendtoaddress(wallet.getnewaddress(), 10) + events = self.get_tracepoints([1, 2, 3, 1, 4]) + success, use_aps, algo, waste, change_pos = self.determine_selection_from_usdt(events) + assert_equal(success, True) + assert_greater_than(change_pos, -1) + + self.log.info("Failing to fund results in 1 tracepoint") + # We should have 1 tracepoints in the order + # 1. normal_create_tx_internal (type 2) + assert_raises_rpc_error(-6, "Insufficient funds", wallet.sendtoaddress, wallet.getnewaddress(), 102 * 50) + events = self.get_tracepoints([2]) + success, use_aps, algo, waste, change_pos = self.determine_selection_from_usdt(events) + assert_equal(success, False) + + self.log.info("Explicitly enabling APS results in 2 tracepoints") + # We should have 2 tracepoints in the order + # 1. selected_coins (type 1) + # 2. normal_create_tx_internal (type 2) + wallet.setwalletflag("avoid_reuse") + wallet.sendtoaddress(address=wallet.getnewaddress(), amount=10, avoid_reuse=True) + events = self.get_tracepoints([1, 2]) + success, use_aps, algo, waste, change_pos = self.determine_selection_from_usdt(events) + assert_equal(success, True) + assert_equal(use_aps, None) + + self.bpf.cleanup() + + +if __name__ == '__main__': + CoinSelectionTracepointTest().main() diff --git a/test/functional/interface_usdt_net.py b/test/functional/interface_usdt_net.py new file mode 100755 index 0000000000..9522cd8c59 --- /dev/null +++ b/test/functional/interface_usdt_net.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" Tests the net:* tracepoint API interface. + See https://github.com/bitcoin/bitcoin/blob/master/doc/tracing.md#context-net +""" + +import ctypes +from io import BytesIO +# Test will be skipped if we don't have bcc installed +try: + from bcc import BPF, USDT # type: ignore[import] +except ImportError: + pass +from test_framework.messages import msg_version +from test_framework.p2p import P2PInterface +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal + +# Tor v3 addresses are 62 chars + 6 chars for the port (':12345'). +MAX_PEER_ADDR_LENGTH = 68 +MAX_PEER_CONN_TYPE_LENGTH = 20 +MAX_MSG_TYPE_LENGTH = 20 +# We won't process messages larger than 150 byte in this test. For reading +# larger messanges see contrib/tracing/log_raw_p2p_msgs.py +MAX_MSG_DATA_LENGTH = 150 + +net_tracepoints_program = """ +#include <uapi/linux/ptrace.h> + +#define MAX_PEER_ADDR_LENGTH {} +#define MAX_PEER_CONN_TYPE_LENGTH {} +#define MAX_MSG_TYPE_LENGTH {} +#define MAX_MSG_DATA_LENGTH {} +""".format( + MAX_PEER_ADDR_LENGTH, + MAX_PEER_CONN_TYPE_LENGTH, + MAX_MSG_TYPE_LENGTH, + MAX_MSG_DATA_LENGTH +) + """ +#define MIN(a,b) ({ __typeof__ (a) _a = (a); __typeof__ (b) _b = (b); _a < _b ? _a : _b; }) + +struct p2p_message +{ + u64 peer_id; + char peer_addr[MAX_PEER_ADDR_LENGTH]; + char peer_conn_type[MAX_PEER_CONN_TYPE_LENGTH]; + char msg_type[MAX_MSG_TYPE_LENGTH]; + u64 msg_size; + u8 msg[MAX_MSG_DATA_LENGTH]; +}; + +BPF_PERF_OUTPUT(inbound_messages); +int trace_inbound_message(struct pt_regs *ctx) { + struct p2p_message msg = {}; + bpf_usdt_readarg(1, ctx, &msg.peer_id); + bpf_usdt_readarg_p(2, ctx, &msg.peer_addr, MAX_PEER_ADDR_LENGTH); + bpf_usdt_readarg_p(3, ctx, &msg.peer_conn_type, MAX_PEER_CONN_TYPE_LENGTH); + bpf_usdt_readarg_p(4, ctx, &msg.msg_type, MAX_MSG_TYPE_LENGTH); + bpf_usdt_readarg(5, ctx, &msg.msg_size); + bpf_usdt_readarg_p(6, ctx, &msg.msg, MIN(msg.msg_size, MAX_MSG_DATA_LENGTH)); + inbound_messages.perf_submit(ctx, &msg, sizeof(msg)); + return 0; +} + +BPF_PERF_OUTPUT(outbound_messages); +int trace_outbound_message(struct pt_regs *ctx) { + struct p2p_message msg = {}; + bpf_usdt_readarg(1, ctx, &msg.peer_id); + bpf_usdt_readarg_p(2, ctx, &msg.peer_addr, MAX_PEER_ADDR_LENGTH); + bpf_usdt_readarg_p(3, ctx, &msg.peer_conn_type, MAX_PEER_CONN_TYPE_LENGTH); + bpf_usdt_readarg_p(4, ctx, &msg.msg_type, MAX_MSG_TYPE_LENGTH); + bpf_usdt_readarg(5, ctx, &msg.msg_size); + bpf_usdt_readarg_p(6, ctx, &msg.msg, MIN(msg.msg_size, MAX_MSG_DATA_LENGTH)); + outbound_messages.perf_submit(ctx, &msg, sizeof(msg)); + return 0; +}; +""" + + +class NetTracepointTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + + def skip_test_if_missing_module(self): + self.skip_if_platform_not_linux() + self.skip_if_no_bitcoind_tracepoints() + self.skip_if_no_python_bcc() + self.skip_if_no_bpf_permissions() + + def run_test(self): + # Tests the net:inbound_message and net:outbound_message tracepoints + # See https://github.com/bitcoin/bitcoin/blob/master/doc/tracing.md#context-net + + class P2PMessage(ctypes.Structure): + _fields_ = [ + ("peer_id", ctypes.c_uint64), + ("peer_addr", ctypes.c_char * MAX_PEER_ADDR_LENGTH), + ("peer_conn_type", ctypes.c_char * MAX_PEER_CONN_TYPE_LENGTH), + ("msg_type", ctypes.c_char * MAX_MSG_TYPE_LENGTH), + ("msg_size", ctypes.c_uint64), + ("msg", ctypes.c_ubyte * MAX_MSG_DATA_LENGTH), + ] + + def __repr__(self): + return f"P2PMessage(peer={self.peer_id}, addr={self.peer_addr.decode('utf-8')}, conn_type={self.peer_conn_type.decode('utf-8')}, msg_type={self.msg_type.decode('utf-8')}, msg_size={self.msg_size})" + + self.log.info( + "hook into the net:inbound_message and net:outbound_message tracepoints") + ctx = USDT(path=str(self.options.bitcoind)) + ctx.enable_probe(probe="net:inbound_message", + fn_name="trace_inbound_message") + ctx.enable_probe(probe="net:outbound_message", + fn_name="trace_outbound_message") + bpf = BPF(text=net_tracepoints_program, usdt_contexts=[ctx], debug=0) + + # The handle_* function is a ctypes callback function called from C. When + # we assert in the handle_* function, the AssertError doesn't propagate + # back to Python. The exception is ignored. We manually count and assert + # that the handle_* functions succeeded. + EXPECTED_INOUTBOUND_VERSION_MSG = 1 + checked_inbound_version_msg = 0 + checked_outbound_version_msg = 0 + + def check_p2p_message(event, inbound): + nonlocal checked_inbound_version_msg, checked_outbound_version_msg + if event.msg_type.decode("utf-8") == "version": + self.log.info( + f"check_p2p_message(): {'inbound' if inbound else 'outbound'} {event}") + peer = self.nodes[0].getpeerinfo()[0] + msg = msg_version() + msg.deserialize(BytesIO(bytes(event.msg[:event.msg_size]))) + assert_equal(peer["id"], event.peer_id, peer["id"]) + assert_equal(peer["addr"], event.peer_addr.decode("utf-8")) + assert_equal(peer["connection_type"], + event.peer_conn_type.decode("utf-8")) + if inbound: + checked_inbound_version_msg += 1 + else: + checked_outbound_version_msg += 1 + + def handle_inbound(_, data, __): + event = ctypes.cast(data, ctypes.POINTER(P2PMessage)).contents + check_p2p_message(event, True) + + def handle_outbound(_, data, __): + event = ctypes.cast(data, ctypes.POINTER(P2PMessage)).contents + check_p2p_message(event, False) + + bpf["inbound_messages"].open_perf_buffer(handle_inbound) + bpf["outbound_messages"].open_perf_buffer(handle_outbound) + + self.log.info("connect a P2P test node to our bitcoind node") + test_node = P2PInterface() + self.nodes[0].add_p2p_connection(test_node) + bpf.perf_buffer_poll(timeout=200) + + self.log.info( + "check that we got both an inbound and outbound version message") + assert_equal(EXPECTED_INOUTBOUND_VERSION_MSG, + checked_inbound_version_msg) + assert_equal(EXPECTED_INOUTBOUND_VERSION_MSG, + checked_outbound_version_msg) + + bpf.cleanup() + + +if __name__ == '__main__': + NetTracepointTest().main() diff --git a/test/functional/interface_usdt_utxocache.py b/test/functional/interface_usdt_utxocache.py new file mode 100755 index 0000000000..0c7f351e66 --- /dev/null +++ b/test/functional/interface_usdt_utxocache.py @@ -0,0 +1,407 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" Tests the utxocache:* tracepoint API interface. + See https://github.com/bitcoin/bitcoin/blob/master/doc/tracing.md#context-utxocache +""" + +import ctypes +# Test will be skipped if we don't have bcc installed +try: + from bcc import BPF, USDT # type: ignore[import] +except ImportError: + pass +from test_framework.messages import COIN +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal +from test_framework.wallet import MiniWallet + +utxocache_changes_program = """ +#include <uapi/linux/ptrace.h> + +typedef signed long long i64; + +struct utxocache_change +{ + char txid[32]; + u32 index; + u32 height; + i64 value; + bool is_coinbase; +}; + +BPF_PERF_OUTPUT(utxocache_add); +int trace_utxocache_add(struct pt_regs *ctx) { + struct utxocache_change add = {}; + bpf_usdt_readarg_p(1, ctx, &add.txid, 32); + bpf_usdt_readarg(2, ctx, &add.index); + bpf_usdt_readarg(3, ctx, &add.height); + bpf_usdt_readarg(4, ctx, &add.value); + bpf_usdt_readarg(5, ctx, &add.is_coinbase); + utxocache_add.perf_submit(ctx, &add, sizeof(add)); + return 0; +} + +BPF_PERF_OUTPUT(utxocache_spent); +int trace_utxocache_spent(struct pt_regs *ctx) { + struct utxocache_change spent = {}; + bpf_usdt_readarg_p(1, ctx, &spent.txid, 32); + bpf_usdt_readarg(2, ctx, &spent.index); + bpf_usdt_readarg(3, ctx, &spent.height); + bpf_usdt_readarg(4, ctx, &spent.value); + bpf_usdt_readarg(5, ctx, &spent.is_coinbase); + utxocache_spent.perf_submit(ctx, &spent, sizeof(spent)); + return 0; +} + +BPF_PERF_OUTPUT(utxocache_uncache); +int trace_utxocache_uncache(struct pt_regs *ctx) { + struct utxocache_change uncache = {}; + bpf_usdt_readarg_p(1, ctx, &uncache.txid, 32); + bpf_usdt_readarg(2, ctx, &uncache.index); + bpf_usdt_readarg(3, ctx, &uncache.height); + bpf_usdt_readarg(4, ctx, &uncache.value); + bpf_usdt_readarg(5, ctx, &uncache.is_coinbase); + utxocache_uncache.perf_submit(ctx, &uncache, sizeof(uncache)); + return 0; +} +""" + +utxocache_flushes_program = """ +#include <uapi/linux/ptrace.h> + +typedef signed long long i64; + +struct utxocache_flush +{ + i64 duration; + u32 mode; + u64 size; + u64 memory; + bool for_prune; +}; + +BPF_PERF_OUTPUT(utxocache_flush); +int trace_utxocache_flush(struct pt_regs *ctx) { + struct utxocache_flush flush = {}; + bpf_usdt_readarg(1, ctx, &flush.duration); + bpf_usdt_readarg(2, ctx, &flush.mode); + bpf_usdt_readarg(3, ctx, &flush.size); + bpf_usdt_readarg(4, ctx, &flush.memory); + bpf_usdt_readarg(5, ctx, &flush.for_prune); + utxocache_flush.perf_submit(ctx, &flush, sizeof(flush)); + return 0; +} +""" + +FLUSHMODE_NAME = { + 0: "NONE", + 1: "IF_NEEDED", + 2: "PERIODIC", + 3: "ALWAYS", +} + + +class UTXOCacheChange(ctypes.Structure): + _fields_ = [ + ("txid", ctypes.c_ubyte * 32), + ("index", ctypes.c_uint32), + ("height", ctypes.c_uint32), + ("value", ctypes.c_uint64), + ("is_coinbase", ctypes.c_bool), + ] + + def __repr__(self): + return f"UTXOCacheChange(outpoint={bytes(self.txid[::-1]).hex()}:{self.index}, height={self.height}, value={self.value}sat, is_coinbase={self.is_coinbase})" + + +class UTXOCacheFlush(ctypes.Structure): + _fields_ = [ + ("duration", ctypes.c_int64), + ("mode", ctypes.c_uint32), + ("size", ctypes.c_uint64), + ("memory", ctypes.c_uint64), + ("for_prune", ctypes.c_bool), + ] + + def __repr__(self): + return f"UTXOCacheFlush(duration={self.duration}, mode={FLUSHMODE_NAME[self.mode]}, size={self.size}, memory={self.memory}, for_prune={self.for_prune})" + + +class UTXOCacheTracepointTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = False + self.num_nodes = 1 + self.extra_args = [["-txindex"]] + + def skip_test_if_missing_module(self): + self.skip_if_platform_not_linux() + self.skip_if_no_bitcoind_tracepoints() + self.skip_if_no_python_bcc() + self.skip_if_no_bpf_permissions() + + def run_test(self): + self.wallet = MiniWallet(self.nodes[0]) + self.generate(self.wallet, 101) + + self.test_uncache() + self.test_add_spent() + self.test_flush() + + def test_uncache(self): + """ Tests the utxocache:uncache tracepoint API. + https://github.com/bitcoin/bitcoin/blob/master/doc/tracing.md#tracepoint-utxocacheuncache + """ + # To trigger an UTXO uncache from the cache, we create an invalid transaction + # spending a not-cached, but existing UTXO. During transaction validation, this + # the UTXO is added to the utxo cache, but as the transaction is invalid, it's + # uncached again. + self.log.info("testing the utxocache:uncache tracepoint API") + + # Retrieve the txid for the UTXO created in the first block. This UTXO is not + # in our UTXO cache. + EARLY_BLOCK_HEIGHT = 1 + block_1_hash = self.nodes[0].getblockhash(EARLY_BLOCK_HEIGHT) + block_1 = self.nodes[0].getblock(block_1_hash) + block_1_coinbase_txid = block_1["tx"][0] + + # Create a transaction and invalidate it by changing the txid of the previous + # output to the coinbase txid of the block at height 1. + invalid_tx = self.wallet.create_self_transfer( + from_node=self.nodes[0])["tx"] + invalid_tx.vin[0].prevout.hash = int(block_1_coinbase_txid, 16) + + self.log.info("hooking into the utxocache:uncache tracepoint") + ctx = USDT(path=str(self.options.bitcoind)) + ctx.enable_probe(probe="utxocache:uncache", + fn_name="trace_utxocache_uncache") + bpf = BPF(text=utxocache_changes_program, usdt_contexts=[ctx], debug=0) + + # The handle_* function is a ctypes callback function called from C. When + # we assert in the handle_* function, the AssertError doesn't propagate + # back to Python. The exception is ignored. We manually count and assert + # that the handle_* functions succeeded. + EXPECTED_HANDLE_UNCACHE_SUCCESS = 1 + handle_uncache_succeeds = 0 + + def handle_utxocache_uncache(_, data, __): + nonlocal handle_uncache_succeeds + event = ctypes.cast(data, ctypes.POINTER(UTXOCacheChange)).contents + self.log.info(f"handle_utxocache_uncache(): {event}") + assert_equal(block_1_coinbase_txid, bytes(event.txid[::-1]).hex()) + assert_equal(0, event.index) # prevout index + assert_equal(EARLY_BLOCK_HEIGHT, event.height) + assert_equal(50 * COIN, event.value) + assert_equal(True, event.is_coinbase) + + handle_uncache_succeeds += 1 + + bpf["utxocache_uncache"].open_perf_buffer(handle_utxocache_uncache) + + self.log.info( + "testmempoolaccept the invalid transaction to trigger an UTXO-cache uncache") + result = self.nodes[0].testmempoolaccept( + [invalid_tx.serialize().hex()])[0] + assert_equal(result["allowed"], False) + + bpf.perf_buffer_poll(timeout=100) + bpf.cleanup() + + self.log.info( + f"check that we successfully traced {EXPECTED_HANDLE_UNCACHE_SUCCESS} uncaches") + assert_equal(EXPECTED_HANDLE_UNCACHE_SUCCESS, handle_uncache_succeeds) + + def test_add_spent(self): + """ Tests the utxocache:add utxocache:spent tracepoint API + See https://github.com/bitcoin/bitcoin/blob/master/doc/tracing.md#tracepoint-utxocacheadd + and https://github.com/bitcoin/bitcoin/blob/master/doc/tracing.md#tracepoint-utxocachespent + """ + + self.log.info( + "test the utxocache:add and utxocache:spent tracepoint API") + + self.log.info("create an unconfirmed transaction") + self.wallet.send_self_transfer(from_node=self.nodes[0]) + + # We mine a block to trace changes (add/spent) to the active in-memory cache + # of the UTXO set (see CoinsTip() of CCoinsViewCache). However, in some cases + # temporary clones of the active cache are made. For example, during mining with + # the generate RPC call, the block is first tested in TestBlockValidity(). There, + # a clone of the active cache is modified during a test ConnectBlock() call. + # These are implementation details we don't want to test here. Thus, after + # mining, we invalidate the block, start the tracing, and then trace the cache + # changes to the active utxo cache. + self.log.info("mine and invalidate a block that is later reconsidered") + block_hash = self.generate(self.wallet, 1)[0] + self.nodes[0].invalidateblock(block_hash) + + self.log.info( + "hook into the utxocache:add and utxocache:spent tracepoints") + ctx = USDT(path=str(self.options.bitcoind)) + ctx.enable_probe(probe="utxocache:add", fn_name="trace_utxocache_add") + ctx.enable_probe(probe="utxocache:spent", + fn_name="trace_utxocache_spent") + bpf = BPF(text=utxocache_changes_program, usdt_contexts=[ctx], debug=0) + + # The handle_* function is a ctypes callback function called from C. When + # we assert in the handle_* function, the AssertError doesn't propagate + # back to Python. The exception is ignored. We manually count and assert + # that the handle_* functions succeeded. + EXPECTED_HANDLE_ADD_SUCCESS = 2 + EXPECTED_HANDLE_SPENT_SUCCESS = 1 + handle_add_succeeds = 0 + handle_spent_succeeds = 0 + + expected_utxocache_spents = [] + expected_utxocache_adds = [] + + def handle_utxocache_add(_, data, __): + nonlocal handle_add_succeeds + event = ctypes.cast(data, ctypes.POINTER(UTXOCacheChange)).contents + self.log.info(f"handle_utxocache_add(): {event}") + add = expected_utxocache_adds.pop(0) + assert_equal(add["txid"], bytes(event.txid[::-1]).hex()) + assert_equal(add["index"], event.index) + assert_equal(add["height"], event.height) + assert_equal(add["value"], event.value) + assert_equal(add["is_coinbase"], event.is_coinbase) + handle_add_succeeds += 1 + + def handle_utxocache_spent(_, data, __): + nonlocal handle_spent_succeeds + event = ctypes.cast(data, ctypes.POINTER(UTXOCacheChange)).contents + self.log.info(f"handle_utxocache_spent(): {event}") + spent = expected_utxocache_spents.pop(0) + assert_equal(spent["txid"], bytes(event.txid[::-1]).hex()) + assert_equal(spent["index"], event.index) + assert_equal(spent["height"], event.height) + assert_equal(spent["value"], event.value) + assert_equal(spent["is_coinbase"], event.is_coinbase) + handle_spent_succeeds += 1 + + bpf["utxocache_add"].open_perf_buffer(handle_utxocache_add) + bpf["utxocache_spent"].open_perf_buffer(handle_utxocache_spent) + + # We trigger a block re-connection. This causes changes (add/spent) + # to the UTXO-cache which in turn triggers the tracepoints. + self.log.info("reconsider the previously invalidated block") + self.nodes[0].reconsiderblock(block_hash) + + block = self.nodes[0].getblock(block_hash, 2) + for (block_index, tx) in enumerate(block["tx"]): + for vin in tx["vin"]: + if "coinbase" not in vin: + prevout_tx = self.nodes[0].getrawtransaction( + vin["txid"], True) + prevout_tx_block = self.nodes[0].getblockheader( + prevout_tx["blockhash"]) + spends_coinbase = "coinbase" in prevout_tx["vin"][0] + expected_utxocache_spents.append({ + "txid": vin["txid"], + "index": vin["vout"], + "height": prevout_tx_block["height"], + "value": int(prevout_tx["vout"][vin["vout"]]["value"] * COIN), + "is_coinbase": spends_coinbase, + }) + for (i, vout) in enumerate(tx["vout"]): + if vout["scriptPubKey"]["type"] != "nulldata": + expected_utxocache_adds.append({ + "txid": tx["txid"], + "index": i, + "height": block["height"], + "value": int(vout["value"] * COIN), + "is_coinbase": block_index == 0, + }) + + assert_equal(EXPECTED_HANDLE_ADD_SUCCESS, len(expected_utxocache_adds)) + assert_equal(EXPECTED_HANDLE_SPENT_SUCCESS, + len(expected_utxocache_spents)) + + bpf.perf_buffer_poll(timeout=200) + bpf.cleanup() + + self.log.info( + f"check that we successfully traced {EXPECTED_HANDLE_ADD_SUCCESS} adds and {EXPECTED_HANDLE_SPENT_SUCCESS} spent") + assert_equal(0, len(expected_utxocache_adds)) + assert_equal(0, len(expected_utxocache_spents)) + assert_equal(EXPECTED_HANDLE_ADD_SUCCESS, handle_add_succeeds) + assert_equal(EXPECTED_HANDLE_SPENT_SUCCESS, handle_spent_succeeds) + + def test_flush(self): + """ Tests the utxocache:flush tracepoint API. + See https://github.com/bitcoin/bitcoin/blob/master/doc/tracing.md#tracepoint-utxocacheflush""" + + self.log.info("test the utxocache:flush tracepoint API") + self.log.info("hook into the utxocache:flush tracepoint") + ctx = USDT(path=str(self.options.bitcoind)) + ctx.enable_probe(probe="utxocache:flush", + fn_name="trace_utxocache_flush") + bpf = BPF(text=utxocache_flushes_program, usdt_contexts=[ctx], debug=0) + + # The handle_* function is a ctypes callback function called from C. When + # we assert in the handle_* function, the AssertError doesn't propagate + # back to Python. The exception is ignored. We manually count and assert + # that the handle_* functions succeeded. + EXPECTED_HANDLE_FLUSH_SUCCESS = 3 + handle_flush_succeeds = 0 + possible_cache_sizes = set() + expected_flushes = [] + + def handle_utxocache_flush(_, data, __): + nonlocal handle_flush_succeeds + event = ctypes.cast(data, ctypes.POINTER(UTXOCacheFlush)).contents + self.log.info(f"handle_utxocache_flush(): {event}") + expected = expected_flushes.pop(0) + assert_equal(expected["mode"], FLUSHMODE_NAME[event.mode]) + possible_cache_sizes.remove(event.size) # fails if size not in set + # sanity checks only + assert(event.memory > 0) + assert(event.duration > 0) + handle_flush_succeeds += 1 + + bpf["utxocache_flush"].open_perf_buffer(handle_utxocache_flush) + + self.log.info("stop the node to flush the UTXO cache") + UTXOS_IN_CACHE = 104 # might need to be changed if the eariler tests are modified + # A node shutdown causes two flushes. One that flushes UTXOS_IN_CACHE + # UTXOs and one that flushes 0 UTXOs. Normally the 0-UTXO-flush is the + # second flush, however it can happen that the order changes. + possible_cache_sizes = {UTXOS_IN_CACHE, 0} + flush_for_shutdown = {"mode": "ALWAYS", "for_prune": False} + expected_flushes.extend([flush_for_shutdown, flush_for_shutdown]) + self.stop_node(0) + + bpf.perf_buffer_poll(timeout=200) + + self.log.info("check that we don't expect additional flushes") + assert_equal(0, len(expected_flushes)) + assert_equal(0, len(possible_cache_sizes)) + + self.log.info("restart the node with -prune") + self.start_node(0, ["-fastprune=1", "-prune=1"]) + + BLOCKS_TO_MINE = 350 + self.log.info(f"mine {BLOCKS_TO_MINE} blocks to be able to prune") + self.generate(self.wallet, BLOCKS_TO_MINE) + # we added BLOCKS_TO_MINE coinbase UTXOs to the cache + possible_cache_sizes = {BLOCKS_TO_MINE} + expected_flushes.append( + {"mode": "NONE", "for_prune": True, "size_fn": lambda x: x == BLOCKS_TO_MINE}) + + self.log.info(f"prune blockchain to trigger a flush for pruning") + self.nodes[0].pruneblockchain(315) + + bpf.perf_buffer_poll(timeout=500) + bpf.cleanup() + + self.log.info( + f"check that we don't expect additional flushes and that the handle_* function succeeded") + assert_equal(0, len(expected_flushes)) + assert_equal(0, len(possible_cache_sizes)) + assert_equal(EXPECTED_HANDLE_FLUSH_SUCCESS, handle_flush_succeeds) + + +if __name__ == '__main__': + UTXOCacheTracepointTest().main() diff --git a/test/functional/interface_usdt_validation.py b/test/functional/interface_usdt_validation.py new file mode 100755 index 0000000000..d11809273b --- /dev/null +++ b/test/functional/interface_usdt_validation.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" Tests the validation:* tracepoint API interface. + See https://github.com/bitcoin/bitcoin/blob/master/doc/tracing.md#context-validation +""" + +import ctypes + +# Test will be skipped if we don't have bcc installed +try: + from bcc import BPF, USDT # type: ignore[import] +except ImportError: + pass + +from test_framework.address import ADDRESS_BCRT1_UNSPENDABLE +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal + + +validation_blockconnected_program = """ +#include <uapi/linux/ptrace.h> + +typedef signed long long i64; + +struct connected_block +{ + char hash[32]; + int height; + i64 transactions; + int inputs; + i64 sigops; + u64 duration; +}; + +BPF_PERF_OUTPUT(block_connected); +int trace_block_connected(struct pt_regs *ctx) { + struct connected_block block = {}; + bpf_usdt_readarg_p(1, ctx, &block.hash, 32); + bpf_usdt_readarg(2, ctx, &block.height); + bpf_usdt_readarg(3, ctx, &block.transactions); + bpf_usdt_readarg(4, ctx, &block.inputs); + bpf_usdt_readarg(5, ctx, &block.sigops); + bpf_usdt_readarg(6, ctx, &block.duration); + block_connected.perf_submit(ctx, &block, sizeof(block)); + return 0; +} +""" + + +class ValidationTracepointTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + + def skip_test_if_missing_module(self): + self.skip_if_platform_not_linux() + self.skip_if_no_bitcoind_tracepoints() + self.skip_if_no_python_bcc() + self.skip_if_no_bpf_permissions() + + def run_test(self): + # Tests the validation:block_connected tracepoint by generating blocks + # and comparing the values passed in the tracepoint arguments with the + # blocks. + # See https://github.com/bitcoin/bitcoin/blob/master/doc/tracing.md#tracepoint-validationblock_connected + + class Block(ctypes.Structure): + _fields_ = [ + ("hash", ctypes.c_ubyte * 32), + ("height", ctypes.c_int), + ("transactions", ctypes.c_int64), + ("inputs", ctypes.c_int), + ("sigops", ctypes.c_int64), + ("duration", ctypes.c_uint64), + ] + + def __repr__(self): + return "ConnectedBlock(hash=%s height=%d, transactions=%d, inputs=%d, sigops=%d, duration=%d)" % ( + bytes(self.hash[::-1]).hex(), + self.height, + self.transactions, + self.inputs, + self.sigops, + self.duration) + + # The handle_* function is a ctypes callback function called from C. When + # we assert in the handle_* function, the AssertError doesn't propagate + # back to Python. The exception is ignored. We manually count and assert + # that the handle_* functions succeeded. + BLOCKS_EXPECTED = 2 + blocks_checked = 0 + expected_blocks = list() + + self.log.info("hook into the validation:block_connected tracepoint") + ctx = USDT(path=str(self.options.bitcoind)) + ctx.enable_probe(probe="validation:block_connected", + fn_name="trace_block_connected") + bpf = BPF(text=validation_blockconnected_program, + usdt_contexts=[ctx], debug=0) + + def handle_blockconnected(_, data, __): + nonlocal expected_blocks, blocks_checked + event = ctypes.cast(data, ctypes.POINTER(Block)).contents + self.log.info(f"handle_blockconnected(): {event}") + block = expected_blocks.pop(0) + assert_equal(block["hash"], bytes(event.hash[::-1]).hex()) + assert_equal(block["height"], event.height) + assert_equal(len(block["tx"]), event.transactions) + assert_equal(len([tx["vin"] for tx in block["tx"]]), event.inputs) + assert_equal(0, event.sigops) # no sigops in coinbase tx + # only plausibility checks + assert(event.duration > 0) + + blocks_checked += 1 + + bpf["block_connected"].open_perf_buffer( + handle_blockconnected) + + self.log.info(f"mine {BLOCKS_EXPECTED} blocks") + block_hashes = self.generatetoaddress( + self.nodes[0], BLOCKS_EXPECTED, ADDRESS_BCRT1_UNSPENDABLE) + for block_hash in block_hashes: + expected_blocks.append(self.nodes[0].getblock(block_hash, 2)) + + bpf.perf_buffer_poll(timeout=200) + bpf.cleanup() + + self.log.info(f"check that we traced {BLOCKS_EXPECTED} blocks") + assert_equal(BLOCKS_EXPECTED, blocks_checked) + assert_equal(0, len(expected_blocks)) + + +if __name__ == '__main__': + ValidationTracepointTest().main() diff --git a/test/functional/interface_zmq.py b/test/functional/interface_zmq.py index 1ee12c0040..7d8d10589b 100755 --- a/test/functional/interface_zmq.py +++ b/test/functional/interface_zmq.py @@ -23,6 +23,9 @@ from test_framework.util import ( assert_equal, assert_raises_rpc_error, ) +from test_framework.wallet import ( + MiniWallet, +) from test_framework.netutil import test_ipv6_local from io import BytesIO from time import sleep @@ -100,8 +103,6 @@ class ZMQTestSetupBlock: class ZMQTest (BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 - if self.is_wallet_compiled(): - self.requires_wallet = True # This test isn't testing txn relay/timing, so set whitelist on the # peers for instant txn relay. This speeds up the test run time 2-3x. self.extra_args = [["-whitelist=noban@127.0.0.1"]] * self.num_nodes @@ -111,6 +112,7 @@ class ZMQTest (BitcoinTestFramework): self.skip_if_no_bitcoind_zmq() def run_test(self): + self.wallet = MiniWallet(self.nodes[0]) self.ctx = zmq.Context() try: self.test_basic() @@ -211,25 +213,25 @@ class ZMQTest (BitcoinTestFramework): assert_equal([txid.hex()], self.nodes[1].getblock(hash)["tx"]) - if self.is_wallet_compiled(): - self.log.info("Wait for tx from second node") - payment_txid = self.nodes[1].sendtoaddress(self.nodes[0].getnewaddress(), 1.0) - self.sync_all() - - # Should receive the broadcasted txid. - txid = hashtx.receive() - assert_equal(payment_txid, txid.hex()) + self.wallet.rescan_utxos() + self.log.info("Wait for tx from second node") + payment_tx = self.wallet.send_self_transfer(from_node=self.nodes[1]) + payment_txid = payment_tx['txid'] + self.sync_all() + # Should receive the broadcasted txid. + txid = hashtx.receive() + assert_equal(payment_txid, txid.hex()) - # Should receive the broadcasted raw transaction. - hex = rawtx.receive() - assert_equal(payment_txid, hash256_reversed(hex).hex()) + # Should receive the broadcasted raw transaction. + hex = rawtx.receive() + assert_equal(payment_tx['wtxid'], hash256_reversed(hex).hex()) - # Mining the block with this tx should result in second notification - # after coinbase tx notification - self.generatetoaddress(self.nodes[0], 1, ADDRESS_BCRT1_UNSPENDABLE) - hashtx.receive() - txid = hashtx.receive() - assert_equal(payment_txid, txid.hex()) + # Mining the block with this tx should result in second notification + # after coinbase tx notification + self.generatetoaddress(self.nodes[0], 1, ADDRESS_BCRT1_UNSPENDABLE) + hashtx.receive() + txid = hashtx.receive() + assert_equal(payment_txid, txid.hex()) self.log.info("Test the getzmqnotifications RPC") @@ -243,9 +245,6 @@ class ZMQTest (BitcoinTestFramework): assert_equal(self.nodes[1].getzmqnotifications(), []) def test_reorg(self): - if not self.is_wallet_compiled(): - self.log.info("Skipping reorg test because wallet is disabled") - return address = 'tcp://127.0.0.1:28333' @@ -256,7 +255,7 @@ class ZMQTest (BitcoinTestFramework): self.disconnect_nodes(0, 1) # Generate 1 block in nodes[0] with 1 mempool tx and receive all notifications - payment_txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 1.0) + payment_txid = self.wallet.send_self_transfer(from_node=self.nodes[0])['txid'] disconnect_block = self.generatetoaddress(self.nodes[0], 1, ADDRESS_BCRT1_UNSPENDABLE, sync_fun=self.no_op)[0] disconnect_cb = self.nodes[0].getblock(disconnect_block)["tx"][0] assert_equal(self.nodes[0].getbestblockhash(), hashblock.receive().hex()) @@ -325,126 +324,124 @@ class ZMQTest (BitcoinTestFramework): assert_equal((self.nodes[1].getblockhash(block_count-1), "C", None), seq.receive_sequence()) assert_equal((self.nodes[1].getblockhash(block_count), "C", None), seq.receive_sequence()) - # Rest of test requires wallet functionality - if self.is_wallet_compiled(): - self.log.info("Wait for tx from second node") - payment_txid = self.nodes[1].sendtoaddress(address=self.nodes[0].getnewaddress(), amount=5.0, replaceable=True) - self.sync_all() - self.log.info("Testing sequence notifications with mempool sequence values") - - # Should receive the broadcasted txid. - assert_equal((payment_txid, "A", seq_num), seq.receive_sequence()) - seq_num += 1 - - self.log.info("Testing RBF notification") - # Replace it to test eviction/addition notification - rbf_info = self.nodes[1].bumpfee(payment_txid) - self.sync_all() - assert_equal((payment_txid, "R", seq_num), seq.receive_sequence()) - seq_num += 1 - assert_equal((rbf_info["txid"], "A", seq_num), seq.receive_sequence()) - seq_num += 1 - - # Doesn't get published when mined, make a block and tx to "flush" the possibility - # though the mempool sequence number does go up by the number of transactions - # removed from the mempool by the block mining it. - mempool_size = len(self.nodes[0].getrawmempool()) - c_block = self.generatetoaddress(self.nodes[0], 1, ADDRESS_BCRT1_UNSPENDABLE)[0] - # Make sure the number of mined transactions matches the number of txs out of mempool - mempool_size_delta = mempool_size - len(self.nodes[0].getrawmempool()) - assert_equal(len(self.nodes[0].getblock(c_block)["tx"])-1, mempool_size_delta) - seq_num += mempool_size_delta - payment_txid_2 = self.nodes[1].sendtoaddress(self.nodes[0].getnewaddress(), 1.0) - self.sync_all() - assert_equal((c_block, "C", None), seq.receive_sequence()) - assert_equal((payment_txid_2, "A", seq_num), seq.receive_sequence()) - seq_num += 1 - - # Spot check getrawmempool results that they only show up when asked for - assert type(self.nodes[0].getrawmempool()) is list - assert type(self.nodes[0].getrawmempool(mempool_sequence=False)) is list - assert "mempool_sequence" not in self.nodes[0].getrawmempool(verbose=True) - assert_raises_rpc_error(-8, "Verbose results cannot contain mempool sequence values.", self.nodes[0].getrawmempool, True, True) - assert_equal(self.nodes[0].getrawmempool(mempool_sequence=True)["mempool_sequence"], seq_num) - - self.log.info("Testing reorg notifications") - # Manually invalidate the last block to test mempool re-entry - # N.B. This part could be made more lenient in exact ordering - # since it greatly depends on inner-workings of blocks/mempool - # during "deep" re-orgs. Probably should "re-construct" - # blockchain/mempool state from notifications instead. - block_count = self.nodes[0].getblockcount() - best_hash = self.nodes[0].getbestblockhash() - self.nodes[0].invalidateblock(best_hash) - sleep(2) # Bit of room to make sure transaction things happened - - # Make sure getrawmempool mempool_sequence results aren't "queued" but immediately reflective - # of the time they were gathered. - assert self.nodes[0].getrawmempool(mempool_sequence=True)["mempool_sequence"] > seq_num - - assert_equal((best_hash, "D", None), seq.receive_sequence()) - assert_equal((rbf_info["txid"], "A", seq_num), seq.receive_sequence()) - seq_num += 1 - - # Other things may happen but aren't wallet-deterministic so we don't test for them currently - self.nodes[0].reconsiderblock(best_hash) - self.generatetoaddress(self.nodes[1], 1, ADDRESS_BCRT1_UNSPENDABLE) - - self.log.info("Evict mempool transaction by block conflict") - orig_txid = self.nodes[0].sendtoaddress(address=self.nodes[0].getnewaddress(), amount=1.0, replaceable=True) - - # More to be simply mined - more_tx = [] - for _ in range(5): - more_tx.append(self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 0.1)) - - raw_tx = self.nodes[0].getrawtransaction(orig_txid) - bump_info = self.nodes[0].bumpfee(orig_txid) - # Mine the pre-bump tx - txs_to_add = [raw_tx] + [self.nodes[0].getrawtransaction(txid) for txid in more_tx] - block = create_block(int(self.nodes[0].getbestblockhash(), 16), create_coinbase(self.nodes[0].getblockcount()+1), txlist=txs_to_add) - add_witness_commitment(block) - block.solve() - assert_equal(self.nodes[0].submitblock(block.serialize().hex()), None) - tip = self.nodes[0].getbestblockhash() - assert_equal(int(tip, 16), block.sha256) - orig_txid_2 = self.nodes[0].sendtoaddress(address=self.nodes[0].getnewaddress(), amount=1.0, replaceable=True) - - # Flush old notifications until evicted tx original entry + self.log.info("Wait for tx from second node") + payment_tx = self.wallet.send_self_transfer(from_node=self.nodes[1]) + payment_txid = payment_tx['txid'] + self.sync_all() + self.log.info("Testing sequence notifications with mempool sequence values") + + # Should receive the broadcasted txid. + assert_equal((payment_txid, "A", seq_num), seq.receive_sequence()) + seq_num += 1 + + self.log.info("Testing RBF notification") + # Replace it to test eviction/addition notification + payment_tx['tx'].vout[0].nValue -= 1000 + rbf_txid = self.nodes[1].sendrawtransaction(payment_tx['tx'].serialize().hex()) + self.sync_all() + assert_equal((payment_txid, "R", seq_num), seq.receive_sequence()) + seq_num += 1 + assert_equal((rbf_txid, "A", seq_num), seq.receive_sequence()) + seq_num += 1 + + # Doesn't get published when mined, make a block and tx to "flush" the possibility + # though the mempool sequence number does go up by the number of transactions + # removed from the mempool by the block mining it. + mempool_size = len(self.nodes[0].getrawmempool()) + c_block = self.generatetoaddress(self.nodes[0], 1, ADDRESS_BCRT1_UNSPENDABLE)[0] + # Make sure the number of mined transactions matches the number of txs out of mempool + mempool_size_delta = mempool_size - len(self.nodes[0].getrawmempool()) + assert_equal(len(self.nodes[0].getblock(c_block)["tx"])-1, mempool_size_delta) + seq_num += mempool_size_delta + payment_txid_2 = self.wallet.send_self_transfer(from_node=self.nodes[1])['txid'] + self.sync_all() + assert_equal((c_block, "C", None), seq.receive_sequence()) + assert_equal((payment_txid_2, "A", seq_num), seq.receive_sequence()) + seq_num += 1 + + # Spot check getrawmempool results that they only show up when asked for + assert type(self.nodes[0].getrawmempool()) is list + assert type(self.nodes[0].getrawmempool(mempool_sequence=False)) is list + assert "mempool_sequence" not in self.nodes[0].getrawmempool(verbose=True) + assert_raises_rpc_error(-8, "Verbose results cannot contain mempool sequence values.", self.nodes[0].getrawmempool, True, True) + assert_equal(self.nodes[0].getrawmempool(mempool_sequence=True)["mempool_sequence"], seq_num) + + self.log.info("Testing reorg notifications") + # Manually invalidate the last block to test mempool re-entry + # N.B. This part could be made more lenient in exact ordering + # since it greatly depends on inner-workings of blocks/mempool + # during "deep" re-orgs. Probably should "re-construct" + # blockchain/mempool state from notifications instead. + block_count = self.nodes[0].getblockcount() + best_hash = self.nodes[0].getbestblockhash() + self.nodes[0].invalidateblock(best_hash) + sleep(2) # Bit of room to make sure transaction things happened + + # Make sure getrawmempool mempool_sequence results aren't "queued" but immediately reflective + # of the time they were gathered. + assert self.nodes[0].getrawmempool(mempool_sequence=True)["mempool_sequence"] > seq_num + + assert_equal((best_hash, "D", None), seq.receive_sequence()) + assert_equal((rbf_txid, "A", seq_num), seq.receive_sequence()) + seq_num += 1 + + # Other things may happen but aren't wallet-deterministic so we don't test for them currently + self.nodes[0].reconsiderblock(best_hash) + self.generatetoaddress(self.nodes[1], 1, ADDRESS_BCRT1_UNSPENDABLE) + + self.log.info("Evict mempool transaction by block conflict") + orig_tx = self.wallet.send_self_transfer(from_node=self.nodes[0]) + orig_txid = orig_tx['txid'] + + # More to be simply mined + more_tx = [] + for _ in range(5): + more_tx.append(self.wallet.send_self_transfer(from_node=self.nodes[0])) + + orig_tx['tx'].vout[0].nValue -= 1000 + bump_txid = self.nodes[0].sendrawtransaction(orig_tx['tx'].serialize().hex()) + # Mine the pre-bump tx + txs_to_add = [orig_tx['hex']] + [tx['hex'] for tx in more_tx] + block = create_block(int(self.nodes[0].getbestblockhash(), 16), create_coinbase(self.nodes[0].getblockcount()+1), txlist=txs_to_add) + add_witness_commitment(block) + block.solve() + assert_equal(self.nodes[0].submitblock(block.serialize().hex()), None) + tip = self.nodes[0].getbestblockhash() + assert_equal(int(tip, 16), block.sha256) + orig_txid_2 = self.wallet.send_self_transfer(from_node=self.nodes[0])['txid'] + + # Flush old notifications until evicted tx original entry + (hash_str, label, mempool_seq) = seq.receive_sequence() + while hash_str != orig_txid: (hash_str, label, mempool_seq) = seq.receive_sequence() - while hash_str != orig_txid: - (hash_str, label, mempool_seq) = seq.receive_sequence() - mempool_seq += 1 + mempool_seq += 1 - # Added original tx - assert_equal(label, "A") - # More transactions to be simply mined - for i in range(len(more_tx)): - assert_equal((more_tx[i], "A", mempool_seq), seq.receive_sequence()) - mempool_seq += 1 - # Bumped by rbf - assert_equal((orig_txid, "R", mempool_seq), seq.receive_sequence()) - mempool_seq += 1 - assert_equal((bump_info["txid"], "A", mempool_seq), seq.receive_sequence()) + # Added original tx + assert_equal(label, "A") + # More transactions to be simply mined + for i in range(len(more_tx)): + assert_equal((more_tx[i]['txid'], "A", mempool_seq), seq.receive_sequence()) mempool_seq += 1 - # Conflict announced first, then block - assert_equal((bump_info["txid"], "R", mempool_seq), seq.receive_sequence()) - mempool_seq += 1 - assert_equal((tip, "C", None), seq.receive_sequence()) - mempool_seq += len(more_tx) - # Last tx - assert_equal((orig_txid_2, "A", mempool_seq), seq.receive_sequence()) - mempool_seq += 1 - self.generatetoaddress(self.nodes[0], 1, ADDRESS_BCRT1_UNSPENDABLE) - self.sync_all() # want to make sure we didn't break "consensus" for other tests + # Bumped by rbf + assert_equal((orig_txid, "R", mempool_seq), seq.receive_sequence()) + mempool_seq += 1 + assert_equal((bump_txid, "A", mempool_seq), seq.receive_sequence()) + mempool_seq += 1 + # Conflict announced first, then block + assert_equal((bump_txid, "R", mempool_seq), seq.receive_sequence()) + mempool_seq += 1 + assert_equal((tip, "C", None), seq.receive_sequence()) + mempool_seq += len(more_tx) + # Last tx + assert_equal((orig_txid_2, "A", mempool_seq), seq.receive_sequence()) + mempool_seq += 1 + self.generatetoaddress(self.nodes[0], 1, ADDRESS_BCRT1_UNSPENDABLE) + self.sync_all() # want to make sure we didn't break "consensus" for other tests def test_mempool_sync(self): """ Use sequence notification plus getrawmempool sequence results to "sync mempool" """ - if not self.is_wallet_compiled(): - self.log.info("Skipping mempool sync test") - return self.log.info("Testing 'mempool sync' usage of sequence notifier") [seq] = self.setup_zmq_test([("sequence", "tcp://127.0.0.1:28333")]) @@ -455,10 +452,10 @@ class ZMQTest (BitcoinTestFramework): # Some transactions have been happening but we aren't consuming zmq notifications yet # or we lost a ZMQ message somehow and want to start over - txids = [] + txs = [] num_txs = 5 for _ in range(num_txs): - txids.append(self.nodes[1].sendtoaddress(address=self.nodes[0].getnewaddress(), amount=1.0, replaceable=True)) + txs.append(self.wallet.send_self_transfer(from_node=self.nodes[1])) self.sync_all() # 1) Consume backlog until we get a mempool sequence number @@ -484,11 +481,12 @@ class ZMQTest (BitcoinTestFramework): # Things continue to happen in the "interim" while waiting for snapshot results # We have node 0 do all these to avoid p2p races with RBF announcements for _ in range(num_txs): - txids.append(self.nodes[0].sendtoaddress(address=self.nodes[0].getnewaddress(), amount=0.1, replaceable=True)) - self.nodes[0].bumpfee(txids[-1]) + txs.append(self.wallet.send_self_transfer(from_node=self.nodes[0])) + txs[-1]['tx'].vout[0].nValue -= 1000 + self.nodes[0].sendrawtransaction(txs[-1]['tx'].serialize().hex()) self.sync_all() self.generatetoaddress(self.nodes[0], 1, ADDRESS_BCRT1_UNSPENDABLE) - final_txid = self.nodes[0].sendtoaddress(address=self.nodes[0].getnewaddress(), amount=0.1, replaceable=True) + final_txid = self.wallet.send_self_transfer(from_node=self.nodes[0])['txid'] # 3) Consume ZMQ backlog until we get to "now" for the mempool snapshot while True: diff --git a/test/functional/mempool_package_onemore.py b/test/functional/mempool_package_onemore.py index a6fb1dcf35..423a5bf2ee 100755 --- a/test/functional/mempool_package_onemore.py +++ b/test/functional/mempool_package_onemore.py @@ -7,74 +7,68 @@ size. """ -from decimal import Decimal - -from test_framework.blocktools import COINBASE_MATURITY from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, assert_raises_rpc_error, - chain_transaction, ) +from test_framework.wallet import MiniWallet + MAX_ANCESTORS = 25 MAX_DESCENDANTS = 25 + class MempoolPackagesTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 self.extra_args = [["-maxorphantx=1000"]] - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() + def chain_tx(self, utxos_to_spend, *, num_outputs=1): + return self.wallet.send_self_transfer_multi( + from_node=self.nodes[0], + utxos_to_spend=utxos_to_spend, + num_outputs=num_outputs)['new_utxos'] def run_test(self): - # Mine some blocks and have them mature. - self.generate(self.nodes[0], COINBASE_MATURITY + 1) - utxo = self.nodes[0].listunspent(10) - txid = utxo[0]['txid'] - vout = utxo[0]['vout'] - value = utxo[0]['amount'] + self.wallet = MiniWallet(self.nodes[0]) + self.wallet.rescan_utxos() - fee = Decimal("0.0002") # MAX_ANCESTORS transactions off a confirmed tx should be fine chain = [] + utxo = self.wallet.get_utxo() for _ in range(4): - (txid, sent_value) = chain_transaction(self.nodes[0], [txid], [vout], value, fee, 2) - vout = 0 - value = sent_value - chain.append([txid, value]) + utxo, utxo2 = self.chain_tx([utxo], num_outputs=2) + chain.append(utxo2) for _ in range(MAX_ANCESTORS - 4): - (txid, sent_value) = chain_transaction(self.nodes[0], [txid], [0], value, fee, 1) - value = sent_value - chain.append([txid, value]) - (second_chain, second_chain_value) = chain_transaction(self.nodes[0], [utxo[1]['txid']], [utxo[1]['vout']], utxo[1]['amount'], fee, 1) + utxo, = self.chain_tx([utxo]) + chain.append(utxo) + second_chain, = self.chain_tx([self.wallet.get_utxo()]) # Check mempool has MAX_ANCESTORS + 1 transactions in it assert_equal(len(self.nodes[0].getrawmempool()), MAX_ANCESTORS + 1) # Adding one more transaction on to the chain should fail. - assert_raises_rpc_error(-26, "too-long-mempool-chain, too many unconfirmed ancestors [limit: 25]", chain_transaction, self.nodes[0], [txid], [0], value, fee, 1) + assert_raises_rpc_error(-26, "too-long-mempool-chain, too many unconfirmed ancestors [limit: 25]", self.chain_tx, [utxo]) # ...even if it chains on from some point in the middle of the chain. - assert_raises_rpc_error(-26, "too-long-mempool-chain, too many descendants", chain_transaction, self.nodes[0], [chain[2][0]], [1], chain[2][1], fee, 1) - assert_raises_rpc_error(-26, "too-long-mempool-chain, too many descendants", chain_transaction, self.nodes[0], [chain[1][0]], [1], chain[1][1], fee, 1) + assert_raises_rpc_error(-26, "too-long-mempool-chain, too many descendants", self.chain_tx, [chain[2]]) + assert_raises_rpc_error(-26, "too-long-mempool-chain, too many descendants", self.chain_tx, [chain[1]]) # ...even if it chains on to two parent transactions with one in the chain. - assert_raises_rpc_error(-26, "too-long-mempool-chain, too many descendants", chain_transaction, self.nodes[0], [chain[0][0], second_chain], [1, 0], chain[0][1] + second_chain_value, fee, 1) + assert_raises_rpc_error(-26, "too-long-mempool-chain, too many descendants", self.chain_tx, [chain[0], second_chain]) # ...especially if its > 40k weight - assert_raises_rpc_error(-26, "too-long-mempool-chain, too many descendants", chain_transaction, self.nodes[0], [chain[0][0]], [1], chain[0][1], fee, 350) + assert_raises_rpc_error(-26, "too-long-mempool-chain, too many descendants", self.chain_tx, [chain[0]], num_outputs=350) # But not if it chains directly off the first transaction - (replacable_txid, replacable_orig_value) = chain_transaction(self.nodes[0], [chain[0][0]], [1], chain[0][1], fee, 1) + replacable_tx = self.wallet.send_self_transfer_multi(from_node=self.nodes[0], utxos_to_spend=[chain[0]])['tx'] # and the second chain should work just fine - chain_transaction(self.nodes[0], [second_chain], [0], second_chain_value, fee, 1) + self.chain_tx([second_chain]) # Make sure we can RBF the chain which used our carve-out rule - second_tx_outputs = {self.nodes[0].getrawtransaction(replacable_txid, True)["vout"][0]['scriptPubKey']['address']: replacable_orig_value - (Decimal(1) / Decimal(100))} - second_tx = self.nodes[0].createrawtransaction([{'txid': chain[0][0], 'vout': 1}], second_tx_outputs) - signed_second_tx = self.nodes[0].signrawtransactionwithwallet(second_tx) - self.nodes[0].sendrawtransaction(signed_second_tx['hex']) + replacable_tx.vout[0].nValue -= 1000000 + self.nodes[0].sendrawtransaction(replacable_tx.serialize().hex()) # Finally, check that we added two transactions assert_equal(len(self.nodes[0].getrawmempool()), MAX_ANCESTORS + 3) + if __name__ == '__main__': MempoolPackagesTest().main() diff --git a/test/functional/mempool_packages.py b/test/functional/mempool_packages.py index 068fdc0b65..a2a2caf324 100755 --- a/test/functional/mempool_packages.py +++ b/test/functional/mempool_packages.py @@ -100,6 +100,12 @@ class MempoolPackagesTest(BitcoinTestFramework): entry = self.nodes[0].getmempoolentry(x) assert_equal(entry, mempool[x]) + # Check that gettxspendingprevout is consistent with getrawmempool + witnesstx = self.nodes[0].gettransaction(txid=x, verbose=True)['decoded'] + for tx_in in witnesstx["vin"]: + spending_result = self.nodes[0].gettxspendingprevout([ {'txid' : tx_in["txid"], 'vout' : tx_in["vout"]} ]) + assert_equal(spending_result, [ {'txid' : tx_in["txid"], 'vout' : tx_in["vout"], 'spendingtxid' : x} ]) + # Check that the descendant calculations are correct assert_equal(entry['descendantcount'], descendant_count) descendant_fees += entry['fees']['base'] diff --git a/test/functional/mempool_unbroadcast.py b/test/functional/mempool_unbroadcast.py index adf7326dac..37ef4a9157 100755 --- a/test/functional/mempool_unbroadcast.py +++ b/test/functional/mempool_unbroadcast.py @@ -9,21 +9,20 @@ import time from test_framework.p2p import P2PTxInvStore from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import ( - assert_equal, - create_confirmed_utxos, -) +from test_framework.util import assert_equal +from test_framework.wallet import MiniWallet MAX_INITIAL_BROADCAST_DELAY = 15 * 60 # 15 minutes in seconds class MempoolUnbroadcastTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 - - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() + if self.is_wallet_compiled(): + self.requires_wallet = True def run_test(self): + self.wallet = MiniWallet(self.nodes[0]) + self.wallet.rescan_utxos() self.test_broadcast() self.test_txn_removal() @@ -31,30 +30,25 @@ class MempoolUnbroadcastTest(BitcoinTestFramework): self.log.info("Test that mempool reattempts delivery of locally submitted transaction") node = self.nodes[0] - min_relay_fee = node.getnetworkinfo()["relayfee"] - utxos = create_confirmed_utxos(self, min_relay_fee, node, 10) - self.disconnect_nodes(0, 1) self.log.info("Generate transactions that only node 0 knows about") - # generate a wallet txn - addr = node.getnewaddress() - wallet_tx_hsh = node.sendtoaddress(addr, 0.0001) + if self.is_wallet_compiled(): + # generate a wallet txn + addr = node.getnewaddress() + wallet_tx_hsh = node.sendtoaddress(addr, 0.0001) # generate a txn using sendrawtransaction - us0 = utxos.pop() - inputs = [{"txid": us0["txid"], "vout": us0["vout"]}] - outputs = {addr: 0.0001} - tx = node.createrawtransaction(inputs, outputs) - node.settxfee(min_relay_fee) - txF = node.fundrawtransaction(tx) - txFS = node.signrawtransactionwithwallet(txF["hex"]) + txFS = self.wallet.create_self_transfer(from_node=node) rpc_tx_hsh = node.sendrawtransaction(txFS["hex"]) # check transactions are in unbroadcast using rpc mempoolinfo = self.nodes[0].getmempoolinfo() - assert_equal(mempoolinfo['unbroadcastcount'], 2) + unbroadcast_count = 1 + if self.is_wallet_compiled(): + unbroadcast_count += 1 + assert_equal(mempoolinfo['unbroadcastcount'], unbroadcast_count) mempool = self.nodes[0].getrawmempool(True) for tx in mempool: assert_equal(mempool[tx]['unbroadcast'], True) @@ -62,7 +56,8 @@ class MempoolUnbroadcastTest(BitcoinTestFramework): # check that second node doesn't have these two txns mempool = self.nodes[1].getrawmempool() assert rpc_tx_hsh not in mempool - assert wallet_tx_hsh not in mempool + if self.is_wallet_compiled(): + assert wallet_tx_hsh not in mempool # ensure that unbroadcast txs are persisted to mempool.dat self.restart_node(0) @@ -75,7 +70,8 @@ class MempoolUnbroadcastTest(BitcoinTestFramework): self.sync_mempools(timeout=30) mempool = self.nodes[1].getrawmempool() assert rpc_tx_hsh in mempool - assert wallet_tx_hsh in mempool + if self.is_wallet_compiled(): + assert wallet_tx_hsh in mempool # check that transactions are no longer in first node's unbroadcast set mempool = self.nodes[0].getrawmempool(True) @@ -102,8 +98,7 @@ class MempoolUnbroadcastTest(BitcoinTestFramework): # since the node doesn't have any connections, it will not receive # any GETDATAs & thus the transaction will remain in the unbroadcast set. - addr = node.getnewaddress() - txhsh = node.sendtoaddress(addr, 0.0001) + txhsh = self.wallet.send_self_transfer(from_node=node)["txid"] # check transaction was removed from unbroadcast set due to presence in # a block diff --git a/test/functional/mempool_updatefromblock.py b/test/functional/mempool_updatefromblock.py index 16c15e3f74..51de582ce0 100755 --- a/test/functional/mempool_updatefromblock.py +++ b/test/functional/mempool_updatefromblock.py @@ -17,7 +17,7 @@ from test_framework.util import assert_equal class MempoolUpdateFromBlockTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 - self.extra_args = [['-limitdescendantsize=1000', '-limitancestorsize=1000']] + self.extra_args = [['-limitdescendantsize=1000', '-limitancestorsize=1000', '-limitancestorcount=100']] def skip_test_if_missing_module(self): self.skip_if_no_wallet() diff --git a/test/functional/mining_prioritisetransaction.py b/test/functional/mining_prioritisetransaction.py index 6f2ac805a0..a15fbe5a24 100755 --- a/test/functional/mining_prioritisetransaction.py +++ b/test/functional/mining_prioritisetransaction.py @@ -4,11 +4,15 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the prioritisetransaction mining RPC.""" +from decimal import Decimal import time +from test_framework.blocktools import COINBASE_MATURITY from test_framework.messages import COIN, MAX_BLOCK_WEIGHT from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal, assert_raises_rpc_error, create_confirmed_utxos, create_lots_of_big_transactions, gen_return_txouts +from test_framework.wallet import MiniWallet + class PrioritiseTransactionTest(BitcoinTestFramework): def set_test_params(self): @@ -23,7 +27,84 @@ class PrioritiseTransactionTest(BitcoinTestFramework): def skip_test_if_missing_module(self): self.skip_if_no_wallet() + def test_diamond(self): + self.log.info("Test diamond-shape package with priority") + self.generate(self.wallet, COINBASE_MATURITY + 1) + mock_time = int(time.time()) + self.nodes[0].setmocktime(mock_time) + + # tx_a + # / \ + # / \ + # tx_b tx_c + # \ / + # \ / + # tx_d + + tx_o_a = self.wallet.send_self_transfer_multi( + from_node=self.nodes[0], + num_outputs=2, + ) + txid_a = tx_o_a["txid"] + + tx_o_b, tx_o_c = [self.wallet.send_self_transfer( + from_node=self.nodes[0], + utxo_to_spend=u, + ) for u in tx_o_a["new_utxos"]] + txid_b = tx_o_b["txid"] + txid_c = tx_o_c["txid"] + + tx_o_d = self.wallet.send_self_transfer_multi( + from_node=self.nodes[0], + utxos_to_spend=[ + self.wallet.get_utxo(txid=txid_b), + self.wallet.get_utxo(txid=txid_c), + ], + ) + txid_d = tx_o_d["txid"] + + self.log.info("Test priority while txs are in mempool") + raw_before = self.nodes[0].getrawmempool(verbose=True) + fee_delta_b = Decimal(9999) / COIN + fee_delta_c_1 = Decimal(-1234) / COIN + fee_delta_c_2 = Decimal(8888) / COIN + self.nodes[0].prioritisetransaction(txid=txid_b, fee_delta=int(fee_delta_b * COIN)) + self.nodes[0].prioritisetransaction(txid=txid_c, fee_delta=int(fee_delta_c_1 * COIN)) + self.nodes[0].prioritisetransaction(txid=txid_c, fee_delta=int(fee_delta_c_2 * COIN)) + raw_before[txid_a]["fees"]["descendant"] += fee_delta_b + fee_delta_c_1 + fee_delta_c_2 + raw_before[txid_b]["fees"]["modified"] += fee_delta_b + raw_before[txid_b]["fees"]["ancestor"] += fee_delta_b + raw_before[txid_b]["fees"]["descendant"] += fee_delta_b + raw_before[txid_c]["fees"]["modified"] += fee_delta_c_1 + fee_delta_c_2 + raw_before[txid_c]["fees"]["ancestor"] += fee_delta_c_1 + fee_delta_c_2 + raw_before[txid_c]["fees"]["descendant"] += fee_delta_c_1 + fee_delta_c_2 + raw_before[txid_d]["fees"]["ancestor"] += fee_delta_b + fee_delta_c_1 + fee_delta_c_2 + raw_after = self.nodes[0].getrawmempool(verbose=True) + assert_equal(raw_before[txid_a], raw_after[txid_a]) + assert_equal(raw_before, raw_after) + + self.log.info("Test priority while txs are not in mempool") + self.restart_node(0, extra_args=["-nopersistmempool"]) + self.nodes[0].setmocktime(mock_time) + assert_equal(self.nodes[0].getmempoolinfo()["size"], 0) + self.nodes[0].prioritisetransaction(txid=txid_b, fee_delta=int(fee_delta_b * COIN)) + self.nodes[0].prioritisetransaction(txid=txid_c, fee_delta=int(fee_delta_c_1 * COIN)) + self.nodes[0].prioritisetransaction(txid=txid_c, fee_delta=int(fee_delta_c_2 * COIN)) + for t in [tx_o_a["hex"], tx_o_b["hex"], tx_o_c["hex"], tx_o_d["hex"]]: + self.nodes[0].sendrawtransaction(t) + raw_after = self.nodes[0].getrawmempool(verbose=True) + assert_equal(raw_before[txid_a], raw_after[txid_a]) + assert_equal(raw_before, raw_after) + + # Clear mempool + self.generate(self.nodes[0], 1) + + # Use default extra_args + self.restart_node(0) + def run_test(self): + self.wallet = MiniWallet(self.nodes[0]) + # Test `prioritisetransaction` required parameters assert_raises_rpc_error(-1, "prioritisetransaction", self.nodes[0].prioritisetransaction) assert_raises_rpc_error(-1, "prioritisetransaction", self.nodes[0].prioritisetransaction, '') @@ -44,6 +125,8 @@ class PrioritiseTransactionTest(BitcoinTestFramework): # Test `prioritisetransaction` invalid `fee_delta` assert_raises_rpc_error(-1, "JSON value is not an integer as expected", self.nodes[0].prioritisetransaction, txid=txid, fee_delta='foo') + self.test_diamond() + self.txouts = gen_return_txouts() self.relayfee = self.nodes[0].getnetworkinfo()['relayfee'] diff --git a/test/functional/p2p_addr_relay.py b/test/functional/p2p_addr_relay.py index 3218a9b14a..e2e9b6dcb2 100755 --- a/test/functional/p2p_addr_relay.py +++ b/test/functional/p2p_addr_relay.py @@ -21,8 +21,19 @@ from test_framework.p2p import ( P2P_SERVICES, ) from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_greater_than +from test_framework.util import ( + assert_equal, + assert_greater_than, + assert_greater_than_or_equal +) + +ONE_MINUTE = 60 +TEN_MINUTES = 10 * ONE_MINUTE +ONE_HOUR = 60 * ONE_MINUTE +TWO_HOURS = 2 * ONE_HOUR +ONE_DAY = 24 * ONE_HOUR +ADDR_DESTINATIONS_THRESHOLD = 4 class AddrReceiver(P2PInterface): num_ipv4_received = 0 @@ -85,6 +96,9 @@ class AddrTest(BitcoinTestFramework): self.relay_tests() self.inbound_blackhole_tests() + self.destination_rotates_once_in_24_hours_test() + self.destination_rotates_more_than_once_over_several_days_test() + # This test populates the addrman, which can impact the node's behavior # in subsequent tests self.getaddr_tests() @@ -362,6 +376,56 @@ class AddrTest(BitcoinTestFramework): self.nodes[0].disconnect_p2ps() + def get_nodes_that_received_addr(self, peer, receiver_peer, addr_receivers, + time_interval_1, time_interval_2): + + # Clean addr response related to the initial getaddr. There is no way to avoid initial + # getaddr because the peer won't self-announce then. + for addr_receiver in addr_receivers: + addr_receiver.num_ipv4_received = 0 + + for _ in range(10): + self.mocktime += time_interval_1 + self.msg.addrs[0].time = self.mocktime + TEN_MINUTES + self.nodes[0].setmocktime(self.mocktime) + with self.nodes[0].assert_debug_log(['received: addr (31 bytes) peer=0']): + peer.send_and_ping(self.msg) + self.mocktime += time_interval_2 + self.nodes[0].setmocktime(self.mocktime) + receiver_peer.sync_with_ping() + return [node for node in addr_receivers if node.addr_received()] + + def destination_rotates_once_in_24_hours_test(self): + self.restart_node(0, []) + + self.log.info('Test within 24 hours an addr relay destination is rotated at most once') + self.mocktime = int(time.time()) + self.msg = self.setup_addr_msg(1) + self.addr_receivers = [] + peer = self.nodes[0].add_p2p_connection(P2PInterface()) + receiver_peer = self.nodes[0].add_p2p_connection(AddrReceiver()) + addr_receivers = [self.nodes[0].add_p2p_connection(AddrReceiver()) for _ in range(20)] + nodes_received_addr = self.get_nodes_that_received_addr(peer, receiver_peer, addr_receivers, 0, TWO_HOURS) # 10 intervals of 2 hours + # Per RelayAddress, we would announce these addrs to 2 destinations per day. + # Since it's at most one rotation, at most 4 nodes can receive ADDR. + assert_greater_than_or_equal(ADDR_DESTINATIONS_THRESHOLD, len(nodes_received_addr)) + self.nodes[0].disconnect_p2ps() + + def destination_rotates_more_than_once_over_several_days_test(self): + self.restart_node(0, []) + + self.log.info('Test after several days an addr relay destination is rotated more than once') + self.msg = self.setup_addr_msg(1) + peer = self.nodes[0].add_p2p_connection(P2PInterface()) + receiver_peer = self.nodes[0].add_p2p_connection(AddrReceiver()) + addr_receivers = [self.nodes[0].add_p2p_connection(AddrReceiver()) for _ in range(20)] + # 10 intervals of 1 day (+ 1 hour, which should be enough to cover 30-min Poisson in most cases) + nodes_received_addr = self.get_nodes_that_received_addr(peer, receiver_peer, addr_receivers, ONE_DAY, ONE_HOUR) + # Now that there should have been more than one rotation, more than + # ADDR_DESTINATIONS_THRESHOLD nodes should have received ADDR. + assert_greater_than(len(nodes_received_addr), ADDR_DESTINATIONS_THRESHOLD) + self.nodes[0].disconnect_p2ps() + if __name__ == '__main__': AddrTest().main() diff --git a/test/functional/p2p_blockfilters.py b/test/functional/p2p_blockfilters.py index e73fad439f..ef12b5f6b7 100755 --- a/test/functional/p2p_blockfilters.py +++ b/test/functional/p2p_blockfilters.py @@ -244,6 +244,23 @@ class CompactFiltersTest(BitcoinTestFramework): peer_0.send_message(request) peer_0.wait_for_disconnect() + self.log.info("Test -peerblockfilters without -blockfilterindex raises an error") + self.stop_node(0) + self.nodes[0].extra_args = ["-peerblockfilters"] + msg = "Error: Cannot set -peerblockfilters without -blockfilterindex." + self.nodes[0].assert_start_raises_init_error(expected_msg=msg) + + self.log.info("Test unknown value to -blockfilterindex raises an error") + self.nodes[0].extra_args = ["-blockfilterindex=abc"] + msg = "Error: Unknown -blockfilterindex value abc." + self.nodes[0].assert_start_raises_init_error(expected_msg=msg) + + self.log.info("Test -blockfilterindex with -reindex-chainstate raises an error") + self.nodes[0].assert_start_raises_init_error( + expected_msg='Error: -reindex-chainstate option is not compatible with -blockfilterindex. ' + 'Please temporarily disable blockfilterindex while using -reindex-chainstate, or replace -reindex-chainstate with -reindex to fully rebuild all indexes.', + extra_args=['-blockfilterindex', '-reindex-chainstate'], + ) def compute_last_header(prev_header, hashes): """Compute the last filter header from a starting header and a sequence of filter hashes.""" diff --git a/test/functional/p2p_blocksonly.py b/test/functional/p2p_blocksonly.py index 6f142f23f2..12ee4b3c27 100755 --- a/test/functional/p2p_blocksonly.py +++ b/test/functional/p2p_blocksonly.py @@ -94,7 +94,7 @@ class P2PBlocksOnly(BitcoinTestFramework): self.nodes[0].sendrawtransaction(tx_hex) - # Bump time forward to ensure nNextInvSend timer pops + # Bump time forward to ensure m_next_inv_send_time timer pops self.nodes[0].setmocktime(int(time.time()) + 60) conn.sync_send_with_ping() diff --git a/test/functional/p2p_compactblocks.py b/test/functional/p2p_compactblocks.py index 364e806e18..b9ac3c32c5 100755 --- a/test/functional/p2p_compactblocks.py +++ b/test/functional/p2p_compactblocks.py @@ -2,11 +2,7 @@ # Copyright (c) 2016-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test compact blocks (BIP 152). - -Version 1 compact blocks are pre-segwit (txids) -Version 2 compact blocks are post-segwit (wtxids) -""" +"""Test compact blocks (BIP 152).""" import random from test_framework.blocktools import ( @@ -31,7 +27,6 @@ from test_framework.messages import ( MSG_BLOCK, MSG_CMPCT_BLOCK, MSG_WITNESS_FLAG, - NODE_NETWORK, P2PHeaderAndShortIDs, PrefilledTransaction, calculate_shortid, @@ -70,7 +65,7 @@ from test_framework.wallet import MiniWallet # TestP2PConn: A peer we use to send messages to bitcoind, and store responses. class TestP2PConn(P2PInterface): - def __init__(self, cmpct_version): + def __init__(self): super().__init__() self.last_sendcmpct = [] self.block_announced = False @@ -78,7 +73,6 @@ class TestP2PConn(P2PInterface): # This is for synchronizing the p2p message traffic, # so we can eg wait until a particular block is announced. self.announced_blockhashes = set() - self.cmpct_version = cmpct_version def on_sendcmpct(self, message): self.last_sendcmpct.append(message) @@ -152,10 +146,8 @@ class CompactBlocksTest(BitcoinTestFramework): ]] self.utxos = [] - def build_block_on_tip(self, node, segwit=False): + def build_block_on_tip(self, node): block = create_block(tmpl=node.getblocktemplate(NORMAL_GBT_REQUEST_PARAMS)) - if segwit: - add_witness_commitment(block) block.solve() return block @@ -185,15 +177,13 @@ class CompactBlocksTest(BitcoinTestFramework): # Test "sendcmpct" (between peers preferring the same version): # - No compact block announcements unless sendcmpct is sent. - # - If sendcmpct is sent with version > preferred_version, the message is ignored. + # - If sendcmpct is sent with version = 1, the message is ignored. + # - If sendcmpct is sent with version > 2, the message is ignored. # - If sendcmpct is sent with boolean 0, then block announcements are not # made with compact blocks. # - If sendcmpct is then sent with boolean 1, then new block announcements # are made with compact blocks. - # If old_node is passed in, request compact blocks with version=preferred-1 - # and verify that it receives block announcements via compact block. - def test_sendcmpct(self, test_node, old_node=None): - preferred_version = test_node.cmpct_version + def test_sendcmpct(self, test_node): node = self.nodes[0] # Make sure we get a SENDCMPCT message from our peer @@ -201,10 +191,8 @@ class CompactBlocksTest(BitcoinTestFramework): return (len(test_node.last_sendcmpct) > 0) test_node.wait_until(received_sendcmpct, timeout=30) with p2p_lock: - # Check that the first version received is the preferred one - assert_equal(test_node.last_sendcmpct[0].version, preferred_version) - # And that we receive versions down to 1. - assert_equal(test_node.last_sendcmpct[-1].version, 1) + # Check that version 2 is received. + assert_equal(test_node.last_sendcmpct[0].version, 2) test_node.last_sendcmpct = [] tip = int(node.getbestblockhash(), 16) @@ -232,22 +220,29 @@ class CompactBlocksTest(BitcoinTestFramework): # Before each test, sync the headers chain. test_node.request_headers_and_sync(locator=[tip]) + # Now try a SENDCMPCT message with too-low version + test_node.send_and_ping(msg_sendcmpct(announce=True, version=1)) + check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message) + + # Headers sync before next test. + test_node.request_headers_and_sync(locator=[tip]) + # Now try a SENDCMPCT message with too-high version - test_node.send_and_ping(msg_sendcmpct(announce=True, version=preferred_version+1)) + test_node.send_and_ping(msg_sendcmpct(announce=True, version=3)) check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message) # Headers sync before next test. test_node.request_headers_and_sync(locator=[tip]) # Now try a SENDCMPCT message with valid version, but announce=False - test_node.send_and_ping(msg_sendcmpct(announce=False, version=preferred_version)) + test_node.send_and_ping(msg_sendcmpct(announce=False, version=2)) check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message) # Headers sync before next test. test_node.request_headers_and_sync(locator=[tip]) # Finally, try a SENDCMPCT message with announce=True - test_node.send_and_ping(msg_sendcmpct(announce=True, version=preferred_version)) + test_node.send_and_ping(msg_sendcmpct(announce=True, version=2)) check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" in p.last_message) # Try one more time (no headers sync should be needed!) @@ -257,22 +252,14 @@ class CompactBlocksTest(BitcoinTestFramework): test_node.send_and_ping(msg_sendheaders()) check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" in p.last_message) - # Try one more time, after sending a version-1, announce=false message. - test_node.send_and_ping(msg_sendcmpct(announce=False, version=preferred_version-1)) + # Try one more time, after sending a version=1, announce=false message. + test_node.send_and_ping(msg_sendcmpct(announce=False, version=1)) check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" in p.last_message) # Now turn off announcements - test_node.send_and_ping(msg_sendcmpct(announce=False, version=preferred_version)) + test_node.send_and_ping(msg_sendcmpct(announce=False, version=2)) check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message and "headers" in p.last_message) - if old_node is not None: - # Verify that a peer using an older protocol version can receive - # announcements from this node. - old_node.send_and_ping(msg_sendcmpct(announce=True, version=preferred_version-1)) - # Header sync - old_node.request_headers_and_sync(locator=[tip]) - check_announcement_of_new_block(node, old_node, lambda p: "cmpctblock" in p.last_message) - # This test actually causes bitcoind to (reasonably!) disconnect us, so do this last. def test_invalid_cmpctblock_message(self): self.generate(self.nodes[0], COINBASE_MATURITY + 1) @@ -289,8 +276,7 @@ class CompactBlocksTest(BitcoinTestFramework): # Compare the generated shortids to what we expect based on BIP 152, given # bitcoind's choice of nonce. - def test_compactblock_construction(self, test_node, use_witness_address=True): - version = test_node.cmpct_version + def test_compactblock_construction(self, test_node): node = self.nodes[0] # Generate a bunch of transactions. self.generate(node, COINBASE_MATURITY + 1) @@ -303,8 +289,7 @@ class CompactBlocksTest(BitcoinTestFramework): if not tx.wit.is_null(): segwit_tx_generated = True - if use_witness_address: - assert segwit_tx_generated # check that our test is not broken + assert segwit_tx_generated # check that our test is not broken # Wait until we've seen the block announcement for the resulting tip tip = int(node.getbestblockhash(), 16) @@ -331,7 +316,7 @@ class CompactBlocksTest(BitcoinTestFramework): with p2p_lock: # Convert the on-the-wire representation to absolute indexes header_and_shortids = HeaderAndShortIDs(test_node.last_message["cmpctblock"].header_and_shortids) - self.check_compactblock_construction_from_block(version, header_and_shortids, block_hash, block) + self.check_compactblock_construction_from_block(header_and_shortids, block_hash, block) # Now fetch the compact block using a normal non-announce getdata test_node.clear_block_announcement() @@ -345,9 +330,9 @@ class CompactBlocksTest(BitcoinTestFramework): with p2p_lock: # Convert the on-the-wire representation to absolute indexes header_and_shortids = HeaderAndShortIDs(test_node.last_message["cmpctblock"].header_and_shortids) - self.check_compactblock_construction_from_block(version, header_and_shortids, block_hash, block) + self.check_compactblock_construction_from_block(header_and_shortids, block_hash, block) - def check_compactblock_construction_from_block(self, version, header_and_shortids, block_hash, block): + def check_compactblock_construction_from_block(self, header_and_shortids, block_hash, block): # Check that we got the right block! header_and_shortids.header.calc_sha256() assert_equal(header_and_shortids.header.sha256, block_hash) @@ -364,11 +349,7 @@ class CompactBlocksTest(BitcoinTestFramework): # And this checks the witness wtxid = entry.tx.calc_sha256(True) - if version == 2: - assert_equal(wtxid, block.vtx[entry.index].calc_sha256(True)) - else: - # Shouldn't have received a witness - assert entry.tx.wit.is_null() + assert_equal(wtxid, block.vtx[entry.index].calc_sha256(True)) # Check that the cmpctblock message announced all the transactions. assert_equal(len(header_and_shortids.prefilled_txn) + len(header_and_shortids.shortids), len(block.vtx)) @@ -384,9 +365,7 @@ class CompactBlocksTest(BitcoinTestFramework): # Already checked prefilled transactions above header_and_shortids.prefilled_txn.pop(0) else: - tx_hash = block.vtx[index].sha256 - if version == 2: - tx_hash = block.vtx[index].calc_sha256(True) + tx_hash = block.vtx[index].calc_sha256(True) shortid = calculate_shortid(k0, k1, tx_hash) assert_equal(shortid, header_and_shortids.shortids[0]) header_and_shortids.shortids.pop(0) @@ -395,16 +374,12 @@ class CompactBlocksTest(BitcoinTestFramework): # Test that bitcoind requests compact blocks when we announce new blocks # via header or inv, and that responding to getblocktxn causes the block # to be successfully reconstructed. - # Post-segwit: upgraded nodes would only make this request of cb-version-2, - # NODE_WITNESS peers. Unupgraded nodes would still make this request of - # any cb-version-1-supporting peer. - def test_compactblock_requests(self, test_node, segwit=True): - version = test_node.cmpct_version + def test_compactblock_requests(self, test_node): node = self.nodes[0] # Try announcing a block with an inv or header, expect a compactblock # request for announce in ["inv", "header"]: - block = self.build_block_on_tip(node, segwit=segwit) + block = self.build_block_on_tip(node) if announce == "inv": test_node.send_message(msg_inv([CInv(MSG_BLOCK, block.sha256)])) @@ -420,9 +395,7 @@ class CompactBlocksTest(BitcoinTestFramework): comp_block.header = CBlockHeader(block) comp_block.nonce = 0 [k0, k1] = comp_block.get_siphash_keys() - coinbase_hash = block.vtx[0].sha256 - if version == 2: - coinbase_hash = block.vtx[0].calc_sha256(True) + coinbase_hash = block.vtx[0].calc_sha256(True) comp_block.shortids = [calculate_shortid(k0, k1, coinbase_hash)] test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) assert_equal(int(node.getbestblockhash(), 16), block.hashPrevBlock) @@ -433,10 +406,7 @@ class CompactBlocksTest(BitcoinTestFramework): assert_equal(absolute_indexes, [0]) # should be a coinbase request # Send the coinbase, and verify that the tip advances. - if version == 2: - msg = msg_blocktxn() - else: - msg = msg_no_witness_blocktxn() + msg = msg_blocktxn() msg.block_transactions.blockhash = block.sha256 msg.block_transactions.transactions = [block.vtx[0]] test_node.send_and_ping(msg) @@ -462,9 +432,7 @@ class CompactBlocksTest(BitcoinTestFramework): # node needs, and that responding to them causes the block to be # reconstructed. def test_getblocktxn_requests(self, test_node): - version = test_node.cmpct_version node = self.nodes[0] - with_witness = (version == 2) def test_getblocktxn_response(compact_block, peer, expected_result): msg = msg_cmpctblock(compact_block.to_p2p()) @@ -485,13 +453,12 @@ class CompactBlocksTest(BitcoinTestFramework): block = self.build_block_with_transactions(node, utxo, 5) self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue]) comp_block = HeaderAndShortIDs() - comp_block.initialize_from_block(block, use_witness=with_witness) + comp_block.initialize_from_block(block, use_witness=True) test_getblocktxn_response(comp_block, test_node, [1, 2, 3, 4, 5]) msg_bt = msg_no_witness_blocktxn() - if with_witness: - msg_bt = msg_blocktxn() # serialize with witnesses + msg_bt = msg_blocktxn() # serialize with witnesses msg_bt.block_transactions = BlockTransactions(block.sha256, block.vtx[1:]) test_tip_after_message(node, test_node, msg_bt, block.sha256) @@ -500,7 +467,7 @@ class CompactBlocksTest(BitcoinTestFramework): self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue]) # Now try interspersing the prefilled transactions - comp_block.initialize_from_block(block, prefill_list=[0, 1, 5], use_witness=with_witness) + comp_block.initialize_from_block(block, prefill_list=[0, 1, 5], use_witness=True) test_getblocktxn_response(comp_block, test_node, [2, 3, 4]) msg_bt.block_transactions = BlockTransactions(block.sha256, block.vtx[2:5]) test_tip_after_message(node, test_node, msg_bt, block.sha256) @@ -514,7 +481,7 @@ class CompactBlocksTest(BitcoinTestFramework): # Prefill 4 out of the 6 transactions, and verify that only the one # that was not in the mempool is requested. - comp_block.initialize_from_block(block, prefill_list=[0, 2, 3, 4], use_witness=with_witness) + comp_block.initialize_from_block(block, prefill_list=[0, 2, 3, 4], use_witness=True) test_getblocktxn_response(comp_block, test_node, [5]) msg_bt.block_transactions = BlockTransactions(block.sha256, [block.vtx[5]]) @@ -538,7 +505,7 @@ class CompactBlocksTest(BitcoinTestFramework): test_node.last_message.pop("getblocktxn", None) # Send compact block - comp_block.initialize_from_block(block, prefill_list=[0], use_witness=with_witness) + comp_block.initialize_from_block(block, prefill_list=[0], use_witness=True) test_tip_after_message(node, test_node, msg_cmpctblock(comp_block.to_p2p()), block.sha256) with p2p_lock: # Shouldn't have gotten a request for any transaction @@ -547,7 +514,6 @@ class CompactBlocksTest(BitcoinTestFramework): # Incorrectly responding to a getblocktxn shouldn't cause the block to be # permanently failed. def test_incorrect_blocktxn_response(self, test_node): - version = test_node.cmpct_version node = self.nodes[0] utxo = self.utxos.pop(0) @@ -564,7 +530,7 @@ class CompactBlocksTest(BitcoinTestFramework): # Send compact block comp_block = HeaderAndShortIDs() - comp_block.initialize_from_block(block, prefill_list=[0], use_witness=(version == 2)) + comp_block.initialize_from_block(block, prefill_list=[0], use_witness=True) test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) absolute_indexes = [] with p2p_lock: @@ -580,9 +546,7 @@ class CompactBlocksTest(BitcoinTestFramework): # different peer provide the block further down, so that we're still # verifying that the block isn't marked bad permanently. This is good # enough for now. - msg = msg_no_witness_blocktxn() - if version == 2: - msg = msg_blocktxn() + msg = msg_blocktxn() msg.block_transactions = BlockTransactions(block.sha256, [block.vtx[5]] + block.vtx[7:]) test_node.send_and_ping(msg) @@ -595,14 +559,10 @@ class CompactBlocksTest(BitcoinTestFramework): test_node.last_message["getdata"].inv[0].type == MSG_BLOCK | MSG_WITNESS_FLAG # Deliver the block - if version == 2: - test_node.send_and_ping(msg_block(block)) - else: - test_node.send_and_ping(msg_no_witness_block(block)) + test_node.send_and_ping(msg_block(block)) assert_equal(int(node.getbestblockhash(), 16), block.sha256) def test_getblocktxn_handler(self, test_node): - version = test_node.cmpct_version node = self.nodes[0] # bitcoind will not send blocktxn responses for blocks whose height is # more than 10 blocks deep. @@ -628,12 +588,8 @@ class CompactBlocksTest(BitcoinTestFramework): tx = test_node.last_message["blocktxn"].block_transactions.transactions.pop(0) tx.calc_sha256() assert_equal(tx.sha256, block.vtx[index].sha256) - if version == 1: - # Witnesses should have been stripped - assert tx.wit.is_null() - else: - # Check that the witness matches - assert_equal(tx.calc_sha256(True), block.vtx[index].calc_sha256(True)) + # Check that the witness matches + assert_equal(tx.calc_sha256(True), block.vtx[index].calc_sha256(True)) test_node.last_message.pop("blocktxn", None) current_height -= 1 @@ -727,7 +683,7 @@ class CompactBlocksTest(BitcoinTestFramework): # Test that we don't get disconnected if we relay a compact block with valid header, # but invalid transactions. - def test_invalid_tx_in_compactblock(self, test_node, use_segwit=True): + def test_invalid_tx_in_compactblock(self, test_node): node = self.nodes[0] assert len(self.utxos) utxo = self.utxos[0] @@ -735,17 +691,15 @@ class CompactBlocksTest(BitcoinTestFramework): block = self.build_block_with_transactions(node, utxo, 5) del block.vtx[3] block.hashMerkleRoot = block.calc_merkle_root() - if use_segwit: - # If we're testing with segwit, also drop the coinbase witness, - # but include the witness commitment. - add_witness_commitment(block) - block.vtx[0].wit.vtxinwit = [] + # Drop the coinbase witness but include the witness commitment. + add_witness_commitment(block) + block.vtx[0].wit.vtxinwit = [] block.solve() # Now send the compact block with all transactions prefilled, and # verify that we don't get disconnected. comp_block = HeaderAndShortIDs() - comp_block.initialize_from_block(block, prefill_list=[0, 1, 2, 3, 4], use_witness=use_segwit) + comp_block.initialize_from_block(block, prefill_list=[0, 1, 2, 3, 4], use_witness=True) msg = msg_cmpctblock(comp_block.to_p2p()) test_node.send_and_ping(msg) @@ -759,7 +713,7 @@ class CompactBlocksTest(BitcoinTestFramework): node = self.nodes[0] tip = node.getbestblockhash() peer.get_headers(locator=[int(tip, 16)], hashstop=0) - peer.send_and_ping(msg_sendcmpct(announce=True, version=peer.cmpct_version)) + peer.send_and_ping(msg_sendcmpct(announce=True, version=2)) def test_compactblock_reconstruction_multiple_peers(self, stalling_peer, delivery_peer): node = self.nodes[0] @@ -813,7 +767,7 @@ class CompactBlocksTest(BitcoinTestFramework): def test_highbandwidth_mode_states_via_getpeerinfo(self): # create new p2p connection for a fresh state w/o any prior sendcmpct messages sent - hb_test_node = self.nodes[0].add_p2p_connection(TestP2PConn(cmpct_version=2)) + hb_test_node = self.nodes[0].add_p2p_connection(TestP2PConn()) # assert the RPC getpeerinfo boolean fields `bip152_hb_{to, from}` # match the given parameters for the last peer of a given node @@ -843,9 +797,8 @@ class CompactBlocksTest(BitcoinTestFramework): self.wallet = MiniWallet(self.nodes[0]) # Setup the p2p connections - self.segwit_node = self.nodes[0].add_p2p_connection(TestP2PConn(cmpct_version=2)) - self.old_node = self.nodes[0].add_p2p_connection(TestP2PConn(cmpct_version=1), services=NODE_NETWORK) - self.additional_segwit_node = self.nodes[0].add_p2p_connection(TestP2PConn(cmpct_version=2)) + self.segwit_node = self.nodes[0].add_p2p_connection(TestP2PConn()) + self.additional_segwit_node = self.nodes[0].add_p2p_connection(TestP2PConn()) # We will need UTXOs to construct transactions in later tests. self.make_utxos() @@ -853,11 +806,10 @@ class CompactBlocksTest(BitcoinTestFramework): assert softfork_active(self.nodes[0], "segwit") self.log.info("Testing SENDCMPCT p2p message... ") - self.test_sendcmpct(self.segwit_node, old_node=self.old_node) + self.test_sendcmpct(self.segwit_node) self.test_sendcmpct(self.additional_segwit_node) self.log.info("Testing compactblock construction...") - self.test_compactblock_construction(self.old_node) self.test_compactblock_construction(self.segwit_node) self.log.info("Testing compactblock requests (segwit node)... ") @@ -868,11 +820,9 @@ class CompactBlocksTest(BitcoinTestFramework): self.log.info("Testing getblocktxn handler (segwit node should return witnesses)...") self.test_getblocktxn_handler(self.segwit_node) - self.test_getblocktxn_handler(self.old_node) self.log.info("Testing compactblock requests/announcements not at chain tip...") self.test_compactblocks_not_at_tip(self.segwit_node) - self.test_compactblocks_not_at_tip(self.old_node) self.log.info("Testing handling of incorrect blocktxn responses...") self.test_incorrect_blocktxn_response(self.segwit_node) @@ -885,13 +835,12 @@ class CompactBlocksTest(BitcoinTestFramework): # (Post-segwit activation, blocks won't propagate from node0 to node1 # automatically, so don't bother testing a block announced to node0.) self.log.info("Testing end-to-end block relay...") - self.request_cb_announcements(self.old_node) self.request_cb_announcements(self.segwit_node) - self.test_end_to_end_block_relay([self.segwit_node, self.old_node]) + self.request_cb_announcements(self.additional_segwit_node) + self.test_end_to_end_block_relay([self.segwit_node, self.additional_segwit_node]) self.log.info("Testing handling of invalid compact blocks...") self.test_invalid_tx_in_compactblock(self.segwit_node) - self.test_invalid_tx_in_compactblock(self.old_node) self.log.info("Testing invalid index in cmpctblock message...") self.test_invalid_cmpctblock_message() diff --git a/test/functional/p2p_compactblocks_blocksonly.py b/test/functional/p2p_compactblocks_blocksonly.py index 6367eb26a3..3d0c421a93 100755 --- a/test/functional/p2p_compactblocks_blocksonly.py +++ b/test/functional/p2p_compactblocks_blocksonly.py @@ -48,7 +48,7 @@ class P2PCompactBlocksBlocksOnly(BitcoinTestFramework): p2p_conn_high_bw = self.nodes[1].add_p2p_connection(P2PInterface()) p2p_conn_low_bw = self.nodes[3].add_p2p_connection(P2PInterface()) for conn in [p2p_conn_blocksonly, p2p_conn_high_bw, p2p_conn_low_bw]: - assert_equal(conn.message_count['sendcmpct'], 2) + assert_equal(conn.message_count['sendcmpct'], 1) conn.send_and_ping(msg_sendcmpct(announce=False, version=2)) # Nodes: @@ -74,14 +74,14 @@ class P2PCompactBlocksBlocksOnly(BitcoinTestFramework): # receiving a new valid block at the tip. p2p_conn_blocksonly.send_and_ping(msg_block(block0)) assert_equal(int(self.nodes[0].getbestblockhash(), 16), block0.sha256) - assert_equal(p2p_conn_blocksonly.message_count['sendcmpct'], 2) + assert_equal(p2p_conn_blocksonly.message_count['sendcmpct'], 1) assert_equal(p2p_conn_blocksonly.last_message['sendcmpct'].announce, False) # A normal node participating in transaction relay should request BIP152 # high bandwidth mode upon receiving a new valid block at the tip. p2p_conn_high_bw.send_and_ping(msg_block(block0)) assert_equal(int(self.nodes[1].getbestblockhash(), 16), block0.sha256) - p2p_conn_high_bw.wait_until(lambda: p2p_conn_high_bw.message_count['sendcmpct'] == 3) + p2p_conn_high_bw.wait_until(lambda: p2p_conn_high_bw.message_count['sendcmpct'] == 2) assert_equal(p2p_conn_high_bw.last_message['sendcmpct'].announce, True) # Don't send a block from the p2p_conn_low_bw so the low bandwidth node diff --git a/test/functional/p2p_message_capture.py b/test/functional/p2p_message_capture.py index edde9a6ecf..87c77f4540 100755 --- a/test/functional/p2p_message_capture.py +++ b/test/functional/p2p_message_capture.py @@ -20,7 +20,7 @@ LENGTH_SIZE = 4 MSGTYPE_SIZE = 12 def mini_parser(dat_file): - """Parse a data file created by CaptureMessage. + """Parse a data file created by CaptureMessageToFile. From the data file we'll only check the structure. @@ -43,12 +43,8 @@ def mini_parser(dat_file): break tmp_header = BytesIO(tmp_header_raw) tmp_header.read(TIME_SIZE) # skip the timestamp field - raw_msgtype = tmp_header.read(MSGTYPE_SIZE) - msgtype: bytes = raw_msgtype.split(b'\x00', 1)[0] - remainder = raw_msgtype.split(b'\x00', 1)[1] - assert(len(msgtype) > 0) + msgtype = tmp_header.read(MSGTYPE_SIZE).rstrip(b'\x00') assert(msgtype in MESSAGEMAP) - assert(len(remainder) == 0 or not remainder.decode().isprintable()) length: int = int.from_bytes(tmp_header.read(LENGTH_SIZE), "little") data = f_in.read(length) assert_equal(len(data), length) diff --git a/test/functional/p2p_segwit.py b/test/functional/p2p_segwit.py index f377fbaaa6..89ddfd3bcf 100755 --- a/test/functional/p2p_segwit.py +++ b/test/functional/p2p_segwit.py @@ -43,7 +43,6 @@ from test_framework.messages import ( ser_uint256, ser_vector, sha256, - tx_from_hex, ) from test_framework.p2p import ( P2PInterface, @@ -89,6 +88,8 @@ from test_framework.util import ( softfork_active, assert_raises_rpc_error, ) +from test_framework.wallet import MiniWallet + MAX_SIGOP_COST = 80000 @@ -221,9 +222,6 @@ class SegWitTest(BitcoinTestFramework): ] self.supports_cli = False - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() - # Helper functions def build_next_block(self): @@ -259,6 +257,7 @@ class SegWitTest(BitcoinTestFramework): self.log.info("Starting tests before segwit activation") self.segwit_active = False + self.wallet = MiniWallet(self.nodes[0]) self.test_non_witness_transaction() self.test_v0_outputs_arent_spendable() @@ -307,7 +306,7 @@ class SegWitTest(BitcoinTestFramework): self.test_node.send_and_ping(msg_no_witness_block(block)) # make sure the block was processed txid = block.vtx[0].sha256 - self.generate(self.nodes[0], 99) # let the block mature + self.generate(self.wallet, 99) # let the block mature # Create a transaction that spends the coinbase tx = CTransaction() @@ -1999,21 +1998,13 @@ class SegWitTest(BitcoinTestFramework): def serialize(self): return serialize_with_bogus_witness(self.tx) - self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(address_type='bech32'), 5) - self.generate(self.nodes[0], 1) - unspent = next(u for u in self.nodes[0].listunspent() if u['spendable'] and u['address'].startswith('bcrt')) - - raw = self.nodes[0].createrawtransaction([{"txid": unspent['txid'], "vout": unspent['vout']}], {self.nodes[0].getnewaddress(): 1}) - tx = tx_from_hex(raw) + tx = self.wallet.create_self_transfer(from_node=self.nodes[0])['tx'] assert_raises_rpc_error(-22, "TX decode failed", self.nodes[0].decoderawtransaction, hexstring=serialize_with_bogus_witness(tx).hex(), iswitness=True) - with self.nodes[0].assert_debug_log(['Superfluous witness record']): + with self.nodes[0].assert_debug_log(['Unknown transaction optional data']): self.test_node.send_and_ping(msg_bogus_tx(tx)) - raw = self.nodes[0].signrawtransactionwithwallet(raw) - assert raw['complete'] - raw = raw['hex'] - tx = tx_from_hex(raw) + tx.wit.vtxinwit = [] # drop witness assert_raises_rpc_error(-22, "TX decode failed", self.nodes[0].decoderawtransaction, hexstring=serialize_with_bogus_witness(tx).hex(), iswitness=True) - with self.nodes[0].assert_debug_log(['Unknown transaction optional data']): + with self.nodes[0].assert_debug_log(['Superfluous witness record']): self.test_node.send_and_ping(msg_bogus_tx(tx)) @subtest diff --git a/test/functional/p2p_unrequested_blocks.py b/test/functional/p2p_unrequested_blocks.py index 9c4e1dd1b1..76d9b045ce 100755 --- a/test/functional/p2p_unrequested_blocks.py +++ b/test/functional/p2p_unrequested_blocks.py @@ -257,16 +257,11 @@ class AcceptBlockTest(BitcoinTestFramework): test_node.send_message(msg_block(block_291)) # At this point we've sent an obviously-bogus block, wait for full processing - # without assuming whether we will be disconnected or not - try: - # Only wait a short while so the test doesn't take forever if we do get - # disconnected - test_node.sync_with_ping(timeout=1) - except AssertionError: - test_node.wait_for_disconnect() - - self.nodes[0].disconnect_p2ps() - test_node = self.nodes[0].add_p2p_connection(P2PInterface()) + # and assume disconnection + test_node.wait_for_disconnect() + + self.nodes[0].disconnect_p2ps() + test_node = self.nodes[0].add_p2p_connection(P2PInterface()) # We should have failed reorg and switched back to 290 (but have block 291) assert_equal(self.nodes[0].getblockcount(), 290) diff --git a/test/functional/rpc_blockchain.py b/test/functional/rpc_blockchain.py index 4dd4899f74..193bd3f1cd 100755 --- a/test/functional/rpc_blockchain.py +++ b/test/functional/rpc_blockchain.py @@ -6,6 +6,7 @@ Test the following RPCs: - getblockchaininfo + - getdeploymentinfo - getchaintxstats - gettxoutsetinfo - getblockheader @@ -68,7 +69,14 @@ class BlockchainTest(BitcoinTestFramework): self.wallet = MiniWallet(self.nodes[0]) self.mine_chain() self._test_max_future_block_time() - self.restart_node(0, extra_args=['-stopatheight=207', '-prune=1']) # Set extra args with pruning after rescan is complete + self.restart_node( + 0, + extra_args=[ + "-stopatheight=207", + "-checkblocks=-1", # Check all blocks + "-prune=1", # Set pruning after rescan is complete + ], + ) self._test_getblockchaininfo() self._test_getchaintxstats() @@ -79,6 +87,7 @@ class BlockchainTest(BitcoinTestFramework): self._test_stopatheight() self._test_waitforblockheight() self._test_getblock() + self._test_getdeploymentinfo() assert self.nodes[0].verifychain(4, 0) def mine_chain(self): @@ -115,7 +124,6 @@ class BlockchainTest(BitcoinTestFramework): 'mediantime', 'pruned', 'size_on_disk', - 'softforks', 'time', 'verificationprogress', 'warnings', @@ -159,11 +167,6 @@ class BlockchainTest(BitcoinTestFramework): self.start_node(0, extra_args=[ '-stopatheight=207', '-prune=550', - '-testactivationheight=bip34@2', - '-testactivationheight=dersig@3', - '-testactivationheight=cltv@4', - '-testactivationheight=csv@5', - '-testactivationheight=segwit@6', ]) res = self.nodes[0].getblockchaininfo() @@ -177,7 +180,13 @@ class BlockchainTest(BitcoinTestFramework): assert_equal(res['prune_target_size'], 576716800) assert_greater_than(res['size_on_disk'], 0) - assert_equal(res['softforks'], { + def check_signalling_deploymentinfo_result(self, gdi_result, height, blockhash, status_next): + assert height >= 144 and height <= 287 + + assert_equal(gdi_result, { + "hash": blockhash, + "height": height, + "deployments": { 'bip34': {'type': 'buried', 'active': True, 'height': 2}, 'bip66': {'type': 'buried', 'active': True, 'height': 3}, 'bip65': {'type': 'buried', 'active': True, 'height': 4}, @@ -186,36 +195,65 @@ class BlockchainTest(BitcoinTestFramework): 'testdummy': { 'type': 'bip9', 'bip9': { - 'status': 'started', 'bit': 28, 'start_time': 0, 'timeout': 0x7fffffffffffffff, # testdummy does not have a timeout so is set to the max int64 value + 'min_activation_height': 0, + 'status': 'started', + 'status_next': status_next, 'since': 144, 'statistics': { 'period': 144, 'threshold': 108, - 'elapsed': HEIGHT - 143, - 'count': HEIGHT - 143, + 'elapsed': height - 143, + 'count': height - 143, 'possible': True, }, - 'min_activation_height': 0, + 'signalling': '#'*(height-143), }, 'active': False }, 'taproot': { 'type': 'bip9', 'bip9': { - 'status': 'active', 'start_time': -1, 'timeout': 9223372036854775807, - 'since': 0, 'min_activation_height': 0, + 'status': 'active', + 'status_next': 'active', + 'since': 0, }, 'height': 0, 'active': True } + } }) + def _test_getdeploymentinfo(self): + # Note: continues past -stopatheight height, so must be invoked + # after _test_stopatheight + + self.log.info("Test getdeploymentinfo") + self.stop_node(0) + self.start_node(0, extra_args=[ + '-testactivationheight=bip34@2', + '-testactivationheight=dersig@3', + '-testactivationheight=cltv@4', + '-testactivationheight=csv@5', + '-testactivationheight=segwit@6', + ]) + + gbci207 = self.nodes[0].getblockchaininfo() + self.check_signalling_deploymentinfo_result(self.nodes[0].getdeploymentinfo(), gbci207["blocks"], gbci207["bestblockhash"], "started") + + # block just prior to lock in + self.generate(self.wallet, 287 - gbci207["blocks"]) + gbci287 = self.nodes[0].getblockchaininfo() + self.check_signalling_deploymentinfo_result(self.nodes[0].getdeploymentinfo(), gbci287["blocks"], gbci287["bestblockhash"], "locked_in") + + # calling with an explicit hash works + self.check_signalling_deploymentinfo_result(self.nodes[0].getdeploymentinfo(gbci207["bestblockhash"]), gbci207["blocks"], gbci207["bestblockhash"], "started") + def _test_getchaintxstats(self): self.log.info("Test getchaintxstats") diff --git a/test/functional/rpc_createmultisig.py b/test/functional/rpc_createmultisig.py index 1a3d14100f..1695acaaa8 100755 --- a/test/functional/rpc_createmultisig.py +++ b/test/functional/rpc_createmultisig.py @@ -18,15 +18,18 @@ from test_framework.util import ( assert_equal, ) from test_framework.wallet_util import bytes_to_wif +from test_framework.wallet import ( + MiniWallet, + getnewdestination, +) class RpcCreateMultiSigTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 3 self.supports_cli = False - - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() + if self.is_bdb_compiled(): + self.requires_wallet = True def get_keys(self): self.pub = [] @@ -37,15 +40,20 @@ class RpcCreateMultiSigTest(BitcoinTestFramework): k.generate() self.pub.append(k.get_pubkey().get_bytes().hex()) self.priv.append(bytes_to_wif(k.get_bytes(), k.is_compressed)) - self.final = node2.getnewaddress() + if self.is_bdb_compiled(): + self.final = node2.getnewaddress() + else: + self.final = getnewdestination()[2] def run_test(self): node0, node1, node2 = self.nodes + self.wallet = MiniWallet(test_node=node0) - self.check_addmultisigaddress_errors() + if self.is_bdb_compiled(): + self.check_addmultisigaddress_errors() self.log.info('Generating blocks ...') - self.generate(node0, 149) + self.generate(self.wallet, 149) self.moved = 0 for self.nkeys in [3, 5]: @@ -53,14 +61,14 @@ class RpcCreateMultiSigTest(BitcoinTestFramework): for self.output_type in ["bech32", "p2sh-segwit", "legacy"]: self.get_keys() self.do_multisig() - - self.checkbalances() + if self.is_bdb_compiled(): + self.checkbalances() # Test mixed compressed and uncompressed pubkeys self.log.info('Mixed compressed and uncompressed multisigs are not allowed') - pk0 = node0.getaddressinfo(node0.getnewaddress())['pubkey'] - pk1 = node1.getaddressinfo(node1.getnewaddress())['pubkey'] - pk2 = node2.getaddressinfo(node2.getnewaddress())['pubkey'] + pk0 = getnewdestination()[0].hex() + pk1 = getnewdestination()[0].hex() + pk2 = getnewdestination()[0].hex() # decompress pk2 pk_obj = ECPubKey() @@ -68,26 +76,30 @@ class RpcCreateMultiSigTest(BitcoinTestFramework): pk_obj.compressed = False pk2 = pk_obj.get_bytes().hex() - node0.createwallet(wallet_name='wmulti0', disable_private_keys=True) - wmulti0 = node0.get_wallet_rpc('wmulti0') + if self.is_bdb_compiled(): + node0.createwallet(wallet_name='wmulti0', disable_private_keys=True) + wmulti0 = node0.get_wallet_rpc('wmulti0') # Check all permutations of keys because order matters apparently for keys in itertools.permutations([pk0, pk1, pk2]): # Results should be the same as this legacy one legacy_addr = node0.createmultisig(2, keys, 'legacy')['address'] - result = wmulti0.addmultisigaddress(2, keys, '', 'legacy') - assert_equal(legacy_addr, result['address']) - assert 'warnings' not in result + + if self.is_bdb_compiled(): + result = wmulti0.addmultisigaddress(2, keys, '', 'legacy') + assert_equal(legacy_addr, result['address']) + assert 'warnings' not in result # Generate addresses with the segwit types. These should all make legacy addresses for addr_type in ['bech32', 'p2sh-segwit']: - result = wmulti0.createmultisig(2, keys, addr_type) + result = self.nodes[0].createmultisig(2, keys, addr_type) assert_equal(legacy_addr, result['address']) assert_equal(result['warnings'], ["Unable to make chosen address type, please ensure no uncompressed public keys are present."]) - result = wmulti0.addmultisigaddress(2, keys, '', addr_type) - assert_equal(legacy_addr, result['address']) - assert_equal(result['warnings'], ["Unable to make chosen address type, please ensure no uncompressed public keys are present."]) + if self.is_bdb_compiled(): + result = wmulti0.addmultisigaddress(2, keys, '', addr_type) + assert_equal(legacy_addr, result['address']) + assert_equal(result['warnings'], ["Unable to make chosen address type, please ensure no uncompressed public keys are present."]) self.log.info('Testing sortedmulti descriptors with BIP 67 test vectors') with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'data/rpc_bip67.json'), encoding='utf-8') as f: @@ -126,26 +138,29 @@ class RpcCreateMultiSigTest(BitcoinTestFramework): bal0 = node0.getbalance() bal1 = node1.getbalance() bal2 = node2.getbalance() + balw = self.wallet.get_balance() height = node0.getblockchaininfo()["blocks"] assert 150 < height < 350 total = 149 * 50 + (height - 149 - 100) * 25 assert bal1 == 0 assert bal2 == self.moved - assert bal0 + bal1 + bal2 == total + assert_equal(bal0 + bal1 + bal2 + balw, total) def do_multisig(self): node0, node1, node2 = self.nodes - if 'wmulti' not in node1.listwallets(): - try: - node1.loadwallet('wmulti') - except JSONRPCException as e: - path = os.path.join(self.options.tmpdir, "node1", "regtest", "wallets", "wmulti") - if e.error['code'] == -18 and "Wallet file verification failed. Failed to load database path '{}'. Path does not exist.".format(path) in e.error['message']: - node1.createwallet(wallet_name='wmulti', disable_private_keys=True) - else: - raise - wmulti = node1.get_wallet_rpc('wmulti') + + if self.is_bdb_compiled(): + if 'wmulti' not in node1.listwallets(): + try: + node1.loadwallet('wmulti') + except JSONRPCException as e: + path = os.path.join(self.options.tmpdir, "node1", "regtest", "wallets", "wmulti") + if e.error['code'] == -18 and "Wallet file verification failed. Failed to load database path '{}'. Path does not exist.".format(path) in e.error['message']: + node1.createwallet(wallet_name='wmulti', disable_private_keys=True) + else: + raise + wmulti = node1.get_wallet_rpc('wmulti') # Construct the expected descriptor desc = 'multi({},{})'.format(self.nsigs, ','.join(self.pub)) @@ -164,17 +179,19 @@ class RpcCreateMultiSigTest(BitcoinTestFramework): if self.output_type == 'bech32': assert madd[0:4] == "bcrt" # actually a bech32 address - # compare against addmultisigaddress - msigw = wmulti.addmultisigaddress(self.nsigs, self.pub, None, self.output_type) - maddw = msigw["address"] - mredeemw = msigw["redeemScript"] - assert_equal(desc, drop_origins(msigw['descriptor'])) - # addmultisigiaddress and createmultisig work the same - assert maddw == madd - assert mredeemw == mredeem - - txid = node0.sendtoaddress(madd, 40) - + if self.is_bdb_compiled(): + # compare against addmultisigaddress + msigw = wmulti.addmultisigaddress(self.nsigs, self.pub, None, self.output_type) + maddw = msigw["address"] + mredeemw = msigw["redeemScript"] + assert_equal(desc, drop_origins(msigw['descriptor'])) + # addmultisigiaddress and createmultisig work the same + assert maddw == madd + assert mredeemw == mredeem + wmulti.unloadwallet() + + spk = bytes.fromhex(node0.validateaddress(madd)["scriptPubKey"]) + txid, _ = self.wallet.send_to(from_node=self.nodes[0], scriptPubKey=spk, amount=1300) tx = node0.getrawtransaction(txid, True) vout = [v["n"] for v in tx["vout"] if madd == v["scriptPubKey"]["address"]] assert len(vout) == 1 @@ -225,8 +242,6 @@ class RpcCreateMultiSigTest(BitcoinTestFramework): txinfo = node0.getrawtransaction(tx, True, blk) self.log.info("n/m=%d/%d %s size=%d vsize=%d weight=%d" % (self.nsigs, self.nkeys, self.output_type, txinfo["size"], txinfo["vsize"], txinfo["weight"])) - wmulti.unloadwallet() - if __name__ == '__main__': RpcCreateMultiSigTest().main() diff --git a/test/functional/rpc_decodescript.py b/test/functional/rpc_decodescript.py index 56f596d419..343cb73989 100755 --- a/test/functional/rpc_decodescript.py +++ b/test/functional/rpc_decodescript.py @@ -52,7 +52,7 @@ class DecodeScriptTest(BitcoinTestFramework): rpc_result = self.nodes[0].decodescript('5100') assert_equal('1 0', rpc_result['asm']) - # null data scriptSig - no such thing because null data scripts can not be spent. + # null data scriptSig - no such thing because null data scripts cannot be spent. # thus, no test case for that standard transaction type is here. def decodescript_script_pub_key(self): diff --git a/test/functional/rpc_dumptxoutset.py b/test/functional/rpc_dumptxoutset.py index 1721b6ffe8..672c9a53dc 100755 --- a/test/functional/rpc_dumptxoutset.py +++ b/test/functional/rpc_dumptxoutset.py @@ -37,21 +37,25 @@ class DumptxoutsetTest(BitcoinTestFramework): # Blockhash should be deterministic based on mocked time. assert_equal( out['base_hash'], - '6fd417acba2a8738b06fee43330c50d58e6a725046c3d843c8dd7e51d46d1ed6') + '09abf0e7b510f61ca6cf33bab104e9ee99b3528b371d27a2d4b39abb800fba7e') with open(str(expected_path), 'rb') as f: digest = hashlib.sha256(f.read()).hexdigest() # UTXO snapshot hash should be deterministic based on mocked time. assert_equal( - digest, '7ae82c986fa5445678d2a21453bb1c86d39e47af13da137640c2b1cf8093691c') + digest, 'b1bacb602eacf5fbc9a7c2ef6eeb0d229c04e98bdf0c2ea5929012cd0eae3830') assert_equal( - out['txoutset_hash'], 'd4b614f476b99a6e569973bf1c0120d88b1a168076f8ce25691fb41dd1cef149') + out['txoutset_hash'], '1f7e3befd45dc13ae198dfbb22869a9c5c4196f8e9ef9735831af1288033f890') assert_equal(out['nchaintx'], 101) - # Specifying a path to an existing file will fail. + # Specifying a path to an existing or invalid file will fail. assert_raises_rpc_error( -8, '{} already exists'.format(FILENAME), node.dumptxoutset, FILENAME) + invalid_path = str(Path(node.datadir) / "invalid" / "path") + assert_raises_rpc_error( + -8, "Couldn't open file {}.incomplete for writing".format(invalid_path), node.dumptxoutset, invalid_path) + if __name__ == '__main__': DumptxoutsetTest().main() diff --git a/test/functional/rpc_fundrawtransaction.py b/test/functional/rpc_fundrawtransaction.py index a8e6acea45..759e43194b 100755 --- a/test/functional/rpc_fundrawtransaction.py +++ b/test/functional/rpc_fundrawtransaction.py @@ -4,8 +4,10 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the fundrawtransaction RPC.""" + from decimal import Decimal from itertools import product +from math import ceil from test_framework.descriptors import descsum_create from test_framework.key import ECKey @@ -1003,7 +1005,7 @@ class RawTransactionsTest(BitcoinTestFramework): ext_utxo = self.nodes[0].listunspent(addresses=[addr])[0] # An external input without solving data should result in an error - raw_tx = wallet.createrawtransaction([ext_utxo], {self.nodes[0].getnewaddress(): 15}) + raw_tx = wallet.createrawtransaction([ext_utxo], {self.nodes[0].getnewaddress(): ext_utxo["amount"] / 2}) assert_raises_rpc_error(-4, "Insufficient funds", wallet.fundrawtransaction, raw_tx) # Error conditions @@ -1011,6 +1013,12 @@ class RawTransactionsTest(BitcoinTestFramework): assert_raises_rpc_error(-5, "'01234567890a0b0c0d0e0f' is not a valid public key", wallet.fundrawtransaction, raw_tx, {"solving_data": {"pubkeys":["01234567890a0b0c0d0e0f"]}}) assert_raises_rpc_error(-5, "'not a script' is not hex", wallet.fundrawtransaction, raw_tx, {"solving_data": {"scripts":["not a script"]}}) assert_raises_rpc_error(-8, "Unable to parse descriptor 'not a descriptor'", wallet.fundrawtransaction, raw_tx, {"solving_data": {"descriptors":["not a descriptor"]}}) + assert_raises_rpc_error(-8, "Invalid parameter, missing vout key", wallet.fundrawtransaction, raw_tx, {"input_weights": [{"txid": ext_utxo["txid"]}]}) + assert_raises_rpc_error(-8, "Invalid parameter, vout cannot be negative", wallet.fundrawtransaction, raw_tx, {"input_weights": [{"txid": ext_utxo["txid"], "vout": -1}]}) + assert_raises_rpc_error(-8, "Invalid parameter, missing weight key", wallet.fundrawtransaction, raw_tx, {"input_weights": [{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"]}]}) + assert_raises_rpc_error(-8, "Invalid parameter, weight cannot be less than 165", wallet.fundrawtransaction, raw_tx, {"input_weights": [{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": 164}]}) + assert_raises_rpc_error(-8, "Invalid parameter, weight cannot be less than 165", wallet.fundrawtransaction, raw_tx, {"input_weights": [{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": -1}]}) + assert_raises_rpc_error(-8, "Invalid parameter, weight cannot be greater than", wallet.fundrawtransaction, raw_tx, {"input_weights": [{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": 400001}]}) # But funding should work when the solving data is provided funded_tx = wallet.fundrawtransaction(raw_tx, {"solving_data": {"pubkeys": [addr_info['pubkey']], "scripts": [addr_info["embedded"]["scriptPubKey"]]}}) @@ -1020,10 +1028,45 @@ class RawTransactionsTest(BitcoinTestFramework): assert signed_tx['complete'] funded_tx = wallet.fundrawtransaction(raw_tx, {"solving_data": {"descriptors": [desc]}}) - signed_tx = wallet.signrawtransactionwithwallet(funded_tx['hex']) - assert not signed_tx['complete'] - signed_tx = self.nodes[0].signrawtransactionwithwallet(signed_tx['hex']) - assert signed_tx['complete'] + signed_tx1 = wallet.signrawtransactionwithwallet(funded_tx['hex']) + assert not signed_tx1['complete'] + signed_tx2 = self.nodes[0].signrawtransactionwithwallet(signed_tx1['hex']) + assert signed_tx2['complete'] + + unsigned_weight = self.nodes[0].decoderawtransaction(signed_tx1["hex"])["weight"] + signed_weight = self.nodes[0].decoderawtransaction(signed_tx2["hex"])["weight"] + # Input's weight is difference between weight of signed and unsigned, + # and the weight of stuff that didn't change (prevout, sequence, 1 byte of scriptSig) + input_weight = signed_weight - unsigned_weight + (41 * 4) + low_input_weight = input_weight // 2 + high_input_weight = input_weight * 2 + + # Funding should also work if the input weight is provided + funded_tx = wallet.fundrawtransaction(raw_tx, {"input_weights": [{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": input_weight}]}) + signed_tx = wallet.signrawtransactionwithwallet(funded_tx["hex"]) + signed_tx = self.nodes[0].signrawtransactionwithwallet(signed_tx["hex"]) + assert_equal(self.nodes[0].testmempoolaccept([signed_tx["hex"]])[0]["allowed"], True) + assert_equal(signed_tx["complete"], True) + # Reducing the weight should have a lower fee + funded_tx2 = wallet.fundrawtransaction(raw_tx, {"input_weights": [{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": low_input_weight}]}) + assert_greater_than(funded_tx["fee"], funded_tx2["fee"]) + # Increasing the weight should have a higher fee + funded_tx2 = wallet.fundrawtransaction(raw_tx, {"input_weights": [{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": high_input_weight}]}) + assert_greater_than(funded_tx2["fee"], funded_tx["fee"]) + # The provided weight should override the calculated weight when solving data is provided + funded_tx3 = wallet.fundrawtransaction(raw_tx, {"solving_data": {"descriptors": [desc]}, "input_weights": [{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": high_input_weight}]}) + assert_equal(funded_tx2["fee"], funded_tx3["fee"]) + # The feerate should be met + funded_tx4 = wallet.fundrawtransaction(raw_tx, {"input_weights": [{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": high_input_weight}], "fee_rate": 10}) + input_add_weight = high_input_weight - (41 * 4) + tx4_weight = wallet.decoderawtransaction(funded_tx4["hex"])["weight"] + input_add_weight + tx4_vsize = int(ceil(tx4_weight / 4)) + assert_fee_amount(funded_tx4["fee"], tx4_vsize, Decimal(0.0001)) + + # Funding with weight at csuint boundaries should not cause problems + funded_tx = wallet.fundrawtransaction(raw_tx, {"input_weights": [{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": 255}]}) + funded_tx = wallet.fundrawtransaction(raw_tx, {"input_weights": [{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": 65539}]}) + self.nodes[2].unloadwallet("extfund") def test_include_unsafe(self): diff --git a/test/functional/rpc_generate.py b/test/functional/rpc_generate.py index 47d7814da3..2b1dd20ea1 100755 --- a/test/functional/rpc_generate.py +++ b/test/functional/rpc_generate.py @@ -2,9 +2,10 @@ # Copyright (c) 2020-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test generate RPC.""" +"""Test generate* RPCs.""" from test_framework.test_framework import BitcoinTestFramework +from test_framework.wallet import MiniWallet from test_framework.util import ( assert_equal, assert_raises_rpc_error, @@ -16,6 +17,94 @@ class RPCGenerateTest(BitcoinTestFramework): self.num_nodes = 1 def run_test(self): + self.test_generatetoaddress() + self.test_generate() + self.test_generateblock() + + def test_generatetoaddress(self): + self.generatetoaddress(self.nodes[0], 1, 'mneYUmWYsuk7kySiURxCi3AGxrAqZxLgPZ') + assert_raises_rpc_error(-5, "Invalid address", self.generatetoaddress, self.nodes[0], 1, '3J98t1WpEZ73CNmQviecrnyiWrnqRhWNLy') + + def test_generateblock(self): + node = self.nodes[0] + miniwallet = MiniWallet(node) + miniwallet.rescan_utxos() + + self.log.info('Generate an empty block to address') + address = miniwallet.get_address() + hash = self.generateblock(node, output=address, transactions=[])['hash'] + block = node.getblock(blockhash=hash, verbose=2) + assert_equal(len(block['tx']), 1) + assert_equal(block['tx'][0]['vout'][0]['scriptPubKey']['address'], address) + + self.log.info('Generate an empty block to a descriptor') + hash = self.generateblock(node, 'addr(' + address + ')', [])['hash'] + block = node.getblock(blockhash=hash, verbosity=2) + assert_equal(len(block['tx']), 1) + assert_equal(block['tx'][0]['vout'][0]['scriptPubKey']['address'], address) + + self.log.info('Generate an empty block to a combo descriptor with compressed pubkey') + combo_key = '0279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798' + combo_address = 'bcrt1qw508d6qejxtdg4y5r3zarvary0c5xw7kygt080' + hash = self.generateblock(node, 'combo(' + combo_key + ')', [])['hash'] + block = node.getblock(hash, 2) + assert_equal(len(block['tx']), 1) + assert_equal(block['tx'][0]['vout'][0]['scriptPubKey']['address'], combo_address) + + self.log.info('Generate an empty block to a combo descriptor with uncompressed pubkey') + combo_key = '0408ef68c46d20596cc3f6ddf7c8794f71913add807f1dc55949fa805d764d191c0b7ce6894c126fce0babc6663042f3dde9b0cf76467ea315514e5a6731149c67' + combo_address = 'mkc9STceoCcjoXEXe6cm66iJbmjM6zR9B2' + hash = self.generateblock(node, 'combo(' + combo_key + ')', [])['hash'] + block = node.getblock(hash, 2) + assert_equal(len(block['tx']), 1) + assert_equal(block['tx'][0]['vout'][0]['scriptPubKey']['address'], combo_address) + + # Generate some extra mempool transactions to verify they don't get mined + for _ in range(10): + miniwallet.send_self_transfer(from_node=node) + + self.log.info('Generate block with txid') + txid = miniwallet.send_self_transfer(from_node=node)['txid'] + hash = self.generateblock(node, address, [txid])['hash'] + block = node.getblock(hash, 1) + assert_equal(len(block['tx']), 2) + assert_equal(block['tx'][1], txid) + + self.log.info('Generate block with raw tx') + rawtx = miniwallet.create_self_transfer()['hex'] + hash = self.generateblock(node, address, [rawtx])['hash'] + + block = node.getblock(hash, 1) + assert_equal(len(block['tx']), 2) + txid = block['tx'][1] + assert_equal(node.getrawtransaction(txid=txid, verbose=False, blockhash=hash), rawtx) + + self.log.info('Fail to generate block with out of order txs') + txid1 = miniwallet.send_self_transfer(from_node=node)['txid'] + utxo1 = miniwallet.get_utxo(txid=txid1) + rawtx2 = miniwallet.create_self_transfer(utxo_to_spend=utxo1)['hex'] + assert_raises_rpc_error(-25, 'TestBlockValidity failed: bad-txns-inputs-missingorspent', self.generateblock, node, address, [rawtx2, txid1]) + + self.log.info('Fail to generate block with txid not in mempool') + missing_txid = '0000000000000000000000000000000000000000000000000000000000000000' + assert_raises_rpc_error(-5, 'Transaction ' + missing_txid + ' not in mempool.', self.generateblock, node, address, [missing_txid]) + + self.log.info('Fail to generate block with invalid raw tx') + invalid_raw_tx = '0000' + assert_raises_rpc_error(-22, 'Transaction decode failed for ' + invalid_raw_tx, self.generateblock, node, address, [invalid_raw_tx]) + + self.log.info('Fail to generate block with invalid address/descriptor') + assert_raises_rpc_error(-5, 'Invalid address or descriptor', self.generateblock, node, '1234', []) + + self.log.info('Fail to generate block with a ranged descriptor') + ranged_descriptor = 'pkh(tpubD6NzVbkrYhZ4XgiXtGrdW5XDAPFCL9h7we1vwNCpn8tGbBcgfVYjXyhWo4E1xkh56hjod1RhGjxbaTLV3X4FyWuejifB9jusQ46QzG87VKp/0/*)' + assert_raises_rpc_error(-8, 'Ranged descriptor not accepted. Maybe pass through deriveaddresses first?', self.generateblock, node, ranged_descriptor, []) + + self.log.info('Fail to generate block with a descriptor missing a private key') + child_descriptor = 'pkh(tpubD6NzVbkrYhZ4XgiXtGrdW5XDAPFCL9h7we1vwNCpn8tGbBcgfVYjXyhWo4E1xkh56hjod1RhGjxbaTLV3X4FyWuejifB9jusQ46QzG87VKp/0\'/0)' + assert_raises_rpc_error(-5, 'Cannot derive script without private keys', self.generateblock, node, child_descriptor, []) + + def test_generate(self): message = ( "generate\n\n" "has been replaced by the -generate " diff --git a/test/functional/rpc_generateblock.py b/test/functional/rpc_generateblock.py deleted file mode 100755 index 7eeb745817..0000000000 --- a/test/functional/rpc_generateblock.py +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2020-2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -'''Test generateblock rpc. -''' - -from test_framework.test_framework import BitcoinTestFramework -from test_framework.wallet import MiniWallet -from test_framework.util import ( - assert_equal, - assert_raises_rpc_error, -) - - -class GenerateBlockTest(BitcoinTestFramework): - def set_test_params(self): - self.num_nodes = 1 - - def run_test(self): - node = self.nodes[0] - miniwallet = MiniWallet(node) - miniwallet.rescan_utxos() - - self.log.info('Generate an empty block to address') - address = miniwallet.get_address() - hash = self.generateblock(node, output=address, transactions=[])['hash'] - block = node.getblock(blockhash=hash, verbose=2) - assert_equal(len(block['tx']), 1) - assert_equal(block['tx'][0]['vout'][0]['scriptPubKey']['address'], address) - - self.log.info('Generate an empty block to a descriptor') - hash = self.generateblock(node, 'addr(' + address + ')', [])['hash'] - block = node.getblock(blockhash=hash, verbosity=2) - assert_equal(len(block['tx']), 1) - assert_equal(block['tx'][0]['vout'][0]['scriptPubKey']['address'], address) - - self.log.info('Generate an empty block to a combo descriptor with compressed pubkey') - combo_key = '0279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798' - combo_address = 'bcrt1qw508d6qejxtdg4y5r3zarvary0c5xw7kygt080' - hash = self.generateblock(node, 'combo(' + combo_key + ')', [])['hash'] - block = node.getblock(hash, 2) - assert_equal(len(block['tx']), 1) - assert_equal(block['tx'][0]['vout'][0]['scriptPubKey']['address'], combo_address) - - self.log.info('Generate an empty block to a combo descriptor with uncompressed pubkey') - combo_key = '0408ef68c46d20596cc3f6ddf7c8794f71913add807f1dc55949fa805d764d191c0b7ce6894c126fce0babc6663042f3dde9b0cf76467ea315514e5a6731149c67' - combo_address = 'mkc9STceoCcjoXEXe6cm66iJbmjM6zR9B2' - hash = self.generateblock(node, 'combo(' + combo_key + ')', [])['hash'] - block = node.getblock(hash, 2) - assert_equal(len(block['tx']), 1) - assert_equal(block['tx'][0]['vout'][0]['scriptPubKey']['address'], combo_address) - - # Generate some extra mempool transactions to verify they don't get mined - for _ in range(10): - miniwallet.send_self_transfer(from_node=node) - - self.log.info('Generate block with txid') - txid = miniwallet.send_self_transfer(from_node=node)['txid'] - hash = self.generateblock(node, address, [txid])['hash'] - block = node.getblock(hash, 1) - assert_equal(len(block['tx']), 2) - assert_equal(block['tx'][1], txid) - - self.log.info('Generate block with raw tx') - rawtx = miniwallet.create_self_transfer()['hex'] - hash = self.generateblock(node, address, [rawtx])['hash'] - - block = node.getblock(hash, 1) - assert_equal(len(block['tx']), 2) - txid = block['tx'][1] - assert_equal(node.getrawtransaction(txid=txid, verbose=False, blockhash=hash), rawtx) - - self.log.info('Fail to generate block with out of order txs') - txid1 = miniwallet.send_self_transfer(from_node=node)['txid'] - utxo1 = miniwallet.get_utxo(txid=txid1) - rawtx2 = miniwallet.create_self_transfer(utxo_to_spend=utxo1)['hex'] - assert_raises_rpc_error(-25, 'TestBlockValidity failed: bad-txns-inputs-missingorspent', self.generateblock, node, address, [rawtx2, txid1]) - - self.log.info('Fail to generate block with txid not in mempool') - missing_txid = '0000000000000000000000000000000000000000000000000000000000000000' - assert_raises_rpc_error(-5, 'Transaction ' + missing_txid + ' not in mempool.', self.generateblock, node, address, [missing_txid]) - - self.log.info('Fail to generate block with invalid raw tx') - invalid_raw_tx = '0000' - assert_raises_rpc_error(-22, 'Transaction decode failed for ' + invalid_raw_tx, self.generateblock, node, address, [invalid_raw_tx]) - - self.log.info('Fail to generate block with invalid address/descriptor') - assert_raises_rpc_error(-5, 'Invalid address or descriptor', self.generateblock, node, '1234', []) - - self.log.info('Fail to generate block with a ranged descriptor') - ranged_descriptor = 'pkh(tpubD6NzVbkrYhZ4XgiXtGrdW5XDAPFCL9h7we1vwNCpn8tGbBcgfVYjXyhWo4E1xkh56hjod1RhGjxbaTLV3X4FyWuejifB9jusQ46QzG87VKp/0/*)' - assert_raises_rpc_error(-8, 'Ranged descriptor not accepted. Maybe pass through deriveaddresses first?', self.generateblock, node, ranged_descriptor, []) - - self.log.info('Fail to generate block with a descriptor missing a private key') - child_descriptor = 'pkh(tpubD6NzVbkrYhZ4XgiXtGrdW5XDAPFCL9h7we1vwNCpn8tGbBcgfVYjXyhWo4E1xkh56hjod1RhGjxbaTLV3X4FyWuejifB9jusQ46QzG87VKp/0\'/0)' - assert_raises_rpc_error(-5, 'Cannot derive script without private keys', self.generateblock, node, child_descriptor, []) - -if __name__ == '__main__': - GenerateBlockTest().main() diff --git a/test/functional/rpc_getblockfrompeer.py b/test/functional/rpc_getblockfrompeer.py index effcebe854..b65322d920 100755 --- a/test/functional/rpc_getblockfrompeer.py +++ b/test/functional/rpc_getblockfrompeer.py @@ -40,12 +40,8 @@ class GetBlockFromPeerTest(BitcoinTestFramework): self.sync_blocks() self.log.info("Node 0 should only have the header for node 1's block 3") - for x in self.nodes[0].getchaintips(): - if x['hash'] == short_tip: - assert_equal(x['status'], "headers-only") - break - else: - raise AssertionError("short tip not synced") + x = next(filter(lambda x: x['hash'] == short_tip, self.nodes[0].getchaintips())) + assert_equal(x['status'], "headers-only") assert_raises_rpc_error(-1, "Block not found on disk", self.nodes[0].getblock, short_tip) self.log.info("Fetch block from node 1") @@ -60,17 +56,15 @@ class GetBlockFromPeerTest(BitcoinTestFramework): assert_raises_rpc_error(-1, "Block header missing", self.nodes[0].getblockfrompeer, "00" * 32, 0) self.log.info("Non-existent peer generates error") - assert_raises_rpc_error(-1, f"Peer nodeid {peer_0_peer_1_id + 1} does not exist", self.nodes[0].getblockfrompeer, short_tip, peer_0_peer_1_id + 1) + assert_raises_rpc_error(-1, "Peer does not exist", self.nodes[0].getblockfrompeer, short_tip, peer_0_peer_1_id + 1) self.log.info("Successful fetch") result = self.nodes[0].getblockfrompeer(short_tip, peer_0_peer_1_id) self.wait_until(lambda: self.check_for_block(short_tip), timeout=1) - assert(not "warnings" in result) + assert_equal(result, {}) self.log.info("Don't fetch blocks we already have") - result = self.nodes[0].getblockfrompeer(short_tip, peer_0_peer_1_id) - assert("warnings" in result) - assert_equal(result["warnings"], "Block already downloaded") + assert_raises_rpc_error(-1, "Block already downloaded", self.nodes[0].getblockfrompeer, short_tip, peer_0_peer_1_id) if __name__ == '__main__': GetBlockFromPeerTest().main() diff --git a/test/functional/rpc_help.py b/test/functional/rpc_help.py index ccb380e25b..3b6413d4a6 100755 --- a/test/functional/rpc_help.py +++ b/test/functional/rpc_help.py @@ -100,7 +100,7 @@ class HelpRpcTest(BitcoinTestFramework): # command titles titles = [line[3:-3] for line in node.help().splitlines() if line.startswith('==')] - components = ['Blockchain', 'Control', 'Generating', 'Mining', 'Network', 'Rawtransactions', 'Util'] + components = ['Blockchain', 'Control', 'Mining', 'Network', 'Rawtransactions', 'Util'] if self.is_wallet_compiled(): components.append('Wallet') diff --git a/test/functional/rpc_mempool_entry_fee_fields_deprecation.py b/test/functional/rpc_mempool_entry_fee_fields_deprecation.py deleted file mode 100755 index 82761ff7c8..0000000000 --- a/test/functional/rpc_mempool_entry_fee_fields_deprecation.py +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test deprecation of fee fields from top level mempool entry object""" - -from test_framework.blocktools import COIN -from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal -from test_framework.wallet import MiniWallet - - -def assertions_helper(new_object, deprecated_object, deprecated_fields): - for field in deprecated_fields: - assert field in deprecated_object - assert field not in new_object - - -class MempoolFeeFieldsDeprecationTest(BitcoinTestFramework): - def set_test_params(self): - self.num_nodes = 2 - self.extra_args = [[], ["-deprecatedrpc=fees"]] - - def run_test(self): - # we get spendable outputs from the premined chain starting - # at block 76. see BitcoinTestFramework._initialize_chain() for details - self.wallet = MiniWallet(self.nodes[0]) - self.wallet.rescan_utxos() - - # we create the tx on the first node and wait until it syncs to node_deprecated - # thus, any differences must be coming from getmempoolentry or getrawmempool - tx = self.wallet.send_self_transfer(from_node=self.nodes[0]) - self.nodes[1].sendrawtransaction(tx["hex"]) - - deprecated_fields = ["ancestorfees", "descendantfees", "modifiedfee", "fee"] - self.test_getmempoolentry(tx["txid"], deprecated_fields) - self.test_getrawmempool(tx["txid"], deprecated_fields) - self.test_deprecated_fields_match(tx["txid"]) - - def test_getmempoolentry(self, txid, deprecated_fields): - - self.log.info("Test getmempoolentry rpc") - entry = self.nodes[0].getmempoolentry(txid) - deprecated_entry = self.nodes[1].getmempoolentry(txid) - assertions_helper(entry, deprecated_entry, deprecated_fields) - - def test_getrawmempool(self, txid, deprecated_fields): - - self.log.info("Test getrawmempool rpc") - entry = self.nodes[0].getrawmempool(verbose=True)[txid] - deprecated_entry = self.nodes[1].getrawmempool(verbose=True)[txid] - assertions_helper(entry, deprecated_entry, deprecated_fields) - - def test_deprecated_fields_match(self, txid): - - self.log.info("Test deprecated fee fields match new fees object") - entry = self.nodes[0].getmempoolentry(txid) - deprecated_entry = self.nodes[1].getmempoolentry(txid) - - assert_equal(deprecated_entry["fee"], entry["fees"]["base"]) - assert_equal(deprecated_entry["modifiedfee"], entry["fees"]["modified"]) - assert_equal(deprecated_entry["descendantfees"], entry["fees"]["descendant"] * COIN) - assert_equal(deprecated_entry["ancestorfees"], entry["fees"]["ancestor"] * COIN) - - -if __name__ == "__main__": - MempoolFeeFieldsDeprecationTest().main() diff --git a/test/functional/rpc_mempool_info.py b/test/functional/rpc_mempool_info.py new file mode 100755 index 0000000000..cd7a48d387 --- /dev/null +++ b/test/functional/rpc_mempool_info.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +# Copyright (c) 2014-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test RPCs that retrieve information from the mempool.""" + +from test_framework.blocktools import COINBASE_MATURITY +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + assert_raises_rpc_error, +) +from test_framework.wallet import MiniWallet + + +class RPCMempoolInfoTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + + def run_test(self): + self.wallet = MiniWallet(self.nodes[0]) + self.generate(self.wallet, COINBASE_MATURITY + 1) + self.wallet.rescan_utxos() + confirmed_utxo = self.wallet.get_utxo() + + # Create a tree of unconfirmed transactions in the mempool: + # txA + # / \ + # / \ + # / \ + # / \ + # / \ + # txB txC + # / \ / \ + # / \ / \ + # txD txE txF txG + # \ / + # \ / + # txH + + def create_tx(**kwargs): + return self.wallet.send_self_transfer_multi( + from_node=self.nodes[0], + **kwargs, + ) + + txA = create_tx(utxos_to_spend=[confirmed_utxo], num_outputs=2) + txB = create_tx(utxos_to_spend=[txA["new_utxos"][0]], num_outputs=2) + txC = create_tx(utxos_to_spend=[txA["new_utxos"][1]], num_outputs=2) + txD = create_tx(utxos_to_spend=[txB["new_utxos"][0]], num_outputs=1) + txE = create_tx(utxos_to_spend=[txB["new_utxos"][1]], num_outputs=1) + txF = create_tx(utxos_to_spend=[txC["new_utxos"][0]], num_outputs=2) + txG = create_tx(utxos_to_spend=[txC["new_utxos"][1]], num_outputs=1) + txH = create_tx(utxos_to_spend=[txE["new_utxos"][0],txF["new_utxos"][0]], num_outputs=1) + txidA, txidB, txidC, txidD, txidE, txidF, txidG, txidH = [ + tx["txid"] for tx in [txA, txB, txC, txD, txE, txF, txG, txH] + ] + + mempool = self.nodes[0].getrawmempool() + assert_equal(len(mempool), 8) + for txid in [txidA, txidB, txidC, txidD, txidE, txidF, txidG, txidH]: + assert_equal(txid in mempool, True) + + self.log.info("Find transactions spending outputs") + result = self.nodes[0].gettxspendingprevout([ {'txid' : confirmed_utxo['txid'], 'vout' : 0}, {'txid' : txidA, 'vout' : 1} ]) + assert_equal(result, [ {'txid' : confirmed_utxo['txid'], 'vout' : 0, 'spendingtxid' : txidA}, {'txid' : txidA, 'vout' : 1, 'spendingtxid' : txidC} ]) + + self.log.info("Find transaction spending multiple outputs") + result = self.nodes[0].gettxspendingprevout([ {'txid' : txidE, 'vout' : 0}, {'txid' : txidF, 'vout' : 0} ]) + assert_equal(result, [ {'txid' : txidE, 'vout' : 0, 'spendingtxid' : txidH}, {'txid' : txidF, 'vout' : 0, 'spendingtxid' : txidH} ]) + + self.log.info("Find no transaction when output is unspent") + result = self.nodes[0].gettxspendingprevout([ {'txid' : txidH, 'vout' : 0} ]) + assert_equal(result, [ {'txid' : txidH, 'vout' : 0} ]) + result = self.nodes[0].gettxspendingprevout([ {'txid' : txidA, 'vout' : 5} ]) + assert_equal(result, [ {'txid' : txidA, 'vout' : 5} ]) + + self.log.info("Mixed spent and unspent outputs") + result = self.nodes[0].gettxspendingprevout([ {'txid' : txidB, 'vout' : 0}, {'txid' : txidG, 'vout' : 3} ]) + assert_equal(result, [ {'txid' : txidB, 'vout' : 0, 'spendingtxid' : txidD}, {'txid' : txidG, 'vout' : 3} ]) + + self.log.info("Unknown input fields") + assert_raises_rpc_error(-3, "Unexpected key unknown", self.nodes[0].gettxspendingprevout, [{'txid' : txidC, 'vout' : 1, 'unknown' : 42}]) + + self.log.info("Invalid vout provided") + assert_raises_rpc_error(-8, "Invalid parameter, vout cannot be negative", self.nodes[0].gettxspendingprevout, [{'txid' : txidA, 'vout' : -1}]) + + self.log.info("Invalid txid provided") + assert_raises_rpc_error(-3, "Expected type string for txid, got number", self.nodes[0].gettxspendingprevout, [{'txid' : 42, 'vout' : 0}]) + + self.log.info("Missing outputs") + assert_raises_rpc_error(-8, "Invalid parameter, outputs are missing", self.nodes[0].gettxspendingprevout, []) + + self.log.info("Missing vout") + assert_raises_rpc_error(-3, "Missing vout", self.nodes[0].gettxspendingprevout, [{'txid' : txidA}]) + + self.log.info("Missing txid") + assert_raises_rpc_error(-3, "Missing txid", self.nodes[0].gettxspendingprevout, [{'vout' : 3}]) + + +if __name__ == '__main__': + RPCMempoolInfoTest().main() diff --git a/test/functional/rpc_misc.py b/test/functional/rpc_misc.py index 2f1796d7cc..f6ee6a5215 100755 --- a/test/functional/rpc_misc.py +++ b/test/functional/rpc_misc.py @@ -27,7 +27,7 @@ class RpcMiscTest(BitcoinTestFramework): self.log.info("test CHECK_NONFATAL") assert_raises_rpc_error( -1, - 'Internal bug detected: \'request.params[9].get_str() != "trigger_internal_bug"\'', + 'Internal bug detected: "request.params[9].get_str() != "trigger_internal_bug""', lambda: node.echo(arg9='trigger_internal_bug'), ) @@ -56,9 +56,6 @@ class RpcMiscTest(BitcoinTestFramework): self.log.info("test logging rpc and help") - # Test logging RPC returns the expected number of logging categories. - assert_equal(len(node.logging()), 27) - # Test toggling a logging category on/off/on with the logging RPC. assert_equal(node.logging()['qt'], True) node.logging(exclude=['qt']) diff --git a/test/functional/rpc_net.py b/test/functional/rpc_net.py index 81a3cfee97..ad8ba06824 100755 --- a/test/functional/rpc_net.py +++ b/test/functional/rpc_net.py @@ -257,6 +257,10 @@ class NetTest(BitcoinTestFramework): assert_equal(node.addpeeraddress(address="", port=8333), {"success": False}) assert_equal(node.getnodeaddresses(count=0), []) + self.log.debug("Test that adding an address with invalid port fails") + assert_raises_rpc_error(-1, "JSON integer out of range", self.nodes[0].addpeeraddress, address="1.2.3.4", port=-1) + assert_raises_rpc_error(-1, "JSON integer out of range", self.nodes[0].addpeeraddress,address="1.2.3.4", port=65536) + self.log.debug("Test that adding a valid address to the tried table succeeds") assert_equal(node.addpeeraddress(address="1.2.3.4", tried=True, port=8333), {"success": True}) with node.assert_debug_log(expected_msgs=["CheckAddrman: new 0, tried 1, total 1 started"]): diff --git a/test/functional/rpc_psbt.py b/test/functional/rpc_psbt.py index 153a201e95..444e56610e 100755 --- a/test/functional/rpc_psbt.py +++ b/test/functional/rpc_psbt.py @@ -10,6 +10,10 @@ from itertools import product from test_framework.descriptors import descsum_create from test_framework.key import ECKey +from test_framework.messages import ( + ser_compact_size, + WITNESS_SCALE_FACTOR, +) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_approx, @@ -606,13 +610,17 @@ class PSBTTest(BitcoinTestFramework): assert_raises_rpc_error(-25, 'Inputs missing or spent', self.nodes[0].walletprocesspsbt, 'cHNidP8BAJoCAAAAAkvEW8NnDtdNtDpsmze+Ht2LH35IJcKv00jKAlUs21RrAwAAAAD/////S8Rbw2cO1020OmybN74e3Ysffkglwq/TSMoCVSzbVGsBAAAAAP7///8CwLYClQAAAAAWABSNJKzjaUb3uOxixsvh1GGE3fW7zQD5ApUAAAAAFgAUKNw0x8HRctAgmvoevm4u1SbN7XIAAAAAAAEAnQIAAAACczMa321tVHuN4GKWKRncycI22aX3uXgwSFUKM2orjRsBAAAAAP7///9zMxrfbW1Ue43gYpYpGdzJwjbZpfe5eDBIVQozaiuNGwAAAAAA/v///wIA+QKVAAAAABl2qRT9zXUVA8Ls5iVqynLHe5/vSe1XyYisQM0ClQAAAAAWABRmWQUcjSjghQ8/uH4Bn/zkakwLtAAAAAAAAQEfQM0ClQAAAAAWABRmWQUcjSjghQ8/uH4Bn/zkakwLtAAAAA==') - # Test that we can fund psbts with external inputs specified + self.log.info("Test that we can fund psbts with external inputs specified") + eckey = ECKey() eckey.generate() privkey = bytes_to_wif(eckey.get_bytes()) - # Make a weird but signable script. sh(pkh()) descriptor accomplishes this - desc = descsum_create("sh(pkh({}))".format(privkey)) + self.nodes[1].createwallet("extfund") + wallet = self.nodes[1].get_wallet_rpc("extfund") + + # Make a weird but signable script. sh(wsh(pkh())) descriptor accomplishes this + desc = descsum_create("sh(wsh(pkh({})))".format(privkey)) if self.options.descriptors: res = self.nodes[0].importdescriptors([{"desc": desc, "timestamp": "now"}]) else: @@ -622,26 +630,98 @@ class PSBTTest(BitcoinTestFramework): addr_info = self.nodes[0].getaddressinfo(addr) self.nodes[0].sendtoaddress(addr, 10) + self.nodes[0].sendtoaddress(wallet.getnewaddress(), 10) self.generate(self.nodes[0], 6) ext_utxo = self.nodes[0].listunspent(addresses=[addr])[0] # An external input without solving data should result in an error - assert_raises_rpc_error(-4, "Insufficient funds", self.nodes[1].walletcreatefundedpsbt, [ext_utxo], {self.nodes[0].getnewaddress(): 10 + ext_utxo['amount']}, 0, {'add_inputs': True}) + assert_raises_rpc_error(-4, "Insufficient funds", wallet.walletcreatefundedpsbt, [ext_utxo], {self.nodes[0].getnewaddress(): 15}) # But funding should work when the solving data is provided - psbt = self.nodes[1].walletcreatefundedpsbt([ext_utxo], {self.nodes[0].getnewaddress(): 15}, 0, {'add_inputs': True, "solving_data": {"pubkeys": [addr_info['pubkey']], "scripts": [addr_info["embedded"]["scriptPubKey"]]}}) - signed = self.nodes[1].walletprocesspsbt(psbt['psbt']) + psbt = wallet.walletcreatefundedpsbt([ext_utxo], {self.nodes[0].getnewaddress(): 15}, 0, {"add_inputs": True, "solving_data": {"pubkeys": [addr_info['pubkey']], "scripts": [addr_info["embedded"]["scriptPubKey"], addr_info["embedded"]["embedded"]["scriptPubKey"]]}}) + signed = wallet.walletprocesspsbt(psbt['psbt']) assert not signed['complete'] signed = self.nodes[0].walletprocesspsbt(signed['psbt']) assert signed['complete'] self.nodes[0].finalizepsbt(signed['psbt']) - psbt = self.nodes[1].walletcreatefundedpsbt([ext_utxo], {self.nodes[0].getnewaddress(): 15}, 0, {'add_inputs': True, "solving_data":{"descriptors": [desc]}}) - signed = self.nodes[1].walletprocesspsbt(psbt['psbt']) + psbt = wallet.walletcreatefundedpsbt([ext_utxo], {self.nodes[0].getnewaddress(): 15}, 0, {"add_inputs": True, "solving_data":{"descriptors": [desc]}}) + signed = wallet.walletprocesspsbt(psbt['psbt']) assert not signed['complete'] signed = self.nodes[0].walletprocesspsbt(signed['psbt']) assert signed['complete'] - self.nodes[0].finalizepsbt(signed['psbt']) + final = self.nodes[0].finalizepsbt(signed['psbt'], False) + + dec = self.nodes[0].decodepsbt(signed["psbt"]) + for i, txin in enumerate(dec["tx"]["vin"]): + if txin["txid"] == ext_utxo["txid"] and txin["vout"] == ext_utxo["vout"]: + input_idx = i + break + psbt_in = dec["inputs"][input_idx] + # Calculate the input weight + # (prevout + sequence + length of scriptSig + scriptsig + 1 byte buffer) * WITNESS_SCALE_FACTOR + num scriptWitness stack items + (length of stack item + stack item) * N stack items + 1 byte buffer + len_scriptsig = len(psbt_in["final_scriptSig"]["hex"]) // 2 if "final_scriptSig" in psbt_in else 0 + len_scriptsig += len(ser_compact_size(len_scriptsig)) + 1 + len_scriptwitness = (sum([(len(x) // 2) + len(ser_compact_size(len(x) // 2)) for x in psbt_in["final_scriptwitness"]]) + len(psbt_in["final_scriptwitness"]) + 1) if "final_scriptwitness" in psbt_in else 0 + input_weight = ((40 + len_scriptsig) * WITNESS_SCALE_FACTOR) + len_scriptwitness + low_input_weight = input_weight // 2 + high_input_weight = input_weight * 2 + + # Input weight error conditions + assert_raises_rpc_error( + -8, + "Input weights should be specified in inputs rather than in options.", + wallet.walletcreatefundedpsbt, + inputs=[ext_utxo], + outputs={self.nodes[0].getnewaddress(): 15}, + options={"input_weights": [{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": 1000}]} + ) + + # Funding should also work if the input weight is provided + psbt = wallet.walletcreatefundedpsbt( + inputs=[{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": input_weight}], + outputs={self.nodes[0].getnewaddress(): 15}, + options={"add_inputs": True} + ) + signed = wallet.walletprocesspsbt(psbt["psbt"]) + signed = self.nodes[0].walletprocesspsbt(signed["psbt"]) + final = self.nodes[0].finalizepsbt(signed["psbt"]) + assert self.nodes[0].testmempoolaccept([final["hex"]])[0]["allowed"] + # Reducing the weight should have a lower fee + psbt2 = wallet.walletcreatefundedpsbt( + inputs=[{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": low_input_weight}], + outputs={self.nodes[0].getnewaddress(): 15}, + options={"add_inputs": True} + ) + assert_greater_than(psbt["fee"], psbt2["fee"]) + # Increasing the weight should have a higher fee + psbt2 = wallet.walletcreatefundedpsbt( + inputs=[{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": high_input_weight}], + outputs={self.nodes[0].getnewaddress(): 15}, + options={"add_inputs": True} + ) + assert_greater_than(psbt2["fee"], psbt["fee"]) + # The provided weight should override the calculated weight when solving data is provided + psbt3 = wallet.walletcreatefundedpsbt( + inputs=[{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": high_input_weight}], + outputs={self.nodes[0].getnewaddress(): 15}, + options={'add_inputs': True, "solving_data":{"descriptors": [desc]}} + ) + assert_equal(psbt2["fee"], psbt3["fee"]) + + # Import the external utxo descriptor so that we can sign for it from the test wallet + if self.options.descriptors: + res = wallet.importdescriptors([{"desc": desc, "timestamp": "now"}]) + else: + res = wallet.importmulti([{"desc": desc, "timestamp": "now"}]) + assert res[0]["success"] + # The provided weight should override the calculated weight for a wallet input + psbt3 = wallet.walletcreatefundedpsbt( + inputs=[{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": high_input_weight}], + outputs={self.nodes[0].getnewaddress(): 15}, + options={"add_inputs": True} + ) + assert_equal(psbt2["fee"], psbt3["fee"]) if __name__ == '__main__': PSBTTest().main() diff --git a/test/functional/rpc_rawtransaction.py b/test/functional/rpc_rawtransaction.py index 96691b2686..fecb8310b9 100755 --- a/test/functional/rpc_rawtransaction.py +++ b/test/functional/rpc_rawtransaction.py @@ -24,7 +24,10 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, assert_raises_rpc_error, - find_vout_for_address, +) +from test_framework.wallet import ( + getnewdestination, + MiniWallet, ) @@ -52,72 +55,71 @@ class multidict(dict): class RawTransactionsTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True - self.num_nodes = 4 + self.num_nodes = 3 self.extra_args = [ ["-txindex"], ["-txindex"], - ["-txindex"], [], ] # whitelist all peers to speed up tx relay / mempool sync for args in self.extra_args: args.append("-whitelist=noban@127.0.0.1") + self.requires_wallet = self.is_specified_wallet_compiled() self.supports_cli = False - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() - def setup_network(self): super().setup_network() self.connect_nodes(0, 2) def run_test(self): + self.wallet = MiniWallet(self.nodes[0]) self.log.info("Prepare some coins for multiple *rawtransaction commands") - self.generate(self.nodes[2], 1) + self.generate(self.wallet, 10) self.generate(self.nodes[0], COINBASE_MATURITY + 1) - for amount in [1.5, 1.0, 5.0]: - self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), amount) - self.sync_all() - self.generate(self.nodes[0], 5) self.getrawtransaction_tests() self.createrawtransaction_tests() - self.signrawtransactionwithwallet_tests() self.sendrawtransaction_tests() self.sendrawtransaction_testmempoolaccept_tests() self.decoderawtransaction_tests() self.transaction_version_number_tests() - if not self.options.descriptors: + if self.requires_wallet and not self.options.descriptors: self.raw_multisig_transaction_legacy_tests() def getrawtransaction_tests(self): - addr = self.nodes[1].getnewaddress() - txid = self.nodes[0].sendtoaddress(addr, 10) - self.generate(self.nodes[0], 1) - vout = find_vout_for_address(self.nodes[1], txid, addr) - rawTx = self.nodes[1].createrawtransaction([{'txid': txid, 'vout': vout}], {self.nodes[1].getnewaddress(): 9.999}) - rawTxSigned = self.nodes[1].signrawtransactionwithwallet(rawTx) - txId = self.nodes[1].sendrawtransaction(rawTxSigned['hex']) + tx = self.wallet.send_self_transfer(from_node=self.nodes[0]) self.generate(self.nodes[0], 1) + txId = tx['txid'] + err_msg = ( + "No such mempool transaction. Use -txindex or provide a block hash to enable" + " blockchain transaction queries. Use gettransaction for wallet transactions." + ) - for n in [0, 3]: + for n in [0, 2]: self.log.info(f"Test getrawtransaction {'with' if n == 0 else 'without'} -txindex") - # 1. valid parameters - only supply txid - assert_equal(self.nodes[n].getrawtransaction(txId), rawTxSigned['hex']) - # 2. valid parameters - supply txid and 0 for non-verbose - assert_equal(self.nodes[n].getrawtransaction(txId, 0), rawTxSigned['hex']) + if n == 0: + # With -txindex. + # 1. valid parameters - only supply txid + assert_equal(self.nodes[n].getrawtransaction(txId), tx['hex']) - # 3. valid parameters - supply txid and False for non-verbose - assert_equal(self.nodes[n].getrawtransaction(txId, False), rawTxSigned['hex']) + # 2. valid parameters - supply txid and 0 for non-verbose + assert_equal(self.nodes[n].getrawtransaction(txId, 0), tx['hex']) - # 4. valid parameters - supply txid and 1 for verbose. - # We only check the "hex" field of the output so we don't need to update this test every time the output format changes. - assert_equal(self.nodes[n].getrawtransaction(txId, 1)["hex"], rawTxSigned['hex']) + # 3. valid parameters - supply txid and False for non-verbose + assert_equal(self.nodes[n].getrawtransaction(txId, False), tx['hex']) - # 5. valid parameters - supply txid and True for non-verbose - assert_equal(self.nodes[n].getrawtransaction(txId, True)["hex"], rawTxSigned['hex']) + # 4. valid parameters - supply txid and 1 for verbose. + # We only check the "hex" field of the output so we don't need to update this test every time the output format changes. + assert_equal(self.nodes[n].getrawtransaction(txId, 1)["hex"], tx['hex']) + + # 5. valid parameters - supply txid and True for non-verbose + assert_equal(self.nodes[n].getrawtransaction(txId, True)["hex"], tx['hex']) + else: + # Without -txindex, expect to raise. + for verbose in [None, 0, False, 1, True]: + assert_raises_rpc_error(-5, err_msg, self.nodes[n].getrawtransaction, txId, verbose) # 6. invalid parameters - supply txid and invalid boolean values (strings) for verbose for value in ["True", "False"]: @@ -130,9 +132,9 @@ class RawTransactionsTest(BitcoinTestFramework): assert_raises_rpc_error(-1, "not a boolean", self.nodes[n].getrawtransaction, txId, {}) # Make a tx by sending, then generate 2 blocks; block1 has the tx in it - tx = self.nodes[2].sendtoaddress(self.nodes[1].getnewaddress(), 1) + tx = self.wallet.send_self_transfer(from_node=self.nodes[2])['txid'] block1, block2 = self.generate(self.nodes[2], 2) - for n in [0, 3]: + for n in [0, 2]: self.log.info(f"Test getrawtransaction {'with' if n == 0 else 'without'} -txindex, with blockhash") # We should be able to get the raw transaction by providing the correct block gottx = self.nodes[n].getrawtransaction(txid=tx, verbose=True, blockhash=block1) @@ -145,10 +147,6 @@ class RawTransactionsTest(BitcoinTestFramework): assert 'in_active_chain' not in gottx else: self.log.info("Test getrawtransaction without -txindex, without blockhash: expect the call to raise") - err_msg = ( - "No such mempool transaction. Use -txindex or provide a block hash to enable" - " blockchain transaction queries. Use gettransaction for wallet transactions." - ) assert_raises_rpc_error(-5, err_msg, self.nodes[n].getrawtransaction, txid=tx, verbose=True) # We should not get the tx if we provide an unrelated block assert_raises_rpc_error(-5, "No such transaction found", self.nodes[n].getrawtransaction, txid=tx, blockhash=block2) @@ -193,20 +191,21 @@ class RawTransactionsTest(BitcoinTestFramework): # sequence number out of range for invalid_seq in [-1, 4294967296]: inputs = [{'txid': TXID, 'vout': 1, 'sequence': invalid_seq}] - outputs = {self.nodes[0].getnewaddress(): 1} + address = getnewdestination()[2] + outputs = {address: 1} assert_raises_rpc_error(-8, 'Invalid parameter, sequence number is out of range', self.nodes[0].createrawtransaction, inputs, outputs) # with valid sequence number for valid_seq in [1000, 4294967294]: inputs = [{'txid': TXID, 'vout': 1, 'sequence': valid_seq}] - outputs = {self.nodes[0].getnewaddress(): 1} + address = getnewdestination()[2] + outputs = {address: 1} rawtx = self.nodes[0].createrawtransaction(inputs, outputs) decrawtx = self.nodes[0].decoderawtransaction(rawtx) assert_equal(decrawtx['vin'][0]['sequence'], valid_seq) # Test `createrawtransaction` invalid `outputs` - address = self.nodes[0].getnewaddress() - address2 = self.nodes[0].getnewaddress() + address = getnewdestination()[2] assert_raises_rpc_error(-1, "JSON value is not an array as expected", self.nodes[0].createrawtransaction, [], 'foo') self.nodes[0].createrawtransaction(inputs=[], outputs={}) # Should not throw for backwards compatibility self.nodes[0].createrawtransaction(inputs=[], outputs=[]) @@ -238,6 +237,7 @@ class RawTransactionsTest(BitcoinTestFramework): self.nodes[2].createrawtransaction(inputs=[{'txid': TXID, 'vout': 9}], outputs=[{address: 99}]), ) # Two outputs + address2 = getnewdestination()[2] tx = tx_from_hex(self.nodes[2].createrawtransaction(inputs=[{'txid': TXID, 'vout': 9}], outputs=OrderedDict([(address, 99), (address2, 99)]))) assert_equal(len(tx.vout), 2) assert_equal( @@ -252,122 +252,53 @@ class RawTransactionsTest(BitcoinTestFramework): self.nodes[2].createrawtransaction(inputs=[{'txid': TXID, 'vout': 9}], outputs=[{address: 99}, {address2: 99}, {'data': '99'}]), ) - def signrawtransactionwithwallet_tests(self): - for type in ["bech32", "p2sh-segwit", "legacy"]: - self.log.info(f"Test signrawtransactionwithwallet with missing prevtx info ({type})") - addr = self.nodes[0].getnewaddress("", type) - addrinfo = self.nodes[0].getaddressinfo(addr) - pubkey = addrinfo["scriptPubKey"] - inputs = [{'txid': TXID, 'vout': 3, 'sequence': 1000}] - outputs = {self.nodes[0].getnewaddress(): 1} - rawtx = self.nodes[0].createrawtransaction(inputs, outputs) - - prevtx = dict(txid=TXID, scriptPubKey=pubkey, vout=3, amount=1) - succ = self.nodes[0].signrawtransactionwithwallet(rawtx, [prevtx]) - assert succ["complete"] - - if type == "legacy": - del prevtx["amount"] - succ = self.nodes[0].signrawtransactionwithwallet(rawtx, [prevtx]) - assert succ["complete"] - else: - assert_raises_rpc_error(-3, "Missing amount", self.nodes[0].signrawtransactionwithwallet, rawtx, [ - { - "txid": TXID, - "scriptPubKey": pubkey, - "vout": 3, - } - ]) - - assert_raises_rpc_error(-3, "Missing vout", self.nodes[0].signrawtransactionwithwallet, rawtx, [ - { - "txid": TXID, - "scriptPubKey": pubkey, - "amount": 1, - } - ]) - assert_raises_rpc_error(-3, "Missing txid", self.nodes[0].signrawtransactionwithwallet, rawtx, [ - { - "scriptPubKey": pubkey, - "vout": 3, - "amount": 1, - } - ]) - assert_raises_rpc_error(-3, "Missing scriptPubKey", self.nodes[0].signrawtransactionwithwallet, rawtx, [ - { - "txid": TXID, - "vout": 3, - "amount": 1 - } - ]) - def sendrawtransaction_tests(self): self.log.info("Test sendrawtransaction with missing input") inputs = [{'txid': TXID, 'vout': 1}] # won't exist - outputs = {self.nodes[0].getnewaddress(): 4.998} + address = getnewdestination()[2] + outputs = {address: 4.998} rawtx = self.nodes[2].createrawtransaction(inputs, outputs) - rawtx = self.nodes[2].signrawtransactionwithwallet(rawtx) - assert_raises_rpc_error(-25, "bad-txns-inputs-missingorspent", self.nodes[2].sendrawtransaction, rawtx['hex']) + assert_raises_rpc_error(-25, "bad-txns-inputs-missingorspent", self.nodes[2].sendrawtransaction, rawtx) def sendrawtransaction_testmempoolaccept_tests(self): self.log.info("Test sendrawtransaction/testmempoolaccept with maxfeerate") fee_exceeds_max = "Fee exceeds maximum configured by user (e.g. -maxtxfee, maxfeerate)" # Test a transaction with a small fee. - txId = self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), 1.0) - rawTx = self.nodes[0].getrawtransaction(txId, True) - vout = next(o for o in rawTx['vout'] if o['value'] == Decimal('1.00000000')) - - self.sync_all() - inputs = [{"txid": txId, "vout": vout['n']}] - # Fee 10,000 satoshis, (1 - (10000 sat * 0.00000001 BTC/sat)) = 0.9999 - outputs = {self.nodes[0].getnewaddress(): Decimal("0.99990000")} - rawTx = self.nodes[2].createrawtransaction(inputs, outputs) - rawTxSigned = self.nodes[2].signrawtransactionwithwallet(rawTx) - assert_equal(rawTxSigned['complete'], True) - # Fee 10,000 satoshis, ~100 b transaction, fee rate should land around 100 sat/byte = 0.00100000 BTC/kB + # Fee rate is 0.00100000 BTC/kvB + tx = self.wallet.create_self_transfer(fee_rate=Decimal('0.00100000')) # Thus, testmempoolaccept should reject - testres = self.nodes[2].testmempoolaccept([rawTxSigned['hex']], 0.00001000)[0] + testres = self.nodes[2].testmempoolaccept([tx['hex']], 0.00001000)[0] assert_equal(testres['allowed'], False) assert_equal(testres['reject-reason'], 'max-fee-exceeded') # and sendrawtransaction should throw - assert_raises_rpc_error(-25, fee_exceeds_max, self.nodes[2].sendrawtransaction, rawTxSigned['hex'], 0.00001000) + assert_raises_rpc_error(-25, fee_exceeds_max, self.nodes[2].sendrawtransaction, tx['hex'], 0.00001000) # and the following calls should both succeed - testres = self.nodes[2].testmempoolaccept(rawtxs=[rawTxSigned['hex']])[0] + testres = self.nodes[2].testmempoolaccept(rawtxs=[tx['hex']])[0] assert_equal(testres['allowed'], True) - self.nodes[2].sendrawtransaction(hexstring=rawTxSigned['hex']) + self.nodes[2].sendrawtransaction(hexstring=tx['hex']) # Test a transaction with a large fee. - txId = self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), 1.0) - rawTx = self.nodes[0].getrawtransaction(txId, True) - vout = next(o for o in rawTx['vout'] if o['value'] == Decimal('1.00000000')) - - self.sync_all() - inputs = [{"txid": txId, "vout": vout['n']}] - # Fee 2,000,000 satoshis, (1 - (2000000 sat * 0.00000001 BTC/sat)) = 0.98 - outputs = {self.nodes[0].getnewaddress() : Decimal("0.98000000")} - rawTx = self.nodes[2].createrawtransaction(inputs, outputs) - rawTxSigned = self.nodes[2].signrawtransactionwithwallet(rawTx) - assert_equal(rawTxSigned['complete'], True) - # Fee 2,000,000 satoshis, ~100 b transaction, fee rate should land around 20,000 sat/byte = 0.20000000 BTC/kB + # Fee rate is 0.20000000 BTC/kvB + tx = self.wallet.create_self_transfer(mempool_valid=False, from_node=self.nodes[0], fee_rate=Decimal('0.20000000')) # Thus, testmempoolaccept should reject - testres = self.nodes[2].testmempoolaccept([rawTxSigned['hex']])[0] + testres = self.nodes[2].testmempoolaccept([tx['hex']])[0] assert_equal(testres['allowed'], False) assert_equal(testres['reject-reason'], 'max-fee-exceeded') # and sendrawtransaction should throw - assert_raises_rpc_error(-25, fee_exceeds_max, self.nodes[2].sendrawtransaction, rawTxSigned['hex']) + assert_raises_rpc_error(-25, fee_exceeds_max, self.nodes[2].sendrawtransaction, tx['hex']) # and the following calls should both succeed - testres = self.nodes[2].testmempoolaccept(rawtxs=[rawTxSigned['hex']], maxfeerate='0.20000000')[0] + testres = self.nodes[2].testmempoolaccept(rawtxs=[tx['hex']], maxfeerate='0.20000000')[0] assert_equal(testres['allowed'], True) - self.nodes[2].sendrawtransaction(hexstring=rawTxSigned['hex'], maxfeerate='0.20000000') + self.nodes[2].sendrawtransaction(hexstring=tx['hex'], maxfeerate='0.20000000') self.log.info("Test sendrawtransaction/testmempoolaccept with tx already in the chain") self.generate(self.nodes[2], 1) for node in self.nodes: - testres = node.testmempoolaccept([rawTxSigned['hex']])[0] + testres = node.testmempoolaccept([tx['hex']])[0] assert_equal(testres['allowed'], False) assert_equal(testres['reject-reason'], 'txn-already-known') - assert_raises_rpc_error(-27, 'Transaction already in block chain', node.sendrawtransaction, rawTxSigned['hex']) + assert_raises_rpc_error(-27, 'Transaction already in block chain', node.sendrawtransaction, tx['hex']) def decoderawtransaction_tests(self): self.log.info("Test decoderawtransaction") diff --git a/test/functional/rpc_signrawtransaction.py b/test/functional/rpc_signrawtransaction.py index e648040278..8da2cfa72b 100755 --- a/test/functional/rpc_signrawtransaction.py +++ b/test/functional/rpc_signrawtransaction.py @@ -270,7 +270,7 @@ class SignRawTransactionsTest(BitcoinTestFramework): getcontext().prec = 8 # Make sure CSV is active - assert self.nodes[0].getblockchaininfo()['softforks']['csv']['active'] + assert self.nodes[0].getdeploymentinfo()['deployments']['csv']['active'] # Create a P2WSH script with CSV script = CScript([1, OP_CHECKSEQUENCEVERIFY, OP_DROP]) @@ -305,7 +305,7 @@ class SignRawTransactionsTest(BitcoinTestFramework): getcontext().prec = 8 # Make sure CLTV is active - assert self.nodes[0].getblockchaininfo()['softforks']['bip65']['active'] + assert self.nodes[0].getdeploymentinfo()['deployments']['bip65']['active'] # Create a P2WSH script with CLTV script = CScript([100, OP_CHECKLOCKTIMEVERIFY, OP_DROP]) @@ -334,6 +334,56 @@ class SignRawTransactionsTest(BitcoinTestFramework): assert_equal(signed["complete"], True) self.nodes[0].sendrawtransaction(signed["hex"]) + def test_signing_with_missing_prevtx_info(self): + txid = "1d1d4e24ed99057e84c3f80fd8fbec79ed9e1acee37da269356ecea000000000" + for type in ["bech32", "p2sh-segwit", "legacy"]: + self.log.info(f"Test signing with missing prevtx info ({type})") + addr = self.nodes[0].getnewaddress("", type) + addrinfo = self.nodes[0].getaddressinfo(addr) + pubkey = addrinfo["scriptPubKey"] + inputs = [{'txid': txid, 'vout': 3, 'sequence': 1000}] + outputs = {self.nodes[0].getnewaddress(): 1} + rawtx = self.nodes[0].createrawtransaction(inputs, outputs) + + prevtx = dict(txid=txid, scriptPubKey=pubkey, vout=3, amount=1) + succ = self.nodes[0].signrawtransactionwithwallet(rawtx, [prevtx]) + assert succ["complete"] + + if type == "legacy": + del prevtx["amount"] + succ = self.nodes[0].signrawtransactionwithwallet(rawtx, [prevtx]) + assert succ["complete"] + else: + assert_raises_rpc_error(-3, "Missing amount", self.nodes[0].signrawtransactionwithwallet, rawtx, [ + { + "txid": txid, + "scriptPubKey": pubkey, + "vout": 3, + } + ]) + + assert_raises_rpc_error(-3, "Missing vout", self.nodes[0].signrawtransactionwithwallet, rawtx, [ + { + "txid": txid, + "scriptPubKey": pubkey, + "amount": 1, + } + ]) + assert_raises_rpc_error(-3, "Missing txid", self.nodes[0].signrawtransactionwithwallet, rawtx, [ + { + "scriptPubKey": pubkey, + "vout": 3, + "amount": 1, + } + ]) + assert_raises_rpc_error(-3, "Missing scriptPubKey", self.nodes[0].signrawtransactionwithwallet, rawtx, [ + { + "txid": txid, + "vout": 3, + "amount": 1 + } + ]) + def run_test(self): self.successful_signing_test() self.script_verification_error_test() @@ -343,6 +393,7 @@ class SignRawTransactionsTest(BitcoinTestFramework): self.test_fully_signed_tx() self.test_signing_with_csv() self.test_signing_with_cltv() + self.test_signing_with_missing_prevtx_info() if __name__ == '__main__': diff --git a/test/functional/rpc_uptime.py b/test/functional/rpc_uptime.py index 1a82d1fa41..024e8aec1a 100755 --- a/test/functional/rpc_uptime.py +++ b/test/functional/rpc_uptime.py @@ -23,7 +23,7 @@ class UptimeTest(BitcoinTestFramework): self._test_uptime() def _test_negative_time(self): - assert_raises_rpc_error(-8, "Mocktime can not be negative: -1.", self.nodes[0].setmocktime, -1) + assert_raises_rpc_error(-8, "Mocktime cannot be negative: -1.", self.nodes[0].setmocktime, -1) def _test_uptime(self): wait_time = 10 diff --git a/test/functional/rpc_users.py b/test/functional/rpc_users.py index 7cedb4336b..1a35a57802 100755 --- a/test/functional/rpc_users.py +++ b/test/functional/rpc_users.py @@ -107,6 +107,9 @@ class HTTPBasicsTest(BitcoinTestFramework): self.stop_node(0) self.nodes[0].assert_start_raises_init_error(expected_msg=init_error, extra_args=['-rpcauth=foo']) self.nodes[0].assert_start_raises_init_error(expected_msg=init_error, extra_args=['-rpcauth=foo:bar']) + self.nodes[0].assert_start_raises_init_error(expected_msg=init_error, extra_args=['-rpcauth=foo:bar:baz']) + self.nodes[0].assert_start_raises_init_error(expected_msg=init_error, extra_args=['-rpcauth=foo$bar:baz']) + self.nodes[0].assert_start_raises_init_error(expected_msg=init_error, extra_args=['-rpcauth=foo$bar$baz']) self.log.info('Check that failure to write cookie file will abort the node gracefully') cookie_file = os.path.join(get_datadir_path(self.options.tmpdir, 0), self.chain, '.cookie.tmp') diff --git a/test/functional/test_framework/address.py b/test/functional/test_framework/address.py index 013522a5e1..fcea24655b 100644 --- a/test/functional/test_framework/address.py +++ b/test/functional/test_framework/address.py @@ -35,7 +35,7 @@ class AddressType(enum.Enum): legacy = 'legacy' # P2PKH -chars = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz' +b58chars = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz' def create_deterministic_address_bcrt1_p2tr_op_true(): @@ -55,17 +55,15 @@ def create_deterministic_address_bcrt1_p2tr_op_true(): def byte_to_base58(b, version): result = '' - str = b.hex() - str = chr(version).encode('latin-1').hex() + str - checksum = hash256(bytes.fromhex(str)).hex() - str += checksum[:8] - value = int('0x' + str, 0) + b = bytes([version]) + b # prepend version + b += hash256(b)[:4] # append checksum + value = int.from_bytes(b, 'big') while value > 0: - result = chars[value % 58] + result + result = b58chars[value % 58] + result value //= 58 - while (str[:2] == '00'): - result = chars[0] + result - str = str[2:] + while b[0] == 0: + result = b58chars[0] + result + b = b[1:] return result @@ -78,8 +76,8 @@ def base58_to_byte(s): n = 0 for c in s: n *= 58 - assert c in chars - digit = chars.index(c) + assert c in b58chars + digit = b58chars.index(c) n += digit h = '%x' % n if len(h) % 2: @@ -87,14 +85,14 @@ def base58_to_byte(s): res = n.to_bytes((n.bit_length() + 7) // 8, 'big') pad = 0 for c in s: - if c == chars[0]: + if c == b58chars[0]: pad += 1 else: break res = b'\x00' * pad + res - # Assert if the checksum is invalid - assert_equal(hash256(res[:-4])[:4], res[-4:]) + if hash256(res[:-4])[:4] != res[-4:]: + raise ValueError('Invalid Base58Check checksum') return res[1:-4], int(res[0]) diff --git a/test/functional/test_framework/messages.py b/test/functional/test_framework/messages.py index f57b6e7494..aae44c0ac0 100755 --- a/test/functional/test_framework/messages.py +++ b/test/functional/test_framework/messages.py @@ -1672,7 +1672,7 @@ class msg_getcfilters: __slots__ = ("filter_type", "start_height", "stop_hash") msgtype = b"getcfilters" - def __init__(self, filter_type, start_height, stop_hash): + def __init__(self, filter_type=None, start_height=None, stop_hash=None): self.filter_type = filter_type self.start_height = start_height self.stop_hash = stop_hash @@ -1722,7 +1722,7 @@ class msg_getcfheaders: __slots__ = ("filter_type", "start_height", "stop_hash") msgtype = b"getcfheaders" - def __init__(self, filter_type, start_height, stop_hash): + def __init__(self, filter_type=None, start_height=None, stop_hash=None): self.filter_type = filter_type self.start_height = start_height self.stop_hash = stop_hash @@ -1775,7 +1775,7 @@ class msg_getcfcheckpt: __slots__ = ("filter_type", "stop_hash") msgtype = b"getcfcheckpt" - def __init__(self, filter_type, stop_hash): + def __init__(self, filter_type=None, stop_hash=None): self.filter_type = filter_type self.stop_hash = stop_hash diff --git a/test/functional/test_framework/netutil.py b/test/functional/test_framework/netutil.py index 174dc44a2a..b64f66e69b 100644 --- a/test/functional/test_framework/netutil.py +++ b/test/functional/test_framework/netutil.py @@ -144,7 +144,6 @@ def test_ipv6_local(): ''' Check for (local) IPv6 support. ''' - import socket # By using SOCK_DGRAM this will not actually make a connection, but it will # fail if there is no route to IPv6 localhost. have_ipv6 = True diff --git a/test/functional/test_framework/p2p.py b/test/functional/test_framework/p2p.py index 251d3d5eae..fc72a9ab73 100755 --- a/test/functional/test_framework/p2p.py +++ b/test/functional/test_framework/p2p.py @@ -47,6 +47,9 @@ from test_framework.messages import ( msg_getaddr, msg_getblocks, msg_getblocktxn, + msg_getcfcheckpt, + msg_getcfheaders, + msg_getcfilters, msg_getdata, msg_getheaders, msg_headers, @@ -108,6 +111,9 @@ MESSAGEMAP = { b"getaddr": msg_getaddr, b"getblocks": msg_getblocks, b"getblocktxn": msg_getblocktxn, + b"getcfcheckpt": msg_getcfcheckpt, + b"getcfheaders": msg_getcfheaders, + b"getcfilters": msg_getcfilters, b"getdata": msg_getdata, b"getheaders": msg_getheaders, b"headers": msg_headers, diff --git a/test/functional/test_framework/script.py b/test/functional/test_framework/script.py index 7791ae5392..2b70eab4e4 100644 --- a/test/functional/test_framework/script.py +++ b/test/functional/test_framework/script.py @@ -27,6 +27,7 @@ from .messages import ( from .ripemd160 import ripemd160 MAX_SCRIPT_ELEMENT_SIZE = 520 +MAX_PUBKEYS_PER_MULTI_A = 999 LOCKTIME_THRESHOLD = 500000000 ANNEX_TAG = 0x50 diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index 8f75255caf..a39ee003ef 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -9,6 +9,7 @@ from enum import Enum import argparse import logging import os +import platform import pdb import random import re @@ -243,8 +244,14 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): "src", "bitcoin-cli" + config["environment"]["EXEEXT"], ) + fname_bitcoinutil = os.path.join( + config["environment"]["BUILDDIR"], + "src", + "bitcoin-util" + config["environment"]["EXEEXT"], + ) self.options.bitcoind = os.getenv("BITCOIND", default=fname_bitcoind) self.options.bitcoincli = os.getenv("BITCOINCLI", default=fname_bitcoincli) + self.options.bitcoinutil = os.getenv("BITCOINUTIL", default=fname_bitcoinutil) os.environ['PATH'] = os.pathsep.join([ os.path.join(config['environment']['BUILDDIR'], 'src'), @@ -447,11 +454,15 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): def get_bin_from_version(version, bin_name, bin_default): if not version: return bin_default + if version > 219999: + # Starting at client version 220000 the first two digits represent + # the major version, e.g. v22.0 instead of v0.22.0. + version *= 100 return os.path.join( self.options.previous_releases_path, re.sub( - r'\.0$', - '', # remove trailing .0 for point releases + r'\.0$' if version <= 219999 else r'(\.0){1,2}$', + '', # Remove trailing dot for point releases, after 22.0 also remove double trailing dot. 'v{}.{}.{}.{}'.format( (version % 100000000) // 1000000, (version % 1000000) // 10000, @@ -473,7 +484,8 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): versions = [None] * num_nodes if self.is_syscall_sandbox_compiled() and not self.disable_syscall_sandbox: for i in range(len(extra_args)): - if versions[i] is None or versions[i] >= 219900: + # The -sandbox argument is not present in the v22.0 release. + if versions[i] is None or versions[i] >= 229900: extra_args[i] = extra_args[i] + ["-sandbox=log-and-abort"] if binary is None: binary = [get_bin_from_version(v, 'bitcoind', self.options.bitcoind) for v in versions] @@ -821,6 +833,29 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): except ImportError: raise SkipTest("python3-zmq module not available.") + def skip_if_no_python_bcc(self): + """Attempt to import the bcc package and skip the tests if the import fails.""" + try: + import bcc # type: ignore[import] # noqa: F401 + except ImportError: + raise SkipTest("bcc python module not available") + + def skip_if_no_bitcoind_tracepoints(self): + """Skip the running test if bitcoind has not been compiled with USDT tracepoint support.""" + if not self.is_usdt_compiled(): + raise SkipTest("bitcoind has not been built with USDT tracepoints enabled.") + + def skip_if_no_bpf_permissions(self): + """Skip the running test if we don't have permissions to do BPF syscalls and load BPF maps.""" + # check for 'root' permissions + if os.geteuid() != 0: + raise SkipTest("no permissions to use BPF (please review the tests carefully before running them with higher privileges)") + + def skip_if_platform_not_linux(self): + """Skip the running test if we are not on a Linux platform""" + if platform.system() != "Linux": + raise SkipTest("not on a Linux system") + def skip_if_no_bitcoind_zmq(self): """Skip the running test if bitcoind has not been compiled with zmq support.""" if not self.is_zmq_compiled(): @@ -851,6 +886,11 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): if not self.is_wallet_tool_compiled(): raise SkipTest("bitcoin-wallet has not been compiled") + def skip_if_no_bitcoin_util(self): + """Skip the running test if bitcoin-util has not been compiled.""" + if not self.is_bitcoin_util_compiled(): + raise SkipTest("bitcoin-util has not been compiled") + def skip_if_no_cli(self): """Skip the running test if bitcoin-cli has not been compiled.""" if not self.is_cli_compiled(): @@ -898,10 +938,18 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): """Checks whether bitcoin-wallet was compiled.""" return self.config["components"].getboolean("ENABLE_WALLET_TOOL") + def is_bitcoin_util_compiled(self): + """Checks whether bitcoin-util was compiled.""" + return self.config["components"].getboolean("ENABLE_BITCOIN_UTIL") + def is_zmq_compiled(self): """Checks whether the zmq module was compiled.""" return self.config["components"].getboolean("ENABLE_ZMQ") + def is_usdt_compiled(self): + """Checks whether the USDT tracepoints were compiled.""" + return self.config["components"].getboolean("ENABLE_USDT_TRACEPOINTS") + def is_sqlite_compiled(self): """Checks whether the wallet module was compiled with Sqlite support.""" return self.config["components"].getboolean("USE_SQLITE") diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index 0b9154a030..7d2db391b6 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -422,7 +422,8 @@ class TestNode(): time.sleep(0.05) self._raise_assertion_error('Expected messages "{}" does not partially match log:\n\n{}\n\n'.format(str(expected_msgs), print_log)) - def wait_for_debug_log(self, expected_msgs, timeout=10, ignore_case=False) -> int: + @contextlib.contextmanager + def wait_for_debug_log(self, expected_msgs, timeout=60, ignore_case=False): """ Block until we see a particular debug log message fragment or until we exceed the timeout. Return: @@ -432,6 +433,8 @@ class TestNode(): prev_size = self.debug_log_bytes() re_flags = re.MULTILINE | (re.IGNORECASE if ignore_case else 0) + yield + while True: found = True with open(self.debug_log_path, encoding='utf-8') as dl: @@ -443,8 +446,7 @@ class TestNode(): found = False if found: - num_logs = len(log.splitlines()) - return num_logs + return if time.time() >= time_end: print_log = " - " + "\n - ".join(log.splitlines()) @@ -456,7 +458,6 @@ class TestNode(): self._raise_assertion_error( 'Expected messages "{}" does not partially match log:\n\n{}\n\n'.format( str(expected_msgs), print_log)) - return -1 # useless return to satisfy linter @contextlib.contextmanager def profile_with_perf(self, profile_name: str): @@ -544,6 +545,7 @@ class TestNode(): Will throw if bitcoind starts without an error. Will throw if an expected_msg is provided and it does not match bitcoind's stdout.""" + assert not self.running with tempfile.NamedTemporaryFile(dir=self.stderr_dir, delete=False) as log_stderr, \ tempfile.NamedTemporaryFile(dir=self.stdout_dir, delete=False) as log_stdout: try: @@ -742,6 +744,9 @@ class RPCOverloadWrapper(): def __getattr__(self, name): return getattr(self.rpc, name) + def createwallet_passthrough(self, *args, **kwargs): + return self.__getattr__("createwallet")(*args, **kwargs) + def createwallet(self, wallet_name, disable_private_keys=None, blank=None, passphrase='', avoid_reuse=None, descriptors=None, load_on_startup=None, external_signer=None): if descriptors is None: descriptors = self.descriptors diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index 195af14914..b043d1a70d 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -36,6 +36,7 @@ def assert_approx(v, vexp, vspan=0.00001): def assert_fee_amount(fee, tx_size, feerate_BTC_kvB): """Assert the fee is in range.""" + assert isinstance(tx_size, int) target_fee = get_fee(tx_size, feerate_BTC_kvB) if fee < target_fee: raise AssertionError("Fee of %s BTC too low! (Should be %s BTC)" % (str(fee), str(target_fee))) @@ -219,7 +220,13 @@ def str_to_b64str(string): def ceildiv(a, b): - """Divide 2 ints and round up to next int rather than round down""" + """ + Divide 2 ints and round up to next int rather than round down + Implementation requires python integers, which have a // operator that does floor division. + Other types like decimal.Decimal whose // operator truncates towards 0 will not work. + """ + assert isinstance(a, int) + assert isinstance(b, int) return -(-a // b) @@ -227,7 +234,7 @@ def get_fee(tx_size, feerate_btc_kvb): """Calculate the fee in BTC given a feerate is BTC/kvB. Reflects CFeeRate::GetFee""" feerate_sat_kvb = int(feerate_btc_kvb * Decimal(1e8)) # Fee in sat/kvb as an int to avoid float precision errors target_fee_sat = ceildiv(feerate_sat_kvb * tx_size, 1000) # Round calculated fee up to nearest sat - return satoshi_round(target_fee_sat / Decimal(1e8)) # Truncate BTC result to nearest sat + return target_fee_sat / Decimal(1e8) # Return result in BTC def satoshi_round(amount): @@ -371,6 +378,7 @@ def write_config(config_path, *, n, chain, extra_config="", disable_autoconnect= f.write("[{}]\n".format(chain_name_conf_section)) f.write("port=" + str(p2p_port(n)) + "\n") f.write("rpcport=" + str(rpc_port(n)) + "\n") + f.write("rpcdoccheck=1\n") f.write("fallbackfee=0.0002\n") f.write("server=1\n") f.write("keypool=1\n") @@ -438,7 +446,7 @@ def delete_cookie_file(datadir, chain): def softfork_active(node, key): """Return whether a softfork is active.""" - return node.getblockchaininfo()['softforks'][key]['active'] + return node.getdeploymentinfo()['deployments'][key]['active'] def set_node_times(nodes, t): @@ -566,17 +574,17 @@ def create_lots_of_big_transactions(node, txouts, utxos, num, fee): return txids -def mine_large_block(test_framework, node, utxos=None): +def mine_large_block(test_framework, mini_wallet, node): # generate a 66k transaction, # and 14 of them is close to the 1MB block limit - num = 14 txouts = gen_return_txouts() - utxos = utxos if utxos is not None else [] - if len(utxos) < num: - utxos.clear() - utxos.extend(node.listunspent()) - fee = 100 * node.getnetworkinfo()["relayfee"] - create_lots_of_big_transactions(node, txouts, utxos, num, fee=fee) + from .messages import COIN + fee = 100 * int(node.getnetworkinfo()["relayfee"] * COIN) + for _ in range(14): + tx = mini_wallet.create_self_transfer(from_node=node, fee_rate=0, mempool_valid=False)['tx'] + tx.vout[0].nValue -= fee + tx.vout.extend(txouts) + mini_wallet.sendrawtransaction(from_node=node, tx_hex=tx.serialize().hex()) test_framework.generate(node, 1) diff --git a/test/functional/test_framework/wallet.py b/test/functional/test_framework/wallet.py index dd41a740ae..6901bcfe66 100644 --- a/test/functional/test_framework/wallet.py +++ b/test/functional/test_framework/wallet.py @@ -8,7 +8,10 @@ from copy import deepcopy from decimal import Decimal from enum import Enum from random import choice -from typing import Optional +from typing import ( + Any, + Optional, +) from test_framework.address import ( base58_to_byte, create_deterministic_address_bcrt1_p2tr_op_true, @@ -93,6 +96,9 @@ class MiniWallet: self._address, self._internal_key = create_deterministic_address_bcrt1_p2tr_op_true() self._scriptPubKey = bytes.fromhex(self._test_node.validateaddress(self._address)['scriptPubKey']) + def get_balance(self): + return sum(u['value'] for u in self._utxos) + def rescan_utxos(self): """Drop all utxos and rescan the utxo set""" self._utxos = [] @@ -121,6 +127,7 @@ class MiniWallet: if not fixed_length: break tx.vin[0].scriptSig = CScript([der_sig + bytes(bytearray([SIGHASH_ALL]))]) + tx.rehash() def generate(self, num_blocks, **kwargs): """Generate blocks with coinbase outputs to the internal address, and append the outputs to the internal list""" @@ -131,24 +138,30 @@ class MiniWallet: self._utxos.append({'txid': cb_tx['txid'], 'vout': 0, 'value': cb_tx['vout'][0]['value'], 'height': block_info['height']}) return blocks + def get_scriptPubKey(self): + return self._scriptPubKey + def get_descriptor(self): return descsum_create(f'raw({self._scriptPubKey.hex()})') def get_address(self): return self._address - def get_utxo(self, *, txid: Optional[str]='', mark_as_spent=True): + def get_utxo(self, *, txid: str = '', vout: Optional[int] = None, mark_as_spent=True): """ Returns a utxo and marks it as spent (pops it from the internal list) Args: txid: get the first utxo we find from a specific transaction """ - index = -1 # by default the last utxo self._utxos = sorted(self._utxos, key=lambda k: (k['value'], -k['height'])) # Put the largest utxo last if txid: - utxo = next(filter(lambda utxo: txid == utxo['txid'], self._utxos)) - index = self._utxos.index(utxo) + utxo_filter: Any = filter(lambda utxo: txid == utxo['txid'], self._utxos) + else: + utxo_filter = reversed(self._utxos) # By default the largest utxo + if vout is not None: + utxo_filter = filter(lambda utxo: vout == utxo['vout'], utxo_filter) + index = self._utxos.index(next(utxo_filter)) if mark_as_spent: return self._utxos.pop(index) else: @@ -179,8 +192,50 @@ class MiniWallet: txid = self.sendrawtransaction(from_node=from_node, tx_hex=tx.serialize().hex()) return txid, 1 + def send_self_transfer_multi(self, **kwargs): + """ + Create and send a transaction that spends the given UTXOs and creates a + certain number of outputs with equal amounts. + + Returns a dictionary with + - txid + - serialized transaction in hex format + - transaction as CTransaction instance + - list of newly created UTXOs, ordered by vout index + """ + tx = self.create_self_transfer_multi(**kwargs) + txid = self.sendrawtransaction(from_node=kwargs['from_node'], tx_hex=tx.serialize().hex()) + return {'new_utxos': [self.get_utxo(txid=txid, vout=vout) for vout in range(len(tx.vout))], + 'txid': txid, 'hex': tx.serialize().hex(), 'tx': tx} + + def create_self_transfer_multi(self, *, from_node, utxos_to_spend=None, num_outputs=1, fee_per_output=1000): + """ + Create and return a transaction that spends the given UTXOs and creates a + certain number of outputs with equal amounts. + """ + utxos_to_spend = utxos_to_spend or [self.get_utxo()] + # create simple tx template (1 input, 1 output) + tx = self.create_self_transfer(fee_rate=0, from_node=from_node, utxo_to_spend=utxos_to_spend[0], mempool_valid=False)['tx'] + + # duplicate inputs, witnesses and outputs + tx.vin = [deepcopy(tx.vin[0]) for _ in range(len(utxos_to_spend))] + tx.wit.vtxinwit = [deepcopy(tx.wit.vtxinwit[0]) for _ in range(len(utxos_to_spend))] + tx.vout = [deepcopy(tx.vout[0]) for _ in range(num_outputs)] + + # adapt input prevouts + for i, utxo in enumerate(utxos_to_spend): + tx.vin[i] = CTxIn(COutPoint(int(utxo['txid'], 16), utxo['vout'])) + + # adapt output amounts (use fixed fee per output) + inputs_value_total = sum([int(COIN * utxo['value']) for utxo in utxos_to_spend]) + outputs_value_total = inputs_value_total - fee_per_output * num_outputs + for o in tx.vout: + o.nValue = outputs_value_total // num_outputs + return tx + def create_self_transfer(self, *, fee_rate=Decimal("0.003"), from_node=None, utxo_to_spend=None, mempool_valid=True, locktime=0, sequence=0): - """Create and return a tx with the specified fee_rate. Fee may be exact or at most one satoshi higher than needed.""" + """Create and return a tx with the specified fee_rate. Fee may be exact or at most one satoshi higher than needed. + Checking mempool validity via the testmempoolaccept RPC can be skipped by setting mempool_valid to False.""" from_node = from_node or self._test_node utxo_to_spend = utxo_to_spend or self.get_utxo() if self._priv_key is None: @@ -207,12 +262,13 @@ class MiniWallet: tx.wit.vtxinwit[0].scriptWitness.stack = [CScript([OP_TRUE]), bytes([LEAF_VERSION_TAPSCRIPT]) + self._internal_key] tx_hex = tx.serialize().hex() - tx_info = from_node.testmempoolaccept([tx_hex])[0] - assert_equal(mempool_valid, tx_info['allowed']) if mempool_valid: + tx_info = from_node.testmempoolaccept([tx_hex])[0] + assert_equal(tx_info['allowed'], True) assert_equal(tx_info['vsize'], vsize) assert_equal(tx_info['fees']['base'], utxo_to_spend['value'] - Decimal(send_value) / COIN) - return {'txid': tx_info['txid'], 'wtxid': tx_info['wtxid'], 'hex': tx_hex, 'tx': tx} + + return {'txid': tx.rehash(), 'wtxid': tx.getwtxid(), 'hex': tx_hex, 'tx': tx} def sendrawtransaction(self, *, from_node, tx_hex, **kwargs): txid = from_node.sendrawtransaction(hexstring=tx_hex, **kwargs) diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index eb2d030f4a..40e08c3f1f 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -28,7 +28,7 @@ import logging import unittest # Formatting. Default colors to empty strings. -BOLD, GREEN, RED, GREY = ("", ""), ("", ""), ("", ""), ("", "") +DEFAULT, BOLD, GREEN, RED = ("", ""), ("", ""), ("", ""), ("", "") try: # Make sure python thinks it can write unicode to its stdout "\u2713".encode("utf_8").decode(sys.stdout.encoding) @@ -59,10 +59,10 @@ if os.name != 'nt' or sys.getwindowsversion() >= (10, 0, 14393): #type:ignore kernel32.SetConsoleMode(stderr, stderr_mode.value | ENABLE_VIRTUAL_TERMINAL_PROCESSING) # primitive formatting on supported # terminal via ANSI escape sequences: + DEFAULT = ('\033[0m', '\033[0m') BOLD = ('\033[0m', '\033[1m') GREEN = ('\033[0m', '\033[0;32m') RED = ('\033[0m', '\033[0;31m') - GREY = ('\033[0m', '\033[1;30m') TEST_EXIT_PASSED = 0 TEST_EXIT_SKIPPED = 77 @@ -82,6 +82,7 @@ EXTENDED_SCRIPTS = [ # Longest test should go first, to favor running tests in parallel 'feature_pruning.py', 'feature_dbcrash.py', + 'feature_index_prune.py', ] BASE_SCRIPTS = [ @@ -111,7 +112,6 @@ BASE_SCRIPTS = [ 'p2p_tx_download.py', 'mempool_updatefromblock.py', 'wallet_dump.py --legacy-wallet', - 'feature_taproot.py --previous_release', 'feature_taproot.py', 'rpc_signer.py', 'wallet_signer.py --descriptors', @@ -145,6 +145,8 @@ BASE_SCRIPTS = [ 'wallet_txn_doublespend.py --mineblock', 'tool_wallet.py --legacy-wallet', 'tool_wallet.py --descriptors', + 'tool_signet_miner.py --legacy-wallet', + 'tool_signet_miner.py --descriptors', 'wallet_txn_clone.py', 'wallet_txn_clone.py --segwit', 'rpc_getchaintips.py', @@ -168,6 +170,10 @@ BASE_SCRIPTS = [ 'wallet_reorgsrestore.py', 'interface_http.py', 'interface_rpc.py', + 'interface_usdt_coinselection.py', + 'interface_usdt_net.py', + 'interface_usdt_utxocache.py', + 'interface_usdt_validation.py', 'rpc_psbt.py --legacy-wallet', 'rpc_psbt.py --descriptors', 'rpc_users.py', @@ -177,7 +183,6 @@ BASE_SCRIPTS = [ 'rpc_signrawtransaction.py --legacy-wallet', 'rpc_signrawtransaction.py --descriptors', 'rpc_rawtransaction.py --legacy-wallet', - 'rpc_rawtransaction.py --descriptors', 'wallet_groups.py --legacy-wallet', 'wallet_transactiontime_rescan.py --descriptors', 'wallet_transactiontime_rescan.py --legacy-wallet', @@ -188,8 +193,7 @@ BASE_SCRIPTS = [ 'rpc_decodescript.py', 'rpc_blockchain.py', 'rpc_deprecated.py', - 'wallet_disable.py --legacy-wallet', - 'wallet_disable.py --descriptors', + 'wallet_disable.py', 'p2p_addr_relay.py', 'p2p_getaddr_caching.py', 'p2p_getdata.py', @@ -198,6 +202,7 @@ BASE_SCRIPTS = [ 'wallet_keypool.py --legacy-wallet', 'wallet_keypool.py --descriptors', 'wallet_descriptor.py --descriptors', + 'feature_maxtipage.py', 'p2p_nobloomfilter_messages.py', 'p2p_filter.py', 'rpc_setban.py', @@ -224,8 +229,7 @@ BASE_SCRIPTS = [ 'feature_rbf.py --descriptors', 'mempool_packages.py', 'mempool_package_onemore.py', - 'rpc_createmultisig.py --legacy-wallet', - 'rpc_createmultisig.py --descriptors', + 'rpc_createmultisig.py', 'rpc_packages.py', 'mempool_package_limits.py', 'feature_versionbits_warning.py', @@ -236,7 +240,6 @@ BASE_SCRIPTS = [ 'p2p_eviction.py', 'wallet_signmessagewithaddress.py', 'rpc_signmessagewithprivkey.py', - 'rpc_generateblock.py', 'rpc_generate.py', 'wallet_balance.py --legacy-wallet', 'wallet_balance.py --descriptors', @@ -251,6 +254,7 @@ BASE_SCRIPTS = [ 'rpc_bind.py --ipv4', 'rpc_bind.py --ipv6', 'rpc_bind.py --nonloopback', + 'wallet_crosschain.py', 'mining_basic.py', 'feature_signet.py', 'wallet_bumpfee.py --legacy-wallet', @@ -275,11 +279,15 @@ BASE_SCRIPTS = [ 'feature_minchainwork.py', 'rpc_estimatefee.py', 'rpc_getblockstats.py', + 'feature_bind_port_externalip.py', 'wallet_create_tx.py --legacy-wallet', 'wallet_send.py --legacy-wallet', 'wallet_send.py --descriptors', + 'wallet_sendall.py --legacy-wallet', + 'wallet_sendall.py --descriptors', 'wallet_create_tx.py --descriptors', 'wallet_taproot.py', + 'wallet_inactive_hdchains.py', 'p2p_fingerprint.py', 'feature_uacomment.py', 'feature_init.py', @@ -289,6 +297,7 @@ BASE_SCRIPTS = [ 'feature_loadblock.py', 'p2p_dos_header_tree.py', 'p2p_add_connections.py', + 'feature_bind_port_discover.py', 'p2p_unrequested_blocks.py', 'p2p_blockfilters.py', 'p2p_message_capture.py', @@ -303,11 +312,12 @@ BASE_SCRIPTS = [ 'p2p_ping.py', 'rpc_scantxoutset.py', 'feature_txindex_compatibility.py', + 'feature_unsupported_utxo_db.py', 'feature_logging.py', 'feature_anchors.py', - 'feature_coinstatsindex.py --legacy-wallet', - 'feature_coinstatsindex.py --descriptors', + 'feature_coinstatsindex.py', 'wallet_orphanedreward.py', + 'wallet_timelock.py', 'p2p_node_network_limited.py', 'p2p_permissions.py', 'feature_blocksdir.py', @@ -317,12 +327,12 @@ BASE_SCRIPTS = [ 'feature_presegwit_node_upgrade.py', 'feature_settings.py', 'rpc_getdescriptorinfo.py', - 'rpc_mempool_entry_fee_fields_deprecation.py', + 'rpc_mempool_info.py', 'rpc_help.py', + 'feature_dirsymlinks.py', 'feature_help.py', 'feature_shutdown.py', 'p2p_ibd_txrelay.py', - 'feature_blockfilterindex_prune.py' # Don't append tests at the end to avoid merge conflicts # Put them in a random line within the section that fits their approximate run-time ] @@ -361,11 +371,11 @@ def main(): args, unknown_args = parser.parse_known_args() if not args.ansi: - global BOLD, GREEN, RED, GREY + global DEFAULT, BOLD, GREEN, RED + DEFAULT = ("", "") BOLD = ("", "") GREEN = ("", "") RED = ("", "") - GREY = ("", "") # args to be passed on always start with two dashes; tests are the remaining unknown args tests = [arg for arg in unknown_args if arg[:2] != "--"] @@ -530,8 +540,11 @@ def run_tests(*, test_list, src_dir, build_dir, tmpdir, jobs=1, enable_coverage= max_len_name = len(max(test_list, key=len)) test_count = len(test_list) + all_passed = True i = 0 while i < test_count: + if failfast and not all_passed: + break for test_result, testdir, stdout, stderr in job_queue.get_next(): test_results.append(test_result) i += 1 @@ -541,6 +554,7 @@ def run_tests(*, test_list, src_dir, build_dir, tmpdir, jobs=1, enable_coverage= elif test_result.status == "Skipped": logging.debug("%s skipped" % (done_str)) else: + all_passed = False print("%s failed, Duration: %s s\n" % (done_str, test_result.time)) print(BOLD[1] + 'stdout:\n' + BOLD[0] + stdout + '\n') print(BOLD[1] + 'stderr:\n' + BOLD[0] + stderr + '\n') @@ -574,16 +588,17 @@ def run_tests(*, test_list, src_dir, build_dir, tmpdir, jobs=1, enable_coverage= if not os.listdir(tmpdir): os.rmdir(tmpdir) - all_passed = all(map(lambda test_result: test_result.was_successful, test_results)) and coverage_passed + all_passed = all_passed and coverage_passed # Clean up dangling processes if any. This may only happen with --failfast option. # Killing the process group will also terminate the current process but that is # not an issue - if len(job_queue.jobs): + if not os.getenv("CI_FAILFAST_TEST_LEAVE_DANGLING") and len(job_queue.jobs): os.killpg(os.getpgid(0), signal.SIGKILL) sys.exit(not all_passed) + def print_results(test_results, max_len_name, runtime): results = "\n" + BOLD[1] + "%s | %s | %s\n\n" % ("TEST".ljust(max_len_name), "STATUS ", "DURATION") + BOLD[0] @@ -704,7 +719,7 @@ class TestResult(): color = RED glyph = CROSS elif self.status == "Skipped": - color = GREY + color = DEFAULT glyph = CIRCLE return color[1] + "%s | %s%s | %s s\n" % (self.name.ljust(self.padding), glyph, self.status.ljust(7), self.time) + color[0] diff --git a/test/functional/tool_signet_miner.py b/test/functional/tool_signet_miner.py new file mode 100755 index 0000000000..e6fc9072ab --- /dev/null +++ b/test/functional/tool_signet_miner.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test signet miner tool""" + +import os.path +import subprocess +import sys +import time + +from test_framework.key import ECKey +from test_framework.script_util import key_to_p2wpkh_script +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal +from test_framework.wallet_util import bytes_to_wif + + +CHALLENGE_PRIVATE_KEY = (42).to_bytes(32, 'big') + + +class SignetMinerTest(BitcoinTestFramework): + def set_test_params(self): + self.chain = "signet" + self.setup_clean_chain = True + self.num_nodes = 1 + + # generate and specify signet challenge (simple p2wpkh script) + privkey = ECKey() + privkey.set(CHALLENGE_PRIVATE_KEY, True) + pubkey = privkey.get_pubkey().get_bytes() + challenge = key_to_p2wpkh_script(pubkey) + self.extra_args = [[f'-signetchallenge={challenge.hex()}']] + + def skip_test_if_missing_module(self): + self.skip_if_no_cli() + self.skip_if_no_wallet() + self.skip_if_no_bitcoin_util() + + def run_test(self): + node = self.nodes[0] + # import private key needed for signing block + node.importprivkey(bytes_to_wif(CHALLENGE_PRIVATE_KEY)) + + # generate block with signet miner tool + base_dir = self.config["environment"]["SRCDIR"] + signet_miner_path = os.path.join(base_dir, "contrib", "signet", "miner") + subprocess.run([ + sys.executable, + signet_miner_path, + f'--cli={node.cli.binary} -datadir={node.cli.datadir}', + 'generate', + f'--address={node.getnewaddress()}', + f'--grind-cmd={self.options.bitcoinutil} grind', + '--nbits=1d00ffff', + f'--set-block-time={int(time.time())}', + ], check=True, stderr=subprocess.STDOUT) + assert_equal(node.getblockcount(), 1) + + +if __name__ == "__main__": + SignetMinerTest().main() diff --git a/test/functional/wallet_abandonconflict.py b/test/functional/wallet_abandonconflict.py index 27d9d8da88..36fcdb36d6 100755 --- a/test/functional/wallet_abandonconflict.py +++ b/test/functional/wallet_abandonconflict.py @@ -29,20 +29,25 @@ class AbandonConflictTest(BitcoinTestFramework): self.skip_if_no_wallet() def run_test(self): + # create two wallets to tests conflicts from both sender's and receiver's sides + alice = self.nodes[0].get_wallet_rpc(self.default_wallet_name) + self.nodes[0].createwallet(wallet_name="bob") + bob = self.nodes[0].get_wallet_rpc("bob") + self.generate(self.nodes[1], COINBASE_MATURITY) - balance = self.nodes[0].getbalance() - txA = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), Decimal("10")) - txB = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), Decimal("10")) - txC = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), Decimal("10")) + balance = alice.getbalance() + txA = alice.sendtoaddress(alice.getnewaddress(), Decimal("10")) + txB = alice.sendtoaddress(alice.getnewaddress(), Decimal("10")) + txC = alice.sendtoaddress(alice.getnewaddress(), Decimal("10")) self.sync_mempools() self.generate(self.nodes[1], 1) # Can not abandon non-wallet transaction - assert_raises_rpc_error(-5, 'Invalid or non-wallet transaction id', lambda: self.nodes[0].abandontransaction(txid='ff' * 32)) + assert_raises_rpc_error(-5, 'Invalid or non-wallet transaction id', lambda: alice.abandontransaction(txid='ff' * 32)) # Can not abandon confirmed transaction - assert_raises_rpc_error(-5, 'Transaction not eligible for abandonment', lambda: self.nodes[0].abandontransaction(txid=txA)) + assert_raises_rpc_error(-5, 'Transaction not eligible for abandonment', lambda: alice.abandontransaction(txid=txA)) - newbalance = self.nodes[0].getbalance() + newbalance = alice.getbalance() assert balance - newbalance < Decimal("0.001") #no more than fees lost balance = newbalance @@ -50,9 +55,9 @@ class AbandonConflictTest(BitcoinTestFramework): self.disconnect_nodes(0, 1) # Identify the 10btc outputs - nA = next(tx_out["vout"] for tx_out in self.nodes[0].gettransaction(txA)["details"] if tx_out["amount"] == Decimal("10")) - nB = next(tx_out["vout"] for tx_out in self.nodes[0].gettransaction(txB)["details"] if tx_out["amount"] == Decimal("10")) - nC = next(tx_out["vout"] for tx_out in self.nodes[0].gettransaction(txC)["details"] if tx_out["amount"] == Decimal("10")) + nA = next(tx_out["vout"] for tx_out in alice.gettransaction(txA)["details"] if tx_out["amount"] == Decimal("10")) + nB = next(tx_out["vout"] for tx_out in alice.gettransaction(txB)["details"] if tx_out["amount"] == Decimal("10")) + nC = next(tx_out["vout"] for tx_out in alice.gettransaction(txC)["details"] if tx_out["amount"] == Decimal("10")) inputs = [] # spend 10btc outputs from txA and txB @@ -60,39 +65,40 @@ class AbandonConflictTest(BitcoinTestFramework): inputs.append({"txid": txB, "vout": nB}) outputs = {} - outputs[self.nodes[0].getnewaddress()] = Decimal("14.99998") - outputs[self.nodes[1].getnewaddress()] = Decimal("5") - signed = self.nodes[0].signrawtransactionwithwallet(self.nodes[0].createrawtransaction(inputs, outputs)) + outputs[alice.getnewaddress()] = Decimal("14.99998") + outputs[bob.getnewaddress()] = Decimal("5") + signed = alice.signrawtransactionwithwallet(alice.createrawtransaction(inputs, outputs)) txAB1 = self.nodes[0].sendrawtransaction(signed["hex"]) # Identify the 14.99998btc output - nAB = next(tx_out["vout"] for tx_out in self.nodes[0].gettransaction(txAB1)["details"] if tx_out["amount"] == Decimal("14.99998")) + nAB = next(tx_out["vout"] for tx_out in alice.gettransaction(txAB1)["details"] if tx_out["amount"] == Decimal("14.99998")) #Create a child tx spending AB1 and C inputs = [] inputs.append({"txid": txAB1, "vout": nAB}) inputs.append({"txid": txC, "vout": nC}) outputs = {} - outputs[self.nodes[0].getnewaddress()] = Decimal("24.9996") - signed2 = self.nodes[0].signrawtransactionwithwallet(self.nodes[0].createrawtransaction(inputs, outputs)) + outputs[alice.getnewaddress()] = Decimal("24.9996") + signed2 = alice.signrawtransactionwithwallet(alice.createrawtransaction(inputs, outputs)) txABC2 = self.nodes[0].sendrawtransaction(signed2["hex"]) # Create a child tx spending ABC2 signed3_change = Decimal("24.999") inputs = [{"txid": txABC2, "vout": 0}] - outputs = {self.nodes[0].getnewaddress(): signed3_change} - signed3 = self.nodes[0].signrawtransactionwithwallet(self.nodes[0].createrawtransaction(inputs, outputs)) + outputs = {alice.getnewaddress(): signed3_change} + signed3 = alice.signrawtransactionwithwallet(alice.createrawtransaction(inputs, outputs)) # note tx is never directly referenced, only abandoned as a child of the above self.nodes[0].sendrawtransaction(signed3["hex"]) # In mempool txs from self should increase balance from change - newbalance = self.nodes[0].getbalance() + newbalance = alice.getbalance() assert_equal(newbalance, balance - Decimal("30") + signed3_change) balance = newbalance # Restart the node with a higher min relay fee so the parent tx is no longer in mempool # TODO: redo with eviction self.restart_node(0, extra_args=["-minrelaytxfee=0.0001"]) + alice = self.nodes[0].get_wallet_rpc(self.default_wallet_name) assert self.nodes[0].getmempoolinfo()['loaded'] # Verify txs no longer in either node's mempool @@ -101,25 +107,25 @@ class AbandonConflictTest(BitcoinTestFramework): # Not in mempool txs from self should only reduce balance # inputs are still spent, but change not received - newbalance = self.nodes[0].getbalance() + newbalance = alice.getbalance() assert_equal(newbalance, balance - signed3_change) # Unconfirmed received funds that are not in mempool, also shouldn't show # up in unconfirmed balance - balances = self.nodes[0].getbalances()['mine'] + balances = alice.getbalances()['mine'] assert_equal(balances['untrusted_pending'] + balances['trusted'], newbalance) # Also shouldn't show up in listunspent - assert not txABC2 in [utxo["txid"] for utxo in self.nodes[0].listunspent(0)] + assert not txABC2 in [utxo["txid"] for utxo in alice.listunspent(0)] balance = newbalance # Abandon original transaction and verify inputs are available again # including that the child tx was also abandoned - self.nodes[0].abandontransaction(txAB1) - newbalance = self.nodes[0].getbalance() + alice.abandontransaction(txAB1) + newbalance = alice.getbalance() assert_equal(newbalance, balance + Decimal("30")) balance = newbalance self.log.info("Check abandoned transactions in listsinceblock") - listsinceblock = self.nodes[0].listsinceblock() + listsinceblock = alice.listsinceblock() txAB1_listsinceblock = [d for d in listsinceblock['transactions'] if d['txid'] == txAB1 and d['category'] == 'send'] for tx in txAB1_listsinceblock: assert_equal(tx['abandoned'], True) @@ -128,49 +134,53 @@ class AbandonConflictTest(BitcoinTestFramework): # Verify that even with a low min relay fee, the tx is not reaccepted from wallet on startup once abandoned self.restart_node(0, extra_args=["-minrelaytxfee=0.00001"]) + alice = self.nodes[0].get_wallet_rpc(self.default_wallet_name) assert self.nodes[0].getmempoolinfo()['loaded'] assert_equal(len(self.nodes[0].getrawmempool()), 0) - assert_equal(self.nodes[0].getbalance(), balance) + assert_equal(alice.getbalance(), balance) # But if it is received again then it is unabandoned # And since now in mempool, the change is available # But its child tx remains abandoned self.nodes[0].sendrawtransaction(signed["hex"]) - newbalance = self.nodes[0].getbalance() + newbalance = alice.getbalance() assert_equal(newbalance, balance - Decimal("20") + Decimal("14.99998")) balance = newbalance # Send child tx again so it is unabandoned self.nodes[0].sendrawtransaction(signed2["hex"]) - newbalance = self.nodes[0].getbalance() + newbalance = alice.getbalance() assert_equal(newbalance, balance - Decimal("10") - Decimal("14.99998") + Decimal("24.9996")) balance = newbalance # Remove using high relay fee again self.restart_node(0, extra_args=["-minrelaytxfee=0.0001"]) + alice = self.nodes[0].get_wallet_rpc(self.default_wallet_name) assert self.nodes[0].getmempoolinfo()['loaded'] assert_equal(len(self.nodes[0].getrawmempool()), 0) - newbalance = self.nodes[0].getbalance() + newbalance = alice.getbalance() assert_equal(newbalance, balance - Decimal("24.9996")) balance = newbalance self.log.info("Test transactions conflicted by a double spend") + self.nodes[0].loadwallet("bob") + bob = self.nodes[0].get_wallet_rpc("bob") + # Create a double spend of AB1 by spending again from only A's 10 output # Mine double spend from node 1 inputs = [] inputs.append({"txid": txA, "vout": nA}) outputs = {} - outputs[self.nodes[1].getnewaddress()] = Decimal("9.9999") - tx = self.nodes[0].createrawtransaction(inputs, outputs) - signed = self.nodes[0].signrawtransactionwithwallet(tx) - self.nodes[1].sendrawtransaction(signed["hex"]) - self.generate(self.nodes[1], 1, sync_fun=self.no_op) - + outputs[self.nodes[1].getnewaddress()] = Decimal("3.9999") + outputs[bob.getnewaddress()] = Decimal("5.9999") + tx = alice.createrawtransaction(inputs, outputs) + signed = alice.signrawtransactionwithwallet(tx) + double_spend_txid = self.nodes[1].sendrawtransaction(signed["hex"]) self.connect_nodes(0, 1) - self.sync_blocks() + self.generate(self.nodes[1], 1) - tx_list = self.nodes[0].listtransactions() + tx_list = alice.listtransactions() conflicted = [tx for tx in tx_list if tx["confirmations"] < 0] assert_equal(4, len(conflicted)) @@ -179,7 +189,7 @@ class AbandonConflictTest(BitcoinTestFramework): assert_equal(2, len(wallet_conflicts)) double_spends = [tx for tx in tx_list if tx["walletconflicts"] and tx["confirmations"] > 0] - assert_equal(1, len(double_spends)) + assert_equal(2, len(double_spends)) # one for each output double_spend = double_spends[0] # Test the properties of the conflicted transactions, i.e. with confirmations < 0. @@ -198,8 +208,19 @@ class AbandonConflictTest(BitcoinTestFramework): assert_equal(double_spend["walletconflicts"], [tx["txid"]]) assert_equal(tx["walletconflicts"], [double_spend["txid"]]) + # Test walletconflicts on the receiver's side + txinfo = bob.gettransaction(txAB1) + assert_equal(txinfo['confirmations'], -1) + assert_equal(txinfo['walletconflicts'], [double_spend['txid']]) + + double_spends = [tx for tx in bob.listtransactions() if tx["walletconflicts"] and tx["confirmations"] > 0] + assert_equal(1, len(double_spends)) + double_spend = double_spends[0] + assert_equal(double_spend_txid, double_spend['txid']) + assert_equal(double_spend["walletconflicts"], [txAB1]) + # Verify that B and C's 10 BTC outputs are available for spending again because AB1 is now conflicted - newbalance = self.nodes[0].getbalance() + newbalance = alice.getbalance() assert_equal(newbalance, balance + Decimal("20")) balance = newbalance @@ -207,7 +228,7 @@ class AbandonConflictTest(BitcoinTestFramework): # Invalidate the block with the double spend and B's 10 BTC output should no longer be available # Don't think C's should either self.nodes[0].invalidateblock(self.nodes[0].getbestblockhash()) - newbalance = self.nodes[0].getbalance() + newbalance = alice.getbalance() #assert_equal(newbalance, balance - Decimal("10")) self.log.info("If balance has not declined after invalidateblock then out of mempool wallet tx which is no longer") self.log.info("conflicted has not resumed causing its inputs to be seen as spent. See Issue #7315") diff --git a/test/functional/wallet_avoidreuse.py b/test/functional/wallet_avoidreuse.py index dc823c2c60..f663666f57 100755 --- a/test/functional/wallet_avoidreuse.py +++ b/test/functional/wallet_avoidreuse.py @@ -118,6 +118,17 @@ class AvoidReuseTest(BitcoinTestFramework): assert_raises_rpc_error(-8, "Wallet flag is already set to false", self.nodes[0].setwalletflag, 'avoid_reuse', False) assert_raises_rpc_error(-8, "Wallet flag is already set to true", self.nodes[1].setwalletflag, 'avoid_reuse', True) + # Create a wallet with avoid reuse, and test that disabling it afterwards persists + self.nodes[1].createwallet(wallet_name="avoid_reuse_persist", avoid_reuse=True) + w = self.nodes[1].get_wallet_rpc("avoid_reuse_persist") + assert_equal(w.getwalletinfo()["avoid_reuse"], True) + w.setwalletflag("avoid_reuse", False) + assert_equal(w.getwalletinfo()["avoid_reuse"], False) + w.unloadwallet() + self.nodes[1].loadwallet("avoid_reuse_persist") + assert_equal(w.getwalletinfo()["avoid_reuse"], False) + w.unloadwallet() + def test_immutable(self): '''Test immutable wallet flags''' self.log.info("Test immutable wallet flags") diff --git a/test/functional/wallet_balance.py b/test/functional/wallet_balance.py index 0cfbefb719..0c93821e7f 100755 --- a/test/functional/wallet_balance.py +++ b/test/functional/wallet_balance.py @@ -50,7 +50,9 @@ class WalletTest(BitcoinTestFramework): self.num_nodes = 2 self.setup_clean_chain = True self.extra_args = [ - ['-limitdescendantcount=3'], # Limit mempool descendants as a hack to have wallet txs rejected from the mempool + # Limit mempool descendants as a hack to have wallet txs rejected from the mempool. + # Set walletrejectlongchains=0 so the wallet still creates the transactions. + ['-limitdescendantcount=3', '-walletrejectlongchains=0'], [], ] diff --git a/test/functional/wallet_basic.py b/test/functional/wallet_basic.py index 69f9df57d8..a6c93ba5f9 100755 --- a/test/functional/wallet_basic.py +++ b/test/functional/wallet_basic.py @@ -25,7 +25,7 @@ class WalletTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 4 self.extra_args = [[ - "-acceptnonstdtxn=1", + "-acceptnonstdtxn=1", "-walletrejectlongchains=0" ]] * self.num_nodes self.setup_clean_chain = True self.supports_cli = False @@ -142,7 +142,7 @@ class WalletTest(BitcoinTestFramework): self.nodes[2].lockunspent(False, [unspent_0], True) # Restarting the node with the lock written to the wallet should keep the lock - self.restart_node(2) + self.restart_node(2, ["-walletrejectlongchains=0"]) assert_raises_rpc_error(-8, "Invalid parameter, output already locked", self.nodes[2].lockunspent, False, [unspent_0]) # Unloading and reloading the wallet with a persistent lock should keep the lock @@ -568,7 +568,7 @@ class WalletTest(BitcoinTestFramework): self.log.info("Test -reindex") self.stop_nodes() # set lower ancestor limit for later - self.start_node(0, ['-reindex', "-limitancestorcount=" + str(chainlimit)]) + self.start_node(0, ['-reindex', "-walletrejectlongchains=0", "-limitancestorcount=" + str(chainlimit)]) self.start_node(1, ['-reindex', "-limitancestorcount=" + str(chainlimit)]) self.start_node(2, ['-reindex', "-limitancestorcount=" + str(chainlimit)]) # reindex will leave rpc warm up "early"; Wait for it to finish @@ -668,7 +668,7 @@ class WalletTest(BitcoinTestFramework): "category": baz["category"], "vout": baz["vout"]} expected_fields = frozenset({'amount', 'bip125-replaceable', 'confirmations', 'details', 'fee', - 'hex', 'time', 'timereceived', 'trusted', 'txid', 'walletconflicts'}) + 'hex', 'time', 'timereceived', 'trusted', 'txid', 'wtxid', 'walletconflicts'}) verbose_field = "decoded" expected_verbose_fields = expected_fields | {verbose_field} diff --git a/test/functional/wallet_bumpfee.py b/test/functional/wallet_bumpfee.py index f6843d597d..3b23ee8e94 100755 --- a/test/functional/wallet_bumpfee.py +++ b/test/functional/wallet_bumpfee.py @@ -442,7 +442,9 @@ def test_watchonly_psbt(self, peer_node, rbf_node, dest_address): self.generate(peer_node, 1) # Create single-input PSBT for transaction to be bumped - psbt = watcher.walletcreatefundedpsbt([], {dest_address: 0.0005}, 0, {"fee_rate": 1}, True)['psbt'] + # Ensure the payment amount + change can be fully funded using one of the 0.001BTC inputs. + psbt = watcher.walletcreatefundedpsbt([watcher.listunspent()[0]], {dest_address: 0.0005}, 0, + {"fee_rate": 1, "add_inputs": False}, True)['psbt'] psbt_signed = signer.walletprocesspsbt(psbt=psbt, sign=True, sighashtype="ALL", bip32derivs=True) psbt_final = watcher.finalizepsbt(psbt_signed["psbt"]) original_txid = watcher.sendrawtransaction(psbt_final["hex"]) diff --git a/test/functional/wallet_createwallet.py b/test/functional/wallet_createwallet.py index 4416a9655f..12480d4d1e 100755 --- a/test/functional/wallet_createwallet.py +++ b/test/functional/wallet_createwallet.py @@ -26,6 +26,11 @@ class CreateWalletTest(BitcoinTestFramework): node = self.nodes[0] self.generate(node, 1) # Leave IBD for sethdseed + self.log.info("Run createwallet with invalid parameters.") + # Run createwallet with invalid parameters. This must not prevent a new wallet with the same name from being created with the correct parameters. + assert_raises_rpc_error(-4, "Passphrase provided but private keys are disabled. A passphrase is only used to encrypt private keys, so cannot be used for wallets with private keys disabled.", + self.nodes[0].createwallet, wallet_name='w0', disable_private_keys=True, passphrase="passphrase") + self.nodes[0].createwallet(wallet_name='w0') w0 = node.get_wallet_rpc('w0') address1 = w0.getnewaddress() @@ -164,5 +169,10 @@ class CreateWalletTest(BitcoinTestFramework): self.log.info('Using a passphrase with private keys disabled returns error') assert_raises_rpc_error(-4, 'Passphrase provided but private keys are disabled. A passphrase is only used to encrypt private keys, so cannot be used for wallets with private keys disabled.', self.nodes[0].createwallet, wallet_name='w9', disable_private_keys=True, passphrase='thisisapassphrase') + if self.is_bdb_compiled(): + self.log.info("Test legacy wallet deprecation") + res = self.nodes[0].createwallet(wallet_name="legacy_w0", descriptors=False, passphrase=None) + assert_equal(res["warning"], "Wallet created successfully. The legacy wallet type is being deprecated and support for creating and opening legacy wallets will be removed in the future.") + if __name__ == '__main__': CreateWalletTest().main() diff --git a/test/functional/wallet_crosschain.py b/test/functional/wallet_crosschain.py new file mode 100755 index 0000000000..b6d0c87985 --- /dev/null +++ b/test/functional/wallet_crosschain.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +import os + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_raises_rpc_error + +class WalletCrossChain(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 2 + self.setup_clean_chain = True + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + + def setup_network(self): + self.add_nodes(self.num_nodes) + + # Switch node 1 to testnet before starting it. + self.nodes[1].chain = 'testnet3' + self.nodes[1].extra_args = ['-maxconnections=0'] # disable testnet sync + with open(self.nodes[1].bitcoinconf, 'r', encoding='utf8') as conf: + conf_data = conf.read() + with open (self.nodes[1].bitcoinconf, 'w', encoding='utf8') as conf: + conf.write(conf_data.replace('regtest=', 'testnet=').replace('[regtest]', '[test]')) + + self.start_nodes() + + def run_test(self): + self.log.info("Creating wallets") + + node0_wallet = os.path.join(self.nodes[0].datadir, 'node0_wallet') + self.nodes[0].createwallet(node0_wallet) + self.nodes[0].unloadwallet(node0_wallet) + node1_wallet = os.path.join(self.nodes[1].datadir, 'node1_wallet') + self.nodes[1].createwallet(node1_wallet) + self.nodes[1].unloadwallet(node1_wallet) + + self.log.info("Loading wallets into nodes with a different genesis blocks") + + if self.options.descriptors: + assert_raises_rpc_error(-18, 'Wallet file verification failed.', self.nodes[0].loadwallet, node1_wallet) + assert_raises_rpc_error(-18, 'Wallet file verification failed.', self.nodes[1].loadwallet, node0_wallet) + else: + assert_raises_rpc_error(-4, 'Wallet files should not be reused across chains.', self.nodes[0].loadwallet, node1_wallet) + assert_raises_rpc_error(-4, 'Wallet files should not be reused across chains.', self.nodes[1].loadwallet, node0_wallet) + + if not self.options.descriptors: + self.log.info("Override cross-chain wallet load protection") + self.stop_nodes() + self.start_nodes([['-walletcrosschain']] * self.num_nodes) + self.nodes[0].loadwallet(node1_wallet) + self.nodes[1].loadwallet(node0_wallet) + + +if __name__ == '__main__': + WalletCrossChain().main() diff --git a/test/functional/wallet_disable.py b/test/functional/wallet_disable.py index 2c7996ca6b..74cddf2738 100755 --- a/test/functional/wallet_disable.py +++ b/test/functional/wallet_disable.py @@ -26,10 +26,6 @@ class DisableWalletTest (BitcoinTestFramework): x = self.nodes[0].validateaddress('mneYUmWYsuk7kySiURxCi3AGxrAqZxLgPZ') assert x['isvalid'] == True - # Checking mining to an address without a wallet. Generating to a valid address should succeed - # but generating to an invalid address will fail. - self.generatetoaddress(self.nodes[0], 1, 'mneYUmWYsuk7kySiURxCi3AGxrAqZxLgPZ') - assert_raises_rpc_error(-5, "Invalid address", self.generatetoaddress, self.nodes[0], 1, '3J98t1WpEZ73CNmQviecrnyiWrnqRhWNLy') if __name__ == '__main__': - DisableWalletTest ().main () + DisableWalletTest().main() diff --git a/test/functional/wallet_encryption.py b/test/functional/wallet_encryption.py index 0d702e44f6..0c9106f800 100755 --- a/test/functional/wallet_encryption.py +++ b/test/functional/wallet_encryption.py @@ -35,14 +35,14 @@ class WalletEncryptionTest(BitcoinTestFramework): assert_raises_rpc_error(-15, "Error: running with an unencrypted wallet, but walletpassphrasechange was called.", self.nodes[0].walletpassphrasechange, 'ff', 'ff') # Encrypt the wallet - assert_raises_rpc_error(-8, "passphrase can not be empty", self.nodes[0].encryptwallet, '') + assert_raises_rpc_error(-8, "passphrase cannot be empty", self.nodes[0].encryptwallet, '') self.nodes[0].encryptwallet(passphrase) # Test that the wallet is encrypted assert_raises_rpc_error(-13, "Please enter the wallet passphrase with walletpassphrase first", self.nodes[0].signmessage, address, msg) assert_raises_rpc_error(-15, "Error: running with an encrypted wallet, but encryptwallet was called.", self.nodes[0].encryptwallet, 'ff') - assert_raises_rpc_error(-8, "passphrase can not be empty", self.nodes[0].walletpassphrase, '', 1) - assert_raises_rpc_error(-8, "passphrase can not be empty", self.nodes[0].walletpassphrasechange, '', 'ff') + assert_raises_rpc_error(-8, "passphrase cannot be empty", self.nodes[0].walletpassphrase, '', 1) + assert_raises_rpc_error(-8, "passphrase cannot be empty", self.nodes[0].walletpassphrasechange, '', 'ff') # Check that walletpassphrase works self.nodes[0].walletpassphrase(passphrase, 2) diff --git a/test/functional/wallet_importprunedfunds.py b/test/functional/wallet_importprunedfunds.py index cdb5823109..2a4d0981c7 100755 --- a/test/functional/wallet_importprunedfunds.py +++ b/test/functional/wallet_importprunedfunds.py @@ -5,9 +5,13 @@ """Test the importprunedfunds and removeprunedfunds RPCs.""" from decimal import Decimal -from test_framework.blocktools import COINBASE_MATURITY from test_framework.address import key_to_p2wpkh +from test_framework.blocktools import COINBASE_MATURITY from test_framework.key import ECKey +from test_framework.messages import ( + CMerkleBlock, + from_hex, +) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, @@ -15,6 +19,7 @@ from test_framework.util import ( ) from test_framework.wallet_util import bytes_to_wif + class ImportPrunedFundsTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True @@ -124,5 +129,18 @@ class ImportPrunedFundsTest(BitcoinTestFramework): w1.removeprunedfunds(txnid3) assert not [tx for tx in w1.listtransactions(include_watchonly=True) if tx['txid'] == txnid3] + # Check various RPC parameter validation errors + assert_raises_rpc_error(-22, "TX decode failed", w1.importprunedfunds, b'invalid tx'.hex(), proof1) + assert_raises_rpc_error(-5, "Transaction given doesn't exist in proof", w1.importprunedfunds, rawtxn2, proof1) + + mb = from_hex(CMerkleBlock(), proof1) + mb.header.hashMerkleRoot = 0xdeadbeef # cause mismatch between merkle root and merkle block + assert_raises_rpc_error(-5, "Something wrong with merkleblock", w1.importprunedfunds, rawtxn1, mb.serialize().hex()) + + mb = from_hex(CMerkleBlock(), proof1) + mb.header.nTime += 1 # modify arbitrary block header field to change block hash + assert_raises_rpc_error(-5, "Block not found in chain", w1.importprunedfunds, rawtxn1, mb.serialize().hex()) + + if __name__ == '__main__': ImportPrunedFundsTest().main() diff --git a/test/functional/wallet_inactive_hdchains.py b/test/functional/wallet_inactive_hdchains.py new file mode 100755 index 0000000000..e1dad00876 --- /dev/null +++ b/test/functional/wallet_inactive_hdchains.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +# Copyright (c) 2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +""" +Test Inactive HD Chains. +""" +import os +import shutil +import time + +from test_framework.authproxy import JSONRPCException +from test_framework.test_framework import BitcoinTestFramework +from test_framework.wallet_util import ( + get_generate_key, +) + + +class InactiveHDChainsTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 2 + self.extra_args = [["-keypool=10"], ["-nowallet", "-keypool=10"]] + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + self.skip_if_no_bdb() + self.skip_if_no_previous_releases() + + def setup_nodes(self): + self.add_nodes(self.num_nodes, extra_args=self.extra_args, versions=[ + None, + 170200, # 0.17.2 Does not have the key metadata upgrade + ]) + + self.start_nodes() + self.init_wallet(node=0) + + def prepare_wallets(self, wallet_basename, encrypt=False): + self.nodes[0].createwallet(wallet_name=f"{wallet_basename}_base", descriptors=False, blank=True) + self.nodes[0].createwallet(wallet_name=f"{wallet_basename}_test", descriptors=False, blank=True) + base_wallet = self.nodes[0].get_wallet_rpc(f"{wallet_basename}_base") + test_wallet = self.nodes[0].get_wallet_rpc(f"{wallet_basename}_test") + + # Setup both wallets with the same HD seed + seed = get_generate_key() + base_wallet.sethdseed(True, seed.privkey) + test_wallet.sethdseed(True, seed.privkey) + + if encrypt: + # Encrypting will generate a new HD seed and flush the keypool + test_wallet.encryptwallet("pass") + else: + # Generate a new HD seed on the test wallet + test_wallet.sethdseed() + + return base_wallet, test_wallet + + def do_inactive_test(self, base_wallet, test_wallet, encrypt=False): + default = self.nodes[0].get_wallet_rpc(self.default_wallet_name) + + # The first address should be known by both wallets. + addr1 = base_wallet.getnewaddress() + assert test_wallet.getaddressinfo(addr1)["ismine"] + # The address at index 9 is the first address that the test wallet will not know initially + for _ in range(0, 9): + base_wallet.getnewaddress() + addr2 = base_wallet.getnewaddress() + assert not test_wallet.getaddressinfo(addr2)["ismine"] + + # Send to first address on the old seed + txid = default.sendtoaddress(addr1, 10) + self.generate(self.nodes[0], 1) + + # Wait for the test wallet to see the transaction + while True: + try: + test_wallet.gettransaction(txid) + break + except JSONRPCException: + time.sleep(0.1) + + if encrypt: + # The test wallet will not be able to generate the topped up keypool + # until it is unlocked. So it still should not know about the second address + assert not test_wallet.getaddressinfo(addr2)["ismine"] + test_wallet.walletpassphrase("pass", 1) + + # The test wallet should now know about the second address as it + # should have generated it in the inactive chain's keypool + assert test_wallet.getaddressinfo(addr2)["ismine"] + + # Send to second address on the old seed + txid = default.sendtoaddress(addr2, 10) + self.generate(self.nodes[0], 1) + test_wallet.gettransaction(txid) + + def test_basic(self): + self.log.info("Test basic case for inactive HD chains") + self.do_inactive_test(*self.prepare_wallets("basic")) + + def test_encrypted_wallet(self): + self.log.info("Test inactive HD chains when wallet is encrypted") + self.do_inactive_test(*self.prepare_wallets("enc", encrypt=True), encrypt=True) + + def test_without_upgraded_keymeta(self): + # Test that it is possible to top up inactive hd chains even if there is no key origin + # in CKeyMetadata. This tests for the segfault reported in + # https://github.com/bitcoin/bitcoin/issues/21605 + self.log.info("Test that topping up inactive HD chains does not need upgraded key origin") + + self.nodes[0].createwallet(wallet_name="keymeta_base", descriptors=False, blank=True) + # Createwallet is overridden in the test framework so that the descriptor option can be filled + # depending on the test's cli args. However we don't want to do that when using old nodes that + # do not support descriptors. So we use the createwallet_passthrough function. + self.nodes[1].createwallet_passthrough(wallet_name="keymeta_test") + base_wallet = self.nodes[0].get_wallet_rpc("keymeta_base") + test_wallet = self.nodes[1].get_wallet_rpc("keymeta_test") + + # Setup both wallets with the same HD seed + seed = get_generate_key() + base_wallet.sethdseed(True, seed.privkey) + test_wallet.sethdseed(True, seed.privkey) + + # Encrypting will generate a new HD seed and flush the keypool + test_wallet.encryptwallet("pass") + + # Copy test wallet to node 0 + test_wallet.unloadwallet() + test_wallet_dir = os.path.join(self.nodes[1].datadir, "regtest/wallets/keymeta_test") + new_test_wallet_dir = os.path.join(self.nodes[0].datadir, "regtest/wallets/keymeta_test") + shutil.copytree(test_wallet_dir, new_test_wallet_dir) + self.nodes[0].loadwallet("keymeta_test") + test_wallet = self.nodes[0].get_wallet_rpc("keymeta_test") + + self.do_inactive_test(base_wallet, test_wallet, encrypt=True) + + def run_test(self): + self.generate(self.nodes[0], 101) + + self.test_basic() + self.test_encrypted_wallet() + self.test_without_upgraded_keymeta() + + +if __name__ == '__main__': + InactiveHDChainsTest().main() diff --git a/test/functional/wallet_listreceivedby.py b/test/functional/wallet_listreceivedby.py index 48b92796fc..db1d8eb54a 100755 --- a/test/functional/wallet_listreceivedby.py +++ b/test/functional/wallet_listreceivedby.py @@ -18,17 +18,12 @@ from test_framework.wallet_util import test_address class ReceivedByTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 - # Test deprecated exclude coinbase on second node - self.extra_args = [[], ["-deprecatedrpc=exclude_coinbase"]] def skip_test_if_missing_module(self): self.skip_if_no_wallet() self.skip_if_no_cli() def run_test(self): - # Generate block to get out of IBD - self.generate(self.nodes[0], 1) - # save the number of coinbase reward addresses so far num_cb_reward_addresses = len(self.nodes[1].listreceivedbyaddress(minconf=0, include_empty=True, include_watchonly=True)) @@ -134,6 +129,9 @@ class ReceivedByTest(BitcoinTestFramework): txid = self.nodes[0].sendtoaddress(addr, 0.1) self.sync_all() + # getreceivedbylabel returns an error if the wallet doesn't own the label + assert_raises_rpc_error(-4, "Label not found in wallet", self.nodes[0].getreceivedbylabel, "dummy") + # listreceivedbylabel should return received_by_label_json because of 0 confirmations assert_array_result(self.nodes[1].listreceivedbylabel(), {"label": label}, @@ -172,7 +170,7 @@ class ReceivedByTest(BitcoinTestFramework): address = self.nodes[0].getnewaddress(label) reward = Decimal("25") - self.generatetoaddress(self.nodes[0], 1, address, sync_fun=self.no_op) + self.generatetoaddress(self.nodes[0], 1, address) hash = self.nodes[0].getbestblockhash() self.log.info("getreceivedbyaddress returns nothing with defaults") @@ -212,7 +210,7 @@ class ReceivedByTest(BitcoinTestFramework): {"label": label, "amount": reward}) self.log.info("Generate 100 more blocks") - self.generate(self.nodes[0], COINBASE_MATURITY, sync_fun=self.no_op) + self.generate(self.nodes[0], COINBASE_MATURITY) self.log.info("getreceivedbyaddress returns reward with defaults") balance = self.nodes[0].getreceivedbyaddress(address) @@ -253,35 +251,6 @@ class ReceivedByTest(BitcoinTestFramework): {"label": label}, {}, True) - # Test exclude_coinbase - address2 = self.nodes[1].getnewaddress(label) - self.generatetoaddress(self.nodes[1], COINBASE_MATURITY + 1, address2, sync_fun=self.no_op) - - self.log.info("getreceivedbyaddress returns nothing when excluding coinbase") - balance = self.nodes[1].getreceivedbyaddress(address2) - assert_equal(balance, 0) - - self.log.info("getreceivedbylabel returns nothing when excluding coinbase") - balance = self.nodes[1].getreceivedbylabel("label") - assert_equal(balance, 0) - - self.log.info("listreceivedbyaddress does not include address when excluding coinbase") - assert_array_result(self.nodes[1].listreceivedbyaddress(), - {"address": address2}, - {}, True) - - self.log.info("listreceivedbylabel does not include label when excluding coinbase") - assert_array_result(self.nodes[1].listreceivedbylabel(), - {"label": label}, - {}, True) - - self.log.info("getreceivedbyaddress throws when setting include_immature_coinbase with deprecated exclude_coinbase") - assert_raises_rpc_error(-8, 'include_immature_coinbase is incompatible with deprecated exclude_coinbase', self.nodes[1].getreceivedbyaddress, address2, 1, True) - - - self.log.info("listreceivedbyaddress throws when setting include_immature_coinbase with deprecated exclude_coinbase") - assert_raises_rpc_error(-8, 'include_immature_coinbase is incompatible with deprecated exclude_coinbase', self.nodes[1].listreceivedbyaddress, 1, False, False, "", True) - if __name__ == '__main__': ReceivedByTest().main() diff --git a/test/functional/wallet_multiwallet.py b/test/functional/wallet_multiwallet.py index 0b868dde6c..dcb82bbbe9 100755 --- a/test/functional/wallet_multiwallet.py +++ b/test/functional/wallet_multiwallet.py @@ -11,6 +11,7 @@ from threading import Thread import os import shutil import stat +import sys import time from test_framework.authproxy import JSONRPCException @@ -141,7 +142,7 @@ class MultiWalletTest(BitcoinTestFramework): # should raise rpc error if wallet path can't be created err_code = -4 if self.options.descriptors else -1 - assert_raises_rpc_error(err_code, "boost::filesystem::create_directory:", self.nodes[0].createwallet, "w8/bad") + assert_raises_rpc_error(err_code, "filesystem error:" if sys.platform != 'win32' else "create_directories:", self.nodes[0].createwallet, "w8/bad") # check that all requested wallets were created self.stop_node(0) diff --git a/test/functional/wallet_resendwallettransactions.py b/test/functional/wallet_resendwallettransactions.py index 5aae2c813a..6552bfe60c 100755 --- a/test/functional/wallet_resendwallettransactions.py +++ b/test/functional/wallet_resendwallettransactions.py @@ -38,7 +38,7 @@ class ResendWalletTransactionsTest(BitcoinTestFramework): # Can take a few seconds due to transaction trickling peer_first.wait_for_broadcast([txid]) - # Add a second peer since txs aren't rebroadcast to the same peer (see filterInventoryKnown) + # Add a second peer since txs aren't rebroadcast to the same peer (see m_tx_inventory_known_filter) peer_second = node.add_p2p_connection(P2PTxInvStore()) self.log.info("Create a block") diff --git a/test/functional/wallet_send.py b/test/functional/wallet_send.py index d77d554baa..07baa0595e 100755 --- a/test/functional/wallet_send.py +++ b/test/functional/wallet_send.py @@ -10,12 +10,17 @@ from itertools import product from test_framework.authproxy import JSONRPCException from test_framework.descriptors import descsum_create from test_framework.key import ECKey +from test_framework.messages import ( + ser_compact_size, + WITNESS_SCALE_FACTOR, +) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, assert_fee_amount, assert_greater_than, assert_raises_rpc_error, + count_bytes, ) from test_framework.wallet_util import bytes_to_wif @@ -320,20 +325,20 @@ class WalletSendTest(BitcoinTestFramework): res = self.test_send(from_wallet=w0, to_wallet=w1, amount=1, fee_rate=7, add_to_wallet=False) fee = self.nodes[1].decodepsbt(res["psbt"])["fee"] - assert_fee_amount(fee, Decimal(len(res["hex"]) / 2), Decimal("0.00007")) + assert_fee_amount(fee, count_bytes(res["hex"]), Decimal("0.00007")) # "unset" and None are treated the same for estimate_mode res = self.test_send(from_wallet=w0, to_wallet=w1, amount=1, fee_rate=2, estimate_mode="unset", add_to_wallet=False) fee = self.nodes[1].decodepsbt(res["psbt"])["fee"] - assert_fee_amount(fee, Decimal(len(res["hex"]) / 2), Decimal("0.00002")) + assert_fee_amount(fee, count_bytes(res["hex"]), Decimal("0.00002")) res = self.test_send(from_wallet=w0, to_wallet=w1, amount=1, arg_fee_rate=4.531, add_to_wallet=False) fee = self.nodes[1].decodepsbt(res["psbt"])["fee"] - assert_fee_amount(fee, Decimal(len(res["hex"]) / 2), Decimal("0.00004531")) + assert_fee_amount(fee, count_bytes(res["hex"]), Decimal("0.00004531")) res = self.test_send(from_wallet=w0, to_wallet=w1, amount=1, arg_fee_rate=3, add_to_wallet=False) fee = self.nodes[1].decodepsbt(res["psbt"])["fee"] - assert_fee_amount(fee, Decimal(len(res["hex"]) / 2), Decimal("0.00003")) + assert_fee_amount(fee, count_bytes(res["hex"]), Decimal("0.00003")) # Test that passing fee_rate as both an argument and an option raises. self.test_send(from_wallet=w0, to_wallet=w1, amount=1, arg_fee_rate=1, fee_rate=1, add_to_wallet=False, @@ -487,8 +492,8 @@ class WalletSendTest(BitcoinTestFramework): self.nodes[1].createwallet("extfund") ext_fund = self.nodes[1].get_wallet_rpc("extfund") - # Make a weird but signable script. sh(pkh()) descriptor accomplishes this - desc = descsum_create("sh(pkh({}))".format(privkey)) + # Make a weird but signable script. sh(wsh(pkh())) descriptor accomplishes this + desc = descsum_create("sh(wsh(pkh({})))".format(privkey)) if self.options.descriptors: res = ext_fund.importdescriptors([{"desc": desc, "timestamp": "now"}]) else: @@ -506,7 +511,7 @@ class WalletSendTest(BitcoinTestFramework): self.test_send(from_wallet=ext_wallet, to_wallet=self.nodes[0], amount=15, inputs=[ext_utxo], add_inputs=True, psbt=True, include_watching=True, expect_error=(-4, "Insufficient funds")) # But funding should work when the solving data is provided - res = self.test_send(from_wallet=ext_wallet, to_wallet=self.nodes[0], amount=15, inputs=[ext_utxo], add_inputs=True, psbt=True, include_watching=True, solving_data={"pubkeys": [addr_info['pubkey']], "scripts": [addr_info["embedded"]["scriptPubKey"]]}) + res = self.test_send(from_wallet=ext_wallet, to_wallet=self.nodes[0], amount=15, inputs=[ext_utxo], add_inputs=True, psbt=True, include_watching=True, solving_data={"pubkeys": [addr_info['pubkey']], "scripts": [addr_info["embedded"]["scriptPubKey"], addr_info["embedded"]["embedded"]["scriptPubKey"]]}) signed = ext_wallet.walletprocesspsbt(res["psbt"]) signed = ext_fund.walletprocesspsbt(res["psbt"]) assert signed["complete"] @@ -518,5 +523,46 @@ class WalletSendTest(BitcoinTestFramework): assert signed["complete"] self.nodes[0].finalizepsbt(signed["psbt"]) + dec = self.nodes[0].decodepsbt(signed["psbt"]) + for i, txin in enumerate(dec["tx"]["vin"]): + if txin["txid"] == ext_utxo["txid"] and txin["vout"] == ext_utxo["vout"]: + input_idx = i + break + psbt_in = dec["inputs"][input_idx] + # Calculate the input weight + # (prevout + sequence + length of scriptSig + scriptsig + 1 byte buffer) * WITNESS_SCALE_FACTOR + num scriptWitness stack items + (length of stack item + stack item) * N stack items + 1 byte buffer + len_scriptsig = len(psbt_in["final_scriptSig"]["hex"]) // 2 if "final_scriptSig" in psbt_in else 0 + len_scriptsig += len(ser_compact_size(len_scriptsig)) + 1 + len_scriptwitness = (sum([(len(x) // 2) + len(ser_compact_size(len(x) // 2)) for x in psbt_in["final_scriptwitness"]]) + len(psbt_in["final_scriptwitness"]) + 1) if "final_scriptwitness" in psbt_in else 0 + input_weight = ((40 + len_scriptsig) * WITNESS_SCALE_FACTOR) + len_scriptwitness + + # Input weight error conditions + assert_raises_rpc_error( + -8, + "Input weights should be specified in inputs rather than in options.", + ext_wallet.send, + outputs={self.nodes[0].getnewaddress(): 15}, + options={"inputs": [ext_utxo], "input_weights": [{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": 1000}]} + ) + + # Funding should also work when input weights are provided + res = self.test_send( + from_wallet=ext_wallet, + to_wallet=self.nodes[0], + amount=15, + inputs=[{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": input_weight}], + add_inputs=True, + psbt=True, + include_watching=True, + fee_rate=10 + ) + signed = ext_wallet.walletprocesspsbt(res["psbt"]) + signed = ext_fund.walletprocesspsbt(res["psbt"]) + assert signed["complete"] + tx = self.nodes[0].finalizepsbt(signed["psbt"]) + testres = self.nodes[0].testmempoolaccept([tx["hex"]])[0] + assert_equal(testres["allowed"], True) + assert_fee_amount(testres["fees"]["base"], testres["vsize"], Decimal(0.0001)) + if __name__ == '__main__': WalletSendTest().main() diff --git a/test/functional/wallet_sendall.py b/test/functional/wallet_sendall.py new file mode 100755 index 0000000000..aa8d2a9d2c --- /dev/null +++ b/test/functional/wallet_sendall.py @@ -0,0 +1,316 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test the sendall RPC command.""" + +from decimal import Decimal, getcontext + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + assert_greater_than, + assert_raises_rpc_error, +) + +# Decorator to reset activewallet to zero utxos +def cleanup(func): + def wrapper(self): + try: + func(self) + finally: + if 0 < self.wallet.getbalances()["mine"]["trusted"]: + self.wallet.sendall([self.remainder_target]) + assert_equal(0, self.wallet.getbalances()["mine"]["trusted"]) # wallet is empty + return wrapper + +class SendallTest(BitcoinTestFramework): + # Setup and helpers + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + + def set_test_params(self): + getcontext().prec=10 + self.num_nodes = 1 + self.setup_clean_chain = True + + def assert_balance_swept_completely(self, tx, balance): + output_sum = sum([o["value"] for o in tx["decoded"]["vout"]]) + assert_equal(output_sum, balance + tx["fee"]) + assert_equal(0, self.wallet.getbalances()["mine"]["trusted"]) # wallet is empty + + def assert_tx_has_output(self, tx, addr, value=None): + for output in tx["decoded"]["vout"]: + if addr == output["scriptPubKey"]["address"] and value is None or value == output["value"]: + return + raise AssertionError("Output to {} not present or wrong amount".format(addr)) + + def assert_tx_has_outputs(self, tx, expected_outputs): + assert_equal(len(expected_outputs), len(tx["decoded"]["vout"])) + for eo in expected_outputs: + self.assert_tx_has_output(tx, eo["address"], eo["value"]) + + def add_utxos(self, amounts): + for a in amounts: + self.def_wallet.sendtoaddress(self.wallet.getnewaddress(), a) + self.generate(self.nodes[0], 1) + assert_greater_than(self.wallet.getbalances()["mine"]["trusted"], 0) + return self.wallet.getbalances()["mine"]["trusted"] + + # Helper schema for success cases + def test_sendall_success(self, sendall_args, remaining_balance = 0): + sendall_tx_receipt = self.wallet.sendall(sendall_args) + self.generate(self.nodes[0], 1) + # wallet has remaining balance (usually empty) + assert_equal(remaining_balance, self.wallet.getbalances()["mine"]["trusted"]) + + assert_equal(sendall_tx_receipt["complete"], True) + return self.wallet.gettransaction(txid = sendall_tx_receipt["txid"], verbose = True) + + @cleanup + def gen_and_clean(self): + self.add_utxos([15, 2, 4]) + + def test_cleanup(self): + self.log.info("Test that cleanup wrapper empties wallet") + self.gen_and_clean() + assert_equal(0, self.wallet.getbalances()["mine"]["trusted"]) # wallet is empty + + # Actual tests + @cleanup + def sendall_two_utxos(self): + self.log.info("Testing basic sendall case without specific amounts") + pre_sendall_balance = self.add_utxos([10,11]) + tx_from_wallet = self.test_sendall_success(sendall_args = [self.remainder_target]) + + self.assert_tx_has_outputs(tx = tx_from_wallet, + expected_outputs = [ + { "address": self.remainder_target, "value": pre_sendall_balance + tx_from_wallet["fee"] } # fee is neg + ] + ) + self.assert_balance_swept_completely(tx_from_wallet, pre_sendall_balance) + + @cleanup + def sendall_split(self): + self.log.info("Testing sendall where two recipients have unspecified amount") + pre_sendall_balance = self.add_utxos([1, 2, 3, 15]) + tx_from_wallet = self.test_sendall_success([self.remainder_target, self.split_target]) + + half = (pre_sendall_balance + tx_from_wallet["fee"]) / 2 + self.assert_tx_has_outputs(tx_from_wallet, + expected_outputs = [ + { "address": self.split_target, "value": half }, + { "address": self.remainder_target, "value": half } + ] + ) + self.assert_balance_swept_completely(tx_from_wallet, pre_sendall_balance) + + @cleanup + def sendall_and_spend(self): + self.log.info("Testing sendall in combination with paying specified amount to recipient") + pre_sendall_balance = self.add_utxos([8, 13]) + tx_from_wallet = self.test_sendall_success([{self.recipient: 5}, self.remainder_target]) + + self.assert_tx_has_outputs(tx_from_wallet, + expected_outputs = [ + { "address": self.recipient, "value": 5 }, + { "address": self.remainder_target, "value": pre_sendall_balance - 5 + tx_from_wallet["fee"] } + ] + ) + self.assert_balance_swept_completely(tx_from_wallet, pre_sendall_balance) + + @cleanup + def sendall_invalid_recipient_addresses(self): + self.log.info("Test having only recipient with specified amount, missing recipient with unspecified amount") + self.add_utxos([12, 9]) + + assert_raises_rpc_error( + -8, + "Must provide at least one address without a specified amount" , + self.wallet.sendall, + [{self.recipient: 5}] + ) + + @cleanup + def sendall_duplicate_recipient(self): + self.log.info("Test duplicate destination") + self.add_utxos([1, 8, 3, 9]) + + assert_raises_rpc_error( + -8, + "Invalid parameter, duplicated address: {}".format(self.remainder_target), + self.wallet.sendall, + [self.remainder_target, self.remainder_target] + ) + + @cleanup + def sendall_invalid_amounts(self): + self.log.info("Test sending more than balance") + pre_sendall_balance = self.add_utxos([7, 14]) + + expected_tx = self.wallet.sendall(recipients=[{self.recipient: 5}, self.remainder_target], options={"add_to_wallet": False}) + tx = self.wallet.decoderawtransaction(expected_tx['hex']) + fee = 21 - sum([o["value"] for o in tx["vout"]]) + + assert_raises_rpc_error(-6, "Assigned more value to outputs than available funds.", self.wallet.sendall, + [{self.recipient: pre_sendall_balance + 1}, self.remainder_target]) + assert_raises_rpc_error(-6, "Insufficient funds for fees after creating specified outputs.", self.wallet.sendall, + [{self.recipient: pre_sendall_balance}, self.remainder_target]) + assert_raises_rpc_error(-8, "Specified output amount to {} is below dust threshold".format(self.recipient), + self.wallet.sendall, [{self.recipient: 0.00000001}, self.remainder_target]) + assert_raises_rpc_error(-6, "Dynamically assigned remainder results in dust output.", self.wallet.sendall, + [{self.recipient: pre_sendall_balance - fee}, self.remainder_target]) + assert_raises_rpc_error(-6, "Dynamically assigned remainder results in dust output.", self.wallet.sendall, + [{self.recipient: pre_sendall_balance - fee - Decimal(0.00000010)}, self.remainder_target]) + + # @cleanup not needed because different wallet used + def sendall_negative_effective_value(self): + self.log.info("Test that sendall fails if all UTXOs have negative effective value") + # Use dedicated wallet for dust amounts and unload wallet at end + self.nodes[0].createwallet("dustwallet") + dust_wallet = self.nodes[0].get_wallet_rpc("dustwallet") + + self.def_wallet.sendtoaddress(dust_wallet.getnewaddress(), 0.00000400) + self.def_wallet.sendtoaddress(dust_wallet.getnewaddress(), 0.00000300) + self.generate(self.nodes[0], 1) + assert_greater_than(dust_wallet.getbalances()["mine"]["trusted"], 0) + + assert_raises_rpc_error(-6, "Total value of UTXO pool too low to pay for transaction." + + " Try using lower feerate or excluding uneconomic UTXOs with 'send_max' option.", + dust_wallet.sendall, recipients=[self.remainder_target], fee_rate=300) + + dust_wallet.unloadwallet() + + @cleanup + def sendall_with_send_max(self): + self.log.info("Check that `send_max` option causes negative value UTXOs to be left behind") + self.add_utxos([0.00000400, 0.00000300, 1]) + + # sendall with send_max + sendall_tx_receipt = self.wallet.sendall(recipients=[self.remainder_target], fee_rate=300, options={"send_max": True}) + tx_from_wallet = self.wallet.gettransaction(txid = sendall_tx_receipt["txid"], verbose = True) + + assert_equal(len(tx_from_wallet["decoded"]["vin"]), 1) + self.assert_tx_has_outputs(tx_from_wallet, [{"address": self.remainder_target, "value": 1 + tx_from_wallet["fee"]}]) + assert_equal(self.wallet.getbalances()["mine"]["trusted"], Decimal("0.00000700")) + + self.def_wallet.sendtoaddress(self.wallet.getnewaddress(), 1) + self.generate(self.nodes[0], 1) + + @cleanup + def sendall_specific_inputs(self): + self.log.info("Test sendall with a subset of UTXO pool") + self.add_utxos([17, 4]) + utxo = self.wallet.listunspent()[0] + + sendall_tx_receipt = self.wallet.sendall(recipients=[self.remainder_target], options={"inputs": [utxo]}) + tx_from_wallet = self.wallet.gettransaction(txid = sendall_tx_receipt["txid"], verbose = True) + assert_equal(len(tx_from_wallet["decoded"]["vin"]), 1) + assert_equal(len(tx_from_wallet["decoded"]["vout"]), 1) + assert_equal(tx_from_wallet["decoded"]["vin"][0]["txid"], utxo["txid"]) + assert_equal(tx_from_wallet["decoded"]["vin"][0]["vout"], utxo["vout"]) + self.assert_tx_has_output(tx_from_wallet, self.remainder_target) + + self.generate(self.nodes[0], 1) + assert_greater_than(self.wallet.getbalances()["mine"]["trusted"], 0) + + @cleanup + def sendall_fails_on_missing_input(self): + # fails because UTXO was previously spent, and wallet is empty + self.log.info("Test sendall fails because specified UTXO is not available") + self.add_utxos([16, 5]) + spent_utxo = self.wallet.listunspent()[0] + + # fails on unconfirmed spent UTXO + self.wallet.sendall(recipients=[self.remainder_target]) + assert_raises_rpc_error(-8, + "Input not available. UTXO ({}:{}) was already spent.".format(spent_utxo["txid"], spent_utxo["vout"]), + self.wallet.sendall, recipients=[self.remainder_target], options={"inputs": [spent_utxo]}) + + # fails on specific previously spent UTXO, while other UTXOs exist + self.generate(self.nodes[0], 1) + self.add_utxos([19, 2]) + assert_raises_rpc_error(-8, + "Input not available. UTXO ({}:{}) was already spent.".format(spent_utxo["txid"], spent_utxo["vout"]), + self.wallet.sendall, recipients=[self.remainder_target], options={"inputs": [spent_utxo]}) + + # fails because UTXO is unknown, while other UTXOs exist + foreign_utxo = self.def_wallet.listunspent()[0] + assert_raises_rpc_error(-8, "Input not found. UTXO ({}:{}) is not part of wallet.".format(foreign_utxo["txid"], + foreign_utxo["vout"]), self.wallet.sendall, recipients=[self.remainder_target], + options={"inputs": [foreign_utxo]}) + + @cleanup + def sendall_fails_on_no_address(self): + self.log.info("Test sendall fails because no address is provided") + self.add_utxos([19, 2]) + + assert_raises_rpc_error( + -8, + "Must provide at least one address without a specified amount" , + self.wallet.sendall, + [] + ) + + @cleanup + def sendall_fails_on_specific_inputs_with_send_max(self): + self.log.info("Test sendall fails because send_max is used while specific inputs are provided") + self.add_utxos([15, 6]) + utxo = self.wallet.listunspent()[0] + + assert_raises_rpc_error(-8, + "Cannot combine send_max with specific inputs.", + self.wallet.sendall, + recipients=[self.remainder_target], + options={"inputs": [utxo], "send_max": True}) + + def run_test(self): + self.nodes[0].createwallet("activewallet") + self.wallet = self.nodes[0].get_wallet_rpc("activewallet") + self.def_wallet = self.nodes[0].get_wallet_rpc(self.default_wallet_name) + self.generate(self.nodes[0], 101) + self.recipient = self.def_wallet.getnewaddress() # payee for a specific amount + self.remainder_target = self.def_wallet.getnewaddress() # address that receives everything left after payments and fees + self.split_target = self.def_wallet.getnewaddress() # 2nd target when splitting rest + + # Test cleanup + self.test_cleanup() + + # Basic sweep: everything to one address + self.sendall_two_utxos() + + # Split remainder to two addresses with equal amounts + self.sendall_split() + + # Pay recipient and sweep remainder + self.sendall_and_spend() + + # sendall fails if no recipient has unspecified amount + self.sendall_invalid_recipient_addresses() + + # Sendall fails if same destination is provided twice + self.sendall_duplicate_recipient() + + # Sendall fails when trying to spend more than the balance + self.sendall_invalid_amounts() + + # Sendall fails when wallet has no economically spendable UTXOs + self.sendall_negative_effective_value() + + # Leave dust behind if using send_max + self.sendall_with_send_max() + + # Sendall succeeds with specific inputs + self.sendall_specific_inputs() + + # Fails for the right reasons on missing or previously spent UTXOs + self.sendall_fails_on_missing_input() + + # Sendall fails when no address is provided + self.sendall_fails_on_no_address() + + # Sendall fails when using send_max while specifying inputs + self.sendall_fails_on_specific_inputs_with_send_max() + +if __name__ == '__main__': + SendallTest().main() diff --git a/test/functional/wallet_signer.py b/test/functional/wallet_signer.py index 9e2db517b6..8e4e1f5d36 100755 --- a/test/functional/wallet_signer.py +++ b/test/functional/wallet_signer.py @@ -72,10 +72,12 @@ class WalletSignerTest(BitcoinTestFramework): self.nodes[1].createwallet(wallet_name='hww', disable_private_keys=True, descriptors=True, external_signer=True) hww = self.nodes[1].get_wallet_rpc('hww') + assert_equal(hww.getwalletinfo()["external_signer"], True) # Flag can't be set afterwards (could be added later for non-blank descriptor based watch-only wallets) self.nodes[1].createwallet(wallet_name='not_hww', disable_private_keys=True, descriptors=True, external_signer=False) not_hww = self.nodes[1].get_wallet_rpc('not_hww') + assert_equal(not_hww.getwalletinfo()["external_signer"], False) assert_raises_rpc_error(-8, "Wallet flag is immutable: external_signer", not_hww.setwalletflag, "external_signer", True) # assert_raises_rpc_error(-4, "Multiple signers found, please specify which to use", wallet_name='not_hww', disable_private_keys=True, descriptors=True, external_signer=True) @@ -192,6 +194,12 @@ class WalletSignerTest(BitcoinTestFramework): assert(res["complete"]) assert_equal(res["hex"], mock_tx) + self.log.info('Test sendall using hww1') + + res = hww.sendall(recipients=[{dest:0.5}, hww.getrawchangeaddress()],options={"add_to_wallet": False}) + assert(res["complete"]) + assert_equal(res["hex"], mock_tx) + # # Handle error thrown by script # self.set_mock_result(self.nodes[4], "2") # assert_raises_rpc_error(-1, 'Unable to parse JSON', diff --git a/test/functional/wallet_taproot.py b/test/functional/wallet_taproot.py index 17eab25457..a4d836c8fe 100755 --- a/test/functional/wallet_taproot.py +++ b/test/functional/wallet_taproot.py @@ -12,8 +12,11 @@ from test_framework.util import assert_equal from test_framework.descriptors import descsum_create from test_framework.script import ( CScript, + MAX_PUBKEYS_PER_MULTI_A, OP_1, OP_CHECKSIG, + OP_CHECKSIGADD, + OP_NUMEQUAL, taproot_construct, ) from test_framework.segwit_addr import encode_segwit_address @@ -167,6 +170,17 @@ def pk(hex_key): """Construct a script expression for taproot_construct for pk(hex_key).""" return (None, CScript([bytes.fromhex(hex_key), OP_CHECKSIG])) +def multi_a(k, hex_keys, sort=False): + """Construct a script expression for taproot_construct for a multi_a script.""" + xkeys = [bytes.fromhex(hex_key) for hex_key in hex_keys] + if sort: + xkeys.sort() + ops = [xkeys[0], OP_CHECKSIG] + for i in range(1, len(hex_keys)): + ops += [xkeys[i], OP_CHECKSIGADD] + ops += [k, OP_NUMEQUAL] + return (None, CScript(ops)) + def compute_taproot_address(pubkey, scripts): """Compute the address for a taproot output with given inner key and scripts.""" tap = taproot_construct(pubkey, scripts) @@ -178,9 +192,9 @@ class WalletTaprootTest(BitcoinTestFramework): """Test generation and spending of P2TR address outputs.""" def set_test_params(self): - self.num_nodes = 3 + self.num_nodes = 2 self.setup_clean_chain = True - self.extra_args = [['-keypool=100'], ['-keypool=100'], ["-vbparams=taproot:1:1"]] + self.extra_args = [['-keypool=100'], ['-keypool=100']] self.supports_cli = False def skip_test_if_missing_module(self): @@ -194,19 +208,6 @@ class WalletTaprootTest(BitcoinTestFramework): pass @staticmethod - def rand_keys(n): - ret = [] - idxes = set() - for _ in range(n): - while True: - i = random.randrange(len(KEYS)) - if not i in idxes: - break - idxes.add(i) - ret.append(KEYS[i]) - return ret - - @staticmethod def make_desc(pattern, privmap, keys, pub_only = False): pat = pattern.replace("$H", H_POINT) for i in range(len(privmap)): @@ -242,15 +243,11 @@ class WalletTaprootTest(BitcoinTestFramework): assert_equal(len(rederive), 1) assert_equal(rederive[0], addr_g) - # tr descriptors can be imported regardless of Taproot status + # tr descriptors can be imported result = self.privs_tr_enabled.importdescriptors([{"desc": desc, "timestamp": "now"}]) assert(result[0]["success"]) result = self.pubs_tr_enabled.importdescriptors([{"desc": desc_pub, "timestamp": "now"}]) assert(result[0]["success"]) - result = self.privs_tr_disabled.importdescriptors([{"desc": desc, "timestamp": "now"}]) - assert result[0]["success"] - result = self.pubs_tr_disabled.importdescriptors([{"desc": desc_pub, "timestamp": "now"}]) - assert result[0]["success"] def do_test_sendtoaddress(self, comment, pattern, privmap, treefn, keys_pay, keys_change): self.log.info("Testing %s through sendtoaddress" % comment) @@ -275,7 +272,8 @@ class WalletTaprootTest(BitcoinTestFramework): self.generatetoaddress(self.nodes[0], 1, self.boring.getnewaddress(), sync_fun=self.no_op) test_balance = int(self.rpc_online.getbalance() * 100000000) ret_amnt = random.randrange(100000, test_balance) - res = self.rpc_online.sendtoaddress(address=self.boring.getnewaddress(), amount=Decimal(ret_amnt) / 100000000, subtractfeefromamount=True) + # Increase fee_rate to compensate for the wallet's inability to estimate fees for script path spends. + res = self.rpc_online.sendtoaddress(address=self.boring.getnewaddress(), amount=Decimal(ret_amnt) / 100000000, subtractfeefromamount=True, fee_rate=200) self.generatetoaddress(self.nodes[0], 1, self.boring.getnewaddress(), sync_fun=self.no_op) assert(self.rpc_online.gettransaction(res)["confirmations"] > 0) @@ -306,7 +304,8 @@ class WalletTaprootTest(BitcoinTestFramework): self.generatetoaddress(self.nodes[0], 1, self.boring.getnewaddress(), sync_fun=self.no_op) test_balance = int(self.psbt_online.getbalance() * 100000000) ret_amnt = random.randrange(100000, test_balance) - psbt = self.psbt_online.walletcreatefundedpsbt([], [{self.boring.getnewaddress(): Decimal(ret_amnt) / 100000000}], None, {"subtractFeeFromOutputs":[0]})['psbt'] + # Increase fee_rate to compensate for the wallet's inability to estimate fees for script path spends. + psbt = self.psbt_online.walletcreatefundedpsbt([], [{self.boring.getnewaddress(): Decimal(ret_amnt) / 100000000}], None, {"subtractFeeFromOutputs":[0], "fee_rate": 200})['psbt'] res = self.psbt_offline.walletprocesspsbt(psbt) assert(res['complete']) rawtx = self.nodes[0].finalizepsbt(res['psbt'])['hex'] @@ -314,8 +313,9 @@ class WalletTaprootTest(BitcoinTestFramework): self.generatetoaddress(self.nodes[0], 1, self.boring.getnewaddress(), sync_fun=self.no_op) assert(self.psbt_online.gettransaction(txid)['confirmations'] > 0) - def do_test(self, comment, pattern, privmap, treefn, nkeys): - keys = self.rand_keys(nkeys * 4) + def do_test(self, comment, pattern, privmap, treefn): + nkeys = len(privmap) + keys = random.sample(KEYS, nkeys * 4) self.do_test_addr(comment, pattern, privmap, treefn, keys[0:nkeys]) self.do_test_sendtoaddress(comment, pattern, privmap, treefn, keys[0:nkeys], keys[nkeys:2*nkeys]) self.do_test_psbt(comment, pattern, privmap, treefn, keys[2*nkeys:3*nkeys], keys[3*nkeys:4*nkeys]) @@ -324,12 +324,8 @@ class WalletTaprootTest(BitcoinTestFramework): self.log.info("Creating wallets...") self.nodes[0].createwallet(wallet_name="privs_tr_enabled", descriptors=True, blank=True) self.privs_tr_enabled = self.nodes[0].get_wallet_rpc("privs_tr_enabled") - self.nodes[2].createwallet(wallet_name="privs_tr_disabled", descriptors=True, blank=True) - self.privs_tr_disabled=self.nodes[2].get_wallet_rpc("privs_tr_disabled") self.nodes[0].createwallet(wallet_name="pubs_tr_enabled", descriptors=True, blank=True, disable_private_keys=True) self.pubs_tr_enabled = self.nodes[0].get_wallet_rpc("pubs_tr_enabled") - self.nodes[2].createwallet(wallet_name="pubs_tr_disabled", descriptors=True, blank=True, disable_private_keys=True) - self.pubs_tr_disabled=self.nodes[2].get_wallet_rpc("pubs_tr_disabled") self.nodes[0].createwallet(wallet_name="boring") self.nodes[0].createwallet(wallet_name="addr_gen", descriptors=True, disable_private_keys=True, blank=True) self.nodes[0].createwallet(wallet_name="rpc_online", descriptors=True, blank=True) @@ -349,73 +345,107 @@ class WalletTaprootTest(BitcoinTestFramework): "tr(XPRV)", "tr($1/*)", [True], - lambda k1: (key(k1), []), - 1 + lambda k1: (key(k1), []) ) self.do_test( "tr(H,XPRV)", "tr($H,pk($1/*))", [True], - lambda k1: (key(H_POINT), [pk(k1)]), - 1 + lambda k1: (key(H_POINT), [pk(k1)]) ) self.do_test( "wpkh(XPRV)", "wpkh($1/*)", [True], - None, - 1 + None ) self.do_test( "tr(XPRV,{H,{H,XPUB}})", "tr($1/*,{pk($H),{pk($H),pk($2/*)}})", [True, False], - lambda k1, k2: (key(k1), [pk(H_POINT), [pk(H_POINT), pk(k2)]]), - 2 + lambda k1, k2: (key(k1), [pk(H_POINT), [pk(H_POINT), pk(k2)]]) ) self.do_test( "wsh(multi(1,XPRV,XPUB))", "wsh(multi(1,$1/*,$2/*))", [True, False], - None, - 2 + None ) self.do_test( "tr(XPRV,{XPUB,XPUB})", "tr($1/*,{pk($2/*),pk($2/*)})", [True, False], - lambda k1, k2: (key(k1), [pk(k2), pk(k2)]), - 2 + lambda k1, k2: (key(k1), [pk(k2), pk(k2)]) ) self.do_test( "tr(XPRV,{{XPUB,H},{H,XPUB}})", "tr($1/*,{{pk($2/*),pk($H)},{pk($H),pk($2/*)}})", [True, False], - lambda k1, k2: (key(k1), [[pk(k2), pk(H_POINT)], [pk(H_POINT), pk(k2)]]), - 2 + lambda k1, k2: (key(k1), [[pk(k2), pk(H_POINT)], [pk(H_POINT), pk(k2)]]) ) self.do_test( "tr(XPUB,{{H,{H,XPUB}},{H,{H,{H,XPRV}}}})", "tr($1/*,{{pk($H),{pk($H),pk($2/*)}},{pk($H),{pk($H),{pk($H),pk($3/*)}}}})", [False, False, True], - lambda k1, k2, k3: (key(k1), [[pk(H_POINT), [pk(H_POINT), pk(k2)]], [pk(H_POINT), [pk(H_POINT), [pk(H_POINT), pk(k3)]]]]), - 3 + lambda k1, k2, k3: (key(k1), [[pk(H_POINT), [pk(H_POINT), pk(k2)]], [pk(H_POINT), [pk(H_POINT), [pk(H_POINT), pk(k3)]]]]) ) self.do_test( "tr(XPRV,{XPUB,{{XPUB,{H,H}},{{H,H},XPUB}}})", "tr($1/*,{pk($2/*),{{pk($2/*),{pk($H),pk($H)}},{{pk($H),pk($H)},pk($2/*)}}})", [True, False], - lambda k1, k2: (key(k1), [pk(k2), [[pk(k2), [pk(H_POINT), pk(H_POINT)]], [[pk(H_POINT), pk(H_POINT)], pk(k2)]]]), - 2 + lambda k1, k2: (key(k1), [pk(k2), [[pk(k2), [pk(H_POINT), pk(H_POINT)]], [[pk(H_POINT), pk(H_POINT)], pk(k2)]]]) + ) + self.do_test( + "tr(H,multi_a(1,XPRV))", + "tr($H,multi_a(1,$1/*))", + [True], + lambda k1: (key(H_POINT), [multi_a(1, [k1])]) + ) + self.do_test( + "tr(H,sortedmulti_a(1,XPRV,XPUB))", + "tr($H,sortedmulti_a(1,$1/*,$2/*))", + [True, False], + lambda k1, k2: (key(H_POINT), [multi_a(1, [k1, k2], True)]) + ) + self.do_test( + "tr(H,{H,multi_a(1,XPUB,XPRV)})", + "tr($H,{pk($H),multi_a(1,$1/*,$2/*)})", + [False, True], + lambda k1, k2: (key(H_POINT), [pk(H_POINT), [multi_a(1, [k1, k2])]]) + ) + self.do_test( + "tr(H,sortedmulti_a(1,XPUB,XPRV,XPRV))", + "tr($H,sortedmulti_a(1,$1/*,$2/*,$3/*))", + [False, True, True], + lambda k1, k2, k3: (key(H_POINT), [multi_a(1, [k1, k2, k3], True)]) + ) + self.do_test( + "tr(H,multi_a(2,XPRV,XPUB,XPRV))", + "tr($H,multi_a(2,$1/*,$2/*,$3/*))", + [True, False, True], + lambda k1, k2, k3: (key(H_POINT), [multi_a(2, [k1, k2, k3])]) + ) + self.do_test( + "tr(XPUB,{{XPUB,{XPUB,sortedmulti_a(2,XPRV,XPUB,XPRV)}})", + "tr($2/*,{pk($2/*),{pk($2/*),sortedmulti_a(2,$1/*,$2/*,$3/*)}})", + [True, False, True], + lambda k1, k2, k3: (key(k2), [pk(k2), [pk(k2), multi_a(2, [k1, k2, k3], True)]]) + ) + rnd_pos = random.randrange(MAX_PUBKEYS_PER_MULTI_A) + self.do_test( + "tr(XPUB,multi_a(1,H...,XPRV,H...))", + "tr($2/*,multi_a(1" + (",$H" * rnd_pos) + ",$1/*" + (",$H" * (MAX_PUBKEYS_PER_MULTI_A - 1 - rnd_pos)) + "))", + [True, False], + lambda k1, k2: (key(k2), [multi_a(1, ([H_POINT] * rnd_pos) + [k1] + ([H_POINT] * (MAX_PUBKEYS_PER_MULTI_A - 1 - rnd_pos)))]) ) self.log.info("Sending everything back...") - txid = self.rpc_online.sendtoaddress(address=self.boring.getnewaddress(), amount=self.rpc_online.getbalance(), subtractfeefromamount=True) + txid = self.rpc_online.sendall(recipients=[self.boring.getnewaddress()])["txid"] self.generatetoaddress(self.nodes[0], 1, self.boring.getnewaddress(), sync_fun=self.no_op) assert(self.rpc_online.gettransaction(txid)["confirmations"] > 0) - psbt = self.psbt_online.walletcreatefundedpsbt([], [{self.boring.getnewaddress(): self.psbt_online.getbalance()}], None, {"subtractFeeFromOutputs": [0]})['psbt'] + psbt = self.psbt_online.sendall(recipients=[self.boring.getnewaddress()], options={"psbt": True})["psbt"] res = self.psbt_offline.walletprocesspsbt(psbt) assert(res['complete']) rawtx = self.nodes[0].finalizepsbt(res['psbt'])['hex'] diff --git a/test/functional/wallet_timelock.py b/test/functional/wallet_timelock.py new file mode 100755 index 0000000000..a71cec6607 --- /dev/null +++ b/test/functional/wallet_timelock.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal + + +class WalletLocktimeTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + + def run_test(self): + node = self.nodes[0] + + mtp_tip = node.getblockheader(node.getbestblockhash())["mediantime"] + + self.log.info("Get new address with label") + label = "timelock⌛🔓" + address = node.getnewaddress(label=label) + + self.log.info("Send to new address with locktime") + node.send( + outputs={address: 5}, + options={"locktime": mtp_tip - 1}, + ) + self.generate(node, 1) + + self.log.info("Check that clock cannot change finality of confirmed txs") + amount_before_ad = node.getreceivedbyaddress(address) + amount_before_lb = node.getreceivedbylabel(label) + list_before_ad = node.listreceivedbyaddress(address_filter=address) + list_before_lb = node.listreceivedbylabel(include_empty=False) + balance_before = node.getbalances()["mine"]["trusted"] + coin_before = node.listunspent(maxconf=1) + node.setmocktime(mtp_tip - 1) + assert_equal(node.getreceivedbyaddress(address), amount_before_ad) + assert_equal(node.getreceivedbylabel(label), amount_before_lb) + assert_equal(node.listreceivedbyaddress(address_filter=address), list_before_ad) + assert_equal(node.listreceivedbylabel(include_empty=False), list_before_lb) + assert_equal(node.getbalances()["mine"]["trusted"], balance_before) + assert_equal(node.listunspent(maxconf=1), coin_before) + + +if __name__ == "__main__": + WalletLocktimeTest().main() diff --git a/test/functional/wallet_upgradewallet.py b/test/functional/wallet_upgradewallet.py index 36e72f2dd9..c452e1eafd 100755 --- a/test/functional/wallet_upgradewallet.py +++ b/test/functional/wallet_upgradewallet.py @@ -345,5 +345,16 @@ class UpgradeWalletTest(BitcoinTestFramework): desc_wallet = self.nodes[0].get_wallet_rpc("desc_upgrade") self.test_upgradewallet(desc_wallet, previous_version=169900, expected_version=169900) + self.log.info("Checking that descriptor wallets without privkeys do nothing, successfully") + self.nodes[0].createwallet(wallet_name="desc_upgrade_nopriv", descriptors=True, disable_private_keys=True) + desc_wallet = self.nodes[0].get_wallet_rpc("desc_upgrade_nopriv") + self.test_upgradewallet(desc_wallet, previous_version=169900, expected_version=169900) + + if self.is_bdb_compiled(): + self.log.info("Upgrading a wallet with private keys disabled") + self.nodes[0].createwallet(wallet_name="privkeys_disabled_upgrade", disable_private_keys=True, descriptors=False) + disabled_wallet = self.nodes[0].get_wallet_rpc("privkeys_disabled_upgrade") + self.test_upgradewallet(disabled_wallet, previous_version=169900, expected_version=169900) + if __name__ == '__main__': UpgradeWalletTest().main() diff --git a/test/get_previous_releases.py b/test/get_previous_releases.py index 177aa74191..cbdb67216c 100755 --- a/test/get_previous_releases.py +++ b/test/get_previous_releases.py @@ -19,36 +19,35 @@ import subprocess import sys import hashlib - SHA256_SUMS = { + "0e2819135366f150d9906e294b61dff58fd1996ebd26c2f8e979d6c0b7a79580": "bitcoin-0.14.3-aarch64-linux-gnu.tar.gz", + "d86fc90824a85c38b25c8488115178d5785dbc975f5ff674f9f5716bc8ad6e65": "bitcoin-0.14.3-arm-linux-gnueabihf.tar.gz", + "1b0a7408c050e3d09a8be8e21e183ef7ee570385dc41216698cc3ab392a484e7": "bitcoin-0.14.3-osx64.tar.gz", + "706e0472dbc933ed2757650d54cbcd780fd3829ebf8f609b32780c7eedebdbc9": "bitcoin-0.14.3-x86_64-linux-gnu.tar.gz", + # "d40f18b4e43c6e6370ef7db9131f584fbb137276ec2e3dba67a4b267f81cb644": "bitcoin-0.15.2-aarch64-linux-gnu.tar.gz", "54fb877a148a6ad189a1e1ab1ff8b11181e58ff2aaf430da55b3fd46ae549a6b": "bitcoin-0.15.2-arm-linux-gnueabihf.tar.gz", - "2b843506c3f1af0eeca5854a920264f9a829f02d0d50328005950ddcbe88874d": "bitcoin-0.15.2-i686-pc-linux-gnu.tar.gz", "87e9340ff3d382d543b2b69112376077f0c8b4f7450d372e83b68f5a1e22b2df": "bitcoin-0.15.2-osx64.tar.gz", "566be44190fd76daa01f13d428939dadfb8e3daacefc8fa17f433cad28f73bd5": "bitcoin-0.15.2-x86_64-linux-gnu.tar.gz", # "0768c6c15caffbaca6524824c9563b42c24f70633c681c2744649158aa3fd484": "bitcoin-0.16.3-aarch64-linux-gnu.tar.gz", "fb2818069854a6ad20ea03b28b55dbd35d8b1f7d453e90b83eace5d0098a2a87": "bitcoin-0.16.3-arm-linux-gnueabihf.tar.gz", - "75a537844313b0a84bdb61ffcdc5c4ce19a738f7ddf71007cd2edf664efd7c37": "bitcoin-0.16.3-i686-pc-linux-gnu.tar.gz", "78c3bff3b619a19aed575961ea43cc9e142959218835cf51aede7f0b764fc25d": "bitcoin-0.16.3-osx64.tar.gz", "5d422a9d544742bc0df12427383f9c2517433ce7b58cf672b9a9b17c2ef51e4f": "bitcoin-0.16.3-x86_64-linux-gnu.tar.gz", # "5a6b35d1a348a402f2d2d6ab5aed653a1a1f13bc63aaaf51605e3501b0733b7a": "bitcoin-0.17.2-aarch64-linux-gnu.tar.gz", "d1913a5d19c8e8da4a67d1bd5205d03c8614dfd2e02bba2fe3087476643a729e": "bitcoin-0.17.2-arm-linux-gnueabihf.tar.gz", - "d295fc93f39bbf0fd937b730a93184899a2eb6c3a6d53f3d857cbe77ef89b98c": "bitcoin-0.17.2-i686-pc-linux-gnu.tar.gz", "a783ba20706dbfd5b47fbedf42165fce70fbbc7d78003305d964f6b3da14887f": "bitcoin-0.17.2-osx64.tar.gz", "943f9362b9f11130177839116f48f809d83478b4c28591d486ee9a7e35179da6": "bitcoin-0.17.2-x86_64-linux-gnu.tar.gz", # "88f343af72803b851c7da13874cc5525026b0b55e63e1b5e1298390c4688adc6": "bitcoin-0.18.1-aarch64-linux-gnu.tar.gz", "cc7d483e4b20c5dabd4dcaf304965214cf4934bcc029ca99cbc9af00d3771a1f": "bitcoin-0.18.1-arm-linux-gnueabihf.tar.gz", - "989e847b3e95fc9fedc0b109cae1b4fa43348f2f712e187a118461876af9bd16": "bitcoin-0.18.1-i686-pc-linux-gnu.tar.gz", "b7bbcee7a7540f711b171d6981f939ca8482005fde22689bc016596d80548bb1": "bitcoin-0.18.1-osx64.tar.gz", "425ee5ec631ae8da71ebc1c3f5c0269c627cf459379b9b030f047107a28e3ef8": "bitcoin-0.18.1-riscv64-linux-gnu.tar.gz", "600d1db5e751fa85903e935a01a74f5cc57e1e7473c15fd3e17ed21e202cfe5a": "bitcoin-0.18.1-x86_64-linux-gnu.tar.gz", # "3a80431717842672df682bdb619e66523b59541483297772a7969413be3502ff": "bitcoin-0.19.1-aarch64-linux-gnu.tar.gz", "657f28213823d240dd3324d14829702f9ad6f0710f8bdd1c379cb3c447197f48": "bitcoin-0.19.1-arm-linux-gnueabihf.tar.gz", - "10d1e53208aa7603022f4acc084a046299ab4ccf25fe01e81b3fb6f856772589": "bitcoin-0.19.1-i686-pc-linux-gnu.tar.gz", "1ae1b87de26487075cd2fd22e0d4ead87d969bd55c44f2f1d873ecdc6147ebb3": "bitcoin-0.19.1-osx64.tar.gz", "aa7a9563b48aa79252c8e7b6a41c07a5441bd9f14c5e4562cc72720ea6cb0ee5": "bitcoin-0.19.1-riscv64-linux-gnu.tar.gz", "5fcac9416e486d4960e1a946145566350ca670f9aaba99de6542080851122e4c": "bitcoin-0.19.1-x86_64-linux-gnu.tar.gz", @@ -56,9 +55,31 @@ SHA256_SUMS = { "60c93e3462c303eb080be7cf623f1a7684b37fd47a018ad3848bc23e13c84e1c": "bitcoin-0.20.1-aarch64-linux-gnu.tar.gz", "55b577e0fb306fb429d4be6c9316607753e8543e5946b542d75d876a2f08654c": "bitcoin-0.20.1-arm-linux-gnueabihf.tar.gz", "b9024dde373ea7dad707363e07ec7e265383204127539ae0c234bff3a61da0d1": "bitcoin-0.20.1-osx64.tar.gz", - "c378d4e21109f09e8829f3591e015c66632dff2925a60b64d259be05a334c30b": "bitcoin-0.20.1-osx.dmg", "fa71cb52ee5e0459cbf5248cdec72df27995840c796f58b304607a1ed4c165af": "bitcoin-0.20.1-riscv64-linux-gnu.tar.gz", "376194f06596ecfa40331167c39bc70c355f960280bd2a645fdbf18f66527397": "bitcoin-0.20.1-x86_64-linux-gnu.tar.gz", + + "43416854330914992bbba2d0e9adf2a6fff4130be9af8ae2ef1186e743d9a3fe": "bitcoin-0.21.0-aarch64-linux-gnu.tar.gz", + "f028af308eda45a3c4c90f9332f96b075bf21e3495c945ebce48597151808176": "bitcoin-0.21.0-arm-linux-gnueabihf.tar.gz", + "695fb624fa6423f5da4f443b60763dd1d77488bfe5ef63760904a7b54e91298d": "bitcoin-0.21.0-osx64.tar.gz", + "f8b2adfeae021a672effbc7bd40d5c48d6b94e53b2dd660f787340bf1a52e4e9": "bitcoin-0.21.0-riscv64-linux-gnu.tar.gz", + "da7766775e3f9c98d7a9145429f2be8297c2672fe5b118fd3dc2411fb48e0032": "bitcoin-0.21.0-x86_64-linux-gnu.tar.gz", + + "ac718fed08570a81b3587587872ad85a25173afa5f9fbbd0c03ba4d1714cfa3e": "bitcoin-22.0-aarch64-linux-gnu.tar.gz", + "b8713c6c5f03f5258b54e9f436e2ed6d85449aa24c2c9972f91963d413e86311": "bitcoin-22.0-arm-linux-gnueabihf.tar.gz", + "2744d199c3343b2d94faffdfb2c94d75a630ba27301a70e47b0ad30a7e0155e9": "bitcoin-22.0-osx64.tar.gz", + "2cca5f99007d060aca9d8c7cbd035dfe2f040dd8200b210ce32cdf858479f70d": "bitcoin-22.0-powerpc64-linux-gnu.tar.gz", + "91b1e012975c5a363b5b5fcc81b5b7495e86ff703ec8262d4b9afcfec633c30d": "bitcoin-22.0-powerpc64le-linux-gnu.tar.gz", + "9cc3a62c469fe57e11485fdd32c916f10ce7a2899299855a2e479256ff49ff3c": "bitcoin-22.0-riscv64-linux-gnu.tar.gz", + "59ebd25dd82a51638b7a6bb914586201e67db67b919b2a1ff08925a7936d1b16": "bitcoin-22.0-x86_64-linux-gnu.tar.gz", + + "06f4c78271a77752ba5990d60d81b1751507f77efda1e5981b4e92fd4d9969fb": "bitcoin-23.0-aarch64-linux-gnu.tar.gz", + "952c574366aff76f6d6ad1c9ee45a361d64fa04155e973e926dfe7e26f9703a3": "bitcoin-23.0-arm-linux-gnueabihf.tar.gz", + "7c8bc63731aa872b7b334a8a7d96e33536ad77d49029bad179b09dca32cd77ac": "bitcoin-23.0-arm64-apple-darwin.tar.gz", + "2caa5898399e415f61d9af80a366a3008e5856efa15aaff74b88acf429674c99": "bitcoin-23.0-powerpc64-linux-gnu.tar.gz", + "217dd0469d0f4962d22818c368358575f6a0abcba8804807bb75325eb2f28b19": "bitcoin-23.0-powerpc64le-linux-gnu.tar.gz", + "078f96b1e92895009c798ab827fb3fde5f6719eee886bd0c0e93acab18ea4865": "bitcoin-23.0-riscv64-linux-gnu.tar.gz", + "c816780583009a9dad426dc0c183c89be9da98906e1e2c7ebae91041c1aaaaf3": "bitcoin-23.0-x86_64-apple-darwin.tar.gz", + "2cca490c1f2842884a3c5b0606f179f9f937177da4eadd628e3f7fd7e25d26d0": "bitcoin-23.0-x86_64-linux-gnu.tar.gz", } @@ -84,8 +105,11 @@ def download_binary(tag, args) -> int: if match: bin_path = 'bin/bitcoin-core-{}/test.{}'.format( match.group(1), match.group(2)) + platform = args.platform + if tag < "v23" and platform in ["x86_64-apple-darwin", "aarch64-apple-darwin"]: + platform = "osx64" tarball = 'bitcoin-{tag}-{platform}.tar.gz'.format( - tag=tag[1:], platform=args.platform) + tag=tag[1:], platform=platform) tarballUrl = 'https://bitcoincore.org/{bin_path}/{tarball}'.format( bin_path=bin_path, tarball=tarball) @@ -189,8 +213,8 @@ def check_host(args) -> int: platforms = { 'aarch64-*-linux*': 'aarch64-linux-gnu', 'x86_64-*-linux*': 'x86_64-linux-gnu', - 'x86_64-apple-darwin*': 'osx64', - 'aarch64-apple-darwin*': 'osx64', + 'x86_64-apple-darwin*': 'x86_64-apple-darwin', + 'aarch64-apple-darwin*': 'aarch64-apple-darwin', } args.platform = '' for pattern, target in platforms.items(): diff --git a/test/lint/README.md b/test/lint/README.md index f4165f908e..1f683c10b3 100644 --- a/test/lint/README.md +++ b/test/lint/README.md @@ -39,6 +39,6 @@ To do so, add the upstream repository as remote: git remote add --fetch secp256k1 https://github.com/bitcoin-core/secp256k1.git ``` -lint-all.sh +lint-all.py =========== Calls other scripts with the `lint-` prefix. diff --git a/test/lint/commit-script-check.sh b/test/lint/commit-script-check.sh index 6a8a15d05c..9449b393f1 100755 --- a/test/lint/commit-script-check.sh +++ b/test/lint/commit-script-check.sh @@ -17,6 +17,11 @@ if test -z "$1"; then exit 1 fi +if ! sed --help 2>&1 | grep -q 'GNU'; then + echo "Error: the installed sed package is not compatible. Please make sure you have GNU sed installed in your system."; + exit 1; +fi + RET=0 PREV_BRANCH=$(git name-rev --name-only HEAD) PREV_HEAD=$(git rev-parse HEAD) diff --git a/test/lint/extended-lint-all.sh b/test/lint/extended-lint-all.sh deleted file mode 100755 index be5d9db4a9..0000000000 --- a/test/lint/extended-lint-all.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2019-2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# This script runs all contrib/devtools/extended-lint-*.sh files, and fails if -# any exit with a non-zero status code. - -# This script is intentionally locale dependent by not setting "export LC_ALL=C" -# in order to allow for the executed lint scripts to opt in or opt out of locale -# dependence themselves. - -set -u - -SCRIPTDIR=$(dirname "${BASH_SOURCE[0]}") -LINTALL=$(basename "${BASH_SOURCE[0]}") - -for f in "${SCRIPTDIR}"/extended-lint-*.sh; do - if [ "$(basename "$f")" != "$LINTALL" ]; then - if ! "$f"; then - echo "^---- failure generated from $f" - exit 1 - fi - fi -done diff --git a/test/lint/extended-lint-cppcheck.sh b/test/lint/extended-lint-cppcheck.sh deleted file mode 100755 index 2af39ed60a..0000000000 --- a/test/lint/extended-lint-cppcheck.sh +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2019-2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# - -export LC_ALL=C - -ENABLED_CHECKS=( - "Class '.*' has a constructor with 1 argument that is not explicit." - "Struct '.*' has a constructor with 1 argument that is not explicit." -) - -IGNORED_WARNINGS=( - "src/arith_uint256.h:.* Class 'arith_uint256' has a constructor with 1 argument that is not explicit." - "src/arith_uint256.h:.* Class 'base_uint < 256 >' has a constructor with 1 argument that is not explicit." - "src/arith_uint256.h:.* Class 'base_uint' has a constructor with 1 argument that is not explicit." - "src/coins.h:.* Class 'CCoinsViewBacked' has a constructor with 1 argument that is not explicit." - "src/coins.h:.* Class 'CCoinsViewCache' has a constructor with 1 argument that is not explicit." - "src/coins.h:.* Class 'CCoinsViewCursor' has a constructor with 1 argument that is not explicit." - "src/net.h:.* Class 'CNetMessage' has a constructor with 1 argument that is not explicit." - "src/policy/feerate.h:.* Class 'CFeeRate' has a constructor with 1 argument that is not explicit." - "src/prevector.h:.* Class 'const_iterator' has a constructor with 1 argument that is not explicit." - "src/prevector.h:.* Class 'const_reverse_iterator' has a constructor with 1 argument that is not explicit." - "src/prevector.h:.* Class 'iterator' has a constructor with 1 argument that is not explicit." - "src/prevector.h:.* Class 'reverse_iterator' has a constructor with 1 argument that is not explicit." - "src/primitives/block.h:.* Class 'CBlock' has a constructor with 1 argument that is not explicit." - "src/primitives/transaction.h:.* Class 'CTransaction' has a constructor with 1 argument that is not explicit." - "src/protocol.h:.* Class 'CMessageHeader' has a constructor with 1 argument that is not explicit." - "src/qt/guiutil.h:.* Class 'ItemDelegate' has a constructor with 1 argument that is not explicit." - "src/rpc/util.h:.* Struct 'RPCResults' has a constructor with 1 argument that is not explicit." - "src/rpc/util.h:.* Struct 'UniValueType' has a constructor with 1 argument that is not explicit." - "src/rpc/util.h:.* style: Struct 'UniValueType' has a constructor with 1 argument that is not explicit." - "src/script/descriptor.cpp:.* Class 'AddressDescriptor' has a constructor with 1 argument that is not explicit." - "src/script/descriptor.cpp:.* Class 'ComboDescriptor' has a constructor with 1 argument that is not explicit." - "src/script/descriptor.cpp:.* Class 'ConstPubkeyProvider' has a constructor with 1 argument that is not explicit." - "src/script/descriptor.cpp:.* Class 'PKDescriptor' has a constructor with 1 argument that is not explicit." - "src/script/descriptor.cpp:.* Class 'PKHDescriptor' has a constructor with 1 argument that is not explicit." - "src/script/descriptor.cpp:.* Class 'RawDescriptor' has a constructor with 1 argument that is not explicit." - "src/script/descriptor.cpp:.* Class 'SHDescriptor' has a constructor with 1 argument that is not explicit." - "src/script/descriptor.cpp:.* Class 'WPKHDescriptor' has a constructor with 1 argument that is not explicit." - "src/script/descriptor.cpp:.* Class 'WSHDescriptor' has a constructor with 1 argument that is not explicit." - "src/script/script.h:.* Class 'CScript' has a constructor with 1 argument that is not explicit." - "src/script/standard.h:.* Class 'CScriptID' has a constructor with 1 argument that is not explicit." - "src/span.h:.* Class 'Span < const CRPCCommand >' has a constructor with 1 argument that is not explicit." - "src/span.h:.* Class 'Span < const char >' has a constructor with 1 argument that is not explicit." - "src/span.h:.* Class 'Span < const std :: vector <unsigned char > >' has a constructor with 1 argument that is not explicit." - "src/span.h:.* Class 'Span < const uint8_t >' has a constructor with 1 argument that is not explicit." - "src/span.h:.* Class 'Span' has a constructor with 1 argument that is not explicit." - "src/support/allocators/secure.h:.* Struct 'secure_allocator < char >' has a constructor with 1 argument that is not explicit." - "src/support/allocators/secure.h:.* Struct 'secure_allocator < RNGState >' has a constructor with 1 argument that is not explicit." - "src/support/allocators/secure.h:.* Struct 'secure_allocator < unsigned char >' has a constructor with 1 argument that is not explicit." - "src/support/allocators/zeroafterfree.h:.* Struct 'zero_after_free_allocator < char >' has a constructor with 1 argument that is not explicit." - "src/test/checkqueue_tests.cpp:.* Struct 'FailingCheck' has a constructor with 1 argument that is not explicit." - "src/test/checkqueue_tests.cpp:.* Struct 'MemoryCheck' has a constructor with 1 argument that is not explicit." - "src/test/checkqueue_tests.cpp:.* Struct 'UniqueCheck' has a constructor with 1 argument that is not explicit." - "src/test/fuzz/util.h:.* Class 'FuzzedFileProvider' has a constructor with 1 argument that is not explicit." - "src/test/fuzz/util.h:.* Class 'FuzzedAutoFileProvider' has a constructor with 1 argument that is not explicit." - "src/wallet/db.h:.* Class 'BerkeleyEnvironment' has a constructor with 1 argument that is not explicit." -) - -if ! command -v cppcheck > /dev/null; then - echo "Skipping cppcheck linting since cppcheck is not installed. Install by running \"apt install cppcheck\"" - exit 0 -fi - -function join_array { - local IFS="$1" - shift - echo "$*" -} - -ENABLED_CHECKS_REGEXP=$(join_array "|" "${ENABLED_CHECKS[@]}") -IGNORED_WARNINGS_REGEXP=$(join_array "|" "${IGNORED_WARNINGS[@]}") -WARNINGS=$(git ls-files -- "*.cpp" "*.h" ":(exclude)src/leveldb/" ":(exclude)src/crc32c/" ":(exclude)src/secp256k1/" ":(exclude)src/minisketch/" ":(exclude)src/univalue/" | \ - xargs cppcheck --enable=all -j "$(getconf _NPROCESSORS_ONLN)" --language=c++ --std=c++17 --template=gcc -D__cplusplus -DCLIENT_VERSION_BUILD -DCLIENT_VERSION_IS_RELEASE -DCLIENT_VERSION_MAJOR -DCLIENT_VERSION_MINOR -DCOPYRIGHT_YEAR -DDEBUG -I src/ -q 2>&1 | sort -u | \ - grep -E "${ENABLED_CHECKS_REGEXP}" | \ - grep -vE "${IGNORED_WARNINGS_REGEXP}") -if [[ ${WARNINGS} != "" ]]; then - echo "${WARNINGS}" - echo - echo "Advice not applicable in this specific case? Add an exception by updating" - echo "IGNORED_WARNINGS in $0" - # Uncomment to enforce the developer note policy "By default, declare single-argument constructors `explicit`" - # exit 1 -fi -exit 0 diff --git a/test/lint/lint-all.py b/test/lint/lint-all.py new file mode 100755 index 0000000000..c280ba2db2 --- /dev/null +++ b/test/lint/lint-all.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2017-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# This script runs all test/lint/lint-* files, and fails if any exit +# with a non-zero status code. + +from glob import glob +from pathlib import Path +from subprocess import run + +exit_code = 0 +mod_path = Path(__file__).parent +for lint in glob(f"{mod_path}/lint-*"): + if lint != __file__: + result = run([lint]) + if result.returncode != 0: + print(f"^---- failure generated from {lint.split('/')[-1]}") + exit_code |= result.returncode + +exit(exit_code) diff --git a/test/lint/lint-all.sh b/test/lint/lint-all.sh deleted file mode 100755 index fabc24c91b..0000000000 --- a/test/lint/lint-all.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2017-2019 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# This script runs all contrib/devtools/lint-*.sh files, and fails if any exit -# with a non-zero status code. - -# This script is intentionally locale dependent by not setting "export LC_ALL=C" -# in order to allow for the executed lint scripts to opt in or opt out of locale -# dependence themselves. - -set -u - -SCRIPTDIR=$(dirname "${BASH_SOURCE[0]}") -LINTALL=$(basename "${BASH_SOURCE[0]}") - -EXIT_CODE=0 - -for f in "${SCRIPTDIR}"/lint-*.sh; do - if [ "$(basename "$f")" != "$LINTALL" ]; then - if ! "$f"; then - echo "^---- failure generated from $f" - EXIT_CODE=1 - fi - fi -done - -exit ${EXIT_CODE} diff --git a/test/lint/lint-assertions.py b/test/lint/lint-assertions.py new file mode 100755 index 0000000000..195ff33d11 --- /dev/null +++ b/test/lint/lint-assertions.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2018-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# Check for assertions with obvious side effects. + +import sys +import subprocess + + +def git_grep(params: [], error_msg: ""): + try: + output = subprocess.check_output(["git", "grep", *params], universal_newlines=True, encoding="utf8") + print(error_msg) + print(output) + return 1 + except subprocess.CalledProcessError as ex1: + if ex1.returncode > 1: + raise ex1 + return 0 + + +def main(): + # PRE31-C (SEI CERT C Coding Standard): + # "Assertions should not contain assignments, increment, or decrement operators." + exit_code = git_grep([ + "-E", + r"[^_]assert\(.*(\+\+|\-\-|[^=!<>]=[^=!<>]).*\);", + "--", + "*.cpp", + "*.h", + ], "Assertions should not have side effects:") + + # Aborting the whole process is undesirable for RPC code. So nonfatal + # checks should be used over assert. See: src/util/check.h + # src/rpc/server.cpp is excluded from this check since it's mostly meta-code. + exit_code |= git_grep([ + "-nE", + r"\<(A|a)ss(ume|ert) *\(.*\);", + "--", + "src/rpc/", + "src/wallet/rpc*", + ":(exclude)src/rpc/server.cpp", + ], "CHECK_NONFATAL(condition) or NONFATAL_UNREACHABLE should be used instead of assert for RPC code.") + + sys.exit(exit_code) + + +if __name__ == "__main__": + main() diff --git a/test/lint/lint-assertions.sh b/test/lint/lint-assertions.sh deleted file mode 100755 index 2860f5621b..0000000000 --- a/test/lint/lint-assertions.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Check for assertions with obvious side effects. - -export LC_ALL=C - -EXIT_CODE=0 - -# PRE31-C (SEI CERT C Coding Standard): -# "Assertions should not contain assignments, increment, or decrement operators." -OUTPUT=$(git grep -E '[^_]assert\(.*(\+\+|\-\-|[^=!<>]=[^=!<>]).*\);' -- "*.cpp" "*.h") -if [[ ${OUTPUT} != "" ]]; then - echo "Assertions should not have side effects:" - echo - echo "${OUTPUT}" - EXIT_CODE=1 -fi - -# Macro CHECK_NONFATAL(condition) should be used instead of assert for RPC code, where it -# is undesirable to crash the whole program. See: src/util/check.h -# src/rpc/server.cpp is excluded from this check since it's mostly meta-code. -OUTPUT=$(git grep -nE '\<(A|a)ssert *\(.*\);' -- "src/rpc/" "src/wallet/rpc*" ":(exclude)src/rpc/server.cpp") -if [[ ${OUTPUT} != "" ]]; then - echo "CHECK_NONFATAL(condition) should be used instead of assert for RPC code." - echo - echo "${OUTPUT}" - EXIT_CODE=1 -fi - -exit ${EXIT_CODE} diff --git a/test/lint/lint-circular-dependencies.py b/test/lint/lint-circular-dependencies.py new file mode 100755 index 0000000000..5d157eb4b1 --- /dev/null +++ b/test/lint/lint-circular-dependencies.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2020-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# Check for circular dependencies + +import os +import re +import subprocess +import sys + +EXPECTED_CIRCULAR_DEPENDENCIES = ( + "chainparamsbase -> util/system -> chainparamsbase", + "node/blockstorage -> validation -> node/blockstorage", + "policy/fees -> txmempool -> policy/fees", + "qt/addresstablemodel -> qt/walletmodel -> qt/addresstablemodel", + "qt/recentrequeststablemodel -> qt/walletmodel -> qt/recentrequeststablemodel", + "qt/sendcoinsdialog -> qt/walletmodel -> qt/sendcoinsdialog", + "qt/transactiontablemodel -> qt/walletmodel -> qt/transactiontablemodel", + "wallet/fees -> wallet/wallet -> wallet/fees", + "wallet/wallet -> wallet/walletdb -> wallet/wallet", + "kernel/coinstats -> validation -> kernel/coinstats", +) + +CODE_DIR = "src" + + +def main(): + circular_dependencies = [] + exit_code = 0 + + os.chdir(CODE_DIR) + files = subprocess.check_output( + ['git', 'ls-files', '--', '*.h', '*.cpp'], + universal_newlines=True, + ).splitlines() + + command = [sys.executable, "../contrib/devtools/circular-dependencies.py", *files] + dependencies_output = subprocess.run( + command, + stdout=subprocess.PIPE, + universal_newlines=True, + ) + + for dependency_str in dependencies_output.stdout.rstrip().split("\n"): + circular_dependencies.append( + re.sub("^Circular dependency: ", "", dependency_str) + ) + + # Check for an unexpected dependencies + for dependency in circular_dependencies: + if dependency not in EXPECTED_CIRCULAR_DEPENDENCIES: + exit_code = 1 + print( + f'A new circular dependency in the form of "{dependency}" appears to have been introduced.\n', + file=sys.stderr, + ) + + # Check for missing expected dependencies + for expected_dependency in EXPECTED_CIRCULAR_DEPENDENCIES: + if expected_dependency not in circular_dependencies: + exit_code = 1 + print( + f'Good job! The circular dependency "{expected_dependency}" is no longer present.', + ) + print( + f"Please remove it from EXPECTED_CIRCULAR_DEPENDENCIES in {__file__}", + ) + print( + "to make sure this circular dependency is not accidentally reintroduced.\n", + ) + + sys.exit(exit_code) + + +if __name__ == "__main__": + main() diff --git a/test/lint/lint-circular-dependencies.sh b/test/lint/lint-circular-dependencies.sh deleted file mode 100755 index 69185090d1..0000000000 --- a/test/lint/lint-circular-dependencies.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Check for circular dependencies - -export LC_ALL=C - -EXPECTED_CIRCULAR_DEPENDENCIES=( - "chainparamsbase -> util/system -> chainparamsbase" - "node/blockstorage -> validation -> node/blockstorage" - "index/blockfilterindex -> node/blockstorage -> validation -> index/blockfilterindex" - "index/base -> validation -> index/blockfilterindex -> index/base" - "index/coinstatsindex -> node/coinstats -> index/coinstatsindex" - "policy/fees -> txmempool -> policy/fees" - "qt/addresstablemodel -> qt/walletmodel -> qt/addresstablemodel" - "qt/recentrequeststablemodel -> qt/walletmodel -> qt/recentrequeststablemodel" - "qt/sendcoinsdialog -> qt/walletmodel -> qt/sendcoinsdialog" - "qt/transactiontablemodel -> qt/walletmodel -> qt/transactiontablemodel" - "wallet/fees -> wallet/wallet -> wallet/fees" - "wallet/wallet -> wallet/walletdb -> wallet/wallet" - "node/coinstats -> validation -> node/coinstats" -) - -EXIT_CODE=0 - -CIRCULAR_DEPENDENCIES=() - -IFS=$'\n' -for CIRC in $(cd src && ../contrib/devtools/circular-dependencies.py {*,*/*,*/*/*}.{h,cpp} | sed -e 's/^Circular dependency: //'); do - CIRCULAR_DEPENDENCIES+=( "$CIRC" ) - IS_EXPECTED_CIRC=0 - for EXPECTED_CIRC in "${EXPECTED_CIRCULAR_DEPENDENCIES[@]}"; do - if [[ "${CIRC}" == "${EXPECTED_CIRC}" ]]; then - IS_EXPECTED_CIRC=1 - break - fi - done - if [[ ${IS_EXPECTED_CIRC} == 0 ]]; then - echo "A new circular dependency in the form of \"${CIRC}\" appears to have been introduced." - echo - EXIT_CODE=1 - fi -done - -for EXPECTED_CIRC in "${EXPECTED_CIRCULAR_DEPENDENCIES[@]}"; do - IS_PRESENT_EXPECTED_CIRC=0 - for CIRC in "${CIRCULAR_DEPENDENCIES[@]}"; do - if [[ "${CIRC}" == "${EXPECTED_CIRC}" ]]; then - IS_PRESENT_EXPECTED_CIRC=1 - break - fi - done - if [[ ${IS_PRESENT_EXPECTED_CIRC} == 0 ]]; then - echo "Good job! The circular dependency \"${EXPECTED_CIRC}\" is no longer present." - echo "Please remove it from EXPECTED_CIRCULAR_DEPENDENCIES in $0" - echo "to make sure this circular dependency is not accidentally reintroduced." - echo - EXIT_CODE=1 - fi -done - -exit ${EXIT_CODE} diff --git a/test/lint/lint-cpp.sh b/test/lint/lint-cpp.sh deleted file mode 100755 index cac57b968d..0000000000 --- a/test/lint/lint-cpp.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Check for various C++ code patterns we want to avoid. - -export LC_ALL=C - -EXIT_CODE=0 - -OUTPUT=$(git grep -E "boost::bind\(" -- "*.cpp" "*.h") -if [[ ${OUTPUT} != "" ]]; then - echo "Use of boost::bind detected. Use std::bind instead." - echo - echo "${OUTPUT}" - EXIT_CODE=1 -fi - -exit ${EXIT_CODE}
\ No newline at end of file diff --git a/test/lint/lint-files.py b/test/lint/lint-files.py index 68b795eef7..dbb51ce54e 100755 --- a/test/lint/lint-files.py +++ b/test/lint/lint-files.py @@ -11,18 +11,20 @@ import os import re import sys from subprocess import check_output -from typing import Optional, NoReturn +from typing import Dict, Optional, NoReturn -CMD_ALL_FILES = "git ls-files -z --full-name" -CMD_SOURCE_FILES = 'git ls-files -z --full-name -- "*.[cC][pP][pP]" "*.[hH]" "*.[pP][yY]" "*.[sS][hH]"' -CMD_SHEBANG_FILES = "git grep --full-name --line-number -I '^#!'" +CMD_TOP_LEVEL = ["git", "rev-parse", "--show-toplevel"] +CMD_ALL_FILES = ["git", "ls-files", "-z", "--full-name", "--stage"] +CMD_SHEBANG_FILES = ["git", "grep", "--full-name", "--line-number", "-I", "^#!"] + +ALL_SOURCE_FILENAMES_REGEXP = r"^.*\.(cpp|h|py|sh)$" ALLOWED_FILENAME_REGEXP = "^[a-zA-Z0-9/_.@][a-zA-Z0-9/_.@-]*$" ALLOWED_SOURCE_FILENAME_REGEXP = "^[a-z0-9_./-]+$" ALLOWED_SOURCE_FILENAME_EXCEPTION_REGEXP = ( "^src/(secp256k1/|minisketch/|univalue/|test/fuzz/FuzzedDataProvider.h)" ) -ALLOWED_PERMISSION_NON_EXECUTABLES = 644 -ALLOWED_PERMISSION_EXECUTABLES = 755 +ALLOWED_PERMISSION_NON_EXECUTABLES = 0o644 +ALLOWED_PERMISSION_EXECUTABLES = 0o755 ALLOWED_EXECUTABLE_SHEBANG = { "py": [b"#!/usr/bin/env python3"], "sh": [b"#!/usr/bin/env bash", b"#!/bin/sh"], @@ -30,8 +32,15 @@ ALLOWED_EXECUTABLE_SHEBANG = { class FileMeta(object): - def __init__(self, file_path: str): - self.file_path = file_path + def __init__(self, file_spec: str): + '''Parse a `git ls files --stage` output line.''' + # 100755 5a150d5f8031fcd75e80a4dd9843afa33655f579 0 ci/test/00_setup_env.sh + meta, self.file_path = file_spec.split('\t', 2) + meta = meta.split() + # The octal file permission of the file. Internally, git only + # keeps an 'executable' bit, so this will always be 0o644 or 0o755. + self.permissions = int(meta[0], 8) & 0o7777 + # We don't currently care about the other fields @property def extension(self) -> Optional[str]: @@ -59,20 +68,24 @@ class FileMeta(object): except IndexError: return None - @property - def permissions(self) -> int: - """ - Returns the octal file permission of the file - """ - return int(oct(os.stat(self.file_path).st_mode)[-3:]) +def get_git_file_metadata() -> Dict[str, FileMeta]: + ''' + Return a dictionary mapping the name of all files in the repository to git tree metadata. + ''' + files_raw = check_output(CMD_ALL_FILES).decode("utf8").rstrip("\0").split("\0") + files = {} + for file_spec in files_raw: + meta = FileMeta(file_spec) + files[meta.file_path] = meta + return files -def check_all_filenames() -> int: +def check_all_filenames(files) -> int: """ Checks every file in the repository against an allowed regexp to make sure only lowercase or uppercase alphanumerics (a-zA-Z0-9), underscores (_), hyphens (-), at (@) and dots (.) are used in repository filenames. """ - filenames = check_output(CMD_ALL_FILES, shell=True).decode("utf8").rstrip("\0").split("\0") + filenames = files.keys() filename_regex = re.compile(ALLOWED_FILENAME_REGEXP) failed_tests = 0 for filename in filenames: @@ -84,14 +97,14 @@ def check_all_filenames() -> int: return failed_tests -def check_source_filenames() -> int: +def check_source_filenames(files) -> int: """ Checks only source files (*.cpp, *.h, *.py, *.sh) against a stricter allowed regexp to make sure only lowercase alphanumerics (a-z0-9), underscores (_), hyphens (-) and dots (.) are used in source code filenames. Additionally there is an exception regexp for directories or files which are excepted from matching this regexp. """ - filenames = check_output(CMD_SOURCE_FILES, shell=True).decode("utf8").rstrip("\0").split("\0") + filenames = [filename for filename in files.keys() if re.match(ALL_SOURCE_FILENAMES_REGEXP, filename, re.IGNORECASE)] filename_regex = re.compile(ALLOWED_SOURCE_FILENAME_REGEXP) filename_exception_regex = re.compile(ALLOWED_SOURCE_FILENAME_EXCEPTION_REGEXP) failed_tests = 0 @@ -104,16 +117,14 @@ def check_source_filenames() -> int: return failed_tests -def check_all_file_permissions() -> int: +def check_all_file_permissions(files) -> int: """ Checks all files in the repository match an allowed executable or non-executable file permission octal. Additionally checks that for executable files, the file contains a shebang line """ - filenames = check_output(CMD_ALL_FILES, shell=True).decode("utf8").rstrip("\0").split("\0") failed_tests = 0 - for filename in filenames: - file_meta = FileMeta(filename) + for filename, file_meta in files.items(): if file_meta.permissions == ALLOWED_PERMISSION_EXECUTABLES: with open(filename, "rb") as f: shebang = f.readline().rstrip(b"\n") @@ -121,7 +132,7 @@ def check_all_file_permissions() -> int: # For any file with executable permissions the first line must contain a shebang if not shebang.startswith(b"#!"): print( - f"""File "{filename}" has permission {ALLOWED_PERMISSION_EXECUTABLES} (executable) and is thus expected to contain a shebang '#!'. Add shebang or do "chmod {ALLOWED_PERMISSION_NON_EXECUTABLES} {filename}" to make it non-executable.""" + f"""File "{filename}" has permission {ALLOWED_PERMISSION_EXECUTABLES:03o} (executable) and is thus expected to contain a shebang '#!'. Add shebang or do "chmod {ALLOWED_PERMISSION_NON_EXECUTABLES:03o} {filename}" to make it non-executable.""" ) failed_tests += 1 @@ -144,18 +155,18 @@ def check_all_file_permissions() -> int: continue else: print( - f"""File "{filename}" has unexpected permission {file_meta.permissions}. Do "chmod {ALLOWED_PERMISSION_NON_EXECUTABLES} {filename}" (if non-executable) or "chmod {ALLOWED_PERMISSION_EXECUTABLES} {filename}" (if executable).""" + f"""File "{filename}" has unexpected permission {file_meta.permissions:03o}. Do "chmod {ALLOWED_PERMISSION_NON_EXECUTABLES:03o} {filename}" (if non-executable) or "chmod {ALLOWED_PERMISSION_EXECUTABLES:03o} {filename}" (if executable).""" ) failed_tests += 1 return failed_tests -def check_shebang_file_permissions() -> int: +def check_shebang_file_permissions(files_meta) -> int: """ Checks every file that contains a shebang line to ensure it has an executable permission """ - filenames = check_output(CMD_SHEBANG_FILES, shell=True).decode("utf8").strip().split("\n") + filenames = check_output(CMD_SHEBANG_FILES).decode("utf8").strip().split("\n") # The git grep command we use returns files which contain a shebang on any line within the file # so we need to filter the list to only files with the shebang on the first line @@ -163,7 +174,7 @@ def check_shebang_file_permissions() -> int: failed_tests = 0 for filename in filenames: - file_meta = FileMeta(filename) + file_meta = files_meta[filename] if file_meta.permissions != ALLOWED_PERMISSION_EXECUTABLES: # These file types are typically expected to be sourced and not executed directly if file_meta.full_extension in ["bash", "init", "openrc", "sh.in"]: @@ -177,18 +188,23 @@ def check_shebang_file_permissions() -> int: continue print( - f"""File "{filename}" contains a shebang line, but has the file permission {file_meta.permissions} instead of the expected executable permission {ALLOWED_PERMISSION_EXECUTABLES}. Do "chmod {ALLOWED_PERMISSION_EXECUTABLES} {filename}" (or remove the shebang line).""" + f"""File "{filename}" contains a shebang line, but has the file permission {file_meta.permissions:03o} instead of the expected executable permission {ALLOWED_PERMISSION_EXECUTABLES:03o}. Do "chmod {ALLOWED_PERMISSION_EXECUTABLES:03o} {filename}" (or remove the shebang line).""" ) failed_tests += 1 return failed_tests def main() -> NoReturn: + root_dir = check_output(CMD_TOP_LEVEL).decode("utf8").strip() + os.chdir(root_dir) + + files = get_git_file_metadata() + failed_tests = 0 - failed_tests += check_all_filenames() - failed_tests += check_source_filenames() - failed_tests += check_all_file_permissions() - failed_tests += check_shebang_file_permissions() + failed_tests += check_all_filenames(files) + failed_tests += check_source_filenames(files) + failed_tests += check_all_file_permissions(files) + failed_tests += check_shebang_file_permissions(files) if failed_tests: print( diff --git a/test/lint/lint-files.sh b/test/lint/lint-files.sh deleted file mode 100755 index 86d7fc724a..0000000000 --- a/test/lint/lint-files.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C - -set -e -cd "$(dirname "$0")/../.." -test/lint/lint-files.py diff --git a/test/lint/lint-format-strings.py b/test/lint/lint-format-strings.py index 2870432bff..28e7b1e4ff 100755 --- a/test/lint/lint-format-strings.py +++ b/test/lint/lint-format-strings.py @@ -1,293 +1,98 @@ #!/usr/bin/env python3 # -# Copyright (c) 2018-2019 The Bitcoin Core developers +# Copyright (c) 2018-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # -# Lint format strings: This program checks that the number of arguments passed -# to a variadic format string function matches the number of format specifiers -# in the format string. -import argparse +""" +Lint format strings: This program checks that the number of arguments passed +to a variadic format string function matches the number of format specifiers +in the format string. +""" + +import subprocess import re import sys -FALSE_POSITIVES = [ - ("src/dbwrapper.cpp", "vsnprintf(p, limit - p, format, backup_ap)"), - ("src/index/base.cpp", "FatalError(const char* fmt, const Args&... args)"), - ("src/netbase.cpp", "LogConnectFailure(bool manual_connection, const char* fmt, const Args&... args)"), - ("src/util/system.cpp", "strprintf(_(COPYRIGHT_HOLDERS).translated, COPYRIGHT_HOLDERS_SUBSTITUTION)"), - ("src/validationinterface.cpp", "LogPrint(BCLog::VALIDATION, fmt \"\\n\", __VA_ARGS__)"), - ("src/wallet/wallet.h", "WalletLogPrintf(std::string fmt, Params... parameters)"), - ("src/wallet/wallet.h", "LogPrintf((\"%s \" + fmt).c_str(), GetDisplayName(), parameters...)"), - ("src/wallet/scriptpubkeyman.h", "WalletLogPrintf(std::string fmt, Params... parameters)"), - ("src/wallet/scriptpubkeyman.h", "LogPrintf((\"%s \" + fmt).c_str(), m_storage.GetDisplayName(), parameters...)"), - ("src/logging.h", "LogPrintf(const char* fmt, const Args&... args)"), - ("src/wallet/scriptpubkeyman.h", "WalletLogPrintf(const std::string& fmt, const Params&... parameters)"), +FUNCTION_NAMES_AND_NUMBER_OF_LEADING_ARGUMENTS = [ + 'FatalError,0', + 'fprintf,1', + 'tfm::format,1', # Assuming tfm::::format(std::ostream&, ... + 'LogConnectFailure,1', + 'LogPrint,1', + 'LogPrintf,0', + 'printf,0', + 'snprintf,2', + 'sprintf,1', + 'strprintf,0', + 'vfprintf,1', + 'vprintf,1', + 'vsnprintf,1', + 'vsprintf,1', + 'WalletLogPrintf,0', ] - - -def parse_function_calls(function_name, source_code): - """Return an array with all calls to function function_name in string source_code. - Preprocessor directives and C++ style comments ("//") in source_code are removed. - - >>> len(parse_function_calls("foo", "foo();bar();foo();bar();")) - 2 - >>> parse_function_calls("foo", "foo(1);bar(1);foo(2);bar(2);")[0].startswith("foo(1);") - True - >>> parse_function_calls("foo", "foo(1);bar(1);foo(2);bar(2);")[1].startswith("foo(2);") - True - >>> len(parse_function_calls("foo", "foo();bar();// foo();bar();")) - 1 - >>> len(parse_function_calls("foo", "#define FOO foo();")) - 0 - """ - assert type(function_name) is str and type(source_code) is str and function_name - lines = [re.sub("// .*", " ", line).strip() - for line in source_code.split("\n") - if not line.strip().startswith("#")] - return re.findall(r"[^a-zA-Z_](?=({}\(.*).*)".format(function_name), " " + " ".join(lines)) - - -def normalize(s): - """Return a normalized version of string s with newlines, tabs and C style comments ("/* ... */") - replaced with spaces. Multiple spaces are replaced with a single space. - - >>> normalize(" /* nothing */ foo\tfoo /* bar */ foo ") - 'foo foo foo' - """ - assert type(s) is str - s = s.replace("\n", " ") - s = s.replace("\t", " ") - s = re.sub(r"/\*.*?\*/", " ", s) - s = re.sub(" {2,}", " ", s) - return s.strip() - - -ESCAPE_MAP = { - r"\n": "[escaped-newline]", - r"\t": "[escaped-tab]", - r'\"': "[escaped-quote]", -} - - -def escape(s): - """Return the escaped version of string s with "\\\"", "\\n" and "\\t" escaped as - "[escaped-backslash]", "[escaped-newline]" and "[escaped-tab]". - - >>> unescape(escape("foo")) == "foo" - True - >>> escape(r'foo \\t foo \\n foo \\\\ foo \\ foo \\"bar\\"') - 'foo [escaped-tab] foo [escaped-newline] foo \\\\\\\\ foo \\\\ foo [escaped-quote]bar[escaped-quote]' - """ - assert type(s) is str - for raw_value, escaped_value in ESCAPE_MAP.items(): - s = s.replace(raw_value, escaped_value) - return s - - -def unescape(s): - """Return the unescaped version of escaped string s. - Reverses the replacements made in function escape(s). - - >>> unescape(escape("bar")) - 'bar' - >>> unescape("foo [escaped-tab] foo [escaped-newline] foo \\\\\\\\ foo \\\\ foo [escaped-quote]bar[escaped-quote]") - 'foo \\\\t foo \\\\n foo \\\\\\\\ foo \\\\ foo \\\\"bar\\\\"' - """ - assert type(s) is str - for raw_value, escaped_value in ESCAPE_MAP.items(): - s = s.replace(escaped_value, raw_value) - return s - - -def parse_function_call_and_arguments(function_name, function_call): - """Split string function_call into an array of strings consisting of: - * the string function_call followed by "(" - * the function call argument #1 - * ... - * the function call argument #n - * a trailing ");" - - The strings returned are in escaped form. See escape(...). - - >>> parse_function_call_and_arguments("foo", 'foo("%s", "foo");') - ['foo(', '"%s",', ' "foo"', ')'] - >>> parse_function_call_and_arguments("foo", 'foo("%s", "foo");') - ['foo(', '"%s",', ' "foo"', ')'] - >>> parse_function_call_and_arguments("foo", 'foo("%s %s", "foo", "bar");') - ['foo(', '"%s %s",', ' "foo",', ' "bar"', ')'] - >>> parse_function_call_and_arguments("fooprintf", 'fooprintf("%050d", i);') - ['fooprintf(', '"%050d",', ' i', ')'] - >>> parse_function_call_and_arguments("foo", 'foo(bar(foobar(barfoo("foo"))), foobar); barfoo') - ['foo(', 'bar(foobar(barfoo("foo"))),', ' foobar', ')'] - >>> parse_function_call_and_arguments("foo", "foo()") - ['foo(', '', ')'] - >>> parse_function_call_and_arguments("foo", "foo(123)") - ['foo(', '123', ')'] - >>> parse_function_call_and_arguments("foo", 'foo("foo")') - ['foo(', '"foo"', ')'] - >>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>,wchar_t>().to_bytes(buf), err);') - ['strprintf(', '"%s (%d)",', ' std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>,wchar_t>().to_bytes(buf),', ' err', ')'] - >>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", foo<wchar_t>().to_bytes(buf), err);') - ['strprintf(', '"%s (%d)",', ' foo<wchar_t>().to_bytes(buf),', ' err', ')'] - >>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", foo().to_bytes(buf), err);') - ['strprintf(', '"%s (%d)",', ' foo().to_bytes(buf),', ' err', ')'] - >>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", foo << 1, err);') - ['strprintf(', '"%s (%d)",', ' foo << 1,', ' err', ')'] - >>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", foo<bar>() >> 1, err);') - ['strprintf(', '"%s (%d)",', ' foo<bar>() >> 1,', ' err', ')'] - >>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", foo < 1 ? bar : foobar, err);') - ['strprintf(', '"%s (%d)",', ' foo < 1 ? bar : foobar,', ' err', ')'] - >>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", foo < 1, err);') - ['strprintf(', '"%s (%d)",', ' foo < 1,', ' err', ')'] - >>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", foo > 1 ? bar : foobar, err);') - ['strprintf(', '"%s (%d)",', ' foo > 1 ? bar : foobar,', ' err', ')'] - >>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", foo > 1, err);') - ['strprintf(', '"%s (%d)",', ' foo > 1,', ' err', ')'] - >>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", foo <= 1, err);') - ['strprintf(', '"%s (%d)",', ' foo <= 1,', ' err', ')'] - >>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", foo <= bar<1, 2>(1, 2), err);') - ['strprintf(', '"%s (%d)",', ' foo <= bar<1, 2>(1, 2),', ' err', ')'] - >>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", foo>foo<1,2>(1,2)?bar:foobar,err)'); - ['strprintf(', '"%s (%d)",', ' foo>foo<1,2>(1,2)?bar:foobar,', 'err', ')'] - >>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", foo>foo<1,2>(1,2),err)'); - ['strprintf(', '"%s (%d)",', ' foo>foo<1,2>(1,2),', 'err', ')'] - """ - assert type(function_name) is str and type(function_call) is str and function_name - remaining = normalize(escape(function_call)) - expected_function_call = "{}(".format(function_name) - assert remaining.startswith(expected_function_call) - parts = [expected_function_call] - remaining = remaining[len(expected_function_call):] - open_parentheses = 1 - open_template_arguments = 0 - in_string = False - parts.append("") - for i, char in enumerate(remaining): - parts.append(parts.pop() + char) - if char == "\"": - in_string = not in_string - continue - if in_string: - continue - if char == "(": - open_parentheses += 1 - continue - if char == ")": - open_parentheses -= 1 - if open_parentheses > 1: - continue - if open_parentheses == 0: - parts.append(parts.pop()[:-1]) - parts.append(char) - break - prev_char = remaining[i - 1] if i - 1 >= 0 else None - next_char = remaining[i + 1] if i + 1 <= len(remaining) - 1 else None - if char == "<" and next_char not in [" ", "<", "="] and prev_char not in [" ", "<"]: - open_template_arguments += 1 - continue - if char == ">" and next_char not in [" ", ">", "="] and prev_char not in [" ", ">"] and open_template_arguments > 0: - open_template_arguments -= 1 - if open_template_arguments > 0: - continue - if char == ",": - parts.append("") - return parts - - -def parse_string_content(argument): - """Return the text within quotes in string argument. - - >>> parse_string_content('1 "foo %d bar" 2') - 'foo %d bar' - >>> parse_string_content('1 foobar 2') - '' - >>> parse_string_content('1 "bar" 2') - 'bar' - >>> parse_string_content('1 "foo" 2 "bar" 3') - 'foobar' - >>> parse_string_content('1 "foo" 2 " " "bar" 3') - 'foo bar' - >>> parse_string_content('""') - '' - >>> parse_string_content('') - '' - >>> parse_string_content('1 2 3') - '' - """ - assert type(argument) is str - string_content = "" - in_string = False - for char in normalize(escape(argument)): - if char == "\"": - in_string = not in_string - elif in_string: - string_content += char - return string_content - - -def count_format_specifiers(format_string): - """Return the number of format specifiers in string format_string. - - >>> count_format_specifiers("foo bar foo") - 0 - >>> count_format_specifiers("foo %d bar foo") - 1 - >>> count_format_specifiers("foo %d bar %i foo") - 2 - >>> count_format_specifiers("foo %d bar %i foo %% foo") - 2 - >>> count_format_specifiers("foo %d bar %i foo %% foo %d foo") - 3 - >>> count_format_specifiers("foo %d bar %i foo %% foo %*d foo") - 4 - """ - assert type(format_string) is str - format_string = format_string.replace('%%', 'X') - n = 0 - in_specifier = False - for i, char in enumerate(format_string): - if char == "%": - in_specifier = True - n += 1 - elif char in "aAcdeEfFgGinopsuxX": - in_specifier = False - elif in_specifier and char == "*": - n += 1 - return n - +RUN_LINT_FILE = 'test/lint/run-lint-format-strings.py' + +def check_doctest(): + command = [ + sys.executable, + '-m', + 'doctest', + RUN_LINT_FILE, + ] + try: + subprocess.run(command, check = True) + except subprocess.CalledProcessError: + sys.exit(1) + +def get_matching_files(function_name): + command = [ + 'git', + 'grep', + '--full-name', + '-l', + function_name, + '--', + '*.c', + '*.cpp', + '*.h', + ] + try: + return subprocess.check_output(command, stderr = subprocess.STDOUT).decode('utf-8').splitlines() + except subprocess.CalledProcessError as e: + if e.returncode > 1: # return code is 1 when match is empty + print(e.output.decode('utf-8'), end='') + sys.exit(1) + return [] def main(): - parser = argparse.ArgumentParser(description="This program checks that the number of arguments passed " - "to a variadic format string function matches the number of format " - "specifiers in the format string.") - parser.add_argument("--skip-arguments", type=int, help="number of arguments before the format string " - "argument (e.g. 1 in the case of fprintf)", default=0) - parser.add_argument("function_name", help="function name (e.g. fprintf)", default=None) - parser.add_argument("file", nargs="*", help="C++ source code file (e.g. foo.cpp)") - args = parser.parse_args() exit_code = 0 - for filename in args.file: - with open(filename, "r", encoding="utf-8") as f: - for function_call_str in parse_function_calls(args.function_name, f.read()): - parts = parse_function_call_and_arguments(args.function_name, function_call_str) - relevant_function_call_str = unescape("".join(parts))[:512] - if (f.name, relevant_function_call_str) in FALSE_POSITIVES: - continue - if len(parts) < 3 + args.skip_arguments: - exit_code = 1 - print("{}: Could not parse function call string \"{}(...)\": {}".format(f.name, args.function_name, relevant_function_call_str)) - continue - argument_count = len(parts) - 3 - args.skip_arguments - format_str = parse_string_content(parts[1 + args.skip_arguments]) - format_specifier_count = count_format_specifiers(format_str) - if format_specifier_count != argument_count: - exit_code = 1 - print("{}: Expected {} argument(s) after format string but found {} argument(s): {}".format(f.name, format_specifier_count, argument_count, relevant_function_call_str)) - continue - sys.exit(exit_code) + check_doctest() + for s in FUNCTION_NAMES_AND_NUMBER_OF_LEADING_ARGUMENTS: + function_name, skip_arguments = s.split(',') + matching_files = get_matching_files(function_name) + + matching_files_filtered = [] + for matching_file in matching_files: + if not re.search('^src/(leveldb|secp256k1|minisketch|tinyformat|univalue|test/fuzz/strprintf.cpp)', matching_file): + matching_files_filtered.append(matching_file) + matching_files_filtered.sort() + + run_lint_args = [ + RUN_LINT_FILE, + '--skip-arguments', + skip_arguments, + function_name, + ] + run_lint_args.extend(matching_files_filtered) + + try: + subprocess.run(run_lint_args, check = True) + except subprocess.CalledProcessError: + exit_code = 1 + sys.exit(exit_code) -if __name__ == "__main__": +if __name__ == '__main__': main() diff --git a/test/lint/lint-format-strings.sh b/test/lint/lint-format-strings.sh deleted file mode 100755 index d98f12b1a1..0000000000 --- a/test/lint/lint-format-strings.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Lint format strings: This program checks that the number of arguments passed -# to a variadic format string function matches the number of format specifiers -# in the format string. - -export LC_ALL=C - -FUNCTION_NAMES_AND_NUMBER_OF_LEADING_ARGUMENTS=( - "FatalError,0" - "fprintf,1" - "tfm::format,1" # Assuming tfm::::format(std::ostream&, ... - "LogConnectFailure,1" - "LogPrint,1" - "LogPrintf,0" - "printf,0" - "snprintf,2" - "sprintf,1" - "strprintf,0" - "vfprintf,1" - "vprintf,1" - "vsnprintf,1" - "vsprintf,1" - "WalletLogPrintf,0" -) - -EXIT_CODE=0 -if ! python3 -m doctest test/lint/lint-format-strings.py; then - EXIT_CODE=1 -fi -for S in "${FUNCTION_NAMES_AND_NUMBER_OF_LEADING_ARGUMENTS[@]}"; do - IFS="," read -r FUNCTION_NAME SKIP_ARGUMENTS <<< "${S}" - for MATCHING_FILE in $(git grep --full-name -l "${FUNCTION_NAME}" -- "*.c" "*.cpp" "*.h" | sort | grep -vE "^src/(leveldb|secp256k1|minisketch|tinyformat|univalue|test/fuzz/strprintf.cpp)"); do - MATCHING_FILES+=("${MATCHING_FILE}") - done - if ! test/lint/lint-format-strings.py --skip-arguments "${SKIP_ARGUMENTS}" "${FUNCTION_NAME}" "${MATCHING_FILES[@]}"; then - EXIT_CODE=1 - fi -done -exit ${EXIT_CODE} diff --git a/test/lint/lint-git-commit-check.py b/test/lint/lint-git-commit-check.py new file mode 100755 index 0000000000..a1d03370e8 --- /dev/null +++ b/test/lint/lint-git-commit-check.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2020-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# Linter to check that commit messages have a new line before the body +# or no body at all + +import argparse +import os +import sys + +from subprocess import check_output + + +def parse_args(): + """Parse command line arguments.""" + parser = argparse.ArgumentParser( + description=""" + Linter to check that commit messages have a new line before + the body or no body at all. + """, + epilog=f""" + You can manually set the commit-range with the COMMIT_RANGE + environment variable (e.g. "COMMIT_RANGE='47ba2c3...ee50c9e' + {sys.argv[0]}"). Defaults to current merge base when neither + prev-commits nor the environment variable is set. + """) + + parser.add_argument("--prev-commits", "-p", required=False, help="The previous n commits to check") + + return parser.parse_args() + + +def main(): + args = parse_args() + exit_code = 0 + + if not os.getenv("COMMIT_RANGE"): + if args.prev_commits: + commit_range = "HEAD~" + args.prev_commits + "...HEAD" + else: + # This assumes that the target branch of the pull request will be master. + merge_base = check_output(["git", "merge-base", "HEAD", "master"], universal_newlines=True, encoding="utf8").rstrip("\n") + commit_range = merge_base + "..HEAD" + else: + commit_range = os.getenv("COMMIT_RANGE") + + commit_hashes = check_output(["git", "log", commit_range, "--format=%H"], universal_newlines=True, encoding="utf8").splitlines() + + for hash in commit_hashes: + commit_info = check_output(["git", "log", "--format=%B", "-n", "1", hash], universal_newlines=True, encoding="utf8").splitlines() + if len(commit_info) >= 2: + if commit_info[1]: + print(f"The subject line of commit hash {hash} is followed by a non-empty line. Subject lines should always be followed by a blank line.") + exit_code = 1 + + sys.exit(exit_code) + + +if __name__ == "__main__": + main() diff --git a/test/lint/lint-git-commit-check.sh b/test/lint/lint-git-commit-check.sh deleted file mode 100755 index f77373ed00..0000000000 --- a/test/lint/lint-git-commit-check.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2020-2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Linter to check that commit messages have a new line before the body -# or no body at all - -export LC_ALL=C - -EXIT_CODE=0 - -while getopts "?" opt; do - case $opt in - ?) - echo "Usage: $0 [N]" - echo " COMMIT_RANGE='<commit range>' $0" - echo " $0 -?" - echo "Checks unmerged commits, the previous N commits, or a commit range." - echo "COMMIT_RANGE='47ba2c3...ee50c9e' $0" - exit ${EXIT_CODE} - ;; - esac -done - -if [ -z "${COMMIT_RANGE}" ]; then - if [ -n "$1" ]; then - COMMIT_RANGE="HEAD~$1...HEAD" - else - # This assumes that the target branch of the pull request will be master. - MERGE_BASE=$(git merge-base HEAD master) - COMMIT_RANGE="$MERGE_BASE..HEAD" - fi -fi - -while IFS= read -r commit_hash || [[ -n "$commit_hash" ]]; do - n_line=0 - while IFS= read -r line || [[ -n "$line" ]]; do - n_line=$((n_line+1)) - length=${#line} - if [ $n_line -eq 2 ] && [ "$length" -ne 0 ]; then - echo "The subject line of commit hash ${commit_hash} is followed by a non-empty line. Subject lines should always be followed by a blank line." - EXIT_CODE=1 - fi - done < <(git log --format=%B -n 1 "$commit_hash") -done < <(git log "${COMMIT_RANGE}" --format=%H) - -exit ${EXIT_CODE} diff --git a/test/lint/lint-include-guards.py b/test/lint/lint-include-guards.py new file mode 100755 index 0000000000..86284517d5 --- /dev/null +++ b/test/lint/lint-include-guards.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2018-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" +Check include guards. +""" + +import re +import sys +from subprocess import check_output +from typing import List + + +HEADER_ID_PREFIX = 'BITCOIN_' +HEADER_ID_SUFFIX = '_H' + +EXCLUDE_FILES_WITH_PREFIX = ['src/crypto/ctaes', + 'src/leveldb', + 'src/crc32c', + 'src/secp256k1', + 'src/minisketch', + 'src/univalue', + 'src/tinyformat.h', + 'src/bench/nanobench.h', + 'src/test/fuzz/FuzzedDataProvider.h'] + + +def _get_header_file_lst() -> List[str]: + """ Helper function to get a list of header filepaths to be + checked for include guards. + """ + git_cmd_lst = ['git', 'ls-files', '--', '*.h'] + header_file_lst = check_output( + git_cmd_lst).decode('utf-8').splitlines() + + header_file_lst = [hf for hf in header_file_lst + if not any(ef in hf for ef + in EXCLUDE_FILES_WITH_PREFIX)] + + return header_file_lst + + +def _get_header_id(header_file: str) -> str: + """ Helper function to get the header id from a header file + string. + + eg: 'src/wallet/walletdb.h' -> 'BITCOIN_WALLET_WALLETDB_H' + + Args: + header_file: Filepath to header file. + + Returns: + The header id. + """ + header_id_base = header_file.split('/')[1:] + header_id_base = '_'.join(header_id_base) + header_id_base = header_id_base.replace('.h', '').replace('-', '_') + header_id_base = header_id_base.upper() + + header_id = f'{HEADER_ID_PREFIX}{header_id_base}{HEADER_ID_SUFFIX}' + + return header_id + + +def main(): + exit_code = 0 + + header_file_lst = _get_header_file_lst() + for header_file in header_file_lst: + header_id = _get_header_id(header_file) + + regex_pattern = f'^#(ifndef|define|endif //) {header_id}' + + with open(header_file, 'r', encoding='utf-8') as f: + header_file_contents = f.readlines() + + count = 0 + for header_file_contents_string in header_file_contents: + include_guard_lst = re.findall( + regex_pattern, header_file_contents_string) + + count += len(include_guard_lst) + + if count != 3: + print(f'{header_file} seems to be missing the expected ' + 'include guard:') + print(f' #ifndef {header_id}') + print(f' #define {header_id}') + print(' ...') + print(f' #endif // {header_id}\n') + exit_code = 1 + + sys.exit(exit_code) + + +if __name__ == '__main__': + main() diff --git a/test/lint/lint-include-guards.sh b/test/lint/lint-include-guards.sh deleted file mode 100755 index f14218aa74..0000000000 --- a/test/lint/lint-include-guards.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Check include guards. - -export LC_ALL=C -HEADER_ID_PREFIX="BITCOIN_" -HEADER_ID_SUFFIX="_H" - -REGEXP_EXCLUDE_FILES_WITH_PREFIX="src/(crypto/ctaes/|leveldb/|crc32c/|secp256k1/|minisketch/|test/fuzz/FuzzedDataProvider.h|tinyformat.h|bench/nanobench.h|univalue/)" - -EXIT_CODE=0 -for HEADER_FILE in $(git ls-files -- "*.h" | grep -vE "^${REGEXP_EXCLUDE_FILES_WITH_PREFIX}") -do - HEADER_ID_BASE=$(cut -f2- -d/ <<< "${HEADER_FILE}" | sed "s/\.h$//g" | tr / _ | tr - _ | tr "[:lower:]" "[:upper:]") - HEADER_ID="${HEADER_ID_PREFIX}${HEADER_ID_BASE}${HEADER_ID_SUFFIX}" - if [[ $(grep --count --extended-regexp "^#(ifndef|define|endif //) ${HEADER_ID}" "${HEADER_FILE}") != 3 ]]; then - echo "${HEADER_FILE} seems to be missing the expected include guard:" - echo " #ifndef ${HEADER_ID}" - echo " #define ${HEADER_ID}" - echo " ..." - echo " #endif // ${HEADER_ID}" - echo - EXIT_CODE=1 - fi -done -exit ${EXIT_CODE} diff --git a/test/lint/lint-includes.py b/test/lint/lint-includes.py new file mode 100755 index 0000000000..ae62994642 --- /dev/null +++ b/test/lint/lint-includes.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2018-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# Check for duplicate includes. +# Guard against accidental introduction of new Boost dependencies. +# Check includes: Check for duplicate includes. Enforce bracket syntax includes. + +import os +import re +import sys + +from subprocess import check_output, CalledProcessError + + +EXCLUDED_DIRS = ["src/leveldb/", + "src/crc32c/", + "src/secp256k1/", + "src/minisketch/", + "src/univalue/"] + +EXPECTED_BOOST_INCLUDES = ["boost/algorithm/string/replace.hpp", + "boost/date_time/posix_time/posix_time.hpp", + "boost/multi_index/hashed_index.hpp", + "boost/multi_index/ordered_index.hpp", + "boost/multi_index/sequenced_index.hpp", + "boost/multi_index_container.hpp", + "boost/process.hpp", + "boost/signals2/connection.hpp", + "boost/signals2/optional_last_value.hpp", + "boost/signals2/signal.hpp", + "boost/test/included/unit_test.hpp", + "boost/test/unit_test.hpp"] + + +def get_toplevel(): + return check_output(["git", "rev-parse", "--show-toplevel"], universal_newlines=True, encoding="utf8").rstrip("\n") + + +def list_files_by_suffix(suffixes): + exclude_args = [":(exclude)" + dir for dir in EXCLUDED_DIRS] + + files_list = check_output(["git", "ls-files", "src"] + exclude_args, universal_newlines=True, encoding="utf8").splitlines() + + return [file for file in files_list if file.endswith(suffixes)] + + +def find_duplicate_includes(include_list): + tempset = set() + duplicates = set() + + for inclusion in include_list: + if inclusion in tempset: + duplicates.add(inclusion) + else: + tempset.add(inclusion) + + return duplicates + + +def find_included_cpps(): + included_cpps = list() + + try: + included_cpps = check_output(["git", "grep", "-E", r"^#include [<\"][^>\"]+\.cpp[>\"]", "--", "*.cpp", "*.h"], universal_newlines=True, encoding="utf8").splitlines() + except CalledProcessError as e: + if e.returncode > 1: + raise e + + return included_cpps + + +def find_extra_boosts(): + included_boosts = list() + filtered_included_boost_set = set() + exclusion_set = set() + + try: + included_boosts = check_output(["git", "grep", "-E", r"^#include <boost/", "--", "*.cpp", "*.h"], universal_newlines=True, encoding="utf8").splitlines() + except CalledProcessError as e: + if e.returncode > 1: + raise e + + for boost in included_boosts: + filtered_included_boost_set.add(re.findall(r'(?<=\<).+?(?=\>)', boost)[0]) + + for expected_boost in EXPECTED_BOOST_INCLUDES: + for boost in filtered_included_boost_set: + if expected_boost in boost: + exclusion_set.add(boost) + + extra_boosts = set(filtered_included_boost_set.difference(exclusion_set)) + + return extra_boosts + + +def find_quote_syntax_inclusions(): + exclude_args = [":(exclude)" + dir for dir in EXCLUDED_DIRS] + quote_syntax_inclusions = list() + + try: + quote_syntax_inclusions = check_output(["git", "grep", r"^#include \"", "--", "*.cpp", "*.h"] + exclude_args, universal_newlines=True, encoding="utf8").splitlines() + except CalledProcessError as e: + if e.returncode > 1: + raise e + + return quote_syntax_inclusions + + +def main(): + exit_code = 0 + + os.chdir(get_toplevel()) + + # Check for duplicate includes + for filename in list_files_by_suffix((".cpp", ".h")): + with open(filename, "r", encoding="utf8") as file: + include_list = [line.rstrip("\n") for line in file if re.match(r"^#include", line)] + + duplicates = find_duplicate_includes(include_list) + + if duplicates: + print(f"Duplicate include(s) in {filename}:") + for duplicate in duplicates: + print(duplicate) + print("") + exit_code = 1 + + # Check if code includes .cpp-files + included_cpps = find_included_cpps() + + if included_cpps: + print("The following files #include .cpp files:") + for included_cpp in included_cpps: + print(included_cpp) + print("") + exit_code = 1 + + # Guard against accidental introduction of new Boost dependencies + extra_boosts = find_extra_boosts() + + if extra_boosts: + for boost in extra_boosts: + print(f"A new Boost dependency in the form of \"{boost}\" appears to have been introduced:") + print(check_output(["git", "grep", boost, "--", "*.cpp", "*.h"], universal_newlines=True, encoding="utf8")) + exit_code = 1 + + # Check if Boost dependencies are no longer used + for expected_boost in EXPECTED_BOOST_INCLUDES: + try: + check_output(["git", "grep", "-q", r"^#include <%s>" % expected_boost, "--", "*.cpp", "*.h"], universal_newlines=True, encoding="utf8") + except CalledProcessError as e: + if e.returncode > 1: + raise e + else: + print(f"Good job! The Boost dependency \"{expected_boost}\" is no longer used. " + "Please remove it from EXPECTED_BOOST_INCLUDES in test/lint/lint-includes.py " + "to make sure this dependency is not accidentally reintroduced.\n") + exit_code = 1 + + # Enforce bracket syntax includes + quote_syntax_inclusions = find_quote_syntax_inclusions() + + if quote_syntax_inclusions: + print("Please use bracket syntax includes (\"#include <foo.h>\") instead of quote syntax includes:") + for quote_syntax_inclusion in quote_syntax_inclusions: + print(quote_syntax_inclusion) + exit_code = 1 + + sys.exit(exit_code) + + +if __name__ == "__main__": + main() diff --git a/test/lint/lint-includes.sh b/test/lint/lint-includes.sh deleted file mode 100755 index 98d5021657..0000000000 --- a/test/lint/lint-includes.sh +++ /dev/null @@ -1,104 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Check for duplicate includes. -# Guard against accidental introduction of new Boost dependencies. -# Check includes: Check for duplicate includes. Enforce bracket syntax includes. - -export LC_ALL=C -IGNORE_REGEXP="/(leveldb|secp256k1|minisketch|univalue|crc32c)/" - -# cd to root folder of git repo for git ls-files to work properly -cd "$(dirname "$0")/../.." || exit 1 - -filter_suffix() { - git ls-files | grep -E "^src/.*\.${1}"'$' | grep -Ev "${IGNORE_REGEXP}" -} - -EXIT_CODE=0 - -for HEADER_FILE in $(filter_suffix h); do - DUPLICATE_INCLUDES_IN_HEADER_FILE=$(grep -E "^#include " < "${HEADER_FILE}" | sort | uniq -d) - if [[ ${DUPLICATE_INCLUDES_IN_HEADER_FILE} != "" ]]; then - echo "Duplicate include(s) in ${HEADER_FILE}:" - echo "${DUPLICATE_INCLUDES_IN_HEADER_FILE}" - echo - EXIT_CODE=1 - fi -done - -for CPP_FILE in $(filter_suffix cpp); do - DUPLICATE_INCLUDES_IN_CPP_FILE=$(grep -E "^#include " < "${CPP_FILE}" | sort | uniq -d) - if [[ ${DUPLICATE_INCLUDES_IN_CPP_FILE} != "" ]]; then - echo "Duplicate include(s) in ${CPP_FILE}:" - echo "${DUPLICATE_INCLUDES_IN_CPP_FILE}" - echo - EXIT_CODE=1 - fi -done - -INCLUDED_CPP_FILES=$(git grep -E "^#include [<\"][^>\"]+\.cpp[>\"]" -- "*.cpp" "*.h") -if [[ ${INCLUDED_CPP_FILES} != "" ]]; then - echo "The following files #include .cpp files:" - echo "${INCLUDED_CPP_FILES}" - echo - EXIT_CODE=1 -fi - -EXPECTED_BOOST_INCLUDES=( - boost/algorithm/string.hpp - boost/algorithm/string/classification.hpp - boost/algorithm/string/replace.hpp - boost/algorithm/string/split.hpp - boost/date_time/posix_time/posix_time.hpp - boost/filesystem.hpp - boost/filesystem/fstream.hpp - boost/multi_index/hashed_index.hpp - boost/multi_index/ordered_index.hpp - boost/multi_index/sequenced_index.hpp - boost/multi_index_container.hpp - boost/process.hpp - boost/signals2/connection.hpp - boost/signals2/optional_last_value.hpp - boost/signals2/signal.hpp - boost/test/unit_test.hpp -) - -for BOOST_INCLUDE in $(git grep '^#include <boost/' -- "*.cpp" "*.h" | cut -f2 -d: | cut -f2 -d'<' | cut -f1 -d'>' | sort -u); do - IS_EXPECTED_INCLUDE=0 - for EXPECTED_BOOST_INCLUDE in "${EXPECTED_BOOST_INCLUDES[@]}"; do - if [[ "${BOOST_INCLUDE}" == "${EXPECTED_BOOST_INCLUDE}" ]]; then - IS_EXPECTED_INCLUDE=1 - break - fi - done - if [[ ${IS_EXPECTED_INCLUDE} == 0 ]]; then - EXIT_CODE=1 - echo "A new Boost dependency in the form of \"${BOOST_INCLUDE}\" appears to have been introduced:" - git grep "${BOOST_INCLUDE}" -- "*.cpp" "*.h" - echo - fi -done - -for EXPECTED_BOOST_INCLUDE in "${EXPECTED_BOOST_INCLUDES[@]}"; do - if ! git grep -q "^#include <${EXPECTED_BOOST_INCLUDE}>" -- "*.cpp" "*.h"; then - echo "Good job! The Boost dependency \"${EXPECTED_BOOST_INCLUDE}\" is no longer used." - echo "Please remove it from EXPECTED_BOOST_INCLUDES in $0" - echo "to make sure this dependency is not accidentally reintroduced." - echo - EXIT_CODE=1 - fi -done - -QUOTE_SYNTAX_INCLUDES=$(git grep '^#include "' -- "*.cpp" "*.h" | grep -Ev "${IGNORE_REGEXP}") -if [[ ${QUOTE_SYNTAX_INCLUDES} != "" ]]; then - echo "Please use bracket syntax includes (\"#include <foo.h>\") instead of quote syntax includes:" - echo "${QUOTE_SYNTAX_INCLUDES}" - echo - EXIT_CODE=1 -fi - -exit ${EXIT_CODE} diff --git a/test/lint/lint-locale-dependence.py b/test/lint/lint-locale-dependence.py new file mode 100755 index 0000000000..9b2cf4587a --- /dev/null +++ b/test/lint/lint-locale-dependence.py @@ -0,0 +1,261 @@ +#!/usr/bin/env python3 +# Copyright (c) 2018-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# Be aware that bitcoind and bitcoin-qt differ in terms of localization: Qt +# opts in to POSIX localization by running setlocale(LC_ALL, "") on startup, +# whereas no such call is made in bitcoind. +# +# Qt runs setlocale(LC_ALL, "") on initialization. This installs the locale +# specified by the user's LC_ALL (or LC_*) environment variable as the new +# C locale. +# +# In contrast, bitcoind does not opt in to localization -- no call to +# setlocale(LC_ALL, "") is made and the environment variables LC_* are +# thus ignored. +# +# This results in situations where bitcoind is guaranteed to be running +# with the classic locale ("C") whereas the locale of bitcoin-qt will vary +# depending on the user's environment variables. +# +# An example: Assuming the environment variable LC_ALL=de_DE then the +# call std::to_string(1.23) will return "1.230000" in bitcoind but +# "1,230000" in bitcoin-qt. +# +# From the Qt documentation: +# "On Unix/Linux Qt is configured to use the system locale settings by default. +# This can cause a conflict when using POSIX functions, for instance, when +# converting between data types such as floats and strings, since the notation +# may differ between locales. To get around this problem, call the POSIX function +# setlocale(LC_NUMERIC,"C") right after initializing QApplication, QGuiApplication +# or QCoreApplication to reset the locale that is used for number formatting to +# "C"-locale." +# +# See https://doc.qt.io/qt-5/qcoreapplication.html#locale-settings and +# https://stackoverflow.com/a/34878283 for more details. +# +# TODO: Reduce KNOWN_VIOLATIONS by replacing uses of locale dependent snprintf with strprintf. + +import re +import sys + +from subprocess import check_output, CalledProcessError + + +KNOWN_VIOLATIONS = [ + "src/dbwrapper.cpp:.*vsnprintf", + "src/test/dbwrapper_tests.cpp:.*snprintf", + "src/test/fuzz/locale.cpp:.*setlocale", + "src/test/fuzz/string.cpp:.*strtol", + "src/test/fuzz/string.cpp:.*strtoul", + "src/test/util_tests.cpp:.*strtoll", + "src/wallet/bdb.cpp:.*DbEnv::strerror", # False positive + "src/util/syserror.cpp:.*strerror", # Outside this function use `SysErrorString` +] + +REGEXP_EXTERNAL_DEPENDENCIES_EXCLUSIONS = [ + "src/crypto/ctaes/", + "src/leveldb/", + "src/secp256k1/", + "src/minisketch/", + "src/tinyformat.h", + "src/univalue/" +] + +LOCALE_DEPENDENT_FUNCTIONS = [ + "alphasort", # LC_COLLATE (via strcoll) + "asctime", # LC_TIME (directly) + "asprintf", # (via vasprintf) + "atof", # LC_NUMERIC (via strtod) + "atoi", # LC_NUMERIC (via strtol) + "atol", # LC_NUMERIC (via strtol) + "atoll", # (via strtoll) + "atoq", + "btowc", # LC_CTYPE (directly) + "ctime", # (via asctime or localtime) + "dprintf", # (via vdprintf) + "fgetwc", + "fgetws", + "fold_case", # boost::locale::fold_case + "fprintf", # (via vfprintf) + "fputwc", + "fputws", + "fscanf", # (via __vfscanf) + "fwprintf", # (via __vfwprintf) + "getdate", # via __getdate_r => isspace // __localtime_r + "getwc", + "getwchar", + "is_digit", # boost::algorithm::is_digit + "is_space", # boost::algorithm::is_space + "isalnum", # LC_CTYPE + "isalpha", # LC_CTYPE + "isblank", # LC_CTYPE + "iscntrl", # LC_CTYPE + "isctype", # LC_CTYPE + "isdigit", # LC_CTYPE + "isgraph", # LC_CTYPE + "islower", # LC_CTYPE + "isprint", # LC_CTYPE + "ispunct", # LC_CTYPE + "isspace", # LC_CTYPE + "isupper", # LC_CTYPE + "iswalnum", # LC_CTYPE + "iswalpha", # LC_CTYPE + "iswblank", # LC_CTYPE + "iswcntrl", # LC_CTYPE + "iswctype", # LC_CTYPE + "iswdigit", # LC_CTYPE + "iswgraph", # LC_CTYPE + "iswlower", # LC_CTYPE + "iswprint", # LC_CTYPE + "iswpunct", # LC_CTYPE + "iswspace", # LC_CTYPE + "iswupper", # LC_CTYPE + "iswxdigit", # LC_CTYPE + "isxdigit", # LC_CTYPE + "localeconv", # LC_NUMERIC + LC_MONETARY + "mblen", # LC_CTYPE + "mbrlen", + "mbrtowc", + "mbsinit", + "mbsnrtowcs", + "mbsrtowcs", + "mbstowcs", # LC_CTYPE + "mbtowc", # LC_CTYPE + "mktime", + "normalize", # boost::locale::normalize + "printf", # LC_NUMERIC + "putwc", + "putwchar", + "scanf", # LC_NUMERIC + "setlocale", + "snprintf", + "sprintf", + "sscanf", + "std::locale::global", + "std::to_string", + "stod", + "stof", + "stoi", + "stol", + "stold", + "stoll", + "stoul", + "stoull", + "strcasecmp", + "strcasestr", + "strcoll", # LC_COLLATE + "strerror", + "strfmon", + "strftime", # LC_TIME + "strncasecmp", + "strptime", + "strtod", # LC_NUMERIC + "strtof", + "strtoimax", + "strtol", # LC_NUMERIC + "strtold", + "strtoll", + "strtoq", + "strtoul", # LC_NUMERIC + "strtoull", + "strtoumax", + "strtouq", + "strxfrm", # LC_COLLATE + "swprintf", + "to_lower", # boost::locale::to_lower + "to_title", # boost::locale::to_title + "to_upper", # boost::locale::to_upper + "tolower", # LC_CTYPE + "toupper", # LC_CTYPE + "towctrans", + "towlower", # LC_CTYPE + "towupper", # LC_CTYPE + "trim", # boost::algorithm::trim + "trim_left", # boost::algorithm::trim_left + "trim_right", # boost::algorithm::trim_right + "ungetwc", + "vasprintf", + "vdprintf", + "versionsort", + "vfprintf", + "vfscanf", + "vfwprintf", + "vprintf", + "vscanf", + "vsnprintf", + "vsprintf", + "vsscanf", + "vswprintf", + "vwprintf", + "wcrtomb", + "wcscasecmp", + "wcscoll", # LC_COLLATE + "wcsftime", # LC_TIME + "wcsncasecmp", + "wcsnrtombs", + "wcsrtombs", + "wcstod", # LC_NUMERIC + "wcstof", + "wcstoimax", + "wcstol", # LC_NUMERIC + "wcstold", + "wcstoll", + "wcstombs", # LC_CTYPE + "wcstoul", # LC_NUMERIC + "wcstoull", + "wcstoumax", + "wcswidth", + "wcsxfrm", # LC_COLLATE + "wctob", + "wctomb", # LC_CTYPE + "wctrans", + "wctype", + "wcwidth", + "wprintf" +] + + +def find_locale_dependent_function_uses(): + regexp_locale_dependent_functions = "|".join(LOCALE_DEPENDENT_FUNCTIONS) + exclude_args = [":(exclude)" + excl for excl in REGEXP_EXTERNAL_DEPENDENCIES_EXCLUSIONS] + git_grep_command = ["git", "grep", "-E", "[^a-zA-Z0-9_\\`'\"<>](" + regexp_locale_dependent_functions + ")(_r|_s)?[^a-zA-Z0-9_\\`'\"<>]", "--", "*.cpp", "*.h"] + exclude_args + git_grep_output = list() + + try: + git_grep_output = check_output(git_grep_command, universal_newlines=True, encoding="utf8").splitlines() + except CalledProcessError as e: + if e.returncode > 1: + raise e + + return git_grep_output + + +def main(): + exit_code = 0 + + regexp_ignore_known_violations = "|".join(KNOWN_VIOLATIONS) + git_grep_output = find_locale_dependent_function_uses() + + for locale_dependent_function in LOCALE_DEPENDENT_FUNCTIONS: + matches = [line for line in git_grep_output + if re.search("[^a-zA-Z0-9_\\`'\"<>]" + locale_dependent_function + "(_r|_s)?[^a-zA-Z0-9_\\`'\"<>]", line) + and not re.search("\\.(c|cpp|h):\\s*(//|\\*|/\\*|\").*" + locale_dependent_function, line) + and not re.search(regexp_ignore_known_violations, line)] + if matches: + print(f"The locale dependent function {locale_dependent_function}(...) appears to be used:") + for match in matches: + print(match) + print("") + exit_code = 1 + + if exit_code == 1: + print("Unnecessary locale depedence can cause bugs that are very tricky to isolate and fix. Please avoid using locale dependent functions if possible.\n") + print(f"Advice not applicable in this specific case? Add an exception by updating the ignore list in {sys.argv[0]}") + + sys.exit(exit_code) + + +if __name__ == "__main__": + main() diff --git a/test/lint/lint-locale-dependence.sh b/test/lint/lint-locale-dependence.sh deleted file mode 100755 index 7d608eed6a..0000000000 --- a/test/lint/lint-locale-dependence.sh +++ /dev/null @@ -1,241 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2018-2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C - -# Be aware that bitcoind and bitcoin-qt differ in terms of localization: Qt -# opts in to POSIX localization by running setlocale(LC_ALL, "") on startup, -# whereas no such call is made in bitcoind. -# -# Qt runs setlocale(LC_ALL, "") on initialization. This installs the locale -# specified by the user's LC_ALL (or LC_*) environment variable as the new -# C locale. -# -# In contrast, bitcoind does not opt in to localization -- no call to -# setlocale(LC_ALL, "") is made and the environment variables LC_* are -# thus ignored. -# -# This results in situations where bitcoind is guaranteed to be running -# with the classic locale ("C") whereas the locale of bitcoin-qt will vary -# depending on the user's environment variables. -# -# An example: Assuming the environment variable LC_ALL=de_DE then the -# call std::to_string(1.23) will return "1.230000" in bitcoind but -# "1,230000" in bitcoin-qt. -# -# From the Qt documentation: -# "On Unix/Linux Qt is configured to use the system locale settings by default. -# This can cause a conflict when using POSIX functions, for instance, when -# converting between data types such as floats and strings, since the notation -# may differ between locales. To get around this problem, call the POSIX function -# setlocale(LC_NUMERIC,"C") right after initializing QApplication, QGuiApplication -# or QCoreApplication to reset the locale that is used for number formatting to -# "C"-locale." -# -# See https://doc.qt.io/qt-5/qcoreapplication.html#locale-settings and -# https://stackoverflow.com/a/34878283 for more details. - -# TODO: Reduce KNOWN_VIOLATIONS by replacing uses of locale dependent snprintf with strprintf. -KNOWN_VIOLATIONS=( - "src/dbwrapper.cpp:.*vsnprintf" - "src/test/dbwrapper_tests.cpp:.*snprintf" - "src/test/fuzz/locale.cpp" - "src/test/fuzz/string.cpp" - "src/test/util_tests.cpp" -) - -REGEXP_IGNORE_EXTERNAL_DEPENDENCIES="^src/(crypto/ctaes/|leveldb/|secp256k1/|minisketch/|tinyformat.h|univalue/)" - -LOCALE_DEPENDENT_FUNCTIONS=( - alphasort # LC_COLLATE (via strcoll) - asctime # LC_TIME (directly) - asprintf # (via vasprintf) - atof # LC_NUMERIC (via strtod) - atoi # LC_NUMERIC (via strtol) - atol # LC_NUMERIC (via strtol) - atoll # (via strtoll) - atoq - btowc # LC_CTYPE (directly) - ctime # (via asctime or localtime) - dprintf # (via vdprintf) - fgetwc - fgetws - fold_case # boost::locale::fold_case - fprintf # (via vfprintf) - fputwc - fputws - fscanf # (via __vfscanf) - fwprintf # (via __vfwprintf) - getdate # via __getdate_r => isspace // __localtime_r - getwc - getwchar - is_digit # boost::algorithm::is_digit - is_space # boost::algorithm::is_space - isalnum # LC_CTYPE - isalpha # LC_CTYPE - isblank # LC_CTYPE - iscntrl # LC_CTYPE - isctype # LC_CTYPE - isdigit # LC_CTYPE - isgraph # LC_CTYPE - islower # LC_CTYPE - isprint # LC_CTYPE - ispunct # LC_CTYPE - isspace # LC_CTYPE - isupper # LC_CTYPE - iswalnum # LC_CTYPE - iswalpha # LC_CTYPE - iswblank # LC_CTYPE - iswcntrl # LC_CTYPE - iswctype # LC_CTYPE - iswdigit # LC_CTYPE - iswgraph # LC_CTYPE - iswlower # LC_CTYPE - iswprint # LC_CTYPE - iswpunct # LC_CTYPE - iswspace # LC_CTYPE - iswupper # LC_CTYPE - iswxdigit # LC_CTYPE - isxdigit # LC_CTYPE - localeconv # LC_NUMERIC + LC_MONETARY - mblen # LC_CTYPE - mbrlen - mbrtowc - mbsinit - mbsnrtowcs - mbsrtowcs - mbstowcs # LC_CTYPE - mbtowc # LC_CTYPE - mktime - normalize # boost::locale::normalize - printf # LC_NUMERIC - putwc - putwchar - scanf # LC_NUMERIC - setlocale - snprintf - sprintf - sscanf - std::locale::global - std::to_string - stod - stof - stoi - stol - stold - stoll - stoul - stoull - strcasecmp - strcasestr - strcoll # LC_COLLATE -# strerror - strfmon - strftime # LC_TIME - strncasecmp - strptime - strtod # LC_NUMERIC - strtof - strtoimax - strtol # LC_NUMERIC - strtold - strtoll - strtoq - strtoul # LC_NUMERIC - strtoull - strtoumax - strtouq - strxfrm # LC_COLLATE - swprintf - to_lower # boost::locale::to_lower - to_title # boost::locale::to_title - to_upper # boost::locale::to_upper - tolower # LC_CTYPE - toupper # LC_CTYPE - towctrans - towlower # LC_CTYPE - towupper # LC_CTYPE - trim # boost::algorithm::trim - trim_left # boost::algorithm::trim_left - trim_right # boost::algorithm::trim_right - ungetwc - vasprintf - vdprintf - versionsort - vfprintf - vfscanf - vfwprintf - vprintf - vscanf - vsnprintf - vsprintf - vsscanf - vswprintf - vwprintf - wcrtomb - wcscasecmp - wcscoll # LC_COLLATE - wcsftime # LC_TIME - wcsncasecmp - wcsnrtombs - wcsrtombs - wcstod # LC_NUMERIC - wcstof - wcstoimax - wcstol # LC_NUMERIC - wcstold - wcstoll - wcstombs # LC_CTYPE - wcstoul # LC_NUMERIC - wcstoull - wcstoumax - wcswidth - wcsxfrm # LC_COLLATE - wctob - wctomb # LC_CTYPE - wctrans - wctype - wcwidth - wprintf -) - -function join_array { - local IFS="$1" - shift - echo "$*" -} - -REGEXP_IGNORE_KNOWN_VIOLATIONS=$(join_array "|" "${KNOWN_VIOLATIONS[@]}") - -# Invoke "git grep" only once in order to minimize run-time -REGEXP_LOCALE_DEPENDENT_FUNCTIONS=$(join_array "|" "${LOCALE_DEPENDENT_FUNCTIONS[@]}") -GIT_GREP_OUTPUT=$(git grep -E "[^a-zA-Z0-9_\`'\"<>](${REGEXP_LOCALE_DEPENDENT_FUNCTIONS}(_r|_s)?)[^a-zA-Z0-9_\`'\"<>]" -- "*.cpp" "*.h") - -EXIT_CODE=0 -for LOCALE_DEPENDENT_FUNCTION in "${LOCALE_DEPENDENT_FUNCTIONS[@]}"; do - MATCHES=$(grep -E "[^a-zA-Z0-9_\`'\"<>]${LOCALE_DEPENDENT_FUNCTION}(_r|_s)?[^a-zA-Z0-9_\`'\"<>]" <<< "${GIT_GREP_OUTPUT}" | \ - grep -vE "\.(c|cpp|h):\s*(//|\*|/\*|\").*${LOCALE_DEPENDENT_FUNCTION}") - if [[ ${REGEXP_IGNORE_EXTERNAL_DEPENDENCIES} != "" ]]; then - MATCHES=$(grep -vE "${REGEXP_IGNORE_EXTERNAL_DEPENDENCIES}" <<< "${MATCHES}") - fi - if [[ ${REGEXP_IGNORE_KNOWN_VIOLATIONS} != "" ]]; then - MATCHES=$(grep -vE "${REGEXP_IGNORE_KNOWN_VIOLATIONS}" <<< "${MATCHES}") - fi - if [[ ${MATCHES} != "" ]]; then - echo "The locale dependent function ${LOCALE_DEPENDENT_FUNCTION}(...) appears to be used:" - echo "${MATCHES}" - echo - EXIT_CODE=1 - fi -done -if [[ ${EXIT_CODE} != 0 ]]; then - echo "Unnecessary locale dependence can cause bugs that are very" - echo "tricky to isolate and fix. Please avoid using locale dependent" - echo "functions if possible." - echo - echo "Advice not applicable in this specific case? Add an exception" - echo "by updating the ignore list in $0" -fi -exit ${EXIT_CODE} diff --git a/test/lint/lint-logs.py b/test/lint/lint-logs.py new file mode 100755 index 0000000000..de53729b4e --- /dev/null +++ b/test/lint/lint-logs.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2018-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# Check that all logs are terminated with '\n' +# +# Some logs are continued over multiple lines. They should be explicitly +# commented with /* Continued */ + +import re +import sys + +from subprocess import check_output + + +def main(): + logs_list = check_output(["git", "grep", "--extended-regexp", r"(LogPrintLevel|LogPrintf?)\(", "--", "*.cpp"], universal_newlines=True, encoding="utf8").splitlines() + + unterminated_logs = [line for line in logs_list if not re.search(r'(\\n"|/\* Continued \*/)', line)] + + if unterminated_logs != []: + print("All calls to LogPrintf(), LogPrint(), LogPrintLevel(), and WalletLogPrintf() should be terminated with \"\\n\".") + print("") + + for line in unterminated_logs: + print(line) + + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/test/lint/lint-logs.sh b/test/lint/lint-logs.sh deleted file mode 100755 index 6d5165f649..0000000000 --- a/test/lint/lint-logs.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Check that all logs are terminated with '\n' -# -# Some logs are continued over multiple lines. They should be explicitly -# commented with /* Continued */ -# -# There are some instances of LogPrintf() in comments. Those can be -# ignored - -export LC_ALL=C -UNTERMINATED_LOGS=$(git grep --extended-regexp "LogPrintf?\(" -- "*.cpp" | \ - grep -v '\\n"' | \ - grep -v '\.\.\.' | \ - grep -v "/\* Continued \*/" | \ - grep -v "LogPrint()" | \ - grep -v "LogPrintf()") -if [[ ${UNTERMINATED_LOGS} != "" ]]; then - # shellcheck disable=SC2028 - echo "All calls to LogPrintf() and LogPrint() should be terminated with \\n" - echo - echo "${UNTERMINATED_LOGS}" - exit 1 -fi diff --git a/test/lint/lint-python-dead-code.py b/test/lint/lint-python-dead-code.py new file mode 100755 index 0000000000..b3f9394788 --- /dev/null +++ b/test/lint/lint-python-dead-code.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" +Find dead Python code. +""" + +from subprocess import check_output, STDOUT, CalledProcessError + +FILES_ARGS = ['git', 'ls-files', '--', '*.py'] + + +def check_vulture_install(): + try: + check_output(["vulture", "--version"]) + except FileNotFoundError: + print("Skipping Python dead code linting since vulture is not installed. Install by running \"pip3 install vulture\"") + exit(0) + + +def main(): + check_vulture_install() + + files = check_output(FILES_ARGS).decode("utf-8").splitlines() + # --min-confidence 100 will only report code that is guaranteed to be unused within the analyzed files. + # Any value below 100 introduces the risk of false positives, which would create an unacceptable maintenance burden. + vulture_args = ['vulture', '--min-confidence=100'] + files + + try: + check_output(vulture_args, stderr=STDOUT) + except CalledProcessError as e: + print(e.output.decode("utf-8"), end="") + print("Python dead code detection found some issues") + exit(1) + + +if __name__ == "__main__": + main() diff --git a/test/lint/lint-python-dead-code.sh b/test/lint/lint-python-dead-code.sh deleted file mode 100755 index 247bfb310a..0000000000 --- a/test/lint/lint-python-dead-code.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Find dead Python code. - -export LC_ALL=C - -if ! command -v vulture > /dev/null; then - echo "Skipping Python dead code linting since vulture is not installed. Install by running \"pip3 install vulture\"" - exit 0 -fi - -# --min-confidence 100 will only report code that is guaranteed to be unused within the analyzed files. -# Any value below 100 introduces the risk of false positives, which would create an unacceptable maintenance burden. -mapfile -t FILES < <(git ls-files -- "*.py") -if ! vulture --min-confidence 100 "${FILES[@]}"; then - echo "Python dead code detection found some issues" - exit 1 -fi diff --git a/test/lint/lint-python-mutable-default-parameters.py b/test/lint/lint-python-mutable-default-parameters.py new file mode 100755 index 0000000000..7991e3630b --- /dev/null +++ b/test/lint/lint-python-mutable-default-parameters.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" +Detect when a mutable list or dict is used as a default parameter value in a Python function. +""" + +import subprocess +import sys + + +def main(): + command = [ + "git", + "grep", + "-E", + r"^\s*def [a-zA-Z0-9_]+\(.*=\s*(\[|\{)", + "--", + "*.py", + ] + output = subprocess.run(command, stdout=subprocess.PIPE, universal_newlines=True) + if len(output.stdout) > 0: + error_msg = ( + "A mutable list or dict seems to be used as default parameter value:\n\n" + f"{output.stdout}\n" + f"{example()}" + ) + print(error_msg) + sys.exit(1) + else: + sys.exit(0) + + +def example(): + return """This is how mutable list and dict default parameter values behave: + +>>> def f(i, j=[], k={}): +... j.append(i) +... k[i] = True +... return j, k +... +>>> f(1) +([1], {1: True}) +>>> f(1) +([1, 1], {1: True}) +>>> f(2) +([1, 1, 2], {1: True, 2: True}) + +The intended behaviour was likely: + +>>> def f(i, j=None, k=None): +... if j is None: +... j = [] +... if k is None: +... k = {} +... j.append(i) +... k[i] = True +... return j, k +... +>>> f(1) +([1], {1: True}) +>>> f(1) +([1], {1: True}) +>>> f(2) +([2], {2: True})""" + + +if __name__ == "__main__": + main() diff --git a/test/lint/lint-python-mutable-default-parameters.sh b/test/lint/lint-python-mutable-default-parameters.sh deleted file mode 100755 index 1f9f035d30..0000000000 --- a/test/lint/lint-python-mutable-default-parameters.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2019 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Detect when a mutable list or dict is used as a default parameter value in a Python function. - -export LC_ALL=C -EXIT_CODE=0 -OUTPUT=$(git grep -E '^\s*def [a-zA-Z0-9_]+\(.*=\s*(\[|\{)' -- "*.py") -if [[ ${OUTPUT} != "" ]]; then - echo "A mutable list or dict seems to be used as default parameter value:" - echo - echo "${OUTPUT}" - echo - cat << EXAMPLE -This is how mutable list and dict default parameter values behave: - ->>> def f(i, j=[], k={}): -... j.append(i) -... k[i] = True -... return j, k -... ->>> f(1) -([1], {1: True}) ->>> f(1) -([1, 1], {1: True}) ->>> f(2) -([1, 1, 2], {1: True, 2: True}) - -The intended behaviour was likely: - ->>> def f(i, j=None, k=None): -... if j is None: -... j = [] -... if k is None: -... k = {} -... j.append(i) -... k[i] = True -... return j, k -... ->>> f(1) -([1], {1: True}) ->>> f(1) -([1], {1: True}) ->>> f(2) -([2], {2: True}) -EXAMPLE - EXIT_CODE=1 -fi -exit ${EXIT_CODE} diff --git a/test/lint/lint-python-utf8-encoding.py b/test/lint/lint-python-utf8-encoding.py new file mode 100755 index 0000000000..62fdc34d50 --- /dev/null +++ b/test/lint/lint-python-utf8-encoding.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2018-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# Make sure we explicitly open all text files using UTF-8 (or ASCII) encoding to +# avoid potential issues on the BSDs where the locale is not always set. + +import sys +import re + +from subprocess import check_output, CalledProcessError + +EXCLUDED_DIRS = ["src/crc32c/"] + + +def get_exclude_args(): + return [":(exclude)" + dir for dir in EXCLUDED_DIRS] + + +def check_fileopens(): + fileopens = list() + + try: + fileopens = check_output(["git", "grep", r" open(", "--", "*.py"] + get_exclude_args(), universal_newlines=True, encoding="utf8").splitlines() + except CalledProcessError as e: + if e.returncode > 1: + raise e + + filtered_fileopens = [fileopen for fileopen in fileopens if not re.search(r"encoding=.(ascii|utf8|utf-8).|open\([^,]*, ['\"][^'\"]*b[^'\"]*['\"]", fileopen)] + + return filtered_fileopens + + +def check_checked_outputs(): + checked_outputs = list() + + try: + checked_outputs = check_output(["git", "grep", "check_output(", "--", "*.py"] + get_exclude_args(), universal_newlines=True, encoding="utf8").splitlines() + except CalledProcessError as e: + if e.returncode > 1: + raise e + + filtered_checked_outputs = [checked_output for checked_output in checked_outputs if re.search(r"universal_newlines=True", checked_output) and not re.search(r"encoding=.(ascii|utf8|utf-8).", checked_output)] + + return filtered_checked_outputs + + +def main(): + exit_code = 0 + + nonexplicit_utf8_fileopens = check_fileopens() + if nonexplicit_utf8_fileopens: + print("Python's open(...) seems to be used to open text files without explicitly specifying encoding='utf8':\n") + for fileopen in nonexplicit_utf8_fileopens: + print(fileopen) + exit_code = 1 + + nonexplicit_utf8_checked_outputs = check_checked_outputs() + if nonexplicit_utf8_checked_outputs: + if nonexplicit_utf8_fileopens: + print("\n") + print("Python's check_output(...) seems to be used to get program outputs without explicitly specifying encoding='utf8':\n") + for checked_output in nonexplicit_utf8_checked_outputs: + print(checked_output) + exit_code = 1 + + sys.exit(exit_code) + + +if __name__ == "__main__": + main() diff --git a/test/lint/lint-python-utf8-encoding.sh b/test/lint/lint-python-utf8-encoding.sh deleted file mode 100755 index 6e5b18fc23..0000000000 --- a/test/lint/lint-python-utf8-encoding.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Make sure we explicitly open all text files using UTF-8 (or ASCII) encoding to -# avoid potential issues on the BSDs where the locale is not always set. - -export LC_ALL=C -EXIT_CODE=0 -OUTPUT=$(git grep " open(" -- "*.py" ":(exclude)src/crc32c/" | grep -vE "encoding=.(ascii|utf8|utf-8)." | grep -vE "open\([^,]*, ['\"][^'\"]*b[^'\"]*['\"]") -if [[ ${OUTPUT} != "" ]]; then - echo "Python's open(...) seems to be used to open text files without explicitly" - echo "specifying encoding=\"utf8\":" - echo - echo "${OUTPUT}" - EXIT_CODE=1 -fi -OUTPUT=$(git grep "check_output(" -- "*.py" ":(exclude)src/crc32c/"| grep "universal_newlines=True" | grep -vE "encoding=.(ascii|utf8|utf-8).") -if [[ ${OUTPUT} != "" ]]; then - echo "Python's check_output(...) seems to be used to get program outputs without explicitly" - echo "specifying encoding=\"utf8\":" - echo - echo "${OUTPUT}" - EXIT_CODE=1 -fi -exit ${EXIT_CODE} diff --git a/test/lint/lint-python.py b/test/lint/lint-python.py new file mode 100755 index 0000000000..4d16facfea --- /dev/null +++ b/test/lint/lint-python.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" +Check for specified flake8 and mypy warnings in python files. +""" + +import os +import pkg_resources +import subprocess +import sys + +DEPS = ['flake8', 'mypy', 'pyzmq'] +MYPY_CACHE_DIR = f"{os.getenv('BASE_ROOT_DIR', '')}/test/.mypy_cache" +FILES_ARGS = ['git', 'ls-files', 'test/functional/*.py', 'contrib/devtools/*.py'] + +ENABLED = ( + 'E101,' # indentation contains mixed spaces and tabs + 'E112,' # expected an indented block + 'E113,' # unexpected indentation + 'E115,' # expected an indented block (comment) + 'E116,' # unexpected indentation (comment) + 'E125,' # continuation line with same indent as next logical line + 'E129,' # visually indented line with same indent as next logical line + 'E131,' # continuation line unaligned for hanging indent + 'E133,' # closing bracket is missing indentation + 'E223,' # tab before operator + 'E224,' # tab after operator + 'E242,' # tab after ',' + 'E266,' # too many leading '#' for block comment + 'E271,' # multiple spaces after keyword + 'E272,' # multiple spaces before keyword + 'E273,' # tab after keyword + 'E274,' # tab before keyword + 'E275,' # missing whitespace after keyword + 'E304,' # blank lines found after function decorator + 'E306,' # expected 1 blank line before a nested definition + 'E401,' # multiple imports on one line + 'E402,' # module level import not at top of file + 'E502,' # the backslash is redundant between brackets + 'E701,' # multiple statements on one line (colon) + 'E702,' # multiple statements on one line (semicolon) + 'E703,' # statement ends with a semicolon + 'E711,' # comparison to None should be 'if cond is None:' + 'E714,' # test for object identity should be "is not" + 'E721,' # do not compare types, use "isinstance()" + 'E742,' # do not define classes named "l", "O", or "I" + 'E743,' # do not define functions named "l", "O", or "I" + 'E901,' # SyntaxError: invalid syntax + 'E902,' # TokenError: EOF in multi-line string + 'F401,' # module imported but unused + 'F402,' # import module from line N shadowed by loop variable + 'F403,' # 'from foo_module import *' used; unable to detect undefined names + 'F404,' # future import(s) name after other statements + 'F405,' # foo_function may be undefined, or defined from star imports: bar_module + 'F406,' # "from module import *" only allowed at module level + 'F407,' # an undefined __future__ feature name was imported + 'F601,' # dictionary key name repeated with different values + 'F602,' # dictionary key variable name repeated with different values + 'F621,' # too many expressions in an assignment with star-unpacking + 'F622,' # two or more starred expressions in an assignment (a, *b, *c = d) + 'F631,' # assertion test is a tuple, which are always True + 'F632,' # use ==/!= to compare str, bytes, and int literals + 'F701,' # a break statement outside of a while or for loop + 'F702,' # a continue statement outside of a while or for loop + 'F703,' # a continue statement in a finally block in a loop + 'F704,' # a yield or yield from statement outside of a function + 'F705,' # a return statement with arguments inside a generator + 'F706,' # a return statement outside of a function/method + 'F707,' # an except: block as not the last exception handler + 'F811,' # redefinition of unused name from line N + 'F812,' # list comprehension redefines 'foo' from line N + 'F821,' # undefined name 'Foo' + 'F822,' # undefined name name in __all__ + 'F823,' # local variable name … referenced before assignment + 'F831,' # duplicate argument name in function definition + 'F841,' # local variable 'foo' is assigned to but never used + 'W191,' # indentation contains tabs + 'W291,' # trailing whitespace + 'W292,' # no newline at end of file + 'W293,' # blank line contains whitespace + 'W601,' # .has_key() is deprecated, use "in" + 'W602,' # deprecated form of raising exception + 'W603,' # "<>" is deprecated, use "!=" + 'W604,' # backticks are deprecated, use "repr()" + 'W605,' # invalid escape sequence "x" + 'W606,' # 'async' and 'await' are reserved keywords starting with Python 3.7 +) + + +def check_dependencies(): + working_set = {pkg.key for pkg in pkg_resources.working_set} + + for dep in DEPS: + if dep not in working_set: + print(f"Skipping Python linting since {dep} is not installed.") + exit(0) + + +def main(): + check_dependencies() + + if len(sys.argv) > 1: + flake8_files = sys.argv[1:] + else: + files_args = ['git', 'ls-files', '*.py'] + flake8_files = subprocess.check_output(files_args).decode("utf-8").splitlines() + + flake8_args = ['flake8', '--ignore=B,C,E,F,I,N,W', f'--select={ENABLED}'] + flake8_files + flake8_env = os.environ.copy() + flake8_env["PYTHONWARNINGS"] = "ignore" + + try: + subprocess.check_call(flake8_args, env=flake8_env) + except subprocess.CalledProcessError: + exit(1) + + mypy_files = subprocess.check_output(FILES_ARGS).decode("utf-8").splitlines() + mypy_args = ['mypy', '--show-error-codes'] + mypy_files + + try: + subprocess.check_call(mypy_args) + except subprocess.CalledProcessError: + exit(1) + + +if __name__ == "__main__": + main() diff --git a/test/lint/lint-python.sh b/test/lint/lint-python.sh deleted file mode 100755 index 7d7857d325..0000000000 --- a/test/lint/lint-python.sh +++ /dev/null @@ -1,111 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2017-2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Check for specified flake8 warnings in python files. - -export LC_ALL=C -export MYPY_CACHE_DIR="${BASE_ROOT_DIR}/test/.mypy_cache" - -enabled=( - E101 # indentation contains mixed spaces and tabs - E112 # expected an indented block - E113 # unexpected indentation - E115 # expected an indented block (comment) - E116 # unexpected indentation (comment) - E125 # continuation line with same indent as next logical line - E129 # visually indented line with same indent as next logical line - E131 # continuation line unaligned for hanging indent - E133 # closing bracket is missing indentation - E223 # tab before operator - E224 # tab after operator - E242 # tab after ',' - E266 # too many leading '#' for block comment - E271 # multiple spaces after keyword - E272 # multiple spaces before keyword - E273 # tab after keyword - E274 # tab before keyword - E275 # missing whitespace after keyword - E304 # blank lines found after function decorator - E306 # expected 1 blank line before a nested definition - E401 # multiple imports on one line - E402 # module level import not at top of file - E502 # the backslash is redundant between brackets - E701 # multiple statements on one line (colon) - E702 # multiple statements on one line (semicolon) - E703 # statement ends with a semicolon - E711 # comparison to None should be 'if cond is None:' - E714 # test for object identity should be "is not" - E721 # do not compare types, use "isinstance()" - E742 # do not define classes named "l", "O", or "I" - E743 # do not define functions named "l", "O", or "I" - E901 # SyntaxError: invalid syntax - E902 # TokenError: EOF in multi-line string - F401 # module imported but unused - F402 # import module from line N shadowed by loop variable - F403 # 'from foo_module import *' used; unable to detect undefined names - F404 # future import(s) name after other statements - F405 # foo_function may be undefined, or defined from star imports: bar_module - F406 # "from module import *" only allowed at module level - F407 # an undefined __future__ feature name was imported - F601 # dictionary key name repeated with different values - F602 # dictionary key variable name repeated with different values - F621 # too many expressions in an assignment with star-unpacking - F622 # two or more starred expressions in an assignment (a, *b, *c = d) - F631 # assertion test is a tuple, which are always True - F632 # use ==/!= to compare str, bytes, and int literals - F701 # a break statement outside of a while or for loop - F702 # a continue statement outside of a while or for loop - F703 # a continue statement in a finally block in a loop - F704 # a yield or yield from statement outside of a function - F705 # a return statement with arguments inside a generator - F706 # a return statement outside of a function/method - F707 # an except: block as not the last exception handler - F811 # redefinition of unused name from line N - F812 # list comprehension redefines 'foo' from line N - F821 # undefined name 'Foo' - F822 # undefined name name in __all__ - F823 # local variable name … referenced before assignment - F831 # duplicate argument name in function definition - F841 # local variable 'foo' is assigned to but never used - W191 # indentation contains tabs - W291 # trailing whitespace - W292 # no newline at end of file - W293 # blank line contains whitespace - W601 # .has_key() is deprecated, use "in" - W602 # deprecated form of raising exception - W603 # "<>" is deprecated, use "!=" - W604 # backticks are deprecated, use "repr()" - W605 # invalid escape sequence "x" - W606 # 'async' and 'await' are reserved keywords starting with Python 3.7 -) - -if ! command -v flake8 > /dev/null; then - echo "Skipping Python linting since flake8 is not installed." - exit 0 -elif PYTHONWARNINGS="ignore" flake8 --version | grep -q "Python 2"; then - echo "Skipping Python linting since flake8 is running under Python 2. Install the Python 3 version of flake8." - exit 0 -fi - -EXIT_CODE=0 - -# shellcheck disable=SC2046 -if ! PYTHONWARNINGS="ignore" flake8 --ignore=B,C,E,F,I,N,W --select=$(IFS=","; echo "${enabled[*]}") $( - if [[ $# == 0 ]]; then - git ls-files "*.py" - else - echo "$@" - fi -); then - EXIT_CODE=1 -fi - -mapfile -t FILES < <(git ls-files "test/functional/*.py" "contrib/devtools/*.py") -if ! mypy --show-error-codes "${FILES[@]}"; then - EXIT_CODE=1 -fi - -exit $EXIT_CODE diff --git a/test/lint/lint-qt.sh b/test/lint/lint-qt.sh deleted file mode 100755 index 2e77682aa2..0000000000 --- a/test/lint/lint-qt.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Check for SIGNAL/SLOT connect style, removed since Qt4 support drop. - -export LC_ALL=C - -EXIT_CODE=0 - -OUTPUT=$(git grep -E '(SIGNAL|, ?SLOT)\(' -- src/qt) -if [[ ${OUTPUT} != "" ]]; then - echo "Use Qt5 connect style in:" - echo "$OUTPUT" - EXIT_CODE=1 -fi - -exit ${EXIT_CODE} diff --git a/test/lint/lint-shell-locale.py b/test/lint/lint-shell-locale.py new file mode 100755 index 0000000000..f3dfe18a95 --- /dev/null +++ b/test/lint/lint-shell-locale.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2018-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" +Make sure all shell scripts are: +a.) explicitly opt out of locale dependence using + "export LC_ALL=C" or "export LC_ALL=C.UTF-8", or +b.) explicitly opt in to locale dependence using the annotation below. +""" + +import subprocess +import sys +import re + +OPT_IN_LINE = '# This script is intentionally locale dependent by not setting \"export LC_ALL=C\"' + +OPT_OUT_LINES = [ + 'export LC_ALL=C', + 'export LC_ALL=C.UTF-8', +] + +def get_shell_files_list(): + command = [ + 'git', + 'ls-files', + '--', + '*.sh', + ] + try: + return subprocess.check_output(command, stderr = subprocess.STDOUT).decode('utf-8').splitlines() + except subprocess.CalledProcessError as e: + if e.returncode > 1: # return code is 1 when match is empty + print(e.output.decode('utf-8'), end='') + sys.exit(1) + return [] + +def main(): + exit_code = 0 + shell_files = get_shell_files_list() + for file_path in shell_files: + if re.search('src/(secp256k1|minisketch|univalue)/', file_path): + continue + + with open(file_path, 'r', encoding='utf-8') as file_obj: + contents = file_obj.read() + + if OPT_IN_LINE in contents: + continue + + non_comment_pattern = re.compile(r'^\s*((?!#).+)$', re.MULTILINE) + non_comment_lines = re.findall(non_comment_pattern, contents) + if not non_comment_lines: + continue + + first_non_comment_line = non_comment_lines[0] + if first_non_comment_line not in OPT_OUT_LINES: + print(f'Missing "export LC_ALL=C" (to avoid locale dependence) as first non-comment non-empty line in {file_path}') + exit_code = 1 + + return sys.exit(exit_code) + +if __name__ == '__main__': + main() + diff --git a/test/lint/lint-shell-locale.sh b/test/lint/lint-shell-locale.sh deleted file mode 100755 index 4c6b8a57e6..0000000000 --- a/test/lint/lint-shell-locale.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Make sure all shell scripts: -# a.) explicitly opt out of locale dependence using -# "export LC_ALL=C" or "export LC_ALL=C.UTF-8", or -# b.) explicitly opt in to locale dependence using the annotation below. - -export LC_ALL=C - -EXIT_CODE=0 -for SHELL_SCRIPT in $(git ls-files -- "*.sh" | grep -vE "src/(secp256k1|minisketch|univalue)/"); do - if grep -q "# This script is intentionally locale dependent by not setting \"export LC_ALL=C\"" "${SHELL_SCRIPT}"; then - continue - fi - FIRST_NON_COMMENT_LINE=$(grep -vE '^(#.*)?$' "${SHELL_SCRIPT}" | head -1) - if [[ ${FIRST_NON_COMMENT_LINE} != "export LC_ALL=C" && ${FIRST_NON_COMMENT_LINE} != "export LC_ALL=C.UTF-8" ]]; then - echo "Missing \"export LC_ALL=C\" (to avoid locale dependence) as first non-comment non-empty line in ${SHELL_SCRIPT}" - EXIT_CODE=1 - fi -done -exit ${EXIT_CODE} diff --git a/test/lint/lint-shell.py b/test/lint/lint-shell.py new file mode 100755 index 0000000000..f1e4494350 --- /dev/null +++ b/test/lint/lint-shell.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2018-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" +Check for shellcheck warnings in shell scripts. +""" + +import subprocess +import re +import sys + +# Disabled warnings: +DISABLED = [ + 'SC2162', # read without -r will mangle backslashes. +] + +def check_shellcheck_install(): + try: + subprocess.run(['shellcheck', '--version'], stdout=subprocess.DEVNULL, check=True) + except FileNotFoundError: + print('Skipping shell linting since shellcheck is not installed.') + sys.exit(0) + +def get_files(command): + output = subprocess.run(command, stdout=subprocess.PIPE, universal_newlines=True) + files = output.stdout.split('\n') + + # remove whitespace element + files = list(filter(None, files)) + return files + +def main(): + check_shellcheck_install() + + # build the `exclude` flag + exclude = '--exclude=' + ','.join(DISABLED) + + # build the `sourced files` list + sourced_files_cmd = [ + 'git', + 'grep', + '-El', + r'^# shellcheck shell=', + ] + sourced_files = get_files(sourced_files_cmd) + + # build the `guix files` list + guix_files_cmd = [ + 'git', + 'grep', + '-El', + r'^#!\/usr\/bin\/env bash', + '--', + 'contrib/guix', + 'contrib/shell', + ] + guix_files = get_files(guix_files_cmd) + + # build the other script files list + files_cmd = [ + 'git', + 'ls-files', + '--', + '*.sh', + ] + files = get_files(files_cmd) + # remove everything that doesn't match this regex + reg = re.compile(r'src/[leveldb,secp256k1,minisketch,univalue]') + files[:] = [file for file in files if not reg.match(file)] + + # build the `shellcheck` command + shellcheck_cmd = [ + 'shellcheck', + '--external-sources', + '--check-sourced', + '--source-path=SCRIPTDIR', + ] + shellcheck_cmd.append(exclude) + shellcheck_cmd.extend(sourced_files) + shellcheck_cmd.extend(guix_files) + shellcheck_cmd.extend(files) + + # run the `shellcheck` command + try: + subprocess.check_call(shellcheck_cmd) + except subprocess.CalledProcessError: + sys.exit(1) + +if __name__ == '__main__': + main() diff --git a/test/lint/lint-shell.sh b/test/lint/lint-shell.sh deleted file mode 100755 index 5fa104fce6..0000000000 --- a/test/lint/lint-shell.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Check for shellcheck warnings in shell scripts. - -export LC_ALL=C - -# Disabled warnings: -disabled=( - SC2162 # read without -r will mangle backslashes. -) - -EXIT_CODE=0 - -if ! command -v shellcheck > /dev/null; then - echo "Skipping shell linting since shellcheck is not installed." - exit $EXIT_CODE -fi - -SHELLCHECK_CMD=(shellcheck --external-sources --check-sourced --source-path=SCRIPTDIR) -EXCLUDE="--exclude=$(IFS=','; echo "${disabled[*]}")" -# Check shellcheck directive used for sourced files -mapfile -t SOURCED_FILES < <(git ls-files | xargs gawk '/^# shellcheck shell=/ {print FILENAME} {nextfile}') -mapfile -t GUIX_FILES < <(git ls-files contrib/guix contrib/shell | xargs gawk '/^#!\/usr\/bin\/env bash/ {print FILENAME} {nextfile}') -mapfile -t FILES < <(git ls-files -- '*.sh' | grep -vE 'src/(leveldb|secp256k1|minisketch|univalue)/') -if ! "${SHELLCHECK_CMD[@]}" "$EXCLUDE" "${SOURCED_FILES[@]}" "${GUIX_FILES[@]}" "${FILES[@]}"; then - EXIT_CODE=1 -fi - -exit $EXIT_CODE diff --git a/test/lint/lint-spelling.py b/test/lint/lint-spelling.py new file mode 100755 index 0000000000..5da1b243f7 --- /dev/null +++ b/test/lint/lint-spelling.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" +Warn in case of spelling errors. +Note: Will exit successfully regardless of spelling errors. +""" + +from subprocess import check_output, STDOUT, CalledProcessError + +IGNORE_WORDS_FILE = 'test/lint/spelling.ignore-words.txt' +FILES_ARGS = ['git', 'ls-files', '--', ":(exclude)build-aux/m4/", ":(exclude)contrib/seeds/*.txt", ":(exclude)depends/", ":(exclude)doc/release-notes/", ":(exclude)src/leveldb/", ":(exclude)src/crc32c/", ":(exclude)src/qt/locale/", ":(exclude)src/qt/*.qrc", ":(exclude)src/secp256k1/", ":(exclude)src/minisketch/", ":(exclude)src/univalue/", ":(exclude)contrib/builder-keys/keys.txt", ":(exclude)contrib/guix/patches"] + + +def check_codespell_install(): + try: + check_output(["codespell", "--version"]) + except FileNotFoundError: + print("Skipping spell check linting since codespell is not installed.") + exit(0) + + +def main(): + check_codespell_install() + + files = check_output(FILES_ARGS).decode("utf-8").splitlines() + codespell_args = ['codespell', '--check-filenames', '--disable-colors', '--quiet-level=7', '--ignore-words={}'.format(IGNORE_WORDS_FILE)] + files + + try: + check_output(codespell_args, stderr=STDOUT) + except CalledProcessError as e: + print(e.output.decode("utf-8"), end="") + print('^ Warning: codespell identified likely spelling errors. Any false positives? Add them to the list of ignored words in {}'.format(IGNORE_WORDS_FILE)) + + +if __name__ == "__main__": + main() diff --git a/test/lint/lint-spelling.sh b/test/lint/lint-spelling.sh deleted file mode 100755 index b3e558b02a..0000000000 --- a/test/lint/lint-spelling.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Warn in case of spelling errors. -# Note: Will exit successfully regardless of spelling errors. - -export LC_ALL=C - -if ! command -v codespell > /dev/null; then - echo "Skipping spell check linting since codespell is not installed." - exit 0 -fi - -IGNORE_WORDS_FILE=test/lint/lint-spelling.ignore-words.txt -mapfile -t FILES < <(git ls-files -- ":(exclude)build-aux/m4/" ":(exclude)contrib/seeds/*.txt" ":(exclude)depends/" ":(exclude)doc/release-notes/" ":(exclude)src/leveldb/" ":(exclude)src/crc32c/" ":(exclude)src/qt/locale/" ":(exclude)src/qt/*.qrc" ":(exclude)src/secp256k1/" ":(exclude)src/minisketch/" ":(exclude)src/univalue/" ":(exclude)contrib/builder-keys/keys.txt" ":(exclude)contrib/guix/patches") -if ! codespell --check-filenames --disable-colors --quiet-level=7 --ignore-words=${IGNORE_WORDS_FILE} "${FILES[@]}"; then - echo "^ Warning: codespell identified likely spelling errors. Any false positives? Add them to the list of ignored words in ${IGNORE_WORDS_FILE}" -fi diff --git a/test/lint/lint-submodule.py b/test/lint/lint-submodule.py new file mode 100755 index 0000000000..89d4c80f55 --- /dev/null +++ b/test/lint/lint-submodule.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" +This script checks for git modules +""" + +import subprocess +import sys + +def main(): + submodules_list = subprocess.check_output(['git', 'submodule', 'status', '--recursive'], + universal_newlines = True, encoding = 'utf8').rstrip('\n') + if submodules_list: + print("These submodules were found, delete them:\n", submodules_list) + sys.exit(1) + sys.exit(0) + +if __name__ == '__main__': + main() diff --git a/test/lint/lint-submodule.sh b/test/lint/lint-submodule.sh deleted file mode 100755 index d9aa021df7..0000000000 --- a/test/lint/lint-submodule.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# This script checks for git modules -export LC_ALL=C -EXIT_CODE=0 - -CMD=$(git submodule status --recursive) -if test -n "$CMD"; -then - echo These submodules were found, delete them: - echo "$CMD" - EXIT_CODE=1 -fi - -exit $EXIT_CODE - diff --git a/test/lint/lint-tests.py b/test/lint/lint-tests.py new file mode 100755 index 0000000000..849ddcb961 --- /dev/null +++ b/test/lint/lint-tests.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2018-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" +Check the test suite naming conventions +""" + +import re +import subprocess +import sys + + +def grep_boost_fixture_test_suite(): + command = [ + "git", + "grep", + "-E", + r"^BOOST_FIXTURE_TEST_SUITE\(", + "--", + "src/test/**.cpp", + "src/wallet/test/**.cpp", + ] + return subprocess.check_output(command, universal_newlines=True, encoding="utf8") + + +def check_matching_test_names(test_suite_list): + not_matching = [ + x + for x in test_suite_list + if re.search(r"/(.*?)\.cpp:BOOST_FIXTURE_TEST_SUITE\(\1, .*\)", x) is None + ] + if len(not_matching) > 0: + not_matching = "\n".join(not_matching) + error_msg = ( + "The test suite in file src/test/foo_tests.cpp should be named\n" + '"foo_tests". Please make sure the following test suites follow\n' + "that convention:\n\n" + f"{not_matching}\n" + ) + print(error_msg) + return 1 + return 0 + + +def get_duplicates(input_list): + """ + From https://stackoverflow.com/a/9835819 + """ + seen = set() + dupes = set() + for x in input_list: + if x in seen: + dupes.add(x) + else: + seen.add(x) + return dupes + + +def check_unique_test_names(test_suite_list): + output = [re.search(r"\((.*?),", x) for x in test_suite_list] + output = [x.group(1) for x in output if x is not None] + output = get_duplicates(output) + output = sorted(list(output)) + + if len(output) > 0: + output = "\n".join(output) + error_msg = ( + "Test suite names must be unique. The following test suite names\n" + f"appear to be used more than once:\n\n{output}" + ) + print(error_msg) + return 1 + return 0 + + +def main(): + test_suite_list = grep_boost_fixture_test_suite().splitlines() + exit_code = check_matching_test_names(test_suite_list) + exit_code |= check_unique_test_names(test_suite_list) + sys.exit(exit_code) + + +if __name__ == "__main__": + main() diff --git a/test/lint/lint-tests.sh b/test/lint/lint-tests.sh deleted file mode 100755 index 35d11023eb..0000000000 --- a/test/lint/lint-tests.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Check the test suite naming conventions - -export LC_ALL=C -EXIT_CODE=0 - -NAMING_INCONSISTENCIES=$(git grep -E '^BOOST_FIXTURE_TEST_SUITE\(' -- \ - "src/test/**.cpp" "src/wallet/test/**.cpp" | \ - grep -vE '/(.*?)\.cpp:BOOST_FIXTURE_TEST_SUITE\(\1, .*\)$') -if [[ ${NAMING_INCONSISTENCIES} != "" ]]; then - echo "The test suite in file src/test/foo_tests.cpp should be named" - echo "\"foo_tests\". Please make sure the following test suites follow" - echo "that convention:" - echo - echo "${NAMING_INCONSISTENCIES}" - EXIT_CODE=1 -fi - -TEST_SUITE_NAME_COLLISSIONS=$(git grep -E '^BOOST_FIXTURE_TEST_SUITE\(' -- \ - "src/test/**.cpp" "src/wallet/test/**.cpp" | cut -f2 -d'(' | cut -f1 -d, | \ - sort | uniq -d) -if [[ ${TEST_SUITE_NAME_COLLISSIONS} != "" ]]; then - echo "Test suite names must be unique. The following test suite names" - echo "appear to be used more than once:" - echo - echo "${TEST_SUITE_NAME_COLLISSIONS}" - EXIT_CODE=1 -fi - -exit ${EXIT_CODE} diff --git a/test/lint/lint-whitespace.py b/test/lint/lint-whitespace.py new file mode 100755 index 0000000000..d98fc8d9a2 --- /dev/null +++ b/test/lint/lint-whitespace.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2017-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# Check for new lines in diff that introduce trailing whitespace or +# tab characters instead of spaces. + +# We can't run this check unless we know the commit range for the PR. + +import argparse +import os +import re +import sys + +from subprocess import check_output + +EXCLUDED_DIRS = ["depends/patches/", + "contrib/guix/patches/", + "src/leveldb/", + "src/crc32c/", + "src/secp256k1/", + "src/minisketch/", + "src/univalue/", + "doc/release-notes/", + "src/qt/locale"] + +def parse_args(): + """Parse command line arguments.""" + parser = argparse.ArgumentParser( + description=""" + Check for new lines in diff that introduce trailing whitespace + or tab characters instead of spaces in unstaged changes, the + previous n commits, or a commit-range. + """, + epilog=f""" + You can manually set the commit-range with the COMMIT_RANGE + environment variable (e.g. "COMMIT_RANGE='47ba2c3...ee50c9e' + {sys.argv[0]}"). Defaults to current merge base when neither + prev-commits nor the environment variable is set. + """) + + parser.add_argument("--prev-commits", "-p", required=False, help="The previous n commits to check") + + return parser.parse_args() + + +def report_diff(selection): + filename = "" + seen = False + seenln = False + + print("The following changes were suspected:") + + for line in selection: + if re.match(r"^diff", line): + filename = line + seen = False + elif re.match(r"^@@", line): + linenumber = line + seenln = False + else: + if not seen: + # The first time a file is seen with trailing whitespace or a tab character, we print the + # filename (preceded by a newline). + print("") + print(filename) + seen = True + if not seenln: + print(linenumber) + seenln = True + print(line) + + +def get_diff(commit_range, check_only_code): + exclude_args = [":(exclude)" + dir for dir in EXCLUDED_DIRS] + + if check_only_code: + what_files = ["*.cpp", "*.h", "*.md", "*.py", "*.sh"] + else: + what_files = ["."] + + diff = check_output(["git", "diff", "-U0", commit_range, "--"] + what_files + exclude_args, universal_newlines=True, encoding="utf8") + + return diff + + +def main(): + args = parse_args() + + if not os.getenv("COMMIT_RANGE"): + if args.prev_commits: + commit_range = "HEAD~" + args.prev_commits + "...HEAD" + else: + # This assumes that the target branch of the pull request will be master. + merge_base = check_output(["git", "merge-base", "HEAD", "master"], universal_newlines=True, encoding="utf8").rstrip("\n") + commit_range = merge_base + "..HEAD" + else: + commit_range = os.getenv("COMMIT_RANGE") + + whitespace_selection = [] + tab_selection = [] + + # Check if trailing whitespace was found in the diff. + for line in get_diff(commit_range, check_only_code=False).splitlines(): + if re.match(r"^(diff --git|\@@|^\+.*\s+$)", line): + whitespace_selection.append(line) + + whitespace_additions = [i for i in whitespace_selection if i.startswith("+")] + + # Check if tab characters were found in the diff. + for line in get_diff(commit_range, check_only_code=True).splitlines(): + if re.match(r"^(diff --git|\@@|^\+.*\t)", line): + tab_selection.append(line) + + tab_additions = [i for i in tab_selection if i.startswith("+")] + + ret = 0 + + if len(whitespace_additions) > 0: + print("This diff appears to have added new lines with trailing whitespace.") + report_diff(whitespace_selection) + ret = 1 + + if len(tab_additions) > 0: + print("This diff appears to have added new lines with tab characters instead of spaces.") + report_diff(tab_selection) + ret = 1 + + sys.exit(ret) + + +if __name__ == "__main__": + main() diff --git a/test/lint/lint-whitespace.sh b/test/lint/lint-whitespace.sh deleted file mode 100755 index 9d55c71eb5..0000000000 --- a/test/lint/lint-whitespace.sh +++ /dev/null @@ -1,115 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2017-2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Check for new lines in diff that introduce trailing whitespace. - -# We can't run this check unless we know the commit range for the PR. - -export LC_ALL=C -while getopts "?" opt; do - case $opt in - ?) - echo "Usage: $0 [N]" - echo " COMMIT_RANGE='<commit range>' $0" - echo " $0 -?" - echo "Checks unstaged changes, the previous N commits, or a commit range." - echo "COMMIT_RANGE='47ba2c3...ee50c9e' $0" - exit 0 - ;; - esac -done - -if [ -z "${COMMIT_RANGE}" ]; then - if [ -n "$1" ]; then - COMMIT_RANGE="HEAD~$1...HEAD" - else - # This assumes that the target branch of the pull request will be master. - MERGE_BASE=$(git merge-base HEAD master) - COMMIT_RANGE="$MERGE_BASE..HEAD" - fi -fi - -showdiff() { - if ! git diff -U0 "${COMMIT_RANGE}" -- "." ":(exclude)depends/patches/" ":(exclude)contrib/guix/patches/" ":(exclude)src/leveldb/" ":(exclude)src/crc32c/" ":(exclude)src/secp256k1/" ":(exclude)src/minisketch/" ":(exclude)src/univalue/" ":(exclude)doc/release-notes/" ":(exclude)src/qt/locale/"; then - echo "Failed to get a diff" - exit 1 - fi -} - -showcodediff() { - if ! git diff -U0 "${COMMIT_RANGE}" -- *.cpp *.h *.md *.py *.sh ":(exclude)src/leveldb/" ":(exclude)src/crc32c/" ":(exclude)src/secp256k1/" ":(exclude)src/minisketch/" ":(exclude)src/univalue/" ":(exclude)doc/release-notes/" ":(exclude)src/qt/locale/"; then - echo "Failed to get a diff" - exit 1 - fi -} - -RET=0 - -# Check if trailing whitespace was found in the diff. -if showdiff | grep -E -q '^\+.*\s+$'; then - echo "This diff appears to have added new lines with trailing whitespace." - echo "The following changes were suspected:" - FILENAME="" - SEEN=0 - SEENLN=0 - while read -r line; do - if [[ "$line" =~ ^diff ]]; then - FILENAME="$line" - SEEN=0 - elif [[ "$line" =~ ^@@ ]]; then - LINENUMBER="$line" - SEENLN=0 - else - if [ "$SEEN" -eq 0 ]; then - # The first time a file is seen with trailing whitespace, we print the - # filename (preceded by a newline). - echo - echo "$FILENAME" - SEEN=1 - fi - if [ "$SEENLN" -eq 0 ]; then - echo "$LINENUMBER" - SEENLN=1 - fi - echo "$line" - fi - done < <(showdiff | grep -E '^(diff --git |@@|\+.*\s+$)') - RET=1 -fi - -# Check if tab characters were found in the diff. -if showcodediff | perl -nle '$MATCH++ if m{^\+.*\t}; END{exit 1 unless $MATCH>0}' > /dev/null; then - echo "This diff appears to have added new lines with tab characters instead of spaces." - echo "The following changes were suspected:" - FILENAME="" - SEEN=0 - SEENLN=0 - while read -r line; do - if [[ "$line" =~ ^diff ]]; then - FILENAME="$line" - SEEN=0 - elif [[ "$line" =~ ^@@ ]]; then - LINENUMBER="$line" - SEENLN=0 - else - if [ "$SEEN" -eq 0 ]; then - # The first time a file is seen with a tab character, we print the - # filename (preceded by a newline). - echo - echo "$FILENAME" - SEEN=1 - fi - if [ "$SEENLN" -eq 0 ]; then - echo "$LINENUMBER" - SEENLN=1 - fi - echo "$line" - fi - done < <(showcodediff | perl -nle 'print if m{^(diff --git |@@|\+.*\t)}') - RET=1 -fi - -exit $RET diff --git a/test/lint/run-lint-format-strings.py b/test/lint/run-lint-format-strings.py new file mode 100755 index 0000000000..b814446125 --- /dev/null +++ b/test/lint/run-lint-format-strings.py @@ -0,0 +1,291 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2018-2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# Lint format strings: This program checks that the number of arguments passed +# to a variadic format string function matches the number of format specifiers +# in the format string. + +import argparse +import re +import sys + +FALSE_POSITIVES = [ + ("src/dbwrapper.cpp", "vsnprintf(p, limit - p, format, backup_ap)"), + ("src/index/base.cpp", "FatalError(const char* fmt, const Args&... args)"), + ("src/netbase.cpp", "LogConnectFailure(bool manual_connection, const char* fmt, const Args&... args)"), + ("src/clientversion.cpp", "strprintf(_(COPYRIGHT_HOLDERS).translated, COPYRIGHT_HOLDERS_SUBSTITUTION)"), + ("src/validationinterface.cpp", "LogPrint(BCLog::VALIDATION, fmt \"\\n\", __VA_ARGS__)"), + ("src/wallet/wallet.h", "WalletLogPrintf(std::string fmt, Params... parameters)"), + ("src/wallet/wallet.h", "LogPrintf((\"%s \" + fmt).c_str(), GetDisplayName(), parameters...)"), + ("src/wallet/scriptpubkeyman.h", "WalletLogPrintf(std::string fmt, Params... parameters)"), + ("src/wallet/scriptpubkeyman.h", "LogPrintf((\"%s \" + fmt).c_str(), m_storage.GetDisplayName(), parameters...)"), +] + + +def parse_function_calls(function_name, source_code): + """Return an array with all calls to function function_name in string source_code. + Preprocessor directives and C++ style comments ("//") in source_code are removed. + + >>> len(parse_function_calls("foo", "foo();bar();foo();bar();")) + 2 + >>> parse_function_calls("foo", "foo(1);bar(1);foo(2);bar(2);")[0].startswith("foo(1);") + True + >>> parse_function_calls("foo", "foo(1);bar(1);foo(2);bar(2);")[1].startswith("foo(2);") + True + >>> len(parse_function_calls("foo", "foo();bar();// foo();bar();")) + 1 + >>> len(parse_function_calls("foo", "#define FOO foo();")) + 0 + """ + assert type(function_name) is str and type(source_code) is str and function_name + lines = [re.sub("// .*", " ", line).strip() + for line in source_code.split("\n") + if not line.strip().startswith("#")] + return re.findall(r"[^a-zA-Z_](?=({}\(.*).*)".format(function_name), " " + " ".join(lines)) + + +def normalize(s): + """Return a normalized version of string s with newlines, tabs and C style comments ("/* ... */") + replaced with spaces. Multiple spaces are replaced with a single space. + + >>> normalize(" /* nothing */ foo\tfoo /* bar */ foo ") + 'foo foo foo' + """ + assert type(s) is str + s = s.replace("\n", " ") + s = s.replace("\t", " ") + s = re.sub(r"/\*.*?\*/", " ", s) + s = re.sub(" {2,}", " ", s) + return s.strip() + + +ESCAPE_MAP = { + r"\n": "[escaped-newline]", + r"\t": "[escaped-tab]", + r'\"': "[escaped-quote]", +} + + +def escape(s): + """Return the escaped version of string s with "\\\"", "\\n" and "\\t" escaped as + "[escaped-backslash]", "[escaped-newline]" and "[escaped-tab]". + + >>> unescape(escape("foo")) == "foo" + True + >>> escape(r'foo \\t foo \\n foo \\\\ foo \\ foo \\"bar\\"') + 'foo [escaped-tab] foo [escaped-newline] foo \\\\\\\\ foo \\\\ foo [escaped-quote]bar[escaped-quote]' + """ + assert type(s) is str + for raw_value, escaped_value in ESCAPE_MAP.items(): + s = s.replace(raw_value, escaped_value) + return s + + +def unescape(s): + """Return the unescaped version of escaped string s. + Reverses the replacements made in function escape(s). + + >>> unescape(escape("bar")) + 'bar' + >>> unescape("foo [escaped-tab] foo [escaped-newline] foo \\\\\\\\ foo \\\\ foo [escaped-quote]bar[escaped-quote]") + 'foo \\\\t foo \\\\n foo \\\\\\\\ foo \\\\ foo \\\\"bar\\\\"' + """ + assert type(s) is str + for raw_value, escaped_value in ESCAPE_MAP.items(): + s = s.replace(escaped_value, raw_value) + return s + + +def parse_function_call_and_arguments(function_name, function_call): + """Split string function_call into an array of strings consisting of: + * the string function_call followed by "(" + * the function call argument #1 + * ... + * the function call argument #n + * a trailing ");" + + The strings returned are in escaped form. See escape(...). + + >>> parse_function_call_and_arguments("foo", 'foo("%s", "foo");') + ['foo(', '"%s",', ' "foo"', ')'] + >>> parse_function_call_and_arguments("foo", 'foo("%s", "foo");') + ['foo(', '"%s",', ' "foo"', ')'] + >>> parse_function_call_and_arguments("foo", 'foo("%s %s", "foo", "bar");') + ['foo(', '"%s %s",', ' "foo",', ' "bar"', ')'] + >>> parse_function_call_and_arguments("fooprintf", 'fooprintf("%050d", i);') + ['fooprintf(', '"%050d",', ' i', ')'] + >>> parse_function_call_and_arguments("foo", 'foo(bar(foobar(barfoo("foo"))), foobar); barfoo') + ['foo(', 'bar(foobar(barfoo("foo"))),', ' foobar', ')'] + >>> parse_function_call_and_arguments("foo", "foo()") + ['foo(', '', ')'] + >>> parse_function_call_and_arguments("foo", "foo(123)") + ['foo(', '123', ')'] + >>> parse_function_call_and_arguments("foo", 'foo("foo")') + ['foo(', '"foo"', ')'] + >>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>,wchar_t>().to_bytes(buf), err);') + ['strprintf(', '"%s (%d)",', ' std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>,wchar_t>().to_bytes(buf),', ' err', ')'] + >>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", foo<wchar_t>().to_bytes(buf), err);') + ['strprintf(', '"%s (%d)",', ' foo<wchar_t>().to_bytes(buf),', ' err', ')'] + >>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", foo().to_bytes(buf), err);') + ['strprintf(', '"%s (%d)",', ' foo().to_bytes(buf),', ' err', ')'] + >>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", foo << 1, err);') + ['strprintf(', '"%s (%d)",', ' foo << 1,', ' err', ')'] + >>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", foo<bar>() >> 1, err);') + ['strprintf(', '"%s (%d)",', ' foo<bar>() >> 1,', ' err', ')'] + >>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", foo < 1 ? bar : foobar, err);') + ['strprintf(', '"%s (%d)",', ' foo < 1 ? bar : foobar,', ' err', ')'] + >>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", foo < 1, err);') + ['strprintf(', '"%s (%d)",', ' foo < 1,', ' err', ')'] + >>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", foo > 1 ? bar : foobar, err);') + ['strprintf(', '"%s (%d)",', ' foo > 1 ? bar : foobar,', ' err', ')'] + >>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", foo > 1, err);') + ['strprintf(', '"%s (%d)",', ' foo > 1,', ' err', ')'] + >>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", foo <= 1, err);') + ['strprintf(', '"%s (%d)",', ' foo <= 1,', ' err', ')'] + >>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", foo <= bar<1, 2>(1, 2), err);') + ['strprintf(', '"%s (%d)",', ' foo <= bar<1, 2>(1, 2),', ' err', ')'] + >>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", foo>foo<1,2>(1,2)?bar:foobar,err)'); + ['strprintf(', '"%s (%d)",', ' foo>foo<1,2>(1,2)?bar:foobar,', 'err', ')'] + >>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", foo>foo<1,2>(1,2),err)'); + ['strprintf(', '"%s (%d)",', ' foo>foo<1,2>(1,2),', 'err', ')'] + """ + assert type(function_name) is str and type(function_call) is str and function_name + remaining = normalize(escape(function_call)) + expected_function_call = "{}(".format(function_name) + assert remaining.startswith(expected_function_call) + parts = [expected_function_call] + remaining = remaining[len(expected_function_call):] + open_parentheses = 1 + open_template_arguments = 0 + in_string = False + parts.append("") + for i, char in enumerate(remaining): + parts.append(parts.pop() + char) + if char == "\"": + in_string = not in_string + continue + if in_string: + continue + if char == "(": + open_parentheses += 1 + continue + if char == ")": + open_parentheses -= 1 + if open_parentheses > 1: + continue + if open_parentheses == 0: + parts.append(parts.pop()[:-1]) + parts.append(char) + break + prev_char = remaining[i - 1] if i - 1 >= 0 else None + next_char = remaining[i + 1] if i + 1 <= len(remaining) - 1 else None + if char == "<" and next_char not in [" ", "<", "="] and prev_char not in [" ", "<"]: + open_template_arguments += 1 + continue + if char == ">" and next_char not in [" ", ">", "="] and prev_char not in [" ", ">"] and open_template_arguments > 0: + open_template_arguments -= 1 + if open_template_arguments > 0: + continue + if char == ",": + parts.append("") + return parts + + +def parse_string_content(argument): + """Return the text within quotes in string argument. + + >>> parse_string_content('1 "foo %d bar" 2') + 'foo %d bar' + >>> parse_string_content('1 foobar 2') + '' + >>> parse_string_content('1 "bar" 2') + 'bar' + >>> parse_string_content('1 "foo" 2 "bar" 3') + 'foobar' + >>> parse_string_content('1 "foo" 2 " " "bar" 3') + 'foo bar' + >>> parse_string_content('""') + '' + >>> parse_string_content('') + '' + >>> parse_string_content('1 2 3') + '' + """ + assert type(argument) is str + string_content = "" + in_string = False + for char in normalize(escape(argument)): + if char == "\"": + in_string = not in_string + elif in_string: + string_content += char + return string_content + + +def count_format_specifiers(format_string): + """Return the number of format specifiers in string format_string. + + >>> count_format_specifiers("foo bar foo") + 0 + >>> count_format_specifiers("foo %d bar foo") + 1 + >>> count_format_specifiers("foo %d bar %i foo") + 2 + >>> count_format_specifiers("foo %d bar %i foo %% foo") + 2 + >>> count_format_specifiers("foo %d bar %i foo %% foo %d foo") + 3 + >>> count_format_specifiers("foo %d bar %i foo %% foo %*d foo") + 4 + """ + assert type(format_string) is str + format_string = format_string.replace('%%', 'X') + n = 0 + in_specifier = False + for i, char in enumerate(format_string): + if char == "%": + in_specifier = True + n += 1 + elif char in "aAcdeEfFgGinopsuxX": + in_specifier = False + elif in_specifier and char == "*": + n += 1 + return n + + +def main(): + parser = argparse.ArgumentParser(description="This program checks that the number of arguments passed " + "to a variadic format string function matches the number of format " + "specifiers in the format string.") + parser.add_argument("--skip-arguments", type=int, help="number of arguments before the format string " + "argument (e.g. 1 in the case of fprintf)", default=0) + parser.add_argument("function_name", help="function name (e.g. fprintf)", default=None) + parser.add_argument("file", nargs="*", help="C++ source code file (e.g. foo.cpp)") + args = parser.parse_args() + exit_code = 0 + for filename in args.file: + with open(filename, "r", encoding="utf-8") as f: + for function_call_str in parse_function_calls(args.function_name, f.read()): + parts = parse_function_call_and_arguments(args.function_name, function_call_str) + relevant_function_call_str = unescape("".join(parts))[:512] + if (f.name, relevant_function_call_str) in FALSE_POSITIVES: + continue + if len(parts) < 3 + args.skip_arguments: + exit_code = 1 + print("{}: Could not parse function call string \"{}(...)\": {}".format(f.name, args.function_name, relevant_function_call_str)) + continue + argument_count = len(parts) - 3 - args.skip_arguments + format_str = parse_string_content(parts[1 + args.skip_arguments]) + format_specifier_count = count_format_specifiers(format_str) + if format_specifier_count != argument_count: + exit_code = 1 + print("{}: Expected {} argument(s) after format string but found {} argument(s): {}".format(f.name, format_specifier_count, argument_count, relevant_function_call_str)) + continue + sys.exit(exit_code) + + +if __name__ == "__main__": + main() diff --git a/test/lint/lint-spelling.ignore-words.txt b/test/lint/spelling.ignore-words.txt index 9906b15e9a..afdb0692d8 100644 --- a/test/lint/lint-spelling.ignore-words.txt +++ b/test/lint/spelling.ignore-words.txt @@ -1,6 +1,8 @@ asend +ba blockin cachable +creat fo fpr hights diff --git a/test/sanitizer_suppressions/tsan b/test/sanitizer_suppressions/tsan index 3c5a15a0c7..3acf575d07 100644 --- a/test/sanitizer_suppressions/tsan +++ b/test/sanitizer_suppressions/tsan @@ -22,10 +22,6 @@ deadlock:sync_tests::potential_deadlock_detected race:src/qt/test/* deadlock:src/qt/test/* -# Race in src/test/main.cpp -# Can be removed once upgraded to boost test 1.74 in depends -race:validation_chainstatemanager_tests - # External libraries deadlock:libdb race:libzmq @@ -43,4 +39,4 @@ race:CZMQAbstractPublishNotifier::SendZmqMessage race:epoll_ctl # https://github.com/bitcoin/bitcoin/issues/23366 -race:std::__1::ios_base::width +race:std::__1::ios_base::* diff --git a/test/sanitizer_suppressions/ubsan b/test/sanitizer_suppressions/ubsan index 4292544dbd..e6cfe5f81a 100644 --- a/test/sanitizer_suppressions/ubsan +++ b/test/sanitizer_suppressions/ubsan @@ -43,62 +43,26 @@ shift-base:test/fuzz/crypto_diff_fuzz_chacha20.cpp # contains files in which we expect unsigned integer overflows to occur. The # list is used to suppress -fsanitize=integer warnings when running our CI UBSan # job. -unsigned-integer-overflow:addrman.cpp unsigned-integer-overflow:arith_uint256.h -unsigned-integer-overflow:bitcoin-tx.cpp unsigned-integer-overflow:common/bloom.cpp -unsigned-integer-overflow:chain.cpp -unsigned-integer-overflow:chain.h unsigned-integer-overflow:coins.cpp unsigned-integer-overflow:compressor.cpp -unsigned-integer-overflow:core_write.cpp unsigned-integer-overflow:crypto/ unsigned-integer-overflow:hash.cpp unsigned-integer-overflow:policy/fees.cpp unsigned-integer-overflow:prevector.h -unsigned-integer-overflow:pubkey.h unsigned-integer-overflow:script/interpreter.cpp unsigned-integer-overflow:txmempool.cpp -unsigned-integer-overflow:util/strencodings.cpp -unsigned-integer-overflow:validation.cpp -implicit-integer-sign-change:addrman.h -implicit-integer-sign-change:bech32.cpp -implicit-integer-sign-change:common/bloom.cpp -implicit-integer-sign-change:chain.cpp -implicit-integer-sign-change:chain.h -implicit-integer-sign-change:coins.h implicit-integer-sign-change:compat/stdin.cpp implicit-integer-sign-change:compressor.h implicit-integer-sign-change:crypto/ -implicit-integer-sign-change:key.cpp -implicit-integer-sign-change:noui.cpp implicit-integer-sign-change:policy/fees.cpp implicit-integer-sign-change:prevector.h implicit-integer-sign-change:script/bitcoinconsensus.cpp implicit-integer-sign-change:script/interpreter.cpp implicit-integer-sign-change:serialize.h -implicit-integer-sign-change:test/arith_uint256_tests.cpp -implicit-integer-sign-change:test/coins_tests.cpp -implicit-integer-sign-change:test/pow_tests.cpp -implicit-integer-sign-change:test/prevector_tests.cpp -implicit-integer-sign-change:test/sighash_tests.cpp -implicit-integer-sign-change:test/skiplist_tests.cpp -implicit-integer-sign-change:test/streams_tests.cpp -implicit-integer-sign-change:test/transaction_tests.cpp implicit-integer-sign-change:txmempool.cpp -implicit-integer-sign-change:zmq/zmqpublishnotifier.cpp -implicit-signed-integer-truncation,implicit-integer-sign-change:chain.h -implicit-signed-integer-truncation,implicit-integer-sign-change:test/skiplist_tests.cpp -implicit-signed-integer-truncation:addrman.cpp -implicit-signed-integer-truncation:addrman.h -implicit-signed-integer-truncation:chain.h implicit-signed-integer-truncation:crypto/ -implicit-signed-integer-truncation:node/miner.cpp -implicit-signed-integer-truncation:net.cpp -implicit-signed-integer-truncation:streams.h -implicit-signed-integer-truncation:test/arith_uint256_tests.cpp -implicit-signed-integer-truncation:test/skiplist_tests.cpp -implicit-signed-integer-truncation:torcontrol.cpp implicit-unsigned-integer-truncation:crypto/ shift-base:arith_uint256.cpp shift-base:crypto/ diff --git a/test/util/data/tt-delin1-out.json b/test/util/data/tt-delin1-out.json index c5b9f6df01..6e053fe2b9 100644 --- a/test/util/data/tt-delin1-out.json +++ b/test/util/data/tt-delin1-out.json @@ -194,6 +194,7 @@ "n": 0, "scriptPubKey": { "asm": "OP_DUP OP_HASH160 8fd139bb39ced713f231c58a4d07bf6954d1c201 OP_EQUALVERIFY OP_CHECKSIG", + "desc": "addr(1E7SGgAZFCHDnVZLuRViX3gUmxpMfdvd2o)#xvg87vgr", "hex": "76a9148fd139bb39ced713f231c58a4d07bf6954d1c20188ac", "address": "1E7SGgAZFCHDnVZLuRViX3gUmxpMfdvd2o", "type": "pubkeyhash" @@ -204,6 +205,7 @@ "n": 1, "scriptPubKey": { "asm": "OP_DUP OP_HASH160 6c772e9cf96371bba3da8cb733da70a2fcf20078 OP_EQUALVERIFY OP_CHECKSIG", + "desc": "addr(1AtWkdmfmYkErU16d3KYykJUbEp9MAj9Sb)#tsyprkms", "hex": "76a9146c772e9cf96371bba3da8cb733da70a2fcf2007888ac", "address": "1AtWkdmfmYkErU16d3KYykJUbEp9MAj9Sb", "type": "pubkeyhash" diff --git a/test/util/data/tt-delout1-out.json b/test/util/data/tt-delout1-out.json index 3863416430..e61b9c79db 100644 --- a/test/util/data/tt-delout1-out.json +++ b/test/util/data/tt-delout1-out.json @@ -203,6 +203,7 @@ "n": 0, "scriptPubKey": { "asm": "OP_DUP OP_HASH160 8fd139bb39ced713f231c58a4d07bf6954d1c201 OP_EQUALVERIFY OP_CHECKSIG", + "desc": "addr(1E7SGgAZFCHDnVZLuRViX3gUmxpMfdvd2o)#xvg87vgr", "hex": "76a9148fd139bb39ced713f231c58a4d07bf6954d1c20188ac", "address": "1E7SGgAZFCHDnVZLuRViX3gUmxpMfdvd2o", "type": "pubkeyhash" diff --git a/test/util/data/tt-locktime317000-out.json b/test/util/data/tt-locktime317000-out.json index 62e785f7d0..873628e124 100644 --- a/test/util/data/tt-locktime317000-out.json +++ b/test/util/data/tt-locktime317000-out.json @@ -203,6 +203,7 @@ "n": 0, "scriptPubKey": { "asm": "OP_DUP OP_HASH160 8fd139bb39ced713f231c58a4d07bf6954d1c201 OP_EQUALVERIFY OP_CHECKSIG", + "desc": "addr(1E7SGgAZFCHDnVZLuRViX3gUmxpMfdvd2o)#xvg87vgr", "hex": "76a9148fd139bb39ced713f231c58a4d07bf6954d1c20188ac", "address": "1E7SGgAZFCHDnVZLuRViX3gUmxpMfdvd2o", "type": "pubkeyhash" @@ -213,6 +214,7 @@ "n": 1, "scriptPubKey": { "asm": "OP_DUP OP_HASH160 6c772e9cf96371bba3da8cb733da70a2fcf20078 OP_EQUALVERIFY OP_CHECKSIG", + "desc": "addr(1AtWkdmfmYkErU16d3KYykJUbEp9MAj9Sb)#tsyprkms", "hex": "76a9146c772e9cf96371bba3da8cb733da70a2fcf2007888ac", "address": "1AtWkdmfmYkErU16d3KYykJUbEp9MAj9Sb", "type": "pubkeyhash" diff --git a/test/util/data/txcreate1.json b/test/util/data/txcreate1.json index 96d77ef273..c4a76f22a6 100644 --- a/test/util/data/txcreate1.json +++ b/test/util/data/txcreate1.json @@ -41,6 +41,7 @@ "n": 0, "scriptPubKey": { "asm": "OP_DUP OP_HASH160 1fc11f39be1729bf973a7ab6a615ca4729d64574 OP_EQUALVERIFY OP_CHECKSIG", + "desc": "addr(13tuJJDR2RgArmgfv6JScSdreahzgc4T6o)#ztmwxg4c", "hex": "76a9141fc11f39be1729bf973a7ab6a615ca4729d6457488ac", "address": "13tuJJDR2RgArmgfv6JScSdreahzgc4T6o", "type": "pubkeyhash" @@ -51,6 +52,7 @@ "n": 1, "scriptPubKey": { "asm": "OP_DUP OP_HASH160 f2d4db28cad6502226ee484ae24505c2885cb12d OP_EQUALVERIFY OP_CHECKSIG", + "desc": "addr(1P8yWvZW8jVihP1bzHeqfE4aoXNX8AVa46)#vdmdu766", "hex": "76a914f2d4db28cad6502226ee484ae24505c2885cb12d88ac", "address": "1P8yWvZW8jVihP1bzHeqfE4aoXNX8AVa46", "type": "pubkeyhash" diff --git a/test/util/data/txcreate2.json b/test/util/data/txcreate2.json index ee9b9c3c17..95953cc90e 100644 --- a/test/util/data/txcreate2.json +++ b/test/util/data/txcreate2.json @@ -14,6 +14,7 @@ "n": 0, "scriptPubKey": { "asm": "", + "desc": "raw()#58lrscpx", "hex": "", "type": "nonstandard" } diff --git a/test/util/data/txcreatedata1.json b/test/util/data/txcreatedata1.json index 87fc7e9cf7..1454ffdab7 100644 --- a/test/util/data/txcreatedata1.json +++ b/test/util/data/txcreatedata1.json @@ -23,6 +23,7 @@ "n": 0, "scriptPubKey": { "asm": "OP_DUP OP_HASH160 1fc11f39be1729bf973a7ab6a615ca4729d64574 OP_EQUALVERIFY OP_CHECKSIG", + "desc": "addr(13tuJJDR2RgArmgfv6JScSdreahzgc4T6o)#ztmwxg4c", "hex": "76a9141fc11f39be1729bf973a7ab6a615ca4729d6457488ac", "address": "13tuJJDR2RgArmgfv6JScSdreahzgc4T6o", "type": "pubkeyhash" @@ -33,6 +34,7 @@ "n": 1, "scriptPubKey": { "asm": "OP_RETURN 54686973204f505f52455455524e207472616e73616374696f6e206f7574707574207761732063726561746564206279206d6f646966696564206372656174657261777472616e73616374696f6e2e", + "desc": "raw(6a4c4f54686973204f505f52455455524e207472616e73616374696f6e206f7574707574207761732063726561746564206279206d6f646966696564206372656174657261777472616e73616374696f6e2e)#zf2avljj", "hex": "6a4c4f54686973204f505f52455455524e207472616e73616374696f6e206f7574707574207761732063726561746564206279206d6f646966696564206372656174657261777472616e73616374696f6e2e", "type": "nulldata" } diff --git a/test/util/data/txcreatedata2.json b/test/util/data/txcreatedata2.json index d03b1c8244..ca20d2aa45 100644 --- a/test/util/data/txcreatedata2.json +++ b/test/util/data/txcreatedata2.json @@ -23,6 +23,7 @@ "n": 0, "scriptPubKey": { "asm": "OP_DUP OP_HASH160 1fc11f39be1729bf973a7ab6a615ca4729d64574 OP_EQUALVERIFY OP_CHECKSIG", + "desc": "addr(13tuJJDR2RgArmgfv6JScSdreahzgc4T6o)#ztmwxg4c", "hex": "76a9141fc11f39be1729bf973a7ab6a615ca4729d6457488ac", "address": "13tuJJDR2RgArmgfv6JScSdreahzgc4T6o", "type": "pubkeyhash" @@ -33,6 +34,7 @@ "n": 1, "scriptPubKey": { "asm": "OP_RETURN 54686973204f505f52455455524e207472616e73616374696f6e206f7574707574207761732063726561746564206279206d6f646966696564206372656174657261777472616e73616374696f6e2e", + "desc": "raw(6a4c4f54686973204f505f52455455524e207472616e73616374696f6e206f7574707574207761732063726561746564206279206d6f646966696564206372656174657261777472616e73616374696f6e2e)#zf2avljj", "hex": "6a4c4f54686973204f505f52455455524e207472616e73616374696f6e206f7574707574207761732063726561746564206279206d6f646966696564206372656174657261777472616e73616374696f6e2e", "type": "nulldata" } diff --git a/test/util/data/txcreatedata_seq0.json b/test/util/data/txcreatedata_seq0.json index 8a123f1ba8..9838383c06 100644 --- a/test/util/data/txcreatedata_seq0.json +++ b/test/util/data/txcreatedata_seq0.json @@ -23,6 +23,7 @@ "n": 0, "scriptPubKey": { "asm": "OP_DUP OP_HASH160 1fc11f39be1729bf973a7ab6a615ca4729d64574 OP_EQUALVERIFY OP_CHECKSIG", + "desc": "addr(13tuJJDR2RgArmgfv6JScSdreahzgc4T6o)#ztmwxg4c", "hex": "76a9141fc11f39be1729bf973a7ab6a615ca4729d6457488ac", "address": "13tuJJDR2RgArmgfv6JScSdreahzgc4T6o", "type": "pubkeyhash" diff --git a/test/util/data/txcreatedata_seq1.json b/test/util/data/txcreatedata_seq1.json index 006fd7259f..c729f8dcfb 100644 --- a/test/util/data/txcreatedata_seq1.json +++ b/test/util/data/txcreatedata_seq1.json @@ -32,6 +32,7 @@ "n": 0, "scriptPubKey": { "asm": "OP_DUP OP_HASH160 1fc11f39be1729bf973a7ab6a615ca4729d64574 OP_EQUALVERIFY OP_CHECKSIG", + "desc": "addr(13tuJJDR2RgArmgfv6JScSdreahzgc4T6o)#ztmwxg4c", "hex": "76a9141fc11f39be1729bf973a7ab6a615ca4729d6457488ac", "address": "13tuJJDR2RgArmgfv6JScSdreahzgc4T6o", "type": "pubkeyhash" diff --git a/test/util/data/txcreatemultisig1.json b/test/util/data/txcreatemultisig1.json index baa290c2b1..9632b20ece 100644 --- a/test/util/data/txcreatemultisig1.json +++ b/test/util/data/txcreatemultisig1.json @@ -14,6 +14,7 @@ "n": 0, "scriptPubKey": { "asm": "2 02a5613bd857b7048924264d1e70e08fb2a7e6527d32b7ab1bb993ac59964ff397 021ac43c7ff740014c3b33737ede99c967e4764553d1b2b83db77c83b8715fa72d 02df2089105c77f266fa11a9d33f05c735234075f2e8780824c6b709415f9fb485 3 OP_CHECKMULTISIG", + "desc": "multi(2,02a5613bd857b7048924264d1e70e08fb2a7e6527d32b7ab1bb993ac59964ff397,021ac43c7ff740014c3b33737ede99c967e4764553d1b2b83db77c83b8715fa72d,02df2089105c77f266fa11a9d33f05c735234075f2e8780824c6b709415f9fb485)#8s88p9pl", "hex": "522102a5613bd857b7048924264d1e70e08fb2a7e6527d32b7ab1bb993ac59964ff39721021ac43c7ff740014c3b33737ede99c967e4764553d1b2b83db77c83b8715fa72d2102df2089105c77f266fa11a9d33f05c735234075f2e8780824c6b709415f9fb48553ae", "type": "multisig" } diff --git a/test/util/data/txcreatemultisig2.json b/test/util/data/txcreatemultisig2.json index 6685512587..021cf539a8 100644 --- a/test/util/data/txcreatemultisig2.json +++ b/test/util/data/txcreatemultisig2.json @@ -14,6 +14,7 @@ "n": 0, "scriptPubKey": { "asm": "OP_HASH160 1c6fbaf46d64221e80cbae182c33ddf81b9294ac OP_EQUAL", + "desc": "addr(34HNh57oBCRKkxNyjTuWAJkTbuGh6jg2Ms)#ngnz8933", "hex": "a9141c6fbaf46d64221e80cbae182c33ddf81b9294ac87", "address": "34HNh57oBCRKkxNyjTuWAJkTbuGh6jg2Ms", "type": "scripthash" diff --git a/test/util/data/txcreatemultisig3.json b/test/util/data/txcreatemultisig3.json index be96f4c704..3c20a88a91 100644 --- a/test/util/data/txcreatemultisig3.json +++ b/test/util/data/txcreatemultisig3.json @@ -14,6 +14,7 @@ "n": 0, "scriptPubKey": { "asm": "0 e15a86a23178f433d514dbbce042e87d72662b8b5edcacfd2e37ab7a2d135f05", + "desc": "addr(bc1qu9dgdg330r6r84g5mw7wqshg04exv2uttmw2elfwx74h5tgntuzs44gyfg)#yvt39j9m", "hex": "0020e15a86a23178f433d514dbbce042e87d72662b8b5edcacfd2e37ab7a2d135f05", "address": "bc1qu9dgdg330r6r84g5mw7wqshg04exv2uttmw2elfwx74h5tgntuzs44gyfg", "type": "witness_v0_scripthash" diff --git a/test/util/data/txcreatemultisig4.json b/test/util/data/txcreatemultisig4.json index 08831ecdca..7ae18fd90a 100644 --- a/test/util/data/txcreatemultisig4.json +++ b/test/util/data/txcreatemultisig4.json @@ -14,6 +14,7 @@ "n": 0, "scriptPubKey": { "asm": "OP_HASH160 6edf12858999f0dae74f9c692e6694ee3621b2ac OP_EQUAL", + "desc": "addr(3BoFUz1StqcNcgUTZE5cC1eFhuYFzj3fGH)#466tx6fn", "hex": "a9146edf12858999f0dae74f9c692e6694ee3621b2ac87", "address": "3BoFUz1StqcNcgUTZE5cC1eFhuYFzj3fGH", "type": "scripthash" diff --git a/test/util/data/txcreatemultisig5.json b/test/util/data/txcreatemultisig5.json index 93048cf261..98a5c2d8d1 100644 --- a/test/util/data/txcreatemultisig5.json +++ b/test/util/data/txcreatemultisig5.json @@ -14,6 +14,7 @@ "n": 0, "scriptPubKey": { "asm": "OP_HASH160 a4051c02398868af83f28f083208fae99a769263 OP_EQUAL", + "desc": "addr(3GeGs1eHUxPz5YyuFe9WPpXid2UsUb5Jos)#juhnnegr", "hex": "a914a4051c02398868af83f28f083208fae99a76926387", "address": "3GeGs1eHUxPz5YyuFe9WPpXid2UsUb5Jos", "type": "scripthash" diff --git a/test/util/data/txcreateoutpubkey1.json b/test/util/data/txcreateoutpubkey1.json index 42b519bb21..3baf479991 100644 --- a/test/util/data/txcreateoutpubkey1.json +++ b/test/util/data/txcreateoutpubkey1.json @@ -14,6 +14,7 @@ "n": 0, "scriptPubKey": { "asm": "02a5613bd857b7048924264d1e70e08fb2a7e6527d32b7ab1bb993ac59964ff397 OP_CHECKSIG", + "desc": "pk(02a5613bd857b7048924264d1e70e08fb2a7e6527d32b7ab1bb993ac59964ff397)#rk5v7uqw", "hex": "2102a5613bd857b7048924264d1e70e08fb2a7e6527d32b7ab1bb993ac59964ff397ac", "type": "pubkey" } diff --git a/test/util/data/txcreateoutpubkey2.json b/test/util/data/txcreateoutpubkey2.json index 52168a889b..78acf1658b 100644 --- a/test/util/data/txcreateoutpubkey2.json +++ b/test/util/data/txcreateoutpubkey2.json @@ -14,6 +14,7 @@ "n": 0, "scriptPubKey": { "asm": "0 a2516e770582864a6a56ed21a102044e388c62e3", + "desc": "addr(bc1q5fgkuac9s2ry56jka5s6zqsyfcugcchry5cwu0)#gm7zhxq2", "hex": "0014a2516e770582864a6a56ed21a102044e388c62e3", "address": "bc1q5fgkuac9s2ry56jka5s6zqsyfcugcchry5cwu0", "type": "witness_v0_keyhash" diff --git a/test/util/data/txcreateoutpubkey3.json b/test/util/data/txcreateoutpubkey3.json index fce210f8a3..632ed52ccf 100644 --- a/test/util/data/txcreateoutpubkey3.json +++ b/test/util/data/txcreateoutpubkey3.json @@ -14,6 +14,7 @@ "n": 0, "scriptPubKey": { "asm": "OP_HASH160 a5ab14c9804d0d8bf02f1aea4e82780733ad0a83 OP_EQUAL", + "desc": "addr(3GnzN8FqgvYGYdhj8NW6UNxxVv3Uj1ApQn)#zsln680u", "hex": "a914a5ab14c9804d0d8bf02f1aea4e82780733ad0a8387", "address": "3GnzN8FqgvYGYdhj8NW6UNxxVv3Uj1ApQn", "type": "scripthash" diff --git a/test/util/data/txcreatescript1.json b/test/util/data/txcreatescript1.json index af1c4c35e2..cdee9dbbfa 100644 --- a/test/util/data/txcreatescript1.json +++ b/test/util/data/txcreatescript1.json @@ -14,6 +14,7 @@ "n": 0, "scriptPubKey": { "asm": "OP_DROP", + "desc": "raw(75)#ppey0zqj", "hex": "75", "type": "nonstandard" } diff --git a/test/util/data/txcreatescript2.json b/test/util/data/txcreatescript2.json index 2cde70fdf7..1fbae62f4b 100644 --- a/test/util/data/txcreatescript2.json +++ b/test/util/data/txcreatescript2.json @@ -14,6 +14,7 @@ "n": 0, "scriptPubKey": { "asm": "OP_HASH160 71ed53322d470bb96657deb786b94f97dd46fb15 OP_EQUAL", + "desc": "addr(3C5QarEGh9feKbDJ3QbMf2YNjnMoiPDhNp)#5mx9waq3", "hex": "a91471ed53322d470bb96657deb786b94f97dd46fb1587", "address": "3C5QarEGh9feKbDJ3QbMf2YNjnMoiPDhNp", "type": "scripthash" diff --git a/test/util/data/txcreatescript3.json b/test/util/data/txcreatescript3.json index 7a282faf4f..502fe91692 100644 --- a/test/util/data/txcreatescript3.json +++ b/test/util/data/txcreatescript3.json @@ -14,6 +14,7 @@ "n": 0, "scriptPubKey": { "asm": "0 0bfe935e70c321c7ca3afc75ce0d0ca2f98b5422e008bb31c00c6d7f1f1c0ad6", + "desc": "addr(bc1qp0lfxhnscvsu0j36l36uurgv5tuck4pzuqytkvwqp3kh78cupttqyf705v)#s4fdh9tu", "hex": "00200bfe935e70c321c7ca3afc75ce0d0ca2f98b5422e008bb31c00c6d7f1f1c0ad6", "address": "bc1qp0lfxhnscvsu0j36l36uurgv5tuck4pzuqytkvwqp3kh78cupttqyf705v", "type": "witness_v0_scripthash" diff --git a/test/util/data/txcreatescript4.json b/test/util/data/txcreatescript4.json index 298b37bb4a..1ed89dfff2 100644 --- a/test/util/data/txcreatescript4.json +++ b/test/util/data/txcreatescript4.json @@ -14,6 +14,7 @@ "n": 0, "scriptPubKey": { "asm": "OP_HASH160 6a2c482f4985f57e702f325816c90e3723ca81ae OP_EQUAL", + "desc": "addr(3BNQbeFeJJGMAyDxPwWPuqxPMrjsFLjk3f)#fdleltnv", "hex": "a9146a2c482f4985f57e702f325816c90e3723ca81ae87", "address": "3BNQbeFeJJGMAyDxPwWPuqxPMrjsFLjk3f", "type": "scripthash" diff --git a/test/util/data/txcreatesignv1.json b/test/util/data/txcreatesignv1.json index ca5e003110..56ef9b195e 100644 --- a/test/util/data/txcreatesignv1.json +++ b/test/util/data/txcreatesignv1.json @@ -23,6 +23,7 @@ "n": 0, "scriptPubKey": { "asm": "OP_DUP OP_HASH160 5834479edbbe0539b31ffd3a8f8ebadc2165ed01 OP_EQUALVERIFY OP_CHECKSIG", + "desc": "addr(193P6LtvS4nCnkDvM9uXn1gsSRqh4aDAz7)#nw04wh58", "hex": "76a9145834479edbbe0539b31ffd3a8f8ebadc2165ed0188ac", "address": "193P6LtvS4nCnkDvM9uXn1gsSRqh4aDAz7", "type": "pubkeyhash" diff --git a/test/util/test_runner.py b/test/util/test_runner.py index aa8fd6eee5..03db05c563 100755 --- a/test/util/test_runner.py +++ b/test/util/test_runner.py @@ -22,7 +22,8 @@ import sys def main(): config = configparser.ConfigParser() config.optionxform = str - config.read_file(open(os.path.join(os.path.dirname(__file__), "../config.ini"), encoding="utf8")) + with open(os.path.join(os.path.dirname(__file__), "../config.ini"), encoding="utf8") as f: + config.read_file(f) env_conf = dict(config.items('environment')) parser = argparse.ArgumentParser(description=__doc__) @@ -43,7 +44,8 @@ def main(): def bctester(testDir, input_basename, buildenv): """ Loads and parses the input file, runs all tests and reports results""" input_filename = os.path.join(testDir, input_basename) - raw_data = open(input_filename, encoding="utf8").read() + with open(input_filename, encoding="utf8") as f: + raw_data = f.read() input_data = json.loads(raw_data) failed_testcases = [] @@ -80,7 +82,8 @@ def bctest(testDir, testObj, buildenv): inputData = None if "input" in testObj: filename = os.path.join(testDir, testObj["input"]) - inputData = open(filename, encoding="utf8").read() + with open(filename, encoding="utf8") as f: + inputData = f.read() stdinCfg = subprocess.PIPE # Read the expected output data (if there is any) @@ -91,9 +94,10 @@ def bctest(testDir, testObj, buildenv): outputFn = testObj['output_cmp'] outputType = os.path.splitext(outputFn)[1][1:] # output type from file extension (determines how to compare) try: - outputData = open(os.path.join(testDir, outputFn), encoding="utf8").read() + with open(os.path.join(testDir, outputFn), encoding="utf8") as f: + outputData = f.read() except: - logging.error("Output file " + outputFn + " can not be opened") + logging.error("Output file " + outputFn + " cannot be opened") raise if not outputData: logging.error("Output data missing for " + outputFn) |