diff options
Diffstat (limited to 'test')
73 files changed, 1526 insertions, 281 deletions
diff --git a/test/functional/README.md b/test/functional/README.md index 1bd618a0c3..a4994f2e7c 100644 --- a/test/functional/README.md +++ b/test/functional/README.md @@ -37,6 +37,10 @@ don't have test cases for. `set_test_params()`, `add_options()` and `setup_xxxx()` methods at the top of the subclass, then locally-defined helper methods, then the `run_test()` method. - Use `f'{x}'` for string formatting in preference to `'{}'.format(x)` or `'%s' % x`. +- Use `platform.system()` for detecting the running operating system and `os.name` to + check whether it's a POSIX system (see also the `skip_if_platform_not_{linux,posix}` + methods in the `BitcoinTestFramework` class, which can be used to skip a whole test + depending on the platform). #### Naming guidelines diff --git a/test/functional/feature_asmap.py b/test/functional/feature_asmap.py index 9cff8042a8..ae483fe449 100755 --- a/test/functional/feature_asmap.py +++ b/test/functional/feature_asmap.py @@ -111,6 +111,14 @@ class AsmapTest(BitcoinTestFramework): self.node.assert_start_raises_init_error(extra_args=['-asmap'], expected_msg=msg) os.remove(self.default_asmap) + def test_asmap_health_check(self): + self.log.info('Test bitcoind -asmap logs ASMap Health Check with basic stats') + shutil.copyfile(self.asmap_raw, self.default_asmap) + msg = "ASMap Health Check: 2 clearnet peers are mapped to 1 ASNs with 0 peers being unmapped" + with self.node.assert_debug_log(expected_msgs=[msg]): + self.start_node(0, extra_args=['-asmap']) + os.remove(self.default_asmap) + def run_test(self): self.node = self.nodes[0] self.datadir = self.node.chain_path @@ -124,6 +132,7 @@ class AsmapTest(BitcoinTestFramework): self.test_asmap_interaction_with_addrman_containing_entries() self.test_default_asmap_with_missing_file() self.test_empty_asmap() + self.test_asmap_health_check() if __name__ == '__main__': diff --git a/test/functional/feature_assumeutxo.py b/test/functional/feature_assumeutxo.py index ab2e6c4d0b..f26a300f70 100755 --- a/test/functional/feature_assumeutxo.py +++ b/test/functional/feature_assumeutxo.py @@ -11,7 +11,6 @@ The assumeutxo value generated and used here is committed to in ## Possible test improvements -- TODO: test submitting a transaction and verifying it appears in mempool - TODO: test what happens with -reindex and -reindex-chainstate before the snapshot is validated, and make sure it's deleted successfully. @@ -35,11 +34,14 @@ Interesting starting states could be loading a snapshot when the current chain t """ from shutil import rmtree +from test_framework.messages import tx_from_hex from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, assert_raises_rpc_error, ) +from test_framework.wallet import getnewdestination + START_HEIGHT = 199 SNAPSHOT_BASE_HEIGHT = 299 @@ -75,7 +77,7 @@ class AssumeutxoTest(BitcoinTestFramework): with self.nodes[1].assert_debug_log([log_msg]): assert_raises_rpc_error(-32603, f"Unable to load UTXO snapshot{rpc_details}", self.nodes[1].loadtxoutset, bad_snapshot_path) - self.log.info(" - snapshot file refering to a block that is not in the assumeutxo parameters") + self.log.info(" - snapshot file referring to a block that is not in the assumeutxo parameters") prev_block_hash = self.nodes[0].getblockhash(SNAPSHOT_BASE_HEIGHT - 1) bogus_block_hash = "0" * 64 # Represents any unknown block hash for bad_block_hash in [bogus_block_hash, prev_block_hash]: @@ -112,14 +114,20 @@ class AssumeutxoTest(BitcoinTestFramework): def test_invalid_chainstate_scenarios(self): self.log.info("Test different scenarios of invalid snapshot chainstate in datadir") - self.log.info(" - snapshot chainstate refering to a block that is not in the assumeutxo parameters") + self.log.info(" - snapshot chainstate referring to a block that is not in the assumeutxo parameters") self.stop_node(0) chainstate_snapshot_path = self.nodes[0].chain_path / "chainstate_snapshot" chainstate_snapshot_path.mkdir() with open(chainstate_snapshot_path / "base_blockhash", 'wb') as f: f.write(b'z' * 32) - expected_error = f"Error: A fatal internal error occurred, see debug.log for details" - self.nodes[0].assert_start_raises_init_error(expected_msg=expected_error) + + def expected_error(log_msg="", error_msg=""): + with self.nodes[0].assert_debug_log([log_msg]): + self.nodes[0].assert_start_raises_init_error(expected_msg=error_msg) + + expected_error_msg = f"Error: A fatal internal error occurred, see debug.log for details" + error_details = f"Assumeutxo data not found for the given blockhash" + expected_error(log_msg=error_details, error_msg=expected_error_msg) # resurrect node again rmtree(chainstate_snapshot_path) @@ -201,6 +209,22 @@ class AssumeutxoTest(BitcoinTestFramework): assert_equal(n1.getblockchaininfo()["blocks"], SNAPSHOT_BASE_HEIGHT) + self.log.info("Submit a spending transaction for a snapshot chainstate coin to the mempool") + # spend the coinbase output of the first block that is not available on node1 + spend_coin_blockhash = n1.getblockhash(START_HEIGHT + 1) + assert_raises_rpc_error(-1, "Block not found on disk", n1.getblock, spend_coin_blockhash) + prev_tx = n0.getblock(spend_coin_blockhash, 3)['tx'][0] + prevout = {"txid": prev_tx['txid'], "vout": 0, "scriptPubKey": prev_tx['vout'][0]['scriptPubKey']['hex']} + privkey = n0.get_deterministic_priv_key().key + raw_tx = n1.createrawtransaction([prevout], {getnewdestination()[2]: 24.99}) + signed_tx = n1.signrawtransactionwithkey(raw_tx, [privkey], [prevout])['hex'] + signed_txid = tx_from_hex(signed_tx).rehash() + + assert n1.gettxout(prev_tx['txid'], 0) is not None + n1.sendrawtransaction(signed_tx) + assert signed_txid in n1.getrawmempool() + assert not n1.gettxout(prev_tx['txid'], 0) + PAUSE_HEIGHT = FINAL_HEIGHT - 40 self.log.info("Restarting node to stop at height %d", PAUSE_HEIGHT) diff --git a/test/functional/feature_bind_extra.py b/test/functional/feature_bind_extra.py index 4a94d2ce7b..5cd031f852 100755 --- a/test/functional/feature_bind_extra.py +++ b/test/functional/feature_bind_extra.py @@ -7,15 +7,12 @@ Test starting bitcoind with -bind and/or -bind=...=onion and confirm that bind happens on the expected ports. """ -import sys - from test_framework.netutil import ( addr_to_hex, get_bind_addrs, ) from test_framework.test_framework import ( BitcoinTestFramework, - SkipTest, ) from test_framework.util import ( assert_equal, @@ -32,12 +29,11 @@ class BindExtraTest(BitcoinTestFramework): self.bind_to_localhost_only = False self.num_nodes = 2 - def setup_network(self): + def skip_test_if_missing_module(self): # Due to OS-specific network stats queries, we only run on Linux. - self.log.info("Checking for Linux") - if not sys.platform.startswith('linux'): - raise SkipTest("This test can only be run on Linux.") + self.skip_if_platform_not_linux() + def setup_network(self): loopback_ipv4 = addr_to_hex("127.0.0.1") # Start custom ports by reusing unused p2p ports diff --git a/test/functional/feature_config_args.py b/test/functional/feature_config_args.py index dcea662089..9e13a3deef 100755 --- a/test/functional/feature_config_args.py +++ b/test/functional/feature_config_args.py @@ -6,8 +6,8 @@ import os from pathlib import Path +import platform import re -import sys import tempfile import time @@ -116,7 +116,7 @@ class ConfArgsTest(BitcoinTestFramework): def test_config_file_log(self): # Disable this test for windows currently because trying to override # the default datadir through the environment does not seem to work. - if sys.platform == "win32": + if platform.system() == "Windows": return self.log.info('Test that correct configuration path is changed when configuration file changes the datadir') @@ -339,7 +339,7 @@ class ConfArgsTest(BitcoinTestFramework): def test_ignored_default_conf(self): # Disable this test for windows currently because trying to override # the default datadir through the environment does not seem to work. - if sys.platform == "win32": + if platform.system() == "Windows": return self.log.info('Test error is triggered when bitcoin.conf in the default data directory sets another datadir ' diff --git a/test/functional/feature_filelock.py b/test/functional/feature_filelock.py index 24a68a04bf..567207915e 100755 --- a/test/functional/feature_filelock.py +++ b/test/functional/feature_filelock.py @@ -30,6 +30,12 @@ class FilelockTest(BitcoinTestFramework): expected_msg = f"Error: Cannot obtain a lock on data directory {datadir}. {self.config['environment']['PACKAGE_NAME']} is probably already running." self.nodes[1].assert_start_raises_init_error(extra_args=[f'-datadir={self.nodes[0].datadir_path}', '-noserver'], expected_msg=expected_msg) + self.log.info("Check that cookie and PID file are not deleted when attempting to start a second bitcoind using the same datadir") + cookie_file = datadir / ".cookie" + assert cookie_file.exists() # should not be deleted during the second bitcoind instance shutdown + pid_file = datadir / "bitcoind.pid" + assert pid_file.exists() + if self.is_wallet_compiled(): def check_wallet_filelock(descriptors): wallet_name = ''.join([random.choice(string.ascii_lowercase) for _ in range(6)]) diff --git a/test/functional/feature_init.py b/test/functional/feature_init.py index 37ef3de4dd..268009b0f4 100755 --- a/test/functional/feature_init.py +++ b/test/functional/feature_init.py @@ -1,11 +1,10 @@ #!/usr/bin/env python3 -# Copyright (c) 2021-2022 The Bitcoin Core developers +# Copyright (c) 2021-present The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Stress tests related to node initialization.""" -import os from pathlib import Path -from random import randint +import platform import shutil from test_framework.test_framework import BitcoinTestFramework, SkipTest @@ -37,7 +36,7 @@ class InitStressTest(BitcoinTestFramework): # and other approaches (like below) don't work: # # os.kill(node.process.pid, signal.CTRL_C_EVENT) - if os.name == 'nt': + if platform.system() == 'Windows': raise SkipTest("can't SIGTERM on Windows") self.stop_node(0) @@ -138,8 +137,8 @@ class InitStressTest(BitcoinTestFramework): # Since the genesis block is not checked by -checkblocks, the # perturbation window must be chosen such that a higher block # in blk*.dat is affected. - tf.seek(randint (150, 15000)) - tf.write(b'1' * randint(20, 2000)) + tf.seek(150) + tf.write(b"1" * 200) start_expecting_error(err_fragment) diff --git a/test/functional/feature_notifications.py b/test/functional/feature_notifications.py index adf6c13973..d2b5315d31 100755 --- a/test/functional/feature_notifications.py +++ b/test/functional/feature_notifications.py @@ -4,6 +4,7 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the -alertnotify, -blocknotify and -walletnotify options.""" import os +import platform from test_framework.address import ADDRESS_BCRT1_UNSPENDABLE from test_framework.descriptors import descsum_create @@ -14,13 +15,13 @@ from test_framework.util import ( # Linux allow all characters other than \x00 # Windows disallow control characters (0-31) and /\?%:|"<> -FILE_CHAR_START = 32 if os.name == 'nt' else 1 +FILE_CHAR_START = 32 if platform.system() == 'Windows' else 1 FILE_CHAR_END = 128 -FILE_CHARS_DISALLOWED = '/\\?%*:|"<>' if os.name == 'nt' else '/' +FILE_CHARS_DISALLOWED = '/\\?%*:|"<>' if platform.system() == 'Windows' else '/' UNCONFIRMED_HASH_STRING = 'unconfirmed' def notify_outputname(walletname, txid): - return txid if os.name == 'nt' else f'{walletname}_{txid}' + return txid if platform.system() == 'Windows' else f'{walletname}_{txid}' class NotificationsTest(BitcoinTestFramework): @@ -181,7 +182,7 @@ class NotificationsTest(BitcoinTestFramework): # Universal newline ensures '\n' on 'nt' assert_equal(text[-1], '\n') text = text[:-1] - if os.name == 'nt': + if platform.system() == 'Windows': # On Windows, echo as above will append a whitespace assert_equal(text[-1], ' ') text = text[:-1] diff --git a/test/functional/feature_remove_pruned_files_on_startup.py b/test/functional/feature_remove_pruned_files_on_startup.py index c128587949..4ee653142a 100755 --- a/test/functional/feature_remove_pruned_files_on_startup.py +++ b/test/functional/feature_remove_pruned_files_on_startup.py @@ -4,6 +4,7 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test removing undeleted pruned blk files on startup.""" +import platform import os from test_framework.test_framework import BitcoinTestFramework @@ -32,7 +33,7 @@ class FeatureRemovePrunedFilesOnStartupTest(BitcoinTestFramework): self.nodes[0].pruneblockchain(600) # Windows systems will not remove files with an open fd - if os.name != 'nt': + if platform.system() != 'Windows': assert not os.path.exists(blk0) assert not os.path.exists(rev0) assert not os.path.exists(blk1) diff --git a/test/functional/feature_segwit.py b/test/functional/feature_segwit.py index 6c467fa613..4dc19222c4 100755 --- a/test/functional/feature_segwit.py +++ b/test/functional/feature_segwit.py @@ -88,14 +88,11 @@ class SegWitTest(BitcoinTestFramework): self.extra_args = [ [ "-acceptnonstdtxn=1", - "-rpcserialversion=0", - "-deprecatedrpc=serialversion", "-testactivationheight=segwit@165", "-addresstype=legacy", ], [ "-acceptnonstdtxn=1", - "-rpcserialversion=1", "-testactivationheight=segwit@165", "-addresstype=legacy", ], @@ -224,18 +221,6 @@ class SegWitTest(BitcoinTestFramework): self.fail_accept(self.nodes[0], "mandatory-script-verify-flag-failed (Witness program hash mismatch)", p2sh_ids[NODE_0][P2WPKH][0], sign=False, redeem_script=witness_script(False, self.pubkey[0])) self.fail_accept(self.nodes[0], "mandatory-script-verify-flag-failed (Witness program was passed an empty witness)", p2sh_ids[NODE_0][P2WSH][0], sign=False, redeem_script=witness_script(True, self.pubkey[0])) - self.log.info("Verify block and transaction serialization rpcs return differing serializations depending on rpc serialization flag") - assert self.nodes[2].getblock(blockhash, False) != self.nodes[0].getblock(blockhash, False) - assert self.nodes[1].getblock(blockhash, False) == self.nodes[2].getblock(blockhash, False) - - for tx_id in segwit_tx_list: - tx = tx_from_hex(self.nodes[2].gettransaction(tx_id)["hex"]) - assert self.nodes[2].getrawtransaction(tx_id, False, blockhash) != self.nodes[0].getrawtransaction(tx_id, False, blockhash) - assert self.nodes[1].getrawtransaction(tx_id, False, blockhash) == self.nodes[2].getrawtransaction(tx_id, False, blockhash) - assert self.nodes[0].getrawtransaction(tx_id, False, blockhash) != self.nodes[2].gettransaction(tx_id)["hex"] - assert self.nodes[1].getrawtransaction(tx_id, False, blockhash) == self.nodes[2].gettransaction(tx_id)["hex"] - assert self.nodes[0].getrawtransaction(tx_id, False, blockhash) == tx.serialize_without_witness().hex() - # Coinbase contains the witness commitment nonce, check that RPC shows us coinbase_txid = self.nodes[2].getblock(blockhash)['tx'][0] coinbase_tx = self.nodes[2].gettransaction(txid=coinbase_txid, verbose=True) @@ -276,9 +261,6 @@ class SegWitTest(BitcoinTestFramework): # tx3 (non-segwit input, paying to a non-segwit output). # tx1 is allowed to appear in the block, but no others. txid1 = send_to_witness(1, self.nodes[0], find_spendable_utxo(self.nodes[0], 50), self.pubkey[0], False, Decimal("49.996")) - hex_tx = self.nodes[0].gettransaction(txid)['hex'] - tx = tx_from_hex(hex_tx) - assert tx.wit.is_null() # This should not be a segwit input assert txid1 in self.nodes[0].getrawmempool() tx1_hex = self.nodes[0].gettransaction(txid1)['hex'] @@ -613,11 +595,6 @@ class SegWitTest(BitcoinTestFramework): assert_equal(self.nodes[1].gettransaction(txid, True)["txid"], txid) assert_equal(self.nodes[1].listtransactions("*", 1, 0, True)[0]["txid"], txid) - self.log.info('Test negative and unknown rpcserialversion throw an init error') - self.stop_node(0) - self.nodes[0].assert_start_raises_init_error(["-rpcserialversion=-1"], "Error: rpcserialversion must be non-negative.") - self.nodes[0].assert_start_raises_init_error(["-rpcserialversion=100"], "Error: Unknown rpcserialversion requested.") - def mine_and_test_listunspent(self, script_list, ismine): utxo = find_spendable_utxo(self.nodes[0], 50) tx = CTransaction() diff --git a/test/functional/feature_taproot.py b/test/functional/feature_taproot.py index 1d8c87feb5..e85541d0ec 100755 --- a/test/functional/feature_taproot.py +++ b/test/functional/feature_taproot.py @@ -104,7 +104,7 @@ from test_framework.key import ( tweak_add_privkey, ECKey, ) -from test_framework import secp256k1 +from test_framework.crypto import secp256k1 from test_framework.address import ( hash160, program_to_witness, diff --git a/test/functional/feature_utxo_set_hash.py b/test/functional/feature_utxo_set_hash.py index ce2a5ab8ac..be154b411f 100755 --- a/test/functional/feature_utxo_set_hash.py +++ b/test/functional/feature_utxo_set_hash.py @@ -11,7 +11,7 @@ from test_framework.messages import ( COutPoint, from_hex, ) -from test_framework.muhash import MuHash3072 +from test_framework.crypto.muhash import MuHash3072 from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal from test_framework.wallet import MiniWallet diff --git a/test/functional/interface_rest.py b/test/functional/interface_rest.py index c0679c5ba9..b81eae2506 100755 --- a/test/functional/interface_rest.py +++ b/test/functional/interface_rest.py @@ -65,7 +65,7 @@ class RESTTest (BitcoinTestFramework): body: str = '', status: int = 200, ret_type: RetType = RetType.JSON, - query_params: Optional[typing.Dict[str, typing.Any]] = None, + query_params: Optional[dict[str, typing.Any]] = None, ) -> typing.Union[http.client.HTTPResponse, bytes, str, None]: rest_uri = '/rest' + uri if req_type in ReqType: diff --git a/test/functional/mempool_compatibility.py b/test/functional/mempool_compatibility.py index fd3e219586..a126f164aa 100755 --- a/test/functional/mempool_compatibility.py +++ b/test/functional/mempool_compatibility.py @@ -28,7 +28,7 @@ class MempoolCompatibilityTest(BitcoinTestFramework): def setup_network(self): self.add_nodes(self.num_nodes, versions=[ - 200100, # Last release with previous mempool format + 200100, # Last release without unbroadcast serialization and without XOR None, ]) self.start_nodes() @@ -59,7 +59,7 @@ class MempoolCompatibilityTest(BitcoinTestFramework): old_node_mempool.rename(new_node_mempool) self.log.info("Start new node and verify mempool contains the tx") - self.start_node(1) + self.start_node(1, extra_args=["-persistmempoolv1=1"]) assert old_tx_hash in new_node.getrawmempool() self.log.info("Add unbroadcasted tx to mempool on new node and shutdown") diff --git a/test/functional/mempool_dust.py b/test/functional/mempool_dust.py index f4e385a112..e0c026207a 100755 --- a/test/functional/mempool_dust.py +++ b/test/functional/mempool_dust.py @@ -40,6 +40,7 @@ DUST_RELAY_TX_FEE = 3000 # default setting [sat/kvB] class DustRelayFeeTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 + self.extra_args = [['-permitbaremultisig']] def test_dust_output(self, node: TestNode, dust_relay_fee: Decimal, output_script: CScript, type_desc: str) -> None: @@ -101,7 +102,7 @@ class DustRelayFeeTest(BitcoinTestFramework): else: dust_parameter = f"-dustrelayfee={dustfee_btc_kvb:.8f}" self.log.info(f"Test dust limit setting {dust_parameter} ({dustfee_sat_kvb} sat/kvB)...") - self.restart_node(0, extra_args=[dust_parameter]) + self.restart_node(0, extra_args=[dust_parameter, "-permitbaremultisig"]) for output_script, description in output_scripts: self.test_dust_output(self.nodes[0], dustfee_btc_kvb, output_script, description) diff --git a/test/functional/mempool_limit.py b/test/functional/mempool_limit.py index a1147f70f3..6215610c31 100755 --- a/test/functional/mempool_limit.py +++ b/test/functional/mempool_limit.py @@ -125,8 +125,9 @@ class MempoolLimitTest(BitcoinTestFramework): utxo_to_spend=tx_B["new_utxo"], confirmed_only=True ) - - assert_raises_rpc_error(-26, "too-long-mempool-chain", node.submitpackage, [tx_B["hex"], tx_C["hex"]]) + res = node.submitpackage([tx_B["hex"], tx_C["hex"]]) + assert_equal(res["package_msg"], "transaction failed") + assert "too-long-mempool-chain" in res["tx-results"][tx_C["wtxid"]]["error"] def test_mid_package_eviction(self): node = self.nodes[0] @@ -205,7 +206,7 @@ class MempoolLimitTest(BitcoinTestFramework): # Package should be submitted, temporarily exceeding maxmempool, and then evicted. with node.assert_debug_log(expected_msgs=["rolling minimum fee bumped"]): - assert_raises_rpc_error(-26, "mempool full", node.submitpackage, package_hex) + assert_equal(node.submitpackage(package_hex)["package_msg"], "transaction failed") # Maximum size must never be exceeded. assert_greater_than(node.getmempoolinfo()["maxmempool"], node.getmempoolinfo()["bytes"]) @@ -273,7 +274,9 @@ class MempoolLimitTest(BitcoinTestFramework): package_hex = [cpfp_parent["hex"], replacement_tx["hex"], child["hex"]] # Package should be submitted, temporarily exceeding maxmempool, and then evicted. - assert_raises_rpc_error(-26, "bad-txns-inputs-missingorspent", node.submitpackage, package_hex) + res = node.submitpackage(package_hex) + assert_equal(res["package_msg"], "transaction failed") + assert len([tx_res for _, tx_res in res["tx-results"].items() if "error" in tx_res and tx_res["error"] == "bad-txns-inputs-missingorspent"]) # Maximum size must never be exceeded. assert_greater_than(node.getmempoolinfo()["maxmempool"], node.getmempoolinfo()["bytes"]) @@ -321,6 +324,7 @@ class MempoolLimitTest(BitcoinTestFramework): package_txns.append(tx_child) submitpackage_result = node.submitpackage([tx["hex"] for tx in package_txns]) + assert_equal(submitpackage_result["package_msg"], "success") rich_parent_result = submitpackage_result["tx-results"][tx_rich["wtxid"]] poor_parent_result = submitpackage_result["tx-results"][tx_poor["wtxid"]] @@ -366,7 +370,9 @@ class MempoolLimitTest(BitcoinTestFramework): assert_greater_than(worst_feerate_btcvb, (parent_fee + child_fee) / (tx_parent_just_below["tx"].get_vsize() + tx_child_just_above["tx"].get_vsize())) assert_greater_than(mempoolmin_feerate, (parent_fee) / (tx_parent_just_below["tx"].get_vsize())) assert_greater_than((parent_fee + child_fee) / (tx_parent_just_below["tx"].get_vsize() + tx_child_just_above["tx"].get_vsize()), mempoolmin_feerate / 1000) - assert_raises_rpc_error(-26, "mempool full", node.submitpackage, [tx_parent_just_below["hex"], tx_child_just_above["hex"]]) + res = node.submitpackage([tx_parent_just_below["hex"], tx_child_just_above["hex"]]) + for wtxid in [tx_parent_just_below["wtxid"], tx_child_just_above["wtxid"]]: + assert_equal(res["tx-results"][wtxid]["error"], "mempool full") self.log.info('Test passing a value below the minimum (5 MB) to -maxmempool throws an error') self.stop_node(0) diff --git a/test/functional/mempool_sigoplimit.py b/test/functional/mempool_sigoplimit.py index fbec6d0dc8..2e7850fb40 100755 --- a/test/functional/mempool_sigoplimit.py +++ b/test/functional/mempool_sigoplimit.py @@ -34,7 +34,6 @@ from test_framework.util import ( assert_equal, assert_greater_than, assert_greater_than_or_equal, - assert_raises_rpc_error, ) from test_framework.wallet import MiniWallet from test_framework.wallet_util import generate_keypair @@ -140,7 +139,7 @@ class BytesPerSigOpTest(BitcoinTestFramework): self.log.info("Test a overly-large sigops-vbyte hits package limits") # Make a 2-transaction package which fails vbyte checks even though # separately they would work. - self.restart_node(0, extra_args=["-bytespersigop=5000"] + self.extra_args[0]) + self.restart_node(0, extra_args=["-bytespersigop=5000","-permitbaremultisig=1"] + self.extra_args[0]) def create_bare_multisig_tx(utxo_to_spend=None): _, pubkey = generate_keypair() @@ -169,7 +168,8 @@ class BytesPerSigOpTest(BitcoinTestFramework): assert_equal([x["package-error"] for x in packet_test], ["package-mempool-limits", "package-mempool-limits"]) # When we actually try to submit, the parent makes it into the mempool, but the child would exceed ancestor vsize limits - assert_raises_rpc_error(-26, "too-long-mempool-chain", self.nodes[0].submitpackage, [tx_parent.serialize().hex(), tx_child.serialize().hex()]) + res = self.nodes[0].submitpackage([tx_parent.serialize().hex(), tx_child.serialize().hex()]) + assert "too-long-mempool-chain" in res["tx-results"][tx_child.getwtxid()]["error"] assert tx_parent.rehash() in self.nodes[0].getrawmempool() # Transactions are tiny in weight diff --git a/test/functional/p2p_filter.py b/test/functional/p2p_filter.py index 665f57365f..62d55cc101 100755 --- a/test/functional/p2p_filter.py +++ b/test/functional/p2p_filter.py @@ -11,6 +11,7 @@ from test_framework.messages import ( COIN, MAX_BLOOM_FILTER_SIZE, MAX_BLOOM_HASH_FUNCS, + MSG_WTX, MSG_BLOCK, MSG_FILTERED_BLOCK, msg_filteradd, @@ -135,14 +136,22 @@ class FilterTest(BitcoinTestFramework): self.log.info("Check that a node with bloom filters enabled services p2p mempool messages") filter_peer = P2PBloomFilter() - self.log.debug("Create a tx relevant to the peer before connecting") - txid = self.wallet.send_to(from_node=self.nodes[0], scriptPubKey=filter_peer.watch_script_pubkey, amount=9 * COIN)["txid"] + self.log.info("Create two tx before connecting, one relevant to the node another that is not") + rel_txid = self.wallet.send_to(from_node=self.nodes[0], scriptPubKey=filter_peer.watch_script_pubkey, amount=1 * COIN)["txid"] + irr_result = self.wallet.send_to(from_node=self.nodes[0], scriptPubKey=getnewdestination()[1], amount=2 * COIN) + irr_txid = irr_result["txid"] + irr_wtxid = irr_result["wtxid"] - self.log.debug("Send a mempool msg after connecting and check that the tx is received") + self.log.info("Send a mempool msg after connecting and check that the relevant tx is announced") self.nodes[0].add_p2p_connection(filter_peer) filter_peer.send_and_ping(filter_peer.watch_filter_init) filter_peer.send_message(msg_mempool()) - filter_peer.wait_for_tx(txid) + filter_peer.wait_for_tx(rel_txid) + + self.log.info("Request the irrelevant transaction even though it was not announced") + filter_peer.send_message(msg_getdata([CInv(t=MSG_WTX, h=int(irr_wtxid, 16))])) + self.log.info("We should get it anyway because it was in the mempool on connection to peer") + filter_peer.wait_for_tx(irr_txid) def test_frelay_false(self, filter_peer): self.log.info("Check that a node with fRelay set to false does not receive invs until the filter is set") diff --git a/test/functional/p2p_node_network_limited.py b/test/functional/p2p_node_network_limited.py index a56afbcf7b..89c35e943b 100755 --- a/test/functional/p2p_node_network_limited.py +++ b/test/functional/p2p_node_network_limited.py @@ -8,7 +8,15 @@ Tests that a node configured with -prune=550 signals NODE_NETWORK_LIMITED correc and that it responds to getdata requests for blocks correctly: - send a block within 288 + 2 of the tip - disconnect peers who request blocks older than that.""" -from test_framework.messages import CInv, MSG_BLOCK, msg_getdata, msg_verack, NODE_NETWORK_LIMITED, NODE_WITNESS +from test_framework.messages import ( + CInv, + MSG_BLOCK, + NODE_NETWORK_LIMITED, + NODE_P2P_V2, + NODE_WITNESS, + msg_getdata, + msg_verack, +) from test_framework.p2p import P2PInterface from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( @@ -50,6 +58,8 @@ class NodeNetworkLimitedTest(BitcoinTestFramework): node = self.nodes[0].add_p2p_connection(P2PIgnoreInv()) expected_services = NODE_WITNESS | NODE_NETWORK_LIMITED + if self.options.v2transport: + expected_services |= NODE_P2P_V2 self.log.info("Check that node has signalled expected services.") assert_equal(node.nServices, expected_services) diff --git a/test/functional/p2p_timeouts.py b/test/functional/p2p_timeouts.py index a308577c02..b4fa5099d8 100755 --- a/test/functional/p2p_timeouts.py +++ b/test/functional/p2p_timeouts.py @@ -68,11 +68,14 @@ class TimeoutsTest(BitcoinTestFramework): with self.nodes[0].assert_debug_log(['Unsupported message "ping" prior to verack from peer=0']): no_verack_node.send_message(msg_ping()) - with self.nodes[0].assert_debug_log(['non-version message before version handshake. Message "ping" from peer=1']): - no_version_node.send_message(msg_ping()) - self.mock_forward(1) + # With v2, non-version messages before the handshake would be interpreted as part of the key exchange. + # Therefore, don't execute this part of the test if v2transport is chosen. + if not self.options.v2transport: + with self.nodes[0].assert_debug_log(['non-version message before version handshake. Message "ping" from peer=1']): + no_version_node.send_message(msg_ping()) + self.mock_forward(1) assert "version" in no_verack_node.last_message assert no_verack_node.is_connected @@ -80,11 +83,12 @@ class TimeoutsTest(BitcoinTestFramework): assert no_send_node.is_connected no_verack_node.send_message(msg_ping()) - no_version_node.send_message(msg_ping()) + if not self.options.v2transport: + no_version_node.send_message(msg_ping()) expected_timeout_logs = [ "version handshake timeout peer=0", - "socket no message in first 3 seconds, 1 0 peer=1", + f"socket no message in first 3 seconds, {'0' if self.options.v2transport else '1'} 0 peer=1", "socket no message in first 3 seconds, 0 0 peer=2", ] @@ -100,5 +104,6 @@ class TimeoutsTest(BitcoinTestFramework): extra_args=['-peertimeout=0'], ) + if __name__ == '__main__': TimeoutsTest().main() diff --git a/test/functional/p2p_v2_transport.py b/test/functional/p2p_v2_transport.py index 5ad2194b84..72d22cb77f 100755 --- a/test/functional/p2p_v2_transport.py +++ b/test/functional/p2p_v2_transport.py @@ -48,7 +48,7 @@ class V2TransportTest(BitcoinTestFramework): assert_equal(self.nodes[1].getblockcount(), 5) # verify there is a v2 connection between node 0 and 1 node_0_info = self.nodes[0].getpeerinfo() - node_1_info = self.nodes[0].getpeerinfo() + node_1_info = self.nodes[1].getpeerinfo() assert_equal(len(node_0_info), 1) assert_equal(len(node_1_info), 1) assert_equal(node_0_info[0]["transport_protocol_type"], "v2") @@ -133,9 +133,8 @@ class V2TransportTest(BitcoinTestFramework): V1_PREFIX = MAGIC_BYTES["regtest"] + b"version\x00\x00\x00\x00\x00" assert_equal(len(V1_PREFIX), 16) with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - num_peers = len(self.nodes[0].getpeerinfo()) - s.connect(("127.0.0.1", p2p_port(0))) - self.wait_until(lambda: len(self.nodes[0].getpeerinfo()) == num_peers + 1) + with self.nodes[0].wait_for_new_peer(): + s.connect(("127.0.0.1", p2p_port(0))) s.sendall(V1_PREFIX[:-1]) assert_equal(self.nodes[0].getpeerinfo()[-1]["transport_protocol_type"], "detecting") s.sendall(bytes([V1_PREFIX[-1]])) # send out last prefix byte @@ -144,22 +143,23 @@ class V2TransportTest(BitcoinTestFramework): # Check wrong network prefix detection (hits if the next 12 bytes correspond to a v1 version message) wrong_network_magic_prefix = MAGIC_BYTES["signet"] + V1_PREFIX[4:] with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - s.connect(("127.0.0.1", p2p_port(0))) + with self.nodes[0].wait_for_new_peer(): + s.connect(("127.0.0.1", p2p_port(0))) with self.nodes[0].assert_debug_log(["V2 transport error: V1 peer with wrong MessageStart"]): s.sendall(wrong_network_magic_prefix + b"somepayload") # Check detection of missing garbage terminator (hits after fixed amount of data if terminator never matches garbage) MAX_KEY_GARB_AND_GARBTERM_LEN = 64 + 4095 + 16 with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - num_peers = len(self.nodes[0].getpeerinfo()) - s.connect(("127.0.0.1", p2p_port(0))) - self.wait_until(lambda: len(self.nodes[0].getpeerinfo()) == num_peers + 1) + with self.nodes[0].wait_for_new_peer(): + s.connect(("127.0.0.1", p2p_port(0))) s.sendall(b'\x00' * (MAX_KEY_GARB_AND_GARBTERM_LEN - 1)) self.wait_until(lambda: self.nodes[0].getpeerinfo()[-1]["bytesrecv"] == MAX_KEY_GARB_AND_GARBTERM_LEN - 1) with self.nodes[0].assert_debug_log(["V2 transport error: missing garbage terminator"]): + peer_id = self.nodes[0].getpeerinfo()[-1]["id"] s.sendall(b'\x00') # send out last byte # should disconnect immediately - self.wait_until(lambda: len(self.nodes[0].getpeerinfo()) == num_peers) + self.wait_until(lambda: not peer_id in [p["id"] for p in self.nodes[0].getpeerinfo()]) if __name__ == '__main__': diff --git a/test/functional/rpc_bind.py b/test/functional/rpc_bind.py index 43cd23fc7a..3106419e5c 100755 --- a/test/functional/rpc_bind.py +++ b/test/functional/rpc_bind.py @@ -4,8 +4,6 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test running bitcoind with the -rpcbind and -rpcallowip options.""" -import sys - from test_framework.netutil import all_interfaces, addr_to_hex, get_bind_addrs, test_ipv6_local from test_framework.test_framework import BitcoinTestFramework, SkipTest from test_framework.util import assert_equal, assert_raises_rpc_error, get_rpc_proxy, rpc_port, rpc_url @@ -17,6 +15,10 @@ class RPCBindTest(BitcoinTestFramework): self.num_nodes = 1 self.supports_cli = False + def skip_test_if_missing_module(self): + # due to OS-specific network stats queries, this test works only on Linux + self.skip_if_platform_not_linux() + def setup_network(self): self.add_nodes(self.num_nodes, None) @@ -61,14 +63,9 @@ class RPCBindTest(BitcoinTestFramework): self.stop_nodes() def run_test(self): - # due to OS-specific network stats queries, this test works only on Linux if sum([self.options.run_ipv4, self.options.run_ipv6, self.options.run_nonloopback]) > 1: raise AssertionError("Only one of --ipv4, --ipv6 and --nonloopback can be set") - self.log.info("Check for linux") - if not sys.platform.startswith('linux'): - raise SkipTest("This test can only be run on linux.") - self.log.info("Check for ipv6") have_ipv6 = test_ipv6_local() if not have_ipv6 and not (self.options.run_ipv4 or self.options.run_nonloopback): diff --git a/test/functional/rpc_blockchain.py b/test/functional/rpc_blockchain.py index 8eb9f3aeb1..9b7743cafa 100755 --- a/test/functional/rpc_blockchain.py +++ b/test/functional/rpc_blockchain.py @@ -437,7 +437,6 @@ class BlockchainTest(BitcoinTestFramework): def _test_getnetworkhashps(self): self.log.info("Test getnetworkhashps") - hashes_per_second = self.nodes[0].getnetworkhashps() assert_raises_rpc_error( -3, textwrap.dedent(""" @@ -449,17 +448,46 @@ class BlockchainTest(BitcoinTestFramework): """).strip(), lambda: self.nodes[0].getnetworkhashps("a", []), ) + assert_raises_rpc_error( + -8, + "Block does not exist at specified height", + lambda: self.nodes[0].getnetworkhashps(100, self.nodes[0].getblockcount() + 1), + ) + assert_raises_rpc_error( + -8, + "Block does not exist at specified height", + lambda: self.nodes[0].getnetworkhashps(100, -10), + ) + assert_raises_rpc_error( + -8, + "Invalid nblocks. Must be a positive number or -1.", + lambda: self.nodes[0].getnetworkhashps(-100), + ) + assert_raises_rpc_error( + -8, + "Invalid nblocks. Must be a positive number or -1.", + lambda: self.nodes[0].getnetworkhashps(0), + ) + + # Genesis block height estimate should return 0 + hashes_per_second = self.nodes[0].getnetworkhashps(100, 0) + assert_equal(hashes_per_second, 0) + # This should be 2 hashes every 10 minutes or 1/300 + hashes_per_second = self.nodes[0].getnetworkhashps() assert abs(hashes_per_second * 300 - 1) < 0.0001 - # Test setting the first param of getnetworkhashps to negative value returns the average network + # Test setting the first param of getnetworkhashps to -1 returns the average network # hashes per second from the last difficulty change. current_block_height = self.nodes[0].getmininginfo()['blocks'] blocks_since_last_diff_change = current_block_height % DIFFICULTY_ADJUSTMENT_INTERVAL + 1 expected_hashes_per_second_since_diff_change = self.nodes[0].getnetworkhashps(blocks_since_last_diff_change) assert_equal(self.nodes[0].getnetworkhashps(-1), expected_hashes_per_second_since_diff_change) - assert_equal(self.nodes[0].getnetworkhashps(-2), expected_hashes_per_second_since_diff_change) + + # Ensure long lookups get truncated to chain length + hashes_per_second = self.nodes[0].getnetworkhashps(self.nodes[0].getblockcount() + 1000) + assert hashes_per_second > 0.003 def _test_stopatheight(self): self.log.info("Test stopping at height") diff --git a/test/functional/rpc_net.py b/test/functional/rpc_net.py index 50a022fc7e..e1820b0f55 100755 --- a/test/functional/rpc_net.py +++ b/test/functional/rpc_net.py @@ -9,6 +9,7 @@ Tests correspond to code in rpc/net.cpp. from decimal import Decimal from itertools import product +import platform import time import test_framework.messages @@ -110,7 +111,7 @@ class NetTest(BitcoinTestFramework): no_version_peer_id = 2 no_version_peer_conntime = int(time.time()) self.nodes[0].setmocktime(no_version_peer_conntime) - with self.nodes[0].assert_debug_log([f"Added connection peer={no_version_peer_id}"]): + with self.nodes[0].wait_for_new_peer(): no_version_peer = self.nodes[0].add_p2p_connection(P2PInterface(), send_version=False, wait_for_verack=False) self.nodes[0].setmocktime(0) peer_info = self.nodes[0].getpeerinfo()[no_version_peer_id] @@ -150,7 +151,7 @@ class NetTest(BitcoinTestFramework): "synced_blocks": -1, "synced_headers": -1, "timeoffset": 0, - "transport_protocol_type": "v1", + "transport_protocol_type": "v1" if not self.options.v2transport else "detecting", "version": 0, }, ) @@ -160,19 +161,23 @@ class NetTest(BitcoinTestFramework): def test_getnettotals(self): self.log.info("Test getnettotals") # Test getnettotals and getpeerinfo by doing a ping. The bytes - # sent/received should increase by at least the size of one ping (32 - # bytes) and one pong (32 bytes). + # sent/received should increase by at least the size of one ping + # and one pong. Both have a payload size of 8 bytes, but the total + # size depends on the used p2p version: + # - p2p v1: 24 bytes (header) + 8 bytes (payload) = 32 bytes + # - p2p v2: 21 bytes (header/tag with short-id) + 8 bytes (payload) = 29 bytes + ping_size = 32 if not self.options.v2transport else 29 net_totals_before = self.nodes[0].getnettotals() peer_info_before = self.nodes[0].getpeerinfo() self.nodes[0].ping() - self.wait_until(lambda: (self.nodes[0].getnettotals()['totalbytessent'] >= net_totals_before['totalbytessent'] + 32 * 2), timeout=1) - self.wait_until(lambda: (self.nodes[0].getnettotals()['totalbytesrecv'] >= net_totals_before['totalbytesrecv'] + 32 * 2), timeout=1) + self.wait_until(lambda: (self.nodes[0].getnettotals()['totalbytessent'] >= net_totals_before['totalbytessent'] + ping_size * 2), timeout=1) + self.wait_until(lambda: (self.nodes[0].getnettotals()['totalbytesrecv'] >= net_totals_before['totalbytesrecv'] + ping_size * 2), timeout=1) for peer_before in peer_info_before: peer_after = lambda: next(p for p in self.nodes[0].getpeerinfo() if p['id'] == peer_before['id']) - self.wait_until(lambda: peer_after()['bytesrecv_per_msg'].get('pong', 0) >= peer_before['bytesrecv_per_msg'].get('pong', 0) + 32, timeout=1) - self.wait_until(lambda: peer_after()['bytessent_per_msg'].get('ping', 0) >= peer_before['bytessent_per_msg'].get('ping', 0) + 32, timeout=1) + self.wait_until(lambda: peer_after()['bytesrecv_per_msg'].get('pong', 0) >= peer_before['bytesrecv_per_msg'].get('pong', 0) + ping_size, timeout=1) + self.wait_until(lambda: peer_after()['bytessent_per_msg'].get('ping', 0) >= peer_before['bytessent_per_msg'].get('ping', 0) + ping_size, timeout=1) def test_getnetworkinfo(self): self.log.info("Test getnetworkinfo") @@ -215,8 +220,13 @@ class NetTest(BitcoinTestFramework): # add a node (node2) to node0 ip_port = "127.0.0.1:{}".format(p2p_port(2)) self.nodes[0].addnode(node=ip_port, command='add') + # try to add an equivalent ip + # (note that OpenBSD doesn't support the IPv4 shorthand notation with omitted zero-bytes) + if platform.system() != "OpenBSD": + ip_port2 = "127.1:{}".format(p2p_port(2)) + assert_raises_rpc_error(-23, "Node already added", self.nodes[0].addnode, node=ip_port2, command='add') # check that the node has indeed been added - added_nodes = self.nodes[0].getaddednodeinfo(ip_port) + added_nodes = self.nodes[0].getaddednodeinfo() assert_equal(len(added_nodes), 1) assert_equal(added_nodes[0]['addednode'], ip_port) # check that node cannot be added again @@ -342,7 +352,10 @@ class NetTest(BitcoinTestFramework): node = self.nodes[0] self.restart_node(0) - self.connect_nodes(0, 1) + # we want to use a p2p v1 connection here in order to ensure + # a peer id of zero (a downgrade from v2 to v1 would lead + # to an increase of the peer id) + self.connect_nodes(0, 1, peer_advertises_v2=False) self.log.info("Test sendmsgtopeer") self.log.debug("Send a valid message") diff --git a/test/functional/rpc_packages.py b/test/functional/rpc_packages.py index 5644a9f5a8..664f2df3f1 100755 --- a/test/functional/rpc_packages.py +++ b/test/functional/rpc_packages.py @@ -304,6 +304,7 @@ class RPCPackagesTest(BitcoinTestFramework): submitpackage_result = node.submitpackage(package=[tx["hex"] for tx in package_txns]) # Check that each result is present, with the correct size and fees + assert_equal(submitpackage_result["package_msg"], "success") for package_txn in package_txns: tx = package_txn["tx"] assert tx.getwtxid() in submitpackage_result["tx-results"] @@ -334,9 +335,26 @@ class RPCPackagesTest(BitcoinTestFramework): self.log.info("Submitpackage only allows packages of 1 child with its parents") # Chain of 3 transactions has too many generations - chain_hex = [t["hex"] for t in self.wallet.create_self_transfer_chain(chain_length=25)] + legacy_pool = node.getrawmempool() + chain_hex = [t["hex"] for t in self.wallet.create_self_transfer_chain(chain_length=3)] assert_raises_rpc_error(-25, "package topology disallowed", node.submitpackage, chain_hex) + assert_equal(legacy_pool, node.getrawmempool()) + # Create a transaction chain such as only the parent gets accepted (by making the child's + # version non-standard). Make sure the parent does get broadcast. + self.log.info("If a package is partially submitted, transactions included in mempool get broadcast") + peer = node.add_p2p_connection(P2PTxInvStore()) + txs = self.wallet.create_self_transfer_chain(chain_length=2) + bad_child = tx_from_hex(txs[1]["hex"]) + bad_child.nVersion = -1 + hex_partial_acceptance = [txs[0]["hex"], bad_child.serialize().hex()] + res = node.submitpackage(hex_partial_acceptance) + assert_equal(res["package_msg"], "transaction failed") + first_wtxid = txs[0]["tx"].getwtxid() + assert "error" not in res["tx-results"][first_wtxid] + sec_wtxid = bad_child.getwtxid() + assert_equal(res["tx-results"][sec_wtxid]["error"], "version") + peer.wait_for_broadcast([first_wtxid]) if __name__ == "__main__": RPCPackagesTest().main() diff --git a/test/functional/rpc_rawtransaction.py b/test/functional/rpc_rawtransaction.py index 2395935620..c12865b5e3 100755 --- a/test/functional/rpc_rawtransaction.py +++ b/test/functional/rpc_rawtransaction.py @@ -32,6 +32,7 @@ from test_framework.script import ( from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, + assert_greater_than, assert_raises_rpc_error, ) from test_framework.wallet import ( @@ -70,7 +71,7 @@ class RawTransactionsTest(BitcoinTestFramework): self.extra_args = [ ["-txindex"], ["-txindex"], - [], + ["-fastprune", "-prune=1"], ] # whitelist all peers to speed up tx relay / mempool sync for args in self.extra_args: @@ -85,7 +86,6 @@ class RawTransactionsTest(BitcoinTestFramework): self.wallet = MiniWallet(self.nodes[0]) self.getrawtransaction_tests() - self.getrawtransaction_verbosity_tests() self.createrawtransaction_tests() self.sendrawtransaction_tests() self.sendrawtransaction_testmempoolaccept_tests() @@ -94,6 +94,8 @@ class RawTransactionsTest(BitcoinTestFramework): if self.is_specified_wallet_compiled() and not self.options.descriptors: self.import_deterministic_coinbase_privkeys() self.raw_multisig_transaction_legacy_tests() + self.getrawtransaction_verbosity_tests() + def getrawtransaction_tests(self): tx = self.wallet.send_self_transfer(from_node=self.nodes[0]) @@ -243,6 +245,13 @@ class RawTransactionsTest(BitcoinTestFramework): coin_base = self.nodes[1].getblock(block1)['tx'][0] gottx = self.nodes[1].getrawtransaction(txid=coin_base, verbosity=2, blockhash=block1) assert 'fee' not in gottx + # check that verbosity 2 for a mempool tx will fallback to verbosity 1 + # Do this with a pruned chain, as a regression test for https://github.com/bitcoin/bitcoin/pull/29003 + self.generate(self.nodes[2], 400) + assert_greater_than(self.nodes[2].pruneblockchain(250), 0) + mempool_tx = self.wallet.send_self_transfer(from_node=self.nodes[2])['txid'] + gottx = self.nodes[2].getrawtransaction(txid=mempool_tx, verbosity=2) + assert 'fee' not in gottx def createrawtransaction_tests(self): self.log.info("Test createrawtransaction") diff --git a/test/functional/test_framework/blockfilter.py b/test/functional/test_framework/blockfilter.py index a30e37ea5b..a16aa3d34f 100644 --- a/test/functional/test_framework/blockfilter.py +++ b/test/functional/test_framework/blockfilter.py @@ -4,7 +4,7 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Helper routines relevant for compact block filters (BIP158). """ -from .siphash import siphash +from .crypto.siphash import siphash def bip158_basic_element_hash(script_pub_key, N, block_hash): @@ -29,7 +29,7 @@ def bip158_basic_element_hash(script_pub_key, N, block_hash): def bip158_relevant_scriptpubkeys(node, block_hash): - """ Determines the basic filter relvant scriptPubKeys as defined in BIP158: + """ Determines the basic filter relevant scriptPubKeys as defined in BIP158: 'A basic filter MUST contain exactly the following items for each transaction in a block: - The previous output script (the script being spent) for each input, except for diff --git a/test/functional/test_framework/crypto/bip324_cipher.py b/test/functional/test_framework/crypto/bip324_cipher.py new file mode 100644 index 0000000000..56190647f2 --- /dev/null +++ b/test/functional/test_framework/crypto/bip324_cipher.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +"""Test-only implementation of ChaCha20 Poly1305 AEAD Construction in RFC 8439 and FSChaCha20Poly1305 for BIP 324 + +It is designed for ease of understanding, not performance. + +WARNING: This code is slow and trivially vulnerable to side channel attacks. Do not use for +anything but tests. +""" + +import unittest + +from .chacha20 import chacha20_block, REKEY_INTERVAL +from .poly1305 import Poly1305 + + +def pad16(x): + if len(x) % 16 == 0: + return b'' + return b'\x00' * (16 - (len(x) % 16)) + + +def aead_chacha20_poly1305_encrypt(key, nonce, aad, plaintext): + """Encrypt a plaintext using ChaCha20Poly1305.""" + ret = bytearray() + msg_len = len(plaintext) + for i in range((msg_len + 63) // 64): + now = min(64, msg_len - 64 * i) + keystream = chacha20_block(key, nonce, i + 1) + for j in range(now): + ret.append(plaintext[j + 64 * i] ^ keystream[j]) + poly1305 = Poly1305(chacha20_block(key, nonce, 0)[:32]) + mac_data = aad + pad16(aad) + mac_data += ret + pad16(ret) + mac_data += len(aad).to_bytes(8, 'little') + msg_len.to_bytes(8, 'little') + ret += poly1305.tag(mac_data) + return bytes(ret) + + +def aead_chacha20_poly1305_decrypt(key, nonce, aad, ciphertext): + """Decrypt a ChaCha20Poly1305 ciphertext.""" + if len(ciphertext) < 16: + return None + msg_len = len(ciphertext) - 16 + poly1305 = Poly1305(chacha20_block(key, nonce, 0)[:32]) + mac_data = aad + pad16(aad) + mac_data += ciphertext[:-16] + pad16(ciphertext[:-16]) + mac_data += len(aad).to_bytes(8, 'little') + msg_len.to_bytes(8, 'little') + if ciphertext[-16:] != poly1305.tag(mac_data): + return None + ret = bytearray() + for i in range((msg_len + 63) // 64): + now = min(64, msg_len - 64 * i) + keystream = chacha20_block(key, nonce, i + 1) + for j in range(now): + ret.append(ciphertext[j + 64 * i] ^ keystream[j]) + return bytes(ret) + + +class FSChaCha20Poly1305: + """Rekeying wrapper AEAD around ChaCha20Poly1305.""" + def __init__(self, initial_key): + self._key = initial_key + self._packet_counter = 0 + + def _crypt(self, aad, text, is_decrypt): + nonce = ((self._packet_counter % REKEY_INTERVAL).to_bytes(4, 'little') + + (self._packet_counter // REKEY_INTERVAL).to_bytes(8, 'little')) + if is_decrypt: + ret = aead_chacha20_poly1305_decrypt(self._key, nonce, aad, text) + else: + ret = aead_chacha20_poly1305_encrypt(self._key, nonce, aad, text) + if (self._packet_counter + 1) % REKEY_INTERVAL == 0: + rekey_nonce = b"\xFF\xFF\xFF\xFF" + nonce[4:] + self._key = aead_chacha20_poly1305_encrypt(self._key, rekey_nonce, b"", b"\x00" * 32)[:32] + self._packet_counter += 1 + return ret + + def decrypt(self, aad, ciphertext): + return self._crypt(aad, ciphertext, True) + + def encrypt(self, aad, plaintext): + return self._crypt(aad, plaintext, False) + + +# Test vectors from RFC8439 consisting of plaintext, aad, 32 byte key, 12 byte nonce and ciphertext +AEAD_TESTS = [ + # RFC 8439 Example from section 2.8.2 + ["4c616469657320616e642047656e746c656d656e206f662074686520636c6173" + "73206f66202739393a204966204920636f756c64206f6666657220796f75206f" + "6e6c79206f6e652074697020666f7220746865206675747572652c2073756e73" + "637265656e20776f756c642062652069742e", + "50515253c0c1c2c3c4c5c6c7", + "808182838485868788898a8b8c8d8e8f909192939495969798999a9b9c9d9e9f", + [7, 0x4746454443424140], + "d31a8d34648e60db7b86afbc53ef7ec2a4aded51296e08fea9e2b5a736ee62d6" + "3dbea45e8ca9671282fafb69da92728b1a71de0a9e060b2905d6a5b67ecd3b36" + "92ddbd7f2d778b8c9803aee328091b58fab324e4fad675945585808b4831d7bc" + "3ff4def08e4b7a9de576d26586cec64b61161ae10b594f09e26a7e902ecbd060" + "0691"], + # RFC 8439 Test vector A.5 + ["496e7465726e65742d4472616674732061726520647261667420646f63756d65" + "6e74732076616c696420666f722061206d6178696d756d206f6620736978206d" + "6f6e74687320616e64206d617920626520757064617465642c207265706c6163" + "65642c206f72206f62736f6c65746564206279206f7468657220646f63756d65" + "6e747320617420616e792074696d652e20497420697320696e617070726f7072" + "6961746520746f2075736520496e7465726e65742d4472616674732061732072" + "65666572656e6365206d6174657269616c206f7220746f206369746520746865" + "6d206f74686572207468616e206173202fe2809c776f726b20696e2070726f67" + "726573732e2fe2809d", + "f33388860000000000004e91", + "1c9240a5eb55d38af333888604f6b5f0473917c1402b80099dca5cbc207075c0", + [0, 0x0807060504030201], + "64a0861575861af460f062c79be643bd5e805cfd345cf389f108670ac76c8cb2" + "4c6cfc18755d43eea09ee94e382d26b0bdb7b73c321b0100d4f03b7f355894cf" + "332f830e710b97ce98c8a84abd0b948114ad176e008d33bd60f982b1ff37c855" + "9797a06ef4f0ef61c186324e2b3506383606907b6a7c02b0f9f6157b53c867e4" + "b9166c767b804d46a59b5216cde7a4e99040c5a40433225ee282a1b0a06c523e" + "af4534d7f83fa1155b0047718cbc546a0d072b04b3564eea1b422273f548271a" + "0bb2316053fa76991955ebd63159434ecebb4e466dae5a1073a6727627097a10" + "49e617d91d361094fa68f0ff77987130305beaba2eda04df997b714d6c6f2c29" + "a6ad5cb4022b02709beead9d67890cbb22392336fea1851f38"], + # Test vectors exercising aad and plaintext which are multiples of 16 bytes. + ["8d2d6a8befd9716fab35819eaac83b33269afb9f1a00fddf66095a6c0cd91951" + "a6b7ad3db580be0674c3f0b55f618e34", + "", + "72ddc73f07101282bbbcf853b9012a9f9695fc5d36b303a97fd0845d0314e0c3", + [0x3432b75f, 0xb3585537eb7f4024], + "f760b8224fb2a317b1b07875092606131232a5b86ae142df5df1c846a7f6341a" + "f2564483dd77f836be45e6230808ffe402a6f0a3e8be074b3d1f4ea8a7b09451"], + ["", + "36970d8a704c065de16250c18033de5a400520ac1b5842b24551e5823a3314f3" + "946285171e04a81ebfbe3566e312e74ab80e94c7dd2ff4e10de0098a58d0f503", + "77adda51d6730b9ad6c995658cbd49f581b2547e7c0c08fcc24ceec797461021", + [0x1f90da88, 0x75dafa3ef84471a4], + "aaae5bb81e8407c94b2ae86ae0c7efbe"], +] + +FSAEAD_TESTS = [ + ["d6a4cb04ef0f7c09c1866ed29dc24d820e75b0491032a51b4c3366f9ca35c19e" + "a3047ec6be9d45f9637b63e1cf9eb4c2523a5aab7b851ebeba87199db0e839cf" + "0d5c25e50168306377aedbe9089fd2463ded88b83211cf51b73b150608cc7a60" + "0d0f11b9a742948482e1b109d8faf15b450aa7322e892fa2208c6691e3fecf4c" + "711191b14d75a72147", + "786cb9b6ebf44288974cf0", + "5c9e1c3951a74fba66708bf9d2c217571684556b6a6a3573bff2847d38612654", + 500, + "9dcebbd3281ea3dd8e9a1ef7d55a97abd6743e56ebc0c190cb2c4e14160b385e" + "0bf508dddf754bd02c7c208447c131ce23e47a4a14dfaf5dd8bc601323950f75" + "4e05d46e9232f83fc5120fbbef6f5347a826ec79a93820718d4ec7a2b7cfaaa4" + "4b21e16d726448b62f803811aff4f6d827ed78e738ce8a507b81a8ae13131192" + "8039213de18a5120dc9b7370baca878f50ff254418de3da50c"], + ["8349b7a2690b63d01204800c288ff1138a1d473c832c90ea8b3fc102d0bb3adc" + "44261b247c7c3d6760bfbe979d061c305f46d94c0582ac3099f0bf249f8cb234", + "", + "3bd2093fcbcb0d034d8c569583c5425c1a53171ea299f8cc3bbf9ae3530adfce", + 60000, + "30a6757ff8439b975363f166a0fa0e36722ab35936abd704297948f45083f4d4" + "99433137ce931f7fca28a0acd3bc30f57b550acbc21cbd45bbef0739d9caf30c" + "14b94829deb27f0b1923a2af704ae5d6"], +] + + +class TestFrameworkAEAD(unittest.TestCase): + def test_aead(self): + """ChaCha20Poly1305 AEAD test vectors.""" + for test_vector in AEAD_TESTS: + hex_plain, hex_aad, hex_key, hex_nonce, hex_cipher = test_vector + plain = bytes.fromhex(hex_plain) + aad = bytes.fromhex(hex_aad) + key = bytes.fromhex(hex_key) + nonce = hex_nonce[0].to_bytes(4, 'little') + hex_nonce[1].to_bytes(8, 'little') + + ciphertext = aead_chacha20_poly1305_encrypt(key, nonce, aad, plain) + self.assertEqual(hex_cipher, ciphertext.hex()) + plaintext = aead_chacha20_poly1305_decrypt(key, nonce, aad, ciphertext) + self.assertEqual(plain, plaintext) + + def test_fschacha20poly1305aead(self): + "FSChaCha20Poly1305 AEAD test vectors." + for test_vector in FSAEAD_TESTS: + hex_plain, hex_aad, hex_key, msg_idx, hex_cipher = test_vector + plain = bytes.fromhex(hex_plain) + aad = bytes.fromhex(hex_aad) + key = bytes.fromhex(hex_key) + + enc_aead = FSChaCha20Poly1305(key) + dec_aead = FSChaCha20Poly1305(key) + + for _ in range(msg_idx): + enc_aead.encrypt(b"", b"") + ciphertext = enc_aead.encrypt(aad, plain) + self.assertEqual(hex_cipher, ciphertext.hex()) + + for _ in range(msg_idx): + dec_aead.decrypt(b"", bytes(16)) + plaintext = dec_aead.decrypt(aad, ciphertext) + self.assertEqual(plain, plaintext) diff --git a/test/functional/test_framework/crypto/chacha20.py b/test/functional/test_framework/crypto/chacha20.py new file mode 100644 index 0000000000..19b6698dfb --- /dev/null +++ b/test/functional/test_framework/crypto/chacha20.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +"""Test-only implementation of ChaCha20 cipher and FSChaCha20 for BIP 324 + +It is designed for ease of understanding, not performance. + +WARNING: This code is slow and trivially vulnerable to side channel attacks. Do not use for +anything but tests. +""" + +import unittest + +CHACHA20_INDICES = ( + (0, 4, 8, 12), (1, 5, 9, 13), (2, 6, 10, 14), (3, 7, 11, 15), + (0, 5, 10, 15), (1, 6, 11, 12), (2, 7, 8, 13), (3, 4, 9, 14) +) + +CHACHA20_CONSTANTS = (0x61707865, 0x3320646e, 0x79622d32, 0x6b206574) +REKEY_INTERVAL = 224 # packets + + +def rotl32(v, bits): + """Rotate the 32-bit value v left by bits bits.""" + bits %= 32 # Make sure the term below does not throw an exception + return ((v << bits) & 0xffffffff) | (v >> (32 - bits)) + + +def chacha20_doubleround(s): + """Apply a ChaCha20 double round to 16-element state array s. + See https://cr.yp.to/chacha/chacha-20080128.pdf and https://tools.ietf.org/html/rfc8439 + """ + for a, b, c, d in CHACHA20_INDICES: + s[a] = (s[a] + s[b]) & 0xffffffff + s[d] = rotl32(s[d] ^ s[a], 16) + s[c] = (s[c] + s[d]) & 0xffffffff + s[b] = rotl32(s[b] ^ s[c], 12) + s[a] = (s[a] + s[b]) & 0xffffffff + s[d] = rotl32(s[d] ^ s[a], 8) + s[c] = (s[c] + s[d]) & 0xffffffff + s[b] = rotl32(s[b] ^ s[c], 7) + + +def chacha20_block(key, nonce, cnt): + """Compute the 64-byte output of the ChaCha20 block function. + Takes as input a 32-byte key, 12-byte nonce, and 32-bit integer counter. + """ + # Initial state. + init = [0] * 16 + init[:4] = CHACHA20_CONSTANTS[:4] + init[4:12] = [int.from_bytes(key[i:i+4], 'little') for i in range(0, 32, 4)] + init[12] = cnt + init[13:16] = [int.from_bytes(nonce[i:i+4], 'little') for i in range(0, 12, 4)] + # Perform 20 rounds. + state = list(init) + for _ in range(10): + chacha20_doubleround(state) + # Add initial values back into state. + for i in range(16): + state[i] = (state[i] + init[i]) & 0xffffffff + # Produce byte output + return b''.join(state[i].to_bytes(4, 'little') for i in range(16)) + +class FSChaCha20: + """Rekeying wrapper stream cipher around ChaCha20.""" + def __init__(self, initial_key, rekey_interval=REKEY_INTERVAL): + self._key = initial_key + self._rekey_interval = rekey_interval + self._block_counter = 0 + self._chunk_counter = 0 + self._keystream = b'' + + def _get_keystream_bytes(self, nbytes): + while len(self._keystream) < nbytes: + nonce = ((0).to_bytes(4, 'little') + (self._chunk_counter // self._rekey_interval).to_bytes(8, 'little')) + self._keystream += chacha20_block(self._key, nonce, self._block_counter) + self._block_counter += 1 + ret = self._keystream[:nbytes] + self._keystream = self._keystream[nbytes:] + return ret + + def crypt(self, chunk): + ks = self._get_keystream_bytes(len(chunk)) + ret = bytes([ks[i] ^ chunk[i] for i in range(len(chunk))]) + if ((self._chunk_counter + 1) % self._rekey_interval) == 0: + self._key = self._get_keystream_bytes(32) + self._block_counter = 0 + self._keystream = b'' + self._chunk_counter += 1 + return ret + + +# Test vectors from RFC7539/8439 consisting of 32 byte key, 12 byte nonce, block counter +# and 64 byte output after applying `chacha20_block` function +CHACHA20_TESTS = [ + ["000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", [0x09000000, 0x4a000000], 1, + "10f1e7e4d13b5915500fdd1fa32071c4c7d1f4c733c068030422aa9ac3d46c4e" + "d2826446079faa0914c2d705d98b02a2b5129cd1de164eb9cbd083e8a2503c4e"], + ["0000000000000000000000000000000000000000000000000000000000000000", [0, 0], 0, + "76b8e0ada0f13d90405d6ae55386bd28bdd219b8a08ded1aa836efcc8b770dc7" + "da41597c5157488d7724e03fb8d84a376a43b8f41518a11cc387b669b2ee6586"], + ["0000000000000000000000000000000000000000000000000000000000000000", [0, 0], 1, + "9f07e7be5551387a98ba977c732d080dcb0f29a048e3656912c6533e32ee7aed" + "29b721769ce64e43d57133b074d839d531ed1f28510afb45ace10a1f4b794d6f"], + ["0000000000000000000000000000000000000000000000000000000000000001", [0, 0], 1, + "3aeb5224ecf849929b9d828db1ced4dd832025e8018b8160b82284f3c949aa5a" + "8eca00bbb4a73bdad192b5c42f73f2fd4e273644c8b36125a64addeb006c13a0"], + ["00ff000000000000000000000000000000000000000000000000000000000000", [0, 0], 2, + "72d54dfbf12ec44b362692df94137f328fea8da73990265ec1bbbea1ae9af0ca" + "13b25aa26cb4a648cb9b9d1be65b2c0924a66c54d545ec1b7374f4872e99f096"], + ["0000000000000000000000000000000000000000000000000000000000000000", [0, 0x200000000000000], 0, + "c2c64d378cd536374ae204b9ef933fcd1a8b2288b3dfa49672ab765b54ee27c7" + "8a970e0e955c14f3a88e741b97c286f75f8fc299e8148362fa198a39531bed6d"], + ["000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", [0, 0x4a000000], 1, + "224f51f3401bd9e12fde276fb8631ded8c131f823d2c06e27e4fcaec9ef3cf78" + "8a3b0aa372600a92b57974cded2b9334794cba40c63e34cdea212c4cf07d41b7"], + ["0000000000000000000000000000000000000000000000000000000000000001", [0, 0], 0, + "4540f05a9f1fb296d7736e7b208e3c96eb4fe1834688d2604f450952ed432d41" + "bbe2a0b6ea7566d2a5d1e7e20d42af2c53d792b1c43fea817e9ad275ae546963"], + ["0000000000000000000000000000000000000000000000000000000000000000", [0, 1], 0, + "ef3fdfd6c61578fbf5cf35bd3dd33b8009631634d21e42ac33960bd138e50d32" + "111e4caf237ee53ca8ad6426194a88545ddc497a0b466e7d6bbdb0041b2f586b"], + ["000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", [0, 0x0706050403020100], 0, + "f798a189f195e66982105ffb640bb7757f579da31602fc93ec01ac56f85ac3c1" + "34a4547b733b46413042c9440049176905d3be59ea1c53f15916155c2be8241a"], +] + +FSCHACHA20_TESTS = [ + ["000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", + "0000000000000000000000000000000000000000000000000000000000000000", 256, + "a93df4ef03011f3db95f60d996e1785df5de38fc39bfcb663a47bb5561928349"], + ["01", "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", 5, "ea"], + ["e93fdb5c762804b9a706816aca31e35b11d2aa3080108ef46a5b1f1508819c0a", + "8ec4c3ccdaea336bdeb245636970be01266509b33f3d2642504eaf412206207a", 4096, + "8bfaa4eacff308fdb4a94a5ff25bd9d0c1f84b77f81239f67ff39d6e1ac280c9"], +] + + +class TestFrameworkChacha(unittest.TestCase): + def test_chacha20(self): + """ChaCha20 test vectors.""" + for test_vector in CHACHA20_TESTS: + hex_key, nonce, counter, hex_output = test_vector + key = bytes.fromhex(hex_key) + nonce_bytes = nonce[0].to_bytes(4, 'little') + nonce[1].to_bytes(8, 'little') + keystream = chacha20_block(key, nonce_bytes, counter) + self.assertEqual(hex_output, keystream.hex()) + + def test_fschacha20(self): + """FSChaCha20 test vectors.""" + for test_vector in FSCHACHA20_TESTS: + hex_plaintext, hex_key, rekey_interval, hex_ciphertext_after_rotation = test_vector + plaintext = bytes.fromhex(hex_plaintext) + key = bytes.fromhex(hex_key) + fsc20 = FSChaCha20(key, rekey_interval) + for _ in range(rekey_interval): + fsc20.crypt(plaintext) + + ciphertext = fsc20.crypt(plaintext) + self.assertEqual(hex_ciphertext_after_rotation, ciphertext.hex()) diff --git a/test/functional/test_framework/ellswift.py b/test/functional/test_framework/crypto/ellswift.py index 97b10118e6..429b7b9f4d 100644 --- a/test/functional/test_framework/ellswift.py +++ b/test/functional/test_framework/crypto/ellswift.py @@ -12,7 +12,7 @@ import os import random import unittest -from test_framework.secp256k1 import FE, G, GE +from test_framework.crypto.secp256k1 import FE, G, GE # Precomputed constant square root of -3 (mod p). MINUS_3_SQRT = FE(-3).sqrt() diff --git a/test/functional/test_framework/ellswift_decode_test_vectors.csv b/test/functional/test_framework/crypto/ellswift_decode_test_vectors.csv index 1bab96b721..1bab96b721 100644 --- a/test/functional/test_framework/ellswift_decode_test_vectors.csv +++ b/test/functional/test_framework/crypto/ellswift_decode_test_vectors.csv diff --git a/test/functional/test_framework/crypto/hkdf.py b/test/functional/test_framework/crypto/hkdf.py new file mode 100644 index 0000000000..7e8958733c --- /dev/null +++ b/test/functional/test_framework/crypto/hkdf.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +# Copyright (c) 2023 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +"""Test-only HKDF-SHA256 implementation + +It is designed for ease of understanding, not performance. + +WARNING: This code is slow and trivially vulnerable to side channel attacks. Do not use for +anything but tests. +""" + +import hashlib +import hmac + + +def hmac_sha256(key, data): + """Compute HMAC-SHA256 from specified byte arrays key and data.""" + return hmac.new(key, data, hashlib.sha256).digest() + + +def hkdf_sha256(length, ikm, salt, info): + """Derive a key using HKDF-SHA256.""" + if len(salt) == 0: + salt = bytes([0] * 32) + prk = hmac_sha256(salt, ikm) + t = b"" + okm = b"" + for i in range((length + 32 - 1) // 32): + t = hmac_sha256(prk, t + info + bytes([i + 1])) + okm += t + return okm[:length] diff --git a/test/functional/test_framework/crypto/muhash.py b/test/functional/test_framework/crypto/muhash.py new file mode 100644 index 0000000000..09241f6203 --- /dev/null +++ b/test/functional/test_framework/crypto/muhash.py @@ -0,0 +1,55 @@ +# Copyright (c) 2020 Pieter Wuille +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Native Python MuHash3072 implementation.""" + +import hashlib +import unittest + +from .chacha20 import chacha20_block + +def data_to_num3072(data): + """Hash a 32-byte array data to a 3072-bit number using 6 Chacha20 operations.""" + bytes384 = b"" + for counter in range(6): + bytes384 += chacha20_block(data, bytes(12), counter) + return int.from_bytes(bytes384, 'little') + +class MuHash3072: + """Class representing the MuHash3072 computation of a set. + + See https://cseweb.ucsd.edu/~mihir/papers/inchash.pdf and https://lists.linuxfoundation.org/pipermail/bitcoin-dev/2017-May/014337.html + """ + + MODULUS = 2**3072 - 1103717 + + def __init__(self): + """Initialize for an empty set.""" + self.numerator = 1 + self.denominator = 1 + + def insert(self, data): + """Insert a byte array data in the set.""" + data_hash = hashlib.sha256(data).digest() + self.numerator = (self.numerator * data_to_num3072(data_hash)) % self.MODULUS + + def remove(self, data): + """Remove a byte array from the set.""" + data_hash = hashlib.sha256(data).digest() + self.denominator = (self.denominator * data_to_num3072(data_hash)) % self.MODULUS + + def digest(self): + """Extract the final hash. Does not modify this object.""" + val = (self.numerator * pow(self.denominator, -1, self.MODULUS)) % self.MODULUS + bytes384 = val.to_bytes(384, 'little') + return hashlib.sha256(bytes384).digest() + +class TestFrameworkMuhash(unittest.TestCase): + def test_muhash(self): + muhash = MuHash3072() + muhash.insert(b'\x00' * 32) + muhash.insert((b'\x01' + b'\x00' * 31)) + muhash.remove((b'\x02' + b'\x00' * 31)) + finalized = muhash.digest() + # This mirrors the result in the C++ MuHash3072 unit test + self.assertEqual(finalized[::-1].hex(), "10d312b100cbd32ada024a6646e40d3482fcff103668d2625f10002a607d5863") diff --git a/test/functional/test_framework/crypto/poly1305.py b/test/functional/test_framework/crypto/poly1305.py new file mode 100644 index 0000000000..967b90254d --- /dev/null +++ b/test/functional/test_framework/crypto/poly1305.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +"""Test-only implementation of Poly1305 authenticator + +It is designed for ease of understanding, not performance. + +WARNING: This code is slow and trivially vulnerable to side channel attacks. Do not use for +anything but tests. +""" + +import unittest + + +class Poly1305: + """Class representing a running poly1305 computation.""" + MODULUS = 2**130 - 5 + + def __init__(self, key): + self.r = int.from_bytes(key[:16], 'little') & 0xffffffc0ffffffc0ffffffc0fffffff + self.s = int.from_bytes(key[16:], 'little') + + def tag(self, data): + """Compute the poly1305 tag.""" + acc, length = 0, len(data) + for i in range((length + 15) // 16): + chunk = data[i * 16:min(length, (i + 1) * 16)] + val = int.from_bytes(chunk, 'little') + 256**len(chunk) + acc = (self.r * (acc + val)) % Poly1305.MODULUS + return ((acc + self.s) & 0xffffffffffffffffffffffffffffffff).to_bytes(16, 'little') + + +# Test vectors from RFC7539/8439 consisting of message to be authenticated, 32 byte key and computed 16 byte tag +POLY1305_TESTS = [ + # RFC 7539, section 2.5.2. + ["43727970746f6772617068696320466f72756d2052657365617263682047726f7570", + "85d6be7857556d337f4452fe42d506a80103808afb0db2fd4abff6af4149f51b", + "a8061dc1305136c6c22b8baf0c0127a9"], + # RFC 7539, section A.3. + ["00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + "000000000000000000000000000", + "0000000000000000000000000000000000000000000000000000000000000000", + "00000000000000000000000000000000"], + ["416e79207375626d697373696f6e20746f20746865204945544620696e74656e6465642062792074686520436f6e747269627" + "5746f7220666f72207075626c69636174696f6e20617320616c6c206f722070617274206f6620616e204945544620496e7465" + "726e65742d4472616674206f722052464320616e6420616e792073746174656d656e74206d6164652077697468696e2074686" + "520636f6e74657874206f6620616e204945544620616374697669747920697320636f6e7369646572656420616e2022494554" + "4620436f6e747269627574696f6e222e20537563682073746174656d656e747320696e636c756465206f72616c20737461746" + "56d656e747320696e20494554462073657373696f6e732c2061732077656c6c206173207772697474656e20616e6420656c65" + "6374726f6e696320636f6d6d756e69636174696f6e73206d61646520617420616e792074696d65206f7220706c6163652c207" + "768696368206172652061646472657373656420746f", + "0000000000000000000000000000000036e5f6b5c5e06070f0efca96227a863e", + "36e5f6b5c5e06070f0efca96227a863e"], + ["416e79207375626d697373696f6e20746f20746865204945544620696e74656e6465642062792074686520436f6e747269627" + "5746f7220666f72207075626c69636174696f6e20617320616c6c206f722070617274206f6620616e204945544620496e7465" + "726e65742d4472616674206f722052464320616e6420616e792073746174656d656e74206d6164652077697468696e2074686" + "520636f6e74657874206f6620616e204945544620616374697669747920697320636f6e7369646572656420616e2022494554" + "4620436f6e747269627574696f6e222e20537563682073746174656d656e747320696e636c756465206f72616c20737461746" + "56d656e747320696e20494554462073657373696f6e732c2061732077656c6c206173207772697474656e20616e6420656c65" + "6374726f6e696320636f6d6d756e69636174696f6e73206d61646520617420616e792074696d65206f7220706c6163652c207" + "768696368206172652061646472657373656420746f", + "36e5f6b5c5e06070f0efca96227a863e00000000000000000000000000000000", + "f3477e7cd95417af89a6b8794c310cf0"], + ["2754776173206272696c6c69672c20616e642074686520736c6974687920746f7665730a446964206779726520616e6420676" + "96d626c6520696e2074686520776162653a0a416c6c206d696d737920776572652074686520626f726f676f7665732c0a416e" + "6420746865206d6f6d65207261746873206f757467726162652e", + "1c9240a5eb55d38af333888604f6b5f0473917c1402b80099dca5cbc207075c0", + "4541669a7eaaee61e708dc7cbcc5eb62"], + ["ffffffffffffffffffffffffffffffff", + "0200000000000000000000000000000000000000000000000000000000000000", + "03000000000000000000000000000000"], + ["02000000000000000000000000000000", + "02000000000000000000000000000000ffffffffffffffffffffffffffffffff", + "03000000000000000000000000000000"], + ["fffffffffffffffffffffffffffffffff0ffffffffffffffffffffffffffffff11000000000000000000000000000000", + "0100000000000000000000000000000000000000000000000000000000000000", + "05000000000000000000000000000000"], + ["fffffffffffffffffffffffffffffffffbfefefefefefefefefefefefefefefe01010101010101010101010101010101", + "0100000000000000000000000000000000000000000000000000000000000000", + "00000000000000000000000000000000"], + ["fdffffffffffffffffffffffffffffff", + "0200000000000000000000000000000000000000000000000000000000000000", + "faffffffffffffffffffffffffffffff"], + ["e33594d7505e43b900000000000000003394d7505e4379cd01000000000000000000000000000000000000000000000001000000000000000000000000000000", + "0100000000000000040000000000000000000000000000000000000000000000", + "14000000000000005500000000000000"], + ["e33594d7505e43b900000000000000003394d7505e4379cd010000000000000000000000000000000000000000000000", + "0100000000000000040000000000000000000000000000000000000000000000", + "13000000000000000000000000000000"], +] + + +class TestFrameworkPoly1305(unittest.TestCase): + def test_poly1305(self): + """Poly1305 test vectors.""" + for test_vector in POLY1305_TESTS: + hex_message, hex_key, hex_tag = test_vector + message = bytes.fromhex(hex_message) + key = bytes.fromhex(hex_key) + tag = bytes.fromhex(hex_tag) + comp_tag = Poly1305(key).tag(message) + self.assertEqual(tag, comp_tag) diff --git a/test/functional/test_framework/ripemd160.py b/test/functional/test_framework/crypto/ripemd160.py index 12801364b4..12801364b4 100644 --- a/test/functional/test_framework/ripemd160.py +++ b/test/functional/test_framework/crypto/ripemd160.py diff --git a/test/functional/test_framework/secp256k1.py b/test/functional/test_framework/crypto/secp256k1.py index 2e9e419da5..2e9e419da5 100644 --- a/test/functional/test_framework/secp256k1.py +++ b/test/functional/test_framework/crypto/secp256k1.py diff --git a/test/functional/test_framework/siphash.py b/test/functional/test_framework/crypto/siphash.py index bd13b2c948..bd13b2c948 100644 --- a/test/functional/test_framework/siphash.py +++ b/test/functional/test_framework/crypto/siphash.py diff --git a/test/functional/test_framework/xswiftec_inv_test_vectors.csv b/test/functional/test_framework/crypto/xswiftec_inv_test_vectors.csv index 138c4cf85c..138c4cf85c 100644 --- a/test/functional/test_framework/xswiftec_inv_test_vectors.csv +++ b/test/functional/test_framework/crypto/xswiftec_inv_test_vectors.csv diff --git a/test/functional/test_framework/key.py b/test/functional/test_framework/key.py index 6c1892539f..06252f8996 100644 --- a/test/functional/test_framework/key.py +++ b/test/functional/test_framework/key.py @@ -13,7 +13,7 @@ import os import random import unittest -from test_framework import secp256k1 +from test_framework.crypto import secp256k1 # Point with no known discrete log. H_POINT = "50929b74c1a04954b78b4b6035e97a5e078a5a0f28ec96d547bfee9ace803ac0" diff --git a/test/functional/test_framework/messages.py b/test/functional/test_framework/messages.py index 8f3aea8785..d008cb39aa 100755 --- a/test/functional/test_framework/messages.py +++ b/test/functional/test_framework/messages.py @@ -29,7 +29,7 @@ import struct import time import unittest -from test_framework.siphash import siphash256 +from test_framework.crypto.siphash import siphash256 from test_framework.util import assert_equal MAX_LOCATOR_SZ = 101 diff --git a/test/functional/test_framework/muhash.py b/test/functional/test_framework/muhash.py deleted file mode 100644 index 0d96114e3e..0000000000 --- a/test/functional/test_framework/muhash.py +++ /dev/null @@ -1,110 +0,0 @@ -# Copyright (c) 2020 Pieter Wuille -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Native Python MuHash3072 implementation.""" - -import hashlib -import unittest - -def rot32(v, bits): - """Rotate the 32-bit value v left by bits bits.""" - bits %= 32 # Make sure the term below does not throw an exception - return ((v << bits) & 0xffffffff) | (v >> (32 - bits)) - -def chacha20_doubleround(s): - """Apply a ChaCha20 double round to 16-element state array s. - - See https://cr.yp.to/chacha/chacha-20080128.pdf and https://tools.ietf.org/html/rfc8439 - """ - QUARTER_ROUNDS = [(0, 4, 8, 12), - (1, 5, 9, 13), - (2, 6, 10, 14), - (3, 7, 11, 15), - (0, 5, 10, 15), - (1, 6, 11, 12), - (2, 7, 8, 13), - (3, 4, 9, 14)] - - for a, b, c, d in QUARTER_ROUNDS: - s[a] = (s[a] + s[b]) & 0xffffffff - s[d] = rot32(s[d] ^ s[a], 16) - s[c] = (s[c] + s[d]) & 0xffffffff - s[b] = rot32(s[b] ^ s[c], 12) - s[a] = (s[a] + s[b]) & 0xffffffff - s[d] = rot32(s[d] ^ s[a], 8) - s[c] = (s[c] + s[d]) & 0xffffffff - s[b] = rot32(s[b] ^ s[c], 7) - -def chacha20_32_to_384(key32): - """Specialized ChaCha20 implementation with 32-byte key, 0 IV, 384-byte output.""" - # See RFC 8439 section 2.3 for chacha20 parameters - CONSTANTS = [0x61707865, 0x3320646e, 0x79622d32, 0x6b206574] - - key_bytes = [0]*8 - for i in range(8): - key_bytes[i] = int.from_bytes(key32[(4 * i):(4 * (i+1))], 'little') - - INITIALIZATION_VECTOR = [0] * 4 - init = CONSTANTS + key_bytes + INITIALIZATION_VECTOR - out = bytearray() - for counter in range(6): - init[12] = counter - s = init.copy() - for _ in range(10): - chacha20_doubleround(s) - for i in range(16): - out.extend(((s[i] + init[i]) & 0xffffffff).to_bytes(4, 'little')) - return bytes(out) - -def data_to_num3072(data): - """Hash a 32-byte array data to a 3072-bit number using 6 Chacha20 operations.""" - bytes384 = chacha20_32_to_384(data) - return int.from_bytes(bytes384, 'little') - -class MuHash3072: - """Class representing the MuHash3072 computation of a set. - - See https://cseweb.ucsd.edu/~mihir/papers/inchash.pdf and https://lists.linuxfoundation.org/pipermail/bitcoin-dev/2017-May/014337.html - """ - - MODULUS = 2**3072 - 1103717 - - def __init__(self): - """Initialize for an empty set.""" - self.numerator = 1 - self.denominator = 1 - - def insert(self, data): - """Insert a byte array data in the set.""" - data_hash = hashlib.sha256(data).digest() - self.numerator = (self.numerator * data_to_num3072(data_hash)) % self.MODULUS - - def remove(self, data): - """Remove a byte array from the set.""" - data_hash = hashlib.sha256(data).digest() - self.denominator = (self.denominator * data_to_num3072(data_hash)) % self.MODULUS - - def digest(self): - """Extract the final hash. Does not modify this object.""" - val = (self.numerator * pow(self.denominator, -1, self.MODULUS)) % self.MODULUS - bytes384 = val.to_bytes(384, 'little') - return hashlib.sha256(bytes384).digest() - -class TestFrameworkMuhash(unittest.TestCase): - def test_muhash(self): - muhash = MuHash3072() - muhash.insert(b'\x00' * 32) - muhash.insert((b'\x01' + b'\x00' * 31)) - muhash.remove((b'\x02' + b'\x00' * 31)) - finalized = muhash.digest() - # This mirrors the result in the C++ MuHash3072 unit test - self.assertEqual(finalized[::-1].hex(), "10d312b100cbd32ada024a6646e40d3482fcff103668d2625f10002a607d5863") - - def test_chacha20(self): - def chacha_check(key, result): - self.assertEqual(chacha20_32_to_384(key)[:64].hex(), result) - - # Test vectors from https://tools.ietf.org/html/draft-agl-tls-chacha20poly1305-04#section-7 - # Since the nonce is hardcoded to 0 in our function we only use those vectors. - chacha_check([0]*32, "76b8e0ada0f13d90405d6ae55386bd28bdd219b8a08ded1aa836efcc8b770dc7da41597c5157488d7724e03fb8d84a376a43b8f41518a11cc387b669b2ee6586") - chacha_check([0]*31 + [1], "4540f05a9f1fb296d7736e7b208e3c96eb4fe1834688d2604f450952ed432d41bbe2a0b6ea7566d2a5d1e7e20d42af2c53d792b1c43fea817e9ad275ae546963") diff --git a/test/functional/test_framework/p2p.py b/test/functional/test_framework/p2p.py index 352becb367..34fe467d23 100755 --- a/test/functional/test_framework/p2p.py +++ b/test/functional/test_framework/p2p.py @@ -24,6 +24,7 @@ import asyncio from collections import defaultdict from io import BytesIO import logging +import platform import struct import sys import threading @@ -364,8 +365,8 @@ class P2PInterface(P2PConnection): vt.addrFrom.port = 0 self.on_connection_send_msg = vt # Will be sent in connection_made callback - def peer_connect(self, *args, services=P2P_SERVICES, send_version=True, **kwargs): - create_conn = super().peer_connect(*args, **kwargs) + def peer_connect(self, *, services=P2P_SERVICES, send_version, **kwargs): + create_conn = super().peer_connect(**kwargs) if send_version: self.peer_connect_send_version(services) @@ -592,7 +593,7 @@ class NetworkThread(threading.Thread): NetworkThread.listeners = {} NetworkThread.protos = {} - if sys.platform == 'win32': + if platform.system() == 'Windows': asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) NetworkThread.network_event_loop = asyncio.new_event_loop() diff --git a/test/functional/test_framework/script.py b/test/functional/test_framework/script.py index 17a954cb22..78d8580794 100644 --- a/test/functional/test_framework/script.py +++ b/test/functional/test_framework/script.py @@ -10,7 +10,6 @@ This file is modified from python-bitcoinlib. from collections import namedtuple import struct import unittest -from typing import List, Dict from .key import TaggedHash, tweak_add_pubkey, compute_xonly_pubkey @@ -24,7 +23,7 @@ from .messages import ( uint256_from_str, ) -from .ripemd160 import ripemd160 +from .crypto.ripemd160 import ripemd160 MAX_SCRIPT_ELEMENT_SIZE = 520 MAX_PUBKEYS_PER_MULTI_A = 999 @@ -110,8 +109,8 @@ class CScriptOp(int): _opcode_instances.append(super().__new__(cls, n)) return _opcode_instances[n] -OPCODE_NAMES: Dict[CScriptOp, str] = {} -_opcode_instances: List[CScriptOp] = [] +OPCODE_NAMES: dict[CScriptOp, str] = {} +_opcode_instances: list[CScriptOp] = [] # Populate opcode instance table for n in range(0xff + 1): diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index 70b3943478..0ee332b75b 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -19,7 +19,6 @@ import sys import tempfile import time -from typing import List from .address import create_deterministic_address_bcrt1_p2tr_op_true from .authproxy import JSONRPCException from . import coverage @@ -97,7 +96,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): """Sets test framework defaults. Do not override this method. Instead, override the set_test_params() method""" self.chain: str = 'regtest' self.setup_clean_chain: bool = False - self.nodes: List[TestNode] = [] + self.nodes: list[TestNode] = [] self.extra_args = None self.network_thread = None self.rpc_timeout = 60 # Wait for up to 60 seconds for the RPC server to respond @@ -508,8 +507,6 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): assert_equal(len(binary_cli), num_nodes) for i in range(num_nodes): args = list(extra_args[i]) - if self.options.v2transport and ("-v2transport=0" not in args): - args.append("-v2transport=1") test_node_i = TestNode( i, get_datadir_path(self.options.tmpdir, i), @@ -528,6 +525,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): start_perf=self.options.perf, use_valgrind=self.options.valgrind, descriptors=self.options.descriptors, + v2transport=self.options.v2transport, ) self.nodes.append(test_node_i) if not test_node_i.version_is_at_least(170000): @@ -602,7 +600,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): ip_port = "127.0.0.1:" + str(p2p_port(b)) if peer_advertises_v2 is None: - peer_advertises_v2 = self.options.v2transport + peer_advertises_v2 = from_connection.use_v2transport if peer_advertises_v2: from_connection.addnode(node=ip_port, command="onetry", v2transport=True) diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index b6af71d85c..77f6e69e98 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -12,6 +12,7 @@ import http.client import json import logging import os +import platform import re import subprocess import tempfile @@ -19,7 +20,6 @@ import time import urllib.parse import collections import shlex -import sys from pathlib import Path from .authproxy import ( @@ -67,7 +67,7 @@ class TestNode(): To make things easier for the test writer, any unrecognised messages will be dispatched to the RPC connection.""" - def __init__(self, i, datadir_path, *, chain, rpchost, timewait, timeout_factor, bitcoind, bitcoin_cli, coverage_dir, cwd, extra_conf=None, extra_args=None, use_cli=False, start_perf=False, use_valgrind=False, version=None, descriptors=False): + def __init__(self, i, datadir_path, *, chain, rpchost, timewait, timeout_factor, bitcoind, bitcoin_cli, coverage_dir, cwd, extra_conf=None, extra_args=None, use_cli=False, start_perf=False, use_valgrind=False, version=None, descriptors=False, v2transport=False): """ Kwargs: start_perf (bool): If True, begin profiling the node with `perf` as soon as @@ -126,6 +126,12 @@ class TestNode(): if self.version_is_at_least(239000): self.args.append("-loglevel=trace") + # Default behavior from global -v2transport flag is added to args to persist it over restarts. + # May be overwritten in individual tests, using extra_args. + self.default_to_v2 = v2transport + if self.default_to_v2: + self.args.append("-v2transport=1") + self.cli = TestNodeCLI(bitcoin_cli, self.datadir_path) self.use_cli = use_cli self.start_perf = start_perf @@ -198,6 +204,8 @@ class TestNode(): if extra_args is None: extra_args = self.extra_args + self.use_v2transport = "-v2transport=1" in extra_args or (self.default_to_v2 and "-v2transport=0" not in extra_args) + # Add a new stdout and stderr file each time bitcoind is started if stderr is None: stderr = tempfile.NamedTemporaryFile(dir=self.stderr_dir, delete=False) @@ -252,7 +260,7 @@ class TestNode(): if self.version_is_at_least(190000): # getmempoolinfo.loaded is available since commit # bb8ae2c (version 0.19.0) - wait_until_helper_internal(lambda: rpc.getmempoolinfo()['loaded'], timeout_factor=self.timeout_factor) + self.wait_until(lambda: rpc.getmempoolinfo()['loaded']) # Wait for the node to finish reindex, block import, and # loading the mempool. Usually importing happens fast or # even "immediate" when the node is started. However, there @@ -406,7 +414,7 @@ class TestNode(): def wait_until_stopped(self, *, timeout=BITCOIND_PROC_WAIT_TIMEOUT, expect_error=False, **kwargs): expected_ret_code = 1 if expect_error else 0 # Whether node shutdown return EXIT_FAILURE or EXIT_SUCCESS - wait_until_helper_internal(lambda: self.is_node_stopped(expected_ret_code=expected_ret_code, **kwargs), timeout=timeout, timeout_factor=self.timeout_factor) + self.wait_until(lambda: self.is_node_stopped(expected_ret_code=expected_ret_code, **kwargs), timeout=timeout) def replace_in_config(self, replacements): """ @@ -512,6 +520,23 @@ class TestNode(): str(expected_msgs), print_log)) @contextlib.contextmanager + def wait_for_new_peer(self, timeout=5): + """ + Wait until the node is connected to at least one new peer. We detect this + by watching for an increased highest peer id, using the `getpeerinfo` RPC call. + Note that the simpler approach of only accounting for the number of peers + suffers from race conditions, as disconnects from unrelated previous peers + could happen anytime in-between. + """ + def get_highest_peer_id(): + peer_info = self.getpeerinfo() + return peer_info[-1]["id"] if peer_info else -1 + + initial_peer_id = get_highest_peer_id() + yield + self.wait_until(lambda: get_highest_peer_id() > initial_peer_id, timeout=timeout) + + @contextlib.contextmanager def profile_with_perf(self, profile_name: str): """ Context manager that allows easy profiling of node activity using `perf`. @@ -541,7 +566,7 @@ class TestNode(): cmd, shell=True, stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL) == 0 - if not sys.platform.startswith('linux'): + if platform.system() != 'Linux': self.log.warning("Can't profile with perf; only available on Linux platforms") return None @@ -634,7 +659,7 @@ class TestNode(): assert_msg += "with expected error " + expected_msg self._raise_assertion_error(assert_msg) - def add_p2p_connection(self, p2p_conn, *, wait_for_verack=True, **kwargs): + def add_p2p_connection(self, p2p_conn, *, wait_for_verack=True, send_version=True, **kwargs): """Add an inbound p2p connection to the node. This method adds the p2p connection to the self.p2ps list and also @@ -645,9 +670,11 @@ class TestNode(): kwargs['dstaddr'] = '127.0.0.1' p2p_conn.p2p_connected_to_node = True - p2p_conn.peer_connect(**kwargs, net=self.chain, timeout_factor=self.timeout_factor)() + p2p_conn.peer_connect(**kwargs, send_version=send_version, net=self.chain, timeout_factor=self.timeout_factor)() self.p2ps.append(p2p_conn) p2p_conn.wait_until(lambda: p2p_conn.is_connected, check_connected=False) + if send_version: + p2p_conn.wait_until(lambda: not p2p_conn.on_connection_send_msg) if wait_for_verack: # Wait for the node to send us the version and verack p2p_conn.wait_for_verack() @@ -701,6 +728,7 @@ class TestNode(): p2p_conn.wait_for_connect() self.p2ps.append(p2p_conn) + p2p_conn.wait_until(lambda: not p2p_conn.on_connection_send_msg) if wait_for_verack: p2p_conn.wait_for_verack() p2p_conn.sync_with_ping() @@ -718,7 +746,7 @@ class TestNode(): p.peer_disconnect() del self.p2ps[:] - wait_until_helper_internal(lambda: self.num_test_p2p_connections() == 0, timeout_factor=self.timeout_factor) + self.wait_until(lambda: self.num_test_p2p_connections() == 0) def bumpmocktime(self, seconds): """Fast forward using setmocktime to self.mocktime + seconds. Requires setmocktime to have @@ -727,6 +755,9 @@ class TestNode(): self.mocktime += seconds self.setmocktime(self.mocktime) + def wait_until(self, test_function, timeout=60): + return wait_until_helper_internal(test_function, timeout=timeout, timeout_factor=self.timeout_factor) + class TestNodeCLIAttr: def __init__(self, cli, command): @@ -779,15 +810,15 @@ class TestNodeCLI(): results.append(dict(error=e)) return results - def send_cli(self, command=None, *args, **kwargs): + def send_cli(self, clicommand=None, *args, **kwargs): """Run bitcoin-cli command. Deserializes returned string as python object.""" pos_args = [arg_to_cli(arg) for arg in args] named_args = [str(key) + "=" + arg_to_cli(value) for (key, value) in kwargs.items()] p_args = [self.binary, f"-datadir={self.datadir}"] + self.options if named_args: p_args += ["-named"] - if command is not None: - p_args += [command] + if clicommand is not None: + p_args += [clicommand] p_args += pos_args + named_args self.log.debug("Running bitcoin-cli {}".format(p_args[2:])) process = subprocess.Popen(p_args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index 104f158d92..b4b05b1597 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -13,13 +13,14 @@ import json import logging import os import pathlib +import platform import re -import sys import time from . import coverage from .authproxy import AuthServiceProxy, JSONRPCException -from typing import Callable, Optional, Tuple +from collections.abc import Callable +from typing import Optional logger = logging.getLogger("TestFramework.utils") @@ -409,16 +410,16 @@ def get_datadir_path(dirname, n): return pathlib.Path(dirname) / f"node{n}" -def get_temp_default_datadir(temp_dir: pathlib.Path) -> Tuple[dict, pathlib.Path]: +def get_temp_default_datadir(temp_dir: pathlib.Path) -> tuple[dict, pathlib.Path]: """Return os-specific environment variables that can be set to make the GetDefaultDataDir() function return a datadir path under the provided temp_dir, as well as the complete path it would return.""" - if sys.platform == "win32": + if platform.system() == "Windows": env = dict(APPDATA=str(temp_dir)) datadir = temp_dir / "Bitcoin" else: env = dict(HOME=str(temp_dir)) - if sys.platform == "darwin": + if platform.system() == "Darwin": datadir = temp_dir / "Library/Application Support/Bitcoin" else: datadir = temp_dir / ".bitcoin" diff --git a/test/functional/test_framework/wallet.py b/test/functional/test_framework/wallet.py index 035a482f4c..53c8e1b0cc 100644 --- a/test/functional/test_framework/wallet.py +++ b/test/functional/test_framework/wallet.py @@ -9,7 +9,6 @@ from decimal import Decimal from enum import Enum from typing import ( Any, - List, Optional, ) from test_framework.address import ( @@ -284,7 +283,7 @@ class MiniWallet: def create_self_transfer_multi( self, *, - utxos_to_spend: Optional[List[dict]] = None, + utxos_to_spend: Optional[list[dict]] = None, num_outputs=1, amount_per_output=0, locktime=0, diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 2b6be93bdf..826705598f 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -17,6 +17,7 @@ from collections import deque import configparser import datetime import os +import platform import time import shutil import signal @@ -42,8 +43,8 @@ except UnicodeDecodeError: CROSS = "x " CIRCLE = "o " -if os.name != 'nt' or sys.getwindowsversion() >= (10, 0, 14393): #type:ignore - if os.name == 'nt': +if platform.system() != 'Windows' or sys.getwindowsversion() >= (10, 0, 14393): #type:ignore + if platform.system() == 'Windows': import ctypes kernel32 = ctypes.windll.kernel32 # type: ignore ENABLE_VIRTUAL_TERMINAL_PROCESSING = 4 @@ -73,12 +74,15 @@ TEST_EXIT_SKIPPED = 77 # the output of `git grep unittest.TestCase ./test/functional/test_framework` TEST_FRAMEWORK_MODULES = [ "address", + "crypto.bip324_cipher", "blocktools", - "ellswift", + "crypto.chacha20", + "crypto.ellswift", "key", "messages", - "muhash", - "ripemd160", + "crypto.muhash", + "crypto.poly1305", + "crypto.ripemd160", "script", "segwit_addr", ] @@ -153,6 +157,7 @@ BASE_SCRIPTS = [ 'p2p_invalid_messages.py', 'rpc_createmultisig.py', 'p2p_timeouts.py', + 'p2p_timeouts.py --v2transport', 'wallet_dump.py --legacy-wallet', 'rpc_signer.py', 'wallet_signer.py --descriptors', @@ -203,6 +208,8 @@ BASE_SCRIPTS = [ 'wallet_createwallet.py --descriptors', 'wallet_watchonly.py --legacy-wallet', 'wallet_watchonly.py --usecli --legacy-wallet', + 'wallet_reindex.py --legacy-wallet', + 'wallet_reindex.py --descriptors', 'wallet_reorgsrestore.py', 'wallet_conflicts.py --legacy-wallet', 'wallet_conflicts.py --descriptors', @@ -240,6 +247,7 @@ BASE_SCRIPTS = [ 'p2p_getdata.py', 'p2p_addrfetch.py', 'rpc_net.py', + 'rpc_net.py --v2transport', 'wallet_keypool.py --legacy-wallet', 'wallet_keypool.py --descriptors', 'wallet_descriptor.py --descriptors', @@ -336,6 +344,7 @@ BASE_SCRIPTS = [ 'feature_filelock.py', 'feature_loadblock.py', 'feature_assumeutxo.py', + 'wallet_assumeutxo.py --descriptors', 'p2p_dos_header_tree.py', 'p2p_add_connections.py', 'feature_bind_port_discover.py', @@ -365,6 +374,7 @@ BASE_SCRIPTS = [ 'wallet_orphanedreward.py', 'wallet_timelock.py', 'p2p_node_network_limited.py', + 'p2p_node_network_limited.py --v2transport', 'p2p_permissions.py', 'feature_blocksdir.py', 'wallet_startup.py', @@ -559,8 +569,7 @@ def run_tests(*, test_list, src_dir, build_dir, tmpdir, jobs=1, enable_coverage= test_framework_tests.addTest(unittest.TestLoader().loadTestsFromName("test_framework.{}".format(module))) result = unittest.TextTestRunner(verbosity=1, failfast=True).run(test_framework_tests) if not result.wasSuccessful(): - logging.debug("Early exiting after failure in TestFramework unit tests") - sys.exit(False) + sys.exit("Early exiting after failure in TestFramework unit tests") flags = ['--cachedir={}'.format(cache_dir)] + args diff --git a/test/functional/wallet_assumeutxo.py b/test/functional/wallet_assumeutxo.py new file mode 100755 index 0000000000..fce4868293 --- /dev/null +++ b/test/functional/wallet_assumeutxo.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 +# Copyright (c) 2023-present The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test for assumeutxo wallet related behavior. +See feature_assumeutxo.py for background. + +## Possible test improvements + +- TODO: test import descriptors while background sync is in progress +- TODO: test loading a wallet (backup) on a pruned node + +""" +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + assert_raises_rpc_error, +) + +START_HEIGHT = 199 +SNAPSHOT_BASE_HEIGHT = 299 +FINAL_HEIGHT = 399 + + +class AssumeutxoTest(BitcoinTestFramework): + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + + def add_options(self, parser): + self.add_wallet_options(parser, legacy=False) + + def set_test_params(self): + """Use the pregenerated, deterministic chain up to height 199.""" + self.num_nodes = 2 + self.rpc_timeout = 120 + self.extra_args = [ + [], + [], + ] + + def setup_network(self): + """Start with the nodes disconnected so that one can generate a snapshot + including blocks the other hasn't yet seen.""" + self.add_nodes(2) + self.start_nodes(extra_args=self.extra_args) + + def run_test(self): + """ + Bring up two (disconnected) nodes, mine some new blocks on the first, + and generate a UTXO snapshot. + + Load the snapshot into the second, ensure it syncs to tip and completes + background validation when connected to the first. + """ + n0 = self.nodes[0] + n1 = self.nodes[1] + + # Mock time for a deterministic chain + for n in self.nodes: + n.setmocktime(n.getblockheader(n.getbestblockhash())['time']) + + self.sync_blocks() + + n0.createwallet('w') + w = n0.get_wallet_rpc("w") + + # Generate a series of blocks that `n0` will have in the snapshot, + # but that n1 doesn't yet see. In order for the snapshot to activate, + # though, we have to ferry over the new headers to n1 so that it + # isn't waiting forever to see the header of the snapshot's base block + # while disconnected from n0. + for _ in range(100): + self.generate(n0, nblocks=1, sync_fun=self.no_op) + newblock = n0.getblock(n0.getbestblockhash(), 0) + + # make n1 aware of the new header, but don't give it the block. + n1.submitheader(newblock) + + # Ensure everyone is seeing the same headers. + for n in self.nodes: + assert_equal(n.getblockchaininfo()[ + "headers"], SNAPSHOT_BASE_HEIGHT) + + w.backupwallet("backup_w.dat") + + self.log.info("-- Testing assumeutxo") + + assert_equal(n0.getblockcount(), SNAPSHOT_BASE_HEIGHT) + assert_equal(n1.getblockcount(), START_HEIGHT) + + self.log.info( + f"Creating a UTXO snapshot at height {SNAPSHOT_BASE_HEIGHT}") + dump_output = n0.dumptxoutset('utxos.dat') + + assert_equal( + dump_output['txoutset_hash'], + '61d9c2b29a2571a5fe285fe2d8554f91f93309666fc9b8223ee96338de25ff53') + assert_equal(dump_output['nchaintx'], 300) + assert_equal(n0.getblockchaininfo()["blocks"], SNAPSHOT_BASE_HEIGHT) + + # Mine more blocks on top of the snapshot that n1 hasn't yet seen. This + # will allow us to test n1's sync-to-tip on top of a snapshot. + self.generate(n0, nblocks=100, sync_fun=self.no_op) + + assert_equal(n0.getblockcount(), FINAL_HEIGHT) + assert_equal(n1.getblockcount(), START_HEIGHT) + + assert_equal(n0.getblockchaininfo()["blocks"], FINAL_HEIGHT) + + self.log.info( + f"Loading snapshot into second node from {dump_output['path']}") + loaded = n1.loadtxoutset(dump_output['path']) + assert_equal(loaded['coins_loaded'], SNAPSHOT_BASE_HEIGHT) + assert_equal(loaded['base_height'], SNAPSHOT_BASE_HEIGHT) + + normal, snapshot = n1.getchainstates()["chainstates"] + assert_equal(normal['blocks'], START_HEIGHT) + assert_equal(normal.get('snapshot_blockhash'), None) + assert_equal(normal['validated'], True) + assert_equal(snapshot['blocks'], SNAPSHOT_BASE_HEIGHT) + assert_equal(snapshot['snapshot_blockhash'], dump_output['base_hash']) + assert_equal(snapshot['validated'], False) + + assert_equal(n1.getblockchaininfo()["blocks"], SNAPSHOT_BASE_HEIGHT) + + self.log.info("Backup can't be loaded during background sync") + assert_raises_rpc_error(-4, "Wallet loading failed. Error loading wallet. Wallet requires blocks to be downloaded, and software does not currently support loading wallets while blocks are being downloaded out of order when using assumeutxo snapshots. Wallet should be able to load successfully after node sync reaches height 299", n1.restorewallet, "w", "backup_w.dat") + + PAUSE_HEIGHT = FINAL_HEIGHT - 40 + + self.log.info("Restarting node to stop at height %d", PAUSE_HEIGHT) + self.restart_node(1, extra_args=[ + f"-stopatheight={PAUSE_HEIGHT}", *self.extra_args[1]]) + + # Finally connect the nodes and let them sync. + # + # Set `wait_for_connect=False` to avoid a race between performing connection + # assertions and the -stopatheight tripping. + self.connect_nodes(0, 1, wait_for_connect=False) + + n1.wait_until_stopped(timeout=5) + + self.log.info( + "Restarted node before snapshot validation completed, reloading...") + self.restart_node(1, extra_args=self.extra_args[1]) + + # TODO: inspect state of e.g. the wallet before reconnecting + self.connect_nodes(0, 1) + + self.log.info( + f"Ensuring snapshot chain syncs to tip. ({FINAL_HEIGHT})") + self.wait_until(lambda: n1.getchainstates()[ + 'chainstates'][-1]['blocks'] == FINAL_HEIGHT) + self.sync_blocks(nodes=(n0, n1)) + + self.log.info("Ensuring background validation completes") + self.wait_until(lambda: len(n1.getchainstates()['chainstates']) == 1) + + self.log.info("Ensuring wallet can be restored from backup") + n1.restorewallet("w", "backup_w.dat") + + +if __name__ == '__main__': + AssumeutxoTest().main() diff --git a/test/functional/wallet_avoidreuse.py b/test/functional/wallet_avoidreuse.py index c257bda452..9d3c55d6b6 100755 --- a/test/functional/wallet_avoidreuse.py +++ b/test/functional/wallet_avoidreuse.py @@ -257,7 +257,7 @@ class AvoidReuseTest(BitcoinTestFramework): if not self.options.descriptors: # For the second send, we transmute it to a related single-key address - # to make sure it's also detected as re-use + # to make sure it's also detected as reuse fund_spk = address_to_scriptpubkey(fundaddr).hex() fund_decoded = self.nodes[0].decodescript(fund_spk) if second_addr_type == "p2sh-segwit": diff --git a/test/functional/wallet_basic.py b/test/functional/wallet_basic.py index 78bfa97212..f798eee365 100755 --- a/test/functional/wallet_basic.py +++ b/test/functional/wallet_basic.py @@ -639,7 +639,7 @@ class WalletTest(BitcoinTestFramework): node0_balance = self.nodes[0].getbalance() # With walletrejectlongchains we will not create the tx and store it in our wallet. - assert_raises_rpc_error(-6, "Transaction has too long of a mempool chain", self.nodes[0].sendtoaddress, sending_addr, node0_balance - Decimal('0.01')) + assert_raises_rpc_error(-6, f"too many unconfirmed ancestors [limit: {chainlimit * 2}]", self.nodes[0].sendtoaddress, sending_addr, node0_balance - Decimal('0.01')) # Verify nothing new in wallet assert_equal(total_txs, len(self.nodes[0].listtransactions("*", 99999))) diff --git a/test/functional/wallet_fast_rescan.py b/test/functional/wallet_fast_rescan.py index 2f9c924e71..1315bccafd 100755 --- a/test/functional/wallet_fast_rescan.py +++ b/test/functional/wallet_fast_rescan.py @@ -4,8 +4,6 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test that fast rescan using block filters for descriptor wallets detects top-ups correctly and finds the same transactions than the slow variant.""" -from typing import List - from test_framework.address import address_to_scriptpubkey from test_framework.descriptors import descsum_create from test_framework.test_framework import BitcoinTestFramework @@ -32,7 +30,7 @@ class WalletFastRescanTest(BitcoinTestFramework): self.skip_if_no_wallet() self.skip_if_no_sqlite() - def get_wallet_txids(self, node: TestNode, wallet_name: str) -> List[str]: + def get_wallet_txids(self, node: TestNode, wallet_name: str) -> list[str]: w = node.get_wallet_rpc(wallet_name) txs = w.listtransactions('*', 1000000) return [tx['txid'] for tx in txs] diff --git a/test/functional/wallet_listtransactions.py b/test/functional/wallet_listtransactions.py index 18bb8a0cd8..064ce12108 100755 --- a/test/functional/wallet_listtransactions.py +++ b/test/functional/wallet_listtransactions.py @@ -111,6 +111,7 @@ class ListTransactionsTest(BitcoinTestFramework): self.run_rbf_opt_in_test() self.run_externally_generated_address_test() + self.run_coinjoin_test() self.run_invalid_parameters_test() self.test_op_return() @@ -281,6 +282,34 @@ class ListTransactionsTest(BitcoinTestFramework): assert_equal(['pizza2'], self.nodes[0].getaddressinfo(addr2)['labels']) assert_equal(['pizza3'], self.nodes[0].getaddressinfo(addr3)['labels']) + def run_coinjoin_test(self): + self.log.info('Check "coin-join" transaction') + input_0 = next(i for i in self.nodes[0].listunspent(query_options={"minimumAmount": 0.2}, include_unsafe=False)) + input_1 = next(i for i in self.nodes[1].listunspent(query_options={"minimumAmount": 0.2}, include_unsafe=False)) + raw_hex = self.nodes[0].createrawtransaction( + inputs=[ + { + "txid": input_0["txid"], + "vout": input_0["vout"], + }, + { + "txid": input_1["txid"], + "vout": input_1["vout"], + }, + ], + outputs={ + self.nodes[0].getnewaddress(): 0.123, + self.nodes[1].getnewaddress(): 0.123, + }, + ) + raw_hex = self.nodes[0].signrawtransactionwithwallet(raw_hex)["hex"] + raw_hex = self.nodes[1].signrawtransactionwithwallet(raw_hex)["hex"] + txid_join = self.nodes[0].sendrawtransaction(hexstring=raw_hex, maxfeerate=0) + fee_join = self.nodes[0].getmempoolentry(txid_join)["fees"]["base"] + # Fee should be correct: assert_equal(fee_join, self.nodes[0].gettransaction(txid_join)['fee']) + # But it is not, see for example https://github.com/bitcoin/bitcoin/issues/14136: + assert fee_join != self.nodes[0].gettransaction(txid_join)["fee"] + def run_invalid_parameters_test(self): self.log.info("Test listtransactions RPC parameter validity") assert_raises_rpc_error(-8, 'Label argument must be a valid label name or "*".', self.nodes[0].listtransactions, label="") diff --git a/test/functional/wallet_migration.py b/test/functional/wallet_migration.py index e2edaef4da..3d68dbe07a 100755 --- a/test/functional/wallet_migration.py +++ b/test/functional/wallet_migration.py @@ -877,6 +877,95 @@ class WalletMigrationTest(BitcoinTestFramework): assert_equal(magic, BTREE_MAGIC) + def test_avoidreuse(self): + self.log.info("Test that avoidreuse persists after migration") + def_wallet = self.nodes[0].get_wallet_rpc(self.default_wallet_name) + + wallet = self.create_legacy_wallet("avoidreuse") + wallet.setwalletflag("avoid_reuse", True) + + # Import a pubkey to the test wallet and send some funds to it + reused_imported_addr = def_wallet.getnewaddress() + wallet.importpubkey(def_wallet.getaddressinfo(reused_imported_addr)["pubkey"]) + imported_utxos = self.create_outpoints(def_wallet, outputs=[{reused_imported_addr: 2}]) + def_wallet.lockunspent(False, imported_utxos) + + # Send funds to the test wallet + reused_addr = wallet.getnewaddress() + def_wallet.sendtoaddress(reused_addr, 2) + + self.generate(self.nodes[0], 1) + + # Send funds from the test wallet with both its own and the imported + wallet.sendall([def_wallet.getnewaddress()]) + def_wallet.sendall(recipients=[def_wallet.getnewaddress()], inputs=imported_utxos) + self.generate(self.nodes[0], 1) + balances = wallet.getbalances() + assert_equal(balances["mine"]["trusted"], 0) + assert_equal(balances["watchonly"]["trusted"], 0) + + # Reuse the addresses + def_wallet.sendtoaddress(reused_addr, 1) + def_wallet.sendtoaddress(reused_imported_addr, 1) + self.generate(self.nodes[0], 1) + balances = wallet.getbalances() + assert_equal(balances["mine"]["used"], 1) + # Reused watchonly will not show up in balances + assert_equal(balances["watchonly"]["trusted"], 0) + assert_equal(balances["watchonly"]["untrusted_pending"], 0) + assert_equal(balances["watchonly"]["immature"], 0) + + utxos = wallet.listunspent() + assert_equal(len(utxos), 2) + for utxo in utxos: + assert_equal(utxo["reused"], True) + + # Migrate + migrate_res = wallet.migratewallet() + watchonly_wallet = self.nodes[0].get_wallet_rpc(migrate_res["watchonly_name"]) + + # One utxo in each wallet, marked used + utxos = wallet.listunspent() + assert_equal(len(utxos), 1) + assert_equal(utxos[0]["reused"], True) + watchonly_utxos = watchonly_wallet.listunspent() + assert_equal(len(watchonly_utxos), 1) + assert_equal(watchonly_utxos[0]["reused"], True) + + def test_preserve_tx_extra_info(self): + self.log.info("Test that tx extra data is preserved after migration") + def_wallet = self.nodes[0].get_wallet_rpc(self.default_wallet_name) + + # Create and fund wallet + wallet = self.create_legacy_wallet("persist_comments") + def_wallet.sendtoaddress(wallet.getnewaddress(), 2) + + self.generate(self.nodes[0], 6) + + # Create tx and bump it to store 'replaced_by_txid' and 'replaces_txid' data within the transactions. + # Additionally, store an extra comment within the original tx. + extra_comment = "don't discard me" + original_tx_id = wallet.sendtoaddress(address=wallet.getnewaddress(), amount=1, comment=extra_comment) + bumped_tx = wallet.bumpfee(txid=original_tx_id) + + def check_comments(): + for record in wallet.listtransactions(): + if record["txid"] == original_tx_id: + assert_equal(record["replaced_by_txid"], bumped_tx["txid"]) + assert_equal(record['comment'], extra_comment) + elif record["txid"] == bumped_tx["txid"]: + assert_equal(record["replaces_txid"], original_tx_id) + + # Pre-migration verification + check_comments() + # Migrate + wallet.migratewallet() + # Post-migration verification + check_comments() + + wallet.unloadwallet() + + def run_test(self): self.generate(self.nodes[0], 101) @@ -896,6 +985,8 @@ class WalletMigrationTest(BitcoinTestFramework): self.test_conflict_txs() self.test_hybrid_pubkey() self.test_failed_migration_cleanup() + self.test_avoidreuse() + self.test_preserve_tx_extra_info() if __name__ == '__main__': WalletMigrationTest().main() diff --git a/test/functional/wallet_miniscript.py b/test/functional/wallet_miniscript.py index e6c8ad545c..67e1283902 100755 --- a/test/functional/wallet_miniscript.py +++ b/test/functional/wallet_miniscript.py @@ -208,6 +208,7 @@ class WalletMiniscriptTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 + self.rpc_timeout = 180 def skip_test_if_missing_module(self): self.skip_if_no_wallet() diff --git a/test/functional/wallet_multiwallet.py b/test/functional/wallet_multiwallet.py index 10bc516d8f..ee866ee59b 100755 --- a/test/functional/wallet_multiwallet.py +++ b/test/functional/wallet_multiwallet.py @@ -9,9 +9,9 @@ Verify that a bitcoind node can load multiple wallet files from decimal import Decimal from threading import Thread import os +import platform import shutil import stat -import sys import time from test_framework.authproxy import JSONRPCException @@ -143,7 +143,7 @@ class MultiWalletTest(BitcoinTestFramework): # should raise rpc error if wallet path can't be created err_code = -4 if self.options.descriptors else -1 - assert_raises_rpc_error(err_code, "filesystem error:" if sys.platform != 'win32' else "create_directories:", self.nodes[0].createwallet, "w8/bad") + assert_raises_rpc_error(err_code, "filesystem error:" if platform.system() != 'Windows' else "create_directories:", self.nodes[0].createwallet, "w8/bad") # check that all requested wallets were created self.stop_node(0) diff --git a/test/functional/wallet_reindex.py b/test/functional/wallet_reindex.py new file mode 100755 index 0000000000..5388de4b71 --- /dev/null +++ b/test/functional/wallet_reindex.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +# Copyright (c) 2023-present The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or https://www.opensource.org/licenses/mit-license.php. + +"""Test wallet-reindex interaction""" + +import time + +from test_framework.blocktools import COINBASE_MATURITY +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, +) +BLOCK_TIME = 60 * 10 + +class WalletReindexTest(BitcoinTestFramework): + def add_options(self, parser): + self.add_wallet_options(parser) + + def set_test_params(self): + self.num_nodes = 1 + self.setup_clean_chain = True + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + + def advance_time(self, node, secs): + self.node_time += secs + node.setmocktime(self.node_time) + + # Verify the wallet updates the birth time accordingly when it detects a transaction + # with a time older than the oldest descriptor timestamp. + # This could happen when the user blindly imports a descriptor with 'timestamp=now'. + def birthtime_test(self, node, miner_wallet): + self.log.info("Test birth time update during tx scanning") + # Fund address to test + wallet_addr = miner_wallet.getnewaddress() + tx_id = miner_wallet.sendtoaddress(wallet_addr, 2) + + # Generate 50 blocks, one every 10 min to surpass the 2 hours rescan window the wallet has + for _ in range(50): + self.generate(node, 1) + self.advance_time(node, BLOCK_TIME) + + # Now create a new wallet, and import the descriptor + node.createwallet(wallet_name='watch_only', disable_private_keys=True, load_on_startup=True) + wallet_watch_only = node.get_wallet_rpc('watch_only') + # Blank wallets don't have a birth time + assert 'birthtime' not in wallet_watch_only.getwalletinfo() + + # For a descriptors wallet: Import address with timestamp=now. + # For legacy wallet: There is no way of importing a script/address with a custom time. The wallet always imports it with birthtime=1. + # In both cases, disable rescan to not detect the transaction. + wallet_watch_only.importaddress(wallet_addr, rescan=False) + assert_equal(len(wallet_watch_only.listtransactions()), 0) + + # Depending on the wallet type, the birth time changes. + wallet_birthtime = wallet_watch_only.getwalletinfo()['birthtime'] + if self.options.descriptors: + # As blocks were generated every 10 min, the chain MTP timestamp is node_time - 60 min. + assert_equal(self.node_time - BLOCK_TIME * 6, wallet_birthtime) + else: + # No way of importing scripts/addresses with a custom time on a legacy wallet. + # It's always set to the beginning of time. + assert_equal(wallet_birthtime, 1) + + # Rescan the wallet to detect the missing transaction + wallet_watch_only.rescanblockchain() + assert_equal(wallet_watch_only.gettransaction(tx_id)['confirmations'], 50) + assert_equal(wallet_watch_only.getbalances()['mine' if self.options.descriptors else 'watchonly']['trusted'], 2) + + # Reindex and wait for it to finish + with node.assert_debug_log(expected_msgs=["initload thread exit"]): + self.restart_node(0, extra_args=['-reindex=1', f'-mocktime={self.node_time}']) + node.syncwithvalidationinterfacequeue() + + # Verify the transaction is still 'confirmed' after reindex + wallet_watch_only = node.get_wallet_rpc('watch_only') + tx_info = wallet_watch_only.gettransaction(tx_id) + assert_equal(tx_info['confirmations'], 50) + + # Depending on the wallet type, the birth time changes. + if self.options.descriptors: + # For descriptors, verify the wallet updated the birth time to the transaction time + assert_equal(tx_info['time'], wallet_watch_only.getwalletinfo()['birthtime']) + else: + # For legacy, as the birth time was set to the beginning of time, verify it did not change + assert_equal(wallet_birthtime, 1) + + wallet_watch_only.unloadwallet() + + def run_test(self): + node = self.nodes[0] + self.node_time = int(time.time()) + node.setmocktime(self.node_time) + + # Fund miner + node.createwallet(wallet_name='miner', load_on_startup=True) + miner_wallet = node.get_wallet_rpc('miner') + self.generatetoaddress(node, COINBASE_MATURITY + 10, miner_wallet.getnewaddress()) + + # Tests + self.birthtime_test(node, miner_wallet) + + +if __name__ == '__main__': + WalletReindexTest().main() diff --git a/test/fuzz/test_runner.py b/test/fuzz/test_runner.py index ec74f7705c..e72977fac0 100755 --- a/test/fuzz/test_runner.py +++ b/test/fuzz/test_runner.py @@ -16,11 +16,14 @@ import sys def get_fuzz_env(*, target, source_dir): + symbolizer = os.environ.get('LLVM_SYMBOLIZER_PATH', "/usr/bin/llvm-symbolizer") return { 'FUZZ': target, 'UBSAN_OPTIONS': f'suppressions={source_dir}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1:report_error_type=1', + 'UBSAN_SYMBOLIZER_PATH':symbolizer, "ASAN_OPTIONS": "detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1", + 'ASAN_SYMBOLIZER_PATH':symbolizer, } diff --git a/test/lint/README.md b/test/lint/README.md index c0889b59af..484008298b 100644 --- a/test/lint/README.md +++ b/test/lint/README.md @@ -13,6 +13,14 @@ DOCKER_BUILDKIT=1 docker build -t bitcoin-linter --file "./ci/lint_imagefile" ./ Building the container can be done every time, because it is fast when the result is cached and it prevents issues when the image changes. +test runner +=========== + +To run the checks in the test runner outside the docker, use: + +```sh +( cd ./test/lint/test_runner/ && cargo fmt && cargo clippy && cargo run ) +``` check-doc.py ============ diff --git a/test/lint/check-doc.py b/test/lint/check-doc.py index d22dd9d996..f55d0f8cb7 100755 --- a/test/lint/check-doc.py +++ b/test/lint/check-doc.py @@ -23,7 +23,7 @@ CMD_GREP_WALLET_ARGS = r"git grep --function-context 'void WalletInit::AddWallet CMD_GREP_WALLET_HIDDEN_ARGS = r"git grep --function-context 'void DummyWalletInit::AddWalletOptions' -- {}".format(CMD_ROOT_DIR) CMD_GREP_DOCS = r"git grep --perl-regexp '{}' {}".format(REGEX_DOC, CMD_ROOT_DIR) # list unsupported, deprecated and duplicate args as they need no documentation -SET_DOC_OPTIONAL = set(['-h', '-help', '-dbcrashratio', '-forcecompactdb', '-zapwallettxes']) +SET_DOC_OPTIONAL = set(['-h', '-help', '-dbcrashratio', '-forcecompactdb']) def lint_missing_argument_documentation(): diff --git a/test/lint/lint-files.py b/test/lint/lint-files.py index f2b5db681b..86fe727b06 100755 --- a/test/lint/lint-files.py +++ b/test/lint/lint-files.py @@ -11,7 +11,7 @@ import os import re import sys from subprocess import check_output -from typing import Dict, Optional, NoReturn +from typing import Optional, NoReturn CMD_TOP_LEVEL = ["git", "rev-parse", "--show-toplevel"] CMD_ALL_FILES = ["git", "ls-files", "-z", "--full-name", "--stage"] @@ -69,7 +69,7 @@ class FileMeta(object): return None -def get_git_file_metadata() -> Dict[str, FileMeta]: +def get_git_file_metadata() -> dict[str, FileMeta]: ''' Return a dictionary mapping the name of all files in the repository to git tree metadata. ''' diff --git a/test/lint/lint-format-strings.py b/test/lint/lint-format-strings.py index 5ac5840ecf..09d858e131 100755 --- a/test/lint/lint-format-strings.py +++ b/test/lint/lint-format-strings.py @@ -20,6 +20,11 @@ FUNCTION_NAMES_AND_NUMBER_OF_LEADING_ARGUMENTS = [ 'fprintf,1', 'tfm::format,1', # Assuming tfm::::format(std::ostream&, ... 'LogConnectFailure,1', + 'LogError,0', + 'LogWarning,0', + 'LogInfo,0', + 'LogDebug,1', + 'LogTrace,1', 'LogPrint,1', 'LogPrintf,0', 'LogPrintfCategory,1', diff --git a/test/lint/lint-include-guards.py b/test/lint/lint-include-guards.py index 48b918e9da..291e528c1d 100755 --- a/test/lint/lint-include-guards.py +++ b/test/lint/lint-include-guards.py @@ -11,7 +11,6 @@ Check include guards. import re import sys from subprocess import check_output -from typing import List HEADER_ID_PREFIX = 'BITCOIN_' @@ -28,7 +27,7 @@ EXCLUDE_FILES_WITH_PREFIX = ['contrib/devtools/bitcoin-tidy', 'src/test/fuzz/FuzzedDataProvider.h'] -def _get_header_file_lst() -> List[str]: +def _get_header_file_lst() -> list[str]: """ Helper function to get a list of header filepaths to be checked for include guards. """ diff --git a/test/lint/spelling.ignore-words.txt b/test/lint/spelling.ignore-words.txt index ecc2a553d2..cb0e0c4d31 100644 --- a/test/lint/spelling.ignore-words.txt +++ b/test/lint/spelling.ignore-words.txt @@ -6,6 +6,7 @@ bu cachable clen crypted +debbugs fo fpr hights diff --git a/test/lint/test_runner/Cargo.lock b/test/lint/test_runner/Cargo.lock new file mode 100644 index 0000000000..ca83aa9331 --- /dev/null +++ b/test/lint/test_runner/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "test_runner" +version = "0.1.0" diff --git a/test/lint/test_runner/Cargo.toml b/test/lint/test_runner/Cargo.toml new file mode 100644 index 0000000000..053ce43d6c --- /dev/null +++ b/test/lint/test_runner/Cargo.toml @@ -0,0 +1,12 @@ +# Copyright (c) The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or https://opensource.org/license/mit/. + +[package] +name = "test_runner" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] diff --git a/test/lint/test_runner/src/main.rs b/test/lint/test_runner/src/main.rs new file mode 100644 index 0000000000..ce94c3b628 --- /dev/null +++ b/test/lint/test_runner/src/main.rs @@ -0,0 +1,132 @@ +// Copyright (c) The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or https://opensource.org/license/mit/. + +use std::env; +use std::path::PathBuf; +use std::process::Command; +use std::process::ExitCode; + +type LintError = String; +type LintResult = Result<(), LintError>; +type LintFn = fn() -> LintResult; + +/// Return the git command +fn git() -> Command { + Command::new("git") +} + +/// Return stdout +fn check_output(cmd: &mut std::process::Command) -> Result<String, LintError> { + let out = cmd.output().expect("command error"); + if !out.status.success() { + return Err(String::from_utf8_lossy(&out.stderr).to_string()); + } + Ok(String::from_utf8(out.stdout) + .map_err(|e| format!("{e}"))? + .trim() + .to_string()) +} + +/// Return the git root as utf8, or panic +fn get_git_root() -> String { + check_output(git().args(["rev-parse", "--show-toplevel"])).unwrap() +} + +fn lint_subtree() -> LintResult { + // This only checks that the trees are pure subtrees, it is not doing a full + // check with -r to not have to fetch all the remotes. + let mut good = true; + for subtree in [ + "src/crypto/ctaes", + "src/secp256k1", + "src/minisketch", + "src/leveldb", + "src/crc32c", + ] { + good &= Command::new("test/lint/git-subtree-check.sh") + .arg(subtree) + .status() + .expect("command_error") + .success(); + } + if good { + Ok(()) + } else { + Err("".to_string()) + } +} + +fn lint_std_filesystem() -> LintResult { + let found = git() + .args([ + "grep", + "std::filesystem", + "--", + "./src/", + ":(exclude)src/util/fs.h", + ]) + .status() + .expect("command error") + .success(); + if found { + Err(r#" +^^^ +Direct use of std::filesystem may be dangerous and buggy. Please include <util/fs.h> and use the +fs:: namespace, which has unsafe filesystem functions marked as deleted. + "# + .to_string()) + } else { + Ok(()) + } +} + +fn lint_doc() -> LintResult { + if Command::new("test/lint/check-doc.py") + .status() + .expect("command error") + .success() + { + Ok(()) + } else { + Err("".to_string()) + } +} + +fn lint_all() -> LintResult { + if Command::new("test/lint/all-lint.py") + .status() + .expect("command error") + .success() + { + Ok(()) + } else { + Err("".to_string()) + } +} + +fn main() -> ExitCode { + let test_list: Vec<(&str, LintFn)> = vec![ + ("subtree check", lint_subtree), + ("std::filesystem check", lint_std_filesystem), + ("-help=1 documentation check", lint_doc), + ("all-lint.py script", lint_all), + ]; + + let git_root = PathBuf::from(get_git_root()); + + let mut test_failed = false; + for (lint_name, lint_fn) in test_list { + // chdir to root before each lint test + env::set_current_dir(&git_root).unwrap(); + if let Err(err) = lint_fn() { + println!("{err}\n^---- Failure generated from {lint_name}!"); + test_failed = true; + } + } + if test_failed { + ExitCode::FAILURE + } else { + ExitCode::SUCCESS + } +} diff --git a/test/sanitizer_suppressions/ubsan b/test/sanitizer_suppressions/ubsan index 533e2eae51..f0ee698909 100644 --- a/test/sanitizer_suppressions/ubsan +++ b/test/sanitizer_suppressions/ubsan @@ -39,28 +39,37 @@ shift-base:test/fuzz/crypto_diff_fuzz_chacha20.cpp # list is used to suppress -fsanitize=integer warnings when running our CI UBSan # job. unsigned-integer-overflow:arith_uint256.h -unsigned-integer-overflow:common/bloom.cpp -unsigned-integer-overflow:coins.cpp -unsigned-integer-overflow:compressor.cpp +unsigned-integer-overflow:CBloomFilter::Hash +unsigned-integer-overflow:CRollingBloomFilter::insert +unsigned-integer-overflow:RollingBloomHash +unsigned-integer-overflow:CCoinsViewCache::AddCoin +unsigned-integer-overflow:CCoinsViewCache::BatchWrite +unsigned-integer-overflow:CCoinsViewCache::DynamicMemoryUsage +unsigned-integer-overflow:CCoinsViewCache::SpendCoin +unsigned-integer-overflow:CCoinsViewCache::Uncache +unsigned-integer-overflow:CompressAmount +unsigned-integer-overflow:DecompressAmount unsigned-integer-overflow:crypto/ -unsigned-integer-overflow:hash.cpp -unsigned-integer-overflow:policy/fees.cpp +unsigned-integer-overflow:MurmurHash3 +unsigned-integer-overflow:CBlockPolicyEstimator::processBlockTx +unsigned-integer-overflow:TxConfirmStats::EstimateMedianVal unsigned-integer-overflow:prevector.h unsigned-integer-overflow:script/interpreter.cpp unsigned-integer-overflow:xoroshiro128plusplus.h -implicit-integer-sign-change:compat/stdin.cpp +implicit-integer-sign-change:CBlockPolicyEstimator::processBlockTx +implicit-integer-sign-change:SetStdinEcho implicit-integer-sign-change:compressor.h implicit-integer-sign-change:crypto/ -implicit-integer-sign-change:policy/fees.cpp +implicit-integer-sign-change:TxConfirmStats::removeTx implicit-integer-sign-change:prevector.h -implicit-integer-sign-change:script/bitcoinconsensus.cpp -implicit-integer-sign-change:script/interpreter.cpp +implicit-integer-sign-change:verify_flags +implicit-integer-sign-change:EvalScript implicit-integer-sign-change:serialize.h implicit-signed-integer-truncation:crypto/ implicit-unsigned-integer-truncation:crypto/ shift-base:arith_uint256.cpp shift-base:crypto/ -shift-base:hash.cpp +shift-base:ROTL32 shift-base:streams.h -shift-base:util/bip32.cpp +shift-base:FormatHDKeypath shift-base:xoroshiro128plusplus.h diff --git a/test/util/data/bitcoin-util-test.json b/test/util/data/bitcoin-util-test.json index c9c64274c6..83b3c430d5 100644 --- a/test/util/data/bitcoin-util-test.json +++ b/test/util/data/bitcoin-util-test.json @@ -163,7 +163,85 @@ "replaceable=0foo"], "return_code": 1, "error_txt": "error: Invalid TX input index", - "description": "Tests the check for an invalid input index with replaceable" + "description": "Tests the check for an invalid string input index with replaceable" + }, + { + "exec": "./bitcoin-tx", + "args": + ["-create", + "in=5897de6bd6027a475eadd57019d4e6872c396d0716c4875a5f1a6fcfdf385c1f:0", + "replaceable=-1"], + "return_code": 1, + "error_txt": "error: Invalid TX input index", + "description": "Tests the check for an invalid negative input index with replaceable" + }, + { + "exec": "./bitcoin-tx", + "args": + ["-create", + "in=5897de6bd6027a475eadd57019d4e6872c396d0716c4875a5f1a6fcfdf385c1f:0", + "replaceable=1"], + "return_code": 1, + "error_txt": "error: Invalid TX input index", + "description": "Tests the check for an invalid positive out-of-bounds input index with replaceable" + }, + { + "exec": "./bitcoin-tx", + "args": + ["-create", + "in=5897de6bd6027a475eadd57019d4e6872c396d0716c4875a5f1a6fcfdf385c1f:0", + "replaceable=0"], + "output_cmp": "txreplacesingleinput.hex", + "description": "Tests that the 'SEQUENCE' value for a single input is set to fdffffff for single input" + }, + { + "exec": "./bitcoin-tx", + "args": + ["-create", + "in=5897de6bd6027a475eadd57019d4e6872c396d0716c4875a5f1a6fcfdf385c1f:0", + "replaceable="], + "output_cmp": "txreplacesingleinput.hex", + "description": "Tests that the 'SEQUENCE' value for a single input is set to fdffffff when N omitted" + }, + { + "exec": "./bitcoin-tx", + "args": + ["-create", + "in=5897de6bd6027a475eadd57019d4e6872c396d0716c4875a5f1a6fcfdf385c1f:0", + "in=bf829c6bcf84579331337659d31f89dfd138f7f7785802d5501c92333145ca7c:18", + "in=22a6f904655d53ae2ff70e701a0bbd90aa3975c0f40bfc6cc996a9049e31cdfc:1", + "replaceable=1"], + "output_cmp": "txreplace1.hex", + "description": "Tests that only the 'SEQUENCE' value of input[1] is set to fdffffff" + }, + { + "exec": "./bitcoin-tx", + "args": + ["-create", + "in=5897de6bd6027a475eadd57019d4e6872c396d0716c4875a5f1a6fcfdf385c1f:0", + "in=bf829c6bcf84579331337659d31f89dfd138f7f7785802d5501c92333145ca7c:18", + "in=22a6f904655d53ae2ff70e701a0bbd90aa3975c0f40bfc6cc996a9049e31cdfc:1", + "replaceable="], + "output_cmp": "txreplaceomittedn.hex", + "description": "Tests that the 'SEQUENCE' value for each input is set to fdffffff when N omitted" + }, + { + "exec": "./bitcoin-tx", + "args": + ["-create", + "replaceable="], + "output_cmp": "txreplacenoinputs.hex", + "description": "Tests behavior when no inputs are provided in the transaction" + }, + { + "exec": "./bitcoin-tx", + "args": + ["-create", + "in=5897de6bd6027a475eadd57019d4e6872c396d0716c4875a5f1a6fcfdf385c1f:0:abcdef00", + "replaceable="], + "return_code": 1, + "error_txt": "error: invalid TX sequence id 'abcdef00'", + "description": "Try to make invalid input replaceable" }, { "exec": "./bitcoin-tx", "args": diff --git a/test/util/data/txreplace1.hex b/test/util/data/txreplace1.hex new file mode 100644 index 0000000000..7401c4e5dd --- /dev/null +++ b/test/util/data/txreplace1.hex @@ -0,0 +1 @@ +02000000031f5c38dfcf6f1a5f5a87c416076d392c87e6d41970d5ad5e477a02d66bde97580000000000ffffffff7cca453133921c50d5025878f7f738d1df891fd359763331935784cf6b9c82bf1200000000fdfffffffccd319e04a996c96cfc0bf4c07539aa90bd0b1a700ef72fae535d6504f9a6220100000000ffffffff0000000000 diff --git a/test/util/data/txreplacenoinputs.hex b/test/util/data/txreplacenoinputs.hex new file mode 100644 index 0000000000..22d830eda1 --- /dev/null +++ b/test/util/data/txreplacenoinputs.hex @@ -0,0 +1 @@ +02000000000000000000 diff --git a/test/util/data/txreplaceomittedn.hex b/test/util/data/txreplaceomittedn.hex new file mode 100644 index 0000000000..a687836a09 --- /dev/null +++ b/test/util/data/txreplaceomittedn.hex @@ -0,0 +1 @@ +02000000031f5c38dfcf6f1a5f5a87c416076d392c87e6d41970d5ad5e477a02d66bde97580000000000fdffffff7cca453133921c50d5025878f7f738d1df891fd359763331935784cf6b9c82bf1200000000fdfffffffccd319e04a996c96cfc0bf4c07539aa90bd0b1a700ef72fae535d6504f9a6220100000000fdffffff0000000000 diff --git a/test/util/data/txreplacesingleinput.hex b/test/util/data/txreplacesingleinput.hex new file mode 100644 index 0000000000..b3e442795e --- /dev/null +++ b/test/util/data/txreplacesingleinput.hex @@ -0,0 +1 @@ +02000000011f5c38dfcf6f1a5f5a87c416076d392c87e6d41970d5ad5e477a02d66bde97580000000000fdffffff0000000000 |