diff options
Diffstat (limited to 'test')
84 files changed, 3657 insertions, 753 deletions
diff --git a/test/functional/data/rpc_psbt.json b/test/functional/data/rpc_psbt.json index 3127350872..1ccc5e0ba0 100644 --- a/test/functional/data/rpc_psbt.json +++ b/test/functional/data/rpc_psbt.json @@ -38,7 +38,9 @@ "cHNidP8BAF4CAAAAAZvUh2UjC/mnLmYgAflyVW5U8Mb5f+tWvLVgDYF/aZUmAQAAAAD/////AUjmBSoBAAAAIlEgAw2k/OT32yjCyylRYx4ANxOFZZf+ljiCy1AOaBEsymMAAAAAAAEBKwDyBSoBAAAAIlEgwiR++/2SrEf29AuNQtFpF1oZ+p+hDkol1/NetN2FtpJBFCyxOsaCSN6AaqajZZzzwD62gh0JyBFKToaP696GW7bSzZcOFfU/wMgvlQ/VYP+pGbdhcr4Bc2iomROvB09ACwlCiXVqo3OczGiewPzzo2C+MswLWbFuk6Hou0YFcmssp6P/cGxBdmSWMrLMaOH5ErileONxnOdxCIXHqWb0m81DywEBAAA=", "cHNidP8BAF4CAAAAAZvUh2UjC/mnLmYgAflyVW5U8Mb5f+tWvLVgDYF/aZUmAQAAAAD/////AUjmBSoBAAAAIlEgAw2k/OT32yjCyylRYx4ANxOFZZf+ljiCy1AOaBEsymMAAAAAAAEBKwDyBSoBAAAAIlEgwiR++/2SrEf29AuNQtFpF1oZ+p+hDkol1/NetN2FtpJBFCyxOsaCSN6AaqajZZzzwD62gh0JyBFKToaP696GW7bSzZcOFfU/wMgvlQ/VYP+pGbdhcr4Bc2iomROvB09ACwk5iXVqo3OczGiewPzzo2C+MswLWbFuk6Hou0YFcmssp6P/cGxBdmSWMrLMaOH5ErileONxnOdxCIXHqWb0m81DywAA", "cHNidP8BAF4CAAAAAZvUh2UjC/mnLmYgAflyVW5U8Mb5f+tWvLVgDYF/aZUmAQAAAAD/////AUjmBSoBAAAAIlEgAw2k/OT32yjCyylRYx4ANxOFZZf+ljiCy1AOaBEsymMAAAAAAAEBKwDyBSoBAAAAIlEgwiR++/2SrEf29AuNQtFpF1oZ+p+hDkol1/NetN2FtpJjFcFQkpt0waBJVLeLS2A16XpeB4paDyjsltVHv+6azoA6wG99YgWelJehpKJnVp2YdtpgEBr/OONSm5uTnOf5GulwEV8uSQr3zEXE94UR82BXzlxaXFYyWin7RN/CA/NW4fgAIyAssTrGgkjegGqmo2Wc88A+toIdCcgRSk6Gj+vehlu20qzAAAA=", - "cHNidP8BAF4CAAAAAZvUh2UjC/mnLmYgAflyVW5U8Mb5f+tWvLVgDYF/aZUmAQAAAAD/////AUjmBSoBAAAAIlEgAw2k/OT32yjCyylRYx4ANxOFZZf+ljiCy1AOaBEsymMAAAAAAAEBKwDyBSoBAAAAIlEgwiR++/2SrEf29AuNQtFpF1oZ+p+hDkol1/NetN2FtpJhFcFQkpt0waBJVLeLS2A16XpeB4paDyjsltVHv+6azoA6wG99YgWelJehpKJnVp2YdtpgEBr/OONSm5uTnOf5GulwEV8uSQr3zEXE94UR82BXzlxaXFYyWin7RN/CA/NW4SMgLLE6xoJI3oBqpqNlnPPAPraCHQnIEUpOho/r3oZbttKswAAA" + "cHNidP8BAF4CAAAAAZvUh2UjC/mnLmYgAflyVW5U8Mb5f+tWvLVgDYF/aZUmAQAAAAD/////AUjmBSoBAAAAIlEgAw2k/OT32yjCyylRYx4ANxOFZZf+ljiCy1AOaBEsymMAAAAAAAEBKwDyBSoBAAAAIlEgwiR++/2SrEf29AuNQtFpF1oZ+p+hDkol1/NetN2FtpJhFcFQkpt0waBJVLeLS2A16XpeB4paDyjsltVHv+6azoA6wG99YgWelJehpKJnVp2YdtpgEBr/OONSm5uTnOf5GulwEV8uSQr3zEXE94UR82BXzlxaXFYyWin7RN/CA/NW4SMgLLE6xoJI3oBqpqNlnPPAPraCHQnIEUpOho/r3oZbttKswAAA", + "cHNidP8BAHUCAAAAAQCBcTce3/KF6Tet7qSze3gADAVmy7OtZGQXE8pCFxv2AAAAAAD+////AtPf9QUAAAAAGXapFNDFmQPFusKGh2DpD9UhpGZap2UgiKwA4fUFAAAAABepFDVF5uM7gyxHBQ8k0+65PJwDlIvHh7MuEwAAAQD9pQEBAAAAAAECiaPHHqtNIOA3G7ukzGmPopXJRjr6Ljl/hTPMti+VZ+UBAAAAFxYAFL4Y0VKpsBIDna89p95PUzSe7LmF/////4b4qkOnHf8USIk6UwpyN+9rRgi7st0tAXHmOuxqSJC0AQAAABcWABT+Pp7xp0XpdNkCxDVZQ6vLNL1TU/////8CAMLrCwAAAAAZdqkUhc/xCX/Z4Ai7NK9wnGIZeziXikiIrHL++E4sAAAAF6kUM5cluiHv1irHU6m80GfWx6ajnQWHAkcwRAIgJxK+IuAnDzlPVoMR3HyppolwuAJf3TskAinwf4pfOiQCIAGLONfc0xTnNMkna9b7QPZzMlvEuqFEyADS8vAtsnZcASED0uFWdJQbrUqZY3LLh+GFbTZSYG2YVi/jnF6efkE/IQUCSDBFAiEA0SuFLYXc2WHS9fSrZgZU327tzHlMDDPOXMMJ/7X85Y0CIGczio4OFyXBl/saiK9Z9R5E5CVbIBZ8hoQDHAXR8lkqASECI7cr7vCWXRC+B3jv7NYfysb3mk6haTkzgHNEZPhPKrMAAAAAAAAA", + "cHNidP8BAHUCAAAAASaBcTce3/KF6Tet7qSze3gADAVmy7OtZGQXE8pCFxv2AAAAAgD+////AtPf9QUAAAAAGXapFNDFmQPFusKGh2DpD9UhpGZap2UgiKwA4fUFAAAAABepFDVF5uM7gyxHBQ8k0+65PJwDlIvHh7MuEwAAAQD9pQEBAAAAAAECiaPHHqtNIOA3G7ukzGmPopXJRjr6Ljl/hTPMti+VZ+UBAAAAFxYAFL4Y0VKpsBIDna89p95PUzSe7LmF/////4b4qkOnHf8USIk6UwpyN+9rRgi7st0tAXHmOuxqSJC0AQAAABcWABT+Pp7xp0XpdNkCxDVZQ6vLNL1TU/////8CAMLrCwAAAAAZdqkUhc/xCX/Z4Ai7NK9wnGIZeziXikiIrHL++E4sAAAAF6kUM5cluiHv1irHU6m80GfWx6ajnQWHAkcwRAIgJxK+IuAnDzlPVoMR3HyppolwuAJf3TskAinwf4pfOiQCIAGLONfc0xTnNMkna9b7QPZzMlvEuqFEyADS8vAtsnZcASED0uFWdJQbrUqZY3LLh+GFbTZSYG2YVi/jnF6efkE/IQUCSDBFAiEA0SuFLYXc2WHS9fSrZgZU327tzHlMDDPOXMMJ/7X85Y0CIGczio4OFyXBl/saiK9Z9R5E5CVbIBZ8hoQDHAXR8lkqASECI7cr7vCWXRC+B3jv7NYfysb3mk6haTkzgHNEZPhPKrMAAAAAAAAA" ], "invalid_with_msg": [ [ diff --git a/test/functional/feature_addrman.py b/test/functional/feature_addrman.py index 95d33d62ea..2efad70900 100755 --- a/test/functional/feature_addrman.py +++ b/test/functional/feature_addrman.py @@ -6,7 +6,6 @@ import os import re -import struct from test_framework.messages import ser_uint256, hash256, MAGIC_BYTES from test_framework.netutil import ADDRMAN_NEW_BUCKET_COUNT, ADDRMAN_TRIED_BUCKET_COUNT, ADDRMAN_BUCKET_SIZE @@ -28,15 +27,15 @@ def serialize_addrman( tried = [] INCOMPATIBILITY_BASE = 32 r = MAGIC_BYTES[net_magic] - r += struct.pack("B", format) - r += struct.pack("B", INCOMPATIBILITY_BASE + lowest_compatible) + r += format.to_bytes(1, "little") + r += (INCOMPATIBILITY_BASE + lowest_compatible).to_bytes(1, "little") r += ser_uint256(bucket_key) - r += struct.pack("<i", len_new or len(new)) - r += struct.pack("<i", len_tried or len(tried)) + r += (len_new or len(new)).to_bytes(4, "little", signed=True) + r += (len_tried or len(tried)).to_bytes(4, "little", signed=True) ADDRMAN_NEW_BUCKET_COUNT = 1 << 10 - r += struct.pack("<i", ADDRMAN_NEW_BUCKET_COUNT ^ (1 << 30)) + r += (ADDRMAN_NEW_BUCKET_COUNT ^ (1 << 30)).to_bytes(4, "little", signed=True) for _ in range(ADDRMAN_NEW_BUCKET_COUNT): - r += struct.pack("<i", 0) + r += (0).to_bytes(4, "little", signed=True) checksum = hash256(r) r += mock_checksum or checksum return r diff --git a/test/functional/feature_asmap.py b/test/functional/feature_asmap.py index 024a8fa18c..e469deef49 100755 --- a/test/functional/feature_asmap.py +++ b/test/functional/feature_asmap.py @@ -27,6 +27,7 @@ import os import shutil from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal DEFAULT_ASMAP_FILENAME = 'ip_asn.map' # defined in src/init.cpp ASMAP = '../../src/test/data/asmap.raw' # path to unit test skeleton asmap @@ -118,6 +119,14 @@ class AsmapTest(BitcoinTestFramework): msg = "ASMap Health Check: 4 clearnet peers are mapped to 3 ASNs with 0 peers being unmapped" with self.node.assert_debug_log(expected_msgs=[msg]): self.start_node(0, extra_args=['-asmap']) + raw_addrman = self.node.getrawaddrman() + asns = [] + for _, entries in raw_addrman.items(): + for _, entry in entries.items(): + asn = entry['mapped_as'] + if asn not in asns: + asns.append(asn) + assert_equal(len(asns), 3) os.remove(self.default_asmap) def run_test(self): diff --git a/test/functional/feature_assumeutxo.py b/test/functional/feature_assumeutxo.py index 19cbbcffdb..688e2866b2 100755 --- a/test/functional/feature_assumeutxo.py +++ b/test/functional/feature_assumeutxo.py @@ -13,8 +13,6 @@ The assumeutxo value generated and used here is committed to in Interesting test cases could be loading an assumeutxo snapshot file with: -- TODO: Valid hash but invalid snapshot file (bad coin height or - bad other serialization) - TODO: Valid snapshot file, but referencing a snapshot block that turns out to be invalid, or has an invalid parent - TODO: Valid snapshot file and snapshot block, but the block is not on the @@ -23,7 +21,6 @@ Interesting test cases could be loading an assumeutxo snapshot file with: Interesting starting states could be loading a snapshot when the current chain tip is: - TODO: An ancestor of snapshot block -- TODO: Not an ancestor of the snapshot block but has less work - TODO: The snapshot block - TODO: A descendant of the snapshot block - TODO: Not an ancestor or a descendant of the snapshot block and has more work @@ -35,6 +32,7 @@ from dataclasses import dataclass from test_framework.messages import tx_from_hex from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( + assert_approx, assert_equal, assert_raises_rpc_error, ) @@ -53,18 +51,19 @@ class AssumeutxoTest(BitcoinTestFramework): def set_test_params(self): """Use the pregenerated, deterministic chain up to height 199.""" - self.num_nodes = 3 + self.num_nodes = 4 self.rpc_timeout = 120 self.extra_args = [ [], ["-fastprune", "-prune=1", "-blockfilterindex=1", "-coinstatsindex=1"], ["-persistmempool=0","-txindex=1", "-blockfilterindex=1", "-coinstatsindex=1"], + [] ] def setup_network(self): """Start with the nodes disconnected so that one can generate a snapshot including blocks the other hasn't yet seen.""" - self.add_nodes(3) + self.add_nodes(4) self.start_nodes(extra_args=self.extra_args) def test_invalid_snapshot_scenarios(self, valid_snapshot_path): @@ -72,50 +71,91 @@ class AssumeutxoTest(BitcoinTestFramework): with open(valid_snapshot_path, 'rb') as f: valid_snapshot_contents = f.read() bad_snapshot_path = valid_snapshot_path + '.mod' + node = self.nodes[1] def expected_error(log_msg="", rpc_details=""): - with self.nodes[1].assert_debug_log([log_msg]): - assert_raises_rpc_error(-32603, f"Unable to load UTXO snapshot{rpc_details}", self.nodes[1].loadtxoutset, bad_snapshot_path) + with node.assert_debug_log([log_msg]): + assert_raises_rpc_error(-32603, f"Unable to load UTXO snapshot{rpc_details}", node.loadtxoutset, bad_snapshot_path) + + self.log.info(" - snapshot file with invalid file magic") + parsing_error_code = -22 + bad_magic = 0xf00f00f000 + with open(bad_snapshot_path, 'wb') as f: + f.write(bad_magic.to_bytes(5, "big") + valid_snapshot_contents[5:]) + assert_raises_rpc_error(parsing_error_code, "Unable to parse metadata: Invalid UTXO set snapshot magic bytes. Please check if this is indeed a snapshot file or if you are using an outdated snapshot format.", node.loadtxoutset, bad_snapshot_path) + + self.log.info(" - snapshot file with unsupported version") + for version in [0, 2]: + with open(bad_snapshot_path, 'wb') as f: + f.write(valid_snapshot_contents[:5] + version.to_bytes(2, "little") + valid_snapshot_contents[7:]) + assert_raises_rpc_error(parsing_error_code, f"Unable to parse metadata: Version of snapshot {version} does not match any of the supported versions.", node.loadtxoutset, bad_snapshot_path) + + self.log.info(" - snapshot file with mismatching network magic") + invalid_magics = [ + # magic, name, real + [0xf9beb4d9, "main", True], + [0x0b110907, "test", True], + [0x0a03cf40, "signet", True], + [0x00000000, "", False], + [0xffffffff, "", False], + ] + for [magic, name, real] in invalid_magics: + with open(bad_snapshot_path, 'wb') as f: + f.write(valid_snapshot_contents[:7] + magic.to_bytes(4, 'big') + valid_snapshot_contents[11:]) + if real: + assert_raises_rpc_error(parsing_error_code, f"Unable to parse metadata: The network of the snapshot ({name}) does not match the network of this node (regtest).", node.loadtxoutset, bad_snapshot_path) + else: + assert_raises_rpc_error(parsing_error_code, "Unable to parse metadata: This snapshot has been created for an unrecognized network. This could be a custom signet, a new testnet or possibly caused by data corruption.", node.loadtxoutset, bad_snapshot_path) self.log.info(" - snapshot file referring to a block that is not in the assumeutxo parameters") prev_block_hash = self.nodes[0].getblockhash(SNAPSHOT_BASE_HEIGHT - 1) bogus_block_hash = "0" * 64 # Represents any unknown block hash + # The height is not used for anything critical currently, so we just + # confirm the manipulation in the error message + bogus_height = 1337 for bad_block_hash in [bogus_block_hash, prev_block_hash]: with open(bad_snapshot_path, 'wb') as f: - # block hash of the snapshot base is stored right at the start (first 32 bytes) - f.write(bytes.fromhex(bad_block_hash)[::-1] + valid_snapshot_contents[32:]) - error_details = f", assumeutxo block hash in snapshot metadata not recognized ({bad_block_hash})" - expected_error(rpc_details=error_details) + f.write(valid_snapshot_contents[:11] + bogus_height.to_bytes(4, "little") + bytes.fromhex(bad_block_hash)[::-1] + valid_snapshot_contents[47:]) + + msg = f"Unable to load UTXO snapshot: assumeutxo block hash in snapshot metadata not recognized (hash: {bad_block_hash}, height: {bogus_height}). The following snapshot heights are available: 110, 200, 299." + assert_raises_rpc_error(-32603, msg, node.loadtxoutset, bad_snapshot_path) self.log.info(" - snapshot file with wrong number of coins") - valid_num_coins = int.from_bytes(valid_snapshot_contents[32:32 + 8], "little") + valid_num_coins = int.from_bytes(valid_snapshot_contents[47:47 + 8], "little") for off in [-1, +1]: with open(bad_snapshot_path, 'wb') as f: - f.write(valid_snapshot_contents[:32]) + f.write(valid_snapshot_contents[:47]) f.write((valid_num_coins + off).to_bytes(8, "little")) - f.write(valid_snapshot_contents[32 + 8:]) + f.write(valid_snapshot_contents[47 + 8:]) expected_error(log_msg=f"bad snapshot - coins left over after deserializing 298 coins" if off == -1 else f"bad snapshot format or truncated snapshot after deserializing 299 coins") - self.log.info(" - snapshot file with alternated UTXO data") + self.log.info(" - snapshot file with alternated but parsable UTXO data results in different hash") cases = [ - [b"\xff" * 32, 0, "7d52155c9a9fdc4525b637ef6170568e5dad6fabd0b1fdbb9432010b8453095b"], # wrong outpoint hash - [(1).to_bytes(4, "little"), 32, "9f4d897031ab8547665b4153317ae2fdbf0130c7840b66427ebc48b881cb80ad"], # wrong outpoint index - [b"\x81", 36, "3da966ba9826fb6d2604260e01607b55ba44e1a5de298606b08704bc62570ea8"], # wrong coin code VARINT((coinbase ? 1 : 0) | (height << 1)) - [b"\x80", 36, "091e893b3ccb4334378709578025356c8bcb0a623f37c7c4e493133c988648e5"], # another wrong coin code + # (content, offset, wrong_hash, custom_message) + [b"\xff" * 32, 0, "7d52155c9a9fdc4525b637ef6170568e5dad6fabd0b1fdbb9432010b8453095b", None], # wrong outpoint hash + [(2).to_bytes(1, "little"), 32, None, "[snapshot] bad snapshot data after deserializing 1 coins"], # wrong txid coins count + [b"\xfd\xff\xff", 32, None, "[snapshot] mismatch in coins count in snapshot metadata and actual snapshot data"], # txid coins count exceeds coins left + [b"\x01", 33, "9f4d897031ab8547665b4153317ae2fdbf0130c7840b66427ebc48b881cb80ad", None], # wrong outpoint index + [b"\x81", 34, "3da966ba9826fb6d2604260e01607b55ba44e1a5de298606b08704bc62570ea8", None], # wrong coin code VARINT + [b"\x80", 34, "091e893b3ccb4334378709578025356c8bcb0a623f37c7c4e493133c988648e5", None], # another wrong coin code + [b"\x84\x58", 34, None, "[snapshot] bad snapshot data after deserializing 0 coins"], # wrong coin case with height 364 and coinbase 0 + [b"\xCA\xD2\x8F\x5A", 39, None, "[snapshot] bad snapshot data after deserializing 0 coins - bad tx out value"], # Amount exceeds MAX_MONEY ] - for content, offset, wrong_hash in cases: + for content, offset, wrong_hash, custom_message in cases: with open(bad_snapshot_path, "wb") as f: - f.write(valid_snapshot_contents[:(32 + 8 + offset)]) + # Prior to offset: Snapshot magic, snapshot version, network magic, height, hash, coins count + f.write(valid_snapshot_contents[:(5 + 2 + 4 + 4 + 32 + 8 + offset)]) f.write(content) - f.write(valid_snapshot_contents[(32 + 8 + offset + len(content)):]) - expected_error(log_msg=f"[snapshot] bad snapshot content hash: expected a4bf3407ccb2cc0145c49ebba8fa91199f8a3903daf0883875941497d2493c27, got {wrong_hash}") + f.write(valid_snapshot_contents[(5 + 2 + 4 + 4 + 32 + 8 + offset + len(content)):]) + + log_msg = custom_message if custom_message is not None else f"[snapshot] bad snapshot content hash: expected a4bf3407ccb2cc0145c49ebba8fa91199f8a3903daf0883875941497d2493c27, got {wrong_hash}" + expected_error(log_msg=log_msg) def test_headers_not_synced(self, valid_snapshot_path): for node in self.nodes[1:]: - assert_raises_rpc_error(-32603, "The base block header (3bb7ce5eba0be48939b7a521ac1ba9316afee2c7bada3a0cca24188e6d7d96c0) must appear in the headers chain. Make sure all headers are syncing, and call this RPC again.", - node.loadtxoutset, - valid_snapshot_path) + msg = "Unable to load UTXO snapshot: The base block header (3bb7ce5eba0be48939b7a521ac1ba9316afee2c7bada3a0cca24188e6d7d96c0) must appear in the headers chain. Make sure all headers are syncing, and call loadtxoutset again." + assert_raises_rpc_error(-32603, msg, node.loadtxoutset, valid_snapshot_path) def test_invalid_chainstate_scenarios(self): self.log.info("Test different scenarios of invalid snapshot chainstate in datadir") @@ -147,11 +187,60 @@ class AssumeutxoTest(BitcoinTestFramework): assert tx['txid'] in node.getrawmempool() # Attempt to load the snapshot on Node 2 and expect it to fail - with node.assert_debug_log(expected_msgs=["[snapshot] can't activate a snapshot when mempool not empty"]): - assert_raises_rpc_error(-32603, "Unable to load UTXO snapshot", node.loadtxoutset, dump_output_path) + msg = "Unable to load UTXO snapshot: Can't activate a snapshot when mempool not empty" + assert_raises_rpc_error(-32603, msg, node.loadtxoutset, dump_output_path) self.restart_node(2, extra_args=self.extra_args[2]) + def test_invalid_file_path(self): + self.log.info("Test bitcoind should fail when file path is invalid.") + node = self.nodes[0] + path = node.datadir_path / node.chain / "invalid" / "path" + assert_raises_rpc_error(-8, "Couldn't open file {} for reading.".format(path), node.loadtxoutset, path) + + def test_snapshot_with_less_work(self, dump_output_path): + self.log.info("Test bitcoind should fail when snapshot has less accumulated work than this node.") + node = self.nodes[0] + assert_equal(node.getblockcount(), FINAL_HEIGHT) + with node.assert_debug_log(expected_msgs=["[snapshot] activation failed - work does not exceed active chainstate"]): + assert_raises_rpc_error(-32603, "Unable to load UTXO snapshot", node.loadtxoutset, dump_output_path) + + def test_snapshot_block_invalidated(self, dump_output_path): + self.log.info("Test snapshot is not loaded when base block is invalid.") + node = self.nodes[0] + # We are testing the case where the base block is invalidated itself + # and also the case where one of its parents is invalidated. + for height in [SNAPSHOT_BASE_HEIGHT, SNAPSHOT_BASE_HEIGHT - 1]: + block_hash = node.getblockhash(height) + node.invalidateblock(block_hash) + assert_equal(node.getblockcount(), height - 1) + msg = "Unable to load UTXO snapshot: The base block header (3bb7ce5eba0be48939b7a521ac1ba9316afee2c7bada3a0cca24188e6d7d96c0) is part of an invalid chain." + assert_raises_rpc_error(-32603, msg, node.loadtxoutset, dump_output_path) + node.reconsiderblock(block_hash) + + def test_snapshot_in_a_divergent_chain(self, dump_output_path): + n0 = self.nodes[0] + n3 = self.nodes[3] + assert_equal(n0.getblockcount(), FINAL_HEIGHT) + assert_equal(n3.getblockcount(), START_HEIGHT) + + self.log.info("Check importing a snapshot where current chain-tip is not an ancestor of the snapshot block but has less work") + # Generate a divergent chain in n3 up to 298 + self.generate(n3, nblocks=99, sync_fun=self.no_op) + assert_equal(n3.getblockcount(), SNAPSHOT_BASE_HEIGHT - 1) + + # Try importing the snapshot and assert its success + loaded = n3.loadtxoutset(dump_output_path) + assert_equal(loaded['base_height'], SNAPSHOT_BASE_HEIGHT) + normal, snapshot = n3.getchainstates()["chainstates"] + assert_equal(normal['blocks'], START_HEIGHT + 99) + assert_equal(snapshot['blocks'], SNAPSHOT_BASE_HEIGHT) + + # Now lets sync the nodes and wait for the background validation to finish + self.connect_nodes(0, 3) + self.sync_blocks(nodes=(n0, n3)) + self.wait_until(lambda: len(n3.getchainstates()['chainstates']) == 1) + def run_test(self): """ Bring up two (disconnected) nodes, mine some new blocks on the first, @@ -163,6 +252,7 @@ class AssumeutxoTest(BitcoinTestFramework): n0 = self.nodes[0] n1 = self.nodes[1] n2 = self.nodes[2] + n3 = self.nodes[3] self.mini_wallet = MiniWallet(n0) @@ -213,6 +303,7 @@ class AssumeutxoTest(BitcoinTestFramework): # block. n1.submitheader(block) n2.submitheader(block) + n3.submitheader(block) # Ensure everyone is seeing the same headers. for n in self.nodes: @@ -233,9 +324,12 @@ class AssumeutxoTest(BitcoinTestFramework): assert_equal(n0.getblockchaininfo()["blocks"], FINAL_HEIGHT) + self.test_snapshot_with_less_work(dump_output['path']) self.test_invalid_mempool_state(dump_output['path']) self.test_invalid_snapshot_scenarios(dump_output['path']) self.test_invalid_chainstate_scenarios() + self.test_invalid_file_path() + self.test_snapshot_block_invalidated(dump_output['path']) self.log.info(f"Loading snapshot into second node from {dump_output['path']}") loaded = n1.loadtxoutset(dump_output['path']) @@ -247,21 +341,35 @@ class AssumeutxoTest(BitcoinTestFramework): the snapshot, and final values after the snapshot is validated.""" for height, block in blocks.items(): tx = n1.getblockheader(block.hash)["nTx"] - chain_tx = n1.getchaintxstats(nblocks=1, blockhash=block.hash)["txcount"] + stats = n1.getchaintxstats(nblocks=1, blockhash=block.hash) + chain_tx = stats.get("txcount", None) + window_tx_count = stats.get("window_tx_count", None) + tx_rate = stats.get("txrate", None) + window_interval = stats.get("window_interval") # Intermediate nTx of the starting block should be set, but nTx of # later blocks should be 0 before they are downloaded. + # The window_tx_count of one block is equal to the blocks tx count. + # If the window tx count is unknown, the value is missing. + # The tx_rate is calculated from window_tx_count and window_interval + # when possible. if final or height == START_HEIGHT: assert_equal(tx, block.tx) + assert_equal(window_tx_count, tx) + if window_interval > 0: + assert_approx(tx_rate, window_tx_count / window_interval, vspan=0.1) + else: + assert_equal(tx_rate, None) else: assert_equal(tx, 0) + assert_equal(window_tx_count, None) # Intermediate nChainTx of the starting block and snapshot block - # should be set, but others should be 0 until they are downloaded. + # should be set, but others should be None until they are downloaded. if final or height in (START_HEIGHT, SNAPSHOT_BASE_HEIGHT): assert_equal(chain_tx, block.chain_tx) else: - assert_equal(chain_tx, 0) + assert_equal(chain_tx, None) check_tx_counts(final=False) @@ -395,9 +503,13 @@ class AssumeutxoTest(BitcoinTestFramework): assert_equal(snapshot['snapshot_blockhash'], dump_output['base_hash']) assert_equal(snapshot['validated'], False) + self.log.info("Check that loading the snapshot again will fail because there is already an active snapshot.") + msg = "Unable to load UTXO snapshot: Can't activate a snapshot-based chainstate more than once" + assert_raises_rpc_error(-32603, msg, n2.loadtxoutset, dump_output['path']) + self.connect_nodes(0, 2) self.wait_until(lambda: n2.getchainstates()['chainstates'][-1]['blocks'] == FINAL_HEIGHT) - self.sync_blocks() + self.sync_blocks(nodes=(n0, n2)) self.log.info("Ensuring background validation completes") self.wait_until(lambda: len(n2.getchainstates()['chainstates']) == 1) @@ -434,6 +546,8 @@ class AssumeutxoTest(BitcoinTestFramework): self.connect_nodes(0, 2) self.wait_until(lambda: n2.getblockcount() == FINAL_HEIGHT) + self.test_snapshot_in_a_divergent_chain(dump_output['path']) + @dataclass class Block: hash: str diff --git a/test/functional/feature_bip68_sequence.py b/test/functional/feature_bip68_sequence.py index 8768d4040d..14b92d6733 100755 --- a/test/functional/feature_bip68_sequence.py +++ b/test/functional/feature_bip68_sequence.py @@ -78,8 +78,8 @@ class BIP68Test(BitcoinTestFramework): self.log.info("Activating BIP68 (and 112/113)") self.activateCSV() - self.log.info("Verifying nVersion=2 transactions are standard.") - self.log.info("Note that nVersion=2 transactions are always standard (independent of BIP68 activation status).") + self.log.info("Verifying version=2 transactions are standard.") + self.log.info("Note that version=2 transactions are always standard (independent of BIP68 activation status).") self.test_version2_relay() self.log.info("Passed") @@ -107,7 +107,7 @@ class BIP68Test(BitcoinTestFramework): # This transaction will enable sequence-locks, so this transaction should # fail tx2 = CTransaction() - tx2.nVersion = 2 + tx2.version = 2 sequence_value = sequence_value & 0x7fffffff tx2.vin = [CTxIn(COutPoint(tx1_id, 0), nSequence=sequence_value)] tx2.wit.vtxinwit = [CTxInWitness()] @@ -119,7 +119,7 @@ class BIP68Test(BitcoinTestFramework): # Setting the version back down to 1 should disable the sequence lock, # so this should be accepted. - tx2.nVersion = 1 + tx2.version = 1 self.wallet.sendrawtransaction(from_node=self.nodes[0], tx_hex=tx2.serialize().hex()) @@ -159,7 +159,7 @@ class BIP68Test(BitcoinTestFramework): using_sequence_locks = False tx = CTransaction() - tx.nVersion = 2 + tx.version = 2 value = 0 for j in range(num_inputs): sequence_value = 0xfffffffe # this disables sequence locks @@ -228,7 +228,7 @@ class BIP68Test(BitcoinTestFramework): # Anyone-can-spend mempool tx. # Sequence lock of 0 should pass. tx2 = CTransaction() - tx2.nVersion = 2 + tx2.version = 2 tx2.vin = [CTxIn(COutPoint(tx1.sha256, 0), nSequence=0)] tx2.vout = [CTxOut(int(tx1.vout[0].nValue - self.relayfee * COIN), SCRIPT_W0_SH_OP_TRUE)] self.wallet.sign_tx(tx=tx2) @@ -246,7 +246,7 @@ class BIP68Test(BitcoinTestFramework): sequence_value |= SEQUENCE_LOCKTIME_TYPE_FLAG tx = CTransaction() - tx.nVersion = 2 + tx.version = 2 tx.vin = [CTxIn(COutPoint(orig_tx.sha256, 0), nSequence=sequence_value)] tx.wit.vtxinwit = [CTxInWitness()] tx.wit.vtxinwit[0].scriptWitness.stack = [CScript([OP_TRUE])] @@ -360,7 +360,7 @@ class BIP68Test(BitcoinTestFramework): # Make an anyone-can-spend transaction tx2 = CTransaction() - tx2.nVersion = 1 + tx2.version = 1 tx2.vin = [CTxIn(COutPoint(tx1.sha256, 0), nSequence=0)] tx2.vout = [CTxOut(int(tx1.vout[0].nValue - self.relayfee * COIN), SCRIPT_W0_SH_OP_TRUE)] @@ -376,7 +376,7 @@ class BIP68Test(BitcoinTestFramework): sequence_value = 100 # 100 block relative locktime tx3 = CTransaction() - tx3.nVersion = 2 + tx3.version = 2 tx3.vin = [CTxIn(COutPoint(tx2.sha256, 0), nSequence=sequence_value)] tx3.wit.vtxinwit = [CTxInWitness()] tx3.wit.vtxinwit[0].scriptWitness.stack = [CScript([OP_TRUE])] diff --git a/test/functional/feature_block.py b/test/functional/feature_block.py index 8a95975184..932f37a083 100755 --- a/test/functional/feature_block.py +++ b/test/functional/feature_block.py @@ -4,7 +4,6 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test block processing.""" import copy -import struct import time from test_framework.blocktools import ( @@ -67,7 +66,7 @@ class CBrokenBlock(CBlock): def serialize(self, with_witness=False): r = b"" r += super(CBlock, self).serialize() - r += struct.pack("<BQ", 255, len(self.vtx)) + r += (255).to_bytes(1, "little") + len(self.vtx).to_bytes(8, "little") for tx in self.vtx: if with_witness: r += tx.serialize_with_witness() diff --git a/test/functional/feature_coinstatsindex.py b/test/functional/feature_coinstatsindex.py index d6c1567e64..691163d053 100755 --- a/test/functional/feature_coinstatsindex.py +++ b/test/functional/feature_coinstatsindex.py @@ -242,6 +242,9 @@ class CoinStatsIndexTest(BitcoinTestFramework): res12 = index_node.gettxoutsetinfo('muhash') assert_equal(res12, res10) + self.log.info("Test obtaining info for a non-existent block hash") + assert_raises_rpc_error(-5, "Block not found", index_node.gettxoutsetinfo, hash_type="none", hash_or_height="ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", use_index=True) + def _test_use_index_option(self): self.log.info("Test use_index option for nodes running the index") diff --git a/test/functional/feature_csv_activation.py b/test/functional/feature_csv_activation.py index bc1f9e8f2f..2db9682931 100755 --- a/test/functional/feature_csv_activation.py +++ b/test/functional/feature_csv_activation.py @@ -110,7 +110,7 @@ class BIP68_112_113Test(BitcoinTestFramework): def create_bip112special(self, input, txversion): tx = self.create_self_transfer_from_utxo(input) - tx.nVersion = txversion + tx.version = txversion self.miniwallet.sign_tx(tx) tx.vin[0].scriptSig = CScript([-1, OP_CHECKSEQUENCEVERIFY, OP_DROP] + list(CScript(tx.vin[0].scriptSig))) tx.rehash() @@ -118,7 +118,7 @@ class BIP68_112_113Test(BitcoinTestFramework): def create_bip112emptystack(self, input, txversion): tx = self.create_self_transfer_from_utxo(input) - tx.nVersion = txversion + tx.version = txversion self.miniwallet.sign_tx(tx) tx.vin[0].scriptSig = CScript([OP_CHECKSEQUENCEVERIFY] + list(CScript(tx.vin[0].scriptSig))) tx.rehash() @@ -136,7 +136,7 @@ class BIP68_112_113Test(BitcoinTestFramework): for i, (sdf, srhb, stf, srlb) in enumerate(product(*[[True, False]] * 4)): locktime = relative_locktime(sdf, srhb, stf, srlb) tx = self.create_self_transfer_from_utxo(bip68inputs[i]) - tx.nVersion = txversion + tx.version = txversion tx.vin[0].nSequence = locktime + locktime_delta self.miniwallet.sign_tx(tx) txs.append({'tx': tx, 'sdf': sdf, 'stf': stf}) @@ -154,7 +154,7 @@ class BIP68_112_113Test(BitcoinTestFramework): tx.vin[0].nSequence = BASE_RELATIVE_LOCKTIME + locktime_delta else: # vary nSequence instead, OP_CSV is fixed tx.vin[0].nSequence = locktime + locktime_delta - tx.nVersion = txversion + tx.version = txversion self.miniwallet.sign_tx(tx) if varyOP_CSV: tx.vin[0].scriptSig = CScript([locktime, OP_CHECKSEQUENCEVERIFY, OP_DROP] + list(CScript(tx.vin[0].scriptSig))) @@ -257,10 +257,10 @@ class BIP68_112_113Test(BitcoinTestFramework): # BIP113 test transaction will be modified before each use to put in appropriate block time bip113tx_v1 = self.create_self_transfer_from_utxo(bip113input) bip113tx_v1.vin[0].nSequence = 0xFFFFFFFE - bip113tx_v1.nVersion = 1 + bip113tx_v1.version = 1 bip113tx_v2 = self.create_self_transfer_from_utxo(bip113input) bip113tx_v2.vin[0].nSequence = 0xFFFFFFFE - bip113tx_v2.nVersion = 2 + bip113tx_v2.version = 2 # For BIP68 test all 16 relative sequence locktimes bip68txs_v1 = self.create_bip68txs(bip68inputs, 1) diff --git a/test/functional/feature_framework_miniwallet.py b/test/functional/feature_framework_miniwallet.py new file mode 100755 index 0000000000..f108289018 --- /dev/null +++ b/test/functional/feature_framework_miniwallet.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +# Copyright (c) 2024 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test MiniWallet.""" +from test_framework.blocktools import COINBASE_MATURITY +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_greater_than_or_equal, +) +from test_framework.wallet import ( + MiniWallet, + MiniWalletMode, +) + + +class FeatureFrameworkMiniWalletTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + + def test_tx_padding(self): + """Verify that MiniWallet's transaction padding (`target_weight` parameter) + works accurately enough (i.e. at most 3 WUs higher) with all modes.""" + for mode_name, wallet in self.wallets: + self.log.info(f"Test tx padding with MiniWallet mode {mode_name}...") + utxo = wallet.get_utxo(mark_as_spent=False) + for target_weight in [1000, 2000, 5000, 10000, 20000, 50000, 100000, 200000, 4000000, + 989, 2001, 4337, 13371, 23219, 49153, 102035, 223419, 3999989]: + tx = wallet.create_self_transfer(utxo_to_spend=utxo, target_weight=target_weight)["tx"] + self.log.debug(f"-> target weight: {target_weight}, actual weight: {tx.get_weight()}") + assert_greater_than_or_equal(tx.get_weight(), target_weight) + assert_greater_than_or_equal(target_weight + 3, tx.get_weight()) + + def run_test(self): + node = self.nodes[0] + self.wallets = [ + ("ADDRESS_OP_TRUE", MiniWallet(node, mode=MiniWalletMode.ADDRESS_OP_TRUE)), + ("RAW_OP_TRUE", MiniWallet(node, mode=MiniWalletMode.RAW_OP_TRUE)), + ("RAW_P2PK", MiniWallet(node, mode=MiniWalletMode.RAW_P2PK)), + ] + for _, wallet in self.wallets: + self.generate(wallet, 10) + self.generate(wallet, COINBASE_MATURITY) + + self.test_tx_padding() + + +if __name__ == '__main__': + FeatureFrameworkMiniWalletTest().main() diff --git a/test/functional/feature_framework_unit_tests.py b/test/functional/feature_framework_unit_tests.py index c9754e083c..14d83f8a70 100755 --- a/test/functional/feature_framework_unit_tests.py +++ b/test/functional/feature_framework_unit_tests.py @@ -25,7 +25,9 @@ TEST_FRAMEWORK_MODULES = [ "crypto.muhash", "crypto.poly1305", "crypto.ripemd160", + "crypto.secp256k1", "script", + "script_util", "segwit_addr", "wallet_util", ] diff --git a/test/functional/feature_init.py b/test/functional/feature_init.py index 268009b0f4..22ae0c307b 100755 --- a/test/functional/feature_init.py +++ b/test/functional/feature_init.py @@ -85,7 +85,7 @@ class InitStressTest(BitcoinTestFramework): for terminate_line in lines_to_terminate_after: self.log.info(f"Starting node and will exit after line {terminate_line}") - with node.wait_for_debug_log([terminate_line]): + with node.busy_wait_for_debug_log([terminate_line]): node.start(extra_args=['-txindex=1', '-blockfilterindex=1', '-coinstatsindex=1']) self.log.debug("Terminating node after terminate line was found") sigterm_node() diff --git a/test/functional/feature_maxtipage.py b/test/functional/feature_maxtipage.py index 51f37ef1e0..a1774a5395 100755 --- a/test/functional/feature_maxtipage.py +++ b/test/functional/feature_maxtipage.py @@ -43,6 +43,10 @@ class MaxTipAgeTest(BitcoinTestFramework): self.generate(node_miner, 1) assert_equal(node_ibd.getblockchaininfo()['initialblockdownload'], False) + # reset time to system time so we don't have a time offset with the ibd node the next + # time we connect to it, ensuring TimeOffsets::WarnIfOutOfSync() doesn't output to stderr + node_miner.setmocktime(0) + def run_test(self): self.log.info("Test IBD with maximum tip age of 24 hours (default).") self.test_maxtipage(DEFAULT_MAX_TIP_AGE, set_parameter=False) diff --git a/test/functional/feature_rbf.py b/test/functional/feature_rbf.py index c5eeaf66e0..739b9b9bb9 100755 --- a/test/functional/feature_rbf.py +++ b/test/functional/feature_rbf.py @@ -28,7 +28,6 @@ class ReplaceByFeeTest(BitcoinTestFramework): self.num_nodes = 2 self.extra_args = [ [ - "-maxorphantx=1000", "-limitancestorcount=50", "-limitancestorsize=101", "-limitdescendantcount=200", diff --git a/test/functional/feature_reindex.py b/test/functional/feature_reindex.py index f0f32a61ab..835cd0c5cf 100755 --- a/test/functional/feature_reindex.py +++ b/test/functional/feature_reindex.py @@ -73,6 +73,25 @@ class ReindexTest(BitcoinTestFramework): # All blocks should be accepted and processed. assert_equal(self.nodes[0].getblockcount(), 12) + def continue_reindex_after_shutdown(self): + node = self.nodes[0] + self.generate(node, 1500) + + # Restart node with reindex and stop reindex as soon as it starts reindexing + self.log.info("Restarting node while reindexing..") + node.stop_node() + with node.busy_wait_for_debug_log([b'initload thread start']): + node.start(['-blockfilterindex', '-reindex']) + node.wait_for_rpc_connection(wait_for_import=False) + node.stop_node() + + # Start node without the reindex flag and verify it does not wipe the indexes data again + db_path = node.chain_path / 'indexes' / 'blockfilter' / 'basic' / 'db' + with node.assert_debug_log(expected_msgs=[f'Opening LevelDB in {db_path}'], unexpected_msgs=[f'Wiping LevelDB in {db_path}']): + node.start(['-blockfilterindex']) + node.wait_for_rpc_connection(wait_for_import=False) + node.stop_node() + def run_test(self): self.reindex(False) self.reindex(True) @@ -80,6 +99,7 @@ class ReindexTest(BitcoinTestFramework): self.reindex(True) self.out_of_order() + self.continue_reindex_after_shutdown() if __name__ == '__main__': diff --git a/test/functional/feature_reindex_readonly.py b/test/functional/feature_reindex_readonly.py index 25cff87a3b..52c0bb26a6 100755 --- a/test/functional/feature_reindex_readonly.py +++ b/test/functional/feature_reindex_readonly.py @@ -75,7 +75,7 @@ class BlockstoreReindexTest(BitcoinTestFramework): if undo_immutable: self.log.debug("Attempt to restart and reindex the node with the unwritable block file") - with self.nodes[0].wait_for_debug_log([b"Reindexing finished"]): + with self.nodes[0].assert_debug_log(["Reindexing finished"], timeout=60): self.start_node(0, extra_args=['-reindex', '-fastprune']) assert block_count == self.nodes[0].getblockcount() undo_immutable() diff --git a/test/functional/feature_settings.py b/test/functional/feature_settings.py index 0214e781de..1cd0aeabd3 100755 --- a/test/functional/feature_settings.py +++ b/test/functional/feature_settings.py @@ -25,7 +25,7 @@ class SettingsTest(BitcoinTestFramework): # Assert default settings file was created self.stop_node(0) - default_settings = {"_warning_": "This file is automatically generated and updated by Bitcoin Core. Please do not edit this file while the node is running, as any changes might be ignored or overwritten."} + default_settings = {"_warning_": f"This file is automatically generated and updated by {self.config['environment']['PACKAGE_NAME']}. Please do not edit this file while the node is running, as any changes might be ignored or overwritten."} with settings.open() as fp: assert_equal(json.load(fp), default_settings) diff --git a/test/functional/feature_taproot.py b/test/functional/feature_taproot.py index e85541d0ec..1a0844d240 100755 --- a/test/functional/feature_taproot.py +++ b/test/functional/feature_taproot.py @@ -10,7 +10,6 @@ from test_framework.blocktools import ( create_block, add_witness_commitment, MAX_BLOCK_SIGOPS_WEIGHT, - WITNESS_SCALE_FACTOR, ) from test_framework.messages import ( COutPoint, @@ -20,6 +19,7 @@ from test_framework.messages import ( CTxOut, SEQUENCE_FINAL, tx_from_hex, + WITNESS_SCALE_FACTOR, ) from test_framework.script import ( ANNEX_TAG, @@ -1408,10 +1408,10 @@ class TaprootTest(BitcoinTestFramework): left = done while left: - # Construct CTransaction with random nVersion, nLocktime + # Construct CTransaction with random version, nLocktime tx = CTransaction() - tx.nVersion = random.choice([1, 2, random.randint(-0x80000000, 0x7fffffff)]) - min_sequence = (tx.nVersion != 1 and tx.nVersion != 0) * 0x80000000 # The minimum sequence number to disable relative locktime + tx.version = random.choice([1, 2, random.getrandbits(32)]) + min_sequence = (tx.version != 1 and tx.version != 0) * 0x80000000 # The minimum sequence number to disable relative locktime if random.choice([True, False]): tx.nLockTime = random.randrange(LOCKTIME_THRESHOLD, self.lastblocktime - 7200) # all absolute locktimes in the past else: @@ -1502,8 +1502,8 @@ class TaprootTest(BitcoinTestFramework): is_standard_tx = ( fail_input is None # Must be valid to be standard and (all(utxo.spender.is_standard for utxo in input_utxos)) # All inputs must be standard - and tx.nVersion >= 1 # The tx version must be standard - and tx.nVersion <= 2) + and tx.version >= 1 # The tx version must be standard + and tx.version <= 2) tx.rehash() msg = ','.join(utxo.spender.comment + ("*" if n == fail_input else "") for n, utxo in enumerate(input_utxos)) if is_standard_tx: @@ -1530,7 +1530,7 @@ class TaprootTest(BitcoinTestFramework): # Deterministically mine coins to OP_TRUE in block 1 assert_equal(self.nodes[0].getblockcount(), 0) coinbase = CTransaction() - coinbase.nVersion = 1 + coinbase.version = 1 coinbase.vin = [CTxIn(COutPoint(0, 0xffffffff), CScript([OP_1, OP_1]), SEQUENCE_FINAL)] coinbase.vout = [CTxOut(5000000000, CScript([OP_1]))] coinbase.nLockTime = 0 @@ -1622,7 +1622,7 @@ class TaprootTest(BitcoinTestFramework): for i, spk in enumerate(old_spks + tap_spks): val = 42000000 * (i + 7) tx = CTransaction() - tx.nVersion = 1 + tx.version = 1 tx.vin = [CTxIn(COutPoint(lasttxid, i & 1), CScript([]), SEQUENCE_FINAL)] tx.vout = [CTxOut(val, spk), CTxOut(amount - val, CScript([OP_1]))] if i & 1: @@ -1679,7 +1679,7 @@ class TaprootTest(BitcoinTestFramework): # Construct a deterministic transaction spending all outputs created above. tx = CTransaction() - tx.nVersion = 2 + tx.version = 2 tx.vin = [] inputs = [] input_spks = [tap_spks[0], tap_spks[1], old_spks[0], tap_spks[2], tap_spks[5], old_spks[2], tap_spks[6], tap_spks[3], tap_spks[4]] diff --git a/test/functional/feature_versionbits_warning.py b/test/functional/feature_versionbits_warning.py index 073d3de812..2c330eb681 100755 --- a/test/functional/feature_versionbits_warning.py +++ b/test/functional/feature_versionbits_warning.py @@ -73,8 +73,8 @@ class VersionBitsWarningTest(BitcoinTestFramework): self.generatetoaddress(node, VB_PERIOD - VB_THRESHOLD + 1, node_deterministic_address) # Check that we're not getting any versionbit-related errors in get*info() - assert not VB_PATTERN.match(node.getmininginfo()["warnings"]) - assert not VB_PATTERN.match(node.getnetworkinfo()["warnings"]) + assert not VB_PATTERN.match(",".join(node.getmininginfo()["warnings"])) + assert not VB_PATTERN.match(",".join(node.getnetworkinfo()["warnings"])) # Build one period of blocks with VB_THRESHOLD blocks signaling some unknown bit self.send_blocks_with_version(peer, VB_THRESHOLD, VB_UNKNOWN_VERSION) @@ -94,8 +94,8 @@ class VersionBitsWarningTest(BitcoinTestFramework): # Generating one more block will be enough to generate an error. self.generatetoaddress(node, 1, node_deterministic_address) # Check that get*info() shows the versionbits unknown rules warning - assert WARN_UNKNOWN_RULES_ACTIVE in node.getmininginfo()["warnings"] - assert WARN_UNKNOWN_RULES_ACTIVE in node.getnetworkinfo()["warnings"] + assert WARN_UNKNOWN_RULES_ACTIVE in ",".join(node.getmininginfo()["warnings"]) + assert WARN_UNKNOWN_RULES_ACTIVE in ",".join(node.getnetworkinfo()["warnings"]) # Check that the alert file shows the versionbits unknown rules warning self.wait_until(lambda: self.versionbits_in_alert_file()) diff --git a/test/functional/interface_bitcoin_cli.py b/test/functional/interface_bitcoin_cli.py index 83bb5121e5..a6628dcbf3 100755 --- a/test/functional/interface_bitcoin_cli.py +++ b/test/functional/interface_bitcoin_cli.py @@ -8,6 +8,7 @@ from decimal import Decimal import re from test_framework.blocktools import COINBASE_MATURITY +from test_framework.netutil import test_ipv6_local from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, @@ -15,6 +16,7 @@ from test_framework.util import ( assert_raises_process_error, assert_raises_rpc_error, get_auth_cookie, + rpc_port, ) import time @@ -107,6 +109,53 @@ class TestBitcoinCli(BitcoinTestFramework): self.log.info("Test connecting to a non-existing server") assert_raises_process_error(1, "Could not connect to the server", self.nodes[0].cli('-rpcport=1').echo) + self.log.info("Test handling of invalid ports in rpcconnect") + assert_raises_process_error(1, "Invalid port provided in -rpcconnect: 127.0.0.1:notaport", self.nodes[0].cli("-rpcconnect=127.0.0.1:notaport").echo) + assert_raises_process_error(1, "Invalid port provided in -rpcconnect: 127.0.0.1:-1", self.nodes[0].cli("-rpcconnect=127.0.0.1:-1").echo) + assert_raises_process_error(1, "Invalid port provided in -rpcconnect: 127.0.0.1:0", self.nodes[0].cli("-rpcconnect=127.0.0.1:0").echo) + assert_raises_process_error(1, "Invalid port provided in -rpcconnect: 127.0.0.1:65536", self.nodes[0].cli("-rpcconnect=127.0.0.1:65536").echo) + + self.log.info("Checking for IPv6") + have_ipv6 = test_ipv6_local() + if not have_ipv6: + self.log.info("Skipping IPv6 tests") + + if have_ipv6: + assert_raises_process_error(1, "Invalid port provided in -rpcconnect: [::1]:notaport", self.nodes[0].cli("-rpcconnect=[::1]:notaport").echo) + assert_raises_process_error(1, "Invalid port provided in -rpcconnect: [::1]:-1", self.nodes[0].cli("-rpcconnect=[::1]:-1").echo) + assert_raises_process_error(1, "Invalid port provided in -rpcconnect: [::1]:0", self.nodes[0].cli("-rpcconnect=[::1]:0").echo) + assert_raises_process_error(1, "Invalid port provided in -rpcconnect: [::1]:65536", self.nodes[0].cli("-rpcconnect=[::1]:65536").echo) + + self.log.info("Test handling of invalid ports in rpcport") + assert_raises_process_error(1, "Invalid port provided in -rpcport: notaport", self.nodes[0].cli("-rpcport=notaport").echo) + assert_raises_process_error(1, "Invalid port provided in -rpcport: -1", self.nodes[0].cli("-rpcport=-1").echo) + assert_raises_process_error(1, "Invalid port provided in -rpcport: 0", self.nodes[0].cli("-rpcport=0").echo) + assert_raises_process_error(1, "Invalid port provided in -rpcport: 65536", self.nodes[0].cli("-rpcport=65536").echo) + + self.log.info("Test port usage preferences") + node_rpc_port = rpc_port(self.nodes[0].index) + # Prevent bitcoin-cli from using existing rpcport in conf + conf_rpcport = "rpcport=" + str(node_rpc_port) + self.nodes[0].replace_in_config([(conf_rpcport, "#" + conf_rpcport)]) + # prefer rpcport over rpcconnect + assert_raises_process_error(1, "Could not connect to the server 127.0.0.1:1", self.nodes[0].cli(f"-rpcconnect=127.0.0.1:{node_rpc_port}", "-rpcport=1").echo) + if have_ipv6: + assert_raises_process_error(1, "Could not connect to the server ::1:1", self.nodes[0].cli(f"-rpcconnect=[::1]:{node_rpc_port}", "-rpcport=1").echo) + + assert_equal(BLOCKS, self.nodes[0].cli("-rpcconnect=127.0.0.1:18999", f'-rpcport={node_rpc_port}').getblockcount()) + if have_ipv6: + assert_equal(BLOCKS, self.nodes[0].cli("-rpcconnect=[::1]:18999", f'-rpcport={node_rpc_port}').getblockcount()) + + # prefer rpcconnect port over default + assert_equal(BLOCKS, self.nodes[0].cli(f"-rpcconnect=127.0.0.1:{node_rpc_port}").getblockcount()) + if have_ipv6: + assert_equal(BLOCKS, self.nodes[0].cli(f"-rpcconnect=[::1]:{node_rpc_port}").getblockcount()) + + # prefer rpcport over default + assert_equal(BLOCKS, self.nodes[0].cli(f'-rpcport={node_rpc_port}').getblockcount()) + # Re-enable rpcport in conf if present + self.nodes[0].replace_in_config([("#" + conf_rpcport, conf_rpcport)]) + self.log.info("Test connecting with non-existing RPC cookie file") assert_raises_process_error(1, "Could not locate RPC credentials", self.nodes[0].cli('-rpccookiefile=does-not-exist', '-rpcpassword=').echo) diff --git a/test/functional/interface_rpc.py b/test/functional/interface_rpc.py index e873e2da0b..6c1855c400 100755 --- a/test/functional/interface_rpc.py +++ b/test/functional/interface_rpc.py @@ -4,22 +4,79 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Tests some generic aspects of the RPC interface.""" +import json import os -from test_framework.authproxy import JSONRPCException +from dataclasses import dataclass from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal, assert_greater_than_or_equal from threading import Thread +from typing import Optional import subprocess -def expect_http_status(expected_http_status, expected_rpc_code, - fcn, *args): - try: - fcn(*args) - raise AssertionError(f"Expected RPC error {expected_rpc_code}, got none") - except JSONRPCException as exc: - assert_equal(exc.error["code"], expected_rpc_code) - assert_equal(exc.http_status, expected_http_status) +RPC_INVALID_PARAMETER = -8 +RPC_METHOD_NOT_FOUND = -32601 +RPC_INVALID_REQUEST = -32600 +RPC_PARSE_ERROR = -32700 + + +@dataclass +class BatchOptions: + version: Optional[int] = None + notification: bool = False + request_fields: Optional[dict] = None + response_fields: Optional[dict] = None + + +def format_request(options, idx, fields): + request = {} + if options.version == 1: + request.update(version="1.1") + elif options.version == 2: + request.update(jsonrpc="2.0") + elif options.version is not None: + raise NotImplementedError(f"Unknown JSONRPC version {options.version}") + if not options.notification: + request.update(id=idx) + request.update(fields) + if options.request_fields: + request.update(options.request_fields) + return request + + +def format_response(options, idx, fields): + if options.version == 2 and options.notification: + return None + response = {} + if not options.notification: + response.update(id=idx) + if options.version == 2: + response.update(jsonrpc="2.0") + else: + response.update(result=None, error=None) + response.update(fields) + if options.response_fields: + response.update(options.response_fields) + return response + + +def send_raw_rpc(node, raw_body: bytes) -> tuple[object, int]: + return node._request("POST", "/", raw_body) + + +def send_json_rpc(node, body: object) -> tuple[object, int]: + raw = json.dumps(body).encode("utf-8") + return send_raw_rpc(node, raw) + + +def expect_http_rpc_status(expected_http_status, expected_rpc_error_code, node, method, params, version=1, notification=False): + req = format_request(BatchOptions(version, notification), 0, {"method": method, "params": params}) + response, status = send_json_rpc(node, req) + + if expected_rpc_error_code is not None: + assert_equal(response["error"]["code"], expected_rpc_error_code) + + assert_equal(status, expected_http_status) def test_work_queue_getblock(node, got_exceeded_error): @@ -48,37 +105,126 @@ class RPCInterfaceTest(BitcoinTestFramework): assert_greater_than_or_equal(command['duration'], 0) assert_equal(info['logpath'], os.path.join(self.nodes[0].chain_path, 'debug.log')) - def test_batch_request(self): - self.log.info("Testing basic JSON-RPC batch request...") - - results = self.nodes[0].batch([ + def test_batch_request(self, call_options): + calls = [ # A basic request that will work fine. - {"method": "getblockcount", "id": 1}, + {"method": "getblockcount"}, # Request that will fail. The whole batch request should still # work fine. - {"method": "invalidmethod", "id": 2}, + {"method": "invalidmethod"}, # Another call that should succeed. - {"method": "getblockhash", "id": 3, "params": [0]}, - ]) - - result_by_id = {} - for res in results: - result_by_id[res["id"]] = res - - assert_equal(result_by_id[1]['error'], None) - assert_equal(result_by_id[1]['result'], 0) - - assert_equal(result_by_id[2]['error']['code'], -32601) - assert_equal(result_by_id[2]['result'], None) - - assert_equal(result_by_id[3]['error'], None) - assert result_by_id[3]['result'] is not None + {"method": "getblockhash", "params": [0]}, + # Invalid request format + {"pizza": "sausage"} + ] + results = [ + {"result": 0}, + {"error": {"code": RPC_METHOD_NOT_FOUND, "message": "Method not found"}}, + {"result": "0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206"}, + {"error": {"code": RPC_INVALID_REQUEST, "message": "Missing method"}}, + ] + + request = [] + response = [] + for idx, (call, result) in enumerate(zip(calls, results), 1): + options = call_options(idx) + if options is None: + continue + request.append(format_request(options, idx, call)) + r = format_response(options, idx, result) + if r is not None: + response.append(r) + + rpc_response, http_status = send_json_rpc(self.nodes[0], request) + if len(response) == 0 and len(request) > 0: + assert_equal(http_status, 204) + assert_equal(rpc_response, None) + else: + assert_equal(http_status, 200) + assert_equal(rpc_response, response) + + def test_batch_requests(self): + self.log.info("Testing empty batch request...") + self.test_batch_request(lambda idx: None) + + self.log.info("Testing basic JSON-RPC 2.0 batch request...") + self.test_batch_request(lambda idx: BatchOptions(version=2)) + + self.log.info("Testing JSON-RPC 2.0 batch with notifications...") + self.test_batch_request(lambda idx: BatchOptions(version=2, notification=idx < 2)) + + self.log.info("Testing JSON-RPC 2.0 batch of ALL notifications...") + self.test_batch_request(lambda idx: BatchOptions(version=2, notification=True)) + + # JSONRPC 1.1 does not support batch requests, but test them for backwards compatibility. + self.log.info("Testing nonstandard JSON-RPC 1.1 batch request...") + self.test_batch_request(lambda idx: BatchOptions(version=1)) + + self.log.info("Testing nonstandard mixed JSON-RPC 1.1/2.0 batch request...") + self.test_batch_request(lambda idx: BatchOptions(version=2 if idx % 2 else 1)) + + self.log.info("Testing nonstandard batch request without version numbers...") + self.test_batch_request(lambda idx: BatchOptions()) + + self.log.info("Testing nonstandard batch request without version numbers or ids...") + self.test_batch_request(lambda idx: BatchOptions(notification=True)) + + self.log.info("Testing nonstandard jsonrpc 1.0 version number is accepted...") + self.test_batch_request(lambda idx: BatchOptions(request_fields={"jsonrpc": "1.0"})) + + self.log.info("Testing unrecognized jsonrpc version number is rejected...") + self.test_batch_request(lambda idx: BatchOptions( + request_fields={"jsonrpc": "2.1"}, + response_fields={"result": None, "error": {"code": RPC_INVALID_REQUEST, "message": "JSON-RPC version not supported"}})) def test_http_status_codes(self): - self.log.info("Testing HTTP status codes for JSON-RPC requests...") - - expect_http_status(404, -32601, self.nodes[0].invalidmethod) - expect_http_status(500, -8, self.nodes[0].getblockhash, 42) + self.log.info("Testing HTTP status codes for JSON-RPC 1.1 requests...") + # OK + expect_http_rpc_status(200, None, self.nodes[0], "getblockhash", [0]) + # Errors + expect_http_rpc_status(404, RPC_METHOD_NOT_FOUND, self.nodes[0], "invalidmethod", []) + expect_http_rpc_status(500, RPC_INVALID_PARAMETER, self.nodes[0], "getblockhash", [42]) + # force-send empty request + response, status = send_raw_rpc(self.nodes[0], b"") + assert_equal(response, {"id": None, "result": None, "error": {"code": RPC_PARSE_ERROR, "message": "Parse error"}}) + assert_equal(status, 500) + # force-send invalidly formatted request + response, status = send_raw_rpc(self.nodes[0], b"this is bad") + assert_equal(response, {"id": None, "result": None, "error": {"code": RPC_PARSE_ERROR, "message": "Parse error"}}) + assert_equal(status, 500) + + self.log.info("Testing HTTP status codes for JSON-RPC 2.0 requests...") + # OK + expect_http_rpc_status(200, None, self.nodes[0], "getblockhash", [0], 2, False) + # RPC errors but not HTTP errors + expect_http_rpc_status(200, RPC_METHOD_NOT_FOUND, self.nodes[0], "invalidmethod", [], 2, False) + expect_http_rpc_status(200, RPC_INVALID_PARAMETER, self.nodes[0], "getblockhash", [42], 2, False) + # force-send invalidly formatted requests + response, status = send_json_rpc(self.nodes[0], {"jsonrpc": 2, "method": "getblockcount"}) + assert_equal(response, {"result": None, "error": {"code": RPC_INVALID_REQUEST, "message": "jsonrpc field must be a string"}}) + assert_equal(status, 400) + response, status = send_json_rpc(self.nodes[0], {"jsonrpc": "3.0", "method": "getblockcount"}) + assert_equal(response, {"result": None, "error": {"code": RPC_INVALID_REQUEST, "message": "JSON-RPC version not supported"}}) + assert_equal(status, 400) + + self.log.info("Testing HTTP status codes for JSON-RPC 2.0 notifications...") + # Not notification: id exists + response, status = send_json_rpc(self.nodes[0], {"jsonrpc": "2.0", "id": None, "method": "getblockcount"}) + assert_equal(response["result"], 0) + assert_equal(status, 200) + # Not notification: JSON 1.1 + expect_http_rpc_status(200, None, self.nodes[0], "getblockcount", [], 1) + # Not notification: has "id" field + expect_http_rpc_status(200, None, self.nodes[0], "getblockcount", [], 2, False) + block_count = self.nodes[0].getblockcount() + # Notification response status code: HTTP_NO_CONTENT + expect_http_rpc_status(204, None, self.nodes[0], "generatetoaddress", [1, "bcrt1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqdku202"], 2, True) + # The command worked even though there was no response + assert_equal(block_count + 1, self.nodes[0].getblockcount()) + # No error response for notifications even if they are invalid + expect_http_rpc_status(204, None, self.nodes[0], "generatetoaddress", [1, "invalid_address"], 2, True) + # Sanity check: command was not executed + assert_equal(block_count + 1, self.nodes[0].getblockcount()) def test_work_queue_exceeded(self): self.log.info("Testing work queue exceeded...") @@ -94,7 +240,7 @@ class RPCInterfaceTest(BitcoinTestFramework): def run_test(self): self.test_getrpcinfo() - self.test_batch_request() + self.test_batch_requests() self.test_http_status_codes() self.test_work_queue_exceeded() diff --git a/test/functional/mempool_accept.py b/test/functional/mempool_accept.py index 272e932fcc..e1cee46839 100755 --- a/test/functional/mempool_accept.py +++ b/test/functional/mempool_accept.py @@ -18,6 +18,7 @@ from test_framework.messages import ( CTxInWitness, CTxOut, MAX_BLOCK_WEIGHT, + WITNESS_SCALE_FACTOR, MAX_MONEY, SEQUENCE_FINAL, tx_from_hex, @@ -28,6 +29,8 @@ from test_framework.script import ( OP_HASH160, OP_RETURN, OP_TRUE, + SIGHASH_ALL, + sign_input_legacy, ) from test_framework.script_util import ( DUMMY_MIN_OP_RETURN_SCRIPT, @@ -226,7 +229,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): self.log.info('A really large transaction') tx = tx_from_hex(raw_tx_reference) - tx.vin = [tx.vin[0]] * math.ceil(MAX_BLOCK_WEIGHT // 4 / len(tx.vin[0].serialize())) + tx.vin = [tx.vin[0]] * math.ceil((MAX_BLOCK_WEIGHT // WITNESS_SCALE_FACTOR) / len(tx.vin[0].serialize())) self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': 'bad-txns-oversize'}], rawtxs=[tx.serialize().hex()], @@ -285,7 +288,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): self.log.info('Some nonstandard transactions') tx = tx_from_hex(raw_tx_reference) - tx.nVersion = 3 # A version currently non-standard + tx.version = 4 # A version currently non-standard self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': 'version'}], rawtxs=[tx.serialize().hex()], @@ -386,5 +389,24 @@ class MempoolAcceptanceTest(BitcoinTestFramework): maxfeerate=0, ) + self.log.info('Spending a confirmed bare multisig is okay') + address = self.wallet.get_address() + tx = tx_from_hex(raw_tx_reference) + privkey, pubkey = generate_keypair() + tx.vout[0].scriptPubKey = keys_to_multisig_script([pubkey] * 3, k=1) # Some bare multisig script (1-of-3) + tx.rehash() + self.generateblock(node, address, [tx.serialize().hex()]) + tx_spend = CTransaction() + tx_spend.vin.append(CTxIn(COutPoint(tx.sha256, 0), b"")) + tx_spend.vout.append(CTxOut(tx.vout[0].nValue - int(fee*COIN), script_to_p2wsh_script(CScript([OP_TRUE])))) + tx_spend.rehash() + sign_input_legacy(tx_spend, 0, tx.vout[0].scriptPubKey, privkey, sighash_type=SIGHASH_ALL) + tx_spend.vin[0].scriptSig = bytes(CScript([OP_0])) + tx_spend.vin[0].scriptSig + self.check_mempool_result( + result_expected=[{'txid': tx_spend.rehash(), 'allowed': True, 'vsize': tx_spend.get_vsize(), 'fees': { 'base': Decimal('0.00000700')}}], + rawtxs=[tx_spend.serialize().hex()], + maxfeerate=0, + ) + if __name__ == '__main__': MempoolAcceptanceTest().main() diff --git a/test/functional/mempool_limit.py b/test/functional/mempool_limit.py index e8a568f7ab..49a0a32c45 100755 --- a/test/functional/mempool_limit.py +++ b/test/functional/mempool_limit.py @@ -6,6 +6,9 @@ from decimal import Decimal +from test_framework.mempool_util import ( + fill_mempool, +) from test_framework.p2p import P2PTxInvStore from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( @@ -13,7 +16,6 @@ from test_framework.util import ( assert_fee_amount, assert_greater_than, assert_raises_rpc_error, - fill_mempool, ) from test_framework.wallet import ( COIN, @@ -57,7 +59,7 @@ class MempoolLimitTest(BitcoinTestFramework): mempoolmin_feerate = node.getmempoolinfo()["mempoolminfee"] tx_A = self.wallet.send_self_transfer( from_node=node, - fee=(mempoolmin_feerate / 1000) * (A_weight // 4) + Decimal('0.000001'), + fee_rate=mempoolmin_feerate, target_weight=A_weight, utxo_to_spend=rbf_utxo, confirmed_only=True @@ -75,7 +77,7 @@ class MempoolLimitTest(BitcoinTestFramework): non_cpfp_carveout_weight = 40001 # EXTRA_DESCENDANT_TX_SIZE_LIMIT + 1 tx_C = self.wallet.create_self_transfer( target_weight=non_cpfp_carveout_weight, - fee = (mempoolmin_feerate / 1000) * (non_cpfp_carveout_weight // 4) + Decimal('0.000001'), + fee_rate=mempoolmin_feerate, utxo_to_spend=tx_B["new_utxo"], confirmed_only=True ) @@ -93,7 +95,7 @@ class MempoolLimitTest(BitcoinTestFramework): assert_equal(node.getmempoolinfo()['minrelaytxfee'], Decimal('0.00001000')) assert_equal(node.getmempoolinfo()['mempoolminfee'], Decimal('0.00001000')) - fill_mempool(self, node, self.wallet) + fill_mempool(self, node) current_info = node.getmempoolinfo() mempoolmin_feerate = current_info["mempoolminfee"] @@ -107,7 +109,7 @@ class MempoolLimitTest(BitcoinTestFramework): # happen in the middle of package evaluation, as it can invalidate the coins cache. mempool_evicted_tx = self.wallet.send_self_transfer( from_node=node, - fee=(mempoolmin_feerate / 1000) * (evicted_weight // 4) + Decimal('0.000001'), + fee_rate=mempoolmin_feerate, target_weight=evicted_weight, confirmed_only=True ) @@ -133,11 +135,11 @@ class MempoolLimitTest(BitcoinTestFramework): parent_weight = 100000 num_big_parents = 3 assert_greater_than(parent_weight * num_big_parents, current_info["maxmempool"] - current_info["bytes"]) - parent_fee = (100 * mempoolmin_feerate / 1000) * (parent_weight // 4) + parent_feerate = 100 * mempoolmin_feerate big_parent_txids = [] for i in range(num_big_parents): - parent = self.wallet.create_self_transfer(fee=parent_fee, target_weight=parent_weight, confirmed_only=True) + parent = self.wallet.create_self_transfer(fee_rate=parent_feerate, target_weight=parent_weight, confirmed_only=True) parent_utxos.append(parent["new_utxo"]) package_hex.append(parent["hex"]) big_parent_txids.append(parent["txid"]) @@ -183,7 +185,7 @@ class MempoolLimitTest(BitcoinTestFramework): assert_equal(node.getmempoolinfo()['minrelaytxfee'], Decimal('0.00001000')) assert_equal(node.getmempoolinfo()['mempoolminfee'], Decimal('0.00001000')) - fill_mempool(self, node, self.wallet) + fill_mempool(self, node) current_info = node.getmempoolinfo() mempoolmin_feerate = current_info["mempoolminfee"] @@ -257,7 +259,7 @@ class MempoolLimitTest(BitcoinTestFramework): assert_equal(node.getmempoolinfo()['minrelaytxfee'], Decimal('0.00001000')) assert_equal(node.getmempoolinfo()['mempoolminfee'], Decimal('0.00001000')) - fill_mempool(self, node, self.wallet) + fill_mempool(self, node) # Deliberately try to create a tx with a fee less than the minimum mempool fee to assert that it does not get added to the mempool self.log.info('Create a mempool tx that will not pass mempoolminfee') @@ -312,18 +314,20 @@ class MempoolLimitTest(BitcoinTestFramework): target_weight_each = 200000 assert_greater_than(target_weight_each * 2, node.getmempoolinfo()["maxmempool"] - node.getmempoolinfo()["bytes"]) # Should be a true CPFP: parent's feerate is just below mempool min feerate - parent_fee = (mempoolmin_feerate / 1000) * (target_weight_each // 4) - Decimal("0.00001") + parent_feerate = mempoolmin_feerate - Decimal("0.000001") # 0.1 sats/vbyte below min feerate # Parent + child is above mempool minimum feerate - child_fee = (worst_feerate_btcvb) * (target_weight_each // 4) - Decimal("0.00001") + child_feerate = (worst_feerate_btcvb * 1000) - Decimal("0.000001") # 0.1 sats/vbyte below worst feerate # However, when eviction is triggered, these transactions should be at the bottom. # This assertion assumes parent and child are the same size. miniwallet.rescan_utxos() - tx_parent_just_below = miniwallet.create_self_transfer(fee=parent_fee, target_weight=target_weight_each) - tx_child_just_above = miniwallet.create_self_transfer(utxo_to_spend=tx_parent_just_below["new_utxo"], fee=child_fee, target_weight=target_weight_each) + tx_parent_just_below = miniwallet.create_self_transfer(fee_rate=parent_feerate, target_weight=target_weight_each) + tx_child_just_above = miniwallet.create_self_transfer(utxo_to_spend=tx_parent_just_below["new_utxo"], fee_rate=child_feerate, target_weight=target_weight_each) # This package ranks below the lowest descendant package in the mempool - assert_greater_than(worst_feerate_btcvb, (parent_fee + child_fee) / (tx_parent_just_below["tx"].get_vsize() + tx_child_just_above["tx"].get_vsize())) - assert_greater_than(mempoolmin_feerate, (parent_fee) / (tx_parent_just_below["tx"].get_vsize())) - assert_greater_than((parent_fee + child_fee) / (tx_parent_just_below["tx"].get_vsize() + tx_child_just_above["tx"].get_vsize()), mempoolmin_feerate / 1000) + package_fee = tx_parent_just_below["fee"] + tx_child_just_above["fee"] + package_vsize = tx_parent_just_below["tx"].get_vsize() + tx_child_just_above["tx"].get_vsize() + assert_greater_than(worst_feerate_btcvb, package_fee / package_vsize) + assert_greater_than(mempoolmin_feerate, tx_parent_just_below["fee"] / (tx_parent_just_below["tx"].get_vsize())) + assert_greater_than(package_fee / package_vsize, mempoolmin_feerate / 1000) res = node.submitpackage([tx_parent_just_below["hex"], tx_child_just_above["hex"]]) for wtxid in [tx_parent_just_below["wtxid"], tx_child_just_above["wtxid"]]: assert_equal(res["tx-results"][wtxid]["error"], "mempool full") diff --git a/test/functional/mempool_package_onemore.py b/test/functional/mempool_package_onemore.py index 921c190668..632425814a 100755 --- a/test/functional/mempool_package_onemore.py +++ b/test/functional/mempool_package_onemore.py @@ -21,7 +21,6 @@ from test_framework.wallet import MiniWallet class MempoolPackagesTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 - self.extra_args = [["-maxorphantx=1000"]] def chain_tx(self, utxos_to_spend, *, num_outputs=1): return self.wallet.send_self_transfer_multi( @@ -41,28 +40,37 @@ class MempoolPackagesTest(BitcoinTestFramework): for _ in range(DEFAULT_ANCESTOR_LIMIT - 4): utxo, = self.chain_tx([utxo]) chain.append(utxo) - second_chain, = self.chain_tx([self.wallet.get_utxo()]) + second_chain, = self.chain_tx([self.wallet.get_utxo(confirmed_only=True)]) # Check mempool has DEFAULT_ANCESTOR_LIMIT + 1 transactions in it assert_equal(len(self.nodes[0].getrawmempool()), DEFAULT_ANCESTOR_LIMIT + 1) # Adding one more transaction on to the chain should fail. assert_raises_rpc_error(-26, "too-long-mempool-chain, too many unconfirmed ancestors [limit: 25]", self.chain_tx, [utxo]) - # ...even if it chains on from some point in the middle of the chain. + # ... or if it chains on from some point in the middle of the chain. assert_raises_rpc_error(-26, "too-long-mempool-chain, too many descendants", self.chain_tx, [chain[2]]) assert_raises_rpc_error(-26, "too-long-mempool-chain, too many descendants", self.chain_tx, [chain[1]]) # ...even if it chains on to two parent transactions with one in the chain. assert_raises_rpc_error(-26, "too-long-mempool-chain, too many descendants", self.chain_tx, [chain[0], second_chain]) # ...especially if its > 40k weight assert_raises_rpc_error(-26, "too-long-mempool-chain, too many descendants", self.chain_tx, [chain[0]], num_outputs=350) + # ...even if it's submitted with other transactions + replaceable_tx = self.wallet.create_self_transfer_multi(utxos_to_spend=[chain[0]]) + txns = [replaceable_tx["tx"], self.wallet.create_self_transfer_multi(utxos_to_spend=replaceable_tx["new_utxos"])["tx"]] + txns_hex = [tx.serialize().hex() for tx in txns] + assert_equal(self.nodes[0].testmempoolaccept(txns_hex)[0]["reject-reason"], "too-long-mempool-chain") + pkg_result = self.nodes[0].submitpackage(txns_hex) + assert "too-long-mempool-chain" in pkg_result["tx-results"][txns[0].getwtxid()]["error"] + assert_equal(pkg_result["tx-results"][txns[1].getwtxid()]["error"], "bad-txns-inputs-missingorspent") # But not if it chains directly off the first transaction - replacable_tx = self.wallet.send_self_transfer_multi(from_node=self.nodes[0], utxos_to_spend=[chain[0]])['tx'] + self.nodes[0].sendrawtransaction(replaceable_tx["hex"]) # and the second chain should work just fine self.chain_tx([second_chain]) - # Make sure we can RBF the chain which used our carve-out rule - replacable_tx.vout[0].nValue -= 1000000 - self.nodes[0].sendrawtransaction(replacable_tx.serialize().hex()) + # Ensure an individual transaction with single direct conflict can RBF the chain which used our carve-out rule + replacement_tx = replaceable_tx["tx"] + replacement_tx.vout[0].nValue -= 1000000 + self.nodes[0].sendrawtransaction(replacement_tx.serialize().hex()) # Finally, check that we added two transactions assert_equal(len(self.nodes[0].getrawmempool()), DEFAULT_ANCESTOR_LIMIT + 3) diff --git a/test/functional/mempool_package_rbf.py b/test/functional/mempool_package_rbf.py new file mode 100755 index 0000000000..ceb9530394 --- /dev/null +++ b/test/functional/mempool_package_rbf.py @@ -0,0 +1,587 @@ +#!/usr/bin/env python3 +# Copyright (c) 2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +from decimal import Decimal + +from test_framework.messages import ( + COIN, + MAX_BIP125_RBF_SEQUENCE, +) +from test_framework.test_framework import BitcoinTestFramework +from test_framework.mempool_util import fill_mempool +from test_framework.util import ( + assert_greater_than_or_equal, + assert_equal, +) +from test_framework.wallet import ( + DEFAULT_FEE, + MiniWallet, +) + +MAX_REPLACEMENT_CANDIDATES = 100 + +# Value high enough to cause evictions in each subtest +# for typical cases +DEFAULT_CHILD_FEE = DEFAULT_FEE * 4 + +class PackageRBFTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 2 + self.setup_clean_chain = True + # Required for fill_mempool() + self.extra_args = [[ + "-datacarriersize=100000", + "-maxmempool=5", + ]] * self.num_nodes + + def assert_mempool_contents(self, expected=None): + """Assert that all transactions in expected are in the mempool, + and no additional ones exist. + """ + if not expected: + expected = [] + mempool = self.nodes[0].getrawmempool(verbose=False) + assert_equal(len(mempool), len(expected)) + for tx in expected: + assert tx.rehash() in mempool + + def create_simple_package(self, parent_coin, parent_fee=DEFAULT_FEE, child_fee=DEFAULT_CHILD_FEE, heavy_child=False): + """Create a 1 parent 1 child package using the coin passed in as the parent's input. The + parent has 1 output, used to fund 1 child transaction. + All transactions signal BIP125 replaceability, but nSequence changes based on self.ctr. This + prevents identical txids between packages when the parents spend the same coin and have the + same fee (i.e. 0sat). + + returns tuple (hex serialized txns, CTransaction objects) + """ + self.ctr += 1 + # Use fee_rate=0 because create_self_transfer will use the default fee_rate value otherwise. + # Passing in fee>0 overrides fee_rate, so this still works for non-zero parent_fee. + parent_result = self.wallet.create_self_transfer( + fee=parent_fee, + utxo_to_spend=parent_coin, + sequence=MAX_BIP125_RBF_SEQUENCE - self.ctr, + ) + + num_child_outputs = 10 if heavy_child else 1 + child_result = self.wallet.create_self_transfer_multi( + utxos_to_spend=[parent_result["new_utxo"]], + num_outputs=num_child_outputs, + fee_per_output=int(child_fee * COIN // num_child_outputs), + sequence=MAX_BIP125_RBF_SEQUENCE - self.ctr, + ) + package_hex = [parent_result["hex"], child_result["hex"]] + package_txns = [parent_result["tx"], child_result["tx"]] + return package_hex, package_txns + + def run_test(self): + # Counter used to count the number of times we constructed packages. Since we're constructing parent transactions with the same + # coins (to create conflicts), and perhaps giving them the same fee, we might accidentally just create the same transaction again. + # To prevent this, set nSequences to MAX_BIP125_RBF_SEQUENCE - self.ctr. + self.ctr = 0 + + self.log.info("Generate blocks to create UTXOs") + self.wallet = MiniWallet(self.nodes[0]) + + # Make more than enough coins for the sum of all tests, + # otherwise a wallet rescan is needed later + self.generate(self.wallet, 300) + self.coins = self.wallet.get_utxos(mark_as_spent=False) + + self.test_package_rbf_basic() + self.test_package_rbf_singleton() + self.test_package_rbf_additional_fees() + self.test_package_rbf_max_conflicts() + self.test_too_numerous_ancestors() + self.test_package_rbf_with_wrong_pkg_size() + self.test_insufficient_feerate() + self.test_wrong_conflict_cluster_size_linear() + self.test_wrong_conflict_cluster_size_parents_child() + self.test_wrong_conflict_cluster_size_parent_children() + self.test_0fee_package_rbf() + self.test_child_conflicts_parent_mempool_ancestor() + + def test_package_rbf_basic(self): + self.log.info("Test that a child can pay to replace its parents' conflicts of cluster size 2") + node = self.nodes[0] + # Reuse the same coins so that the transactions conflict with one another. + parent_coin = self.coins.pop() + package_hex1, package_txns1 = self.create_simple_package(parent_coin, DEFAULT_FEE, DEFAULT_FEE) + package_hex2, package_txns2 = self.create_simple_package(parent_coin, DEFAULT_FEE, DEFAULT_CHILD_FEE) + node.submitpackage(package_hex1) + self.assert_mempool_contents(expected=package_txns1) + + # Make sure 2nd node gets set up for basic package RBF + self.sync_all() + + # Test run rejected because conflicts are not allowed in subpackage evaluation + testres = node.testmempoolaccept(package_hex2) + assert_equal(testres[0]["reject-reason"], "bip125-replacement-disallowed") + + # But accepted during normal submission + submitres = node.submitpackage(package_hex2) + assert_equal(set(submitres["replaced-transactions"]), set([tx.rehash() for tx in package_txns1])) + self.assert_mempool_contents(expected=package_txns2) + + # Make sure 2nd node gets a basic package RBF over p2p + self.sync_all() + + self.generate(node, 1) + + def test_package_rbf_singleton(self): + self.log.info("Test child can pay to replace a parent's single conflicted tx") + node = self.nodes[0] + + # Make singleton tx to conflict with in next batch + singleton_coin = self.coins.pop() + singleton_tx = self.wallet.create_self_transfer(utxo_to_spend=singleton_coin) + node.sendrawtransaction(singleton_tx["hex"]) + self.assert_mempool_contents(expected=[singleton_tx["tx"]]) + + package_hex, package_txns = self.create_simple_package(singleton_coin, DEFAULT_FEE, singleton_tx["fee"] * 2) + + submitres = node.submitpackage(package_hex) + assert_equal(submitres["replaced-transactions"], [singleton_tx["tx"].rehash()]) + self.assert_mempool_contents(expected=package_txns) + + self.generate(node, 1) + + def test_package_rbf_additional_fees(self): + self.log.info("Check Package RBF must increase the absolute fee") + node = self.nodes[0] + coin = self.coins.pop() + + package_hex1, package_txns1 = self.create_simple_package(coin, parent_fee=DEFAULT_FEE, child_fee=DEFAULT_CHILD_FEE, heavy_child=True) + assert_greater_than_or_equal(1000, package_txns1[-1].get_vsize()) + node.submitpackage(package_hex1) + self.assert_mempool_contents(expected=package_txns1) + + PACKAGE_FEE = DEFAULT_FEE + DEFAULT_CHILD_FEE + PACKAGE_FEE_MINUS_ONE = PACKAGE_FEE - Decimal("0.00000001") + + # Package 2 has a higher feerate but lower absolute fee + package_hex2, package_txns2 = self.create_simple_package(coin, parent_fee=DEFAULT_FEE, child_fee=DEFAULT_CHILD_FEE - Decimal("0.00000001")) + pkg_results2 = node.submitpackage(package_hex2) + assert_equal(f"package RBF failed: insufficient anti-DoS fees, rejecting replacement {package_txns2[1].rehash()}, less fees than conflicting txs; {PACKAGE_FEE_MINUS_ONE} < {PACKAGE_FEE}", pkg_results2["package_msg"]) + self.assert_mempool_contents(expected=package_txns1) + + self.log.info("Check replacement pays for incremental bandwidth") + package_hex3, package_txns3 = self.create_simple_package(coin, parent_fee=DEFAULT_FEE, child_fee=DEFAULT_CHILD_FEE) + pkg_results3 = node.submitpackage(package_hex3) + assert_equal(f"package RBF failed: insufficient anti-DoS fees, rejecting replacement {package_txns3[1].rehash()}, not enough additional fees to relay; 0.00 < 0.00000{sum([tx.get_vsize() for tx in package_txns3])}", pkg_results3["package_msg"]) + + self.assert_mempool_contents(expected=package_txns1) + self.generate(node, 1) + + self.log.info("Check Package RBF must have strict cpfp structure") + coin = self.coins.pop() + package_hex4, package_txns4 = self.create_simple_package(coin, parent_fee=DEFAULT_FEE, child_fee=DEFAULT_CHILD_FEE) + node.submitpackage(package_hex4) + self.assert_mempool_contents(expected=package_txns4) + package_hex5, package_txns5 = self.create_simple_package(coin, parent_fee=DEFAULT_CHILD_FEE, child_fee=DEFAULT_CHILD_FEE - Decimal("0.00000001")) + pkg_results5 = node.submitpackage(package_hex5) + assert 'package RBF failed: package feerate is less than parent feerate' in pkg_results5["package_msg"] + + self.assert_mempool_contents(expected=package_txns4) + self.generate(node, 1) + + def test_package_rbf_max_conflicts(self): + node = self.nodes[0] + self.log.info("Check Package RBF cannot replace more than MAX_REPLACEMENT_CANDIDATES transactions") + num_coins = 51 + parent_coins = self.coins[:num_coins] + del self.coins[:num_coins] + + # Original transactions: 51 transactions with 1 descendants each -> 102 total transactions + size_two_clusters = [] + for coin in parent_coins: + size_two_clusters.append(self.wallet.send_self_transfer_chain(from_node=node, chain_length=2, utxo_to_spend=coin)) + expected_txns = [txn["tx"] for parent_child_txns in size_two_clusters for txn in parent_child_txns] + assert_equal(len(expected_txns), num_coins * 2) + self.assert_mempool_contents(expected=expected_txns) + + # parent feeerate needs to be high enough for minrelay + # child feerate needs to be large enough to trigger package rbf with a very large parent and + # pay for all evicted fees. maxfeerate turned off for all submissions since child feerate + # is extremely high + parent_fee_per_conflict = 10000 + child_feerate = 10000 * DEFAULT_FEE + + # Conflict against all transactions by double-spending each parent, causing 102 evictions + package_parent = self.wallet.create_self_transfer_multi(utxos_to_spend=parent_coins, fee_per_output=parent_fee_per_conflict) + package_child = self.wallet.create_self_transfer(fee_rate=child_feerate, utxo_to_spend=package_parent["new_utxos"][0]) + + pkg_results = node.submitpackage([package_parent["hex"], package_child["hex"]], maxfeerate=0) + assert_equal(f"package RBF failed: too many potential replacements, rejecting replacement {package_child['tx'].rehash()}; too many potential replacements (102 > 100)\n", pkg_results["package_msg"]) + self.assert_mempool_contents(expected=expected_txns) + + # Make singleton tx to conflict with in next batch + singleton_coin = self.coins.pop() + singleton_tx = self.wallet.create_self_transfer(utxo_to_spend=singleton_coin) + node.sendrawtransaction(singleton_tx["hex"]) + expected_txns.append(singleton_tx["tx"]) + + # Double-spend same set minus last, and double-spend singleton. This hits 101 evictions; should still fail. + # N.B. we can't RBF just a child tx in the clusters, as that would make resulting cluster of size 3. + double_spending_coins = parent_coins[:-1] + [singleton_coin] + package_parent = self.wallet.create_self_transfer_multi(utxos_to_spend=double_spending_coins, fee_per_output=parent_fee_per_conflict) + package_child = self.wallet.create_self_transfer(fee_rate=child_feerate, utxo_to_spend=package_parent["new_utxos"][0]) + pkg_results = node.submitpackage([package_parent["hex"], package_child["hex"]], maxfeerate=0) + assert_equal(f"package RBF failed: too many potential replacements, rejecting replacement {package_child['tx'].rehash()}; too many potential replacements (101 > 100)\n", pkg_results["package_msg"]) + self.assert_mempool_contents(expected=expected_txns) + + # Finally, evict MAX_REPLACEMENT_CANDIDATES + package_parent = self.wallet.create_self_transfer_multi(utxos_to_spend=parent_coins[:-1], fee_per_output=parent_fee_per_conflict) + package_child = self.wallet.create_self_transfer(fee_rate=child_feerate, utxo_to_spend=package_parent["new_utxos"][0]) + pkg_results = node.submitpackage([package_parent["hex"], package_child["hex"]], maxfeerate=0) + assert_equal(pkg_results["package_msg"], "success") + self.assert_mempool_contents(expected=[singleton_tx["tx"], size_two_clusters[-1][0]["tx"], size_two_clusters[-1][1]["tx"], package_parent["tx"], package_child["tx"]] ) + + self.generate(node, 1) + + def test_too_numerous_ancestors(self): + self.log.info("Test that package RBF doesn't work with packages larger than 2 due to ancestors") + node = self.nodes[0] + coin = self.coins.pop() + + package_hex1, package_txns1 = self.create_simple_package(coin, DEFAULT_FEE, DEFAULT_CHILD_FEE) + node.submitpackage(package_hex1) + self.assert_mempool_contents(expected=package_txns1) + + # Double-spends the original package + self.ctr += 1 + parent_result1 = self.wallet.create_self_transfer( + fee=DEFAULT_FEE, + utxo_to_spend=coin, + sequence=MAX_BIP125_RBF_SEQUENCE - self.ctr, + ) + + coin2 = self.coins.pop() + + # Added to make package too large for package RBF; + # it will enter mempool individually + self.ctr += 1 + parent_result2 = self.wallet.create_self_transfer( + fee=DEFAULT_FEE, + utxo_to_spend=coin2, + sequence=MAX_BIP125_RBF_SEQUENCE - self.ctr, + ) + + # Child that spends both, violating cluster size rule due + # to in-mempool ancestry + self.ctr += 1 + child_result = self.wallet.create_self_transfer_multi( + fee_per_output=int(DEFAULT_CHILD_FEE * COIN), + utxos_to_spend=[parent_result1["new_utxo"], parent_result2["new_utxo"]], + sequence=MAX_BIP125_RBF_SEQUENCE - self.ctr, + ) + + package_hex2 = [parent_result1["hex"], parent_result2["hex"], child_result["hex"]] + package_txns2_succeed = [parent_result2["tx"]] + + pkg_result = node.submitpackage(package_hex2) + assert_equal(pkg_result["package_msg"], 'package RBF failed: new transaction cannot have mempool ancestors') + self.assert_mempool_contents(expected=package_txns1 + package_txns2_succeed) + self.generate(node, 1) + + def test_wrong_conflict_cluster_size_linear(self): + self.log.info("Test that conflicting with a cluster not sized two is rejected: linear chain") + node = self.nodes[0] + + # Coins we will conflict with + coin1 = self.coins.pop() + coin2 = self.coins.pop() + coin3 = self.coins.pop() + + # Three transactions chained; package RBF against any of these + # should be rejected + self.ctr += 1 + parent_result = self.wallet.create_self_transfer( + fee=DEFAULT_FEE, + utxo_to_spend=coin1, + sequence=MAX_BIP125_RBF_SEQUENCE - self.ctr, + ) + + self.ctr += 1 + child_result = self.wallet.create_self_transfer_multi( + fee_per_output=int(DEFAULT_FEE * COIN), + utxos_to_spend=[parent_result["new_utxo"], coin2], + sequence=MAX_BIP125_RBF_SEQUENCE - self.ctr, + ) + + self.ctr += 1 + grandchild_result = self.wallet.create_self_transfer_multi( + fee_per_output=int(DEFAULT_FEE * COIN), + utxos_to_spend=[child_result["new_utxos"][0], coin3], + sequence=MAX_BIP125_RBF_SEQUENCE - self.ctr, + ) + + expected_txns = [parent_result["tx"], child_result["tx"], grandchild_result["tx"]] + for tx in expected_txns: + node.sendrawtransaction(tx.serialize().hex()) + self.assert_mempool_contents(expected=expected_txns) + + # Now make conflicting packages for each coin + package_hex1, package_txns1 = self.create_simple_package(coin1, DEFAULT_FEE, DEFAULT_CHILD_FEE) + + package_result = node.submitpackage(package_hex1) + assert_equal(f"package RBF failed: {parent_result['tx'].rehash()} has 2 descendants, max 1 allowed", package_result["package_msg"]) + + package_hex2, package_txns2 = self.create_simple_package(coin2, DEFAULT_FEE, DEFAULT_CHILD_FEE) + package_result = node.submitpackage(package_hex2) + assert_equal(f"package RBF failed: {child_result['tx'].rehash()} has both ancestor and descendant, exceeding cluster limit of 2", package_result["package_msg"]) + + package_hex3, package_txns3 = self.create_simple_package(coin3, DEFAULT_FEE, DEFAULT_CHILD_FEE) + package_result = node.submitpackage(package_hex3) + assert_equal(f"package RBF failed: {grandchild_result['tx'].rehash()} has 2 ancestors, max 1 allowed", package_result["package_msg"]) + + # Check that replacements were actually rejected + self.assert_mempool_contents(expected=expected_txns) + self.generate(node, 1) + + def test_wrong_conflict_cluster_size_parents_child(self): + self.log.info("Test that conflicting with a cluster not sized two is rejected: two parents one child") + node = self.nodes[0] + + # Coins we will conflict with + coin1 = self.coins.pop() + coin2 = self.coins.pop() + coin3 = self.coins.pop() + + self.ctr += 1 + parent1_result = self.wallet.create_self_transfer( + fee=DEFAULT_FEE, + utxo_to_spend=coin1, + sequence=MAX_BIP125_RBF_SEQUENCE - self.ctr, + ) + + self.ctr += 1 + parent2_result = self.wallet.create_self_transfer_multi( + fee_per_output=int(DEFAULT_FEE * COIN), + utxos_to_spend=[coin2], + sequence=MAX_BIP125_RBF_SEQUENCE - self.ctr, + ) + + self.ctr += 1 + child_result = self.wallet.create_self_transfer_multi( + fee_per_output=int(DEFAULT_FEE * COIN), + utxos_to_spend=[parent1_result["new_utxo"], parent2_result["new_utxos"][0], coin3], + sequence=MAX_BIP125_RBF_SEQUENCE - self.ctr, + ) + + expected_txns = [parent1_result["tx"], parent2_result["tx"], child_result["tx"]] + for tx in expected_txns: + node.sendrawtransaction(tx.serialize().hex()) + self.assert_mempool_contents(expected=expected_txns) + + # Now make conflicting packages for each coin + package_hex1, package_txns1 = self.create_simple_package(coin1, DEFAULT_FEE, DEFAULT_CHILD_FEE) + package_result = node.submitpackage(package_hex1) + assert_equal(f"package RBF failed: {parent1_result['tx'].rehash()} is not the only parent of child {child_result['tx'].rehash()}", package_result["package_msg"]) + + package_hex2, package_txns2 = self.create_simple_package(coin2, DEFAULT_FEE, DEFAULT_CHILD_FEE) + package_result = node.submitpackage(package_hex2) + assert_equal(f"package RBF failed: {parent2_result['tx'].rehash()} is not the only parent of child {child_result['tx'].rehash()}", package_result["package_msg"]) + + package_hex3, package_txns3 = self.create_simple_package(coin3, DEFAULT_FEE, DEFAULT_CHILD_FEE) + package_result = node.submitpackage(package_hex3) + assert_equal(f"package RBF failed: {child_result['tx'].rehash()} has 2 ancestors, max 1 allowed", package_result["package_msg"]) + + # Check that replacements were actually rejected + self.assert_mempool_contents(expected=expected_txns) + self.generate(node, 1) + + def test_wrong_conflict_cluster_size_parent_children(self): + self.log.info("Test that conflicting with a cluster not sized two is rejected: one parent two children") + node = self.nodes[0] + + # Coins we will conflict with + coin1 = self.coins.pop() + coin2 = self.coins.pop() + coin3 = self.coins.pop() + + self.ctr += 1 + parent_result = self.wallet.create_self_transfer_multi( + fee_per_output=int(DEFAULT_FEE * COIN), + num_outputs=2, + utxos_to_spend=[coin1], + sequence=MAX_BIP125_RBF_SEQUENCE - self.ctr, + ) + + self.ctr += 1 + child1_result = self.wallet.create_self_transfer_multi( + fee_per_output=int(DEFAULT_FEE * COIN), + utxos_to_spend=[parent_result["new_utxos"][0], coin2], + sequence=MAX_BIP125_RBF_SEQUENCE - self.ctr, + ) + + self.ctr += 1 + child2_result = self.wallet.create_self_transfer_multi( + fee_per_output=int(DEFAULT_FEE * COIN), + utxos_to_spend=[parent_result["new_utxos"][1], coin3], + sequence=MAX_BIP125_RBF_SEQUENCE - self.ctr, + ) + + # Submit them to mempool + expected_txns = [parent_result["tx"], child1_result["tx"], child2_result["tx"]] + for tx in expected_txns: + node.sendrawtransaction(tx.serialize().hex()) + self.assert_mempool_contents(expected=expected_txns) + + # Now make conflicting packages for each coin + package_hex1, package_txns1 = self.create_simple_package(coin1, DEFAULT_FEE, DEFAULT_CHILD_FEE) + package_result = node.submitpackage(package_hex1) + assert_equal(f"package RBF failed: {parent_result['tx'].rehash()} has 2 descendants, max 1 allowed", package_result["package_msg"]) + + package_hex2, package_txns2 = self.create_simple_package(coin2, DEFAULT_FEE, DEFAULT_CHILD_FEE) + package_result = node.submitpackage(package_hex2) + assert_equal(f"package RBF failed: {child1_result['tx'].rehash()} is not the only child of parent {parent_result['tx'].rehash()}", package_result["package_msg"]) + + package_hex3, package_txns3 = self.create_simple_package(coin3, DEFAULT_FEE, DEFAULT_CHILD_FEE) + package_result = node.submitpackage(package_hex3) + assert_equal(f"package RBF failed: {child2_result['tx'].rehash()} is not the only child of parent {parent_result['tx'].rehash()}", package_result["package_msg"]) + + # Check that replacements were actually rejected + self.assert_mempool_contents(expected=expected_txns) + self.generate(node, 1) + + def test_package_rbf_with_wrong_pkg_size(self): + self.log.info("Test that package RBF doesn't work with packages larger than 2 due to pkg size") + node = self.nodes[0] + coin1 = self.coins.pop() + coin2 = self.coins.pop() + + # Two packages to require multiple direct conflicts, easier to set up illicit pkg size + package_hex1, package_txns1 = self.create_simple_package(coin1, DEFAULT_FEE, DEFAULT_CHILD_FEE) + package_hex2, package_txns2 = self.create_simple_package(coin2, DEFAULT_FEE, DEFAULT_CHILD_FEE) + + node.submitpackage(package_hex1) + node.submitpackage(package_hex2) + + self.assert_mempool_contents(expected=package_txns1 + package_txns2) + assert_equal(len(node.getrawmempool()), 4) + + # Double-spends the first package + self.ctr += 1 + parent_result1 = self.wallet.create_self_transfer( + fee=DEFAULT_FEE, + utxo_to_spend=coin1, + sequence=MAX_BIP125_RBF_SEQUENCE - self.ctr, + ) + + # Double-spends the second package + self.ctr += 1 + parent_result2 = self.wallet.create_self_transfer( + fee=DEFAULT_FEE, + utxo_to_spend=coin2, + sequence=MAX_BIP125_RBF_SEQUENCE - self.ctr, + ) + + # Child that spends both, violating cluster size rule due + # to pkg size + self.ctr += 1 + child_result = self.wallet.create_self_transfer_multi( + fee_per_output=int(DEFAULT_CHILD_FEE * COIN), + utxos_to_spend=[parent_result1["new_utxo"], parent_result2["new_utxo"]], + sequence=MAX_BIP125_RBF_SEQUENCE - self.ctr, + ) + + package_hex3 = [parent_result1["hex"], parent_result2["hex"], child_result["hex"]] + + pkg_result = node.submitpackage(package_hex3) + assert_equal(pkg_result["package_msg"], 'package RBF failed: package must be 1-parent-1-child') + self.assert_mempool_contents(expected=package_txns1 + package_txns2) + self.generate(node, 1) + + def test_insufficient_feerate(self): + self.log.info("Check Package RBF must beat feerate of direct conflict") + node = self.nodes[0] + coin = self.coins.pop() + + # Non-cpfp structure + package_hex1, package_txns1 = self.create_simple_package(coin, parent_fee=DEFAULT_CHILD_FEE, child_fee=DEFAULT_FEE) + node.submitpackage(package_hex1) + self.assert_mempool_contents(expected=package_txns1) + + # Package 2 feerate is below the feerate of directly conflicted parent, so it fails even though + # total fees are higher than the original package + package_hex2, package_txns2 = self.create_simple_package(coin, parent_fee=DEFAULT_CHILD_FEE - Decimal("0.00000001"), child_fee=DEFAULT_CHILD_FEE) + pkg_results2 = node.submitpackage(package_hex2) + assert_equal(pkg_results2["package_msg"], 'package RBF failed: insufficient feerate: does not improve feerate diagram') + self.assert_mempool_contents(expected=package_txns1) + self.generate(node, 1) + + def test_0fee_package_rbf(self): + self.log.info("Test package RBF: TRUC 0-fee parent + high-fee child replaces parent's conflicts") + node = self.nodes[0] + # Reuse the same coins so that the transactions conflict with one another. + self.wallet.rescan_utxos() + parent_coin = self.wallet.get_utxo(confirmed_only=True) + + # package1 pays default fee on both transactions + parent1 = self.wallet.create_self_transfer(utxo_to_spend=parent_coin, version=3) + child1 = self.wallet.create_self_transfer(utxo_to_spend=parent1["new_utxo"], version=3) + package_hex1 = [parent1["hex"], child1["hex"]] + fees_package1 = parent1["fee"] + child1["fee"] + submitres1 = node.submitpackage(package_hex1) + assert_equal(submitres1["package_msg"], "success") + self.assert_mempool_contents([parent1["tx"], child1["tx"]]) + + # package2 has a 0-fee parent (conflicting with package1) and very high fee child + parent2 = self.wallet.create_self_transfer(utxo_to_spend=parent_coin, fee=0, fee_rate=0, version=3) + child2 = self.wallet.create_self_transfer(utxo_to_spend=parent2["new_utxo"], fee=fees_package1*10, version=3) + package_hex2 = [parent2["hex"], child2["hex"]] + + submitres2 = node.submitpackage(package_hex2) + assert_equal(submitres2["package_msg"], "success") + assert_equal(set(submitres2["replaced-transactions"]), set([parent1["txid"], child1["txid"]])) + self.assert_mempool_contents([parent2["tx"], child2["tx"]]) + + self.generate(node, 1) + + def test_child_conflicts_parent_mempool_ancestor(self): + fill_mempool(self, self.nodes[0]) + # Reset coins since we filled the mempool with current coins + self.coins = self.wallet.get_utxos(mark_as_spent=False, confirmed_only=True) + + self.log.info("Test that package RBF doesn't have issues with mempool<->package conflicts via inconsistency") + node = self.nodes[0] + coin = self.coins.pop() + + self.ctr += 1 + grandparent_result = self.wallet.create_self_transfer( + fee=DEFAULT_FEE, + utxo_to_spend=coin, + sequence=MAX_BIP125_RBF_SEQUENCE - self.ctr, + ) + + node.sendrawtransaction(grandparent_result["hex"]) + + # Now make package of two descendants that looks + # like a cpfp where the parent can't get in on its own + self.ctr += 1 + parent_result = self.wallet.create_self_transfer( + fee_rate=Decimal('0.00001000'), + utxo_to_spend=grandparent_result["new_utxo"], + sequence=MAX_BIP125_RBF_SEQUENCE - self.ctr, + ) + # Last tx double-spends grandparent's coin, + # which is not inside the current package + self.ctr += 1 + child_result = self.wallet.create_self_transfer_multi( + fee_per_output=int(DEFAULT_CHILD_FEE * COIN), + utxos_to_spend=[parent_result["new_utxo"], coin], + sequence=MAX_BIP125_RBF_SEQUENCE - self.ctr, + ) + + pkg_result = node.submitpackage([parent_result["hex"], child_result["hex"]]) + assert_equal(pkg_result["package_msg"], 'package RBF failed: new transaction cannot have mempool ancestors') + mempool_info = node.getrawmempool() + assert grandparent_result["txid"] in mempool_info + assert parent_result["txid"] not in mempool_info + assert child_result["txid"] not in mempool_info + +if __name__ == "__main__": + PackageRBFTest().main() diff --git a/test/functional/mempool_packages.py b/test/functional/mempool_packages.py index dcb66b2ca1..4be6594de6 100755 --- a/test/functional/mempool_packages.py +++ b/test/functional/mempool_packages.py @@ -31,10 +31,8 @@ class MempoolPackagesTest(BitcoinTestFramework): self.noban_tx_relay = True self.extra_args = [ [ - "-maxorphantx=1000", ], [ - "-maxorphantx=1000", "-limitancestorcount={}".format(CUSTOM_ANCESTOR_LIMIT), "-limitdescendantcount={}".format(CUSTOM_DESCENDANT_LIMIT), ], @@ -199,13 +197,13 @@ class MempoolPackagesTest(BitcoinTestFramework): assert set(mempool1).issubset(set(mempool0)) for tx in chain[:CUSTOM_ANCESTOR_LIMIT]: assert tx in mempool1 - # TODO: more detailed check of node1's mempool (fees etc.) - # check transaction unbroadcast info (should be false if in both mempools) - mempool = self.nodes[0].getrawmempool(True) - for tx in mempool: - assert_equal(mempool[tx]['unbroadcast'], False) - - # TODO: test ancestor size limits + entry0 = self.nodes[0].getmempoolentry(tx) + entry1 = self.nodes[1].getmempoolentry(tx) + assert not entry0['unbroadcast'] + assert not entry1['unbroadcast'] + assert_equal(entry1['fees']['base'], entry0['fees']['base']) + assert_equal(entry1['vsize'], entry0['vsize']) + assert_equal(entry1['depends'], entry0['depends']) # Now test descendant chain limits @@ -251,10 +249,14 @@ class MempoolPackagesTest(BitcoinTestFramework): assert tx in mempool1 for tx in chain[CUSTOM_DESCENDANT_LIMIT:]: assert tx not in mempool1 - # TODO: more detailed check of node1's mempool (fees etc.) - - # TODO: test descendant size limits - + for tx in mempool1: + entry0 = self.nodes[0].getmempoolentry(tx) + entry1 = self.nodes[1].getmempoolentry(tx) + assert not entry0['unbroadcast'] + assert not entry1['unbroadcast'] + assert_equal(entry1['fees']['base'], entry0['fees']['base']) + assert_equal(entry1['vsize'], entry0['vsize']) + assert_equal(entry1['depends'], entry0['depends']) # Test reorg handling # First, the basics: self.generate(self.nodes[0], 1) diff --git a/test/functional/mempool_accept_v3.py b/test/functional/mempool_truc.py index 1b55cd0a0d..e1f3d77201 100755 --- a/test/functional/mempool_accept_v3.py +++ b/test/functional/mempool_truc.py @@ -6,6 +6,7 @@ from decimal import Decimal from test_framework.messages import ( MAX_BIP125_RBF_SEQUENCE, + WITNESS_SCALE_FACTOR, ) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( @@ -21,6 +22,7 @@ from test_framework.wallet import ( ) MAX_REPLACEMENT_CANDIDATES = 100 +TRUC_MAX_VSIZE = 10000 def cleanup(extra_args=None): def decorator(func): @@ -37,10 +39,10 @@ def cleanup(extra_args=None): return wrapper return decorator -class MempoolAcceptV3(BitcoinTestFramework): +class MempoolTRUC(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 - self.extra_args = [["-acceptnonstdtxn=1"]] + self.extra_args = [[]] self.setup_clean_chain = True def check_mempool(self, txids): @@ -49,10 +51,24 @@ class MempoolAcceptV3(BitcoinTestFramework): assert_equal(len(txids), len(mempool_contents)) assert all([txid in txids for txid in mempool_contents]) - @cleanup(extra_args=["-datacarriersize=1000", "-acceptnonstdtxn=1"]) - def test_v3_acceptance(self): + @cleanup(extra_args=["-datacarriersize=20000"]) + def test_truc_max_vsize(self): node = self.nodes[0] - self.log.info("Test a child of a v3 transaction cannot be more than 1000vB") + self.log.info("Test TRUC-specific maximum transaction vsize") + tx_v3_heavy = self.wallet.create_self_transfer(target_weight=(TRUC_MAX_VSIZE + 1) * WITNESS_SCALE_FACTOR, version=3) + assert_greater_than_or_equal(tx_v3_heavy["tx"].get_vsize(), TRUC_MAX_VSIZE) + expected_error_heavy = f"TRUC-violation, version=3 tx {tx_v3_heavy['txid']} (wtxid={tx_v3_heavy['wtxid']}) is too big" + assert_raises_rpc_error(-26, expected_error_heavy, node.sendrawtransaction, tx_v3_heavy["hex"]) + self.check_mempool([]) + + # Ensure we are hitting the TRUC-specific limit and not something else + tx_v2_heavy = self.wallet.send_self_transfer(from_node=node, target_weight=(TRUC_MAX_VSIZE + 1) * WITNESS_SCALE_FACTOR, version=2) + self.check_mempool([tx_v2_heavy["txid"]]) + + @cleanup(extra_args=["-datacarriersize=1000"]) + def test_truc_acceptance(self): + node = self.nodes[0] + self.log.info("Test a child of a TRUC transaction cannot be more than 1000vB") tx_v3_parent_normal = self.wallet.send_self_transfer(from_node=node, version=3) self.check_mempool([tx_v3_parent_normal["txid"]]) tx_v3_child_heavy = self.wallet.create_self_transfer( @@ -61,13 +77,13 @@ class MempoolAcceptV3(BitcoinTestFramework): version=3 ) assert_greater_than_or_equal(tx_v3_child_heavy["tx"].get_vsize(), 1000) - expected_error_child_heavy = f"v3-rule-violation, v3 child tx {tx_v3_child_heavy['txid']} (wtxid={tx_v3_child_heavy['wtxid']}) is too big" + expected_error_child_heavy = f"TRUC-violation, version=3 child tx {tx_v3_child_heavy['txid']} (wtxid={tx_v3_child_heavy['wtxid']}) is too big" assert_raises_rpc_error(-26, expected_error_child_heavy, node.sendrawtransaction, tx_v3_child_heavy["hex"]) self.check_mempool([tx_v3_parent_normal["txid"]]) # tx has no descendants assert_equal(node.getmempoolentry(tx_v3_parent_normal["txid"])["descendantcount"], 1) - self.log.info("Test that, during replacements, only the new transaction counts for v3 descendant limit") + self.log.info("Test that, during replacements, only the new transaction counts for TRUC descendant limit") tx_v3_child_almost_heavy = self.wallet.send_self_transfer( from_node=node, fee_rate=DEFAULT_FEE, @@ -89,10 +105,10 @@ class MempoolAcceptV3(BitcoinTestFramework): self.check_mempool([tx_v3_parent_normal["txid"], tx_v3_child_almost_heavy_rbf["txid"]]) assert_equal(node.getmempoolentry(tx_v3_parent_normal["txid"])["descendantcount"], 2) - @cleanup(extra_args=["-acceptnonstdtxn=1"]) - def test_v3_replacement(self): + @cleanup(extra_args=None) + def test_truc_replacement(self): node = self.nodes[0] - self.log.info("Test v3 transactions may be replaced by v3 transactions") + self.log.info("Test TRUC transactions may be replaced by TRUC transactions") utxo_v3_bip125 = self.wallet.get_utxo() tx_v3_bip125 = self.wallet.send_self_transfer( from_node=node, @@ -111,7 +127,7 @@ class MempoolAcceptV3(BitcoinTestFramework): ) self.check_mempool([tx_v3_bip125_rbf["txid"]]) - self.log.info("Test v3 transactions may be replaced by V2 transactions") + self.log.info("Test TRUC transactions may be replaced by non-TRUC (BIP125) transactions") tx_v3_bip125_rbf_v2 = self.wallet.send_self_transfer( from_node=node, fee_rate=DEFAULT_FEE * 3, @@ -120,7 +136,7 @@ class MempoolAcceptV3(BitcoinTestFramework): ) self.check_mempool([tx_v3_bip125_rbf_v2["txid"]]) - self.log.info("Test that replacements cannot cause violation of inherited v3") + self.log.info("Test that replacements cannot cause violation of inherited TRUC") utxo_v3_parent = self.wallet.get_utxo() tx_v3_parent = self.wallet.send_self_transfer( from_node=node, @@ -141,15 +157,15 @@ class MempoolAcceptV3(BitcoinTestFramework): utxo_to_spend=tx_v3_parent["new_utxo"], version=2 ) - expected_error_v2_v3 = f"v3-rule-violation, non-v3 tx {tx_v3_child_rbf_v2['txid']} (wtxid={tx_v3_child_rbf_v2['wtxid']}) cannot spend from v3 tx {tx_v3_parent['txid']} (wtxid={tx_v3_parent['wtxid']})" + expected_error_v2_v3 = f"TRUC-violation, non-version=3 tx {tx_v3_child_rbf_v2['txid']} (wtxid={tx_v3_child_rbf_v2['wtxid']}) cannot spend from version=3 tx {tx_v3_parent['txid']} (wtxid={tx_v3_parent['wtxid']})" assert_raises_rpc_error(-26, expected_error_v2_v3, node.sendrawtransaction, tx_v3_child_rbf_v2["hex"]) self.check_mempool([tx_v3_bip125_rbf_v2["txid"], tx_v3_parent["txid"], tx_v3_child["txid"]]) - @cleanup(extra_args=["-acceptnonstdtxn=1"]) - def test_v3_bip125(self): + @cleanup(extra_args=None) + def test_truc_bip125(self): node = self.nodes[0] - self.log.info("Test v3 transactions that don't signal BIP125 are replaceable") + self.log.info("Test TRUC transactions that don't signal BIP125 are replaceable") assert_equal(node.getmempoolinfo()["fullrbf"], False) utxo_v3_no_bip125 = self.wallet.get_utxo() tx_v3_no_bip125 = self.wallet.send_self_transfer( @@ -170,10 +186,10 @@ class MempoolAcceptV3(BitcoinTestFramework): ) self.check_mempool([tx_v3_no_bip125_rbf["txid"]]) - @cleanup(extra_args=["-datacarriersize=40000", "-acceptnonstdtxn=1"]) - def test_v3_reorg(self): + @cleanup(extra_args=["-datacarriersize=40000"]) + def test_truc_reorg(self): node = self.nodes[0] - self.log.info("Test that, during a reorg, v3 rules are not enforced") + self.log.info("Test that, during a reorg, TRUC rules are not enforced") tx_v2_block = self.wallet.send_self_transfer(from_node=node, version=2) tx_v3_block = self.wallet.send_self_transfer(from_node=node, version=3) tx_v3_block2 = self.wallet.send_self_transfer(from_node=node, version=3) @@ -192,36 +208,62 @@ class MempoolAcceptV3(BitcoinTestFramework): node.reconsiderblock(block[0]) - @cleanup(extra_args=["-limitdescendantsize=10", "-datacarriersize=40000", "-acceptnonstdtxn=1"]) + @cleanup(extra_args=["-limitdescendantsize=10", "-datacarriersize=40000"]) def test_nondefault_package_limits(self): """ - Max standard tx size + v3 rules imply the ancestor/descendant rules (at their default + Max standard tx size + TRUC rules imply the ancestor/descendant rules (at their default values), but those checks must not be skipped. Ensure both sets of checks are done by changing the ancestor/descendant limit configurations. """ node = self.nodes[0] - self.log.info("Test that a decreased limitdescendantsize also applies to v3 child") - tx_v3_parent_large1 = self.wallet.send_self_transfer(from_node=node, target_weight=99900, version=3) - tx_v3_child_large1 = self.wallet.create_self_transfer(utxo_to_spend=tx_v3_parent_large1["new_utxo"], version=3) - # Child is within v3 limits, but parent's descendant limit is exceeded - assert_greater_than(1000, tx_v3_child_large1["tx"].get_vsize()) + self.log.info("Test that a decreased limitdescendantsize also applies to TRUC child") + parent_target_weight = 9990 * WITNESS_SCALE_FACTOR + child_target_weight = 500 * WITNESS_SCALE_FACTOR + tx_v3_parent_large1 = self.wallet.send_self_transfer( + from_node=node, + target_weight=parent_target_weight, + version=3 + ) + tx_v3_child_large1 = self.wallet.create_self_transfer( + utxo_to_spend=tx_v3_parent_large1["new_utxo"], + target_weight=child_target_weight, + version=3 + ) + + # Parent and child are within v3 limits, but parent's 10kvB descendant limit is exceeded + assert_greater_than_or_equal(TRUC_MAX_VSIZE, tx_v3_parent_large1["tx"].get_vsize()) + assert_greater_than_or_equal(1000, tx_v3_child_large1["tx"].get_vsize()) + assert_greater_than(tx_v3_parent_large1["tx"].get_vsize() + tx_v3_child_large1["tx"].get_vsize(), 10000) + assert_raises_rpc_error(-26, f"too-long-mempool-chain, exceeds descendant size limit for tx {tx_v3_parent_large1['txid']}", node.sendrawtransaction, tx_v3_child_large1["hex"]) self.check_mempool([tx_v3_parent_large1["txid"]]) assert_equal(node.getmempoolentry(tx_v3_parent_large1["txid"])["descendantcount"], 1) self.generate(node, 1) self.log.info("Test that a decreased limitancestorsize also applies to v3 parent") - self.restart_node(0, extra_args=["-limitancestorsize=10", "-datacarriersize=40000", "-acceptnonstdtxn=1"]) - tx_v3_parent_large2 = self.wallet.send_self_transfer(from_node=node, target_weight=99900, version=3) - tx_v3_child_large2 = self.wallet.create_self_transfer(utxo_to_spend=tx_v3_parent_large2["new_utxo"], version=3) - # Child is within v3 limits + self.restart_node(0, extra_args=["-limitancestorsize=10", "-datacarriersize=40000"]) + tx_v3_parent_large2 = self.wallet.send_self_transfer( + from_node=node, + target_weight=parent_target_weight, + version=3 + ) + tx_v3_child_large2 = self.wallet.create_self_transfer( + utxo_to_spend=tx_v3_parent_large2["new_utxo"], + target_weight=child_target_weight, + version=3 + ) + + # Parent and child are within TRUC limits + assert_greater_than_or_equal(TRUC_MAX_VSIZE, tx_v3_parent_large2["tx"].get_vsize()) assert_greater_than_or_equal(1000, tx_v3_child_large2["tx"].get_vsize()) + assert_greater_than(tx_v3_parent_large2["tx"].get_vsize() + tx_v3_child_large2["tx"].get_vsize(), 10000) + assert_raises_rpc_error(-26, f"too-long-mempool-chain, exceeds ancestor size limit", node.sendrawtransaction, tx_v3_child_large2["hex"]) self.check_mempool([tx_v3_parent_large2["txid"]]) - @cleanup(extra_args=["-datacarriersize=1000", "-acceptnonstdtxn=1"]) - def test_v3_ancestors_package(self): - self.log.info("Test that v3 ancestor limits are checked within the package") + @cleanup(extra_args=["-datacarriersize=1000"]) + def test_truc_ancestors_package(self): + self.log.info("Test that TRUC ancestor limits are checked within the package") node = self.nodes[0] tx_v3_parent_normal = self.wallet.create_self_transfer( fee_rate=0, @@ -247,34 +289,34 @@ class MempoolAcceptV3(BitcoinTestFramework): self.check_mempool([]) result = node.submitpackage([tx_v3_parent_normal["hex"], tx_v3_parent_2_normal["hex"], tx_v3_child_multiparent["hex"]]) - assert_equal(result['package_msg'], f"v3-violation, tx {tx_v3_child_multiparent['txid']} (wtxid={tx_v3_child_multiparent['wtxid']}) would have too many ancestors") + assert_equal(result['package_msg'], f"TRUC-violation, tx {tx_v3_child_multiparent['txid']} (wtxid={tx_v3_child_multiparent['wtxid']}) would have too many ancestors") self.check_mempool([]) self.check_mempool([]) result = node.submitpackage([tx_v3_parent_normal["hex"], tx_v3_child_heavy["hex"]]) # tx_v3_child_heavy is heavy based on weight, not sigops. - assert_equal(result['package_msg'], f"v3-violation, v3 child tx {tx_v3_child_heavy['txid']} (wtxid={tx_v3_child_heavy['wtxid']}) is too big: {tx_v3_child_heavy['tx'].get_vsize()} > 1000 virtual bytes") + assert_equal(result['package_msg'], f"TRUC-violation, version=3 child tx {tx_v3_child_heavy['txid']} (wtxid={tx_v3_child_heavy['wtxid']}) is too big: {tx_v3_child_heavy['tx'].get_vsize()} > 1000 virtual bytes") self.check_mempool([]) tx_v3_parent = self.wallet.create_self_transfer(version=3) tx_v3_child = self.wallet.create_self_transfer(utxo_to_spend=tx_v3_parent["new_utxo"], version=3) tx_v3_grandchild = self.wallet.create_self_transfer(utxo_to_spend=tx_v3_child["new_utxo"], version=3) result = node.testmempoolaccept([tx_v3_parent["hex"], tx_v3_child["hex"], tx_v3_grandchild["hex"]]) - assert all([txresult["package-error"] == f"v3-violation, tx {tx_v3_grandchild['txid']} (wtxid={tx_v3_grandchild['wtxid']}) would have too many ancestors" for txresult in result]) + assert all([txresult["package-error"] == f"TRUC-violation, tx {tx_v3_grandchild['txid']} (wtxid={tx_v3_grandchild['wtxid']}) would have too many ancestors" for txresult in result]) - @cleanup(extra_args=["-acceptnonstdtxn=1"]) - def test_v3_ancestors_package_and_mempool(self): + @cleanup(extra_args=None) + def test_truc_ancestors_package_and_mempool(self): """ - A v3 transaction in a package cannot have 2 v3 parents. + A TRUC transaction in a package cannot have 2 TRUC parents. Test that if we have a transaction graph A -> B -> C, where A, B, C are - all v3 transactions, that we cannot use submitpackage to get the + all TRUC transactions, that we cannot use submitpackage to get the transactions all into the mempool. Verify, in particular, that if A is already in the mempool, then submitpackage(B, C) will fail. """ node = self.nodes[0] - self.log.info("Test that v3 ancestor limits include transactions within the package and all in-mempool ancestors") + self.log.info("Test that TRUC ancestor limits include transactions within the package and all in-mempool ancestors") # This is our transaction "A": tx_in_mempool = self.wallet.send_self_transfer(from_node=node, version=3) @@ -289,17 +331,17 @@ class MempoolAcceptV3(BitcoinTestFramework): # submitpackage(B, C) should fail result = node.submitpackage([tx_0fee_parent["hex"], tx_child_violator["hex"]]) - assert_equal(result['package_msg'], f"v3-violation, tx {tx_child_violator['txid']} (wtxid={tx_child_violator['wtxid']}) would have too many ancestors") + assert_equal(result['package_msg'], f"TRUC-violation, tx {tx_child_violator['txid']} (wtxid={tx_child_violator['wtxid']}) would have too many ancestors") self.check_mempool([tx_in_mempool["txid"]]) - @cleanup(extra_args=["-acceptnonstdtxn=1"]) + @cleanup(extra_args=None) def test_sibling_eviction_package(self): """ When a transaction has a mempool sibling, it may be eligible for sibling eviction. However, this option is only available in single transaction acceptance. It doesn't work in a multi-testmempoolaccept (where RBF is disabled) or when doing package CPFP. """ - self.log.info("Test v3 sibling eviction in submitpackage and multi-testmempoolaccept") + self.log.info("Test TRUC sibling eviction in submitpackage and multi-testmempoolaccept") node = self.nodes[0] # Add a parent + child to mempool tx_mempool_parent = self.wallet.send_self_transfer_multi( @@ -342,17 +384,17 @@ class MempoolAcceptV3(BitcoinTestFramework): # Fails with another non-related transaction via testmempoolaccept tx_unrelated = self.wallet.create_self_transfer(version=3) result_test_unrelated = node.testmempoolaccept([tx_sibling_1["hex"], tx_unrelated["hex"]]) - assert_equal(result_test_unrelated[0]["reject-reason"], "v3-rule-violation") + assert_equal(result_test_unrelated[0]["reject-reason"], "TRUC-violation") # Fails in a package via testmempoolaccept result_test_1p1c = node.testmempoolaccept([tx_sibling_1["hex"], tx_has_mempool_uncle["hex"]]) - assert_equal(result_test_1p1c[0]["reject-reason"], "v3-rule-violation") + assert_equal(result_test_1p1c[0]["reject-reason"], "TRUC-violation") # Allowed when tx is submitted in a package and evaluated individually. # Note that the child failed since it would be the 3rd generation. result_package_indiv = node.submitpackage([tx_sibling_1["hex"], tx_has_mempool_uncle["hex"]]) self.check_mempool([tx_mempool_parent["txid"], tx_sibling_1["txid"]]) - expected_error_gen3 = f"v3-rule-violation, tx {tx_has_mempool_uncle['txid']} (wtxid={tx_has_mempool_uncle['wtxid']}) would have too many ancestors" + expected_error_gen3 = f"TRUC-violation, tx {tx_has_mempool_uncle['txid']} (wtxid={tx_has_mempool_uncle['wtxid']}) would have too many ancestors" assert_equal(result_package_indiv["tx-results"][tx_has_mempool_uncle['wtxid']]['error'], expected_error_gen3) @@ -360,17 +402,17 @@ class MempoolAcceptV3(BitcoinTestFramework): node.submitpackage([tx_mempool_parent["hex"], tx_sibling_2["hex"]]) self.check_mempool([tx_mempool_parent["txid"], tx_sibling_2["txid"]]) - # Child cannot pay for sibling eviction for parent, as it violates v3 topology limits + # Child cannot pay for sibling eviction for parent, as it violates TRUC topology limits result_package_cpfp = node.submitpackage([tx_sibling_3["hex"], tx_bumps_parent_with_sibling["hex"]]) self.check_mempool([tx_mempool_parent["txid"], tx_sibling_2["txid"]]) - expected_error_cpfp = f"v3-rule-violation, tx {tx_mempool_parent['txid']} (wtxid={tx_mempool_parent['wtxid']}) would exceed descendant count limit" + expected_error_cpfp = f"TRUC-violation, tx {tx_mempool_parent['txid']} (wtxid={tx_mempool_parent['wtxid']}) would exceed descendant count limit" assert_equal(result_package_cpfp["tx-results"][tx_sibling_3['wtxid']]['error'], expected_error_cpfp) - @cleanup(extra_args=["-datacarriersize=1000", "-acceptnonstdtxn=1"]) - def test_v3_package_inheritance(self): - self.log.info("Test that v3 inheritance is checked within package") + @cleanup(extra_args=["-datacarriersize=1000"]) + def test_truc_package_inheritance(self): + self.log.info("Test that TRUC inheritance is checked within package") node = self.nodes[0] tx_v3_parent = self.wallet.create_self_transfer( fee_rate=0, @@ -384,14 +426,14 @@ class MempoolAcceptV3(BitcoinTestFramework): ) self.check_mempool([]) result = node.submitpackage([tx_v3_parent["hex"], tx_v2_child["hex"]]) - assert_equal(result['package_msg'], f"v3-violation, non-v3 tx {tx_v2_child['txid']} (wtxid={tx_v2_child['wtxid']}) cannot spend from v3 tx {tx_v3_parent['txid']} (wtxid={tx_v3_parent['wtxid']})") + assert_equal(result['package_msg'], f"TRUC-violation, non-version=3 tx {tx_v2_child['txid']} (wtxid={tx_v2_child['wtxid']}) cannot spend from version=3 tx {tx_v3_parent['txid']} (wtxid={tx_v3_parent['wtxid']})") self.check_mempool([]) - @cleanup(extra_args=["-acceptnonstdtxn=1"]) - def test_v3_in_testmempoolaccept(self): + @cleanup(extra_args=None) + def test_truc_in_testmempoolaccept(self): node = self.nodes[0] - self.log.info("Test that v3 inheritance is accurately assessed in testmempoolaccept") + self.log.info("Test that TRUC inheritance is accurately assessed in testmempoolaccept") tx_v2 = self.wallet.create_self_transfer(version=2) tx_v2_from_v2 = self.wallet.create_self_transfer(utxo_to_spend=tx_v2["new_utxo"], version=2) tx_v3_from_v2 = self.wallet.create_self_transfer(utxo_to_spend=tx_v2["new_utxo"], version=3) @@ -405,11 +447,11 @@ class MempoolAcceptV3(BitcoinTestFramework): assert all([result["allowed"] for result in test_accept_v2_and_v3]) test_accept_v3_from_v2 = node.testmempoolaccept([tx_v2["hex"], tx_v3_from_v2["hex"]]) - expected_error_v3_from_v2 = f"v3-violation, v3 tx {tx_v3_from_v2['txid']} (wtxid={tx_v3_from_v2['wtxid']}) cannot spend from non-v3 tx {tx_v2['txid']} (wtxid={tx_v2['wtxid']})" + expected_error_v3_from_v2 = f"TRUC-violation, version=3 tx {tx_v3_from_v2['txid']} (wtxid={tx_v3_from_v2['wtxid']}) cannot spend from non-version=3 tx {tx_v2['txid']} (wtxid={tx_v2['wtxid']})" assert all([result["package-error"] == expected_error_v3_from_v2 for result in test_accept_v3_from_v2]) test_accept_v2_from_v3 = node.testmempoolaccept([tx_v3["hex"], tx_v2_from_v3["hex"]]) - expected_error_v2_from_v3 = f"v3-violation, non-v3 tx {tx_v2_from_v3['txid']} (wtxid={tx_v2_from_v3['wtxid']}) cannot spend from v3 tx {tx_v3['txid']} (wtxid={tx_v3['wtxid']})" + expected_error_v2_from_v3 = f"TRUC-violation, non-version=3 tx {tx_v2_from_v3['txid']} (wtxid={tx_v2_from_v3['wtxid']}) cannot spend from version=3 tx {tx_v3['txid']} (wtxid={tx_v3['wtxid']})" assert all([result["package-error"] == expected_error_v2_from_v3 for result in test_accept_v2_from_v3]) test_accept_pairs = node.testmempoolaccept([tx_v2["hex"], tx_v3["hex"], tx_v2_from_v2["hex"], tx_v3_from_v3["hex"]]) @@ -421,26 +463,26 @@ class MempoolAcceptV3(BitcoinTestFramework): tx_v3_child_1 = self.wallet.create_self_transfer(utxo_to_spend=tx_v3_parent["new_utxos"][0], version=3) tx_v3_child_2 = self.wallet.create_self_transfer(utxo_to_spend=tx_v3_parent["new_utxos"][1], version=3) test_accept_2children = node.testmempoolaccept([tx_v3_parent["hex"], tx_v3_child_1["hex"], tx_v3_child_2["hex"]]) - expected_error_2children = f"v3-violation, tx {tx_v3_parent['txid']} (wtxid={tx_v3_parent['wtxid']}) would exceed descendant count limit" + expected_error_2children = f"TRUC-violation, tx {tx_v3_parent['txid']} (wtxid={tx_v3_parent['wtxid']}) would exceed descendant count limit" assert all([result["package-error"] == expected_error_2children for result in test_accept_2children]) - # Extra v3 transaction does not get incorrectly marked as extra descendant + # Extra TRUC transaction does not get incorrectly marked as extra descendant test_accept_1child_with_exra = node.testmempoolaccept([tx_v3_parent["hex"], tx_v3_child_1["hex"], tx_v3_independent["hex"]]) assert all([result["allowed"] for result in test_accept_1child_with_exra]) - # Extra v3 transaction does not make us ignore the extra descendant + # Extra TRUC transaction does not make us ignore the extra descendant test_accept_2children_with_exra = node.testmempoolaccept([tx_v3_parent["hex"], tx_v3_child_1["hex"], tx_v3_child_2["hex"], tx_v3_independent["hex"]]) - expected_error_extra = f"v3-violation, tx {tx_v3_parent['txid']} (wtxid={tx_v3_parent['wtxid']}) would exceed descendant count limit" + expected_error_extra = f"TRUC-violation, tx {tx_v3_parent['txid']} (wtxid={tx_v3_parent['wtxid']}) would exceed descendant count limit" assert all([result["package-error"] == expected_error_extra for result in test_accept_2children_with_exra]) # Same result if the parent is already in mempool node.sendrawtransaction(tx_v3_parent["hex"]) test_accept_2children_with_in_mempool_parent = node.testmempoolaccept([tx_v3_child_1["hex"], tx_v3_child_2["hex"]]) assert all([result["package-error"] == expected_error_extra for result in test_accept_2children_with_in_mempool_parent]) - @cleanup(extra_args=["-acceptnonstdtxn=1"]) + @cleanup(extra_args=None) def test_reorg_2child_rbf(self): node = self.nodes[0] - self.log.info("Test that children of a v3 transaction can be replaced individually, even if there are multiple due to reorg") + self.log.info("Test that children of a TRUC transaction can be replaced individually, even if there are multiple due to reorg") ancestor_tx = self.wallet.send_self_transfer_multi(from_node=node, num_outputs=2, version=3) self.check_mempool([ancestor_tx["txid"]]) @@ -468,9 +510,9 @@ class MempoolAcceptV3(BitcoinTestFramework): self.check_mempool([ancestor_tx["txid"], child_1_conflict["txid"], child_2["txid"]]) assert_equal(node.getmempoolentry(ancestor_tx["txid"])["descendantcount"], 3) - @cleanup(extra_args=["-acceptnonstdtxn=1"]) - def test_v3_sibling_eviction(self): - self.log.info("Test sibling eviction for v3") + @cleanup(extra_args=None) + def test_truc_sibling_eviction(self): + self.log.info("Test sibling eviction for TRUC") node = self.nodes[0] tx_v3_parent = self.wallet.send_self_transfer_multi(from_node=node, num_outputs=2, version=3) # This is the sibling to replace @@ -533,15 +575,15 @@ class MempoolAcceptV3(BitcoinTestFramework): tx_unrelated_replacee = self.wallet.send_self_transfer(from_node=node, utxo_to_spend=utxo_unrelated_conflict) assert tx_unrelated_replacee["txid"] in node.getrawmempool() - fee_to_beat_child2 = int(tx_v3_child_2["fee"] * COIN) + fee_to_beat = max(int(tx_v3_child_2["fee"] * COIN), int(tx_unrelated_replacee["fee"]*COIN)) tx_v3_child_3 = self.wallet.create_self_transfer_multi( - utxos_to_spend=[tx_v3_parent["new_utxos"][0], utxo_unrelated_conflict], fee_per_output=fee_to_beat_child2*5, version=3 + utxos_to_spend=[tx_v3_parent["new_utxos"][0], utxo_unrelated_conflict], fee_per_output=fee_to_beat*2, version=3 ) node.sendrawtransaction(tx_v3_child_3["hex"]) self.check_mempool(txids_v2_100 + [tx_v3_parent["txid"], tx_v3_child_3["txid"]]) - @cleanup(extra_args=["-acceptnonstdtxn=1"]) + @cleanup(extra_args=None) def test_reorg_sibling_eviction_1p2c(self): node = self.nodes[0] self.log.info("Test that sibling eviction is not allowed when multiple siblings exist") @@ -567,7 +609,7 @@ class MempoolAcceptV3(BitcoinTestFramework): utxo_to_spend=tx_with_multi_children["new_utxos"][2], fee_rate=DEFAULT_FEE*50 ) - expected_error_2siblings = f"v3-rule-violation, tx {tx_with_multi_children['txid']} (wtxid={tx_with_multi_children['wtxid']}) would exceed descendant count limit" + expected_error_2siblings = f"TRUC-violation, tx {tx_with_multi_children['txid']} (wtxid={tx_with_multi_children['wtxid']}) would exceed descendant count limit" assert_raises_rpc_error(-26, expected_error_2siblings, node.sendrawtransaction, tx_with_sibling3["hex"]) # However, an RBF (with conflicting inputs) is possible even if the resulting cluster size exceeds 2 @@ -585,20 +627,21 @@ class MempoolAcceptV3(BitcoinTestFramework): node = self.nodes[0] self.wallet = MiniWallet(node) self.generate(self.wallet, 120) - self.test_v3_acceptance() - self.test_v3_replacement() - self.test_v3_bip125() - self.test_v3_reorg() + self.test_truc_max_vsize() + self.test_truc_acceptance() + self.test_truc_replacement() + self.test_truc_bip125() + self.test_truc_reorg() self.test_nondefault_package_limits() - self.test_v3_ancestors_package() - self.test_v3_ancestors_package_and_mempool() + self.test_truc_ancestors_package() + self.test_truc_ancestors_package_and_mempool() self.test_sibling_eviction_package() - self.test_v3_package_inheritance() - self.test_v3_in_testmempoolaccept() + self.test_truc_package_inheritance() + self.test_truc_in_testmempoolaccept() self.test_reorg_2child_rbf() - self.test_v3_sibling_eviction() + self.test_truc_sibling_eviction() self.test_reorg_sibling_eviction_1p2c() if __name__ == "__main__": - MempoolAcceptV3().main() + MempoolTRUC().main() diff --git a/test/functional/p2p_1p1c_network.py b/test/functional/p2p_1p1c_network.py new file mode 100755 index 0000000000..ea59248506 --- /dev/null +++ b/test/functional/p2p_1p1c_network.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python3 +# Copyright (c) 2024-present The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +""" +Test that 1p1c package submission allows a 1p1c package to propagate in a "network" of nodes. Send +various packages from different nodes on a network in which some nodes have already received some of +the transactions (and submitted them to mempool, kept them as orphans or rejected them as +too-low-feerate transactions). The packages should be received and accepted by all nodes. +""" + +from decimal import Decimal +from math import ceil + +from test_framework.mempool_util import ( + fill_mempool, +) +from test_framework.messages import ( + msg_tx, +) +from test_framework.p2p import ( + P2PInterface, +) +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + assert_greater_than, +) +from test_framework.wallet import ( + MiniWallet, + MiniWalletMode, +) + +# 1sat/vB feerate denominated in BTC/KvB +FEERATE_1SAT_VB = Decimal("0.00001000") + +class PackageRelayTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 4 + # hugely speeds up the test, as it involves multiple hops of tx relay. + self.noban_tx_relay = True + self.extra_args = [[ + "-datacarriersize=100000", + "-maxmempool=5", + ]] * self.num_nodes + self.supports_cli = False + + def raise_network_minfee(self): + fill_mempool(self, self.nodes[0]) + + self.log.debug("Wait for the network to sync mempools") + self.sync_mempools() + + self.log.debug("Check that all nodes' mempool minimum feerates are above min relay feerate") + for node in self.nodes: + assert_equal(node.getmempoolinfo()['minrelaytxfee'], FEERATE_1SAT_VB) + assert_greater_than(node.getmempoolinfo()['mempoolminfee'], FEERATE_1SAT_VB) + + def create_basic_1p1c(self, wallet): + low_fee_parent = wallet.create_self_transfer(fee_rate=FEERATE_1SAT_VB, confirmed_only=True) + high_fee_child = wallet.create_self_transfer(utxo_to_spend=low_fee_parent["new_utxo"], fee_rate=999*FEERATE_1SAT_VB) + package_hex_basic = [low_fee_parent["hex"], high_fee_child["hex"]] + return package_hex_basic, low_fee_parent["tx"], high_fee_child["tx"] + + def create_package_2outs(self, wallet): + # First create a tester tx to see the vsize, and then adjust the fees + utxo_for_2outs = wallet.get_utxo(confirmed_only=True) + + low_fee_parent_2outs_tester = wallet.create_self_transfer_multi( + utxos_to_spend=[utxo_for_2outs], + num_outputs=2, + ) + + # Target 1sat/vB so the number of satoshis is equal to the vsize. + # Round up. The goal is to be between min relay feerate and mempool min feerate. + fee_2outs = ceil(low_fee_parent_2outs_tester["tx"].get_vsize() / 2) + + low_fee_parent_2outs = wallet.create_self_transfer_multi( + utxos_to_spend=[utxo_for_2outs], + num_outputs=2, + fee_per_output=fee_2outs, + ) + + # Now create the child + high_fee_child_2outs = wallet.create_self_transfer_multi( + utxos_to_spend=low_fee_parent_2outs["new_utxos"][::-1], + fee_per_output=fee_2outs*100, + ) + return [low_fee_parent_2outs["hex"], high_fee_child_2outs["hex"]], low_fee_parent_2outs["tx"], high_fee_child_2outs["tx"] + + def create_package_2p1c(self, wallet): + parent1 = wallet.create_self_transfer(fee_rate=FEERATE_1SAT_VB*10, confirmed_only=True) + parent2 = wallet.create_self_transfer(fee_rate=FEERATE_1SAT_VB*20, confirmed_only=True) + child = wallet.create_self_transfer_multi( + utxos_to_spend=[parent1["new_utxo"], parent2["new_utxo"]], + fee_per_output=999*parent1["tx"].get_vsize(), + ) + return [parent1["hex"], parent2["hex"], child["hex"]], parent1["tx"], parent2["tx"], child["tx"] + + def create_packages(self): + # 1: Basic 1-parent-1-child package, parent 1sat/vB, child 999sat/vB + package_hex_1, parent_1, child_1 = self.create_basic_1p1c(self.wallet) + + # 2: same as 1, parent's txid is the same as its wtxid. + package_hex_2, parent_2, child_2 = self.create_basic_1p1c(self.wallet_nonsegwit) + + # 3: 2-parent-1-child package. Both parents are above mempool min feerate. No package submission happens. + # We require packages to be child-with-unconfirmed-parents and only allow 1-parent-1-child packages. + package_hex_3, parent_31, parent_32, child_3 = self.create_package_2p1c(self.wallet) + + # 4: parent + child package where the child spends 2 different outputs from the parent. + package_hex_4, parent_4, child_4 = self.create_package_2outs(self.wallet) + + # Assemble return results + packages_to_submit = [package_hex_1, package_hex_2, package_hex_3, package_hex_4] + # node0: sender + # node1: pre-received the children (orphan) + # node3: pre-received the parents (too low fee) + # All nodes receive parent_31 ahead of time. + txns_to_send = [ + [], + [child_1, child_2, parent_31, child_3, child_4], + [parent_31], + [parent_1, parent_2, parent_31, parent_4] + ] + + return packages_to_submit, txns_to_send + + def run_test(self): + self.wallet = MiniWallet(self.nodes[1]) + self.wallet_nonsegwit = MiniWallet(self.nodes[2], mode=MiniWalletMode.RAW_P2PK) + self.generate(self.wallet_nonsegwit, 10) + self.generate(self.wallet, 120) + + self.log.info("Fill mempools with large transactions to raise mempool minimum feerates") + self.raise_network_minfee() + + # Create the transactions. + self.wallet.rescan_utxos(include_mempool=True) + packages_to_submit, transactions_to_presend = self.create_packages() + + self.peers = [self.nodes[i].add_p2p_connection(P2PInterface()) for i in range(self.num_nodes)] + + self.log.info("Pre-send some transactions to nodes") + for (i, peer) in enumerate(self.peers): + for tx in transactions_to_presend[i]: + peer.send_and_ping(msg_tx(tx)) + # This disconnect removes any sent orphans from the orphanage (EraseForPeer) and times + # out the in-flight requests. It is currently required for the test to pass right now, + # because the node will not reconsider an orphan tx and will not (re)try requesting + # orphan parents from multiple peers if the first one didn't respond. + # TODO: remove this in the future if the node tries orphan resolution with multiple peers. + peer.peer_disconnect() + + self.log.info("Submit full packages to node0") + for package_hex in packages_to_submit: + submitpackage_result = self.nodes[0].submitpackage(package_hex) + assert_equal(submitpackage_result["package_msg"], "success") + + self.log.info("Wait for mempools to sync") + self.sync_mempools(timeout=20) + + +if __name__ == '__main__': + PackageRelayTest().main() diff --git a/test/functional/p2p_addr_relay.py b/test/functional/p2p_addr_relay.py index b23ec1028b..d10e47e036 100755 --- a/test/functional/p2p_addr_relay.py +++ b/test/functional/p2p_addr_relay.py @@ -142,7 +142,8 @@ class AddrTest(BitcoinTestFramework): msg = self.setup_addr_msg(1010) with self.nodes[0].assert_debug_log(['addr message size = 1010']): - addr_source.send_and_ping(msg) + addr_source.send_message(msg) + addr_source.wait_for_disconnect() self.nodes[0].disconnect_p2ps() diff --git a/test/functional/p2p_addrv2_relay.py b/test/functional/p2p_addrv2_relay.py index ea114e7d70..4ec8e0bc04 100755 --- a/test/functional/p2p_addrv2_relay.py +++ b/test/functional/p2p_addrv2_relay.py @@ -86,11 +86,6 @@ class AddrTest(BitcoinTestFramework): addr_source = self.nodes[0].add_p2p_connection(P2PInterface()) msg = msg_addrv2() - self.log.info('Send too-large addrv2 message') - msg.addrs = ADDRS * 101 - with self.nodes[0].assert_debug_log(['addrv2 message size = 1010']): - addr_source.send_and_ping(msg) - self.log.info('Check that addrv2 message content is relayed and added to addrman') addr_receiver = self.nodes[0].add_p2p_connection(AddrReceiver()) msg.addrs = ADDRS @@ -106,6 +101,13 @@ class AddrTest(BitcoinTestFramework): assert addr_receiver.addrv2_received_and_checked assert_equal(len(self.nodes[0].getnodeaddresses(count=0, network="i2p")), 0) + self.log.info('Send too-large addrv2 message') + msg.addrs = ADDRS * 101 + with self.nodes[0].assert_debug_log(['addrv2 message size = 1010']): + addr_source.send_message(msg) + addr_source.wait_for_disconnect() + + if __name__ == '__main__': AddrTest().main() diff --git a/test/functional/p2p_disconnect_ban.py b/test/functional/p2p_disconnect_ban.py index 678b006886..e47f9c732b 100755 --- a/test/functional/p2p_disconnect_ban.py +++ b/test/functional/p2p_disconnect_ban.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2022 The Bitcoin Core developers +# Copyright (c) 2014-present The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test node disconnect and ban behavior""" @@ -18,7 +18,7 @@ class DisconnectBanTest(BitcoinTestFramework): self.supports_cli = False def run_test(self): - self.log.info("Connect nodes both way") + self.log.info("Connect nodes both ways") # By default, the test framework sets up an addnode connection from # node 1 --> node0. By connecting node0 --> node 1, we're left with # the two nodes being connected both ways. @@ -84,7 +84,7 @@ class DisconnectBanTest(BitcoinTestFramework): assert_equal("192.168.0.1/32", listBeforeShutdown[2]['address']) self.log.info("setban: test banning with absolute timestamp") - self.nodes[1].setban("192.168.0.2", "add", old_time + 120, True) + self.nodes[1].setban("192.168.0.2", "add", old_time + 120, absolute=True) # Move time forward by 3 seconds so the fourth ban has expired self.nodes[1].setmocktime(old_time + 3) @@ -102,7 +102,9 @@ class DisconnectBanTest(BitcoinTestFramework): assert_equal(ban["ban_duration"], 120) assert_equal(ban["time_remaining"], 117) - self.restart_node(1) + # Keep mocktime, to avoid ban expiry when restart takes longer than + # time_remaining + self.restart_node(1, extra_args=[f"-mocktime={old_time+4}"]) listAfterShutdown = self.nodes[1].listbanned() assert_equal("127.0.0.0/24", listAfterShutdown[0]['address']) @@ -113,7 +115,7 @@ class DisconnectBanTest(BitcoinTestFramework): # Clear ban lists self.nodes[1].clearbanned() - self.log.info("Connect nodes both way") + self.log.info("Connect nodes both ways") self.connect_nodes(0, 1) self.connect_nodes(1, 0) diff --git a/test/functional/p2p_handshake.py b/test/functional/p2p_handshake.py index dd19fe9333..21959ae522 100755 --- a/test/functional/p2p_handshake.py +++ b/test/functional/p2p_handshake.py @@ -88,6 +88,10 @@ class P2PHandshakeTest(BitcoinTestFramework): with node.assert_debug_log([f"feeler connection completed"]): self.add_outbound_connection(node, "feeler", NODE_NONE, wait_for_disconnect=True) + # TODO: re-add test introduced in commit 5d2fb14bafe4e80c0a482d99e5ebde07c477f000 + # ("test: p2p: check that connecting to ourself leads to disconnect") once + # the race condition causing issue #30368 is fixed + if __name__ == '__main__': P2PHandshakeTest().main() diff --git a/test/functional/p2p_invalid_messages.py b/test/functional/p2p_invalid_messages.py index 40a69936bc..8e459ba676 100755 --- a/test/functional/p2p_invalid_messages.py +++ b/test/functional/p2p_invalid_messages.py @@ -5,7 +5,6 @@ """Test node responses to invalid network messages.""" import random -import struct import time from test_framework.messages import ( @@ -233,7 +232,7 @@ class InvalidMessagesTest(BitcoinTestFramework): '208d')) # port def test_addrv2_unrecognized_network(self): - now_hex = struct.pack('<I', int(time.time())).hex() + now_hex = int(time.time()).to_bytes(4, "little").hex() self.test_addrv2('unrecognized network', [ 'received: addrv2 (25 bytes)', @@ -261,7 +260,9 @@ class InvalidMessagesTest(BitcoinTestFramework): msg_type = msg.msgtype.decode('ascii') self.log.info("Test {} message of size {} is logged as misbehaving".format(msg_type, size)) with self.nodes[0].assert_debug_log(['Misbehaving', '{} message size = {}'.format(msg_type, size)]): - self.nodes[0].add_p2p_connection(P2PInterface()).send_and_ping(msg) + conn = self.nodes[0].add_p2p_connection(P2PInterface()) + conn.send_message(msg) + conn.wait_for_disconnect() self.nodes[0].disconnect_p2ps() def test_oversized_inv_msg(self): @@ -322,7 +323,8 @@ class InvalidMessagesTest(BitcoinTestFramework): # delete arbitrary block header somewhere in the middle to break link del block_headers[random.randrange(1, len(block_headers)-1)] with self.nodes[0].assert_debug_log(expected_msgs=MISBEHAVING_NONCONTINUOUS_HEADERS_MSGS): - peer.send_and_ping(msg_headers(block_headers)) + peer.send_message(msg_headers(block_headers)) + peer.wait_for_disconnect() self.nodes[0].disconnect_p2ps() def test_resource_exhaustion(self): diff --git a/test/functional/p2p_invalid_tx.py b/test/functional/p2p_invalid_tx.py index ae9dc816ab..0ae05d4b0b 100755 --- a/test/functional/p2p_invalid_tx.py +++ b/test/functional/p2p_invalid_tx.py @@ -165,7 +165,7 @@ class InvalidTxRequestTest(BitcoinTestFramework): node.p2ps[0].send_txs_and_test([rejected_parent], node, success=False) self.log.info('Test that a peer disconnection causes erase its transactions from the orphan pool') - with node.assert_debug_log(['Erased 100 orphan tx from peer=25']): + with node.assert_debug_log(['Erased 100 orphan transaction(s) from peer=25']): self.reconnect_p2p(num_connections=1) self.log.info('Test that a transaction in the orphan pool is included in a new tip block causes erase this transaction from the orphan pool') @@ -190,7 +190,7 @@ class InvalidTxRequestTest(BitcoinTestFramework): block_A.solve() self.log.info('Send the block that includes the previous orphan ... ') - with node.assert_debug_log(["Erased 1 orphan tx included or conflicted by block"]): + with node.assert_debug_log(["Erased 1 orphan transaction(s) included or conflicted by block"]): node.p2ps[0].send_blocks_and_test([block_A], node, success=True) self.log.info('Test that a transaction in the orphan pool conflicts with a new tip block causes erase this transaction from the orphan pool') @@ -219,7 +219,7 @@ class InvalidTxRequestTest(BitcoinTestFramework): block_B.solve() self.log.info('Send the block that includes a transaction which conflicts with the previous orphan ... ') - with node.assert_debug_log(["Erased 1 orphan tx included or conflicted by block"]): + with node.assert_debug_log(["Erased 1 orphan transaction(s) included or conflicted by block"]): node.p2ps[0].send_blocks_and_test([block_B], node, success=True) diff --git a/test/functional/p2p_mutated_blocks.py b/test/functional/p2p_mutated_blocks.py index 737edaf5bf..708b19b1e5 100755 --- a/test/functional/p2p_mutated_blocks.py +++ b/test/functional/p2p_mutated_blocks.py @@ -55,7 +55,7 @@ class MutatedBlocksTest(BitcoinTestFramework): # Create mutated version of the block by changing the transaction # version on the self-transfer. mutated_block = copy.deepcopy(block) - mutated_block.vtx[1].nVersion = 4 + mutated_block.vtx[1].version = 4 # Announce the new block via a compact block through the honest relayer cmpctblock = HeaderAndShortIDs() @@ -104,11 +104,10 @@ class MutatedBlocksTest(BitcoinTestFramework): block_missing_prev.hashPrevBlock = 123 block_missing_prev.solve() - # Attacker gets a DoS score of 10, not immediately disconnected, so we do it 10 times to get to 100 - for _ in range(10): - assert_equal(len(self.nodes[0].getpeerinfo()), 2) - with self.nodes[0].assert_debug_log(expected_msgs=["AcceptBlock FAILED (prev-blk-not-found)"]): - attacker.send_message(msg_block(block_missing_prev)) + # Check that non-connecting block causes disconnect + assert_equal(len(self.nodes[0].getpeerinfo()), 2) + with self.nodes[0].assert_debug_log(expected_msgs=["AcceptBlock FAILED (prev-blk-not-found)"]): + attacker.send_message(msg_block(block_missing_prev)) attacker.wait_for_disconnect(timeout=5) diff --git a/test/functional/p2p_opportunistic_1p1c.py b/test/functional/p2p_opportunistic_1p1c.py new file mode 100755 index 0000000000..aec6e95fbc --- /dev/null +++ b/test/functional/p2p_opportunistic_1p1c.py @@ -0,0 +1,415 @@ +#!/usr/bin/env python3 +# Copyright (c) 2024-present The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +""" +Test opportunistic 1p1c package submission logic. +""" + +from decimal import Decimal +import time +from test_framework.mempool_util import ( + fill_mempool, +) +from test_framework.messages import ( + CInv, + CTxInWitness, + MAX_BIP125_RBF_SEQUENCE, + MSG_WTX, + msg_inv, + msg_tx, + tx_from_hex, +) +from test_framework.p2p import ( + P2PInterface, +) +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + assert_greater_than, +) +from test_framework.wallet import ( + MiniWallet, + MiniWalletMode, +) + +# 1sat/vB feerate denominated in BTC/KvB +FEERATE_1SAT_VB = Decimal("0.00001000") +# Number of seconds to wait to ensure no getdata is received +GETDATA_WAIT = 60 + +def cleanup(func): + def wrapper(self, *args, **kwargs): + try: + func(self, *args, **kwargs) + finally: + self.nodes[0].disconnect_p2ps() + # Do not clear the node's mempool, as each test requires mempool min feerate > min + # relay feerate. However, do check that this is the case. + assert self.nodes[0].getmempoolinfo()["mempoolminfee"] > self.nodes[0].getnetworkinfo()["relayfee"] + # Ensure we do not try to spend the same UTXOs in subsequent tests, as they will look like RBF attempts. + self.wallet.rescan_utxos(include_mempool=True) + + # Resets if mocktime was used + self.nodes[0].setmocktime(0) + return wrapper + +class PackageRelayTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 1 + self.extra_args = [[ + "-datacarriersize=100000", + "-maxmempool=5", + ]] + self.supports_cli = False + + def create_tx_below_mempoolminfee(self, wallet): + """Create a 1-input 1sat/vB transaction using a confirmed UTXO. Decrement and use + self.sequence so that subsequent calls to this function result in unique transactions.""" + + self.sequence -= 1 + assert_greater_than(self.nodes[0].getmempoolinfo()["mempoolminfee"], FEERATE_1SAT_VB) + + return wallet.create_self_transfer(fee_rate=FEERATE_1SAT_VB, sequence=self.sequence, confirmed_only=True) + + @cleanup + def test_basic_child_then_parent(self): + node = self.nodes[0] + self.log.info("Check that opportunistic 1p1c logic works when child is received before parent") + + low_fee_parent = self.create_tx_below_mempoolminfee(self.wallet) + high_fee_child = self.wallet.create_self_transfer(utxo_to_spend=low_fee_parent["new_utxo"], fee_rate=20*FEERATE_1SAT_VB) + + peer_sender = node.add_p2p_connection(P2PInterface()) + + # 1. Child is received first (perhaps the low feerate parent didn't meet feefilter or the requests were sent to different nodes). It is missing an input. + high_child_wtxid_int = int(high_fee_child["tx"].getwtxid(), 16) + peer_sender.send_and_ping(msg_inv([CInv(t=MSG_WTX, h=high_child_wtxid_int)])) + peer_sender.wait_for_getdata([high_child_wtxid_int]) + peer_sender.send_and_ping(msg_tx(high_fee_child["tx"])) + + # 2. Node requests the missing parent by txid. + parent_txid_int = int(low_fee_parent["txid"], 16) + peer_sender.wait_for_getdata([parent_txid_int]) + + # 3. Sender relays the parent. Parent+Child are evaluated as a package and accepted. + peer_sender.send_and_ping(msg_tx(low_fee_parent["tx"])) + + # 4. Both transactions should now be in mempool. + node_mempool = node.getrawmempool() + assert low_fee_parent["txid"] in node_mempool + assert high_fee_child["txid"] in node_mempool + + node.disconnect_p2ps() + + @cleanup + def test_basic_parent_then_child(self, wallet): + node = self.nodes[0] + low_fee_parent = self.create_tx_below_mempoolminfee(wallet) + high_fee_child = wallet.create_self_transfer(utxo_to_spend=low_fee_parent["new_utxo"], fee_rate=20*FEERATE_1SAT_VB) + + peer_sender = node.add_outbound_p2p_connection(P2PInterface(), p2p_idx=1, connection_type="outbound-full-relay") + peer_ignored = node.add_outbound_p2p_connection(P2PInterface(), p2p_idx=2, connection_type="outbound-full-relay") + + # 1. Parent is relayed first. It is too low feerate. + parent_wtxid_int = int(low_fee_parent["tx"].getwtxid(), 16) + peer_sender.send_and_ping(msg_inv([CInv(t=MSG_WTX, h=parent_wtxid_int)])) + peer_sender.wait_for_getdata([parent_wtxid_int]) + peer_sender.send_and_ping(msg_tx(low_fee_parent["tx"])) + assert low_fee_parent["txid"] not in node.getrawmempool() + + # Send again from peer_ignored, check that it is ignored + peer_ignored.send_and_ping(msg_inv([CInv(t=MSG_WTX, h=parent_wtxid_int)])) + assert "getdata" not in peer_ignored.last_message + + # 2. Child is relayed next. It is missing an input. + high_child_wtxid_int = int(high_fee_child["tx"].getwtxid(), 16) + peer_sender.send_and_ping(msg_inv([CInv(t=MSG_WTX, h=high_child_wtxid_int)])) + peer_sender.wait_for_getdata([high_child_wtxid_int]) + peer_sender.send_and_ping(msg_tx(high_fee_child["tx"])) + + # 3. Node requests the missing parent by txid. + # It should do so even if it has previously rejected that parent for being too low feerate. + parent_txid_int = int(low_fee_parent["txid"], 16) + peer_sender.wait_for_getdata([parent_txid_int]) + + # 4. Sender re-relays the parent. Parent+Child are evaluated as a package and accepted. + peer_sender.send_and_ping(msg_tx(low_fee_parent["tx"])) + + # 5. Both transactions should now be in mempool. + node_mempool = node.getrawmempool() + assert low_fee_parent["txid"] in node_mempool + assert high_fee_child["txid"] in node_mempool + + @cleanup + def test_low_and_high_child(self, wallet): + node = self.nodes[0] + low_fee_parent = self.create_tx_below_mempoolminfee(wallet) + # This feerate is above mempoolminfee, but not enough to also bump the low feerate parent. + feerate_just_above = node.getmempoolinfo()["mempoolminfee"] + med_fee_child = wallet.create_self_transfer(utxo_to_spend=low_fee_parent["new_utxo"], fee_rate=feerate_just_above) + high_fee_child = wallet.create_self_transfer(utxo_to_spend=low_fee_parent["new_utxo"], fee_rate=999*FEERATE_1SAT_VB) + + peer_sender = node.add_outbound_p2p_connection(P2PInterface(), p2p_idx=1, connection_type="outbound-full-relay") + peer_ignored = node.add_outbound_p2p_connection(P2PInterface(), p2p_idx=2, connection_type="outbound-full-relay") + + self.log.info("Check that tx caches low fee parent + low fee child package rejections") + + # 1. Send parent, rejected for being low feerate. + parent_wtxid_int = int(low_fee_parent["tx"].getwtxid(), 16) + peer_sender.send_and_ping(msg_inv([CInv(t=MSG_WTX, h=parent_wtxid_int)])) + peer_sender.wait_for_getdata([parent_wtxid_int]) + peer_sender.send_and_ping(msg_tx(low_fee_parent["tx"])) + assert low_fee_parent["txid"] not in node.getrawmempool() + + # Send again from peer_ignored, check that it is ignored + peer_ignored.send_and_ping(msg_inv([CInv(t=MSG_WTX, h=parent_wtxid_int)])) + assert "getdata" not in peer_ignored.last_message + + # 2. Send an (orphan) child that has a higher feerate, but not enough to bump the parent. + med_child_wtxid_int = int(med_fee_child["tx"].getwtxid(), 16) + peer_sender.send_and_ping(msg_inv([CInv(t=MSG_WTX, h=med_child_wtxid_int)])) + peer_sender.wait_for_getdata([med_child_wtxid_int]) + peer_sender.send_and_ping(msg_tx(med_fee_child["tx"])) + + # 3. Node requests the orphan's missing parent. + parent_txid_int = int(low_fee_parent["txid"], 16) + peer_sender.wait_for_getdata([parent_txid_int]) + + # 4. The low parent + low child are submitted as a package. They are not accepted due to low package feerate. + peer_sender.send_and_ping(msg_tx(low_fee_parent["tx"])) + + assert low_fee_parent["txid"] not in node.getrawmempool() + assert med_fee_child["txid"] not in node.getrawmempool() + + # If peer_ignored announces the low feerate child, it should be ignored + peer_ignored.send_and_ping(msg_inv([CInv(t=MSG_WTX, h=med_child_wtxid_int)])) + assert "getdata" not in peer_ignored.last_message + # If either peer sends the parent again, package evaluation should not be attempted + peer_sender.send_and_ping(msg_tx(low_fee_parent["tx"])) + peer_ignored.send_and_ping(msg_tx(low_fee_parent["tx"])) + + assert low_fee_parent["txid"] not in node.getrawmempool() + assert med_fee_child["txid"] not in node.getrawmempool() + + # 5. Send the high feerate (orphan) child + high_child_wtxid_int = int(high_fee_child["tx"].getwtxid(), 16) + peer_sender.send_and_ping(msg_inv([CInv(t=MSG_WTX, h=high_child_wtxid_int)])) + peer_sender.wait_for_getdata([high_child_wtxid_int]) + peer_sender.send_and_ping(msg_tx(high_fee_child["tx"])) + + # 6. Node requests the orphan's parent, even though it has already been rejected, both by + # itself and with a child. This is necessary, otherwise high_fee_child can be censored. + parent_txid_int = int(low_fee_parent["txid"], 16) + peer_sender.wait_for_getdata([parent_txid_int]) + + # 7. The low feerate parent + high feerate child are submitted as a package. + peer_sender.send_and_ping(msg_tx(low_fee_parent["tx"])) + + # 8. Both transactions should now be in mempool + node_mempool = node.getrawmempool() + assert low_fee_parent["txid"] in node_mempool + assert high_fee_child["txid"] in node_mempool + assert med_fee_child["txid"] not in node_mempool + + @cleanup + def test_orphan_consensus_failure(self): + self.log.info("Check opportunistic 1p1c logic with consensus-invalid orphan causes disconnect of the correct peer") + node = self.nodes[0] + low_fee_parent = self.create_tx_below_mempoolminfee(self.wallet) + coin = low_fee_parent["new_utxo"] + address = node.get_deterministic_priv_key().address + # Create raw transaction spending the parent, but with no signature (a consensus error). + hex_orphan_no_sig = node.createrawtransaction([{"txid": coin["txid"], "vout": coin["vout"]}], {address : coin["value"] - Decimal("0.0001")}) + tx_orphan_bad_wit = tx_from_hex(hex_orphan_no_sig) + tx_orphan_bad_wit.wit.vtxinwit.append(CTxInWitness()) + tx_orphan_bad_wit.wit.vtxinwit[0].scriptWitness.stack = [b'garbage'] + + bad_orphan_sender = node.add_p2p_connection(P2PInterface()) + parent_sender = node.add_p2p_connection(P2PInterface()) + + # 1. Child is received first. It is missing an input. + child_wtxid_int = int(tx_orphan_bad_wit.getwtxid(), 16) + bad_orphan_sender.send_and_ping(msg_inv([CInv(t=MSG_WTX, h=child_wtxid_int)])) + bad_orphan_sender.wait_for_getdata([child_wtxid_int]) + bad_orphan_sender.send_and_ping(msg_tx(tx_orphan_bad_wit)) + + # 2. Node requests the missing parent by txid. + parent_txid_int = int(low_fee_parent["txid"], 16) + bad_orphan_sender.wait_for_getdata([parent_txid_int]) + + # 3. A different peer relays the parent. Parent+Child are evaluated as a package and rejected. + parent_sender.send_message(msg_tx(low_fee_parent["tx"])) + + # 4. Transactions should not be in mempool. + node_mempool = node.getrawmempool() + assert low_fee_parent["txid"] not in node_mempool + assert tx_orphan_bad_wit.rehash() not in node_mempool + + # 5. Peer that sent a consensus-invalid transaction should be disconnected. + bad_orphan_sender.wait_for_disconnect() + + # The peer that didn't provide the orphan should not be disconnected. + parent_sender.sync_with_ping() + + @cleanup + def test_parent_consensus_failure(self): + self.log.info("Check opportunistic 1p1c logic with consensus-invalid parent causes disconnect of the correct peer") + node = self.nodes[0] + low_fee_parent = self.create_tx_below_mempoolminfee(self.wallet) + high_fee_child = self.wallet.create_self_transfer(utxo_to_spend=low_fee_parent["new_utxo"], fee_rate=999*FEERATE_1SAT_VB) + + # Create invalid version of parent with a bad signature. + tx_parent_bad_wit = tx_from_hex(low_fee_parent["hex"]) + tx_parent_bad_wit.wit.vtxinwit.append(CTxInWitness()) + tx_parent_bad_wit.wit.vtxinwit[0].scriptWitness.stack = [b'garbage'] + + package_sender = node.add_p2p_connection(P2PInterface()) + fake_parent_sender = node.add_p2p_connection(P2PInterface()) + + # 1. Child is received first. It is missing an input. + child_wtxid_int = int(high_fee_child["tx"].getwtxid(), 16) + package_sender.send_and_ping(msg_inv([CInv(t=MSG_WTX, h=child_wtxid_int)])) + package_sender.wait_for_getdata([child_wtxid_int]) + package_sender.send_and_ping(msg_tx(high_fee_child["tx"])) + + # 2. Node requests the missing parent by txid. + parent_txid_int = int(tx_parent_bad_wit.rehash(), 16) + package_sender.wait_for_getdata([parent_txid_int]) + + # 3. A different node relays the parent. The parent is first evaluated by itself and + # rejected for being too low feerate. Then it is evaluated as a package and, after passing + # feerate checks, rejected for having a bad signature (consensus error). + fake_parent_sender.send_message(msg_tx(tx_parent_bad_wit)) + + # 4. Transactions should not be in mempool. + node_mempool = node.getrawmempool() + assert tx_parent_bad_wit.rehash() not in node_mempool + assert high_fee_child["txid"] not in node_mempool + + # 5. Peer sent a consensus-invalid transaction. + fake_parent_sender.wait_for_disconnect() + + self.log.info("Check that fake parent does not cause orphan to be deleted and real package can still be submitted") + # 6. Child-sending should not have been punished and the orphan should remain in orphanage. + # It can send the "real" parent transaction, and the package is accepted. + parent_wtxid_int = int(low_fee_parent["tx"].getwtxid(), 16) + package_sender.send_and_ping(msg_inv([CInv(t=MSG_WTX, h=parent_wtxid_int)])) + package_sender.wait_for_getdata([parent_wtxid_int]) + package_sender.send_and_ping(msg_tx(low_fee_parent["tx"])) + + node_mempool = node.getrawmempool() + assert low_fee_parent["txid"] in node_mempool + assert high_fee_child["txid"] in node_mempool + + @cleanup + def test_multiple_parents(self): + self.log.info("Check that node does not request more than 1 previously-rejected low feerate parent") + + node = self.nodes[0] + node.setmocktime(int(time.time())) + + # 2-parent-1-child package where both parents are below mempool min feerate + parent_low_1 = self.create_tx_below_mempoolminfee(self.wallet_nonsegwit) + parent_low_2 = self.create_tx_below_mempoolminfee(self.wallet_nonsegwit) + child_bumping = self.wallet_nonsegwit.create_self_transfer_multi( + utxos_to_spend=[parent_low_1["new_utxo"], parent_low_2["new_utxo"]], + fee_per_output=999*parent_low_1["tx"].get_vsize(), + ) + + peer_sender = node.add_outbound_p2p_connection(P2PInterface(), p2p_idx=1, connection_type="outbound-full-relay") + + # 1. Send both parents. Each should be rejected for being too low feerate. + # Send unsolicited so that we can later check that no "getdata" was ever received. + peer_sender.send_and_ping(msg_tx(parent_low_1["tx"])) + peer_sender.send_and_ping(msg_tx(parent_low_2["tx"])) + + # parent_low_1 and parent_low_2 are rejected for being low feerate. + assert parent_low_1["txid"] not in node.getrawmempool() + assert parent_low_2["txid"] not in node.getrawmempool() + + # 2. Send child. + peer_sender.send_and_ping(msg_tx(child_bumping["tx"])) + + # 3. Node should not request any parents, as it should recognize that it will not accept + # multi-parent-1-child packages. + node.bumpmocktime(GETDATA_WAIT) + peer_sender.sync_with_ping() + assert "getdata" not in peer_sender.last_message + + @cleanup + def test_other_parent_in_mempool(self): + self.log.info("Check opportunistic 1p1c fails if child already has another parent in mempool") + node = self.nodes[0] + + # This parent needs CPFP + parent_low = self.create_tx_below_mempoolminfee(self.wallet) + # This parent does not need CPFP and can be submitted alone ahead of time + parent_high = self.wallet.create_self_transfer(fee_rate=FEERATE_1SAT_VB*10, confirmed_only=True) + child = self.wallet.create_self_transfer_multi( + utxos_to_spend=[parent_high["new_utxo"], parent_low["new_utxo"]], + fee_per_output=999*parent_low["tx"].get_vsize(), + ) + + peer_sender = node.add_outbound_p2p_connection(P2PInterface(), p2p_idx=1, connection_type="outbound-full-relay") + + # 1. Send first parent which will be accepted. + peer_sender.send_and_ping(msg_tx(parent_high["tx"])) + assert parent_high["txid"] in node.getrawmempool() + + # 2. Send child. + peer_sender.send_and_ping(msg_tx(child["tx"])) + + # 3. Node requests parent_low. However, 1p1c fails because package-not-child-with-unconfirmed-parents + parent_low_txid_int = int(parent_low["txid"], 16) + peer_sender.wait_for_getdata([parent_low_txid_int]) + peer_sender.send_and_ping(msg_tx(parent_low["tx"])) + + node_mempool = node.getrawmempool() + assert parent_high["txid"] in node_mempool + assert parent_low["txid"] not in node_mempool + assert child["txid"] not in node_mempool + + # Same error if submitted through submitpackage without parent_high + package_hex_missing_parent = [parent_low["hex"], child["hex"]] + result_missing_parent = node.submitpackage(package_hex_missing_parent) + assert_equal(result_missing_parent["package_msg"], "package-not-child-with-unconfirmed-parents") + + def run_test(self): + node = self.nodes[0] + # To avoid creating transactions with the same txid (can happen if we set the same feerate + # and reuse the same input as a previous transaction that wasn't successfully submitted), + # we give each subtest a different nSequence for its transactions. + self.sequence = MAX_BIP125_RBF_SEQUENCE + + self.wallet = MiniWallet(node) + self.wallet_nonsegwit = MiniWallet(node, mode=MiniWalletMode.RAW_P2PK) + self.generate(self.wallet_nonsegwit, 10) + self.generate(self.wallet, 20) + + fill_mempool(self, node) + + self.log.info("Check opportunistic 1p1c logic when parent (txid != wtxid) is received before child") + self.test_basic_parent_then_child(self.wallet) + + self.log.info("Check opportunistic 1p1c logic when parent (txid == wtxid) is received before child") + self.test_basic_parent_then_child(self.wallet_nonsegwit) + + self.log.info("Check opportunistic 1p1c logic when child is received before parent") + self.test_basic_child_then_parent() + + self.log.info("Check opportunistic 1p1c logic when 2 candidate children exist (parent txid != wtxid)") + self.test_low_and_high_child(self.wallet) + + self.log.info("Check opportunistic 1p1c logic when 2 candidate children exist (parent txid == wtxid)") + self.test_low_and_high_child(self.wallet_nonsegwit) + + self.test_orphan_consensus_failure() + self.test_parent_consensus_failure() + self.test_multiple_parents() + self.test_other_parent_in_mempool() + + +if __name__ == '__main__': + PackageRelayTest().main() diff --git a/test/functional/p2p_orphan_handling.py b/test/functional/p2p_orphan_handling.py index 6166c62aa2..f525d87cca 100755 --- a/test/functional/p2p_orphan_handling.py +++ b/test/functional/p2p_orphan_handling.py @@ -7,6 +7,7 @@ import time from test_framework.messages import ( CInv, + CTxInWitness, MSG_TX, MSG_WITNESS_TX, MSG_WTX, @@ -21,6 +22,7 @@ from test_framework.p2p import ( NONPREF_PEER_TX_DELAY, OVERLOADED_PEER_TX_DELAY, p2p_lock, + P2PInterface, P2PTxInvStore, TXID_RELAY_DELAY, ) @@ -127,6 +129,22 @@ class OrphanHandlingTest(BitcoinTestFramework): peer.wait_for_getdata([wtxid]) peer.send_and_ping(msg_tx(tx)) + def create_malleated_version(self, tx): + """ + Create a malleated version of the tx where the witness is replaced with garbage data. + Returns a CTransaction object. + """ + tx_bad_wit = tx_from_hex(tx["hex"]) + tx_bad_wit.wit.vtxinwit = [CTxInWitness()] + # Add garbage data to witness 0. We cannot simply strip the witness, as the node would + # classify it as a transaction in which the witness was missing rather than wrong. + tx_bad_wit.wit.vtxinwit[0].scriptWitness.stack = [b'garbage'] + + assert_equal(tx["txid"], tx_bad_wit.rehash()) + assert tx["wtxid"] != tx_bad_wit.getwtxid() + + return tx_bad_wit + @cleanup def test_arrival_timing_orphan(self): self.log.info("Test missing parents that arrive during delay are not requested") @@ -284,8 +302,8 @@ class OrphanHandlingTest(BitcoinTestFramework): # doesn't give up on the orphan. Once all of the missing parents are received, it should be # submitted to mempool. peer.send_message(msg_notfound(vec=[CInv(MSG_WITNESS_TX, int(txid_conf_old, 16))])) + # Sync with ping to ensure orphans are reconsidered peer.send_and_ping(msg_tx(missing_tx["tx"])) - peer.sync_with_ping() assert_equal(node.getmempoolentry(orphan["txid"])["ancestorcount"], 3) @cleanup @@ -394,10 +412,161 @@ class OrphanHandlingTest(BitcoinTestFramework): peer2.assert_never_requested(child["tx"].getwtxid()) # The child should never be requested, even if announced again with potentially different witness. + # Sync with ping to ensure orphans are reconsidered peer3.send_and_ping(msg_inv([CInv(t=MSG_TX, h=int(child["txid"], 16))])) self.nodes[0].bumpmocktime(TXREQUEST_TIME_SKIP) peer3.assert_never_requested(child["txid"]) + @cleanup + def test_same_txid_orphan(self): + self.log.info("Check what happens when orphan with same txid is already in orphanage") + node = self.nodes[0] + + tx_parent = self.wallet.create_self_transfer() + + # Create the real child + tx_child = self.wallet.create_self_transfer(utxo_to_spend=tx_parent["new_utxo"]) + + # Create a fake version of the child + tx_orphan_bad_wit = self.create_malleated_version(tx_child) + + bad_peer = node.add_p2p_connection(P2PInterface()) + honest_peer = node.add_p2p_connection(P2PInterface()) + + # 1. Fake orphan is received first. It is missing an input. + bad_peer.send_and_ping(msg_tx(tx_orphan_bad_wit)) + + # 2. Node requests the missing parent by txid. + parent_txid_int = int(tx_parent["txid"], 16) + node.bumpmocktime(NONPREF_PEER_TX_DELAY + TXID_RELAY_DELAY) + bad_peer.wait_for_getdata([parent_txid_int]) + + # 3. Honest peer relays the real child, which is also missing parents and should be placed + # in the orphanage. + with node.assert_debug_log(["missingorspent", "stored orphan tx"]): + honest_peer.send_and_ping(msg_tx(tx_child["tx"])) + + # Time out the previous request for the parent (node will not request the same transaction + # from multiple nodes at the same time) + node.bumpmocktime(GETDATA_TX_INTERVAL) + + # 4. The parent is requested. Honest peer sends it. + honest_peer.wait_for_getdata([parent_txid_int]) + # Sync with ping to ensure orphans are reconsidered + honest_peer.send_and_ping(msg_tx(tx_parent["tx"])) + + # 5. After parent is accepted, orphans should be reconsidered. + # The real child should be accepted and the fake one rejected. + node_mempool = node.getrawmempool() + assert tx_parent["txid"] in node_mempool + assert tx_child["txid"] in node_mempool + assert_equal(node.getmempoolentry(tx_child["txid"])["wtxid"], tx_child["wtxid"]) + + @cleanup + def test_same_txid_orphan_of_orphan(self): + self.log.info("Check what happens when orphan's parent with same txid is already in orphanage") + node = self.nodes[0] + + tx_grandparent = self.wallet.create_self_transfer() + + # Create middle tx (both parent and child) which will be in orphanage. + tx_middle = self.wallet.create_self_transfer(utxo_to_spend=tx_grandparent["new_utxo"]) + + # Create a fake version of the middle tx + tx_orphan_bad_wit = self.create_malleated_version(tx_middle) + + # Create grandchild spending from tx_middle (and spending from tx_orphan_bad_wit since they + # have the same txid). + tx_grandchild = self.wallet.create_self_transfer(utxo_to_spend=tx_middle["new_utxo"]) + + bad_peer = node.add_p2p_connection(P2PInterface()) + honest_peer = node.add_p2p_connection(P2PInterface()) + + # 1. Fake orphan is received first. It is missing an input. + bad_peer.send_and_ping(msg_tx(tx_orphan_bad_wit)) + + # 2. Node requests missing tx_grandparent by txid. + grandparent_txid_int = int(tx_grandparent["txid"], 16) + node.bumpmocktime(NONPREF_PEER_TX_DELAY + TXID_RELAY_DELAY) + bad_peer.wait_for_getdata([grandparent_txid_int]) + + # 3. Honest peer relays the grandchild, which is missing a parent. The parent by txid already + # exists in orphanage, but should be re-requested because the node shouldn't assume that the + # witness data is the same. In this case, a same-txid-different-witness transaction exists! + with node.assert_debug_log(["stored orphan tx"]): + honest_peer.send_and_ping(msg_tx(tx_grandchild["tx"])) + middle_txid_int = int(tx_middle["txid"], 16) + node.bumpmocktime(NONPREF_PEER_TX_DELAY + TXID_RELAY_DELAY) + honest_peer.wait_for_getdata([middle_txid_int]) + + # 4. Honest peer relays the real child, which is also missing parents and should be placed + # in the orphanage. + with node.assert_debug_log(["stored orphan tx"]): + honest_peer.send_and_ping(msg_tx(tx_middle["tx"])) + assert_equal(len(node.getrawmempool()), 0) + + # 5. Honest peer sends tx_grandparent + honest_peer.send_and_ping(msg_tx(tx_grandparent["tx"])) + + # 6. After parent is accepted, orphans should be reconsidered. + # The real child should be accepted and the fake one rejected. + node_mempool = node.getrawmempool() + assert tx_grandparent["txid"] in node_mempool + assert tx_middle["txid"] in node_mempool + assert tx_grandchild["txid"] in node_mempool + assert_equal(node.getmempoolentry(tx_middle["txid"])["wtxid"], tx_middle["wtxid"]) + + @cleanup + def test_orphan_txid_inv(self): + self.log.info("Check node does not ignore announcement with same txid as tx in orphanage") + node = self.nodes[0] + + tx_parent = self.wallet.create_self_transfer() + + # Create the real child and fake version + tx_child = self.wallet.create_self_transfer(utxo_to_spend=tx_parent["new_utxo"]) + tx_orphan_bad_wit = self.create_malleated_version(tx_child) + + bad_peer = node.add_p2p_connection(PeerTxRelayer()) + # Must not send wtxidrelay because otherwise the inv(TX) will be ignored later + honest_peer = node.add_p2p_connection(P2PInterface(wtxidrelay=False)) + + # 1. Fake orphan is received first. It is missing an input. + bad_peer.send_and_ping(msg_tx(tx_orphan_bad_wit)) + + # 2. Node requests the missing parent by txid. + parent_txid_int = int(tx_parent["txid"], 16) + node.bumpmocktime(NONPREF_PEER_TX_DELAY + TXID_RELAY_DELAY) + bad_peer.wait_for_getdata([parent_txid_int]) + + # 3. Honest peer announces the real child, by txid (this isn't common but the node should + # still keep track of it). + child_txid_int = int(tx_child["txid"], 16) + honest_peer.send_and_ping(msg_inv([CInv(t=MSG_TX, h=child_txid_int)])) + + # 4. The child is requested. Honest peer sends it. + node.bumpmocktime(TXREQUEST_TIME_SKIP) + honest_peer.wait_for_getdata([child_txid_int]) + with node.assert_debug_log(["stored orphan tx"]): + honest_peer.send_and_ping(msg_tx(tx_child["tx"])) + + # 5. After first parent request times out, the node sends another one for the missing parent + # of the real orphan child. + node.bumpmocktime(GETDATA_TX_INTERVAL) + honest_peer.wait_for_getdata([parent_txid_int]) + honest_peer.send_and_ping(msg_tx(tx_parent["tx"])) + + # 6. After parent is accepted, orphans should be reconsidered. + # The real child should be accepted and the fake one rejected. This may happen in either + # order since the message-processing is randomized. If tx_orphan_bad_wit is validated first, + # its consensus error leads to disconnection of bad_peer. If tx_child is validated first, + # tx_orphan_bad_wit is rejected for txn-same-nonwitness-data-in-mempool (no punishment). + node_mempool = node.getrawmempool() + assert tx_parent["txid"] in node_mempool + assert tx_child["txid"] in node_mempool + assert_equal(node.getmempoolentry(tx_child["txid"])["wtxid"], tx_child["wtxid"]) + + def run_test(self): self.nodes[0].setmocktime(int(time.time())) self.wallet_nonsegwit = MiniWallet(self.nodes[0], mode=MiniWalletMode.RAW_P2PK) @@ -410,6 +579,9 @@ class OrphanHandlingTest(BitcoinTestFramework): self.test_orphans_overlapping_parents() self.test_orphan_of_orphan() self.test_orphan_inherit_rejection() + self.test_same_txid_orphan() + self.test_same_txid_orphan_of_orphan() + self.test_orphan_txid_inv() if __name__ == '__main__': diff --git a/test/functional/p2p_outbound_eviction.py b/test/functional/p2p_outbound_eviction.py new file mode 100755 index 0000000000..8d89688999 --- /dev/null +++ b/test/functional/p2p_outbound_eviction.py @@ -0,0 +1,253 @@ +#!/usr/bin/env python3 +# Copyright (c) 2019-2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" Test node outbound peer eviction logic + +A subset of our outbound peers are subject to eviction logic if they cannot keep up +with our vision of the best chain. This criteria applies only to non-protected peers, +and can be triggered by either not learning about any blocks from an outbound peer after +a certain deadline, or by them not being able to catch up fast enough (under the same deadline). + +This tests the different eviction paths based on the peer's behavior and on whether they are protected +or not. +""" +import time + +from test_framework.messages import ( + from_hex, + msg_headers, + CBlockHeader, +) +from test_framework.p2p import P2PInterface +from test_framework.test_framework import BitcoinTestFramework + +# Timeouts (in seconds) +CHAIN_SYNC_TIMEOUT = 20 * 60 +HEADERS_RESPONSE_TIME = 2 * 60 + + +class P2POutEvict(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + + def test_outbound_eviction_unprotected(self): + # This tests the eviction logic for **unprotected** outbound peers (that is, PeerManagerImpl::ConsiderEviction) + node = self.nodes[0] + cur_mock_time = node.mocktime + + # Get our tip header and its parent + tip_header = from_hex(CBlockHeader(), node.getblockheader(node.getbestblockhash(), False)) + prev_header = from_hex(CBlockHeader(), node.getblockheader(f"{tip_header.hashPrevBlock:064x}", False)) + + self.log.info("Create an outbound connection and don't send any headers") + # Test disconnect due to no block being announced in 22+ minutes (headers are not even exchanged) + peer = node.add_outbound_p2p_connection(P2PInterface(), p2p_idx=0, connection_type="outbound-full-relay") + # Wait for over 20 min to trigger the first eviction timeout. This sets the last call past 2 min in the future. + cur_mock_time += (CHAIN_SYNC_TIMEOUT + 1) + node.setmocktime(cur_mock_time) + peer.sync_with_ping() + # Wait for over 2 more min to trigger the disconnection + peer.wait_for_getheaders(block_hash=tip_header.hashPrevBlock) + cur_mock_time += (HEADERS_RESPONSE_TIME + 1) + node.setmocktime(cur_mock_time) + self.log.info("Test that the peer gets evicted") + peer.wait_for_disconnect() + + self.log.info("Create an outbound connection and send header but never catch up") + # Mimic a node that just falls behind for long enough + # This should also apply for a node doing IBD that does not catch up in time + # Connect a peer and make it send us headers ending in our tip's parent + peer = node.add_outbound_p2p_connection(P2PInterface(), p2p_idx=0, connection_type="outbound-full-relay") + peer.send_and_ping(msg_headers([prev_header])) + + # Trigger the timeouts + cur_mock_time += (CHAIN_SYNC_TIMEOUT + 1) + node.setmocktime(cur_mock_time) + peer.sync_with_ping() + peer.wait_for_getheaders(block_hash=tip_header.hashPrevBlock) + cur_mock_time += (HEADERS_RESPONSE_TIME + 1) + node.setmocktime(cur_mock_time) + self.log.info("Test that the peer gets evicted") + peer.wait_for_disconnect() + + self.log.info("Create an outbound connection and keep lagging behind, but not too much") + # Test that if the peer never catches up with our current tip, but it does with the + # expected work that we set when setting the timer (that is, our tip at the time) + # we do not disconnect the peer + peer = node.add_outbound_p2p_connection(P2PInterface(), p2p_idx=0, connection_type="outbound-full-relay") + + self.log.info("Mine a block so our peer starts lagging") + prev_prev_hash = tip_header.hashPrevBlock + best_block_hash = self.generateblock(node, output="raw(42)", transactions=[])["hash"] + peer.sync_with_ping() + + self.log.info("Keep catching up with the old tip and check that we are not evicted") + for i in range(10): + # Generate an additional block so the peers is 2 blocks behind + prev_header = from_hex(CBlockHeader(), node.getblockheader(best_block_hash, False)) + best_block_hash = self.generateblock(node, output="raw(42)", transactions=[])["hash"] + peer.sync_with_ping() + + # Advance time but not enough to evict the peer + cur_mock_time += (CHAIN_SYNC_TIMEOUT + 1) + node.setmocktime(cur_mock_time) + peer.sync_with_ping() + + # Wait until we get out last call (by receiving a getheaders) + peer.wait_for_getheaders(block_hash=prev_prev_hash) + + # Send a header with the previous tip (so we go back to 1 block behind) + peer.send_and_ping(msg_headers([prev_header])) + prev_prev_hash = tip_header.hash + + self.log.info("Create an outbound connection and take some time to catch up, but do it in time") + # Check that if the peer manages to catch up within time, the timeouts are removed (and the peer is not disconnected) + # We are reusing the peer from the previous case which already sent us a valid (but old) block and whose timer is ticking + + # Send an updated headers message matching our tip + peer.send_and_ping(msg_headers([from_hex(CBlockHeader(), node.getblockheader(best_block_hash, False))])) + + # Wait for long enough for the timeouts to have triggered and check that we are still connected + cur_mock_time += (CHAIN_SYNC_TIMEOUT + 1) + node.setmocktime(cur_mock_time) + peer.sync_with_ping() + cur_mock_time += (HEADERS_RESPONSE_TIME + 1) + node.setmocktime(cur_mock_time) + self.log.info("Test that the peer does not get evicted") + peer.sync_with_ping() + + node.disconnect_p2ps() + + def test_outbound_eviction_protected(self): + # This tests the eviction logic for **protected** outbound peers (that is, PeerManagerImpl::ConsiderEviction) + # Outbound connections are flagged as protected as long as they have sent us a connecting block with at least as + # much work as our current tip and we have enough empty protected_peers slots. + node = self.nodes[0] + cur_mock_time = node.mocktime + tip_header = from_hex(CBlockHeader(), node.getblockheader(node.getbestblockhash(), False)) + + self.log.info("Create an outbound connection to a peer that shares our tip so it gets granted protection") + peer = node.add_outbound_p2p_connection(P2PInterface(), p2p_idx=0, connection_type="outbound-full-relay") + peer.send_and_ping(msg_headers([tip_header])) + + self.log.info("Mine a new block and sync with our peer") + self.generateblock(node, output="raw(42)", transactions=[]) + peer.sync_with_ping() + + self.log.info("Let enough time pass for the timeouts to go off") + # Trigger the timeouts and check how we are still connected + cur_mock_time += (CHAIN_SYNC_TIMEOUT + 1) + node.setmocktime(cur_mock_time) + peer.sync_with_ping() + peer.wait_for_getheaders(block_hash=tip_header.hashPrevBlock) + cur_mock_time += (HEADERS_RESPONSE_TIME + 1) + node.setmocktime(cur_mock_time) + self.log.info("Test that the node does not get evicted") + peer.sync_with_ping() + + node.disconnect_p2ps() + + def test_outbound_eviction_mixed(self): + # This tests the outbound eviction logic for a mix of protected and unprotected peers. + node = self.nodes[0] + cur_mock_time = node.mocktime + + self.log.info("Create a mix of protected and unprotected outbound connections to check against eviction") + + # Let's try this logic having multiple peers, some protected and some unprotected + # We protect up to 4 peers as long as they have provided a block with the same amount of work as our tip + self.log.info("The first 4 peers are protected by sending us a valid block with enough work") + tip_header = from_hex(CBlockHeader(), node.getblockheader(node.getbestblockhash(), False)) + headers_message = msg_headers([tip_header]) + protected_peers = [] + for i in range(4): + peer = node.add_outbound_p2p_connection(P2PInterface(), p2p_idx=i, connection_type="outbound-full-relay") + peer.send_and_ping(headers_message) + protected_peers.append(peer) + + # We can create 4 additional outbound connections to peers that are unprotected. 2 of them will be well behaved, + # whereas the other 2 will misbehave (1 sending no headers, 1 sending old ones) + self.log.info("The remaining 4 peers will be mixed between honest (2) and misbehaving peers (2)") + prev_header = from_hex(CBlockHeader(), node.getblockheader(f"{tip_header.hashPrevBlock:064x}", False)) + headers_message = msg_headers([prev_header]) + honest_unprotected_peers = [] + for i in range(2): + peer = node.add_outbound_p2p_connection(P2PInterface(), p2p_idx=4+i, connection_type="outbound-full-relay") + peer.send_and_ping(headers_message) + honest_unprotected_peers.append(peer) + + misbehaving_unprotected_peers = [] + for i in range(2): + peer = node.add_outbound_p2p_connection(P2PInterface(), p2p_idx=6+i, connection_type="outbound-full-relay") + if i%2==0: + peer.send_and_ping(headers_message) + misbehaving_unprotected_peers.append(peer) + + self.log.info("Mine a new block and keep the unprotected honest peer on sync, all the rest off-sync") + # Mine a block so all peers become outdated + target_hash = prev_header.rehash() + tip_hash = self.generateblock(node, output="raw(42)", transactions=[])["hash"] + tip_header = from_hex(CBlockHeader(), node.getblockheader(tip_hash, False)) + tip_headers_message = msg_headers([tip_header]) + + # Let the timeouts hit and check back + cur_mock_time += (CHAIN_SYNC_TIMEOUT + 1) + node.setmocktime(cur_mock_time) + for peer in protected_peers + misbehaving_unprotected_peers: + peer.sync_with_ping() + peer.wait_for_getheaders(block_hash=target_hash) + for peer in honest_unprotected_peers: + peer.send_and_ping(tip_headers_message) + peer.wait_for_getheaders(block_hash=target_hash) + + cur_mock_time += (HEADERS_RESPONSE_TIME + 1) + node.setmocktime(cur_mock_time) + self.log.info("Check how none of the honest nor protected peers was evicted but all the misbehaving unprotected were") + for peer in protected_peers + honest_unprotected_peers: + peer.sync_with_ping() + for peer in misbehaving_unprotected_peers: + peer.wait_for_disconnect() + + node.disconnect_p2ps() + + def test_outbound_eviction_blocks_relay_only(self): + # The logic for outbound eviction protection only applies to outbound-full-relay peers + # This tests that other types of peers (blocks-relay-only for instance) are not granted protection + node = self.nodes[0] + cur_mock_time = node.mocktime + tip_header = from_hex(CBlockHeader(), node.getblockheader(node.getbestblockhash(), False)) + + self.log.info("Create an blocks-only outbound connection to a peer that shares our tip. This would usually grant protection") + peer = node.add_outbound_p2p_connection(P2PInterface(), p2p_idx=0, connection_type="block-relay-only") + peer.send_and_ping(msg_headers([tip_header])) + + self.log.info("Mine a new block and sync with our peer") + self.generateblock(node, output="raw(42)", transactions=[]) + peer.sync_with_ping() + + self.log.info("Let enough time pass for the timeouts to go off") + # Trigger the timeouts and check how the peer gets evicted, since protection is only given to outbound-full-relay peers + cur_mock_time += (CHAIN_SYNC_TIMEOUT + 1) + node.setmocktime(cur_mock_time) + peer.sync_with_ping() + peer.wait_for_getheaders(block_hash=tip_header.hash) + cur_mock_time += (HEADERS_RESPONSE_TIME + 1) + node.setmocktime(cur_mock_time) + self.log.info("Test that the peer gets evicted") + peer.wait_for_disconnect() + + node.disconnect_p2ps() + + + def run_test(self): + self.nodes[0].setmocktime(int(time.time())) + self.test_outbound_eviction_unprotected() + self.test_outbound_eviction_protected() + self.test_outbound_eviction_mixed() + self.test_outbound_eviction_blocks_relay_only() + + +if __name__ == '__main__': + P2POutEvict().main() diff --git a/test/functional/p2p_segwit.py b/test/functional/p2p_segwit.py index 213c748eda..d20cf41a72 100755 --- a/test/functional/p2p_segwit.py +++ b/test/functional/p2p_segwit.py @@ -5,7 +5,6 @@ """Test segwit transactions and blocks on P2P network.""" from decimal import Decimal import random -import struct import time from test_framework.blocktools import ( @@ -1054,7 +1053,7 @@ class SegWitTest(BitcoinTestFramework): @subtest def test_max_witness_push_length(self): - """Test that witness stack can only allow up to 520 byte pushes.""" + """Test that witness stack can only allow up to MAX_SCRIPT_ELEMENT_SIZE byte pushes.""" block = self.build_next_block() @@ -1165,16 +1164,16 @@ class SegWitTest(BitcoinTestFramework): if not self.wit.is_null(): flags |= 1 r = b"" - r += struct.pack("<i", self.nVersion) + r += self.version.to_bytes(4, "little") if flags: dummy = [] r += ser_vector(dummy) - r += struct.pack("<B", flags) + r += flags.to_bytes(1, "little") r += ser_vector(self.vin) r += ser_vector(self.vout) if flags & 1: r += self.wit.serialize() - r += struct.pack("<I", self.nLockTime) + r += self.nLockTime.to_bytes(4, "little") return r tx2 = BrokenCTransaction() @@ -1976,11 +1975,11 @@ class SegWitTest(BitcoinTestFramework): def serialize_with_bogus_witness(tx): flags = 3 r = b"" - r += struct.pack("<i", tx.nVersion) + r += tx.version.to_bytes(4, "little") if flags: dummy = [] r += ser_vector(dummy) - r += struct.pack("<B", flags) + r += flags.to_bytes(1, "little") r += ser_vector(tx.vin) r += ser_vector(tx.vout) if flags & 1: @@ -1990,7 +1989,7 @@ class SegWitTest(BitcoinTestFramework): for _ in range(len(tx.wit.vtxinwit), len(tx.vin)): tx.wit.vtxinwit.append(CTxInWitness()) r += tx.wit.serialize() - r += struct.pack("<I", tx.nLockTime) + r += tx.nLockTime.to_bytes(4, "little") return r class msg_bogus_tx(msg_tx): diff --git a/test/functional/p2p_sendheaders.py b/test/functional/p2p_sendheaders.py index 27a3aa8fb9..5c463267a1 100755 --- a/test/functional/p2p_sendheaders.py +++ b/test/functional/p2p_sendheaders.py @@ -71,19 +71,13 @@ f. Announce 1 more header that builds on that fork. Expect: no response. Part 5: Test handling of headers that don't connect. -a. Repeat 10 times: +a. Repeat 100 times: 1. Announce a header that doesn't connect. Expect: getheaders message 2. Send headers chain. Expect: getdata for the missing blocks, tip update. -b. Then send 9 more headers that don't connect. +b. Then send 99 more headers that don't connect. Expect: getheaders message each time. -c. Announce a header that does connect. - Expect: no response. -d. Announce 49 headers that don't connect. - Expect: getheaders message each time. -e. Announce one more that doesn't connect. - Expect: disconnect. """ from test_framework.blocktools import create_block, create_coinbase from test_framework.messages import CInv @@ -526,7 +520,8 @@ class SendHeadersTest(BitcoinTestFramework): # First we test that receipt of an unconnecting header doesn't prevent # chain sync. expected_hash = tip - for i in range(10): + NUM_HEADERS = 100 + for i in range(NUM_HEADERS): self.log.debug("Part 5.{}: starting...".format(i)) test_node.last_message.pop("getdata", None) blocks = [] @@ -550,41 +545,24 @@ class SendHeadersTest(BitcoinTestFramework): blocks = [] # Now we test that if we repeatedly don't send connecting headers, we # don't go into an infinite loop trying to get them to connect. - MAX_NUM_UNCONNECTING_HEADERS_MSGS = 10 - for _ in range(MAX_NUM_UNCONNECTING_HEADERS_MSGS + 1): + for _ in range(NUM_HEADERS + 1): blocks.append(create_block(tip, create_coinbase(height), block_time)) blocks[-1].solve() tip = blocks[-1].sha256 block_time += 1 height += 1 - for i in range(1, MAX_NUM_UNCONNECTING_HEADERS_MSGS): - # Send a header that doesn't connect, check that we get a getheaders. + for i in range(1, NUM_HEADERS): + with p2p_lock: + test_node.last_message.pop("getheaders", None) + # Send an empty header as a failed response to the received getheaders + # (from the previous iteration). Otherwise, the new headers will be + # treated as a response instead of as an announcement. + test_node.send_header_for_blocks([]) + # Send the actual unconnecting header, which should trigger a new getheaders. test_node.send_header_for_blocks([blocks[i]]) test_node.wait_for_getheaders(block_hash=expected_hash) - # Next header will connect, should re-set our count: - test_node.send_header_for_blocks([blocks[0]]) - expected_hash = blocks[0].sha256 - - # Remove the first two entries (blocks[1] would connect): - blocks = blocks[2:] - - # Now try to see how many unconnecting headers we can send - # before we get disconnected. Should be 5*MAX_NUM_UNCONNECTING_HEADERS_MSGS - for i in range(5 * MAX_NUM_UNCONNECTING_HEADERS_MSGS - 1): - # Send a header that doesn't connect, check that we get a getheaders. - test_node.send_header_for_blocks([blocks[i % len(blocks)]]) - test_node.wait_for_getheaders(block_hash=expected_hash) - - # Eventually this stops working. - test_node.send_header_for_blocks([blocks[-1]]) - - # Should get disconnected - test_node.wait_for_disconnect() - - self.log.info("Part 5: success!") - # Finally, check that the inv node never received a getdata request, # throughout the test assert "getdata" not in inv_node.last_message diff --git a/test/functional/p2p_tx_download.py b/test/functional/p2p_tx_download.py index 7a50f1e605..0af6b1d2c9 100755 --- a/test/functional/p2p_tx_download.py +++ b/test/functional/p2p_tx_download.py @@ -8,6 +8,9 @@ Test transaction download behavior from decimal import Decimal import time +from test_framework.mempool_util import ( + fill_mempool, +) from test_framework.messages import ( CInv, MSG_TX, @@ -24,7 +27,6 @@ from test_framework.p2p import ( from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, - fill_mempool, ) from test_framework.wallet import MiniWallet @@ -248,7 +250,7 @@ class TxDownloadTest(BitcoinTestFramework): def test_rejects_filter_reset(self): self.log.info('Check that rejected tx is not requested again') node = self.nodes[0] - fill_mempool(self, node, self.wallet) + fill_mempool(self, node) self.wallet.rescan_utxos() mempoolminfee = node.getmempoolinfo()['mempoolminfee'] peer = node.add_p2p_connection(TestP2PConn()) diff --git a/test/functional/p2p_unrequested_blocks.py b/test/functional/p2p_unrequested_blocks.py index f368434895..776eaf5255 100755 --- a/test/functional/p2p_unrequested_blocks.py +++ b/test/functional/p2p_unrequested_blocks.py @@ -170,9 +170,11 @@ class AcceptBlockTest(BitcoinTestFramework): tip = next_block # Now send the block at height 5 and check that it wasn't accepted (missing header) - test_node.send_and_ping(msg_block(all_blocks[1])) + test_node.send_message(msg_block(all_blocks[1])) + test_node.wait_for_disconnect() assert_raises_rpc_error(-5, "Block not found", self.nodes[0].getblock, all_blocks[1].hash) assert_raises_rpc_error(-5, "Block not found", self.nodes[0].getblockheader, all_blocks[1].hash) + test_node = self.nodes[0].add_p2p_connection(P2PInterface()) # The block at height 5 should be accepted if we provide the missing header, though headers_message = msg_headers() diff --git a/test/functional/p2p_v2_earlykeyresponse.py b/test/functional/p2p_v2_earlykeyresponse.py deleted file mode 100755 index 32d2e1148a..0000000000 --- a/test/functional/p2p_v2_earlykeyresponse.py +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2022 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -import random - -from test_framework.test_framework import BitcoinTestFramework -from test_framework.crypto.ellswift import ellswift_create -from test_framework.p2p import P2PInterface -from test_framework.v2_p2p import EncryptedP2PState - - -class TestEncryptedP2PState(EncryptedP2PState): - """ Modify v2 P2P protocol functions for testing that "The responder waits until one byte is received which does - not match the 16 bytes consisting of the network magic followed by "version\x00\x00\x00\x00\x00"." (see BIP 324) - - - if `send_net_magic` is True, send first 4 bytes of ellswift (match network magic) else send remaining 60 bytes - - `can_data_be_received` is a variable used to assert if data is received on recvbuf. - - v2 TestNode shouldn't respond back if we send V1_PREFIX and data shouldn't be received on recvbuf. - This state is represented using `can_data_be_received` = False. - - v2 TestNode responds back when mismatch from V1_PREFIX happens and data can be received on recvbuf. - This state is represented using `can_data_be_received` = True. - """ - - def __init__(self): - super().__init__(initiating=True, net='regtest') - self.send_net_magic = True - self.can_data_be_received = False - - def initiate_v2_handshake(self, garbage_len=random.randrange(4096)): - """Initiator begins the v2 handshake by sending its ellswift bytes and garbage. - Here, the 64 bytes ellswift is assumed to have it's 4 bytes match network magic bytes. It is sent in 2 phases: - 1. when `send_network_magic` = True, send first 4 bytes of ellswift (matches network magic bytes) - 2. when `send_network_magic` = False, send remaining 60 bytes of ellswift - """ - if self.send_net_magic: - self.privkey_ours, self.ellswift_ours = ellswift_create() - self.sent_garbage = random.randbytes(garbage_len) - self.send_net_magic = False - return b"\xfa\xbf\xb5\xda" - else: - self.can_data_be_received = True - return self.ellswift_ours[4:] + self.sent_garbage - - -class PeerEarlyKey(P2PInterface): - """Custom implementation of P2PInterface which uses modified v2 P2P protocol functions for testing purposes.""" - def __init__(self): - super().__init__() - self.v2_state = None - self.connection_opened = False - - def connection_made(self, transport): - """64 bytes ellswift is sent in 2 parts during `initial_v2_handshake()`""" - self.v2_state = TestEncryptedP2PState() - super().connection_made(transport) - - def data_received(self, t): - # check that data can be received on recvbuf only when mismatch from V1_PREFIX happens (send_net_magic = False) - assert self.v2_state.can_data_be_received and not self.v2_state.send_net_magic - - def on_open(self): - self.connection_opened = True - -class P2PEarlyKey(BitcoinTestFramework): - def set_test_params(self): - self.num_nodes = 1 - self.extra_args = [["-v2transport=1", "-peertimeout=3"]] - - def run_test(self): - self.log.info('Sending ellswift bytes in parts to ensure that response from responder is received only when') - self.log.info('ellswift bytes have a mismatch from the 16 bytes(network magic followed by "version\\x00\\x00\\x00\\x00\\x00")') - node0 = self.nodes[0] - self.log.info('Sending first 4 bytes of ellswift which match network magic') - self.log.info('If a response is received, assertion failure would happen in our custom data_received() function') - # send happens in `initiate_v2_handshake()` in `connection_made()` - peer1 = node0.add_p2p_connection(PeerEarlyKey(), wait_for_verack=False, send_version=False, supports_v2_p2p=True, wait_for_v2_handshake=False) - self.wait_until(lambda: peer1.connection_opened) - self.log.info('Sending remaining ellswift and garbage which are different from V1_PREFIX. Since a response is') - self.log.info('expected now, our custom data_received() function wouldn\'t result in assertion failure') - ellswift_and_garbage_data = peer1.v2_state.initiate_v2_handshake() - peer1.send_raw_message(ellswift_and_garbage_data) - peer1.wait_for_disconnect(timeout=5) - self.log.info('successful disconnection when MITM happens in the key exchange phase') - - -if __name__ == '__main__': - P2PEarlyKey().main() diff --git a/test/functional/p2p_v2_misbehaving.py b/test/functional/p2p_v2_misbehaving.py new file mode 100755 index 0000000000..e45a63b3b0 --- /dev/null +++ b/test/functional/p2p_v2_misbehaving.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +import random +from enum import Enum + +from test_framework.messages import MAGIC_BYTES +from test_framework.p2p import P2PInterface +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import random_bitflip +from test_framework.v2_p2p import ( + EncryptedP2PState, + MAX_GARBAGE_LEN, +) + + +class TestType(Enum): + """ Scenarios to be tested: + + 1. EARLY_KEY_RESPONSE - The responder needs to wait until one byte is received which does not match the 16 bytes + consisting of network magic followed by "version\x00\x00\x00\x00\x00" before sending out its ellswift + garbage bytes + 2. EXCESS_GARBAGE - Disconnection happens when > MAX_GARBAGE_LEN bytes garbage is sent + 3. WRONG_GARBAGE_TERMINATOR - Disconnection happens when incorrect garbage terminator is sent + 4. WRONG_GARBAGE - Disconnection happens when garbage bytes that is sent is different from what the peer receives + 5. SEND_NO_AAD - Disconnection happens when AAD of first encrypted packet after the garbage terminator is not filled + 6. SEND_NON_EMPTY_VERSION_PACKET - non-empty version packet is simply ignored + """ + EARLY_KEY_RESPONSE = 0 + EXCESS_GARBAGE = 1 + WRONG_GARBAGE_TERMINATOR = 2 + WRONG_GARBAGE = 3 + SEND_NO_AAD = 4 + SEND_NON_EMPTY_VERSION_PACKET = 5 + + +class EarlyKeyResponseState(EncryptedP2PState): + """ Modify v2 P2P protocol functions for testing EARLY_KEY_RESPONSE scenario""" + def __init__(self, initiating, net): + super().__init__(initiating=initiating, net=net) + self.can_data_be_received = False # variable used to assert if data is received on recvbuf. + + def initiate_v2_handshake(self): + """Send ellswift and garbage bytes in 2 parts when TestType = (EARLY_KEY_RESPONSE)""" + self.generate_keypair_and_garbage() + return b"" + + +class ExcessGarbageState(EncryptedP2PState): + """Generate > MAX_GARBAGE_LEN garbage bytes""" + def generate_keypair_and_garbage(self): + garbage_len = MAX_GARBAGE_LEN + random.randrange(1, MAX_GARBAGE_LEN + 1) + return super().generate_keypair_and_garbage(garbage_len) + + +class WrongGarbageTerminatorState(EncryptedP2PState): + """Add option for sending wrong garbage terminator""" + def generate_keypair_and_garbage(self): + garbage_len = random.randrange(MAX_GARBAGE_LEN//2) + return super().generate_keypair_and_garbage(garbage_len) + + def complete_handshake(self, response): + length, handshake_bytes = super().complete_handshake(response) + # first 16 bytes returned by complete_handshake() is the garbage terminator + wrong_garbage_terminator = random_bitflip(handshake_bytes[:16]) + return length, wrong_garbage_terminator + handshake_bytes[16:] + + +class WrongGarbageState(EncryptedP2PState): + """Generate tampered garbage bytes""" + def generate_keypair_and_garbage(self): + garbage_len = random.randrange(1, MAX_GARBAGE_LEN) + ellswift_garbage_bytes = super().generate_keypair_and_garbage(garbage_len) + # assume that garbage bytes sent to TestNode were tampered with + return ellswift_garbage_bytes[:64] + random_bitflip(ellswift_garbage_bytes[64:]) + + +class NoAADState(EncryptedP2PState): + """Add option for not filling first encrypted packet after garbage terminator with AAD""" + def generate_keypair_and_garbage(self): + garbage_len = random.randrange(1, MAX_GARBAGE_LEN) + return super().generate_keypair_and_garbage(garbage_len) + + def complete_handshake(self, response): + self.sent_garbage = b'' # do not authenticate the garbage which is sent + return super().complete_handshake(response) + + +class NonEmptyVersionPacketState(EncryptedP2PState): + """"Add option for sending non-empty transport version packet.""" + def complete_handshake(self, response): + self.transport_version = random.randbytes(5) + return super().complete_handshake(response) + + +class MisbehavingV2Peer(P2PInterface): + """Custom implementation of P2PInterface which uses modified v2 P2P protocol functions for testing purposes.""" + def __init__(self, test_type): + super().__init__() + self.test_type = test_type + + def connection_made(self, transport): + if self.test_type == TestType.EARLY_KEY_RESPONSE: + self.v2_state = EarlyKeyResponseState(initiating=True, net='regtest') + elif self.test_type == TestType.EXCESS_GARBAGE: + self.v2_state = ExcessGarbageState(initiating=True, net='regtest') + elif self.test_type == TestType.WRONG_GARBAGE_TERMINATOR: + self.v2_state = WrongGarbageTerminatorState(initiating=True, net='regtest') + elif self.test_type == TestType.WRONG_GARBAGE: + self.v2_state = WrongGarbageState(initiating=True, net='regtest') + elif self.test_type == TestType.SEND_NO_AAD: + self.v2_state = NoAADState(initiating=True, net='regtest') + elif TestType.SEND_NON_EMPTY_VERSION_PACKET: + self.v2_state = NonEmptyVersionPacketState(initiating=True, net='regtest') + super().connection_made(transport) + + def data_received(self, t): + if self.test_type == TestType.EARLY_KEY_RESPONSE: + # check that data can be received on recvbuf only when mismatch from V1_PREFIX happens + assert self.v2_state.can_data_be_received + else: + super().data_received(t) + + +class EncryptedP2PMisbehaving(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + self.extra_args = [["-v2transport=1", "-peertimeout=3"]] + + def run_test(self): + self.test_earlykeyresponse() + self.test_v2disconnection() + + def test_earlykeyresponse(self): + self.log.info('Sending ellswift bytes in parts to ensure that response from responder is received only when') + self.log.info('ellswift bytes have a mismatch from the 16 bytes(network magic followed by "version\\x00\\x00\\x00\\x00\\x00")') + node0 = self.nodes[0] + self.log.info('Sending first 4 bytes of ellswift which match network magic') + self.log.info('If a response is received, assertion failure would happen in our custom data_received() function') + peer1 = node0.add_p2p_connection(MisbehavingV2Peer(TestType.EARLY_KEY_RESPONSE), wait_for_verack=False, send_version=False, supports_v2_p2p=True, wait_for_v2_handshake=False) + peer1.send_raw_message(MAGIC_BYTES['regtest']) + self.log.info('Sending remaining ellswift and garbage which are different from V1_PREFIX. Since a response is') + self.log.info('expected now, our custom data_received() function wouldn\'t result in assertion failure') + peer1.v2_state.can_data_be_received = True + peer1.send_raw_message(peer1.v2_state.ellswift_ours[4:] + peer1.v2_state.sent_garbage) + with node0.assert_debug_log(['V2 handshake timeout peer=0']): + peer1.wait_for_disconnect(timeout=5) + self.log.info('successful disconnection since modified ellswift was sent as response') + + def test_v2disconnection(self): + # test v2 disconnection scenarios + node0 = self.nodes[0] + expected_debug_message = [ + [], # EARLY_KEY_RESPONSE + ["V2 transport error: missing garbage terminator, peer=1"], # EXCESS_GARBAGE + ["V2 handshake timeout peer=2"], # WRONG_GARBAGE_TERMINATOR + ["V2 transport error: packet decryption failure"], # WRONG_GARBAGE + ["V2 transport error: packet decryption failure"], # SEND_NO_AAD + [], # SEND_NON_EMPTY_VERSION_PACKET + ] + for test_type in TestType: + if test_type == TestType.EARLY_KEY_RESPONSE: + continue + elif test_type == TestType.SEND_NON_EMPTY_VERSION_PACKET: + node0.add_p2p_connection(MisbehavingV2Peer(test_type), wait_for_verack=True, send_version=True, supports_v2_p2p=True) + self.log.info(f"No disconnection for {test_type.name}") + else: + with node0.assert_debug_log(expected_debug_message[test_type.value], timeout=5): + peer = node0.add_p2p_connection(MisbehavingV2Peer(test_type), wait_for_verack=False, send_version=False, supports_v2_p2p=True, expect_success=False) + peer.wait_for_disconnect() + self.log.info(f"Expected disconnection for {test_type.name}") + + +if __name__ == '__main__': + EncryptedP2PMisbehaving().main() diff --git a/test/functional/rpc_createmultisig.py b/test/functional/rpc_createmultisig.py index 65d7b4c422..37656341d2 100755 --- a/test/functional/rpc_createmultisig.py +++ b/test/functional/rpc_createmultisig.py @@ -9,10 +9,10 @@ import json import os from test_framework.address import address_to_scriptpubkey -from test_framework.blocktools import COINBASE_MATURITY -from test_framework.authproxy import JSONRPCException from test_framework.descriptors import descsum_create, drop_origins from test_framework.key import ECPubKey +from test_framework.messages import COIN +from test_framework.script_util import keys_to_multisig_script from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_raises_rpc_error, @@ -32,92 +32,54 @@ class RpcCreateMultiSigTest(BitcoinTestFramework): self.setup_clean_chain = True self.num_nodes = 3 self.supports_cli = False + self.enable_wallet_if_possible() - def get_keys(self): + def create_keys(self, num_keys): self.pub = [] self.priv = [] - node0, node1, node2 = self.nodes - for _ in range(self.nkeys): + for _ in range(num_keys): privkey, pubkey = generate_keypair(wif=True) self.pub.append(pubkey.hex()) self.priv.append(privkey) - if self.is_bdb_compiled(): - self.final = node2.getnewaddress() - else: - self.final = getnewdestination('bech32')[2] + + def create_wallet(self, node, wallet_name): + node.createwallet(wallet_name=wallet_name, disable_private_keys=True) + return node.get_wallet_rpc(wallet_name) def run_test(self): node0, node1, node2 = self.nodes self.wallet = MiniWallet(test_node=node0) - if self.is_bdb_compiled(): - self.import_deterministic_coinbase_privkeys() + if self.is_wallet_compiled(): self.check_addmultisigaddress_errors() self.log.info('Generating blocks ...') self.generate(self.wallet, 149) - self.moved = 0 - for self.nkeys in [3, 5]: - for self.nsigs in [2, 3]: - for self.output_type in ["bech32", "p2sh-segwit", "legacy"]: - self.get_keys() - self.do_multisig() - if self.is_bdb_compiled(): - self.checkbalances() - - # Test mixed compressed and uncompressed pubkeys - self.log.info('Mixed compressed and uncompressed multisigs are not allowed') - pk0, pk1, pk2 = [getnewdestination('bech32')[0].hex() for _ in range(3)] - - # decompress pk2 - pk_obj = ECPubKey() - pk_obj.set(bytes.fromhex(pk2)) - pk_obj.compressed = False - pk2 = pk_obj.get_bytes().hex() - - if self.is_bdb_compiled(): - node0.createwallet(wallet_name='wmulti0', disable_private_keys=True) - wmulti0 = node0.get_wallet_rpc('wmulti0') - - # Check all permutations of keys because order matters apparently - for keys in itertools.permutations([pk0, pk1, pk2]): - # Results should be the same as this legacy one - legacy_addr = node0.createmultisig(2, keys, 'legacy')['address'] - - if self.is_bdb_compiled(): - result = wmulti0.addmultisigaddress(2, keys, '', 'legacy') - assert_equal(legacy_addr, result['address']) - assert 'warnings' not in result - - # Generate addresses with the segwit types. These should all make legacy addresses - err_msg = ["Unable to make chosen address type, please ensure no uncompressed public keys are present."] - - for addr_type in ['bech32', 'p2sh-segwit']: - result = self.nodes[0].createmultisig(nrequired=2, keys=keys, address_type=addr_type) - assert_equal(legacy_addr, result['address']) - assert_equal(result['warnings'], err_msg) - - if self.is_bdb_compiled(): - result = wmulti0.addmultisigaddress(nrequired=2, keys=keys, address_type=addr_type) - assert_equal(legacy_addr, result['address']) - assert_equal(result['warnings'], err_msg) - - self.log.info('Testing sortedmulti descriptors with BIP 67 test vectors') - with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'data/rpc_bip67.json'), encoding='utf-8') as f: - vectors = json.load(f) + wallet_multi = self.create_wallet(node1, 'wmulti') if self._requires_wallet else None + self.create_keys(21) # max number of allowed keys + 1 + m_of_n = [(2, 3), (3, 3), (2, 5), (3, 5), (10, 15), (15, 15)] + for (sigs, keys) in m_of_n: + for output_type in ["bech32", "p2sh-segwit", "legacy"]: + self.do_multisig(keys, sigs, output_type, wallet_multi) - for t in vectors: - key_str = ','.join(t['keys']) - desc = descsum_create('sh(sortedmulti(2,{}))'.format(key_str)) - assert_equal(self.nodes[0].deriveaddresses(desc)[0], t['address']) - sorted_key_str = ','.join(t['sorted_keys']) - sorted_key_desc = descsum_create('sh(multi(2,{}))'.format(sorted_key_str)) - assert_equal(self.nodes[0].deriveaddresses(sorted_key_desc)[0], t['address']) + self.test_multisig_script_limit(wallet_multi) + self.test_mixing_uncompressed_and_compressed_keys(node0, wallet_multi) + self.test_sortedmulti_descriptors_bip67() # Check that bech32m is currently not allowed assert_raises_rpc_error(-5, "createmultisig cannot create bech32m multisig addresses", self.nodes[0].createmultisig, 2, self.pub, "bech32m") + self.log.info('Check correct encoding of multisig script for all n (1..20)') + for nkeys in range(1, 20+1): + keys = [self.pub[0]]*nkeys + expected_ms_script = keys_to_multisig_script(keys, k=nkeys) # simply use n-of-n + # note that the 'legacy' address type fails for n values larger than 15 + # due to exceeding the P2SH size limit (520 bytes), so we use 'bech32' instead + # (for the purpose of this encoding test, we don't care about the resulting address) + res = self.nodes[0].createmultisig(nrequired=nkeys, keys=keys, address_type='bech32') + assert_equal(res['redeemScript'], expected_ms_script.hex()) + def check_addmultisigaddress_errors(self): if self.options.descriptors: return @@ -133,117 +95,165 @@ class RpcCreateMultiSigTest(BitcoinTestFramework): pubs = [self.nodes[1].getaddressinfo(addr)["pubkey"] for addr in addresses] assert_raises_rpc_error(-5, "Bech32m multisig addresses cannot be created with legacy wallets", self.nodes[0].addmultisigaddress, 2, pubs, "", "bech32m") - def checkbalances(self): - node0, node1, node2 = self.nodes - self.generate(node0, COINBASE_MATURITY) + def test_multisig_script_limit(self, wallet_multi): + node1 = self.nodes[1] + pubkeys = self.pub[0:20] - bal0 = node0.getbalance() - bal1 = node1.getbalance() - bal2 = node2.getbalance() - balw = self.wallet.get_balance() + self.log.info('Test legacy redeem script max size limit') + assert_raises_rpc_error(-8, "redeemScript exceeds size limit: 684 > 520", node1.createmultisig, 16, pubkeys, 'legacy') - height = node0.getblockchaininfo()["blocks"] - assert 150 < height < 350 - total = 149 * 50 + (height - 149 - 100) * 25 - assert bal1 == 0 - assert bal2 == self.moved - assert_equal(bal0 + bal1 + bal2 + balw, total) + self.log.info('Test valid 16-20 multisig p2sh-legacy and bech32 (no wallet)') + self.do_multisig(nkeys=20, nsigs=16, output_type="p2sh-segwit", wallet_multi=None) + self.do_multisig(nkeys=20, nsigs=16, output_type="bech32", wallet_multi=None) - def do_multisig(self): - node0, node1, node2 = self.nodes + self.log.info('Test invalid 16-21 multisig p2sh-legacy and bech32 (no wallet)') + assert_raises_rpc_error(-8, "Number of keys involved in the multisignature address creation > 20", node1.createmultisig, 16, self.pub, 'p2sh-segwit') + assert_raises_rpc_error(-8, "Number of keys involved in the multisignature address creation > 20", node1.createmultisig, 16, self.pub, 'bech32') - if self.is_bdb_compiled(): - if 'wmulti' not in node1.listwallets(): - try: - node1.loadwallet('wmulti') - except JSONRPCException as e: - path = self.nodes[1].wallets_path / "wmulti" - if e.error['code'] == -18 and "Wallet file verification failed. Failed to load database path '{}'. Path does not exist.".format(path) in e.error['message']: - node1.createwallet(wallet_name='wmulti', disable_private_keys=True) - else: - raise - wmulti = node1.get_wallet_rpc('wmulti') + # Check legacy wallet related command + self.log.info('Test legacy redeem script max size limit (with wallet)') + if wallet_multi is not None and not self.options.descriptors: + assert_raises_rpc_error(-8, "redeemScript exceeds size limit: 684 > 520", wallet_multi.addmultisigaddress, 16, pubkeys, '', 'legacy') + + self.log.info('Test legacy wallet unsupported operation. 16-20 multisig p2sh-legacy and bech32 generation') + # Due an internal limitation on legacy wallets, the redeem script limit also applies to p2sh-segwit and bech32 (even when the scripts are valid) + # We take this as a "good thing" to tell users to upgrade to descriptors. + assert_raises_rpc_error(-4, "Unsupported multisig script size for legacy wallet. Upgrade to descriptors to overcome this limitation for p2sh-segwit or bech32 scripts", wallet_multi.addmultisigaddress, 16, pubkeys, '', 'p2sh-segwit') + assert_raises_rpc_error(-4, "Unsupported multisig script size for legacy wallet. Upgrade to descriptors to overcome this limitation for p2sh-segwit or bech32 scripts", wallet_multi.addmultisigaddress, 16, pubkeys, '', 'bech32') + + def do_multisig(self, nkeys, nsigs, output_type, wallet_multi): + node0, node1, node2 = self.nodes + pub_keys = self.pub[0: nkeys] + priv_keys = self.priv[0: nkeys] # Construct the expected descriptor - desc = 'multi({},{})'.format(self.nsigs, ','.join(self.pub)) - if self.output_type == 'legacy': + desc = 'multi({},{})'.format(nsigs, ','.join(pub_keys)) + if output_type == 'legacy': desc = 'sh({})'.format(desc) - elif self.output_type == 'p2sh-segwit': + elif output_type == 'p2sh-segwit': desc = 'sh(wsh({}))'.format(desc) - elif self.output_type == 'bech32': + elif output_type == 'bech32': desc = 'wsh({})'.format(desc) desc = descsum_create(desc) - msig = node2.createmultisig(self.nsigs, self.pub, self.output_type) + msig = node2.createmultisig(nsigs, pub_keys, output_type) assert 'warnings' not in msig madd = msig["address"] mredeem = msig["redeemScript"] assert_equal(desc, msig['descriptor']) - if self.output_type == 'bech32': + if output_type == 'bech32': assert madd[0:4] == "bcrt" # actually a bech32 address - if self.is_bdb_compiled(): + if wallet_multi is not None: # compare against addmultisigaddress - msigw = wmulti.addmultisigaddress(self.nsigs, self.pub, None, self.output_type) + msigw = wallet_multi.addmultisigaddress(nsigs, pub_keys, None, output_type) maddw = msigw["address"] mredeemw = msigw["redeemScript"] assert_equal(desc, drop_origins(msigw['descriptor'])) # addmultisigiaddress and createmultisig work the same assert maddw == madd assert mredeemw == mredeem - wmulti.unloadwallet() spk = address_to_scriptpubkey(madd) - txid = self.wallet.send_to(from_node=self.nodes[0], scriptPubKey=spk, amount=1300)["txid"] - tx = node0.getrawtransaction(txid, True) - vout = [v["n"] for v in tx["vout"] if madd == v["scriptPubKey"]["address"]] - assert len(vout) == 1 - vout = vout[0] - scriptPubKey = tx["vout"][vout]["scriptPubKey"]["hex"] - value = tx["vout"][vout]["value"] - prevtxs = [{"txid": txid, "vout": vout, "scriptPubKey": scriptPubKey, "redeemScript": mredeem, "amount": value}] + value = decimal.Decimal("0.00004000") + tx = self.wallet.send_to(from_node=self.nodes[0], scriptPubKey=spk, amount=int(value * COIN)) + prevtxs = [{"txid": tx["txid"], "vout": tx["sent_vout"], "scriptPubKey": spk.hex(), "redeemScript": mredeem, "amount": value}] self.generate(node0, 1) - outval = value - decimal.Decimal("0.00001000") - rawtx = node2.createrawtransaction([{"txid": txid, "vout": vout}], [{self.final: outval}]) + outval = value - decimal.Decimal("0.00002000") # deduce fee (must be higher than the min relay fee) + # send coins to node2 when wallet is enabled + node2_balance = node2.getbalances()['mine']['trusted'] if self.is_wallet_compiled() else 0 + out_addr = node2.getnewaddress() if self.is_wallet_compiled() else getnewdestination('bech32')[2] + rawtx = node2.createrawtransaction([{"txid": tx["txid"], "vout": tx["sent_vout"]}], [{out_addr: outval}]) prevtx_err = dict(prevtxs[0]) del prevtx_err["redeemScript"] - assert_raises_rpc_error(-8, "Missing redeemScript/witnessScript", node2.signrawtransactionwithkey, rawtx, self.priv[0:self.nsigs-1], [prevtx_err]) + assert_raises_rpc_error(-8, "Missing redeemScript/witnessScript", node2.signrawtransactionwithkey, rawtx, priv_keys[0:nsigs-1], [prevtx_err]) # if witnessScript specified, all ok prevtx_err["witnessScript"] = prevtxs[0]["redeemScript"] - node2.signrawtransactionwithkey(rawtx, self.priv[0:self.nsigs-1], [prevtx_err]) + node2.signrawtransactionwithkey(rawtx, priv_keys[0:nsigs-1], [prevtx_err]) # both specified, also ok prevtx_err["redeemScript"] = prevtxs[0]["redeemScript"] - node2.signrawtransactionwithkey(rawtx, self.priv[0:self.nsigs-1], [prevtx_err]) + node2.signrawtransactionwithkey(rawtx, priv_keys[0:nsigs-1], [prevtx_err]) # redeemScript mismatch to witnessScript prevtx_err["redeemScript"] = "6a" # OP_RETURN - assert_raises_rpc_error(-8, "redeemScript does not correspond to witnessScript", node2.signrawtransactionwithkey, rawtx, self.priv[0:self.nsigs-1], [prevtx_err]) + assert_raises_rpc_error(-8, "redeemScript does not correspond to witnessScript", node2.signrawtransactionwithkey, rawtx, priv_keys[0:nsigs-1], [prevtx_err]) # redeemScript does not match scriptPubKey del prevtx_err["witnessScript"] - assert_raises_rpc_error(-8, "redeemScript/witnessScript does not match scriptPubKey", node2.signrawtransactionwithkey, rawtx, self.priv[0:self.nsigs-1], [prevtx_err]) + assert_raises_rpc_error(-8, "redeemScript/witnessScript does not match scriptPubKey", node2.signrawtransactionwithkey, rawtx, priv_keys[0:nsigs-1], [prevtx_err]) # witnessScript does not match scriptPubKey prevtx_err["witnessScript"] = prevtx_err["redeemScript"] del prevtx_err["redeemScript"] - assert_raises_rpc_error(-8, "redeemScript/witnessScript does not match scriptPubKey", node2.signrawtransactionwithkey, rawtx, self.priv[0:self.nsigs-1], [prevtx_err]) + assert_raises_rpc_error(-8, "redeemScript/witnessScript does not match scriptPubKey", node2.signrawtransactionwithkey, rawtx, priv_keys[0:nsigs-1], [prevtx_err]) - rawtx2 = node2.signrawtransactionwithkey(rawtx, self.priv[0:self.nsigs - 1], prevtxs) - rawtx3 = node2.signrawtransactionwithkey(rawtx2["hex"], [self.priv[-1]], prevtxs) + rawtx2 = node2.signrawtransactionwithkey(rawtx, priv_keys[0:nsigs - 1], prevtxs) + rawtx3 = node2.signrawtransactionwithkey(rawtx2["hex"], [priv_keys[-1]], prevtxs) + assert rawtx3['complete'] - self.moved += outval tx = node0.sendrawtransaction(rawtx3["hex"], 0) blk = self.generate(node0, 1)[0] assert tx in node0.getblock(blk)["tx"] + # When the wallet is enabled, assert node2 sees the incoming amount + if self.is_wallet_compiled(): + assert_equal(node2.getbalances()['mine']['trusted'], node2_balance + outval) + txinfo = node0.getrawtransaction(tx, True, blk) - self.log.info("n/m=%d/%d %s size=%d vsize=%d weight=%d" % (self.nsigs, self.nkeys, self.output_type, txinfo["size"], txinfo["vsize"], txinfo["weight"])) + self.log.info("n/m=%d/%d %s size=%d vsize=%d weight=%d" % (nsigs, nkeys, output_type, txinfo["size"], txinfo["vsize"], txinfo["weight"])) + + def test_mixing_uncompressed_and_compressed_keys(self, node, wallet_multi): + self.log.info('Mixed compressed and uncompressed multisigs are not allowed') + pk0, pk1, pk2 = [getnewdestination('bech32')[0].hex() for _ in range(3)] + + # decompress pk2 + pk_obj = ECPubKey() + pk_obj.set(bytes.fromhex(pk2)) + pk_obj.compressed = False + pk2 = pk_obj.get_bytes().hex() + + # Check all permutations of keys because order matters apparently + for keys in itertools.permutations([pk0, pk1, pk2]): + # Results should be the same as this legacy one + legacy_addr = node.createmultisig(2, keys, 'legacy')['address'] + + if wallet_multi is not None: + # 'addmultisigaddress' should return the same address + result = wallet_multi.addmultisigaddress(2, keys, '', 'legacy') + assert_equal(legacy_addr, result['address']) + assert 'warnings' not in result + + # Generate addresses with the segwit types. These should all make legacy addresses + err_msg = ["Unable to make chosen address type, please ensure no uncompressed public keys are present."] + + for addr_type in ['bech32', 'p2sh-segwit']: + result = self.nodes[0].createmultisig(nrequired=2, keys=keys, address_type=addr_type) + assert_equal(legacy_addr, result['address']) + assert_equal(result['warnings'], err_msg) + + if wallet_multi is not None: + result = wallet_multi.addmultisigaddress(nrequired=2, keys=keys, address_type=addr_type) + assert_equal(legacy_addr, result['address']) + assert_equal(result['warnings'], err_msg) + + def test_sortedmulti_descriptors_bip67(self): + self.log.info('Testing sortedmulti descriptors with BIP 67 test vectors') + with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'data/rpc_bip67.json'), encoding='utf-8') as f: + vectors = json.load(f) + + for t in vectors: + key_str = ','.join(t['keys']) + desc = descsum_create('sh(sortedmulti(2,{}))'.format(key_str)) + assert_equal(self.nodes[0].deriveaddresses(desc)[0], t['address']) + sorted_key_str = ','.join(t['sorted_keys']) + sorted_key_desc = descsum_create('sh(multi(2,{}))'.format(sorted_key_str)) + assert_equal(self.nodes[0].deriveaddresses(sorted_key_desc)[0], t['address']) if __name__ == '__main__': diff --git a/test/functional/rpc_dumptxoutset.py b/test/functional/rpc_dumptxoutset.py index 1ea6cf52d1..c92c8da357 100755 --- a/test/functional/rpc_dumptxoutset.py +++ b/test/functional/rpc_dumptxoutset.py @@ -43,7 +43,7 @@ class DumptxoutsetTest(BitcoinTestFramework): # UTXO snapshot hash should be deterministic based on mocked time. assert_equal( sha256sum_file(str(expected_path)).hex(), - 'b1bacb602eacf5fbc9a7c2ef6eeb0d229c04e98bdf0c2ea5929012cd0eae3830') + '2f775f82811150d310527b5ff773f81fb0fb517e941c543c1f7c4d38fd2717b3') assert_equal( out['txoutset_hash'], 'a0b7baa3bf5ccbd3279728f230d7ca0c44a76e9923fca8f32dbfd08d65ea496a') diff --git a/test/functional/rpc_generate.py b/test/functional/rpc_generate.py index 20f62079fd..3e250925e7 100755 --- a/test/functional/rpc_generate.py +++ b/test/functional/rpc_generate.py @@ -87,7 +87,7 @@ class RPCGenerateTest(BitcoinTestFramework): txid1 = miniwallet.send_self_transfer(from_node=node)['txid'] utxo1 = miniwallet.get_utxo(txid=txid1) rawtx2 = miniwallet.create_self_transfer(utxo_to_spend=utxo1)['hex'] - assert_raises_rpc_error(-25, 'TestBlockValidity failed: bad-txns-inputs-missingorspent', self.generateblock, node, address, [rawtx2, txid1]) + assert_raises_rpc_error(-25, 'testBlockValidity failed: bad-txns-inputs-missingorspent', self.generateblock, node, address, [rawtx2, txid1]) self.log.info('Fail to generate block with txid not in mempool') missing_txid = '0000000000000000000000000000000000000000000000000000000000000000' diff --git a/test/functional/rpc_net.py b/test/functional/rpc_net.py index 2701d2471d..37e2c1fb71 100755 --- a/test/functional/rpc_net.py +++ b/test/functional/rpc_net.py @@ -237,28 +237,35 @@ class NetTest(BitcoinTestFramework): def test_addnode_getaddednodeinfo(self): self.log.info("Test addnode and getaddednodeinfo") assert_equal(self.nodes[0].getaddednodeinfo(), []) - # add a node (node2) to node0 + self.log.info("Add a node (node2) to node0") ip_port = "127.0.0.1:{}".format(p2p_port(2)) self.nodes[0].addnode(node=ip_port, command='add') - # try to add an equivalent ip - # (note that OpenBSD doesn't support the IPv4 shorthand notation with omitted zero-bytes) + self.log.info("Try to add an equivalent ip and check it fails") + self.log.debug("(note that OpenBSD doesn't support the IPv4 shorthand notation with omitted zero-bytes)") if platform.system() != "OpenBSD": ip_port2 = "127.1:{}".format(p2p_port(2)) assert_raises_rpc_error(-23, "Node already added", self.nodes[0].addnode, node=ip_port2, command='add') - # check that the node has indeed been added + self.log.info("Check that the node has indeed been added") added_nodes = self.nodes[0].getaddednodeinfo() assert_equal(len(added_nodes), 1) assert_equal(added_nodes[0]['addednode'], ip_port) - # check that node cannot be added again + self.log.info("Check that filtering by node works") + self.nodes[0].addnode(node="11.22.33.44", command='add') + first_added_node = self.nodes[0].getaddednodeinfo(node=ip_port) + assert_equal(added_nodes, first_added_node) + assert_equal(len(self.nodes[0].getaddednodeinfo()), 2) + self.log.info("Check that node cannot be added again") assert_raises_rpc_error(-23, "Node already added", self.nodes[0].addnode, node=ip_port, command='add') - # check that node can be removed + self.log.info("Check that node can be removed") self.nodes[0].addnode(node=ip_port, command='remove') - assert_equal(self.nodes[0].getaddednodeinfo(), []) - # check that an invalid command returns an error + added_nodes = self.nodes[0].getaddednodeinfo() + assert_equal(len(added_nodes), 1) + assert_equal(added_nodes[0]['addednode'], "11.22.33.44") + self.log.info("Check that an invalid command returns an error") assert_raises_rpc_error(-1, 'addnode "node" "command"', self.nodes[0].addnode, node=ip_port, command='abc') - # check that trying to remove the node again returns an error + self.log.info("Check that trying to remove the node again returns an error") assert_raises_rpc_error(-24, "Node could not be removed", self.nodes[0].addnode, node=ip_port, command='remove') - # check that a non-existent node returns an error + self.log.info("Check that a non-existent node returns an error") assert_raises_rpc_error(-24, "Node has not been added", self.nodes[0].getaddednodeinfo, '1.1.1.1') def test_service_flags(self): diff --git a/test/functional/rpc_packages.py b/test/functional/rpc_packages.py index 37c42f2533..1acd586d2c 100755 --- a/test/functional/rpc_packages.py +++ b/test/functional/rpc_packages.py @@ -8,6 +8,9 @@ from decimal import Decimal import random from test_framework.blocktools import COINBASE_MATURITY +from test_framework.mempool_util import ( + fill_mempool, +) from test_framework.messages import ( MAX_BIP125_RBF_SEQUENCE, tx_from_hex, @@ -18,14 +21,17 @@ from test_framework.util import ( assert_equal, assert_fee_amount, assert_raises_rpc_error, - fill_mempool, ) from test_framework.wallet import ( + COIN, DEFAULT_FEE, MiniWallet, ) +MAX_PACKAGE_COUNT = 25 + + class RPCPackagesTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 @@ -237,6 +243,37 @@ class RPCPackagesTest(BitcoinTestFramework): {"txid": tx2["txid"], "wtxid": tx2["wtxid"], "package-error": "conflict-in-package"} ]) + # Add a child that spends both at high feerate to submit via submitpackage + tx_child = self.wallet.create_self_transfer_multi( + fee_per_output=int(DEFAULT_FEE * 5 * COIN), + utxos_to_spend=[tx1["new_utxo"], tx2["new_utxo"]], + ) + + testres = node.testmempoolaccept([tx1["hex"], tx2["hex"], tx_child["hex"]]) + + assert_equal(testres, [ + {"txid": tx1["txid"], "wtxid": tx1["wtxid"], "package-error": "conflict-in-package"}, + {"txid": tx2["txid"], "wtxid": tx2["wtxid"], "package-error": "conflict-in-package"}, + {"txid": tx_child["txid"], "wtxid": tx_child["wtxid"], "package-error": "conflict-in-package"} + ]) + + submitres = node.submitpackage([tx1["hex"], tx2["hex"], tx_child["hex"]]) + assert_equal(submitres, {'package_msg': 'conflict-in-package', 'tx-results': {}, 'replaced-transactions': []}) + + # Submit tx1 to mempool, then try the same package again + node.sendrawtransaction(tx1["hex"]) + + submitres = node.submitpackage([tx1["hex"], tx2["hex"], tx_child["hex"]]) + assert_equal(submitres, {'package_msg': 'conflict-in-package', 'tx-results': {}, 'replaced-transactions': []}) + assert tx_child["txid"] not in node.getrawmempool() + + # ... and without the in-mempool ancestor tx1 included in the call + submitres = node.submitpackage([tx2["hex"], tx_child["hex"]]) + assert_equal(submitres, {'package_msg': 'package-not-child-with-unconfirmed-parents', 'tx-results': {}, 'replaced-transactions': []}) + + # Regardless of error type, the child can never enter the mempool + assert tx_child["txid"] not in node.getrawmempool() + def test_rbf(self): node = self.nodes[0] @@ -344,13 +381,20 @@ class RPCPackagesTest(BitcoinTestFramework): assert_raises_rpc_error(-25, "package topology disallowed", node.submitpackage, chain_hex) assert_equal(legacy_pool, node.getrawmempool()) + assert_raises_rpc_error(-8, f"Array must contain between 2 and {MAX_PACKAGE_COUNT} transactions.", node.submitpackage, []) + assert_raises_rpc_error(-8, f"Array must contain between 2 and {MAX_PACKAGE_COUNT} transactions.", node.submitpackage, [chain_hex[0]] * 1) + assert_raises_rpc_error( + -8, f"Array must contain between 2 and {MAX_PACKAGE_COUNT} transactions.", + node.submitpackage, [chain_hex[0]] * (MAX_PACKAGE_COUNT + 1) + ) + # Create a transaction chain such as only the parent gets accepted (by making the child's # version non-standard). Make sure the parent does get broadcast. self.log.info("If a package is partially submitted, transactions included in mempool get broadcast") peer = node.add_p2p_connection(P2PTxInvStore()) txs = self.wallet.create_self_transfer_chain(chain_length=2) bad_child = tx_from_hex(txs[1]["hex"]) - bad_child.nVersion = -1 + bad_child.version = 0xffffffff hex_partial_acceptance = [txs[0]["hex"], bad_child.serialize().hex()] res = node.submitpackage(hex_partial_acceptance) assert_equal(res["package_msg"], "transaction failed") @@ -388,7 +432,7 @@ class RPCPackagesTest(BitcoinTestFramework): ]) self.wallet.rescan_utxos() - fill_mempool(self, node, self.wallet) + fill_mempool(self, node) minrelay = node.getmempoolinfo()["minrelaytxfee"] parent = self.wallet.create_self_transfer( diff --git a/test/functional/rpc_psbt.py b/test/functional/rpc_psbt.py index 6ee7e56886..a56960adff 100755 --- a/test/functional/rpc_psbt.py +++ b/test/functional/rpc_psbt.py @@ -700,11 +700,9 @@ class PSBTTest(BitcoinTestFramework): assert_equal(analysis['next'], 'creator') assert_equal(analysis['error'], 'PSBT is not valid. Output amount invalid') - analysis = self.nodes[0].analyzepsbt('cHNidP8BAJoCAAAAAkvEW8NnDtdNtDpsmze+Ht2LH35IJcKv00jKAlUs21RrAwAAAAD/////S8Rbw2cO1020OmybN74e3Ysffkglwq/TSMoCVSzbVGsBAAAAAP7///8CwLYClQAAAAAWABSNJKzjaUb3uOxixsvh1GGE3fW7zQD5ApUAAAAAFgAUKNw0x8HRctAgmvoevm4u1SbN7XIAAAAAAAEAnQIAAAACczMa321tVHuN4GKWKRncycI22aX3uXgwSFUKM2orjRsBAAAAAP7///9zMxrfbW1Ue43gYpYpGdzJwjbZpfe5eDBIVQozaiuNGwAAAAAA/v///wIA+QKVAAAAABl2qRT9zXUVA8Ls5iVqynLHe5/vSe1XyYisQM0ClQAAAAAWABRmWQUcjSjghQ8/uH4Bn/zkakwLtAAAAAAAAQEfQM0ClQAAAAAWABRmWQUcjSjghQ8/uH4Bn/zkakwLtAAAAA==') - assert_equal(analysis['next'], 'creator') - assert_equal(analysis['error'], 'PSBT is not valid. Input 0 specifies invalid prevout') + assert_raises_rpc_error(-22, "TX decode failed", self.nodes[0].analyzepsbt, "cHNidP8BAJoCAAAAAkvEW8NnDtdNtDpsmze+Ht2LH35IJcKv00jKAlUs21RrAwAAAAD/////S8Rbw2cO1020OmybN74e3Ysffkglwq/TSMoCVSzbVGsBAAAAAP7///8CwLYClQAAAAAWABSNJKzjaUb3uOxixsvh1GGE3fW7zQD5ApUAAAAAFgAUKNw0x8HRctAgmvoevm4u1SbN7XIAAAAAAAEAnQIAAAACczMa321tVHuN4GKWKRncycI22aX3uXgwSFUKM2orjRsBAAAAAP7///9zMxrfbW1Ue43gYpYpGdzJwjbZpfe5eDBIVQozaiuNGwAAAAAA/v///wIA+QKVAAAAABl2qRT9zXUVA8Ls5iVqynLHe5/vSe1XyYisQM0ClQAAAAAWABRmWQUcjSjghQ8/uH4Bn/zkakwLtAAAAAAAAQEfQM0ClQAAAAAWABRmWQUcjSjghQ8/uH4Bn/zkakwLtAAAAA==") - assert_raises_rpc_error(-25, 'Inputs missing or spent', self.nodes[0].walletprocesspsbt, 'cHNidP8BAJoCAAAAAkvEW8NnDtdNtDpsmze+Ht2LH35IJcKv00jKAlUs21RrAwAAAAD/////S8Rbw2cO1020OmybN74e3Ysffkglwq/TSMoCVSzbVGsBAAAAAP7///8CwLYClQAAAAAWABSNJKzjaUb3uOxixsvh1GGE3fW7zQD5ApUAAAAAFgAUKNw0x8HRctAgmvoevm4u1SbN7XIAAAAAAAEAnQIAAAACczMa321tVHuN4GKWKRncycI22aX3uXgwSFUKM2orjRsBAAAAAP7///9zMxrfbW1Ue43gYpYpGdzJwjbZpfe5eDBIVQozaiuNGwAAAAAA/v///wIA+QKVAAAAABl2qRT9zXUVA8Ls5iVqynLHe5/vSe1XyYisQM0ClQAAAAAWABRmWQUcjSjghQ8/uH4Bn/zkakwLtAAAAAAAAQEfQM0ClQAAAAAWABRmWQUcjSjghQ8/uH4Bn/zkakwLtAAAAA==') + assert_raises_rpc_error(-22, "TX decode failed", self.nodes[0].walletprocesspsbt, "cHNidP8BAJoCAAAAAkvEW8NnDtdNtDpsmze+Ht2LH35IJcKv00jKAlUs21RrAwAAAAD/////S8Rbw2cO1020OmybN74e3Ysffkglwq/TSMoCVSzbVGsBAAAAAP7///8CwLYClQAAAAAWABSNJKzjaUb3uOxixsvh1GGE3fW7zQD5ApUAAAAAFgAUKNw0x8HRctAgmvoevm4u1SbN7XIAAAAAAAEAnQIAAAACczMa321tVHuN4GKWKRncycI22aX3uXgwSFUKM2orjRsBAAAAAP7///9zMxrfbW1Ue43gYpYpGdzJwjbZpfe5eDBIVQozaiuNGwAAAAAA/v///wIA+QKVAAAAABl2qRT9zXUVA8Ls5iVqynLHe5/vSe1XyYisQM0ClQAAAAAWABRmWQUcjSjghQ8/uH4Bn/zkakwLtAAAAAAAAQEfQM0ClQAAAAAWABRmWQUcjSjghQ8/uH4Bn/zkakwLtAAAAA==") self.log.info("Test that we can fund psbts with external inputs specified") diff --git a/test/functional/rpc_rawtransaction.py b/test/functional/rpc_rawtransaction.py index 12697e9d0c..f974a05f7b 100755 --- a/test/functional/rpc_rawtransaction.py +++ b/test/functional/rpc_rawtransaction.py @@ -463,20 +463,34 @@ class RawTransactionsTest(BitcoinTestFramework): self.log.info("Test transaction version numbers") # Test the minimum transaction version number that fits in a signed 32-bit integer. - # As transaction version is unsigned, this should convert to its unsigned equivalent. + # As transaction version is serialized unsigned, this should convert to its unsigned equivalent. tx = CTransaction() - tx.nVersion = -0x80000000 + tx.version = 0x80000000 rawtx = tx.serialize().hex() decrawtx = self.nodes[0].decoderawtransaction(rawtx) assert_equal(decrawtx['version'], 0x80000000) # Test the maximum transaction version number that fits in a signed 32-bit integer. tx = CTransaction() - tx.nVersion = 0x7fffffff + tx.version = 0x7fffffff rawtx = tx.serialize().hex() decrawtx = self.nodes[0].decoderawtransaction(rawtx) assert_equal(decrawtx['version'], 0x7fffffff) + # Test the minimum transaction version number that fits in an unsigned 32-bit integer. + tx = CTransaction() + tx.version = 0 + rawtx = tx.serialize().hex() + decrawtx = self.nodes[0].decoderawtransaction(rawtx) + assert_equal(decrawtx['version'], 0) + + # Test the maximum transaction version number that fits in an unsigned 32-bit integer. + tx = CTransaction() + tx.version = 0xffffffff + rawtx = tx.serialize().hex() + decrawtx = self.nodes[0].decoderawtransaction(rawtx) + assert_equal(decrawtx['version'], 0xffffffff) + def raw_multisig_transaction_legacy_tests(self): self.log.info("Test raw multisig transactions (legacy)") # The traditional multisig workflow does not work with descriptor wallets so these are legacy only. @@ -490,11 +504,11 @@ class RawTransactionsTest(BitcoinTestFramework): addr2Obj = self.nodes[2].getaddressinfo(addr2) # Tests for createmultisig and addmultisigaddress - assert_raises_rpc_error(-5, "Invalid public key", self.nodes[0].createmultisig, 1, ["01020304"]) + assert_raises_rpc_error(-5, 'Pubkey "01020304" must have a length of either 33 or 65 bytes', self.nodes[0].createmultisig, 1, ["01020304"]) # createmultisig can only take public keys self.nodes[0].createmultisig(2, [addr1Obj['pubkey'], addr2Obj['pubkey']]) # addmultisigaddress can take both pubkeys and addresses so long as they are in the wallet, which is tested here - assert_raises_rpc_error(-5, "Invalid public key", self.nodes[0].createmultisig, 2, [addr1Obj['pubkey'], addr1]) + assert_raises_rpc_error(-5, f'Pubkey "{addr1}" must be a hex string', self.nodes[0].createmultisig, 2, [addr1Obj['pubkey'], addr1]) mSigObj = self.nodes[2].addmultisigaddress(2, [addr1Obj['pubkey'], addr1])['address'] @@ -585,6 +599,8 @@ class RawTransactionsTest(BitcoinTestFramework): rawTxPartialSigned2 = self.nodes[2].signrawtransactionwithwallet(rawTx2, inputs) self.log.debug(rawTxPartialSigned2) assert_equal(rawTxPartialSigned2['complete'], False) # node2 only has one key, can't comp. sign the tx + assert_raises_rpc_error(-22, "TX decode failed", self.nodes[0].combinerawtransaction, [rawTxPartialSigned1['hex'], rawTxPartialSigned2['hex'] + "00"]) + assert_raises_rpc_error(-22, "Missing transactions", self.nodes[0].combinerawtransaction, []) rawTxComb = self.nodes[2].combinerawtransaction([rawTxPartialSigned1['hex'], rawTxPartialSigned2['hex']]) self.log.debug(rawTxComb) self.nodes[2].sendrawtransaction(rawTxComb) @@ -592,6 +608,7 @@ class RawTransactionsTest(BitcoinTestFramework): self.sync_all() self.generate(self.nodes[0], 1) assert_equal(self.nodes[0].getbalance(), bal + Decimal('50.00000000') + Decimal('2.19000000')) # block reward + tx + assert_raises_rpc_error(-25, "Input not found or already spent", self.nodes[0].combinerawtransaction, [rawTxPartialSigned1['hex'], rawTxPartialSigned2['hex']]) if __name__ == '__main__': diff --git a/test/functional/rpc_signrawtransactionwithkey.py b/test/functional/rpc_signrawtransactionwithkey.py index 268584331e..f4fec13495 100755 --- a/test/functional/rpc_signrawtransactionwithkey.py +++ b/test/functional/rpc_signrawtransactionwithkey.py @@ -126,10 +126,20 @@ class SignRawTransactionWithKeyTest(BitcoinTestFramework): privkeys = [self.nodes[0].get_deterministic_priv_key().key] assert_raises_rpc_error(-8, "'all' is not a valid sighash parameter.", self.nodes[0].signrawtransactionwithkey, tx, privkeys, sighashtype="all") + def invalid_private_key_and_tx(self): + self.log.info("Test signing transaction with an invalid private key") + tx = self.nodes[0].createrawtransaction(INPUTS, OUTPUTS) + privkeys = ["123"] + assert_raises_rpc_error(-5, "Invalid private key", self.nodes[0].signrawtransactionwithkey, tx, privkeys) + self.log.info("Test signing transaction with an invalid tx hex") + privkeys = [self.nodes[0].get_deterministic_priv_key().key] + assert_raises_rpc_error(-22, "TX decode failed. Make sure the tx has at least one input.", self.nodes[0].signrawtransactionwithkey, tx + "00", privkeys) + def run_test(self): self.successful_signing_test() self.witness_script_test() self.invalid_sighashtype_test() + self.invalid_private_key_and_tx() if __name__ == '__main__': diff --git a/test/functional/rpc_users.py b/test/functional/rpc_users.py index 66cdd7cf9a..153493fbab 100755 --- a/test/functional/rpc_users.py +++ b/test/functional/rpc_users.py @@ -11,12 +11,15 @@ from test_framework.util import ( ) import http.client +import os +import platform import urllib.parse import subprocess from random import SystemRandom import string import configparser import sys +from typing import Optional def call_with_auth(node, user, password): @@ -84,6 +87,40 @@ class HTTPBasicsTest(BitcoinTestFramework): self.log.info('Wrong...') assert_equal(401, call_with_auth(node, user + 'wrong', password + 'wrong').status) + def test_rpccookieperms(self): + p = {"owner": 0o600, "group": 0o640, "all": 0o644} + + if platform.system() == 'Windows': + self.log.info(f"Skip cookie file permissions checks as OS detected as: {platform.system()=}") + return + + self.log.info('Check cookie file permissions can be set using -rpccookieperms') + + cookie_file_path = self.nodes[1].chain_path / '.cookie' + PERM_BITS_UMASK = 0o777 + + def test_perm(perm: Optional[str]): + if not perm: + perm = 'owner' + self.restart_node(1) + else: + self.restart_node(1, extra_args=[f"-rpccookieperms={perm}"]) + + file_stat = os.stat(cookie_file_path) + actual_perms = file_stat.st_mode & PERM_BITS_UMASK + expected_perms = p[perm] + assert_equal(expected_perms, actual_perms) + + # Remove any leftover rpc{user|password} config options from previous tests + self.nodes[1].replace_in_config([("rpcuser", "#rpcuser"), ("rpcpassword", "#rpcpassword")]) + + self.log.info('Check default cookie permission') + test_perm(None) + + self.log.info('Check custom cookie permissions') + for perm in ["owner", "group", "all"]: + test_perm(perm) + def run_test(self): self.conf_setup() self.log.info('Check correctness of the rpcauth config option') @@ -115,6 +152,8 @@ class HTTPBasicsTest(BitcoinTestFramework): (self.nodes[0].chain_path / ".cookie.tmp").mkdir() self.nodes[0].assert_start_raises_init_error(expected_msg=init_error) + self.test_rpccookieperms() + if __name__ == '__main__': HTTPBasicsTest().main() diff --git a/test/functional/test-shell.md b/test/functional/test-shell.md index b89b40f13d..4cd62c4ef3 100644 --- a/test/functional/test-shell.md +++ b/test/functional/test-shell.md @@ -123,11 +123,11 @@ We can also log custom events to the logger. ``` **Note: Please also consider the functional test -[readme](../test/functional/README.md), which provides an overview of the +[readme](/test/functional/README.md), which provides an overview of the test-framework**. Modules such as -[key.py](../test/functional/test_framework/key.py), -[script.py](../test/functional/test_framework/script.py) and -[messages.py](../test/functional/test_framework/messages.py) are particularly +[key.py](/test/functional/test_framework/key.py), +[script.py](/test/functional/test_framework/script.py) and +[messages.py](/test/functional/test_framework/messages.py) are particularly useful in constructing objects which can be passed to the bitcoind nodes managed by a running `TestShell` object. diff --git a/test/functional/test_framework/address.py b/test/functional/test_framework/address.py index 5b2e3289a9..bcb38b21cd 100644 --- a/test/functional/test_framework/address.py +++ b/test/functional/test_framework/address.py @@ -47,7 +47,7 @@ class AddressType(enum.Enum): b58chars = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz' -def create_deterministic_address_bcrt1_p2tr_op_true(): +def create_deterministic_address_bcrt1_p2tr_op_true(explicit_internal_key=None): """ Generates a deterministic bech32m address (segwit v1 output) that can be spent with a witness stack of OP_TRUE and the control block @@ -55,9 +55,10 @@ def create_deterministic_address_bcrt1_p2tr_op_true(): Returns a tuple with the generated address and the internal key. """ - internal_key = (1).to_bytes(32, 'big') + internal_key = explicit_internal_key or (1).to_bytes(32, 'big') address = output_key_to_p2tr(taproot_construct(internal_key, [(None, CScript([OP_TRUE]))]).output_pubkey) - assert_equal(address, 'bcrt1p9yfmy5h72durp7zrhlw9lf7jpwjgvwdg0jr0lqmmjtgg83266lqsekaqka') + if explicit_internal_key is None: + assert_equal(address, 'bcrt1p9yfmy5h72durp7zrhlw9lf7jpwjgvwdg0jr0lqmmjtgg83266lqsekaqka') return (address, internal_key) diff --git a/test/functional/test_framework/authproxy.py b/test/functional/test_framework/authproxy.py index 03042877b2..a357ae4d34 100644 --- a/test/functional/test_framework/authproxy.py +++ b/test/functional/test_framework/authproxy.py @@ -26,7 +26,7 @@ ServiceProxy class: - HTTP connections persist for the life of the AuthServiceProxy object (if server supports HTTP/1.1) -- sends protocol 'version', per JSON-RPC 1.1 +- sends "jsonrpc":"2.0", per JSON-RPC 2.0 - sends proper, incrementing 'id' - sends Basic HTTP authentication headers - parses all JSON numbers that look like floats as Decimal @@ -117,7 +117,7 @@ class AuthServiceProxy(): params = dict(args=args, **argsn) else: params = args or argsn - return {'version': '1.1', + return {'jsonrpc': '2.0', 'method': self._service_name, 'params': params, 'id': AuthServiceProxy.__id_count} @@ -125,15 +125,28 @@ class AuthServiceProxy(): def __call__(self, *args, **argsn): postdata = json.dumps(self.get_request(*args, **argsn), default=serialization_fallback, ensure_ascii=self.ensure_ascii) response, status = self._request('POST', self.__url.path, postdata.encode('utf-8')) - if response['error'] is not None: - raise JSONRPCException(response['error'], status) - elif 'result' not in response: - raise JSONRPCException({ - 'code': -343, 'message': 'missing JSON-RPC result'}, status) - elif status != HTTPStatus.OK: - raise JSONRPCException({ - 'code': -342, 'message': 'non-200 HTTP status code but no JSON-RPC error'}, status) + # For backwards compatibility tests, accept JSON RPC 1.1 responses + if 'jsonrpc' not in response: + if response['error'] is not None: + raise JSONRPCException(response['error'], status) + elif 'result' not in response: + raise JSONRPCException({ + 'code': -343, 'message': 'missing JSON-RPC result'}, status) + elif status != HTTPStatus.OK: + raise JSONRPCException({ + 'code': -342, 'message': 'non-200 HTTP status code but no JSON-RPC error'}, status) + else: + return response['result'] else: + assert response['jsonrpc'] == '2.0' + if status != HTTPStatus.OK: + raise JSONRPCException({ + 'code': -342, 'message': 'non-200 HTTP status code'}, status) + if 'error' in response: + raise JSONRPCException(response['error'], status) + elif 'result' not in response: + raise JSONRPCException({ + 'code': -343, 'message': 'missing JSON-RPC 2.0 result and error'}, status) return response['result'] def batch(self, rpc_call_list): @@ -142,7 +155,7 @@ class AuthServiceProxy(): response, status = self._request('POST', self.__url.path, postdata.encode('utf-8')) if status != HTTPStatus.OK: raise JSONRPCException({ - 'code': -342, 'message': 'non-200 HTTP status code but no JSON-RPC error'}, status) + 'code': -342, 'message': 'non-200 HTTP status code'}, status) return response def _get_response(self): @@ -160,6 +173,15 @@ class AuthServiceProxy(): raise JSONRPCException({ 'code': -342, 'message': 'missing HTTP response from server'}) + # Check for no-content HTTP status code, which can be returned when an + # RPC client requests a JSON-RPC 2.0 "notification" with no response. + # Currently this is only possible if clients call the _request() method + # directly to send a raw request. + if http_response.status == HTTPStatus.NO_CONTENT: + if len(http_response.read()) != 0: + raise JSONRPCException({'code': -342, 'message': 'Content received with NO CONTENT status code'}) + return None, http_response.status + content_type = http_response.getheader('Content-Type') if content_type != 'application/json': raise JSONRPCException( diff --git a/test/functional/test_framework/blocktools.py b/test/functional/test_framework/blocktools.py index cfd923bab3..f0dc866f69 100644 --- a/test/functional/test_framework/blocktools.py +++ b/test/functional/test_framework/blocktools.py @@ -28,6 +28,7 @@ from .messages import ( ser_uint256, tx_from_hex, uint256_from_str, + WITNESS_SCALE_FACTOR, ) from .script import ( CScript, @@ -45,7 +46,6 @@ from .script_util import ( ) from .util import assert_equal -WITNESS_SCALE_FACTOR = 4 MAX_BLOCK_SIGOPS = 20000 MAX_BLOCK_SIGOPS_WEIGHT = MAX_BLOCK_SIGOPS * WITNESS_SCALE_FACTOR diff --git a/test/functional/test_framework/crypto/secp256k1.py b/test/functional/test_framework/crypto/secp256k1.py index 2e9e419da5..50a46dce37 100644 --- a/test/functional/test_framework/crypto/secp256k1.py +++ b/test/functional/test_framework/crypto/secp256k1.py @@ -15,6 +15,8 @@ Exports: * G: the secp256k1 generator point """ +import unittest +from hashlib import sha256 class FE: """Objects of this class represent elements of the field GF(2**256 - 2**32 - 977). @@ -344,3 +346,9 @@ class FastGEMul: # Precomputed table with multiples of G for fast multiplication FAST_G = FastGEMul(G) + +class TestFrameworkSecp256k1(unittest.TestCase): + def test_H(self): + H = sha256(G.to_bytes_uncompressed()).digest() + assert GE.lift_x(FE.from_bytes(H)) is not None + self.assertEqual(H.hex(), "50929b74c1a04954b78b4b6035e97a5e078a5a0f28ec96d547bfee9ace803ac0") diff --git a/test/functional/test_framework/key.py b/test/functional/test_framework/key.py index 06252f8996..939c7cbef6 100644 --- a/test/functional/test_framework/key.py +++ b/test/functional/test_framework/key.py @@ -14,6 +14,7 @@ import random import unittest from test_framework.crypto import secp256k1 +from test_framework.util import random_bitflip # Point with no known discrete log. H_POINT = "50929b74c1a04954b78b4b6035e97a5e078a5a0f28ec96d547bfee9ace803ac0" @@ -292,11 +293,6 @@ def sign_schnorr(key, msg, aux=None, flip_p=False, flip_r=False): class TestFrameworkKey(unittest.TestCase): def test_ecdsa_and_schnorr(self): """Test the Python ECDSA and Schnorr implementations.""" - def random_bitflip(sig): - sig = list(sig) - sig[random.randrange(len(sig))] ^= (1 << (random.randrange(8))) - return bytes(sig) - byte_arrays = [generate_privkey() for _ in range(3)] + [v.to_bytes(32, 'big') for v in [0, ORDER - 1, ORDER, 2**256 - 1]] keys = {} for privkey_bytes in byte_arrays: # build array of key/pubkey pairs diff --git a/test/functional/test_framework/mempool_util.py b/test/functional/test_framework/mempool_util.py new file mode 100644 index 0000000000..148cc935ed --- /dev/null +++ b/test/functional/test_framework/mempool_util.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +# Copyright (c) 2024 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Helpful routines for mempool testing.""" +from decimal import Decimal + +from .blocktools import ( + COINBASE_MATURITY, +) +from .util import ( + assert_equal, + assert_greater_than, + create_lots_of_big_transactions, + gen_return_txouts, +) +from .wallet import ( + MiniWallet, +) + + +def fill_mempool(test_framework, node): + """Fill mempool until eviction. + + Allows for simpler testing of scenarios with floating mempoolminfee > minrelay + Requires -datacarriersize=100000 and + -maxmempool=5. + It will not ensure mempools become synced as it + is based on a single node and assumes -minrelaytxfee + is 1 sat/vbyte. + To avoid unintentional tx dependencies, the mempool filling txs are created with a + tagged ephemeral miniwallet instance. + """ + test_framework.log.info("Fill the mempool until eviction is triggered and the mempoolminfee rises") + txouts = gen_return_txouts() + relayfee = node.getnetworkinfo()['relayfee'] + + assert_equal(relayfee, Decimal('0.00001000')) + + tx_batch_size = 1 + num_of_batches = 75 + # Generate UTXOs to flood the mempool + # 1 to create a tx initially that will be evicted from the mempool later + # 75 transactions each with a fee rate higher than the previous one + ephemeral_miniwallet = MiniWallet(node, tag_name="fill_mempool_ephemeral_wallet") + test_framework.generate(ephemeral_miniwallet, 1 + num_of_batches * tx_batch_size) + + # Mine enough blocks so that the UTXOs are allowed to be spent + test_framework.generate(node, COINBASE_MATURITY - 1) + + # Get all UTXOs up front to ensure none of the transactions spend from each other, as that may + # change their effective feerate and thus the order in which they are selected for eviction. + confirmed_utxos = [ephemeral_miniwallet.get_utxo(confirmed_only=True) for _ in range(num_of_batches * tx_batch_size + 1)] + assert_equal(len(confirmed_utxos), num_of_batches * tx_batch_size + 1) + + test_framework.log.debug("Create a mempool tx that will be evicted") + tx_to_be_evicted_id = ephemeral_miniwallet.send_self_transfer( + from_node=node, utxo_to_spend=confirmed_utxos.pop(0), fee_rate=relayfee)["txid"] + + # Increase the tx fee rate to give the subsequent transactions a higher priority in the mempool + # The tx has an approx. vsize of 65k, i.e. multiplying the previous fee rate (in sats/kvB) + # by 130 should result in a fee that corresponds to 2x of that fee rate + base_fee = relayfee * 130 + + test_framework.log.debug("Fill up the mempool with txs with higher fee rate") + with node.assert_debug_log(["rolling minimum fee bumped"]): + for batch_of_txid in range(num_of_batches): + fee = (batch_of_txid + 1) * base_fee + utxos = confirmed_utxos[:tx_batch_size] + create_lots_of_big_transactions(ephemeral_miniwallet, node, fee, tx_batch_size, txouts, utxos) + del confirmed_utxos[:tx_batch_size] + + test_framework.log.debug("The tx should be evicted by now") + # The number of transactions created should be greater than the ones present in the mempool + assert_greater_than(tx_batch_size * num_of_batches, len(node.getrawmempool())) + # Initial tx created should not be present in the mempool anymore as it had a lower fee rate + assert tx_to_be_evicted_id not in node.getrawmempool() + + test_framework.log.debug("Check that mempoolminfee is larger than minrelaytxfee") + assert_equal(node.getmempoolinfo()['minrelaytxfee'], Decimal('0.00001000')) + assert_greater_than(node.getmempoolinfo()['mempoolminfee'], Decimal('0.00001000')) diff --git a/test/functional/test_framework/messages.py b/test/functional/test_framework/messages.py index 4e496a9275..005f7546a8 100755 --- a/test/functional/test_framework/messages.py +++ b/test/functional/test_framework/messages.py @@ -560,12 +560,12 @@ class CTxWitness: class CTransaction: - __slots__ = ("hash", "nLockTime", "nVersion", "sha256", "vin", "vout", + __slots__ = ("hash", "nLockTime", "version", "sha256", "vin", "vout", "wit") def __init__(self, tx=None): if tx is None: - self.nVersion = 2 + self.version = 2 self.vin = [] self.vout = [] self.wit = CTxWitness() @@ -573,7 +573,7 @@ class CTransaction: self.sha256 = None self.hash = None else: - self.nVersion = tx.nVersion + self.version = tx.version self.vin = copy.deepcopy(tx.vin) self.vout = copy.deepcopy(tx.vout) self.nLockTime = tx.nLockTime @@ -582,7 +582,7 @@ class CTransaction: self.wit = copy.deepcopy(tx.wit) def deserialize(self, f): - self.nVersion = int.from_bytes(f.read(4), "little", signed=True) + self.version = int.from_bytes(f.read(4), "little") self.vin = deser_vector(f, CTxIn) flags = 0 if len(self.vin) == 0: @@ -605,7 +605,7 @@ class CTransaction: def serialize_without_witness(self): r = b"" - r += self.nVersion.to_bytes(4, "little", signed=True) + r += self.version.to_bytes(4, "little") r += ser_vector(self.vin) r += ser_vector(self.vout) r += self.nLockTime.to_bytes(4, "little") @@ -617,7 +617,7 @@ class CTransaction: if not self.wit.is_null(): flags |= 1 r = b"" - r += self.nVersion.to_bytes(4, "little", signed=True) + r += self.version.to_bytes(4, "little") if flags: dummy = [] r += ser_vector(dummy) @@ -677,8 +677,8 @@ class CTransaction: return math.ceil(self.get_weight() / WITNESS_SCALE_FACTOR) def __repr__(self): - return "CTransaction(nVersion=%i vin=%s vout=%s wit=%s nLockTime=%i)" \ - % (self.nVersion, repr(self.vin), repr(self.vout), repr(self.wit), self.nLockTime) + return "CTransaction(version=%i vin=%s vout=%s wit=%s nLockTime=%i)" \ + % (self.version, repr(self.vin), repr(self.vout), repr(self.wit), self.nLockTime) class CBlockHeader: diff --git a/test/functional/test_framework/p2p.py b/test/functional/test_framework/p2p.py index 00bd1e4017..4f1265eb54 100755 --- a/test/functional/test_framework/p2p.py +++ b/test/functional/test_framework/p2p.py @@ -223,6 +223,7 @@ class P2PConnection(asyncio.Protocol): # send the initial handshake immediately if self.supports_v2_p2p and self.v2_state.initiating and not self.v2_state.tried_v2_handshake: send_handshake_bytes = self.v2_state.initiate_v2_handshake() + logger.debug(f"sending {len(self.v2_state.sent_garbage)} bytes of garbage data") self.send_raw_message(send_handshake_bytes) # for v1 outbound connections, send version message immediately after opening # (for v2 outbound connections, send it after the initial v2 handshake) @@ -262,6 +263,7 @@ class P2PConnection(asyncio.Protocol): self.v2_state = None return elif send_handshake_bytes: + logger.debug(f"sending {len(self.v2_state.sent_garbage)} bytes of garbage data") self.send_raw_message(send_handshake_bytes) elif send_handshake_bytes == b"": return # only after send_handshake_bytes are sent can `complete_handshake()` be done @@ -411,7 +413,7 @@ class P2PConnection(asyncio.Protocol): tmsg = self.magic_bytes tmsg += msgtype tmsg += b"\x00" * (12 - len(msgtype)) - tmsg += struct.pack("<I", len(data)) + tmsg += len(data).to_bytes(4, "little") th = sha256(data) h = sha256(th) tmsg += h[:4] diff --git a/test/functional/test_framework/script.py b/test/functional/test_framework/script.py index 7b19d31e17..97d62f957b 100644 --- a/test/functional/test_framework/script.py +++ b/test/functional/test_framework/script.py @@ -8,7 +8,6 @@ This file is modified from python-bitcoinlib. """ from collections import namedtuple -import struct import unittest from .key import TaggedHash, tweak_add_pubkey, compute_xonly_pubkey @@ -58,9 +57,9 @@ class CScriptOp(int): elif len(d) <= 0xff: return b'\x4c' + bytes([len(d)]) + d # OP_PUSHDATA1 elif len(d) <= 0xffff: - return b'\x4d' + struct.pack(b'<H', len(d)) + d # OP_PUSHDATA2 + return b'\x4d' + len(d).to_bytes(2, "little") + d # OP_PUSHDATA2 elif len(d) <= 0xffffffff: - return b'\x4e' + struct.pack(b'<I', len(d)) + d # OP_PUSHDATA4 + return b'\x4e' + len(d).to_bytes(4, "little") + d # OP_PUSHDATA4 else: raise ValueError("Data too long to encode in a PUSHDATA op") @@ -670,7 +669,7 @@ def LegacySignatureMsg(script, txTo, inIdx, hashtype): txtmp.vin.append(tmp) s = txtmp.serialize_without_witness() - s += struct.pack(b"<I", hashtype) + s += hashtype.to_bytes(4, "little") return (s, None) @@ -726,7 +725,7 @@ def SegwitV0SignatureMsg(script, txTo, inIdx, hashtype, amount): if (not (hashtype & SIGHASH_ANYONECANPAY) and (hashtype & 0x1f) != SIGHASH_SINGLE and (hashtype & 0x1f) != SIGHASH_NONE): serialize_sequence = bytes() for i in txTo.vin: - serialize_sequence += struct.pack("<I", i.nSequence) + serialize_sequence += i.nSequence.to_bytes(4, "little") hashSequence = uint256_from_str(hash256(serialize_sequence)) if ((hashtype & 0x1f) != SIGHASH_SINGLE and (hashtype & 0x1f) != SIGHASH_NONE): @@ -739,16 +738,16 @@ def SegwitV0SignatureMsg(script, txTo, inIdx, hashtype, amount): hashOutputs = uint256_from_str(hash256(serialize_outputs)) ss = bytes() - ss += struct.pack("<i", txTo.nVersion) + ss += txTo.version.to_bytes(4, "little") ss += ser_uint256(hashPrevouts) ss += ser_uint256(hashSequence) ss += txTo.vin[inIdx].prevout.serialize() ss += ser_string(script) - ss += struct.pack("<q", amount) - ss += struct.pack("<I", txTo.vin[inIdx].nSequence) + ss += amount.to_bytes(8, "little", signed=True) + ss += txTo.vin[inIdx].nSequence.to_bytes(4, "little") ss += ser_uint256(hashOutputs) ss += txTo.nLockTime.to_bytes(4, "little") - ss += struct.pack("<I", hashtype) + ss += hashtype.to_bytes(4, "little") return ss def SegwitV0SignatureHash(*args, **kwargs): @@ -800,13 +799,13 @@ def BIP341_sha_prevouts(txTo): return sha256(b"".join(i.prevout.serialize() for i in txTo.vin)) def BIP341_sha_amounts(spent_utxos): - return sha256(b"".join(struct.pack("<q", u.nValue) for u in spent_utxos)) + return sha256(b"".join(u.nValue.to_bytes(8, "little", signed=True) for u in spent_utxos)) def BIP341_sha_scriptpubkeys(spent_utxos): return sha256(b"".join(ser_string(u.scriptPubKey) for u in spent_utxos)) def BIP341_sha_sequences(txTo): - return sha256(b"".join(struct.pack("<I", i.nSequence) for i in txTo.vin)) + return sha256(b"".join(i.nSequence.to_bytes(4, "little") for i in txTo.vin)) def BIP341_sha_outputs(txTo): return sha256(b"".join(o.serialize() for o in txTo.vout)) @@ -818,8 +817,8 @@ def TaprootSignatureMsg(txTo, spent_utxos, hash_type, input_index = 0, scriptpat in_type = hash_type & SIGHASH_ANYONECANPAY spk = spent_utxos[input_index].scriptPubKey ss = bytes([0, hash_type]) # epoch, hash_type - ss += struct.pack("<i", txTo.nVersion) - ss += struct.pack("<I", txTo.nLockTime) + ss += txTo.version.to_bytes(4, "little") + ss += txTo.nLockTime.to_bytes(4, "little") if in_type != SIGHASH_ANYONECANPAY: ss += BIP341_sha_prevouts(txTo) ss += BIP341_sha_amounts(spent_utxos) @@ -835,11 +834,11 @@ def TaprootSignatureMsg(txTo, spent_utxos, hash_type, input_index = 0, scriptpat ss += bytes([spend_type]) if in_type == SIGHASH_ANYONECANPAY: ss += txTo.vin[input_index].prevout.serialize() - ss += struct.pack("<q", spent_utxos[input_index].nValue) + ss += spent_utxos[input_index].nValue.to_bytes(8, "little", signed=True) ss += ser_string(spk) - ss += struct.pack("<I", txTo.vin[input_index].nSequence) + ss += txTo.vin[input_index].nSequence.to_bytes(4, "little") else: - ss += struct.pack("<I", input_index) + ss += input_index.to_bytes(4, "little") if (spend_type & 1): ss += sha256(ser_string(annex)) if out_type == SIGHASH_SINGLE: @@ -850,7 +849,7 @@ def TaprootSignatureMsg(txTo, spent_utxos, hash_type, input_index = 0, scriptpat if (scriptpath): ss += TaggedHash("TapLeaf", bytes([leaf_ver]) + ser_string(script)) ss += bytes([0]) - ss += struct.pack("<i", codeseparator_pos) + ss += codeseparator_pos.to_bytes(4, "little", signed=True) assert len(ss) == 175 - (in_type == SIGHASH_ANYONECANPAY) * 49 - (out_type != SIGHASH_ALL and out_type != SIGHASH_SINGLE) * 32 + (annex is not None) * 32 + scriptpath * 37 return ss diff --git a/test/functional/test_framework/script_util.py b/test/functional/test_framework/script_util.py index 62894cc0f4..855f3b8cf5 100755 --- a/test/functional/test_framework/script_util.py +++ b/test/functional/test_framework/script_util.py @@ -3,10 +3,13 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Useful Script constants and utils.""" +import unittest + from test_framework.script import ( CScript, - CScriptOp, OP_0, + OP_15, + OP_16, OP_CHECKMULTISIG, OP_CHECKSIG, OP_DUP, @@ -49,10 +52,8 @@ def keys_to_multisig_script(keys, *, k=None): if k is None: # n-of-n multisig by default k = n assert k <= n - op_k = CScriptOp.encode_op_n(k) - op_n = CScriptOp.encode_op_n(n) checked_keys = [check_key(key) for key in keys] - return CScript([op_k] + checked_keys + [op_n, OP_CHECKMULTISIG]) + return CScript([k] + checked_keys + [n, OP_CHECKMULTISIG]) def keyhash_to_p2pkh_script(hash): @@ -125,3 +126,19 @@ def check_script(script): if isinstance(script, bytes) or isinstance(script, CScript): return script assert False + + +class TestFrameworkScriptUtil(unittest.TestCase): + def test_multisig(self): + fake_pubkey = bytes([0]*33) + # check correct encoding of P2MS script with n,k <= 16 + normal_ms_script = keys_to_multisig_script([fake_pubkey]*16, k=15) + self.assertEqual(len(normal_ms_script), 1 + 16*34 + 1 + 1) + self.assertTrue(normal_ms_script.startswith(bytes([OP_15]))) + self.assertTrue(normal_ms_script.endswith(bytes([OP_16, OP_CHECKMULTISIG]))) + + # check correct encoding of P2MS script with n,k > 16 + max_ms_script = keys_to_multisig_script([fake_pubkey]*20, k=19) + self.assertEqual(len(max_ms_script), 2 + 20*34 + 2 + 1) + self.assertTrue(max_ms_script.startswith(bytes([1, 19]))) # using OP_PUSH1 + self.assertTrue(max_ms_script.endswith(bytes([1, 20, OP_CHECKMULTISIG]))) diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index a2f767cc98..9e44a11143 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2022 The Bitcoin Core developers +# Copyright (c) 2014-present The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Base class for RPC testing.""" @@ -444,6 +444,10 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): n.createwallet(wallet_name=wallet_name, descriptors=self.options.descriptors, load_on_startup=True) n.importprivkey(privkey=n.get_deterministic_priv_key().key, label='coinbase', rescan=True) + # Only enables wallet support when the module is available + def enable_wallet_if_possible(self): + self._requires_wallet = self.is_wallet_compiled() + def run_test(self): """Tests must override this method to define test logic""" raise NotImplementedError @@ -610,8 +614,6 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): """ from_connection = self.nodes[a] to_connection = self.nodes[b] - from_num_peers = 1 + len(from_connection.getpeerinfo()) - to_num_peers = 1 + len(to_connection.getpeerinfo()) ip_port = "127.0.0.1:" + str(p2p_port(b)) if peer_advertises_v2 is None: @@ -627,19 +629,28 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): if not wait_for_connect: return - # poll until version handshake complete to avoid race conditions - # with transaction relaying - # See comments in net_processing: - # * Must have a version message before anything else - # * Must have a verack message before anything else - self.wait_until(lambda: sum(peer['version'] != 0 for peer in from_connection.getpeerinfo()) == from_num_peers) - self.wait_until(lambda: sum(peer['version'] != 0 for peer in to_connection.getpeerinfo()) == to_num_peers) - self.wait_until(lambda: sum(peer['bytesrecv_per_msg'].pop('verack', 0) >= 21 for peer in from_connection.getpeerinfo()) == from_num_peers) - self.wait_until(lambda: sum(peer['bytesrecv_per_msg'].pop('verack', 0) >= 21 for peer in to_connection.getpeerinfo()) == to_num_peers) - # The message bytes are counted before processing the message, so make - # sure it was fully processed by waiting for a ping. - self.wait_until(lambda: sum(peer["bytesrecv_per_msg"].pop("pong", 0) >= 29 for peer in from_connection.getpeerinfo()) == from_num_peers) - self.wait_until(lambda: sum(peer["bytesrecv_per_msg"].pop("pong", 0) >= 29 for peer in to_connection.getpeerinfo()) == to_num_peers) + # Use subversion as peer id. Test nodes have their node number appended to the user agent string + from_connection_subver = from_connection.getnetworkinfo()['subversion'] + to_connection_subver = to_connection.getnetworkinfo()['subversion'] + + def find_conn(node, peer_subversion, inbound): + return next(filter(lambda peer: peer['subver'] == peer_subversion and peer['inbound'] == inbound, node.getpeerinfo()), None) + + self.wait_until(lambda: find_conn(from_connection, to_connection_subver, inbound=False) is not None) + self.wait_until(lambda: find_conn(to_connection, from_connection_subver, inbound=True) is not None) + + def check_bytesrecv(peer, msg_type, min_bytes_recv): + assert peer is not None, "Error: peer disconnected" + return peer['bytesrecv_per_msg'].pop(msg_type, 0) >= min_bytes_recv + + # Poll until version handshake (fSuccessfullyConnected) is complete to + # avoid race conditions, because some message types are blocked from + # being sent or received before fSuccessfullyConnected. + # + # As the flag fSuccessfullyConnected is not exposed, check it by + # waiting for a pong, which can only happen after the flag was set. + self.wait_until(lambda: check_bytesrecv(find_conn(from_connection, to_connection_subver, inbound=False), 'pong', 29)) + self.wait_until(lambda: check_bytesrecv(find_conn(to_connection, from_connection_subver, inbound=True), 'pong', 29)) def disconnect_nodes(self, a, b): def disconnect_nodes_helper(node_a, node_b): diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index 67e0be5280..0f0083191d 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -106,7 +106,7 @@ class TestNode(): "-debugexclude=libevent", "-debugexclude=leveldb", "-debugexclude=rand", - "-uacomment=testnode%d" % i, + "-uacomment=testnode%d" % i, # required for subversion uniqueness across peers ] if self.descriptors is None: self.args.append("-disablewallet") @@ -241,7 +241,7 @@ class TestNode(): if self.start_perf: self._start_perf() - def wait_for_rpc_connection(self): + def wait_for_rpc_connection(self, *, wait_for_import=True): """Sets up an RPC connection to the bitcoind process. Returns False if unable to connect.""" # Poll at a rate of four times per second poll_per_s = 4 @@ -263,7 +263,7 @@ class TestNode(): ) rpc.getblockcount() # If the call to getblockcount() succeeds then the RPC connection is up - if self.version_is_at_least(190000): + if self.version_is_at_least(190000) and wait_for_import: # getmempoolinfo.loaded is available since commit # bb8ae2c (version 0.19.0) self.wait_until(lambda: rpc.getmempoolinfo()['loaded']) @@ -419,8 +419,9 @@ class TestNode(): return True def wait_until_stopped(self, *, timeout=BITCOIND_PROC_WAIT_TIMEOUT, expect_error=False, **kwargs): - expected_ret_code = 1 if expect_error else 0 # Whether node shutdown return EXIT_FAILURE or EXIT_SUCCESS - self.wait_until(lambda: self.is_node_stopped(expected_ret_code=expected_ret_code, **kwargs), timeout=timeout) + if "expected_ret_code" not in kwargs: + kwargs["expected_ret_code"] = 1 if expect_error else 0 # Whether node shutdown return EXIT_FAILURE or EXIT_SUCCESS + self.wait_until(lambda: self.is_node_stopped(**kwargs), timeout=timeout) def replace_in_config(self, replacements): """ @@ -490,7 +491,7 @@ class TestNode(): self._raise_assertion_error('Expected messages "{}" does not partially match log:\n\n{}\n\n'.format(str(expected_msgs), print_log)) @contextlib.contextmanager - def wait_for_debug_log(self, expected_msgs, timeout=60): + def busy_wait_for_debug_log(self, expected_msgs, timeout=60): """ Block until we see a particular debug log message fragment or until we exceed the timeout. Return: @@ -665,7 +666,7 @@ class TestNode(): assert_msg += "with expected error " + expected_msg self._raise_assertion_error(assert_msg) - def add_p2p_connection(self, p2p_conn, *, wait_for_verack=True, send_version=True, supports_v2_p2p=None, wait_for_v2_handshake=True, **kwargs): + def add_p2p_connection(self, p2p_conn, *, wait_for_verack=True, send_version=True, supports_v2_p2p=None, wait_for_v2_handshake=True, expect_success=True, **kwargs): """Add an inbound p2p connection to the node. This method adds the p2p connection to the self.p2ps list and also @@ -685,7 +686,6 @@ class TestNode(): if supports_v2_p2p is None: supports_v2_p2p = self.use_v2transport - p2p_conn.p2p_connected_to_node = True if self.use_v2transport: kwargs['services'] = kwargs.get('services', P2P_SERVICES) | NODE_P2P_V2 @@ -693,6 +693,8 @@ class TestNode(): p2p_conn.peer_connect(**kwargs, send_version=send_version, net=self.chain, timeout_factor=self.timeout_factor, supports_v2_p2p=supports_v2_p2p)() self.p2ps.append(p2p_conn) + if not expect_success: + return p2p_conn p2p_conn.wait_until(lambda: p2p_conn.is_connected, check_connected=False) if supports_v2_p2p and wait_for_v2_handshake: p2p_conn.wait_until(lambda: p2p_conn.v2_state.tried_v2_handshake) diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index 0de09b6440..f3d080fdde 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -14,6 +14,7 @@ import logging import os import pathlib import platform +import random import re import time @@ -247,6 +248,12 @@ def ceildiv(a, b): return -(-a // b) +def random_bitflip(data): + data = list(data) + data[random.randrange(len(data))] ^= (1 << (random.randrange(8))) + return bytes(data) + + def get_fee(tx_size, feerate_btc_kvb): """Calculate the fee in BTC given a feerate is BTC/kvB. Reflects CFeeRate::GetFee""" feerate_sat_kvb = int(feerate_btc_kvb * Decimal(1e8)) # Fee in sat/kvb as an int to avoid float precision errors @@ -496,65 +503,6 @@ def check_node_connections(*, node, num_in, num_out): assert_equal(info["connections_in"], num_in) assert_equal(info["connections_out"], num_out) -def fill_mempool(test_framework, node, miniwallet): - """Fill mempool until eviction. - - Allows for simpler testing of scenarios with floating mempoolminfee > minrelay - Requires -datacarriersize=100000 and - -maxmempool=5. - It will not ensure mempools become synced as it - is based on a single node and assumes -minrelaytxfee - is 1 sat/vbyte. - To avoid unintentional tx dependencies, it is recommended to use separate miniwallets for - mempool filling vs transactions in tests. - """ - test_framework.log.info("Fill the mempool until eviction is triggered and the mempoolminfee rises") - txouts = gen_return_txouts() - relayfee = node.getnetworkinfo()['relayfee'] - - assert_equal(relayfee, Decimal('0.00001000')) - - tx_batch_size = 1 - num_of_batches = 75 - # Generate UTXOs to flood the mempool - # 1 to create a tx initially that will be evicted from the mempool later - # 75 transactions each with a fee rate higher than the previous one - test_framework.generate(miniwallet, 1 + (num_of_batches * tx_batch_size)) - - # Mine COINBASE_MATURITY - 1 blocks so that the UTXOs are allowed to be spent - test_framework.generate(node, 100 - 1) - - # Get all UTXOs up front to ensure none of the transactions spend from each other, as that may - # change their effective feerate and thus the order in which they are selected for eviction. - confirmed_utxos = [miniwallet.get_utxo(confirmed_only=True) for _ in range(num_of_batches * tx_batch_size + 1)] - assert_equal(len(confirmed_utxos), num_of_batches * tx_batch_size + 1) - - test_framework.log.debug("Create a mempool tx that will be evicted") - tx_to_be_evicted_id = miniwallet.send_self_transfer(from_node=node, utxo_to_spend=confirmed_utxos[0], fee_rate=relayfee)["txid"] - del confirmed_utxos[0] - - # Increase the tx fee rate to give the subsequent transactions a higher priority in the mempool - # The tx has an approx. vsize of 65k, i.e. multiplying the previous fee rate (in sats/kvB) - # by 130 should result in a fee that corresponds to 2x of that fee rate - base_fee = relayfee * 130 - - test_framework.log.debug("Fill up the mempool with txs with higher fee rate") - with node.assert_debug_log(["rolling minimum fee bumped"]): - for batch_of_txid in range(num_of_batches): - fee = (batch_of_txid + 1) * base_fee - utxos = confirmed_utxos[:tx_batch_size] - create_lots_of_big_transactions(miniwallet, node, fee, tx_batch_size, txouts, utxos) - del confirmed_utxos[:tx_batch_size] - - test_framework.log.debug("The tx should be evicted by now") - # The number of transactions created should be greater than the ones present in the mempool - assert_greater_than(tx_batch_size * num_of_batches, len(node.getrawmempool())) - # Initial tx created should not be present in the mempool anymore as it had a lower fee rate - assert tx_to_be_evicted_id not in node.getrawmempool() - - test_framework.log.debug("Check that mempoolminfee is larger than minrelaytxfee") - assert_equal(node.getmempoolinfo()['minrelaytxfee'], Decimal('0.00001000')) - assert_greater_than(node.getmempoolinfo()['mempoolminfee'], Decimal('0.00001000')) # Transaction/Block functions ############################# diff --git a/test/functional/test_framework/v2_p2p.py b/test/functional/test_framework/v2_p2p.py index 8f79623bd8..87600c36de 100644 --- a/test/functional/test_framework/v2_p2p.py +++ b/test/functional/test_framework/v2_p2p.py @@ -4,7 +4,6 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Class for v2 P2P protocol (see BIP 324)""" -import logging import random from .crypto.bip324_cipher import FSChaCha20Poly1305 @@ -14,14 +13,12 @@ from .crypto.hkdf import hkdf_sha256 from .key import TaggedHash from .messages import MAGIC_BYTES -logger = logging.getLogger("TestFramework.v2_p2p") CHACHA20POLY1305_EXPANSION = 16 HEADER_LEN = 1 IGNORE_BIT_POS = 7 LENGTH_FIELD_LEN = 3 MAX_GARBAGE_LEN = 4095 -TRANSPORT_VERSION = b'' SHORTID = { 1: b"addr", @@ -95,6 +92,7 @@ class EncryptedP2PState: # has been decrypted. set to -1 if decryption hasn't been done yet. self.contents_len = -1 self.found_garbage_terminator = False + self.transport_version = b'' @staticmethod def v2_ecdh(priv, ellswift_theirs, ellswift_ours, initiating): @@ -111,12 +109,12 @@ class EncryptedP2PState: # Responding, place their public key encoding first. return TaggedHash("bip324_ellswift_xonly_ecdh", ellswift_theirs + ellswift_ours + ecdh_point_x32) - def generate_keypair_and_garbage(self): + def generate_keypair_and_garbage(self, garbage_len=None): """Generates ellswift keypair and 4095 bytes garbage at max""" self.privkey_ours, self.ellswift_ours = ellswift_create() - garbage_len = random.randrange(MAX_GARBAGE_LEN + 1) + if garbage_len is None: + garbage_len = random.randrange(MAX_GARBAGE_LEN + 1) self.sent_garbage = random.randbytes(garbage_len) - logger.debug(f"sending {garbage_len} bytes of garbage data") return self.ellswift_ours + self.sent_garbage def initiate_v2_handshake(self): @@ -172,7 +170,7 @@ class EncryptedP2PState: msg_to_send += self.v2_enc_packet(decoy_content_len * b'\x00', aad=aad, ignore=True) aad = b'' # Send version packet. - msg_to_send += self.v2_enc_packet(TRANSPORT_VERSION, aad=aad) + msg_to_send += self.v2_enc_packet(self.transport_version, aad=aad) return 64 - len(self.received_prefix), msg_to_send def authenticate_handshake(self, response): diff --git a/test/functional/test_framework/wallet.py b/test/functional/test_framework/wallet.py index 470ed08ed4..cb0d291361 100644 --- a/test/functional/test_framework/wallet.py +++ b/test/functional/test_framework/wallet.py @@ -7,6 +7,7 @@ from copy import deepcopy from decimal import Decimal from enum import Enum +import math from typing import ( Any, Optional, @@ -32,10 +33,14 @@ from test_framework.messages import ( CTxIn, CTxInWitness, CTxOut, + hash256, + ser_compact_size, + WITNESS_SCALE_FACTOR, ) from test_framework.script import ( CScript, LEAF_VERSION_TAPSCRIPT, + OP_1, OP_NOP, OP_RETURN, OP_TRUE, @@ -51,6 +56,7 @@ from test_framework.script_util import ( from test_framework.util import ( assert_equal, assert_greater_than_or_equal, + get_fee, ) from test_framework.wallet_util import generate_keypair @@ -65,7 +71,10 @@ class MiniWalletMode(Enum): However, if the transactions need to be modified by the user (e.g. prepending scriptSig for testing opcodes that are activated by a soft-fork), or the txs should contain an actual signature, the raw modes RAW_OP_TRUE and RAW_P2PK - can be useful. Summary of modes: + can be useful. In order to avoid mixing of UTXOs between different MiniWallet + instances, a tag name can be passed to the default mode, to create different + output scripts. Note that the UTXOs from the pre-generated test chain can + only be spent if no tag is passed. Summary of modes: | output | | tx is | can modify | needs mode | description | address | standard | scriptSig | signing @@ -80,22 +89,25 @@ class MiniWalletMode(Enum): class MiniWallet: - def __init__(self, test_node, *, mode=MiniWalletMode.ADDRESS_OP_TRUE): + def __init__(self, test_node, *, mode=MiniWalletMode.ADDRESS_OP_TRUE, tag_name=None): self._test_node = test_node self._utxos = [] self._mode = mode assert isinstance(mode, MiniWalletMode) if mode == MiniWalletMode.RAW_OP_TRUE: + assert tag_name is None self._scriptPubKey = bytes(CScript([OP_TRUE])) elif mode == MiniWalletMode.RAW_P2PK: # use simple deterministic private key (k=1) + assert tag_name is None self._priv_key = ECKey() self._priv_key.set((1).to_bytes(32, 'big'), True) pub_key = self._priv_key.get_pubkey() self._scriptPubKey = key_to_p2pk_script(pub_key.get_bytes()) elif mode == MiniWalletMode.ADDRESS_OP_TRUE: - self._address, self._internal_key = create_deterministic_address_bcrt1_p2tr_op_true() + internal_key = None if tag_name is None else hash256(tag_name.encode()) + self._address, self._internal_key = create_deterministic_address_bcrt1_p2tr_op_true(internal_key) self._scriptPubKey = address_to_scriptpubkey(self._address) # When the pre-mined test framework chain is used, it contains coinbase @@ -112,13 +124,16 @@ class MiniWallet: """Pad a transaction with extra outputs until it reaches a target weight (or higher). returns the tx """ - tx.vout.append(CTxOut(nValue=0, scriptPubKey=CScript([OP_RETURN, b'a']))) + tx.vout.append(CTxOut(nValue=0, scriptPubKey=CScript([OP_RETURN]))) + # determine number of needed padding bytes by converting weight difference to vbytes dummy_vbytes = (target_weight - tx.get_weight() + 3) // 4 - tx.vout[-1].scriptPubKey = CScript([OP_RETURN, b'a' * dummy_vbytes]) - # Lower bound should always be off by at most 3 + # compensate for the increase of the compact-size encoded script length + # (note that the length encoding of the unpadded output script needs one byte) + dummy_vbytes -= len(ser_compact_size(dummy_vbytes)) - 1 + tx.vout[-1].scriptPubKey = CScript([OP_RETURN] + [OP_1] * dummy_vbytes) + # Actual weight should be at most 3 higher than target weight assert_greater_than_or_equal(tx.get_weight(), target_weight) - # Higher bound should always be off by at most 3 + 12 weight (for encoding the length) - assert_greater_than_or_equal(target_weight + 15, tx.get_weight()) + assert_greater_than_or_equal(target_weight + 3, tx.get_weight()) def get_balance(self): return sum(u['value'] for u in self._utxos) @@ -314,7 +329,7 @@ class MiniWallet: tx = CTransaction() tx.vin = [CTxIn(COutPoint(int(utxo_to_spend['txid'], 16), utxo_to_spend['vout']), nSequence=seq) for utxo_to_spend, seq in zip(utxos_to_spend, sequence)] tx.vout = [CTxOut(amount_per_output, bytearray(self._scriptPubKey)) for _ in range(num_outputs)] - tx.nVersion = version + tx.version = version tx.nLockTime = locktime self.sign_tx(tx) @@ -360,6 +375,10 @@ class MiniWallet: vsize = Decimal(168) # P2PK (73 bytes scriptSig + 35 bytes scriptPubKey + 60 bytes other) else: assert False + if target_weight and not fee: # respect fee_rate if target weight is passed + # the actual weight might be off by 3 WUs, so calculate based on that (see self._bulk_tx) + max_actual_weight = target_weight + 3 + fee = get_fee(math.ceil(max_actual_weight / WITNESS_SCALE_FACTOR), fee_rate) send_value = utxo_to_spend["value"] - (fee or (fee_rate * vsize / 1000)) # create tx diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index f6eccab2b3..67693259d3 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -15,8 +15,10 @@ For a description of arguments recognized by test scripts, see import argparse from collections import deque import configparser +import csv import datetime import os +import pathlib import platform import time import shutil @@ -29,6 +31,13 @@ import logging os.environ["REQUIRE_WALLET_TYPE_SET"] = "1" +# Minimum amount of space to run the tests. +MIN_FREE_SPACE = 1.1 * 1024 * 1024 * 1024 +# Additional space to run an extra job. +ADDITIONAL_SPACE_PER_JOB = 100 * 1024 * 1024 +# Minimum amount of space required for --nocleanup +MIN_NO_CLEANUP_SPACE = 12 * 1024 * 1024 * 1024 + # Formatting. Default colors to empty strings. DEFAULT, BOLD, GREEN, RED = ("", ""), ("", ""), ("", ""), ("", "") try: @@ -176,6 +185,8 @@ BASE_SCRIPTS = [ 'mempool_resurrect.py', 'wallet_txn_doublespend.py --mineblock', 'tool_wallet.py --legacy-wallet', + 'tool_wallet.py --legacy-wallet --bdbro', + 'tool_wallet.py --legacy-wallet --bdbro --swap-bdb-endian', 'tool_wallet.py --descriptors', 'tool_signet_miner.py --legacy-wallet', 'tool_signet_miner.py --descriptors', @@ -183,6 +194,8 @@ BASE_SCRIPTS = [ 'wallet_txn_clone.py --segwit', 'rpc_getchaintips.py', 'rpc_misc.py', + 'p2p_1p1c_network.py', + 'p2p_opportunistic_1p1c.py', 'interface_rest.py', 'mempool_spend_coinbase.py', 'wallet_avoid_mixing_output_types.py --descriptors', @@ -251,9 +264,9 @@ BASE_SCRIPTS = [ 'p2p_invalid_tx.py --v2transport', 'p2p_v2_transport.py', 'p2p_v2_encrypted.py', - 'p2p_v2_earlykeyresponse.py', + 'p2p_v2_misbehaving.py', 'example_test.py', - 'mempool_accept_v3.py', + 'mempool_truc.py', 'wallet_txn_doublespend.py --legacy-wallet', 'wallet_multisig_descriptor_psbt.py --descriptors', 'wallet_txn_doublespend.py --descriptors', @@ -269,6 +282,7 @@ BASE_SCRIPTS = [ 'mempool_packages.py', 'mempool_package_onemore.py', 'mempool_package_limits.py', + 'mempool_package_rbf.py', 'feature_versionbits_warning.py', 'rpc_preciousblock.py', 'wallet_importprunedfunds.py --legacy-wallet', @@ -276,6 +290,7 @@ BASE_SCRIPTS = [ 'p2p_leak_tx.py --v1transport', 'p2p_leak_tx.py --v2transport', 'p2p_eviction.py', + 'p2p_outbound_eviction.py', 'p2p_ibd_stalling.py --v1transport', 'p2p_ibd_stalling.py --v2transport', 'p2p_net_deadlock.py --v1transport', @@ -349,6 +364,7 @@ BASE_SCRIPTS = [ 'feature_addrman.py', 'feature_asmap.py', 'feature_fastprune.py', + 'feature_framework_miniwallet.py', 'mempool_unbroadcast.py', 'mempool_compatibility.py', 'mempool_accept_wtxid.py', @@ -424,6 +440,9 @@ def main(): parser.add_argument('--tmpdirprefix', '-t', default=tempfile.gettempdir(), help="Root directory for datadirs") parser.add_argument('--failfast', '-F', action='store_true', help='stop execution after the first test failure') parser.add_argument('--filter', help='filter scripts to run by regular expression') + parser.add_argument("--nocleanup", dest="nocleanup", default=False, action="store_true", + help="Leave bitcoinds and test.* datadir on exit or error") + parser.add_argument('--resultsfile', '-r', help='store test results (as CSV) to the provided file') args, unknown_args = parser.parse_known_args() @@ -456,6 +475,13 @@ def main(): logging.debug("Temporary test directory at %s" % tmpdir) + results_filepath = None + if args.resultsfile: + results_filepath = pathlib.Path(args.resultsfile) + # Stop early if the parent directory doesn't exist + assert results_filepath.parent.exists(), "Results file parent directory does not exist" + logging.debug("Test results will be written to " + str(results_filepath)) + enable_bitcoind = config["components"].getboolean("ENABLE_BITCOIND") if not enable_bitcoind: @@ -518,6 +544,13 @@ def main(): subprocess.check_call([sys.executable, os.path.join(config["environment"]["SRCDIR"], 'test', 'functional', test_list[0].split()[0]), '-h']) sys.exit(0) + # Warn if there is not enough space on tmpdir to run the tests with --nocleanup + if args.nocleanup: + if shutil.disk_usage(tmpdir).free < MIN_NO_CLEANUP_SPACE: + print(f"{BOLD[1]}WARNING!{BOLD[0]} There may be insufficient free space in {tmpdir} to run the functional test suite with --nocleanup. " + f"A minimum of {MIN_NO_CLEANUP_SPACE // (1024 * 1024 * 1024)} GB of free space is required.") + passon_args.append("--nocleanup") + check_script_list(src_dir=config["environment"]["SRCDIR"], fail_on_warn=args.ci) check_script_prefixes() @@ -535,9 +568,10 @@ def main(): combined_logs_len=args.combinedlogslen, failfast=args.failfast, use_term_control=args.ansi, + results_filepath=results_filepath, ) -def run_tests(*, test_list, src_dir, build_dir, tmpdir, jobs=1, enable_coverage=False, args=None, combined_logs_len=0, failfast=False, use_term_control): +def run_tests(*, test_list, src_dir, build_dir, tmpdir, jobs=1, enable_coverage=False, args=None, combined_logs_len=0, failfast=False, use_term_control, results_filepath=None): args = args or [] # Warn if bitcoind is already running @@ -554,6 +588,11 @@ def run_tests(*, test_list, src_dir, build_dir, tmpdir, jobs=1, enable_coverage= if os.path.isdir(cache_dir): print("%sWARNING!%s There is a cache directory here: %s. If tests fail unexpectedly, try deleting the cache directory." % (BOLD[1], BOLD[0], cache_dir)) + # Warn if there is not enough space on the testing dir + min_space = MIN_FREE_SPACE + (jobs - 1) * ADDITIONAL_SPACE_PER_JOB + if shutil.disk_usage(tmpdir).free < min_space: + print(f"{BOLD[1]}WARNING!{BOLD[0]} There may be insufficient free space in {tmpdir} to run the Bitcoin functional test suite. " + f"Running the test suite with fewer than {min_space // (1024 * 1024)} MB of free space might cause tests to fail.") tests_dir = src_dir + '/test/functional/' # This allows `test_runner.py` to work from an out-of-source build directory using a symlink, @@ -623,7 +662,15 @@ def run_tests(*, test_list, src_dir, build_dir, tmpdir, jobs=1, enable_coverage= logging.debug("Early exiting after test failure") break - print_results(test_results, max_len_name, (int(time.time() - start_time))) + if "[Errno 28] No space left on device" in stdout: + sys.exit(f"Early exiting after test failure due to insufficient free space in {tmpdir}\n" + f"Test execution data left in {tmpdir}.\n" + f"Additional storage is needed to execute testing.") + + runtime = int(time.time() - start_time) + print_results(test_results, max_len_name, runtime) + if results_filepath: + write_results(test_results, results_filepath, runtime) if coverage: coverage_passed = coverage.report_rpc_coverage() @@ -670,6 +717,17 @@ def print_results(test_results, max_len_name, runtime): results += "Runtime: %s s\n" % (runtime) print(results) + +def write_results(test_results, filepath, total_runtime): + with open(filepath, mode="w", encoding="utf8") as results_file: + results_writer = csv.writer(results_file) + results_writer.writerow(['test', 'status', 'duration(seconds)']) + all_passed = True + for test_result in test_results: + all_passed = all_passed and test_result.was_successful + results_writer.writerow([test_result.name, test_result.status, str(test_result.time)]) + results_writer.writerow(['ALL', ("Passed" if all_passed else "Failed"), str(total_runtime)]) + class TestHandler: """ Trigger the test scripts passed in via the list. diff --git a/test/functional/tool_wallet.py b/test/functional/tool_wallet.py index fc042bca66..dcf74f6075 100755 --- a/test/functional/tool_wallet.py +++ b/test/functional/tool_wallet.py @@ -5,6 +5,7 @@ """Test bitcoin-wallet.""" import os +import platform import stat import subprocess import textwrap @@ -14,6 +15,7 @@ from collections import OrderedDict from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, + assert_greater_than, sha256sum_file, ) @@ -21,11 +23,15 @@ from test_framework.util import ( class ToolWalletTest(BitcoinTestFramework): def add_options(self, parser): self.add_wallet_options(parser) + parser.add_argument("--bdbro", action="store_true", help="Use the BerkeleyRO internal parser when dumping a Berkeley DB wallet file") + parser.add_argument("--swap-bdb-endian", action="store_true",help="When making Legacy BDB wallets, always make then byte swapped internally") def set_test_params(self): self.num_nodes = 1 self.setup_clean_chain = True self.rpc_timeout = 120 + if self.options.swap_bdb_endian: + self.extra_args = [["-swapbdbendian"]] def skip_test_if_missing_module(self): self.skip_if_no_wallet() @@ -35,15 +41,21 @@ class ToolWalletTest(BitcoinTestFramework): default_args = ['-datadir={}'.format(self.nodes[0].datadir_path), '-chain=%s' % self.chain] if not self.options.descriptors and 'create' in args: default_args.append('-legacy') + if "dump" in args and self.options.bdbro: + default_args.append("-withinternalbdb") return subprocess.Popen([self.options.bitcoinwallet] + default_args + list(args), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) def assert_raises_tool_error(self, error, *args): p = self.bitcoin_wallet_process(*args) stdout, stderr = p.communicate() - assert_equal(p.poll(), 1) assert_equal(stdout, '') - assert_equal(stderr.strip(), error) + if isinstance(error, tuple): + assert_equal(p.poll(), error[0]) + assert error[1] in stderr.strip() + else: + assert_equal(p.poll(), 1) + assert error in stderr.strip() def assert_tool_output(self, output, *args): p = self.bitcoin_wallet_process(*args) @@ -451,6 +463,88 @@ class ToolWalletTest(BitcoinTestFramework): ''') self.assert_tool_output(expected_output, "-wallet=conflicts", "info") + def test_dump_endianness(self): + self.log.info("Testing dumps of the same contents with different BDB endianness") + + self.start_node(0) + self.nodes[0].createwallet("endian") + self.stop_node(0) + + wallet_dump = self.nodes[0].datadir_path / "endian.dump" + self.assert_tool_output("The dumpfile may contain private keys. To ensure the safety of your Bitcoin, do not share the dumpfile.\n", "-wallet=endian", f"-dumpfile={wallet_dump}", "dump") + expected_dump = self.read_dump(wallet_dump) + + self.do_tool_createfromdump("native_endian", "endian.dump", "bdb") + native_dump = self.read_dump(self.nodes[0].datadir_path / "rt-native_endian.dump") + self.assert_dump(expected_dump, native_dump) + + self.do_tool_createfromdump("other_endian", "endian.dump", "bdb_swap") + other_dump = self.read_dump(self.nodes[0].datadir_path / "rt-other_endian.dump") + self.assert_dump(expected_dump, other_dump) + + def test_dump_very_large_records(self): + self.log.info("Test that wallets with large records are successfully dumped") + + self.start_node(0) + self.nodes[0].createwallet("bigrecords") + wallet = self.nodes[0].get_wallet_rpc("bigrecords") + + # Both BDB and sqlite have maximum page sizes of 65536 bytes, with defaults of 4096 + # When a record exceeds some size threshold, both BDB and SQLite will store the data + # in one or more overflow pages. We want to make sure that our tooling can dump such + # records, even when they span multiple pages. To make a large record, we just need + # to make a very big transaction. + self.generate(self.nodes[0], 101) + def_wallet = self.nodes[0].get_wallet_rpc(self.default_wallet_name) + outputs = {} + for i in range(500): + outputs[wallet.getnewaddress(address_type="p2sh-segwit")] = 0.01 + def_wallet.sendmany(amounts=outputs) + self.generate(self.nodes[0], 1) + send_res = wallet.sendall([def_wallet.getnewaddress()]) + self.generate(self.nodes[0], 1) + assert_equal(send_res["complete"], True) + tx = wallet.gettransaction(txid=send_res["txid"], verbose=True) + assert_greater_than(tx["decoded"]["size"], 70000) + + self.stop_node(0) + + wallet_dump = self.nodes[0].datadir_path / "bigrecords.dump" + self.assert_tool_output("The dumpfile may contain private keys. To ensure the safety of your Bitcoin, do not share the dumpfile.\n", "-wallet=bigrecords", f"-dumpfile={wallet_dump}", "dump") + dump = self.read_dump(wallet_dump) + for k,v in dump.items(): + if tx["hex"] in v: + break + else: + assert False, "Big transaction was not found in wallet dump" + + def test_dump_unclean_lsns(self): + if not self.options.bdbro: + return + self.log.info("Test that a legacy wallet that has not been compacted is not dumped by bdbro") + + self.start_node(0, extra_args=["-flushwallet=0"]) + self.nodes[0].createwallet("unclean_lsn") + wallet = self.nodes[0].get_wallet_rpc("unclean_lsn") + # First unload and load normally to make sure everything is written + wallet.unloadwallet() + self.nodes[0].loadwallet("unclean_lsn") + # Next cause a bunch of writes by filling the keypool + wallet.keypoolrefill(wallet.getwalletinfo()["keypoolsize"] + 100) + # Lastly kill bitcoind so that the LSNs don't get reset + self.nodes[0].process.kill() + self.nodes[0].wait_until_stopped(expected_ret_code=1 if platform.system() == "Windows" else -9) + assert self.nodes[0].is_node_stopped() + + wallet_dump = self.nodes[0].datadir_path / "unclean_lsn.dump" + self.assert_raises_tool_error("LSNs are not reset, this database is not completely flushed. Please reopen then close the database with a version that has BDB support", "-wallet=unclean_lsn", f"-dumpfile={wallet_dump}", "dump") + + # File can be dumped after reload it normally + self.start_node(0) + self.nodes[0].loadwallet("unclean_lsn") + self.stop_node(0) + self.assert_tool_output("The dumpfile may contain private keys. To ensure the safety of your Bitcoin, do not share the dumpfile.\n", "-wallet=unclean_lsn", f"-dumpfile={wallet_dump}", "dump") + def run_test(self): self.wallet_path = self.nodes[0].wallets_path / self.default_wallet_name / self.wallet_data_filename self.test_invalid_tool_commands_and_args() @@ -462,8 +556,11 @@ class ToolWalletTest(BitcoinTestFramework): if not self.options.descriptors: # Salvage is a legacy wallet only thing self.test_salvage() + self.test_dump_endianness() + self.test_dump_unclean_lsns() self.test_dump_createfromdump() self.test_chainless_conflicts() + self.test_dump_very_large_records() if __name__ == '__main__': ToolWalletTest().main() diff --git a/test/functional/wallet_balance.py b/test/functional/wallet_balance.py index c322ae52c1..2c85773bf3 100755 --- a/test/functional/wallet_balance.py +++ b/test/functional/wallet_balance.py @@ -4,7 +4,6 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the wallet balance RPC methods.""" from decimal import Decimal -import struct from test_framework.address import ADDRESS_BCRT1_UNSPENDABLE as ADDRESS_WATCHONLY from test_framework.blocktools import COINBASE_MATURITY @@ -266,8 +265,8 @@ class WalletTest(BitcoinTestFramework): tx_orig = self.nodes[0].gettransaction(txid)['hex'] # Increase fee by 1 coin tx_replace = tx_orig.replace( - struct.pack("<q", 99 * 10**8).hex(), - struct.pack("<q", 98 * 10**8).hex(), + (99 * 10**8).to_bytes(8, "little", signed=True).hex(), + (98 * 10**8).to_bytes(8, "little", signed=True).hex(), ) tx_replace = self.nodes[0].signrawtransactionwithwallet(tx_replace)['hex'] # Total balance is given by the sum of outputs of the tx diff --git a/test/functional/wallet_basic.py b/test/functional/wallet_basic.py index 56228d2bad..1b2b8ec1f3 100755 --- a/test/functional/wallet_basic.py +++ b/test/functional/wallet_basic.py @@ -461,10 +461,13 @@ class WalletTest(BitcoinTestFramework): assert_raises_rpc_error(-5, "Invalid Bitcoin address or script", self.nodes[0].importaddress, "invalid") # This will raise an exception for attempting to import a pubkey that isn't in hex - assert_raises_rpc_error(-5, "Pubkey must be a hex string", self.nodes[0].importpubkey, "not hex") + assert_raises_rpc_error(-5, 'Pubkey "not hex" must be a hex string', self.nodes[0].importpubkey, "not hex") - # This will raise an exception for importing an invalid pubkey - assert_raises_rpc_error(-5, "Pubkey is not a valid public key", self.nodes[0].importpubkey, "5361746f736869204e616b616d6f746f") + # This will raise exceptions for importing a pubkeys with invalid length / invalid coordinates + too_short_pubkey = "5361746f736869204e616b616d6f746f" + assert_raises_rpc_error(-5, f'Pubkey "{too_short_pubkey}" must have a length of either 33 or 65 bytes', self.nodes[0].importpubkey, too_short_pubkey) + not_on_curve_pubkey = bytes([4] + [0]*64).hex() # pubkey with coordinates (0,0) is not on curve + assert_raises_rpc_error(-5, f'Pubkey "{not_on_curve_pubkey}" must be cryptographically valid', self.nodes[0].importpubkey, not_on_curve_pubkey) # Bech32m addresses cannot be imported into a legacy wallet assert_raises_rpc_error(-5, "Bech32m addresses cannot be imported into legacy wallets", self.nodes[0].importaddress, "bcrt1p0xlxvlhemja6c4dqv22uapctqupfhlxm9h8z3k2e72q4k9hcz7vqc8gma6") diff --git a/test/functional/wallet_bumpfee.py b/test/functional/wallet_bumpfee.py index 5b7db55f45..6d45adc823 100755 --- a/test/functional/wallet_bumpfee.py +++ b/test/functional/wallet_bumpfee.py @@ -117,6 +117,7 @@ class BumpFeeTest(BitcoinTestFramework): # Context independent tests test_feerate_checks_replaced_outputs(self, rbf_node, peer_node) + test_bumpfee_with_feerate_ignores_walletincrementalrelayfee(self, rbf_node, peer_node) def test_invalid_parameters(self, rbf_node, peer_node, dest_address): self.log.info('Test invalid parameters') @@ -816,7 +817,7 @@ def test_feerate_checks_replaced_outputs(self, rbf_node, peer_node): # Since the bumped tx will replace all of the outputs with a single output, we can estimate that its size will 31 * (len(outputs) - 1) bytes smaller tx_size = tx_details["decoded"]["vsize"] est_bumped_size = tx_size - (len(tx_details["decoded"]["vout"]) - 1) * 31 - inc_fee_rate = max(rbf_node.getmempoolinfo()["incrementalrelayfee"], Decimal(0.00005000)) # Wallet has a fixed incremental relay fee of 5 sat/vb + inc_fee_rate = rbf_node.getmempoolinfo()["incrementalrelayfee"] # RPC gives us fee as negative min_fee = (-tx_details["fee"] + get_fee(est_bumped_size, inc_fee_rate)) * Decimal(1e8) min_fee_rate = (min_fee / est_bumped_size).quantize(Decimal("1.000")) @@ -830,5 +831,27 @@ def test_feerate_checks_replaced_outputs(self, rbf_node, peer_node): self.clear_mempool() +def test_bumpfee_with_feerate_ignores_walletincrementalrelayfee(self, rbf_node, peer_node): + self.log.info('Test that bumpfee with fee_rate ignores walletincrementalrelayfee') + # Make sure there is enough balance + peer_node.sendtoaddress(rbf_node.getnewaddress(), 2) + self.generate(peer_node, 1) + + dest_address = peer_node.getnewaddress(address_type="bech32") + tx = rbf_node.send(outputs=[{dest_address: 1}], fee_rate=2) + + # Ensure you can not fee bump with a fee_rate below or equal to the original fee_rate + assert_raises_rpc_error(-8, "Insufficient total fee", rbf_node.bumpfee, tx["txid"], {"fee_rate": 1}) + assert_raises_rpc_error(-8, "Insufficient total fee", rbf_node.bumpfee, tx["txid"], {"fee_rate": 2}) + + # Ensure you can not fee bump if the fee_rate is more than original fee_rate but the total fee from new fee_rate is + # less than (original fee + incrementalrelayfee) + assert_raises_rpc_error(-8, "Insufficient total fee", rbf_node.bumpfee, tx["txid"], {"fee_rate": 2.8}) + + # You can fee bump as long as the new fee set from fee_rate is atleast (original fee + incrementalrelayfee) + rbf_node.bumpfee(tx["txid"], {"fee_rate": 3}) + self.clear_mempool() + + if __name__ == "__main__": BumpFeeTest().main() diff --git a/test/functional/wallet_conflicts.py b/test/functional/wallet_conflicts.py index e5739a6a59..25a95aa954 100755 --- a/test/functional/wallet_conflicts.py +++ b/test/functional/wallet_conflicts.py @@ -9,7 +9,6 @@ Test that wallet correctly tracks transactions that have been conflicted by bloc from decimal import Decimal -from test_framework.blocktools import COINBASE_MATURITY from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, @@ -37,7 +36,6 @@ class TxConflicts(BitcoinTestFramework): """ self.test_block_conflicts() - self.generatetoaddress(self.nodes[0], COINBASE_MATURITY + 7, self.nodes[2].getnewaddress()) self.test_mempool_conflict() self.test_mempool_and_block_conflicts() self.test_descendants_with_mempool_conflicts() diff --git a/test/functional/wallet_create_tx.py b/test/functional/wallet_create_tx.py index 4e31b48ec0..41ddb2bc69 100755 --- a/test/functional/wallet_create_tx.py +++ b/test/functional/wallet_create_tx.py @@ -3,6 +3,9 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. +from test_framework.messages import ( + tx_from_hex, +) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, @@ -33,6 +36,7 @@ class CreateTxWalletTest(BitcoinTestFramework): self.test_anti_fee_sniping() self.test_tx_size_too_large() self.test_create_too_long_mempool_chain() + self.test_version3() def test_anti_fee_sniping(self): self.log.info('Check that we have some (old) blocks and that anti-fee-sniping is disabled') @@ -106,6 +110,23 @@ class CreateTxWalletTest(BitcoinTestFramework): test_wallet.unloadwallet() + def test_version3(self): + self.log.info('Check wallet does not create transactions with version=3 yet') + wallet_rpc = self.nodes[0].get_wallet_rpc(self.default_wallet_name) + + self.nodes[0].createwallet("version3") + wallet_v3 = self.nodes[0].get_wallet_rpc("version3") + + tx_data = wallet_rpc.send(outputs=[{wallet_v3.getnewaddress(): 25}], options={"change_position": 0}) + wallet_tx_data = wallet_rpc.gettransaction(tx_data["txid"]) + tx_current_version = tx_from_hex(wallet_tx_data["hex"]) + + # While version=3 transactions are standard, the CURRENT_VERSION is 2. + # This test can be removed if CURRENT_VERSION is changed, and replaced with tests that the + # wallet handles TRUC rules properly. + assert_equal(tx_current_version.version, 2) + wallet_v3.unloadwallet() + if __name__ == '__main__': CreateTxWalletTest().main() diff --git a/test/functional/wallet_fundrawtransaction.py b/test/functional/wallet_fundrawtransaction.py index ff4648e638..3c1b2deb1d 100755 --- a/test/functional/wallet_fundrawtransaction.py +++ b/test/functional/wallet_fundrawtransaction.py @@ -114,6 +114,7 @@ class RawTransactionsTest(BitcoinTestFramework): self.test_add_inputs_default_value() self.test_preset_inputs_selection() self.test_weight_calculation() + self.test_weight_limits() self.test_change_position() self.test_simple() self.test_simple_two_coins() @@ -1054,8 +1055,8 @@ class RawTransactionsTest(BitcoinTestFramework): assert_raises_rpc_error(-4, "Not solvable pre-selected input COutPoint(%s, %s)" % (ext_utxo["txid"][0:10], ext_utxo["vout"]), wallet.fundrawtransaction, raw_tx) # Error conditions - assert_raises_rpc_error(-5, "'not a pubkey' is not hex", wallet.fundrawtransaction, raw_tx, solving_data={"pubkeys":["not a pubkey"]}) - assert_raises_rpc_error(-5, "'01234567890a0b0c0d0e0f' is not a valid public key", wallet.fundrawtransaction, raw_tx, solving_data={"pubkeys":["01234567890a0b0c0d0e0f"]}) + assert_raises_rpc_error(-5, 'Pubkey "not a pubkey" must be a hex string', wallet.fundrawtransaction, raw_tx, solving_data={"pubkeys":["not a pubkey"]}) + assert_raises_rpc_error(-5, 'Pubkey "01234567890a0b0c0d0e0f" must have a length of either 33 or 65 bytes', wallet.fundrawtransaction, raw_tx, solving_data={"pubkeys":["01234567890a0b0c0d0e0f"]}) assert_raises_rpc_error(-5, "'not a script' is not hex", wallet.fundrawtransaction, raw_tx, solving_data={"scripts":["not a script"]}) assert_raises_rpc_error(-8, "Unable to parse descriptor 'not a descriptor'", wallet.fundrawtransaction, raw_tx, solving_data={"descriptors":["not a descriptor"]}) assert_raises_rpc_error(-8, "Invalid parameter, missing vout key", wallet.fundrawtransaction, raw_tx, input_weights=[{"txid": ext_utxo["txid"]}]) @@ -1312,6 +1313,38 @@ class RawTransactionsTest(BitcoinTestFramework): self.nodes[2].unloadwallet("test_weight_calculation") + def test_weight_limits(self): + self.log.info("Test weight limits") + + self.nodes[2].createwallet("test_weight_limits") + wallet = self.nodes[2].get_wallet_rpc("test_weight_limits") + + outputs = [] + for _ in range(1472): + outputs.append({wallet.getnewaddress(address_type="legacy"): 0.1}) + txid = self.nodes[0].send(outputs=outputs)["txid"] + self.generate(self.nodes[0], 1) + + # 272 WU per input (273 when high-s); picking 1471 inputs will exceed the max standard tx weight. + rawtx = wallet.createrawtransaction([], [{wallet.getnewaddress(): 0.1 * 1471}]) + + # 1) Try to fund transaction only using the preset inputs + input_weights = [] + for i in range(1471): + input_weights.append({"txid": txid, "vout": i, "weight": 273}) + assert_raises_rpc_error(-4, "Transaction too large", wallet.fundrawtransaction, hexstring=rawtx, input_weights=input_weights) + + # 2) Let the wallet fund the transaction + assert_raises_rpc_error(-4, "The inputs size exceeds the maximum weight. Please try sending a smaller amount or manually consolidating your wallet's UTXOs", + wallet.fundrawtransaction, hexstring=rawtx) + + # 3) Pre-select some inputs and let the wallet fill-up the remaining amount + inputs = input_weights[0:1000] + assert_raises_rpc_error(-4, "The combination of the pre-selected inputs and the wallet automatic inputs selection exceeds the transaction maximum weight. Please try sending a smaller amount or manually consolidating your wallet's UTXOs", + wallet.fundrawtransaction, hexstring=rawtx, input_weights=inputs) + + self.nodes[2].unloadwallet("test_weight_limits") + def test_include_unsafe(self): self.log.info("Test fundrawtxn with unsafe inputs") diff --git a/test/functional/wallet_listsinceblock.py b/test/functional/wallet_listsinceblock.py index fd586d546e..15214539a9 100755 --- a/test/functional/wallet_listsinceblock.py +++ b/test/functional/wallet_listsinceblock.py @@ -40,6 +40,7 @@ class ListSinceBlockTest(BitcoinTestFramework): self.test_no_blockhash() self.test_invalid_blockhash() self.test_reorg() + self.test_cant_read_block() self.test_double_spend() self.test_double_send() self.double_spends_filtered() @@ -167,6 +168,31 @@ class ListSinceBlockTest(BitcoinTestFramework): found = next(tx for tx in transactions if tx['txid'] == senttx) assert_equal(found['blockheight'], self.nodes[0].getblockheader(nodes2_first_blockhash)['height']) + def test_cant_read_block(self): + self.log.info('Test the RPC error "Can\'t read block from disk"') + + # Split network into two + self.split_network() + + # generate on both sides + nodes1_last_blockhash = self.generate(self.nodes[1], 6, sync_fun=lambda: self.sync_all(self.nodes[:2]))[-1] + self.generate(self.nodes[2], 7, sync_fun=lambda: self.sync_all(self.nodes[2:]))[0] + + self.join_network() + + # Renaming the block file to induce unsuccessful block read + blk_dat = (self.nodes[0].blocks_path / "blk00000.dat") + blk_dat_moved = blk_dat.rename(self.nodes[0].blocks_path / "blk00000.dat.moved") + assert not blk_dat.exists() + + # listsinceblock(nodes1_last_blockhash) should now fail as blocks are not accessible + assert_raises_rpc_error(-32603, "Can't read block from disk", + self.nodes[0].listsinceblock, nodes1_last_blockhash) + + # Restoring block file + blk_dat_moved.rename(self.nodes[0].blocks_path / "blk00000.dat") + assert blk_dat.exists() + def test_double_spend(self): ''' This tests the case where the same UTXO is spent twice on two separate diff --git a/test/functional/wallet_multisig_descriptor_psbt.py b/test/functional/wallet_multisig_descriptor_psbt.py index 68bf45f7e3..145912025f 100755 --- a/test/functional/wallet_multisig_descriptor_psbt.py +++ b/test/functional/wallet_multisig_descriptor_psbt.py @@ -7,7 +7,6 @@ This is meant to be documentation as much as functional tests, so it is kept as simple and readable as possible. """ -from test_framework.address import base58_to_byte from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_approx, @@ -30,10 +29,12 @@ class WalletMultisigDescriptorPSBTTest(BitcoinTestFramework): self.skip_if_no_sqlite() @staticmethod - def _get_xpub(wallet): + def _get_xpub(wallet, internal): """Extract the wallet's xpubs using `listdescriptors` and pick the one from the `pkh` descriptor since it's least likely to be accidentally reused (legacy addresses).""" - descriptor = next(filter(lambda d: d["desc"].startswith("pkh"), wallet.listdescriptors()["descriptors"])) - return descriptor["desc"].split("]")[-1].split("/")[0] + pkh_descriptor = next(filter(lambda d: d["desc"].startswith("pkh(") and d["internal"] == internal, wallet.listdescriptors()["descriptors"])) + # Keep all key origin information (master key fingerprint and all derivation steps) for proper support of hardware devices + # See section 'Key origin identification' in 'doc/descriptors.md' for more details... + return pkh_descriptor["desc"].split("pkh(")[1].split(")")[0] @staticmethod def _check_psbt(psbt, to, value, multisig): @@ -47,19 +48,13 @@ class WalletMultisigDescriptorPSBTTest(BitcoinTestFramework): amount += vout["value"] assert_approx(amount, float(value), vspan=0.001) - def participants_create_multisigs(self, xpubs): + def participants_create_multisigs(self, external_xpubs, internal_xpubs): """The multisig is created by importing the following descriptors. The resulting wallet is watch-only and every participant can do this.""" - # some simple validation - assert_equal(len(xpubs), self.N) - # a sanity-check/assertion, this will throw if the base58 checksum of any of the provided xpubs are invalid - for xpub in xpubs: - base58_to_byte(xpub) - for i, node in enumerate(self.nodes): node.createwallet(wallet_name=f"{self.name}_{i}", blank=True, descriptors=True, disable_private_keys=True) multisig = node.get_wallet_rpc(f"{self.name}_{i}") - external = multisig.getdescriptorinfo(f"wsh(sortedmulti({self.M},{f'/0/*,'.join(xpubs)}/0/*))") - internal = multisig.getdescriptorinfo(f"wsh(sortedmulti({self.M},{f'/1/*,'.join(xpubs)}/1/*))") + external = multisig.getdescriptorinfo(f"wsh(sortedmulti({self.M},{f','.join(external_xpubs)}))") + internal = multisig.getdescriptorinfo(f"wsh(sortedmulti({self.M},{f','.join(internal_xpubs)}))") result = multisig.importdescriptors([ { # receiving addresses (internal: False) "desc": external["descriptor"], @@ -93,10 +88,10 @@ class WalletMultisigDescriptorPSBTTest(BitcoinTestFramework): } self.log.info("Generate and exchange xpubs...") - xpubs = [self._get_xpub(signer) for signer in participants["signers"]] + external_xpubs, internal_xpubs = [[self._get_xpub(signer, internal) for signer in participants["signers"]] for internal in [False, True]] self.log.info("Every participant imports the following descriptors to create the watch-only multisig...") - participants["multisigs"] = list(self.participants_create_multisigs(xpubs)) + participants["multisigs"] = list(self.participants_create_multisigs(external_xpubs, internal_xpubs)) self.log.info("Check that every participant's multisig generates the same addresses...") for _ in range(10): # we check that the first 10 generated addresses are the same for all participant's multisigs diff --git a/test/functional/wallet_send.py b/test/functional/wallet_send.py index 0a0a8dba0d..bbb0d658d9 100755 --- a/test/functional/wallet_send.py +++ b/test/functional/wallet_send.py @@ -577,5 +577,39 @@ class WalletSendTest(BitcoinTestFramework): # but rounded to nearest integer, it should be the same as the target fee rate assert_equal(round(actual_fee_rate_sat_vb), target_fee_rate_sat_vb) + # Check tx creation size limits + self.test_weight_limits() + + def test_weight_limits(self): + self.log.info("Test weight limits") + + self.nodes[1].createwallet("test_weight_limits") + wallet = self.nodes[1].get_wallet_rpc("test_weight_limits") + + # Generate future inputs; 272 WU per input (273 when high-s). + # Picking 1471 inputs will exceed the max standard tx weight. + outputs = [] + for _ in range(1472): + outputs.append({wallet.getnewaddress(address_type="legacy"): 0.1}) + self.nodes[0].send(outputs=outputs) + self.generate(self.nodes[0], 1) + + # 1) Try to fund transaction only using the preset inputs + inputs = wallet.listunspent() + assert_raises_rpc_error(-4, "Transaction too large", + wallet.send, outputs=[{wallet.getnewaddress(): 0.1 * 1471}], options={"inputs": inputs, "add_inputs": False}) + + # 2) Let the wallet fund the transaction + assert_raises_rpc_error(-4, "The inputs size exceeds the maximum weight. Please try sending a smaller amount or manually consolidating your wallet's UTXOs", + wallet.send, outputs=[{wallet.getnewaddress(): 0.1 * 1471}]) + + # 3) Pre-select some inputs and let the wallet fill-up the remaining amount + inputs = inputs[0:1000] + assert_raises_rpc_error(-4, "The combination of the pre-selected inputs and the wallet automatic inputs selection exceeds the transaction maximum weight. Please try sending a smaller amount or manually consolidating your wallet's UTXOs", + wallet.send, outputs=[{wallet.getnewaddress(): 0.1 * 1471}], options={"inputs": inputs, "add_inputs": True}) + + self.nodes[1].unloadwallet("test_weight_limits") + + if __name__ == '__main__': WalletSendTest().main() diff --git a/test/functional/wallet_sendall.py b/test/functional/wallet_sendall.py index c2b800df21..1d308c225d 100755 --- a/test/functional/wallet_sendall.py +++ b/test/functional/wallet_sendall.py @@ -379,6 +379,64 @@ class SendallTest(BitcoinTestFramework): assert_equal(len(self.wallet.listunspent()), 1) assert_equal(self.wallet.listunspent()[0]['confirmations'], 6) + @cleanup + def sendall_spends_unconfirmed_change(self): + self.log.info("Test that sendall spends unconfirmed change") + self.add_utxos([17]) + self.wallet.sendtoaddress(self.remainder_target, 10) + assert_greater_than(self.wallet.getbalances()["mine"]["trusted"], 6) + self.test_sendall_success(sendall_args = [self.remainder_target]) + + assert_equal(self.wallet.getbalance(), 0) + + @cleanup + def sendall_spends_unconfirmed_inputs_if_specified(self): + self.log.info("Test that sendall spends specified unconfirmed inputs") + self.def_wallet.sendtoaddress(self.wallet.getnewaddress(), 17) + self.wallet.syncwithvalidationinterfacequeue() + assert_equal(self.wallet.getbalances()["mine"]["untrusted_pending"], 17) + unspent = self.wallet.listunspent(minconf=0)[0] + + self.wallet.sendall(recipients=[self.remainder_target], inputs=[unspent]) + assert_equal(self.wallet.getbalance(), 0) + + @cleanup + def sendall_does_ancestor_aware_funding(self): + self.log.info("Test that sendall does ancestor aware funding for unconfirmed inputs") + + # higher parent feerate + self.def_wallet.sendtoaddress(address=self.wallet.getnewaddress(), amount=17, fee_rate=20) + self.wallet.syncwithvalidationinterfacequeue() + + assert_equal(self.wallet.getbalances()["mine"]["untrusted_pending"], 17) + unspent = self.wallet.listunspent(minconf=0)[0] + + parent_txid = unspent["txid"] + assert_equal(self.wallet.gettransaction(parent_txid)["confirmations"], 0) + + res_1 = self.wallet.sendall(recipients=[self.def_wallet.getnewaddress()], inputs=[unspent], fee_rate=20, add_to_wallet=False, lock_unspents=True) + child_hex = res_1["hex"] + + child_tx = self.wallet.decoderawtransaction(child_hex) + higher_parent_feerate_amount = child_tx["vout"][0]["value"] + + # lower parent feerate + self.def_wallet.sendtoaddress(address=self.wallet.getnewaddress(), amount=17, fee_rate=10) + self.wallet.syncwithvalidationinterfacequeue() + assert_equal(self.wallet.getbalances()["mine"]["untrusted_pending"], 34) + unspent = self.wallet.listunspent(minconf=0)[0] + + parent_txid = unspent["txid"] + assert_equal(self.wallet.gettransaction(parent_txid)["confirmations"], 0) + + res_2 = self.wallet.sendall(recipients=[self.def_wallet.getnewaddress()], inputs=[unspent], fee_rate=20, add_to_wallet=False, lock_unspents=True) + child_hex = res_2["hex"] + + child_tx = self.wallet.decoderawtransaction(child_hex) + lower_parent_feerate_amount = child_tx["vout"][0]["value"] + + assert_greater_than(higher_parent_feerate_amount, lower_parent_feerate_amount) + # This tests needs to be the last one otherwise @cleanup will fail with "Transaction too large" error def sendall_fails_with_transaction_too_large(self): self.log.info("Test that sendall fails if resulting transaction is too large") @@ -460,6 +518,15 @@ class SendallTest(BitcoinTestFramework): # Sendall only uses outputs with less than a given number of confirmation when using minconf self.sendall_with_maxconf() + # Sendall spends unconfirmed change + self.sendall_spends_unconfirmed_change() + + # Sendall spends unconfirmed inputs if they are specified + self.sendall_spends_unconfirmed_inputs_if_specified() + + # Sendall does ancestor aware funding when spending an unconfirmed UTXO + self.sendall_does_ancestor_aware_funding() + # Sendall fails when many inputs result to too large transaction self.sendall_fails_with_transaction_too_large() diff --git a/test/fuzz/test_runner.py b/test/fuzz/test_runner.py index a635175e7c..c74246ef45 100755 --- a/test/fuzz/test_runner.py +++ b/test/fuzz/test_runner.py @@ -215,12 +215,12 @@ def transform_process_message_target(targets, src_dir): p2p_msg_target = "process_message" if (p2p_msg_target, {}) in targets: lines = subprocess.run( - ["git", "grep", "--function-context", "g_all_net_message_types{", src_dir / "src" / "protocol.cpp"], + ["git", "grep", "--function-context", "ALL_NET_MESSAGE_TYPES{", src_dir / "src" / "protocol.h"], check=True, stdout=subprocess.PIPE, text=True, ).stdout.splitlines() - lines = [l.split("::", 1)[1].split(",")[0].lower() for l in lines if l.startswith("src/protocol.cpp- NetMsgType::")] + lines = [l.split("::", 1)[1].split(",")[0].lower() for l in lines if l.startswith("src/protocol.h- NetMsgType::")] assert len(lines) targets += [(p2p_msg_target, {"LIMIT_TO_MESSAGE_TYPE": m}) for m in lines] return targets diff --git a/test/lint/README.md b/test/lint/README.md index 9d167bac72..04a836c4d2 100644 --- a/test/lint/README.md +++ b/test/lint/README.md @@ -26,17 +26,33 @@ Then you can use: ( cd ./test/lint/test_runner/ && cargo fmt && cargo clippy && RUST_BACKTRACE=1 cargo run ) ``` +If you wish to run individual lint checks, run the test_runner with +`--lint=TEST_TO_RUN` arguments. If running with `cargo run`, arguments after +`--` are passed to the binary you are running e.g.: + +```sh +( cd ./test/lint/test_runner/ && RUST_BACKTRACE=1 cargo run -- --lint=doc --lint=trailing_whitespace ) +``` + +To see a list of all individual lint checks available in test_runner, use `-h` +or `--help`: + +```sh +( cd ./test/lint/test_runner/ && RUST_BACKTRACE=1 cargo run -- --help ) +``` + #### Dependencies | Lint test | Dependency | |-----------|:----------:| -| [`lint-python.py`](lint/lint-python.py) | [flake8](https://gitlab.com/pycqa/flake8) -| [`lint-python.py`](lint/lint-python.py) | [lief](https://github.com/lief-project/LIEF) -| [`lint-python.py`](lint/lint-python.py) | [mypy](https://github.com/python/mypy) -| [`lint-python.py`](lint/lint-python.py) | [pyzmq](https://github.com/zeromq/pyzmq) -| [`lint-python-dead-code.py`](lint/lint-python-dead-code.py) | [vulture](https://github.com/jendrikseipp/vulture) -| [`lint-shell.py`](lint/lint-shell.py) | [ShellCheck](https://github.com/koalaman/shellcheck) -| [`lint-spelling.py`](lint/lint-spelling.py) | [codespell](https://github.com/codespell-project/codespell) +| [`lint-python.py`](/test/lint/lint-python.py) | [flake8](https://github.com/PyCQA/flake8) +| [`lint-python.py`](/test/lint/lint-python.py) | [lief](https://github.com/lief-project/LIEF) +| [`lint-python.py`](/test/lint/lint-python.py) | [mypy](https://github.com/python/mypy) +| [`lint-python.py`](/test/lint/lint-python.py) | [pyzmq](https://github.com/zeromq/pyzmq) +| [`lint-python-dead-code.py`](/test/lint/lint-python-dead-code.py) | [vulture](https://github.com/jendrikseipp/vulture) +| [`lint-shell.py`](/test/lint/lint-shell.py) | [ShellCheck](https://github.com/koalaman/shellcheck) +| [`lint-spelling.py`](/test/lint/lint-spelling.py) | [codespell](https://github.com/codespell-project/codespell) +| markdown link check | [mlc](https://github.com/becheran/mlc) In use versions and install instructions are available in the [CI setup](../../ci/lint/04_install.sh). diff --git a/test/lint/test_runner/src/main.rs b/test/lint/test_runner/src/main.rs index e22e047e4b..2ba58a6da2 100644 --- a/test/lint/test_runner/src/main.rs +++ b/test/lint/test_runner/src/main.rs @@ -4,14 +4,121 @@ use std::env; use std::fs; +use std::io::ErrorKind; use std::path::{Path, PathBuf}; -use std::process::Command; -use std::process::ExitCode; +use std::process::{Command, ExitCode, Stdio}; type LintError = String; type LintResult = Result<(), LintError>; type LintFn = fn() -> LintResult; +struct Linter { + pub description: &'static str, + pub name: &'static str, + pub lint_fn: LintFn, +} + +fn get_linter_list() -> Vec<&'static Linter> { + vec![ + &Linter { + description: "Check that all command line arguments are documented.", + name: "doc", + lint_fn: lint_doc + }, + &Linter { + description: "Check that no symbol from bitcoin-config.h is used without the header being included", + name: "includes_build_config", + lint_fn: lint_includes_build_config + }, + &Linter { + description: "Check that markdown links resolve", + name: "markdown", + lint_fn: lint_markdown + }, + &Linter { + description: "Check that std::filesystem is not used directly", + name: "std_filesystem", + lint_fn: lint_std_filesystem + }, + &Linter { + description: "Check that subtrees are pure subtrees", + name: "subtree", + lint_fn: lint_subtree + }, + &Linter { + description: "Check that tabs are not used as whitespace", + name: "tabs_whitespace", + lint_fn: lint_tabs_whitespace + }, + &Linter { + description: "Check for trailing whitespace", + name: "trailing_whitespace", + lint_fn: lint_trailing_whitespace + }, + &Linter { + description: "Run all linters of the form: test/lint/lint-*.py", + name: "all_python_linters", + lint_fn: run_all_python_linters + }, + ] +} + +fn print_help_and_exit() { + print!( + r#" +Usage: test_runner [--lint=LINTER_TO_RUN] +Runs all linters in the lint test suite, printing any errors +they detect. + +If you wish to only run some particular lint tests, pass +'--lint=' with the name of the lint test you wish to run. +You can set as many '--lint=' values as you wish, e.g.: +test_runner --lint=doc --lint=subtree + +The individual linters available to run are: +"# + ); + for linter in get_linter_list() { + println!("{}: \"{}\"", linter.name, linter.description) + } + + std::process::exit(1); +} + +fn parse_lint_args(args: &[String]) -> Vec<&'static Linter> { + let linter_list = get_linter_list(); + let mut lint_values = Vec::new(); + + for arg in args { + #[allow(clippy::if_same_then_else)] + if arg.starts_with("--lint=") { + let lint_arg_value = arg + .trim_start_matches("--lint=") + .trim_matches('"') + .trim_matches('\''); + + let try_find_linter = linter_list + .iter() + .find(|linter| linter.name == lint_arg_value); + match try_find_linter { + Some(linter) => { + lint_values.push(*linter); + } + None => { + println!("No linter {lint_arg_value} found!"); + print_help_and_exit(); + } + } + } else if arg.eq("--help") || arg.eq("-h") { + print_help_and_exit(); + } else { + print_help_and_exit(); + } + } + + lint_values +} + /// Return the git command fn git() -> Command { let mut git = Command::new("git"); @@ -137,9 +244,9 @@ fn lint_trailing_whitespace() -> LintResult { if trailing_space { Err(r#" ^^^ -Trailing whitespace is problematic, because git may warn about it, or editors may remove it by -default, forcing developers in the future to either undo the changes manually or spend time on -review. +Trailing whitespace (including Windows line endings [CR LF]) is problematic, because git may warn +about it, or editors may remove it by default, forcing developers in the future to either undo the +changes manually or spend time on review. Thus, it is best to remove the trailing space now. @@ -178,7 +285,6 @@ Please add any false positives, such as subtrees, or externally sourced files to fn lint_includes_build_config() -> LintResult { let config_path = "./src/config/bitcoin-config.h.in"; - let include_directive = "#include <config/bitcoin-config.h>"; if !Path::new(config_path).is_file() { assert!(Command::new("./autogen.sh") .status() @@ -235,7 +341,11 @@ fn lint_includes_build_config() -> LintResult { } else { "--files-with-matches" }, - include_directive, + if mode { + "^#include <config/bitcoin-config.h> // IWYU pragma: keep$" + } else { + "#include <config/bitcoin-config.h>" // Catch redundant includes with and without the IWYU pragma + }, "--", ]) .args(defines_files.lines()) @@ -256,6 +366,11 @@ even though bitcoin-config.h indicates that a faster feature is available and sh If you are unsure which symbol is used, you can find it with this command: git grep --perl-regexp '{}' -- file_name + +Make sure to include it with the IWYU pragma. Otherwise, IWYU may falsely instruct to remove the +include again. + +#include <config/bitcoin-config.h> // IWYU pragma: keep "#, defines_regex )); @@ -284,7 +399,48 @@ fn lint_doc() -> LintResult { } } -fn lint_all() -> LintResult { +fn lint_markdown() -> LintResult { + let bin_name = "mlc"; + let mut md_ignore_paths = get_subtrees(); + md_ignore_paths.push("./doc/README_doxygen.md"); + let md_ignore_path_str = md_ignore_paths.join(","); + + let mut cmd = Command::new(bin_name); + cmd.args([ + "--offline", + "--ignore-path", + md_ignore_path_str.as_str(), + "--gitignore", + "--root-dir", + ".", + ]) + .stdout(Stdio::null()); // Suppress overly-verbose output + + match cmd.output() { + Ok(output) if output.status.success() => Ok(()), + Ok(output) => { + let stderr = String::from_utf8_lossy(&output.stderr); + Err(format!( + r#" +One or more markdown links are broken. + +Relative links are preferred (but not required) as jumping to file works natively within Emacs. + +Markdown link errors found: +{} + "#, + stderr + )) + } + Err(e) if e.kind() == ErrorKind::NotFound => { + println!("`mlc` was not found in $PATH, skipping markdown lint check."); + Ok(()) + } + Err(e) => Err(format!("Error running mlc: {}", e)), // Misc errors + } +} + +fn run_all_python_linters() -> LintResult { let mut good = true; let lint_dir = get_git_root().join("test/lint"); for entry in fs::read_dir(lint_dir).unwrap() { @@ -299,7 +455,7 @@ fn lint_all() -> LintResult { .success() { good = false; - println!("^---- failure generated from {}", entry_fn); + println!("^---- ⚠️ Failure generated from {}", entry_fn); } } if good { @@ -310,24 +466,26 @@ fn lint_all() -> LintResult { } fn main() -> ExitCode { - let test_list: Vec<(&str, LintFn)> = vec![ - ("subtree check", lint_subtree), - ("std::filesystem check", lint_std_filesystem), - ("trailing whitespace check", lint_trailing_whitespace), - ("no-tabs check", lint_tabs_whitespace), - ("build config includes check", lint_includes_build_config), - ("-help=1 documentation check", lint_doc), - ("lint-*.py scripts", lint_all), - ]; + let linters_to_run: Vec<&Linter> = if env::args().count() > 1 { + let args: Vec<String> = env::args().skip(1).collect(); + parse_lint_args(&args) + } else { + // If no arguments are passed, run all linters. + get_linter_list() + }; let git_root = get_git_root(); let mut test_failed = false; - for (lint_name, lint_fn) in test_list { + for linter in linters_to_run { // chdir to root before each lint test env::set_current_dir(&git_root).unwrap(); - if let Err(err) = lint_fn() { - println!("{err}\n^---- ⚠️ Failure generated from {lint_name}!"); + if let Err(err) = (linter.lint_fn)() { + println!( + "{err}\n^---- ⚠️ Failure generated from lint check '{}'!", + linter.name + ); + println!("{}", linter.description); test_failed = true; } } diff --git a/test/sanitizer_suppressions/ubsan b/test/sanitizer_suppressions/ubsan index 482667a26a..94bd14e6c3 100644 --- a/test/sanitizer_suppressions/ubsan +++ b/test/sanitizer_suppressions/ubsan @@ -51,18 +51,18 @@ unsigned-integer-overflow:CCoinsViewCache::Uncache unsigned-integer-overflow:CompressAmount unsigned-integer-overflow:DecompressAmount unsigned-integer-overflow:crypto/ -unsigned-integer-overflow:getchaintxstats* unsigned-integer-overflow:MurmurHash3 unsigned-integer-overflow:CBlockPolicyEstimator::processBlockTx unsigned-integer-overflow:TxConfirmStats::EstimateMedianVal unsigned-integer-overflow:prevector.h unsigned-integer-overflow:EvalScript -unsigned-integer-overflow:xoroshiro128plusplus.h +unsigned-integer-overflow:InsecureRandomContext::rand64 +unsigned-integer-overflow:InsecureRandomContext::SplitMix64 +unsigned-integer-overflow:bitset_detail::PopCount implicit-integer-sign-change:CBlockPolicyEstimator::processBlockTx implicit-integer-sign-change:SetStdinEcho implicit-integer-sign-change:compressor.h implicit-integer-sign-change:crypto/ -implicit-integer-sign-change:getchaintxstats* implicit-integer-sign-change:TxConfirmStats::removeTx implicit-integer-sign-change:prevector.h implicit-integer-sign-change:verify_flags @@ -74,4 +74,6 @@ shift-base:arith_uint256.cpp shift-base:crypto/ shift-base:streams.h shift-base:FormatHDKeypath -shift-base:xoroshiro128plusplus.h +shift-base:InsecureRandomContext::rand64 +shift-base:RandomMixin<*>::randbits +shift-base:RandomMixin<*>::randbits<*> |