diff options
Diffstat (limited to 'test')
131 files changed, 2955 insertions, 902 deletions
diff --git a/test/README.md b/test/README.md index fdbb91832a..0eddb72e1f 100644 --- a/test/README.md +++ b/test/README.md @@ -109,34 +109,57 @@ how many jobs to run, append `--jobs=n` The individual tests and the test_runner harness have many command-line options. Run `test/functional/test_runner.py -h` to see them all. -#### Speed up test runs with a ramdisk +#### Speed up test runs with a RAM disk -If you have available RAM on your system you can create a ramdisk to use as the `cache` and `tmp` directories for the functional tests in order to speed them up. -Speed-up amount varies on each system (and according to your ram speed and other variables), but a 2-3x speed-up is not uncommon. +If you have available RAM on your system you can create a RAM disk to use as the `cache` and `tmp` directories for the functional tests in order to speed them up. +Speed-up amount varies on each system (and according to your RAM speed and other variables), but a 2-3x speed-up is not uncommon. -To create a 4GB ramdisk on Linux at `/mnt/tmp/`: +**Linux** + +To create a 4 GiB RAM disk at `/mnt/tmp/`: ```bash sudo mkdir -p /mnt/tmp sudo mount -t tmpfs -o size=4g tmpfs /mnt/tmp/ ``` -Configure the size of the ramdisk using the `size=` option. -The size of the ramdisk needed is relative to the number of concurrent jobs the test suite runs. -For example running the test suite with `--jobs=100` might need a 4GB ramdisk, but running with `--jobs=32` will only need a 2.5GB ramdisk. +Configure the size of the RAM disk using the `size=` option. +The size of the RAM disk needed is relative to the number of concurrent jobs the test suite runs. +For example running the test suite with `--jobs=100` might need a 4 GiB RAM disk, but running with `--jobs=32` will only need a 2.5 GiB RAM disk. -To use, run the test suite specifying the ramdisk as the `cachedir` and `tmpdir`: +To use, run the test suite specifying the RAM disk as the `cachedir` and `tmpdir`: ```bash test/functional/test_runner.py --cachedir=/mnt/tmp/cache --tmpdir=/mnt/tmp ``` -Once finished with the tests and the disk, and to free the ram, simply unmount the disk: +Once finished with the tests and the disk, and to free the RAM, simply unmount the disk: ```bash sudo umount /mnt/tmp ``` +**macOS** + +To create a 4 GiB RAM disk named "ramdisk" at `/Volumes/ramdisk/`: + +```bash +diskutil erasevolume HFS+ ramdisk $(hdiutil attach -nomount ram://8388608) +``` + +Configure the RAM disk size, expressed as the number of blocks, at the end of the command +(`4096 MiB * 2048 blocks/MiB = 8388608 blocks` for 4 GiB). To run the tests using the RAM disk: + +```bash +test/functional/test_runner.py --cachedir=/Volumes/ramdisk/cache --tmpdir=/Volumes/ramdisk/tmp +``` + +To unmount: + +```bash +umount /Volumes/ramdisk +``` + #### Troubleshooting and debugging test failures ##### Resource contention diff --git a/test/functional/data/rpc_decodescript.json b/test/functional/data/rpc_decodescript.json index 4a15ae8792..5f3e725d4c 100644 --- a/test/functional/data/rpc_decodescript.json +++ b/test/functional/data/rpc_decodescript.json @@ -69,7 +69,7 @@ "p2sh": "2N34iiGoUUkVSPiaaTFpJjB1FR9TXQu3PGM", "segwit": { "asm": "0 96c2368fc30514a438a8bd909f93c49a1549d77198ccbdb792043b666cb24f42", - "desc": "addr(bcrt1qjmprdr7rq522gw9ghkgfly7yng25n4m3nrxtmdujqsakvm9jfapqk795l5)#5akkdska", + "desc": "wsh(raw(02eeee))#gtay4y0z", "hex": "002096c2368fc30514a438a8bd909f93c49a1549d77198ccbdb792043b666cb24f42", "address": "bcrt1qjmprdr7rq522gw9ghkgfly7yng25n4m3nrxtmdujqsakvm9jfapqk795l5", "type": "witness_v0_scripthash", diff --git a/test/functional/feature_abortnode.py b/test/functional/feature_abortnode.py index 32cf4a47f4..fa1bb6506a 100755 --- a/test/functional/feature_abortnode.py +++ b/test/functional/feature_abortnode.py @@ -19,7 +19,6 @@ class AbortNodeTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 2 - self.rpc_timeout = 240 def setup_network(self): self.setup_nodes() @@ -41,7 +40,7 @@ class AbortNodeTest(BitcoinTestFramework): # Check that node0 aborted self.log.info("Waiting for crash") - self.nodes[0].wait_until_stopped(timeout=200) + self.nodes[0].wait_until_stopped(timeout=5) self.log.info("Node crashed - now verifying restart fails") self.nodes[0].assert_start_raises_init_error() diff --git a/test/functional/feature_bip68_sequence.py b/test/functional/feature_bip68_sequence.py index d63821fbbc..894afffc79 100755 --- a/test/functional/feature_bip68_sequence.py +++ b/test/functional/feature_bip68_sequence.py @@ -32,6 +32,7 @@ from test_framework.util import ( assert_raises_rpc_error, softfork_active, ) +from test_framework.wallet import MiniWallet SCRIPT_W0_SH_OP_TRUE = script_to_p2wsh_script(CScript([OP_TRUE])) @@ -58,14 +59,9 @@ class BIP68Test(BitcoinTestFramework): ], ] - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() - def run_test(self): self.relayfee = self.nodes[0].getnetworkinfo()["relayfee"] - - # Generate some coins - self.generate(self.nodes[0], 110) + self.wallet = MiniWallet(self.nodes[0]) self.log.info("Running test disable flag") self.test_disable_flag() @@ -92,16 +88,10 @@ class BIP68Test(BitcoinTestFramework): # the first sequence bit is set. def test_disable_flag(self): # Create some unconfirmed inputs - new_addr = self.nodes[0].getnewaddress() - self.nodes[0].sendtoaddress(new_addr, 2) # send 2 BTC - - utxos = self.nodes[0].listunspent(0, 0) - assert len(utxos) > 0 - - utxo = utxos[0] + utxo = self.wallet.send_self_transfer(from_node=self.nodes[0])["new_utxo"] tx1 = CTransaction() - value = int((utxo["amount"] - self.relayfee) * COIN) + value = int((utxo["value"] - self.relayfee) * COIN) # Check that the disable flag disables relative locktime. # If sequence locks were used, this would require 1 block for the @@ -110,8 +100,8 @@ class BIP68Test(BitcoinTestFramework): tx1.vin = [CTxIn(COutPoint(int(utxo["txid"], 16), utxo["vout"]), nSequence=sequence_value)] tx1.vout = [CTxOut(value, SCRIPT_W0_SH_OP_TRUE)] - tx1_signed = self.nodes[0].signrawtransactionwithwallet(tx1.serialize().hex())["hex"] - tx1_id = self.nodes[0].sendrawtransaction(tx1_signed) + self.wallet.sign_tx(tx=tx1) + tx1_id = self.wallet.sendrawtransaction(from_node=self.nodes[0], tx_hex=tx1.serialize().hex()) tx1_id = int(tx1_id, 16) # This transaction will enable sequence-locks, so this transaction should @@ -125,13 +115,13 @@ class BIP68Test(BitcoinTestFramework): tx2.vout = [CTxOut(int(value - self.relayfee * COIN), SCRIPT_W0_SH_OP_TRUE)] tx2.rehash() - assert_raises_rpc_error(-26, NOT_FINAL_ERROR, self.nodes[0].sendrawtransaction, tx2.serialize().hex()) + assert_raises_rpc_error(-26, NOT_FINAL_ERROR, self.wallet.sendrawtransaction, from_node=self.nodes[0], tx_hex=tx2.serialize().hex()) # Setting the version back down to 1 should disable the sequence lock, # so this should be accepted. tx2.nVersion = 1 - self.nodes[0].sendrawtransaction(tx2.serialize().hex()) + self.wallet.sendrawtransaction(from_node=self.nodes[0], tx_hex=tx2.serialize().hex()) # Calculate the median time past of a prior block ("confirmations" before # the current tip). @@ -144,20 +134,13 @@ class BIP68Test(BitcoinTestFramework): # Create lots of confirmed utxos, and use them to generate lots of random # transactions. max_outputs = 50 - addresses = [] - while len(addresses) < max_outputs: - addresses.append(self.nodes[0].getnewaddress()) - while len(self.nodes[0].listunspent()) < 200: + while len(self.wallet.get_utxos(include_immature_coinbase=False, mark_as_spent=False)) < 200: import random - random.shuffle(addresses) num_outputs = random.randint(1, max_outputs) - outputs = {} - for i in range(num_outputs): - outputs[addresses[i]] = random.randint(1, 20)*0.01 - self.nodes[0].sendmany("", outputs) - self.generate(self.nodes[0], 1) + self.wallet.send_self_transfer_multi(from_node=self.nodes[0], num_outputs=num_outputs) + self.generate(self.wallet, 1) - utxos = self.nodes[0].listunspent() + utxos = self.wallet.get_utxos(include_immature_coinbase=False) # Try creating a lot of random transactions. # Each time, choose a random number of inputs, and randomly set @@ -214,19 +197,20 @@ class BIP68Test(BitcoinTestFramework): sequence_value = ((cur_time - orig_time) >> SEQUENCE_LOCKTIME_GRANULARITY)+1 sequence_value |= SEQUENCE_LOCKTIME_TYPE_FLAG tx.vin.append(CTxIn(COutPoint(int(utxos[j]["txid"], 16), utxos[j]["vout"]), nSequence=sequence_value)) - value += utxos[j]["amount"]*COIN + value += utxos[j]["value"]*COIN # Overestimate the size of the tx - signatures should be less than 120 bytes, and leave 50 for the output tx_size = len(tx.serialize().hex())//2 + 120*num_inputs + 50 tx.vout.append(CTxOut(int(value - self.relayfee * tx_size * COIN / 1000), SCRIPT_W0_SH_OP_TRUE)) - rawtx = self.nodes[0].signrawtransactionwithwallet(tx.serialize().hex())["hex"] + self.wallet.sign_tx(tx=tx) if (using_sequence_locks and not should_pass): # This transaction should be rejected - assert_raises_rpc_error(-26, NOT_FINAL_ERROR, self.nodes[0].sendrawtransaction, rawtx) + assert_raises_rpc_error(-26, NOT_FINAL_ERROR, self.wallet.sendrawtransaction, from_node=self.nodes[0], tx_hex=tx.serialize().hex()) else: # This raw transaction should be accepted - self.nodes[0].sendrawtransaction(rawtx) - utxos = self.nodes[0].listunspent() + self.wallet.sendrawtransaction(from_node=self.nodes[0], tx_hex=tx.serialize().hex()) + self.wallet.rescan_utxos() + utxos = self.wallet.get_utxos(include_immature_coinbase=False) # Test that sequence locks on unconfirmed inputs must have nSequence # height or time of 0 to be accepted. @@ -237,8 +221,8 @@ class BIP68Test(BitcoinTestFramework): cur_height = self.nodes[0].getblockcount() # Create a mempool tx. - txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 2) - tx1 = tx_from_hex(self.nodes[0].getrawtransaction(txid)) + self.wallet.rescan_utxos() + tx1 = self.wallet.send_self_transfer(from_node=self.nodes[0])["tx"] tx1.rehash() # Anyone-can-spend mempool tx. @@ -247,11 +231,11 @@ class BIP68Test(BitcoinTestFramework): tx2.nVersion = 2 tx2.vin = [CTxIn(COutPoint(tx1.sha256, 0), nSequence=0)] tx2.vout = [CTxOut(int(tx1.vout[0].nValue - self.relayfee * COIN), SCRIPT_W0_SH_OP_TRUE)] - tx2_raw = self.nodes[0].signrawtransactionwithwallet(tx2.serialize().hex())["hex"] - tx2 = tx_from_hex(tx2_raw) + self.wallet.sign_tx(tx=tx2) + tx2_raw = tx2.serialize().hex() tx2.rehash() - self.nodes[0].sendrawtransaction(tx2_raw) + self.wallet.sendrawtransaction(from_node=self.nodes[0], tx_hex=tx2_raw) # Create a spend of the 0th output of orig_tx with a sequence lock # of 1, and test what happens when submitting. @@ -271,10 +255,10 @@ class BIP68Test(BitcoinTestFramework): if (orig_tx.hash in node.getrawmempool()): # sendrawtransaction should fail if the tx is in the mempool - assert_raises_rpc_error(-26, NOT_FINAL_ERROR, node.sendrawtransaction, tx.serialize().hex()) + assert_raises_rpc_error(-26, NOT_FINAL_ERROR, self.wallet.sendrawtransaction, from_node=node, tx_hex=tx.serialize().hex()) else: # sendrawtransaction should succeed if the tx is not in the mempool - node.sendrawtransaction(tx.serialize().hex()) + self.wallet.sendrawtransaction(from_node=node, tx_hex=tx.serialize().hex()) return tx @@ -287,7 +271,7 @@ class BIP68Test(BitcoinTestFramework): cur_time = int(time.time()) for _ in range(10): self.nodes[0].setmocktime(cur_time + 600) - self.generate(self.nodes[0], 1, sync_fun=self.no_op) + self.generate(self.wallet, 1, sync_fun=self.no_op) cur_time += 600 assert tx2.hash in self.nodes[0].getrawmempool() @@ -321,12 +305,12 @@ class BIP68Test(BitcoinTestFramework): tx5 = test_nonzero_locks(tx4, self.nodes[0], self.relayfee, use_height_lock=True) assert tx5.hash not in self.nodes[0].getrawmempool() - utxos = self.nodes[0].listunspent() - tx5.vin.append(CTxIn(COutPoint(int(utxos[0]["txid"], 16), utxos[0]["vout"]), nSequence=1)) - tx5.vout[0].nValue += int(utxos[0]["amount"]*COIN) - raw_tx5 = self.nodes[0].signrawtransactionwithwallet(tx5.serialize().hex())["hex"] + utxo = self.wallet.get_utxo() + tx5.vin.append(CTxIn(COutPoint(int(utxo["txid"], 16), utxo["vout"]), nSequence=1)) + tx5.vout[0].nValue += int(utxo["value"]*COIN) + self.wallet.sign_tx(tx=tx5) - assert_raises_rpc_error(-26, NOT_FINAL_ERROR, self.nodes[0].sendrawtransaction, raw_tx5) + assert_raises_rpc_error(-26, NOT_FINAL_ERROR, self.wallet.sendrawtransaction, from_node=self.nodes[0], tx_hex=tx5.serialize().hex()) # Test mempool-BIP68 consistency after reorg # @@ -362,7 +346,7 @@ class BIP68Test(BitcoinTestFramework): # Reset the chain and get rid of the mocktimed-blocks self.nodes[0].setmocktime(0) self.nodes[0].invalidateblock(self.nodes[0].getblockhash(cur_height+1)) - self.generate(self.nodes[0], 10, sync_fun=self.no_op) + self.generate(self.wallet, 10, sync_fun=self.no_op) # Make sure that BIP68 isn't being used to validate blocks prior to # activation height. If more blocks are mined prior to this test @@ -370,9 +354,8 @@ class BIP68Test(BitcoinTestFramework): # this test should be moved to run earlier, or deleted. def test_bip68_not_consensus(self): assert not softfork_active(self.nodes[0], 'csv') - txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 2) - tx1 = tx_from_hex(self.nodes[0].getrawtransaction(txid)) + tx1 = self.wallet.send_self_transfer(from_node=self.nodes[0])["tx"] tx1.rehash() # Make an anyone-can-spend transaction @@ -382,11 +365,12 @@ class BIP68Test(BitcoinTestFramework): tx2.vout = [CTxOut(int(tx1.vout[0].nValue - self.relayfee * COIN), SCRIPT_W0_SH_OP_TRUE)] # sign tx2 - tx2_raw = self.nodes[0].signrawtransactionwithwallet(tx2.serialize().hex())["hex"] + self.wallet.sign_tx(tx=tx2) + tx2_raw = tx2.serialize().hex() tx2 = tx_from_hex(tx2_raw) tx2.rehash() - self.nodes[0].sendrawtransaction(tx2.serialize().hex()) + self.wallet.sendrawtransaction(from_node=self.nodes[0], tx_hex=tx2_raw) # Now make an invalid spend of tx2 according to BIP68 sequence_value = 100 # 100 block relative locktime @@ -399,7 +383,7 @@ class BIP68Test(BitcoinTestFramework): tx3.vout = [CTxOut(int(tx2.vout[0].nValue - self.relayfee * COIN), SCRIPT_W0_SH_OP_TRUE)] tx3.rehash() - assert_raises_rpc_error(-26, NOT_FINAL_ERROR, self.nodes[0].sendrawtransaction, tx3.serialize().hex()) + assert_raises_rpc_error(-26, NOT_FINAL_ERROR, self.wallet.sendrawtransaction, from_node=self.nodes[0], tx_hex=tx3.serialize().hex()) # make a block that violates bip68; ensure that the tip updates block = create_block(tmpl=self.nodes[0].getblocktemplate(NORMAL_GBT_REQUEST_PARAMS), txlist=[tx1, tx2, tx3]) @@ -415,22 +399,19 @@ class BIP68Test(BitcoinTestFramework): min_activation_height = 432 height = self.nodes[0].getblockcount() assert_greater_than(min_activation_height - height, 2) - self.generate(self.nodes[0], min_activation_height - height - 2, sync_fun=self.no_op) + self.generate(self.wallet, min_activation_height - height - 2, sync_fun=self.no_op) assert not softfork_active(self.nodes[0], 'csv') - self.generate(self.nodes[0], 1, sync_fun=self.no_op) + self.generate(self.wallet, 1, sync_fun=self.no_op) assert softfork_active(self.nodes[0], 'csv') self.sync_blocks() # Use self.nodes[1] to test that version 2 transactions are standard. def test_version2_relay(self): - inputs = [ ] - outputs = { self.nodes[1].getnewaddress() : 1.0 } - rawtx = self.nodes[1].createrawtransaction(inputs, outputs) - rawtxfund = self.nodes[1].fundrawtransaction(rawtx)['hex'] - tx = tx_from_hex(rawtxfund) + mini_wallet = MiniWallet(self.nodes[1]) + mini_wallet.rescan_utxos() + tx = mini_wallet.create_self_transfer()["tx"] tx.nVersion = 2 - tx_signed = self.nodes[1].signrawtransactionwithwallet(tx.serialize().hex())["hex"] - self.nodes[1].sendrawtransaction(tx_signed) + mini_wallet.sendrawtransaction(from_node=self.nodes[1], tx_hex=tx.serialize().hex()) if __name__ == '__main__': BIP68Test().main() diff --git a/test/functional/feature_block.py b/test/functional/feature_block.py index 22b1918b85..1080e77c40 100755 --- a/test/functional/feature_block.py +++ b/test/functional/feature_block.py @@ -161,7 +161,7 @@ class FullBlockTest(BitcoinTestFramework): self.log.info(f"Reject block with invalid tx: {TxTemplate.__name__}") blockname = f"for_invalid.{TxTemplate.__name__}" - badblock = self.next_block(blockname) + self.next_block(blockname) badtx = template.get_tx() if TxTemplate != invalid_txs.InputMissing: self.sign_tx(badtx, attempt_spend_tx) @@ -473,7 +473,7 @@ class FullBlockTest(BitcoinTestFramework): # self.log.info("Check P2SH SIGOPS are correctly counted") self.move_tip(35) - b39 = self.next_block(39) + self.next_block(39) b39_outputs = 0 b39_sigops_per_output = 6 @@ -672,7 +672,7 @@ class FullBlockTest(BitcoinTestFramework): self.log.info("Reject a block with two coinbase transactions") self.move_tip(44) - b51 = self.next_block(51) + self.next_block(51) cb2 = create_coinbase(51, self.coinbase_pubkey) b51 = self.update_block(51, [cb2]) self.send_blocks([b51], success=False, reject_reason='bad-cb-multiple', reconnect=True) @@ -752,7 +752,7 @@ class FullBlockTest(BitcoinTestFramework): # b57 - a good block with 2 txs, don't submit until end self.move_tip(55) - b57 = self.next_block(57) + self.next_block(57) tx = self.create_and_sign_transaction(out[16], 1) tx1 = self.create_tx(tx, 0, 1) b57 = self.update_block(57, [tx, tx1]) @@ -769,7 +769,7 @@ class FullBlockTest(BitcoinTestFramework): # b57p2 - a good block with 6 tx'es, don't submit until end self.move_tip(55) - b57p2 = self.next_block("57p2") + self.next_block("57p2") tx = self.create_and_sign_transaction(out[16], 1) tx1 = self.create_tx(tx, 0, 1) tx2 = self.create_tx(tx1, 0, 1) @@ -803,7 +803,7 @@ class FullBlockTest(BitcoinTestFramework): # tx with prevout.n out of range self.log.info("Reject a block with a transaction with prevout.n out of range") self.move_tip(57) - b58 = self.next_block(58, spend=out[17]) + self.next_block(58, spend=out[17]) tx = CTransaction() assert len(out[17].vout) < 42 tx.vin.append(CTxIn(COutPoint(out[17].sha256, 42), CScript([OP_TRUE]), SEQUENCE_FINAL)) @@ -815,7 +815,7 @@ class FullBlockTest(BitcoinTestFramework): # tx with output value > input value self.log.info("Reject a block with a transaction with outputs > inputs") self.move_tip(57) - b59 = self.next_block(59) + self.next_block(59) tx = self.create_and_sign_transaction(out[17], 51 * COIN) b59 = self.update_block(59, [tx]) self.send_blocks([b59], success=False, reject_reason='bad-txns-in-belowout', reconnect=True) @@ -851,7 +851,7 @@ class FullBlockTest(BitcoinTestFramework): # \-> b_spend_dup_cb (b_dup_cb) -> b_dup_2 () # self.move_tip(57) - b_spend_dup_cb = self.next_block('spend_dup_cb') + self.next_block('spend_dup_cb') tx = CTransaction() tx.vin.append(CTxIn(COutPoint(duplicate_tx.sha256, 0))) tx.vout.append(CTxOut(0, CScript([OP_TRUE]))) @@ -876,7 +876,7 @@ class FullBlockTest(BitcoinTestFramework): # self.log.info("Reject a block with a transaction with a nonfinal locktime") self.move_tip('dup_2') - b62 = self.next_block(62) + self.next_block(62) tx = CTransaction() tx.nLockTime = 0xffffffff # this locktime is non-final tx.vin.append(CTxIn(COutPoint(out[18].sha256, 0))) # don't set nSequence @@ -957,7 +957,7 @@ class FullBlockTest(BitcoinTestFramework): # self.log.info("Accept a block with a transaction spending an output created in the same block") self.move_tip(64) - b65 = self.next_block(65) + self.next_block(65) tx1 = self.create_and_sign_transaction(out[19], out[19].vout[0].nValue) tx2 = self.create_and_sign_transaction(tx1, 0) b65 = self.update_block(65, [tx1, tx2]) @@ -970,7 +970,7 @@ class FullBlockTest(BitcoinTestFramework): # \-> b66 (20) self.log.info("Reject a block with a transaction spending an output created later in the same block") self.move_tip(65) - b66 = self.next_block(66) + self.next_block(66) tx1 = self.create_and_sign_transaction(out[20], out[20].vout[0].nValue) tx2 = self.create_and_sign_transaction(tx1, 1) b66 = self.update_block(66, [tx2, tx1]) @@ -984,7 +984,7 @@ class FullBlockTest(BitcoinTestFramework): # self.log.info("Reject a block with a transaction double spending a transaction created in the same block") self.move_tip(65) - b67 = self.next_block(67) + self.next_block(67) tx1 = self.create_and_sign_transaction(out[20], out[20].vout[0].nValue) tx2 = self.create_and_sign_transaction(tx1, 1) tx3 = self.create_and_sign_transaction(tx1, 2) @@ -1005,7 +1005,7 @@ class FullBlockTest(BitcoinTestFramework): # self.log.info("Reject a block trying to claim too much subsidy in the coinbase transaction") self.move_tip(65) - b68 = self.next_block(68, additional_coinbase_value=10) + self.next_block(68, additional_coinbase_value=10) tx = self.create_and_sign_transaction(out[20], out[20].vout[0].nValue - 9) b68 = self.update_block(68, [tx]) self.send_blocks([b68], success=False, reject_reason='bad-cb-amount', reconnect=True) @@ -1025,7 +1025,7 @@ class FullBlockTest(BitcoinTestFramework): # self.log.info("Reject a block containing a transaction spending from a non-existent input") self.move_tip(69) - b70 = self.next_block(70, spend=out[21]) + self.next_block(70, spend=out[21]) bogus_tx = CTransaction() bogus_tx.sha256 = uint256_from_str(b"23c70ed7c0506e9178fc1a987f40a33946d4ad4c962b5ae3a52546da53af0c5c") tx = CTransaction() @@ -1043,7 +1043,7 @@ class FullBlockTest(BitcoinTestFramework): # b71 is a copy of 72, but re-adds one of its transactions. However, it has the same hash as b72. self.log.info("Reject a block containing a duplicate transaction but with the same Merkle root (Merkle tree malleability") self.move_tip(69) - b72 = self.next_block(72) + self.next_block(72) tx1 = self.create_and_sign_transaction(out[21], 2) tx2 = self.create_and_sign_transaction(tx1, 1) b72 = self.update_block(72, [tx1, tx2]) # now tip is 72 @@ -1081,7 +1081,7 @@ class FullBlockTest(BitcoinTestFramework): # bytearray[20,526] : OP_CHECKSIG (this puts us over the limit) self.log.info("Reject a block containing too many sigops after a large script element") self.move_tip(72) - b73 = self.next_block(73) + self.next_block(73) size = MAX_BLOCK_SIGOPS - 1 + MAX_SCRIPT_ELEMENT_SIZE + 1 + 5 + 1 a = bytearray([OP_CHECKSIG] * size) a[MAX_BLOCK_SIGOPS - 1] = int("4e", 16) # OP_PUSHDATA4 @@ -1109,7 +1109,7 @@ class FullBlockTest(BitcoinTestFramework): # b75 succeeds because we put MAX_BLOCK_SIGOPS before the element self.log.info("Check sigops are counted correctly after an invalid script element") self.move_tip(72) - b74 = self.next_block(74) + self.next_block(74) size = MAX_BLOCK_SIGOPS - 1 + MAX_SCRIPT_ELEMENT_SIZE + 42 # total = 20,561 a = bytearray([OP_CHECKSIG] * size) a[MAX_BLOCK_SIGOPS] = 0x4e @@ -1122,7 +1122,7 @@ class FullBlockTest(BitcoinTestFramework): self.send_blocks([b74], success=False, reject_reason='bad-blk-sigops', reconnect=True) self.move_tip(72) - b75 = self.next_block(75) + self.next_block(75) size = MAX_BLOCK_SIGOPS - 1 + MAX_SCRIPT_ELEMENT_SIZE + 42 a = bytearray([OP_CHECKSIG] * size) a[MAX_BLOCK_SIGOPS - 1] = 0x4e @@ -1137,7 +1137,7 @@ class FullBlockTest(BitcoinTestFramework): # Check that if we push an element filled with CHECKSIGs, they are not counted self.move_tip(75) - b76 = self.next_block(76) + self.next_block(76) size = MAX_BLOCK_SIGOPS - 1 + MAX_SCRIPT_ELEMENT_SIZE + 1 + 5 a = bytearray([OP_CHECKSIG] * size) a[MAX_BLOCK_SIGOPS - 1] = 0x4e # PUSHDATA4, but leave the following bytes as just checksigs @@ -1165,18 +1165,18 @@ class FullBlockTest(BitcoinTestFramework): # updated. (Perhaps to spend to a P2SH OP_TRUE script) self.log.info("Test transaction resurrection during a re-org") self.move_tip(76) - b77 = self.next_block(77) + self.next_block(77) tx77 = self.create_and_sign_transaction(out[24], 10 * COIN) b77 = self.update_block(77, [tx77]) self.send_blocks([b77], True) self.save_spendable_output() - b78 = self.next_block(78) + self.next_block(78) tx78 = self.create_tx(tx77, 0, 9 * COIN) b78 = self.update_block(78, [tx78]) self.send_blocks([b78], True) - b79 = self.next_block(79) + self.next_block(79) tx79 = self.create_tx(tx78, 0, 8 * COIN) b79 = self.update_block(79, [tx79]) self.send_blocks([b79], True) @@ -1208,7 +1208,7 @@ class FullBlockTest(BitcoinTestFramework): # -> b81 (26) -> b82 (27) -> b83 (28) # self.log.info("Accept a block with invalid opcodes in dead execution paths") - b83 = self.next_block(83) + self.next_block(83) op_codes = [OP_IF, OP_INVALIDOPCODE, OP_ELSE, OP_TRUE, OP_ENDIF] script = CScript(op_codes) tx1 = self.create_and_sign_transaction(out[28], out[28].vout[0].nValue, script) @@ -1227,7 +1227,7 @@ class FullBlockTest(BitcoinTestFramework): # \-> b85 (29) -> b86 (30) \-> b89a (32) # self.log.info("Test re-orging blocks with OP_RETURN in them") - b84 = self.next_block(84) + self.next_block(84) tx1 = self.create_tx(out[29], 0, 0, CScript([OP_RETURN])) tx1.vout.append(CTxOut(0, CScript([OP_TRUE]))) tx1.vout.append(CTxOut(0, CScript([OP_TRUE]))) @@ -1265,7 +1265,7 @@ class FullBlockTest(BitcoinTestFramework): self.save_spendable_output() # trying to spend the OP_RETURN output is rejected - b89a = self.next_block("89a", spend=out[32]) + self.next_block("89a", spend=out[32]) tx = self.create_tx(tx1, 0, 0, CScript([OP_TRUE])) b89a = self.update_block("89a", [tx]) self.send_blocks([b89a], success=False, reject_reason='bad-txns-inputs-missingorspent', reconnect=True) diff --git a/test/functional/feature_coinstatsindex.py b/test/functional/feature_coinstatsindex.py index eff4d9b149..4f8541a5d7 100755 --- a/test/functional/feature_coinstatsindex.py +++ b/test/functional/feature_coinstatsindex.py @@ -156,9 +156,10 @@ class CoinStatsIndexTest(BitcoinTestFramework): # Generate and send another tx with an OP_RETURN output (which is unspendable) tx2 = self.wallet.create_self_transfer(utxo_to_spend=tx1_out_21)['tx'] - tx2.vout = [CTxOut(int(Decimal('20.99') * COIN), CScript([OP_RETURN] + [OP_FALSE] * 30))] + tx2_val = '20.99' + tx2.vout = [CTxOut(int(Decimal(tx2_val) * COIN), CScript([OP_RETURN] + [OP_FALSE] * 30))] tx2_hex = tx2.serialize().hex() - self.nodes[0].sendrawtransaction(tx2_hex) + self.nodes[0].sendrawtransaction(tx2_hex, 0, tx2_val) # Include both txs in a block self.generate(self.nodes[0], 1) diff --git a/test/functional/feature_config_args.py b/test/functional/feature_config_args.py index d5e5ed47d6..f9730b48c5 100755 --- a/test/functional/feature_config_args.py +++ b/test/functional/feature_config_args.py @@ -126,7 +126,6 @@ class ConfArgsTest(BitcoinTestFramework): expected_msgs=[ 'Command-line arg: addnode="some.node"', 'Command-line arg: rpcauth=****', - 'Command-line arg: rpcbind=****', 'Command-line arg: rpcpassword=****', 'Command-line arg: rpcuser=****', 'Command-line arg: torpassword=****', @@ -135,14 +134,17 @@ class ConfArgsTest(BitcoinTestFramework): ], unexpected_msgs=[ 'alice:f7efda5c189b999524f151318c0c86$d5b51b3beffbc0', - '127.1.1.1', 'secret-rpcuser', 'secret-torpassword', + 'Command-line arg: rpcbind=****', + 'Command-line arg: rpcallowip=****', ]): self.start_node(0, extra_args=[ '-addnode=some.node', '-rpcauth=alice:f7efda5c189b999524f151318c0c86$d5b51b3beffbc0', '-rpcbind=127.1.1.1', + '-rpcbind=127.0.0.1', + "-rpcallowip=127.0.0.1", '-rpcpassword=', '-rpcuser=secret-rpcuser', '-torpassword=secret-torpassword', @@ -249,11 +251,43 @@ class ConfArgsTest(BitcoinTestFramework): ]): self.nodes[0].setmocktime(start + 65) + def test_connect_with_seednode(self): + self.log.info('Test -connect with -seednode') + seednode_ignored = ['-seednode is ignored when -connect is used\n'] + dnsseed_ignored = ['-dnsseed is ignored when -connect is used and -proxy is specified\n'] + addcon_thread_started = ['addcon thread start\n'] + self.stop_node(0) + + # When -connect is supplied, expanding addrman via getaddr calls to ADDR_FETCH(-seednode) + # nodes is irrelevant and -seednode is ignored. + with self.nodes[0].assert_debug_log(expected_msgs=seednode_ignored): + self.start_node(0, extra_args=['-connect=fakeaddress1', '-seednode=fakeaddress2']) + + # With -proxy, an ADDR_FETCH connection is made to a peer that the dns seed resolves to. + # ADDR_FETCH connections are not used when -connect is used. + with self.nodes[0].assert_debug_log(expected_msgs=dnsseed_ignored): + self.restart_node(0, extra_args=['-connect=fakeaddress1', '-dnsseed=1', '-proxy=1.2.3.4']) + + # If the user did not disable -dnsseed, but it was soft-disabled because they provided -connect, + # they shouldn't see a warning about -dnsseed being ignored. + with self.nodes[0].assert_debug_log(expected_msgs=addcon_thread_started, + unexpected_msgs=dnsseed_ignored): + self.restart_node(0, extra_args=['-connect=fakeaddress1', '-proxy=1.2.3.4']) + + # We have to supply expected_msgs as it's a required argument + # The expected_msg must be something we are confident will be logged after the unexpected_msg + # These cases test for -connect being supplied but only to disable it + for connect_arg in ['-connect=0', '-noconnect']: + with self.nodes[0].assert_debug_log(expected_msgs=addcon_thread_started, + unexpected_msgs=seednode_ignored): + self.restart_node(0, extra_args=[connect_arg, '-seednode=fakeaddress2']) + def run_test(self): self.test_log_buffer() self.test_args_log() self.test_seed_peers() self.test_networkactive() + self.test_connect_with_seednode() self.test_config_file_parser() self.test_invalid_command_line_options() diff --git a/test/functional/feature_dbcrash.py b/test/functional/feature_dbcrash.py index ff770e7707..3f94bbc9d1 100755 --- a/test/functional/feature_dbcrash.py +++ b/test/functional/feature_dbcrash.py @@ -51,9 +51,11 @@ class ChainstateWriteCrashTest(BitcoinTestFramework): self.supports_cli = False # Set -maxmempool=0 to turn off mempool memory sharing with dbcache - # Set -rpcservertimeout=900 to reduce socket disconnects in this - # long-running test - self.base_args = ["-limitdescendantsize=0", "-maxmempool=0", "-rpcservertimeout=900", "-dbbatchsize=200000"] + self.base_args = [ + "-limitdescendantsize=0", + "-maxmempool=0", + "-dbbatchsize=200000", + ] # Set different crash ratios and cache sizes. Note that not all of # -dbcache goes to the in-memory coins cache. @@ -85,7 +87,7 @@ class ChainstateWriteCrashTest(BitcoinTestFramework): self.nodes[node_index].waitforblock(expected_tip) utxo_hash = self.nodes[node_index].gettxoutsetinfo()['hash_serialized_2'] return utxo_hash - except: + except Exception: # An exception here should mean the node is about to crash. # If bitcoind exits, then try again. wait_for_node_exit() # should raise an exception if bitcoind doesn't exit. @@ -202,7 +204,6 @@ class ChainstateWriteCrashTest(BitcoinTestFramework): def run_test(self): self.wallet = MiniWallet(self.nodes[3]) - self.wallet.rescan_utxos() initial_height = self.nodes[3].getblockcount() self.generate(self.nodes[3], COINBASE_MATURITY, sync_fun=self.no_op) diff --git a/test/functional/feature_fee_estimation.py b/test/functional/feature_fee_estimation.py index 6b2d5b9455..05ee556ece 100755 --- a/test/functional/feature_fee_estimation.py +++ b/test/functional/feature_fee_estimation.py @@ -297,7 +297,6 @@ class EstimateFeeTest(BitcoinTestFramework): # Split two coinbases into many small utxos self.start_node(0) self.wallet = MiniWallet(self.nodes[0]) - self.wallet.rescan_utxos() self.initial_split(self.nodes[0]) self.log.info("Finished splitting") diff --git a/test/functional/feature_maxuploadtarget.py b/test/functional/feature_maxuploadtarget.py index 28a8959e93..c551c0b449 100755 --- a/test/functional/feature_maxuploadtarget.py +++ b/test/functional/feature_maxuploadtarget.py @@ -164,6 +164,9 @@ class MaxUploadTest(BitcoinTestFramework): assert_equal(len(peer_info), 1) # node is still connected assert_equal(peer_info[0]['permissions'], ['download']) + self.log.info("Test passing an unparsable value to -maxuploadtarget throws an error") + self.stop_node(0) + self.nodes[0].assert_start_raises_init_error(extra_args=["-maxuploadtarget=abc"], expected_msg="Error: Unable to parse -maxuploadtarget: 'abc'") if __name__ == '__main__': MaxUploadTest().main() diff --git a/test/functional/feature_notifications.py b/test/functional/feature_notifications.py index 32fea18f37..8cb633d454 100755 --- a/test/functional/feature_notifications.py +++ b/test/functional/feature_notifications.py @@ -31,7 +31,7 @@ class NotificationsTest(BitcoinTestFramework): self.num_nodes = 2 self.setup_clean_chain = True # The experimental syscall sandbox feature (-sandbox) is not compatible with -alertnotify, - # -blocknotify or -walletnotify (which all invoke execve). + # -blocknotify, -walletnotify or -shutdownnotify (which all invoke execve). self.disable_syscall_sandbox = True def setup_network(self): @@ -39,14 +39,18 @@ class NotificationsTest(BitcoinTestFramework): self.alertnotify_dir = os.path.join(self.options.tmpdir, "alertnotify") self.blocknotify_dir = os.path.join(self.options.tmpdir, "blocknotify") self.walletnotify_dir = os.path.join(self.options.tmpdir, "walletnotify") + self.shutdownnotify_dir = os.path.join(self.options.tmpdir, "shutdownnotify") + self.shutdownnotify_file = os.path.join(self.shutdownnotify_dir, "shutdownnotify.txt") os.mkdir(self.alertnotify_dir) os.mkdir(self.blocknotify_dir) os.mkdir(self.walletnotify_dir) + os.mkdir(self.shutdownnotify_dir) # -alertnotify and -blocknotify on node0, walletnotify on node1 self.extra_args = [[ f"-alertnotify=echo > {os.path.join(self.alertnotify_dir, '%s')}", f"-blocknotify=echo > {os.path.join(self.blocknotify_dir, '%s')}", + f"-shutdownnotify=echo > {self.shutdownnotify_file}", ], [ f"-walletnotify=echo %h_%b > {os.path.join(self.walletnotify_dir, notify_outputname('%w', '%s'))}", ]] @@ -162,6 +166,10 @@ class NotificationsTest(BitcoinTestFramework): # TODO: add test for `-alertnotify` large fork notifications + self.log.info("test -shutdownnotify") + self.stop_nodes() + self.wait_until(lambda: os.path.isfile(self.shutdownnotify_file), timeout=10) + def expect_wallet_notify(self, tx_details): self.wait_until(lambda: len(os.listdir(self.walletnotify_dir)) >= len(tx_details), timeout=10) # Should have no more and no less files than expected diff --git a/test/functional/feature_nulldummy.py b/test/functional/feature_nulldummy.py index d7558c830e..c95657dbbb 100755 --- a/test/functional/feature_nulldummy.py +++ b/test/functional/feature_nulldummy.py @@ -14,6 +14,7 @@ Generate COINBASE_MATURITY (CB) more blocks to ensure the coinbases are mature. """ import time +from test_framework.address import address_to_scriptpubkey from test_framework.blocktools import ( COINBASE_MATURITY, NORMAL_GBT_REQUEST_PARAMS, @@ -77,7 +78,7 @@ class NULLDUMMYTest(BitcoinTestFramework): cms = self.nodes[0].createmultisig(1, [self.pubkey]) wms = self.nodes[0].createmultisig(1, [self.pubkey], 'p2sh-segwit') self.ms_address = cms["address"] - ms_unlock_details = {"scriptPubKey": self.nodes[0].validateaddress(self.ms_address)["scriptPubKey"], + ms_unlock_details = {"scriptPubKey": address_to_scriptpubkey(self.ms_address).hex(), "redeemScript": cms["redeemScript"]} self.wit_ms_address = wms['address'] diff --git a/test/functional/feature_posix_fs_permissions.py b/test/functional/feature_posix_fs_permissions.py new file mode 100755 index 0000000000..c5a543e97a --- /dev/null +++ b/test/functional/feature_posix_fs_permissions.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test file system permissions for POSIX platforms. +""" + +import os +import stat + +from test_framework.test_framework import BitcoinTestFramework + + +class PosixFsPermissionsTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 1 + + def skip_test_if_missing_module(self): + self.skip_if_platform_not_posix() + + def check_directory_permissions(self, dir): + mode = os.lstat(dir).st_mode + self.log.info(f"{stat.filemode(mode)} {dir}") + assert mode == (stat.S_IFDIR | stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) + + def check_file_permissions(self, file): + mode = os.lstat(file).st_mode + self.log.info(f"{stat.filemode(mode)} {file}") + assert mode == (stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR) + + def run_test(self): + self.stop_node(0) + datadir = os.path.join(self.nodes[0].datadir, self.chain) + self.check_directory_permissions(datadir) + walletsdir = os.path.join(datadir, "wallets") + self.check_directory_permissions(walletsdir) + debuglog = os.path.join(datadir, "debug.log") + self.check_file_permissions(debuglog) + + +if __name__ == '__main__': + PosixFsPermissionsTest().main() diff --git a/test/functional/feature_pruning.py b/test/functional/feature_pruning.py index f68416dc03..519877ac5b 100755 --- a/test/functional/feature_pruning.py +++ b/test/functional/feature_pruning.py @@ -84,7 +84,7 @@ class PruneTest(BitcoinTestFramework): ["-maxreceivebuffer=20000", "-prune=550"], ["-maxreceivebuffer=20000"], ["-maxreceivebuffer=20000"], - ["-prune=550"], + ["-prune=550", "-blockfilterindex=1"], ] self.rpc_timeout = 120 @@ -223,8 +223,8 @@ class PruneTest(BitcoinTestFramework): def reorg_back(self): # Verify that a block on the old main chain fork has been pruned away assert_raises_rpc_error(-1, "Block not available (pruned data)", self.nodes[2].getblock, self.forkhash) - with self.nodes[2].assert_debug_log(expected_msgs=['block verification stopping at height', '(pruning, no data)']): - self.nodes[2].verifychain(checklevel=4, nblocks=0) + with self.nodes[2].assert_debug_log(expected_msgs=['block verification stopping at height', '(no data)']): + assert not self.nodes[2].verifychain(checklevel=4, nblocks=0) self.log.info(f"Will need to redownload block {self.forkheight}") # Verify that we have enough history to reorg back to the fork point @@ -356,7 +356,7 @@ class PruneTest(BitcoinTestFramework): self.connect_nodes(0, 5) nds = [self.nodes[0], self.nodes[5]] self.sync_blocks(nds, wait=5, timeout=300) - self.restart_node(5, extra_args=["-prune=550"]) # restart to trigger rescan + self.restart_node(5, extra_args=["-prune=550", "-blockfilterindex=1"]) # restart to trigger rescan self.log.info("Success") def run_test(self): @@ -472,7 +472,20 @@ class PruneTest(BitcoinTestFramework): self.log.info("Test invalid pruning command line options") self.test_invalid_command_line_options() + self.test_scanblocks_pruned() + self.log.info("Done") + def test_scanblocks_pruned(self): + node = self.nodes[5] + genesis_blockhash = node.getblockhash(0) + false_positive_spk = bytes.fromhex("001400000000000000000000000000000000000cadcb") + + assert genesis_blockhash in node.scanblocks( + "start", [{"desc": f"raw({false_positive_spk.hex()})"}], 0, 0)['relevant_blocks'] + + assert_raises_rpc_error(-1, "Block not available (pruned data)", node.scanblocks, + "start", [{"desc": f"raw({false_positive_spk.hex()})"}], 0, 0, "basic", {"filter_false_positives": True}) + if __name__ == '__main__': PruneTest().main() diff --git a/test/functional/feature_rbf.py b/test/functional/feature_rbf.py index 085ff3a2e3..947d2e8273 100755 --- a/test/functional/feature_rbf.py +++ b/test/functional/feature_rbf.py @@ -42,10 +42,6 @@ class ReplaceByFeeTest(BitcoinTestFramework): def run_test(self): self.wallet = MiniWallet(self.nodes[0]) - # the pre-mined test framework chain contains coinbase outputs to the - # MiniWallet's default address in blocks 76-100 (see method - # BitcoinTestFramework._initialize_chain()) - self.wallet.rescan_utxos() self.log.info("Running test simple doublespend...") self.test_simple_doublespend() @@ -396,12 +392,11 @@ class ReplaceByFeeTest(BitcoinTestFramework): enough transactions off of each root UTXO to exceed the MAX_REPLACEMENT_LIMIT. Then create a conflicting RBF replacement transaction. """ - normal_node = self.nodes[1] - wallet = MiniWallet(normal_node) - wallet.rescan_utxos() # Clear mempools to avoid cross-node sync failure. for node in self.nodes: self.generate(node, 1) + normal_node = self.nodes[1] + wallet = MiniWallet(normal_node) # This has to be chosen so that the total number of transactions can exceed # MAX_REPLACEMENT_LIMIT without having any one tx graph run into the descendant diff --git a/test/functional/feature_remove_pruned_files_on_startup.py b/test/functional/feature_remove_pruned_files_on_startup.py new file mode 100755 index 0000000000..ca0e5ace9f --- /dev/null +++ b/test/functional/feature_remove_pruned_files_on_startup.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test removing undeleted pruned blk files on startup.""" + +import os +from test_framework.test_framework import BitcoinTestFramework + +class FeatureRemovePrunedFilesOnStartupTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + self.extra_args = [["-fastprune", "-prune=1"]] + + def mine_batches(self, blocks): + n = blocks // 250 + for _ in range(n): + self.generate(self.nodes[0], 250) + self.generate(self.nodes[0], blocks % 250) + self.sync_blocks() + + def run_test(self): + blk0 = os.path.join(self.nodes[0].datadir, self.nodes[0].chain, 'blocks', 'blk00000.dat') + rev0 = os.path.join(self.nodes[0].datadir, self.nodes[0].chain, 'blocks', 'rev00000.dat') + blk1 = os.path.join(self.nodes[0].datadir, self.nodes[0].chain, 'blocks', 'blk00001.dat') + rev1 = os.path.join(self.nodes[0].datadir, self.nodes[0].chain, 'blocks', 'rev00001.dat') + self.mine_batches(800) + fo1 = os.open(blk0, os.O_RDONLY) + fo2 = os.open(rev1, os.O_RDONLY) + fd1 = os.fdopen(fo1) + fd2 = os.fdopen(fo2) + self.nodes[0].pruneblockchain(600) + + # Windows systems will not remove files with an open fd + if os.name != 'nt': + assert not os.path.exists(blk0) + assert not os.path.exists(rev0) + assert not os.path.exists(blk1) + assert not os.path.exists(rev1) + else: + assert os.path.exists(blk0) + assert not os.path.exists(rev0) + assert not os.path.exists(blk1) + assert os.path.exists(rev1) + + # Check that the files are removed on restart once the fds are closed + fd1.close() + fd2.close() + self.restart_node(0) + assert not os.path.exists(blk0) + assert not os.path.exists(rev1) + +if __name__ == '__main__': + FeatureRemovePrunedFilesOnStartupTest().main() diff --git a/test/functional/feature_startupnotify.py b/test/functional/feature_startupnotify.py index f4672a2705..ff5272b281 100755 --- a/test/functional/feature_startupnotify.py +++ b/test/functional/feature_startupnotify.py @@ -29,9 +29,14 @@ class StartupNotifyTest(BitcoinTestFramework): self.wait_until(lambda: os.path.exists(tmpdir_file)) self.log.info("Test -startupnotify is executed once") - with open(tmpdir_file, "r", encoding="utf8") as f: - file_content = f.read() - assert_equal(file_content.count(FILE_NAME), 1) + + def get_count(): + with open(tmpdir_file, "r", encoding="utf8") as f: + file_content = f.read() + return file_content.count(FILE_NAME) + + self.wait_until(lambda: get_count() > 0) + assert_equal(get_count(), 1) self.log.info("Test node is fully started") assert_equal(self.nodes[0].getblockcount(), 200) diff --git a/test/functional/feature_taproot.py b/test/functional/feature_taproot.py index 144e01c367..8be2040d91 100755 --- a/test/functional/feature_taproot.py +++ b/test/functional/feature_taproot.py @@ -19,6 +19,7 @@ from test_framework.messages import ( CTxInWitness, CTxOut, SEQUENCE_FINAL, + tx_from_hex, ) from test_framework.script import ( ANNEX_TAG, @@ -109,7 +110,6 @@ from test_framework.address import ( program_to_witness, ) from collections import OrderedDict, namedtuple -from io import BytesIO import json import hashlib import os @@ -750,7 +750,7 @@ def spenders_taproot_active(): # Reusing the scripts above, test that various features affect the sighash. add_spender(spenders, "sighash/annex", tap=tap, leaf="pk_codesep", key=secs[1], hashtype=hashtype, standard=False, **SINGLE_SIG, annex=bytes([ANNEX_TAG]), failure={"sighash": override(default_sighash, annex=None)}, **ERR_SIG_SCHNORR) add_spender(spenders, "sighash/script", tap=tap, leaf="pk_codesep", key=secs[1], **common, **SINGLE_SIG, failure={"sighash": override(default_sighash, script_taproot=tap.leaves["codesep_pk"].script)}, **ERR_SIG_SCHNORR) - add_spender(spenders, "sighash/leafver", tap=tap, leaf="pk_codesep", key=secs[1], **common, **SINGLE_SIG, failure={"sighash": override(default_sighash, leafversion=random.choice([x & 0xFE for x in range(0x100) if x & 0xFE != 0xC0]))}, **ERR_SIG_SCHNORR) + add_spender(spenders, "sighash/leafver", tap=tap, leaf="pk_codesep", key=secs[1], **common, **SINGLE_SIG, failure={"sighash": override(default_sighash, leafversion=random.choice([x & 0xFE for x in range(0x100) if x & 0xFE != LEAF_VERSION_TAPSCRIPT]))}, **ERR_SIG_SCHNORR) add_spender(spenders, "sighash/scriptpath", tap=tap, leaf="pk_codesep", key=secs[1], **common, **SINGLE_SIG, failure={"sighash": override(default_sighash, leaf=None)}, **ERR_SIG_SCHNORR) add_spender(spenders, "sighash/keypath", tap=tap, key=secs[0], **common, failure={"sighash": override(default_sighash, leaf="pk_codesep")}, **ERR_SIG_SCHNORR) @@ -1386,8 +1386,7 @@ class TaprootTest(BitcoinTestFramework): # Add change fund_tx.vout.append(CTxOut(balance - 10000, random.choice(host_spks))) # Ask the wallet to sign - ss = BytesIO(bytes.fromhex(node.signrawtransactionwithwallet(fund_tx.serialize().hex())["hex"])) - fund_tx.deserialize(ss) + fund_tx = tx_from_hex(node.signrawtransactionwithwallet(fund_tx.serialize().hex())["hex"]) # Construct UTXOData entries fund_tx.rehash() for i in range(count_this_tx): @@ -1555,12 +1554,16 @@ class TaprootTest(BitcoinTestFramework): script_lists = [ None, - [("0", CScript([pubs[50], OP_CHECKSIG]), 0xc0)], - [("0", CScript([pubs[51], OP_CHECKSIG]), 0xc0)], - [("0", CScript([pubs[52], OP_CHECKSIG]), 0xc0), ("1", CScript([b"BIP341"]), VALID_LEAF_VERS[pubs[99][0] % 41])], - [("0", CScript([pubs[53], OP_CHECKSIG]), 0xc0), ("1", CScript([b"Taproot"]), VALID_LEAF_VERS[pubs[99][1] % 41])], - [("0", CScript([pubs[54], OP_CHECKSIG]), 0xc0), [("1", CScript([pubs[55], OP_CHECKSIG]), 0xc0), ("2", CScript([pubs[56], OP_CHECKSIG]), 0xc0)]], - [("0", CScript([pubs[57], OP_CHECKSIG]), 0xc0), [("1", CScript([pubs[58], OP_CHECKSIG]), 0xc0), ("2", CScript([pubs[59], OP_CHECKSIG]), 0xc0)]], + [("0", CScript([pubs[50], OP_CHECKSIG]), LEAF_VERSION_TAPSCRIPT)], + [("0", CScript([pubs[51], OP_CHECKSIG]), LEAF_VERSION_TAPSCRIPT)], + [("0", CScript([pubs[52], OP_CHECKSIG]), LEAF_VERSION_TAPSCRIPT), ("1", CScript([b"BIP341"]), VALID_LEAF_VERS[pubs[99][0] % 41])], + [("0", CScript([pubs[53], OP_CHECKSIG]), LEAF_VERSION_TAPSCRIPT), ("1", CScript([b"Taproot"]), VALID_LEAF_VERS[pubs[99][1] % 41])], + [("0", CScript([pubs[54], OP_CHECKSIG]), LEAF_VERSION_TAPSCRIPT), + [("1", CScript([pubs[55], OP_CHECKSIG]), LEAF_VERSION_TAPSCRIPT), ("2", CScript([pubs[56], OP_CHECKSIG]), LEAF_VERSION_TAPSCRIPT)] + ], + [("0", CScript([pubs[57], OP_CHECKSIG]), LEAF_VERSION_TAPSCRIPT), + [("1", CScript([pubs[58], OP_CHECKSIG]), LEAF_VERSION_TAPSCRIPT), ("2", CScript([pubs[59], OP_CHECKSIG]), LEAF_VERSION_TAPSCRIPT)] + ], ] taps = [taproot_construct(inner_keys[i], script_lists[i]) for i in range(len(inner_keys))] diff --git a/test/functional/feature_txindex_compatibility.py b/test/functional/feature_txindex_compatibility.py index dd18c5fd99..48fefaa0ba 100755 --- a/test/functional/feature_txindex_compatibility.py +++ b/test/functional/feature_txindex_compatibility.py @@ -42,7 +42,6 @@ class TxindexCompatibilityTest(BitcoinTestFramework): def run_test(self): mini_wallet = MiniWallet(self.nodes[1]) - mini_wallet.rescan_utxos() spend_utxo = mini_wallet.get_utxo() mini_wallet.send_self_transfer(from_node=self.nodes[1], utxo_to_spend=spend_utxo) self.generate(self.nodes[1], 1) diff --git a/test/functional/interface_rest.py b/test/functional/interface_rest.py index db2ff19b44..5017f77d18 100755 --- a/test/functional/interface_rest.py +++ b/test/functional/interface_rest.py @@ -7,9 +7,7 @@ from decimal import Decimal from enum import Enum import http.client -from io import BytesIO import json -from struct import pack, unpack import typing import urllib.parse @@ -96,7 +94,6 @@ class RESTTest (BitcoinTestFramework): def run_test(self): self.url = urllib.parse.urlparse(self.nodes[0].url) self.wallet = MiniWallet(self.nodes[0]) - self.wallet.rescan_utxos() self.log.info("Broadcast test transaction and sync nodes") txid, _ = self.wallet.send_to(from_node=self.nodes[0], scriptPubKey=getnewdestination()[1], amount=int(0.1 * COIN)) @@ -161,12 +158,11 @@ class RESTTest (BitcoinTestFramework): bin_request = b'\x01\x02' for txid, n in [spending, spent]: bin_request += bytes.fromhex(txid) - bin_request += pack("i", n) + bin_request += n.to_bytes(4, 'little') bin_response = self.test_rest_request("/getutxos", http_method='POST', req_type=ReqType.BIN, body=bin_request, ret_type=RetType.BYTES) - output = BytesIO(bin_response) - chain_height, = unpack("<i", output.read(4)) - response_hash = output.read(32)[::-1].hex() + chain_height = int.from_bytes(bin_response[0:4], 'little') + response_hash = bin_response[4:36][::-1].hex() assert_equal(bb_hash, response_hash) # check if getutxo's chaintip during calculation was fine assert_equal(chain_height, 201) # chain height must be 201 (pre-mined chain [200] + generated block [1]) @@ -281,6 +277,11 @@ class RESTTest (BitcoinTestFramework): assert_equal(len(json_obj), 1) # ensure that there is one header in the json response assert_equal(json_obj[0]['hash'], bb_hash) # request/response hash should be the same + # Check invalid uri (% symbol at the end of the request) + for invalid_uri in [f"/headers/{bb_hash}%", f"/blockfilterheaders/basic/{bb_hash}%", "/mempool/contents.json?%"]: + resp = self.test_rest_request(invalid_uri, ret_type=RetType.OBJ, status=400) + assert_equal(resp.read().decode('utf-8').rstrip(), "URI parsing failed, it likely contained RFC 3986 invalid characters") + # Compare with normal RPC block response rpc_block_json = self.nodes[0].getblock(bb_hash) for key in ['hash', 'confirmations', 'height', 'version', 'merkleroot', 'time', 'nonce', 'bits', 'difficulty', 'chainwork', 'previousblockhash']: @@ -348,6 +349,34 @@ class RESTTest (BitcoinTestFramework): assert_equal(json_obj[tx]['spentby'], txs[i + 1:i + 2]) assert_equal(json_obj[tx]['depends'], txs[i - 1:i]) + # Check the mempool response for explicit parameters + json_obj = self.test_rest_request("/mempool/contents", query_params={"verbose": "true", "mempool_sequence": "false"}) + assert_equal(json_obj, raw_mempool_verbose) + + # Check the mempool response for not verbose + json_obj = self.test_rest_request("/mempool/contents", query_params={"verbose": "false"}) + raw_mempool = self.nodes[0].getrawmempool(verbose=False) + + assert_equal(json_obj, raw_mempool) + + # Check the mempool response for sequence + json_obj = self.test_rest_request("/mempool/contents", query_params={"verbose": "false", "mempool_sequence": "true"}) + raw_mempool = self.nodes[0].getrawmempool(verbose=False, mempool_sequence=True) + + assert_equal(json_obj, raw_mempool) + + # Check for error response if verbose=true and mempool_sequence=true + resp = self.test_rest_request("/mempool/contents", ret_type=RetType.OBJ, status=400, query_params={"verbose": "true", "mempool_sequence": "true"}) + assert_equal(resp.read().decode('utf-8').strip(), 'Verbose results cannot contain mempool sequence values. (hint: set "verbose=false")') + + # Check for error response if verbose is not "true" or "false" + resp = self.test_rest_request("/mempool/contents", ret_type=RetType.OBJ, status=400, query_params={"verbose": "TRUE"}) + assert_equal(resp.read().decode('utf-8').strip(), 'The "verbose" query parameter must be either "true" or "false".') + + # Check for error response if mempool_sequence is not "true" or "false" + resp = self.test_rest_request("/mempool/contents", ret_type=RetType.OBJ, status=400, query_params={"verbose": "false", "mempool_sequence": "TRUE"}) + assert_equal(resp.read().decode('utf-8').strip(), 'The "mempool_sequence" query parameter must be either "true" or "false".') + # Now mine the transactions newblockhash = self.generate(self.nodes[1], 1) diff --git a/test/functional/interface_usdt_mempool.py b/test/functional/interface_usdt_mempool.py new file mode 100755 index 0000000000..ec2f9e4e77 --- /dev/null +++ b/test/functional/interface_usdt_mempool.py @@ -0,0 +1,343 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" Tests the mempool:* tracepoint API interface. + See https://github.com/bitcoin/bitcoin/blob/master/doc/tracing.md#context-mempool +""" + +from decimal import Decimal + +# Test will be skipped if we don't have bcc installed +try: + from bcc import BPF, USDT # type: ignore[import] +except ImportError: + pass + +from test_framework.blocktools import COINBASE_MATURITY +from test_framework.messages import COIN, DEFAULT_MEMPOOL_EXPIRY_HOURS +from test_framework.p2p import P2PDataStore +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal +from test_framework.wallet import MiniWallet + +MEMPOOL_TRACEPOINTS_PROGRAM = """ +# include <uapi/linux/ptrace.h> + +// The longest rejection reason is 118 chars and is generated in case of SCRIPT_ERR_EVAL_FALSE by +// strprintf("mandatory-script-verify-flag-failed (%s)", ScriptErrorString(check.GetScriptError())) +#define MAX_REJECT_REASON_LENGTH 118 +// The longest string returned by RemovalReasonToString() is 'sizelimit' +#define MAX_REMOVAL_REASON_LENGTH 9 +#define HASH_LENGTH 32 + +struct added_event +{ + u8 hash[HASH_LENGTH]; + u64 vsize; + s64 fee; +}; + +struct removed_event +{ + u8 hash[HASH_LENGTH]; + char reason[MAX_REMOVAL_REASON_LENGTH]; + u64 vsize; + s64 fee; + u64 entry_time; +}; + +struct rejected_event +{ + u8 hash[HASH_LENGTH]; + char reason[MAX_REJECT_REASON_LENGTH]; +}; + +struct replaced_event +{ + u8 replaced_hash[HASH_LENGTH]; + u64 replaced_vsize; + s64 replaced_fee; + u64 replaced_entry_time; + u8 replacement_hash[HASH_LENGTH]; + u64 replacement_vsize; + s64 replacement_fee; +}; + +// BPF perf buffer to push the data to user space. +BPF_PERF_OUTPUT(added_events); +BPF_PERF_OUTPUT(removed_events); +BPF_PERF_OUTPUT(rejected_events); +BPF_PERF_OUTPUT(replaced_events); + +int trace_added(struct pt_regs *ctx) { + struct added_event added = {}; + + bpf_usdt_readarg_p(1, ctx, &added.hash, HASH_LENGTH); + bpf_usdt_readarg(2, ctx, &added.vsize); + bpf_usdt_readarg(3, ctx, &added.fee); + + added_events.perf_submit(ctx, &added, sizeof(added)); + return 0; +} + +int trace_removed(struct pt_regs *ctx) { + struct removed_event removed = {}; + + bpf_usdt_readarg_p(1, ctx, &removed.hash, HASH_LENGTH); + bpf_usdt_readarg_p(2, ctx, &removed.reason, MAX_REMOVAL_REASON_LENGTH); + bpf_usdt_readarg(3, ctx, &removed.vsize); + bpf_usdt_readarg(4, ctx, &removed.fee); + bpf_usdt_readarg(5, ctx, &removed.entry_time); + + removed_events.perf_submit(ctx, &removed, sizeof(removed)); + return 0; +} + +int trace_rejected(struct pt_regs *ctx) { + struct rejected_event rejected = {}; + + bpf_usdt_readarg_p(1, ctx, &rejected.hash, HASH_LENGTH); + bpf_usdt_readarg_p(2, ctx, &rejected.reason, MAX_REJECT_REASON_LENGTH); + + rejected_events.perf_submit(ctx, &rejected, sizeof(rejected)); + return 0; +} + +int trace_replaced(struct pt_regs *ctx) { + struct replaced_event replaced = {}; + + bpf_usdt_readarg_p(1, ctx, &replaced.replaced_hash, HASH_LENGTH); + bpf_usdt_readarg(2, ctx, &replaced.replaced_vsize); + bpf_usdt_readarg(3, ctx, &replaced.replaced_fee); + bpf_usdt_readarg(4, ctx, &replaced.replaced_entry_time); + bpf_usdt_readarg_p(5, ctx, &replaced.replacement_hash, HASH_LENGTH); + bpf_usdt_readarg(6, ctx, &replaced.replacement_vsize); + bpf_usdt_readarg(7, ctx, &replaced.replacement_fee); + + replaced_events.perf_submit(ctx, &replaced, sizeof(replaced)); + return 0; +} +""" + + +class MempoolTracepointTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + self.setup_clean_chain = True + + def skip_test_if_missing_module(self): + self.skip_if_platform_not_linux() + self.skip_if_no_bitcoind_tracepoints() + self.skip_if_no_python_bcc() + self.skip_if_no_bpf_permissions() + + def added_test(self): + """Add a transaction to the mempool and make sure the tracepoint returns + the expected txid, vsize, and fee.""" + + EXPECTED_ADDED_EVENTS = 1 + handled_added_events = 0 + + self.log.info("Hooking into mempool:added tracepoint...") + node = self.nodes[0] + ctx = USDT(pid=node.process.pid) + ctx.enable_probe(probe="mempool:added", fn_name="trace_added") + bpf = BPF(text=MEMPOOL_TRACEPOINTS_PROGRAM, usdt_contexts=[ctx], debug=0) + + def handle_added_event(_, data, __): + nonlocal handled_added_events + event = bpf["added_events"].event(data) + assert_equal(txid, bytes(event.hash)[::-1].hex()) + assert_equal(vsize, event.vsize) + assert_equal(fee, event.fee) + handled_added_events += 1 + + bpf["added_events"].open_perf_buffer(handle_added_event) + + self.log.info("Sending transaction...") + fee = Decimal(31200) + tx = self.wallet.send_self_transfer(from_node=node, fee=fee / COIN) + # expected data + txid = tx["txid"] + vsize = tx["tx"].get_vsize() + + self.log.info("Polling buffer...") + bpf.perf_buffer_poll(timeout=200) + + self.log.info("Cleaning up mempool...") + self.generate(node, 1) + + bpf.cleanup() + + self.log.info("Ensuring mempool:added event was handled successfully...") + assert_equal(EXPECTED_ADDED_EVENTS, handled_added_events) + + def removed_test(self): + """Expire a transaction from the mempool and make sure the tracepoint returns + the expected txid, expiry reason, vsize, and fee.""" + + EXPECTED_REMOVED_EVENTS = 1 + handled_removed_events = 0 + + self.log.info("Hooking into mempool:removed tracepoint...") + node = self.nodes[0] + ctx = USDT(pid=node.process.pid) + ctx.enable_probe(probe="mempool:removed", fn_name="trace_removed") + bpf = BPF(text=MEMPOOL_TRACEPOINTS_PROGRAM, usdt_contexts=[ctx], debug=0) + + def handle_removed_event(_, data, __): + nonlocal handled_removed_events + event = bpf["removed_events"].event(data) + assert_equal(txid, bytes(event.hash)[::-1].hex()) + assert_equal(reason, event.reason.decode("UTF-8")) + assert_equal(vsize, event.vsize) + assert_equal(fee, event.fee) + assert_equal(entry_time, event.entry_time) + handled_removed_events += 1 + + bpf["removed_events"].open_perf_buffer(handle_removed_event) + + self.log.info("Sending transaction...") + fee = Decimal(31200) + tx = self.wallet.send_self_transfer(from_node=node, fee=fee / COIN) + # expected data + txid = tx["txid"] + reason = "expiry" + vsize = tx["tx"].get_vsize() + + self.log.info("Fast-forwarding time to mempool expiry...") + entry_time = node.getmempoolentry(txid)["time"] + expiry_time = entry_time + 60 * 60 * DEFAULT_MEMPOOL_EXPIRY_HOURS + 5 + node.setmocktime(expiry_time) + + self.log.info("Triggering expiry...") + self.wallet.get_utxo(txid=txid) + self.wallet.send_self_transfer(from_node=node) + + self.log.info("Polling buffer...") + bpf.perf_buffer_poll(timeout=200) + + bpf.cleanup() + + self.log.info("Ensuring mempool:removed event was handled successfully...") + assert_equal(EXPECTED_REMOVED_EVENTS, handled_removed_events) + + def replaced_test(self): + """Replace one and two transactions in the mempool and make sure the tracepoint + returns the expected txids, vsizes, and fees.""" + + EXPECTED_REPLACED_EVENTS = 1 + handled_replaced_events = 0 + + self.log.info("Hooking into mempool:replaced tracepoint...") + node = self.nodes[0] + ctx = USDT(pid=node.process.pid) + ctx.enable_probe(probe="mempool:replaced", fn_name="trace_replaced") + bpf = BPF(text=MEMPOOL_TRACEPOINTS_PROGRAM, usdt_contexts=[ctx], debug=0) + + def handle_replaced_event(_, data, __): + nonlocal handled_replaced_events + event = bpf["replaced_events"].event(data) + assert_equal(replaced_txid, bytes(event.replaced_hash)[::-1].hex()) + assert_equal(replaced_vsize, event.replaced_vsize) + assert_equal(replaced_fee, event.replaced_fee) + assert_equal(replaced_entry_time, event.replaced_entry_time) + assert_equal(replacement_txid, bytes(event.replacement_hash)[::-1].hex()) + assert_equal(replacement_vsize, event.replacement_vsize) + assert_equal(replacement_fee, event.replacement_fee) + handled_replaced_events += 1 + + bpf["replaced_events"].open_perf_buffer(handle_replaced_event) + + self.log.info("Sending RBF transaction...") + utxo = self.wallet.get_utxo(mark_as_spent=True) + original_fee = Decimal(40000) + original_tx = self.wallet.send_self_transfer( + from_node=node, utxo_to_spend=utxo, fee=original_fee / COIN + ) + entry_time = node.getmempoolentry(original_tx["txid"])["time"] + + self.log.info("Sending replacement transaction...") + replacement_fee = Decimal(45000) + replacement_tx = self.wallet.send_self_transfer( + from_node=node, utxo_to_spend=utxo, fee=replacement_fee / COIN + ) + + # expected data + replaced_txid = original_tx["txid"] + replaced_vsize = original_tx["tx"].get_vsize() + replaced_fee = original_fee + replaced_entry_time = entry_time + replacement_txid = replacement_tx["txid"] + replacement_vsize = replacement_tx["tx"].get_vsize() + + self.log.info("Polling buffer...") + bpf.perf_buffer_poll(timeout=200) + + bpf.cleanup() + + self.log.info("Ensuring mempool:replaced event was handled successfully...") + assert_equal(EXPECTED_REPLACED_EVENTS, handled_replaced_events) + + def rejected_test(self): + """Create an invalid transaction and make sure the tracepoint returns + the expected txid, rejection reason, peer id, and peer address.""" + + EXPECTED_REJECTED_EVENTS = 1 + handled_rejected_events = 0 + + self.log.info("Adding P2P connection...") + node = self.nodes[0] + node.add_p2p_connection(P2PDataStore()) + + self.log.info("Hooking into mempool:rejected tracepoint...") + ctx = USDT(pid=node.process.pid) + ctx.enable_probe(probe="mempool:rejected", fn_name="trace_rejected") + bpf = BPF(text=MEMPOOL_TRACEPOINTS_PROGRAM, usdt_contexts=[ctx], debug=0) + + def handle_rejected_event(_, data, __): + nonlocal handled_rejected_events + event = bpf["rejected_events"].event(data) + assert_equal(txid, bytes(event.hash)[::-1].hex()) + assert_equal(reason, event.reason.decode("UTF-8")) + handled_rejected_events += 1 + + bpf["rejected_events"].open_perf_buffer(handle_rejected_event) + + self.log.info("Sending invalid transaction...") + tx = self.wallet.create_self_transfer(fee_rate=Decimal(0)) + node.p2ps[0].send_txs_and_test([tx["tx"]], node, success=False) + + # expected data + txid = tx["tx"].hash + reason = "min relay fee not met" + + self.log.info("Polling buffer...") + bpf.perf_buffer_poll(timeout=200) + + bpf.cleanup() + + self.log.info("Ensuring mempool:rejected event was handled successfully...") + assert_equal(EXPECTED_REJECTED_EVENTS, handled_rejected_events) + + def run_test(self): + """Tests the mempool:added, mempool:removed, mempool:replaced, + and mempool:rejected tracepoints.""" + + # Create some coinbase transactions and mature them so they can be spent + node = self.nodes[0] + self.wallet = MiniWallet(node) + self.generate(self.wallet, 4) + self.generate(node, COINBASE_MATURITY) + + # Test individual tracepoints + self.added_test() + self.removed_test() + self.replaced_test() + self.rejected_test() + + +if __name__ == "__main__": + MempoolTracepointTest().main() diff --git a/test/functional/interface_usdt_utxocache.py b/test/functional/interface_usdt_utxocache.py index e3b0b32f0d..23785b1e8a 100755 --- a/test/functional/interface_usdt_utxocache.py +++ b/test/functional/interface_usdt_utxocache.py @@ -144,7 +144,6 @@ class UTXOCacheTracepointTest(BitcoinTestFramework): def run_test(self): self.wallet = MiniWallet(self.nodes[0]) - self.generate(self.wallet, 101) self.test_uncache() self.test_add_spent() diff --git a/test/functional/interface_zmq.py b/test/functional/interface_zmq.py index 00cd3b4894..2358dd4387 100755 --- a/test/functional/interface_zmq.py +++ b/test/functional/interface_zmq.py @@ -4,6 +4,7 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the ZMQ notification interface.""" import struct +from time import sleep from test_framework.address import ( ADDRESS_BCRT1_P2WSH_OP_TRUE, @@ -16,19 +17,19 @@ from test_framework.blocktools import ( ) from test_framework.test_framework import BitcoinTestFramework from test_framework.messages import ( - CTransaction, hash256, + tx_from_hex, ) from test_framework.util import ( assert_equal, assert_raises_rpc_error, + p2p_port, ) from test_framework.wallet import ( MiniWallet, ) from test_framework.netutil import test_ipv6_local -from io import BytesIO -from time import sleep + # Test may be skipped and not have zmq installed try: @@ -106,6 +107,7 @@ class ZMQTest (BitcoinTestFramework): # This test isn't testing txn relay/timing, so set whitelist on the # peers for instant txn relay. This speeds up the test run time 2-3x. self.extra_args = [["-whitelist=noban@127.0.0.1"]] * self.num_nodes + self.zmq_port_base = p2p_port(self.num_nodes + 1) def skip_test_if_missing_module(self): self.skip_if_no_py3_zmq() @@ -179,7 +181,7 @@ class ZMQTest (BitcoinTestFramework): # Invalid zmq arguments don't take down the node, see #17185. self.restart_node(0, ["-zmqpubrawtx=foo", "-zmqpubhashtx=bar"]) - address = 'tcp://127.0.0.1:28332' + address = f"tcp://127.0.0.1:{self.zmq_port_base}" subs = self.setup_zmq_test([(topic, address) for topic in ["hashblock", "hashtx", "rawblock", "rawtx"]]) hashblock = subs[0] @@ -196,9 +198,7 @@ class ZMQTest (BitcoinTestFramework): txid = hashtx.receive() # Should receive the coinbase raw transaction. - hex = rawtx.receive() - tx = CTransaction() - tx.deserialize(BytesIO(hex)) + tx = tx_from_hex(rawtx.receive().hex()) tx.calc_sha256() assert_equal(tx.hash, txid.hex()) @@ -213,7 +213,6 @@ class ZMQTest (BitcoinTestFramework): assert_equal([txid.hex()], self.nodes[1].getblock(hash)["tx"]) - self.wallet.rescan_utxos() self.log.info("Wait for tx from second node") payment_tx = self.wallet.send_self_transfer(from_node=self.nodes[1]) payment_txid = payment_tx['txid'] @@ -246,7 +245,7 @@ class ZMQTest (BitcoinTestFramework): def test_reorg(self): - address = 'tcp://127.0.0.1:28333' + address = f"tcp://127.0.0.1:{self.zmq_port_base}" # Should only notify the tip if a reorg occurs hashblock, hashtx = self.setup_zmq_test( @@ -300,7 +299,7 @@ class ZMQTest (BitcoinTestFramework): <32-byte hash>A<8-byte LE uint> : Transactionhash added mempool """ self.log.info("Testing 'sequence' publisher") - [seq] = self.setup_zmq_test([("sequence", "tcp://127.0.0.1:28333")]) + [seq] = self.setup_zmq_test([("sequence", f"tcp://127.0.0.1:{self.zmq_port_base}")]) self.disconnect_nodes(0, 1) # Mempool sequence number starts at 1 @@ -444,7 +443,7 @@ class ZMQTest (BitcoinTestFramework): """ self.log.info("Testing 'mempool sync' usage of sequence notifier") - [seq] = self.setup_zmq_test([("sequence", "tcp://127.0.0.1:28333")]) + [seq] = self.setup_zmq_test([("sequence", f"tcp://127.0.0.1:{self.zmq_port_base}")]) # In-memory counter, should always start at 1 next_mempool_seq = self.nodes[0].getrawmempool(mempool_sequence=True)["mempool_sequence"] @@ -549,8 +548,8 @@ class ZMQTest (BitcoinTestFramework): # chain lengths on node0 and node1; for this test we only need node0, so # we can disable syncing blocks on the setup) subscribers = self.setup_zmq_test([ - ("hashblock", "tcp://127.0.0.1:28334"), - ("hashblock", "tcp://127.0.0.1:28335"), + ("hashblock", f"tcp://127.0.0.1:{self.zmq_port_base + 1}"), + ("hashblock", f"tcp://127.0.0.1:{self.zmq_port_base + 2}"), ], sync_blocks=False) # Generate 1 block in nodes[0] and receive all notifications @@ -567,7 +566,7 @@ class ZMQTest (BitcoinTestFramework): self.log.info("Testing IPv6") # Set up subscriber using IPv6 loopback address subscribers = self.setup_zmq_test([ - ("hashblock", "tcp://[::1]:28332") + ("hashblock", f"tcp://[::1]:{self.zmq_port_base}") ], ipv6=True) # Generate 1 block in nodes[0] diff --git a/test/functional/mempool_accept.py b/test/functional/mempool_accept.py index abbca15bed..362b407062 100755 --- a/test/functional/mempool_accept.py +++ b/test/functional/mempool_accept.py @@ -58,14 +58,17 @@ class MempoolAcceptanceTest(BitcoinTestFramework): """Wrapper to check result of testmempoolaccept on node_0's mempool""" result_test = self.nodes[0].testmempoolaccept(*args, **kwargs) for r in result_test: - r.pop('wtxid') # Skip check for now + # Skip these checks for now + r.pop('wtxid') + if "fees" in r: + r["fees"].pop("effective-feerate") + r["fees"].pop("effective-includes") assert_equal(result_expected, result_test) assert_equal(self.nodes[0].getmempoolinfo()['size'], self.mempool_size) # Must not change mempool state def run_test(self): node = self.nodes[0] self.wallet = MiniWallet(node) - self.wallet.rescan_utxos() self.log.info('Start with empty mempool, and 200 blocks') self.mempool_size = 0 diff --git a/test/functional/mempool_datacarrier.py b/test/functional/mempool_datacarrier.py index 9c82964a24..c370d8fa91 100755 --- a/test/functional/mempool_datacarrier.py +++ b/test/functional/mempool_datacarrier.py @@ -44,7 +44,6 @@ class DataCarrierTest(BitcoinTestFramework): def run_test(self): self.wallet = MiniWallet(self.nodes[0]) - self.wallet.rescan_utxos() # By default, only 80 bytes are used for data (+1 for OP_RETURN, +2 for the pushdata opcodes). default_size_data = random_bytes(MAX_OP_RETURN_RELAY - 3) diff --git a/test/functional/mempool_dust.py b/test/functional/mempool_dust.py new file mode 100755 index 0000000000..41a26e82da --- /dev/null +++ b/test/functional/mempool_dust.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test dust limit mempool policy (`-dustrelayfee` parameter)""" +from decimal import Decimal + +from test_framework.key import ECKey +from test_framework.messages import ( + COIN, + CTxOut, +) +from test_framework.script import ( + CScript, + OP_RETURN, + OP_TRUE, +) +from test_framework.script_util import ( + key_to_p2pk_script, + key_to_p2pkh_script, + key_to_p2wpkh_script, + keys_to_multisig_script, + output_key_to_p2tr_script, + program_to_witness_script, + script_to_p2sh_script, + script_to_p2wsh_script, +) +from test_framework.test_framework import BitcoinTestFramework +from test_framework.test_node import TestNode +from test_framework.util import ( + assert_equal, + get_fee, +) +from test_framework.wallet import MiniWallet + + +DUST_RELAY_TX_FEE = 3000 # default setting [sat/kvB] + + +class DustRelayFeeTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + + def test_dust_output(self, node: TestNode, dust_relay_fee: Decimal, + output_script: CScript, type_desc: str) -> None: + # determine dust threshold (see `GetDustThreshold`) + if output_script[0] == OP_RETURN: + dust_threshold = 0 + else: + tx_size = len(CTxOut(nValue=0, scriptPubKey=output_script).serialize()) + tx_size += 67 if output_script.IsWitnessProgram() else 148 + dust_threshold = int(get_fee(tx_size, dust_relay_fee) * COIN) + self.log.info(f"-> Test {type_desc} output (size {len(output_script)}, limit {dust_threshold})") + + # amount right on the dust threshold should pass + tx = self.wallet.create_self_transfer()["tx"] + tx.vout.append(CTxOut(nValue=dust_threshold, scriptPubKey=output_script)) + tx.vout[0].nValue -= dust_threshold # keep total output value constant + tx_good_hex = tx.serialize().hex() + res = node.testmempoolaccept([tx_good_hex])[0] + assert_equal(res['allowed'], True) + + # amount just below the dust threshold should fail + if dust_threshold > 0: + tx.vout[1].nValue -= 1 + res = node.testmempoolaccept([tx.serialize().hex()])[0] + assert_equal(res['allowed'], False) + assert_equal(res['reject-reason'], 'dust') + + # finally send the transaction to avoid running out of MiniWallet UTXOs + self.wallet.sendrawtransaction(from_node=node, tx_hex=tx_good_hex) + + def run_test(self): + self.wallet = MiniWallet(self.nodes[0]) + + # prepare output scripts of each standard type + key = ECKey() + key.generate(compressed=False) + uncompressed_pubkey = key.get_pubkey().get_bytes() + key.generate(compressed=True) + pubkey = key.get_pubkey().get_bytes() + + output_scripts = ( + (key_to_p2pk_script(uncompressed_pubkey), "P2PK (uncompressed)"), + (key_to_p2pk_script(pubkey), "P2PK (compressed)"), + (key_to_p2pkh_script(pubkey), "P2PKH"), + (script_to_p2sh_script(CScript([OP_TRUE])), "P2SH"), + (key_to_p2wpkh_script(pubkey), "P2WPKH"), + (script_to_p2wsh_script(CScript([OP_TRUE])), "P2WSH"), + (output_key_to_p2tr_script(pubkey[1:]), "P2TR"), + # witness programs for segwitv2+ can be between 2 and 40 bytes + (program_to_witness_script(2, b'\x66' * 2), "P2?? (future witness version 2)"), + (program_to_witness_script(16, b'\x77' * 40), "P2?? (future witness version 16)"), + # largest possible output script considered standard + (keys_to_multisig_script([uncompressed_pubkey]*3), "bare multisig (m-of-3)"), + (CScript([OP_RETURN, b'superimportanthash']), "null data (OP_RETURN)"), + ) + + # test default (no parameter), disabled (=0) and a bunch of arbitrary dust fee rates [sat/kvB] + for dustfee_sat_kvb in (DUST_RELAY_TX_FEE, 0, 1, 66, 500, 1337, 12345, 21212, 333333): + dustfee_btc_kvb = dustfee_sat_kvb / Decimal(COIN) + if dustfee_sat_kvb == DUST_RELAY_TX_FEE: + self.log.info(f"Test default dust limit setting ({dustfee_sat_kvb} sat/kvB)...") + else: + dust_parameter = f"-dustrelayfee={dustfee_btc_kvb:.8f}" + self.log.info(f"Test dust limit setting {dust_parameter} ({dustfee_sat_kvb} sat/kvB)...") + self.restart_node(0, extra_args=[dust_parameter]) + + for output_script, description in output_scripts: + self.test_dust_output(self.nodes[0], dustfee_btc_kvb, output_script, description) + self.generate(self.nodes[0], 1) + + +if __name__ == '__main__': + DustRelayFeeTest().main() diff --git a/test/functional/mempool_expiry.py b/test/functional/mempool_expiry.py index 0c91da901f..15a5f765df 100755 --- a/test/functional/mempool_expiry.py +++ b/test/functional/mempool_expiry.py @@ -12,7 +12,6 @@ definable expiry timeout via the '-mempoolexpiry=<n>' command line argument from datetime import timedelta -from test_framework.blocktools import COINBASE_MATURITY from test_framework.messages import DEFAULT_MEMPOOL_EXPIRY_HOURS from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( @@ -27,17 +26,11 @@ CUSTOM_MEMPOOL_EXPIRY = 10 # hours class MempoolExpiryTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 - self.setup_clean_chain = True def test_transaction_expiry(self, timeout): """Tests that a transaction expires after the expiry timeout and its children are removed as well.""" node = self.nodes[0] - self.wallet = MiniWallet(node) - - # Add enough mature utxos to the wallet so that all txs spend confirmed coins. - self.generate(self.wallet, 4) - self.generate(node, COINBASE_MATURITY) # Send a parent transaction that will expire. parent_txid = self.wallet.send_self_transfer(from_node=node)['txid'] @@ -97,6 +90,8 @@ class MempoolExpiryTest(BitcoinTestFramework): assert_equal(half_expiry_time, node.getmempoolentry(independent_txid)['time']) def run_test(self): + self.wallet = MiniWallet(self.nodes[0]) + self.log.info('Test default mempool expiry timeout of %d hours.' % DEFAULT_MEMPOOL_EXPIRY_HOURS) self.test_transaction_expiry(DEFAULT_MEMPOOL_EXPIRY_HOURS) diff --git a/test/functional/mempool_limit.py b/test/functional/mempool_limit.py index 07636439d0..f3f4b42ad0 100755 --- a/test/functional/mempool_limit.py +++ b/test/functional/mempool_limit.py @@ -7,15 +7,21 @@ from decimal import Decimal from test_framework.blocktools import COINBASE_MATURITY +from test_framework.p2p import P2PTxInvStore from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, + assert_fee_amount, assert_greater_than, assert_raises_rpc_error, create_lots_of_big_transactions, gen_return_txouts, ) -from test_framework.wallet import MiniWallet +from test_framework.wallet import ( + COIN, + DEFAULT_FEE, + MiniWallet, +) class MempoolLimitTest(BitcoinTestFramework): @@ -25,7 +31,6 @@ class MempoolLimitTest(BitcoinTestFramework): self.extra_args = [[ "-datacarriersize=100000", "-maxmempool=5", - "-spendzeroconfchange=0", ]] self.supports_cli = False @@ -39,13 +44,14 @@ class MempoolLimitTest(BitcoinTestFramework): assert_equal(node.getmempoolinfo()['minrelaytxfee'], Decimal('0.00001000')) assert_equal(node.getmempoolinfo()['mempoolminfee'], Decimal('0.00001000')) - tx_batch_size = 25 - num_of_batches = 3 + tx_batch_size = 1 + num_of_batches = 75 # Generate UTXOs to flood the mempool # 1 to create a tx initially that will be evicted from the mempool later # 3 batches of multiple transactions with a fee rate much higher than the previous UTXO # And 1 more to verify that this tx does not get added to the mempool with a fee rate less than the mempoolminfee - self.generate(miniwallet, 1 + (num_of_batches * tx_batch_size) + 1) + # And 2 more for the package cpfp test + self.generate(miniwallet, 1 + (num_of_batches * tx_batch_size) + 1 + 2) # Mine 99 blocks so that the UTXOs are allowed to be spent self.generate(node, COINBASE_MATURITY - 1) @@ -77,6 +83,68 @@ class MempoolLimitTest(BitcoinTestFramework): self.log.info('Create a mempool tx that will not pass mempoolminfee') assert_raises_rpc_error(-26, "mempool min fee not met", miniwallet.send_self_transfer, from_node=node, fee_rate=relayfee) + self.log.info("Check that submitpackage allows cpfp of a parent below mempool min feerate") + node = self.nodes[0] + peer = node.add_p2p_connection(P2PTxInvStore()) + + # Package with 2 parents and 1 child. One parent has a high feerate due to modified fees, + # another is below the mempool minimum feerate but bumped by the child. + tx_poor = miniwallet.create_self_transfer(fee_rate=relayfee) + tx_rich = miniwallet.create_self_transfer(fee=0, fee_rate=0) + node.prioritisetransaction(tx_rich["txid"], 0, int(DEFAULT_FEE * COIN)) + package_txns = [tx_rich, tx_poor] + coins = [tx["new_utxo"] for tx in package_txns] + tx_child = miniwallet.create_self_transfer_multi(utxos_to_spend=coins, fee_per_output=10000) #DEFAULT_FEE + package_txns.append(tx_child) + + submitpackage_result = node.submitpackage([tx["hex"] for tx in package_txns]) + + rich_parent_result = submitpackage_result["tx-results"][tx_rich["wtxid"]] + poor_parent_result = submitpackage_result["tx-results"][tx_poor["wtxid"]] + child_result = submitpackage_result["tx-results"][tx_child["tx"].getwtxid()] + assert_fee_amount(poor_parent_result["fees"]["base"], tx_poor["tx"].get_vsize(), relayfee) + assert_equal(rich_parent_result["fees"]["base"], 0) + assert_equal(child_result["fees"]["base"], DEFAULT_FEE) + # The "rich" parent does not require CPFP so its effective feerate is just its individual feerate. + assert_fee_amount(DEFAULT_FEE, tx_rich["tx"].get_vsize(), rich_parent_result["fees"]["effective-feerate"]) + assert_equal(rich_parent_result["fees"]["effective-includes"], [tx_rich["wtxid"]]) + # The "poor" parent and child's effective feerates are the same, composed of their total + # fees divided by their combined vsize. + package_fees = poor_parent_result["fees"]["base"] + child_result["fees"]["base"] + package_vsize = tx_poor["tx"].get_vsize() + tx_child["tx"].get_vsize() + assert_fee_amount(package_fees, package_vsize, poor_parent_result["fees"]["effective-feerate"]) + assert_fee_amount(package_fees, package_vsize, child_result["fees"]["effective-feerate"]) + assert_equal([tx_poor["wtxid"], tx_child["tx"].getwtxid()], poor_parent_result["fees"]["effective-includes"]) + assert_equal([tx_poor["wtxid"], tx_child["tx"].getwtxid()], child_result["fees"]["effective-includes"]) + + # The node will broadcast each transaction, still abiding by its peer's fee filter + peer.wait_for_broadcast([tx["tx"].getwtxid() for tx in package_txns]) + + self.log.info("Check a package that passes mempoolminfee but is evicted immediately after submission") + mempoolmin_feerate = node.getmempoolinfo()["mempoolminfee"] + current_mempool = node.getrawmempool(verbose=False) + worst_feerate_btcvb = Decimal("21000000") + for txid in current_mempool: + entry = node.getmempoolentry(txid) + worst_feerate_btcvb = min(worst_feerate_btcvb, entry["fees"]["descendant"] / entry["descendantsize"]) + # Needs to be large enough to trigger eviction + target_weight_each = 200000 + assert_greater_than(target_weight_each * 2, node.getmempoolinfo()["maxmempool"] - node.getmempoolinfo()["bytes"]) + # Should be a true CPFP: parent's feerate is just below mempool min feerate + parent_fee = (mempoolmin_feerate / 1000) * (target_weight_each // 4) - Decimal("0.00001") + # Parent + child is above mempool minimum feerate + child_fee = (worst_feerate_btcvb) * (target_weight_each // 4) - Decimal("0.00001") + # However, when eviction is triggered, these transactions should be at the bottom. + # This assertion assumes parent and child are the same size. + miniwallet.rescan_utxos() + tx_parent_just_below = miniwallet.create_self_transfer(fee=parent_fee, target_weight=target_weight_each) + tx_child_just_above = miniwallet.create_self_transfer(utxo_to_spend=tx_parent_just_below["new_utxo"], fee=child_fee, target_weight=target_weight_each) + # This package ranks below the lowest descendant package in the mempool + assert_greater_than(worst_feerate_btcvb, (parent_fee + child_fee) / (tx_parent_just_below["tx"].get_vsize() + tx_child_just_above["tx"].get_vsize())) + assert_greater_than(mempoolmin_feerate, (parent_fee) / (tx_parent_just_below["tx"].get_vsize())) + assert_greater_than((parent_fee + child_fee) / (tx_parent_just_below["tx"].get_vsize() + tx_child_just_above["tx"].get_vsize()), mempoolmin_feerate / 1000) + assert_raises_rpc_error(-26, "mempool full", node.submitpackage, [tx_parent_just_below["hex"], tx_child_just_above["hex"]]) + self.log.info('Test passing a value below the minimum (5 MB) to -maxmempool throws an error') self.stop_node(0) self.nodes[0].assert_start_raises_init_error(["-maxmempool=4"], "Error: -maxmempool must be at least 5 MB") diff --git a/test/functional/mempool_package_limits.py b/test/functional/mempool_package_limits.py index 47b7be7d88..6143ae83de 100755 --- a/test/functional/mempool_package_limits.py +++ b/test/functional/mempool_package_limits.py @@ -3,20 +3,41 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test logic for limiting mempool and package ancestors/descendants.""" - -from decimal import Decimal - from test_framework.blocktools import COINBASE_MATURITY -from test_framework.test_framework import BitcoinTestFramework from test_framework.messages import ( - COIN, WITNESS_SCALE_FACTOR, ) +from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, ) from test_framework.wallet import MiniWallet +# Decorator to +# 1) check that mempool is empty at the start of a subtest +# 2) run the subtest, which may submit some transaction(s) to the mempool and +# create a list of hex transactions +# 3) testmempoolaccept the package hex and check that it fails with the error +# "package-mempool-limits" for each tx +# 4) after mining a block, clearing the pre-submitted transactions from mempool, +# check that submitting the created package succeeds +def check_package_limits(func): + def func_wrapper(self, *args, **kwargs): + node = self.nodes[0] + assert_equal(0, node.getmempoolinfo()["size"]) + package_hex = func(self, *args, **kwargs) + testres_error_expected = node.testmempoolaccept(rawtxs=package_hex) + assert_equal(len(testres_error_expected), len(package_hex)) + for txres in testres_error_expected: + assert_equal(txres["package-error"], "package-mempool-limits") + + # Clear mempool and check that the package passes now + self.generate(node, 1) + assert all([res["allowed"] for res in node.testmempoolaccept(rawtxs=package_hex)]) + + return func_wrapper + + class MempoolPackageLimitsTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 @@ -40,24 +61,18 @@ class MempoolPackageLimitsTest(BitcoinTestFramework): self.test_anc_size_limits() self.test_desc_size_limits() + @check_package_limits def test_chain_limits_helper(self, mempool_count, package_count): node = self.nodes[0] - assert_equal(0, node.getmempoolinfo()["size"]) chain_hex = [] - chaintip_utxo = self.wallet.send_self_transfer_chain(from_node=node, chain_length=mempool_count) + chaintip_utxo = self.wallet.send_self_transfer_chain(from_node=node, chain_length=mempool_count)[-1]["new_utxo"] # in-package transactions for _ in range(package_count): tx = self.wallet.create_self_transfer(utxo_to_spend=chaintip_utxo) chaintip_utxo = tx["new_utxo"] chain_hex.append(tx["hex"]) - testres_too_long = node.testmempoolaccept(rawtxs=chain_hex) - for txres in testres_too_long: - assert_equal(txres["package-error"], "package-mempool-limits") - - # Clear mempool and check that the package passes now - self.generate(node, 1) - assert all([res["allowed"] for res in node.testmempoolaccept(rawtxs=chain_hex)]) + return chain_hex def test_chain_limits(self): """Create chains from mempool and package transactions that are longer than 25, @@ -76,6 +91,7 @@ class MempoolPackageLimitsTest(BitcoinTestFramework): # 13 transactions in the mempool and 13 in the package. self.test_chain_limits_helper(13, 13) + @check_package_limits def test_desc_count_limits(self): """Create an 'A' shaped package with 24 transactions in the mempool and 2 in the package: M1 @@ -93,34 +109,28 @@ class MempoolPackageLimitsTest(BitcoinTestFramework): package transactions). """ node = self.nodes[0] - assert_equal(0, node.getmempoolinfo()["size"]) self.log.info("Check that in-mempool and in-package descendants are calculated properly in packages") # Top parent in mempool, M1 m1_utxos = self.wallet.send_self_transfer_multi(from_node=node, num_outputs=2)['new_utxos'] package_hex = [] # Chain A (M2a... M12a) - chain_a_tip_utxo = self.wallet.send_self_transfer_chain(from_node=node, chain_length=11, utxo_to_spend=m1_utxos[0]) + chain_a_tip_utxo = self.wallet.send_self_transfer_chain(from_node=node, chain_length=11, utxo_to_spend=m1_utxos[0])[-1]["new_utxo"] # Pa pa_hex = self.wallet.create_self_transfer(utxo_to_spend=chain_a_tip_utxo)["hex"] package_hex.append(pa_hex) # Chain B (M2b... M13b) - chain_b_tip_utxo = self.wallet.send_self_transfer_chain(from_node=node, chain_length=12, utxo_to_spend=m1_utxos[1]) + chain_b_tip_utxo = self.wallet.send_self_transfer_chain(from_node=node, chain_length=12, utxo_to_spend=m1_utxos[1])[-1]["new_utxo"] # Pb pb_hex = self.wallet.create_self_transfer(utxo_to_spend=chain_b_tip_utxo)["hex"] package_hex.append(pb_hex) assert_equal(24, node.getmempoolinfo()["size"]) assert_equal(2, len(package_hex)) - testres_too_long = node.testmempoolaccept(rawtxs=package_hex) - for txres in testres_too_long: - assert_equal(txres["package-error"], "package-mempool-limits") - - # Clear mempool and check that the package passes now - self.generate(node, 1) - assert all([res["allowed"] for res in node.testmempoolaccept(rawtxs=package_hex)]) + return package_hex + @check_package_limits def test_desc_count_limits_2(self): """Create a Package with 24 transaction in mempool and 2 transaction in package: M1 @@ -145,7 +155,7 @@ class MempoolPackageLimitsTest(BitcoinTestFramework): m1_utxos = self.wallet.send_self_transfer_multi(from_node=node, num_outputs=2)['new_utxos'] # Chain M2...M24 - self.wallet.send_self_transfer_chain(from_node=node, chain_length=23, utxo_to_spend=m1_utxos[0]) + self.wallet.send_self_transfer_chain(from_node=node, chain_length=23, utxo_to_spend=m1_utxos[0])[-1]["new_utxo"] # P1 p1_tx = self.wallet.create_self_transfer(utxo_to_spend=m1_utxos[1]) @@ -157,15 +167,9 @@ class MempoolPackageLimitsTest(BitcoinTestFramework): assert_equal(24, node.getmempoolinfo()["size"]) assert_equal(2, len(package_hex)) - testres = node.testmempoolaccept(rawtxs=package_hex) - assert_equal(len(testres), len(package_hex)) - for txres in testres: - assert_equal(txres["package-error"], "package-mempool-limits") - - # Clear mempool and check that the package passes now - self.generate(node, 1) - assert all([res["allowed"] for res in node.testmempoolaccept(rawtxs=package_hex)]) + return package_hex + @check_package_limits def test_anc_count_limits(self): """Create a 'V' shaped chain with 24 transactions in the mempool and 3 in the package: M1a M1b @@ -183,7 +187,6 @@ class MempoolPackageLimitsTest(BitcoinTestFramework): and in-package ancestors are all considered together. """ node = self.nodes[0] - assert_equal(0, node.getmempoolinfo()["size"]) package_hex = [] pc_parent_utxos = [] @@ -191,7 +194,7 @@ class MempoolPackageLimitsTest(BitcoinTestFramework): # Two chains of 13 transactions each for _ in range(2): - chain_tip_utxo = self.wallet.send_self_transfer_chain(from_node=node, chain_length=12) + chain_tip_utxo = self.wallet.send_self_transfer_chain(from_node=node, chain_length=12)[-1]["new_utxo"] # Save the 13th transaction for the package tx = self.wallet.create_self_transfer(utxo_to_spend=chain_tip_utxo) package_hex.append(tx["hex"]) @@ -203,14 +206,9 @@ class MempoolPackageLimitsTest(BitcoinTestFramework): assert_equal(24, node.getmempoolinfo()["size"]) assert_equal(3, len(package_hex)) - testres_too_long = node.testmempoolaccept(rawtxs=package_hex) - for txres in testres_too_long: - assert_equal(txres["package-error"], "package-mempool-limits") - - # Clear mempool and check that the package passes now - self.generate(node, 1) - assert all([res["allowed"] for res in node.testmempoolaccept(rawtxs=package_hex)]) + return package_hex + @check_package_limits def test_anc_count_limits_2(self): """Create a 'Y' shaped chain with 24 transactions in the mempool and 2 in the package: M1a M1b @@ -228,13 +226,12 @@ class MempoolPackageLimitsTest(BitcoinTestFramework): and in-package ancestors are all considered together. """ node = self.nodes[0] - assert_equal(0, node.getmempoolinfo()["size"]) pc_parent_utxos = [] self.log.info("Check that in-mempool and in-package ancestors are calculated properly in packages") # Two chains of 12 transactions each for _ in range(2): - chaintip_utxo = self.wallet.send_self_transfer_chain(from_node=node, chain_length=12) + chaintip_utxo = self.wallet.send_self_transfer_chain(from_node=node, chain_length=12)[-1]["new_utxo"] # last 2 transactions will be the parents of Pc pc_parent_utxos.append(chaintip_utxo) @@ -245,14 +242,9 @@ class MempoolPackageLimitsTest(BitcoinTestFramework): pd_tx = self.wallet.create_self_transfer(utxo_to_spend=pc_tx["new_utxos"][0]) assert_equal(24, node.getmempoolinfo()["size"]) - testres_too_long = node.testmempoolaccept(rawtxs=[pc_tx["hex"], pd_tx["hex"]]) - for txres in testres_too_long: - assert_equal(txres["package-error"], "package-mempool-limits") - - # Clear mempool and check that the package passes now - self.generate(node, 1) - assert all([res["allowed"] for res in node.testmempoolaccept(rawtxs=[pc_tx["hex"], pd_tx["hex"]])]) + return [pc_tx["hex"], pd_tx["hex"]] + @check_package_limits def test_anc_count_limits_bushy(self): """Create a tree with 20 transactions in the mempool and 6 in the package: M1...M4 M5...M8 M9...M12 M13...M16 M17...M20 @@ -265,7 +257,6 @@ class MempoolPackageLimitsTest(BitcoinTestFramework): combined, PC has 25 in-mempool and in-package parents. """ node = self.nodes[0] - assert_equal(0, node.getmempoolinfo()["size"]) package_hex = [] pc_parent_utxos = [] for _ in range(5): # Make package transactions P0 ... P4 @@ -282,14 +273,9 @@ class MempoolPackageLimitsTest(BitcoinTestFramework): assert_equal(20, node.getmempoolinfo()["size"]) assert_equal(6, len(package_hex)) - testres = node.testmempoolaccept(rawtxs=package_hex) - for txres in testres: - assert_equal(txres["package-error"], "package-mempool-limits") - - # Clear mempool and check that the package passes now - self.generate(node, 1) - assert all([res["allowed"] for res in node.testmempoolaccept(rawtxs=package_hex)]) + return package_hex + @check_package_limits def test_anc_size_limits(self): """Test Case with 2 independent transactions in the mempool and a parent + child in the package, where the package parent is the child of both mempool transactions (30KvB each): @@ -302,10 +288,10 @@ class MempoolPackageLimitsTest(BitcoinTestFramework): and in-package ancestors are all considered together. """ node = self.nodes[0] - assert_equal(0, node.getmempoolinfo()["size"]) parent_utxos = [] - target_weight = WITNESS_SCALE_FACTOR * 1000 * 30 # 30KvB - high_fee = Decimal("0.003") # 10 sats/vB + target_vsize = 30_000 + high_fee = 10 * target_vsize # 10 sats/vB + target_weight = target_vsize * WITNESS_SCALE_FACTOR self.log.info("Check that in-mempool and in-package ancestor size limits are calculated properly in packages") # Mempool transactions A and B for _ in range(2): @@ -314,22 +300,17 @@ class MempoolPackageLimitsTest(BitcoinTestFramework): parent_utxos.append(bulked_tx["new_utxo"]) # Package transaction C - pc_tx = self.wallet.create_self_transfer_multi(utxos_to_spend=parent_utxos, fee_per_output=int(high_fee * COIN), target_weight=target_weight) + pc_tx = self.wallet.create_self_transfer_multi(utxos_to_spend=parent_utxos, fee_per_output=high_fee, target_weight=target_weight) # Package transaction D pd_tx = self.wallet.create_self_transfer(utxo_to_spend=pc_tx["new_utxos"][0], target_weight=target_weight) assert_equal(2, node.getmempoolinfo()["size"]) - testres_too_heavy = node.testmempoolaccept(rawtxs=[pc_tx["hex"], pd_tx["hex"]]) - for txres in testres_too_heavy: - assert_equal(txres["package-error"], "package-mempool-limits") - - # Clear mempool and check that the package passes now - self.generate(node, 1) - assert all([res["allowed"] for res in node.testmempoolaccept(rawtxs=[pc_tx["hex"], pd_tx["hex"]])]) + return [pc_tx["hex"], pd_tx["hex"]] + @check_package_limits def test_desc_size_limits(self): - """Create 3 mempool transactions and 2 package transactions (25KvB each): + """Create 3 mempool transactions and 2 package transactions (21KvB each): Ma ^ ^ Mb Mc @@ -339,12 +320,12 @@ class MempoolPackageLimitsTest(BitcoinTestFramework): and in-package descendants are all considered together. """ node = self.nodes[0] - assert_equal(0, node.getmempoolinfo()["size"]) - target_weight = 21 * 1000 * WITNESS_SCALE_FACTOR - high_fee = Decimal("0.0021") # 10 sats/vB + target_vsize = 21_000 + high_fee = 10 * target_vsize # 10 sats/vB + target_weight = target_vsize * WITNESS_SCALE_FACTOR self.log.info("Check that in-mempool and in-package descendant sizes are calculated properly in packages") # Top parent in mempool, Ma - ma_tx = self.wallet.create_self_transfer_multi(num_outputs=2, fee_per_output=int(high_fee / 2 * COIN), target_weight=target_weight) + ma_tx = self.wallet.create_self_transfer_multi(num_outputs=2, fee_per_output=high_fee // 2, target_weight=target_weight) self.wallet.sendrawtransaction(from_node=node, tx_hex=ma_tx["hex"]) package_hex = [] @@ -359,13 +340,8 @@ class MempoolPackageLimitsTest(BitcoinTestFramework): assert_equal(3, node.getmempoolinfo()["size"]) assert_equal(2, len(package_hex)) - testres_too_heavy = node.testmempoolaccept(rawtxs=package_hex) - for txres in testres_too_heavy: - assert_equal(txres["package-error"], "package-mempool-limits") + return package_hex - # Clear mempool and check that the package passes now - self.generate(node, 1) - assert all([res["allowed"] for res in node.testmempoolaccept(rawtxs=package_hex)]) if __name__ == "__main__": MempoolPackageLimitsTest().main() diff --git a/test/functional/mempool_package_onemore.py b/test/functional/mempool_package_onemore.py index 23ee587098..921c190668 100755 --- a/test/functional/mempool_package_onemore.py +++ b/test/functional/mempool_package_onemore.py @@ -31,7 +31,6 @@ class MempoolPackagesTest(BitcoinTestFramework): def run_test(self): self.wallet = MiniWallet(self.nodes[0]) - self.wallet.rescan_utxos() # DEFAULT_ANCESTOR_LIMIT transactions off a confirmed tx should be fine chain = [] diff --git a/test/functional/mempool_packages.py b/test/functional/mempool_packages.py index c89528101e..0387282862 100755 --- a/test/functional/mempool_packages.py +++ b/test/functional/mempool_packages.py @@ -6,7 +6,6 @@ from decimal import Decimal -from test_framework.blocktools import COINBASE_MATURITY from test_framework.messages import ( COIN, DEFAULT_ANCESTOR_LIMIT, @@ -17,9 +16,8 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, assert_raises_rpc_error, - chain_transaction, ) - +from test_framework.wallet import MiniWallet # custom limits for node1 CUSTOM_ANCESTOR_LIMIT = 5 @@ -45,43 +43,33 @@ class MempoolPackagesTest(BitcoinTestFramework): ], ] - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() - def run_test(self): - # Mine some blocks and have them mature. + self.wallet = MiniWallet(self.nodes[0]) + self.wallet.rescan_utxos() + + if self.is_specified_wallet_compiled(): + self.nodes[0].createwallet("watch_wallet", disable_private_keys=True) + self.nodes[0].importaddress(self.wallet.get_address()) + peer_inv_store = self.nodes[0].add_p2p_connection(P2PTxInvStore()) # keep track of invs - self.generate(self.nodes[0], COINBASE_MATURITY + 1) - utxo = self.nodes[0].listunspent(10) - txid = utxo[0]['txid'] - vout = utxo[0]['vout'] - value = utxo[0]['amount'] - assert 'ancestorcount' not in utxo[0] - assert 'ancestorsize' not in utxo[0] - assert 'ancestorfees' not in utxo[0] - - fee = Decimal("0.0001") + # DEFAULT_ANCESTOR_LIMIT transactions off a confirmed tx should be fine - chain = [] - witness_chain = [] + chain = self.wallet.create_self_transfer_chain(chain_length=DEFAULT_ANCESTOR_LIMIT) + witness_chain = [t["wtxid"] for t in chain] ancestor_vsize = 0 ancestor_fees = Decimal(0) - for i in range(DEFAULT_ANCESTOR_LIMIT): - (txid, sent_value) = chain_transaction(self.nodes[0], [txid], [0], value, fee, 1) - value = sent_value - chain.append(txid) - # We need the wtxids to check P2P announcements - witnesstx = self.nodes[0].gettransaction(txid=txid, verbose=True)['decoded'] - witness_chain.append(witnesstx['hash']) + for i, t in enumerate(chain): + ancestor_vsize += t["tx"].get_vsize() + ancestor_fees += t["fee"] + self.wallet.sendrawtransaction(from_node=self.nodes[0], tx_hex=t["hex"]) # Check that listunspent ancestor{count, size, fees} yield the correct results - wallet_unspent = self.nodes[0].listunspent(minconf=0) - this_unspent = next(utxo_info for utxo_info in wallet_unspent if utxo_info['txid'] == txid) - assert_equal(this_unspent['ancestorcount'], i + 1) - ancestor_vsize += self.nodes[0].getrawtransaction(txid=txid, verbose=True)['vsize'] - assert_equal(this_unspent['ancestorsize'], ancestor_vsize) - ancestor_fees -= self.nodes[0].gettransaction(txid=txid)['fee'] - assert_equal(this_unspent['ancestorfees'], ancestor_fees * COIN) + if self.is_specified_wallet_compiled(): + wallet_unspent = self.nodes[0].listunspent(minconf=0) + this_unspent = next(utxo_info for utxo_info in wallet_unspent if utxo_info["txid"] == t["txid"]) + assert_equal(this_unspent['ancestorcount'], i + 1) + assert_equal(this_unspent['ancestorsize'], ancestor_vsize) + assert_equal(this_unspent['ancestorfees'], ancestor_fees * COIN) # Wait until mempool transactions have passed initial broadcast (sent inv and received getdata) # Otherwise, getrawmempool may be inconsistent with getmempoolentry if unbroadcast changes in between @@ -99,15 +87,20 @@ class MempoolPackagesTest(BitcoinTestFramework): ancestor_count = DEFAULT_ANCESTOR_LIMIT assert_equal(ancestor_fees, sum([mempool[tx]['fees']['base'] for tx in mempool])) + # Adding one more transaction on to the chain should fail. + next_hop = self.wallet.create_self_transfer(utxo_to_spend=chain[-1]["new_utxo"])["hex"] + assert_raises_rpc_error(-26, "too-long-mempool-chain", lambda: self.nodes[0].sendrawtransaction(next_hop)) + descendants = [] - ancestors = list(chain) + ancestors = [t["txid"] for t in chain] + chain = [t["txid"] for t in chain] for x in reversed(chain): # Check that getmempoolentry is consistent with getrawmempool entry = self.nodes[0].getmempoolentry(x) assert_equal(entry, mempool[x]) # Check that gettxspendingprevout is consistent with getrawmempool - witnesstx = self.nodes[0].gettransaction(txid=x, verbose=True)['decoded'] + witnesstx = self.nodes[0].getrawtransaction(txid=x, verbose=True) for tx_in in witnesstx["vin"]: spending_result = self.nodes[0].gettxspendingprevout([ {'txid' : tx_in["txid"], 'vout' : tx_in["vout"]} ]) assert_equal(spending_result, [ {'txid' : tx_in["txid"], 'vout' : tx_in["vout"], 'spendingtxid' : x} ]) @@ -193,9 +186,6 @@ class MempoolPackagesTest(BitcoinTestFramework): descendant_fees += entry['fees']['base'] assert_equal(entry['fees']['descendant'], descendant_fees + Decimal('0.00001')) - # Adding one more transaction on to the chain should fail. - assert_raises_rpc_error(-26, "too-long-mempool-chain", chain_transaction, self.nodes[0], [txid], [vout], value, fee, 1) - # Check that prioritising a tx before it's added to the mempool works # First clear the mempool by mining a block. self.generate(self.nodes[0], 1) @@ -232,28 +222,23 @@ class MempoolPackagesTest(BitcoinTestFramework): # TODO: test ancestor size limits # Now test descendant chain limits - txid = utxo[1]['txid'] - value = utxo[1]['amount'] - vout = utxo[1]['vout'] - transaction_package = [] tx_children = [] # First create one parent tx with 10 children - (txid, sent_value) = chain_transaction(self.nodes[0], [txid], [vout], value, fee, 10) - parent_transaction = txid - for i in range(10): - transaction_package.append({'txid': txid, 'vout': i, 'amount': sent_value}) + tx_with_children = self.wallet.send_self_transfer_multi(from_node=self.nodes[0], num_outputs=10) + parent_transaction = tx_with_children["txid"] + transaction_package = tx_with_children["new_utxos"] # Sign and send up to MAX_DESCENDANT transactions chained off the parent tx chain = [] # save sent txs for the purpose of checking node1's mempool later (see below) for _ in range(DEFAULT_DESCENDANT_LIMIT - 1): utxo = transaction_package.pop(0) - (txid, sent_value) = chain_transaction(self.nodes[0], [utxo['txid']], [utxo['vout']], utxo['amount'], fee, 10) + new_tx = self.wallet.send_self_transfer_multi(from_node=self.nodes[0], num_outputs=10, utxos_to_spend=[utxo]) + txid = new_tx["txid"] chain.append(txid) if utxo['txid'] is parent_transaction: tx_children.append(txid) - for j in range(10): - transaction_package.append({'txid': txid, 'vout': j, 'amount': sent_value}) + transaction_package.extend(new_tx["new_utxos"]) mempool = self.nodes[0].getrawmempool(True) assert_equal(mempool[parent_transaction]['descendantcount'], DEFAULT_DESCENDANT_LIMIT) @@ -263,8 +248,8 @@ class MempoolPackagesTest(BitcoinTestFramework): assert_equal(mempool[child]['depends'], [parent_transaction]) # Sending one more chained transaction will fail - utxo = transaction_package.pop(0) - assert_raises_rpc_error(-26, "too-long-mempool-chain", chain_transaction, self.nodes[0], [utxo['txid']], [utxo['vout']], utxo['amount'], fee, 10) + next_hop = self.wallet.create_self_transfer(utxo_to_spend=transaction_package.pop(0))["hex"] + assert_raises_rpc_error(-26, "too-long-mempool-chain", lambda: self.nodes[0].sendrawtransaction(next_hop)) # Check that node1's mempool is as expected, containing: # - txs from previous ancestor test (-> custom ancestor limit) @@ -304,42 +289,19 @@ class MempoolPackagesTest(BitcoinTestFramework): # last block. # Create tx0 with 2 outputs - utxo = self.nodes[0].listunspent() - txid = utxo[0]['txid'] - value = utxo[0]['amount'] - vout = utxo[0]['vout'] - - send_value = (value - fee) / 2 - inputs = [ {'txid' : txid, 'vout' : vout} ] - outputs = {} - for _ in range(2): - outputs[self.nodes[0].getnewaddress()] = send_value - rawtx = self.nodes[0].createrawtransaction(inputs, outputs) - signedtx = self.nodes[0].signrawtransactionwithwallet(rawtx) - txid = self.nodes[0].sendrawtransaction(signedtx['hex']) - tx0_id = txid - value = send_value + tx0 = self.wallet.send_self_transfer_multi(from_node=self.nodes[0], num_outputs=2) # Create tx1 - tx1_id, _ = chain_transaction(self.nodes[0], [tx0_id], [0], value, fee, 1) + tx1 = self.wallet.send_self_transfer(from_node=self.nodes[0], utxo_to_spend=tx0["new_utxos"][0]) # Create tx2-7 - vout = 1 - txid = tx0_id - for _ in range(6): - (txid, sent_value) = chain_transaction(self.nodes[0], [txid], [vout], value, fee, 1) - vout = 0 - value = sent_value + tx7 = self.wallet.send_self_transfer_chain(from_node=self.nodes[0], utxo_to_spend=tx0["new_utxos"][1], chain_length=6)[-1] # Mine these in a block self.generate(self.nodes[0], 1) # Now generate tx8, with a big fee - inputs = [ {'txid' : tx1_id, 'vout': 0}, {'txid' : txid, 'vout': 0} ] - outputs = { self.nodes[0].getnewaddress() : send_value + value - 4*fee } - rawtx = self.nodes[0].createrawtransaction(inputs, outputs) - signedtx = self.nodes[0].signrawtransactionwithwallet(rawtx) - txid = self.nodes[0].sendrawtransaction(signedtx['hex']) + self.wallet.send_self_transfer_multi(from_node=self.nodes[0], utxos_to_spend=[tx1["new_utxo"], tx7["new_utxo"]], fee_per_output=40000) self.sync_mempools() # Now try to disconnect the tip on each node... diff --git a/test/functional/mempool_persist.py b/test/functional/mempool_persist.py index dca4a71bd0..f818801136 100755 --- a/test/functional/mempool_persist.py +++ b/test/functional/mempool_persist.py @@ -59,7 +59,6 @@ class MempoolPersistTest(BitcoinTestFramework): def run_test(self): self.mini_wallet = MiniWallet(self.nodes[2]) - self.mini_wallet.rescan_utxos() if self.is_sqlite_compiled(): self.nodes[2].createwallet( wallet_name="watch", diff --git a/test/functional/mempool_reorg.py b/test/functional/mempool_reorg.py index 83ab65f1ba..3a5bc1ebcd 100755 --- a/test/functional/mempool_reorg.py +++ b/test/functional/mempool_reorg.py @@ -31,7 +31,6 @@ class MempoolCoinbaseTest(BitcoinTestFramework): self.log.info("Add 4 coinbase utxos to the miniwallet") # Block 76 contains the first spendable coinbase txs. first_block = 76 - wallet.rescan_utxos() # Three scenarios for re-orging coinbase spends in the memory pool: # 1. Direct coinbase spend : spend_1 diff --git a/test/functional/mempool_resurrect.py b/test/functional/mempool_resurrect.py index 3e610d02ac..c10052372d 100755 --- a/test/functional/mempool_resurrect.py +++ b/test/functional/mempool_resurrect.py @@ -4,7 +4,6 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test resurrection of mined transactions when the blockchain is re-organized.""" -from test_framework.blocktools import COINBASE_MATURITY from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal from test_framework.wallet import MiniWallet @@ -13,16 +12,11 @@ from test_framework.wallet import MiniWallet class MempoolCoinbaseTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 - self.setup_clean_chain = True def run_test(self): node = self.nodes[0] wallet = MiniWallet(node) - # Add enough mature utxos to the wallet so that all txs spend confirmed coins - self.generate(wallet, 3) - self.generate(node, COINBASE_MATURITY) - # Spend block 1/2/3's coinbase transactions # Mine a block # Create three more transactions, spending the spends diff --git a/test/functional/mempool_sigoplimit.py b/test/functional/mempool_sigoplimit.py new file mode 100755 index 0000000000..b178b9feda --- /dev/null +++ b/test/functional/mempool_sigoplimit.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +# Copyright (c) 2023 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test sigop limit mempool policy (`-bytespersigop` parameter)""" +from math import ceil + +from test_framework.messages import ( + COutPoint, + CTransaction, + CTxIn, + CTxInWitness, + CTxOut, + WITNESS_SCALE_FACTOR, + tx_from_hex, +) +from test_framework.script import ( + CScript, + OP_CHECKMULTISIG, + OP_CHECKSIG, + OP_ENDIF, + OP_FALSE, + OP_IF, + OP_RETURN, + OP_TRUE, +) +from test_framework.script_util import ( + script_to_p2wsh_script, +) +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + assert_greater_than, + assert_greater_than_or_equal, +) +from test_framework.wallet import MiniWallet + + +DEFAULT_BYTES_PER_SIGOP = 20 # default setting + + +class BytesPerSigOpTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + # allow large datacarrier output to pad transactions + self.extra_args = [['-datacarriersize=100000']] + + def create_p2wsh_spending_tx(self, witness_script, output_script): + """Create a 1-input-1-output P2WSH spending transaction with only the + witness script in the witness stack and the given output script.""" + # create P2WSH address and fund it via MiniWallet first + txid, vout = self.wallet.send_to( + from_node=self.nodes[0], + scriptPubKey=script_to_p2wsh_script(witness_script), + amount=1000000, + ) + + # create spending transaction + tx = CTransaction() + tx.vin = [CTxIn(COutPoint(int(txid, 16), vout))] + tx.wit.vtxinwit = [CTxInWitness()] + tx.wit.vtxinwit[0].scriptWitness.stack = [bytes(witness_script)] + tx.vout = [CTxOut(500000, output_script)] + return tx + + def test_sigops_limit(self, bytes_per_sigop, num_sigops): + sigop_equivalent_vsize = ceil(num_sigops * bytes_per_sigop / WITNESS_SCALE_FACTOR) + self.log.info(f"- {num_sigops} sigops (equivalent size of {sigop_equivalent_vsize} vbytes)") + + # create a template tx with the specified sigop cost in the witness script + # (note that the sigops count even though being in a branch that's not executed) + num_multisigops = num_sigops // 20 + num_singlesigops = num_sigops % 20 + witness_script = CScript( + [OP_FALSE, OP_IF] + + [OP_CHECKMULTISIG]*num_multisigops + + [OP_CHECKSIG]*num_singlesigops + + [OP_ENDIF, OP_TRUE] + ) + # use a 256-byte data-push as lower bound in the output script, in order + # to avoid having to compensate for tx size changes caused by varying + # length serialization sizes (both for scriptPubKey and data-push lengths) + tx = self.create_p2wsh_spending_tx(witness_script, CScript([OP_RETURN, b'X'*256])) + + # bump the tx to reach the sigop-limit equivalent size by padding the datacarrier output + assert_greater_than_or_equal(sigop_equivalent_vsize, tx.get_vsize()) + vsize_to_pad = sigop_equivalent_vsize - tx.get_vsize() + tx.vout[0].scriptPubKey = CScript([OP_RETURN, b'X'*(256+vsize_to_pad)]) + assert_equal(sigop_equivalent_vsize, tx.get_vsize()) + + res = self.nodes[0].testmempoolaccept([tx.serialize().hex()])[0] + assert_equal(res['allowed'], True) + assert_equal(res['vsize'], sigop_equivalent_vsize) + + # increase the tx's vsize to be right above the sigop-limit equivalent size + # => tx's vsize in mempool should also grow accordingly + tx.vout[0].scriptPubKey = CScript([OP_RETURN, b'X'*(256+vsize_to_pad+1)]) + res = self.nodes[0].testmempoolaccept([tx.serialize().hex()])[0] + assert_equal(res['allowed'], True) + assert_equal(res['vsize'], sigop_equivalent_vsize+1) + + # decrease the tx's vsize to be right below the sigop-limit equivalent size + # => tx's vsize in mempool should stick at the sigop-limit equivalent + # bytes level, as it is higher than the tx's serialized vsize + # (the maximum of both is taken) + tx.vout[0].scriptPubKey = CScript([OP_RETURN, b'X'*(256+vsize_to_pad-1)]) + res = self.nodes[0].testmempoolaccept([tx.serialize().hex()])[0] + assert_equal(res['allowed'], True) + assert_equal(res['vsize'], sigop_equivalent_vsize) + + # check that the ancestor and descendant size calculations in the mempool + # also use the same max(sigop_equivalent_vsize, serialized_vsize) logic + # (to keep it simple, we only test the case here where the sigop vsize + # is much larger than the serialized vsize, i.e. we create a small child + # tx by getting rid of the large padding output) + tx.vout[0].scriptPubKey = CScript([OP_RETURN, b'test123']) + assert_greater_than(sigop_equivalent_vsize, tx.get_vsize()) + self.nodes[0].sendrawtransaction(hexstring=tx.serialize().hex(), maxburnamount='1.0') + + # fetch parent tx, which doesn't contain any sigops + parent_txid = tx.vin[0].prevout.hash.to_bytes(32, 'big').hex() + parent_tx = tx_from_hex(self.nodes[0].getrawtransaction(txid=parent_txid)) + + entry_child = self.nodes[0].getmempoolentry(tx.rehash()) + assert_equal(entry_child['descendantcount'], 1) + assert_equal(entry_child['descendantsize'], sigop_equivalent_vsize) + assert_equal(entry_child['ancestorcount'], 2) + assert_equal(entry_child['ancestorsize'], sigop_equivalent_vsize + parent_tx.get_vsize()) + + entry_parent = self.nodes[0].getmempoolentry(parent_tx.rehash()) + assert_equal(entry_parent['ancestorcount'], 1) + assert_equal(entry_parent['ancestorsize'], parent_tx.get_vsize()) + assert_equal(entry_parent['descendantcount'], 2) + assert_equal(entry_parent['descendantsize'], parent_tx.get_vsize() + sigop_equivalent_vsize) + + def run_test(self): + self.wallet = MiniWallet(self.nodes[0]) + + for bytes_per_sigop in (DEFAULT_BYTES_PER_SIGOP, 43, 81, 165, 327, 649, 1072): + if bytes_per_sigop == DEFAULT_BYTES_PER_SIGOP: + self.log.info(f"Test default sigops limit setting ({bytes_per_sigop} bytes per sigop)...") + else: + bytespersigop_parameter = f"-bytespersigop={bytes_per_sigop}" + self.log.info(f"Test sigops limit setting {bytespersigop_parameter}...") + self.restart_node(0, extra_args=[bytespersigop_parameter] + self.extra_args[0]) + + for num_sigops in (69, 101, 142, 183, 222): + self.test_sigops_limit(bytes_per_sigop, num_sigops) + + self.generate(self.wallet, 1) + + +if __name__ == '__main__': + BytesPerSigOpTest().main() diff --git a/test/functional/mempool_spend_coinbase.py b/test/functional/mempool_spend_coinbase.py index bca512445c..a7cb2ba602 100755 --- a/test/functional/mempool_spend_coinbase.py +++ b/test/functional/mempool_spend_coinbase.py @@ -28,7 +28,6 @@ class MempoolSpendCoinbaseTest(BitcoinTestFramework): chain_height = 198 self.nodes[0].invalidateblock(self.nodes[0].getblockhash(chain_height + 1)) assert_equal(chain_height, self.nodes[0].getblockcount()) - wallet.rescan_utxos() # Coinbase at height chain_height-100+1 ok in mempool, should # get mined. Coinbase at height chain_height-100+2 is diff --git a/test/functional/mempool_unbroadcast.py b/test/functional/mempool_unbroadcast.py index 1b0097d578..12de750731 100755 --- a/test/functional/mempool_unbroadcast.py +++ b/test/functional/mempool_unbroadcast.py @@ -23,7 +23,6 @@ class MempoolUnbroadcastTest(BitcoinTestFramework): def run_test(self): self.wallet = MiniWallet(self.nodes[0]) - self.wallet.rescan_utxos() self.test_broadcast() self.test_txn_removal() diff --git a/test/functional/mempool_updatefromblock.py b/test/functional/mempool_updatefromblock.py index 68cbb5dbed..8350e9c91e 100755 --- a/test/functional/mempool_updatefromblock.py +++ b/test/functional/mempool_updatefromblock.py @@ -7,14 +7,12 @@ Test mempool update of transaction descendants/ancestors information (count, size) when transactions have been re-added from a disconnected block to the mempool. """ +from math import ceil import time -from decimal import Decimal from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal -from test_framework.address import key_to_p2pkh -from test_framework.wallet_util import bytes_to_wif -from test_framework.key import ECKey +from test_framework.wallet import MiniWallet class MempoolUpdateFromBlockTest(BitcoinTestFramework): @@ -22,15 +20,7 @@ class MempoolUpdateFromBlockTest(BitcoinTestFramework): self.num_nodes = 1 self.extra_args = [['-limitdescendantsize=1000', '-limitancestorsize=1000', '-limitancestorcount=100']] - def get_new_address(self): - key = ECKey() - key.generate() - pubkey = key.get_pubkey().get_bytes() - address = key_to_p2pkh(pubkey) - self.priv_keys.append(bytes_to_wif(key.get_bytes())) - return address - - def transaction_graph_test(self, size, n_tx_to_mine=None, start_input_txid='', end_address='', fee=Decimal(0.00100000)): + def transaction_graph_test(self, size, n_tx_to_mine=None, fee=100_000): """Create an acyclic tournament (a type of directed graph) of transactions and use it for testing. Keyword arguments: @@ -45,14 +35,7 @@ class MempoolUpdateFromBlockTest(BitcoinTestFramework): More details: https://en.wikipedia.org/wiki/Tournament_(graph_theory) """ - - self.priv_keys = [self.nodes[0].get_deterministic_priv_key().key] - if not start_input_txid: - start_input_txid = self.nodes[0].getblock(self.nodes[0].getblockhash(1))['tx'][0] - - if not end_address: - end_address = self.get_new_address() - + wallet = MiniWallet(self.nodes[0]) first_block_hash = '' tx_id = [] tx_size = [] @@ -61,41 +44,31 @@ class MempoolUpdateFromBlockTest(BitcoinTestFramework): self.log.debug('Preparing transaction #{}...'.format(i)) # Prepare inputs. if i == 0: - inputs = [{'txid': start_input_txid, 'vout': 0}] - inputs_value = self.nodes[0].gettxout(start_input_txid, 0)['value'] + inputs = [wallet.get_utxo()] # let MiniWallet provide a start UTXO else: inputs = [] - inputs_value = 0 for j, tx in enumerate(tx_id[0:i]): # Transaction tx[K] is a child of each of previous transactions tx[0]..tx[K-1] at their output K-1. vout = i - j - 1 - inputs.append({'txid': tx_id[j], 'vout': vout}) - inputs_value += self.nodes[0].gettxout(tx, vout)['value'] - - self.log.debug('inputs={}'.format(inputs)) - self.log.debug('inputs_value={}'.format(inputs_value)) + inputs.append(wallet.get_utxo(txid=tx_id[j], vout=vout)) # Prepare outputs. tx_count = i + 1 if tx_count < size: # Transaction tx[K] is an ancestor of each of subsequent transactions tx[K+1]..tx[N-1]. n_outputs = size - tx_count - output_value = ((inputs_value - fee) / Decimal(n_outputs)).quantize(Decimal('0.00000001')) - outputs = {} - for _ in range(n_outputs): - outputs[self.get_new_address()] = output_value else: - output_value = (inputs_value - fee).quantize(Decimal('0.00000001')) - outputs = {end_address: output_value} - - self.log.debug('output_value={}'.format(output_value)) - self.log.debug('outputs={}'.format(outputs)) + n_outputs = 1 # Create a new transaction. - unsigned_raw_tx = self.nodes[0].createrawtransaction(inputs, outputs) - signed_raw_tx = self.nodes[0].signrawtransactionwithkey(unsigned_raw_tx, self.priv_keys) - tx_id.append(self.nodes[0].sendrawtransaction(signed_raw_tx['hex'])) - tx_size.append(self.nodes[0].getmempoolentry(tx_id[-1])['vsize']) + new_tx = wallet.send_self_transfer_multi( + from_node=self.nodes[0], + utxos_to_spend=inputs, + num_outputs=n_outputs, + fee_per_output=ceil(fee / n_outputs) + ) + tx_id.append(new_tx['txid']) + tx_size.append(new_tx['tx'].get_vsize()) if tx_count in n_tx_to_mine: # The created transactions are mined into blocks by batches. diff --git a/test/functional/mining_getblocktemplate_longpoll.py b/test/functional/mining_getblocktemplate_longpoll.py index e928ee4936..53182eb79e 100755 --- a/test/functional/mining_getblocktemplate_longpoll.py +++ b/test/functional/mining_getblocktemplate_longpoll.py @@ -4,11 +4,9 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test longpolling with getblocktemplate.""" -from decimal import Decimal import random import threading -from test_framework.blocktools import COINBASE_MATURITY from test_framework.test_framework import BitcoinTestFramework from test_framework.util import get_rpc_proxy from test_framework.wallet import MiniWallet @@ -48,9 +46,9 @@ class GetBlockTemplateLPTest(BitcoinTestFramework): thr.join(5) # wait 5 seconds or until thread exits assert thr.is_alive() - miniwallets = [MiniWallet(node) for node in self.nodes] + self.miniwallet = MiniWallet(self.nodes[0]) self.log.info("Test that longpoll will terminate if another node generates a block") - self.generate(miniwallets[1], 1) # generate a block on another node + self.generate(self.nodes[1], 1) # generate a block on another node # check that thread will exit now that new transaction entered mempool thr.join(5) # wait 5 seconds or until thread exits assert not thr.is_alive() @@ -58,21 +56,15 @@ class GetBlockTemplateLPTest(BitcoinTestFramework): self.log.info("Test that longpoll will terminate if we generate a block ourselves") thr = LongpollThread(self.nodes[0]) thr.start() - self.generate(miniwallets[0], 1) # generate a block on own node + self.generate(self.nodes[0], 1) # generate a block on own node thr.join(5) # wait 5 seconds or until thread exits assert not thr.is_alive() - # Add enough mature utxos to the wallets, so that all txs spend confirmed coins - self.generate(self.nodes[0], COINBASE_MATURITY) - self.log.info("Test that introducing a new transaction into the mempool will terminate the longpoll") thr = LongpollThread(self.nodes[0]) thr.start() - # generate a random transaction and submit it - min_relay_fee = self.nodes[0].getnetworkinfo()["relayfee"] - fee_rate = min_relay_fee + Decimal('0.00000010') * random.randint(0,20) - miniwallets[0].send_self_transfer(from_node=random.choice(self.nodes), - fee_rate=fee_rate) + # generate a transaction and submit it + self.miniwallet.send_self_transfer(from_node=random.choice(self.nodes)) # after one minute, every 10 seconds the mempool is probed, so in 80 seconds it should have returned thr.join(60 + 20) assert not thr.is_alive() diff --git a/test/functional/mining_prioritisetransaction.py b/test/functional/mining_prioritisetransaction.py index 4a54f82b58..a4481c15a0 100755 --- a/test/functional/mining_prioritisetransaction.py +++ b/test/functional/mining_prioritisetransaction.py @@ -106,7 +106,6 @@ class PrioritiseTransactionTest(BitcoinTestFramework): def run_test(self): self.wallet = MiniWallet(self.nodes[0]) - self.wallet.rescan_utxos() # Test `prioritisetransaction` required parameters assert_raises_rpc_error(-1, "prioritisetransaction", self.nodes[0].prioritisetransaction) diff --git a/test/functional/p2p_blocksonly.py b/test/functional/p2p_blocksonly.py index fa9ddf7ebe..110a1bd03f 100755 --- a/test/functional/p2p_blocksonly.py +++ b/test/functional/p2p_blocksonly.py @@ -20,8 +20,6 @@ class P2PBlocksOnly(BitcoinTestFramework): def run_test(self): self.miniwallet = MiniWallet(self.nodes[0]) - # Add enough mature utxos to the wallet, so that all txs spend confirmed coins - self.miniwallet.rescan_utxos() self.blocksonly_mode_tests() self.blocks_relay_conn_tests() diff --git a/test/functional/p2p_disconnect_ban.py b/test/functional/p2p_disconnect_ban.py index 91c2a43932..394009f30f 100755 --- a/test/functional/p2p_disconnect_ban.py +++ b/test/functional/p2p_disconnect_ban.py @@ -46,6 +46,9 @@ class DisconnectBanTest(BitcoinTestFramework): assert_raises_rpc_error(-30, "Error: Invalid IP/Subnet", self.nodes[1].setban, "127.0.0.1/42", "add") assert_equal(len(self.nodes[1].listbanned()), 1) # still only one banned ip because 127.0.0.1 is within the range of 127.0.0.0/24 + self.log.info("setban: fail to ban with past absolute timestamp") + assert_raises_rpc_error(-8, "Error: Absolute timestamp is in the past", self.nodes[1].setban, "127.27.0.1", "add", 123, True) + self.log.info("setban remove: fail to unban a non-banned subnet") assert_raises_rpc_error(-30, "Error: Unban failed", self.nodes[1].setban, "127.0.0.1", "remove") assert_equal(len(self.nodes[1].listbanned()), 1) @@ -66,9 +69,13 @@ class DisconnectBanTest(BitcoinTestFramework): self.nodes[1].setban("2001:4d48:ac57:400:cacf:e9ff:fe1d:9c63/19", "add", 1000) # ban for 1000 seconds listBeforeShutdown = self.nodes[1].listbanned() assert_equal("192.168.0.1/32", listBeforeShutdown[2]['address']) + + self.log.info("setban: test banning with absolute timestamp") + self.nodes[1].setban("192.168.0.2", "add", old_time + 120, True) + # Move time forward by 3 seconds so the third ban has expired self.nodes[1].setmocktime(old_time + 3) - assert_equal(len(self.nodes[1].listbanned()), 3) + assert_equal(len(self.nodes[1].listbanned()), 4) self.log.info("Test ban_duration and time_remaining") for ban in self.nodes[1].listbanned(): @@ -78,13 +85,17 @@ class DisconnectBanTest(BitcoinTestFramework): elif ban["address"] == "2001:4d48:ac57:400:cacf:e9ff:fe1d:9c63/19": assert_equal(ban["ban_duration"], 1000) assert_equal(ban["time_remaining"], 997) + elif ban["address"] == "192.168.0.2/32": + assert_equal(ban["ban_duration"], 120) + assert_equal(ban["time_remaining"], 117) self.restart_node(1) listAfterShutdown = self.nodes[1].listbanned() assert_equal("127.0.0.0/24", listAfterShutdown[0]['address']) assert_equal("127.0.0.0/32", listAfterShutdown[1]['address']) - assert_equal("/19" in listAfterShutdown[2]['address'], True) + assert_equal("192.168.0.2/32", listAfterShutdown[2]['address']) + assert_equal("/19" in listAfterShutdown[3]['address'], True) # Clear ban lists self.nodes[1].clearbanned() @@ -96,7 +107,7 @@ class DisconnectBanTest(BitcoinTestFramework): self.log.info("disconnectnode: fail to disconnect when calling with address and nodeid") address1 = self.nodes[0].getpeerinfo()[0]['addr'] - node1 = self.nodes[0].getpeerinfo()[0]['addr'] + node1 = self.nodes[0].getpeerinfo()[0]["id"] assert_raises_rpc_error(-32602, "Only one of address and nodeid should be provided.", self.nodes[0].disconnectnode, address=address1, nodeid=node1) self.log.info("disconnectnode: fail to disconnect when calling with junk address") @@ -105,7 +116,7 @@ class DisconnectBanTest(BitcoinTestFramework): self.log.info("disconnectnode: successfully disconnect node by address") address1 = self.nodes[0].getpeerinfo()[0]['addr'] self.nodes[0].disconnectnode(address=address1) - self.wait_until(lambda: len(self.nodes[0].getpeerinfo()) == 1, timeout=10) + self.wait_until(lambda: len(self.nodes[1].getpeerinfo()) == 1, timeout=10) assert not [node for node in self.nodes[0].getpeerinfo() if node['addr'] == address1] self.log.info("disconnectnode: successfully reconnect node") @@ -116,7 +127,7 @@ class DisconnectBanTest(BitcoinTestFramework): self.log.info("disconnectnode: successfully disconnect node by node id") id1 = self.nodes[0].getpeerinfo()[0]['id'] self.nodes[0].disconnectnode(nodeid=id1) - self.wait_until(lambda: len(self.nodes[0].getpeerinfo()) == 1, timeout=10) + self.wait_until(lambda: len(self.nodes[1].getpeerinfo()) == 1, timeout=10) assert not [node for node in self.nodes[0].getpeerinfo() if node['id'] == id1] if __name__ == '__main__': diff --git a/test/functional/p2p_eviction.py b/test/functional/p2p_eviction.py index 1f4797a89d..8b31dfa549 100755 --- a/test/functional/p2p_eviction.py +++ b/test/functional/p2p_eviction.py @@ -12,22 +12,23 @@ address/netgroup since in the current framework, all peers are connecting from the same local address. See Issue #14210 for more info. Therefore, this test is limited to the remaining protection criteria. """ - import time from test_framework.blocktools import ( - COINBASE_MATURITY, create_block, create_coinbase, ) from test_framework.messages import ( msg_pong, msg_tx, - tx_from_hex, ) -from test_framework.p2p import P2PDataStore, P2PInterface +from test_framework.p2p import ( + P2PDataStore, + P2PInterface, +) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal +from test_framework.wallet import MiniWallet class SlowP2PDataStore(P2PDataStore): @@ -35,14 +36,15 @@ class SlowP2PDataStore(P2PDataStore): time.sleep(0.1) self.send_message(msg_pong(message.nonce)) + class SlowP2PInterface(P2PInterface): def on_ping(self, message): time.sleep(0.1) self.send_message(msg_pong(message.nonce)) + class P2PEvict(BitcoinTestFramework): def set_test_params(self): - self.setup_clean_chain = True self.num_nodes = 1 # The choice of maxconnections=32 results in a maximum of 21 inbound connections # (32 - 10 outbound - 1 feeler). 20 inbound peers are protected from eviction: @@ -53,7 +55,7 @@ class P2PEvict(BitcoinTestFramework): protected_peers = set() # peers that we expect to be protected from eviction current_peer = -1 node = self.nodes[0] - self.generatetoaddress(node, COINBASE_MATURITY + 1, node.get_deterministic_priv_key().address) + self.wallet = MiniWallet(node) self.log.info("Create 4 peers and protect them from eviction by sending us a block") for _ in range(4): @@ -79,21 +81,8 @@ class P2PEvict(BitcoinTestFramework): current_peer += 1 txpeer.sync_with_ping() - prevtx = node.getblock(node.getblockhash(i + 1), 2)['tx'][0] - rawtx = node.createrawtransaction( - inputs=[{'txid': prevtx['txid'], 'vout': 0}], - outputs=[{node.get_deterministic_priv_key().address: 50 - 0.00125}], - ) - sigtx = node.signrawtransactionwithkey( - hexstring=rawtx, - privkeys=[node.get_deterministic_priv_key().key], - prevtxs=[{ - 'txid': prevtx['txid'], - 'vout': 0, - 'scriptPubKey': prevtx['vout'][0]['scriptPubKey']['hex'], - }], - )['hex'] - txpeer.send_message(msg_tx(tx_from_hex(sigtx))) + tx = self.wallet.create_self_transfer()['tx'] + txpeer.send_message(msg_tx(tx)) protected_peers.add(current_peer) self.log.info("Create 8 peers and protect them from eviction by having faster pings") @@ -133,5 +122,6 @@ class P2PEvict(BitcoinTestFramework): self.log.debug("{} protected peers: {}".format(len(protected_peers), protected_peers)) assert evicted_peers[0] not in protected_peers + if __name__ == '__main__': P2PEvict().main() diff --git a/test/functional/p2p_feefilter.py b/test/functional/p2p_feefilter.py index b65e927d5b..6b03cdf877 100755 --- a/test/functional/p2p_feefilter.py +++ b/test/functional/p2p_feefilter.py @@ -6,7 +6,6 @@ from decimal import Decimal -from test_framework.blocktools import COINBASE_MATURITY from test_framework.messages import MSG_TX, MSG_WTX, msg_feefilter from test_framework.p2p import P2PInterface, p2p_lock from test_framework.test_framework import BitcoinTestFramework @@ -80,9 +79,6 @@ class FeeFilterTest(BitcoinTestFramework): node1 = self.nodes[1] node0 = self.nodes[0] miniwallet = MiniWallet(node1) - # Add enough mature utxos to the wallet, so that all txs spend confirmed coins - self.generate(miniwallet, 5) - self.generate(node1, COINBASE_MATURITY) conn = self.nodes[0].add_p2p_connection(TestP2PConn()) diff --git a/test/functional/p2p_filter.py b/test/functional/p2p_filter.py index 3cf92b0316..b3e68ca536 100755 --- a/test/functional/p2p_filter.py +++ b/test/functional/p2p_filter.py @@ -214,7 +214,6 @@ class FilterTest(BitcoinTestFramework): def run_test(self): self.wallet = MiniWallet(self.nodes[0]) - self.wallet.rescan_utxos() filter_peer = self.nodes[0].add_p2p_connection(P2PBloomFilter()) self.log.info('Test filter size limits') diff --git a/test/functional/p2p_headers_sync_with_minchainwork.py b/test/functional/p2p_headers_sync_with_minchainwork.py index b07077c668..832fd7e0e9 100755 --- a/test/functional/p2p_headers_sync_with_minchainwork.py +++ b/test/functional/p2p_headers_sync_with_minchainwork.py @@ -27,6 +27,7 @@ NODE2_BLOCKS_REQUIRED = 2047 class RejectLowDifficultyHeadersTest(BitcoinTestFramework): def set_test_params(self): + self.rpc_timeout *= 4 # To avoid timeout when generating BLOCKS_TO_MINE self.setup_clean_chain = True self.num_nodes = 4 # Node0 has no required chainwork; node1 requires 15 blocks on top of the genesis block; node2 requires 2047 diff --git a/test/functional/p2p_i2p_sessions.py b/test/functional/p2p_i2p_sessions.py index 4e52522b81..9e7fdc6e14 100755 --- a/test/functional/p2p_i2p_sessions.py +++ b/test/functional/p2p_i2p_sessions.py @@ -23,12 +23,12 @@ class I2PSessions(BitcoinTestFramework): self.log.info("Ensure we create a persistent session when -i2pacceptincoming=1") node0 = self.nodes[0] - with node0.assert_debug_log(expected_msgs=[f"Creating persistent SAM session"]): + with node0.assert_debug_log(expected_msgs=["Creating persistent SAM session"]): node0.addnode(node=addr, command="onetry") self.log.info("Ensure we create a transient session when -i2pacceptincoming=0") node1 = self.nodes[1] - with node1.assert_debug_log(expected_msgs=[f"Creating transient SAM session"]): + with node1.assert_debug_log(expected_msgs=["Creating transient SAM session"]): node1.addnode(node=addr, command="onetry") diff --git a/test/functional/p2p_ibd_stalling.py b/test/functional/p2p_ibd_stalling.py new file mode 100755 index 0000000000..aca98ceb3f --- /dev/null +++ b/test/functional/p2p_ibd_stalling.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022- The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +""" +Test stalling logic during IBD +""" + +import time + +from test_framework.blocktools import ( + create_block, + create_coinbase +) +from test_framework.messages import ( + MSG_BLOCK, + MSG_TYPE_MASK, +) +from test_framework.p2p import ( + CBlockHeader, + msg_block, + msg_headers, + P2PDataStore, +) +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, +) + + +class P2PStaller(P2PDataStore): + def __init__(self, stall_block): + self.stall_block = stall_block + super().__init__() + + def on_getdata(self, message): + for inv in message.inv: + self.getdata_requests.append(inv.hash) + if (inv.type & MSG_TYPE_MASK) == MSG_BLOCK: + if (inv.hash != self.stall_block): + self.send_message(msg_block(self.block_store[inv.hash])) + + def on_getheaders(self, message): + pass + + +class P2PIBDStallingTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 1 + + def run_test(self): + NUM_BLOCKS = 1025 + NUM_PEERS = 4 + node = self.nodes[0] + tip = int(node.getbestblockhash(), 16) + blocks = [] + height = 1 + block_time = node.getblock(node.getbestblockhash())['time'] + 1 + self.log.info("Prepare blocks without sending them to the node") + block_dict = {} + for _ in range(NUM_BLOCKS): + blocks.append(create_block(tip, create_coinbase(height), block_time)) + blocks[-1].solve() + tip = blocks[-1].sha256 + block_time += 1 + height += 1 + block_dict[blocks[-1].sha256] = blocks[-1] + stall_block = blocks[0].sha256 + + headers_message = msg_headers() + headers_message.headers = [CBlockHeader(b) for b in blocks[:NUM_BLOCKS-1]] + peers = [] + + self.log.info("Check that a staller does not get disconnected if the 1024 block lookahead buffer is filled") + for id in range(NUM_PEERS): + peers.append(node.add_outbound_p2p_connection(P2PStaller(stall_block), p2p_idx=id, connection_type="outbound-full-relay")) + peers[-1].block_store = block_dict + peers[-1].send_message(headers_message) + + # Need to wait until 1023 blocks are received - the magic total bytes number is a workaround in lack of an rpc + # returning the number of downloaded (but not connected) blocks. + self.wait_until(lambda: self.total_bytes_recv_for_blocks() == 172761) + + self.all_sync_send_with_ping(peers) + # If there was a peer marked for stalling, it would get disconnected + self.mocktime = int(time.time()) + 3 + node.setmocktime(self.mocktime) + self.all_sync_send_with_ping(peers) + assert_equal(node.num_test_p2p_connections(), NUM_PEERS) + + self.log.info("Check that increasing the window beyond 1024 blocks triggers stalling logic") + headers_message.headers = [CBlockHeader(b) for b in blocks] + with node.assert_debug_log(expected_msgs=['Stall started']): + for p in peers: + p.send_message(headers_message) + self.all_sync_send_with_ping(peers) + + self.log.info("Check that the stalling peer is disconnected after 2 seconds") + self.mocktime += 3 + node.setmocktime(self.mocktime) + peers[0].wait_for_disconnect() + assert_equal(node.num_test_p2p_connections(), NUM_PEERS - 1) + self.wait_until(lambda: self.is_block_requested(peers, stall_block)) + # Make sure that SendMessages() is invoked, which assigns the missing block + # to another peer and starts the stalling logic for them + self.all_sync_send_with_ping(peers) + + self.log.info("Check that the stalling timeout gets doubled to 4 seconds for the next staller") + # No disconnect after just 3 seconds + self.mocktime += 3 + node.setmocktime(self.mocktime) + self.all_sync_send_with_ping(peers) + assert_equal(node.num_test_p2p_connections(), NUM_PEERS - 1) + + self.mocktime += 2 + node.setmocktime(self.mocktime) + self.wait_until(lambda: sum(x.is_connected for x in node.p2ps) == NUM_PEERS - 2) + self.wait_until(lambda: self.is_block_requested(peers, stall_block)) + self.all_sync_send_with_ping(peers) + + self.log.info("Check that the stalling timeout gets doubled to 8 seconds for the next staller") + # No disconnect after just 7 seconds + self.mocktime += 7 + node.setmocktime(self.mocktime) + self.all_sync_send_with_ping(peers) + assert_equal(node.num_test_p2p_connections(), NUM_PEERS - 2) + + self.mocktime += 2 + node.setmocktime(self.mocktime) + self.wait_until(lambda: sum(x.is_connected for x in node.p2ps) == NUM_PEERS - 3) + self.wait_until(lambda: self.is_block_requested(peers, stall_block)) + self.all_sync_send_with_ping(peers) + + self.log.info("Provide the withheld block and check that stalling timeout gets reduced back to 2 seconds") + with node.assert_debug_log(expected_msgs=['Decreased stalling timeout to 2 seconds']): + for p in peers: + if p.is_connected and (stall_block in p.getdata_requests): + p.send_message(msg_block(block_dict[stall_block])) + + self.log.info("Check that all outstanding blocks get connected") + self.wait_until(lambda: node.getblockcount() == NUM_BLOCKS) + + def total_bytes_recv_for_blocks(self): + total = 0 + for info in self.nodes[0].getpeerinfo(): + if ("block" in info["bytesrecv_per_msg"].keys()): + total += info["bytesrecv_per_msg"]["block"] + return total + + def all_sync_send_with_ping(self, peers): + for p in peers: + if p.is_connected: + p.sync_send_with_ping() + + def is_block_requested(self, peers, hash): + for p in peers: + if p.is_connected and (hash in p.getdata_requests): + return True + return False + + +if __name__ == '__main__': + P2PIBDStallingTest().main() diff --git a/test/functional/p2p_invalid_messages.py b/test/functional/p2p_invalid_messages.py index 3109ad2b56..644abda914 100755 --- a/test/functional/p2p_invalid_messages.py +++ b/test/functional/p2p_invalid_messages.py @@ -13,11 +13,12 @@ from test_framework.messages import ( MAX_HEADERS_RESULTS, MAX_INV_SIZE, MAX_PROTOCOL_MESSAGE_LENGTH, + MSG_TX, + from_hex, msg_getdata, msg_headers, msg_inv, msg_ping, - MSG_TX, msg_version, ser_string, ) @@ -73,11 +74,17 @@ class InvalidMessagesTest(BitcoinTestFramework): self.test_oversized_inv_msg() self.test_oversized_getdata_msg() self.test_oversized_headers_msg() + self.test_invalid_pow_headers_msg() self.test_resource_exhaustion() def test_buffer(self): self.log.info("Test message with header split across two buffers is received") conn = self.nodes[0].add_p2p_connection(P2PDataStore()) + # After add_p2p_connection both sides have the verack processed. + # However the pong from conn in reply to the ping from the node has not + # been processed and recorded in totalbytesrecv. + # Flush the pong from conn by sending a ping from conn. + conn.sync_with_ping(timeout=1) # Create valid message msg = conn.build_message(msg_ping(nonce=12345)) cut_pos = 12 # Chosen at an arbitrary position within the header @@ -87,8 +94,6 @@ class InvalidMessagesTest(BitcoinTestFramework): # Wait until node has processed the first half of the message self.wait_until(lambda: self.nodes[0].getnettotals()['totalbytesrecv'] != before) middle = self.nodes[0].getnettotals()['totalbytesrecv'] - # If this assert fails, we've hit an unlikely race - # where the test framework sent a message in between the two halves assert_equal(middle, before + cut_pos) conn.send_raw_message(msg[cut_pos:]) conn.sync_with_ping(timeout=1) @@ -248,6 +253,36 @@ class InvalidMessagesTest(BitcoinTestFramework): size = MAX_HEADERS_RESULTS + 1 self.test_oversized_msg(msg_headers([CBlockHeader()] * size), size) + def test_invalid_pow_headers_msg(self): + self.log.info("Test headers message with invalid proof-of-work is logged as misbehaving and disconnects peer") + blockheader_tip_hash = self.nodes[0].getbestblockhash() + blockheader_tip = from_hex(CBlockHeader(), self.nodes[0].getblockheader(blockheader_tip_hash, False)) + + # send valid headers message first + assert_equal(self.nodes[0].getblockchaininfo()['headers'], 0) + blockheader = CBlockHeader() + blockheader.hashPrevBlock = int(blockheader_tip_hash, 16) + blockheader.nTime = int(time.time()) + blockheader.nBits = blockheader_tip.nBits + blockheader.rehash() + while not blockheader.hash.startswith('0'): + blockheader.nNonce += 1 + blockheader.rehash() + peer = self.nodes[0].add_p2p_connection(P2PInterface()) + peer.send_and_ping(msg_headers([blockheader])) + assert_equal(self.nodes[0].getblockchaininfo()['headers'], 1) + chaintips = self.nodes[0].getchaintips() + assert_equal(chaintips[0]['status'], 'headers-only') + assert_equal(chaintips[0]['hash'], blockheader.hash) + + # invalidate PoW + while not blockheader.hash.startswith('f'): + blockheader.nNonce += 1 + blockheader.rehash() + with self.nodes[0].assert_debug_log(['Misbehaving', 'header with invalid proof of work']): + peer.send_message(msg_headers([blockheader])) + peer.wait_for_disconnect() + def test_resource_exhaustion(self): self.log.info("Test node stays up despite many large junk messages") conn = self.nodes[0].add_p2p_connection(P2PDataStore()) diff --git a/test/functional/p2p_leak_tx.py b/test/functional/p2p_leak_tx.py index 4c064b359e..6283dd89ac 100755 --- a/test/functional/p2p_leak_tx.py +++ b/test/functional/p2p_leak_tx.py @@ -4,7 +4,6 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test that we don't leak txs to inbound peers that we haven't yet announced to""" -from test_framework.blocktools import COINBASE_MATURITY from test_framework.messages import msg_getdata, CInv, MSG_TX from test_framework.p2p import p2p_lock, P2PDataStore from test_framework.test_framework import BitcoinTestFramework @@ -26,9 +25,6 @@ class P2PLeakTxTest(BitcoinTestFramework): def run_test(self): gen_node = self.nodes[0] # The block and tx generating node miniwallet = MiniWallet(gen_node) - # Add enough mature utxos to the wallet, so that all txs spend confirmed coins - self.generate(miniwallet, 1) - self.generate(gen_node, COINBASE_MATURITY) inbound_peer = self.nodes[0].add_p2p_connection(P2PNode()) # An "attacking" inbound peer diff --git a/test/functional/p2p_node_network_limited.py b/test/functional/p2p_node_network_limited.py index 5a0003d3ef..a56afbcf7b 100755 --- a/test/functional/p2p_node_network_limited.py +++ b/test/functional/p2p_node_network_limited.py @@ -85,7 +85,7 @@ class NodeNetworkLimitedTest(BitcoinTestFramework): self.connect_nodes(0, 2) try: self.sync_blocks([self.nodes[0], self.nodes[2]], timeout=5) - except: + except Exception: pass # node2 must remain at height 0 assert_equal(self.nodes[2].getblockheader(self.nodes[2].getbestblockhash())['height'], 0) diff --git a/test/functional/p2p_permissions.py b/test/functional/p2p_permissions.py index f8d3fd919d..f84bbf67e6 100755 --- a/test/functional/p2p_permissions.py +++ b/test/functional/p2p_permissions.py @@ -7,30 +7,26 @@ Test that permissions are correctly calculated and applied """ -from test_framework.address import ADDRESS_BCRT1_P2WSH_OP_TRUE from test_framework.messages import ( - CTxInWitness, - tx_from_hex, + SEQUENCE_FINAL, ) from test_framework.p2p import P2PDataStore -from test_framework.script import ( - CScript, - OP_TRUE, -) from test_framework.test_node import ErrorMatch from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, p2p_port, ) +from test_framework.wallet import MiniWallet class P2PPermissionsTests(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 - self.setup_clean_chain = True def run_test(self): + self.wallet = MiniWallet(self.nodes[0]) + self.check_tx_relay() self.checkpermission( @@ -60,12 +56,12 @@ class P2PPermissionsTests(BitcoinTestFramework): # For this, we need to use whitebind instead of bind # by modifying the configuration file. ip_port = "127.0.0.1:{}".format(p2p_port(1)) - self.replaceinconfig(1, "bind=127.0.0.1", "whitebind=bloomfilter,forcerelay@" + ip_port) + self.nodes[1].replace_in_config([("bind=127.0.0.1", "whitebind=bloomfilter,forcerelay@" + ip_port)]) self.checkpermission( ["-whitelist=noban@127.0.0.1"], # Check parameter interaction forcerelay should activate relay ["noban", "bloomfilter", "forcerelay", "relay", "download"]) - self.replaceinconfig(1, "whitebind=bloomfilter,forcerelay@" + ip_port, "bind=127.0.0.1") + self.nodes[1].replace_in_config([("whitebind=bloomfilter,forcerelay@" + ip_port, "bind=127.0.0.1")]) self.checkpermission( # legacy whitelistrelay should be ignored @@ -94,8 +90,6 @@ class P2PPermissionsTests(BitcoinTestFramework): self.nodes[1].assert_start_raises_init_error(["-whitebind=noban@127.0.0.1", "-bind=127.0.0.1", "-listen=0"], "Cannot set -bind or -whitebind together with -listen=0", match=ErrorMatch.PARTIAL_REGEX) def check_tx_relay(self): - block_op_true = self.nodes[0].getblock(self.generatetoaddress(self.nodes[0], 100, ADDRESS_BCRT1_P2WSH_OP_TRUE)[0]) - self.log.debug("Create a connection from a forcerelay peer that rebroadcasts raw txs") # A test framework p2p connection is needed to send the raw transaction directly. If a full node was used, it could only # rebroadcast via the inv-getdata mechanism. However, even for forcerelay connections, a full node would @@ -104,18 +98,7 @@ class P2PPermissionsTests(BitcoinTestFramework): p2p_rebroadcast_wallet = self.nodes[1].add_p2p_connection(P2PDataStore()) self.log.debug("Send a tx from the wallet initially") - tx = tx_from_hex( - self.nodes[0].createrawtransaction( - inputs=[{ - 'txid': block_op_true['tx'][0], - 'vout': 0, - }], outputs=[{ - ADDRESS_BCRT1_P2WSH_OP_TRUE: 5, - }], - replaceable=False), - ) - tx.wit.vtxinwit = [CTxInWitness()] - tx.wit.vtxinwit[0].scriptWitness.stack = [CScript([OP_TRUE])] + tx = self.wallet.create_self_transfer(sequence=SEQUENCE_FINAL)['tx'] txid = tx.rehash() self.log.debug("Wait until tx is in node[1]'s mempool") @@ -155,12 +138,6 @@ class P2PPermissionsTests(BitcoinTestFramework): if p not in peerinfo['permissions']: raise AssertionError("Expected permissions %r is not granted." % p) - def replaceinconfig(self, nodeid, old, new): - with open(self.nodes[nodeid].bitcoinconf, encoding="utf8") as f: - newText = f.read().replace(old, new) - with open(self.nodes[nodeid].bitcoinconf, 'w', encoding="utf8") as f: - f.write(newText) - if __name__ == '__main__': P2PPermissionsTests().main() diff --git a/test/functional/p2p_segwit.py b/test/functional/p2p_segwit.py index 2ddc09b13b..b0900e49b8 100755 --- a/test/functional/p2p_segwit.py +++ b/test/functional/p2p_segwit.py @@ -622,8 +622,10 @@ class SegWitTest(BitcoinTestFramework): if not self.segwit_active: # Just check mempool acceptance, but don't add the transaction to the mempool, since witness is disallowed # in blocks and the tx is impossible to mine right now. - assert_equal( - self.nodes[0].testmempoolaccept([tx3.serialize_with_witness().hex()]), + testres3 = self.nodes[0].testmempoolaccept([tx3.serialize_with_witness().hex()]) + testres3[0]["fees"].pop("effective-feerate") + testres3[0]["fees"].pop("effective-includes") + assert_equal(testres3, [{ 'txid': tx3.hash, 'wtxid': tx3.getwtxid(), @@ -639,8 +641,10 @@ class SegWitTest(BitcoinTestFramework): tx3 = tx tx3.vout = [tx3_out] tx3.rehash() - assert_equal( - self.nodes[0].testmempoolaccept([tx3.serialize_with_witness().hex()]), + testres3_replaced = self.nodes[0].testmempoolaccept([tx3.serialize_with_witness().hex()]) + testres3_replaced[0]["fees"].pop("effective-feerate") + testres3_replaced[0]["fees"].pop("effective-includes") + assert_equal(testres3_replaced, [{ 'txid': tx3.hash, 'wtxid': tx3.getwtxid(), diff --git a/test/functional/p2p_sendheaders.py b/test/functional/p2p_sendheaders.py index 1ccc447b89..508d6fe403 100755 --- a/test/functional/p2p_sendheaders.py +++ b/test/functional/p2p_sendheaders.py @@ -546,15 +546,15 @@ class SendHeadersTest(BitcoinTestFramework): blocks = [] # Now we test that if we repeatedly don't send connecting headers, we # don't go into an infinite loop trying to get them to connect. - MAX_UNCONNECTING_HEADERS = 10 - for _ in range(MAX_UNCONNECTING_HEADERS + 1): + MAX_NUM_UNCONNECTING_HEADERS_MSGS = 10 + for _ in range(MAX_NUM_UNCONNECTING_HEADERS_MSGS + 1): blocks.append(create_block(tip, create_coinbase(height), block_time)) blocks[-1].solve() tip = blocks[-1].sha256 block_time += 1 height += 1 - for i in range(1, MAX_UNCONNECTING_HEADERS): + for i in range(1, MAX_NUM_UNCONNECTING_HEADERS_MSGS): # Send a header that doesn't connect, check that we get a getheaders. with p2p_lock: test_node.last_message.pop("getheaders", None) @@ -568,8 +568,8 @@ class SendHeadersTest(BitcoinTestFramework): blocks = blocks[2:] # Now try to see how many unconnecting headers we can send - # before we get disconnected. Should be 5*MAX_UNCONNECTING_HEADERS - for i in range(5 * MAX_UNCONNECTING_HEADERS - 1): + # before we get disconnected. Should be 5*MAX_NUM_UNCONNECTING_HEADERS_MSGS + for i in range(5 * MAX_NUM_UNCONNECTING_HEADERS_MSGS - 1): # Send a header that doesn't connect, check that we get a getheaders. with p2p_lock: test_node.last_message.pop("getheaders", None) diff --git a/test/functional/p2p_tx_download.py b/test/functional/p2p_tx_download.py index 7356b8bbb3..0e463c5072 100755 --- a/test/functional/p2p_tx_download.py +++ b/test/functional/p2p_tx_download.py @@ -5,6 +5,7 @@ """ Test transaction download behavior """ +import time from test_framework.messages import ( CInv, @@ -13,7 +14,6 @@ from test_framework.messages import ( MSG_WTX, msg_inv, msg_notfound, - tx_from_hex, ) from test_framework.p2p import ( P2PInterface, @@ -23,9 +23,7 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, ) -from test_framework.address import ADDRESS_BCRT1_UNSPENDABLE - -import time +from test_framework.wallet import MiniWallet class TestP2PConn(P2PInterface): @@ -88,19 +86,8 @@ class TxDownloadTest(BitcoinTestFramework): def test_inv_block(self): self.log.info("Generate a transaction on node 0") - tx = self.nodes[0].createrawtransaction( - inputs=[{ # coinbase - "txid": self.nodes[0].getblock(self.nodes[0].getblockhash(1))['tx'][0], - "vout": 0 - }], - outputs={ADDRESS_BCRT1_UNSPENDABLE: 50 - 0.00025}, - ) - tx = self.nodes[0].signrawtransactionwithkey( - hexstring=tx, - privkeys=[self.nodes[0].get_deterministic_priv_key().key], - )['hex'] - ctx = tx_from_hex(tx) - txid = int(ctx.rehash(), 16) + tx = self.wallet.create_self_transfer() + txid = int(tx['txid'], 16) self.log.info( "Announce the transaction to all nodes from all {} incoming peers, but never send it".format(NUM_INBOUND)) @@ -109,7 +96,7 @@ class TxDownloadTest(BitcoinTestFramework): p.send_and_ping(msg) self.log.info("Put the tx in node 0's mempool") - self.nodes[0].sendrawtransaction(tx) + self.nodes[0].sendrawtransaction(tx['hex']) # Since node 1 is connected outbound to an honest peer (node 0), it # should get the tx within a timeout. (Assuming that node 0 @@ -255,6 +242,8 @@ class TxDownloadTest(BitcoinTestFramework): self.nodes[0].p2ps[0].send_message(msg_notfound(vec=[CInv(MSG_TX, 1)])) def run_test(self): + self.wallet = MiniWallet(self.nodes[0]) + # Run tests without mocktime that only need one peer-connection first, to avoid restarting the nodes self.test_expiry_fallback() self.test_disconnect_fallback() diff --git a/test/functional/p2p_tx_privacy.py b/test/functional/p2p_tx_privacy.py index b885ccdf5d..e674f6c3eb 100755 --- a/test/functional/p2p_tx_privacy.py +++ b/test/functional/p2p_tx_privacy.py @@ -53,7 +53,6 @@ class TxPrivacyTest(BitcoinTestFramework): def run_test(self): self.wallet = MiniWallet(self.nodes[0]) - self.wallet.rescan_utxos() tx_originator = self.nodes[0].add_p2p_connection(P2PInterface()) spy = self.nodes[0].add_p2p_connection(P2PTxSpy(), wait_for_verack=False) diff --git a/test/functional/rpc_blockchain.py b/test/functional/rpc_blockchain.py index 19c73eebf0..6022042c11 100755 --- a/test/functional/rpc_blockchain.py +++ b/test/functional/rpc_blockchain.py @@ -25,6 +25,7 @@ from decimal import Decimal import http.client import os import subprocess +import textwrap from test_framework.blocktools import ( MAX_FUTURE_BLOCK_TIME, @@ -68,6 +69,7 @@ class BlockchainTest(BitcoinTestFramework): def run_test(self): self.wallet = MiniWallet(self.nodes[0]) + self._test_prune_disk_space() self.mine_chain() self._test_max_future_block_time() self.restart_node( @@ -99,6 +101,13 @@ class BlockchainTest(BitcoinTestFramework): self.generate(self.wallet, 1) assert_equal(self.nodes[0].getblockchaininfo()['blocks'], HEIGHT) + def _test_prune_disk_space(self): + self.log.info("Test that a manually pruned node does not run into " + "integer overflow on first start up") + self.restart_node(0, extra_args=["-prune=1"]) + self.log.info("Avoid warning when assumed chain size is enough") + self.restart_node(0, extra_args=["-prune=123456789"]) + def _test_max_future_block_time(self): self.stop_node(0) self.log.info("A block tip of more than MAX_FUTURE_BLOCK_TIME in the future raises an error") @@ -429,6 +438,17 @@ class BlockchainTest(BitcoinTestFramework): def _test_getnetworkhashps(self): self.log.info("Test getnetworkhashps") hashes_per_second = self.nodes[0].getnetworkhashps() + assert_raises_rpc_error( + -3, + textwrap.dedent(""" + Wrong type passed: + { + "Position 1 (nblocks)": "JSON value of type string is not of expected type number", + "Position 2 (height)": "JSON value of type array is not of expected type number" + } + """).strip(), + lambda: self.nodes[0].getnetworkhashps("a", []), + ) # This should be 2 hashes every 10 minutes or 1/300 assert abs(hashes_per_second * 300 - 1) < 0.0001 diff --git a/test/functional/rpc_createmultisig.py b/test/functional/rpc_createmultisig.py index 7d03ed2951..bec499107f 100755 --- a/test/functional/rpc_createmultisig.py +++ b/test/functional/rpc_createmultisig.py @@ -8,6 +8,7 @@ import itertools import json import os +from test_framework.address import address_to_scriptpubkey from test_framework.blocktools import COINBASE_MATURITY from test_framework.authproxy import JSONRPCException from test_framework.descriptors import descsum_create, drop_origins @@ -193,7 +194,7 @@ class RpcCreateMultiSigTest(BitcoinTestFramework): assert mredeemw == mredeem wmulti.unloadwallet() - spk = bytes.fromhex(node0.validateaddress(madd)["scriptPubKey"]) + spk = address_to_scriptpubkey(madd) txid, _ = self.wallet.send_to(from_node=self.nodes[0], scriptPubKey=spk, amount=1300) tx = node0.getrawtransaction(txid, True) vout = [v["n"] for v in tx["vout"] if madd == v["scriptPubKey"]["address"]] diff --git a/test/functional/rpc_decodescript.py b/test/functional/rpc_decodescript.py index a61710b739..673836bd04 100755 --- a/test/functional/rpc_decodescript.py +++ b/test/functional/rpc_decodescript.py @@ -263,6 +263,19 @@ class DecodeScriptTest(BitcoinTestFramework): rpc_result = self.nodes[0].decodescript(script) assert_equal(result, rpc_result) + def decodescript_miniscript(self): + """Check that a Miniscript is decoded when possible under P2WSH context.""" + # Sourced from https://github.com/bitcoin/bitcoin/pull/27037#issuecomment-1416151907. + # Miniscript-compatible offered HTLC + res = self.nodes[0].decodescript("82012088a914ffffffffffffffffffffffffffffffffffffffff88210250929b74c1a04954b78b4b6035e97a5e078a5a0f28ec96d547bfee9ace803ac0ad51b2") + assert res["segwit"]["desc"] == "wsh(and_v(and_v(v:hash160(ffffffffffffffffffffffffffffffffffffffff),v:pk(0250929b74c1a04954b78b4b6035e97a5e078a5a0f28ec96d547bfee9ace803ac0)),older(1)))#gm8xz4fl" + # Miniscript-incompatible offered HTLC + res = self.nodes[0].decodescript("82012088a914ffffffffffffffffffffffffffffffffffffffff882102ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffacb2") + assert res["segwit"]["desc"] == "wsh(raw(82012088a914ffffffffffffffffffffffffffffffffffffffff882102ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffacb2))#ra6w2xa7" + # Miniscript-compatible multisig bigger than 520 byte P2SH limit. + res = self.nodes[0].decodescript("5b21020e0338c96a8870479f2396c373cc7696ba124e8635d41b0ea581112b678172612102675333a4e4b8fb51d9d4e22fa5a8eaced3fdac8a8cbf9be8c030f75712e6af992102896807d54bc55c24981f24a453c60ad3e8993d693732288068a23df3d9f50d4821029e51a5ef5db3137051de8323b001749932f2ff0d34c82e96a2c2461de96ae56c2102a4e1a9638d46923272c266631d94d36bdb03a64ee0e14c7518e49d2f29bc401021031c41fdbcebe17bec8d49816e00ca1b5ac34766b91c9f2ac37d39c63e5e008afb2103079e252e85abffd3c401a69b087e590a9b86f33f574f08129ccbd3521ecf516b2103111cf405b627e22135b3b3733a4a34aa5723fb0f58379a16d32861bf576b0ec2210318f331b3e5d38156da6633b31929c5b220349859cc9ca3d33fb4e68aa08401742103230dae6b4ac93480aeab26d000841298e3b8f6157028e47b0897c1e025165de121035abff4281ff00660f99ab27bb53e6b33689c2cd8dcd364bc3c90ca5aea0d71a62103bd45cddfacf2083b14310ae4a84e25de61e451637346325222747b157446614c2103cc297026b06c71cbfa52089149157b5ff23de027ac5ab781800a578192d175462103d3bde5d63bdb3a6379b461be64dad45eabff42f758543a9645afd42f6d4248282103ed1e8d5109c9ed66f7941bc53cc71137baa76d50d274bda8d5e8ffbd6e61fe9a5fae736402c00fb269522103aab896d53a8e7d6433137bbba940f9c521e085dd07e60994579b64a6d992cf79210291b7d0b1b692f8f524516ed950872e5da10fb1b808b5a526dedc6fed1cf29807210386aa9372fbab374593466bc5451dc59954e90787f08060964d95c87ef34ca5bb53ae68") + assert_equal(res["segwit"]["desc"], "wsh(or_d(multi(11,020e0338c96a8870479f2396c373cc7696ba124e8635d41b0ea581112b67817261,02675333a4e4b8fb51d9d4e22fa5a8eaced3fdac8a8cbf9be8c030f75712e6af99,02896807d54bc55c24981f24a453c60ad3e8993d693732288068a23df3d9f50d48,029e51a5ef5db3137051de8323b001749932f2ff0d34c82e96a2c2461de96ae56c,02a4e1a9638d46923272c266631d94d36bdb03a64ee0e14c7518e49d2f29bc4010,031c41fdbcebe17bec8d49816e00ca1b5ac34766b91c9f2ac37d39c63e5e008afb,03079e252e85abffd3c401a69b087e590a9b86f33f574f08129ccbd3521ecf516b,03111cf405b627e22135b3b3733a4a34aa5723fb0f58379a16d32861bf576b0ec2,0318f331b3e5d38156da6633b31929c5b220349859cc9ca3d33fb4e68aa0840174,03230dae6b4ac93480aeab26d000841298e3b8f6157028e47b0897c1e025165de1,035abff4281ff00660f99ab27bb53e6b33689c2cd8dcd364bc3c90ca5aea0d71a6,03bd45cddfacf2083b14310ae4a84e25de61e451637346325222747b157446614c,03cc297026b06c71cbfa52089149157b5ff23de027ac5ab781800a578192d17546,03d3bde5d63bdb3a6379b461be64dad45eabff42f758543a9645afd42f6d424828,03ed1e8d5109c9ed66f7941bc53cc71137baa76d50d274bda8d5e8ffbd6e61fe9a),and_v(v:older(4032),multi(2,03aab896d53a8e7d6433137bbba940f9c521e085dd07e60994579b64a6d992cf79,0291b7d0b1b692f8f524516ed950872e5da10fb1b808b5a526dedc6fed1cf29807,0386aa9372fbab374593466bc5451dc59954e90787f08060964d95c87ef34ca5bb))))#7jwwklk4") + def run_test(self): self.log.info("Test decoding of standard input scripts [scriptSig]") self.decodescript_script_sig() @@ -272,6 +285,8 @@ class DecodeScriptTest(BitcoinTestFramework): self.decoderawtransaction_asm_sighashtype() self.log.info("Data-driven tests") self.decodescript_datadriven_tests() + self.log.info("Miniscript descriptor decoding") + self.decodescript_miniscript() if __name__ == '__main__': DecodeScriptTest().main() diff --git a/test/functional/rpc_generate.py b/test/functional/rpc_generate.py index 89b410e37e..20f62079fd 100755 --- a/test/functional/rpc_generate.py +++ b/test/functional/rpc_generate.py @@ -28,10 +28,14 @@ class RPCGenerateTest(BitcoinTestFramework): def test_generateblock(self): node = self.nodes[0] miniwallet = MiniWallet(node) - miniwallet.rescan_utxos() - self.log.info('Generate an empty block to address') + self.log.info('Mine an empty block to address and return the hex') address = miniwallet.get_address() + generated_block = self.generateblock(node, output=address, transactions=[], submit=False) + node.submitblock(hexdata=generated_block['hex']) + assert_equal(generated_block['hash'], node.getbestblockhash()) + + self.log.info('Generate an empty block to address') hash = self.generateblock(node, output=address, transactions=[])['hash'] block = node.getblock(blockhash=hash, verbose=2) assert_equal(len(block['tx']), 1) diff --git a/test/functional/rpc_getblockfrompeer.py b/test/functional/rpc_getblockfrompeer.py index dddc779763..2f093bebff 100755 --- a/test/functional/rpc_getblockfrompeer.py +++ b/test/functional/rpc_getblockfrompeer.py @@ -24,14 +24,19 @@ from test_framework.util import ( class GetBlockFromPeerTest(BitcoinTestFramework): def set_test_params(self): - self.num_nodes = 2 + self.num_nodes = 3 + self.extra_args = [ + [], + [], + ["-fastprune", "-prune=1"] + ] def setup_network(self): self.setup_nodes() - def check_for_block(self, hash): + def check_for_block(self, node, hash): try: - self.nodes[0].getblock(hash) + self.nodes[node].getblock(hash) return True except JSONRPCException: return False @@ -48,7 +53,7 @@ class GetBlockFromPeerTest(BitcoinTestFramework): self.log.info("Connect nodes to sync headers") self.connect_nodes(0, 1) - self.sync_blocks() + self.sync_blocks(self.nodes[0:2]) self.log.info("Node 0 should only have the header for node 1's block 3") x = next(filter(lambda x: x['hash'] == short_tip, self.nodes[0].getchaintips())) @@ -81,7 +86,7 @@ class GetBlockFromPeerTest(BitcoinTestFramework): self.log.info("Successful fetch") result = self.nodes[0].getblockfrompeer(short_tip, peer_0_peer_1_id) - self.wait_until(lambda: self.check_for_block(short_tip), timeout=1) + self.wait_until(lambda: self.check_for_block(node=0, hash=short_tip), timeout=1) assert_equal(result, {}) self.log.info("Don't fetch blocks we already have") @@ -111,6 +116,40 @@ class GetBlockFromPeerTest(BitcoinTestFramework): error_msg = "In prune mode, only blocks that the node has already synced previously can be fetched from a peer" assert_raises_rpc_error(-1, error_msg, self.nodes[1].getblockfrompeer, blockhash, node1_interface_id) + self.log.info("Connect pruned node") + # We need to generate more blocks to be able to prune + self.connect_nodes(0, 2) + pruned_node = self.nodes[2] + self.generate(self.nodes[0], 400, sync_fun=self.no_op) + self.sync_blocks([self.nodes[0], pruned_node]) + pruneheight = pruned_node.pruneblockchain(300) + assert_equal(pruneheight, 248) + # Ensure the block is actually pruned + pruned_block = self.nodes[0].getblockhash(2) + assert_raises_rpc_error(-1, "Block not available (pruned data)", pruned_node.getblock, pruned_block) + + self.log.info("Fetch pruned block") + peers = pruned_node.getpeerinfo() + assert_equal(len(peers), 1) + pruned_node_peer_0_id = peers[0]["id"] + result = pruned_node.getblockfrompeer(pruned_block, pruned_node_peer_0_id) + self.wait_until(lambda: self.check_for_block(node=2, hash=pruned_block), timeout=1) + assert_equal(result, {}) + + self.log.info("Fetched block persists after next pruning event") + self.generate(self.nodes[0], 250, sync_fun=self.no_op) + self.sync_blocks([self.nodes[0], pruned_node]) + pruneheight += 251 + assert_equal(pruned_node.pruneblockchain(700), pruneheight) + assert_equal(pruned_node.getblock(pruned_block)["hash"], "36c56c5b5ebbaf90d76b0d1a074dcb32d42abab75b7ec6fa0ffd9b4fbce8f0f7") + + self.log.info("Fetched block can be pruned again when prune height exceeds the height of the tip at the time when the block was fetched") + self.generate(self.nodes[0], 250, sync_fun=self.no_op) + self.sync_blocks([self.nodes[0], pruned_node]) + pruneheight += 250 + assert_equal(pruned_node.pruneblockchain(1000), pruneheight) + assert_raises_rpc_error(-1, "Block not available (pruned data)", pruned_node.getblock, pruned_block) + if __name__ == '__main__': GetBlockFromPeerTest().main() diff --git a/test/functional/rpc_invalid_address_message.py b/test/functional/rpc_invalid_address_message.py index 452f857a44..fd282a9bc1 100755 --- a/test/functional/rpc_invalid_address_message.py +++ b/test/functional/rpc_invalid_address_message.py @@ -64,7 +64,7 @@ class InvalidAddressErrorMessageTest(BitcoinTestFramework): def test_validateaddress(self): # Invalid Bech32 self.check_invalid(BECH32_INVALID_SIZE, 'Invalid Bech32 address data size') - self.check_invalid(BECH32_INVALID_PREFIX, 'Not a valid Bech32 or Base58 encoding') + self.check_invalid(BECH32_INVALID_PREFIX, 'Invalid or unsupported Segwit (Bech32) or Base58 encoding.') self.check_invalid(BECH32_INVALID_BECH32, 'Version 1+ witness address must use Bech32m checksum') self.check_invalid(BECH32_INVALID_BECH32M, 'Version 0 witness address must use Bech32 checksum') self.check_invalid(BECH32_INVALID_VERSION, 'Invalid Bech32 address witness version') @@ -84,27 +84,31 @@ class InvalidAddressErrorMessageTest(BitcoinTestFramework): self.check_valid(BECH32_VALID_MULTISIG) # Invalid Base58 - self.check_invalid(BASE58_INVALID_PREFIX, 'Invalid prefix for Base58-encoded address') - self.check_invalid(BASE58_INVALID_CHECKSUM, 'Invalid checksum or length of Base58 address') - self.check_invalid(BASE58_INVALID_LENGTH, 'Invalid checksum or length of Base58 address') + self.check_invalid(BASE58_INVALID_PREFIX, 'Invalid or unsupported Base58-encoded address.') + self.check_invalid(BASE58_INVALID_CHECKSUM, 'Invalid checksum or length of Base58 address (P2PKH or P2SH)') + self.check_invalid(BASE58_INVALID_LENGTH, 'Invalid checksum or length of Base58 address (P2PKH or P2SH)') # Valid Base58 self.check_valid(BASE58_VALID) # Invalid address format - self.check_invalid(INVALID_ADDRESS, 'Not a valid Bech32 or Base58 encoding') - self.check_invalid(INVALID_ADDRESS_2, 'Not a valid Bech32 or Base58 encoding') + self.check_invalid(INVALID_ADDRESS, 'Invalid or unsupported Segwit (Bech32) or Base58 encoding.') + self.check_invalid(INVALID_ADDRESS_2, 'Invalid or unsupported Segwit (Bech32) or Base58 encoding.') - def test_getaddressinfo(self): node = self.nodes[0] - assert_raises_rpc_error(-5, "Invalid Bech32 address data size", node.getaddressinfo, BECH32_INVALID_SIZE) - - assert_raises_rpc_error(-5, "Not a valid Bech32 or Base58 encoding", node.getaddressinfo, BECH32_INVALID_PREFIX) + # Missing arg returns the help text + assert_raises_rpc_error(-1, "Return information about the given bitcoin address.", node.validateaddress) + # Explicit None is not allowed for required parameters + assert_raises_rpc_error(-3, "JSON value of type null is not of expected type string", node.validateaddress, None) - assert_raises_rpc_error(-5, "Invalid prefix for Base58-encoded address", node.getaddressinfo, BASE58_INVALID_PREFIX) + def test_getaddressinfo(self): + node = self.nodes[0] - assert_raises_rpc_error(-5, "Not a valid Bech32 or Base58 encoding", node.getaddressinfo, INVALID_ADDRESS) + assert_raises_rpc_error(-5, "Invalid Bech32 address data size", node.getaddressinfo, BECH32_INVALID_SIZE) + assert_raises_rpc_error(-5, "Invalid or unsupported Segwit (Bech32) or Base58 encoding.", node.getaddressinfo, BECH32_INVALID_PREFIX) + assert_raises_rpc_error(-5, "Invalid or unsupported Base58-encoded address.", node.getaddressinfo, BASE58_INVALID_PREFIX) + assert_raises_rpc_error(-5, "Invalid or unsupported Segwit (Bech32) or Base58 encoding.", node.getaddressinfo, INVALID_ADDRESS) def run_test(self): self.test_validateaddress() diff --git a/test/functional/rpc_mempool_info.py b/test/functional/rpc_mempool_info.py index ae9c6572cf..246af22e50 100755 --- a/test/functional/rpc_mempool_info.py +++ b/test/functional/rpc_mempool_info.py @@ -18,7 +18,6 @@ class RPCMempoolInfoTest(BitcoinTestFramework): def run_test(self): self.wallet = MiniWallet(self.nodes[0]) - self.wallet.rescan_utxos() confirmed_utxo = self.wallet.get_utxo() # Create a tree of unconfirmed transactions in the mempool: diff --git a/test/functional/rpc_net.py b/test/functional/rpc_net.py index af8b2ad72b..5fdd5daddf 100755 --- a/test/functional/rpc_net.py +++ b/test/functional/rpc_net.py @@ -11,7 +11,6 @@ from decimal import Decimal from itertools import product import time -from test_framework.blocktools import COINBASE_MATURITY import test_framework.messages from test_framework.p2p import ( P2PInterface, @@ -43,7 +42,6 @@ def assert_net_servicesnames(servicesflag, servicenames): class NetTest(BitcoinTestFramework): def set_test_params(self): - self.setup_clean_chain = True self.num_nodes = 2 self.extra_args = [["-minrelaytxfee=0.00001000"], ["-minrelaytxfee=0.00000500"]] self.supports_cli = False @@ -51,9 +49,6 @@ class NetTest(BitcoinTestFramework): def run_test(self): # We need miniwallet to make a transaction self.wallet = MiniWallet(self.nodes[0]) - self.generate(self.wallet, 1) - # Get out of IBD for the minfeefilter and getpeerinfo tests. - self.generate(self.nodes[0], COINBASE_MATURITY + 1) # By default, the test framework sets up an addnode connection from # node 1 --> node0. By connecting node0 --> node 1, we're left with diff --git a/test/functional/rpc_packages.py b/test/functional/rpc_packages.py index f1352f88d8..ae1a498e28 100755 --- a/test/functional/rpc_packages.py +++ b/test/functional/rpc_packages.py @@ -127,8 +127,8 @@ class RPCPackagesTest(BitcoinTestFramework): node = self.nodes[0] chain = self.wallet.create_self_transfer_chain(chain_length=25) - chain_hex = chain["chain_hex"] - chain_txns = chain["chain_txns"] + chain_hex = [t["hex"] for t in chain] + chain_txns = [t["tx"] for t in chain] self.log.info("Check that testmempoolaccept requires packages to be sorted by dependency") assert_equal(node.testmempoolaccept(rawtxs=chain_hex[::-1]), @@ -239,11 +239,14 @@ class RPCPackagesTest(BitcoinTestFramework): coin = self.wallet.get_utxo() fee = Decimal("0.00125000") replaceable_tx = self.wallet.create_self_transfer(utxo_to_spend=coin, sequence=MAX_BIP125_RBF_SEQUENCE, fee = fee) - testres_replaceable = node.testmempoolaccept([replaceable_tx["hex"]]) - assert_equal(testres_replaceable, [ - {"txid": replaceable_tx["txid"], "wtxid": replaceable_tx["wtxid"], - "allowed": True, "vsize": replaceable_tx["tx"].get_vsize(), "fees": { "base": fee }} - ]) + testres_replaceable = node.testmempoolaccept([replaceable_tx["hex"]])[0] + assert_equal(testres_replaceable["txid"], replaceable_tx["txid"]) + assert_equal(testres_replaceable["wtxid"], replaceable_tx["wtxid"]) + assert testres_replaceable["allowed"] + assert_equal(testres_replaceable["vsize"], replaceable_tx["tx"].get_vsize()) + assert_equal(testres_replaceable["fees"]["base"], fee) + assert_fee_amount(fee, replaceable_tx["tx"].get_vsize(), testres_replaceable["fees"]["effective-feerate"]) + assert_equal(testres_replaceable["fees"]["effective-includes"], [replaceable_tx["wtxid"]]) # Replacement transaction is identical except has double the fee replacement_tx = self.wallet.create_self_transfer(utxo_to_spend=coin, sequence=MAX_BIP125_RBF_SEQUENCE, fee = 2 * fee) @@ -287,11 +290,13 @@ class RPCPackagesTest(BitcoinTestFramework): peer = node.add_p2p_connection(P2PTxInvStore()) package_txns = [] + presubmitted_wtxids = set() for _ in range(num_parents): parent_tx = self.wallet.create_self_transfer(fee=DEFAULT_FEE) package_txns.append(parent_tx) if partial_submit and random.choice([True, False]): node.sendrawtransaction(parent_tx["hex"]) + presubmitted_wtxids.add(parent_tx["wtxid"]) child_tx = self.wallet.create_self_transfer_multi(utxos_to_spend=[tx["new_utxo"] for tx in package_txns], fee_per_output=10000) #DEFAULT_FEE package_txns.append(child_tx) @@ -302,57 +307,23 @@ class RPCPackagesTest(BitcoinTestFramework): for package_txn in package_txns: tx = package_txn["tx"] assert tx.getwtxid() in submitpackage_result["tx-results"] - tx_result = submitpackage_result["tx-results"][tx.getwtxid()] - assert_equal(tx_result, { - "txid": package_txn["txid"], - "vsize": tx.get_vsize(), - "fees": { - "base": DEFAULT_FEE, - } - }) + wtxid = tx.getwtxid() + assert wtxid in submitpackage_result["tx-results"] + tx_result = submitpackage_result["tx-results"][wtxid] + assert_equal(tx_result["txid"], tx.rehash()) + assert_equal(tx_result["vsize"], tx.get_vsize()) + assert_equal(tx_result["fees"]["base"], DEFAULT_FEE) + if wtxid not in presubmitted_wtxids: + assert_fee_amount(DEFAULT_FEE, tx.get_vsize(), tx_result["fees"]["effective-feerate"]) + assert_equal(tx_result["fees"]["effective-includes"], [wtxid]) # submitpackage result should be consistent with testmempoolaccept and getmempoolentry self.assert_equal_package_results(node, testmempoolaccept_result, submitpackage_result) - # Package feerate is calculated for the remaining transactions after deduplication and - # individual submission. If only 0 or 1 transaction is left, e.g. because all transactions - # had high-feerates or were already in the mempool, no package feerate is provided. - # In this case, since all of the parents have high fees, each is accepted individually. - assert "package-feerate" not in submitpackage_result - # The node should announce each transaction. No guarantees for propagation. peer.wait_for_broadcast([tx["tx"].getwtxid() for tx in package_txns]) self.generate(node, 1) - def test_submit_cpfp(self): - node = self.nodes[0] - peer = node.add_p2p_connection(P2PTxInvStore()) - - tx_poor = self.wallet.create_self_transfer(fee=0, fee_rate=0) - tx_rich = self.wallet.create_self_transfer(fee=DEFAULT_FEE) - package_txns = [tx_rich, tx_poor] - coins = [tx["new_utxo"] for tx in package_txns] - tx_child = self.wallet.create_self_transfer_multi(utxos_to_spend=coins, fee_per_output=10000) #DEFAULT_FEE - package_txns.append(tx_child) - - submitpackage_result = node.submitpackage([tx["hex"] for tx in package_txns]) - - rich_parent_result = submitpackage_result["tx-results"][tx_rich["wtxid"]] - poor_parent_result = submitpackage_result["tx-results"][tx_poor["wtxid"]] - child_result = submitpackage_result["tx-results"][tx_child["tx"].getwtxid()] - assert_equal(rich_parent_result["fees"]["base"], DEFAULT_FEE) - assert_equal(poor_parent_result["fees"]["base"], 0) - assert_equal(child_result["fees"]["base"], DEFAULT_FEE) - # Package feerate is calculated for the remaining transactions after deduplication and - # individual submission. Since this package had a 0-fee parent, package feerate must have - # been used and returned. - assert "package-feerate" in submitpackage_result - assert_fee_amount(DEFAULT_FEE, rich_parent_result["vsize"] + child_result["vsize"], submitpackage_result["package-feerate"]) - - # The node will broadcast each transaction, still abiding by its peer's fee filter - peer.wait_for_broadcast([tx["tx"].getwtxid() for tx in package_txns]) - self.generate(node, 1) - def test_submitpackage(self): node = self.nodes[0] @@ -361,12 +332,9 @@ class RPCPackagesTest(BitcoinTestFramework): self.test_submit_child_with_parents(num_parents, False) self.test_submit_child_with_parents(num_parents, True) - self.log.info("Submitpackage valid packages with CPFP") - self.test_submit_cpfp() - self.log.info("Submitpackage only allows packages of 1 child with its parents") # Chain of 3 transactions has too many generations - chain_hex = self.wallet.create_self_transfer_chain(chain_length=25)["chain_hex"] + chain_hex = [t["hex"] for t in self.wallet.create_self_transfer_chain(chain_length=25)] assert_raises_rpc_error(-25, "not-child-with-parents", node.submitpackage, chain_hex) diff --git a/test/functional/rpc_preciousblock.py b/test/functional/rpc_preciousblock.py index 91298937fd..3062a86565 100755 --- a/test/functional/rpc_preciousblock.py +++ b/test/functional/rpc_preciousblock.py @@ -16,7 +16,7 @@ def unidirectional_node_sync_via_rpc(node_src, node_dest): try: assert len(node_dest.getblock(blockhash, False)) > 0 break - except: + except Exception: blocks_to_copy.append(blockhash) blockhash = node_src.getblockheader(blockhash, True)['previousblockhash'] blocks_to_copy.reverse() diff --git a/test/functional/rpc_psbt.py b/test/functional/rpc_psbt.py index a50e0fb244..ef773463d8 100755 --- a/test/functional/rpc_psbt.py +++ b/test/functional/rpc_psbt.py @@ -4,7 +4,6 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the Partially Signed Transaction RPCs. """ - from decimal import Decimal from itertools import product @@ -27,6 +26,7 @@ from test_framework.psbt import ( PSBT_IN_SHA256, PSBT_IN_HASH160, PSBT_IN_HASH256, + PSBT_IN_NON_WITNESS_UTXO, PSBT_IN_WITNESS_UTXO, PSBT_OUT_TAP_TREE, ) @@ -36,6 +36,7 @@ from test_framework.util import ( assert_approx, assert_equal, assert_greater_than, + assert_greater_than_or_equal, assert_raises_rpc_error, find_output, find_vout_for_address, @@ -58,13 +59,16 @@ class PSBTTest(BitcoinTestFramework): ["-walletrbf=0", "-changetype=legacy"], [] ] + # whitelist peers to speed up tx relay / mempool sync + for args in self.extra_args: + args.append("-whitelist=noban@127.0.0.1") self.supports_cli = False def skip_test_if_missing_module(self): self.skip_if_no_wallet() - # TODO: Re-enable this test with segwit v1 def test_utxo_conversion(self): + self.log.info("Check that non-witness UTXOs are removed for segwit v1+ inputs") mining_node = self.nodes[2] offline_node = self.nodes[0] online_node = self.nodes[1] @@ -76,36 +80,102 @@ class PSBTTest(BitcoinTestFramework): # Create watchonly on online_node online_node.createwallet(wallet_name='wonline', disable_private_keys=True) wonline = online_node.get_wallet_rpc('wonline') - w2 = online_node.get_wallet_rpc('') + w2 = online_node.get_wallet_rpc(self.default_wallet_name) # Mine a transaction that credits the offline address - offline_addr = offline_node.getnewaddress(address_type="p2sh-segwit") - online_addr = w2.getnewaddress(address_type="p2sh-segwit") + offline_addr = offline_node.getnewaddress(address_type="bech32m") + online_addr = w2.getnewaddress(address_type="bech32m") wonline.importaddress(offline_addr, "", False) - mining_node.sendtoaddress(address=offline_addr, amount=1.0) - self.generate(mining_node, nblocks=1) + mining_wallet = mining_node.get_wallet_rpc(self.default_wallet_name) + mining_wallet.sendtoaddress(address=offline_addr, amount=1.0) + self.generate(mining_node, nblocks=1, sync_fun=lambda: self.sync_all([online_node, mining_node])) - # Construct an unsigned PSBT on the online node (who doesn't know the output is Segwit, so will include a non-witness UTXO) + # Construct an unsigned PSBT on the online node utxos = wonline.listunspent(addresses=[offline_addr]) raw = wonline.createrawtransaction([{"txid":utxos[0]["txid"], "vout":utxos[0]["vout"]}],[{online_addr:0.9999}]) psbt = wonline.walletprocesspsbt(online_node.converttopsbt(raw))["psbt"] - assert "non_witness_utxo" in mining_node.decodepsbt(psbt)["inputs"][0] + assert not "not_witness_utxo" in mining_node.decodepsbt(psbt)["inputs"][0] + + # add non-witness UTXO manually + psbt_new = PSBT.from_base64(psbt) + prev_tx = wonline.gettransaction(utxos[0]["txid"])["hex"] + psbt_new.i[0].map[PSBT_IN_NON_WITNESS_UTXO] = bytes.fromhex(prev_tx) + assert "non_witness_utxo" in mining_node.decodepsbt(psbt_new.to_base64())["inputs"][0] - # Have the offline node sign the PSBT (which will update the UTXO to segwit) - signed_psbt = offline_node.walletprocesspsbt(psbt)["psbt"] - assert "witness_utxo" in mining_node.decodepsbt(signed_psbt)["inputs"][0] + # Have the offline node sign the PSBT (which will remove the non-witness UTXO) + signed_psbt = offline_node.walletprocesspsbt(psbt_new.to_base64())["psbt"] + assert not "non_witness_utxo" in mining_node.decodepsbt(signed_psbt)["inputs"][0] # Make sure we can mine the resulting transaction txid = mining_node.sendrawtransaction(mining_node.finalizepsbt(signed_psbt)["hex"]) - self.generate(mining_node, 1) + self.generate(mining_node, nblocks=1, sync_fun=lambda: self.sync_all([online_node, mining_node])) assert_equal(online_node.gettxout(txid,0)["confirmations"], 1) wonline.unloadwallet() # Reconnect - self.connect_nodes(0, 1) + self.connect_nodes(1, 0) self.connect_nodes(0, 2) + def test_input_confs_control(self): + self.nodes[0].createwallet("minconf") + wallet = self.nodes[0].get_wallet_rpc("minconf") + + # Fund the wallet with different chain heights + for _ in range(2): + self.nodes[1].sendmany("", {wallet.getnewaddress():1, wallet.getnewaddress():1}) + self.generate(self.nodes[1], 1) + + unconfirmed_txid = wallet.sendtoaddress(wallet.getnewaddress(), 0.5) + + self.log.info("Crafting PSBT using an unconfirmed input") + target_address = self.nodes[1].getnewaddress() + psbtx1 = wallet.walletcreatefundedpsbt([], {target_address: 0.1}, 0, {'fee_rate': 1, 'maxconf': 0})['psbt'] + + # Make sure we only had the one input + tx1_inputs = self.nodes[0].decodepsbt(psbtx1)['tx']['vin'] + assert_equal(len(tx1_inputs), 1) + + utxo1 = tx1_inputs[0] + assert_equal(unconfirmed_txid, utxo1['txid']) + + signed_tx1 = wallet.walletprocesspsbt(psbtx1)['psbt'] + final_tx1 = wallet.finalizepsbt(signed_tx1)['hex'] + txid1 = self.nodes[0].sendrawtransaction(final_tx1) + + mempool = self.nodes[0].getrawmempool() + assert txid1 in mempool + + self.log.info("Fail to craft a new PSBT that sends more funds with add_inputs = False") + assert_raises_rpc_error(-4, "The preselected coins total amount does not cover the transaction target. Please allow other inputs to be automatically selected or include more coins manually", wallet.walletcreatefundedpsbt, [{'txid': utxo1['txid'], 'vout': utxo1['vout']}], {target_address: 1}, 0, {'add_inputs': False}) + + self.log.info("Fail to craft a new PSBT with minconf above highest one") + assert_raises_rpc_error(-4, "Insufficient funds", wallet.walletcreatefundedpsbt, [{'txid': utxo1['txid'], 'vout': utxo1['vout']}], {target_address: 1}, 0, {'add_inputs': True, 'minconf': 3, 'fee_rate': 10}) + + self.log.info("Fail to broadcast a new PSBT with maxconf 0 due to BIP125 rules to verify it actually chose unconfirmed outputs") + psbt_invalid = wallet.walletcreatefundedpsbt([{'txid': utxo1['txid'], 'vout': utxo1['vout']}], {target_address: 1}, 0, {'add_inputs': True, 'maxconf': 0, 'fee_rate': 10})['psbt'] + signed_invalid = wallet.walletprocesspsbt(psbt_invalid)['psbt'] + final_invalid = wallet.finalizepsbt(signed_invalid)['hex'] + assert_raises_rpc_error(-26, "bad-txns-spends-conflicting-tx", self.nodes[0].sendrawtransaction, final_invalid) + + self.log.info("Craft a replacement adding inputs with highest confs possible") + psbtx2 = wallet.walletcreatefundedpsbt([{'txid': utxo1['txid'], 'vout': utxo1['vout']}], {target_address: 1}, 0, {'add_inputs': True, 'minconf': 2, 'fee_rate': 10})['psbt'] + tx2_inputs = self.nodes[0].decodepsbt(psbtx2)['tx']['vin'] + assert_greater_than_or_equal(len(tx2_inputs), 2) + for vin in tx2_inputs: + if vin['txid'] != unconfirmed_txid: + assert_greater_than_or_equal(self.nodes[0].gettxout(vin['txid'], vin['vout'])['confirmations'], 2) + + signed_tx2 = wallet.walletprocesspsbt(psbtx2)['psbt'] + final_tx2 = wallet.finalizepsbt(signed_tx2)['hex'] + txid2 = self.nodes[0].sendrawtransaction(final_tx2) + + mempool = self.nodes[0].getrawmempool() + assert txid1 not in mempool + assert txid2 in mempool + + wallet.unloadwallet() + def assert_change_type(self, psbtx, expected_type): """Assert that the given PSBT has a change output with the given type.""" @@ -511,8 +581,10 @@ class PSBTTest(BitcoinTestFramework): for i, signer in enumerate(signers): self.nodes[2].unloadwallet("wallet{}".format(i)) - # TODO: Re-enable this for segwit v1 - # self.test_utxo_conversion() + if self.options.descriptors: + self.test_utxo_conversion() + + self.test_input_confs_control() # Test that psbts with p2pkh outputs are created properly p2pkh = self.nodes[0].getnewaddress(address_type='legacy') @@ -549,17 +621,17 @@ class PSBTTest(BitcoinTestFramework): # Bech32 inputs should be filled with witness UTXO. Other inputs should not be filled because they are non-witness updated = self.nodes[1].utxoupdatepsbt(psbt) decoded = self.nodes[1].decodepsbt(updated) - test_psbt_input_keys(decoded['inputs'][0], ['witness_utxo']) - test_psbt_input_keys(decoded['inputs'][1], []) - test_psbt_input_keys(decoded['inputs'][2], []) + test_psbt_input_keys(decoded['inputs'][0], ['witness_utxo', 'non_witness_utxo']) + test_psbt_input_keys(decoded['inputs'][1], ['non_witness_utxo']) + test_psbt_input_keys(decoded['inputs'][2], ['non_witness_utxo']) # Try again, now while providing descriptors, making P2SH-segwit work, and causing bip32_derivs and redeem_script to be filled in descs = [self.nodes[1].getaddressinfo(addr)['desc'] for addr in [addr1,addr2,addr3]] updated = self.nodes[1].utxoupdatepsbt(psbt=psbt, descriptors=descs) decoded = self.nodes[1].decodepsbt(updated) - test_psbt_input_keys(decoded['inputs'][0], ['witness_utxo', 'bip32_derivs']) - test_psbt_input_keys(decoded['inputs'][1], []) - test_psbt_input_keys(decoded['inputs'][2], ['witness_utxo', 'bip32_derivs', 'redeem_script']) + test_psbt_input_keys(decoded['inputs'][0], ['witness_utxo', 'non_witness_utxo', 'bip32_derivs']) + test_psbt_input_keys(decoded['inputs'][1], ['non_witness_utxo', 'bip32_derivs']) + test_psbt_input_keys(decoded['inputs'][2], ['non_witness_utxo','witness_utxo', 'bip32_derivs', 'redeem_script']) # Two PSBTs with a common input should not be joinable psbt1 = self.nodes[1].createpsbt([{"txid":txid1, "vout":vout1}], {self.nodes[0].getnewaddress():Decimal('10.999')}) @@ -869,6 +941,9 @@ class PSBTTest(BitcoinTestFramework): assert_equal(self.nodes[0].finalizepsbt(psbt.to_base64()), {'hex': '0200000001dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd0000000000000000000100000000000000000000000000', 'complete': True}) + self.log.info("Test we don't crash when making a 0-value funded transaction at 0 fee without forcing an input selection") + assert_raises_rpc_error(-4, "Transaction requires one destination of non-0 value, a non-0 feerate, or a pre-selected input", self.nodes[0].walletcreatefundedpsbt, [], [{"data": "deadbeef"}], 0, {"fee_rate": "0"}) + if __name__ == '__main__': PSBTTest().main() diff --git a/test/functional/rpc_rawtransaction.py b/test/functional/rpc_rawtransaction.py index 1c91ab6f5f..2395935620 100755 --- a/test/functional/rpc_rawtransaction.py +++ b/test/functional/rpc_rawtransaction.py @@ -16,12 +16,19 @@ from collections import OrderedDict from decimal import Decimal from itertools import product -from test_framework.blocktools import COINBASE_MATURITY from test_framework.messages import ( MAX_BIP125_RBF_SEQUENCE, + COIN, CTransaction, + CTxOut, tx_from_hex, ) +from test_framework.script import ( + CScript, + OP_FALSE, + OP_INVALIDOPCODE, + OP_RETURN, +) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, @@ -59,7 +66,6 @@ class RawTransactionsTest(BitcoinTestFramework): self.add_wallet_options(parser, descriptors=False) def set_test_params(self): - self.setup_clean_chain = True self.num_nodes = 3 self.extra_args = [ ["-txindex"], @@ -77,9 +83,6 @@ class RawTransactionsTest(BitcoinTestFramework): def run_test(self): self.wallet = MiniWallet(self.nodes[0]) - self.log.info("Prepare some coins for multiple *rawtransaction commands") - self.generate(self.wallet, 10) - self.generate(self.nodes[0], COINBASE_MATURITY + 1) self.getrawtransaction_tests() self.getrawtransaction_verbosity_tests() @@ -336,6 +339,57 @@ class RawTransactionsTest(BitcoinTestFramework): rawtx = self.nodes[2].createrawtransaction(inputs, outputs) assert_raises_rpc_error(-25, "bad-txns-inputs-missingorspent", self.nodes[2].sendrawtransaction, rawtx) + self.log.info("Test sendrawtransaction exceeding, falling short of, and equaling maxburnamount") + max_burn_exceeded = "Unspendable output exceeds maximum configured by user (maxburnamount)" + + + # Test that spendable transaction with default maxburnamount (0) gets sent + tx = self.wallet.create_self_transfer()['tx'] + tx_hex = tx.serialize().hex() + self.nodes[2].sendrawtransaction(hexstring=tx_hex) + + # Test that datacarrier transaction with default maxburnamount (0) does not get sent + tx = self.wallet.create_self_transfer()['tx'] + tx_val = 0.001 + tx.vout = [CTxOut(int(Decimal(tx_val) * COIN), CScript([OP_RETURN] + [OP_FALSE] * 30))] + tx_hex = tx.serialize().hex() + assert_raises_rpc_error(-25, max_burn_exceeded, self.nodes[2].sendrawtransaction, tx_hex) + + # Test that oversized script gets rejected by sendrawtransaction + tx = self.wallet.create_self_transfer()['tx'] + tx_val = 0.001 + tx.vout = [CTxOut(int(Decimal(tx_val) * COIN), CScript([OP_FALSE] * 10001))] + tx_hex = tx.serialize().hex() + assert_raises_rpc_error(-25, max_burn_exceeded, self.nodes[2].sendrawtransaction, tx_hex) + + # Test that script containing invalid opcode gets rejected by sendrawtransaction + tx = self.wallet.create_self_transfer()['tx'] + tx_val = 0.01 + tx.vout = [CTxOut(int(Decimal(tx_val) * COIN), CScript([OP_INVALIDOPCODE]))] + tx_hex = tx.serialize().hex() + assert_raises_rpc_error(-25, max_burn_exceeded, self.nodes[2].sendrawtransaction, tx_hex) + + # Test a transaction where our burn exceeds maxburnamount + tx = self.wallet.create_self_transfer()['tx'] + tx_val = 0.001 + tx.vout = [CTxOut(int(Decimal(tx_val) * COIN), CScript([OP_RETURN] + [OP_FALSE] * 30))] + tx_hex = tx.serialize().hex() + assert_raises_rpc_error(-25, max_burn_exceeded, self.nodes[2].sendrawtransaction, tx_hex, 0, 0.0009) + + # Test a transaction where our burn falls short of maxburnamount + tx = self.wallet.create_self_transfer()['tx'] + tx_val = 0.001 + tx.vout = [CTxOut(int(Decimal(tx_val) * COIN), CScript([OP_RETURN] + [OP_FALSE] * 30))] + tx_hex = tx.serialize().hex() + self.nodes[2].sendrawtransaction(hexstring=tx_hex, maxfeerate='0', maxburnamount='0.0011') + + # Test a transaction where our burn equals maxburnamount + tx = self.wallet.create_self_transfer()['tx'] + tx_val = 0.001 + tx.vout = [CTxOut(int(Decimal(tx_val) * COIN), CScript([OP_RETURN] + [OP_FALSE] * 30))] + tx_hex = tx.serialize().hex() + self.nodes[2].sendrawtransaction(hexstring=tx_hex, maxfeerate='0', maxburnamount='0.001') + def sendrawtransaction_testmempoolaccept_tests(self): self.log.info("Test sendrawtransaction/testmempoolaccept with maxfeerate") fee_exceeds_max = "Fee exceeds maximum configured by user (e.g. -maxtxfee, maxfeerate)" diff --git a/test/functional/rpc_scanblocks.py b/test/functional/rpc_scanblocks.py index 9a00518150..9354709dfb 100755 --- a/test/functional/rpc_scanblocks.py +++ b/test/functional/rpc_scanblocks.py @@ -3,6 +3,7 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the scanblocks RPC call.""" +from test_framework.address import address_to_scriptpubkey from test_framework.blockfilter import ( bip158_basic_element_hash, bip158_relevant_scriptpubkeys, @@ -27,7 +28,6 @@ class ScanblocksTest(BitcoinTestFramework): def run_test(self): node = self.nodes[0] wallet = MiniWallet(node) - wallet.rescan_utxos() # send 1.0, mempool only _, spk_1, addr_1 = getnewdestination() @@ -37,7 +37,7 @@ class ScanblocksTest(BitcoinTestFramework): # send 1.0, mempool only # childkey 5 of `parent_key` wallet.send_to(from_node=node, - scriptPubKey=bytes.fromhex(node.validateaddress("mkS4HXoTYWRTescLGaUTGbtTTYX5EjJyEE")['scriptPubKey']), + scriptPubKey=address_to_scriptpubkey("mkS4HXoTYWRTescLGaUTGbtTTYX5EjJyEE"), amount=1 * COIN) # mine a block and assure that the mined blockhash is in the filterresult @@ -62,6 +62,12 @@ class ScanblocksTest(BitcoinTestFramework): # make sure the blockhash is present when using the first mined block as start_height assert blockhash in node.scanblocks( "start", [f"addr({addr_1})"], height)['relevant_blocks'] + for v in [False, True]: + assert blockhash in node.scanblocks( + action="start", + scanobjects=[f"addr({addr_1})"], + start_height=height, + options={"filter_false_positives": v})['relevant_blocks'] # also test the stop height assert blockhash in node.scanblocks( @@ -94,8 +100,11 @@ class ScanblocksTest(BitcoinTestFramework): assert genesis_blockhash in node.scanblocks( "start", [{"desc": f"raw({false_positive_spk.hex()})"}], 0, 0)['relevant_blocks'] - # TODO: after an "accurate" mode for scanblocks is implemented (e.g. PR #26325) - # check here that it filters out the false-positive + # check that the filter_false_positives option works + assert genesis_blockhash in node.scanblocks( + "start", [{"desc": f"raw({genesis_coinbase_spk.hex()})"}], 0, 0, "basic", {"filter_false_positives": True})['relevant_blocks'] + assert genesis_blockhash not in node.scanblocks( + "start", [{"desc": f"raw({false_positive_spk.hex()})"}], 0, 0, "basic", {"filter_false_positives": True})['relevant_blocks'] # test node with disabled blockfilterindex assert_raises_rpc_error(-1, "Index is not enabled for filtertype basic", diff --git a/test/functional/rpc_scantxoutset.py b/test/functional/rpc_scantxoutset.py index af3e7a6d19..dca965aacb 100755 --- a/test/functional/rpc_scantxoutset.py +++ b/test/functional/rpc_scantxoutset.py @@ -3,12 +3,12 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the scantxoutset rpc call.""" +from test_framework.address import address_to_scriptpubkey from test_framework.messages import COIN from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal, assert_raises_rpc_error from test_framework.wallet import ( MiniWallet, - address_to_scriptpubkey, getnewdestination, ) @@ -31,7 +31,6 @@ class ScantxoutsetTest(BitcoinTestFramework): def run_test(self): self.wallet = MiniWallet(self.nodes[0]) - self.wallet.rescan_utxos() self.log.info("Test if we find coinbase outputs.") assert_equal(sum(u["coinbase"] for u in self.nodes[0].scantxoutset("start", [self.wallet.get_descriptor()])["unspents"]), 49) diff --git a/test/functional/rpc_signrawtransactionwithkey.py b/test/functional/rpc_signrawtransactionwithkey.py index 0da5a99fdb..580f63063d 100755 --- a/test/functional/rpc_signrawtransactionwithkey.py +++ b/test/functional/rpc_signrawtransactionwithkey.py @@ -8,6 +8,7 @@ from test_framework.blocktools import ( COINBASE_MATURITY, ) from test_framework.address import ( + address_to_scriptpubkey, script_to_p2sh, ) from test_framework.key import ECKey @@ -118,7 +119,7 @@ class SignRawTransactionWithKeyTest(BitcoinTestFramework): }.get(tx_type, "Invalid tx_type") redeem_script = script_to_p2wsh_script(witness_script).hex() addr = script_to_p2sh(redeem_script) - script_pub_key = self.nodes[1].validateaddress(addr)['scriptPubKey'] + script_pub_key = address_to_scriptpubkey(addr).hex() # Fund that address txid = self.send_to_address(addr, 10) vout = find_vout_for_address(self.nodes[0], txid, addr) diff --git a/test/functional/rpc_txoutproof.py b/test/functional/rpc_txoutproof.py index d04d05962f..60b7ce8d20 100755 --- a/test/functional/rpc_txoutproof.py +++ b/test/functional/rpc_txoutproof.py @@ -4,7 +4,6 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test gettxoutproof and verifytxoutproof RPCs.""" -from test_framework.blocktools import COINBASE_MATURITY from test_framework.messages import ( CMerkleBlock, from_hex, @@ -20,7 +19,6 @@ from test_framework.wallet import MiniWallet class MerkleBlockTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 - self.setup_clean_chain = True self.extra_args = [ [], ["-txindex"], @@ -28,12 +26,9 @@ class MerkleBlockTest(BitcoinTestFramework): def run_test(self): miniwallet = MiniWallet(self.nodes[0]) - # Add enough mature utxos to the wallet, so that all txs spend confirmed coins - self.generate(miniwallet, 5) - self.generate(self.nodes[0], COINBASE_MATURITY) chain_height = self.nodes[1].getblockcount() - assert_equal(chain_height, 105) + assert_equal(chain_height, 200) txid1 = miniwallet.send_self_transfer(from_node=self.nodes[0])['txid'] txid2 = miniwallet.send_self_transfer(from_node=self.nodes[0])['txid'] diff --git a/test/functional/rpc_users.py b/test/functional/rpc_users.py index 560f226469..8cc3ec401e 100755 --- a/test/functional/rpc_users.py +++ b/test/functional/rpc_users.py @@ -53,13 +53,13 @@ class HTTPBasicsTest(BitcoinTestFramework): # Generate RPCAUTH with specified password self.rt2password = "8/F3uMDw4KSEbw96U3CA1C4X05dkHDN2BPFjTgZW4KI=" - p = subprocess.Popen([sys.executable, gen_rpcauth, 'rt2', self.rt2password], stdout=subprocess.PIPE, universal_newlines=True) + p = subprocess.Popen([sys.executable, gen_rpcauth, 'rt2', self.rt2password], stdout=subprocess.PIPE, text=True) lines = p.stdout.read().splitlines() rpcauth2 = lines[1] # Generate RPCAUTH without specifying password self.user = ''.join(SystemRandom().choice(string.ascii_letters + string.digits) for _ in range(10)) - p = subprocess.Popen([sys.executable, gen_rpcauth, self.user], stdout=subprocess.PIPE, universal_newlines=True) + p = subprocess.Popen([sys.executable, gen_rpcauth, self.user], stdout=subprocess.PIPE, text=True) lines = p.stdout.read().splitlines() rpcauth3 = lines[1] self.password = lines[3] diff --git a/test/functional/test_framework/address.py b/test/functional/test_framework/address.py index 959a2a65bd..5b2e3289a9 100644 --- a/test/functional/test_framework/address.py +++ b/test/functional/test_framework/address.py @@ -20,8 +20,17 @@ from .script import ( sha256, taproot_construct, ) -from .segwit_addr import encode_segwit_address from .util import assert_equal +from test_framework.script_util import ( + keyhash_to_p2pkh_script, + program_to_witness_script, + scripthash_to_p2sh_script, +) +from test_framework.segwit_addr import ( + decode_segwit_address, + encode_segwit_address, +) + ADDRESS_BCRT1_UNSPENDABLE = 'bcrt1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq3xueyj' ADDRESS_BCRT1_UNSPENDABLE_DESCRIPTOR = 'addr(bcrt1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq3xueyj)#juyq9d97' @@ -159,6 +168,31 @@ def check_script(script): assert False +def bech32_to_bytes(address): + hrp = address.split('1')[0] + if hrp not in ['bc', 'tb', 'bcrt']: + return (None, None) + version, payload = decode_segwit_address(hrp, address) + if version is None: + return (None, None) + return version, bytearray(payload) + + +def address_to_scriptpubkey(address): + """Converts a given address to the corresponding output script (scriptPubKey).""" + version, payload = bech32_to_bytes(address) + if version is not None: + return program_to_witness_script(version, payload) # testnet segwit scriptpubkey + payload, version = base58_to_byte(address) + if version == 111: # testnet pubkey hash + return keyhash_to_p2pkh_script(payload) + elif version == 196: # testnet script hash + return scripthash_to_p2sh_script(payload) + # TODO: also support other address formats + else: + assert False + + class TestFrameworkScript(unittest.TestCase): def test_base58encodedecode(self): def check_base58(data, version): @@ -176,3 +210,18 @@ class TestFrameworkScript(unittest.TestCase): check_base58(bytes.fromhex('0041c1eaf111802559bad61b60d62b1f897c63928a'), 0) check_base58(bytes.fromhex('000041c1eaf111802559bad61b60d62b1f897c63928a'), 0) check_base58(bytes.fromhex('00000041c1eaf111802559bad61b60d62b1f897c63928a'), 0) + + + def test_bech32_decode(self): + def check_bech32_decode(payload, version): + hrp = "tb" + self.assertEqual(bech32_to_bytes(encode_segwit_address(hrp, version, payload)), (version, payload)) + + check_bech32_decode(bytes.fromhex('36e3e2a33f328de12e4b43c515a75fba2632ecc3'), 0) + check_bech32_decode(bytes.fromhex('823e9790fc1d1782321140d4f4aa61aabd5e045b'), 0) + check_bech32_decode(bytes.fromhex('79be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798'), 1) + check_bech32_decode(bytes.fromhex('39cf8ebd95134f431c39db0220770bd127f5dd3cc103c988b7dcd577ae34e354'), 1) + check_bech32_decode(bytes.fromhex('708244006d27c757f6f1fc6f853b6ec26268b727866f7ce632886e34eb5839a3'), 1) + check_bech32_decode(bytes.fromhex('616211ab00dffe0adcb6ce258d6d3fd8cbd901e2'), 0) + check_bech32_decode(bytes.fromhex('b6a7c98b482d7fb21c9fa8e65692a0890410ff22'), 0) + check_bech32_decode(bytes.fromhex('f0c2109cb1008cfa7b5a09cc56f7267cd8e50929'), 0) diff --git a/test/functional/test_framework/authproxy.py b/test/functional/test_framework/authproxy.py index dd20b28550..f7765a9dfa 100644 --- a/test/functional/test_framework/authproxy.py +++ b/test/functional/test_framework/authproxy.py @@ -39,7 +39,6 @@ from http import HTTPStatus import http.client import json import logging -import os import socket import time import urllib.parse @@ -78,7 +77,10 @@ class AuthServiceProxy(): passwd = None if self.__url.password is None else self.__url.password.encode('utf8') authpair = user + b':' + passwd self.__auth_header = b'Basic ' + base64.b64encode(authpair) - self.timeout = timeout + # clamp the socket timeout, since larger values can cause an + # "Invalid argument" exception in Python's HTTP(S) client + # library on some operating systems (e.g. OpenBSD, FreeBSD) + self.timeout = min(timeout, 2147483) self._set_conn(connection) def __getattr__(self, name): @@ -91,36 +93,14 @@ class AuthServiceProxy(): def _request(self, method, path, postdata): ''' - Do a HTTP request, with retry if we get disconnected (e.g. due to a timeout). - This is a workaround for https://bugs.python.org/issue3566 which is fixed in Python 3.5. + Do a HTTP request. ''' headers = {'Host': self.__url.hostname, 'User-Agent': USER_AGENT, 'Authorization': self.__auth_header, 'Content-type': 'application/json'} - if os.name == 'nt': - # Windows somehow does not like to re-use connections - # TODO: Find out why the connection would disconnect occasionally and make it reusable on Windows - # Avoid "ConnectionAbortedError: [WinError 10053] An established connection was aborted by the software in your host machine" - self._set_conn() - try: - self.__conn.request(method, path, postdata, headers) - return self._get_response() - except (BrokenPipeError, ConnectionResetError): - # Python 3.5+ raises BrokenPipeError when the connection was reset - # ConnectionResetError happens on FreeBSD - self.__conn.close() - self.__conn.request(method, path, postdata, headers) - return self._get_response() - except OSError as e: - # Workaround for a bug on macOS. See https://bugs.python.org/issue33450 - retry = '[Errno 41] Protocol wrong type for socket' in str(e) - if retry: - self.__conn.close() - self.__conn.request(method, path, postdata, headers) - return self._get_response() - else: - raise + self.__conn.request(method, path, postdata, headers) + return self._get_response() def get_request(self, *args, **argsn): AuthServiceProxy.__id_count += 1 diff --git a/test/functional/test_framework/blocktools.py b/test/functional/test_framework/blocktools.py index b08cc6a3f9..cfd923bab3 100644 --- a/test/functional/test_framework/blocktools.py +++ b/test/functional/test_framework/blocktools.py @@ -9,6 +9,7 @@ import time import unittest from .address import ( + address_to_scriptpubkey, key_to_p2sh_p2wpkh, key_to_p2wpkh, script_to_p2sh_p2wsh, @@ -205,7 +206,7 @@ def create_witness_tx(node, use_p2wsh, utxo, pubkey, encode_p2sh, amount): else: addr = key_to_p2sh_p2wpkh(pubkey) if encode_p2sh else key_to_p2wpkh(pubkey) if not encode_p2sh: - assert_equal(node.getaddressinfo(addr)['scriptPubKey'], witness_script(use_p2wsh, pubkey)) + assert_equal(address_to_scriptpubkey(addr).hex(), witness_script(use_p2wsh, pubkey)) return node.createrawtransaction([utxo], {addr: amount}) def send_to_witness(use_p2wsh, node, utxo, pubkey, encode_p2sh, amount, sign=True, insert_redeem_script=""): diff --git a/test/functional/test_framework/messages.py b/test/functional/test_framework/messages.py index 8c6f68cacb..a6764365c5 100755 --- a/test/functional/test_framework/messages.py +++ b/test/functional/test_framework/messages.py @@ -93,6 +93,7 @@ def ser_compact_size(l): r = struct.pack("<BQ", 255, l) return r + def deser_compact_size(f): nit = struct.unpack("<B", f.read(1))[0] if nit == 253: @@ -103,35 +104,26 @@ def deser_compact_size(f): nit = struct.unpack("<Q", f.read(8))[0] return nit + def deser_string(f): nit = deser_compact_size(f) return f.read(nit) + def ser_string(s): return ser_compact_size(len(s)) + s + def deser_uint256(f): - r = 0 - for i in range(8): - t = struct.unpack("<I", f.read(4))[0] - r += t << (i * 32) - return r + return int.from_bytes(f.read(32), 'little') def ser_uint256(u): - rs = b"" - for _ in range(8): - rs += struct.pack("<I", u & 0xFFFFFFFF) - u >>= 32 - return rs + return u.to_bytes(32, 'little') def uint256_from_str(s): - r = 0 - t = struct.unpack("<IIIIIIII", s[:32]) - for i in range(8): - r += t[i] << (i * 32) - return r + return int.from_bytes(s[:32], 'little') def uint256_from_compact(c): diff --git a/test/functional/test_framework/p2p.py b/test/functional/test_framework/p2p.py index 59157f4755..2433e52671 100755 --- a/test/functional/test_framework/p2p.py +++ b/test/functional/test_framework/p2p.py @@ -385,7 +385,7 @@ class P2PInterface(P2PConnection): self.message_count[msgtype] += 1 self.last_message[msgtype] = message getattr(self, 'on_' + msgtype)(message) - except: + except Exception: print("ERROR delivering %s (%s)" % (repr(message), sys.exc_info()[0])) raise @@ -464,7 +464,7 @@ class P2PInterface(P2PConnection): def wait_for_connect(self, timeout=60): test_function = lambda: self.is_connected - wait_until_helper(test_function, timeout=timeout, lock=p2p_lock) + self.wait_until(test_function, timeout=timeout, check_connected=False) def wait_for_disconnect(self, timeout=60): test_function = lambda: not self.is_connected diff --git a/test/functional/test_framework/psbt.py b/test/functional/test_framework/psbt.py index 3a5b4ec74d..1eff4a250e 100644 --- a/test/functional/test_framework/psbt.py +++ b/test/functional/test_framework/psbt.py @@ -105,8 +105,8 @@ class PSBT: def deserialize(self, f): assert f.read(5) == b"psbt\xff" self.g = from_binary(PSBTMap, f) - assert 0 in self.g.map - self.tx = from_binary(CTransaction, self.g.map[0]) + assert PSBT_GLOBAL_UNSIGNED_TX in self.g.map + self.tx = from_binary(CTransaction, self.g.map[PSBT_GLOBAL_UNSIGNED_TX]) self.i = [from_binary(PSBTMap, f) for _ in self.tx.vin] self.o = [from_binary(PSBTMap, f) for _ in self.tx.vout] return self @@ -115,8 +115,8 @@ class PSBT: assert isinstance(self.g, PSBTMap) assert isinstance(self.i, list) and all(isinstance(x, PSBTMap) for x in self.i) assert isinstance(self.o, list) and all(isinstance(x, PSBTMap) for x in self.o) - assert 0 in self.g.map - tx = from_binary(CTransaction, self.g.map[0]) + assert PSBT_GLOBAL_UNSIGNED_TX in self.g.map + tx = from_binary(CTransaction, self.g.map[PSBT_GLOBAL_UNSIGNED_TX]) assert len(tx.vin) == len(self.i) assert len(tx.vout) == len(self.o) @@ -130,7 +130,7 @@ class PSBT: for m in self.i + self.o: m.map.clear() - self.g = PSBTMap(map={0: self.g.map[0]}) + self.g = PSBTMap(map={PSBT_GLOBAL_UNSIGNED_TX: self.g.map[PSBT_GLOBAL_UNSIGNED_TX]}) def to_base64(self): return base64.b64encode(self.serialize()).decode("utf8") diff --git a/test/functional/test_framework/script.py b/test/functional/test_framework/script.py index f345bf02db..443cae86a1 100644 --- a/test/functional/test_framework/script.py +++ b/test/functional/test_framework/script.py @@ -597,6 +597,13 @@ class CScript(bytes): lastOpcode = opcode return n + def IsWitnessProgram(self): + """A witness program is any valid CScript that consists of a 1-byte + push opcode followed by a data push between 2 and 40 bytes.""" + return ((4 <= len(self) <= 42) and + (self[0] == OP_0 or (OP_1 <= self[0] <= OP_16)) and + (self[1] + 2 == len(self))) + SIGHASH_DEFAULT = 0 # Taproot-only default, semantics same as SIGHASH_ALL SIGHASH_ALL = 1 diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index f7dd4551c8..66a23b443c 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -120,8 +120,6 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): self.disable_autoconnect = True self.set_test_params() assert self.wallet_names is None or len(self.wallet_names) <= self.num_nodes - if self.options.timeout_factor == 0 : - self.options.timeout_factor = 99999 self.rpc_timeout = int(self.rpc_timeout * self.options.timeout_factor) # optionally, increase timeout by a factor def main(self): @@ -193,7 +191,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): help="run nodes under the valgrind memory error detector: expect at least a ~10x slowdown. valgrind 3.14 or later required. Forces --nosandbox.") parser.add_argument("--randomseed", type=int, help="set a random seed for deterministically reproducing a previous test run") - parser.add_argument('--timeout-factor', dest="timeout_factor", type=float, default=1.0, help='adjust test timeouts by a factor. Setting it to 0 disables all timeouts') + parser.add_argument("--timeout-factor", dest="timeout_factor", type=float, help="adjust test timeouts by a factor. Setting it to 0 disables all timeouts") self.add_options(parser) # Running TestShell in a Jupyter notebook causes an additional -f argument @@ -201,6 +199,9 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): # source: https://stackoverflow.com/questions/48796169/how-to-fix-ipykernel-launcher-py-error-unrecognized-arguments-in-jupyter/56349168#56349168 parser.add_argument("-f", "--fff", help="a dummy argument to fool ipython", default="1") self.options = parser.parse_args() + if self.options.timeout_factor == 0: + self.options.timeout_factor = 99999 + self.options.timeout_factor = self.options.timeout_factor or (4 if self.options.valgrind else 1) self.options.previous_releases_path = previous_releases_path config = configparser.ConfigParser() @@ -279,10 +280,10 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): if seed is None: seed = random.randrange(sys.maxsize) else: - self.log.debug("User supplied random seed {}".format(seed)) + self.log.info("User supplied random seed {}".format(seed)) random.seed(seed) - self.log.debug("PRNG seed is: {}".format(seed)) + self.log.info("PRNG seed is: {}".format(seed)) self.log.debug('Setting up network thread') self.network_thread = NetworkThread() @@ -533,11 +534,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): self.nodes.append(test_node_i) if not test_node_i.version_is_at_least(170000): # adjust conf for pre 17 - conf_file = test_node_i.bitcoinconf - with open(conf_file, 'r', encoding='utf8') as conf: - conf_data = conf.read() - with open(conf_file, 'w', encoding='utf8') as conf: - conf.write(conf_data.replace('[regtest]', '')) + test_node_i.replace_in_config([('[regtest]', '')]) def start_node(self, i, *args, **kwargs): """Start a bitcoind""" @@ -561,7 +558,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): node.start(extra_args[i], *args, **kwargs) for node in self.nodes: node.wait_for_rpc_connection() - except: + except Exception: # If one node failed to start, stop the others self.stop_nodes() raise @@ -608,6 +605,10 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): self.wait_until(lambda: sum(peer['version'] != 0 for peer in to_connection.getpeerinfo()) == to_num_peers) self.wait_until(lambda: sum(peer['bytesrecv_per_msg'].pop('verack', 0) == 24 for peer in from_connection.getpeerinfo()) == from_num_peers) self.wait_until(lambda: sum(peer['bytesrecv_per_msg'].pop('verack', 0) == 24 for peer in to_connection.getpeerinfo()) == to_num_peers) + # The message bytes are counted before processing the message, so make + # sure it was fully processed by waiting for a ping. + self.wait_until(lambda: sum(peer["bytesrecv_per_msg"].pop("pong", 0) >= 32 for peer in from_connection.getpeerinfo()) == from_num_peers) + self.wait_until(lambda: sum(peer["bytesrecv_per_msg"].pop("pong", 0) >= 32 for peer in to_connection.getpeerinfo()) == to_num_peers) def disconnect_nodes(self, a, b): def disconnect_nodes_helper(node_a, node_b): @@ -850,6 +851,13 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): except ImportError: raise SkipTest("python3-zmq module not available.") + def skip_if_no_py_sqlite3(self): + """Attempt to import the sqlite3 package and skip the test if the import fails.""" + try: + import sqlite3 # noqa + except ImportError: + raise SkipTest("sqlite3 module not available.") + def skip_if_no_python_bcc(self): """Attempt to import the bcc package and skip the tests if the import fails.""" try: @@ -873,6 +881,11 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): if platform.system() != "Linux": raise SkipTest("not on a Linux system") + def skip_if_platform_not_posix(self): + """Skip the running test if we are not on a POSIX platform""" + if os.name != 'posix': + raise SkipTest("not on a POSIX system") + def skip_if_no_bitcoind_zmq(self): """Skip the running test if bitcoind has not been compiled with zmq support.""" if not self.is_zmq_compiled(): diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index 8585972cb3..56abe5f26a 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -387,6 +387,21 @@ class TestNode(): def wait_until_stopped(self, timeout=BITCOIND_PROC_WAIT_TIMEOUT): wait_until_helper(self.is_node_stopped, timeout=timeout, timeout_factor=self.timeout_factor) + def replace_in_config(self, replacements): + """ + Perform replacements in the configuration file. + The substitutions are passed as a list of search-replace-tuples, e.g. + [("old", "new"), ("foo", "bar"), ...] + """ + with open(self.bitcoinconf, 'r', encoding='utf8') as conf: + conf_data = conf.read() + for replacement in replacements: + assert_equal(len(replacement), 2) + old, new = replacement[0], replacement[1] + conf_data = conf_data.replace(old, new) + with open(self.bitcoinconf, 'w', encoding='utf8') as conf: + conf.write(conf_data) + @property def chain_path(self) -> Path: return Path(self.datadir) / self.chain @@ -454,7 +469,7 @@ class TestNode(): return if time.time() >= time_end: - print_log = " - " + "\n - ".join(log.splitlines()) + print_log = " - " + "\n - ".join(log.decode("utf8", errors="replace").splitlines()) break # No sleep here because we want to detect the message fragment as fast as @@ -730,7 +745,7 @@ class TestNodeCLI(): p_args += [command] p_args += pos_args + named_args self.log.debug("Running bitcoin-cli {}".format(p_args[2:])) - process = subprocess.Popen(p_args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) + process = subprocess.Popen(p_args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) cli_stdout, cli_stderr = process.communicate(input=self.input) returncode = process.poll() if returncode: @@ -814,7 +829,7 @@ class RPCOverloadWrapper(): int(address ,16) is_hex = True desc = descsum_create('raw(' + address + ')') - except: + except Exception: desc = descsum_create('addr(' + address + ')') reqs = [{ 'desc': desc, diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index 1d5108c31a..a1b90860f6 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -390,6 +390,8 @@ def write_config(config_path, *, n, chain, extra_config="", disable_autoconnect= f.write("[{}]\n".format(chain_name_conf_section)) f.write("port=" + str(p2p_port(n)) + "\n") f.write("rpcport=" + str(rpc_port(n)) + "\n") + # Disable server-side timeouts to avoid intermittent issues + f.write("rpcservertimeout=99000\n") f.write("rpcdoccheck=1\n") f.write("fallbackfee=0.0002\n") f.write("server=1\n") @@ -488,28 +490,6 @@ def find_output(node, txid, amount, *, blockhash=None): raise RuntimeError("find_output txid %s : %s not found" % (txid, str(amount))) -def chain_transaction(node, parent_txids, vouts, value, fee, num_outputs): - """Build and send a transaction that spends the given inputs (specified - by lists of parent_txid:vout each), with the desired total value and fee, - equally divided up to the desired number of outputs. - - Returns a tuple with the txid and the amount sent per output. - """ - send_value = satoshi_round((value - fee)/num_outputs) - inputs = [] - for (txid, vout) in zip(parent_txids, vouts): - inputs.append({'txid' : txid, 'vout' : vout}) - outputs = {} - for _ in range(num_outputs): - outputs[node.getnewaddress()] = send_value - rawtx = node.createrawtransaction(inputs, outputs, 0, True) - signedtx = node.signrawtransactionwithwallet(rawtx) - txid = node.sendrawtransaction(signedtx['hex']) - fulltx = node.getrawtransaction(txid, 1) - assert len(fulltx['vout']) == num_outputs # make sure we didn't generate a change output - return (txid, send_value) - - # Create large OP_RETURN txouts that can be appended to a transaction # to make it large (helper for constructing large transactions). The # total serialized size of the txouts is about 66k vbytes. diff --git a/test/functional/test_framework/wallet.py b/test/functional/test_framework/wallet.py index a72b5e5891..fcd396c700 100644 --- a/test/functional/test_framework/wallet.py +++ b/test/functional/test_framework/wallet.py @@ -13,7 +13,7 @@ from typing import ( Optional, ) from test_framework.address import ( - base58_to_byte, + address_to_scriptpubkey, create_deterministic_address_bcrt1_p2tr_op_true, key_to_p2pkh, key_to_p2sh_p2wpkh, @@ -48,13 +48,12 @@ from test_framework.script_util import ( key_to_p2pkh_script, key_to_p2sh_p2wpkh_script, key_to_p2wpkh_script, - keyhash_to_p2pkh_script, - scripthash_to_p2sh_script, ) from test_framework.util import ( assert_equal, assert_greater_than_or_equal, ) +from test_framework.blocktools import COINBASE_MATURITY DEFAULT_FEE = Decimal("0.0001") @@ -98,10 +97,17 @@ class MiniWallet: self._scriptPubKey = key_to_p2pk_script(pub_key.get_bytes()) elif mode == MiniWalletMode.ADDRESS_OP_TRUE: self._address, self._internal_key = create_deterministic_address_bcrt1_p2tr_op_true() - self._scriptPubKey = bytes.fromhex(self._test_node.validateaddress(self._address)['scriptPubKey']) + self._scriptPubKey = address_to_scriptpubkey(self._address) - def _create_utxo(self, *, txid, vout, value, height): - return {"txid": txid, "vout": vout, "value": value, "height": height} + # When the pre-mined test framework chain is used, it contains coinbase + # outputs to the MiniWallet's default address in blocks 76-100 + # (see method BitcoinTestFramework._initialize_chain()) + # The MiniWallet needs to rescan_utxos() in order to account + # for those mature UTXOs, so that all txs spend confirmed coins + self.rescan_utxos() + + def _create_utxo(self, *, txid, vout, value, height, coinbase, confirmations): + return {"txid": txid, "vout": vout, "value": value, "height": height, "coinbase": coinbase, "confirmations": confirmations} def _bulk_tx(self, tx, target_weight): """Pad a transaction with extra outputs until it reaches a target weight (or higher). @@ -118,13 +124,25 @@ class MiniWallet: def get_balance(self): return sum(u['value'] for u in self._utxos) - def rescan_utxos(self): + def rescan_utxos(self, *, include_mempool=True): """Drop all utxos and rescan the utxo set""" self._utxos = [] res = self._test_node.scantxoutset(action="start", scanobjects=[self.get_descriptor()]) assert_equal(True, res['success']) for utxo in res['unspents']: - self._utxos.append(self._create_utxo(txid=utxo["txid"], vout=utxo["vout"], value=utxo["amount"], height=utxo["height"])) + self._utxos.append( + self._create_utxo(txid=utxo["txid"], + vout=utxo["vout"], + value=utxo["amount"], + height=utxo["height"], + coinbase=utxo["coinbase"], + confirmations=res["height"] - utxo["height"] + 1)) + if include_mempool: + mempool = self._test_node.getrawmempool(verbose=True) + # Sort tx by ancestor count. See BlockAssembler::SortForBlock in src/node/miner.cpp + sorted_mempool = sorted(mempool.items(), key=lambda item: (item[1]["ancestorcount"], int(item[0], 16))) + for txid, _ in sorted_mempool: + self.scan_tx(self._test_node.getrawtransaction(txid=txid, verbose=True)) def scan_tx(self, tx): """Scan the tx and adjust the internal list of owned utxos""" @@ -139,27 +157,35 @@ class MiniWallet: pass for out in tx['vout']: if out['scriptPubKey']['hex'] == self._scriptPubKey.hex(): - self._utxos.append(self._create_utxo(txid=tx["txid"], vout=out["n"], value=out["value"], height=0)) + self._utxos.append(self._create_utxo(txid=tx["txid"], vout=out["n"], value=out["value"], height=0, coinbase=False, confirmations=0)) def scan_txs(self, txs): for tx in txs: self.scan_tx(tx) def sign_tx(self, tx, fixed_length=True): - """Sign tx that has been created by MiniWallet in P2PK mode""" - assert_equal(self._mode, MiniWalletMode.RAW_P2PK) - (sighash, err) = LegacySignatureHash(CScript(self._scriptPubKey), tx, 0, SIGHASH_ALL) - assert err is None - # for exact fee calculation, create only signatures with fixed size by default (>49.89% probability): - # 65 bytes: high-R val (33 bytes) + low-S val (32 bytes) - # with the DER header/skeleton data of 6 bytes added, this leads to a target size of 71 bytes - der_sig = b'' - while not len(der_sig) == 71: - der_sig = self._priv_key.sign_ecdsa(sighash) - if not fixed_length: - break - tx.vin[0].scriptSig = CScript([der_sig + bytes(bytearray([SIGHASH_ALL]))]) - tx.rehash() + if self._mode == MiniWalletMode.RAW_P2PK: + (sighash, err) = LegacySignatureHash(CScript(self._scriptPubKey), tx, 0, SIGHASH_ALL) + assert err is None + # for exact fee calculation, create only signatures with fixed size by default (>49.89% probability): + # 65 bytes: high-R val (33 bytes) + low-S val (32 bytes) + # with the DER header/skeleton data of 6 bytes added, this leads to a target size of 71 bytes + der_sig = b'' + while not len(der_sig) == 71: + der_sig = self._priv_key.sign_ecdsa(sighash) + if not fixed_length: + break + tx.vin[0].scriptSig = CScript([der_sig + bytes(bytearray([SIGHASH_ALL]))]) + tx.rehash() + elif self._mode == MiniWalletMode.RAW_OP_TRUE: + for i in tx.vin: + i.scriptSig = CScript([OP_NOP] * 43) # pad to identical size + elif self._mode == MiniWalletMode.ADDRESS_OP_TRUE: + tx.wit.vtxinwit = [CTxInWitness()] * len(tx.vin) + for i in tx.wit.vtxinwit: + i.scriptWitness.stack = [CScript([OP_TRUE]), bytes([LEAF_VERSION_TAPSCRIPT]) + self._internal_key] + else: + assert False def generate(self, num_blocks, **kwargs): """Generate blocks with coinbase outputs to the internal address, and call rescan_utxos""" @@ -204,9 +230,13 @@ class MiniWallet: else: return self._utxos[index] - def get_utxos(self, *, mark_as_spent=True): + def get_utxos(self, *, include_immature_coinbase=False, mark_as_spent=True): """Returns the list of all utxos and optionally mark them as spent""" - utxos = deepcopy(self._utxos) + if not include_immature_coinbase: + utxo_filter = filter(lambda utxo: not utxo['coinbase'] or COINBASE_MATURITY <= utxo['confirmations'], self._utxos) + else: + utxo_filter = self._utxos + utxos = deepcopy(list(utxo_filter)) if mark_as_spent: self._utxos = [] return utxos @@ -266,24 +296,17 @@ class MiniWallet: inputs_value_total = sum([int(COIN * utxo['value']) for utxo in utxos_to_spend]) outputs_value_total = inputs_value_total - fee_per_output * num_outputs amount_per_output = amount_per_output or (outputs_value_total // num_outputs) + assert amount_per_output > 0 + outputs_value_total = amount_per_output * num_outputs + fee = Decimal(inputs_value_total - outputs_value_total) / COIN # create tx tx = CTransaction() - tx.vin = [CTxIn(COutPoint(int(utxo_to_spend['txid'], 16), utxo_to_spend['vout']), nSequence=seq) for utxo_to_spend,seq in zip(utxos_to_spend, sequence)] + tx.vin = [CTxIn(COutPoint(int(utxo_to_spend['txid'], 16), utxo_to_spend['vout']), nSequence=seq) for utxo_to_spend, seq in zip(utxos_to_spend, sequence)] tx.vout = [CTxOut(amount_per_output, bytearray(self._scriptPubKey)) for _ in range(num_outputs)] tx.nLockTime = locktime - if self._mode == MiniWalletMode.RAW_P2PK: - self.sign_tx(tx) - elif self._mode == MiniWalletMode.RAW_OP_TRUE: - for i in range(len(utxos_to_spend)): - tx.vin[i].scriptSig = CScript([OP_NOP] * 43) # pad to identical size - elif self._mode == MiniWalletMode.ADDRESS_OP_TRUE: - tx.wit.vtxinwit = [CTxInWitness()] * len(utxos_to_spend) - for i in range(len(utxos_to_spend)): - tx.wit.vtxinwit[i].scriptWitness.stack = [CScript([OP_TRUE]), bytes([LEAF_VERSION_TAPSCRIPT]) + self._internal_key] - else: - assert False + self.sign_tx(tx) if target_weight: self._bulk_tx(tx, target_weight) @@ -295,8 +318,12 @@ class MiniWallet: vout=i, value=Decimal(tx.vout[i].nValue) / COIN, height=0, + coinbase=False, + confirmations=0, ) for i in range(len(tx.vout))], + "fee": fee, "txid": txid, + "wtxid": tx.getwtxid(), "hex": tx.serialize().hex(), "tx": tx, } @@ -314,52 +341,45 @@ class MiniWallet: else: assert False send_value = utxo_to_spend["value"] - (fee or (fee_rate * vsize / 1000)) - assert send_value > 0 # create tx tx = self.create_self_transfer_multi(utxos_to_spend=[utxo_to_spend], locktime=locktime, sequence=sequence, amount_per_output=int(COIN * send_value), target_weight=target_weight) if not target_weight: assert_equal(tx["tx"].get_vsize(), vsize) + tx["new_utxo"] = tx.pop("new_utxos")[0] - return {"txid": tx["txid"], "wtxid": tx["tx"].getwtxid(), "hex": tx["hex"], "tx": tx["tx"], "new_utxo": tx["new_utxos"][0]} + return tx def sendrawtransaction(self, *, from_node, tx_hex, maxfeerate=0, **kwargs): txid = from_node.sendrawtransaction(hexstring=tx_hex, maxfeerate=maxfeerate, **kwargs) self.scan_tx(from_node.decoderawtransaction(tx_hex)) return txid - def create_self_transfer_chain(self, *, chain_length): + def create_self_transfer_chain(self, *, chain_length, utxo_to_spend=None): """ Create a "chain" of chain_length transactions. The nth transaction in the chain is a child of the n-1th transaction and parent of the n+1th transaction. - - Returns a dic {"chain_hex": chain_hex, "chain_txns" : chain_txns} - - "chain_hex" is a list representing the chain's transactions in hexadecimal. - "chain_txns" is a list representing the chain's transactions in the CTransaction object. """ - chaintip_utxo = self.get_utxo() - chain_hex = [] - chain_txns = [] + chaintip_utxo = utxo_to_spend or self.get_utxo() + chain = [] for _ in range(chain_length): tx = self.create_self_transfer(utxo_to_spend=chaintip_utxo) chaintip_utxo = tx["new_utxo"] - chain_hex.append(tx["hex"]) - chain_txns.append(tx["tx"]) + chain.append(tx) - return {"chain_hex": chain_hex, "chain_txns" : chain_txns} + return chain - def send_self_transfer_chain(self, *, from_node, chain_length, utxo_to_spend=None): + def send_self_transfer_chain(self, *, from_node, **kwargs): """Create and send a "chain" of chain_length transactions. The nth transaction in the chain is a child of the n-1th transaction and parent of the n+1th transaction. - Returns the chaintip (nth) utxo + Returns a list of objects for each tx (see create_self_transfer_multi). """ - chaintip_utxo = utxo_to_spend or self.get_utxo() - for _ in range(chain_length): - chaintip_utxo = self.send_self_transfer(utxo_to_spend=chaintip_utxo, from_node=from_node)["new_utxo"] - return chaintip_utxo + chain = self.create_self_transfer_chain(**kwargs) + for t in chain: + self.sendrawtransaction(from_node=from_node, tx_hex=t["hex"]) + return chain def getnewdestination(address_type='bech32m'): @@ -388,15 +408,3 @@ def getnewdestination(address_type='bech32m'): else: assert False return pubkey, scriptpubkey, address - - -def address_to_scriptpubkey(address): - """Converts a given address to the corresponding output script (scriptPubKey).""" - payload, version = base58_to_byte(address) - if version == 111: # testnet pubkey hash - return keyhash_to_p2pkh_script(payload) - elif version == 196: # testnet script hash - return scripthash_to_p2sh_script(payload) - # TODO: also support other address formats - else: - assert False diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 86a9e4cd9a..bcedc0c9af 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -195,6 +195,7 @@ BASE_SCRIPTS = [ 'interface_http.py', 'interface_rpc.py', 'interface_usdt_coinselection.py', + 'interface_usdt_mempool.py', 'interface_usdt_net.py', 'interface_usdt_utxocache.py', 'interface_usdt_validation.py', @@ -211,10 +212,13 @@ BASE_SCRIPTS = [ 'p2p_addrv2_relay.py', 'p2p_compactblocks_hb.py', 'p2p_disconnect_ban.py', + 'feature_posix_fs_permissions.py', 'rpc_decodescript.py', 'rpc_blockchain.py', 'rpc_deprecated.py', 'wallet_disable.py', + 'wallet_change_address.py --legacy-wallet', + 'wallet_change_address.py --descriptors', 'p2p_addr_relay.py', 'p2p_getaddr_caching.py', 'p2p_getdata.py', @@ -254,6 +258,7 @@ BASE_SCRIPTS = [ 'wallet_importprunedfunds.py --descriptors', 'p2p_leak_tx.py', 'p2p_eviction.py', + 'p2p_ibd_stalling.py', 'wallet_signmessagewithaddress.py', 'rpc_signmessagewithprivkey.py', 'rpc_generate.py', @@ -318,6 +323,8 @@ BASE_SCRIPTS = [ 'mempool_unbroadcast.py', 'mempool_compatibility.py', 'mempool_accept_wtxid.py', + 'mempool_dust.py', + 'mempool_sigoplimit.py', 'rpc_deriveaddresses.py', 'rpc_deriveaddresses.py --usecli', 'p2p_ping.py', @@ -337,6 +344,7 @@ BASE_SCRIPTS = [ 'p2p_permissions.py', 'feature_blocksdir.py', 'wallet_startup.py', + 'feature_remove_pruned_files_on_startup.py', 'p2p_i2p_ports.py', 'p2p_i2p_sessions.py', 'feature_config_args.py', @@ -584,7 +592,7 @@ def run_tests(*, test_list, src_dir, build_dir, tmpdir, jobs=1, enable_coverage= combined_logs_args = [sys.executable, os.path.join(tests_dir, 'combine_logs.py'), testdir] if BOLD[0]: combined_logs_args += ['--color'] - combined_logs, _ = subprocess.Popen(combined_logs_args, universal_newlines=True, stdout=subprocess.PIPE).communicate() + combined_logs, _ = subprocess.Popen(combined_logs_args, text=True, stdout=subprocess.PIPE).communicate() print("\n".join(deque(combined_logs.splitlines(), combined_logs_len))) if failfast: @@ -669,7 +677,7 @@ class TestHandler: self.jobs.append((test, time.time(), subprocess.Popen([sys.executable, self.tests_dir + test_argv[0]] + test_argv[1:] + self.flags + portseed_arg + tmpdir_arg, - universal_newlines=True, + text=True, stdout=log_stdout, stderr=log_stderr), testdir, diff --git a/test/functional/tool_wallet.py b/test/functional/tool_wallet.py index 4a321c2fc4..a888f93b03 100755 --- a/test/functional/tool_wallet.py +++ b/test/functional/tool_wallet.py @@ -37,7 +37,7 @@ class ToolWalletTest(BitcoinTestFramework): if not self.options.descriptors and 'create' in args: default_args.append('-legacy') - return subprocess.Popen([binary] + default_args + list(args), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) + return subprocess.Popen([binary] + default_args + list(args), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) def assert_raises_tool_error(self, error, *args): p = self.bitcoin_wallet_process(*args) diff --git a/test/functional/wallet_avoidreuse.py b/test/functional/wallet_avoidreuse.py index 5601d81227..c257bda452 100755 --- a/test/functional/wallet_avoidreuse.py +++ b/test/functional/wallet_avoidreuse.py @@ -4,6 +4,7 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the avoid_reuse and setwalletflag features.""" +from test_framework.address import address_to_scriptpubkey from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_approx, @@ -257,7 +258,7 @@ class AvoidReuseTest(BitcoinTestFramework): if not self.options.descriptors: # For the second send, we transmute it to a related single-key address # to make sure it's also detected as re-use - fund_spk = self.nodes[0].getaddressinfo(fundaddr)["scriptPubKey"] + fund_spk = address_to_scriptpubkey(fundaddr).hex() fund_decoded = self.nodes[0].decodescript(fund_spk) if second_addr_type == "p2sh-segwit": new_fundaddr = fund_decoded["segwit"]["p2sh-segwit"] diff --git a/test/functional/wallet_backwards_compatibility.py b/test/functional/wallet_backwards_compatibility.py index f55a3758ce..a6401a76c1 100755 --- a/test/functional/wallet_backwards_compatibility.py +++ b/test/functional/wallet_backwards_compatibility.py @@ -33,11 +33,12 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True - self.num_nodes = 10 + self.num_nodes = 11 # Add new version after each release: self.extra_args = [ ["-addresstype=bech32", "-whitelist=noban@127.0.0.1"], # Pre-release: use to mine blocks. noban for immediate tx relay ["-nowallet", "-walletrbf=1", "-addresstype=bech32", "-whitelist=noban@127.0.0.1"], # Pre-release: use to receive coins, swap wallets, etc + ["-nowallet", "-walletrbf=1", "-addresstype=bech32", "-whitelist=noban@127.0.0.1"], # v24.0.1 ["-nowallet", "-walletrbf=1", "-addresstype=bech32", "-whitelist=noban@127.0.0.1"], # v23.0 ["-nowallet", "-walletrbf=1", "-addresstype=bech32", "-whitelist=noban@127.0.0.1"], # v22.0 ["-nowallet", "-walletrbf=1", "-addresstype=bech32", "-whitelist=noban@127.0.0.1"], # v0.21.0 @@ -57,6 +58,7 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): self.add_nodes(self.num_nodes, extra_args=self.extra_args, versions=[ None, None, + 240001, 230000, 220000, 210000, @@ -263,7 +265,12 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): ) load_res = node_master.loadwallet("u1_v16") # Make sure this wallet opens without warnings. See https://github.com/bitcoin/bitcoin/pull/19054 - assert_equal(load_res['warning'], '') + if int(node_master.getnetworkinfo()["version"]) >= 249900: + # loadwallet#warnings (added in v25) -- only present if there is a warning + assert "warnings" not in load_res + else: + # loadwallet#warning (deprecated in v25) -- always present, but empty string if no warning + assert_equal(load_res["warning"], '') wallet = node_master.get_wallet_rpc("u1_v16") info = wallet.getaddressinfo(v16_addr) descriptor = f"wpkh([{info['hdmasterfingerprint']}{hdkeypath[1:]}]{v16_pubkey})" diff --git a/test/functional/wallet_basic.py b/test/functional/wallet_basic.py index 86cfd4a230..53ac01686a 100755 --- a/test/functional/wallet_basic.py +++ b/test/functional/wallet_basic.py @@ -632,7 +632,7 @@ class WalletTest(BitcoinTestFramework): assert_equal(total_txs, len(self.nodes[0].listtransactions("*", 99999))) # Test getaddressinfo on external address. Note that these addresses are taken from disablewallet.py - assert_raises_rpc_error(-5, "Invalid prefix for Base58-encoded address", self.nodes[0].getaddressinfo, "3J98t1WpEZ73CNmQviecrnyiWrnqRhWNLy") + assert_raises_rpc_error(-5, "Invalid or unsupported Base58-encoded address.", self.nodes[0].getaddressinfo, "3J98t1WpEZ73CNmQviecrnyiWrnqRhWNLy") address_info = self.nodes[0].getaddressinfo("mneYUmWYsuk7kySiURxCi3AGxrAqZxLgPZ") assert_equal(address_info['address'], "mneYUmWYsuk7kySiURxCi3AGxrAqZxLgPZ") assert_equal(address_info["scriptPubKey"], "76a9144e3854046c7bd1594ac904e4793b6a45b36dea0988ac") @@ -731,6 +731,45 @@ class WalletTest(BitcoinTestFramework): assert_equal(coin_b["parent_descs"][0], multi_b) self.nodes[0].unloadwallet("wo") + self.log.info("Test -spendzeroconfchange") + self.restart_node(0, ["-spendzeroconfchange=0"]) + + # create new wallet and fund it with a confirmed UTXO + self.nodes[0].createwallet(wallet_name="zeroconf", load_on_startup=True) + zeroconf_wallet = self.nodes[0].get_wallet_rpc("zeroconf") + default_wallet = self.nodes[0].get_wallet_rpc(self.default_wallet_name) + default_wallet.sendtoaddress(zeroconf_wallet.getnewaddress(), Decimal('1.0')) + self.generate(self.nodes[0], 1, sync_fun=self.no_op) + utxos = zeroconf_wallet.listunspent(minconf=0) + assert_equal(len(utxos), 1) + assert_equal(utxos[0]['confirmations'], 1) + + # spend confirmed UTXO to ourselves + zeroconf_wallet.sendall(recipients=[zeroconf_wallet.getnewaddress()]) + utxos = zeroconf_wallet.listunspent(minconf=0) + assert_equal(len(utxos), 1) + assert_equal(utxos[0]['confirmations'], 0) + # accounts for untrusted pending balance + bal = zeroconf_wallet.getbalances() + assert_equal(bal['mine']['trusted'], 0) + assert_equal(bal['mine']['untrusted_pending'], utxos[0]['amount']) + + # spending an unconfirmed UTXO sent to ourselves should fail + assert_raises_rpc_error(-6, "Insufficient funds", zeroconf_wallet.sendtoaddress, zeroconf_wallet.getnewaddress(), Decimal('0.5')) + + # check that it works again with -spendzeroconfchange set (=default) + self.restart_node(0, ["-spendzeroconfchange=1"]) + zeroconf_wallet = self.nodes[0].get_wallet_rpc("zeroconf") + utxos = zeroconf_wallet.listunspent(minconf=0) + assert_equal(len(utxos), 1) + assert_equal(utxos[0]['confirmations'], 0) + # accounts for trusted balance + bal = zeroconf_wallet.getbalances() + assert_equal(bal['mine']['trusted'], utxos[0]['amount']) + assert_equal(bal['mine']['untrusted_pending'], 0) + + zeroconf_wallet.sendtoaddress(zeroconf_wallet.getnewaddress(), Decimal('0.5')) + if __name__ == '__main__': WalletTest().main() diff --git a/test/functional/wallet_bumpfee.py b/test/functional/wallet_bumpfee.py index f1e7869d91..5ed7b0ccaa 100755 --- a/test/functional/wallet_bumpfee.py +++ b/test/functional/wallet_bumpfee.py @@ -26,6 +26,7 @@ from test_framework.util import ( assert_equal, assert_greater_than, assert_raises_rpc_error, + get_fee, ) from test_framework.wallet import MiniWallet @@ -81,13 +82,14 @@ class BumpFeeTest(BitcoinTestFramework): self.log.info("Running tests") dest_address = peer_node.getnewaddress() - for mode in ["default", "fee_rate"]: + for mode in ["default", "fee_rate", "new_outputs"]: test_simple_bumpfee_succeeds(self, mode, rbf_node, peer_node, dest_address) self.test_invalid_parameters(rbf_node, peer_node, dest_address) test_segwit_bumpfee_succeeds(self, rbf_node, dest_address) test_nonrbf_bumpfee_fails(self, peer_node, dest_address) test_notmine_bumpfee(self, rbf_node, peer_node, dest_address) test_bumpfee_with_descendant_fails(self, rbf_node, rbf_node_address, dest_address) + test_bumpfee_with_abandoned_descendant_succeeds(self, rbf_node, rbf_node_address, dest_address) test_dust_to_fee(self, rbf_node, dest_address) test_watchonly_psbt(self, peer_node, rbf_node, dest_address) test_rebumping(self, rbf_node, dest_address) @@ -103,6 +105,9 @@ class BumpFeeTest(BitcoinTestFramework): test_small_output_with_feerate_succeeds(self, rbf_node, dest_address) test_no_more_inputs_fails(self, rbf_node, dest_address) + # Context independent tests + test_feerate_checks_replaced_outputs(self, rbf_node, peer_node) + def test_invalid_parameters(self, rbf_node, peer_node, dest_address): self.log.info('Test invalid parameters') rbfid = spend_one_input(rbf_node, dest_address) @@ -156,6 +161,14 @@ class BumpFeeTest(BitcoinTestFramework): assert_raises_rpc_error(-8, 'Invalid estimate_mode parameter, must be one of: "unset", "economical", "conservative"', rbf_node.bumpfee, rbfid, {"estimate_mode": mode}) + self.log.info("Test invalid outputs values") + assert_raises_rpc_error(-8, "Invalid parameter, output argument cannot be an empty array", + rbf_node.bumpfee, rbfid, {"outputs": []}) + assert_raises_rpc_error(-8, "Invalid parameter, duplicated address: " + dest_address, + rbf_node.bumpfee, rbfid, {"outputs": [{dest_address: 0.1}, {dest_address: 0.2}]}) + assert_raises_rpc_error(-8, "Invalid parameter, duplicate key: data", + rbf_node.bumpfee, rbfid, {"outputs": [{"data": "deadbeef"}, {"data": "deadbeef"}]}) + self.clear_mempool() @@ -168,6 +181,10 @@ def test_simple_bumpfee_succeeds(self, mode, rbf_node, peer_node, dest_address): if mode == "fee_rate": bumped_psbt = rbf_node.psbtbumpfee(rbfid, {"fee_rate": str(NORMAL)}) bumped_tx = rbf_node.bumpfee(rbfid, {"fee_rate": NORMAL}) + elif mode == "new_outputs": + new_address = peer_node.getnewaddress() + bumped_psbt = rbf_node.psbtbumpfee(rbfid, {"outputs": {new_address: 0.0003}}) + bumped_tx = rbf_node.bumpfee(rbfid, {"outputs": {new_address: 0.0003}}) else: bumped_psbt = rbf_node.psbtbumpfee(rbfid) bumped_tx = rbf_node.bumpfee(rbfid) @@ -191,6 +208,10 @@ def test_simple_bumpfee_succeeds(self, mode, rbf_node, peer_node, dest_address): bumpedwtx = rbf_node.gettransaction(bumped_tx["txid"]) assert_equal(oldwtx["replaced_by_txid"], bumped_tx["txid"]) assert_equal(bumpedwtx["replaces_txid"], rbfid) + # if this is a new_outputs test, check that outputs were indeed replaced + if mode == "new_outputs": + assert len(bumpedwtx["details"]) == 1 + assert bumpedwtx["details"][0]["address"] == new_address self.clear_mempool() @@ -286,6 +307,35 @@ def test_bumpfee_with_descendant_fails(self, rbf_node, rbf_node_address, dest_ad self.clear_mempool() +def test_bumpfee_with_abandoned_descendant_succeeds(self, rbf_node, rbf_node_address, dest_address): + self.log.info('Test that fee can be bumped when it has abandoned descendant') + # parent is send-to-self, so we don't have to check which output is change when creating the child tx + parent_id = spend_one_input(rbf_node, rbf_node_address) + # Submit child transaction with low fee + child_id = rbf_node.send(outputs={dest_address: 0.00020000}, + options={"inputs": [{"txid": parent_id, "vout": 0}], "fee_rate": 2})["txid"] + assert child_id in rbf_node.getrawmempool() + + # Restart the node with higher min relay fee so the descendant tx is no longer in mempool so that we can abandon it + self.restart_node(1, ['-minrelaytxfee=0.00005'] + self.extra_args[1]) + rbf_node.walletpassphrase(WALLET_PASSPHRASE, WALLET_PASSPHRASE_TIMEOUT) + self.connect_nodes(1, 0) + assert parent_id in rbf_node.getrawmempool() + assert child_id not in rbf_node.getrawmempool() + # Should still raise an error even if not in mempool + assert_raises_rpc_error(-8, "Transaction has descendants in the wallet", rbf_node.bumpfee, parent_id) + # Now abandon the child transaction and bump the original + rbf_node.abandontransaction(child_id) + bumped_result = rbf_node.bumpfee(parent_id, {"fee_rate": HIGH}) + assert bumped_result['txid'] in rbf_node.getrawmempool() + assert parent_id not in rbf_node.getrawmempool() + # Cleanup + self.restart_node(1, self.extra_args[1]) + rbf_node.walletpassphrase(WALLET_PASSPHRASE, WALLET_PASSPHRASE_TIMEOUT) + self.connect_nodes(1, 0) + self.clear_mempool() + + def test_small_output_with_feerate_succeeds(self, rbf_node, dest_address): self.log.info('Testing small output with feerate bump succeeds') @@ -603,12 +653,14 @@ def test_change_script_match(self, rbf_node, dest_address): self.clear_mempool() -def spend_one_input(node, dest_address, change_size=Decimal("0.00049000")): +def spend_one_input(node, dest_address, change_size=Decimal("0.00049000"), data=None): tx_input = dict( sequence=MAX_BIP125_RBF_SEQUENCE, **next(u for u in node.listunspent() if u["amount"] == Decimal("0.00100000"))) destinations = {dest_address: Decimal("0.00050000")} if change_size > 0: destinations[node.getrawchangeaddress()] = change_size + if data: + destinations['data'] = data rawtx = node.createrawtransaction([tx_input], destinations) signedtx = node.signrawtransactionwithwallet(rawtx) txid = node.sendrawtransaction(signedtx["hex"]) @@ -625,5 +677,35 @@ def test_no_more_inputs_fails(self, rbf_node, dest_address): self.clear_mempool() +def test_feerate_checks_replaced_outputs(self, rbf_node, peer_node): + # Make sure there is enough balance + peer_node.sendtoaddress(rbf_node.getnewaddress(), 60) + self.generate(peer_node, 1) + + self.log.info("Test that feerate checks use replaced outputs") + outputs = [] + for i in range(50): + outputs.append({rbf_node.getnewaddress(address_type="bech32"): 1}) + tx_res = rbf_node.send(outputs=outputs, fee_rate=5) + tx_details = rbf_node.gettransaction(txid=tx_res["txid"], verbose=True) + + # Calculate the minimum feerate required for the bump to work. + # Since the bumped tx will replace all of the outputs with a single output, we can estimate that its size will 31 * (len(outputs) - 1) bytes smaller + tx_size = tx_details["decoded"]["vsize"] + est_bumped_size = tx_size - (len(tx_details["decoded"]["vout"]) - 1) * 31 + inc_fee_rate = max(rbf_node.getmempoolinfo()["incrementalrelayfee"], Decimal(0.00005000)) # Wallet has a fixed incremental relay fee of 5 sat/vb + # RPC gives us fee as negative + min_fee = (-tx_details["fee"] + get_fee(est_bumped_size, inc_fee_rate)) * Decimal(1e8) + min_fee_rate = (min_fee / est_bumped_size).quantize(Decimal("1.000")) + + # Attempt to bumpfee and replace all outputs with a single one using a feerate slightly less than the minimum + new_outputs = [{rbf_node.getnewaddress(address_type="bech32"): 49}] + assert_raises_rpc_error(-8, "Insufficient total fee", rbf_node.bumpfee, tx_res["txid"], {"fee_rate": min_fee_rate - 1, "outputs": new_outputs}) + + # Bumpfee and replace all outputs with a single one using the minimum feerate + rbf_node.bumpfee(tx_res["txid"], {"fee_rate": min_fee_rate, "outputs": new_outputs}) + self.clear_mempool() + + if __name__ == "__main__": BumpFeeTest().main() diff --git a/test/functional/wallet_change_address.py b/test/functional/wallet_change_address.py new file mode 100755 index 0000000000..f8bfe9eebf --- /dev/null +++ b/test/functional/wallet_change_address.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +# Copyright (c) 2023 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test wallet change address selection""" + +import re + +from test_framework.blocktools import COINBASE_MATURITY +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, +) + + +class WalletChangeAddressTest(BitcoinTestFramework): + def add_options(self, parser): + self.add_wallet_options(parser) + + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 3 + # discardfee is used to make change outputs less likely in the change_pos test + self.extra_args = [ + [], + ["-discardfee=1"], + ["-avoidpartialspends", "-discardfee=1"] + ] + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + + def assert_change_index(self, node, tx, index): + change_index = None + for vout in tx["vout"]: + info = node.getaddressinfo(vout["scriptPubKey"]["address"]) + if (info["ismine"] and info["ischange"]): + change_index = int(re.findall(r'\d+', info["hdkeypath"])[-1]) + break + assert_equal(change_index, index) + + def assert_change_pos(self, wallet, tx, pos): + change_pos = None + for index, output in enumerate(tx["vout"]): + info = wallet.getaddressinfo(output["scriptPubKey"]["address"]) + if (info["ismine"] and info["ischange"]): + change_pos = index + break + assert_equal(change_pos, pos) + + def run_test(self): + self.log.info("Setting up") + # Mine some coins + self.generate(self.nodes[0], COINBASE_MATURITY + 1) + + # Get some addresses from the two nodes + addr1 = [self.nodes[1].getnewaddress() for _ in range(3)] + addr2 = [self.nodes[2].getnewaddress() for _ in range(3)] + addrs = addr1 + addr2 + + # Send 1 + 0.5 coin to each address + [self.nodes[0].sendtoaddress(addr, 1.0) for addr in addrs] + [self.nodes[0].sendtoaddress(addr, 0.5) for addr in addrs] + self.generate(self.nodes[0], 1) + + for i in range(20): + for n in [1, 2]: + self.log.debug(f"Send transaction from node {n}: expected change index {i}") + txid = self.nodes[n].sendtoaddress(self.nodes[0].getnewaddress(), 0.2) + tx = self.nodes[n].getrawtransaction(txid, True) + # find the change output and ensure that expected change index was used + self.assert_change_index(self.nodes[n], tx, i) + + # Start next test with fresh wallets and new coins + self.nodes[1].createwallet("w1") + self.nodes[2].createwallet("w2") + w1 = self.nodes[1].get_wallet_rpc("w1") + w2 = self.nodes[2].get_wallet_rpc("w2") + addr1 = w1.getnewaddress() + addr2 = w2.getnewaddress() + self.nodes[0].sendtoaddress(addr1, 3.0) + self.nodes[0].sendtoaddress(addr1, 0.1) + self.nodes[0].sendtoaddress(addr2, 3.0) + self.nodes[0].sendtoaddress(addr2, 0.1) + self.generate(self.nodes[0], 1) + + sendTo1 = self.nodes[0].getnewaddress() + sendTo2 = self.nodes[0].getnewaddress() + sendTo3 = self.nodes[0].getnewaddress() + + # The avoid partial spends wallet will always create a change output + node = self.nodes[2] + res = w2.send({sendTo1: "1.0", sendTo2: "1.0", sendTo3: "0.9999"}, options={"change_position": 0}) + tx = node.getrawtransaction(res["txid"], True) + self.assert_change_pos(w2, tx, 0) + + # The default wallet will internally create a tx without change first, + # then create a second candidate using APS that requires a change output. + # Ensure that the user-configured change position is kept + node = self.nodes[1] + res = w1.send({sendTo1: "1.0", sendTo2: "1.0", sendTo3: "0.9999"}, options={"change_position": 0}) + tx = node.getrawtransaction(res["txid"], True) + # If the wallet ignores the user's change_position there is still a 25% + # that the random change position passes the test + self.assert_change_pos(w1, tx, 0) + +if __name__ == '__main__': + WalletChangeAddressTest().main() diff --git a/test/functional/wallet_create_tx.py b/test/functional/wallet_create_tx.py index 2d9bb38fcc..4e31b48ec0 100755 --- a/test/functional/wallet_create_tx.py +++ b/test/functional/wallet_create_tx.py @@ -32,6 +32,7 @@ class CreateTxWalletTest(BitcoinTestFramework): self.test_anti_fee_sniping() self.test_tx_size_too_large() + self.test_create_too_long_mempool_chain() def test_anti_fee_sniping(self): self.log.info('Check that we have some (old) blocks and that anti-fee-sniping is disabled') @@ -80,6 +81,31 @@ class CreateTxWalletTest(BitcoinTestFramework): ) self.nodes[0].settxfee(0) + def test_create_too_long_mempool_chain(self): + self.log.info('Check too-long mempool chain error') + df_wallet = self.nodes[0].get_wallet_rpc(self.default_wallet_name) + + self.nodes[0].createwallet("too_long") + test_wallet = self.nodes[0].get_wallet_rpc("too_long") + + tx_data = df_wallet.send(outputs=[{test_wallet.getnewaddress(): 25}], options={"change_position": 0}) + txid = tx_data['txid'] + vout = 1 + + self.nodes[0].syncwithvalidationinterfacequeue() + options = {"change_position": 0, "add_inputs": False} + for i in range(1, 25): + options['inputs'] = [{'txid': txid, 'vout': vout}] + tx_data = test_wallet.send(outputs=[{test_wallet.getnewaddress(): 25 - i}], options=options) + txid = tx_data['txid'] + + # Sending one more chained transaction will fail + options = {"minconf": 0, "include_unsafe": True, 'add_inputs': True} + assert_raises_rpc_error(-4, "Unconfirmed UTXOs are available, but spending them creates a chain of transactions that will be rejected by the mempool", + test_wallet.send, outputs=[{test_wallet.getnewaddress(): 0.3}], options=options) + + test_wallet.unloadwallet() + if __name__ == '__main__': CreateTxWalletTest().main() diff --git a/test/functional/wallet_createwallet.py b/test/functional/wallet_createwallet.py index 22c491441b..58cc6befbd 100755 --- a/test/functional/wallet_createwallet.py +++ b/test/functional/wallet_createwallet.py @@ -15,12 +15,17 @@ from test_framework.util import ( ) from test_framework.wallet_util import bytes_to_wif, generate_wif_key +EMPTY_PASSPHRASE_MSG = "Empty string given as passphrase, wallet will not be encrypted." +LEGACY_WALLET_MSG = "Wallet created successfully. The legacy wallet type is being deprecated and support for creating and opening legacy wallets will be removed in the future." + + class CreateWalletTest(BitcoinTestFramework): def add_options(self, parser): self.add_wallet_options(parser) def set_test_params(self): self.num_nodes = 1 + self.extra_args = [["-deprecatedrpc=walletwarningfield"]] def skip_test_if_missing_module(self): self.skip_if_no_wallet() @@ -55,7 +60,7 @@ class CreateWalletTest(BitcoinTestFramework): else: result = w1.importmulti([{'scriptPubKey': {'address': key_to_p2wpkh(eckey.get_pubkey().get_bytes())}, 'timestamp': 'now', 'keys': [privkey]}]) assert not result[0]['success'] - assert 'warning' not in result[0] + assert 'warnings' not in result[0] assert_equal(result[0]['error']['code'], -4) assert_equal(result[0]['error']['message'], 'Cannot import private keys to a wallet with private keys disabled') @@ -159,7 +164,9 @@ class CreateWalletTest(BitcoinTestFramework): assert_equal(walletinfo['keypoolsize_hd_internal'], keys) # Allow empty passphrase, but there should be a warning resp = self.nodes[0].createwallet(wallet_name='w7', disable_private_keys=False, blank=False, passphrase='') - assert 'Empty string given as passphrase, wallet will not be encrypted.' in resp['warning'] + assert_equal(resp["warning"], EMPTY_PASSPHRASE_MSG if self.options.descriptors else f"{EMPTY_PASSPHRASE_MSG}\n{LEGACY_WALLET_MSG}") + assert_equal(resp["warnings"], [EMPTY_PASSPHRASE_MSG] if self.options.descriptors else [EMPTY_PASSPHRASE_MSG, LEGACY_WALLET_MSG]) + w7 = node.get_wallet_rpc('w7') assert_raises_rpc_error(-15, 'Error: running with an unencrypted wallet, but walletpassphrase was called.', w7.walletpassphrase, '', 60) @@ -174,8 +181,24 @@ class CreateWalletTest(BitcoinTestFramework): if self.is_bdb_compiled(): self.log.info("Test legacy wallet deprecation") - res = self.nodes[0].createwallet(wallet_name="legacy_w0", descriptors=False, passphrase=None) - assert_equal(res["warning"], "Wallet created successfully. The legacy wallet type is being deprecated and support for creating and opening legacy wallets will be removed in the future.") + result = self.nodes[0].createwallet(wallet_name="legacy_w0", descriptors=False, passphrase=None) + assert_equal(result, { + "name": "legacy_w0", + "warning": LEGACY_WALLET_MSG, + "warnings": [LEGACY_WALLET_MSG], + }) + result = self.nodes[0].createwallet(wallet_name="legacy_w1", descriptors=False, passphrase="") + assert_equal(result, { + "name": "legacy_w1", + "warning": f"{EMPTY_PASSPHRASE_MSG}\n{LEGACY_WALLET_MSG}", + "warnings": [EMPTY_PASSPHRASE_MSG, LEGACY_WALLET_MSG], + }) + + self.log.info('Test "warning" field deprecation, i.e. not returned without -deprecatedrpc=walletwarningfield') + self.restart_node(0, extra_args=[]) + result = self.nodes[0].createwallet(wallet_name="w7_again", disable_private_keys=False, blank=False, passphrase="") + assert "warning" not in result + if __name__ == '__main__': CreateWalletTest().main() diff --git a/test/functional/wallet_crosschain.py b/test/functional/wallet_crosschain.py index 6f93ad4e3b..7a1297e65f 100755 --- a/test/functional/wallet_crosschain.py +++ b/test/functional/wallet_crosschain.py @@ -25,11 +25,7 @@ class WalletCrossChain(BitcoinTestFramework): # Switch node 1 to testnet before starting it. self.nodes[1].chain = 'testnet3' self.nodes[1].extra_args = ['-maxconnections=0', '-prune=550'] # disable testnet sync - with open(self.nodes[1].bitcoinconf, 'r', encoding='utf8') as conf: - conf_data = conf.read() - with open (self.nodes[1].bitcoinconf, 'w', encoding='utf8') as conf: - conf.write(conf_data.replace('regtest=', 'testnet=').replace('[regtest]', '[test]')) - + self.nodes[1].replace_in_config([('regtest=', 'testnet='), ('[regtest]', '[test]')]) self.start_nodes() def run_test(self): diff --git a/test/functional/wallet_descriptor.py b/test/functional/wallet_descriptor.py index 2b70e5ecc9..b0e93df36a 100755 --- a/test/functional/wallet_descriptor.py +++ b/test/functional/wallet_descriptor.py @@ -4,7 +4,11 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test descriptor wallet function.""" import os -import sqlite3 + +try: + import sqlite3 +except ImportError: + pass from test_framework.blocktools import COINBASE_MATURITY from test_framework.test_framework import BitcoinTestFramework @@ -27,6 +31,7 @@ class WalletDescriptorTest(BitcoinTestFramework): def skip_test_if_missing_module(self): self.skip_if_no_wallet() self.skip_if_no_sqlite() + self.skip_if_no_py_sqlite3() def run_test(self): if self.is_bdb_compiled(): @@ -109,7 +114,7 @@ class WalletDescriptorTest(BitcoinTestFramework): # Make sure things are disabled self.log.info("Test disabled RPCs") assert_raises_rpc_error(-4, "Only legacy wallets are supported by this command", recv_wrpc.rpc.importprivkey, "cVpF924EspNh8KjYsfhgY96mmxvT6DgdWiTYMtMjuM74hJaU5psW") - assert_raises_rpc_error(-4, "Only legacy wallets are supported by this command", recv_wrpc.rpc.importpubkey, send_wrpc.getaddressinfo(send_wrpc.getnewaddress())) + assert_raises_rpc_error(-4, "Only legacy wallets are supported by this command", recv_wrpc.rpc.importpubkey, send_wrpc.getaddressinfo(send_wrpc.getnewaddress())["pubkey"]) assert_raises_rpc_error(-4, "Only legacy wallets are supported by this command", recv_wrpc.rpc.importaddress, recv_wrpc.getnewaddress()) assert_raises_rpc_error(-4, "Only legacy wallets are supported by this command", recv_wrpc.rpc.importmulti, []) assert_raises_rpc_error(-4, "Only legacy wallets are supported by this command", recv_wrpc.rpc.addmultisigaddress, 1, [recv_wrpc.getnewaddress()]) diff --git a/test/functional/wallet_encryption.py b/test/functional/wallet_encryption.py index 885c52cf2e..88b9ebbddd 100755 --- a/test/functional/wallet_encryption.py +++ b/test/functional/wallet_encryption.py @@ -90,6 +90,17 @@ class WalletEncryptionTest(BitcoinTestFramework): self.nodes[0].walletpassphrase(passphrase2, MAX_VALUE + 1000) actual_time = self.nodes[0].getwalletinfo()['unlocked_until'] assert_equal(actual_time, expected_time) + self.nodes[0].walletlock() + + # Test passphrase with null characters + passphrase_with_nulls = "Phrase\0With\0Nulls" + self.nodes[0].walletpassphrasechange(passphrase2, passphrase_with_nulls) + # walletpassphrasechange should not stop at null characters + assert_raises_rpc_error(-14, "wallet passphrase entered was incorrect", self.nodes[0].walletpassphrase, passphrase_with_nulls.partition("\0")[0], 10) + self.nodes[0].walletpassphrase(passphrase_with_nulls, 10) + sig = self.nodes[0].signmessage(address, msg) + assert self.nodes[0].verifymessage(address, sig, msg) + self.nodes[0].walletlock() if __name__ == '__main__': diff --git a/test/functional/wallet_fast_rescan.py b/test/functional/wallet_fast_rescan.py index 52e33acb24..112aa25e86 100755 --- a/test/functional/wallet_fast_rescan.py +++ b/test/functional/wallet_fast_rescan.py @@ -7,6 +7,7 @@ import os from typing import List +from test_framework.address import address_to_scriptpubkey from test_framework.descriptors import descsum_create from test_framework.test_framework import BitcoinTestFramework from test_framework.test_node import TestNode @@ -40,7 +41,6 @@ class WalletFastRescanTest(BitcoinTestFramework): def run_test(self): node = self.nodes[0] wallet = MiniWallet(node) - wallet.rescan_utxos() self.log.info("Create descriptor wallet with backup") WALLET_BACKUP_FILENAME = os.path.join(node.datadir, 'wallet.bak') @@ -59,7 +59,7 @@ class WalletFastRescanTest(BitcoinTestFramework): if 'range' in desc_info: start_range, end_range = desc_info['range'] addr = w.deriveaddresses(desc_info['desc'], [end_range, end_range])[0] - spk = bytes.fromhex(w.getaddressinfo(addr)['scriptPubKey']) + spk = address_to_scriptpubkey(addr) self.log.info(f"-> range [{start_range},{end_range}], last address {addr}") else: spk = bytes.fromhex(fixed_key.p2wpkh_script) diff --git a/test/functional/wallet_fundrawtransaction.py b/test/functional/wallet_fundrawtransaction.py index 98b0f70b01..29ddb77b41 100755 --- a/test/functional/wallet_fundrawtransaction.py +++ b/test/functional/wallet_fundrawtransaction.py @@ -148,6 +148,7 @@ class RawTransactionsTest(BitcoinTestFramework): self.test_external_inputs() self.test_22670() self.test_feerate_rounding() + self.test_input_confs_control() def test_change_position(self): """Ensure setting changePosition in fundraw with an exact match is handled properly.""" @@ -1403,6 +1404,66 @@ class RawTransactionsTest(BitcoinTestFramework): rawtx = w.createrawtransaction(inputs=[], outputs=[{self.nodes[0].getnewaddress(address_type="bech32"): 1 - 0.00000202}]) assert_raises_rpc_error(-4, "Insufficient funds", w.fundrawtransaction, rawtx, {"fee_rate": 1.85}) + def test_input_confs_control(self): + self.nodes[0].createwallet("minconf") + wallet = self.nodes[0].get_wallet_rpc("minconf") + + # Fund the wallet with different chain heights + for _ in range(2): + self.nodes[2].sendmany("", {wallet.getnewaddress():1, wallet.getnewaddress():1}) + self.generate(self.nodes[2], 1) + + unconfirmed_txid = wallet.sendtoaddress(wallet.getnewaddress(), 0.5) + + self.log.info("Crafting TX using an unconfirmed input") + target_address = self.nodes[2].getnewaddress() + raw_tx1 = wallet.createrawtransaction([], {target_address: 0.1}, 0, True) + funded_tx1 = wallet.fundrawtransaction(raw_tx1, {'fee_rate': 1, 'maxconf': 0})['hex'] + + # Make sure we only had the one input + tx1_inputs = self.nodes[0].decoderawtransaction(funded_tx1)['vin'] + assert_equal(len(tx1_inputs), 1) + + utxo1 = tx1_inputs[0] + assert unconfirmed_txid == utxo1['txid'] + + final_tx1 = wallet.signrawtransactionwithwallet(funded_tx1)['hex'] + txid1 = self.nodes[0].sendrawtransaction(final_tx1) + + mempool = self.nodes[0].getrawmempool() + assert txid1 in mempool + + self.log.info("Fail to craft a new TX with minconf above highest one") + # Create a replacement tx to 'final_tx1' that has 1 BTC target instead of 0.1. + raw_tx2 = wallet.createrawtransaction([{'txid': utxo1['txid'], 'vout': utxo1['vout']}], {target_address: 1}) + assert_raises_rpc_error(-4, "Insufficient funds", wallet.fundrawtransaction, raw_tx2, {'add_inputs': True, 'minconf': 3, 'fee_rate': 10}) + + self.log.info("Fail to broadcast a new TX with maxconf 0 due to BIP125 rules to verify it actually chose unconfirmed outputs") + # Now fund 'raw_tx2' to fulfill the total target (1 BTC) by using all the wallet unconfirmed outputs. + # As it was created with the first unconfirmed output, 'raw_tx2' only has 0.1 BTC covered (need to fund 0.9 BTC more). + # So, the selection process, to cover the amount, will pick up the 'final_tx1' output as well, which is an output of the tx that this + # new tx is replacing!. So, once we send it to the mempool, it will return a "bad-txns-spends-conflicting-tx" + # because the input will no longer exist once the first tx gets replaced by this new one). + funded_invalid = wallet.fundrawtransaction(raw_tx2, {'add_inputs': True, 'maxconf': 0, 'fee_rate': 10})['hex'] + final_invalid = wallet.signrawtransactionwithwallet(funded_invalid)['hex'] + assert_raises_rpc_error(-26, "bad-txns-spends-conflicting-tx", self.nodes[0].sendrawtransaction, final_invalid) + + self.log.info("Craft a replacement adding inputs with highest depth possible") + funded_tx2 = wallet.fundrawtransaction(raw_tx2, {'add_inputs': True, 'minconf': 2, 'fee_rate': 10})['hex'] + tx2_inputs = self.nodes[0].decoderawtransaction(funded_tx2)['vin'] + assert_greater_than_or_equal(len(tx2_inputs), 2) + for vin in tx2_inputs: + if vin['txid'] != unconfirmed_txid: + assert_greater_than_or_equal(self.nodes[0].gettxout(vin['txid'], vin['vout'])['confirmations'], 2) + + final_tx2 = wallet.signrawtransactionwithwallet(funded_tx2)['hex'] + txid2 = self.nodes[0].sendrawtransaction(final_tx2) + + mempool = self.nodes[0].getrawmempool() + assert txid1 not in mempool + assert txid2 in mempool + + wallet.unloadwallet() if __name__ == '__main__': RawTransactionsTest().main() diff --git a/test/functional/wallet_groups.py b/test/functional/wallet_groups.py index 83c1826a41..bdb9081261 100755 --- a/test/functional/wallet_groups.py +++ b/test/functional/wallet_groups.py @@ -41,6 +41,11 @@ class WalletGroupTest(BitcoinTestFramework): def run_test(self): self.log.info("Setting up") + # To take full use of immediate tx relay, all nodes need to be reachable + # via inbound peers, i.e. connect first to last to close the circle + # (the default test network topology looks like this: + # node0 <-- node1 <-- node2 <-- node3 <-- node4 <-- node5) + self.connect_nodes(0, self.num_nodes - 1) # Mine some coins self.generate(self.nodes[0], COINBASE_MATURITY + 1) diff --git a/test/functional/wallet_importdescriptors.py b/test/functional/wallet_importdescriptors.py index 9e813166c5..4f2db2018a 100755 --- a/test/functional/wallet_importdescriptors.py +++ b/test/functional/wallet_importdescriptors.py @@ -15,7 +15,9 @@ variants. - `test_address()` is called to call getaddressinfo for an address on node1 and test the values returned.""" -from test_framework.address import key_to_p2pkh +import concurrent.futures + +from test_framework.authproxy import JSONRPCException from test_framework.blocktools import COINBASE_MATURITY from test_framework.test_framework import BitcoinTestFramework from test_framework.descriptors import descsum_create @@ -120,12 +122,11 @@ class ImportDescriptorsTest(BitcoinTestFramework): self.log.info("Internal addresses should be detected as such") key = get_generate_key() - addr = key_to_p2pkh(key.pubkey) self.test_importdesc({"desc": descsum_create("pkh(" + key.pubkey + ")"), "timestamp": "now", "internal": True}, success=True) - info = w1.getaddressinfo(addr) + info = w1.getaddressinfo(key.p2pkh_addr) assert_equal(info["ismine"], True) assert_equal(info["ischange"], True) @@ -450,14 +451,14 @@ class ImportDescriptorsTest(BitcoinTestFramework): wallet=wmulti_priv) assert_equal(wmulti_priv.getwalletinfo()['keypoolsize'], 1001) # Range end (1000) is inclusive, so 1001 addresses generated - addr = wmulti_priv.getnewaddress('', 'bech32') + addr = wmulti_priv.getnewaddress('', 'bech32') # uses receive 0 assert_equal(addr, 'bcrt1qdt0qy5p7dzhxzmegnn4ulzhard33s2809arjqgjndx87rv5vd0fq2czhy8') # Derived at m/84'/0'/0'/0 - change_addr = wmulti_priv.getrawchangeaddress('bech32') - assert_equal(change_addr, 'bcrt1qt9uhe3a9hnq7vajl7a094z4s3crm9ttf8zw3f5v9gr2nyd7e3lnsy44n8e') + change_addr = wmulti_priv.getrawchangeaddress('bech32') # uses change 0 + assert_equal(change_addr, 'bcrt1qt9uhe3a9hnq7vajl7a094z4s3crm9ttf8zw3f5v9gr2nyd7e3lnsy44n8e') # Derived at m/84'/1'/0'/0 assert_equal(wmulti_priv.getwalletinfo()['keypoolsize'], 1000) txid = w0.sendtoaddress(addr, 10) self.generate(self.nodes[0], 6) - send_txid = wmulti_priv.sendtoaddress(w0.getnewaddress(), 8) + send_txid = wmulti_priv.sendtoaddress(w0.getnewaddress(), 8) # uses change 1 decoded = wmulti_priv.gettransaction(txid=send_txid, verbose=True)['decoded'] assert_equal(len(decoded['vin'][0]['txinwitness']), 4) self.sync_all() @@ -483,10 +484,10 @@ class ImportDescriptorsTest(BitcoinTestFramework): wallet=wmulti_pub) assert_equal(wmulti_pub.getwalletinfo()['keypoolsize'], 1000) # The first one was already consumed by previous import and is detected as used - addr = wmulti_pub.getnewaddress('', 'bech32') + addr = wmulti_pub.getnewaddress('', 'bech32') # uses receive 1 assert_equal(addr, 'bcrt1qp8s25ckjl7gr6x2q3dx3tn2pytwp05upkjztk6ey857tt50r5aeqn6mvr9') # Derived at m/84'/0'/0'/1 - change_addr = wmulti_pub.getrawchangeaddress('bech32') - assert_equal(change_addr, 'bcrt1qzxl0qz2t88kljdnkzg4n4gapr6kte26390gttrg79x66nt4p04fssj53nl') + change_addr = wmulti_pub.getrawchangeaddress('bech32') # uses change 2 + assert_equal(change_addr, 'bcrt1qp6j3jw8yetefte7kw6v5pc89rkgakzy98p6gf7ayslaveaxqyjusnw580c') # Derived at m/84'/1'/0'/2 assert send_txid in self.nodes[0].getrawmempool(True) assert send_txid in (x['txid'] for x in wmulti_pub.listunspent(0)) assert_equal(wmulti_pub.getwalletinfo()['keypoolsize'], 999) @@ -669,5 +670,47 @@ class ImportDescriptorsTest(BitcoinTestFramework): success=True, warnings=["Unknown output type, cannot set descriptor to active."]) + self.log.info("Test importing a descriptor to an encrypted wallet") + + descriptor = {"desc": descsum_create("pkh(" + xpriv + "/1h/*h)"), + "timestamp": "now", + "active": True, + "range": [0,4000], + "next_index": 4000} + + self.nodes[0].createwallet("temp_wallet", blank=True, descriptors=True) + temp_wallet = self.nodes[0].get_wallet_rpc("temp_wallet") + temp_wallet.importdescriptors([descriptor]) + self.generatetoaddress(self.nodes[0], COINBASE_MATURITY + 1, temp_wallet.getnewaddress()) + self.generatetoaddress(self.nodes[0], COINBASE_MATURITY + 1, temp_wallet.getnewaddress()) + + self.nodes[0].createwallet("encrypted_wallet", blank=True, descriptors=True, passphrase="passphrase") + encrypted_wallet = self.nodes[0].get_wallet_rpc("encrypted_wallet") + + descriptor["timestamp"] = 0 + descriptor["next_index"] = 0 + + encrypted_wallet.walletpassphrase("passphrase", 99999) + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as thread: + with self.nodes[0].assert_debug_log(expected_msgs=["Rescan started from block 0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206... (slow variant inspecting all blocks)"], timeout=5): + importing = thread.submit(encrypted_wallet.importdescriptors, requests=[descriptor]) + + # Set the passphrase timeout to 1 to test that the wallet remains unlocked during the rescan + self.nodes[0].cli("-rpcwallet=encrypted_wallet").walletpassphrase("passphrase", 1) + + try: + self.nodes[0].cli("-rpcwallet=encrypted_wallet").walletlock() + except JSONRPCException as e: + assert e.error["code"] == -4 and "Error: the wallet is currently being used to rescan the blockchain for related transactions. Please call `abortrescan` before locking the wallet." in e.error["message"] + + try: + self.nodes[0].cli("-rpcwallet=encrypted_wallet").walletpassphrasechange("passphrase", "newpassphrase") + except JSONRPCException as e: + assert e.error["code"] == -4 and "Error: the wallet is currently being used to rescan the blockchain for related transactions. Please call `abortrescan` before changing the passphrase." in e.error["message"] + + assert_equal(importing.result(), [{"success": True}]) + + assert_equal(temp_wallet.getbalance(), encrypted_wallet.getbalance()) + if __name__ == '__main__': ImportDescriptorsTest().main() diff --git a/test/functional/wallet_labels.py b/test/functional/wallet_labels.py index 34acaee563..f074339a2b 100755 --- a/test/functional/wallet_labels.py +++ b/test/functional/wallet_labels.py @@ -23,16 +23,58 @@ class WalletLabelsTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True - self.num_nodes = 1 + self.num_nodes = 2 def skip_test_if_missing_module(self): self.skip_if_no_wallet() + def invalid_label_name_test(self): + node = self.nodes[0] + address = node.getnewaddress() + pubkey = node.getaddressinfo(address)['pubkey'] + rpc_calls = [ + [node.getnewaddress], + [node.setlabel, address], + [node.getaddressesbylabel], + [node.importpubkey, pubkey], + [node.addmultisigaddress, 1, [pubkey]], + [node.getreceivedbylabel], + [node.listsinceblock, node.getblockhash(0), 1, False, True, False], + ] + if self.options.descriptors: + response = node.importdescriptors([{ + 'desc': f'pkh({pubkey})', + 'label': '*', + 'timestamp': 'now', + }]) + else: + rpc_calls.extend([ + [node.importprivkey, node.dumpprivkey(address)], + [node.importaddress, address], + ]) + + response = node.importmulti([{ + 'scriptPubKey': {'address': address}, + 'label': '*', + 'timestamp': 'now', + }]) + + assert_equal(response[0]['success'], False) + assert_equal(response[0]['error']['code'], -11) + assert_equal(response[0]['error']['message'], "Invalid label name") + + for rpc_call in rpc_calls: + assert_raises_rpc_error(-11, "Invalid label name", *rpc_call, "*") + def run_test(self): # Check that there's no UTXO on the node node = self.nodes[0] assert_equal(len(node.listunspent()), 0) + self.log.info("Checking listlabels' invalid parameters") + assert_raises_rpc_error(-8, "Invalid 'purpose' argument, must be a known purpose string, typically 'send', or 'receive'.", node.listlabels, "notavalidpurpose") + assert_raises_rpc_error(-8, "Invalid 'purpose' argument, must be a known purpose string, typically 'send', or 'receive'.", node.listlabels, "unknown") + # Note each time we call generate, all generated coins go into # the same address, so we call twice to get two addresses w/50 each self.generatetoaddress(node, nblocks=1, address=node.getnewaddress(label='coinbase')) @@ -83,8 +125,14 @@ class WalletLabelsTest(BitcoinTestFramework): label.add_receive_address(address) label.verify(node) + # Check listlabels when passing 'purpose' + node2_addr = self.nodes[1].getnewaddress() + node.setlabel(node2_addr, "node2_addr") + assert_equal(node.listlabels(purpose="send"), ["node2_addr"]) + assert_equal(node.listlabels(purpose="receive"), sorted(['coinbase'] + [label.name for label in labels])) + # Check all labels are returned by listlabels. - assert_equal(node.listlabels(), sorted(['coinbase'] + [label.name for label in labels])) + assert_equal(node.listlabels(), sorted(['coinbase'] + [label.name for label in labels] + ["node2_addr"])) # Send a transaction to each label. for label in labels: @@ -138,6 +186,8 @@ class WalletLabelsTest(BitcoinTestFramework): # in the label. This is a no-op. change_label(node, labels[2].addresses[0], labels[2], labels[2]) + self.invalid_label_name_test() + if self.options.descriptors: # This is a descriptor wallet test because of segwit v1+ addresses self.log.info('Check watchonly labels') diff --git a/test/functional/wallet_listdescriptors.py b/test/functional/wallet_listdescriptors.py index fb2156bda1..c5479089c6 100755 --- a/test/functional/wallet_listdescriptors.py +++ b/test/functional/wallet_listdescriptors.py @@ -54,7 +54,7 @@ class ListDescriptorsTest(BitcoinTestFramework): assert_equal(4, len([d for d in result['descriptors'] if d['internal']])) for item in result['descriptors']: assert item['desc'] != '' - assert item['next'] == 0 + assert item['next_index'] == 0 assert item['range'] == [0, 0] assert item['timestamp'] is not None @@ -78,7 +78,8 @@ class ListDescriptorsTest(BitcoinTestFramework): 'timestamp': TIME_GENESIS_BLOCK, 'active': False, 'range': [0, 0], - 'next': 0}, + 'next': 0, + 'next_index': 0}, ], } assert_equal(expected, wallet.listdescriptors()) @@ -92,7 +93,8 @@ class ListDescriptorsTest(BitcoinTestFramework): 'timestamp': TIME_GENESIS_BLOCK, 'active': False, 'range': [0, 0], - 'next': 0}, + 'next': 0, + 'next_index': 0}, ], } assert_equal(expected_private, wallet.listdescriptors(True)) diff --git a/test/functional/wallet_migration.py b/test/functional/wallet_migration.py index 93bd5b481a..7c2959bb89 100755 --- a/test/functional/wallet_migration.py +++ b/test/functional/wallet_migration.py @@ -163,6 +163,10 @@ class WalletMigrationTest(BitcoinTestFramework): assert_equal(basic2.getbalance(), basic2_balance) self.assert_list_txs_equal(basic2.listtransactions(), basic2_txs) + # Now test migration on a descriptor wallet + self.log.info("Test \"nothing to migrate\" when the user tries to migrate a wallet with no legacy data") + assert_raises_rpc_error(-4, "Error: This wallet is already a descriptor wallet", basic2.migratewallet) + def test_multisig(self): default = self.nodes[0].get_wallet_rpc(self.default_wallet_name) @@ -258,7 +262,7 @@ class WalletMigrationTest(BitcoinTestFramework): self.log.info("Test migration of a wallet with watchonly imports") imports0 = self.create_legacy_wallet("imports0") - # Exteranl address label + # External address label imports0.setlabel(default.getnewaddress(), "external") # Normal non-watchonly tx @@ -311,6 +315,13 @@ class WalletMigrationTest(BitcoinTestFramework): assert_raises_rpc_error(-5, "Invalid or non-wallet transaction id", watchonly.gettransaction, received_txid) assert_equal(len(watchonly.listtransactions(include_watchonly=True)), 3) + # Check that labels were migrated and persisted to watchonly wallet + self.nodes[0].unloadwallet("imports0_watchonly") + self.nodes[0].loadwallet("imports0_watchonly") + labels = watchonly.listlabels() + assert "external" in labels + assert "imported" in labels + def test_no_privkeys(self): default = self.nodes[0].get_wallet_rpc(self.default_wallet_name) @@ -389,11 +400,75 @@ class WalletMigrationTest(BitcoinTestFramework): def test_encrypted(self): self.log.info("Test migration of an encrypted wallet") wallet = self.create_legacy_wallet("encrypted") + default = self.nodes[0].get_wallet_rpc(self.default_wallet_name) wallet.encryptwallet("pass") + addr = wallet.getnewaddress() + txid = default.sendtoaddress(addr, 1) + self.generate(self.nodes[0], 1) + bals = wallet.getbalances() + + assert_raises_rpc_error(-4, "Error: Wallet decryption failed, the wallet passphrase was not provided or was incorrect", wallet.migratewallet) + assert_raises_rpc_error(-4, "Error: Wallet decryption failed, the wallet passphrase was not provided or was incorrect", wallet.migratewallet, None, "badpass") + assert_raises_rpc_error(-4, "The passphrase contains a null character", wallet.migratewallet, None, "pass\0with\0null") + + wallet.migratewallet(passphrase="pass") + + info = wallet.getwalletinfo() + assert_equal(info["descriptors"], True) + assert_equal(info["format"], "sqlite") + assert_equal(info["unlocked_until"], 0) + wallet.gettransaction(txid) + + assert_equal(bals, wallet.getbalances()) + + def test_unloaded(self): + self.log.info("Test migration of a wallet that isn't loaded") + wallet = self.create_legacy_wallet("notloaded") + default = self.nodes[0].get_wallet_rpc(self.default_wallet_name) + + addr = wallet.getnewaddress() + txid = default.sendtoaddress(addr, 1) + self.generate(self.nodes[0], 1) + bals = wallet.getbalances() + + wallet.unloadwallet() - assert_raises_rpc_error(-15, "Error: migratewallet on encrypted wallets is currently unsupported.", wallet.migratewallet) - # TODO: Fix migratewallet so that we can actually migrate encrypted wallets + assert_raises_rpc_error(-8, "RPC endpoint wallet and wallet_name parameter specify different wallets", wallet.migratewallet, "someotherwallet") + assert_raises_rpc_error(-8, "Either RPC endpoint wallet or wallet_name parameter must be provided", self.nodes[0].migratewallet) + self.nodes[0].migratewallet("notloaded") + + info = wallet.getwalletinfo() + assert_equal(info["descriptors"], True) + assert_equal(info["format"], "sqlite") + wallet.gettransaction(txid) + + assert_equal(bals, wallet.getbalances()) + + def test_unloaded_by_path(self): + self.log.info("Test migration of a wallet that isn't loaded, specified by path") + wallet = self.create_legacy_wallet("notloaded2") + default = self.nodes[0].get_wallet_rpc(self.default_wallet_name) + + addr = wallet.getnewaddress() + txid = default.sendtoaddress(addr, 1) + self.generate(self.nodes[0], 1) + bals = wallet.getbalances() + + wallet.unloadwallet() + + wallet_file_path = os.path.join(self.nodes[0].datadir, "regtest", "wallets", "notloaded2") + self.nodes[0].migratewallet(wallet_file_path) + + # Because we gave the name by full path, the loaded wallet's name is that path too. + wallet = self.nodes[0].get_wallet_rpc(wallet_file_path) + + info = wallet.getwalletinfo() + assert_equal(info["descriptors"], True) + assert_equal(info["format"], "sqlite") + wallet.gettransaction(txid) + + assert_equal(bals, wallet.getbalances()) def run_test(self): self.generate(self.nodes[0], 101) @@ -405,6 +480,8 @@ class WalletMigrationTest(BitcoinTestFramework): self.test_no_privkeys() self.test_pk_coinbases() self.test_encrypted() + self.test_unloaded() + self.test_unloaded_by_path() if __name__ == '__main__': WalletMigrationTest().main() diff --git a/test/functional/wallet_miniscript.py b/test/functional/wallet_miniscript.py index cefcaf4dc7..7bc3424bf4 100755 --- a/test/functional/wallet_miniscript.py +++ b/test/functional/wallet_miniscript.py @@ -5,19 +5,137 @@ """Test Miniscript descriptors integration in the wallet.""" from test_framework.descriptors import descsum_create +from test_framework.psbt import PSBT, PSBT_IN_SHA256 from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal +TPRVS = [ + "tprv8ZgxMBicQKsPerQj6m35no46amfKQdjY7AhLnmatHYXs8S4MTgeZYkWAn4edSGwwL3vkSiiGqSZQrmy5D3P5gBoqgvYP2fCUpBwbKTMTAkL", + "tprv8ZgxMBicQKsPd3cbrKjE5GKKJLDEidhtzSSmPVtSPyoHQGL2LZw49yt9foZsN9BeiC5VqRaESUSDV2PS9w7zAVBSK6EQH3CZW9sMKxSKDwD", + "tprv8iF7W37EHnVEtDr9EFeyFjQJFL6SfGby2AnZ2vQARxTQHQXy9tdzZvBBVp8a19e5vXhskczLkJ1AZjqgScqWL4FpmXVp8LLjiorcrFK63Sr", +] +TPUBS = [ + "tpubD6NzVbkrYhZ4YPAbyf6urxqqnmJF79PzQtyERAmvkSVS9fweCTjxjDh22Z5St9fGb1a5DUCv8G27nYupKP1Ctr1pkamJossoetzws1moNRn", + "tpubD6NzVbkrYhZ4YMQC15JS7QcrsAyfGrGiykweqMmPxTkEVScu7vCZLNpPXW1XphHwzsgmqdHWDQAfucbM72EEB1ZEyfgZxYvkZjYVXx1xS9p", + "tpubD6NzVbkrYhZ4YU9vM1s53UhD75UyJatx8EMzMZ3VUjR2FciNfLLkAw6a4pWACChzobTseNqdWk4G7ZdBqRDLtLSACKykTScmqibb1ZrCvJu", + "tpubD6NzVbkrYhZ4XRMcMFMMFvzVt6jaDAtjZhD7JLwdPdMm9xa76DnxYYP7w9TZGJDVFkek3ArwVsuacheqqPog8TH5iBCX1wuig8PLXim4n9a", + "tpubD6NzVbkrYhZ4WsqRzDmkL82SWcu42JzUvKWzrJHQ8EC2vEHRHkXj1De93sD3biLrKd8XGnamXURGjMbYavbszVDXpjXV2cGUERucLJkE6cy", + "tpubDEFLeBkKTm8aiYkySz8hXAXPVnPSfxMi7Fxhg9sejUrkwJuRWvPdLEiXjTDbhGbjLKCZUDUUibLxTnK5UP1q7qYrSnPqnNe7M8mvAW1STcc", +] +PUBKEYS = [ + "02aebf2d10b040eb936a6f02f44ee82f8b34f5c1ccb20ff3949c2b28206b7c1068", + "030f64b922aee2fd597f104bc6cb3b670f1ca2c6c49b1071a1a6c010575d94fe5a", + "02abe475b199ec3d62fa576faee16a334fdb86ffb26dce75becebaaedf328ac3fe", + "0314f3dc33595b0d016bb522f6fe3a67680723d842c1b9b8ae6b59fdd8ab5cccb4", + "025eba3305bd3c829e4e1551aac7358e4178832c739e4fc4729effe428de0398ab", + "029ffbe722b147f3035c87cb1c60b9a5947dd49c774cc31e94773478711a929ac0", + "0211c7b2e18b6fd330f322de087da62da92ae2ae3d0b7cec7e616479cce175f183", +] + MINISCRIPTS = [ # One of two keys - "or_b(pk(tpubD6NzVbkrYhZ4XRMcMFMMFvzVt6jaDAtjZhD7JLwdPdMm9xa76DnxYYP7w9TZGJDVFkek3ArwVsuacheqqPog8TH5iBCX1wuig8PLXim4n9a/*),s:pk(tpubD6NzVbkrYhZ4WsqRzDmkL82SWcu42JzUvKWzrJHQ8EC2vEHRHkXj1De93sD3biLrKd8XGnamXURGjMbYavbszVDXpjXV2cGUERucLJkE6cy/*))", + f"or_b(pk({TPUBS[0]}/*),s:pk({TPUBS[1]}/*))", # A script similar (same spending policy) to BOLT3's offered HTLC (with anchor outputs) - "or_d(pk(tpubD6NzVbkrYhZ4XRMcMFMMFvzVt6jaDAtjZhD7JLwdPdMm9xa76DnxYYP7w9TZGJDVFkek3ArwVsuacheqqPog8TH5iBCX1wuig8PLXim4n9a/*),and_v(and_v(v:pk(tpubD6NzVbkrYhZ4WsqRzDmkL82SWcu42JzUvKWzrJHQ8EC2vEHRHkXj1De93sD3biLrKd8XGnamXURGjMbYavbszVDXpjXV2cGUERucLJkE6cy/*),or_c(pk(tpubD6NzVbkrYhZ4YNwtTWrKRJQzQX3PjPKeUQg1gYh1hiLMkk1cw8SRLgB1yb7JzE8bHKNt6EcZXkJ6AqpCZL1aaRSjnG36mLgbQvJZBNsjWnG/*),v:hash160(7f999c905d5e35cefd0a37673f746eb13fba3640))),older(1)))", + f"or_d(pk({TPUBS[0]}/*),and_v(and_v(v:pk({TPUBS[1]}/*),or_c(pk({TPUBS[2]}/*),v:hash160(7f999c905d5e35cefd0a37673f746eb13fba3640))),older(1)))", # A Revault Unvault policy with the older() replaced by an after() - "andor(multi(2,tpubD6NzVbkrYhZ4YMQC15JS7QcrsAyfGrGiykweqMmPxTkEVScu7vCZLNpPXW1XphHwzsgmqdHWDQAfucbM72EEB1ZEyfgZxYvkZjYVXx1xS9p/*,tpubD6NzVbkrYhZ4WkCyc7E3z6g6NkypHMiecnwc4DpWHTPqFdteRGkEKukdrSSyJGNnGrHNMfy4BCw2UXo5soYRCtCDDfy4q8pc8oyB7RgTFv8/*),and_v(v:multi(4,030f64b922aee2fd597f104bc6cb3b670f1ca2c6c49b1071a1a6c010575d94fe5a,02abe475b199ec3d62fa576faee16a334fdb86ffb26dce75becebaaedf328ac3fe,0314f3dc33595b0d016bb522f6fe3a67680723d842c1b9b8ae6b59fdd8ab5cccb4,025eba3305bd3c829e4e1551aac7358e4178832c739e4fc4729effe428de0398ab),after(424242)),thresh(4,pkh(tpubD6NzVbkrYhZ4YVrNggiT2ptVHwnFbLBqDkCtV5HkxR4WtcRLAQReKTkqZGNcV6GE7cQsmpBzzSzhk16DUwB1gn1L7ZPnJF2dnNePP1uMBCY/*),a:pkh(tpubD6NzVbkrYhZ4YU9vM1s53UhD75UyJatx8EMzMZ3VUjR2FciNfLLkAw6a4pWACChzobTseNqdWk4G7ZdBqRDLtLSACKykTScmqibb1ZrCvJu/*),a:pkh(tpubD6NzVbkrYhZ4YUHcFfuH9iEBLiH8CBRJTpS7X3qjHmh82m1KCNbzs6w9gyK8oWHSZmKHWcakAXCGfbKg6xoCvKzQCWAHyxaC7QcWfmzyBf4/*),a:pkh(tpubD6NzVbkrYhZ4XXEmQtS3sgxpJbMyMg4McqRR1Af6ULzyrTRnhwjyr1etPD7svap9oFtJf4MM72brUb5o7uvF2Jyszc5c1t836fJW7SX2e8D/*)))", + f"andor(multi(2,{TPUBS[0]}/*,{TPUBS[1]}/*),and_v(v:multi(4,{PUBKEYS[0]},{PUBKEYS[1]},{PUBKEYS[2]},{PUBKEYS[3]}),after(424242)),thresh(4,pkh({TPUBS[2]}/*),a:pkh({TPUBS[3]}/*),a:pkh({TPUBS[4]}/*),a:pkh({TPUBS[5]}/*)))", # Liquid-like federated pegin with emergency recovery keys - "or_i(and_b(pk(029ffbe722b147f3035c87cb1c60b9a5947dd49c774cc31e94773478711a929ac0),a:and_b(pk(025f05815e3a1a8a83bfbb03ce016c9a2ee31066b98f567f6227df1d76ec4bd143),a:and_b(pk(025625f41e4a065efc06d5019cbbd56fe8c07595af1231e7cbc03fafb87ebb71ec),a:and_b(pk(02a27c8b850a00f67da3499b60562673dcf5fdfb82b7e17652a7ac54416812aefd),s:pk(03e618ec5f384d6e19ca9ebdb8e2119e5bef978285076828ce054e55c4daf473e2))))),and_v(v:thresh(2,pkh(tpubD6NzVbkrYhZ4YK67cd5fDe4fBVmGB2waTDrAt1q4ey9HPq9veHjWkw3VpbaCHCcWozjkhgAkWpFrxuPMUrmXVrLHMfEJ9auoZA6AS1g3grC/*),a:pkh(033841045a531e1adf9910a6ec279589a90b3b8a904ee64ffd692bd08a8996c1aa),a:pkh(02aebf2d10b040eb936a6f02f44ee82f8b34f5c1ccb20ff3949c2b28206b7c1068)),older(4209713)))", + f"or_i(and_b(pk({PUBKEYS[0]}),a:and_b(pk({PUBKEYS[1]}),a:and_b(pk({PUBKEYS[2]}),a:and_b(pk({PUBKEYS[3]}),s:pk({PUBKEYS[4]}))))),and_v(v:thresh(2,pkh({TPUBS[0]}/*),a:pkh({PUBKEYS[5]}),a:pkh({PUBKEYS[6]})),older(4209713)))", +] + +MINISCRIPTS_PRIV = [ + # One of two keys, of which one private key is known + { + "ms": f"or_i(pk({TPRVS[0]}/*),pk({TPUBS[0]}/*))", + "sequence": None, + "locktime": None, + "sigs_count": 1, + "stack_size": 3, + }, + # A more complex policy, that can't be satisfied through the first branch (need for a preimage) + { + "ms": f"andor(ndv:older(2),and_v(v:pk({TPRVS[0]}),sha256(2a8ce30189b2ec3200b47aeb4feaac8fcad7c0ba170389729f4898b0b7933bcb)),and_v(v:pkh({TPRVS[1]}),pk({TPRVS[2]}/*)))", + "sequence": 2, + "locktime": None, + "sigs_count": 3, + "stack_size": 5, + }, + # The same policy but we provide the preimage. This path will be chosen as it's a smaller witness. + { + "ms": f"andor(ndv:older(2),and_v(v:pk({TPRVS[0]}),sha256(61e33e9dbfefc45f6a194187684d278f789fd4d5e207a357e79971b6519a8b12)),and_v(v:pkh({TPRVS[1]}),pk({TPRVS[2]}/*)))", + "sequence": 2, + "locktime": None, + "sigs_count": 3, + "stack_size": 4, + "sha256_preimages": { + "61e33e9dbfefc45f6a194187684d278f789fd4d5e207a357e79971b6519a8b12": "e8774f330f5f330c23e8bbefc5595cb87009ddb7ac3b8deaaa8e9e41702d919c" + }, + }, + # Signature with a relative timelock + { + "ms": f"and_v(v:older(2),pk({TPRVS[0]}/*))", + "sequence": 2, + "locktime": None, + "sigs_count": 1, + "stack_size": 2, + }, + # Signature with an absolute timelock + { + "ms": f"and_v(v:after(20),pk({TPRVS[0]}/*))", + "sequence": None, + "locktime": 20, + "sigs_count": 1, + "stack_size": 2, + }, + # Signature with both + { + "ms": f"and_v(v:older(4),and_v(v:after(30),pk({TPRVS[0]}/*)))", + "sequence": 4, + "locktime": 30, + "sigs_count": 1, + "stack_size": 2, + }, + # We have one key on each branch; Core signs both (can't finalize) + { + "ms": f"c:andor(pk({TPRVS[0]}/*),pk_k({TPUBS[0]}),and_v(v:pk({TPRVS[1]}),pk_k({TPUBS[1]})))", + "sequence": None, + "locktime": None, + "sigs_count": 2, + "stack_size": None, + }, + # We have all the keys, wallet selects the timeout path to sign since it's smaller and sequence is set + { + "ms": f"andor(pk({TPRVS[0]}/*),pk({TPRVS[2]}),and_v(v:pk({TPRVS[1]}),older(10)))", + "sequence": 10, + "locktime": None, + "sigs_count": 3, + "stack_size": 3, + }, + # We have all the keys, wallet selects the primary path to sign unconditionally since nsequence wasn't set to be valid for timeout path + { + "ms": f"andor(pk({TPRVS[0]}/*),pk({TPRVS[2]}),and_v(v:pkh({TPRVS[1]}),older(10)))", + "sequence": None, + "locktime": None, + "sigs_count": 3, + "stack_size": 3, + }, + # Finalizes to the smallest valid witness, regardless of sequence + { + "ms": f"or_d(pk({TPRVS[0]}/*),and_v(v:pk({TPRVS[1]}),and_v(v:pk({TPRVS[2]}),older(10))))", + "sequence": 12, + "locktime": None, + "sigs_count": 3, + "stack_size": 2, + }, + # Liquid-like federated pegin with emergency recovery privkeys + { + "ms": f"or_i(and_b(pk({TPUBS[0]}/*),a:and_b(pk({TPUBS[1]}),a:and_b(pk({TPUBS[2]}),a:and_b(pk({TPUBS[3]}),s:pk({PUBKEYS[0]}))))),and_v(v:thresh(2,pkh({TPRVS[0]}),a:pkh({TPRVS[1]}),a:pkh({TPUBS[4]})),older(42)))", + "sequence": 42, + "locktime": None, + "sigs_count": 2, + "stack_size": 8, + }, ] @@ -62,7 +180,77 @@ class WalletMiniscriptTest(BitcoinTestFramework): lambda: len(self.ms_wo_wallet.listunspent(minconf=0, addresses=[addr])) == 1 ) utxo = self.ms_wo_wallet.listunspent(minconf=0, addresses=[addr])[0] - assert utxo["txid"] == txid and not utxo["solvable"] # No satisfaction logic (yet) + assert utxo["txid"] == txid and utxo["solvable"] + + def signing_test( + self, ms, sequence, locktime, sigs_count, stack_size, sha256_preimages + ): + self.log.info(f"Importing private Miniscript '{ms}'") + desc = descsum_create(f"wsh({ms})") + res = self.ms_sig_wallet.importdescriptors( + [ + { + "desc": desc, + "active": True, + "range": 0, + "next_index": 0, + "timestamp": "now", + } + ] + ) + assert res[0]["success"], res + + self.log.info("Generating an address for it and testing it detects funds") + addr = self.ms_sig_wallet.getnewaddress() + txid = self.funder.sendtoaddress(addr, 0.01) + self.wait_until(lambda: txid in self.funder.getrawmempool()) + self.funder.generatetoaddress(1, self.funder.getnewaddress()) + utxo = self.ms_sig_wallet.listunspent(addresses=[addr])[0] + assert txid == utxo["txid"] and utxo["solvable"] + + self.log.info("Creating a transaction spending these funds") + dest_addr = self.funder.getnewaddress() + seq = sequence if sequence is not None else 0xFFFFFFFF - 2 + lt = locktime if locktime is not None else 0 + psbt = self.ms_sig_wallet.createpsbt( + [ + { + "txid": txid, + "vout": utxo["vout"], + "sequence": seq, + } + ], + [{dest_addr: 0.009}], + lt, + ) + + self.log.info("Signing it and checking the satisfaction.") + if sha256_preimages is not None: + psbt = PSBT.from_base64(psbt) + for (h, preimage) in sha256_preimages.items(): + k = PSBT_IN_SHA256.to_bytes(1, "big") + bytes.fromhex(h) + psbt.i[0].map[k] = bytes.fromhex(preimage) + psbt = psbt.to_base64() + res = self.ms_sig_wallet.walletprocesspsbt(psbt=psbt, finalize=False) + psbtin = self.nodes[0].rpc.decodepsbt(res["psbt"])["inputs"][0] + assert len(psbtin["partial_signatures"]) == sigs_count + res = self.ms_sig_wallet.finalizepsbt(res["psbt"]) + assert res["complete"] == (stack_size is not None) + + if stack_size is not None: + txin = self.nodes[0].rpc.decoderawtransaction(res["hex"])["vin"][0] + assert len(txin["txinwitness"]) == stack_size, txin["txinwitness"] + self.log.info("Broadcasting the transaction.") + # If necessary, satisfy a relative timelock + if sequence is not None: + self.funder.generatetoaddress(sequence, self.funder.getnewaddress()) + # If necessary, satisfy an absolute timelock + height = self.funder.getblockcount() + if locktime is not None and height < locktime: + self.funder.generatetoaddress( + locktime - height, self.funder.getnewaddress() + ) + self.ms_sig_wallet.sendrawtransaction(res["hex"]) def run_test(self): self.log.info("Making a descriptor wallet") @@ -71,6 +259,8 @@ class WalletMiniscriptTest(BitcoinTestFramework): wallet_name="ms_wo", descriptors=True, disable_private_keys=True ) self.ms_wo_wallet = self.nodes[0].get_wallet_rpc("ms_wo") + self.nodes[0].createwallet(wallet_name="ms_sig", descriptors=True) + self.ms_sig_wallet = self.nodes[0].get_wallet_rpc("ms_sig") # Sanity check we wouldn't let an insane Miniscript descriptor in res = self.ms_wo_wallet.importdescriptors( @@ -91,6 +281,17 @@ class WalletMiniscriptTest(BitcoinTestFramework): for ms in MINISCRIPTS: self.watchonly_test(ms) + # Test we can sign for any Miniscript. + for ms in MINISCRIPTS_PRIV: + self.signing_test( + ms["ms"], + ms["sequence"], + ms["locktime"], + ms["sigs_count"], + ms["stack_size"], + ms.get("sha256_preimages"), + ) + if __name__ == "__main__": WalletMiniscriptTest().main() diff --git a/test/functional/wallet_orphanedreward.py b/test/functional/wallet_orphanedreward.py index d9f7c14ded..d8931fa620 100755 --- a/test/functional/wallet_orphanedreward.py +++ b/test/functional/wallet_orphanedreward.py @@ -34,29 +34,40 @@ class OrphanedBlockRewardTest(BitcoinTestFramework): # the existing balance and the block reward. self.generate(self.nodes[0], 150) assert_equal(self.nodes[1].getbalance(), 10 + 25) + pre_reorg_conf_bals = self.nodes[1].getbalances() txid = self.nodes[1].sendtoaddress(self.nodes[0].getnewaddress(), 30) + orig_chain_tip = self.nodes[0].getbestblockhash() + self.sync_mempools() # Orphan the block reward and make sure that the original coins # from the wallet can still be spent. self.nodes[0].invalidateblock(blk) - self.generate(self.nodes[0], 152) - # Without the following abandontransaction call, the coins are - # not considered available yet. - assert_equal(self.nodes[1].getbalances()["mine"], { - "trusted": 0, - "untrusted_pending": 0, - "immature": 0, - }) - # The following abandontransaction is necessary to make the later - # lines succeed, and probably should not be needed; see - # https://github.com/bitcoin/bitcoin/issues/14148. - self.nodes[1].abandontransaction(txid) + blocks = self.generate(self.nodes[0], 152) + conflict_block = blocks[0] + # We expect the descendants of orphaned rewards to no longer be considered assert_equal(self.nodes[1].getbalances()["mine"], { "trusted": 10, "untrusted_pending": 0, "immature": 0, }) - self.nodes[1].sendtoaddress(self.nodes[0].getnewaddress(), 9) + # And the unconfirmed tx to be abandoned + assert_equal(self.nodes[1].gettransaction(txid)["details"][0]["abandoned"], True) + + # The abandoning should persist through reloading + self.nodes[1].unloadwallet(self.default_wallet_name) + self.nodes[1].loadwallet(self.default_wallet_name) + assert_equal(self.nodes[1].gettransaction(txid)["details"][0]["abandoned"], True) + + # If the orphaned reward is reorged back into the main chain, any unconfirmed + # descendant txs at the time of the original reorg remain abandoned. + self.nodes[0].invalidateblock(conflict_block) + self.nodes[0].reconsiderblock(blk) + assert_equal(self.nodes[0].getbestblockhash(), orig_chain_tip) + self.generate(self.nodes[0], 3) + + assert_equal(self.nodes[1].getbalances(), pre_reorg_conf_bals) + assert_equal(self.nodes[1].gettransaction(txid)["details"][0]["abandoned"], True) + if __name__ == '__main__': OrphanedBlockRewardTest().main() diff --git a/test/functional/wallet_pruning.py b/test/functional/wallet_pruning.py index 6d8475ce8d..1ceceaee93 100755 --- a/test/functional/wallet_pruning.py +++ b/test/functional/wallet_pruning.py @@ -39,11 +39,15 @@ class WalletPruningTest(BitcoinTestFramework): def mine_large_blocks(self, node, n): # Get the block parameters for the first block - best_block = node.getblock(node.getbestblockhash()) + best_block = node.getblockheader(node.getbestblockhash()) height = int(best_block["height"]) + 1 self.nTime = max(self.nTime, int(best_block["time"])) + 1 previousblockhash = int(best_block["hash"], 16) big_script = CScript([OP_RETURN] + [OP_TRUE] * 950000) + # Set mocktime to accept all future blocks + for i in self.nodes: + if i.running: + i.setmocktime(self.nTime + 600 * n) for _ in range(n): block = create_block(hashprev=previousblockhash, ntime=self.nTime, coinbase=create_coinbase(height, script_pubkey=big_script)) block.solve() @@ -57,9 +61,6 @@ class WalletPruningTest(BitcoinTestFramework): # Simulate 10 minutes of work time per block # Important for matching a timestamp with a block +- some window self.nTime += 600 - for n in self.nodes: - if n.running: - n.setmocktime(self.nTime) # Update node's time to accept future blocks self.sync_all() def test_wallet_import_pruned(self, wallet_name): @@ -122,7 +123,7 @@ class WalletPruningTest(BitcoinTestFramework): # A blk*.dat file is 128MB # Generate 250 light blocks - self.generate(self.nodes[0], 250, sync_fun=self.no_op) + self.generate(self.nodes[0], 250) # Generate 50MB worth of large blocks in the blk00000.dat file self.mine_large_blocks(self.nodes[0], 50) diff --git a/test/functional/wallet_send.py b/test/functional/wallet_send.py index 424834323f..ac3ec06eec 100755 --- a/test/functional/wallet_send.py +++ b/test/functional/wallet_send.py @@ -45,7 +45,7 @@ class WalletSendTest(BitcoinTestFramework): conf_target=None, estimate_mode=None, fee_rate=None, add_to_wallet=None, psbt=None, inputs=None, add_inputs=None, include_unsafe=None, change_address=None, change_position=None, change_type=None, include_watching=None, locktime=None, lock_unspents=None, replaceable=None, subtract_fee_from_outputs=None, - expect_error=None, solving_data=None): + expect_error=None, solving_data=None, minconf=None): assert (amount is None) != (data is None) from_balance_before = from_wallet.getbalances()["mine"]["trusted"] @@ -106,6 +106,8 @@ class WalletSendTest(BitcoinTestFramework): options["subtract_fee_from_outputs"] = subtract_fee_from_outputs if solving_data is not None: options["solving_data"] = solving_data + if minconf is not None: + options["minconf"] = minconf if len(options.keys()) == 0: options = None @@ -487,6 +489,16 @@ class WalletSendTest(BitcoinTestFramework): res = self.test_send(from_wallet=w5, to_wallet=w0, amount=1, include_unsafe=True) assert res["complete"] + self.log.info("Minconf") + self.nodes[1].createwallet(wallet_name="minconfw") + minconfw= self.nodes[1].get_wallet_rpc("minconfw") + self.test_send(from_wallet=w0, to_wallet=minconfw, amount=2) + self.generate(self.nodes[0], 3) + self.test_send(from_wallet=minconfw, to_wallet=w0, amount=1, minconf=4, expect_error=(-4, "Insufficient funds")) + self.test_send(from_wallet=minconfw, to_wallet=w0, amount=1, minconf=-4, expect_error=(-8, "Negative minconf")) + res = self.test_send(from_wallet=minconfw, to_wallet=w0, amount=1, minconf=3) + assert res["complete"] + self.log.info("External outputs") eckey = ECKey() eckey.generate() diff --git a/test/functional/wallet_sendall.py b/test/functional/wallet_sendall.py index 778c8a5b9e..f6440f07d7 100755 --- a/test/functional/wallet_sendall.py +++ b/test/functional/wallet_sendall.py @@ -317,6 +317,68 @@ class SendallTest(BitcoinTestFramework): assert_equal(decoded["tx"]["vin"][0]["vout"], utxo["vout"]) assert_equal(decoded["tx"]["vout"][0]["scriptPubKey"]["address"], self.remainder_target) + @cleanup + def sendall_with_minconf(self): + # utxo of 17 bicoin has 6 confirmations, utxo of 4 has 3 + self.add_utxos([17]) + self.generate(self.nodes[0], 2) + self.add_utxos([4]) + self.generate(self.nodes[0], 2) + + self.log.info("Test sendall fails because minconf is negative") + + assert_raises_rpc_error(-8, + "Invalid minconf (minconf cannot be negative): -2", + self.wallet.sendall, + recipients=[self.remainder_target], + options={"minconf": -2}) + self.log.info("Test sendall fails because minconf is used while specific inputs are provided") + + utxo = self.wallet.listunspent()[0] + assert_raises_rpc_error(-8, + "Cannot combine minconf or maxconf with specific inputs.", + self.wallet.sendall, + recipients=[self.remainder_target], + options={"inputs": [utxo], "minconf": 2}) + + self.log.info("Test sendall fails because there are no utxos with enough confirmations specified by minconf") + + assert_raises_rpc_error(-6, + "Total value of UTXO pool too low to pay for transaction. Try using lower feerate or excluding uneconomic UTXOs with 'send_max' option.", + self.wallet.sendall, + recipients=[self.remainder_target], + options={"minconf": 7}) + + self.log.info("Test sendall only spends utxos with a specified number of confirmations when minconf is used") + self.wallet.sendall(recipients=[self.remainder_target], fee_rate=300, options={"minconf": 6}) + + assert_equal(len(self.wallet.listunspent()), 1) + assert_equal(self.wallet.listunspent()[0]['confirmations'], 3) + + # decrease minconf and show the remaining utxo is picked up + self.wallet.sendall(recipients=[self.remainder_target], fee_rate=300, options={"minconf": 3}) + assert_equal(self.wallet.getbalance(), 0) + + @cleanup + def sendall_with_maxconf(self): + # utxo of 17 bicoin has 6 confirmations, utxo of 4 has 3 + self.add_utxos([17]) + self.generate(self.nodes[0], 2) + self.add_utxos([4]) + self.generate(self.nodes[0], 2) + + self.log.info("Test sendall fails because there are no utxos with enough confirmations specified by maxconf") + assert_raises_rpc_error(-6, + "Total value of UTXO pool too low to pay for transaction. Try using lower feerate or excluding uneconomic UTXOs with 'send_max' option.", + self.wallet.sendall, + recipients=[self.remainder_target], + options={"maxconf": 1}) + + self.log.info("Test sendall only spends utxos with a specified number of confirmations when maxconf is used") + self.wallet.sendall(recipients=[self.remainder_target], fee_rate=300, options={"maxconf":4}) + assert_equal(len(self.wallet.listunspent()), 1) + assert_equal(self.wallet.listunspent()[0]['confirmations'], 6) + # This tests needs to be the last one otherwise @cleanup will fail with "Transaction too large" error def sendall_fails_with_transaction_too_large(self): self.log.info("Test that sendall fails if resulting transaction is too large") @@ -392,6 +454,12 @@ class SendallTest(BitcoinTestFramework): # Sendall succeeds with watchonly wallets spending specific UTXOs self.sendall_watchonly_specific_inputs() + # Sendall only uses outputs with at least a give number of confirmations when using minconf + self.sendall_with_minconf() + + # Sendall only uses outputs with less than a given number of confirmation when using minconf + self.sendall_with_maxconf() + # Sendall fails when many inputs result to too large transaction self.sendall_fails_with_transaction_too_large() diff --git a/test/functional/wallet_transactiontime_rescan.py b/test/functional/wallet_transactiontime_rescan.py index de9616b4a1..ea99992084 100755 --- a/test/functional/wallet_transactiontime_rescan.py +++ b/test/functional/wallet_transactiontime_rescan.py @@ -5,8 +5,10 @@ """Test transaction time during old block rescanning """ +import concurrent.futures import time +from test_framework.authproxy import JSONRPCException from test_framework.blocktools import COINBASE_MATURITY from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( @@ -14,6 +16,9 @@ from test_framework.util import ( assert_raises_rpc_error, set_node_times, ) +from test_framework.wallet_util import ( + get_generate_key, +) class TransactionTimeRescanTest(BitcoinTestFramework): @@ -23,6 +28,10 @@ class TransactionTimeRescanTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = False self.num_nodes = 3 + self.extra_args = [["-keypool=400"], + ["-keypool=400"], + [] + ] def skip_test_if_missing_module(self): self.skip_if_no_wallet() @@ -167,6 +176,51 @@ class TransactionTimeRescanTest(BitcoinTestFramework): assert_raises_rpc_error(-8, "Invalid stop_height", restorewo_wallet.rescanblockchain, 1, -1) assert_raises_rpc_error(-8, "stop_height must be greater than start_height", restorewo_wallet.rescanblockchain, 20, 10) + self.log.info("Test `rescanblockchain` fails when wallet is encrypted and locked") + usernode.createwallet(wallet_name="enc_wallet", passphrase="passphrase") + enc_wallet = usernode.get_wallet_rpc("enc_wallet") + assert_raises_rpc_error(-13, "Error: Please enter the wallet passphrase with walletpassphrase first.", enc_wallet.rescanblockchain) + + if not self.options.descriptors: + self.log.info("Test rescanning an encrypted wallet") + hd_seed = get_generate_key().privkey + + usernode.createwallet(wallet_name="temp_wallet", blank=True, descriptors=False) + temp_wallet = usernode.get_wallet_rpc("temp_wallet") + temp_wallet.sethdseed(seed=hd_seed) + + for i in range(399): + temp_wallet.getnewaddress() + + self.generatetoaddress(usernode, COINBASE_MATURITY + 1, temp_wallet.getnewaddress()) + self.generatetoaddress(usernode, COINBASE_MATURITY + 1, temp_wallet.getnewaddress()) + + minernode.createwallet("encrypted_wallet", blank=True, passphrase="passphrase", descriptors=False) + encrypted_wallet = minernode.get_wallet_rpc("encrypted_wallet") + + encrypted_wallet.walletpassphrase("passphrase", 99999) + encrypted_wallet.sethdseed(seed=hd_seed) + + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as thread: + with minernode.assert_debug_log(expected_msgs=["Rescan started from block 0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206... (slow variant inspecting all blocks)"], timeout=5): + rescanning = thread.submit(encrypted_wallet.rescanblockchain) + + # set the passphrase timeout to 1 to test that the wallet remains unlocked during the rescan + minernode.cli("-rpcwallet=encrypted_wallet").walletpassphrase("passphrase", 1) + + try: + minernode.cli("-rpcwallet=encrypted_wallet").walletlock() + except JSONRPCException as e: + assert e.error["code"] == -4 and "Error: the wallet is currently being used to rescan the blockchain for related transactions. Please call `abortrescan` before locking the wallet." in e.error["message"] + + try: + minernode.cli("-rpcwallet=encrypted_wallet").walletpassphrasechange("passphrase", "newpassphrase") + except JSONRPCException as e: + assert e.error["code"] == -4 and "Error: the wallet is currently being used to rescan the blockchain for related transactions. Please call `abortrescan` before changing the passphrase." in e.error["message"] + + assert_equal(rescanning.result(), {"start_height": 0, "stop_height": 803}) + + assert_equal(encrypted_wallet.getbalance(), temp_wallet.getbalance()) if __name__ == '__main__': TransactionTimeRescanTest().main() diff --git a/test/fuzz/test_runner.py b/test/fuzz/test_runner.py index e2eab2a0fe..af21e7b956 100755 --- a/test/fuzz/test_runner.py +++ b/test/fuzz/test_runner.py @@ -143,7 +143,7 @@ def main(): timeout=20, check=True, stderr=subprocess.PIPE, - universal_newlines=True, + text=True, ).stderr if "libFuzzer" not in help_output: logging.error("Must be built with libFuzzer") @@ -200,7 +200,7 @@ def generate_corpus(*, fuzz_pool, src_dir, build_dir, corpus_dir, targets): env=get_fuzz_env(target=t, source_dir=src_dir), check=True, stderr=subprocess.PIPE, - universal_newlines=True, + text=True, ).stderr)) futures = [] @@ -241,7 +241,7 @@ def merge_inputs(*, fuzz_pool, corpus, test_list, src_dir, build_dir, merge_dir) env=get_fuzz_env(target=t, source_dir=src_dir), check=True, stderr=subprocess.PIPE, - universal_newlines=True, + text=True, ).stderr logging.debug(output) @@ -270,7 +270,7 @@ def run_once(*, fuzz_pool, corpus, test_list, src_dir, build_dir, use_valgrind): args, env=get_fuzz_env(target=t, source_dir=src_dir), stderr=subprocess.PIPE, - universal_newlines=True, + text=True, ) output += result.stderr return output, result @@ -299,7 +299,7 @@ def parse_test_list(*, fuzz_bin): }, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, - universal_newlines=True, + text=True, ).stdout.splitlines() return test_list_all diff --git a/test/get_previous_releases.py b/test/get_previous_releases.py index 7f5f15655c..60c868ca04 100755 --- a/test/get_previous_releases.py +++ b/test/get_previous_releases.py @@ -80,6 +80,15 @@ SHA256_SUMS = { "078f96b1e92895009c798ab827fb3fde5f6719eee886bd0c0e93acab18ea4865": {"tag": "v23.0", "tarball": "bitcoin-23.0-riscv64-linux-gnu.tar.gz"}, "c816780583009a9dad426dc0c183c89be9da98906e1e2c7ebae91041c1aaaaf3": {"tag": "v23.0", "tarball": "bitcoin-23.0-x86_64-apple-darwin.tar.gz"}, "2cca490c1f2842884a3c5b0606f179f9f937177da4eadd628e3f7fd7e25d26d0": {"tag": "v23.0", "tarball": "bitcoin-23.0-x86_64-linux-gnu.tar.gz"}, + + "0b48b9e69b30037b41a1e6b78fb7cbcc48c7ad627908c99686e81f3802454609": {"tag": "v24.0.1", "tarball": "bitcoin-24.0.1-aarch64-linux-gnu.tar.gz"}, + "37d7660f0277301744e96426bbb001d2206b8d4505385dfdeedf50c09aaaef60": {"tag": "v24.0.1", "tarball": "bitcoin-24.0.1-arm-linux-gnueabihf.tar.gz"}, + "90ed59e86bfda1256f4b4cad8cc1dd77ee0efec2492bcb5af61402709288b62c": {"tag": "v24.0.1", "tarball": "bitcoin-24.0.1-arm64-apple-darwin.tar.gz"}, + "7590645e8676f8b5fda62dc20174474c4ac8fd0defc83a19ed908ebf2e94dc11": {"tag": "v24.0.1", "tarball": "bitcoin-24.0.1-powerpc64-linux-gnu.tar.gz"}, + "79e89a101f23ff87816675b98769cd1ee91059f95c5277f38f48f21a9f7f8509": {"tag": "v24.0.1", "tarball": "bitcoin-24.0.1-powerpc64le-linux-gnu.tar.gz"}, + "6b163cef7de4beb07b8cb3347095e0d76a584019b1891135cd1268a1f05b9d88": {"tag": "v24.0.1", "tarball": "bitcoin-24.0.1-riscv64-linux-gnu.tar.gz"}, + "e2f751512f3c0f00eb68ba946d9c829e6cf99422a61e8f5e0a7c109c318674d0": {"tag": "v24.0.1", "tarball": "bitcoin-24.0.1-x86_64-apple-darwin.tar.gz"}, + "49df6e444515d457ea0b885d66f521f2a26ca92ccf73d5296082e633544253bf": {"tag": "v24.0.1", "tarball": "bitcoin-24.0.1-x86_64-linux-gnu.tar.gz"}, } diff --git a/test/lint/README.md b/test/lint/README.md index 8d592c3282..704922d7ab 100644 --- a/test/lint/README.md +++ b/test/lint/README.md @@ -1,5 +1,23 @@ This folder contains lint scripts. +Running locally +=============== + +To run linters locally with the same versions as the CI environment, use the included +Dockerfile: + +```sh +cd ./ci/lint +docker build -t bitcoin-linter . + +cd /root/of/bitcoin/repo +docker run --rm -v $(pwd):/bitcoin -it bitcoin-linter +``` + +After building the container once, you can simply run the last command any time you +want to lint. + + check-doc.py ============ Check for missing documentation of command line options. diff --git a/test/lint/lint-assertions.py b/test/lint/lint-assertions.py index 195ff33d11..e7eecebce5 100755 --- a/test/lint/lint-assertions.py +++ b/test/lint/lint-assertions.py @@ -12,7 +12,7 @@ import subprocess def git_grep(params: [], error_msg: ""): try: - output = subprocess.check_output(["git", "grep", *params], universal_newlines=True, encoding="utf8") + output = subprocess.check_output(["git", "grep", *params], text=True, encoding="utf8") print(error_msg) print(output) return 1 diff --git a/test/lint/lint-circular-dependencies.py b/test/lint/lint-circular-dependencies.py index b69bbe7cd0..307b4dca5a 100755 --- a/test/lint/lint-circular-dependencies.py +++ b/test/lint/lint-circular-dependencies.py @@ -12,7 +12,7 @@ import subprocess import sys EXPECTED_CIRCULAR_DEPENDENCIES = ( - "chainparamsbase -> util/system -> chainparamsbase", + "chainparamsbase -> common/args -> chainparamsbase", "node/blockstorage -> validation -> node/blockstorage", "node/utxo_snapshot -> validation -> node/utxo_snapshot", "qt/addresstablemodel -> qt/walletmodel -> qt/addresstablemodel", @@ -38,14 +38,14 @@ def main(): os.chdir(CODE_DIR) files = subprocess.check_output( ['git', 'ls-files', '--', '*.h', '*.cpp'], - universal_newlines=True, + text=True, ).splitlines() command = [sys.executable, "../contrib/devtools/circular-dependencies.py", *files] dependencies_output = subprocess.run( command, stdout=subprocess.PIPE, - universal_newlines=True, + text=True, ) for dependency_str in dependencies_output.stdout.rstrip().split("\n"): diff --git a/test/lint/lint-git-commit-check.py b/test/lint/lint-git-commit-check.py index 049104398a..5897a17e70 100755 --- a/test/lint/lint-git-commit-check.py +++ b/test/lint/lint-git-commit-check.py @@ -42,17 +42,17 @@ def main(): commit_range = "HEAD~" + args.prev_commits + "...HEAD" else: # This assumes that the target branch of the pull request will be master. - merge_base = check_output(["git", "merge-base", "HEAD", "master"], universal_newlines=True, encoding="utf8").rstrip("\n") + merge_base = check_output(["git", "merge-base", "HEAD", "master"], text=True, encoding="utf8").rstrip("\n") commit_range = merge_base + "..HEAD" else: commit_range = os.getenv("COMMIT_RANGE") if commit_range == "SKIP_EMPTY_NOT_A_PR": sys.exit(0) - commit_hashes = check_output(["git", "log", commit_range, "--format=%H"], universal_newlines=True, encoding="utf8").splitlines() + commit_hashes = check_output(["git", "log", commit_range, "--format=%H"], text=True, encoding="utf8").splitlines() for hash in commit_hashes: - commit_info = check_output(["git", "log", "--format=%B", "-n", "1", hash], universal_newlines=True, encoding="utf8").splitlines() + commit_info = check_output(["git", "log", "--format=%B", "-n", "1", hash], text=True, encoding="utf8").splitlines() if len(commit_info) >= 2: if commit_info[1]: print(f"The subject line of commit hash {hash} is followed by a non-empty line. Subject lines should always be followed by a blank line.") diff --git a/test/lint/lint-includes.py b/test/lint/lint-includes.py index b3fa4b9303..459030bb0b 100755 --- a/test/lint/lint-includes.py +++ b/test/lint/lint-includes.py @@ -35,13 +35,13 @@ EXPECTED_BOOST_INCLUDES = ["boost/date_time/posix_time/posix_time.hpp", def get_toplevel(): - return check_output(["git", "rev-parse", "--show-toplevel"], universal_newlines=True, encoding="utf8").rstrip("\n") + return check_output(["git", "rev-parse", "--show-toplevel"], text=True, encoding="utf8").rstrip("\n") def list_files_by_suffix(suffixes): exclude_args = [":(exclude)" + dir for dir in EXCLUDED_DIRS] - files_list = check_output(["git", "ls-files", "src"] + exclude_args, universal_newlines=True, encoding="utf8").splitlines() + files_list = check_output(["git", "ls-files", "src"] + exclude_args, text=True, encoding="utf8").splitlines() return [file for file in files_list if file.endswith(suffixes)] @@ -63,7 +63,7 @@ def find_included_cpps(): included_cpps = list() try: - included_cpps = check_output(["git", "grep", "-E", r"^#include [<\"][^>\"]+\.cpp[>\"]", "--", "*.cpp", "*.h"], universal_newlines=True, encoding="utf8").splitlines() + included_cpps = check_output(["git", "grep", "-E", r"^#include [<\"][^>\"]+\.cpp[>\"]", "--", "*.cpp", "*.h"], text=True, encoding="utf8").splitlines() except CalledProcessError as e: if e.returncode > 1: raise e @@ -77,7 +77,7 @@ def find_extra_boosts(): exclusion_set = set() try: - included_boosts = check_output(["git", "grep", "-E", r"^#include <boost/", "--", "*.cpp", "*.h"], universal_newlines=True, encoding="utf8").splitlines() + included_boosts = check_output(["git", "grep", "-E", r"^#include <boost/", "--", "*.cpp", "*.h"], text=True, encoding="utf8").splitlines() except CalledProcessError as e: if e.returncode > 1: raise e @@ -100,7 +100,7 @@ def find_quote_syntax_inclusions(): quote_syntax_inclusions = list() try: - quote_syntax_inclusions = check_output(["git", "grep", r"^#include \"", "--", "*.cpp", "*.h"] + exclude_args, universal_newlines=True, encoding="utf8").splitlines() + quote_syntax_inclusions = check_output(["git", "grep", r"^#include \"", "--", "*.cpp", "*.h"] + exclude_args, text=True, encoding="utf8").splitlines() except CalledProcessError as e: if e.returncode > 1: raise e @@ -143,13 +143,13 @@ def main(): if extra_boosts: for boost in extra_boosts: print(f"A new Boost dependency in the form of \"{boost}\" appears to have been introduced:") - print(check_output(["git", "grep", boost, "--", "*.cpp", "*.h"], universal_newlines=True, encoding="utf8")) + print(check_output(["git", "grep", boost, "--", "*.cpp", "*.h"], text=True, encoding="utf8")) exit_code = 1 # Check if Boost dependencies are no longer used for expected_boost in EXPECTED_BOOST_INCLUDES: try: - check_output(["git", "grep", "-q", r"^#include <%s>" % expected_boost, "--", "*.cpp", "*.h"], universal_newlines=True, encoding="utf8") + check_output(["git", "grep", "-q", r"^#include <%s>" % expected_boost, "--", "*.cpp", "*.h"], text=True, encoding="utf8") except CalledProcessError as e: if e.returncode > 1: raise e diff --git a/test/lint/lint-locale-dependence.py b/test/lint/lint-locale-dependence.py index ce7444cd1a..d84e458bb1 100755 --- a/test/lint/lint-locale-dependence.py +++ b/test/lint/lint-locale-dependence.py @@ -34,8 +34,6 @@ # # See https://doc.qt.io/qt-5/qcoreapplication.html#locale-settings and # https://stackoverflow.com/a/34878283 for more details. -# -# TODO: Reduce KNOWN_VIOLATIONS by replacing uses of locale dependent snprintf with strprintf. import re import sys @@ -45,10 +43,7 @@ from subprocess import check_output, CalledProcessError KNOWN_VIOLATIONS = [ "src/dbwrapper.cpp:.*vsnprintf", - "src/test/dbwrapper_tests.cpp:.*snprintf", "src/test/fuzz/locale.cpp:.*setlocale", - "src/test/fuzz/string.cpp:.*strtol", - "src/test/fuzz/string.cpp:.*strtoul", "src/test/util_tests.cpp:.*strtoll", "src/wallet/bdb.cpp:.*DbEnv::strerror", # False positive "src/util/syserror.cpp:.*strerror", # Outside this function use `SysErrorString` @@ -223,7 +218,7 @@ def find_locale_dependent_function_uses(): git_grep_output = list() try: - git_grep_output = check_output(git_grep_command, universal_newlines=True, encoding="utf8").splitlines() + git_grep_output = check_output(git_grep_command, text=True, encoding="utf8").splitlines() except CalledProcessError as e: if e.returncode > 1: raise e diff --git a/test/lint/lint-logs.py b/test/lint/lint-logs.py index aaf697467d..de04a1aeca 100755 --- a/test/lint/lint-logs.py +++ b/test/lint/lint-logs.py @@ -16,7 +16,7 @@ from subprocess import check_output def main(): - logs_list = check_output(["git", "grep", "--extended-regexp", r"(LogPrintLevel|LogPrintfCategory|LogPrintf?)\(", "--", "*.cpp"], universal_newlines=True, encoding="utf8").splitlines() + logs_list = check_output(["git", "grep", "--extended-regexp", r"(LogPrintLevel|LogPrintfCategory|LogPrintf?)\(", "--", "*.cpp"], text=True, encoding="utf8").splitlines() unterminated_logs = [line for line in logs_list if not re.search(r'(\\n"|/\* Continued \*/)', line)] diff --git a/test/lint/lint-python-mutable-default-parameters.py b/test/lint/lint-python-mutable-default-parameters.py index 3dfc5940f7..820595ea34 100755 --- a/test/lint/lint-python-mutable-default-parameters.py +++ b/test/lint/lint-python-mutable-default-parameters.py @@ -21,7 +21,7 @@ def main(): "--", "*.py", ] - output = subprocess.run(command, stdout=subprocess.PIPE, universal_newlines=True) + output = subprocess.run(command, stdout=subprocess.PIPE, text=True) if len(output.stdout) > 0: error_msg = ( "A mutable list or dict seems to be used as default parameter value:\n\n" diff --git a/test/lint/lint-python-utf8-encoding.py b/test/lint/lint-python-utf8-encoding.py index 62fdc34d50..64d04bff57 100755 --- a/test/lint/lint-python-utf8-encoding.py +++ b/test/lint/lint-python-utf8-encoding.py @@ -12,7 +12,7 @@ import re from subprocess import check_output, CalledProcessError -EXCLUDED_DIRS = ["src/crc32c/"] +EXCLUDED_DIRS = ["src/crc32c/", "src/secp256k1/"] def get_exclude_args(): @@ -23,7 +23,7 @@ def check_fileopens(): fileopens = list() try: - fileopens = check_output(["git", "grep", r" open(", "--", "*.py"] + get_exclude_args(), universal_newlines=True, encoding="utf8").splitlines() + fileopens = check_output(["git", "grep", r" open(", "--", "*.py"] + get_exclude_args(), text=True, encoding="utf8").splitlines() except CalledProcessError as e: if e.returncode > 1: raise e @@ -37,12 +37,12 @@ def check_checked_outputs(): checked_outputs = list() try: - checked_outputs = check_output(["git", "grep", "check_output(", "--", "*.py"] + get_exclude_args(), universal_newlines=True, encoding="utf8").splitlines() + checked_outputs = check_output(["git", "grep", "check_output(", "--", "*.py"] + get_exclude_args(), text=True, encoding="utf8").splitlines() except CalledProcessError as e: if e.returncode > 1: raise e - filtered_checked_outputs = [checked_output for checked_output in checked_outputs if re.search(r"universal_newlines=True", checked_output) and not re.search(r"encoding=.(ascii|utf8|utf-8).", checked_output)] + filtered_checked_outputs = [checked_output for checked_output in checked_outputs if re.search(r"text=True", checked_output) and not re.search(r"encoding=.(ascii|utf8|utf-8).", checked_output)] return filtered_checked_outputs diff --git a/test/lint/lint-python.py b/test/lint/lint-python.py index 4d16facfea..9de13e44e2 100755 --- a/test/lint/lint-python.py +++ b/test/lint/lint-python.py @@ -15,7 +15,13 @@ import sys DEPS = ['flake8', 'mypy', 'pyzmq'] MYPY_CACHE_DIR = f"{os.getenv('BASE_ROOT_DIR', '')}/test/.mypy_cache" -FILES_ARGS = ['git', 'ls-files', 'test/functional/*.py', 'contrib/devtools/*.py'] + +# All .py files, except those in src/ (to exclude subtrees there) +FLAKE_FILES_ARGS = ['git', 'ls-files', '*.py', ':!:src/*.py'] + +# Only .py files in test/functional and contrib/devtools have type annotations +# enforced. +MYPY_FILES_ARGS = ['git', 'ls-files', 'test/functional/*.py', 'contrib/devtools/*.py'] ENABLED = ( 'E101,' # indentation contains mixed spaces and tabs @@ -47,6 +53,7 @@ ENABLED = ( 'E711,' # comparison to None should be 'if cond is None:' 'E714,' # test for object identity should be "is not" 'E721,' # do not compare types, use "isinstance()" + 'E722,' # do not use bare 'except' 'E742,' # do not define classes named "l", "O", or "I" 'E743,' # do not define functions named "l", "O", or "I" 'E901,' # SyntaxError: invalid syntax @@ -106,8 +113,7 @@ def main(): if len(sys.argv) > 1: flake8_files = sys.argv[1:] else: - files_args = ['git', 'ls-files', '*.py'] - flake8_files = subprocess.check_output(files_args).decode("utf-8").splitlines() + flake8_files = subprocess.check_output(FLAKE_FILES_ARGS).decode("utf-8").splitlines() flake8_args = ['flake8', '--ignore=B,C,E,F,I,N,W', f'--select={ENABLED}'] + flake8_files flake8_env = os.environ.copy() @@ -118,7 +124,7 @@ def main(): except subprocess.CalledProcessError: exit(1) - mypy_files = subprocess.check_output(FILES_ARGS).decode("utf-8").splitlines() + mypy_files = subprocess.check_output(MYPY_FILES_ARGS).decode("utf-8").splitlines() mypy_args = ['mypy', '--show-error-codes'] + mypy_files try: diff --git a/test/lint/lint-shell.py b/test/lint/lint-shell.py index ed95024ef5..1646bf0d3e 100755 --- a/test/lint/lint-shell.py +++ b/test/lint/lint-shell.py @@ -25,7 +25,7 @@ def check_shellcheck_install(): sys.exit(0) def get_files(command): - output = subprocess.run(command, stdout=subprocess.PIPE, universal_newlines=True) + output = subprocess.run(command, stdout=subprocess.PIPE, text=True) files = output.stdout.split('\n') # remove whitespace element diff --git a/test/lint/lint-submodule.py b/test/lint/lint-submodule.py index 89d4c80f55..4d2fbf088f 100755 --- a/test/lint/lint-submodule.py +++ b/test/lint/lint-submodule.py @@ -13,7 +13,7 @@ import sys def main(): submodules_list = subprocess.check_output(['git', 'submodule', 'status', '--recursive'], - universal_newlines = True, encoding = 'utf8').rstrip('\n') + text = True, encoding = 'utf8').rstrip('\n') if submodules_list: print("These submodules were found, delete them:\n", submodules_list) sys.exit(1) diff --git a/test/lint/lint-tests.py b/test/lint/lint-tests.py index 849ddcb961..1eeb7bb014 100755 --- a/test/lint/lint-tests.py +++ b/test/lint/lint-tests.py @@ -23,7 +23,7 @@ def grep_boost_fixture_test_suite(): "src/test/**.cpp", "src/wallet/test/**.cpp", ] - return subprocess.check_output(command, universal_newlines=True, encoding="utf8") + return subprocess.check_output(command, text=True, encoding="utf8") def check_matching_test_names(test_suite_list): diff --git a/test/lint/lint-whitespace.py b/test/lint/lint-whitespace.py index 72b7ebc394..f5e4a776d0 100755 --- a/test/lint/lint-whitespace.py +++ b/test/lint/lint-whitespace.py @@ -80,7 +80,7 @@ def get_diff(commit_range, check_only_code): else: what_files = ["."] - diff = check_output(["git", "diff", "-U0", commit_range, "--"] + what_files + exclude_args, universal_newlines=True, encoding="utf8") + diff = check_output(["git", "diff", "-U0", commit_range, "--"] + what_files + exclude_args, text=True, encoding="utf8") return diff @@ -93,7 +93,7 @@ def main(): commit_range = "HEAD~" + args.prev_commits + "...HEAD" else: # This assumes that the target branch of the pull request will be master. - merge_base = check_output(["git", "merge-base", "HEAD", "master"], universal_newlines=True, encoding="utf8").rstrip("\n") + merge_base = check_output(["git", "merge-base", "HEAD", "master"], text=True, encoding="utf8").rstrip("\n") commit_range = merge_base + "..HEAD" else: commit_range = os.getenv("COMMIT_RANGE") diff --git a/test/lint/run-lint-format-strings.py b/test/lint/run-lint-format-strings.py index 57eefb00f2..91915f05f9 100755 --- a/test/lint/run-lint-format-strings.py +++ b/test/lint/run-lint-format-strings.py @@ -17,6 +17,7 @@ FALSE_POSITIVES = [ ("src/index/base.cpp", "FatalError(const char* fmt, const Args&... args)"), ("src/netbase.cpp", "LogConnectFailure(bool manual_connection, const char* fmt, const Args&... args)"), ("src/clientversion.cpp", "strprintf(_(COPYRIGHT_HOLDERS).translated, COPYRIGHT_HOLDERS_SUBSTITUTION)"), + ("src/test/translation_tests.cpp", "strprintf(format, arg)"), ("src/validationinterface.cpp", "LogPrint(BCLog::VALIDATION, fmt \"\\n\", __VA_ARGS__)"), ("src/wallet/wallet.h", "WalletLogPrintf(std::string fmt, Params... parameters)"), ("src/wallet/wallet.h", "LogPrintf((\"%s \" + fmt).c_str(), GetDisplayName(), parameters...)"), diff --git a/test/sanitizer_suppressions/lsan b/test/sanitizer_suppressions/lsan index 828b1676f6..7ccb22515f 100644 --- a/test/sanitizer_suppressions/lsan +++ b/test/sanitizer_suppressions/lsan @@ -1,5 +1,2 @@ # Suppress warnings triggered in dependencies leak:libQt5Widgets - -# false-positive due to use of secure_allocator<> -leak:GetRNGState diff --git a/test/sanitizer_suppressions/tsan b/test/sanitizer_suppressions/tsan index d331991273..5a790b72f2 100644 --- a/test/sanitizer_suppressions/tsan +++ b/test/sanitizer_suppressions/tsan @@ -13,6 +13,8 @@ race:zmq::* race:bitcoin-qt # deadlock (TODO fix) +# To reproduce, see: +# https://github.com/bitcoin/bitcoin/issues/19303#issuecomment-1514926359 deadlock:Chainstate::ConnectTip # Intentional deadlock in tests @@ -35,7 +37,7 @@ race:libzmq # https://github.com/bitcoin/bitcoin/issues/20618 race:CZMQAbstractPublishNotifier::SendZmqMessage -# https://github.com/bitcoin/bitcoin/pull/20218, https://github.com/bitcoin/bitcoin/pull/20745 +# https://github.com/bitcoin/bitcoin/pull/27498#issuecomment-1517410478 race:epoll_ctl # https://github.com/bitcoin/bitcoin/issues/23366 diff --git a/test/sanitizer_suppressions/ubsan b/test/sanitizer_suppressions/ubsan index 67ef512895..6b891c462e 100644 --- a/test/sanitizer_suppressions/ubsan +++ b/test/sanitizer_suppressions/ubsan @@ -5,16 +5,12 @@ # names can be used. # See https://github.com/google/sanitizers/issues/1364 -# https://github.com/bitcoin/bitcoin/pull/21798#issuecomment-829180719 -signed-integer-overflow:policy/feerate.cpp - # -fsanitize=integer suppressions # =============================== # Dependencies # ------------ # Suppressions in dependencies that are developed outside this repository. unsigned-integer-overflow:*/include/c++/ -unsigned-integer-overflow:bench/bench.h # unsigned-integer-overflow in FuzzedDataProvider's ConsumeIntegralInRange unsigned-integer-overflow:FuzzedDataProvider.h unsigned-integer-overflow:leveldb/ @@ -31,8 +27,6 @@ implicit-signed-integer-truncation:leveldb/ implicit-unsigned-integer-truncation:*/include/c++/ implicit-unsigned-integer-truncation:leveldb/ implicit-unsigned-integer-truncation:test/fuzz/crypto_diff_fuzz_chacha20.cpp -# std::variant warning fixed in https://github.com/gcc-mirror/gcc/commit/074436cf8cdd2a9ce75cadd36deb8301f00e55b9 -implicit-unsigned-integer-truncation:std::__detail::__variant::_Variant_storage shift-base:*/include/c++/ shift-base:leveldb/ shift-base:minisketch/ @@ -53,6 +47,7 @@ unsigned-integer-overflow:policy/fees.cpp unsigned-integer-overflow:prevector.h unsigned-integer-overflow:script/interpreter.cpp unsigned-integer-overflow:txmempool.cpp +unsigned-integer-overflow:xoroshiro128plusplus.h implicit-integer-sign-change:compat/stdin.cpp implicit-integer-sign-change:compressor.h implicit-integer-sign-change:crypto/ @@ -69,3 +64,4 @@ shift-base:crypto/ shift-base:hash.cpp shift-base:streams.h shift-base:util/bip32.cpp +shift-base:xoroshiro128plusplus.h diff --git a/test/util/rpcauth-test.py b/test/util/rpcauth-test.py index 53058dc394..8a7ff26dcb 100755 --- a/test/util/rpcauth-test.py +++ b/test/util/rpcauth-test.py @@ -4,7 +4,7 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test share/rpcauth/rpcauth.py """ -import base64 +import re import configparser import hmac import importlib @@ -28,18 +28,17 @@ class TestRPCAuth(unittest.TestCase): self.assertEqual(len(self.rpcauth.generate_salt(i)), i * 2) def test_generate_password(self): + """Test that generated passwords only consist of urlsafe characters.""" + r = re.compile(r"[0-9a-zA-Z_-]*") password = self.rpcauth.generate_password() - expected_password = base64.urlsafe_b64encode( - base64.urlsafe_b64decode(password)).decode('utf-8') - self.assertEqual(expected_password, password) + self.assertTrue(r.fullmatch(password)) def test_check_password_hmac(self): salt = self.rpcauth.generate_salt(16) password = self.rpcauth.generate_password() password_hmac = self.rpcauth.password_to_hmac(salt, password) - m = hmac.new(bytearray(salt, 'utf-8'), - bytearray(password, 'utf-8'), 'SHA256') + m = hmac.new(salt.encode('utf-8'), password.encode('utf-8'), 'SHA256') expected_password_hmac = m.hexdigest() self.assertEqual(expected_password_hmac, password_hmac) diff --git a/test/util/test_runner.py b/test/util/test_runner.py index 03db05c563..e5cdd0bc3a 100755 --- a/test/util/test_runner.py +++ b/test/util/test_runner.py @@ -54,7 +54,7 @@ def bctester(testDir, input_basename, buildenv): try: bctest(testDir, testObj, buildenv) logging.info("PASSED: " + testObj["description"]) - except: + except Exception: logging.info("FAILED: " + testObj["description"]) failed_testcases.append(testObj["description"]) @@ -96,7 +96,7 @@ def bctest(testDir, testObj, buildenv): try: with open(os.path.join(testDir, outputFn), encoding="utf8") as f: outputData = f.read() - except: + except Exception: logging.error("Output file " + outputFn + " cannot be opened") raise if not outputData: @@ -107,7 +107,7 @@ def bctest(testDir, testObj, buildenv): raise Exception # Run the test - proc = subprocess.Popen(execrun, stdin=stdinCfg, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) + proc = subprocess.Popen(execrun, stdin=stdinCfg, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) try: outs = proc.communicate(input=inputData) except OSError: |