diff options
Diffstat (limited to 'test')
68 files changed, 875 insertions, 355 deletions
diff --git a/test/functional/README.md b/test/functional/README.md index aff5f714f2..0d85a74074 100644 --- a/test/functional/README.md +++ b/test/functional/README.md @@ -127,8 +127,8 @@ Base class for functional tests. #### [util.py](test_framework/util.py) Generally useful functions. -#### [mininode.py](test_framework/mininode.py) -Basic code to support P2P connectivity to a bitcoind. +#### [p2p.py](test_framework/p2p.py) +Test objects for interacting with a bitcoind node over the p2p interface. #### [script.py](test_framework/script.py) Utilities for manipulating transaction scripts (originally from python-bitcoinlib) diff --git a/test/functional/example_test.py b/test/functional/example_test.py index 34e4999329..1832043989 100755 --- a/test/functional/example_test.py +++ b/test/functional/example_test.py @@ -16,17 +16,16 @@ from collections import defaultdict # Avoid wildcard * imports from test_framework.blocktools import (create_block, create_coinbase) from test_framework.messages import CInv, MSG_BLOCK -from test_framework.mininode import ( +from test_framework.p2p import ( P2PInterface, - mininode_lock, msg_block, msg_getdata, + p2p_lock, ) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, connect_nodes, - wait_until, ) # P2PInterface is a class containing callbacks to be executed when a P2P @@ -167,7 +166,7 @@ class ExampleTest(BitcoinTestFramework): height = self.nodes[0].getblockcount() for _ in range(10): - # Use the mininode and blocktools functionality to manually build a block + # Use the blocktools functionality to manually build a block. # Calling the generate() rpc is easier, but this allows us to exactly # control the blocks and transactions. block = create_block(self.tip, create_coinbase(height+1), self.block_time) @@ -203,13 +202,13 @@ class ExampleTest(BitcoinTestFramework): # wait_until() will loop until a predicate condition is met. Use it to test properties of the # P2PInterface objects. - wait_until(lambda: sorted(blocks) == sorted(list(self.nodes[2].p2p.block_receive_map.keys())), timeout=5, lock=mininode_lock) + self.nodes[2].p2p.wait_until(lambda: sorted(blocks) == sorted(list(self.nodes[2].p2p.block_receive_map.keys())), timeout=5) self.log.info("Check that each block was received only once") # The network thread uses a global lock on data access to the P2PConnection objects when sending and receiving # messages. The test thread should acquire the global lock before accessing any P2PConnection data to avoid locking # and synchronization issues. Note wait_until() acquires this global lock when testing the predicate. - with mininode_lock: + with p2p_lock: for block in self.nodes[2].p2p.block_receive_map.values(): assert_equal(block, 1) diff --git a/test/functional/feature_abortnode.py b/test/functional/feature_abortnode.py index 75267de80b..17fbf50cc8 100755 --- a/test/functional/feature_abortnode.py +++ b/test/functional/feature_abortnode.py @@ -11,7 +11,7 @@ """ from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import wait_until, get_datadir_path, connect_nodes +from test_framework.util import get_datadir_path, connect_nodes import os @@ -41,7 +41,7 @@ class AbortNodeTest(BitcoinTestFramework): # Check that node0 aborted self.log.info("Waiting for crash") - wait_until(lambda: self.nodes[0].is_node_stopped(), timeout=200) + self.nodes[0].wait_until_stopped(timeout=200) self.log.info("Node crashed - now verifying restart fails") self.nodes[0].assert_start_raises_init_error() diff --git a/test/functional/feature_assumevalid.py b/test/functional/feature_assumevalid.py index f19ee12f95..603d7f5d3b 100755 --- a/test/functional/feature_assumevalid.py +++ b/test/functional/feature_assumevalid.py @@ -42,7 +42,7 @@ from test_framework.messages import ( msg_block, msg_headers, ) -from test_framework.mininode import P2PInterface +from test_framework.p2p import P2PInterface from test_framework.script import (CScript, OP_TRUE) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal diff --git a/test/functional/feature_backwards_compatibility.py b/test/functional/feature_backwards_compatibility.py index 07dd0f8f82..126c9fe929 100755 --- a/test/functional/feature_backwards_compatibility.py +++ b/test/functional/feature_backwards_compatibility.py @@ -6,7 +6,7 @@ Test various backwards compatibility scenarios. Download the previous node binaries: -contrib/devtools/previous_release.py -b v0.19.1 v0.18.1 v0.17.1 v0.16.3 v0.15.2 +test/get_previous_releases.py -b v0.19.1 v0.18.1 v0.17.1 v0.16.3 v0.15.2 v0.15.2 is not required by this test, but it is used in wallet_upgradewallet.py. Due to a hardfork in regtest, it can't be used to sync nodes. diff --git a/test/functional/feature_block.py b/test/functional/feature_block.py index c74761869b..efafcfaec3 100755 --- a/test/functional/feature_block.py +++ b/test/functional/feature_block.py @@ -26,7 +26,7 @@ from test_framework.messages import ( uint256_from_compact, uint256_from_str, ) -from test_framework.mininode import P2PDataStore +from test_framework.p2p import P2PDataStore from test_framework.script import ( CScript, MAX_SCRIPT_ELEMENT_SIZE, @@ -53,7 +53,7 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal from data import invalid_txs -# Use this class for tests that require behavior other than normal "mininode" behavior. +# Use this class for tests that require behavior other than normal p2p behavior. # For now, it is used to serialize a bloated varint (b64). class CBrokenBlock(CBlock): def initialize(self, base_block): diff --git a/test/functional/feature_cltv.py b/test/functional/feature_cltv.py index fd0330924d..2919b0ea0b 100755 --- a/test/functional/feature_cltv.py +++ b/test/functional/feature_cltv.py @@ -10,7 +10,7 @@ Test that the CHECKLOCKTIMEVERIFY soft-fork activates at (regtest) block height from test_framework.blocktools import create_coinbase, create_block, create_transaction from test_framework.messages import CTransaction, msg_block, ToHex -from test_framework.mininode import P2PInterface +from test_framework.p2p import P2PInterface from test_framework.script import CScript, OP_1NEGATE, OP_CHECKLOCKTIMEVERIFY, OP_DROP, CScriptNum from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( diff --git a/test/functional/feature_csv_activation.py b/test/functional/feature_csv_activation.py index dfb3683143..38e95f00e9 100755 --- a/test/functional/feature_csv_activation.py +++ b/test/functional/feature_csv_activation.py @@ -44,7 +44,7 @@ import time from test_framework.blocktools import create_coinbase, create_block, create_transaction from test_framework.messages import ToHex, CTransaction -from test_framework.mininode import P2PDataStore +from test_framework.p2p import P2PDataStore from test_framework.script import ( CScript, OP_CHECKSEQUENCEVERIFY, diff --git a/test/functional/feature_dersig.py b/test/functional/feature_dersig.py index 05fdacd451..f263c93c8a 100755 --- a/test/functional/feature_dersig.py +++ b/test/functional/feature_dersig.py @@ -9,7 +9,7 @@ Test that the DERSIG soft-fork activates at (regtest) height 1251. from test_framework.blocktools import create_coinbase, create_block, create_transaction from test_framework.messages import msg_block -from test_framework.mininode import P2PInterface +from test_framework.p2p import P2PInterface from test_framework.script import CScript from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( diff --git a/test/functional/feature_maxuploadtarget.py b/test/functional/feature_maxuploadtarget.py index 0dc2839191..e5c62d1ea7 100755 --- a/test/functional/feature_maxuploadtarget.py +++ b/test/functional/feature_maxuploadtarget.py @@ -14,7 +14,7 @@ from collections import defaultdict import time from test_framework.messages import CInv, MSG_BLOCK, msg_getdata -from test_framework.mininode import P2PInterface +from test_framework.p2p import P2PInterface from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal, mine_large_block diff --git a/test/functional/feature_notifications.py b/test/functional/feature_notifications.py index dd4c318cee..3497b49a19 100755 --- a/test/functional/feature_notifications.py +++ b/test/functional/feature_notifications.py @@ -9,7 +9,6 @@ from test_framework.address import ADDRESS_BCRT1_UNSPENDABLE, keyhash_to_p2pkh from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, - wait_until, connect_nodes, disconnect_nodes, hex_str_to_bytes, @@ -56,7 +55,7 @@ class NotificationsTest(BitcoinTestFramework): blocks = self.nodes[1].generatetoaddress(block_count, self.nodes[1].getnewaddress() if self.is_wallet_compiled() else ADDRESS_BCRT1_UNSPENDABLE) # wait at most 10 seconds for expected number of files before reading the content - wait_until(lambda: len(os.listdir(self.blocknotify_dir)) == block_count, timeout=10) + self.wait_until(lambda: len(os.listdir(self.blocknotify_dir)) == block_count, timeout=10) # directory content should equal the generated blocks hashes assert_equal(sorted(blocks), sorted(os.listdir(self.blocknotify_dir))) @@ -64,7 +63,7 @@ class NotificationsTest(BitcoinTestFramework): if self.is_wallet_compiled(): self.log.info("test -walletnotify") # wait at most 10 seconds for expected number of files before reading the content - wait_until(lambda: len(os.listdir(self.walletnotify_dir)) == block_count, timeout=10) + self.wait_until(lambda: len(os.listdir(self.walletnotify_dir)) == block_count, timeout=10) # directory content should equal the generated transaction hashes txids_rpc = list(map(lambda t: notify_outputname(self.wallet, t['txid']), self.nodes[1].listtransactions("*", block_count))) @@ -78,7 +77,7 @@ class NotificationsTest(BitcoinTestFramework): self.start_node(1) connect_nodes(self.nodes[0], 1) - wait_until(lambda: len(os.listdir(self.walletnotify_dir)) == block_count, timeout=10) + self.wait_until(lambda: len(os.listdir(self.walletnotify_dir)) == block_count, timeout=10) # directory content should equal the generated transaction hashes txids_rpc = list(map(lambda t: notify_outputname(self.wallet, t['txid']), self.nodes[1].listtransactions("*", block_count))) @@ -140,7 +139,7 @@ class NotificationsTest(BitcoinTestFramework): # TODO: add test for `-alertnotify` large fork notifications def expect_wallet_notify(self, tx_ids): - wait_until(lambda: len(os.listdir(self.walletnotify_dir)) >= len(tx_ids), timeout=10) + self.wait_until(lambda: len(os.listdir(self.walletnotify_dir)) >= len(tx_ids), timeout=10) assert_equal(sorted(notify_outputname(self.wallet, tx_id) for tx_id in tx_ids), sorted(os.listdir(self.walletnotify_dir))) for tx_file in os.listdir(self.walletnotify_dir): os.remove(os.path.join(self.walletnotify_dir, tx_file)) diff --git a/test/functional/feature_pruning.py b/test/functional/feature_pruning.py index 02fa88f7c8..db408ab67a 100755 --- a/test/functional/feature_pruning.py +++ b/test/functional/feature_pruning.py @@ -20,7 +20,6 @@ from test_framework.util import ( assert_raises_rpc_error, connect_nodes, disconnect_nodes, - wait_until, ) # Rescans start at the earliest block up to 2 hours before a key timestamp, so @@ -136,7 +135,7 @@ class PruneTest(BitcoinTestFramework): mine_large_blocks(self.nodes[0], 25) # Wait for blk00000.dat to be pruned - wait_until(lambda: not os.path.isfile(os.path.join(self.prunedir, "blk00000.dat")), timeout=30) + self.wait_until(lambda: not os.path.isfile(os.path.join(self.prunedir, "blk00000.dat")), timeout=30) self.log.info("Success") usage = calc_usage(self.prunedir) @@ -250,7 +249,7 @@ class PruneTest(BitcoinTestFramework): self.log.info("Verify node 2 reorged back to the main chain, some blocks of which it had to redownload") # Wait for Node 2 to reorg to proper height - wait_until(lambda: self.nodes[2].getblockcount() >= goalbestheight, timeout=900) + self.wait_until(lambda: self.nodes[2].getblockcount() >= goalbestheight, timeout=900) assert_equal(self.nodes[2].getbestblockhash(), goalbesthash) # Verify we can now have the data for a block previously pruned assert_equal(self.nodes[2].getblock(self.forkhash)["height"], self.forkheight) diff --git a/test/functional/feature_shutdown.py b/test/functional/feature_shutdown.py index d782d3b1d8..a76e0f1b50 100755 --- a/test/functional/feature_shutdown.py +++ b/test/functional/feature_shutdown.py @@ -5,7 +5,7 @@ """Test bitcoind shutdown.""" from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, get_rpc_proxy, wait_until +from test_framework.util import assert_equal, get_rpc_proxy from threading import Thread def test_long_call(node): @@ -25,7 +25,7 @@ class ShutdownTest(BitcoinTestFramework): node.getblockcount() Thread(target=test_long_call, args=(node,)).start() # Wait until the server is executing the above `waitfornewblock`. - wait_until(lambda: len(self.nodes[0].getrpcinfo()['active_commands']) == 2) + self.wait_until(lambda: len(self.nodes[0].getrpcinfo()['active_commands']) == 2) # Wait 1 second after requesting shutdown but not before the `stop` call # finishes. This is to ensure event loop waits for current connections # to close. diff --git a/test/functional/feature_versionbits_warning.py b/test/functional/feature_versionbits_warning.py index 0713925141..e045adac32 100755 --- a/test/functional/feature_versionbits_warning.py +++ b/test/functional/feature_versionbits_warning.py @@ -12,9 +12,8 @@ import re from test_framework.blocktools import create_block, create_coinbase from test_framework.messages import msg_block -from test_framework.mininode import P2PInterface, mininode_lock +from test_framework.p2p import p2p_lock, P2PInterface from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import wait_until VB_PERIOD = 144 # versionbits period length for regtest VB_THRESHOLD = 108 # versionbits activation threshold for regtest @@ -91,14 +90,14 @@ class VersionBitsWarningTest(BitcoinTestFramework): # Generating one block guarantees that we'll get out of IBD node.generatetoaddress(1, node_deterministic_address) - wait_until(lambda: not node.getblockchaininfo()['initialblockdownload'], timeout=10, lock=mininode_lock) + self.wait_until(lambda: not node.getblockchaininfo()['initialblockdownload'], timeout=10, lock=p2p_lock) # Generating one more block will be enough to generate an error. node.generatetoaddress(1, node_deterministic_address) # Check that get*info() shows the versionbits unknown rules warning assert WARN_UNKNOWN_RULES_ACTIVE in node.getmininginfo()["warnings"] assert WARN_UNKNOWN_RULES_ACTIVE in node.getnetworkinfo()["warnings"] # Check that the alert file shows the versionbits unknown rules warning - wait_until(lambda: self.versionbits_in_alert_file(), timeout=60) + self.wait_until(lambda: self.versionbits_in_alert_file()) if __name__ == '__main__': VersionBitsWarningTest().main() diff --git a/test/functional/mempool_compatibility.py b/test/functional/mempool_compatibility.py index 31fb751904..75ca8d3236 100755 --- a/test/functional/mempool_compatibility.py +++ b/test/functional/mempool_compatibility.py @@ -8,7 +8,7 @@ NOTE: The test is designed to prevent cases when compatibility is broken acciden In case we need to break mempool compatibility we can continue to use the test by just bumping the version number. Download node binaries: -contrib/devtools/previous_release.py -b v0.19.1 v0.18.1 v0.17.1 v0.16.3 v0.15.2 +test/get_previous_releases.py -b v0.19.1 v0.18.1 v0.17.1 v0.16.3 v0.15.2 Only v0.15.2 is required by this test. The rest is used in other backwards compatibility tests. """ diff --git a/test/functional/mempool_packages.py b/test/functional/mempool_packages.py index 98dac30ace..e74ef8cf16 100755 --- a/test/functional/mempool_packages.py +++ b/test/functional/mempool_packages.py @@ -7,13 +7,12 @@ from decimal import Decimal from test_framework.messages import COIN -from test_framework.mininode import P2PTxInvStore +from test_framework.p2p import P2PTxInvStore from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, assert_raises_rpc_error, satoshi_round, - wait_until, ) # default limits @@ -269,8 +268,8 @@ class MempoolPackagesTest(BitcoinTestFramework): # - txs from previous ancestor test (-> custom ancestor limit) # - parent tx for descendant test # - txs chained off parent tx (-> custom descendant limit) - wait_until(lambda: len(self.nodes[1].getrawmempool(False)) == - MAX_ANCESTORS_CUSTOM + 1 + MAX_DESCENDANTS_CUSTOM, timeout=10) + self.wait_until(lambda: len(self.nodes[1].getrawmempool(False)) == + MAX_ANCESTORS_CUSTOM + 1 + MAX_DESCENDANTS_CUSTOM, timeout=10) mempool0 = self.nodes[0].getrawmempool(False) mempool1 = self.nodes[1].getrawmempool(False) assert set(mempool1).issubset(set(mempool0)) diff --git a/test/functional/mempool_persist.py b/test/functional/mempool_persist.py index 85c4d6d570..f73f1a02a2 100755 --- a/test/functional/mempool_persist.py +++ b/test/functional/mempool_persist.py @@ -39,15 +39,14 @@ from decimal import Decimal import os import time +from test_framework.p2p import P2PTxInvStore from test_framework.test_framework import BitcoinTestFramework -from test_framework.mininode import P2PTxInvStore from test_framework.util import ( assert_equal, assert_greater_than_or_equal, assert_raises_rpc_error, connect_nodes, disconnect_nodes, - wait_until, ) @@ -172,7 +171,7 @@ class MempoolPersistTest(BitcoinTestFramework): # check that txn gets broadcast due to unbroadcast logic conn = node0.add_p2p_connection(P2PTxInvStore()) node0.mockscheduler(16*60) # 15 min + 1 for buffer - wait_until(lambda: len(conn.get_invs()) == 1) + self.wait_until(lambda: len(conn.get_invs()) == 1) if __name__ == '__main__': MempoolPersistTest().main() diff --git a/test/functional/mempool_unbroadcast.py b/test/functional/mempool_unbroadcast.py index 365d011157..abd5a03d95 100755 --- a/test/functional/mempool_unbroadcast.py +++ b/test/functional/mempool_unbroadcast.py @@ -7,7 +7,7 @@ to peers until a GETDATA is received.""" import time -from test_framework.mininode import P2PTxInvStore +from test_framework.p2p import P2PTxInvStore from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, diff --git a/test/functional/mining_basic.py b/test/functional/mining_basic.py index 63d1ccfb36..b13740750f 100755 --- a/test/functional/mining_basic.py +++ b/test/functional/mining_basic.py @@ -20,7 +20,7 @@ from test_framework.messages import ( CBlockHeader, BLOCK_HEADER_SIZE, ) -from test_framework.mininode import P2PDataStore +from test_framework.p2p import P2PDataStore from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, diff --git a/test/functional/p2p_addr_relay.py b/test/functional/p2p_addr_relay.py index 5c7e27a3a8..80f262d0d3 100755 --- a/test/functional/p2p_addr_relay.py +++ b/test/functional/p2p_addr_relay.py @@ -12,9 +12,7 @@ from test_framework.messages import ( NODE_WITNESS, msg_addr, ) -from test_framework.mininode import ( - P2PInterface, -) +from test_framework.p2p import P2PInterface from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, diff --git a/test/functional/p2p_blockfilters.py b/test/functional/p2p_blockfilters.py index a9e86bd2fc..84178d0dd7 100755 --- a/test/functional/p2p_blockfilters.py +++ b/test/functional/p2p_blockfilters.py @@ -18,13 +18,12 @@ from test_framework.messages import ( ser_uint256, uint256_from_str, ) -from test_framework.mininode import P2PInterface +from test_framework.p2p import P2PInterface from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, connect_nodes, disconnect_nodes, - wait_until, ) class CFiltersClient(P2PInterface): @@ -65,11 +64,11 @@ class CompactFiltersTest(BitcoinTestFramework): disconnect_nodes(self.nodes[0], 1) self.nodes[0].generate(1) - wait_until(lambda: self.nodes[0].getblockcount() == 1000) + self.wait_until(lambda: self.nodes[0].getblockcount() == 1000) stale_block_hash = self.nodes[0].getblockhash(1000) self.nodes[1].generate(1001) - wait_until(lambda: self.nodes[1].getblockcount() == 2000) + self.wait_until(lambda: self.nodes[1].getblockcount() == 2000) # Check that nodes have signalled NODE_COMPACT_FILTERS correctly. assert node0.nServices & NODE_COMPACT_FILTERS != 0 diff --git a/test/functional/p2p_blocksonly.py b/test/functional/p2p_blocksonly.py index 27e6b669f6..65259f1869 100755 --- a/test/functional/p2p_blocksonly.py +++ b/test/functional/p2p_blocksonly.py @@ -5,7 +5,7 @@ """Test p2p blocksonly""" from test_framework.messages import msg_tx, CTransaction, FromHex -from test_framework.mininode import P2PInterface +from test_framework.p2p import P2PInterface from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal diff --git a/test/functional/p2p_compactblocks.py b/test/functional/p2p_compactblocks.py index 225d393e1b..fdae7fb68b 100755 --- a/test/functional/p2p_compactblocks.py +++ b/test/functional/p2p_compactblocks.py @@ -11,10 +11,10 @@ import random from test_framework.blocktools import create_block, create_coinbase, add_witness_commitment from test_framework.messages import BlockTransactions, BlockTransactionsRequest, calculate_shortid, CBlock, CBlockHeader, CInv, COutPoint, CTransaction, CTxIn, CTxInWitness, CTxOut, FromHex, HeaderAndShortIDs, msg_no_witness_block, msg_no_witness_blocktxn, msg_cmpctblock, msg_getblocktxn, msg_getdata, msg_getheaders, msg_headers, msg_inv, msg_sendcmpct, msg_sendheaders, msg_tx, msg_block, msg_blocktxn, MSG_BLOCK, MSG_CMPCT_BLOCK, MSG_WITNESS_FLAG, NODE_NETWORK, P2PHeaderAndShortIDs, PrefilledTransaction, ser_uint256, ToHex -from test_framework.mininode import mininode_lock, P2PInterface +from test_framework.p2p import p2p_lock, P2PInterface from test_framework.script import CScript, OP_TRUE, OP_DROP from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, wait_until, softfork_active +from test_framework.util import assert_equal, softfork_active # TestP2PConn: A peer we use to send messages to bitcoind, and store responses. class TestP2PConn(P2PInterface): @@ -48,12 +48,12 @@ class TestP2PConn(P2PInterface): self.block_announced = True self.announced_blockhashes.add(x.hash) - # Requires caller to hold mininode_lock + # Requires caller to hold p2p_lock def received_block_announcement(self): return self.block_announced def clear_block_announcement(self): - with mininode_lock: + with p2p_lock: self.block_announced = False self.last_message.pop("inv", None) self.last_message.pop("headers", None) @@ -73,7 +73,7 @@ class TestP2PConn(P2PInterface): def request_headers_and_sync(self, locator, hashstop=0): self.clear_block_announcement() self.get_headers(locator, hashstop) - wait_until(self.received_block_announcement, timeout=30, lock=mininode_lock) + self.wait_until(self.received_block_announcement, timeout=30) self.clear_block_announcement() # Block until a block announcement for a particular block hash is @@ -81,7 +81,7 @@ class TestP2PConn(P2PInterface): def wait_for_block_announcement(self, block_hash, timeout=30): def received_hash(): return (block_hash in self.announced_blockhashes) - wait_until(received_hash, timeout=timeout, lock=mininode_lock) + self.wait_until(received_hash, timeout=timeout) def send_await_disconnect(self, message, timeout=30): """Sends a message to the node and wait for disconnect. @@ -89,7 +89,7 @@ class TestP2PConn(P2PInterface): This is used when we want to send a message into the node that we expect will get us disconnected, eg an invalid block.""" self.send_message(message) - wait_until(lambda: not self.is_connected, timeout=timeout, lock=mininode_lock) + self.wait_for_disconnect(timeout) class CompactBlocksTest(BitcoinTestFramework): def set_test_params(self): @@ -154,8 +154,8 @@ class CompactBlocksTest(BitcoinTestFramework): # Make sure we get a SENDCMPCT message from our peer def received_sendcmpct(): return (len(test_node.last_sendcmpct) > 0) - wait_until(received_sendcmpct, timeout=30, lock=mininode_lock) - with mininode_lock: + test_node.wait_until(received_sendcmpct, timeout=30) + with p2p_lock: # Check that the first version received is the preferred one assert_equal(test_node.last_sendcmpct[0].version, preferred_version) # And that we receive versions down to 1. @@ -170,7 +170,7 @@ class CompactBlocksTest(BitcoinTestFramework): peer.wait_for_block_announcement(block_hash, timeout=30) assert peer.block_announced - with mininode_lock: + with p2p_lock: assert predicate(peer), ( "block_hash={!r}, cmpctblock={!r}, inv={!r}".format( block_hash, peer.last_message.get("cmpctblock", None), peer.last_message.get("inv", None))) @@ -294,11 +294,11 @@ class CompactBlocksTest(BitcoinTestFramework): block.rehash() # Wait until the block was announced (via compact blocks) - wait_until(lambda: "cmpctblock" in test_node.last_message, timeout=30, lock=mininode_lock) + test_node.wait_until(lambda: "cmpctblock" in test_node.last_message, timeout=30) # Now fetch and check the compact block header_and_shortids = None - with mininode_lock: + with p2p_lock: # Convert the on-the-wire representation to absolute indexes header_and_shortids = HeaderAndShortIDs(test_node.last_message["cmpctblock"].header_and_shortids) self.check_compactblock_construction_from_block(version, header_and_shortids, block_hash, block) @@ -308,11 +308,11 @@ class CompactBlocksTest(BitcoinTestFramework): inv = CInv(MSG_CMPCT_BLOCK, block_hash) test_node.send_message(msg_getdata([inv])) - wait_until(lambda: "cmpctblock" in test_node.last_message, timeout=30, lock=mininode_lock) + test_node.wait_until(lambda: "cmpctblock" in test_node.last_message, timeout=30) # Now fetch and check the compact block header_and_shortids = None - with mininode_lock: + with p2p_lock: # Convert the on-the-wire representation to absolute indexes header_and_shortids = HeaderAndShortIDs(test_node.last_message["cmpctblock"].header_and_shortids) self.check_compactblock_construction_from_block(version, header_and_shortids, block_hash, block) @@ -378,7 +378,7 @@ class CompactBlocksTest(BitcoinTestFramework): if announce == "inv": test_node.send_message(msg_inv([CInv(MSG_BLOCK, block.sha256)])) - wait_until(lambda: "getheaders" in test_node.last_message, timeout=30, lock=mininode_lock) + test_node.wait_until(lambda: "getheaders" in test_node.last_message, timeout=30) test_node.send_header_for_blocks([block]) else: test_node.send_header_for_blocks([block]) @@ -397,7 +397,7 @@ class CompactBlocksTest(BitcoinTestFramework): test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) assert_equal(int(node.getbestblockhash(), 16), block.hashPrevBlock) # Expect a getblocktxn message. - with mininode_lock: + with p2p_lock: assert "getblocktxn" in test_node.last_message absolute_indexes = test_node.last_message["getblocktxn"].block_txn_request.to_absolute() assert_equal(absolute_indexes, [0]) # should be a coinbase request @@ -439,7 +439,7 @@ class CompactBlocksTest(BitcoinTestFramework): def test_getblocktxn_response(compact_block, peer, expected_result): msg = msg_cmpctblock(compact_block.to_p2p()) peer.send_and_ping(msg) - with mininode_lock: + with p2p_lock: assert "getblocktxn" in peer.last_message absolute_indexes = peer.last_message["getblocktxn"].block_txn_request.to_absolute() assert_equal(absolute_indexes, expected_result) @@ -504,13 +504,13 @@ class CompactBlocksTest(BitcoinTestFramework): assert tx.hash in mempool # Clear out last request. - with mininode_lock: + with p2p_lock: test_node.last_message.pop("getblocktxn", None) # Send compact block comp_block.initialize_from_block(block, prefill_list=[0], use_witness=with_witness) test_tip_after_message(node, test_node, msg_cmpctblock(comp_block.to_p2p()), block.sha256) - with mininode_lock: + with p2p_lock: # Shouldn't have gotten a request for any transaction assert "getblocktxn" not in test_node.last_message @@ -537,7 +537,7 @@ class CompactBlocksTest(BitcoinTestFramework): comp_block.initialize_from_block(block, prefill_list=[0], use_witness=(version == 2)) test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) absolute_indexes = [] - with mininode_lock: + with p2p_lock: assert "getblocktxn" in test_node.last_message absolute_indexes = test_node.last_message["getblocktxn"].block_txn_request.to_absolute() assert_equal(absolute_indexes, [6, 7, 8, 9, 10]) @@ -588,10 +588,10 @@ class CompactBlocksTest(BitcoinTestFramework): num_to_request = random.randint(1, len(block.vtx)) msg.block_txn_request.from_absolute(sorted(random.sample(range(len(block.vtx)), num_to_request))) test_node.send_message(msg) - wait_until(lambda: "blocktxn" in test_node.last_message, timeout=10, lock=mininode_lock) + test_node.wait_until(lambda: "blocktxn" in test_node.last_message, timeout=10) [tx.calc_sha256() for tx in block.vtx] - with mininode_lock: + with p2p_lock: assert_equal(test_node.last_message["blocktxn"].block_transactions.blockhash, int(block_hash, 16)) all_indices = msg.block_txn_request.to_absolute() for index in all_indices: @@ -611,11 +611,11 @@ class CompactBlocksTest(BitcoinTestFramework): # allowed depth for a blocktxn response. block_hash = node.getblockhash(current_height) msg.block_txn_request = BlockTransactionsRequest(int(block_hash, 16), [0]) - with mininode_lock: + with p2p_lock: test_node.last_message.pop("block", None) test_node.last_message.pop("blocktxn", None) test_node.send_and_ping(msg) - with mininode_lock: + with p2p_lock: test_node.last_message["block"].block.calc_sha256() assert_equal(test_node.last_message["block"].block.sha256, int(block_hash, 16)) assert "blocktxn" not in test_node.last_message @@ -628,21 +628,21 @@ class CompactBlocksTest(BitcoinTestFramework): for _ in range(MAX_CMPCTBLOCK_DEPTH + 1): test_node.clear_block_announcement() new_blocks.append(node.generate(1)[0]) - wait_until(test_node.received_block_announcement, timeout=30, lock=mininode_lock) + test_node.wait_until(test_node.received_block_announcement, timeout=30) test_node.clear_block_announcement() test_node.send_message(msg_getdata([CInv(MSG_CMPCT_BLOCK, int(new_blocks[0], 16))])) - wait_until(lambda: "cmpctblock" in test_node.last_message, timeout=30, lock=mininode_lock) + test_node.wait_until(lambda: "cmpctblock" in test_node.last_message, timeout=30) test_node.clear_block_announcement() node.generate(1) - wait_until(test_node.received_block_announcement, timeout=30, lock=mininode_lock) + test_node.wait_until(test_node.received_block_announcement, timeout=30) test_node.clear_block_announcement() - with mininode_lock: + with p2p_lock: test_node.last_message.pop("block", None) test_node.send_message(msg_getdata([CInv(MSG_CMPCT_BLOCK, int(new_blocks[0], 16))])) - wait_until(lambda: "block" in test_node.last_message, timeout=30, lock=mininode_lock) - with mininode_lock: + test_node.wait_until(lambda: "block" in test_node.last_message, timeout=30) + with p2p_lock: test_node.last_message["block"].block.calc_sha256() assert_equal(test_node.last_message["block"].block.sha256, int(new_blocks[0], 16)) @@ -670,10 +670,10 @@ class CompactBlocksTest(BitcoinTestFramework): # (to avoid fingerprinting attacks). msg = msg_getblocktxn() msg.block_txn_request = BlockTransactionsRequest(block.sha256, [0]) - with mininode_lock: + with p2p_lock: test_node.last_message.pop("blocktxn", None) test_node.send_and_ping(msg) - with mininode_lock: + with p2p_lock: assert "blocktxn" not in test_node.last_message def test_end_to_end_block_relay(self, listeners): @@ -689,8 +689,8 @@ class CompactBlocksTest(BitcoinTestFramework): node.submitblock(ToHex(block)) for l in listeners: - wait_until(lambda: "cmpctblock" in l.last_message, timeout=30, lock=mininode_lock) - with mininode_lock: + l.wait_until(lambda: "cmpctblock" in l.last_message, timeout=30) + with p2p_lock: for l in listeners: l.last_message["cmpctblock"].header_and_shortids.header.calc_sha256() assert_equal(l.last_message["cmpctblock"].header_and_shortids.header.sha256, block.sha256) @@ -747,7 +747,7 @@ class CompactBlocksTest(BitcoinTestFramework): cmpct_block.initialize_from_block(block) msg = msg_cmpctblock(cmpct_block.to_p2p()) peer.send_and_ping(msg) - with mininode_lock: + with p2p_lock: assert "getblocktxn" in peer.last_message return block, cmpct_block diff --git a/test/functional/p2p_disconnect_ban.py b/test/functional/p2p_disconnect_ban.py index 09b9ebeb2d..b7c2a306eb 100755 --- a/test/functional/p2p_disconnect_ban.py +++ b/test/functional/p2p_disconnect_ban.py @@ -10,7 +10,6 @@ from test_framework.util import ( assert_equal, assert_raises_rpc_error, connect_nodes, - wait_until, ) class DisconnectBanTest(BitcoinTestFramework): @@ -28,7 +27,7 @@ class DisconnectBanTest(BitcoinTestFramework): self.log.info("setban: successfully ban single IP address") assert_equal(len(self.nodes[1].getpeerinfo()), 2) # node1 should have 2 connections to node0 at this point self.nodes[1].setban(subnet="127.0.0.1", command="add") - wait_until(lambda: len(self.nodes[1].getpeerinfo()) == 0, timeout=10) + self.wait_until(lambda: len(self.nodes[1].getpeerinfo()) == 0, timeout=10) assert_equal(len(self.nodes[1].getpeerinfo()), 0) # all nodes must be disconnected at this point assert_equal(len(self.nodes[1].listbanned()), 1) @@ -95,7 +94,7 @@ class DisconnectBanTest(BitcoinTestFramework): self.log.info("disconnectnode: successfully disconnect node by address") address1 = self.nodes[0].getpeerinfo()[0]['addr'] self.nodes[0].disconnectnode(address=address1) - wait_until(lambda: len(self.nodes[0].getpeerinfo()) == 1, timeout=10) + self.wait_until(lambda: len(self.nodes[0].getpeerinfo()) == 1, timeout=10) assert not [node for node in self.nodes[0].getpeerinfo() if node['addr'] == address1] self.log.info("disconnectnode: successfully reconnect node") @@ -106,7 +105,7 @@ class DisconnectBanTest(BitcoinTestFramework): self.log.info("disconnectnode: successfully disconnect node by node id") id1 = self.nodes[0].getpeerinfo()[0]['id'] self.nodes[0].disconnectnode(nodeid=id1) - wait_until(lambda: len(self.nodes[0].getpeerinfo()) == 1, timeout=10) + self.wait_until(lambda: len(self.nodes[0].getpeerinfo()) == 1, timeout=10) assert not [node for node in self.nodes[0].getpeerinfo() if node['id'] == id1] if __name__ == '__main__': diff --git a/test/functional/p2p_dos_header_tree.py b/test/functional/p2p_dos_header_tree.py index f8552cf53d..7dd8c3146b 100755 --- a/test/functional/p2p_dos_header_tree.py +++ b/test/functional/p2p_dos_header_tree.py @@ -8,7 +8,7 @@ from test_framework.messages import ( CBlockHeader, FromHex, ) -from test_framework.mininode import ( +from test_framework.p2p import ( P2PInterface, msg_headers, ) diff --git a/test/functional/p2p_eviction.py b/test/functional/p2p_eviction.py index b2b3a89aab..72a255991c 100755 --- a/test/functional/p2p_eviction.py +++ b/test/functional/p2p_eviction.py @@ -15,11 +15,11 @@ Therefore, this test is limited to the remaining protection criteria. import time -from test_framework.test_framework import BitcoinTestFramework -from test_framework.mininode import P2PInterface, P2PDataStore -from test_framework.util import assert_equal, wait_until from test_framework.blocktools import create_block, create_coinbase from test_framework.messages import CTransaction, FromHex, msg_pong, msg_tx +from test_framework.p2p import P2PDataStore, P2PInterface +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal class SlowP2PDataStore(P2PDataStore): @@ -92,7 +92,7 @@ class P2PEvict(BitcoinTestFramework): for _ in range(8): fastpeer = node.add_p2p_connection(P2PInterface()) current_peer += 1 - wait_until(lambda: "ping" in fastpeer.last_message, timeout=10) + self.wait_until(lambda: "ping" in fastpeer.last_message, timeout=10) # Make sure by asking the node what the actual min pings are peerinfo = node.getpeerinfo() diff --git a/test/functional/p2p_feefilter.py b/test/functional/p2p_feefilter.py index 0b51d8f4bb..0c07b56a69 100755 --- a/test/functional/p2p_feefilter.py +++ b/test/functional/p2p_feefilter.py @@ -5,10 +5,9 @@ """Test processing of feefilter messages.""" from decimal import Decimal -import time from test_framework.messages import MSG_TX, MSG_WTX, msg_feefilter -from test_framework.mininode import mininode_lock, P2PInterface +from test_framework.p2p import P2PInterface, p2p_lock from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal @@ -17,16 +16,6 @@ def hashToHex(hash): return format(hash, '064x') -# Wait up to 60 secs to see if the testnode has received all the expected invs -def allInvsMatch(invsExpected, testnode): - for _ in range(60): - with mininode_lock: - if (sorted(invsExpected) == sorted(testnode.txinvs)): - return True - time.sleep(1) - return False - - class FeefilterConn(P2PInterface): feefilter_received = False @@ -34,7 +23,7 @@ class FeefilterConn(P2PInterface): self.feefilter_received = True def assert_feefilter_received(self, recv: bool): - with mininode_lock: + with p2p_lock: assert_equal(self.feefilter_received, recv) @@ -48,8 +37,12 @@ class TestP2PConn(P2PInterface): if (i.type == MSG_TX) or (i.type == MSG_WTX): self.txinvs.append(hashToHex(i.hash)) + def wait_for_invs_to_match(self, invs_expected): + invs_expected.sort() + self.wait_until(lambda: invs_expected == sorted(self.txinvs)) + def clear_invs(self): - with mininode_lock: + with p2p_lock: self.txinvs = [] @@ -61,7 +54,12 @@ class FeeFilterTest(BitcoinTestFramework): # mempool and wallet feerate calculation based on GetFee # rounding down 3 places, leading to stranded transactions. # See issue #16499 - self.extra_args = [["-minrelaytxfee=0.00000100", "-mintxfee=0.00000100"]] * self.num_nodes + # grant noban permission to all peers to speed up tx relay / mempool sync + self.extra_args = [[ + "-minrelaytxfee=0.00000100", + "-mintxfee=0.00000100", + "-whitelist=noban@127.0.0.1", + ]] * self.num_nodes def skip_test_if_missing_module(self): self.skip_if_no_wallet() @@ -88,24 +86,22 @@ class FeeFilterTest(BitcoinTestFramework): conn = self.nodes[0].add_p2p_connection(TestP2PConn()) - # Test that invs are received by test connection for all txs at - # feerate of .2 sat/byte + self.log.info("Test txs paying 0.2 sat/byte are received by test connection") node1.settxfee(Decimal("0.00000200")) txids = [node1.sendtoaddress(node1.getnewaddress(), 1) for _ in range(3)] - assert allInvsMatch(txids, conn) + conn.wait_for_invs_to_match(txids) conn.clear_invs() - # Set a filter of .15 sat/byte on test connection + # Set a fee filter of 0.15 sat/byte on test connection conn.send_and_ping(msg_feefilter(150)) - # Test that txs are still being received by test connection (paying .15 sat/byte) + self.log.info("Test txs paying 0.15 sat/byte are received by test connection") node1.settxfee(Decimal("0.00000150")) txids = [node1.sendtoaddress(node1.getnewaddress(), 1) for _ in range(3)] - assert allInvsMatch(txids, conn) + conn.wait_for_invs_to_match(txids) conn.clear_invs() - # Change tx fee rate to .1 sat/byte and test they are no longer received - # by the test connection + self.log.info("Test txs paying 0.1 sat/byte are no longer received by test connection") node1.settxfee(Decimal("0.00000100")) [node1.sendtoaddress(node1.getnewaddress(), 1) for _ in range(3)] self.sync_mempools() # must be sure node 0 has received all txs @@ -119,13 +115,13 @@ class FeeFilterTest(BitcoinTestFramework): # as well. node0.settxfee(Decimal("0.00020000")) txids = [node0.sendtoaddress(node0.getnewaddress(), 1)] - assert allInvsMatch(txids, conn) + conn.wait_for_invs_to_match(txids) conn.clear_invs() - # Remove fee filter and check that txs are received again + self.log.info("Remove fee filter and check txs are received again") conn.send_and_ping(msg_feefilter(0)) txids = [node1.sendtoaddress(node1.getnewaddress(), 1) for _ in range(3)] - assert allInvsMatch(txids, conn) + conn.wait_for_invs_to_match(txids) conn.clear_invs() diff --git a/test/functional/p2p_filter.py b/test/functional/p2p_filter.py index ce3856fc95..613d96eaad 100755 --- a/test/functional/p2p_filter.py +++ b/test/functional/p2p_filter.py @@ -19,7 +19,7 @@ from test_framework.messages import ( msg_mempool, msg_version, ) -from test_framework.mininode import P2PInterface, mininode_lock +from test_framework.p2p import P2PInterface, p2p_lock from test_framework.script import MAX_SCRIPT_ELEMENT_SIZE from test_framework.test_framework import BitcoinTestFramework @@ -60,22 +60,22 @@ class P2PBloomFilter(P2PInterface): @property def tx_received(self): - with mininode_lock: + with p2p_lock: return self._tx_received @tx_received.setter def tx_received(self, value): - with mininode_lock: + with p2p_lock: self._tx_received = value @property def merkleblock_received(self): - with mininode_lock: + with p2p_lock: return self._merkleblock_received @merkleblock_received.setter def merkleblock_received(self, value): - with mininode_lock: + with p2p_lock: self._merkleblock_received = value diff --git a/test/functional/p2p_fingerprint.py b/test/functional/p2p_fingerprint.py index d743abe681..aaf862e6c8 100755 --- a/test/functional/p2p_fingerprint.py +++ b/test/functional/p2p_fingerprint.py @@ -12,7 +12,7 @@ import time from test_framework.blocktools import (create_block, create_coinbase) from test_framework.messages import CInv, MSG_BLOCK -from test_framework.mininode import ( +from test_framework.p2p import ( P2PInterface, msg_headers, msg_block, @@ -22,9 +22,9 @@ from test_framework.mininode import ( from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, - wait_until, ) + class P2PFingerprintTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True @@ -102,12 +102,12 @@ class P2PFingerprintTest(BitcoinTestFramework): # Check that getdata request for stale block succeeds self.send_block_request(stale_hash, node0) test_function = lambda: self.last_block_equals(stale_hash, node0) - wait_until(test_function, timeout=3) + self.wait_until(test_function, timeout=3) # Check that getheader request for stale block header succeeds self.send_header_request(stale_hash, node0) test_function = lambda: self.last_header_equals(stale_hash, node0) - wait_until(test_function, timeout=3) + self.wait_until(test_function, timeout=3) # Longest chain is extended so stale is much older than chain tip self.nodes[0].setmocktime(0) @@ -138,11 +138,11 @@ class P2PFingerprintTest(BitcoinTestFramework): self.send_block_request(block_hash, node0) test_function = lambda: self.last_block_equals(block_hash, node0) - wait_until(test_function, timeout=3) + self.wait_until(test_function, timeout=3) self.send_header_request(block_hash, node0) test_function = lambda: self.last_header_equals(block_hash, node0) - wait_until(test_function, timeout=3) + self.wait_until(test_function, timeout=3) if __name__ == '__main__': P2PFingerprintTest().main() diff --git a/test/functional/p2p_getaddr_caching.py b/test/functional/p2p_getaddr_caching.py index c9278eab92..6622ea9ec2 100755 --- a/test/functional/p2p_getaddr_caching.py +++ b/test/functional/p2p_getaddr_caching.py @@ -12,9 +12,9 @@ from test_framework.messages import ( msg_addr, msg_getaddr, ) -from test_framework.mininode import ( +from test_framework.p2p import ( P2PInterface, - mininode_lock + p2p_lock ) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( @@ -44,7 +44,7 @@ class AddrReceiver(P2PInterface): self.received_addrs = None def get_received_addrs(self): - with mininode_lock: + with p2p_lock: return self.received_addrs def on_addr(self, message): diff --git a/test/functional/p2p_getdata.py b/test/functional/p2p_getdata.py index d1b11c2c61..51921a8ab5 100755 --- a/test/functional/p2p_getdata.py +++ b/test/functional/p2p_getdata.py @@ -9,7 +9,7 @@ from test_framework.messages import ( CInv, msg_getdata, ) -from test_framework.mininode import P2PInterface +from test_framework.p2p import P2PInterface from test_framework.test_framework import BitcoinTestFramework diff --git a/test/functional/p2p_invalid_block.py b/test/functional/p2p_invalid_block.py index e280a62997..b2c3c5d45f 100755 --- a/test/functional/p2p_invalid_block.py +++ b/test/functional/p2p_invalid_block.py @@ -14,7 +14,7 @@ import copy from test_framework.blocktools import create_block, create_coinbase, create_tx_with_script from test_framework.messages import COIN -from test_framework.mininode import P2PDataStore +from test_framework.p2p import P2PDataStore from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal diff --git a/test/functional/p2p_invalid_locator.py b/test/functional/p2p_invalid_locator.py index 0155eb21f0..24328c2919 100755 --- a/test/functional/p2p_invalid_locator.py +++ b/test/functional/p2p_invalid_locator.py @@ -6,7 +6,7 @@ """ from test_framework.messages import msg_getheaders, msg_getblocks, MAX_LOCATOR_SZ -from test_framework.mininode import P2PInterface +from test_framework.p2p import P2PInterface from test_framework.test_framework import BitcoinTestFramework diff --git a/test/functional/p2p_invalid_messages.py b/test/functional/p2p_invalid_messages.py index d9a9ae5188..fe57057a83 100755 --- a/test/functional/p2p_invalid_messages.py +++ b/test/functional/p2p_invalid_messages.py @@ -17,14 +17,13 @@ from test_framework.messages import ( MSG_TX, ser_string, ) -from test_framework.mininode import ( +from test_framework.p2p import ( P2PDataStore, P2PInterface, ) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, - wait_until, ) VALID_DATA_LIMIT = MAX_PROTOCOL_MESSAGE_LENGTH - 5 # Account for the 5-byte length prefix @@ -70,7 +69,7 @@ class InvalidMessagesTest(BitcoinTestFramework): before = int(self.nodes[0].getnettotals()['totalbytesrecv']) conn.send_raw_message(msg[:cut_pos]) # Wait until node has processed the first half of the message - wait_until(lambda: int(self.nodes[0].getnettotals()['totalbytesrecv']) != before) + self.wait_until(lambda: int(self.nodes[0].getnettotals()['totalbytesrecv']) != before) middle = int(self.nodes[0].getnettotals()['totalbytesrecv']) # If this assert fails, we've hit an unlikely race # where the test framework sent a message in between the two halves diff --git a/test/functional/p2p_invalid_tx.py b/test/functional/p2p_invalid_tx.py index c70a892463..a0ef6c9d6e 100755 --- a/test/functional/p2p_invalid_tx.py +++ b/test/functional/p2p_invalid_tx.py @@ -13,11 +13,10 @@ from test_framework.messages import ( CTxIn, CTxOut, ) -from test_framework.mininode import P2PDataStore +from test_framework.p2p import P2PDataStore from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, - wait_until, ) from data import invalid_txs @@ -146,7 +145,7 @@ class InvalidTxRequestTest(BitcoinTestFramework): # tx_orphan_no_fee, because it has too low fee (p2ps[0] is not disconnected for relaying that tx) # tx_orphan_invaid, because it has negative fee (p2ps[1] is disconnected for relaying that tx) - wait_until(lambda: 1 == len(node.getpeerinfo()), timeout=12) # p2ps[1] is no longer connected + self.wait_until(lambda: 1 == len(node.getpeerinfo()), timeout=12) # p2ps[1] is no longer connected assert_equal(expected_mempool, set(node.getrawmempool())) self.log.info('Test orphan pool overflow') diff --git a/test/functional/p2p_leak.py b/test/functional/p2p_leak.py index 2fc5245241..4978aa3845 100755 --- a/test/functional/p2p_leak.py +++ b/test/functional/p2p_leak.py @@ -15,21 +15,19 @@ import time from test_framework.messages import ( msg_getaddr, msg_ping, - msg_verack, msg_version, ) -from test_framework.mininode import mininode_lock, P2PInterface +from test_framework.p2p import p2p_lock, P2PInterface from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, assert_greater_than_or_equal, - wait_until, ) DISCOURAGEMENT_THRESHOLD = 100 -class CLazyNode(P2PInterface): +class LazyPeer(P2PInterface): def __init__(self): super().__init__() self.unexpected_msg = False @@ -42,6 +40,7 @@ class CLazyNode(P2PInterface): def on_open(self): self.ever_connected = True + # Does not respond to "version" with "verack" def on_version(self, message): self.bad_message(message) def on_verack(self, message): self.bad_message(message) def on_inv(self, message): self.bad_message(message) @@ -64,21 +63,8 @@ class CLazyNode(P2PInterface): def on_blocktxn(self, message): self.bad_message(message) -# Node that never sends a version. We'll use this to send a bunch of messages -# anyway, and eventually get disconnected. -class CNodeNoVersionMisbehavior(CLazyNode): - pass - - -# Node that never sends a version. This one just sits idle and hopes to receive -# any message (it shouldn't!) -class CNodeNoVersionIdle(CLazyNode): - def __init__(self): - super().__init__() - - -# Node that sends a version but not a verack. -class CNodeNoVerackIdle(CLazyNode): +# Peer that sends a version but not a verack. +class NoVerackIdlePeer(LazyPeer): def __init__(self): self.version_received = False super().__init__() @@ -97,6 +83,7 @@ class P2PVersionStore(P2PInterface): version_received = None def on_version(self, msg): + # Responds with an appropriate verack super().on_version(msg) self.version_received = msg @@ -106,39 +93,45 @@ class P2PLeakTest(BitcoinTestFramework): self.num_nodes = 1 def run_test(self): - no_version_disconnect_node = self.nodes[0].add_p2p_connection( - CNodeNoVersionMisbehavior(), send_version=False, wait_for_verack=False) - no_version_idlenode = self.nodes[0].add_p2p_connection(CNodeNoVersionIdle(), send_version=False, wait_for_verack=False) - no_verack_idlenode = self.nodes[0].add_p2p_connection(CNodeNoVerackIdle(), wait_for_verack=False) + # Peer that never sends a version. We will send a bunch of messages + # from this peer anyway and verify eventual disconnection. + no_version_disconnect_peer = self.nodes[0].add_p2p_connection( + LazyPeer(), send_version=False, wait_for_verack=False) + + # Another peer that never sends a version, nor any other messages. It shouldn't receive anything from the node. + no_version_idle_peer = self.nodes[0].add_p2p_connection(LazyPeer(), send_version=False, wait_for_verack=False) + + # Peer that sends a version but not a verack. + no_verack_idle_peer = self.nodes[0].add_p2p_connection(NoVerackIdlePeer(), wait_for_verack=False) - # Send enough veracks without a message to reach the peer discouragement - # threshold. This should get us disconnected. + # Send enough ping messages (any non-version message will do) prior to sending + # version to reach the peer discouragement threshold. This should get us disconnected. for _ in range(DISCOURAGEMENT_THRESHOLD): - no_version_disconnect_node.send_message(msg_verack()) + no_version_disconnect_peer.send_message(msg_ping()) - # Wait until we got the verack in response to the version. Though, don't wait for the other node to receive the + # Wait until we got the verack in response to the version. Though, don't wait for the node to receive the # verack, since we never sent one - no_verack_idlenode.wait_for_verack() + no_verack_idle_peer.wait_for_verack() - wait_until(lambda: no_version_disconnect_node.ever_connected, timeout=10, lock=mininode_lock) - wait_until(lambda: no_version_idlenode.ever_connected, timeout=10, lock=mininode_lock) - wait_until(lambda: no_verack_idlenode.version_received, timeout=10, lock=mininode_lock) + self.wait_until(lambda: no_version_disconnect_peer.ever_connected, timeout=10, lock=p2p_lock) + self.wait_until(lambda: no_version_idle_peer.ever_connected, timeout=10, lock=p2p_lock) + self.wait_until(lambda: no_verack_idle_peer.version_received, timeout=10, lock=p2p_lock) - # Mine a block and make sure that it's not sent to the connected nodes - self.nodes[0].generatetoaddress(1, self.nodes[0].get_deterministic_priv_key().address) + # Mine a block and make sure that it's not sent to the connected peers + self.nodes[0].generate(nblocks=1) #Give the node enough time to possibly leak out a message time.sleep(5) - # Expect this node to be disconnected for misbehavior - assert not no_version_disconnect_node.is_connected + # Expect this peer to be disconnected for misbehavior + assert not no_version_disconnect_peer.is_connected self.nodes[0].disconnect_p2ps() # Make sure no unexpected messages came in - assert no_version_disconnect_node.unexpected_msg == False - assert no_version_idlenode.unexpected_msg == False - assert no_verack_idlenode.unexpected_msg == False + assert no_version_disconnect_peer.unexpected_msg == False + assert no_version_idle_peer.unexpected_msg == False + assert no_verack_idle_peer.unexpected_msg == False self.log.info('Check that the version message does not leak the local address of the node') p2p_version_store = self.nodes[0].add_p2p_connection(P2PVersionStore()) @@ -151,13 +144,13 @@ class P2PLeakTest(BitcoinTestFramework): assert_equal(ver.nStartingHeight, 201) assert_equal(ver.nRelay, 1) - self.log.info('Check that old nodes are disconnected') - p2p_old_node = self.nodes[0].add_p2p_connection(P2PInterface(), send_version=False, wait_for_verack=False) + self.log.info('Check that old peers are disconnected') + p2p_old_peer = self.nodes[0].add_p2p_connection(P2PInterface(), send_version=False, wait_for_verack=False) old_version_msg = msg_version() old_version_msg.nVersion = 31799 with self.nodes[0].assert_debug_log(['peer=4 using obsolete version 31799; disconnecting']): - p2p_old_node.send_message(old_version_msg) - p2p_old_node.wait_for_disconnect() + p2p_old_peer.send_message(old_version_msg) + p2p_old_peer.wait_for_disconnect() if __name__ == '__main__': diff --git a/test/functional/p2p_leak_tx.py b/test/functional/p2p_leak_tx.py index da30ad5977..9e761db03f 100755 --- a/test/functional/p2p_leak_tx.py +++ b/test/functional/p2p_leak_tx.py @@ -5,7 +5,7 @@ """Test that we don't leak txs to inbound peers that we haven't yet announced to""" from test_framework.messages import msg_getdata, CInv, MSG_TX -from test_framework.mininode import P2PDataStore +from test_framework.p2p import P2PDataStore from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, diff --git a/test/functional/p2p_nobloomfilter_messages.py b/test/functional/p2p_nobloomfilter_messages.py index accc5dc23c..c2311cb197 100755 --- a/test/functional/p2p_nobloomfilter_messages.py +++ b/test/functional/p2p_nobloomfilter_messages.py @@ -12,7 +12,7 @@ Test that, when bloom filters are not enabled, peers are disconnected if: """ from test_framework.messages import msg_mempool, msg_filteradd, msg_filterload, msg_filterclear -from test_framework.mininode import P2PInterface +from test_framework.p2p import P2PInterface from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal diff --git a/test/functional/p2p_node_network_limited.py b/test/functional/p2p_node_network_limited.py index a2f6ea538c..2c9cbea5e4 100755 --- a/test/functional/p2p_node_network_limited.py +++ b/test/functional/p2p_node_network_limited.py @@ -9,13 +9,12 @@ and that it responds to getdata requests for blocks correctly: - send a block within 288 + 2 of the tip - disconnect peers who request blocks older than that.""" from test_framework.messages import CInv, MSG_BLOCK, msg_getdata, msg_verack, NODE_NETWORK_LIMITED, NODE_WITNESS -from test_framework.mininode import P2PInterface, mininode_lock +from test_framework.p2p import P2PInterface from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, disconnect_nodes, connect_nodes, - wait_until, ) @@ -28,7 +27,7 @@ class P2PIgnoreInv(P2PInterface): self.firstAddrnServices = message.addrs[0].nServices def wait_for_addr(self, timeout=5): test_function = lambda: self.last_message.get("addr") - wait_until(test_function, timeout=timeout, lock=mininode_lock) + self.wait_until(test_function, timeout=timeout) def send_getdata_for_block(self, blockhash): getdata_request = msg_getdata() getdata_request.inv.append(CInv(MSG_BLOCK, int(blockhash, 16))) diff --git a/test/functional/p2p_permissions.py b/test/functional/p2p_permissions.py index 254352c816..3ec36edb41 100755 --- a/test/functional/p2p_permissions.py +++ b/test/functional/p2p_permissions.py @@ -13,7 +13,7 @@ from test_framework.messages import ( CTxInWitness, FromHex, ) -from test_framework.mininode import P2PDataStore +from test_framework.p2p import P2PDataStore from test_framework.script import ( CScript, OP_TRUE, @@ -24,7 +24,6 @@ from test_framework.util import ( assert_equal, connect_nodes, p2p_port, - wait_until, ) @@ -109,7 +108,7 @@ class P2PPermissionsTests(BitcoinTestFramework): self.sync_all() self.log.debug("Create a connection from a forcerelay peer that rebroadcasts raw txs") - # A python mininode is needed to send the raw transaction directly. If a full node was used, it could only + # A test framework p2p connection is needed to send the raw transaction directly. If a full node was used, it could only # rebroadcast via the inv-getdata mechanism. However, even for forcerelay connections, a full node would # currently not request a txid that is already in the mempool. self.restart_node(1, extra_args=["-whitelist=forcerelay@127.0.0.1"]) @@ -137,7 +136,7 @@ class P2PPermissionsTests(BitcoinTestFramework): connect_nodes(self.nodes[1], 0) with self.nodes[1].assert_debug_log(["Force relaying tx {} from peer=0".format(txid)]): p2p_rebroadcast_wallet.send_txs_and_test([tx], self.nodes[1]) - wait_until(lambda: txid in self.nodes[0].getrawmempool()) + self.wait_until(lambda: txid in self.nodes[0].getrawmempool()) self.log.debug("Check that node[1] will not send an invalid tx to node[0]") tx.vout[0].nValue += 1 diff --git a/test/functional/p2p_ping.py b/test/functional/p2p_ping.py index 5f5fd3e104..888e986fba 100755 --- a/test/functional/p2p_ping.py +++ b/test/functional/p2p_ping.py @@ -8,7 +8,7 @@ import time from test_framework.messages import msg_pong -from test_framework.mininode import P2PInterface +from test_framework.p2p import P2PInterface from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal diff --git a/test/functional/p2p_segwit.py b/test/functional/p2p_segwit.py index 564e49f3d8..2155c1d0e7 100755 --- a/test/functional/p2p_segwit.py +++ b/test/functional/p2p_segwit.py @@ -42,9 +42,9 @@ from test_framework.messages import ( uint256_from_str, FromHex, ) -from test_framework.mininode import ( +from test_framework.p2p import ( P2PInterface, - mininode_lock, + p2p_lock, ) from test_framework.script import ( CScript, @@ -83,7 +83,6 @@ from test_framework.util import ( softfork_active, hex_str_to_bytes, assert_raises_rpc_error, - wait_until, ) # The versionbit bit used to signal activation of SegWit @@ -153,8 +152,8 @@ class TestP2PConn(P2PInterface): self.lastgetdata = [] self.wtxidrelay = wtxidrelay - # Avoid sending out msg_getdata in the mininode thread as a reply to invs. - # They are not needed and would only lead to races because we send msg_getdata out in the test thread + # Don't send getdata message replies to invs automatically. + # We'll send the getdata messages explicitly in the test logic. def on_inv(self, message): pass @@ -177,7 +176,7 @@ class TestP2PConn(P2PInterface): if success: # sanity check assert (self.wtxidrelay and use_wtxid) or (not self.wtxidrelay and not use_wtxid) - with mininode_lock: + with p2p_lock: self.last_message.pop("getdata", None) if use_wtxid: wtxid = tx.calc_sha256(True) @@ -195,7 +194,7 @@ class TestP2PConn(P2PInterface): assert not self.last_message.get("getdata") def announce_block_and_wait_for_getdata(self, block, use_header, timeout=60): - with mininode_lock: + with p2p_lock: self.last_message.pop("getdata", None) self.last_message.pop("getheaders", None) msg = msg_headers() @@ -209,7 +208,7 @@ class TestP2PConn(P2PInterface): self.wait_for_getdata([block.sha256]) def request_block(self, blockhash, inv_type, timeout=60): - with mininode_lock: + with p2p_lock: self.last_message.pop("block", None) self.send_message(msg_getdata(inv=[CInv(inv_type, blockhash)])) self.wait_for_block(blockhash, timeout) @@ -2114,7 +2113,7 @@ class SegWitTest(BitcoinTestFramework): # Check wtxidrelay feature negotiation message through connecting a new peer def received_wtxidrelay(): return (len(self.wtx_node.last_wtxidrelay) > 0) - wait_until(received_wtxidrelay, timeout=60, lock=mininode_lock) + self.wtx_node.wait_until(received_wtxidrelay) # Create a Segwit output from the latest UTXO # and announce it to the network @@ -2138,25 +2137,25 @@ class SegWitTest(BitcoinTestFramework): # Announce Segwit transaction with wtxid # and wait for getdata self.wtx_node.announce_tx_and_wait_for_getdata(tx2, use_wtxid=True) - with mininode_lock: + with p2p_lock: lgd = self.wtx_node.lastgetdata[:] assert_equal(lgd, [CInv(MSG_WTX, tx2.calc_sha256(True))]) # Announce Segwit transaction from non wtxidrelay peer # and wait for getdata self.tx_node.announce_tx_and_wait_for_getdata(tx2, use_wtxid=False) - with mininode_lock: + with p2p_lock: lgd = self.tx_node.lastgetdata[:] assert_equal(lgd, [CInv(MSG_TX|MSG_WITNESS_FLAG, tx2.sha256)]) # Send tx2 through; it's an orphan so won't be accepted - with mininode_lock: + with p2p_lock: self.wtx_node.last_message.pop("getdata", None) test_transaction_acceptance(self.nodes[0], self.wtx_node, tx2, with_witness=True, accepted=False) # Expect a request for parent (tx) by txid despite use of WTX peer self.wtx_node.wait_for_getdata([tx.sha256], 60) - with mininode_lock: + with p2p_lock: lgd = self.wtx_node.lastgetdata[:] assert_equal(lgd, [CInv(MSG_TX|MSG_WITNESS_FLAG, tx.sha256)]) diff --git a/test/functional/p2p_sendheaders.py b/test/functional/p2p_sendheaders.py index 126a46bd53..04e6ec4172 100755 --- a/test/functional/p2p_sendheaders.py +++ b/test/functional/p2p_sendheaders.py @@ -87,11 +87,11 @@ e. Announce one more that doesn't connect. """ from test_framework.blocktools import create_block, create_coinbase from test_framework.messages import CInv -from test_framework.mininode import ( +from test_framework.p2p import ( CBlockHeader, NODE_WITNESS, P2PInterface, - mininode_lock, + p2p_lock, MSG_BLOCK, msg_block, msg_getblocks, @@ -104,7 +104,6 @@ from test_framework.mininode import ( from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, - wait_until, ) DIRECT_FETCH_RESPONSE_TIME = 0.05 @@ -147,7 +146,7 @@ class BaseNode(P2PInterface): def wait_for_block_announcement(self, block_hash, timeout=60): test_function = lambda: self.last_blockhash_announced == block_hash - wait_until(test_function, timeout=timeout, lock=mininode_lock) + self.wait_until(test_function, timeout=timeout) def on_inv(self, message): self.block_announced = True @@ -163,7 +162,7 @@ class BaseNode(P2PInterface): self.last_blockhash_announced = message.headers[-1].sha256 def clear_block_announcements(self): - with mininode_lock: + with p2p_lock: self.block_announced = False self.last_message.pop("inv", None) self.last_message.pop("headers", None) @@ -174,8 +173,8 @@ class BaseNode(P2PInterface): """Test whether the last headers announcements received are right. Headers may be announced across more than one message.""" test_function = lambda: (len(self.recent_headers_announced) >= len(headers)) - wait_until(test_function, timeout=60, lock=mininode_lock) - with mininode_lock: + self.wait_until(test_function) + with p2p_lock: assert_equal(self.recent_headers_announced, headers) self.block_announced = False self.last_message.pop("headers", None) @@ -186,9 +185,9 @@ class BaseNode(P2PInterface): inv should be a list of block hashes.""" test_function = lambda: self.block_announced - wait_until(test_function, timeout=60, lock=mininode_lock) + self.wait_until(test_function) - with mininode_lock: + with p2p_lock: compare_inv = [] if "inv" in self.last_message: compare_inv = [x.hash for x in self.last_message["inv"].inv] @@ -298,7 +297,7 @@ class SendHeadersTest(BitcoinTestFramework): test_node.send_header_for_blocks([new_block]) test_node.wait_for_getdata([new_block.sha256]) test_node.send_and_ping(msg_block(new_block)) # make sure this block is processed - wait_until(lambda: inv_node.block_announced, timeout=60, lock=mininode_lock) + inv_node.wait_until(lambda: inv_node.block_announced) inv_node.clear_block_announcements() test_node.clear_block_announcements() @@ -456,7 +455,7 @@ class SendHeadersTest(BitcoinTestFramework): test_node.send_header_for_blocks(blocks) test_node.sync_with_ping() # should not have received any getdata messages - with mininode_lock: + with p2p_lock: assert "getdata" not in test_node.last_message # This time, direct fetch should work @@ -494,7 +493,7 @@ class SendHeadersTest(BitcoinTestFramework): test_node.last_message.pop("getdata", None) test_node.send_header_for_blocks(blocks[0:1]) test_node.sync_with_ping() - with mininode_lock: + with p2p_lock: assert "getdata" not in test_node.last_message # Announcing one more block on fork should trigger direct fetch for @@ -513,7 +512,7 @@ class SendHeadersTest(BitcoinTestFramework): test_node.last_message.pop("getdata", None) test_node.send_header_for_blocks(blocks[18:19]) test_node.sync_with_ping() - with mininode_lock: + with p2p_lock: assert "getdata" not in test_node.last_message self.log.info("Part 4: success!") @@ -536,7 +535,7 @@ class SendHeadersTest(BitcoinTestFramework): block_time += 1 height += 1 # Send the header of the second block -> this won't connect. - with mininode_lock: + with p2p_lock: test_node.last_message.pop("getheaders", None) test_node.send_header_for_blocks([blocks[1]]) test_node.wait_for_getheaders() @@ -559,7 +558,7 @@ class SendHeadersTest(BitcoinTestFramework): for i in range(1, MAX_UNCONNECTING_HEADERS): # Send a header that doesn't connect, check that we get a getheaders. - with mininode_lock: + with p2p_lock: test_node.last_message.pop("getheaders", None) test_node.send_header_for_blocks([blocks[i]]) test_node.wait_for_getheaders() @@ -574,7 +573,7 @@ class SendHeadersTest(BitcoinTestFramework): # before we get disconnected. Should be 5*MAX_UNCONNECTING_HEADERS for i in range(5 * MAX_UNCONNECTING_HEADERS - 1): # Send a header that doesn't connect, check that we get a getheaders. - with mininode_lock: + with p2p_lock: test_node.last_message.pop("getheaders", None) test_node.send_header_for_blocks([blocks[i % len(blocks)]]) test_node.wait_for_getheaders() diff --git a/test/functional/p2p_timeouts.py b/test/functional/p2p_timeouts.py index 5a4fa42988..ce12ce26ce 100755 --- a/test/functional/p2p_timeouts.py +++ b/test/functional/p2p_timeouts.py @@ -24,7 +24,7 @@ from time import sleep from test_framework.messages import msg_ping -from test_framework.mininode import P2PInterface +from test_framework.p2p import P2PInterface from test_framework.test_framework import BitcoinTestFramework diff --git a/test/functional/p2p_tx_download.py b/test/functional/p2p_tx_download.py index 3ea1c6e5e7..653c7ae43f 100755 --- a/test/functional/p2p_tx_download.py +++ b/test/functional/p2p_tx_download.py @@ -16,14 +16,13 @@ from test_framework.messages import ( msg_inv, msg_notfound, ) -from test_framework.mininode import ( +from test_framework.p2p import ( P2PInterface, - mininode_lock, + p2p_lock, ) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, - wait_until, ) from test_framework.address import ADDRESS_BCRT1_UNSPENDABLE @@ -73,14 +72,14 @@ class TxDownloadTest(BitcoinTestFramework): def getdata_found(peer_index): p = self.nodes[0].p2ps[peer_index] - with mininode_lock: + with p2p_lock: return p.last_message.get("getdata") and p.last_message["getdata"].inv[-1].hash == txid node_0_mocktime = int(time.time()) while outstanding_peer_index: node_0_mocktime += MAX_GETDATA_INBOUND_WAIT self.nodes[0].setmocktime(node_0_mocktime) - wait_until(lambda: any(getdata_found(i) for i in outstanding_peer_index)) + self.wait_until(lambda: any(getdata_found(i) for i in outstanding_peer_index)) for i in outstanding_peer_index: if getdata_found(i): outstanding_peer_index.remove(i) @@ -134,24 +133,24 @@ class TxDownloadTest(BitcoinTestFramework): p = self.nodes[0].p2ps[0] - with mininode_lock: + with p2p_lock: p.tx_getdata_count = 0 p.send_message(msg_inv([CInv(t=MSG_WTX, h=i) for i in txids])) - wait_until(lambda: p.tx_getdata_count >= MAX_GETDATA_IN_FLIGHT, lock=mininode_lock) - with mininode_lock: + p.wait_until(lambda: p.tx_getdata_count >= MAX_GETDATA_IN_FLIGHT) + with p2p_lock: assert_equal(p.tx_getdata_count, MAX_GETDATA_IN_FLIGHT) self.log.info("Now check that if we send a NOTFOUND for a transaction, we'll get one more request") p.send_message(msg_notfound(vec=[CInv(t=MSG_WTX, h=txids[0])])) - wait_until(lambda: p.tx_getdata_count >= MAX_GETDATA_IN_FLIGHT + 1, timeout=10, lock=mininode_lock) - with mininode_lock: + p.wait_until(lambda: p.tx_getdata_count >= MAX_GETDATA_IN_FLIGHT + 1, timeout=10) + with p2p_lock: assert_equal(p.tx_getdata_count, MAX_GETDATA_IN_FLIGHT + 1) WAIT_TIME = TX_EXPIRY_INTERVAL // 2 + TX_EXPIRY_INTERVAL self.log.info("if we wait about {} minutes, we should eventually get more requests".format(WAIT_TIME / 60)) self.nodes[0].setmocktime(int(time.time() + WAIT_TIME)) - wait_until(lambda: p.tx_getdata_count == MAX_GETDATA_IN_FLIGHT + 2) + p.wait_until(lambda: p.tx_getdata_count == MAX_GETDATA_IN_FLIGHT + 2) self.nodes[0].setmocktime(0) def test_spurious_notfound(self): diff --git a/test/functional/p2p_unrequested_blocks.py b/test/functional/p2p_unrequested_blocks.py index 71b0b0f63a..36b434bce3 100755 --- a/test/functional/p2p_unrequested_blocks.py +++ b/test/functional/p2p_unrequested_blocks.py @@ -55,7 +55,7 @@ import time from test_framework.blocktools import create_block, create_coinbase, create_tx_with_script from test_framework.messages import CBlockHeader, CInv, MSG_BLOCK, msg_block, msg_headers, msg_inv -from test_framework.mininode import mininode_lock, P2PInterface +from test_framework.p2p import p2p_lock, P2PInterface from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, @@ -199,13 +199,13 @@ class AcceptBlockTest(BitcoinTestFramework): # 6. Try to get node to request the missing block. # Poke the node with an inv for block at height 3 and see if that # triggers a getdata on block 2 (it should if block 2 is missing). - with mininode_lock: + with p2p_lock: # Clear state so we can check the getdata request test_node.last_message.pop("getdata", None) test_node.send_message(msg_inv([CInv(MSG_BLOCK, block_h3.sha256)])) test_node.sync_with_ping() - with mininode_lock: + with p2p_lock: getdata = test_node.last_message["getdata"] # Check that the getdata includes the right block diff --git a/test/functional/rpc_blockchain.py b/test/functional/rpc_blockchain.py index 7c70f30ca3..c005584485 100755 --- a/test/functional/rpc_blockchain.py +++ b/test/functional/rpc_blockchain.py @@ -22,6 +22,17 @@ from decimal import Decimal import http.client import subprocess +from test_framework.blocktools import ( + create_block, + create_coinbase, + TIME_GENESIS_BLOCK, +) +from test_framework.messages import ( + CBlockHeader, + FromHex, + msg_block, +) +from test_framework.p2p import P2PInterface from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, @@ -32,17 +43,6 @@ from test_framework.util import ( assert_is_hex_string, assert_is_hash_string, ) -from test_framework.blocktools import ( - create_block, - create_coinbase, - TIME_GENESIS_BLOCK, -) -from test_framework.messages import ( - msg_block, -) -from test_framework.mininode import ( - P2PInterface, -) class BlockchainTest(BitcoinTestFramework): @@ -280,6 +280,14 @@ class BlockchainTest(BitcoinTestFramework): assert isinstance(int(header['versionHex'], 16), int) assert isinstance(header['difficulty'], Decimal) + # Test with verbose=False, which should return the header as hex. + header_hex = node.getblockheader(blockhash=besthash, verbose=False) + assert_is_hex_string(header_hex) + + header = FromHex(CBlockHeader(), header_hex) + header.calc_sha256() + assert_equal(header.hash, besthash) + def _test_getdifficulty(self): difficulty = self.nodes[0].getdifficulty() # 1 hash in 2 should be valid, so difficulty should be 1/2**31 diff --git a/test/functional/rpc_invalidateblock.py b/test/functional/rpc_invalidateblock.py index 1fdc134f97..e788e75557 100755 --- a/test/functional/rpc_invalidateblock.py +++ b/test/functional/rpc_invalidateblock.py @@ -9,7 +9,6 @@ from test_framework.address import ADDRESS_BCRT1_UNSPENDABLE_DESCRIPTOR from test_framework.util import ( assert_equal, connect_nodes, - wait_until, ) @@ -57,9 +56,9 @@ class InvalidateTest(BitcoinTestFramework): self.log.info("..and then mine a block") self.nodes[2].generatetoaddress(1, self.nodes[2].get_deterministic_priv_key().address) self.log.info("Verify all nodes are at the right height") - wait_until(lambda: self.nodes[2].getblockcount() == 3, timeout=5) - wait_until(lambda: self.nodes[0].getblockcount() == 4, timeout=5) - wait_until(lambda: self.nodes[1].getblockcount() == 4, timeout=5) + self.wait_until(lambda: self.nodes[2].getblockcount() == 3, timeout=5) + self.wait_until(lambda: self.nodes[0].getblockcount() == 4, timeout=5) + self.wait_until(lambda: self.nodes[1].getblockcount() == 4, timeout=5) self.log.info("Verify that we reconsider all ancestors as well") blocks = self.nodes[1].generatetodescriptor(10, ADDRESS_BCRT1_UNSPENDABLE_DESCRIPTOR) diff --git a/test/functional/rpc_misc.py b/test/functional/rpc_misc.py index cc5a264adb..0493ceeb64 100755 --- a/test/functional/rpc_misc.py +++ b/test/functional/rpc_misc.py @@ -61,6 +61,34 @@ class RpcMiscTest(BitcoinTestFramework): node.logging(include=['qt']) assert_equal(node.logging()['qt'], True) + self.log.info("test getindexinfo") + # Without any indices running the RPC returns an empty object + assert_equal(node.getindexinfo(), {}) + + # Restart the node with indices and wait for them to sync + self.restart_node(0, ["-txindex", "-blockfilterindex"]) + self.wait_until(lambda: all(i["synced"] for i in node.getindexinfo().values())) + + # Returns a list of all running indices by default + assert_equal( + node.getindexinfo(), + { + "txindex": {"synced": True, "best_block_height": 200}, + "basic block filter index": {"synced": True, "best_block_height": 200} + } + ) + + # Specifying an index by name returns only the status of that index + assert_equal( + node.getindexinfo("txindex"), + { + "txindex": {"synced": True, "best_block_height": 200}, + } + ) + + # Specifying an unknown index name returns an empty result + assert_equal(node.getindexinfo("foo"), {}) + if __name__ == '__main__': RpcMiscTest().main() diff --git a/test/functional/rpc_net.py b/test/functional/rpc_net.py index 192b60e5d2..506c77c567 100755 --- a/test/functional/rpc_net.py +++ b/test/functional/rpc_net.py @@ -8,22 +8,24 @@ Tests correspond to code in rpc/net.cpp. """ from decimal import Decimal +from itertools import product +import time +from test_framework.p2p import P2PInterface +import test_framework.messages +from test_framework.messages import ( + NODE_NETWORK, + NODE_WITNESS, +) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( + assert_approx, assert_equal, assert_greater_than_or_equal, assert_greater_than, assert_raises_rpc_error, connect_nodes, p2p_port, - wait_until, -) -from test_framework.mininode import P2PInterface -import test_framework.messages -from test_framework.messages import ( - NODE_NETWORK, - NODE_WITNESS, ) @@ -48,25 +50,27 @@ class NetTest(BitcoinTestFramework): self.supports_cli = False def run_test(self): - self.log.info('Get out of IBD for the minfeefilter test') - self.nodes[0].generate(1) - self.log.info('Connect nodes both way') + # Get out of IBD for the minfeefilter and getpeerinfo tests. + self.nodes[0].generate(101) + # Connect nodes both ways. connect_nodes(self.nodes[0], 1) connect_nodes(self.nodes[1], 0) - self._test_connection_count() - self._test_getnettotals() - self._test_getnetworkinfo() - self._test_getaddednodeinfo() - self._test_getpeerinfo() + self.test_connection_count() + self.test_getpeerinfo() + self.test_getnettotals() + self.test_getnetworkinfo() + self.test_getaddednodeinfo() self.test_service_flags() - self._test_getnodeaddresses() + self.test_getnodeaddresses() - def _test_connection_count(self): - # connect_nodes connects each node to the other + def test_connection_count(self): + self.log.info("Test getconnectioncount") + # After using `connect_nodes` to connect nodes 0 and 1 to each other. assert_equal(self.nodes[0].getconnectioncount(), 2) - def _test_getnettotals(self): + def test_getnettotals(self): + self.log.info("Test getnettotals") # getnettotals totalbytesrecv and totalbytessent should be # consistent with getpeerinfo. Since the RPC calls are not atomic, # and messages might have been recvd or sent between RPC calls, call @@ -88,15 +92,16 @@ class NetTest(BitcoinTestFramework): # the bytes sent/received should change # note ping and pong are 32 bytes each self.nodes[0].ping() - wait_until(lambda: (self.nodes[0].getnettotals()['totalbytessent'] >= net_totals_after['totalbytessent'] + 32 * 2), timeout=1) - wait_until(lambda: (self.nodes[0].getnettotals()['totalbytesrecv'] >= net_totals_after['totalbytesrecv'] + 32 * 2), timeout=1) + self.wait_until(lambda: (self.nodes[0].getnettotals()['totalbytessent'] >= net_totals_after['totalbytessent'] + 32 * 2), timeout=1) + self.wait_until(lambda: (self.nodes[0].getnettotals()['totalbytesrecv'] >= net_totals_after['totalbytesrecv'] + 32 * 2), timeout=1) peer_info_after_ping = self.nodes[0].getpeerinfo() for before, after in zip(peer_info, peer_info_after_ping): assert_greater_than_or_equal(after['bytesrecv_per_msg'].get('pong', 0), before['bytesrecv_per_msg'].get('pong', 0) + 32) assert_greater_than_or_equal(after['bytessent_per_msg'].get('ping', 0), before['bytessent_per_msg'].get('ping', 0) + 32) - def _test_getnetworkinfo(self): + def test_getnetworkinfo(self): + self.log.info("Test getnetworkinfo") assert_equal(self.nodes[0].getnetworkinfo()['networkactive'], True) assert_equal(self.nodes[0].getnetworkinfo()['connections'], 2) @@ -104,11 +109,11 @@ class NetTest(BitcoinTestFramework): self.nodes[0].setnetworkactive(state=False) assert_equal(self.nodes[0].getnetworkinfo()['networkactive'], False) # Wait a bit for all sockets to close - wait_until(lambda: self.nodes[0].getnetworkinfo()['connections'] == 0, timeout=3) + self.wait_until(lambda: self.nodes[0].getnetworkinfo()['connections'] == 0, timeout=3) with self.nodes[0].assert_debug_log(expected_msgs=['SetNetworkActive: true\n']): self.nodes[0].setnetworkactive(state=True) - self.log.info('Connect nodes both way') + # Connect nodes both ways. connect_nodes(self.nodes[0], 1) connect_nodes(self.nodes[1], 0) @@ -120,7 +125,8 @@ class NetTest(BitcoinTestFramework): for info in network_info: assert_net_servicesnames(int(info["localservices"], 0x10), info["localservicesnames"]) - def _test_getaddednodeinfo(self): + def test_getaddednodeinfo(self): + self.log.info("Test getaddednodeinfo") assert_equal(self.nodes[0].getaddednodeinfo(), []) # add a node (node2) to node0 ip_port = "127.0.0.1:{}".format(p2p_port(2)) @@ -139,8 +145,20 @@ class NetTest(BitcoinTestFramework): # check that a non-existent node returns an error assert_raises_rpc_error(-24, "Node has not been added", self.nodes[0].getaddednodeinfo, '1.1.1.1') - def _test_getpeerinfo(self): + def test_getpeerinfo(self): + self.log.info("Test getpeerinfo") + # Create a few getpeerinfo last_block/last_transaction values. + if self.is_wallet_compiled(): + self.nodes[0].sendtoaddress(self.nodes[1].getnewaddress(), 1) + self.nodes[1].generate(1) + self.sync_all() + time_now = int(time.time()) peer_info = [x.getpeerinfo() for x in self.nodes] + # Verify last_block and last_transaction keys/values. + for node, peer, field in product(range(self.num_nodes), range(2), ['last_block', 'last_transaction']): + assert field in peer_info[node][peer].keys() + if peer_info[node][peer][field] != 0: + assert_approx(peer_info[node][peer][field], time_now, vspan=60) # check both sides of bidirectional connection between nodes # the address bound to on one side will be the source address for the other node assert_equal(peer_info[0][0]['addrbind'], peer_info[1][0]['addr']) @@ -152,11 +170,13 @@ class NetTest(BitcoinTestFramework): assert_net_servicesnames(int(info[0]["services"], 0x10), info[0]["servicesnames"]) def test_service_flags(self): + self.log.info("Test service flags") self.nodes[0].add_p2p_connection(P2PInterface(), services=(1 << 4) | (1 << 63)) assert_equal(['UNKNOWN[2^4]', 'UNKNOWN[2^63]'], self.nodes[0].getpeerinfo()[-1]['servicesnames']) self.nodes[0].disconnect_p2ps() - def _test_getnodeaddresses(self): + def test_getnodeaddresses(self): + self.log.info("Test getnodeaddresses") self.nodes[0].add_p2p_connection(P2PInterface()) # Add some addresses to the Address Manager over RPC. Due to the way diff --git a/test/functional/rpc_psbt.py b/test/functional/rpc_psbt.py index f7f23bc8f4..1c7dc98d16 100755 --- a/test/functional/rpc_psbt.py +++ b/test/functional/rpc_psbt.py @@ -103,7 +103,16 @@ class PSBTTest(BitcoinTestFramework): final_tx = self.nodes[0].finalizepsbt(signed_tx)['hex'] self.nodes[0].sendrawtransaction(final_tx) - # Get pubkeys + # Manually selected inputs can be locked: + assert_equal(len(self.nodes[0].listlockunspent()), 0) + utxo1 = self.nodes[0].listunspent()[0] + psbtx1 = self.nodes[0].walletcreatefundedpsbt([{"txid": utxo1['txid'], "vout": utxo1['vout']}], {self.nodes[2].getnewaddress():1}, 0,{"lockUnspents": True})["psbt"] + assert_equal(len(self.nodes[0].listlockunspent()), 1) + + # Locks are ignored for manually selected inputs + self.nodes[0].walletcreatefundedpsbt([{"txid": utxo1['txid'], "vout": utxo1['vout']}], {self.nodes[2].getnewaddress():1}, 0) + + # Create p2sh, p2wpkh, and p2wsh addresses pubkey0 = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress())['pubkey'] pubkey1 = self.nodes[1].getaddressinfo(self.nodes[1].getnewaddress())['pubkey'] pubkey2 = self.nodes[2].getaddressinfo(self.nodes[2].getnewaddress())['pubkey'] diff --git a/test/functional/test_framework/messages.py b/test/functional/test_framework/messages.py index 5207b563a1..bd4a53876e 100755 --- a/test/functional/test_framework/messages.py +++ b/test/functional/test_framework/messages.py @@ -32,7 +32,7 @@ from test_framework.util import hex_str_to_bytes, assert_equal MIN_VERSION_SUPPORTED = 60001 MY_VERSION = 70016 # past wtxid relay -MY_SUBVERSION = b"/python-mininode-tester:0.0.3/" +MY_SUBVERSION = b"/python-p2p-tester:0.0.3/" MY_RELAY = 1 # from version 70001 onwards, fRelay should be appended to version messages (BIP37) MAX_LOCATOR_SZ = 101 diff --git a/test/functional/test_framework/mininode.py b/test/functional/test_framework/p2p.py index eaf637fbb8..57c77e60b5 100755 --- a/test/functional/test_framework/mininode.py +++ b/test/functional/test_framework/p2p.py @@ -4,10 +4,14 @@ # Copyright (c) 2010-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Bitcoin P2P network half-a-node. +"""Test objects for interacting with a bitcoind node over the p2p protocol. -This python code was modified from ArtForz' public domain half-a-node, as -found in the mini-node branch of http://github.com/jgarzik/pynode. +The P2PInterface objects interact with the bitcoind nodes under test using the +node's p2p interface. They can be used to send messages to the node, and +callbacks can be registered that execute when messages are received from the +node. Messages are sent to/received from the node on an asyncio event loop. +State held inside the objects must be guarded by the p2p_lock to avoid data +races between the main testing thread and the event loop. P2PConnection: A low-level connection object to a node's P2P interface P2PInterface: A high-level interface object for communicating to a node over P2P @@ -67,7 +71,7 @@ from test_framework.messages import ( ) from test_framework.util import wait_until -logger = logging.getLogger("TestFramework.mininode") +logger = logging.getLogger("TestFramework.p2p") MESSAGEMAP = { b"addr": msg_addr, @@ -320,7 +324,7 @@ class P2PInterface(P2PConnection): We keep a count of how many of each message type has been received and the most recent message of each type.""" - with mininode_lock: + with p2p_lock: try: msgtype = message.msgtype.decode('ascii') self.message_count[msgtype] += 1 @@ -394,7 +398,7 @@ class P2PInterface(P2PConnection): assert self.is_connected return test_function_in() - wait_until(test_function, timeout=timeout, lock=mininode_lock, timeout_factor=self.timeout_factor) + wait_until(test_function, timeout=timeout, lock=p2p_lock, timeout_factor=self.timeout_factor) def wait_for_disconnect(self, timeout=60): test_function = lambda: not self.is_connected @@ -498,7 +502,7 @@ class P2PInterface(P2PConnection): # P2PConnection acquires this lock whenever delivering a message to a P2PInterface. # This lock should be acquired in the thread running the test logic to synchronize # access to any data shared with the P2PInterface or P2PConnection. -mininode_lock = threading.Lock() +p2p_lock = threading.Lock() class NetworkThread(threading.Thread): @@ -592,7 +596,7 @@ class P2PDataStore(P2PInterface): - if success is False: assert that the node's tip doesn't advance - if reject_reason is set: assert that the correct reject message is logged""" - with mininode_lock: + with p2p_lock: for block in blocks: self.block_store[block.sha256] = block self.last_block_hash = block.sha256 @@ -629,7 +633,7 @@ class P2PDataStore(P2PInterface): - if expect_disconnect is True: Skip the sync with ping - if reject_reason is set: assert that the correct reject message is logged.""" - with mininode_lock: + with p2p_lock: for tx in txs: self.tx_store[tx.sha256] = tx @@ -668,7 +672,7 @@ class P2PTxInvStore(P2PInterface): self.tx_invs_received[i.hash] += 1 def get_invs(self): - with mininode_lock: + with p2p_lock: return list(self.tx_invs_received.keys()) def wait_for_broadcast(self, txns, timeout=60): diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index 8d402d4888..2a60f8e0c1 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -20,8 +20,8 @@ import time from .authproxy import JSONRPCException from . import coverage +from .p2p import NetworkThread from .test_node import TestNode -from .mininode import NetworkThread from .util import ( MAX_NODES, PortSeed, diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index 8f0d45c7f9..5c7a883c43 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -551,7 +551,7 @@ class TestNode(): assert self.p2ps, self._node_msg("No p2p connection") return self.p2ps[0] - def num_connected_mininodes(self): + def num_test_p2p_connections(self): """Return number of test framework p2p connections to the node.""" return len([peer for peer in self.getpeerinfo() if peer['subver'] == MY_SUBVERSION]) @@ -560,7 +560,7 @@ class TestNode(): for p in self.p2ps: p.peer_disconnect() del self.p2ps[:] - wait_until(lambda: self.num_connected_mininodes() == 0) + wait_until(lambda: self.num_test_p2p_connections() == 0) class TestNodeCLIAttr: @@ -650,10 +650,10 @@ class RPCOverloadWrapper(): def __getattr__(self, name): return getattr(self.rpc, name) - def createwallet(self, wallet_name, disable_private_keys=None, blank=None, passphrase='', avoid_reuse=None, descriptors=None): + def createwallet(self, wallet_name, disable_private_keys=None, blank=None, passphrase='', avoid_reuse=None, descriptors=None, load_on_startup=None): if descriptors is None: descriptors = self.descriptors - return self.__getattr__('createwallet')(wallet_name, disable_private_keys, blank, passphrase, avoid_reuse, descriptors) + return self.__getattr__('createwallet')(wallet_name, disable_private_keys, blank, passphrase, avoid_reuse, descriptors, load_on_startup) def importprivkey(self, privkey, label=None, rescan=None): wallet_info = self.getwalletinfo() diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index 3362b41209..cfc4ee65d4 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -226,6 +226,14 @@ def satoshi_round(amount): def wait_until(predicate, *, attempts=float('inf'), timeout=float('inf'), lock=None, timeout_factor=1.0): + """Sleep until the predicate resolves to be True. + + Warning: Note that this method is not recommended to be used in tests as it is + not aware of the context of the test framework. Using `wait_until()` counterpart + from `BitcoinTestFramework` or `P2PInterface` class ensures an understandable + amount of timeout and a common shared timeout_factor. Furthermore, `wait_until()` + from `P2PInterface` class in `mininode.py` has a preset lock. + """ if attempts == float('inf') and timeout == float('inf'): timeout = 60 timeout = timeout * timeout_factor diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 67b9050123..01232bda3c 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -244,6 +244,7 @@ BASE_SCRIPTS = [ 'p2p_node_network_limited.py', 'p2p_permissions.py', 'feature_blocksdir.py', + 'wallet_startup.py', 'feature_config_args.py', 'feature_settings.py', 'rpc_getdescriptorinfo.py', diff --git a/test/functional/wallet_basic.py b/test/functional/wallet_basic.py index d9a8b58a84..c4db8e2119 100755 --- a/test/functional/wallet_basic.py +++ b/test/functional/wallet_basic.py @@ -12,7 +12,6 @@ from test_framework.util import ( assert_fee_amount, assert_raises_rpc_error, connect_nodes, - wait_until, ) from test_framework.wallet_util import test_address @@ -136,11 +135,19 @@ class WalletTest(BitcoinTestFramework): self.nodes[2].lockunspent, False, [{"txid": unspent_0["txid"], "vout": 999}]) - # An output should be unlocked when spent + # The lock on a manually selected output is ignored unspent_0 = self.nodes[1].listunspent()[0] self.nodes[1].lockunspent(False, [unspent_0]) tx = self.nodes[1].createrawtransaction([unspent_0], { self.nodes[1].getnewaddress() : 1 }) - tx = self.nodes[1].fundrawtransaction(tx)['hex'] + self.nodes[1].fundrawtransaction(tx,{"lockUnspents": True}) + + # fundrawtransaction can lock an input + self.nodes[1].lockunspent(True, [unspent_0]) + assert_equal(len(self.nodes[1].listlockunspent()), 0) + tx = self.nodes[1].fundrawtransaction(tx,{"lockUnspents": True})['hex'] + assert_equal(len(self.nodes[1].listlockunspent()), 1) + + # Send transaction tx = self.nodes[1].signrawtransactionwithwallet(tx)["hex"] self.nodes[1].sendrawtransaction(tx) assert_equal(len(self.nodes[1].listlockunspent()), 0) @@ -540,7 +547,7 @@ class WalletTest(BitcoinTestFramework): self.start_node(2, [m, "-limitancestorcount=" + str(chainlimit)]) if m == '-reindex': # reindex will leave rpc warm up "early"; Wait for it to finish - wait_until(lambda: [block_count] * 3 == [self.nodes[i].getblockcount() for i in range(3)]) + self.wait_until(lambda: [block_count] * 3 == [self.nodes[i].getblockcount() for i in range(3)]) assert_equal(balance_nodes, [self.nodes[i].getbalance() for i in range(3)]) # Exercise listsinceblock with the last two blocks @@ -589,7 +596,7 @@ class WalletTest(BitcoinTestFramework): self.start_node(0, extra_args=extra_args) # wait until the wallet has submitted all transactions to the mempool - wait_until(lambda: len(self.nodes[0].getrawmempool()) == chainlimit * 2) + self.wait_until(lambda: len(self.nodes[0].getrawmempool()) == chainlimit * 2) node0_balance = self.nodes[0].getbalance() # With walletrejectlongchains we will not create the tx and store it in our wallet. diff --git a/test/functional/wallet_bumpfee.py b/test/functional/wallet_bumpfee.py index 53496084ef..56d1da60b7 100755 --- a/test/functional/wallet_bumpfee.py +++ b/test/functional/wallet_bumpfee.py @@ -50,6 +50,11 @@ class BumpFeeTest(BitcoinTestFramework): def skip_test_if_missing_module(self): self.skip_if_no_wallet() + def clear_mempool(self): + # Clear mempool between subtests. The subtests may only depend on chainstate (utxos) + self.nodes[1].generate(1) + self.sync_all() + def run_test(self): # Encrypt wallet for test_locked_wallet_fails test self.nodes[1].encryptwallet(WALLET_PASSPHRASE) @@ -71,7 +76,7 @@ class BumpFeeTest(BitcoinTestFramework): self.log.info("Running tests") dest_address = peer_node.getnewaddress() - test_invalid_parameters(rbf_node, dest_address) + self.test_invalid_parameters(rbf_node, dest_address) test_simple_bumpfee_succeeds(self, "default", rbf_node, peer_node, dest_address) test_simple_bumpfee_succeeds(self, "fee_rate", rbf_node, peer_node, dest_address) test_feerate_args(self, rbf_node, peer_node, dest_address) @@ -93,28 +98,30 @@ class BumpFeeTest(BitcoinTestFramework): test_small_output_with_feerate_succeeds(self, rbf_node, dest_address) test_no_more_inputs_fails(self, rbf_node, dest_address) -def test_invalid_parameters(node, dest_address): - txid = spend_one_input(node, dest_address) - # invalid estimate mode - assert_raises_rpc_error(-8, "Invalid estimate_mode parameter", node.bumpfee, txid, { - "estimate_mode": "moo", - }) - assert_raises_rpc_error(-3, "Expected type string", node.bumpfee, txid, { - "estimate_mode": 38, - }) - assert_raises_rpc_error(-3, "Expected type string", node.bumpfee, txid, { - "estimate_mode": { - "foo": "bar", - }, - }) - assert_raises_rpc_error(-8, "Invalid estimate_mode parameter", node.bumpfee, txid, { - "estimate_mode": Decimal("3.141592"), - }) - # confTarget and conf_target - assert_raises_rpc_error(-8, "confTarget and conf_target options should not both be set", node.bumpfee, txid, { - "confTarget": 123, - "conf_target": 456, - }) + def test_invalid_parameters(self, node, dest_address): + txid = spend_one_input(node, dest_address) + # invalid estimate mode + assert_raises_rpc_error(-8, "Invalid estimate_mode parameter", node.bumpfee, txid, { + "estimate_mode": "moo", + }) + assert_raises_rpc_error(-3, "Expected type string", node.bumpfee, txid, { + "estimate_mode": 38, + }) + assert_raises_rpc_error(-3, "Expected type string", node.bumpfee, txid, { + "estimate_mode": { + "foo": "bar", + }, + }) + assert_raises_rpc_error(-8, "Invalid estimate_mode parameter", node.bumpfee, txid, { + "estimate_mode": Decimal("3.141592"), + }) + # confTarget and conf_target + assert_raises_rpc_error(-8, "confTarget and conf_target options should not both be set", node.bumpfee, txid, { + "confTarget": 123, + "conf_target": 456, + }) + self.clear_mempool() + def test_simple_bumpfee_succeeds(self, mode, rbf_node, peer_node, dest_address): self.log.info('Test simple bumpfee: {}'.format(mode)) @@ -148,6 +155,7 @@ def test_simple_bumpfee_succeeds(self, mode, rbf_node, peer_node, dest_address): bumpedwtx = rbf_node.gettransaction(bumped_tx["txid"]) assert_equal(oldwtx["replaced_by_txid"], bumped_tx["txid"]) assert_equal(bumpedwtx["replaces_txid"], rbfid) + self.clear_mempool() def test_feerate_args(self, rbf_node, peer_node, dest_address): @@ -167,6 +175,7 @@ def test_feerate_args(self, rbf_node, peer_node, dest_address): assert_raises_rpc_error(-3, "Amount out of range", rbf_node.bumpfee, rbfid, {"fee_rate": -1}) assert_raises_rpc_error(-4, "is too high (cannot be higher than", rbf_node.bumpfee, rbfid, {"fee_rate": TOO_HIGH}) + self.clear_mempool() def test_segwit_bumpfee_succeeds(self, rbf_node, dest_address): @@ -198,12 +207,14 @@ def test_segwit_bumpfee_succeeds(self, rbf_node, dest_address): bumped_tx = rbf_node.bumpfee(rbfid) assert bumped_tx["txid"] in rbf_node.getrawmempool() assert rbfid not in rbf_node.getrawmempool() + self.clear_mempool() def test_nonrbf_bumpfee_fails(self, peer_node, dest_address): self.log.info('Test that we cannot replace a non RBF transaction') not_rbfid = peer_node.sendtoaddress(dest_address, Decimal("0.00090000")) assert_raises_rpc_error(-4, "not BIP 125 replaceable", peer_node.bumpfee, not_rbfid) + self.clear_mempool() def test_notmine_bumpfee_fails(self, rbf_node, peer_node, dest_address): @@ -211,20 +222,22 @@ def test_notmine_bumpfee_fails(self, rbf_node, peer_node, dest_address): # here, the rbftx has a peer_node coin and then adds a rbf_node input # Note that this test depends upon the RPC code checking input ownership prior to change outputs # (since it can't use fundrawtransaction, it lacks a proper change output) - utxos = [node.listunspent()[-1] for node in (rbf_node, peer_node)] + fee = Decimal("0.001") + utxos = [node.listunspent(query_options={'minimumAmount': fee})[-1] for node in (rbf_node, peer_node)] inputs = [{ "txid": utxo["txid"], "vout": utxo["vout"], "address": utxo["address"], "sequence": BIP125_SEQUENCE_NUMBER } for utxo in utxos] - output_val = sum(utxo["amount"] for utxo in utxos) - Decimal("0.001") + output_val = sum(utxo["amount"] for utxo in utxos) - fee rawtx = rbf_node.createrawtransaction(inputs, {dest_address: output_val}) signedtx = rbf_node.signrawtransactionwithwallet(rawtx) signedtx = peer_node.signrawtransactionwithwallet(signedtx["hex"]) rbfid = rbf_node.sendrawtransaction(signedtx["hex"]) assert_raises_rpc_error(-4, "Transaction contains inputs that don't belong to this wallet", rbf_node.bumpfee, rbfid) + self.clear_mempool() def test_bumpfee_with_descendant_fails(self, rbf_node, rbf_node_address, dest_address): @@ -235,6 +248,7 @@ def test_bumpfee_with_descendant_fails(self, rbf_node, rbf_node_address, dest_ad tx = rbf_node.signrawtransactionwithwallet(tx) rbf_node.sendrawtransaction(tx["hex"]) assert_raises_rpc_error(-8, "Transaction has descendants in the wallet", rbf_node.bumpfee, parent_id) + self.clear_mempool() def test_small_output_with_feerate_succeeds(self, rbf_node, dest_address): @@ -276,6 +290,7 @@ def test_small_output_with_feerate_succeeds(self, rbf_node, dest_address): rbf_node.generatetoaddress(1, rbf_node.getnewaddress()) assert_equal(rbf_node.gettransaction(rbfid)["confirmations"], 1) + self.clear_mempool() def test_dust_to_fee(self, rbf_node, dest_address): @@ -298,6 +313,7 @@ def test_dust_to_fee(self, rbf_node, dest_address): assert_equal(len(fulltx["vout"]), 2) assert_equal(len(full_bumped_tx["vout"]), 1) # change output is eliminated assert_equal(full_bumped_tx["vout"][0]['value'], Decimal("0.00050000")) + self.clear_mempool() def test_settxfee(self, rbf_node, dest_address): @@ -320,6 +336,8 @@ def test_settxfee(self, rbf_node, dest_address): assert_raises_rpc_error(-8, "txfee cannot be more than wallet max tx fee", rbf_node.settxfee, Decimal('0.00003')) self.restart_node(1, self.extra_args[1]) rbf_node.walletpassphrase(WALLET_PASSPHRASE, WALLET_PASSPHRASE_TIMEOUT) + self.connect_nodes(1, 0) + self.clear_mempool() def test_maxtxfee_fails(self, rbf_node, dest_address): @@ -333,6 +351,8 @@ def test_maxtxfee_fails(self, rbf_node, dest_address): assert_raises_rpc_error(-4, "Unable to create transaction. Fee exceeds maximum configured by -maxtxfee", rbf_node.bumpfee, rbfid) self.restart_node(1, self.extra_args[1]) rbf_node.walletpassphrase(WALLET_PASSPHRASE, WALLET_PASSPHRASE_TIMEOUT) + self.connect_nodes(1, 0) + self.clear_mempool() def test_watchonly_psbt(self, peer_node, rbf_node, dest_address): @@ -415,6 +435,7 @@ def test_watchonly_psbt(self, peer_node, rbf_node, dest_address): rbf_node.unloadwallet("watcher") rbf_node.unloadwallet("signer") + self.clear_mempool() def test_rebumping(self, rbf_node, dest_address): @@ -423,6 +444,7 @@ def test_rebumping(self, rbf_node, dest_address): bumped = rbf_node.bumpfee(rbfid, {"fee_rate": ECONOMICAL}) assert_raises_rpc_error(-4, "already bumped", rbf_node.bumpfee, rbfid, {"fee_rate": NORMAL}) rbf_node.bumpfee(bumped["txid"], {"fee_rate": NORMAL}) + self.clear_mempool() def test_rebumping_not_replaceable(self, rbf_node, dest_address): @@ -431,6 +453,7 @@ def test_rebumping_not_replaceable(self, rbf_node, dest_address): bumped = rbf_node.bumpfee(rbfid, {"fee_rate": ECONOMICAL, "replaceable": False}) assert_raises_rpc_error(-4, "Transaction is not BIP 125 replaceable", rbf_node.bumpfee, bumped["txid"], {"fee_rate": NORMAL}) + self.clear_mempool() def test_unconfirmed_not_spendable(self, rbf_node, rbf_node_address): @@ -470,6 +493,7 @@ def test_unconfirmed_not_spendable(self, rbf_node, rbf_node_address): assert_equal( sum(1 for t in rbf_node.listunspent(minconf=0, include_unsafe=False) if t["txid"] == rbfid and t["address"] == rbf_node_address and t["spendable"]), 1) + self.clear_mempool() def test_bumpfee_metadata(self, rbf_node, dest_address): @@ -481,6 +505,7 @@ def test_bumpfee_metadata(self, rbf_node, dest_address): bumped_wtx = rbf_node.gettransaction(bumped_tx["txid"]) assert_equal(bumped_wtx["comment"], "comment value") assert_equal(bumped_wtx["to"], "to value") + self.clear_mempool() def test_locked_wallet_fails(self, rbf_node, dest_address): @@ -490,6 +515,7 @@ def test_locked_wallet_fails(self, rbf_node, dest_address): assert_raises_rpc_error(-13, "Please enter the wallet passphrase with walletpassphrase first.", rbf_node.bumpfee, rbfid) rbf_node.walletpassphrase(WALLET_PASSPHRASE, WALLET_PASSPHRASE_TIMEOUT) + self.clear_mempool() def test_change_script_match(self, rbf_node, dest_address): @@ -510,6 +536,7 @@ def test_change_script_match(self, rbf_node, dest_address): assert_equal(change_addresses, get_change_address(bumped_total_tx['txid'])) bumped_rate_tx = rbf_node.bumpfee(bumped_total_tx["txid"]) assert_equal(change_addresses, get_change_address(bumped_rate_tx['txid'])) + self.clear_mempool() def spend_one_input(node, dest_address, change_size=Decimal("0.00049000")): @@ -548,6 +575,7 @@ def test_no_more_inputs_fails(self, rbf_node, dest_address): # spend all funds, no change output rbfid = rbf_node.sendtoaddress(rbf_node.getnewaddress(), rbf_node.getbalance(), "", "", True) assert_raises_rpc_error(-4, "Unable to create transaction. Insufficient funds", rbf_node.bumpfee, rbfid) + self.clear_mempool() if __name__ == "__main__": diff --git a/test/functional/wallet_groups.py b/test/functional/wallet_groups.py index b6fe295127..e5c4f12f20 100755 --- a/test/functional/wallet_groups.py +++ b/test/functional/wallet_groups.py @@ -15,8 +15,14 @@ from test_framework.util import ( class WalletGroupTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True - self.num_nodes = 3 - self.extra_args = [[], [], ['-avoidpartialspends']] + self.num_nodes = 5 + self.extra_args = [ + [], + [], + ["-avoidpartialspends"], + ["-maxapsfee=0.00002719"], + ["-maxapsfee=0.00002720"], + ] self.rpc_timeout = 480 def skip_test_if_missing_module(self): @@ -50,8 +56,8 @@ class WalletGroupTest(BitcoinTestFramework): # one output should be 0.2, the other should be ~0.3 v = [vout["value"] for vout in tx1["vout"]] v.sort() - assert_approx(v[0], 0.2) - assert_approx(v[1], 0.3, 0.0001) + assert_approx(v[0], vexp=0.2, vspan=0.0001) + assert_approx(v[1], vexp=0.3, vspan=0.0001) txid2 = self.nodes[2].sendtoaddress(self.nodes[0].getnewaddress(), 0.2) tx2 = self.nodes[2].getrawtransaction(txid2, True) @@ -61,8 +67,80 @@ class WalletGroupTest(BitcoinTestFramework): # one output should be 0.2, the other should be ~1.3 v = [vout["value"] for vout in tx2["vout"]] v.sort() - assert_approx(v[0], 0.2) - assert_approx(v[1], 1.3, 0.0001) + assert_approx(v[0], vexp=0.2, vspan=0.0001) + assert_approx(v[1], vexp=1.3, vspan=0.0001) + + # Test 'avoid partial if warranted, even if disabled' + self.sync_all() + self.nodes[0].generate(1) + # Nodes 1-2 now have confirmed UTXOs (letters denote destinations): + # Node #1: Node #2: + # - A 1.0 - D0 1.0 + # - B0 1.0 - D1 0.5 + # - B1 0.5 - E0 1.0 + # - C0 1.0 - E1 0.5 + # - C1 0.5 - F ~1.3 + # - D ~0.3 + assert_approx(self.nodes[1].getbalance(), vexp=4.3, vspan=0.0001) + assert_approx(self.nodes[2].getbalance(), vexp=4.3, vspan=0.0001) + # Sending 1.4 btc should pick one 1.0 + one more. For node #1, + # this could be (A / B0 / C0) + (B1 / C1 / D). We ensure that it is + # B0 + B1 or C0 + C1, because this avoids partial spends while not being + # detrimental to transaction cost + txid3 = self.nodes[1].sendtoaddress(self.nodes[0].getnewaddress(), 1.4) + tx3 = self.nodes[1].getrawtransaction(txid3, True) + # tx3 should have 2 inputs and 2 outputs + assert_equal(2, len(tx3["vin"])) + assert_equal(2, len(tx3["vout"])) + # the accumulated value should be 1.5, so the outputs should be + # ~0.1 and 1.4 and should come from the same destination + values = [vout["value"] for vout in tx3["vout"]] + values.sort() + assert_approx(values[0], vexp=0.1, vspan=0.0001) + assert_approx(values[1], vexp=1.4, vspan=0.0001) + + input_txids = [vin["txid"] for vin in tx3["vin"]] + input_addrs = [self.nodes[1].gettransaction(txid)['details'][0]['address'] for txid in input_txids] + assert_equal(input_addrs[0], input_addrs[1]) + # Node 2 enforces avoidpartialspends so needs no checking here + + # Test wallet option maxapsfee with Node 3 + addr_aps = self.nodes[3].getnewaddress() + self.nodes[0].sendtoaddress(addr_aps, 1.0) + self.nodes[0].sendtoaddress(addr_aps, 1.0) + self.nodes[0].generate(1) + self.sync_all() + with self.nodes[3].assert_debug_log(['Fee non-grouped = 2820, grouped = 4160, using grouped']): + txid4 = self.nodes[3].sendtoaddress(self.nodes[0].getnewaddress(), 0.1) + tx4 = self.nodes[3].getrawtransaction(txid4, True) + # tx4 should have 2 inputs and 2 outputs although one output would + # have been enough and the transaction caused higher fees + assert_equal(2, len(tx4["vin"])) + assert_equal(2, len(tx4["vout"])) + + addr_aps2 = self.nodes[3].getnewaddress() + [self.nodes[0].sendtoaddress(addr_aps2, 1.0) for _ in range(5)] + self.nodes[0].generate(1) + self.sync_all() + with self.nodes[3].assert_debug_log(['Fee non-grouped = 5520, grouped = 8240, using non-grouped']): + txid5 = self.nodes[3].sendtoaddress(self.nodes[0].getnewaddress(), 2.95) + tx5 = self.nodes[3].getrawtransaction(txid5, True) + # tx5 should have 3 inputs (1.0, 1.0, 1.0) and 2 outputs + assert_equal(3, len(tx5["vin"])) + assert_equal(2, len(tx5["vout"])) + + # Test wallet option maxapsfee with node 4, which sets maxapsfee + # 1 sat higher, crossing the threshold from non-grouped to grouped. + addr_aps3 = self.nodes[4].getnewaddress() + [self.nodes[0].sendtoaddress(addr_aps3, 1.0) for _ in range(5)] + self.nodes[0].generate(1) + self.sync_all() + with self.nodes[4].assert_debug_log(['Fee non-grouped = 5520, grouped = 8240, using grouped']): + txid6 = self.nodes[4].sendtoaddress(self.nodes[0].getnewaddress(), 2.95) + tx6 = self.nodes[4].getrawtransaction(txid6, True) + # tx6 should have 5 inputs and 2 outputs + assert_equal(5, len(tx6["vin"])) + assert_equal(2, len(tx6["vout"])) # Empty out node2's wallet self.nodes[2].sendtoaddress(address=self.nodes[0].getnewaddress(), amount=self.nodes[2].getbalance(), subtractfeefromamount=True) diff --git a/test/functional/wallet_resendwallettransactions.py b/test/functional/wallet_resendwallettransactions.py index 3417616d77..0327c9e070 100755 --- a/test/functional/wallet_resendwallettransactions.py +++ b/test/functional/wallet_resendwallettransactions.py @@ -7,9 +7,9 @@ import time from test_framework.blocktools import create_block, create_coinbase from test_framework.messages import ToHex -from test_framework.mininode import P2PTxInvStore, mininode_lock +from test_framework.p2p import P2PTxInvStore from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, wait_until +from test_framework.util import assert_equal class ResendWalletTransactionsTest(BitcoinTestFramework): def set_test_params(self): @@ -24,7 +24,7 @@ class ResendWalletTransactionsTest(BitcoinTestFramework): node.add_p2p_connection(P2PTxInvStore()) self.log.info("Create a new transaction and wait until it's broadcast") - txid = int(node.sendtoaddress(node.getnewaddress(), 1), 16) + txid = node.sendtoaddress(node.getnewaddress(), 1) # Wallet rebroadcast is first scheduled 1 sec after startup (see # nNextResend in ResendWalletTransactions()). Sleep for just over a @@ -33,7 +33,7 @@ class ResendWalletTransactionsTest(BitcoinTestFramework): time.sleep(1.1) # Can take a few seconds due to transaction trickling - wait_until(lambda: node.p2p.tx_invs_received[txid] >= 1, lock=mininode_lock) + node.p2p.wait_for_broadcast([txid]) # Add a second peer since txs aren't rebroadcast to the same peer (see filterInventoryKnown) node.add_p2p_connection(P2PTxInvStore()) @@ -58,13 +58,13 @@ class ResendWalletTransactionsTest(BitcoinTestFramework): two_min = 2 * 60 node.setmocktime(now + twelve_hrs - two_min) time.sleep(2) # ensure enough time has passed for rebroadcast attempt to occur - assert_equal(txid in node.p2ps[1].get_invs(), False) + assert_equal(int(txid, 16) in node.p2ps[1].get_invs(), False) self.log.info("Bump time & check that transaction is rebroadcast") # Transaction should be rebroadcast approximately 24 hours in the future, # but can range from 12-36. So bump 36 hours to be sure. node.setmocktime(now + 36 * 60 * 60) - wait_until(lambda: node.p2ps[1].tx_invs_received[txid] >= 1, lock=mininode_lock) + node.p2p.wait_for_broadcast([txid]) if __name__ == '__main__': diff --git a/test/functional/wallet_startup.py b/test/functional/wallet_startup.py new file mode 100755 index 0000000000..cfc4edb8ee --- /dev/null +++ b/test/functional/wallet_startup.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +# Copyright (c) 2017-2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test wallet load on startup. + +Verify that a bitcoind node can maintain list of wallets loading on startup +""" +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, +) + + +class WalletStartupTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 1 + self.supports_cli = True + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + + def setup_nodes(self): + self.add_nodes(self.num_nodes) + self.start_nodes() + + def run_test(self): + self.nodes[0].createwallet(wallet_name='w0', load_on_startup=True) + self.nodes[0].createwallet(wallet_name='w1', load_on_startup=False) + self.nodes[0].createwallet(wallet_name='w2', load_on_startup=True) + self.nodes[0].createwallet(wallet_name='w3', load_on_startup=False) + self.nodes[0].createwallet(wallet_name='w4', load_on_startup=False) + self.nodes[0].unloadwallet(wallet_name='w0', load_on_startup=False) + self.nodes[0].unloadwallet(wallet_name='w4', load_on_startup=False) + self.nodes[0].loadwallet(filename='w4', load_on_startup=True) + assert_equal(set(self.nodes[0].listwallets()), set(('', 'w1', 'w2', 'w3', 'w4'))) + self.restart_node(0) + assert_equal(set(self.nodes[0].listwallets()), set(('', 'w2', 'w4'))) + self.nodes[0].unloadwallet(wallet_name='', load_on_startup=False) + self.nodes[0].unloadwallet(wallet_name='w4', load_on_startup=False) + self.nodes[0].loadwallet(filename='w3', load_on_startup=True) + self.nodes[0].loadwallet(filename='') + self.restart_node(0) + assert_equal(set(self.nodes[0].listwallets()), set(('w2', 'w3'))) + +if __name__ == '__main__': + WalletStartupTest().main() diff --git a/test/functional/wallet_upgradewallet.py b/test/functional/wallet_upgradewallet.py index 1a76f65215..f3d6e74829 100755 --- a/test/functional/wallet_upgradewallet.py +++ b/test/functional/wallet_upgradewallet.py @@ -6,7 +6,7 @@ Test upgradewallet RPC. Download node binaries: -contrib/devtools/previous_release.py -b v0.19.1 v0.18.1 v0.17.1 v0.16.3 v0.15.2 +test/get_previous_releases.py -b v0.19.1 v0.18.1 v0.17.1 v0.16.3 v0.15.2 Only v0.15.2 and v0.16.3 are required by this test. The others are used in feature_backwards_compatibility.py """ diff --git a/test/functional/wallet_zapwallettxes.py b/test/functional/wallet_zapwallettxes.py index 7f1cdbd20b..1287092cac 100755 --- a/test/functional/wallet_zapwallettxes.py +++ b/test/functional/wallet_zapwallettxes.py @@ -18,9 +18,9 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, assert_raises_rpc_error, - wait_until, ) + class ZapWalletTXesTest (BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True @@ -59,7 +59,7 @@ class ZapWalletTXesTest (BitcoinTestFramework): # transaction is zapped from the wallet, but is re-added when the mempool is reloaded. self.restart_node(0, ["-persistmempool=1", "-zapwallettxes=2"]) - wait_until(lambda: self.nodes[0].getmempoolinfo()['size'] == 1, timeout=3) + self.wait_until(lambda: self.nodes[0].getmempoolinfo()['size'] == 1, timeout=3) self.nodes[0].syncwithvalidationinterfacequeue() # Flush mempool to wallet assert_equal(self.nodes[0].gettransaction(txid1)['txid'], txid1) diff --git a/test/fuzz/test_runner.py b/test/fuzz/test_runner.py index 56b18752ec..c7895edbcc 100755 --- a/test/fuzz/test_runner.py +++ b/test/fuzz/test_runner.py @@ -56,6 +56,14 @@ def main(): '--m_dir', help='Merge inputs from this directory into the seed_dir. Needs /target subdirectory.', ) + parser.add_argument( + '-g', + '--generate', + action='store_true', + help='Create new corpus seeds (or extend the existing ones) by running' + ' the given targets for a finite number of times. Outputs them to' + ' the passed seed_dir.' + ) args = parser.parse_args() @@ -100,19 +108,20 @@ def main(): logging.info("{} of {} detected fuzz target(s) selected: {}".format(len(test_list_selection), len(test_list_all), " ".join(test_list_selection))) - test_list_seedless = [] - for t in test_list_selection: - corpus_path = os.path.join(args.seed_dir, t) - if not os.path.exists(corpus_path) or len(os.listdir(corpus_path)) == 0: - test_list_seedless.append(t) - test_list_seedless.sort() - if test_list_seedless: - logging.info( - "Fuzzing harnesses lacking a seed corpus: {}".format( - " ".join(test_list_seedless) + if not args.generate: + test_list_seedless = [] + for t in test_list_selection: + corpus_path = os.path.join(args.seed_dir, t) + if not os.path.exists(corpus_path) or len(os.listdir(corpus_path)) == 0: + test_list_seedless.append(t) + test_list_seedless.sort() + if test_list_seedless: + logging.info( + "Fuzzing harnesses lacking a seed corpus: {}".format( + " ".join(test_list_seedless) + ) ) - ) - logging.info("Please consider adding a fuzz seed corpus at https://github.com/bitcoin-core/qa-assets") + logging.info("Please consider adding a fuzz seed corpus at https://github.com/bitcoin-core/qa-assets") try: help_output = subprocess.run( @@ -133,6 +142,14 @@ def main(): sys.exit(1) with ThreadPoolExecutor(max_workers=args.par) as fuzz_pool: + if args.generate: + return generate_corpus_seeds( + fuzz_pool=fuzz_pool, + build_dir=config["environment"]["BUILDDIR"], + seed_dir=args.seed_dir, + targets=test_list_selection, + ) + if args.m_dir: merge_inputs( fuzz_pool=fuzz_pool, @@ -152,6 +169,37 @@ def main(): ) +def generate_corpus_seeds(*, fuzz_pool, build_dir, seed_dir, targets): + """Generates new corpus seeds. + + Run {targets} without input, and outputs the generated corpus seeds to + {seed_dir}. + """ + logging.info("Generating corpus seeds to {}".format(seed_dir)) + + def job(command): + logging.debug("Running '{}'\n".format(" ".join(command))) + logging.debug("Command '{}' output:\n'{}'\n".format( + ' '.join(command), + subprocess.run(command, check=True, stderr=subprocess.PIPE, + universal_newlines=True).stderr + )) + + futures = [] + for target in targets: + target_seed_dir = os.path.join(seed_dir, target) + os.makedirs(target_seed_dir, exist_ok=True) + command = [ + os.path.join(build_dir, "src", "test", "fuzz", target), + "-runs=100000", + target_seed_dir, + ] + futures.append(fuzz_pool.submit(job, command)) + + for future in as_completed(futures): + future.result() + + def merge_inputs(*, fuzz_pool, corpus, test_list, build_dir, merge_dir): logging.info("Merge the inputs in the passed dir into the seed_dir. Passed dir {}".format(merge_dir)) jobs = [] diff --git a/test/get_previous_releases.py b/test/get_previous_releases.py new file mode 100755 index 0000000000..1348b8246b --- /dev/null +++ b/test/get_previous_releases.py @@ -0,0 +1,228 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2018-2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# Download or build previous releases. +# Needs curl and tar to download a release, or the build dependencies when +# building a release. + +import argparse +import contextlib +from fnmatch import fnmatch +import os +from pathlib import Path +import re +import shutil +import subprocess +import sys +import hashlib + + +SHA256_SUMS = { +"d40f18b4e43c6e6370ef7db9131f584fbb137276ec2e3dba67a4b267f81cb644": "bitcoin-0.15.2-aarch64-linux-gnu.tar.gz", +"54fb877a148a6ad189a1e1ab1ff8b11181e58ff2aaf430da55b3fd46ae549a6b": "bitcoin-0.15.2-arm-linux-gnueabihf.tar.gz", +"2b843506c3f1af0eeca5854a920264f9a829f02d0d50328005950ddcbe88874d": "bitcoin-0.15.2-i686-pc-linux-gnu.tar.gz", +"87e9340ff3d382d543b2b69112376077f0c8b4f7450d372e83b68f5a1e22b2df": "bitcoin-0.15.2-osx64.tar.gz", +"566be44190fd76daa01f13d428939dadfb8e3daacefc8fa17f433cad28f73bd5": "bitcoin-0.15.2-x86_64-linux-gnu.tar.gz", + +"0768c6c15caffbaca6524824c9563b42c24f70633c681c2744649158aa3fd484": "bitcoin-0.16.3-aarch64-linux-gnu.tar.gz", +"fb2818069854a6ad20ea03b28b55dbd35d8b1f7d453e90b83eace5d0098a2a87": "bitcoin-0.16.3-arm-linux-gnueabihf.tar.gz", +"75a537844313b0a84bdb61ffcdc5c4ce19a738f7ddf71007cd2edf664efd7c37": "bitcoin-0.16.3-i686-pc-linux-gnu.tar.gz", +"78c3bff3b619a19aed575961ea43cc9e142959218835cf51aede7f0b764fc25d": "bitcoin-0.16.3-osx64.tar.gz", +"5d422a9d544742bc0df12427383f9c2517433ce7b58cf672b9a9b17c2ef51e4f": "bitcoin-0.16.3-x86_64-linux-gnu.tar.gz", + +"5a6b35d1a348a402f2d2d6ab5aed653a1a1f13bc63aaaf51605e3501b0733b7a": "bitcoin-0.17.2-aarch64-linux-gnu.tar.gz", +"d1913a5d19c8e8da4a67d1bd5205d03c8614dfd2e02bba2fe3087476643a729e": "bitcoin-0.17.2-arm-linux-gnueabihf.tar.gz", +"d295fc93f39bbf0fd937b730a93184899a2eb6c3a6d53f3d857cbe77ef89b98c": "bitcoin-0.17.2-i686-pc-linux-gnu.tar.gz", +"a783ba20706dbfd5b47fbedf42165fce70fbbc7d78003305d964f6b3da14887f": "bitcoin-0.17.2-osx64.tar.gz", +"943f9362b9f11130177839116f48f809d83478b4c28591d486ee9a7e35179da6": "bitcoin-0.17.2-x86_64-linux-gnu.tar.gz", + +"88f343af72803b851c7da13874cc5525026b0b55e63e1b5e1298390c4688adc6": "bitcoin-0.18.1-aarch64-linux-gnu.tar.gz", +"cc7d483e4b20c5dabd4dcaf304965214cf4934bcc029ca99cbc9af00d3771a1f": "bitcoin-0.18.1-arm-linux-gnueabihf.tar.gz", +"989e847b3e95fc9fedc0b109cae1b4fa43348f2f712e187a118461876af9bd16": "bitcoin-0.18.1-i686-pc-linux-gnu.tar.gz", +"b7bbcee7a7540f711b171d6981f939ca8482005fde22689bc016596d80548bb1": "bitcoin-0.18.1-osx64.tar.gz", +"425ee5ec631ae8da71ebc1c3f5c0269c627cf459379b9b030f047107a28e3ef8": "bitcoin-0.18.1-riscv64-linux-gnu.tar.gz", +"600d1db5e751fa85903e935a01a74f5cc57e1e7473c15fd3e17ed21e202cfe5a": "bitcoin-0.18.1-x86_64-linux-gnu.tar.gz", + +"3a80431717842672df682bdb619e66523b59541483297772a7969413be3502ff": "bitcoin-0.19.1-aarch64-linux-gnu.tar.gz", +"657f28213823d240dd3324d14829702f9ad6f0710f8bdd1c379cb3c447197f48": "bitcoin-0.19.1-arm-linux-gnueabihf.tar.gz", +"10d1e53208aa7603022f4acc084a046299ab4ccf25fe01e81b3fb6f856772589": "bitcoin-0.19.1-i686-pc-linux-gnu.tar.gz", +"1ae1b87de26487075cd2fd22e0d4ead87d969bd55c44f2f1d873ecdc6147ebb3": "bitcoin-0.19.1-osx64.tar.gz", +"aa7a9563b48aa79252c8e7b6a41c07a5441bd9f14c5e4562cc72720ea6cb0ee5": "bitcoin-0.19.1-riscv64-linux-gnu.tar.gz", +"5fcac9416e486d4960e1a946145566350ca670f9aaba99de6542080851122e4c": "bitcoin-0.19.1-x86_64-linux-gnu.tar.gz" +} + +@contextlib.contextmanager +def pushd(new_dir) -> None: + previous_dir = os.getcwd() + os.chdir(new_dir) + try: + yield + finally: + os.chdir(previous_dir) + + +def download_binary(tag, args) -> int: + if Path(tag).is_dir(): + if not args.remove_dir: + print('Using cached {}'.format(tag)) + return 0 + shutil.rmtree(tag) + Path(tag).mkdir() + bin_path = 'bin/bitcoin-core-{}'.format(tag[1:]) + match = re.compile('v(.*)(rc[0-9]+)$').search(tag) + if match: + bin_path = 'bin/bitcoin-core-{}/test.{}'.format( + match.group(1), match.group(2)) + tarball = 'bitcoin-{tag}-{platform}.tar.gz'.format( + tag=tag[1:], platform=args.platform) + tarballUrl = 'https://bitcoincore.org/{bin_path}/{tarball}'.format( + bin_path=bin_path, tarball=tarball) + + print('Fetching: {tarballUrl}'.format(tarballUrl=tarballUrl)) + + header, status = subprocess.Popen( + ['curl', '--head', tarballUrl], stdout=subprocess.PIPE).communicate() + if re.search("404 Not Found", header.decode("utf-8")): + print("Binary tag was not found") + return 1 + + curlCmds = [ + ['curl', '--remote-name', tarballUrl] + ] + + for cmd in curlCmds: + ret = subprocess.run(cmd).returncode + if ret: + return ret + + hasher = hashlib.sha256() + with open(tarball, "rb") as afile: + hasher.update(afile.read()) + tarballHash = hasher.hexdigest() + + if tarballHash not in SHA256_SUMS or SHA256_SUMS[tarballHash] != tarball: + print("Checksum did not match") + return 1 + print("Checksum matched") + + # Extract tarball + ret = subprocess.run(['tar', '-zxf', tarball, '-C', tag, + '--strip-components=1', + 'bitcoin-{tag}'.format(tag=tag[1:])]).returncode + if ret: + return ret + + Path(tarball).unlink() + return 0 + + +def build_release(tag, args) -> int: + githubUrl = "https://github.com/bitcoin/bitcoin" + if args.remove_dir: + if Path(tag).is_dir(): + shutil.rmtree(tag) + if not Path(tag).is_dir(): + # fetch new tags + subprocess.run( + ["git", "fetch", githubUrl, "--tags"]) + output = subprocess.check_output(['git', 'tag', '-l', tag]) + if not output: + print('Tag {} not found'.format(tag)) + return 1 + ret = subprocess.run([ + 'git', 'clone', githubUrl, tag + ]).returncode + if ret: + return ret + with pushd(tag): + ret = subprocess.run(['git', 'checkout', tag]).returncode + if ret: + return ret + host = args.host + if args.depends: + with pushd('depends'): + ret = subprocess.run(['make', 'NO_QT=1']).returncode + if ret: + return ret + host = os.environ.get( + 'HOST', subprocess.check_output(['./config.guess'])) + config_flags = '--prefix={pwd}/depends/{host} '.format( + pwd=os.getcwd(), + host=host) + args.config_flags + cmds = [ + './autogen.sh', + './configure {}'.format(config_flags), + 'make', + ] + for cmd in cmds: + ret = subprocess.run(cmd.split()).returncode + if ret: + return ret + # Move binaries, so they're in the same place as in the + # release download + Path('bin').mkdir(exist_ok=True) + files = ['bitcoind', 'bitcoin-cli', 'bitcoin-tx'] + for f in files: + Path('src/'+f).rename('bin/'+f) + return 0 + + +def check_host(args) -> int: + args.host = os.environ.get('HOST', subprocess.check_output( + './depends/config.guess').decode()) + if args.download_binary: + platforms = { + 'x86_64-*-linux*': 'x86_64-linux-gnu', + 'x86_64-apple-darwin*': 'osx64', + } + args.platform = '' + for pattern, target in platforms.items(): + if fnmatch(args.host, pattern): + args.platform = target + if not args.platform: + print('Not sure which binary to download for {}'.format(args.host)) + return 1 + return 0 + + +def main(args) -> int: + Path(args.target_dir).mkdir(exist_ok=True, parents=True) + print("Releases directory: {}".format(args.target_dir)) + ret = check_host(args) + if ret: + return ret + if args.download_binary: + with pushd(args.target_dir): + for tag in args.tags: + ret = download_binary(tag, args) + if ret: + return ret + return 0 + args.config_flags = os.environ.get('CONFIG_FLAGS', '') + args.config_flags += ' --without-gui --disable-tests --disable-bench' + with pushd(args.target_dir): + for tag in args.tags: + ret = build_release(tag, args) + if ret: + return ret + return 0 + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument('-r', '--remove-dir', action='store_true', + help='remove existing directory.') + parser.add_argument('-d', '--depends', action='store_true', + help='use depends.') + parser.add_argument('-b', '--download-binary', action='store_true', + help='download release binary.') + parser.add_argument('-t', '--target-dir', action='store', + help='target directory.', default='releases') + parser.add_argument('tags', nargs='+', + help="release tags. e.g.: v0.18.1 v0.20.0rc2") + args = parser.parse_args() + sys.exit(main(args)) diff --git a/test/lint/lint-locale-dependence.sh b/test/lint/lint-locale-dependence.sh index 2e5b801849..e5657f7555 100755 --- a/test/lint/lint-locale-dependence.sh +++ b/test/lint/lint-locale-dependence.sh @@ -4,6 +4,39 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C + +# Be aware that bitcoind and bitcoin-qt differ in terms of localization: Qt +# opts in to POSIX localization by running setlocale(LC_ALL, "") on startup, +# whereas no such call is made in bitcoind. +# +# Qt runs setlocale(LC_ALL, "") on initialization. This installs the locale +# specified by the user's LC_ALL (or LC_*) environment variable as the new +# C locale. +# +# In contrast, bitcoind does not opt in to localization -- no call to +# setlocale(LC_ALL, "") is made and the environment variables LC_* are +# thus ignored. +# +# This results in situations where bitcoind is guaranteed to be running +# with the classic locale ("C") whereas the locale of bitcoin-qt will vary +# depending on the user's environment variables. +# +# An example: Assuming the environment variable LC_ALL=de_DE then the +# call std::to_string(1.23) will return "1.230000" in bitcoind but +# "1,230000" in bitcoin-qt. +# +# From the Qt documentation: +# "On Unix/Linux Qt is configured to use the system locale settings by default. +# This can cause a conflict when using POSIX functions, for instance, when +# converting between data types such as floats and strings, since the notation +# may differ between locales. To get around this problem, call the POSIX function +# setlocale(LC_NUMERIC,"C") right after initializing QApplication, QGuiApplication +# or QCoreApplication to reset the locale that is used for number formatting to +# "C"-locale." +# +# See https://doc.qt.io/qt-5/qcoreapplication.html#locale-settings and +# https://stackoverflow.com/a/34878283 for more details. + KNOWN_VIOLATIONS=( "src/bitcoin-tx.cpp.*stoul" "src/bitcoin-tx.cpp.*trim_right" diff --git a/test/sanitizer_suppressions/tsan b/test/sanitizer_suppressions/tsan index 3d0ac7f995..3ba2b2a103 100644 --- a/test/sanitizer_suppressions/tsan +++ b/test/sanitizer_suppressions/tsan @@ -24,6 +24,7 @@ race:WalletBatch::WriteHDChain race:BerkeleyBatch race:BerkeleyDatabase race:DatabaseBatch +race:leveldb::DBImpl::DeleteObsoleteFiles race:zmq::* race:bitcoin-qt # deadlock (TODO fix) |