From 61b8f7f273022d3163f998ff5d66d53ca0460c8b Mon Sep 17 00:00:00 2001 From: Anthony Towns Date: Thu, 25 Jan 2018 09:44:29 +1000 Subject: [tests] Rename p2p_* functional tests. --- test/functional/README.md | 4 +- test/functional/disconnect_ban.py | 107 -- test/functional/invalidblockrequest.py | 117 -- test/functional/invalidtxrequest.py | 73 - test/functional/node_network_limited.py | 57 - test/functional/p2p-acceptblock.py | 323 ----- test/functional/p2p-compactblocks.py | 921 ------------- test/functional/p2p-feefilter.py | 93 -- test/functional/p2p-fingerprint.py | 152 --- test/functional/p2p-leaktests.py | 153 --- test/functional/p2p-mempool.py | 35 - test/functional/p2p-segwit.py | 1952 --------------------------- test/functional/p2p-timeouts.py | 75 - test/functional/p2p_compactblocks.py | 921 +++++++++++++ test/functional/p2p_disconnect_ban.py | 107 ++ test/functional/p2p_feefilter.py | 93 ++ test/functional/p2p_fingerprint.py | 152 +++ test/functional/p2p_invalid_block.py | 117 ++ test/functional/p2p_invalid_tx.py | 73 + test/functional/p2p_leak.py | 153 +++ test/functional/p2p_mempool.py | 35 + test/functional/p2p_node_network_limited.py | 57 + test/functional/p2p_segwit.py | 1952 +++++++++++++++++++++++++++ test/functional/p2p_sendheaders.py | 598 ++++++++ test/functional/p2p_timeouts.py | 75 + test/functional/p2p_unrequested_blocks.py | 323 +++++ test/functional/sendheaders.py | 598 -------- test/functional/test_runner.py | 28 +- 28 files changed, 4672 insertions(+), 4672 deletions(-) delete mode 100755 test/functional/disconnect_ban.py delete mode 100755 test/functional/invalidblockrequest.py delete mode 100755 test/functional/invalidtxrequest.py delete mode 100755 test/functional/node_network_limited.py delete mode 100755 test/functional/p2p-acceptblock.py delete mode 100755 test/functional/p2p-compactblocks.py delete mode 100755 test/functional/p2p-feefilter.py delete mode 100755 test/functional/p2p-fingerprint.py delete mode 100755 test/functional/p2p-leaktests.py delete mode 100755 test/functional/p2p-mempool.py delete mode 100755 test/functional/p2p-segwit.py delete mode 100755 test/functional/p2p-timeouts.py create mode 100755 test/functional/p2p_compactblocks.py create mode 100755 test/functional/p2p_disconnect_ban.py create mode 100755 test/functional/p2p_feefilter.py create mode 100755 test/functional/p2p_fingerprint.py create mode 100755 test/functional/p2p_invalid_block.py create mode 100755 test/functional/p2p_invalid_tx.py create mode 100755 test/functional/p2p_leak.py create mode 100755 test/functional/p2p_mempool.py create mode 100755 test/functional/p2p_node_network_limited.py create mode 100755 test/functional/p2p_segwit.py create mode 100755 test/functional/p2p_sendheaders.py create mode 100755 test/functional/p2p_timeouts.py create mode 100755 test/functional/p2p_unrequested_blocks.py delete mode 100755 test/functional/sendheaders.py (limited to 'test/functional') diff --git a/test/functional/README.md b/test/functional/README.md index b3d34c72fd..662b4b44d5 100644 --- a/test/functional/README.md +++ b/test/functional/README.md @@ -87,7 +87,7 @@ start the networking thread. (Continue with the test logic in your existing thread.) - Can be used to write tests where specific P2P protocol behavior is tested. -Examples tests are `p2p-acceptblock.py`, `p2p-compactblocks.py`. +Examples tests are `p2p_unrequested_blocks.py`, `p2p_compactblocks.py`. #### Comptool @@ -133,7 +133,7 @@ Each `TestInstance` consists of: acceptance is tested against the given outcome. - For examples of tests written in this framework, see - `invalidblockrequest.py` and `feature_block.py`. + `p2p_invalid_block.py` and `feature_block.py`. ### test-framework modules diff --git a/test/functional/disconnect_ban.py b/test/functional/disconnect_ban.py deleted file mode 100755 index c6067befb2..0000000000 --- a/test/functional/disconnect_ban.py +++ /dev/null @@ -1,107 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2014-2017 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test node disconnect and ban behavior""" -import time - -from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import ( - assert_equal, - assert_raises_rpc_error, - connect_nodes_bi, - wait_until, -) - -class DisconnectBanTest(BitcoinTestFramework): - def set_test_params(self): - self.num_nodes = 2 - - def run_test(self): - self.log.info("Test setban and listbanned RPCs") - - self.log.info("setban: successfully ban single IP address") - assert_equal(len(self.nodes[1].getpeerinfo()), 2) # node1 should have 2 connections to node0 at this point - self.nodes[1].setban("127.0.0.1", "add") - wait_until(lambda: len(self.nodes[1].getpeerinfo()) == 0, timeout=10) - assert_equal(len(self.nodes[1].getpeerinfo()), 0) # all nodes must be disconnected at this point - assert_equal(len(self.nodes[1].listbanned()), 1) - - self.log.info("clearbanned: successfully clear ban list") - self.nodes[1].clearbanned() - assert_equal(len(self.nodes[1].listbanned()), 0) - self.nodes[1].setban("127.0.0.0/24", "add") - - self.log.info("setban: fail to ban an already banned subnet") - assert_equal(len(self.nodes[1].listbanned()), 1) - assert_raises_rpc_error(-23, "IP/Subnet already banned", self.nodes[1].setban, "127.0.0.1", "add") - - self.log.info("setban: fail to ban an invalid subnet") - assert_raises_rpc_error(-30, "Error: Invalid IP/Subnet", self.nodes[1].setban, "127.0.0.1/42", "add") - assert_equal(len(self.nodes[1].listbanned()), 1) # still only one banned ip because 127.0.0.1 is within the range of 127.0.0.0/24 - - self.log.info("setban remove: fail to unban a non-banned subnet") - assert_raises_rpc_error(-30, "Error: Unban failed", self.nodes[1].setban, "127.0.0.1", "remove") - assert_equal(len(self.nodes[1].listbanned()), 1) - - self.log.info("setban remove: successfully unban subnet") - self.nodes[1].setban("127.0.0.0/24", "remove") - assert_equal(len(self.nodes[1].listbanned()), 0) - self.nodes[1].clearbanned() - assert_equal(len(self.nodes[1].listbanned()), 0) - - self.log.info("setban: test persistence across node restart") - self.nodes[1].setban("127.0.0.0/32", "add") - self.nodes[1].setban("127.0.0.0/24", "add") - # Set the mocktime so we can control when bans expire - old_time = int(time.time()) - self.nodes[1].setmocktime(old_time) - self.nodes[1].setban("192.168.0.1", "add", 1) # ban for 1 seconds - self.nodes[1].setban("2001:4d48:ac57:400:cacf:e9ff:fe1d:9c63/19", "add", 1000) # ban for 1000 seconds - listBeforeShutdown = self.nodes[1].listbanned() - assert_equal("192.168.0.1/32", listBeforeShutdown[2]['address']) - # Move time forward by 3 seconds so the third ban has expired - self.nodes[1].setmocktime(old_time + 3) - assert_equal(len(self.nodes[1].listbanned()), 3) - - self.stop_node(1) - self.start_node(1) - - listAfterShutdown = self.nodes[1].listbanned() - assert_equal("127.0.0.0/24", listAfterShutdown[0]['address']) - assert_equal("127.0.0.0/32", listAfterShutdown[1]['address']) - assert_equal("/19" in listAfterShutdown[2]['address'], True) - - # Clear ban lists - self.nodes[1].clearbanned() - connect_nodes_bi(self.nodes, 0, 1) - - self.log.info("Test disconnectnode RPCs") - - self.log.info("disconnectnode: fail to disconnect when calling with address and nodeid") - address1 = self.nodes[0].getpeerinfo()[0]['addr'] - node1 = self.nodes[0].getpeerinfo()[0]['addr'] - assert_raises_rpc_error(-32602, "Only one of address and nodeid should be provided.", self.nodes[0].disconnectnode, address=address1, nodeid=node1) - - self.log.info("disconnectnode: fail to disconnect when calling with junk address") - assert_raises_rpc_error(-29, "Node not found in connected nodes", self.nodes[0].disconnectnode, address="221B Baker Street") - - self.log.info("disconnectnode: successfully disconnect node by address") - address1 = self.nodes[0].getpeerinfo()[0]['addr'] - self.nodes[0].disconnectnode(address=address1) - wait_until(lambda: len(self.nodes[0].getpeerinfo()) == 1, timeout=10) - assert not [node for node in self.nodes[0].getpeerinfo() if node['addr'] == address1] - - self.log.info("disconnectnode: successfully reconnect node") - connect_nodes_bi(self.nodes, 0, 1) # reconnect the node - assert_equal(len(self.nodes[0].getpeerinfo()), 2) - assert [node for node in self.nodes[0].getpeerinfo() if node['addr'] == address1] - - self.log.info("disconnectnode: successfully disconnect node by node id") - id1 = self.nodes[0].getpeerinfo()[0]['id'] - self.nodes[0].disconnectnode(nodeid=id1) - wait_until(lambda: len(self.nodes[0].getpeerinfo()) == 1, timeout=10) - assert not [node for node in self.nodes[0].getpeerinfo() if node['id'] == id1] - -if __name__ == '__main__': - DisconnectBanTest().main() diff --git a/test/functional/invalidblockrequest.py b/test/functional/invalidblockrequest.py deleted file mode 100755 index edcade63c1..0000000000 --- a/test/functional/invalidblockrequest.py +++ /dev/null @@ -1,117 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2015-2017 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test node responses to invalid blocks. - -In this test we connect to one node over p2p, and test block requests: -1) Valid blocks should be requested and become chain tip. -2) Invalid block with duplicated transaction should be re-requested. -3) Invalid block with bad coinbase value should be rejected and not -re-requested. -""" - -from test_framework.test_framework import ComparisonTestFramework -from test_framework.util import * -from test_framework.comptool import TestManager, TestInstance, RejectResult -from test_framework.blocktools import * -from test_framework.mininode import network_thread_start -import copy -import time - -# Use the ComparisonTestFramework with 1 node: only use --testbinary. -class InvalidBlockRequestTest(ComparisonTestFramework): - - ''' Can either run this test as 1 node with expected answers, or two and compare them. - Change the "outcome" variable from each TestInstance object to only do the comparison. ''' - def set_test_params(self): - self.num_nodes = 1 - self.setup_clean_chain = True - - def run_test(self): - test = TestManager(self, self.options.tmpdir) - test.add_all_connections(self.nodes) - self.tip = None - self.block_time = None - network_thread_start() - test.run() - - def get_tests(self): - if self.tip is None: - self.tip = int("0x" + self.nodes[0].getbestblockhash(), 0) - self.block_time = int(time.time())+1 - - ''' - Create a new block with an anyone-can-spend coinbase - ''' - height = 1 - block = create_block(self.tip, create_coinbase(height), self.block_time) - self.block_time += 1 - block.solve() - # Save the coinbase for later - self.block1 = block - self.tip = block.sha256 - height += 1 - yield TestInstance([[block, True]]) - - ''' - Now we need that block to mature so we can spend the coinbase. - ''' - test = TestInstance(sync_every_block=False) - for i in range(100): - block = create_block(self.tip, create_coinbase(height), self.block_time) - block.solve() - self.tip = block.sha256 - self.block_time += 1 - test.blocks_and_transactions.append([block, True]) - height += 1 - yield test - - ''' - Now we use merkle-root malleability to generate an invalid block with - same blockheader. - Manufacture a block with 3 transactions (coinbase, spend of prior - coinbase, spend of that spend). Duplicate the 3rd transaction to - leave merkle root and blockheader unchanged but invalidate the block. - ''' - block2 = create_block(self.tip, create_coinbase(height), self.block_time) - self.block_time += 1 - - # b'0x51' is OP_TRUE - tx1 = create_transaction(self.block1.vtx[0], 0, b'\x51', 50 * COIN) - tx2 = create_transaction(tx1, 0, b'\x51', 50 * COIN) - - block2.vtx.extend([tx1, tx2]) - block2.hashMerkleRoot = block2.calc_merkle_root() - block2.rehash() - block2.solve() - orig_hash = block2.sha256 - block2_orig = copy.deepcopy(block2) - - # Mutate block 2 - block2.vtx.append(tx2) - assert_equal(block2.hashMerkleRoot, block2.calc_merkle_root()) - assert_equal(orig_hash, block2.rehash()) - assert(block2_orig.vtx != block2.vtx) - - self.tip = block2.sha256 - yield TestInstance([[block2, RejectResult(16, b'bad-txns-duplicate')], [block2_orig, True]]) - height += 1 - - ''' - Make sure that a totally screwed up block is not valid. - ''' - block3 = create_block(self.tip, create_coinbase(height), self.block_time) - self.block_time += 1 - block3.vtx[0].vout[0].nValue = 100 * COIN # Too high! - block3.vtx[0].sha256=None - block3.vtx[0].calc_sha256() - block3.hashMerkleRoot = block3.calc_merkle_root() - block3.rehash() - block3.solve() - - yield TestInstance([[block3, RejectResult(16, b'bad-cb-amount')]]) - - -if __name__ == '__main__': - InvalidBlockRequestTest().main() diff --git a/test/functional/invalidtxrequest.py b/test/functional/invalidtxrequest.py deleted file mode 100755 index 9c1100e070..0000000000 --- a/test/functional/invalidtxrequest.py +++ /dev/null @@ -1,73 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2015-2017 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test node responses to invalid transactions. - -In this test we connect to one node over p2p, and test tx requests. -""" - -from test_framework.test_framework import ComparisonTestFramework -from test_framework.comptool import TestManager, TestInstance, RejectResult -from test_framework.blocktools import * -import time - - - -# Use the ComparisonTestFramework with 1 node: only use --testbinary. -class InvalidTxRequestTest(ComparisonTestFramework): - - ''' Can either run this test as 1 node with expected answers, or two and compare them. - Change the "outcome" variable from each TestInstance object to only do the comparison. ''' - def set_test_params(self): - self.num_nodes = 1 - self.setup_clean_chain = True - - def run_test(self): - test = TestManager(self, self.options.tmpdir) - test.add_all_connections(self.nodes) - self.tip = None - self.block_time = None - network_thread_start() - test.run() - - def get_tests(self): - if self.tip is None: - self.tip = int("0x" + self.nodes[0].getbestblockhash(), 0) - self.block_time = int(time.time())+1 - - ''' - Create a new block with an anyone-can-spend coinbase - ''' - height = 1 - block = create_block(self.tip, create_coinbase(height), self.block_time) - self.block_time += 1 - block.solve() - # Save the coinbase for later - self.block1 = block - self.tip = block.sha256 - height += 1 - yield TestInstance([[block, True]]) - - ''' - Now we need that block to mature so we can spend the coinbase. - ''' - test = TestInstance(sync_every_block=False) - for i in range(100): - block = create_block(self.tip, create_coinbase(height), self.block_time) - block.solve() - self.tip = block.sha256 - self.block_time += 1 - test.blocks_and_transactions.append([block, True]) - height += 1 - yield test - - # b'\x64' is OP_NOTIF - # Transaction will be rejected with code 16 (REJECT_INVALID) - tx1 = create_transaction(self.block1.vtx[0], 0, b'\x64', 50 * COIN - 12000) - yield TestInstance([[tx1, RejectResult(16, b'mandatory-script-verify-flag-failed')]]) - - # TODO: test further transactions... - -if __name__ == '__main__': - InvalidTxRequestTest().main() diff --git a/test/functional/node_network_limited.py b/test/functional/node_network_limited.py deleted file mode 100755 index 70415e0168..0000000000 --- a/test/functional/node_network_limited.py +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2017 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Tests NODE_NETWORK_LIMITED. - -Tests that a node configured with -prune=550 signals NODE_NETWORK_LIMITED correctly -and that it responds to getdata requests for blocks correctly: - - send a block within 288 + 2 of the tip - - disconnect peers who request blocks older than that.""" -from test_framework.messages import CInv, msg_getdata -from test_framework.mininode import NODE_BLOOM, NODE_NETWORK_LIMITED, NODE_WITNESS, NetworkThread, P2PInterface -from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal - -class P2PIgnoreInv(P2PInterface): - def on_inv(self, message): - # The node will send us invs for other blocks. Ignore them. - pass - - def send_getdata_for_block(self, blockhash): - getdata_request = msg_getdata() - getdata_request.inv.append(CInv(2, int(blockhash, 16))) - self.send_message(getdata_request) - -class NodeNetworkLimitedTest(BitcoinTestFramework): - def set_test_params(self): - self.setup_clean_chain = True - self.num_nodes = 1 - self.extra_args = [['-prune=550']] - - def run_test(self): - node = self.nodes[0].add_p2p_connection(P2PIgnoreInv()) - NetworkThread().start() - node.wait_for_verack() - - expected_services = NODE_BLOOM | NODE_WITNESS | NODE_NETWORK_LIMITED - - self.log.info("Check that node has signalled expected services.") - assert_equal(node.nServices, expected_services) - - self.log.info("Check that the localservices is as expected.") - assert_equal(int(self.nodes[0].getnetworkinfo()['localservices'], 16), expected_services) - - self.log.info("Mine enough blocks to reach the NODE_NETWORK_LIMITED range.") - blocks = self.nodes[0].generate(292) - - self.log.info("Make sure we can max retrive block at tip-288.") - node.send_getdata_for_block(blocks[1]) # last block in valid range - node.wait_for_block(int(blocks[1], 16), timeout=3) - - self.log.info("Requesting block at height 2 (tip-289) must fail (ignored).") - node.send_getdata_for_block(blocks[0]) # first block outside of the 288+2 limit - node.wait_for_disconnect(5) - -if __name__ == '__main__': - NodeNetworkLimitedTest().main() diff --git a/test/functional/p2p-acceptblock.py b/test/functional/p2p-acceptblock.py deleted file mode 100755 index 672626f15b..0000000000 --- a/test/functional/p2p-acceptblock.py +++ /dev/null @@ -1,323 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2015-2017 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test processing of unrequested blocks. - -Setup: two nodes, node0+node1, not connected to each other. Node1 will have -nMinimumChainWork set to 0x10, so it won't process low-work unrequested blocks. - -We have one P2PInterface connection to node0 called test_node, and one to node1 -called min_work_node. - -The test: -1. Generate one block on each node, to leave IBD. - -2. Mine a new block on each tip, and deliver to each node from node's peer. - The tip should advance for node0, but node1 should skip processing due to - nMinimumChainWork. - -Node1 is unused in tests 3-7: - -3. Mine a block that forks from the genesis block, and deliver to test_node. - Node0 should not process this block (just accept the header), because it - is unrequested and doesn't have more or equal work to the tip. - -4a,b. Send another two blocks that build on the forking block. - Node0 should process the second block but be stuck on the shorter chain, - because it's missing an intermediate block. - -4c.Send 288 more blocks on the longer chain (the number of blocks ahead - we currently store). - Node0 should process all but the last block (too far ahead in height). - -5. Send a duplicate of the block in #3 to Node0. - Node0 should not process the block because it is unrequested, and stay on - the shorter chain. - -6. Send Node0 an inv for the height 3 block produced in #4 above. - Node0 should figure out that Node0 has the missing height 2 block and send a - getdata. - -7. Send Node0 the missing block again. - Node0 should process and the tip should advance. - -8. Create a fork which is invalid at a height longer than the current chain - (ie to which the node will try to reorg) but which has headers built on top - of the invalid block. Check that we get disconnected if we send more headers - on the chain the node now knows to be invalid. - -9. Test Node1 is able to sync when connected to node0 (which should have sufficient - work on its chain). -""" - -from test_framework.mininode import * -from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import * -import time -from test_framework.blocktools import create_block, create_coinbase, create_transaction - -class AcceptBlockTest(BitcoinTestFramework): - def add_options(self, parser): - parser.add_option("--testbinary", dest="testbinary", - default=os.getenv("BITCOIND", "bitcoind"), - help="bitcoind binary to test") - - def set_test_params(self): - self.setup_clean_chain = True - self.num_nodes = 2 - self.extra_args = [[], ["-minimumchainwork=0x10"]] - - def setup_network(self): - # Node0 will be used to test behavior of processing unrequested blocks - # from peers which are not whitelisted, while Node1 will be used for - # the whitelisted case. - # Node2 will be used for non-whitelisted peers to test the interaction - # with nMinimumChainWork. - self.setup_nodes() - - def run_test(self): - # Setup the p2p connections and start up the network thread. - # test_node connects to node0 (not whitelisted) - test_node = self.nodes[0].add_p2p_connection(P2PInterface()) - # min_work_node connects to node1 (whitelisted) - min_work_node = self.nodes[1].add_p2p_connection(P2PInterface()) - - network_thread_start() - - # Test logic begins here - test_node.wait_for_verack() - min_work_node.wait_for_verack() - - # 1. Have nodes mine a block (leave IBD) - [ n.generate(1) for n in self.nodes ] - tips = [ int("0x" + n.getbestblockhash(), 0) for n in self.nodes ] - - # 2. Send one block that builds on each tip. - # This should be accepted by node0 - blocks_h2 = [] # the height 2 blocks on each node's chain - block_time = int(time.time()) + 1 - for i in range(2): - blocks_h2.append(create_block(tips[i], create_coinbase(2), block_time)) - blocks_h2[i].solve() - block_time += 1 - test_node.send_message(msg_block(blocks_h2[0])) - min_work_node.send_message(msg_block(blocks_h2[1])) - - for x in [test_node, min_work_node]: - x.sync_with_ping() - assert_equal(self.nodes[0].getblockcount(), 2) - assert_equal(self.nodes[1].getblockcount(), 1) - self.log.info("First height 2 block accepted by node0; correctly rejected by node1") - - # 3. Send another block that builds on genesis. - block_h1f = create_block(int("0x" + self.nodes[0].getblockhash(0), 0), create_coinbase(1), block_time) - block_time += 1 - block_h1f.solve() - test_node.send_message(msg_block(block_h1f)) - - test_node.sync_with_ping() - tip_entry_found = False - for x in self.nodes[0].getchaintips(): - if x['hash'] == block_h1f.hash: - assert_equal(x['status'], "headers-only") - tip_entry_found = True - assert(tip_entry_found) - assert_raises_rpc_error(-1, "Block not found on disk", self.nodes[0].getblock, block_h1f.hash) - - # 4. Send another two block that build on the fork. - block_h2f = create_block(block_h1f.sha256, create_coinbase(2), block_time) - block_time += 1 - block_h2f.solve() - test_node.send_message(msg_block(block_h2f)) - - test_node.sync_with_ping() - # Since the earlier block was not processed by node, the new block - # can't be fully validated. - tip_entry_found = False - for x in self.nodes[0].getchaintips(): - if x['hash'] == block_h2f.hash: - assert_equal(x['status'], "headers-only") - tip_entry_found = True - assert(tip_entry_found) - - # But this block should be accepted by node since it has equal work. - self.nodes[0].getblock(block_h2f.hash) - self.log.info("Second height 2 block accepted, but not reorg'ed to") - - # 4b. Now send another block that builds on the forking chain. - block_h3 = create_block(block_h2f.sha256, create_coinbase(3), block_h2f.nTime+1) - block_h3.solve() - test_node.send_message(msg_block(block_h3)) - - test_node.sync_with_ping() - # Since the earlier block was not processed by node, the new block - # can't be fully validated. - tip_entry_found = False - for x in self.nodes[0].getchaintips(): - if x['hash'] == block_h3.hash: - assert_equal(x['status'], "headers-only") - tip_entry_found = True - assert(tip_entry_found) - self.nodes[0].getblock(block_h3.hash) - - # But this block should be accepted by node since it has more work. - self.nodes[0].getblock(block_h3.hash) - self.log.info("Unrequested more-work block accepted") - - # 4c. Now mine 288 more blocks and deliver; all should be processed but - # the last (height-too-high) on node (as long as its not missing any headers) - tip = block_h3 - all_blocks = [] - for i in range(288): - next_block = create_block(tip.sha256, create_coinbase(i + 4), tip.nTime+1) - next_block.solve() - all_blocks.append(next_block) - tip = next_block - - # Now send the block at height 5 and check that it wasn't accepted (missing header) - test_node.send_message(msg_block(all_blocks[1])) - test_node.sync_with_ping() - assert_raises_rpc_error(-5, "Block not found", self.nodes[0].getblock, all_blocks[1].hash) - assert_raises_rpc_error(-5, "Block not found", self.nodes[0].getblockheader, all_blocks[1].hash) - - # The block at height 5 should be accepted if we provide the missing header, though - headers_message = msg_headers() - headers_message.headers.append(CBlockHeader(all_blocks[0])) - test_node.send_message(headers_message) - test_node.send_message(msg_block(all_blocks[1])) - test_node.sync_with_ping() - self.nodes[0].getblock(all_blocks[1].hash) - - # Now send the blocks in all_blocks - for i in range(288): - test_node.send_message(msg_block(all_blocks[i])) - test_node.sync_with_ping() - - # Blocks 1-287 should be accepted, block 288 should be ignored because it's too far ahead - for x in all_blocks[:-1]: - self.nodes[0].getblock(x.hash) - assert_raises_rpc_error(-1, "Block not found on disk", self.nodes[0].getblock, all_blocks[-1].hash) - - # 5. Test handling of unrequested block on the node that didn't process - # Should still not be processed (even though it has a child that has more - # work). - - # The node should have requested the blocks at some point, so - # disconnect/reconnect first - - self.nodes[0].disconnect_p2ps() - self.nodes[1].disconnect_p2ps() - network_thread_join() - - test_node = self.nodes[0].add_p2p_connection(P2PInterface()) - network_thread_start() - test_node.wait_for_verack() - - test_node.send_message(msg_block(block_h1f)) - - test_node.sync_with_ping() - assert_equal(self.nodes[0].getblockcount(), 2) - self.log.info("Unrequested block that would complete more-work chain was ignored") - - # 6. Try to get node to request the missing block. - # Poke the node with an inv for block at height 3 and see if that - # triggers a getdata on block 2 (it should if block 2 is missing). - with mininode_lock: - # Clear state so we can check the getdata request - test_node.last_message.pop("getdata", None) - test_node.send_message(msg_inv([CInv(2, block_h3.sha256)])) - - test_node.sync_with_ping() - with mininode_lock: - getdata = test_node.last_message["getdata"] - - # Check that the getdata includes the right block - assert_equal(getdata.inv[0].hash, block_h1f.sha256) - self.log.info("Inv at tip triggered getdata for unprocessed block") - - # 7. Send the missing block for the third time (now it is requested) - test_node.send_message(msg_block(block_h1f)) - - test_node.sync_with_ping() - assert_equal(self.nodes[0].getblockcount(), 290) - self.nodes[0].getblock(all_blocks[286].hash) - assert_equal(self.nodes[0].getbestblockhash(), all_blocks[286].hash) - assert_raises_rpc_error(-1, "Block not found on disk", self.nodes[0].getblock, all_blocks[287].hash) - self.log.info("Successfully reorged to longer chain from non-whitelisted peer") - - # 8. Create a chain which is invalid at a height longer than the - # current chain, but which has more blocks on top of that - block_289f = create_block(all_blocks[284].sha256, create_coinbase(289), all_blocks[284].nTime+1) - block_289f.solve() - block_290f = create_block(block_289f.sha256, create_coinbase(290), block_289f.nTime+1) - block_290f.solve() - block_291 = create_block(block_290f.sha256, create_coinbase(291), block_290f.nTime+1) - # block_291 spends a coinbase below maturity! - block_291.vtx.append(create_transaction(block_290f.vtx[0], 0, b"42", 1)) - block_291.hashMerkleRoot = block_291.calc_merkle_root() - block_291.solve() - block_292 = create_block(block_291.sha256, create_coinbase(292), block_291.nTime+1) - block_292.solve() - - # Now send all the headers on the chain and enough blocks to trigger reorg - headers_message = msg_headers() - headers_message.headers.append(CBlockHeader(block_289f)) - headers_message.headers.append(CBlockHeader(block_290f)) - headers_message.headers.append(CBlockHeader(block_291)) - headers_message.headers.append(CBlockHeader(block_292)) - test_node.send_message(headers_message) - - test_node.sync_with_ping() - tip_entry_found = False - for x in self.nodes[0].getchaintips(): - if x['hash'] == block_292.hash: - assert_equal(x['status'], "headers-only") - tip_entry_found = True - assert(tip_entry_found) - assert_raises_rpc_error(-1, "Block not found on disk", self.nodes[0].getblock, block_292.hash) - - test_node.send_message(msg_block(block_289f)) - test_node.send_message(msg_block(block_290f)) - - test_node.sync_with_ping() - self.nodes[0].getblock(block_289f.hash) - self.nodes[0].getblock(block_290f.hash) - - test_node.send_message(msg_block(block_291)) - - # At this point we've sent an obviously-bogus block, wait for full processing - # without assuming whether we will be disconnected or not - try: - # Only wait a short while so the test doesn't take forever if we do get - # disconnected - test_node.sync_with_ping(timeout=1) - except AssertionError: - test_node.wait_for_disconnect() - - self.nodes[0].disconnect_p2ps() - test_node = self.nodes[0].add_p2p_connection(P2PInterface()) - - network_thread_start() - test_node.wait_for_verack() - - # We should have failed reorg and switched back to 290 (but have block 291) - assert_equal(self.nodes[0].getblockcount(), 290) - assert_equal(self.nodes[0].getbestblockhash(), all_blocks[286].hash) - assert_equal(self.nodes[0].getblock(block_291.hash)["confirmations"], -1) - - # Now send a new header on the invalid chain, indicating we're forked off, and expect to get disconnected - block_293 = create_block(block_292.sha256, create_coinbase(293), block_292.nTime+1) - block_293.solve() - headers_message = msg_headers() - headers_message.headers.append(CBlockHeader(block_293)) - test_node.send_message(headers_message) - test_node.wait_for_disconnect() - - # 9. Connect node1 to node0 and ensure it is able to sync - connect_nodes(self.nodes[0], 1) - sync_blocks([self.nodes[0], self.nodes[1]]) - self.log.info("Successfully synced nodes 1 and 0") - -if __name__ == '__main__': - AcceptBlockTest().main() diff --git a/test/functional/p2p-compactblocks.py b/test/functional/p2p-compactblocks.py deleted file mode 100755 index d9f461a049..0000000000 --- a/test/functional/p2p-compactblocks.py +++ /dev/null @@ -1,921 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2016-2017 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test compact blocks (BIP 152). - -Version 1 compact blocks are pre-segwit (txids) -Version 2 compact blocks are post-segwit (wtxids) -""" - -from test_framework.mininode import * -from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import * -from test_framework.blocktools import create_block, create_coinbase, add_witness_commitment -from test_framework.script import CScript, OP_TRUE - -# TestNode: A peer we use to send messages to bitcoind, and store responses. -class TestNode(P2PInterface): - def __init__(self): - super().__init__() - self.last_sendcmpct = [] - self.block_announced = False - # Store the hashes of blocks we've seen announced. - # This is for synchronizing the p2p message traffic, - # so we can eg wait until a particular block is announced. - self.announced_blockhashes = set() - - def on_sendcmpct(self, message): - self.last_sendcmpct.append(message) - - def on_cmpctblock(self, message): - self.block_announced = True - self.last_message["cmpctblock"].header_and_shortids.header.calc_sha256() - self.announced_blockhashes.add(self.last_message["cmpctblock"].header_and_shortids.header.sha256) - - def on_headers(self, message): - self.block_announced = True - for x in self.last_message["headers"].headers: - x.calc_sha256() - self.announced_blockhashes.add(x.sha256) - - def on_inv(self, message): - for x in self.last_message["inv"].inv: - if x.type == 2: - self.block_announced = True - self.announced_blockhashes.add(x.hash) - - # Requires caller to hold mininode_lock - def received_block_announcement(self): - return self.block_announced - - def clear_block_announcement(self): - with mininode_lock: - self.block_announced = False - self.last_message.pop("inv", None) - self.last_message.pop("headers", None) - self.last_message.pop("cmpctblock", None) - - def get_headers(self, locator, hashstop): - msg = msg_getheaders() - msg.locator.vHave = locator - msg.hashstop = hashstop - self.send_message(msg) - - def send_header_for_blocks(self, new_blocks): - headers_message = msg_headers() - headers_message.headers = [CBlockHeader(b) for b in new_blocks] - self.send_message(headers_message) - - def request_headers_and_sync(self, locator, hashstop=0): - self.clear_block_announcement() - self.get_headers(locator, hashstop) - wait_until(self.received_block_announcement, timeout=30, lock=mininode_lock) - self.clear_block_announcement() - - # Block until a block announcement for a particular block hash is - # received. - def wait_for_block_announcement(self, block_hash, timeout=30): - def received_hash(): - return (block_hash in self.announced_blockhashes) - wait_until(received_hash, timeout=timeout, lock=mininode_lock) - - def send_await_disconnect(self, message, timeout=30): - """Sends a message to the node and wait for disconnect. - - This is used when we want to send a message into the node that we expect - will get us disconnected, eg an invalid block.""" - self.send_message(message) - wait_until(lambda: self.state != "connected", timeout=timeout, lock=mininode_lock) - -class CompactBlocksTest(BitcoinTestFramework): - def set_test_params(self): - self.setup_clean_chain = True - # Node0 = pre-segwit, node1 = segwit-aware - self.num_nodes = 2 - # This test was written assuming SegWit is activated using BIP9 at height 432 (3x confirmation window). - # TODO: Rewrite this test to support SegWit being always active. - self.extra_args = [["-vbparams=segwit:0:0"], ["-vbparams=segwit:0:999999999999", "-txindex", "-deprecatedrpc=addwitnessaddress"]] - self.utxos = [] - - def build_block_on_tip(self, node, segwit=False): - height = node.getblockcount() - tip = node.getbestblockhash() - mtp = node.getblockheader(tip)['mediantime'] - block = create_block(int(tip, 16), create_coinbase(height + 1), mtp + 1) - block.nVersion = 4 - if segwit: - add_witness_commitment(block) - block.solve() - return block - - # Create 10 more anyone-can-spend utxo's for testing. - def make_utxos(self): - # Doesn't matter which node we use, just use node0. - block = self.build_block_on_tip(self.nodes[0]) - self.test_node.send_and_ping(msg_block(block)) - assert(int(self.nodes[0].getbestblockhash(), 16) == block.sha256) - self.nodes[0].generate(100) - - total_value = block.vtx[0].vout[0].nValue - out_value = total_value // 10 - tx = CTransaction() - tx.vin.append(CTxIn(COutPoint(block.vtx[0].sha256, 0), b'')) - for i in range(10): - tx.vout.append(CTxOut(out_value, CScript([OP_TRUE]))) - tx.rehash() - - block2 = self.build_block_on_tip(self.nodes[0]) - block2.vtx.append(tx) - block2.hashMerkleRoot = block2.calc_merkle_root() - block2.solve() - self.test_node.send_and_ping(msg_block(block2)) - assert_equal(int(self.nodes[0].getbestblockhash(), 16), block2.sha256) - self.utxos.extend([[tx.sha256, i, out_value] for i in range(10)]) - return - - # Test "sendcmpct" (between peers preferring the same version): - # - No compact block announcements unless sendcmpct is sent. - # - If sendcmpct is sent with version > preferred_version, the message is ignored. - # - If sendcmpct is sent with boolean 0, then block announcements are not - # made with compact blocks. - # - If sendcmpct is then sent with boolean 1, then new block announcements - # are made with compact blocks. - # If old_node is passed in, request compact blocks with version=preferred-1 - # and verify that it receives block announcements via compact block. - def test_sendcmpct(self, node, test_node, preferred_version, old_node=None): - # Make sure we get a SENDCMPCT message from our peer - def received_sendcmpct(): - return (len(test_node.last_sendcmpct) > 0) - wait_until(received_sendcmpct, timeout=30, lock=mininode_lock) - with mininode_lock: - # Check that the first version received is the preferred one - assert_equal(test_node.last_sendcmpct[0].version, preferred_version) - # And that we receive versions down to 1. - assert_equal(test_node.last_sendcmpct[-1].version, 1) - test_node.last_sendcmpct = [] - - tip = int(node.getbestblockhash(), 16) - - def check_announcement_of_new_block(node, peer, predicate): - peer.clear_block_announcement() - block_hash = int(node.generate(1)[0], 16) - peer.wait_for_block_announcement(block_hash, timeout=30) - assert(peer.block_announced) - - with mininode_lock: - assert predicate(peer), ( - "block_hash={!r}, cmpctblock={!r}, inv={!r}".format( - block_hash, peer.last_message.get("cmpctblock", None), peer.last_message.get("inv", None))) - - # We shouldn't get any block announcements via cmpctblock yet. - check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message) - - # Try one more time, this time after requesting headers. - test_node.request_headers_and_sync(locator=[tip]) - check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message and "inv" in p.last_message) - - # Test a few ways of using sendcmpct that should NOT - # result in compact block announcements. - # Before each test, sync the headers chain. - test_node.request_headers_and_sync(locator=[tip]) - - # Now try a SENDCMPCT message with too-high version - sendcmpct = msg_sendcmpct() - sendcmpct.version = preferred_version+1 - sendcmpct.announce = True - test_node.send_and_ping(sendcmpct) - check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message) - - # Headers sync before next test. - test_node.request_headers_and_sync(locator=[tip]) - - # Now try a SENDCMPCT message with valid version, but announce=False - sendcmpct.version = preferred_version - sendcmpct.announce = False - test_node.send_and_ping(sendcmpct) - check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message) - - # Headers sync before next test. - test_node.request_headers_and_sync(locator=[tip]) - - # Finally, try a SENDCMPCT message with announce=True - sendcmpct.version = preferred_version - sendcmpct.announce = True - test_node.send_and_ping(sendcmpct) - check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" in p.last_message) - - # Try one more time (no headers sync should be needed!) - check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" in p.last_message) - - # Try one more time, after turning on sendheaders - test_node.send_and_ping(msg_sendheaders()) - check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" in p.last_message) - - # Try one more time, after sending a version-1, announce=false message. - sendcmpct.version = preferred_version-1 - sendcmpct.announce = False - test_node.send_and_ping(sendcmpct) - check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" in p.last_message) - - # Now turn off announcements - sendcmpct.version = preferred_version - sendcmpct.announce = False - test_node.send_and_ping(sendcmpct) - check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message and "headers" in p.last_message) - - if old_node is not None: - # Verify that a peer using an older protocol version can receive - # announcements from this node. - sendcmpct.version = preferred_version-1 - sendcmpct.announce = True - old_node.send_and_ping(sendcmpct) - # Header sync - old_node.request_headers_and_sync(locator=[tip]) - check_announcement_of_new_block(node, old_node, lambda p: "cmpctblock" in p.last_message) - - # This test actually causes bitcoind to (reasonably!) disconnect us, so do this last. - def test_invalid_cmpctblock_message(self): - self.nodes[0].generate(101) - block = self.build_block_on_tip(self.nodes[0]) - - cmpct_block = P2PHeaderAndShortIDs() - cmpct_block.header = CBlockHeader(block) - cmpct_block.prefilled_txn_length = 1 - # This index will be too high - prefilled_txn = PrefilledTransaction(1, block.vtx[0]) - cmpct_block.prefilled_txn = [prefilled_txn] - self.test_node.send_await_disconnect(msg_cmpctblock(cmpct_block)) - assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.hashPrevBlock) - - # Compare the generated shortids to what we expect based on BIP 152, given - # bitcoind's choice of nonce. - def test_compactblock_construction(self, node, test_node, version, use_witness_address): - # Generate a bunch of transactions. - node.generate(101) - num_transactions = 25 - address = node.getnewaddress() - if use_witness_address: - # Want at least one segwit spend, so move all funds to - # a witness address. - address = node.addwitnessaddress(address) - value_to_send = node.getbalance() - node.sendtoaddress(address, satoshi_round(value_to_send-Decimal(0.1))) - node.generate(1) - - segwit_tx_generated = False - for i in range(num_transactions): - txid = node.sendtoaddress(address, 0.1) - hex_tx = node.gettransaction(txid)["hex"] - tx = FromHex(CTransaction(), hex_tx) - if not tx.wit.is_null(): - segwit_tx_generated = True - - if use_witness_address: - assert(segwit_tx_generated) # check that our test is not broken - - # Wait until we've seen the block announcement for the resulting tip - tip = int(node.getbestblockhash(), 16) - test_node.wait_for_block_announcement(tip) - - # Make sure we will receive a fast-announce compact block - self.request_cb_announcements(test_node, node, version) - - # Now mine a block, and look at the resulting compact block. - test_node.clear_block_announcement() - block_hash = int(node.generate(1)[0], 16) - - # Store the raw block in our internal format. - block = FromHex(CBlock(), node.getblock("%02x" % block_hash, False)) - for tx in block.vtx: - tx.calc_sha256() - block.rehash() - - # Wait until the block was announced (via compact blocks) - wait_until(test_node.received_block_announcement, timeout=30, lock=mininode_lock) - - # Now fetch and check the compact block - header_and_shortids = None - with mininode_lock: - assert("cmpctblock" in test_node.last_message) - # Convert the on-the-wire representation to absolute indexes - header_and_shortids = HeaderAndShortIDs(test_node.last_message["cmpctblock"].header_and_shortids) - self.check_compactblock_construction_from_block(version, header_and_shortids, block_hash, block) - - # Now fetch the compact block using a normal non-announce getdata - with mininode_lock: - test_node.clear_block_announcement() - inv = CInv(4, block_hash) # 4 == "CompactBlock" - test_node.send_message(msg_getdata([inv])) - - wait_until(test_node.received_block_announcement, timeout=30, lock=mininode_lock) - - # Now fetch and check the compact block - header_and_shortids = None - with mininode_lock: - assert("cmpctblock" in test_node.last_message) - # Convert the on-the-wire representation to absolute indexes - header_and_shortids = HeaderAndShortIDs(test_node.last_message["cmpctblock"].header_and_shortids) - self.check_compactblock_construction_from_block(version, header_and_shortids, block_hash, block) - - def check_compactblock_construction_from_block(self, version, header_and_shortids, block_hash, block): - # Check that we got the right block! - header_and_shortids.header.calc_sha256() - assert_equal(header_and_shortids.header.sha256, block_hash) - - # Make sure the prefilled_txn appears to have included the coinbase - assert(len(header_and_shortids.prefilled_txn) >= 1) - assert_equal(header_and_shortids.prefilled_txn[0].index, 0) - - # Check that all prefilled_txn entries match what's in the block. - for entry in header_and_shortids.prefilled_txn: - entry.tx.calc_sha256() - # This checks the non-witness parts of the tx agree - assert_equal(entry.tx.sha256, block.vtx[entry.index].sha256) - - # And this checks the witness - wtxid = entry.tx.calc_sha256(True) - if version == 2: - assert_equal(wtxid, block.vtx[entry.index].calc_sha256(True)) - else: - # Shouldn't have received a witness - assert(entry.tx.wit.is_null()) - - # Check that the cmpctblock message announced all the transactions. - assert_equal(len(header_and_shortids.prefilled_txn) + len(header_and_shortids.shortids), len(block.vtx)) - - # And now check that all the shortids are as expected as well. - # Determine the siphash keys to use. - [k0, k1] = header_and_shortids.get_siphash_keys() - - index = 0 - while index < len(block.vtx): - if (len(header_and_shortids.prefilled_txn) > 0 and - header_and_shortids.prefilled_txn[0].index == index): - # Already checked prefilled transactions above - header_and_shortids.prefilled_txn.pop(0) - else: - tx_hash = block.vtx[index].sha256 - if version == 2: - tx_hash = block.vtx[index].calc_sha256(True) - shortid = calculate_shortid(k0, k1, tx_hash) - assert_equal(shortid, header_and_shortids.shortids[0]) - header_and_shortids.shortids.pop(0) - index += 1 - - # Test that bitcoind requests compact blocks when we announce new blocks - # via header or inv, and that responding to getblocktxn causes the block - # to be successfully reconstructed. - # Post-segwit: upgraded nodes would only make this request of cb-version-2, - # NODE_WITNESS peers. Unupgraded nodes would still make this request of - # any cb-version-1-supporting peer. - def test_compactblock_requests(self, node, test_node, version, segwit): - # Try announcing a block with an inv or header, expect a compactblock - # request - for announce in ["inv", "header"]: - block = self.build_block_on_tip(node, segwit=segwit) - with mininode_lock: - test_node.last_message.pop("getdata", None) - - if announce == "inv": - test_node.send_message(msg_inv([CInv(2, block.sha256)])) - wait_until(lambda: "getheaders" in test_node.last_message, timeout=30, lock=mininode_lock) - test_node.send_header_for_blocks([block]) - else: - test_node.send_header_for_blocks([block]) - wait_until(lambda: "getdata" in test_node.last_message, timeout=30, lock=mininode_lock) - assert_equal(len(test_node.last_message["getdata"].inv), 1) - assert_equal(test_node.last_message["getdata"].inv[0].type, 4) - assert_equal(test_node.last_message["getdata"].inv[0].hash, block.sha256) - - # Send back a compactblock message that omits the coinbase - comp_block = HeaderAndShortIDs() - comp_block.header = CBlockHeader(block) - comp_block.nonce = 0 - [k0, k1] = comp_block.get_siphash_keys() - coinbase_hash = block.vtx[0].sha256 - if version == 2: - coinbase_hash = block.vtx[0].calc_sha256(True) - comp_block.shortids = [ - calculate_shortid(k0, k1, coinbase_hash) ] - test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) - assert_equal(int(node.getbestblockhash(), 16), block.hashPrevBlock) - # Expect a getblocktxn message. - with mininode_lock: - assert("getblocktxn" in test_node.last_message) - absolute_indexes = test_node.last_message["getblocktxn"].block_txn_request.to_absolute() - assert_equal(absolute_indexes, [0]) # should be a coinbase request - - # Send the coinbase, and verify that the tip advances. - if version == 2: - msg = msg_witness_blocktxn() - else: - msg = msg_blocktxn() - msg.block_transactions.blockhash = block.sha256 - msg.block_transactions.transactions = [block.vtx[0]] - test_node.send_and_ping(msg) - assert_equal(int(node.getbestblockhash(), 16), block.sha256) - - # Create a chain of transactions from given utxo, and add to a new block. - def build_block_with_transactions(self, node, utxo, num_transactions): - block = self.build_block_on_tip(node) - - for i in range(num_transactions): - tx = CTransaction() - tx.vin.append(CTxIn(COutPoint(utxo[0], utxo[1]), b'')) - tx.vout.append(CTxOut(utxo[2] - 1000, CScript([OP_TRUE]))) - tx.rehash() - utxo = [tx.sha256, 0, tx.vout[0].nValue] - block.vtx.append(tx) - - block.hashMerkleRoot = block.calc_merkle_root() - block.solve() - return block - - # Test that we only receive getblocktxn requests for transactions that the - # node needs, and that responding to them causes the block to be - # reconstructed. - def test_getblocktxn_requests(self, node, test_node, version): - with_witness = (version==2) - - def test_getblocktxn_response(compact_block, peer, expected_result): - msg = msg_cmpctblock(compact_block.to_p2p()) - peer.send_and_ping(msg) - with mininode_lock: - assert("getblocktxn" in peer.last_message) - absolute_indexes = peer.last_message["getblocktxn"].block_txn_request.to_absolute() - assert_equal(absolute_indexes, expected_result) - - def test_tip_after_message(node, peer, msg, tip): - peer.send_and_ping(msg) - assert_equal(int(node.getbestblockhash(), 16), tip) - - # First try announcing compactblocks that won't reconstruct, and verify - # that we receive getblocktxn messages back. - utxo = self.utxos.pop(0) - - block = self.build_block_with_transactions(node, utxo, 5) - self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue]) - comp_block = HeaderAndShortIDs() - comp_block.initialize_from_block(block, use_witness=with_witness) - - test_getblocktxn_response(comp_block, test_node, [1, 2, 3, 4, 5]) - - msg_bt = msg_blocktxn() - if with_witness: - msg_bt = msg_witness_blocktxn() # serialize with witnesses - msg_bt.block_transactions = BlockTransactions(block.sha256, block.vtx[1:]) - test_tip_after_message(node, test_node, msg_bt, block.sha256) - - utxo = self.utxos.pop(0) - block = self.build_block_with_transactions(node, utxo, 5) - self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue]) - - # Now try interspersing the prefilled transactions - comp_block.initialize_from_block(block, prefill_list=[0, 1, 5], use_witness=with_witness) - test_getblocktxn_response(comp_block, test_node, [2, 3, 4]) - msg_bt.block_transactions = BlockTransactions(block.sha256, block.vtx[2:5]) - test_tip_after_message(node, test_node, msg_bt, block.sha256) - - # Now try giving one transaction ahead of time. - utxo = self.utxos.pop(0) - block = self.build_block_with_transactions(node, utxo, 5) - self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue]) - test_node.send_and_ping(msg_tx(block.vtx[1])) - assert(block.vtx[1].hash in node.getrawmempool()) - - # Prefill 4 out of the 6 transactions, and verify that only the one - # that was not in the mempool is requested. - comp_block.initialize_from_block(block, prefill_list=[0, 2, 3, 4], use_witness=with_witness) - test_getblocktxn_response(comp_block, test_node, [5]) - - msg_bt.block_transactions = BlockTransactions(block.sha256, [block.vtx[5]]) - test_tip_after_message(node, test_node, msg_bt, block.sha256) - - # Now provide all transactions to the node before the block is - # announced and verify reconstruction happens immediately. - utxo = self.utxos.pop(0) - block = self.build_block_with_transactions(node, utxo, 10) - self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue]) - for tx in block.vtx[1:]: - test_node.send_message(msg_tx(tx)) - test_node.sync_with_ping() - # Make sure all transactions were accepted. - mempool = node.getrawmempool() - for tx in block.vtx[1:]: - assert(tx.hash in mempool) - - # Clear out last request. - with mininode_lock: - test_node.last_message.pop("getblocktxn", None) - - # Send compact block - comp_block.initialize_from_block(block, prefill_list=[0], use_witness=with_witness) - test_tip_after_message(node, test_node, msg_cmpctblock(comp_block.to_p2p()), block.sha256) - with mininode_lock: - # Shouldn't have gotten a request for any transaction - assert("getblocktxn" not in test_node.last_message) - - # Incorrectly responding to a getblocktxn shouldn't cause the block to be - # permanently failed. - def test_incorrect_blocktxn_response(self, node, test_node, version): - if (len(self.utxos) == 0): - self.make_utxos() - utxo = self.utxos.pop(0) - - block = self.build_block_with_transactions(node, utxo, 10) - self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue]) - # Relay the first 5 transactions from the block in advance - for tx in block.vtx[1:6]: - test_node.send_message(msg_tx(tx)) - test_node.sync_with_ping() - # Make sure all transactions were accepted. - mempool = node.getrawmempool() - for tx in block.vtx[1:6]: - assert(tx.hash in mempool) - - # Send compact block - comp_block = HeaderAndShortIDs() - comp_block.initialize_from_block(block, prefill_list=[0], use_witness=(version == 2)) - test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) - absolute_indexes = [] - with mininode_lock: - assert("getblocktxn" in test_node.last_message) - absolute_indexes = test_node.last_message["getblocktxn"].block_txn_request.to_absolute() - assert_equal(absolute_indexes, [6, 7, 8, 9, 10]) - - # Now give an incorrect response. - # Note that it's possible for bitcoind to be smart enough to know we're - # lying, since it could check to see if the shortid matches what we're - # sending, and eg disconnect us for misbehavior. If that behavior - # change were made, we could just modify this test by having a - # different peer provide the block further down, so that we're still - # verifying that the block isn't marked bad permanently. This is good - # enough for now. - msg = msg_blocktxn() - if version==2: - msg = msg_witness_blocktxn() - msg.block_transactions = BlockTransactions(block.sha256, [block.vtx[5]] + block.vtx[7:]) - test_node.send_and_ping(msg) - - # Tip should not have updated - assert_equal(int(node.getbestblockhash(), 16), block.hashPrevBlock) - - # We should receive a getdata request - wait_until(lambda: "getdata" in test_node.last_message, timeout=10, lock=mininode_lock) - assert_equal(len(test_node.last_message["getdata"].inv), 1) - assert(test_node.last_message["getdata"].inv[0].type == 2 or test_node.last_message["getdata"].inv[0].type == 2|MSG_WITNESS_FLAG) - assert_equal(test_node.last_message["getdata"].inv[0].hash, block.sha256) - - # Deliver the block - if version==2: - test_node.send_and_ping(msg_witness_block(block)) - else: - test_node.send_and_ping(msg_block(block)) - assert_equal(int(node.getbestblockhash(), 16), block.sha256) - - def test_getblocktxn_handler(self, node, test_node, version): - # bitcoind will not send blocktxn responses for blocks whose height is - # more than 10 blocks deep. - MAX_GETBLOCKTXN_DEPTH = 10 - chain_height = node.getblockcount() - current_height = chain_height - while (current_height >= chain_height - MAX_GETBLOCKTXN_DEPTH): - block_hash = node.getblockhash(current_height) - block = FromHex(CBlock(), node.getblock(block_hash, False)) - - msg = msg_getblocktxn() - msg.block_txn_request = BlockTransactionsRequest(int(block_hash, 16), []) - num_to_request = random.randint(1, len(block.vtx)) - msg.block_txn_request.from_absolute(sorted(random.sample(range(len(block.vtx)), num_to_request))) - test_node.send_message(msg) - wait_until(lambda: "blocktxn" in test_node.last_message, timeout=10, lock=mininode_lock) - - [tx.calc_sha256() for tx in block.vtx] - with mininode_lock: - assert_equal(test_node.last_message["blocktxn"].block_transactions.blockhash, int(block_hash, 16)) - all_indices = msg.block_txn_request.to_absolute() - for index in all_indices: - tx = test_node.last_message["blocktxn"].block_transactions.transactions.pop(0) - tx.calc_sha256() - assert_equal(tx.sha256, block.vtx[index].sha256) - if version == 1: - # Witnesses should have been stripped - assert(tx.wit.is_null()) - else: - # Check that the witness matches - assert_equal(tx.calc_sha256(True), block.vtx[index].calc_sha256(True)) - test_node.last_message.pop("blocktxn", None) - current_height -= 1 - - # Next request should send a full block response, as we're past the - # allowed depth for a blocktxn response. - block_hash = node.getblockhash(current_height) - msg.block_txn_request = BlockTransactionsRequest(int(block_hash, 16), [0]) - with mininode_lock: - test_node.last_message.pop("block", None) - test_node.last_message.pop("blocktxn", None) - test_node.send_and_ping(msg) - with mininode_lock: - test_node.last_message["block"].block.calc_sha256() - assert_equal(test_node.last_message["block"].block.sha256, int(block_hash, 16)) - assert "blocktxn" not in test_node.last_message - - def test_compactblocks_not_at_tip(self, node, test_node): - # Test that requesting old compactblocks doesn't work. - MAX_CMPCTBLOCK_DEPTH = 5 - new_blocks = [] - for i in range(MAX_CMPCTBLOCK_DEPTH + 1): - test_node.clear_block_announcement() - new_blocks.append(node.generate(1)[0]) - wait_until(test_node.received_block_announcement, timeout=30, lock=mininode_lock) - - test_node.clear_block_announcement() - test_node.send_message(msg_getdata([CInv(4, int(new_blocks[0], 16))])) - wait_until(lambda: "cmpctblock" in test_node.last_message, timeout=30, lock=mininode_lock) - - test_node.clear_block_announcement() - node.generate(1) - wait_until(test_node.received_block_announcement, timeout=30, lock=mininode_lock) - test_node.clear_block_announcement() - with mininode_lock: - test_node.last_message.pop("block", None) - test_node.send_message(msg_getdata([CInv(4, int(new_blocks[0], 16))])) - wait_until(lambda: "block" in test_node.last_message, timeout=30, lock=mininode_lock) - with mininode_lock: - test_node.last_message["block"].block.calc_sha256() - assert_equal(test_node.last_message["block"].block.sha256, int(new_blocks[0], 16)) - - # Generate an old compactblock, and verify that it's not accepted. - cur_height = node.getblockcount() - hashPrevBlock = int(node.getblockhash(cur_height-5), 16) - block = self.build_block_on_tip(node) - block.hashPrevBlock = hashPrevBlock - block.solve() - - comp_block = HeaderAndShortIDs() - comp_block.initialize_from_block(block) - test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) - - tips = node.getchaintips() - found = False - for x in tips: - if x["hash"] == block.hash: - assert_equal(x["status"], "headers-only") - found = True - break - assert(found) - - # Requesting this block via getblocktxn should silently fail - # (to avoid fingerprinting attacks). - msg = msg_getblocktxn() - msg.block_txn_request = BlockTransactionsRequest(block.sha256, [0]) - with mininode_lock: - test_node.last_message.pop("blocktxn", None) - test_node.send_and_ping(msg) - with mininode_lock: - assert "blocktxn" not in test_node.last_message - - def activate_segwit(self, node): - node.generate(144*3) - assert_equal(get_bip9_status(node, "segwit")["status"], 'active') - - def test_end_to_end_block_relay(self, node, listeners): - utxo = self.utxos.pop(0) - - block = self.build_block_with_transactions(node, utxo, 10) - - [l.clear_block_announcement() for l in listeners] - - # ToHex() won't serialize with witness, but this block has no witnesses - # anyway. TODO: repeat this test with witness tx's to a segwit node. - node.submitblock(ToHex(block)) - - for l in listeners: - wait_until(lambda: l.received_block_announcement(), timeout=30, lock=mininode_lock) - with mininode_lock: - for l in listeners: - assert "cmpctblock" in l.last_message - l.last_message["cmpctblock"].header_and_shortids.header.calc_sha256() - assert_equal(l.last_message["cmpctblock"].header_and_shortids.header.sha256, block.sha256) - - # Test that we don't get disconnected if we relay a compact block with valid header, - # but invalid transactions. - def test_invalid_tx_in_compactblock(self, node, test_node, use_segwit): - assert(len(self.utxos)) - utxo = self.utxos[0] - - block = self.build_block_with_transactions(node, utxo, 5) - del block.vtx[3] - block.hashMerkleRoot = block.calc_merkle_root() - if use_segwit: - # If we're testing with segwit, also drop the coinbase witness, - # but include the witness commitment. - add_witness_commitment(block) - block.vtx[0].wit.vtxinwit = [] - block.solve() - - # Now send the compact block with all transactions prefilled, and - # verify that we don't get disconnected. - comp_block = HeaderAndShortIDs() - comp_block.initialize_from_block(block, prefill_list=[0, 1, 2, 3, 4], use_witness=use_segwit) - msg = msg_cmpctblock(comp_block.to_p2p()) - test_node.send_and_ping(msg) - - # Check that the tip didn't advance - assert(int(node.getbestblockhash(), 16) is not block.sha256) - test_node.sync_with_ping() - - # Helper for enabling cb announcements - # Send the sendcmpct request and sync headers - def request_cb_announcements(self, peer, node, version): - tip = node.getbestblockhash() - peer.get_headers(locator=[int(tip, 16)], hashstop=0) - - msg = msg_sendcmpct() - msg.version = version - msg.announce = True - peer.send_and_ping(msg) - - def test_compactblock_reconstruction_multiple_peers(self, node, stalling_peer, delivery_peer): - assert(len(self.utxos)) - - def announce_cmpct_block(node, peer): - utxo = self.utxos.pop(0) - block = self.build_block_with_transactions(node, utxo, 5) - - cmpct_block = HeaderAndShortIDs() - cmpct_block.initialize_from_block(block) - msg = msg_cmpctblock(cmpct_block.to_p2p()) - peer.send_and_ping(msg) - with mininode_lock: - assert "getblocktxn" in peer.last_message - return block, cmpct_block - - block, cmpct_block = announce_cmpct_block(node, stalling_peer) - - for tx in block.vtx[1:]: - delivery_peer.send_message(msg_tx(tx)) - delivery_peer.sync_with_ping() - mempool = node.getrawmempool() - for tx in block.vtx[1:]: - assert(tx.hash in mempool) - - delivery_peer.send_and_ping(msg_cmpctblock(cmpct_block.to_p2p())) - assert_equal(int(node.getbestblockhash(), 16), block.sha256) - - self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue]) - - # Now test that delivering an invalid compact block won't break relay - - block, cmpct_block = announce_cmpct_block(node, stalling_peer) - for tx in block.vtx[1:]: - delivery_peer.send_message(msg_tx(tx)) - delivery_peer.sync_with_ping() - - cmpct_block.prefilled_txn[0].tx.wit.vtxinwit = [ CTxInWitness() ] - cmpct_block.prefilled_txn[0].tx.wit.vtxinwit[0].scriptWitness.stack = [ser_uint256(0)] - - cmpct_block.use_witness = True - delivery_peer.send_and_ping(msg_cmpctblock(cmpct_block.to_p2p())) - assert(int(node.getbestblockhash(), 16) != block.sha256) - - msg = msg_blocktxn() - msg.block_transactions.blockhash = block.sha256 - msg.block_transactions.transactions = block.vtx[1:] - stalling_peer.send_and_ping(msg) - assert_equal(int(node.getbestblockhash(), 16), block.sha256) - - def run_test(self): - # Setup the p2p connections and start up the network thread. - self.test_node = self.nodes[0].add_p2p_connection(TestNode()) - self.segwit_node = self.nodes[1].add_p2p_connection(TestNode(), services=NODE_NETWORK|NODE_WITNESS) - self.old_node = self.nodes[1].add_p2p_connection(TestNode(), services=NODE_NETWORK) - - network_thread_start() - - self.test_node.wait_for_verack() - - # We will need UTXOs to construct transactions in later tests. - self.make_utxos() - - self.log.info("Running tests, pre-segwit activation:") - - self.log.info("Testing SENDCMPCT p2p message... ") - self.test_sendcmpct(self.nodes[0], self.test_node, 1) - sync_blocks(self.nodes) - self.test_sendcmpct(self.nodes[1], self.segwit_node, 2, old_node=self.old_node) - sync_blocks(self.nodes) - - self.log.info("Testing compactblock construction...") - self.test_compactblock_construction(self.nodes[0], self.test_node, 1, False) - sync_blocks(self.nodes) - self.test_compactblock_construction(self.nodes[1], self.segwit_node, 2, False) - sync_blocks(self.nodes) - - self.log.info("Testing compactblock requests... ") - self.test_compactblock_requests(self.nodes[0], self.test_node, 1, False) - sync_blocks(self.nodes) - self.test_compactblock_requests(self.nodes[1], self.segwit_node, 2, False) - sync_blocks(self.nodes) - - self.log.info("Testing getblocktxn requests...") - self.test_getblocktxn_requests(self.nodes[0], self.test_node, 1) - sync_blocks(self.nodes) - self.test_getblocktxn_requests(self.nodes[1], self.segwit_node, 2) - sync_blocks(self.nodes) - - self.log.info("Testing getblocktxn handler...") - self.test_getblocktxn_handler(self.nodes[0], self.test_node, 1) - sync_blocks(self.nodes) - self.test_getblocktxn_handler(self.nodes[1], self.segwit_node, 2) - self.test_getblocktxn_handler(self.nodes[1], self.old_node, 1) - sync_blocks(self.nodes) - - self.log.info("Testing compactblock requests/announcements not at chain tip...") - self.test_compactblocks_not_at_tip(self.nodes[0], self.test_node) - sync_blocks(self.nodes) - self.test_compactblocks_not_at_tip(self.nodes[1], self.segwit_node) - self.test_compactblocks_not_at_tip(self.nodes[1], self.old_node) - sync_blocks(self.nodes) - - self.log.info("Testing handling of incorrect blocktxn responses...") - self.test_incorrect_blocktxn_response(self.nodes[0], self.test_node, 1) - sync_blocks(self.nodes) - self.test_incorrect_blocktxn_response(self.nodes[1], self.segwit_node, 2) - sync_blocks(self.nodes) - - # End-to-end block relay tests - self.log.info("Testing end-to-end block relay...") - self.request_cb_announcements(self.test_node, self.nodes[0], 1) - self.request_cb_announcements(self.old_node, self.nodes[1], 1) - self.request_cb_announcements(self.segwit_node, self.nodes[1], 2) - self.test_end_to_end_block_relay(self.nodes[0], [self.segwit_node, self.test_node, self.old_node]) - self.test_end_to_end_block_relay(self.nodes[1], [self.segwit_node, self.test_node, self.old_node]) - - self.log.info("Testing handling of invalid compact blocks...") - self.test_invalid_tx_in_compactblock(self.nodes[0], self.test_node, False) - self.test_invalid_tx_in_compactblock(self.nodes[1], self.segwit_node, False) - self.test_invalid_tx_in_compactblock(self.nodes[1], self.old_node, False) - - self.log.info("Testing reconstructing compact blocks from all peers...") - self.test_compactblock_reconstruction_multiple_peers(self.nodes[1], self.segwit_node, self.old_node) - sync_blocks(self.nodes) - - # Advance to segwit activation - self.log.info("Advancing to segwit activation") - self.activate_segwit(self.nodes[1]) - self.log.info("Running tests, post-segwit activation...") - - self.log.info("Testing compactblock construction...") - self.test_compactblock_construction(self.nodes[1], self.old_node, 1, True) - self.test_compactblock_construction(self.nodes[1], self.segwit_node, 2, True) - sync_blocks(self.nodes) - - self.log.info("Testing compactblock requests (unupgraded node)... ") - self.test_compactblock_requests(self.nodes[0], self.test_node, 1, True) - - self.log.info("Testing getblocktxn requests (unupgraded node)...") - self.test_getblocktxn_requests(self.nodes[0], self.test_node, 1) - - # Need to manually sync node0 and node1, because post-segwit activation, - # node1 will not download blocks from node0. - self.log.info("Syncing nodes...") - assert(self.nodes[0].getbestblockhash() != self.nodes[1].getbestblockhash()) - while (self.nodes[0].getblockcount() > self.nodes[1].getblockcount()): - block_hash = self.nodes[0].getblockhash(self.nodes[1].getblockcount()+1) - self.nodes[1].submitblock(self.nodes[0].getblock(block_hash, False)) - assert_equal(self.nodes[0].getbestblockhash(), self.nodes[1].getbestblockhash()) - - self.log.info("Testing compactblock requests (segwit node)... ") - self.test_compactblock_requests(self.nodes[1], self.segwit_node, 2, True) - - self.log.info("Testing getblocktxn requests (segwit node)...") - self.test_getblocktxn_requests(self.nodes[1], self.segwit_node, 2) - sync_blocks(self.nodes) - - self.log.info("Testing getblocktxn handler (segwit node should return witnesses)...") - self.test_getblocktxn_handler(self.nodes[1], self.segwit_node, 2) - self.test_getblocktxn_handler(self.nodes[1], self.old_node, 1) - - # Test that if we submitblock to node1, we'll get a compact block - # announcement to all peers. - # (Post-segwit activation, blocks won't propagate from node0 to node1 - # automatically, so don't bother testing a block announced to node0.) - self.log.info("Testing end-to-end block relay...") - self.request_cb_announcements(self.test_node, self.nodes[0], 1) - self.request_cb_announcements(self.old_node, self.nodes[1], 1) - self.request_cb_announcements(self.segwit_node, self.nodes[1], 2) - self.test_end_to_end_block_relay(self.nodes[1], [self.segwit_node, self.test_node, self.old_node]) - - self.log.info("Testing handling of invalid compact blocks...") - self.test_invalid_tx_in_compactblock(self.nodes[0], self.test_node, False) - self.test_invalid_tx_in_compactblock(self.nodes[1], self.segwit_node, True) - self.test_invalid_tx_in_compactblock(self.nodes[1], self.old_node, True) - - self.log.info("Testing invalid index in cmpctblock message...") - self.test_invalid_cmpctblock_message() - - -if __name__ == '__main__': - CompactBlocksTest().main() diff --git a/test/functional/p2p-feefilter.py b/test/functional/p2p-feefilter.py deleted file mode 100755 index 47d9c55160..0000000000 --- a/test/functional/p2p-feefilter.py +++ /dev/null @@ -1,93 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2016-2017 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test processing of feefilter messages.""" - -from test_framework.mininode import * -from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import * -import time - - -def hashToHex(hash): - return format(hash, '064x') - -# Wait up to 60 secs to see if the testnode has received all the expected invs -def allInvsMatch(invsExpected, testnode): - for x in range(60): - with mininode_lock: - if (sorted(invsExpected) == sorted(testnode.txinvs)): - return True - time.sleep(1) - return False - -class TestNode(P2PInterface): - def __init__(self): - super().__init__() - self.txinvs = [] - - def on_inv(self, message): - for i in message.inv: - if (i.type == 1): - self.txinvs.append(hashToHex(i.hash)) - - def clear_invs(self): - with mininode_lock: - self.txinvs = [] - -class FeeFilterTest(BitcoinTestFramework): - def set_test_params(self): - self.num_nodes = 2 - - def run_test(self): - node1 = self.nodes[1] - node0 = self.nodes[0] - # Get out of IBD - node1.generate(1) - sync_blocks(self.nodes) - - # Setup the p2p connections and start up the network thread. - self.nodes[0].add_p2p_connection(TestNode()) - network_thread_start() - self.nodes[0].p2p.wait_for_verack() - - # Test that invs are received for all txs at feerate of 20 sat/byte - node1.settxfee(Decimal("0.00020000")) - txids = [node1.sendtoaddress(node1.getnewaddress(), 1) for x in range(3)] - assert(allInvsMatch(txids, self.nodes[0].p2p)) - self.nodes[0].p2p.clear_invs() - - # Set a filter of 15 sat/byte - self.nodes[0].p2p.send_and_ping(msg_feefilter(15000)) - - # Test that txs are still being received (paying 20 sat/byte) - txids = [node1.sendtoaddress(node1.getnewaddress(), 1) for x in range(3)] - assert(allInvsMatch(txids, self.nodes[0].p2p)) - self.nodes[0].p2p.clear_invs() - - # Change tx fee rate to 10 sat/byte and test they are no longer received - node1.settxfee(Decimal("0.00010000")) - [node1.sendtoaddress(node1.getnewaddress(), 1) for x in range(3)] - sync_mempools(self.nodes) # must be sure node 0 has received all txs - - # Send one transaction from node0 that should be received, so that we - # we can sync the test on receipt (if node1's txs were relayed, they'd - # be received by the time this node0 tx is received). This is - # unfortunately reliant on the current relay behavior where we batch up - # to 35 entries in an inv, which means that when this next transaction - # is eligible for relay, the prior transactions from node1 are eligible - # as well. - node0.settxfee(Decimal("0.00020000")) - txids = [node0.sendtoaddress(node0.getnewaddress(), 1)] - assert(allInvsMatch(txids, self.nodes[0].p2p)) - self.nodes[0].p2p.clear_invs() - - # Remove fee filter and check that txs are received again - self.nodes[0].p2p.send_and_ping(msg_feefilter(0)) - txids = [node1.sendtoaddress(node1.getnewaddress(), 1) for x in range(3)] - assert(allInvsMatch(txids, self.nodes[0].p2p)) - self.nodes[0].p2p.clear_invs() - -if __name__ == '__main__': - FeeFilterTest().main() diff --git a/test/functional/p2p-fingerprint.py b/test/functional/p2p-fingerprint.py deleted file mode 100755 index 93ef73e25e..0000000000 --- a/test/functional/p2p-fingerprint.py +++ /dev/null @@ -1,152 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2017 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test various fingerprinting protections. - -If an stale block more than a month old or its header are requested by a peer, -the node should pretend that it does not have it to avoid fingerprinting. -""" - -import time - -from test_framework.blocktools import (create_block, create_coinbase) -from test_framework.mininode import ( - CInv, - P2PInterface, - msg_headers, - msg_block, - msg_getdata, - msg_getheaders, - network_thread_start, - wait_until, -) -from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import ( - assert_equal, -) - -class P2PFingerprintTest(BitcoinTestFramework): - def set_test_params(self): - self.setup_clean_chain = True - self.num_nodes = 1 - - # Build a chain of blocks on top of given one - def build_chain(self, nblocks, prev_hash, prev_height, prev_median_time): - blocks = [] - for _ in range(nblocks): - coinbase = create_coinbase(prev_height + 1) - block_time = prev_median_time + 1 - block = create_block(int(prev_hash, 16), coinbase, block_time) - block.solve() - - blocks.append(block) - prev_hash = block.hash - prev_height += 1 - prev_median_time = block_time - return blocks - - # Send a getdata request for a given block hash - def send_block_request(self, block_hash, node): - msg = msg_getdata() - msg.inv.append(CInv(2, block_hash)) # 2 == "Block" - node.send_message(msg) - - # Send a getheaders request for a given single block hash - def send_header_request(self, block_hash, node): - msg = msg_getheaders() - msg.hashstop = block_hash - node.send_message(msg) - - # Check whether last block received from node has a given hash - def last_block_equals(self, expected_hash, node): - block_msg = node.last_message.get("block") - return block_msg and block_msg.block.rehash() == expected_hash - - # Check whether last block header received from node has a given hash - def last_header_equals(self, expected_hash, node): - headers_msg = node.last_message.get("headers") - return (headers_msg and - headers_msg.headers and - headers_msg.headers[0].rehash() == expected_hash) - - # Checks that stale blocks timestamped more than a month ago are not served - # by the node while recent stale blocks and old active chain blocks are. - # This does not currently test that stale blocks timestamped within the - # last month but that have over a month's worth of work are also withheld. - def run_test(self): - node0 = self.nodes[0].add_p2p_connection(P2PInterface()) - - network_thread_start() - node0.wait_for_verack() - - # Set node time to 60 days ago - self.nodes[0].setmocktime(int(time.time()) - 60 * 24 * 60 * 60) - - # Generating a chain of 10 blocks - block_hashes = self.nodes[0].generate(nblocks=10) - - # Create longer chain starting 2 blocks before current tip - height = len(block_hashes) - 2 - block_hash = block_hashes[height - 1] - block_time = self.nodes[0].getblockheader(block_hash)["mediantime"] + 1 - new_blocks = self.build_chain(5, block_hash, height, block_time) - - # Force reorg to a longer chain - node0.send_message(msg_headers(new_blocks)) - node0.wait_for_getdata() - for block in new_blocks: - node0.send_and_ping(msg_block(block)) - - # Check that reorg succeeded - assert_equal(self.nodes[0].getblockcount(), 13) - - stale_hash = int(block_hashes[-1], 16) - - # Check that getdata request for stale block succeeds - self.send_block_request(stale_hash, node0) - test_function = lambda: self.last_block_equals(stale_hash, node0) - wait_until(test_function, timeout=3) - - # Check that getheader request for stale block header succeeds - self.send_header_request(stale_hash, node0) - test_function = lambda: self.last_header_equals(stale_hash, node0) - wait_until(test_function, timeout=3) - - # Longest chain is extended so stale is much older than chain tip - self.nodes[0].setmocktime(0) - tip = self.nodes[0].generate(nblocks=1)[0] - assert_equal(self.nodes[0].getblockcount(), 14) - - # Send getdata & getheaders to refresh last received getheader message - block_hash = int(tip, 16) - self.send_block_request(block_hash, node0) - self.send_header_request(block_hash, node0) - node0.sync_with_ping() - - # Request for very old stale block should now fail - self.send_block_request(stale_hash, node0) - time.sleep(3) - assert not self.last_block_equals(stale_hash, node0) - - # Request for very old stale block header should now fail - self.send_header_request(stale_hash, node0) - time.sleep(3) - assert not self.last_header_equals(stale_hash, node0) - - # Verify we can fetch very old blocks and headers on the active chain - block_hash = int(block_hashes[2], 16) - self.send_block_request(block_hash, node0) - self.send_header_request(block_hash, node0) - node0.sync_with_ping() - - self.send_block_request(block_hash, node0) - test_function = lambda: self.last_block_equals(block_hash, node0) - wait_until(test_function, timeout=3) - - self.send_header_request(block_hash, node0) - test_function = lambda: self.last_header_equals(block_hash, node0) - wait_until(test_function, timeout=3) - -if __name__ == '__main__': - P2PFingerprintTest().main() diff --git a/test/functional/p2p-leaktests.py b/test/functional/p2p-leaktests.py deleted file mode 100755 index ce4e6e9144..0000000000 --- a/test/functional/p2p-leaktests.py +++ /dev/null @@ -1,153 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2017 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test message sending before handshake completion. - -A node should never send anything other than VERSION/VERACK/REJECT until it's -received a VERACK. - -This test connects to a node and sends it a few messages, trying to intice it -into sending us something it shouldn't. - -Also test that nodes that send unsupported service bits to bitcoind are disconnected -and don't receive a VERACK. Unsupported service bits are currently 1 << 5 and -1 << 7 (until August 1st 2018).""" - -from test_framework.mininode import * -from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import * - -banscore = 10 - -class CLazyNode(P2PInterface): - def __init__(self): - super().__init__() - self.unexpected_msg = False - self.ever_connected = False - - def bad_message(self, message): - self.unexpected_msg = True - self.log.info("should not have received message: %s" % message.command) - - def on_open(self): - self.ever_connected = True - - def on_version(self, message): self.bad_message(message) - def on_verack(self, message): self.bad_message(message) - def on_reject(self, message): self.bad_message(message) - def on_inv(self, message): self.bad_message(message) - def on_addr(self, message): self.bad_message(message) - def on_getdata(self, message): self.bad_message(message) - def on_getblocks(self, message): self.bad_message(message) - def on_tx(self, message): self.bad_message(message) - def on_block(self, message): self.bad_message(message) - def on_getaddr(self, message): self.bad_message(message) - def on_headers(self, message): self.bad_message(message) - def on_getheaders(self, message): self.bad_message(message) - def on_ping(self, message): self.bad_message(message) - def on_mempool(self, message): self.bad_message(message) - def on_pong(self, message): self.bad_message(message) - def on_feefilter(self, message): self.bad_message(message) - def on_sendheaders(self, message): self.bad_message(message) - def on_sendcmpct(self, message): self.bad_message(message) - def on_cmpctblock(self, message): self.bad_message(message) - def on_getblocktxn(self, message): self.bad_message(message) - def on_blocktxn(self, message): self.bad_message(message) - -# Node that never sends a version. We'll use this to send a bunch of messages -# anyway, and eventually get disconnected. -class CNodeNoVersionBan(CLazyNode): - # send a bunch of veracks without sending a message. This should get us disconnected. - # NOTE: implementation-specific check here. Remove if bitcoind ban behavior changes - def on_open(self): - super().on_open() - for i in range(banscore): - self.send_message(msg_verack()) - - def on_reject(self, message): pass - -# Node that never sends a version. This one just sits idle and hopes to receive -# any message (it shouldn't!) -class CNodeNoVersionIdle(CLazyNode): - def __init__(self): - super().__init__() - -# Node that sends a version but not a verack. -class CNodeNoVerackIdle(CLazyNode): - def __init__(self): - self.version_received = False - super().__init__() - - def on_reject(self, message): pass - def on_verack(self, message): pass - # When version is received, don't reply with a verack. Instead, see if the - # node will give us a message that it shouldn't. This is not an exhaustive - # list! - def on_version(self, message): - self.version_received = True - self.send_message(msg_ping()) - self.send_message(msg_getaddr()) - -class P2PLeakTest(BitcoinTestFramework): - def set_test_params(self): - self.num_nodes = 1 - self.extra_args = [['-banscore='+str(banscore)]] - - def run_test(self): - self.nodes[0].setmocktime(1501545600) # August 1st 2017 - - no_version_bannode = self.nodes[0].add_p2p_connection(CNodeNoVersionBan(), send_version=False) - no_version_idlenode = self.nodes[0].add_p2p_connection(CNodeNoVersionIdle(), send_version=False) - no_verack_idlenode = self.nodes[0].add_p2p_connection(CNodeNoVerackIdle()) - unsupported_service_bit5_node = self.nodes[0].add_p2p_connection(CLazyNode(), services=NODE_NETWORK|NODE_UNSUPPORTED_SERVICE_BIT_5) - unsupported_service_bit7_node = self.nodes[0].add_p2p_connection(CLazyNode(), services=NODE_NETWORK|NODE_UNSUPPORTED_SERVICE_BIT_7) - - network_thread_start() - - wait_until(lambda: no_version_bannode.ever_connected, timeout=10, lock=mininode_lock) - wait_until(lambda: no_version_idlenode.ever_connected, timeout=10, lock=mininode_lock) - wait_until(lambda: no_verack_idlenode.version_received, timeout=10, lock=mininode_lock) - wait_until(lambda: unsupported_service_bit5_node.ever_connected, timeout=10, lock=mininode_lock) - wait_until(lambda: unsupported_service_bit7_node.ever_connected, timeout=10, lock=mininode_lock) - - # Mine a block and make sure that it's not sent to the connected nodes - self.nodes[0].generate(1) - - #Give the node enough time to possibly leak out a message - time.sleep(5) - - #This node should have been banned - assert no_version_bannode.state != "connected" - - # These nodes should have been disconnected - assert unsupported_service_bit5_node.state != "connected" - assert unsupported_service_bit7_node.state != "connected" - - self.nodes[0].disconnect_p2ps() - - # Wait until all connections are closed and the network thread has terminated - wait_until(lambda: len(self.nodes[0].getpeerinfo()) == 0) - network_thread_join() - - # Make sure no unexpected messages came in - assert(no_version_bannode.unexpected_msg == False) - assert(no_version_idlenode.unexpected_msg == False) - assert(no_verack_idlenode.unexpected_msg == False) - assert not unsupported_service_bit5_node.unexpected_msg - assert not unsupported_service_bit7_node.unexpected_msg - - self.log.info("Service bits 5 and 7 are allowed after August 1st 2018") - self.nodes[0].setmocktime(1533168000) # August 2nd 2018 - - allowed_service_bit5_node = self.nodes[0].add_p2p_connection(P2PInterface(), services=NODE_NETWORK|NODE_UNSUPPORTED_SERVICE_BIT_5) - allowed_service_bit7_node = self.nodes[0].add_p2p_connection(P2PInterface(), services=NODE_NETWORK|NODE_UNSUPPORTED_SERVICE_BIT_7) - - # Network thread stopped when all previous P2PInterfaces disconnected. Restart it - network_thread_start() - - wait_until(lambda: allowed_service_bit5_node.message_count["verack"], lock=mininode_lock) - wait_until(lambda: allowed_service_bit7_node.message_count["verack"], lock=mininode_lock) - -if __name__ == '__main__': - P2PLeakTest().main() diff --git a/test/functional/p2p-mempool.py b/test/functional/p2p-mempool.py deleted file mode 100755 index 485a8af3d0..0000000000 --- a/test/functional/p2p-mempool.py +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2015-2017 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test p2p mempool message. - -Test that nodes are disconnected if they send mempool messages when bloom -filters are not enabled. -""" - -from test_framework.mininode import * -from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import * - -class P2PMempoolTests(BitcoinTestFramework): - def set_test_params(self): - self.setup_clean_chain = True - self.num_nodes = 1 - self.extra_args = [["-peerbloomfilters=0"]] - - def run_test(self): - # Add a p2p connection - self.nodes[0].add_p2p_connection(P2PInterface()) - network_thread_start() - self.nodes[0].p2p.wait_for_verack() - - #request mempool - self.nodes[0].p2p.send_message(msg_mempool()) - self.nodes[0].p2p.wait_for_disconnect() - - #mininode must be disconnected at this point - assert_equal(len(self.nodes[0].getpeerinfo()), 0) - -if __name__ == '__main__': - P2PMempoolTests().main() diff --git a/test/functional/p2p-segwit.py b/test/functional/p2p-segwit.py deleted file mode 100755 index 20e4805df0..0000000000 --- a/test/functional/p2p-segwit.py +++ /dev/null @@ -1,1952 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2016-2017 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test segwit transactions and blocks on P2P network.""" - -from test_framework.mininode import * -from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import * -from test_framework.script import * -from test_framework.blocktools import create_block, create_coinbase, add_witness_commitment, get_witness_script, WITNESS_COMMITMENT_HEADER -from test_framework.key import CECKey, CPubKey -import time -import random -from binascii import hexlify - -# The versionbit bit used to signal activation of SegWit -VB_WITNESS_BIT = 1 -VB_PERIOD = 144 -VB_TOP_BITS = 0x20000000 - -MAX_SIGOP_COST = 80000 - - -# Calculate the virtual size of a witness block: -# (base + witness/4) -def get_virtual_size(witness_block): - base_size = len(witness_block.serialize(with_witness=False)) - total_size = len(witness_block.serialize(with_witness=True)) - # the "+3" is so we round up - vsize = int((3*base_size + total_size + 3)/4) - return vsize - -def test_transaction_acceptance(rpc, p2p, tx, with_witness, accepted, reason=None): - """Send a transaction to the node and check that it's accepted to the mempool - - - Submit the transaction over the p2p interface - - use the getrawmempool rpc to check for acceptance.""" - tx_message = msg_tx(tx) - if with_witness: - tx_message = msg_witness_tx(tx) - p2p.send_message(tx_message) - p2p.sync_with_ping() - assert_equal(tx.hash in rpc.getrawmempool(), accepted) - if (reason != None and not accepted): - # Check the rejection reason as well. - with mininode_lock: - assert_equal(p2p.last_message["reject"].reason, reason) - -def test_witness_block(rpc, p2p, block, accepted, with_witness=True): - """Send a block to the node and check that it's accepted - - - Submit the block over the p2p interface - - use the getbestblockhash rpc to check for acceptance.""" - if with_witness: - p2p.send_message(msg_witness_block(block)) - else: - p2p.send_message(msg_block(block)) - p2p.sync_with_ping() - assert_equal(rpc.getbestblockhash() == block.hash, accepted) - -class TestNode(P2PInterface): - def __init__(self): - super().__init__() - self.getdataset = set() - - def on_getdata(self, message): - for inv in message.inv: - self.getdataset.add(inv.hash) - - def announce_tx_and_wait_for_getdata(self, tx, timeout=60): - with mininode_lock: - self.last_message.pop("getdata", None) - self.send_message(msg_inv(inv=[CInv(1, tx.sha256)])) - self.wait_for_getdata(timeout) - - def announce_block_and_wait_for_getdata(self, block, use_header, timeout=60): - with mininode_lock: - self.last_message.pop("getdata", None) - self.last_message.pop("getheaders", None) - msg = msg_headers() - msg.headers = [ CBlockHeader(block) ] - if use_header: - self.send_message(msg) - else: - self.send_message(msg_inv(inv=[CInv(2, block.sha256)])) - self.wait_for_getheaders() - self.send_message(msg) - self.wait_for_getdata() - - def request_block(self, blockhash, inv_type, timeout=60): - with mininode_lock: - self.last_message.pop("block", None) - self.send_message(msg_getdata(inv=[CInv(inv_type, blockhash)])) - self.wait_for_block(blockhash, timeout) - return self.last_message["block"].block - -# Used to keep track of anyone-can-spend outputs that we can use in the tests -class UTXO(): - def __init__(self, sha256, n, nValue): - self.sha256 = sha256 - self.n = n - self.nValue = nValue - -# Helper for getting the script associated with a P2PKH -def GetP2PKHScript(pubkeyhash): - return CScript([CScriptOp(OP_DUP), CScriptOp(OP_HASH160), pubkeyhash, CScriptOp(OP_EQUALVERIFY), CScriptOp(OP_CHECKSIG)]) - -# Add signature for a P2PK witness program. -def sign_P2PK_witness_input(script, txTo, inIdx, hashtype, value, key): - tx_hash = SegwitVersion1SignatureHash(script, txTo, inIdx, hashtype, value) - signature = key.sign(tx_hash) + chr(hashtype).encode('latin-1') - txTo.wit.vtxinwit[inIdx].scriptWitness.stack = [signature, script] - txTo.rehash() - - -class SegWitTest(BitcoinTestFramework): - def set_test_params(self): - self.setup_clean_chain = True - self.num_nodes = 3 - # This test tests SegWit both pre and post-activation, so use the normal BIP9 activation. - self.extra_args = [["-whitelist=127.0.0.1", "-vbparams=segwit:0:999999999999"], ["-whitelist=127.0.0.1", "-acceptnonstdtxn=0", "-vbparams=segwit:0:999999999999"], ["-whitelist=127.0.0.1", "-vbparams=segwit:0:0"]] - - def setup_network(self): - self.setup_nodes() - connect_nodes(self.nodes[0], 1) - connect_nodes(self.nodes[0], 2) - self.sync_all() - - ''' Helpers ''' - # Build a block on top of node0's tip. - def build_next_block(self, nVersion=4): - tip = self.nodes[0].getbestblockhash() - height = self.nodes[0].getblockcount() + 1 - block_time = self.nodes[0].getblockheader(tip)["mediantime"] + 1 - block = create_block(int(tip, 16), create_coinbase(height), block_time) - block.nVersion = nVersion - block.rehash() - return block - - # Adds list of transactions to block, adds witness commitment, then solves. - def update_witness_block_with_transactions(self, block, tx_list, nonce=0): - block.vtx.extend(tx_list) - add_witness_commitment(block, nonce) - block.solve() - return - - ''' Individual tests ''' - def test_witness_services(self): - self.log.info("Verifying NODE_WITNESS service bit") - assert((self.test_node.nServices & NODE_WITNESS) != 0) - - - # See if sending a regular transaction works, and create a utxo - # to use in later tests. - def test_non_witness_transaction(self): - # Mine a block with an anyone-can-spend coinbase, - # let it mature, then try to spend it. - self.log.info("Testing non-witness transaction") - block = self.build_next_block(nVersion=1) - block.solve() - self.test_node.send_message(msg_block(block)) - self.test_node.sync_with_ping() # make sure the block was processed - txid = block.vtx[0].sha256 - - self.nodes[0].generate(99) # let the block mature - - # Create a transaction that spends the coinbase - tx = CTransaction() - tx.vin.append(CTxIn(COutPoint(txid, 0), b"")) - tx.vout.append(CTxOut(49*100000000, CScript([OP_TRUE]))) - tx.calc_sha256() - - # Check that serializing it with or without witness is the same - # This is a sanity check of our testing framework. - assert_equal(msg_tx(tx).serialize(), msg_witness_tx(tx).serialize()) - - self.test_node.send_message(msg_witness_tx(tx)) - self.test_node.sync_with_ping() # make sure the tx was processed - assert(tx.hash in self.nodes[0].getrawmempool()) - # Save this transaction for later - self.utxo.append(UTXO(tx.sha256, 0, 49*100000000)) - self.nodes[0].generate(1) - - - # Verify that blocks with witnesses are rejected before activation. - def test_unnecessary_witness_before_segwit_activation(self): - self.log.info("Testing behavior of unnecessary witnesses") - # For now, rely on earlier tests to have created at least one utxo for - # us to use - assert(len(self.utxo) > 0) - assert(get_bip9_status(self.nodes[0], 'segwit')['status'] != 'active') - - tx = CTransaction() - tx.vin.append(CTxIn(COutPoint(self.utxo[0].sha256, self.utxo[0].n), b"")) - tx.vout.append(CTxOut(self.utxo[0].nValue-1000, CScript([OP_TRUE]))) - tx.wit.vtxinwit.append(CTxInWitness()) - tx.wit.vtxinwit[0].scriptWitness.stack = [CScript([CScriptNum(1)])] - - # Verify the hash with witness differs from the txid - # (otherwise our testing framework must be broken!) - tx.rehash() - assert(tx.sha256 != tx.calc_sha256(with_witness=True)) - - # Construct a segwit-signaling block that includes the transaction. - block = self.build_next_block(nVersion=(VB_TOP_BITS|(1 << VB_WITNESS_BIT))) - self.update_witness_block_with_transactions(block, [tx]) - # Sending witness data before activation is not allowed (anti-spam - # rule). - test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=False) - # TODO: fix synchronization so we can test reject reason - # Right now, bitcoind delays sending reject messages for blocks - # until the future, making synchronization here difficult. - #assert_equal(self.test_node.last_message["reject"].reason, "unexpected-witness") - - # But it should not be permanently marked bad... - # Resend without witness information. - self.test_node.send_message(msg_block(block)) - self.test_node.sync_with_ping() - assert_equal(self.nodes[0].getbestblockhash(), block.hash) - - sync_blocks(self.nodes) - - # Create a p2sh output -- this is so we can pass the standardness - # rules (an anyone-can-spend OP_TRUE would be rejected, if not wrapped - # in P2SH). - p2sh_program = CScript([OP_TRUE]) - p2sh_pubkey = hash160(p2sh_program) - scriptPubKey = CScript([OP_HASH160, p2sh_pubkey, OP_EQUAL]) - - # Now check that unnecessary witnesses can't be used to blind a node - # to a transaction, eg by violating standardness checks. - tx2 = CTransaction() - tx2.vin.append(CTxIn(COutPoint(tx.sha256, 0), b"")) - tx2.vout.append(CTxOut(tx.vout[0].nValue-1000, scriptPubKey)) - tx2.rehash() - test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx2, False, True) - self.nodes[0].generate(1) - sync_blocks(self.nodes) - - # We'll add an unnecessary witness to this transaction that would cause - # it to be non-standard, to test that violating policy with a witness before - # segwit activation doesn't blind a node to a transaction. Transactions - # rejected for having a witness before segwit activation shouldn't be added - # to the rejection cache. - tx3 = CTransaction() - tx3.vin.append(CTxIn(COutPoint(tx2.sha256, 0), CScript([p2sh_program]))) - tx3.vout.append(CTxOut(tx2.vout[0].nValue-1000, scriptPubKey)) - tx3.wit.vtxinwit.append(CTxInWitness()) - tx3.wit.vtxinwit[0].scriptWitness.stack = [b'a'*400000] - tx3.rehash() - # Note that this should be rejected for the premature witness reason, - # rather than a policy check, since segwit hasn't activated yet. - test_transaction_acceptance(self.nodes[1].rpc, self.std_node, tx3, True, False, b'no-witness-yet') - - # If we send without witness, it should be accepted. - test_transaction_acceptance(self.nodes[1].rpc, self.std_node, tx3, False, True) - - # Now create a new anyone-can-spend utxo for the next test. - tx4 = CTransaction() - tx4.vin.append(CTxIn(COutPoint(tx3.sha256, 0), CScript([p2sh_program]))) - tx4.vout.append(CTxOut(tx3.vout[0].nValue-1000, CScript([OP_TRUE]))) - tx4.rehash() - test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx3, False, True) - test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx4, False, True) - - self.nodes[0].generate(1) - sync_blocks(self.nodes) - - # Update our utxo list; we spent the first entry. - self.utxo.pop(0) - self.utxo.append(UTXO(tx4.sha256, 0, tx4.vout[0].nValue)) - - - # Mine enough blocks for segwit's vb state to be 'started'. - def advance_to_segwit_started(self): - height = self.nodes[0].getblockcount() - # Will need to rewrite the tests here if we are past the first period - assert(height < VB_PERIOD - 1) - # Genesis block is 'defined'. - assert_equal(get_bip9_status(self.nodes[0], 'segwit')['status'], 'defined') - # Advance to end of period, status should now be 'started' - self.nodes[0].generate(VB_PERIOD-height-1) - assert_equal(get_bip9_status(self.nodes[0], 'segwit')['status'], 'started') - - # Mine enough blocks to lock in segwit, but don't activate. - # TODO: we could verify that lockin only happens at the right threshold of - # signalling blocks, rather than just at the right period boundary. - def advance_to_segwit_lockin(self): - height = self.nodes[0].getblockcount() - assert_equal(get_bip9_status(self.nodes[0], 'segwit')['status'], 'started') - # Advance to end of period, and verify lock-in happens at the end - self.nodes[0].generate(VB_PERIOD-1) - height = self.nodes[0].getblockcount() - assert((height % VB_PERIOD) == VB_PERIOD - 2) - assert_equal(get_bip9_status(self.nodes[0], 'segwit')['status'], 'started') - self.nodes[0].generate(1) - assert_equal(get_bip9_status(self.nodes[0], 'segwit')['status'], 'locked_in') - - - # Mine enough blocks to activate segwit. - # TODO: we could verify that activation only happens at the right threshold - # of signalling blocks, rather than just at the right period boundary. - def advance_to_segwit_active(self): - assert_equal(get_bip9_status(self.nodes[0], 'segwit')['status'], 'locked_in') - height = self.nodes[0].getblockcount() - self.nodes[0].generate(VB_PERIOD - (height%VB_PERIOD) - 2) - assert_equal(get_bip9_status(self.nodes[0], 'segwit')['status'], 'locked_in') - self.nodes[0].generate(1) - assert_equal(get_bip9_status(self.nodes[0], 'segwit')['status'], 'active') - - - # This test can only be run after segwit has activated - def test_witness_commitments(self): - self.log.info("Testing witness commitments") - - # First try a correct witness commitment. - block = self.build_next_block() - add_witness_commitment(block) - block.solve() - - # Test the test -- witness serialization should be different - assert(msg_witness_block(block).serialize() != msg_block(block).serialize()) - - # This empty block should be valid. - test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) - - # Try to tweak the nonce - block_2 = self.build_next_block() - add_witness_commitment(block_2, nonce=28) - block_2.solve() - - # The commitment should have changed! - assert(block_2.vtx[0].vout[-1] != block.vtx[0].vout[-1]) - - # This should also be valid. - test_witness_block(self.nodes[0].rpc, self.test_node, block_2, accepted=True) - - # Now test commitments with actual transactions - assert (len(self.utxo) > 0) - tx = CTransaction() - tx.vin.append(CTxIn(COutPoint(self.utxo[0].sha256, self.utxo[0].n), b"")) - - # Let's construct a witness program - witness_program = CScript([OP_TRUE]) - witness_hash = sha256(witness_program) - scriptPubKey = CScript([OP_0, witness_hash]) - tx.vout.append(CTxOut(self.utxo[0].nValue-1000, scriptPubKey)) - tx.rehash() - - # tx2 will spend tx1, and send back to a regular anyone-can-spend address - tx2 = CTransaction() - tx2.vin.append(CTxIn(COutPoint(tx.sha256, 0), b"")) - tx2.vout.append(CTxOut(tx.vout[0].nValue-1000, witness_program)) - tx2.wit.vtxinwit.append(CTxInWitness()) - tx2.wit.vtxinwit[0].scriptWitness.stack = [witness_program] - tx2.rehash() - - block_3 = self.build_next_block() - self.update_witness_block_with_transactions(block_3, [tx, tx2], nonce=1) - # Add an extra OP_RETURN output that matches the witness commitment template, - # even though it has extra data after the incorrect commitment. - # This block should fail. - block_3.vtx[0].vout.append(CTxOut(0, CScript([OP_RETURN, WITNESS_COMMITMENT_HEADER + ser_uint256(2), 10]))) - block_3.vtx[0].rehash() - block_3.hashMerkleRoot = block_3.calc_merkle_root() - block_3.rehash() - block_3.solve() - - test_witness_block(self.nodes[0].rpc, self.test_node, block_3, accepted=False) - - # Add a different commitment with different nonce, but in the - # right location, and with some funds burned(!). - # This should succeed (nValue shouldn't affect finding the - # witness commitment). - add_witness_commitment(block_3, nonce=0) - block_3.vtx[0].vout[0].nValue -= 1 - block_3.vtx[0].vout[-1].nValue += 1 - block_3.vtx[0].rehash() - block_3.hashMerkleRoot = block_3.calc_merkle_root() - block_3.rehash() - assert(len(block_3.vtx[0].vout) == 4) # 3 OP_returns - block_3.solve() - test_witness_block(self.nodes[0].rpc, self.test_node, block_3, accepted=True) - - # Finally test that a block with no witness transactions can - # omit the commitment. - block_4 = self.build_next_block() - tx3 = CTransaction() - tx3.vin.append(CTxIn(COutPoint(tx2.sha256, 0), b"")) - tx3.vout.append(CTxOut(tx.vout[0].nValue-1000, witness_program)) - tx3.rehash() - block_4.vtx.append(tx3) - block_4.hashMerkleRoot = block_4.calc_merkle_root() - block_4.solve() - test_witness_block(self.nodes[0].rpc, self.test_node, block_4, with_witness=False, accepted=True) - - # Update available utxo's for use in later test. - self.utxo.pop(0) - self.utxo.append(UTXO(tx3.sha256, 0, tx3.vout[0].nValue)) - - - def test_block_malleability(self): - self.log.info("Testing witness block malleability") - - # Make sure that a block that has too big a virtual size - # because of a too-large coinbase witness is not permanently - # marked bad. - block = self.build_next_block() - add_witness_commitment(block) - block.solve() - - block.vtx[0].wit.vtxinwit[0].scriptWitness.stack.append(b'a'*5000000) - assert(get_virtual_size(block) > MAX_BLOCK_BASE_SIZE) - - # We can't send over the p2p network, because this is too big to relay - # TODO: repeat this test with a block that can be relayed - self.nodes[0].submitblock(bytes_to_hex_str(block.serialize(True))) - - assert(self.nodes[0].getbestblockhash() != block.hash) - - block.vtx[0].wit.vtxinwit[0].scriptWitness.stack.pop() - assert(get_virtual_size(block) < MAX_BLOCK_BASE_SIZE) - self.nodes[0].submitblock(bytes_to_hex_str(block.serialize(True))) - - assert(self.nodes[0].getbestblockhash() == block.hash) - - # Now make sure that malleating the witness nonce doesn't - # result in a block permanently marked bad. - block = self.build_next_block() - add_witness_commitment(block) - block.solve() - - # Change the nonce -- should not cause the block to be permanently - # failed - block.vtx[0].wit.vtxinwit[0].scriptWitness.stack = [ ser_uint256(1) ] - test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=False) - - # Changing the witness nonce doesn't change the block hash - block.vtx[0].wit.vtxinwit[0].scriptWitness.stack = [ ser_uint256(0) ] - test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) - - - def test_witness_block_size(self): - self.log.info("Testing witness block size limit") - # TODO: Test that non-witness carrying blocks can't exceed 1MB - # Skipping this test for now; this is covered in p2p-fullblocktest.py - - # Test that witness-bearing blocks are limited at ceil(base + wit/4) <= 1MB. - block = self.build_next_block() - - assert(len(self.utxo) > 0) - - # Create a P2WSH transaction. - # The witness program will be a bunch of OP_2DROP's, followed by OP_TRUE. - # This should give us plenty of room to tweak the spending tx's - # virtual size. - NUM_DROPS = 200 # 201 max ops per script! - NUM_OUTPUTS = 50 - - witness_program = CScript([OP_2DROP]*NUM_DROPS + [OP_TRUE]) - witness_hash = uint256_from_str(sha256(witness_program)) - scriptPubKey = CScript([OP_0, ser_uint256(witness_hash)]) - - prevout = COutPoint(self.utxo[0].sha256, self.utxo[0].n) - value = self.utxo[0].nValue - - parent_tx = CTransaction() - parent_tx.vin.append(CTxIn(prevout, b"")) - child_value = int(value/NUM_OUTPUTS) - for i in range(NUM_OUTPUTS): - parent_tx.vout.append(CTxOut(child_value, scriptPubKey)) - parent_tx.vout[0].nValue -= 50000 - assert(parent_tx.vout[0].nValue > 0) - parent_tx.rehash() - - child_tx = CTransaction() - for i in range(NUM_OUTPUTS): - child_tx.vin.append(CTxIn(COutPoint(parent_tx.sha256, i), b"")) - child_tx.vout = [CTxOut(value - 100000, CScript([OP_TRUE]))] - for i in range(NUM_OUTPUTS): - child_tx.wit.vtxinwit.append(CTxInWitness()) - child_tx.wit.vtxinwit[-1].scriptWitness.stack = [b'a'*195]*(2*NUM_DROPS) + [witness_program] - child_tx.rehash() - self.update_witness_block_with_transactions(block, [parent_tx, child_tx]) - - vsize = get_virtual_size(block) - additional_bytes = (MAX_BLOCK_BASE_SIZE - vsize)*4 - i = 0 - while additional_bytes > 0: - # Add some more bytes to each input until we hit MAX_BLOCK_BASE_SIZE+1 - extra_bytes = min(additional_bytes+1, 55) - block.vtx[-1].wit.vtxinwit[int(i/(2*NUM_DROPS))].scriptWitness.stack[i%(2*NUM_DROPS)] = b'a'*(195+extra_bytes) - additional_bytes -= extra_bytes - i += 1 - - block.vtx[0].vout.pop() # Remove old commitment - add_witness_commitment(block) - block.solve() - vsize = get_virtual_size(block) - assert_equal(vsize, MAX_BLOCK_BASE_SIZE + 1) - # Make sure that our test case would exceed the old max-network-message - # limit - assert(len(block.serialize(True)) > 2*1024*1024) - - test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=False) - - # Now resize the second transaction to make the block fit. - cur_length = len(block.vtx[-1].wit.vtxinwit[0].scriptWitness.stack[0]) - block.vtx[-1].wit.vtxinwit[0].scriptWitness.stack[0] = b'a'*(cur_length-1) - block.vtx[0].vout.pop() - add_witness_commitment(block) - block.solve() - assert(get_virtual_size(block) == MAX_BLOCK_BASE_SIZE) - - test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) - - # Update available utxo's - self.utxo.pop(0) - self.utxo.append(UTXO(block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue)) - - - # submitblock will try to add the nonce automatically, so that mining - # software doesn't need to worry about doing so itself. - def test_submit_block(self): - block = self.build_next_block() - - # Try using a custom nonce and then don't supply it. - # This shouldn't possibly work. - add_witness_commitment(block, nonce=1) - block.vtx[0].wit = CTxWitness() # drop the nonce - block.solve() - self.nodes[0].submitblock(bytes_to_hex_str(block.serialize(True))) - assert(self.nodes[0].getbestblockhash() != block.hash) - - # Now redo commitment with the standard nonce, but let bitcoind fill it in. - add_witness_commitment(block, nonce=0) - block.vtx[0].wit = CTxWitness() - block.solve() - self.nodes[0].submitblock(bytes_to_hex_str(block.serialize(True))) - assert_equal(self.nodes[0].getbestblockhash(), block.hash) - - # This time, add a tx with non-empty witness, but don't supply - # the commitment. - block_2 = self.build_next_block() - - add_witness_commitment(block_2) - - block_2.solve() - - # Drop commitment and nonce -- submitblock should not fill in. - block_2.vtx[0].vout.pop() - block_2.vtx[0].wit = CTxWitness() - - self.nodes[0].submitblock(bytes_to_hex_str(block_2.serialize(True))) - # Tip should not advance! - assert(self.nodes[0].getbestblockhash() != block_2.hash) - - - # Consensus tests of extra witness data in a transaction. - def test_extra_witness_data(self): - self.log.info("Testing extra witness data in tx") - - assert(len(self.utxo) > 0) - - block = self.build_next_block() - - witness_program = CScript([OP_DROP, OP_TRUE]) - witness_hash = sha256(witness_program) - scriptPubKey = CScript([OP_0, witness_hash]) - - # First try extra witness data on a tx that doesn't require a witness - tx = CTransaction() - tx.vin.append(CTxIn(COutPoint(self.utxo[0].sha256, self.utxo[0].n), b"")) - tx.vout.append(CTxOut(self.utxo[0].nValue-2000, scriptPubKey)) - tx.vout.append(CTxOut(1000, CScript([OP_TRUE]))) # non-witness output - tx.wit.vtxinwit.append(CTxInWitness()) - tx.wit.vtxinwit[0].scriptWitness.stack = [CScript([])] - tx.rehash() - self.update_witness_block_with_transactions(block, [tx]) - - # Extra witness data should not be allowed. - test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=False) - - # Try extra signature data. Ok if we're not spending a witness output. - block.vtx[1].wit.vtxinwit = [] - block.vtx[1].vin[0].scriptSig = CScript([OP_0]) - block.vtx[1].rehash() - add_witness_commitment(block) - block.solve() - - test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) - - # Now try extra witness/signature data on an input that DOES require a - # witness - tx2 = CTransaction() - tx2.vin.append(CTxIn(COutPoint(tx.sha256, 0), b"")) # witness output - tx2.vin.append(CTxIn(COutPoint(tx.sha256, 1), b"")) # non-witness - tx2.vout.append(CTxOut(tx.vout[0].nValue, CScript([OP_TRUE]))) - tx2.wit.vtxinwit.extend([CTxInWitness(), CTxInWitness()]) - tx2.wit.vtxinwit[0].scriptWitness.stack = [ CScript([CScriptNum(1)]), CScript([CScriptNum(1)]), witness_program ] - tx2.wit.vtxinwit[1].scriptWitness.stack = [ CScript([OP_TRUE]) ] - - block = self.build_next_block() - self.update_witness_block_with_transactions(block, [tx2]) - - # This has extra witness data, so it should fail. - test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=False) - - # Now get rid of the extra witness, but add extra scriptSig data - tx2.vin[0].scriptSig = CScript([OP_TRUE]) - tx2.vin[1].scriptSig = CScript([OP_TRUE]) - tx2.wit.vtxinwit[0].scriptWitness.stack.pop(0) - tx2.wit.vtxinwit[1].scriptWitness.stack = [] - tx2.rehash() - add_witness_commitment(block) - block.solve() - - # This has extra signature data for a witness input, so it should fail. - test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=False) - - # Now get rid of the extra scriptsig on the witness input, and verify - # success (even with extra scriptsig data in the non-witness input) - tx2.vin[0].scriptSig = b"" - tx2.rehash() - add_witness_commitment(block) - block.solve() - - test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) - - # Update utxo for later tests - self.utxo.pop(0) - self.utxo.append(UTXO(tx2.sha256, 0, tx2.vout[0].nValue)) - - - def test_max_witness_push_length(self): - ''' Should only allow up to 520 byte pushes in witness stack ''' - self.log.info("Testing maximum witness push size") - MAX_SCRIPT_ELEMENT_SIZE = 520 - assert(len(self.utxo)) - - block = self.build_next_block() - - witness_program = CScript([OP_DROP, OP_TRUE]) - witness_hash = sha256(witness_program) - scriptPubKey = CScript([OP_0, witness_hash]) - - tx = CTransaction() - tx.vin.append(CTxIn(COutPoint(self.utxo[0].sha256, self.utxo[0].n), b"")) - tx.vout.append(CTxOut(self.utxo[0].nValue-1000, scriptPubKey)) - tx.rehash() - - tx2 = CTransaction() - tx2.vin.append(CTxIn(COutPoint(tx.sha256, 0), b"")) - tx2.vout.append(CTxOut(tx.vout[0].nValue-1000, CScript([OP_TRUE]))) - tx2.wit.vtxinwit.append(CTxInWitness()) - # First try a 521-byte stack element - tx2.wit.vtxinwit[0].scriptWitness.stack = [ b'a'*(MAX_SCRIPT_ELEMENT_SIZE+1), witness_program ] - tx2.rehash() - - self.update_witness_block_with_transactions(block, [tx, tx2]) - test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=False) - - # Now reduce the length of the stack element - tx2.wit.vtxinwit[0].scriptWitness.stack[0] = b'a'*(MAX_SCRIPT_ELEMENT_SIZE) - - add_witness_commitment(block) - block.solve() - test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) - - # Update the utxo for later tests - self.utxo.pop() - self.utxo.append(UTXO(tx2.sha256, 0, tx2.vout[0].nValue)) - - def test_max_witness_program_length(self): - # Can create witness outputs that are long, but can't be greater than - # 10k bytes to successfully spend - self.log.info("Testing maximum witness program length") - assert(len(self.utxo)) - MAX_PROGRAM_LENGTH = 10000 - - # This program is 19 max pushes (9937 bytes), then 64 more opcode-bytes. - long_witness_program = CScript([b'a'*520]*19 + [OP_DROP]*63 + [OP_TRUE]) - assert(len(long_witness_program) == MAX_PROGRAM_LENGTH+1) - long_witness_hash = sha256(long_witness_program) - long_scriptPubKey = CScript([OP_0, long_witness_hash]) - - block = self.build_next_block() - - tx = CTransaction() - tx.vin.append(CTxIn(COutPoint(self.utxo[0].sha256, self.utxo[0].n), b"")) - tx.vout.append(CTxOut(self.utxo[0].nValue-1000, long_scriptPubKey)) - tx.rehash() - - tx2 = CTransaction() - tx2.vin.append(CTxIn(COutPoint(tx.sha256, 0), b"")) - tx2.vout.append(CTxOut(tx.vout[0].nValue-1000, CScript([OP_TRUE]))) - tx2.wit.vtxinwit.append(CTxInWitness()) - tx2.wit.vtxinwit[0].scriptWitness.stack = [b'a']*44 + [long_witness_program] - tx2.rehash() - - self.update_witness_block_with_transactions(block, [tx, tx2]) - - test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=False) - - # Try again with one less byte in the witness program - witness_program = CScript([b'a'*520]*19 + [OP_DROP]*62 + [OP_TRUE]) - assert(len(witness_program) == MAX_PROGRAM_LENGTH) - witness_hash = sha256(witness_program) - scriptPubKey = CScript([OP_0, witness_hash]) - - tx.vout[0] = CTxOut(tx.vout[0].nValue, scriptPubKey) - tx.rehash() - tx2.vin[0].prevout.hash = tx.sha256 - tx2.wit.vtxinwit[0].scriptWitness.stack = [b'a']*43 + [witness_program] - tx2.rehash() - block.vtx = [block.vtx[0]] - self.update_witness_block_with_transactions(block, [tx, tx2]) - test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) - - self.utxo.pop() - self.utxo.append(UTXO(tx2.sha256, 0, tx2.vout[0].nValue)) - - - def test_witness_input_length(self): - ''' Ensure that vin length must match vtxinwit length ''' - self.log.info("Testing witness input length") - assert(len(self.utxo)) - - witness_program = CScript([OP_DROP, OP_TRUE]) - witness_hash = sha256(witness_program) - scriptPubKey = CScript([OP_0, witness_hash]) - - # Create a transaction that splits our utxo into many outputs - tx = CTransaction() - tx.vin.append(CTxIn(COutPoint(self.utxo[0].sha256, self.utxo[0].n), b"")) - nValue = self.utxo[0].nValue - for i in range(10): - tx.vout.append(CTxOut(int(nValue/10), scriptPubKey)) - tx.vout[0].nValue -= 1000 - assert(tx.vout[0].nValue >= 0) - - block = self.build_next_block() - self.update_witness_block_with_transactions(block, [tx]) - test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) - - # Try various ways to spend tx that should all break. - # This "broken" transaction serializer will not normalize - # the length of vtxinwit. - class BrokenCTransaction(CTransaction): - def serialize_with_witness(self): - flags = 0 - if not self.wit.is_null(): - flags |= 1 - r = b"" - r += struct.pack(" version 1 transactions - # are non-standard - scriptPubKey = CScript([CScriptOp(OP_1), witness_hash]) - tx2 = CTransaction() - tx2.vin = [CTxIn(COutPoint(tx.sha256, 0), b"")] - tx2.vout = [CTxOut(tx.vout[0].nValue-1000, scriptPubKey)] - tx2.wit.vtxinwit.append(CTxInWitness()) - tx2.wit.vtxinwit[0].scriptWitness.stack = [ witness_program ] - tx2.rehash() - # Gets accepted to test_node, because standardness of outputs isn't - # checked with fRequireStandard - test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx2, with_witness=True, accepted=True) - test_transaction_acceptance(self.nodes[1].rpc, self.std_node, tx2, with_witness=True, accepted=False) - temp_utxo.pop() # last entry in temp_utxo was the output we just spent - temp_utxo.append(UTXO(tx2.sha256, 0, tx2.vout[0].nValue)) - - # Spend everything in temp_utxo back to an OP_TRUE output. - tx3 = CTransaction() - total_value = 0 - for i in temp_utxo: - tx3.vin.append(CTxIn(COutPoint(i.sha256, i.n), b"")) - tx3.wit.vtxinwit.append(CTxInWitness()) - total_value += i.nValue - tx3.wit.vtxinwit[-1].scriptWitness.stack = [witness_program] - tx3.vout.append(CTxOut(total_value - 1000, CScript([OP_TRUE]))) - tx3.rehash() - # Spending a higher version witness output is not allowed by policy, - # even with fRequireStandard=false. - test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx3, with_witness=True, accepted=False) - self.test_node.sync_with_ping() - with mininode_lock: - assert(b"reserved for soft-fork upgrades" in self.test_node.last_message["reject"].reason) - - # Building a block with the transaction must be valid, however. - block = self.build_next_block() - self.update_witness_block_with_transactions(block, [tx2, tx3]) - test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) - sync_blocks(self.nodes) - - # Add utxo to our list - self.utxo.append(UTXO(tx3.sha256, 0, tx3.vout[0].nValue)) - - - def test_premature_coinbase_witness_spend(self): - self.log.info("Testing premature coinbase witness spend") - block = self.build_next_block() - # Change the output of the block to be a witness output. - witness_program = CScript([OP_TRUE]) - witness_hash = sha256(witness_program) - scriptPubKey = CScript([OP_0, witness_hash]) - block.vtx[0].vout[0].scriptPubKey = scriptPubKey - # This next line will rehash the coinbase and update the merkle - # root, and solve. - self.update_witness_block_with_transactions(block, []) - test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) - - spend_tx = CTransaction() - spend_tx.vin = [CTxIn(COutPoint(block.vtx[0].sha256, 0), b"")] - spend_tx.vout = [CTxOut(block.vtx[0].vout[0].nValue, witness_program)] - spend_tx.wit.vtxinwit.append(CTxInWitness()) - spend_tx.wit.vtxinwit[0].scriptWitness.stack = [ witness_program ] - spend_tx.rehash() - - # Now test a premature spend. - self.nodes[0].generate(98) - sync_blocks(self.nodes) - block2 = self.build_next_block() - self.update_witness_block_with_transactions(block2, [spend_tx]) - test_witness_block(self.nodes[0].rpc, self.test_node, block2, accepted=False) - - # Advancing one more block should allow the spend. - self.nodes[0].generate(1) - block2 = self.build_next_block() - self.update_witness_block_with_transactions(block2, [spend_tx]) - test_witness_block(self.nodes[0].rpc, self.test_node, block2, accepted=True) - sync_blocks(self.nodes) - - - def test_signature_version_1(self): - self.log.info("Testing segwit signature hash version 1") - key = CECKey() - key.set_secretbytes(b"9") - pubkey = CPubKey(key.get_pubkey()) - - witness_program = CScript([pubkey, CScriptOp(OP_CHECKSIG)]) - witness_hash = sha256(witness_program) - scriptPubKey = CScript([OP_0, witness_hash]) - - # First create a witness output for use in the tests. - assert(len(self.utxo)) - tx = CTransaction() - tx.vin.append(CTxIn(COutPoint(self.utxo[0].sha256, self.utxo[0].n), b"")) - tx.vout.append(CTxOut(self.utxo[0].nValue-1000, scriptPubKey)) - tx.rehash() - - test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx, with_witness=True, accepted=True) - # Mine this transaction in preparation for following tests. - block = self.build_next_block() - self.update_witness_block_with_transactions(block, [tx]) - test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) - sync_blocks(self.nodes) - self.utxo.pop(0) - - # Test each hashtype - prev_utxo = UTXO(tx.sha256, 0, tx.vout[0].nValue) - for sigflag in [ 0, SIGHASH_ANYONECANPAY ]: - for hashtype in [SIGHASH_ALL, SIGHASH_NONE, SIGHASH_SINGLE]: - hashtype |= sigflag - block = self.build_next_block() - tx = CTransaction() - tx.vin.append(CTxIn(COutPoint(prev_utxo.sha256, prev_utxo.n), b"")) - tx.vout.append(CTxOut(prev_utxo.nValue - 1000, scriptPubKey)) - tx.wit.vtxinwit.append(CTxInWitness()) - # Too-large input value - sign_P2PK_witness_input(witness_program, tx, 0, hashtype, prev_utxo.nValue+1, key) - self.update_witness_block_with_transactions(block, [tx]) - test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=False) - - # Too-small input value - sign_P2PK_witness_input(witness_program, tx, 0, hashtype, prev_utxo.nValue-1, key) - block.vtx.pop() # remove last tx - self.update_witness_block_with_transactions(block, [tx]) - test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=False) - - # Now try correct value - sign_P2PK_witness_input(witness_program, tx, 0, hashtype, prev_utxo.nValue, key) - block.vtx.pop() - self.update_witness_block_with_transactions(block, [tx]) - test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) - - prev_utxo = UTXO(tx.sha256, 0, tx.vout[0].nValue) - - # Test combinations of signature hashes. - # Split the utxo into a lot of outputs. - # Randomly choose up to 10 to spend, sign with different hashtypes, and - # output to a random number of outputs. Repeat NUM_TESTS times. - # Ensure that we've tested a situation where we use SIGHASH_SINGLE with - # an input index > number of outputs. - NUM_TESTS = 500 - temp_utxos = [] - tx = CTransaction() - tx.vin.append(CTxIn(COutPoint(prev_utxo.sha256, prev_utxo.n), b"")) - split_value = prev_utxo.nValue // NUM_TESTS - for i in range(NUM_TESTS): - tx.vout.append(CTxOut(split_value, scriptPubKey)) - tx.wit.vtxinwit.append(CTxInWitness()) - sign_P2PK_witness_input(witness_program, tx, 0, SIGHASH_ALL, prev_utxo.nValue, key) - for i in range(NUM_TESTS): - temp_utxos.append(UTXO(tx.sha256, i, split_value)) - - block = self.build_next_block() - self.update_witness_block_with_transactions(block, [tx]) - test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) - - block = self.build_next_block() - used_sighash_single_out_of_bounds = False - for i in range(NUM_TESTS): - # Ping regularly to keep the connection alive - if (not i % 100): - self.test_node.sync_with_ping() - # Choose random number of inputs to use. - num_inputs = random.randint(1, 10) - # Create a slight bias for producing more utxos - num_outputs = random.randint(1, 11) - random.shuffle(temp_utxos) - assert(len(temp_utxos) > num_inputs) - tx = CTransaction() - total_value = 0 - for i in range(num_inputs): - tx.vin.append(CTxIn(COutPoint(temp_utxos[i].sha256, temp_utxos[i].n), b"")) - tx.wit.vtxinwit.append(CTxInWitness()) - total_value += temp_utxos[i].nValue - split_value = total_value // num_outputs - for i in range(num_outputs): - tx.vout.append(CTxOut(split_value, scriptPubKey)) - for i in range(num_inputs): - # Now try to sign each input, using a random hashtype. - anyonecanpay = 0 - if random.randint(0, 1): - anyonecanpay = SIGHASH_ANYONECANPAY - hashtype = random.randint(1, 3) | anyonecanpay - sign_P2PK_witness_input(witness_program, tx, i, hashtype, temp_utxos[i].nValue, key) - if (hashtype == SIGHASH_SINGLE and i >= num_outputs): - used_sighash_single_out_of_bounds = True - tx.rehash() - for i in range(num_outputs): - temp_utxos.append(UTXO(tx.sha256, i, split_value)) - temp_utxos = temp_utxos[num_inputs:] - - block.vtx.append(tx) - - # Test the block periodically, if we're close to maxblocksize - if (get_virtual_size(block) > MAX_BLOCK_BASE_SIZE - 1000): - self.update_witness_block_with_transactions(block, []) - test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) - block = self.build_next_block() - - if (not used_sighash_single_out_of_bounds): - self.log.info("WARNING: this test run didn't attempt SIGHASH_SINGLE with out-of-bounds index value") - # Test the transactions we've added to the block - if (len(block.vtx) > 1): - self.update_witness_block_with_transactions(block, []) - test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) - - # Now test witness version 0 P2PKH transactions - pubkeyhash = hash160(pubkey) - scriptPKH = CScript([OP_0, pubkeyhash]) - tx = CTransaction() - tx.vin.append(CTxIn(COutPoint(temp_utxos[0].sha256, temp_utxos[0].n), b"")) - tx.vout.append(CTxOut(temp_utxos[0].nValue, scriptPKH)) - tx.wit.vtxinwit.append(CTxInWitness()) - sign_P2PK_witness_input(witness_program, tx, 0, SIGHASH_ALL, temp_utxos[0].nValue, key) - tx2 = CTransaction() - tx2.vin.append(CTxIn(COutPoint(tx.sha256, 0), b"")) - tx2.vout.append(CTxOut(tx.vout[0].nValue, CScript([OP_TRUE]))) - - script = GetP2PKHScript(pubkeyhash) - sig_hash = SegwitVersion1SignatureHash(script, tx2, 0, SIGHASH_ALL, tx.vout[0].nValue) - signature = key.sign(sig_hash) + b'\x01' # 0x1 is SIGHASH_ALL - - # Check that we can't have a scriptSig - tx2.vin[0].scriptSig = CScript([signature, pubkey]) - block = self.build_next_block() - self.update_witness_block_with_transactions(block, [tx, tx2]) - test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=False) - - # Move the signature to the witness. - block.vtx.pop() - tx2.wit.vtxinwit.append(CTxInWitness()) - tx2.wit.vtxinwit[0].scriptWitness.stack = [signature, pubkey] - tx2.vin[0].scriptSig = b"" - tx2.rehash() - - self.update_witness_block_with_transactions(block, [tx2]) - test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) - - temp_utxos.pop(0) - - # Update self.utxos for later tests by creating two outputs - # that consolidate all the coins in temp_utxos. - output_value = sum(i.nValue for i in temp_utxos) // 2 - - tx = CTransaction() - index = 0 - # Just spend to our usual anyone-can-spend output - tx.vout = [CTxOut(output_value, CScript([OP_TRUE]))] * 2 - for i in temp_utxos: - # Use SIGHASH_ALL|SIGHASH_ANYONECANPAY so we can build up - # the signatures as we go. - tx.vin.append(CTxIn(COutPoint(i.sha256, i.n), b"")) - tx.wit.vtxinwit.append(CTxInWitness()) - sign_P2PK_witness_input(witness_program, tx, index, SIGHASH_ALL|SIGHASH_ANYONECANPAY, i.nValue, key) - index += 1 - block = self.build_next_block() - self.update_witness_block_with_transactions(block, [tx]) - test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) - - for i in range(len(tx.vout)): - self.utxo.append(UTXO(tx.sha256, i, tx.vout[i].nValue)) - - - # Test P2SH wrapped witness programs. - def test_p2sh_witness(self, segwit_activated): - self.log.info("Testing P2SH witness transactions") - - assert(len(self.utxo)) - - # Prepare the p2sh-wrapped witness output - witness_program = CScript([OP_DROP, OP_TRUE]) - witness_hash = sha256(witness_program) - p2wsh_pubkey = CScript([OP_0, witness_hash]) - p2sh_witness_hash = hash160(p2wsh_pubkey) - scriptPubKey = CScript([OP_HASH160, p2sh_witness_hash, OP_EQUAL]) - scriptSig = CScript([p2wsh_pubkey]) # a push of the redeem script - - # Fund the P2SH output - tx = CTransaction() - tx.vin.append(CTxIn(COutPoint(self.utxo[0].sha256, self.utxo[0].n), b"")) - tx.vout.append(CTxOut(self.utxo[0].nValue-1000, scriptPubKey)) - tx.rehash() - - # Verify mempool acceptance and block validity - test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx, with_witness=False, accepted=True) - block = self.build_next_block() - self.update_witness_block_with_transactions(block, [tx]) - test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True, with_witness=segwit_activated) - sync_blocks(self.nodes) - - # Now test attempts to spend the output. - spend_tx = CTransaction() - spend_tx.vin.append(CTxIn(COutPoint(tx.sha256, 0), scriptSig)) - spend_tx.vout.append(CTxOut(tx.vout[0].nValue-1000, CScript([OP_TRUE]))) - spend_tx.rehash() - - # This transaction should not be accepted into the mempool pre- or - # post-segwit. Mempool acceptance will use SCRIPT_VERIFY_WITNESS which - # will require a witness to spend a witness program regardless of - # segwit activation. Note that older bitcoind's that are not - # segwit-aware would also reject this for failing CLEANSTACK. - test_transaction_acceptance(self.nodes[0].rpc, self.test_node, spend_tx, with_witness=False, accepted=False) - - # Try to put the witness script in the scriptSig, should also fail. - spend_tx.vin[0].scriptSig = CScript([p2wsh_pubkey, b'a']) - spend_tx.rehash() - test_transaction_acceptance(self.nodes[0].rpc, self.test_node, spend_tx, with_witness=False, accepted=False) - - # Now put the witness script in the witness, should succeed after - # segwit activates. - spend_tx.vin[0].scriptSig = scriptSig - spend_tx.rehash() - spend_tx.wit.vtxinwit.append(CTxInWitness()) - spend_tx.wit.vtxinwit[0].scriptWitness.stack = [ b'a', witness_program ] - - # Verify mempool acceptance - test_transaction_acceptance(self.nodes[0].rpc, self.test_node, spend_tx, with_witness=True, accepted=segwit_activated) - block = self.build_next_block() - self.update_witness_block_with_transactions(block, [spend_tx]) - - # If we're before activation, then sending this without witnesses - # should be valid. If we're after activation, then sending this with - # witnesses should be valid. - if segwit_activated: - test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) - else: - test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True, with_witness=False) - - # Update self.utxo - self.utxo.pop(0) - self.utxo.append(UTXO(spend_tx.sha256, 0, spend_tx.vout[0].nValue)) - - # Test the behavior of starting up a segwit-aware node after the softfork - # has activated. As segwit requires different block data than pre-segwit - # nodes would have stored, this requires special handling. - # To enable this test, pass --oldbinary= to - # the test. - def test_upgrade_after_activation(self, node_id): - self.log.info("Testing software upgrade after softfork activation") - - assert(node_id != 0) # node0 is assumed to be a segwit-active bitcoind - - # Make sure the nodes are all up - sync_blocks(self.nodes) - - # Restart with the new binary - self.stop_node(node_id) - self.start_node(node_id, extra_args=["-vbparams=segwit:0:999999999999"]) - connect_nodes(self.nodes[0], node_id) - - sync_blocks(self.nodes) - - # Make sure that this peer thinks segwit has activated. - assert(get_bip9_status(self.nodes[node_id], 'segwit')['status'] == "active") - - # Make sure this peers blocks match those of node0. - height = self.nodes[node_id].getblockcount() - while height >= 0: - block_hash = self.nodes[node_id].getblockhash(height) - assert_equal(block_hash, self.nodes[0].getblockhash(height)) - assert_equal(self.nodes[0].getblock(block_hash), self.nodes[node_id].getblock(block_hash)) - height -= 1 - - - def test_witness_sigops(self): - '''Ensure sigop counting is correct inside witnesses.''' - self.log.info("Testing sigops limit") - - assert(len(self.utxo)) - - # Keep this under MAX_OPS_PER_SCRIPT (201) - witness_program = CScript([OP_TRUE, OP_IF, OP_TRUE, OP_ELSE] + [OP_CHECKMULTISIG]*5 + [OP_CHECKSIG]*193 + [OP_ENDIF]) - witness_hash = sha256(witness_program) - scriptPubKey = CScript([OP_0, witness_hash]) - - sigops_per_script = 20*5 + 193*1 - # We'll produce 2 extra outputs, one with a program that would take us - # over max sig ops, and one with a program that would exactly reach max - # sig ops - outputs = (MAX_SIGOP_COST // sigops_per_script) + 2 - extra_sigops_available = MAX_SIGOP_COST % sigops_per_script - - # We chose the number of checkmultisigs/checksigs to make this work: - assert(extra_sigops_available < 100) # steer clear of MAX_OPS_PER_SCRIPT - - # This script, when spent with the first - # N(=MAX_SIGOP_COST//sigops_per_script) outputs of our transaction, - # would push us just over the block sigop limit. - witness_program_toomany = CScript([OP_TRUE, OP_IF, OP_TRUE, OP_ELSE] + [OP_CHECKSIG]*(extra_sigops_available + 1) + [OP_ENDIF]) - witness_hash_toomany = sha256(witness_program_toomany) - scriptPubKey_toomany = CScript([OP_0, witness_hash_toomany]) - - # If we spend this script instead, we would exactly reach our sigop - # limit (for witness sigops). - witness_program_justright = CScript([OP_TRUE, OP_IF, OP_TRUE, OP_ELSE] + [OP_CHECKSIG]*(extra_sigops_available) + [OP_ENDIF]) - witness_hash_justright = sha256(witness_program_justright) - scriptPubKey_justright = CScript([OP_0, witness_hash_justright]) - - # First split our available utxo into a bunch of outputs - split_value = self.utxo[0].nValue // outputs - tx = CTransaction() - tx.vin.append(CTxIn(COutPoint(self.utxo[0].sha256, self.utxo[0].n), b"")) - for i in range(outputs): - tx.vout.append(CTxOut(split_value, scriptPubKey)) - tx.vout[-2].scriptPubKey = scriptPubKey_toomany - tx.vout[-1].scriptPubKey = scriptPubKey_justright - tx.rehash() - - block_1 = self.build_next_block() - self.update_witness_block_with_transactions(block_1, [tx]) - test_witness_block(self.nodes[0].rpc, self.test_node, block_1, accepted=True) - - tx2 = CTransaction() - # If we try to spend the first n-1 outputs from tx, that should be - # too many sigops. - total_value = 0 - for i in range(outputs-1): - tx2.vin.append(CTxIn(COutPoint(tx.sha256, i), b"")) - tx2.wit.vtxinwit.append(CTxInWitness()) - tx2.wit.vtxinwit[-1].scriptWitness.stack = [ witness_program ] - total_value += tx.vout[i].nValue - tx2.wit.vtxinwit[-1].scriptWitness.stack = [ witness_program_toomany ] - tx2.vout.append(CTxOut(total_value, CScript([OP_TRUE]))) - tx2.rehash() - - block_2 = self.build_next_block() - self.update_witness_block_with_transactions(block_2, [tx2]) - test_witness_block(self.nodes[0].rpc, self.test_node, block_2, accepted=False) - - # Try dropping the last input in tx2, and add an output that has - # too many sigops (contributing to legacy sigop count). - checksig_count = (extra_sigops_available // 4) + 1 - scriptPubKey_checksigs = CScript([OP_CHECKSIG]*checksig_count) - tx2.vout.append(CTxOut(0, scriptPubKey_checksigs)) - tx2.vin.pop() - tx2.wit.vtxinwit.pop() - tx2.vout[0].nValue -= tx.vout[-2].nValue - tx2.rehash() - block_3 = self.build_next_block() - self.update_witness_block_with_transactions(block_3, [tx2]) - test_witness_block(self.nodes[0].rpc, self.test_node, block_3, accepted=False) - - # If we drop the last checksig in this output, the tx should succeed. - block_4 = self.build_next_block() - tx2.vout[-1].scriptPubKey = CScript([OP_CHECKSIG]*(checksig_count-1)) - tx2.rehash() - self.update_witness_block_with_transactions(block_4, [tx2]) - test_witness_block(self.nodes[0].rpc, self.test_node, block_4, accepted=True) - - # Reset the tip back down for the next test - sync_blocks(self.nodes) - for x in self.nodes: - x.invalidateblock(block_4.hash) - - # Try replacing the last input of tx2 to be spending the last - # output of tx - block_5 = self.build_next_block() - tx2.vout.pop() - tx2.vin.append(CTxIn(COutPoint(tx.sha256, outputs-1), b"")) - tx2.wit.vtxinwit.append(CTxInWitness()) - tx2.wit.vtxinwit[-1].scriptWitness.stack = [ witness_program_justright ] - tx2.rehash() - self.update_witness_block_with_transactions(block_5, [tx2]) - test_witness_block(self.nodes[0].rpc, self.test_node, block_5, accepted=True) - - # TODO: test p2sh sigop counting - - def test_getblocktemplate_before_lockin(self): - self.log.info("Testing getblocktemplate setting of segwit versionbit (before lockin)") - # Node0 is segwit aware, node2 is not. - for node in [self.nodes[0], self.nodes[2]]: - gbt_results = node.getblocktemplate() - block_version = gbt_results['version'] - # If we're not indicating segwit support, we will still be - # signalling for segwit activation. - assert_equal((block_version & (1 << VB_WITNESS_BIT) != 0), node == self.nodes[0]) - # If we don't specify the segwit rule, then we won't get a default - # commitment. - assert('default_witness_commitment' not in gbt_results) - - # Workaround: - # Can either change the tip, or change the mempool and wait 5 seconds - # to trigger a recomputation of getblocktemplate. - txid = int(self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 1), 16) - # Using mocktime lets us avoid sleep() - sync_mempools(self.nodes) - self.nodes[0].setmocktime(int(time.time())+10) - self.nodes[2].setmocktime(int(time.time())+10) - - for node in [self.nodes[0], self.nodes[2]]: - gbt_results = node.getblocktemplate({"rules" : ["segwit"]}) - block_version = gbt_results['version'] - if node == self.nodes[2]: - # If this is a non-segwit node, we should still not get a witness - # commitment, nor a version bit signalling segwit. - assert_equal(block_version & (1 << VB_WITNESS_BIT), 0) - assert('default_witness_commitment' not in gbt_results) - else: - # For segwit-aware nodes, check the version bit and the witness - # commitment are correct. - assert(block_version & (1 << VB_WITNESS_BIT) != 0) - assert('default_witness_commitment' in gbt_results) - witness_commitment = gbt_results['default_witness_commitment'] - - # Check that default_witness_commitment is present. - witness_root = CBlock.get_merkle_root([ser_uint256(0), - ser_uint256(txid)]) - script = get_witness_script(witness_root, 0) - assert_equal(witness_commitment, bytes_to_hex_str(script)) - - # undo mocktime - self.nodes[0].setmocktime(0) - self.nodes[2].setmocktime(0) - - # Uncompressed pubkeys are no longer supported in default relay policy, - # but (for now) are still valid in blocks. - def test_uncompressed_pubkey(self): - self.log.info("Testing uncompressed pubkeys") - # Segwit transactions using uncompressed pubkeys are not accepted - # under default policy, but should still pass consensus. - key = CECKey() - key.set_secretbytes(b"9") - key.set_compressed(False) - pubkey = CPubKey(key.get_pubkey()) - assert_equal(len(pubkey), 65) # This should be an uncompressed pubkey - - assert(len(self.utxo) > 0) - utxo = self.utxo.pop(0) - - # Test 1: P2WPKH - # First create a P2WPKH output that uses an uncompressed pubkey - pubkeyhash = hash160(pubkey) - scriptPKH = CScript([OP_0, pubkeyhash]) - tx = CTransaction() - tx.vin.append(CTxIn(COutPoint(utxo.sha256, utxo.n), b"")) - tx.vout.append(CTxOut(utxo.nValue-1000, scriptPKH)) - tx.rehash() - - # Confirm it in a block. - block = self.build_next_block() - self.update_witness_block_with_transactions(block, [tx]) - test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) - - # Now try to spend it. Send it to a P2WSH output, which we'll - # use in the next test. - witness_program = CScript([pubkey, CScriptOp(OP_CHECKSIG)]) - witness_hash = sha256(witness_program) - scriptWSH = CScript([OP_0, witness_hash]) - - tx2 = CTransaction() - tx2.vin.append(CTxIn(COutPoint(tx.sha256, 0), b"")) - tx2.vout.append(CTxOut(tx.vout[0].nValue-1000, scriptWSH)) - script = GetP2PKHScript(pubkeyhash) - sig_hash = SegwitVersion1SignatureHash(script, tx2, 0, SIGHASH_ALL, tx.vout[0].nValue) - signature = key.sign(sig_hash) + b'\x01' # 0x1 is SIGHASH_ALL - tx2.wit.vtxinwit.append(CTxInWitness()) - tx2.wit.vtxinwit[0].scriptWitness.stack = [ signature, pubkey ] - tx2.rehash() - - # Should fail policy test. - test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx2, True, False, b'non-mandatory-script-verify-flag (Using non-compressed keys in segwit)') - # But passes consensus. - block = self.build_next_block() - self.update_witness_block_with_transactions(block, [tx2]) - test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) - - # Test 2: P2WSH - # Try to spend the P2WSH output created in last test. - # Send it to a P2SH(P2WSH) output, which we'll use in the next test. - p2sh_witness_hash = hash160(scriptWSH) - scriptP2SH = CScript([OP_HASH160, p2sh_witness_hash, OP_EQUAL]) - scriptSig = CScript([scriptWSH]) - - tx3 = CTransaction() - tx3.vin.append(CTxIn(COutPoint(tx2.sha256, 0), b"")) - tx3.vout.append(CTxOut(tx2.vout[0].nValue-1000, scriptP2SH)) - tx3.wit.vtxinwit.append(CTxInWitness()) - sign_P2PK_witness_input(witness_program, tx3, 0, SIGHASH_ALL, tx2.vout[0].nValue, key) - - # Should fail policy test. - test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx3, True, False, b'non-mandatory-script-verify-flag (Using non-compressed keys in segwit)') - # But passes consensus. - block = self.build_next_block() - self.update_witness_block_with_transactions(block, [tx3]) - test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) - - # Test 3: P2SH(P2WSH) - # Try to spend the P2SH output created in the last test. - # Send it to a P2PKH output, which we'll use in the next test. - scriptPubKey = GetP2PKHScript(pubkeyhash) - tx4 = CTransaction() - tx4.vin.append(CTxIn(COutPoint(tx3.sha256, 0), scriptSig)) - tx4.vout.append(CTxOut(tx3.vout[0].nValue-1000, scriptPubKey)) - tx4.wit.vtxinwit.append(CTxInWitness()) - sign_P2PK_witness_input(witness_program, tx4, 0, SIGHASH_ALL, tx3.vout[0].nValue, key) - - # Should fail policy test. - test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx4, True, False, b'non-mandatory-script-verify-flag (Using non-compressed keys in segwit)') - block = self.build_next_block() - self.update_witness_block_with_transactions(block, [tx4]) - test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) - - # Test 4: Uncompressed pubkeys should still be valid in non-segwit - # transactions. - tx5 = CTransaction() - tx5.vin.append(CTxIn(COutPoint(tx4.sha256, 0), b"")) - tx5.vout.append(CTxOut(tx4.vout[0].nValue-1000, CScript([OP_TRUE]))) - (sig_hash, err) = SignatureHash(scriptPubKey, tx5, 0, SIGHASH_ALL) - signature = key.sign(sig_hash) + b'\x01' # 0x1 is SIGHASH_ALL - tx5.vin[0].scriptSig = CScript([signature, pubkey]) - tx5.rehash() - # Should pass policy and consensus. - test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx5, True, True) - block = self.build_next_block() - self.update_witness_block_with_transactions(block, [tx5]) - test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) - self.utxo.append(UTXO(tx5.sha256, 0, tx5.vout[0].nValue)) - - def test_non_standard_witness(self): - self.log.info("Testing detection of non-standard P2WSH witness") - pad = chr(1).encode('latin-1') - - # Create scripts for tests - scripts = [] - scripts.append(CScript([OP_DROP] * 100)) - scripts.append(CScript([OP_DROP] * 99)) - scripts.append(CScript([pad * 59] * 59 + [OP_DROP] * 60)) - scripts.append(CScript([pad * 59] * 59 + [OP_DROP] * 61)) - - p2wsh_scripts = [] - - assert(len(self.utxo)) - tx = CTransaction() - tx.vin.append(CTxIn(COutPoint(self.utxo[0].sha256, self.utxo[0].n), b"")) - - # For each script, generate a pair of P2WSH and P2SH-P2WSH output. - outputvalue = (self.utxo[0].nValue - 1000) // (len(scripts) * 2) - for i in scripts: - p2wsh = CScript([OP_0, sha256(i)]) - p2sh = hash160(p2wsh) - p2wsh_scripts.append(p2wsh) - tx.vout.append(CTxOut(outputvalue, p2wsh)) - tx.vout.append(CTxOut(outputvalue, CScript([OP_HASH160, p2sh, OP_EQUAL]))) - tx.rehash() - txid = tx.sha256 - test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx, with_witness=False, accepted=True) - - self.nodes[0].generate(1) - sync_blocks(self.nodes) - - # Creating transactions for tests - p2wsh_txs = [] - p2sh_txs = [] - for i in range(len(scripts)): - p2wsh_tx = CTransaction() - p2wsh_tx.vin.append(CTxIn(COutPoint(txid,i*2))) - p2wsh_tx.vout.append(CTxOut(outputvalue - 5000, CScript([OP_0, hash160(hex_str_to_bytes(""))]))) - p2wsh_tx.wit.vtxinwit.append(CTxInWitness()) - p2wsh_tx.rehash() - p2wsh_txs.append(p2wsh_tx) - p2sh_tx = CTransaction() - p2sh_tx.vin.append(CTxIn(COutPoint(txid,i*2+1), CScript([p2wsh_scripts[i]]))) - p2sh_tx.vout.append(CTxOut(outputvalue - 5000, CScript([OP_0, hash160(hex_str_to_bytes(""))]))) - p2sh_tx.wit.vtxinwit.append(CTxInWitness()) - p2sh_tx.rehash() - p2sh_txs.append(p2sh_tx) - - # Testing native P2WSH - # Witness stack size, excluding witnessScript, over 100 is non-standard - p2wsh_txs[0].wit.vtxinwit[0].scriptWitness.stack = [pad] * 101 + [scripts[0]] - test_transaction_acceptance(self.nodes[1].rpc, self.std_node, p2wsh_txs[0], True, False, b'bad-witness-nonstandard') - # Non-standard nodes should accept - test_transaction_acceptance(self.nodes[0].rpc, self.test_node, p2wsh_txs[0], True, True) - - # Stack element size over 80 bytes is non-standard - p2wsh_txs[1].wit.vtxinwit[0].scriptWitness.stack = [pad * 81] * 100 + [scripts[1]] - test_transaction_acceptance(self.nodes[1].rpc, self.std_node, p2wsh_txs[1], True, False, b'bad-witness-nonstandard') - # Non-standard nodes should accept - test_transaction_acceptance(self.nodes[0].rpc, self.test_node, p2wsh_txs[1], True, True) - # Standard nodes should accept if element size is not over 80 bytes - p2wsh_txs[1].wit.vtxinwit[0].scriptWitness.stack = [pad * 80] * 100 + [scripts[1]] - test_transaction_acceptance(self.nodes[1].rpc, self.std_node, p2wsh_txs[1], True, True) - - # witnessScript size at 3600 bytes is standard - p2wsh_txs[2].wit.vtxinwit[0].scriptWitness.stack = [pad, pad, scripts[2]] - test_transaction_acceptance(self.nodes[0].rpc, self.test_node, p2wsh_txs[2], True, True) - test_transaction_acceptance(self.nodes[1].rpc, self.std_node, p2wsh_txs[2], True, True) - - # witnessScript size at 3601 bytes is non-standard - p2wsh_txs[3].wit.vtxinwit[0].scriptWitness.stack = [pad, pad, pad, scripts[3]] - test_transaction_acceptance(self.nodes[1].rpc, self.std_node, p2wsh_txs[3], True, False, b'bad-witness-nonstandard') - # Non-standard nodes should accept - test_transaction_acceptance(self.nodes[0].rpc, self.test_node, p2wsh_txs[3], True, True) - - # Repeating the same tests with P2SH-P2WSH - p2sh_txs[0].wit.vtxinwit[0].scriptWitness.stack = [pad] * 101 + [scripts[0]] - test_transaction_acceptance(self.nodes[1].rpc, self.std_node, p2sh_txs[0], True, False, b'bad-witness-nonstandard') - test_transaction_acceptance(self.nodes[0].rpc, self.test_node, p2sh_txs[0], True, True) - p2sh_txs[1].wit.vtxinwit[0].scriptWitness.stack = [pad * 81] * 100 + [scripts[1]] - test_transaction_acceptance(self.nodes[1].rpc, self.std_node, p2sh_txs[1], True, False, b'bad-witness-nonstandard') - test_transaction_acceptance(self.nodes[0].rpc, self.test_node, p2sh_txs[1], True, True) - p2sh_txs[1].wit.vtxinwit[0].scriptWitness.stack = [pad * 80] * 100 + [scripts[1]] - test_transaction_acceptance(self.nodes[1].rpc, self.std_node, p2sh_txs[1], True, True) - p2sh_txs[2].wit.vtxinwit[0].scriptWitness.stack = [pad, pad, scripts[2]] - test_transaction_acceptance(self.nodes[0].rpc, self.test_node, p2sh_txs[2], True, True) - test_transaction_acceptance(self.nodes[1].rpc, self.std_node, p2sh_txs[2], True, True) - p2sh_txs[3].wit.vtxinwit[0].scriptWitness.stack = [pad, pad, pad, scripts[3]] - test_transaction_acceptance(self.nodes[1].rpc, self.std_node, p2sh_txs[3], True, False, b'bad-witness-nonstandard') - test_transaction_acceptance(self.nodes[0].rpc, self.test_node, p2sh_txs[3], True, True) - - self.nodes[0].generate(1) # Mine and clean up the mempool of non-standard node - # Valid but non-standard transactions in a block should be accepted by standard node - sync_blocks(self.nodes) - assert_equal(len(self.nodes[0].getrawmempool()), 0) - assert_equal(len(self.nodes[1].getrawmempool()), 0) - - self.utxo.pop(0) - - - def run_test(self): - # Setup the p2p connections and start up the network thread. - # self.test_node sets NODE_WITNESS|NODE_NETWORK - self.test_node = self.nodes[0].add_p2p_connection(TestNode(), services=NODE_NETWORK|NODE_WITNESS) - # self.old_node sets only NODE_NETWORK - self.old_node = self.nodes[0].add_p2p_connection(TestNode(), services=NODE_NETWORK) - # self.std_node is for testing node1 (fRequireStandard=true) - self.std_node = self.nodes[1].add_p2p_connection(TestNode(), services=NODE_NETWORK|NODE_WITNESS) - - network_thread_start() - - # Keep a place to store utxo's that can be used in later tests - self.utxo = [] - - # Test logic begins here - self.test_node.wait_for_verack() - - self.log.info("Starting tests before segwit lock in:") - - self.test_witness_services() # Verifies NODE_WITNESS - self.test_non_witness_transaction() # non-witness tx's are accepted - self.test_unnecessary_witness_before_segwit_activation() - self.test_block_relay(segwit_activated=False) - - # Advance to segwit being 'started' - self.advance_to_segwit_started() - sync_blocks(self.nodes) - self.test_getblocktemplate_before_lockin() - - sync_blocks(self.nodes) - - # At lockin, nothing should change. - self.log.info("Testing behavior post lockin, pre-activation") - self.advance_to_segwit_lockin() - - # Retest unnecessary witnesses - self.test_unnecessary_witness_before_segwit_activation() - self.test_witness_tx_relay_before_segwit_activation() - self.test_block_relay(segwit_activated=False) - self.test_p2sh_witness(segwit_activated=False) - self.test_standardness_v0(segwit_activated=False) - - sync_blocks(self.nodes) - - # Now activate segwit - self.log.info("Testing behavior after segwit activation") - self.advance_to_segwit_active() - - sync_blocks(self.nodes) - - # Test P2SH witness handling again - self.test_p2sh_witness(segwit_activated=True) - self.test_witness_commitments() - self.test_block_malleability() - self.test_witness_block_size() - self.test_submit_block() - self.test_extra_witness_data() - self.test_max_witness_push_length() - self.test_max_witness_program_length() - self.test_witness_input_length() - self.test_block_relay(segwit_activated=True) - self.test_tx_relay_after_segwit_activation() - self.test_standardness_v0(segwit_activated=True) - self.test_segwit_versions() - self.test_premature_coinbase_witness_spend() - self.test_uncompressed_pubkey() - self.test_signature_version_1() - self.test_non_standard_witness() - sync_blocks(self.nodes) - self.test_upgrade_after_activation(node_id=2) - self.test_witness_sigops() - - -if __name__ == '__main__': - SegWitTest().main() diff --git a/test/functional/p2p-timeouts.py b/test/functional/p2p-timeouts.py deleted file mode 100755 index 6d21095cc6..0000000000 --- a/test/functional/p2p-timeouts.py +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2016-2017 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test various net timeouts. - -- Create three bitcoind nodes: - - no_verack_node - we never send a verack in response to their version - no_version_node - we never send a version (only a ping) - no_send_node - we never send any P2P message. - -- Start all three nodes -- Wait 1 second -- Assert that we're connected -- Send a ping to no_verack_node and no_version_node -- Wait 30 seconds -- Assert that we're still connected -- Send a ping to no_verack_node and no_version_node -- Wait 31 seconds -- Assert that we're no longer connected (timeout to receive version/verack is 60 seconds) -""" - -from time import sleep - -from test_framework.mininode import * -from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import * - -class TestNode(P2PInterface): - def on_version(self, message): - # Don't send a verack in response - pass - -class TimeoutsTest(BitcoinTestFramework): - def set_test_params(self): - self.setup_clean_chain = True - self.num_nodes = 1 - - def run_test(self): - # Setup the p2p connections and start up the network thread. - no_verack_node = self.nodes[0].add_p2p_connection(TestNode()) - no_version_node = self.nodes[0].add_p2p_connection(TestNode(), send_version=False) - no_send_node = self.nodes[0].add_p2p_connection(TestNode(), send_version=False) - - network_thread_start() - - sleep(1) - - assert no_verack_node.connected - assert no_version_node.connected - assert no_send_node.connected - - no_verack_node.send_message(msg_ping()) - no_version_node.send_message(msg_ping()) - - sleep(30) - - assert "version" in no_verack_node.last_message - - assert no_verack_node.connected - assert no_version_node.connected - assert no_send_node.connected - - no_verack_node.send_message(msg_ping()) - no_version_node.send_message(msg_ping()) - - sleep(31) - - assert not no_verack_node.connected - assert not no_version_node.connected - assert not no_send_node.connected - -if __name__ == '__main__': - TimeoutsTest().main() diff --git a/test/functional/p2p_compactblocks.py b/test/functional/p2p_compactblocks.py new file mode 100755 index 0000000000..d9f461a049 --- /dev/null +++ b/test/functional/p2p_compactblocks.py @@ -0,0 +1,921 @@ +#!/usr/bin/env python3 +# Copyright (c) 2016-2017 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test compact blocks (BIP 152). + +Version 1 compact blocks are pre-segwit (txids) +Version 2 compact blocks are post-segwit (wtxids) +""" + +from test_framework.mininode import * +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import * +from test_framework.blocktools import create_block, create_coinbase, add_witness_commitment +from test_framework.script import CScript, OP_TRUE + +# TestNode: A peer we use to send messages to bitcoind, and store responses. +class TestNode(P2PInterface): + def __init__(self): + super().__init__() + self.last_sendcmpct = [] + self.block_announced = False + # Store the hashes of blocks we've seen announced. + # This is for synchronizing the p2p message traffic, + # so we can eg wait until a particular block is announced. + self.announced_blockhashes = set() + + def on_sendcmpct(self, message): + self.last_sendcmpct.append(message) + + def on_cmpctblock(self, message): + self.block_announced = True + self.last_message["cmpctblock"].header_and_shortids.header.calc_sha256() + self.announced_blockhashes.add(self.last_message["cmpctblock"].header_and_shortids.header.sha256) + + def on_headers(self, message): + self.block_announced = True + for x in self.last_message["headers"].headers: + x.calc_sha256() + self.announced_blockhashes.add(x.sha256) + + def on_inv(self, message): + for x in self.last_message["inv"].inv: + if x.type == 2: + self.block_announced = True + self.announced_blockhashes.add(x.hash) + + # Requires caller to hold mininode_lock + def received_block_announcement(self): + return self.block_announced + + def clear_block_announcement(self): + with mininode_lock: + self.block_announced = False + self.last_message.pop("inv", None) + self.last_message.pop("headers", None) + self.last_message.pop("cmpctblock", None) + + def get_headers(self, locator, hashstop): + msg = msg_getheaders() + msg.locator.vHave = locator + msg.hashstop = hashstop + self.send_message(msg) + + def send_header_for_blocks(self, new_blocks): + headers_message = msg_headers() + headers_message.headers = [CBlockHeader(b) for b in new_blocks] + self.send_message(headers_message) + + def request_headers_and_sync(self, locator, hashstop=0): + self.clear_block_announcement() + self.get_headers(locator, hashstop) + wait_until(self.received_block_announcement, timeout=30, lock=mininode_lock) + self.clear_block_announcement() + + # Block until a block announcement for a particular block hash is + # received. + def wait_for_block_announcement(self, block_hash, timeout=30): + def received_hash(): + return (block_hash in self.announced_blockhashes) + wait_until(received_hash, timeout=timeout, lock=mininode_lock) + + def send_await_disconnect(self, message, timeout=30): + """Sends a message to the node and wait for disconnect. + + This is used when we want to send a message into the node that we expect + will get us disconnected, eg an invalid block.""" + self.send_message(message) + wait_until(lambda: self.state != "connected", timeout=timeout, lock=mininode_lock) + +class CompactBlocksTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + # Node0 = pre-segwit, node1 = segwit-aware + self.num_nodes = 2 + # This test was written assuming SegWit is activated using BIP9 at height 432 (3x confirmation window). + # TODO: Rewrite this test to support SegWit being always active. + self.extra_args = [["-vbparams=segwit:0:0"], ["-vbparams=segwit:0:999999999999", "-txindex", "-deprecatedrpc=addwitnessaddress"]] + self.utxos = [] + + def build_block_on_tip(self, node, segwit=False): + height = node.getblockcount() + tip = node.getbestblockhash() + mtp = node.getblockheader(tip)['mediantime'] + block = create_block(int(tip, 16), create_coinbase(height + 1), mtp + 1) + block.nVersion = 4 + if segwit: + add_witness_commitment(block) + block.solve() + return block + + # Create 10 more anyone-can-spend utxo's for testing. + def make_utxos(self): + # Doesn't matter which node we use, just use node0. + block = self.build_block_on_tip(self.nodes[0]) + self.test_node.send_and_ping(msg_block(block)) + assert(int(self.nodes[0].getbestblockhash(), 16) == block.sha256) + self.nodes[0].generate(100) + + total_value = block.vtx[0].vout[0].nValue + out_value = total_value // 10 + tx = CTransaction() + tx.vin.append(CTxIn(COutPoint(block.vtx[0].sha256, 0), b'')) + for i in range(10): + tx.vout.append(CTxOut(out_value, CScript([OP_TRUE]))) + tx.rehash() + + block2 = self.build_block_on_tip(self.nodes[0]) + block2.vtx.append(tx) + block2.hashMerkleRoot = block2.calc_merkle_root() + block2.solve() + self.test_node.send_and_ping(msg_block(block2)) + assert_equal(int(self.nodes[0].getbestblockhash(), 16), block2.sha256) + self.utxos.extend([[tx.sha256, i, out_value] for i in range(10)]) + return + + # Test "sendcmpct" (between peers preferring the same version): + # - No compact block announcements unless sendcmpct is sent. + # - If sendcmpct is sent with version > preferred_version, the message is ignored. + # - If sendcmpct is sent with boolean 0, then block announcements are not + # made with compact blocks. + # - If sendcmpct is then sent with boolean 1, then new block announcements + # are made with compact blocks. + # If old_node is passed in, request compact blocks with version=preferred-1 + # and verify that it receives block announcements via compact block. + def test_sendcmpct(self, node, test_node, preferred_version, old_node=None): + # Make sure we get a SENDCMPCT message from our peer + def received_sendcmpct(): + return (len(test_node.last_sendcmpct) > 0) + wait_until(received_sendcmpct, timeout=30, lock=mininode_lock) + with mininode_lock: + # Check that the first version received is the preferred one + assert_equal(test_node.last_sendcmpct[0].version, preferred_version) + # And that we receive versions down to 1. + assert_equal(test_node.last_sendcmpct[-1].version, 1) + test_node.last_sendcmpct = [] + + tip = int(node.getbestblockhash(), 16) + + def check_announcement_of_new_block(node, peer, predicate): + peer.clear_block_announcement() + block_hash = int(node.generate(1)[0], 16) + peer.wait_for_block_announcement(block_hash, timeout=30) + assert(peer.block_announced) + + with mininode_lock: + assert predicate(peer), ( + "block_hash={!r}, cmpctblock={!r}, inv={!r}".format( + block_hash, peer.last_message.get("cmpctblock", None), peer.last_message.get("inv", None))) + + # We shouldn't get any block announcements via cmpctblock yet. + check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message) + + # Try one more time, this time after requesting headers. + test_node.request_headers_and_sync(locator=[tip]) + check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message and "inv" in p.last_message) + + # Test a few ways of using sendcmpct that should NOT + # result in compact block announcements. + # Before each test, sync the headers chain. + test_node.request_headers_and_sync(locator=[tip]) + + # Now try a SENDCMPCT message with too-high version + sendcmpct = msg_sendcmpct() + sendcmpct.version = preferred_version+1 + sendcmpct.announce = True + test_node.send_and_ping(sendcmpct) + check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message) + + # Headers sync before next test. + test_node.request_headers_and_sync(locator=[tip]) + + # Now try a SENDCMPCT message with valid version, but announce=False + sendcmpct.version = preferred_version + sendcmpct.announce = False + test_node.send_and_ping(sendcmpct) + check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message) + + # Headers sync before next test. + test_node.request_headers_and_sync(locator=[tip]) + + # Finally, try a SENDCMPCT message with announce=True + sendcmpct.version = preferred_version + sendcmpct.announce = True + test_node.send_and_ping(sendcmpct) + check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" in p.last_message) + + # Try one more time (no headers sync should be needed!) + check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" in p.last_message) + + # Try one more time, after turning on sendheaders + test_node.send_and_ping(msg_sendheaders()) + check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" in p.last_message) + + # Try one more time, after sending a version-1, announce=false message. + sendcmpct.version = preferred_version-1 + sendcmpct.announce = False + test_node.send_and_ping(sendcmpct) + check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" in p.last_message) + + # Now turn off announcements + sendcmpct.version = preferred_version + sendcmpct.announce = False + test_node.send_and_ping(sendcmpct) + check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message and "headers" in p.last_message) + + if old_node is not None: + # Verify that a peer using an older protocol version can receive + # announcements from this node. + sendcmpct.version = preferred_version-1 + sendcmpct.announce = True + old_node.send_and_ping(sendcmpct) + # Header sync + old_node.request_headers_and_sync(locator=[tip]) + check_announcement_of_new_block(node, old_node, lambda p: "cmpctblock" in p.last_message) + + # This test actually causes bitcoind to (reasonably!) disconnect us, so do this last. + def test_invalid_cmpctblock_message(self): + self.nodes[0].generate(101) + block = self.build_block_on_tip(self.nodes[0]) + + cmpct_block = P2PHeaderAndShortIDs() + cmpct_block.header = CBlockHeader(block) + cmpct_block.prefilled_txn_length = 1 + # This index will be too high + prefilled_txn = PrefilledTransaction(1, block.vtx[0]) + cmpct_block.prefilled_txn = [prefilled_txn] + self.test_node.send_await_disconnect(msg_cmpctblock(cmpct_block)) + assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.hashPrevBlock) + + # Compare the generated shortids to what we expect based on BIP 152, given + # bitcoind's choice of nonce. + def test_compactblock_construction(self, node, test_node, version, use_witness_address): + # Generate a bunch of transactions. + node.generate(101) + num_transactions = 25 + address = node.getnewaddress() + if use_witness_address: + # Want at least one segwit spend, so move all funds to + # a witness address. + address = node.addwitnessaddress(address) + value_to_send = node.getbalance() + node.sendtoaddress(address, satoshi_round(value_to_send-Decimal(0.1))) + node.generate(1) + + segwit_tx_generated = False + for i in range(num_transactions): + txid = node.sendtoaddress(address, 0.1) + hex_tx = node.gettransaction(txid)["hex"] + tx = FromHex(CTransaction(), hex_tx) + if not tx.wit.is_null(): + segwit_tx_generated = True + + if use_witness_address: + assert(segwit_tx_generated) # check that our test is not broken + + # Wait until we've seen the block announcement for the resulting tip + tip = int(node.getbestblockhash(), 16) + test_node.wait_for_block_announcement(tip) + + # Make sure we will receive a fast-announce compact block + self.request_cb_announcements(test_node, node, version) + + # Now mine a block, and look at the resulting compact block. + test_node.clear_block_announcement() + block_hash = int(node.generate(1)[0], 16) + + # Store the raw block in our internal format. + block = FromHex(CBlock(), node.getblock("%02x" % block_hash, False)) + for tx in block.vtx: + tx.calc_sha256() + block.rehash() + + # Wait until the block was announced (via compact blocks) + wait_until(test_node.received_block_announcement, timeout=30, lock=mininode_lock) + + # Now fetch and check the compact block + header_and_shortids = None + with mininode_lock: + assert("cmpctblock" in test_node.last_message) + # Convert the on-the-wire representation to absolute indexes + header_and_shortids = HeaderAndShortIDs(test_node.last_message["cmpctblock"].header_and_shortids) + self.check_compactblock_construction_from_block(version, header_and_shortids, block_hash, block) + + # Now fetch the compact block using a normal non-announce getdata + with mininode_lock: + test_node.clear_block_announcement() + inv = CInv(4, block_hash) # 4 == "CompactBlock" + test_node.send_message(msg_getdata([inv])) + + wait_until(test_node.received_block_announcement, timeout=30, lock=mininode_lock) + + # Now fetch and check the compact block + header_and_shortids = None + with mininode_lock: + assert("cmpctblock" in test_node.last_message) + # Convert the on-the-wire representation to absolute indexes + header_and_shortids = HeaderAndShortIDs(test_node.last_message["cmpctblock"].header_and_shortids) + self.check_compactblock_construction_from_block(version, header_and_shortids, block_hash, block) + + def check_compactblock_construction_from_block(self, version, header_and_shortids, block_hash, block): + # Check that we got the right block! + header_and_shortids.header.calc_sha256() + assert_equal(header_and_shortids.header.sha256, block_hash) + + # Make sure the prefilled_txn appears to have included the coinbase + assert(len(header_and_shortids.prefilled_txn) >= 1) + assert_equal(header_and_shortids.prefilled_txn[0].index, 0) + + # Check that all prefilled_txn entries match what's in the block. + for entry in header_and_shortids.prefilled_txn: + entry.tx.calc_sha256() + # This checks the non-witness parts of the tx agree + assert_equal(entry.tx.sha256, block.vtx[entry.index].sha256) + + # And this checks the witness + wtxid = entry.tx.calc_sha256(True) + if version == 2: + assert_equal(wtxid, block.vtx[entry.index].calc_sha256(True)) + else: + # Shouldn't have received a witness + assert(entry.tx.wit.is_null()) + + # Check that the cmpctblock message announced all the transactions. + assert_equal(len(header_and_shortids.prefilled_txn) + len(header_and_shortids.shortids), len(block.vtx)) + + # And now check that all the shortids are as expected as well. + # Determine the siphash keys to use. + [k0, k1] = header_and_shortids.get_siphash_keys() + + index = 0 + while index < len(block.vtx): + if (len(header_and_shortids.prefilled_txn) > 0 and + header_and_shortids.prefilled_txn[0].index == index): + # Already checked prefilled transactions above + header_and_shortids.prefilled_txn.pop(0) + else: + tx_hash = block.vtx[index].sha256 + if version == 2: + tx_hash = block.vtx[index].calc_sha256(True) + shortid = calculate_shortid(k0, k1, tx_hash) + assert_equal(shortid, header_and_shortids.shortids[0]) + header_and_shortids.shortids.pop(0) + index += 1 + + # Test that bitcoind requests compact blocks when we announce new blocks + # via header or inv, and that responding to getblocktxn causes the block + # to be successfully reconstructed. + # Post-segwit: upgraded nodes would only make this request of cb-version-2, + # NODE_WITNESS peers. Unupgraded nodes would still make this request of + # any cb-version-1-supporting peer. + def test_compactblock_requests(self, node, test_node, version, segwit): + # Try announcing a block with an inv or header, expect a compactblock + # request + for announce in ["inv", "header"]: + block = self.build_block_on_tip(node, segwit=segwit) + with mininode_lock: + test_node.last_message.pop("getdata", None) + + if announce == "inv": + test_node.send_message(msg_inv([CInv(2, block.sha256)])) + wait_until(lambda: "getheaders" in test_node.last_message, timeout=30, lock=mininode_lock) + test_node.send_header_for_blocks([block]) + else: + test_node.send_header_for_blocks([block]) + wait_until(lambda: "getdata" in test_node.last_message, timeout=30, lock=mininode_lock) + assert_equal(len(test_node.last_message["getdata"].inv), 1) + assert_equal(test_node.last_message["getdata"].inv[0].type, 4) + assert_equal(test_node.last_message["getdata"].inv[0].hash, block.sha256) + + # Send back a compactblock message that omits the coinbase + comp_block = HeaderAndShortIDs() + comp_block.header = CBlockHeader(block) + comp_block.nonce = 0 + [k0, k1] = comp_block.get_siphash_keys() + coinbase_hash = block.vtx[0].sha256 + if version == 2: + coinbase_hash = block.vtx[0].calc_sha256(True) + comp_block.shortids = [ + calculate_shortid(k0, k1, coinbase_hash) ] + test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) + assert_equal(int(node.getbestblockhash(), 16), block.hashPrevBlock) + # Expect a getblocktxn message. + with mininode_lock: + assert("getblocktxn" in test_node.last_message) + absolute_indexes = test_node.last_message["getblocktxn"].block_txn_request.to_absolute() + assert_equal(absolute_indexes, [0]) # should be a coinbase request + + # Send the coinbase, and verify that the tip advances. + if version == 2: + msg = msg_witness_blocktxn() + else: + msg = msg_blocktxn() + msg.block_transactions.blockhash = block.sha256 + msg.block_transactions.transactions = [block.vtx[0]] + test_node.send_and_ping(msg) + assert_equal(int(node.getbestblockhash(), 16), block.sha256) + + # Create a chain of transactions from given utxo, and add to a new block. + def build_block_with_transactions(self, node, utxo, num_transactions): + block = self.build_block_on_tip(node) + + for i in range(num_transactions): + tx = CTransaction() + tx.vin.append(CTxIn(COutPoint(utxo[0], utxo[1]), b'')) + tx.vout.append(CTxOut(utxo[2] - 1000, CScript([OP_TRUE]))) + tx.rehash() + utxo = [tx.sha256, 0, tx.vout[0].nValue] + block.vtx.append(tx) + + block.hashMerkleRoot = block.calc_merkle_root() + block.solve() + return block + + # Test that we only receive getblocktxn requests for transactions that the + # node needs, and that responding to them causes the block to be + # reconstructed. + def test_getblocktxn_requests(self, node, test_node, version): + with_witness = (version==2) + + def test_getblocktxn_response(compact_block, peer, expected_result): + msg = msg_cmpctblock(compact_block.to_p2p()) + peer.send_and_ping(msg) + with mininode_lock: + assert("getblocktxn" in peer.last_message) + absolute_indexes = peer.last_message["getblocktxn"].block_txn_request.to_absolute() + assert_equal(absolute_indexes, expected_result) + + def test_tip_after_message(node, peer, msg, tip): + peer.send_and_ping(msg) + assert_equal(int(node.getbestblockhash(), 16), tip) + + # First try announcing compactblocks that won't reconstruct, and verify + # that we receive getblocktxn messages back. + utxo = self.utxos.pop(0) + + block = self.build_block_with_transactions(node, utxo, 5) + self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue]) + comp_block = HeaderAndShortIDs() + comp_block.initialize_from_block(block, use_witness=with_witness) + + test_getblocktxn_response(comp_block, test_node, [1, 2, 3, 4, 5]) + + msg_bt = msg_blocktxn() + if with_witness: + msg_bt = msg_witness_blocktxn() # serialize with witnesses + msg_bt.block_transactions = BlockTransactions(block.sha256, block.vtx[1:]) + test_tip_after_message(node, test_node, msg_bt, block.sha256) + + utxo = self.utxos.pop(0) + block = self.build_block_with_transactions(node, utxo, 5) + self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue]) + + # Now try interspersing the prefilled transactions + comp_block.initialize_from_block(block, prefill_list=[0, 1, 5], use_witness=with_witness) + test_getblocktxn_response(comp_block, test_node, [2, 3, 4]) + msg_bt.block_transactions = BlockTransactions(block.sha256, block.vtx[2:5]) + test_tip_after_message(node, test_node, msg_bt, block.sha256) + + # Now try giving one transaction ahead of time. + utxo = self.utxos.pop(0) + block = self.build_block_with_transactions(node, utxo, 5) + self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue]) + test_node.send_and_ping(msg_tx(block.vtx[1])) + assert(block.vtx[1].hash in node.getrawmempool()) + + # Prefill 4 out of the 6 transactions, and verify that only the one + # that was not in the mempool is requested. + comp_block.initialize_from_block(block, prefill_list=[0, 2, 3, 4], use_witness=with_witness) + test_getblocktxn_response(comp_block, test_node, [5]) + + msg_bt.block_transactions = BlockTransactions(block.sha256, [block.vtx[5]]) + test_tip_after_message(node, test_node, msg_bt, block.sha256) + + # Now provide all transactions to the node before the block is + # announced and verify reconstruction happens immediately. + utxo = self.utxos.pop(0) + block = self.build_block_with_transactions(node, utxo, 10) + self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue]) + for tx in block.vtx[1:]: + test_node.send_message(msg_tx(tx)) + test_node.sync_with_ping() + # Make sure all transactions were accepted. + mempool = node.getrawmempool() + for tx in block.vtx[1:]: + assert(tx.hash in mempool) + + # Clear out last request. + with mininode_lock: + test_node.last_message.pop("getblocktxn", None) + + # Send compact block + comp_block.initialize_from_block(block, prefill_list=[0], use_witness=with_witness) + test_tip_after_message(node, test_node, msg_cmpctblock(comp_block.to_p2p()), block.sha256) + with mininode_lock: + # Shouldn't have gotten a request for any transaction + assert("getblocktxn" not in test_node.last_message) + + # Incorrectly responding to a getblocktxn shouldn't cause the block to be + # permanently failed. + def test_incorrect_blocktxn_response(self, node, test_node, version): + if (len(self.utxos) == 0): + self.make_utxos() + utxo = self.utxos.pop(0) + + block = self.build_block_with_transactions(node, utxo, 10) + self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue]) + # Relay the first 5 transactions from the block in advance + for tx in block.vtx[1:6]: + test_node.send_message(msg_tx(tx)) + test_node.sync_with_ping() + # Make sure all transactions were accepted. + mempool = node.getrawmempool() + for tx in block.vtx[1:6]: + assert(tx.hash in mempool) + + # Send compact block + comp_block = HeaderAndShortIDs() + comp_block.initialize_from_block(block, prefill_list=[0], use_witness=(version == 2)) + test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) + absolute_indexes = [] + with mininode_lock: + assert("getblocktxn" in test_node.last_message) + absolute_indexes = test_node.last_message["getblocktxn"].block_txn_request.to_absolute() + assert_equal(absolute_indexes, [6, 7, 8, 9, 10]) + + # Now give an incorrect response. + # Note that it's possible for bitcoind to be smart enough to know we're + # lying, since it could check to see if the shortid matches what we're + # sending, and eg disconnect us for misbehavior. If that behavior + # change were made, we could just modify this test by having a + # different peer provide the block further down, so that we're still + # verifying that the block isn't marked bad permanently. This is good + # enough for now. + msg = msg_blocktxn() + if version==2: + msg = msg_witness_blocktxn() + msg.block_transactions = BlockTransactions(block.sha256, [block.vtx[5]] + block.vtx[7:]) + test_node.send_and_ping(msg) + + # Tip should not have updated + assert_equal(int(node.getbestblockhash(), 16), block.hashPrevBlock) + + # We should receive a getdata request + wait_until(lambda: "getdata" in test_node.last_message, timeout=10, lock=mininode_lock) + assert_equal(len(test_node.last_message["getdata"].inv), 1) + assert(test_node.last_message["getdata"].inv[0].type == 2 or test_node.last_message["getdata"].inv[0].type == 2|MSG_WITNESS_FLAG) + assert_equal(test_node.last_message["getdata"].inv[0].hash, block.sha256) + + # Deliver the block + if version==2: + test_node.send_and_ping(msg_witness_block(block)) + else: + test_node.send_and_ping(msg_block(block)) + assert_equal(int(node.getbestblockhash(), 16), block.sha256) + + def test_getblocktxn_handler(self, node, test_node, version): + # bitcoind will not send blocktxn responses for blocks whose height is + # more than 10 blocks deep. + MAX_GETBLOCKTXN_DEPTH = 10 + chain_height = node.getblockcount() + current_height = chain_height + while (current_height >= chain_height - MAX_GETBLOCKTXN_DEPTH): + block_hash = node.getblockhash(current_height) + block = FromHex(CBlock(), node.getblock(block_hash, False)) + + msg = msg_getblocktxn() + msg.block_txn_request = BlockTransactionsRequest(int(block_hash, 16), []) + num_to_request = random.randint(1, len(block.vtx)) + msg.block_txn_request.from_absolute(sorted(random.sample(range(len(block.vtx)), num_to_request))) + test_node.send_message(msg) + wait_until(lambda: "blocktxn" in test_node.last_message, timeout=10, lock=mininode_lock) + + [tx.calc_sha256() for tx in block.vtx] + with mininode_lock: + assert_equal(test_node.last_message["blocktxn"].block_transactions.blockhash, int(block_hash, 16)) + all_indices = msg.block_txn_request.to_absolute() + for index in all_indices: + tx = test_node.last_message["blocktxn"].block_transactions.transactions.pop(0) + tx.calc_sha256() + assert_equal(tx.sha256, block.vtx[index].sha256) + if version == 1: + # Witnesses should have been stripped + assert(tx.wit.is_null()) + else: + # Check that the witness matches + assert_equal(tx.calc_sha256(True), block.vtx[index].calc_sha256(True)) + test_node.last_message.pop("blocktxn", None) + current_height -= 1 + + # Next request should send a full block response, as we're past the + # allowed depth for a blocktxn response. + block_hash = node.getblockhash(current_height) + msg.block_txn_request = BlockTransactionsRequest(int(block_hash, 16), [0]) + with mininode_lock: + test_node.last_message.pop("block", None) + test_node.last_message.pop("blocktxn", None) + test_node.send_and_ping(msg) + with mininode_lock: + test_node.last_message["block"].block.calc_sha256() + assert_equal(test_node.last_message["block"].block.sha256, int(block_hash, 16)) + assert "blocktxn" not in test_node.last_message + + def test_compactblocks_not_at_tip(self, node, test_node): + # Test that requesting old compactblocks doesn't work. + MAX_CMPCTBLOCK_DEPTH = 5 + new_blocks = [] + for i in range(MAX_CMPCTBLOCK_DEPTH + 1): + test_node.clear_block_announcement() + new_blocks.append(node.generate(1)[0]) + wait_until(test_node.received_block_announcement, timeout=30, lock=mininode_lock) + + test_node.clear_block_announcement() + test_node.send_message(msg_getdata([CInv(4, int(new_blocks[0], 16))])) + wait_until(lambda: "cmpctblock" in test_node.last_message, timeout=30, lock=mininode_lock) + + test_node.clear_block_announcement() + node.generate(1) + wait_until(test_node.received_block_announcement, timeout=30, lock=mininode_lock) + test_node.clear_block_announcement() + with mininode_lock: + test_node.last_message.pop("block", None) + test_node.send_message(msg_getdata([CInv(4, int(new_blocks[0], 16))])) + wait_until(lambda: "block" in test_node.last_message, timeout=30, lock=mininode_lock) + with mininode_lock: + test_node.last_message["block"].block.calc_sha256() + assert_equal(test_node.last_message["block"].block.sha256, int(new_blocks[0], 16)) + + # Generate an old compactblock, and verify that it's not accepted. + cur_height = node.getblockcount() + hashPrevBlock = int(node.getblockhash(cur_height-5), 16) + block = self.build_block_on_tip(node) + block.hashPrevBlock = hashPrevBlock + block.solve() + + comp_block = HeaderAndShortIDs() + comp_block.initialize_from_block(block) + test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) + + tips = node.getchaintips() + found = False + for x in tips: + if x["hash"] == block.hash: + assert_equal(x["status"], "headers-only") + found = True + break + assert(found) + + # Requesting this block via getblocktxn should silently fail + # (to avoid fingerprinting attacks). + msg = msg_getblocktxn() + msg.block_txn_request = BlockTransactionsRequest(block.sha256, [0]) + with mininode_lock: + test_node.last_message.pop("blocktxn", None) + test_node.send_and_ping(msg) + with mininode_lock: + assert "blocktxn" not in test_node.last_message + + def activate_segwit(self, node): + node.generate(144*3) + assert_equal(get_bip9_status(node, "segwit")["status"], 'active') + + def test_end_to_end_block_relay(self, node, listeners): + utxo = self.utxos.pop(0) + + block = self.build_block_with_transactions(node, utxo, 10) + + [l.clear_block_announcement() for l in listeners] + + # ToHex() won't serialize with witness, but this block has no witnesses + # anyway. TODO: repeat this test with witness tx's to a segwit node. + node.submitblock(ToHex(block)) + + for l in listeners: + wait_until(lambda: l.received_block_announcement(), timeout=30, lock=mininode_lock) + with mininode_lock: + for l in listeners: + assert "cmpctblock" in l.last_message + l.last_message["cmpctblock"].header_and_shortids.header.calc_sha256() + assert_equal(l.last_message["cmpctblock"].header_and_shortids.header.sha256, block.sha256) + + # Test that we don't get disconnected if we relay a compact block with valid header, + # but invalid transactions. + def test_invalid_tx_in_compactblock(self, node, test_node, use_segwit): + assert(len(self.utxos)) + utxo = self.utxos[0] + + block = self.build_block_with_transactions(node, utxo, 5) + del block.vtx[3] + block.hashMerkleRoot = block.calc_merkle_root() + if use_segwit: + # If we're testing with segwit, also drop the coinbase witness, + # but include the witness commitment. + add_witness_commitment(block) + block.vtx[0].wit.vtxinwit = [] + block.solve() + + # Now send the compact block with all transactions prefilled, and + # verify that we don't get disconnected. + comp_block = HeaderAndShortIDs() + comp_block.initialize_from_block(block, prefill_list=[0, 1, 2, 3, 4], use_witness=use_segwit) + msg = msg_cmpctblock(comp_block.to_p2p()) + test_node.send_and_ping(msg) + + # Check that the tip didn't advance + assert(int(node.getbestblockhash(), 16) is not block.sha256) + test_node.sync_with_ping() + + # Helper for enabling cb announcements + # Send the sendcmpct request and sync headers + def request_cb_announcements(self, peer, node, version): + tip = node.getbestblockhash() + peer.get_headers(locator=[int(tip, 16)], hashstop=0) + + msg = msg_sendcmpct() + msg.version = version + msg.announce = True + peer.send_and_ping(msg) + + def test_compactblock_reconstruction_multiple_peers(self, node, stalling_peer, delivery_peer): + assert(len(self.utxos)) + + def announce_cmpct_block(node, peer): + utxo = self.utxos.pop(0) + block = self.build_block_with_transactions(node, utxo, 5) + + cmpct_block = HeaderAndShortIDs() + cmpct_block.initialize_from_block(block) + msg = msg_cmpctblock(cmpct_block.to_p2p()) + peer.send_and_ping(msg) + with mininode_lock: + assert "getblocktxn" in peer.last_message + return block, cmpct_block + + block, cmpct_block = announce_cmpct_block(node, stalling_peer) + + for tx in block.vtx[1:]: + delivery_peer.send_message(msg_tx(tx)) + delivery_peer.sync_with_ping() + mempool = node.getrawmempool() + for tx in block.vtx[1:]: + assert(tx.hash in mempool) + + delivery_peer.send_and_ping(msg_cmpctblock(cmpct_block.to_p2p())) + assert_equal(int(node.getbestblockhash(), 16), block.sha256) + + self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue]) + + # Now test that delivering an invalid compact block won't break relay + + block, cmpct_block = announce_cmpct_block(node, stalling_peer) + for tx in block.vtx[1:]: + delivery_peer.send_message(msg_tx(tx)) + delivery_peer.sync_with_ping() + + cmpct_block.prefilled_txn[0].tx.wit.vtxinwit = [ CTxInWitness() ] + cmpct_block.prefilled_txn[0].tx.wit.vtxinwit[0].scriptWitness.stack = [ser_uint256(0)] + + cmpct_block.use_witness = True + delivery_peer.send_and_ping(msg_cmpctblock(cmpct_block.to_p2p())) + assert(int(node.getbestblockhash(), 16) != block.sha256) + + msg = msg_blocktxn() + msg.block_transactions.blockhash = block.sha256 + msg.block_transactions.transactions = block.vtx[1:] + stalling_peer.send_and_ping(msg) + assert_equal(int(node.getbestblockhash(), 16), block.sha256) + + def run_test(self): + # Setup the p2p connections and start up the network thread. + self.test_node = self.nodes[0].add_p2p_connection(TestNode()) + self.segwit_node = self.nodes[1].add_p2p_connection(TestNode(), services=NODE_NETWORK|NODE_WITNESS) + self.old_node = self.nodes[1].add_p2p_connection(TestNode(), services=NODE_NETWORK) + + network_thread_start() + + self.test_node.wait_for_verack() + + # We will need UTXOs to construct transactions in later tests. + self.make_utxos() + + self.log.info("Running tests, pre-segwit activation:") + + self.log.info("Testing SENDCMPCT p2p message... ") + self.test_sendcmpct(self.nodes[0], self.test_node, 1) + sync_blocks(self.nodes) + self.test_sendcmpct(self.nodes[1], self.segwit_node, 2, old_node=self.old_node) + sync_blocks(self.nodes) + + self.log.info("Testing compactblock construction...") + self.test_compactblock_construction(self.nodes[0], self.test_node, 1, False) + sync_blocks(self.nodes) + self.test_compactblock_construction(self.nodes[1], self.segwit_node, 2, False) + sync_blocks(self.nodes) + + self.log.info("Testing compactblock requests... ") + self.test_compactblock_requests(self.nodes[0], self.test_node, 1, False) + sync_blocks(self.nodes) + self.test_compactblock_requests(self.nodes[1], self.segwit_node, 2, False) + sync_blocks(self.nodes) + + self.log.info("Testing getblocktxn requests...") + self.test_getblocktxn_requests(self.nodes[0], self.test_node, 1) + sync_blocks(self.nodes) + self.test_getblocktxn_requests(self.nodes[1], self.segwit_node, 2) + sync_blocks(self.nodes) + + self.log.info("Testing getblocktxn handler...") + self.test_getblocktxn_handler(self.nodes[0], self.test_node, 1) + sync_blocks(self.nodes) + self.test_getblocktxn_handler(self.nodes[1], self.segwit_node, 2) + self.test_getblocktxn_handler(self.nodes[1], self.old_node, 1) + sync_blocks(self.nodes) + + self.log.info("Testing compactblock requests/announcements not at chain tip...") + self.test_compactblocks_not_at_tip(self.nodes[0], self.test_node) + sync_blocks(self.nodes) + self.test_compactblocks_not_at_tip(self.nodes[1], self.segwit_node) + self.test_compactblocks_not_at_tip(self.nodes[1], self.old_node) + sync_blocks(self.nodes) + + self.log.info("Testing handling of incorrect blocktxn responses...") + self.test_incorrect_blocktxn_response(self.nodes[0], self.test_node, 1) + sync_blocks(self.nodes) + self.test_incorrect_blocktxn_response(self.nodes[1], self.segwit_node, 2) + sync_blocks(self.nodes) + + # End-to-end block relay tests + self.log.info("Testing end-to-end block relay...") + self.request_cb_announcements(self.test_node, self.nodes[0], 1) + self.request_cb_announcements(self.old_node, self.nodes[1], 1) + self.request_cb_announcements(self.segwit_node, self.nodes[1], 2) + self.test_end_to_end_block_relay(self.nodes[0], [self.segwit_node, self.test_node, self.old_node]) + self.test_end_to_end_block_relay(self.nodes[1], [self.segwit_node, self.test_node, self.old_node]) + + self.log.info("Testing handling of invalid compact blocks...") + self.test_invalid_tx_in_compactblock(self.nodes[0], self.test_node, False) + self.test_invalid_tx_in_compactblock(self.nodes[1], self.segwit_node, False) + self.test_invalid_tx_in_compactblock(self.nodes[1], self.old_node, False) + + self.log.info("Testing reconstructing compact blocks from all peers...") + self.test_compactblock_reconstruction_multiple_peers(self.nodes[1], self.segwit_node, self.old_node) + sync_blocks(self.nodes) + + # Advance to segwit activation + self.log.info("Advancing to segwit activation") + self.activate_segwit(self.nodes[1]) + self.log.info("Running tests, post-segwit activation...") + + self.log.info("Testing compactblock construction...") + self.test_compactblock_construction(self.nodes[1], self.old_node, 1, True) + self.test_compactblock_construction(self.nodes[1], self.segwit_node, 2, True) + sync_blocks(self.nodes) + + self.log.info("Testing compactblock requests (unupgraded node)... ") + self.test_compactblock_requests(self.nodes[0], self.test_node, 1, True) + + self.log.info("Testing getblocktxn requests (unupgraded node)...") + self.test_getblocktxn_requests(self.nodes[0], self.test_node, 1) + + # Need to manually sync node0 and node1, because post-segwit activation, + # node1 will not download blocks from node0. + self.log.info("Syncing nodes...") + assert(self.nodes[0].getbestblockhash() != self.nodes[1].getbestblockhash()) + while (self.nodes[0].getblockcount() > self.nodes[1].getblockcount()): + block_hash = self.nodes[0].getblockhash(self.nodes[1].getblockcount()+1) + self.nodes[1].submitblock(self.nodes[0].getblock(block_hash, False)) + assert_equal(self.nodes[0].getbestblockhash(), self.nodes[1].getbestblockhash()) + + self.log.info("Testing compactblock requests (segwit node)... ") + self.test_compactblock_requests(self.nodes[1], self.segwit_node, 2, True) + + self.log.info("Testing getblocktxn requests (segwit node)...") + self.test_getblocktxn_requests(self.nodes[1], self.segwit_node, 2) + sync_blocks(self.nodes) + + self.log.info("Testing getblocktxn handler (segwit node should return witnesses)...") + self.test_getblocktxn_handler(self.nodes[1], self.segwit_node, 2) + self.test_getblocktxn_handler(self.nodes[1], self.old_node, 1) + + # Test that if we submitblock to node1, we'll get a compact block + # announcement to all peers. + # (Post-segwit activation, blocks won't propagate from node0 to node1 + # automatically, so don't bother testing a block announced to node0.) + self.log.info("Testing end-to-end block relay...") + self.request_cb_announcements(self.test_node, self.nodes[0], 1) + self.request_cb_announcements(self.old_node, self.nodes[1], 1) + self.request_cb_announcements(self.segwit_node, self.nodes[1], 2) + self.test_end_to_end_block_relay(self.nodes[1], [self.segwit_node, self.test_node, self.old_node]) + + self.log.info("Testing handling of invalid compact blocks...") + self.test_invalid_tx_in_compactblock(self.nodes[0], self.test_node, False) + self.test_invalid_tx_in_compactblock(self.nodes[1], self.segwit_node, True) + self.test_invalid_tx_in_compactblock(self.nodes[1], self.old_node, True) + + self.log.info("Testing invalid index in cmpctblock message...") + self.test_invalid_cmpctblock_message() + + +if __name__ == '__main__': + CompactBlocksTest().main() diff --git a/test/functional/p2p_disconnect_ban.py b/test/functional/p2p_disconnect_ban.py new file mode 100755 index 0000000000..c6067befb2 --- /dev/null +++ b/test/functional/p2p_disconnect_ban.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +# Copyright (c) 2014-2017 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test node disconnect and ban behavior""" +import time + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + assert_raises_rpc_error, + connect_nodes_bi, + wait_until, +) + +class DisconnectBanTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 2 + + def run_test(self): + self.log.info("Test setban and listbanned RPCs") + + self.log.info("setban: successfully ban single IP address") + assert_equal(len(self.nodes[1].getpeerinfo()), 2) # node1 should have 2 connections to node0 at this point + self.nodes[1].setban("127.0.0.1", "add") + wait_until(lambda: len(self.nodes[1].getpeerinfo()) == 0, timeout=10) + assert_equal(len(self.nodes[1].getpeerinfo()), 0) # all nodes must be disconnected at this point + assert_equal(len(self.nodes[1].listbanned()), 1) + + self.log.info("clearbanned: successfully clear ban list") + self.nodes[1].clearbanned() + assert_equal(len(self.nodes[1].listbanned()), 0) + self.nodes[1].setban("127.0.0.0/24", "add") + + self.log.info("setban: fail to ban an already banned subnet") + assert_equal(len(self.nodes[1].listbanned()), 1) + assert_raises_rpc_error(-23, "IP/Subnet already banned", self.nodes[1].setban, "127.0.0.1", "add") + + self.log.info("setban: fail to ban an invalid subnet") + assert_raises_rpc_error(-30, "Error: Invalid IP/Subnet", self.nodes[1].setban, "127.0.0.1/42", "add") + assert_equal(len(self.nodes[1].listbanned()), 1) # still only one banned ip because 127.0.0.1 is within the range of 127.0.0.0/24 + + self.log.info("setban remove: fail to unban a non-banned subnet") + assert_raises_rpc_error(-30, "Error: Unban failed", self.nodes[1].setban, "127.0.0.1", "remove") + assert_equal(len(self.nodes[1].listbanned()), 1) + + self.log.info("setban remove: successfully unban subnet") + self.nodes[1].setban("127.0.0.0/24", "remove") + assert_equal(len(self.nodes[1].listbanned()), 0) + self.nodes[1].clearbanned() + assert_equal(len(self.nodes[1].listbanned()), 0) + + self.log.info("setban: test persistence across node restart") + self.nodes[1].setban("127.0.0.0/32", "add") + self.nodes[1].setban("127.0.0.0/24", "add") + # Set the mocktime so we can control when bans expire + old_time = int(time.time()) + self.nodes[1].setmocktime(old_time) + self.nodes[1].setban("192.168.0.1", "add", 1) # ban for 1 seconds + self.nodes[1].setban("2001:4d48:ac57:400:cacf:e9ff:fe1d:9c63/19", "add", 1000) # ban for 1000 seconds + listBeforeShutdown = self.nodes[1].listbanned() + assert_equal("192.168.0.1/32", listBeforeShutdown[2]['address']) + # Move time forward by 3 seconds so the third ban has expired + self.nodes[1].setmocktime(old_time + 3) + assert_equal(len(self.nodes[1].listbanned()), 3) + + self.stop_node(1) + self.start_node(1) + + listAfterShutdown = self.nodes[1].listbanned() + assert_equal("127.0.0.0/24", listAfterShutdown[0]['address']) + assert_equal("127.0.0.0/32", listAfterShutdown[1]['address']) + assert_equal("/19" in listAfterShutdown[2]['address'], True) + + # Clear ban lists + self.nodes[1].clearbanned() + connect_nodes_bi(self.nodes, 0, 1) + + self.log.info("Test disconnectnode RPCs") + + self.log.info("disconnectnode: fail to disconnect when calling with address and nodeid") + address1 = self.nodes[0].getpeerinfo()[0]['addr'] + node1 = self.nodes[0].getpeerinfo()[0]['addr'] + assert_raises_rpc_error(-32602, "Only one of address and nodeid should be provided.", self.nodes[0].disconnectnode, address=address1, nodeid=node1) + + self.log.info("disconnectnode: fail to disconnect when calling with junk address") + assert_raises_rpc_error(-29, "Node not found in connected nodes", self.nodes[0].disconnectnode, address="221B Baker Street") + + self.log.info("disconnectnode: successfully disconnect node by address") + address1 = self.nodes[0].getpeerinfo()[0]['addr'] + self.nodes[0].disconnectnode(address=address1) + wait_until(lambda: len(self.nodes[0].getpeerinfo()) == 1, timeout=10) + assert not [node for node in self.nodes[0].getpeerinfo() if node['addr'] == address1] + + self.log.info("disconnectnode: successfully reconnect node") + connect_nodes_bi(self.nodes, 0, 1) # reconnect the node + assert_equal(len(self.nodes[0].getpeerinfo()), 2) + assert [node for node in self.nodes[0].getpeerinfo() if node['addr'] == address1] + + self.log.info("disconnectnode: successfully disconnect node by node id") + id1 = self.nodes[0].getpeerinfo()[0]['id'] + self.nodes[0].disconnectnode(nodeid=id1) + wait_until(lambda: len(self.nodes[0].getpeerinfo()) == 1, timeout=10) + assert not [node for node in self.nodes[0].getpeerinfo() if node['id'] == id1] + +if __name__ == '__main__': + DisconnectBanTest().main() diff --git a/test/functional/p2p_feefilter.py b/test/functional/p2p_feefilter.py new file mode 100755 index 0000000000..47d9c55160 --- /dev/null +++ b/test/functional/p2p_feefilter.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +# Copyright (c) 2016-2017 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test processing of feefilter messages.""" + +from test_framework.mininode import * +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import * +import time + + +def hashToHex(hash): + return format(hash, '064x') + +# Wait up to 60 secs to see if the testnode has received all the expected invs +def allInvsMatch(invsExpected, testnode): + for x in range(60): + with mininode_lock: + if (sorted(invsExpected) == sorted(testnode.txinvs)): + return True + time.sleep(1) + return False + +class TestNode(P2PInterface): + def __init__(self): + super().__init__() + self.txinvs = [] + + def on_inv(self, message): + for i in message.inv: + if (i.type == 1): + self.txinvs.append(hashToHex(i.hash)) + + def clear_invs(self): + with mininode_lock: + self.txinvs = [] + +class FeeFilterTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 2 + + def run_test(self): + node1 = self.nodes[1] + node0 = self.nodes[0] + # Get out of IBD + node1.generate(1) + sync_blocks(self.nodes) + + # Setup the p2p connections and start up the network thread. + self.nodes[0].add_p2p_connection(TestNode()) + network_thread_start() + self.nodes[0].p2p.wait_for_verack() + + # Test that invs are received for all txs at feerate of 20 sat/byte + node1.settxfee(Decimal("0.00020000")) + txids = [node1.sendtoaddress(node1.getnewaddress(), 1) for x in range(3)] + assert(allInvsMatch(txids, self.nodes[0].p2p)) + self.nodes[0].p2p.clear_invs() + + # Set a filter of 15 sat/byte + self.nodes[0].p2p.send_and_ping(msg_feefilter(15000)) + + # Test that txs are still being received (paying 20 sat/byte) + txids = [node1.sendtoaddress(node1.getnewaddress(), 1) for x in range(3)] + assert(allInvsMatch(txids, self.nodes[0].p2p)) + self.nodes[0].p2p.clear_invs() + + # Change tx fee rate to 10 sat/byte and test they are no longer received + node1.settxfee(Decimal("0.00010000")) + [node1.sendtoaddress(node1.getnewaddress(), 1) for x in range(3)] + sync_mempools(self.nodes) # must be sure node 0 has received all txs + + # Send one transaction from node0 that should be received, so that we + # we can sync the test on receipt (if node1's txs were relayed, they'd + # be received by the time this node0 tx is received). This is + # unfortunately reliant on the current relay behavior where we batch up + # to 35 entries in an inv, which means that when this next transaction + # is eligible for relay, the prior transactions from node1 are eligible + # as well. + node0.settxfee(Decimal("0.00020000")) + txids = [node0.sendtoaddress(node0.getnewaddress(), 1)] + assert(allInvsMatch(txids, self.nodes[0].p2p)) + self.nodes[0].p2p.clear_invs() + + # Remove fee filter and check that txs are received again + self.nodes[0].p2p.send_and_ping(msg_feefilter(0)) + txids = [node1.sendtoaddress(node1.getnewaddress(), 1) for x in range(3)] + assert(allInvsMatch(txids, self.nodes[0].p2p)) + self.nodes[0].p2p.clear_invs() + +if __name__ == '__main__': + FeeFilterTest().main() diff --git a/test/functional/p2p_fingerprint.py b/test/functional/p2p_fingerprint.py new file mode 100755 index 0000000000..93ef73e25e --- /dev/null +++ b/test/functional/p2p_fingerprint.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +# Copyright (c) 2017 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test various fingerprinting protections. + +If an stale block more than a month old or its header are requested by a peer, +the node should pretend that it does not have it to avoid fingerprinting. +""" + +import time + +from test_framework.blocktools import (create_block, create_coinbase) +from test_framework.mininode import ( + CInv, + P2PInterface, + msg_headers, + msg_block, + msg_getdata, + msg_getheaders, + network_thread_start, + wait_until, +) +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, +) + +class P2PFingerprintTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 1 + + # Build a chain of blocks on top of given one + def build_chain(self, nblocks, prev_hash, prev_height, prev_median_time): + blocks = [] + for _ in range(nblocks): + coinbase = create_coinbase(prev_height + 1) + block_time = prev_median_time + 1 + block = create_block(int(prev_hash, 16), coinbase, block_time) + block.solve() + + blocks.append(block) + prev_hash = block.hash + prev_height += 1 + prev_median_time = block_time + return blocks + + # Send a getdata request for a given block hash + def send_block_request(self, block_hash, node): + msg = msg_getdata() + msg.inv.append(CInv(2, block_hash)) # 2 == "Block" + node.send_message(msg) + + # Send a getheaders request for a given single block hash + def send_header_request(self, block_hash, node): + msg = msg_getheaders() + msg.hashstop = block_hash + node.send_message(msg) + + # Check whether last block received from node has a given hash + def last_block_equals(self, expected_hash, node): + block_msg = node.last_message.get("block") + return block_msg and block_msg.block.rehash() == expected_hash + + # Check whether last block header received from node has a given hash + def last_header_equals(self, expected_hash, node): + headers_msg = node.last_message.get("headers") + return (headers_msg and + headers_msg.headers and + headers_msg.headers[0].rehash() == expected_hash) + + # Checks that stale blocks timestamped more than a month ago are not served + # by the node while recent stale blocks and old active chain blocks are. + # This does not currently test that stale blocks timestamped within the + # last month but that have over a month's worth of work are also withheld. + def run_test(self): + node0 = self.nodes[0].add_p2p_connection(P2PInterface()) + + network_thread_start() + node0.wait_for_verack() + + # Set node time to 60 days ago + self.nodes[0].setmocktime(int(time.time()) - 60 * 24 * 60 * 60) + + # Generating a chain of 10 blocks + block_hashes = self.nodes[0].generate(nblocks=10) + + # Create longer chain starting 2 blocks before current tip + height = len(block_hashes) - 2 + block_hash = block_hashes[height - 1] + block_time = self.nodes[0].getblockheader(block_hash)["mediantime"] + 1 + new_blocks = self.build_chain(5, block_hash, height, block_time) + + # Force reorg to a longer chain + node0.send_message(msg_headers(new_blocks)) + node0.wait_for_getdata() + for block in new_blocks: + node0.send_and_ping(msg_block(block)) + + # Check that reorg succeeded + assert_equal(self.nodes[0].getblockcount(), 13) + + stale_hash = int(block_hashes[-1], 16) + + # Check that getdata request for stale block succeeds + self.send_block_request(stale_hash, node0) + test_function = lambda: self.last_block_equals(stale_hash, node0) + wait_until(test_function, timeout=3) + + # Check that getheader request for stale block header succeeds + self.send_header_request(stale_hash, node0) + test_function = lambda: self.last_header_equals(stale_hash, node0) + wait_until(test_function, timeout=3) + + # Longest chain is extended so stale is much older than chain tip + self.nodes[0].setmocktime(0) + tip = self.nodes[0].generate(nblocks=1)[0] + assert_equal(self.nodes[0].getblockcount(), 14) + + # Send getdata & getheaders to refresh last received getheader message + block_hash = int(tip, 16) + self.send_block_request(block_hash, node0) + self.send_header_request(block_hash, node0) + node0.sync_with_ping() + + # Request for very old stale block should now fail + self.send_block_request(stale_hash, node0) + time.sleep(3) + assert not self.last_block_equals(stale_hash, node0) + + # Request for very old stale block header should now fail + self.send_header_request(stale_hash, node0) + time.sleep(3) + assert not self.last_header_equals(stale_hash, node0) + + # Verify we can fetch very old blocks and headers on the active chain + block_hash = int(block_hashes[2], 16) + self.send_block_request(block_hash, node0) + self.send_header_request(block_hash, node0) + node0.sync_with_ping() + + self.send_block_request(block_hash, node0) + test_function = lambda: self.last_block_equals(block_hash, node0) + wait_until(test_function, timeout=3) + + self.send_header_request(block_hash, node0) + test_function = lambda: self.last_header_equals(block_hash, node0) + wait_until(test_function, timeout=3) + +if __name__ == '__main__': + P2PFingerprintTest().main() diff --git a/test/functional/p2p_invalid_block.py b/test/functional/p2p_invalid_block.py new file mode 100755 index 0000000000..edcade63c1 --- /dev/null +++ b/test/functional/p2p_invalid_block.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +# Copyright (c) 2015-2017 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test node responses to invalid blocks. + +In this test we connect to one node over p2p, and test block requests: +1) Valid blocks should be requested and become chain tip. +2) Invalid block with duplicated transaction should be re-requested. +3) Invalid block with bad coinbase value should be rejected and not +re-requested. +""" + +from test_framework.test_framework import ComparisonTestFramework +from test_framework.util import * +from test_framework.comptool import TestManager, TestInstance, RejectResult +from test_framework.blocktools import * +from test_framework.mininode import network_thread_start +import copy +import time + +# Use the ComparisonTestFramework with 1 node: only use --testbinary. +class InvalidBlockRequestTest(ComparisonTestFramework): + + ''' Can either run this test as 1 node with expected answers, or two and compare them. + Change the "outcome" variable from each TestInstance object to only do the comparison. ''' + def set_test_params(self): + self.num_nodes = 1 + self.setup_clean_chain = True + + def run_test(self): + test = TestManager(self, self.options.tmpdir) + test.add_all_connections(self.nodes) + self.tip = None + self.block_time = None + network_thread_start() + test.run() + + def get_tests(self): + if self.tip is None: + self.tip = int("0x" + self.nodes[0].getbestblockhash(), 0) + self.block_time = int(time.time())+1 + + ''' + Create a new block with an anyone-can-spend coinbase + ''' + height = 1 + block = create_block(self.tip, create_coinbase(height), self.block_time) + self.block_time += 1 + block.solve() + # Save the coinbase for later + self.block1 = block + self.tip = block.sha256 + height += 1 + yield TestInstance([[block, True]]) + + ''' + Now we need that block to mature so we can spend the coinbase. + ''' + test = TestInstance(sync_every_block=False) + for i in range(100): + block = create_block(self.tip, create_coinbase(height), self.block_time) + block.solve() + self.tip = block.sha256 + self.block_time += 1 + test.blocks_and_transactions.append([block, True]) + height += 1 + yield test + + ''' + Now we use merkle-root malleability to generate an invalid block with + same blockheader. + Manufacture a block with 3 transactions (coinbase, spend of prior + coinbase, spend of that spend). Duplicate the 3rd transaction to + leave merkle root and blockheader unchanged but invalidate the block. + ''' + block2 = create_block(self.tip, create_coinbase(height), self.block_time) + self.block_time += 1 + + # b'0x51' is OP_TRUE + tx1 = create_transaction(self.block1.vtx[0], 0, b'\x51', 50 * COIN) + tx2 = create_transaction(tx1, 0, b'\x51', 50 * COIN) + + block2.vtx.extend([tx1, tx2]) + block2.hashMerkleRoot = block2.calc_merkle_root() + block2.rehash() + block2.solve() + orig_hash = block2.sha256 + block2_orig = copy.deepcopy(block2) + + # Mutate block 2 + block2.vtx.append(tx2) + assert_equal(block2.hashMerkleRoot, block2.calc_merkle_root()) + assert_equal(orig_hash, block2.rehash()) + assert(block2_orig.vtx != block2.vtx) + + self.tip = block2.sha256 + yield TestInstance([[block2, RejectResult(16, b'bad-txns-duplicate')], [block2_orig, True]]) + height += 1 + + ''' + Make sure that a totally screwed up block is not valid. + ''' + block3 = create_block(self.tip, create_coinbase(height), self.block_time) + self.block_time += 1 + block3.vtx[0].vout[0].nValue = 100 * COIN # Too high! + block3.vtx[0].sha256=None + block3.vtx[0].calc_sha256() + block3.hashMerkleRoot = block3.calc_merkle_root() + block3.rehash() + block3.solve() + + yield TestInstance([[block3, RejectResult(16, b'bad-cb-amount')]]) + + +if __name__ == '__main__': + InvalidBlockRequestTest().main() diff --git a/test/functional/p2p_invalid_tx.py b/test/functional/p2p_invalid_tx.py new file mode 100755 index 0000000000..9c1100e070 --- /dev/null +++ b/test/functional/p2p_invalid_tx.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +# Copyright (c) 2015-2017 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test node responses to invalid transactions. + +In this test we connect to one node over p2p, and test tx requests. +""" + +from test_framework.test_framework import ComparisonTestFramework +from test_framework.comptool import TestManager, TestInstance, RejectResult +from test_framework.blocktools import * +import time + + + +# Use the ComparisonTestFramework with 1 node: only use --testbinary. +class InvalidTxRequestTest(ComparisonTestFramework): + + ''' Can either run this test as 1 node with expected answers, or two and compare them. + Change the "outcome" variable from each TestInstance object to only do the comparison. ''' + def set_test_params(self): + self.num_nodes = 1 + self.setup_clean_chain = True + + def run_test(self): + test = TestManager(self, self.options.tmpdir) + test.add_all_connections(self.nodes) + self.tip = None + self.block_time = None + network_thread_start() + test.run() + + def get_tests(self): + if self.tip is None: + self.tip = int("0x" + self.nodes[0].getbestblockhash(), 0) + self.block_time = int(time.time())+1 + + ''' + Create a new block with an anyone-can-spend coinbase + ''' + height = 1 + block = create_block(self.tip, create_coinbase(height), self.block_time) + self.block_time += 1 + block.solve() + # Save the coinbase for later + self.block1 = block + self.tip = block.sha256 + height += 1 + yield TestInstance([[block, True]]) + + ''' + Now we need that block to mature so we can spend the coinbase. + ''' + test = TestInstance(sync_every_block=False) + for i in range(100): + block = create_block(self.tip, create_coinbase(height), self.block_time) + block.solve() + self.tip = block.sha256 + self.block_time += 1 + test.blocks_and_transactions.append([block, True]) + height += 1 + yield test + + # b'\x64' is OP_NOTIF + # Transaction will be rejected with code 16 (REJECT_INVALID) + tx1 = create_transaction(self.block1.vtx[0], 0, b'\x64', 50 * COIN - 12000) + yield TestInstance([[tx1, RejectResult(16, b'mandatory-script-verify-flag-failed')]]) + + # TODO: test further transactions... + +if __name__ == '__main__': + InvalidTxRequestTest().main() diff --git a/test/functional/p2p_leak.py b/test/functional/p2p_leak.py new file mode 100755 index 0000000000..ce4e6e9144 --- /dev/null +++ b/test/functional/p2p_leak.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +# Copyright (c) 2017 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test message sending before handshake completion. + +A node should never send anything other than VERSION/VERACK/REJECT until it's +received a VERACK. + +This test connects to a node and sends it a few messages, trying to intice it +into sending us something it shouldn't. + +Also test that nodes that send unsupported service bits to bitcoind are disconnected +and don't receive a VERACK. Unsupported service bits are currently 1 << 5 and +1 << 7 (until August 1st 2018).""" + +from test_framework.mininode import * +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import * + +banscore = 10 + +class CLazyNode(P2PInterface): + def __init__(self): + super().__init__() + self.unexpected_msg = False + self.ever_connected = False + + def bad_message(self, message): + self.unexpected_msg = True + self.log.info("should not have received message: %s" % message.command) + + def on_open(self): + self.ever_connected = True + + def on_version(self, message): self.bad_message(message) + def on_verack(self, message): self.bad_message(message) + def on_reject(self, message): self.bad_message(message) + def on_inv(self, message): self.bad_message(message) + def on_addr(self, message): self.bad_message(message) + def on_getdata(self, message): self.bad_message(message) + def on_getblocks(self, message): self.bad_message(message) + def on_tx(self, message): self.bad_message(message) + def on_block(self, message): self.bad_message(message) + def on_getaddr(self, message): self.bad_message(message) + def on_headers(self, message): self.bad_message(message) + def on_getheaders(self, message): self.bad_message(message) + def on_ping(self, message): self.bad_message(message) + def on_mempool(self, message): self.bad_message(message) + def on_pong(self, message): self.bad_message(message) + def on_feefilter(self, message): self.bad_message(message) + def on_sendheaders(self, message): self.bad_message(message) + def on_sendcmpct(self, message): self.bad_message(message) + def on_cmpctblock(self, message): self.bad_message(message) + def on_getblocktxn(self, message): self.bad_message(message) + def on_blocktxn(self, message): self.bad_message(message) + +# Node that never sends a version. We'll use this to send a bunch of messages +# anyway, and eventually get disconnected. +class CNodeNoVersionBan(CLazyNode): + # send a bunch of veracks without sending a message. This should get us disconnected. + # NOTE: implementation-specific check here. Remove if bitcoind ban behavior changes + def on_open(self): + super().on_open() + for i in range(banscore): + self.send_message(msg_verack()) + + def on_reject(self, message): pass + +# Node that never sends a version. This one just sits idle and hopes to receive +# any message (it shouldn't!) +class CNodeNoVersionIdle(CLazyNode): + def __init__(self): + super().__init__() + +# Node that sends a version but not a verack. +class CNodeNoVerackIdle(CLazyNode): + def __init__(self): + self.version_received = False + super().__init__() + + def on_reject(self, message): pass + def on_verack(self, message): pass + # When version is received, don't reply with a verack. Instead, see if the + # node will give us a message that it shouldn't. This is not an exhaustive + # list! + def on_version(self, message): + self.version_received = True + self.send_message(msg_ping()) + self.send_message(msg_getaddr()) + +class P2PLeakTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + self.extra_args = [['-banscore='+str(banscore)]] + + def run_test(self): + self.nodes[0].setmocktime(1501545600) # August 1st 2017 + + no_version_bannode = self.nodes[0].add_p2p_connection(CNodeNoVersionBan(), send_version=False) + no_version_idlenode = self.nodes[0].add_p2p_connection(CNodeNoVersionIdle(), send_version=False) + no_verack_idlenode = self.nodes[0].add_p2p_connection(CNodeNoVerackIdle()) + unsupported_service_bit5_node = self.nodes[0].add_p2p_connection(CLazyNode(), services=NODE_NETWORK|NODE_UNSUPPORTED_SERVICE_BIT_5) + unsupported_service_bit7_node = self.nodes[0].add_p2p_connection(CLazyNode(), services=NODE_NETWORK|NODE_UNSUPPORTED_SERVICE_BIT_7) + + network_thread_start() + + wait_until(lambda: no_version_bannode.ever_connected, timeout=10, lock=mininode_lock) + wait_until(lambda: no_version_idlenode.ever_connected, timeout=10, lock=mininode_lock) + wait_until(lambda: no_verack_idlenode.version_received, timeout=10, lock=mininode_lock) + wait_until(lambda: unsupported_service_bit5_node.ever_connected, timeout=10, lock=mininode_lock) + wait_until(lambda: unsupported_service_bit7_node.ever_connected, timeout=10, lock=mininode_lock) + + # Mine a block and make sure that it's not sent to the connected nodes + self.nodes[0].generate(1) + + #Give the node enough time to possibly leak out a message + time.sleep(5) + + #This node should have been banned + assert no_version_bannode.state != "connected" + + # These nodes should have been disconnected + assert unsupported_service_bit5_node.state != "connected" + assert unsupported_service_bit7_node.state != "connected" + + self.nodes[0].disconnect_p2ps() + + # Wait until all connections are closed and the network thread has terminated + wait_until(lambda: len(self.nodes[0].getpeerinfo()) == 0) + network_thread_join() + + # Make sure no unexpected messages came in + assert(no_version_bannode.unexpected_msg == False) + assert(no_version_idlenode.unexpected_msg == False) + assert(no_verack_idlenode.unexpected_msg == False) + assert not unsupported_service_bit5_node.unexpected_msg + assert not unsupported_service_bit7_node.unexpected_msg + + self.log.info("Service bits 5 and 7 are allowed after August 1st 2018") + self.nodes[0].setmocktime(1533168000) # August 2nd 2018 + + allowed_service_bit5_node = self.nodes[0].add_p2p_connection(P2PInterface(), services=NODE_NETWORK|NODE_UNSUPPORTED_SERVICE_BIT_5) + allowed_service_bit7_node = self.nodes[0].add_p2p_connection(P2PInterface(), services=NODE_NETWORK|NODE_UNSUPPORTED_SERVICE_BIT_7) + + # Network thread stopped when all previous P2PInterfaces disconnected. Restart it + network_thread_start() + + wait_until(lambda: allowed_service_bit5_node.message_count["verack"], lock=mininode_lock) + wait_until(lambda: allowed_service_bit7_node.message_count["verack"], lock=mininode_lock) + +if __name__ == '__main__': + P2PLeakTest().main() diff --git a/test/functional/p2p_mempool.py b/test/functional/p2p_mempool.py new file mode 100755 index 0000000000..485a8af3d0 --- /dev/null +++ b/test/functional/p2p_mempool.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +# Copyright (c) 2015-2017 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test p2p mempool message. + +Test that nodes are disconnected if they send mempool messages when bloom +filters are not enabled. +""" + +from test_framework.mininode import * +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import * + +class P2PMempoolTests(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 1 + self.extra_args = [["-peerbloomfilters=0"]] + + def run_test(self): + # Add a p2p connection + self.nodes[0].add_p2p_connection(P2PInterface()) + network_thread_start() + self.nodes[0].p2p.wait_for_verack() + + #request mempool + self.nodes[0].p2p.send_message(msg_mempool()) + self.nodes[0].p2p.wait_for_disconnect() + + #mininode must be disconnected at this point + assert_equal(len(self.nodes[0].getpeerinfo()), 0) + +if __name__ == '__main__': + P2PMempoolTests().main() diff --git a/test/functional/p2p_node_network_limited.py b/test/functional/p2p_node_network_limited.py new file mode 100755 index 0000000000..70415e0168 --- /dev/null +++ b/test/functional/p2p_node_network_limited.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +# Copyright (c) 2017 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Tests NODE_NETWORK_LIMITED. + +Tests that a node configured with -prune=550 signals NODE_NETWORK_LIMITED correctly +and that it responds to getdata requests for blocks correctly: + - send a block within 288 + 2 of the tip + - disconnect peers who request blocks older than that.""" +from test_framework.messages import CInv, msg_getdata +from test_framework.mininode import NODE_BLOOM, NODE_NETWORK_LIMITED, NODE_WITNESS, NetworkThread, P2PInterface +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal + +class P2PIgnoreInv(P2PInterface): + def on_inv(self, message): + # The node will send us invs for other blocks. Ignore them. + pass + + def send_getdata_for_block(self, blockhash): + getdata_request = msg_getdata() + getdata_request.inv.append(CInv(2, int(blockhash, 16))) + self.send_message(getdata_request) + +class NodeNetworkLimitedTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 1 + self.extra_args = [['-prune=550']] + + def run_test(self): + node = self.nodes[0].add_p2p_connection(P2PIgnoreInv()) + NetworkThread().start() + node.wait_for_verack() + + expected_services = NODE_BLOOM | NODE_WITNESS | NODE_NETWORK_LIMITED + + self.log.info("Check that node has signalled expected services.") + assert_equal(node.nServices, expected_services) + + self.log.info("Check that the localservices is as expected.") + assert_equal(int(self.nodes[0].getnetworkinfo()['localservices'], 16), expected_services) + + self.log.info("Mine enough blocks to reach the NODE_NETWORK_LIMITED range.") + blocks = self.nodes[0].generate(292) + + self.log.info("Make sure we can max retrive block at tip-288.") + node.send_getdata_for_block(blocks[1]) # last block in valid range + node.wait_for_block(int(blocks[1], 16), timeout=3) + + self.log.info("Requesting block at height 2 (tip-289) must fail (ignored).") + node.send_getdata_for_block(blocks[0]) # first block outside of the 288+2 limit + node.wait_for_disconnect(5) + +if __name__ == '__main__': + NodeNetworkLimitedTest().main() diff --git a/test/functional/p2p_segwit.py b/test/functional/p2p_segwit.py new file mode 100755 index 0000000000..20e4805df0 --- /dev/null +++ b/test/functional/p2p_segwit.py @@ -0,0 +1,1952 @@ +#!/usr/bin/env python3 +# Copyright (c) 2016-2017 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test segwit transactions and blocks on P2P network.""" + +from test_framework.mininode import * +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import * +from test_framework.script import * +from test_framework.blocktools import create_block, create_coinbase, add_witness_commitment, get_witness_script, WITNESS_COMMITMENT_HEADER +from test_framework.key import CECKey, CPubKey +import time +import random +from binascii import hexlify + +# The versionbit bit used to signal activation of SegWit +VB_WITNESS_BIT = 1 +VB_PERIOD = 144 +VB_TOP_BITS = 0x20000000 + +MAX_SIGOP_COST = 80000 + + +# Calculate the virtual size of a witness block: +# (base + witness/4) +def get_virtual_size(witness_block): + base_size = len(witness_block.serialize(with_witness=False)) + total_size = len(witness_block.serialize(with_witness=True)) + # the "+3" is so we round up + vsize = int((3*base_size + total_size + 3)/4) + return vsize + +def test_transaction_acceptance(rpc, p2p, tx, with_witness, accepted, reason=None): + """Send a transaction to the node and check that it's accepted to the mempool + + - Submit the transaction over the p2p interface + - use the getrawmempool rpc to check for acceptance.""" + tx_message = msg_tx(tx) + if with_witness: + tx_message = msg_witness_tx(tx) + p2p.send_message(tx_message) + p2p.sync_with_ping() + assert_equal(tx.hash in rpc.getrawmempool(), accepted) + if (reason != None and not accepted): + # Check the rejection reason as well. + with mininode_lock: + assert_equal(p2p.last_message["reject"].reason, reason) + +def test_witness_block(rpc, p2p, block, accepted, with_witness=True): + """Send a block to the node and check that it's accepted + + - Submit the block over the p2p interface + - use the getbestblockhash rpc to check for acceptance.""" + if with_witness: + p2p.send_message(msg_witness_block(block)) + else: + p2p.send_message(msg_block(block)) + p2p.sync_with_ping() + assert_equal(rpc.getbestblockhash() == block.hash, accepted) + +class TestNode(P2PInterface): + def __init__(self): + super().__init__() + self.getdataset = set() + + def on_getdata(self, message): + for inv in message.inv: + self.getdataset.add(inv.hash) + + def announce_tx_and_wait_for_getdata(self, tx, timeout=60): + with mininode_lock: + self.last_message.pop("getdata", None) + self.send_message(msg_inv(inv=[CInv(1, tx.sha256)])) + self.wait_for_getdata(timeout) + + def announce_block_and_wait_for_getdata(self, block, use_header, timeout=60): + with mininode_lock: + self.last_message.pop("getdata", None) + self.last_message.pop("getheaders", None) + msg = msg_headers() + msg.headers = [ CBlockHeader(block) ] + if use_header: + self.send_message(msg) + else: + self.send_message(msg_inv(inv=[CInv(2, block.sha256)])) + self.wait_for_getheaders() + self.send_message(msg) + self.wait_for_getdata() + + def request_block(self, blockhash, inv_type, timeout=60): + with mininode_lock: + self.last_message.pop("block", None) + self.send_message(msg_getdata(inv=[CInv(inv_type, blockhash)])) + self.wait_for_block(blockhash, timeout) + return self.last_message["block"].block + +# Used to keep track of anyone-can-spend outputs that we can use in the tests +class UTXO(): + def __init__(self, sha256, n, nValue): + self.sha256 = sha256 + self.n = n + self.nValue = nValue + +# Helper for getting the script associated with a P2PKH +def GetP2PKHScript(pubkeyhash): + return CScript([CScriptOp(OP_DUP), CScriptOp(OP_HASH160), pubkeyhash, CScriptOp(OP_EQUALVERIFY), CScriptOp(OP_CHECKSIG)]) + +# Add signature for a P2PK witness program. +def sign_P2PK_witness_input(script, txTo, inIdx, hashtype, value, key): + tx_hash = SegwitVersion1SignatureHash(script, txTo, inIdx, hashtype, value) + signature = key.sign(tx_hash) + chr(hashtype).encode('latin-1') + txTo.wit.vtxinwit[inIdx].scriptWitness.stack = [signature, script] + txTo.rehash() + + +class SegWitTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 3 + # This test tests SegWit both pre and post-activation, so use the normal BIP9 activation. + self.extra_args = [["-whitelist=127.0.0.1", "-vbparams=segwit:0:999999999999"], ["-whitelist=127.0.0.1", "-acceptnonstdtxn=0", "-vbparams=segwit:0:999999999999"], ["-whitelist=127.0.0.1", "-vbparams=segwit:0:0"]] + + def setup_network(self): + self.setup_nodes() + connect_nodes(self.nodes[0], 1) + connect_nodes(self.nodes[0], 2) + self.sync_all() + + ''' Helpers ''' + # Build a block on top of node0's tip. + def build_next_block(self, nVersion=4): + tip = self.nodes[0].getbestblockhash() + height = self.nodes[0].getblockcount() + 1 + block_time = self.nodes[0].getblockheader(tip)["mediantime"] + 1 + block = create_block(int(tip, 16), create_coinbase(height), block_time) + block.nVersion = nVersion + block.rehash() + return block + + # Adds list of transactions to block, adds witness commitment, then solves. + def update_witness_block_with_transactions(self, block, tx_list, nonce=0): + block.vtx.extend(tx_list) + add_witness_commitment(block, nonce) + block.solve() + return + + ''' Individual tests ''' + def test_witness_services(self): + self.log.info("Verifying NODE_WITNESS service bit") + assert((self.test_node.nServices & NODE_WITNESS) != 0) + + + # See if sending a regular transaction works, and create a utxo + # to use in later tests. + def test_non_witness_transaction(self): + # Mine a block with an anyone-can-spend coinbase, + # let it mature, then try to spend it. + self.log.info("Testing non-witness transaction") + block = self.build_next_block(nVersion=1) + block.solve() + self.test_node.send_message(msg_block(block)) + self.test_node.sync_with_ping() # make sure the block was processed + txid = block.vtx[0].sha256 + + self.nodes[0].generate(99) # let the block mature + + # Create a transaction that spends the coinbase + tx = CTransaction() + tx.vin.append(CTxIn(COutPoint(txid, 0), b"")) + tx.vout.append(CTxOut(49*100000000, CScript([OP_TRUE]))) + tx.calc_sha256() + + # Check that serializing it with or without witness is the same + # This is a sanity check of our testing framework. + assert_equal(msg_tx(tx).serialize(), msg_witness_tx(tx).serialize()) + + self.test_node.send_message(msg_witness_tx(tx)) + self.test_node.sync_with_ping() # make sure the tx was processed + assert(tx.hash in self.nodes[0].getrawmempool()) + # Save this transaction for later + self.utxo.append(UTXO(tx.sha256, 0, 49*100000000)) + self.nodes[0].generate(1) + + + # Verify that blocks with witnesses are rejected before activation. + def test_unnecessary_witness_before_segwit_activation(self): + self.log.info("Testing behavior of unnecessary witnesses") + # For now, rely on earlier tests to have created at least one utxo for + # us to use + assert(len(self.utxo) > 0) + assert(get_bip9_status(self.nodes[0], 'segwit')['status'] != 'active') + + tx = CTransaction() + tx.vin.append(CTxIn(COutPoint(self.utxo[0].sha256, self.utxo[0].n), b"")) + tx.vout.append(CTxOut(self.utxo[0].nValue-1000, CScript([OP_TRUE]))) + tx.wit.vtxinwit.append(CTxInWitness()) + tx.wit.vtxinwit[0].scriptWitness.stack = [CScript([CScriptNum(1)])] + + # Verify the hash with witness differs from the txid + # (otherwise our testing framework must be broken!) + tx.rehash() + assert(tx.sha256 != tx.calc_sha256(with_witness=True)) + + # Construct a segwit-signaling block that includes the transaction. + block = self.build_next_block(nVersion=(VB_TOP_BITS|(1 << VB_WITNESS_BIT))) + self.update_witness_block_with_transactions(block, [tx]) + # Sending witness data before activation is not allowed (anti-spam + # rule). + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=False) + # TODO: fix synchronization so we can test reject reason + # Right now, bitcoind delays sending reject messages for blocks + # until the future, making synchronization here difficult. + #assert_equal(self.test_node.last_message["reject"].reason, "unexpected-witness") + + # But it should not be permanently marked bad... + # Resend without witness information. + self.test_node.send_message(msg_block(block)) + self.test_node.sync_with_ping() + assert_equal(self.nodes[0].getbestblockhash(), block.hash) + + sync_blocks(self.nodes) + + # Create a p2sh output -- this is so we can pass the standardness + # rules (an anyone-can-spend OP_TRUE would be rejected, if not wrapped + # in P2SH). + p2sh_program = CScript([OP_TRUE]) + p2sh_pubkey = hash160(p2sh_program) + scriptPubKey = CScript([OP_HASH160, p2sh_pubkey, OP_EQUAL]) + + # Now check that unnecessary witnesses can't be used to blind a node + # to a transaction, eg by violating standardness checks. + tx2 = CTransaction() + tx2.vin.append(CTxIn(COutPoint(tx.sha256, 0), b"")) + tx2.vout.append(CTxOut(tx.vout[0].nValue-1000, scriptPubKey)) + tx2.rehash() + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx2, False, True) + self.nodes[0].generate(1) + sync_blocks(self.nodes) + + # We'll add an unnecessary witness to this transaction that would cause + # it to be non-standard, to test that violating policy with a witness before + # segwit activation doesn't blind a node to a transaction. Transactions + # rejected for having a witness before segwit activation shouldn't be added + # to the rejection cache. + tx3 = CTransaction() + tx3.vin.append(CTxIn(COutPoint(tx2.sha256, 0), CScript([p2sh_program]))) + tx3.vout.append(CTxOut(tx2.vout[0].nValue-1000, scriptPubKey)) + tx3.wit.vtxinwit.append(CTxInWitness()) + tx3.wit.vtxinwit[0].scriptWitness.stack = [b'a'*400000] + tx3.rehash() + # Note that this should be rejected for the premature witness reason, + # rather than a policy check, since segwit hasn't activated yet. + test_transaction_acceptance(self.nodes[1].rpc, self.std_node, tx3, True, False, b'no-witness-yet') + + # If we send without witness, it should be accepted. + test_transaction_acceptance(self.nodes[1].rpc, self.std_node, tx3, False, True) + + # Now create a new anyone-can-spend utxo for the next test. + tx4 = CTransaction() + tx4.vin.append(CTxIn(COutPoint(tx3.sha256, 0), CScript([p2sh_program]))) + tx4.vout.append(CTxOut(tx3.vout[0].nValue-1000, CScript([OP_TRUE]))) + tx4.rehash() + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx3, False, True) + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx4, False, True) + + self.nodes[0].generate(1) + sync_blocks(self.nodes) + + # Update our utxo list; we spent the first entry. + self.utxo.pop(0) + self.utxo.append(UTXO(tx4.sha256, 0, tx4.vout[0].nValue)) + + + # Mine enough blocks for segwit's vb state to be 'started'. + def advance_to_segwit_started(self): + height = self.nodes[0].getblockcount() + # Will need to rewrite the tests here if we are past the first period + assert(height < VB_PERIOD - 1) + # Genesis block is 'defined'. + assert_equal(get_bip9_status(self.nodes[0], 'segwit')['status'], 'defined') + # Advance to end of period, status should now be 'started' + self.nodes[0].generate(VB_PERIOD-height-1) + assert_equal(get_bip9_status(self.nodes[0], 'segwit')['status'], 'started') + + # Mine enough blocks to lock in segwit, but don't activate. + # TODO: we could verify that lockin only happens at the right threshold of + # signalling blocks, rather than just at the right period boundary. + def advance_to_segwit_lockin(self): + height = self.nodes[0].getblockcount() + assert_equal(get_bip9_status(self.nodes[0], 'segwit')['status'], 'started') + # Advance to end of period, and verify lock-in happens at the end + self.nodes[0].generate(VB_PERIOD-1) + height = self.nodes[0].getblockcount() + assert((height % VB_PERIOD) == VB_PERIOD - 2) + assert_equal(get_bip9_status(self.nodes[0], 'segwit')['status'], 'started') + self.nodes[0].generate(1) + assert_equal(get_bip9_status(self.nodes[0], 'segwit')['status'], 'locked_in') + + + # Mine enough blocks to activate segwit. + # TODO: we could verify that activation only happens at the right threshold + # of signalling blocks, rather than just at the right period boundary. + def advance_to_segwit_active(self): + assert_equal(get_bip9_status(self.nodes[0], 'segwit')['status'], 'locked_in') + height = self.nodes[0].getblockcount() + self.nodes[0].generate(VB_PERIOD - (height%VB_PERIOD) - 2) + assert_equal(get_bip9_status(self.nodes[0], 'segwit')['status'], 'locked_in') + self.nodes[0].generate(1) + assert_equal(get_bip9_status(self.nodes[0], 'segwit')['status'], 'active') + + + # This test can only be run after segwit has activated + def test_witness_commitments(self): + self.log.info("Testing witness commitments") + + # First try a correct witness commitment. + block = self.build_next_block() + add_witness_commitment(block) + block.solve() + + # Test the test -- witness serialization should be different + assert(msg_witness_block(block).serialize() != msg_block(block).serialize()) + + # This empty block should be valid. + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) + + # Try to tweak the nonce + block_2 = self.build_next_block() + add_witness_commitment(block_2, nonce=28) + block_2.solve() + + # The commitment should have changed! + assert(block_2.vtx[0].vout[-1] != block.vtx[0].vout[-1]) + + # This should also be valid. + test_witness_block(self.nodes[0].rpc, self.test_node, block_2, accepted=True) + + # Now test commitments with actual transactions + assert (len(self.utxo) > 0) + tx = CTransaction() + tx.vin.append(CTxIn(COutPoint(self.utxo[0].sha256, self.utxo[0].n), b"")) + + # Let's construct a witness program + witness_program = CScript([OP_TRUE]) + witness_hash = sha256(witness_program) + scriptPubKey = CScript([OP_0, witness_hash]) + tx.vout.append(CTxOut(self.utxo[0].nValue-1000, scriptPubKey)) + tx.rehash() + + # tx2 will spend tx1, and send back to a regular anyone-can-spend address + tx2 = CTransaction() + tx2.vin.append(CTxIn(COutPoint(tx.sha256, 0), b"")) + tx2.vout.append(CTxOut(tx.vout[0].nValue-1000, witness_program)) + tx2.wit.vtxinwit.append(CTxInWitness()) + tx2.wit.vtxinwit[0].scriptWitness.stack = [witness_program] + tx2.rehash() + + block_3 = self.build_next_block() + self.update_witness_block_with_transactions(block_3, [tx, tx2], nonce=1) + # Add an extra OP_RETURN output that matches the witness commitment template, + # even though it has extra data after the incorrect commitment. + # This block should fail. + block_3.vtx[0].vout.append(CTxOut(0, CScript([OP_RETURN, WITNESS_COMMITMENT_HEADER + ser_uint256(2), 10]))) + block_3.vtx[0].rehash() + block_3.hashMerkleRoot = block_3.calc_merkle_root() + block_3.rehash() + block_3.solve() + + test_witness_block(self.nodes[0].rpc, self.test_node, block_3, accepted=False) + + # Add a different commitment with different nonce, but in the + # right location, and with some funds burned(!). + # This should succeed (nValue shouldn't affect finding the + # witness commitment). + add_witness_commitment(block_3, nonce=0) + block_3.vtx[0].vout[0].nValue -= 1 + block_3.vtx[0].vout[-1].nValue += 1 + block_3.vtx[0].rehash() + block_3.hashMerkleRoot = block_3.calc_merkle_root() + block_3.rehash() + assert(len(block_3.vtx[0].vout) == 4) # 3 OP_returns + block_3.solve() + test_witness_block(self.nodes[0].rpc, self.test_node, block_3, accepted=True) + + # Finally test that a block with no witness transactions can + # omit the commitment. + block_4 = self.build_next_block() + tx3 = CTransaction() + tx3.vin.append(CTxIn(COutPoint(tx2.sha256, 0), b"")) + tx3.vout.append(CTxOut(tx.vout[0].nValue-1000, witness_program)) + tx3.rehash() + block_4.vtx.append(tx3) + block_4.hashMerkleRoot = block_4.calc_merkle_root() + block_4.solve() + test_witness_block(self.nodes[0].rpc, self.test_node, block_4, with_witness=False, accepted=True) + + # Update available utxo's for use in later test. + self.utxo.pop(0) + self.utxo.append(UTXO(tx3.sha256, 0, tx3.vout[0].nValue)) + + + def test_block_malleability(self): + self.log.info("Testing witness block malleability") + + # Make sure that a block that has too big a virtual size + # because of a too-large coinbase witness is not permanently + # marked bad. + block = self.build_next_block() + add_witness_commitment(block) + block.solve() + + block.vtx[0].wit.vtxinwit[0].scriptWitness.stack.append(b'a'*5000000) + assert(get_virtual_size(block) > MAX_BLOCK_BASE_SIZE) + + # We can't send over the p2p network, because this is too big to relay + # TODO: repeat this test with a block that can be relayed + self.nodes[0].submitblock(bytes_to_hex_str(block.serialize(True))) + + assert(self.nodes[0].getbestblockhash() != block.hash) + + block.vtx[0].wit.vtxinwit[0].scriptWitness.stack.pop() + assert(get_virtual_size(block) < MAX_BLOCK_BASE_SIZE) + self.nodes[0].submitblock(bytes_to_hex_str(block.serialize(True))) + + assert(self.nodes[0].getbestblockhash() == block.hash) + + # Now make sure that malleating the witness nonce doesn't + # result in a block permanently marked bad. + block = self.build_next_block() + add_witness_commitment(block) + block.solve() + + # Change the nonce -- should not cause the block to be permanently + # failed + block.vtx[0].wit.vtxinwit[0].scriptWitness.stack = [ ser_uint256(1) ] + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=False) + + # Changing the witness nonce doesn't change the block hash + block.vtx[0].wit.vtxinwit[0].scriptWitness.stack = [ ser_uint256(0) ] + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) + + + def test_witness_block_size(self): + self.log.info("Testing witness block size limit") + # TODO: Test that non-witness carrying blocks can't exceed 1MB + # Skipping this test for now; this is covered in p2p-fullblocktest.py + + # Test that witness-bearing blocks are limited at ceil(base + wit/4) <= 1MB. + block = self.build_next_block() + + assert(len(self.utxo) > 0) + + # Create a P2WSH transaction. + # The witness program will be a bunch of OP_2DROP's, followed by OP_TRUE. + # This should give us plenty of room to tweak the spending tx's + # virtual size. + NUM_DROPS = 200 # 201 max ops per script! + NUM_OUTPUTS = 50 + + witness_program = CScript([OP_2DROP]*NUM_DROPS + [OP_TRUE]) + witness_hash = uint256_from_str(sha256(witness_program)) + scriptPubKey = CScript([OP_0, ser_uint256(witness_hash)]) + + prevout = COutPoint(self.utxo[0].sha256, self.utxo[0].n) + value = self.utxo[0].nValue + + parent_tx = CTransaction() + parent_tx.vin.append(CTxIn(prevout, b"")) + child_value = int(value/NUM_OUTPUTS) + for i in range(NUM_OUTPUTS): + parent_tx.vout.append(CTxOut(child_value, scriptPubKey)) + parent_tx.vout[0].nValue -= 50000 + assert(parent_tx.vout[0].nValue > 0) + parent_tx.rehash() + + child_tx = CTransaction() + for i in range(NUM_OUTPUTS): + child_tx.vin.append(CTxIn(COutPoint(parent_tx.sha256, i), b"")) + child_tx.vout = [CTxOut(value - 100000, CScript([OP_TRUE]))] + for i in range(NUM_OUTPUTS): + child_tx.wit.vtxinwit.append(CTxInWitness()) + child_tx.wit.vtxinwit[-1].scriptWitness.stack = [b'a'*195]*(2*NUM_DROPS) + [witness_program] + child_tx.rehash() + self.update_witness_block_with_transactions(block, [parent_tx, child_tx]) + + vsize = get_virtual_size(block) + additional_bytes = (MAX_BLOCK_BASE_SIZE - vsize)*4 + i = 0 + while additional_bytes > 0: + # Add some more bytes to each input until we hit MAX_BLOCK_BASE_SIZE+1 + extra_bytes = min(additional_bytes+1, 55) + block.vtx[-1].wit.vtxinwit[int(i/(2*NUM_DROPS))].scriptWitness.stack[i%(2*NUM_DROPS)] = b'a'*(195+extra_bytes) + additional_bytes -= extra_bytes + i += 1 + + block.vtx[0].vout.pop() # Remove old commitment + add_witness_commitment(block) + block.solve() + vsize = get_virtual_size(block) + assert_equal(vsize, MAX_BLOCK_BASE_SIZE + 1) + # Make sure that our test case would exceed the old max-network-message + # limit + assert(len(block.serialize(True)) > 2*1024*1024) + + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=False) + + # Now resize the second transaction to make the block fit. + cur_length = len(block.vtx[-1].wit.vtxinwit[0].scriptWitness.stack[0]) + block.vtx[-1].wit.vtxinwit[0].scriptWitness.stack[0] = b'a'*(cur_length-1) + block.vtx[0].vout.pop() + add_witness_commitment(block) + block.solve() + assert(get_virtual_size(block) == MAX_BLOCK_BASE_SIZE) + + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) + + # Update available utxo's + self.utxo.pop(0) + self.utxo.append(UTXO(block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue)) + + + # submitblock will try to add the nonce automatically, so that mining + # software doesn't need to worry about doing so itself. + def test_submit_block(self): + block = self.build_next_block() + + # Try using a custom nonce and then don't supply it. + # This shouldn't possibly work. + add_witness_commitment(block, nonce=1) + block.vtx[0].wit = CTxWitness() # drop the nonce + block.solve() + self.nodes[0].submitblock(bytes_to_hex_str(block.serialize(True))) + assert(self.nodes[0].getbestblockhash() != block.hash) + + # Now redo commitment with the standard nonce, but let bitcoind fill it in. + add_witness_commitment(block, nonce=0) + block.vtx[0].wit = CTxWitness() + block.solve() + self.nodes[0].submitblock(bytes_to_hex_str(block.serialize(True))) + assert_equal(self.nodes[0].getbestblockhash(), block.hash) + + # This time, add a tx with non-empty witness, but don't supply + # the commitment. + block_2 = self.build_next_block() + + add_witness_commitment(block_2) + + block_2.solve() + + # Drop commitment and nonce -- submitblock should not fill in. + block_2.vtx[0].vout.pop() + block_2.vtx[0].wit = CTxWitness() + + self.nodes[0].submitblock(bytes_to_hex_str(block_2.serialize(True))) + # Tip should not advance! + assert(self.nodes[0].getbestblockhash() != block_2.hash) + + + # Consensus tests of extra witness data in a transaction. + def test_extra_witness_data(self): + self.log.info("Testing extra witness data in tx") + + assert(len(self.utxo) > 0) + + block = self.build_next_block() + + witness_program = CScript([OP_DROP, OP_TRUE]) + witness_hash = sha256(witness_program) + scriptPubKey = CScript([OP_0, witness_hash]) + + # First try extra witness data on a tx that doesn't require a witness + tx = CTransaction() + tx.vin.append(CTxIn(COutPoint(self.utxo[0].sha256, self.utxo[0].n), b"")) + tx.vout.append(CTxOut(self.utxo[0].nValue-2000, scriptPubKey)) + tx.vout.append(CTxOut(1000, CScript([OP_TRUE]))) # non-witness output + tx.wit.vtxinwit.append(CTxInWitness()) + tx.wit.vtxinwit[0].scriptWitness.stack = [CScript([])] + tx.rehash() + self.update_witness_block_with_transactions(block, [tx]) + + # Extra witness data should not be allowed. + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=False) + + # Try extra signature data. Ok if we're not spending a witness output. + block.vtx[1].wit.vtxinwit = [] + block.vtx[1].vin[0].scriptSig = CScript([OP_0]) + block.vtx[1].rehash() + add_witness_commitment(block) + block.solve() + + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) + + # Now try extra witness/signature data on an input that DOES require a + # witness + tx2 = CTransaction() + tx2.vin.append(CTxIn(COutPoint(tx.sha256, 0), b"")) # witness output + tx2.vin.append(CTxIn(COutPoint(tx.sha256, 1), b"")) # non-witness + tx2.vout.append(CTxOut(tx.vout[0].nValue, CScript([OP_TRUE]))) + tx2.wit.vtxinwit.extend([CTxInWitness(), CTxInWitness()]) + tx2.wit.vtxinwit[0].scriptWitness.stack = [ CScript([CScriptNum(1)]), CScript([CScriptNum(1)]), witness_program ] + tx2.wit.vtxinwit[1].scriptWitness.stack = [ CScript([OP_TRUE]) ] + + block = self.build_next_block() + self.update_witness_block_with_transactions(block, [tx2]) + + # This has extra witness data, so it should fail. + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=False) + + # Now get rid of the extra witness, but add extra scriptSig data + tx2.vin[0].scriptSig = CScript([OP_TRUE]) + tx2.vin[1].scriptSig = CScript([OP_TRUE]) + tx2.wit.vtxinwit[0].scriptWitness.stack.pop(0) + tx2.wit.vtxinwit[1].scriptWitness.stack = [] + tx2.rehash() + add_witness_commitment(block) + block.solve() + + # This has extra signature data for a witness input, so it should fail. + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=False) + + # Now get rid of the extra scriptsig on the witness input, and verify + # success (even with extra scriptsig data in the non-witness input) + tx2.vin[0].scriptSig = b"" + tx2.rehash() + add_witness_commitment(block) + block.solve() + + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) + + # Update utxo for later tests + self.utxo.pop(0) + self.utxo.append(UTXO(tx2.sha256, 0, tx2.vout[0].nValue)) + + + def test_max_witness_push_length(self): + ''' Should only allow up to 520 byte pushes in witness stack ''' + self.log.info("Testing maximum witness push size") + MAX_SCRIPT_ELEMENT_SIZE = 520 + assert(len(self.utxo)) + + block = self.build_next_block() + + witness_program = CScript([OP_DROP, OP_TRUE]) + witness_hash = sha256(witness_program) + scriptPubKey = CScript([OP_0, witness_hash]) + + tx = CTransaction() + tx.vin.append(CTxIn(COutPoint(self.utxo[0].sha256, self.utxo[0].n), b"")) + tx.vout.append(CTxOut(self.utxo[0].nValue-1000, scriptPubKey)) + tx.rehash() + + tx2 = CTransaction() + tx2.vin.append(CTxIn(COutPoint(tx.sha256, 0), b"")) + tx2.vout.append(CTxOut(tx.vout[0].nValue-1000, CScript([OP_TRUE]))) + tx2.wit.vtxinwit.append(CTxInWitness()) + # First try a 521-byte stack element + tx2.wit.vtxinwit[0].scriptWitness.stack = [ b'a'*(MAX_SCRIPT_ELEMENT_SIZE+1), witness_program ] + tx2.rehash() + + self.update_witness_block_with_transactions(block, [tx, tx2]) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=False) + + # Now reduce the length of the stack element + tx2.wit.vtxinwit[0].scriptWitness.stack[0] = b'a'*(MAX_SCRIPT_ELEMENT_SIZE) + + add_witness_commitment(block) + block.solve() + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) + + # Update the utxo for later tests + self.utxo.pop() + self.utxo.append(UTXO(tx2.sha256, 0, tx2.vout[0].nValue)) + + def test_max_witness_program_length(self): + # Can create witness outputs that are long, but can't be greater than + # 10k bytes to successfully spend + self.log.info("Testing maximum witness program length") + assert(len(self.utxo)) + MAX_PROGRAM_LENGTH = 10000 + + # This program is 19 max pushes (9937 bytes), then 64 more opcode-bytes. + long_witness_program = CScript([b'a'*520]*19 + [OP_DROP]*63 + [OP_TRUE]) + assert(len(long_witness_program) == MAX_PROGRAM_LENGTH+1) + long_witness_hash = sha256(long_witness_program) + long_scriptPubKey = CScript([OP_0, long_witness_hash]) + + block = self.build_next_block() + + tx = CTransaction() + tx.vin.append(CTxIn(COutPoint(self.utxo[0].sha256, self.utxo[0].n), b"")) + tx.vout.append(CTxOut(self.utxo[0].nValue-1000, long_scriptPubKey)) + tx.rehash() + + tx2 = CTransaction() + tx2.vin.append(CTxIn(COutPoint(tx.sha256, 0), b"")) + tx2.vout.append(CTxOut(tx.vout[0].nValue-1000, CScript([OP_TRUE]))) + tx2.wit.vtxinwit.append(CTxInWitness()) + tx2.wit.vtxinwit[0].scriptWitness.stack = [b'a']*44 + [long_witness_program] + tx2.rehash() + + self.update_witness_block_with_transactions(block, [tx, tx2]) + + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=False) + + # Try again with one less byte in the witness program + witness_program = CScript([b'a'*520]*19 + [OP_DROP]*62 + [OP_TRUE]) + assert(len(witness_program) == MAX_PROGRAM_LENGTH) + witness_hash = sha256(witness_program) + scriptPubKey = CScript([OP_0, witness_hash]) + + tx.vout[0] = CTxOut(tx.vout[0].nValue, scriptPubKey) + tx.rehash() + tx2.vin[0].prevout.hash = tx.sha256 + tx2.wit.vtxinwit[0].scriptWitness.stack = [b'a']*43 + [witness_program] + tx2.rehash() + block.vtx = [block.vtx[0]] + self.update_witness_block_with_transactions(block, [tx, tx2]) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) + + self.utxo.pop() + self.utxo.append(UTXO(tx2.sha256, 0, tx2.vout[0].nValue)) + + + def test_witness_input_length(self): + ''' Ensure that vin length must match vtxinwit length ''' + self.log.info("Testing witness input length") + assert(len(self.utxo)) + + witness_program = CScript([OP_DROP, OP_TRUE]) + witness_hash = sha256(witness_program) + scriptPubKey = CScript([OP_0, witness_hash]) + + # Create a transaction that splits our utxo into many outputs + tx = CTransaction() + tx.vin.append(CTxIn(COutPoint(self.utxo[0].sha256, self.utxo[0].n), b"")) + nValue = self.utxo[0].nValue + for i in range(10): + tx.vout.append(CTxOut(int(nValue/10), scriptPubKey)) + tx.vout[0].nValue -= 1000 + assert(tx.vout[0].nValue >= 0) + + block = self.build_next_block() + self.update_witness_block_with_transactions(block, [tx]) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) + + # Try various ways to spend tx that should all break. + # This "broken" transaction serializer will not normalize + # the length of vtxinwit. + class BrokenCTransaction(CTransaction): + def serialize_with_witness(self): + flags = 0 + if not self.wit.is_null(): + flags |= 1 + r = b"" + r += struct.pack(" version 1 transactions + # are non-standard + scriptPubKey = CScript([CScriptOp(OP_1), witness_hash]) + tx2 = CTransaction() + tx2.vin = [CTxIn(COutPoint(tx.sha256, 0), b"")] + tx2.vout = [CTxOut(tx.vout[0].nValue-1000, scriptPubKey)] + tx2.wit.vtxinwit.append(CTxInWitness()) + tx2.wit.vtxinwit[0].scriptWitness.stack = [ witness_program ] + tx2.rehash() + # Gets accepted to test_node, because standardness of outputs isn't + # checked with fRequireStandard + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx2, with_witness=True, accepted=True) + test_transaction_acceptance(self.nodes[1].rpc, self.std_node, tx2, with_witness=True, accepted=False) + temp_utxo.pop() # last entry in temp_utxo was the output we just spent + temp_utxo.append(UTXO(tx2.sha256, 0, tx2.vout[0].nValue)) + + # Spend everything in temp_utxo back to an OP_TRUE output. + tx3 = CTransaction() + total_value = 0 + for i in temp_utxo: + tx3.vin.append(CTxIn(COutPoint(i.sha256, i.n), b"")) + tx3.wit.vtxinwit.append(CTxInWitness()) + total_value += i.nValue + tx3.wit.vtxinwit[-1].scriptWitness.stack = [witness_program] + tx3.vout.append(CTxOut(total_value - 1000, CScript([OP_TRUE]))) + tx3.rehash() + # Spending a higher version witness output is not allowed by policy, + # even with fRequireStandard=false. + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx3, with_witness=True, accepted=False) + self.test_node.sync_with_ping() + with mininode_lock: + assert(b"reserved for soft-fork upgrades" in self.test_node.last_message["reject"].reason) + + # Building a block with the transaction must be valid, however. + block = self.build_next_block() + self.update_witness_block_with_transactions(block, [tx2, tx3]) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) + sync_blocks(self.nodes) + + # Add utxo to our list + self.utxo.append(UTXO(tx3.sha256, 0, tx3.vout[0].nValue)) + + + def test_premature_coinbase_witness_spend(self): + self.log.info("Testing premature coinbase witness spend") + block = self.build_next_block() + # Change the output of the block to be a witness output. + witness_program = CScript([OP_TRUE]) + witness_hash = sha256(witness_program) + scriptPubKey = CScript([OP_0, witness_hash]) + block.vtx[0].vout[0].scriptPubKey = scriptPubKey + # This next line will rehash the coinbase and update the merkle + # root, and solve. + self.update_witness_block_with_transactions(block, []) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) + + spend_tx = CTransaction() + spend_tx.vin = [CTxIn(COutPoint(block.vtx[0].sha256, 0), b"")] + spend_tx.vout = [CTxOut(block.vtx[0].vout[0].nValue, witness_program)] + spend_tx.wit.vtxinwit.append(CTxInWitness()) + spend_tx.wit.vtxinwit[0].scriptWitness.stack = [ witness_program ] + spend_tx.rehash() + + # Now test a premature spend. + self.nodes[0].generate(98) + sync_blocks(self.nodes) + block2 = self.build_next_block() + self.update_witness_block_with_transactions(block2, [spend_tx]) + test_witness_block(self.nodes[0].rpc, self.test_node, block2, accepted=False) + + # Advancing one more block should allow the spend. + self.nodes[0].generate(1) + block2 = self.build_next_block() + self.update_witness_block_with_transactions(block2, [spend_tx]) + test_witness_block(self.nodes[0].rpc, self.test_node, block2, accepted=True) + sync_blocks(self.nodes) + + + def test_signature_version_1(self): + self.log.info("Testing segwit signature hash version 1") + key = CECKey() + key.set_secretbytes(b"9") + pubkey = CPubKey(key.get_pubkey()) + + witness_program = CScript([pubkey, CScriptOp(OP_CHECKSIG)]) + witness_hash = sha256(witness_program) + scriptPubKey = CScript([OP_0, witness_hash]) + + # First create a witness output for use in the tests. + assert(len(self.utxo)) + tx = CTransaction() + tx.vin.append(CTxIn(COutPoint(self.utxo[0].sha256, self.utxo[0].n), b"")) + tx.vout.append(CTxOut(self.utxo[0].nValue-1000, scriptPubKey)) + tx.rehash() + + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx, with_witness=True, accepted=True) + # Mine this transaction in preparation for following tests. + block = self.build_next_block() + self.update_witness_block_with_transactions(block, [tx]) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) + sync_blocks(self.nodes) + self.utxo.pop(0) + + # Test each hashtype + prev_utxo = UTXO(tx.sha256, 0, tx.vout[0].nValue) + for sigflag in [ 0, SIGHASH_ANYONECANPAY ]: + for hashtype in [SIGHASH_ALL, SIGHASH_NONE, SIGHASH_SINGLE]: + hashtype |= sigflag + block = self.build_next_block() + tx = CTransaction() + tx.vin.append(CTxIn(COutPoint(prev_utxo.sha256, prev_utxo.n), b"")) + tx.vout.append(CTxOut(prev_utxo.nValue - 1000, scriptPubKey)) + tx.wit.vtxinwit.append(CTxInWitness()) + # Too-large input value + sign_P2PK_witness_input(witness_program, tx, 0, hashtype, prev_utxo.nValue+1, key) + self.update_witness_block_with_transactions(block, [tx]) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=False) + + # Too-small input value + sign_P2PK_witness_input(witness_program, tx, 0, hashtype, prev_utxo.nValue-1, key) + block.vtx.pop() # remove last tx + self.update_witness_block_with_transactions(block, [tx]) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=False) + + # Now try correct value + sign_P2PK_witness_input(witness_program, tx, 0, hashtype, prev_utxo.nValue, key) + block.vtx.pop() + self.update_witness_block_with_transactions(block, [tx]) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) + + prev_utxo = UTXO(tx.sha256, 0, tx.vout[0].nValue) + + # Test combinations of signature hashes. + # Split the utxo into a lot of outputs. + # Randomly choose up to 10 to spend, sign with different hashtypes, and + # output to a random number of outputs. Repeat NUM_TESTS times. + # Ensure that we've tested a situation where we use SIGHASH_SINGLE with + # an input index > number of outputs. + NUM_TESTS = 500 + temp_utxos = [] + tx = CTransaction() + tx.vin.append(CTxIn(COutPoint(prev_utxo.sha256, prev_utxo.n), b"")) + split_value = prev_utxo.nValue // NUM_TESTS + for i in range(NUM_TESTS): + tx.vout.append(CTxOut(split_value, scriptPubKey)) + tx.wit.vtxinwit.append(CTxInWitness()) + sign_P2PK_witness_input(witness_program, tx, 0, SIGHASH_ALL, prev_utxo.nValue, key) + for i in range(NUM_TESTS): + temp_utxos.append(UTXO(tx.sha256, i, split_value)) + + block = self.build_next_block() + self.update_witness_block_with_transactions(block, [tx]) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) + + block = self.build_next_block() + used_sighash_single_out_of_bounds = False + for i in range(NUM_TESTS): + # Ping regularly to keep the connection alive + if (not i % 100): + self.test_node.sync_with_ping() + # Choose random number of inputs to use. + num_inputs = random.randint(1, 10) + # Create a slight bias for producing more utxos + num_outputs = random.randint(1, 11) + random.shuffle(temp_utxos) + assert(len(temp_utxos) > num_inputs) + tx = CTransaction() + total_value = 0 + for i in range(num_inputs): + tx.vin.append(CTxIn(COutPoint(temp_utxos[i].sha256, temp_utxos[i].n), b"")) + tx.wit.vtxinwit.append(CTxInWitness()) + total_value += temp_utxos[i].nValue + split_value = total_value // num_outputs + for i in range(num_outputs): + tx.vout.append(CTxOut(split_value, scriptPubKey)) + for i in range(num_inputs): + # Now try to sign each input, using a random hashtype. + anyonecanpay = 0 + if random.randint(0, 1): + anyonecanpay = SIGHASH_ANYONECANPAY + hashtype = random.randint(1, 3) | anyonecanpay + sign_P2PK_witness_input(witness_program, tx, i, hashtype, temp_utxos[i].nValue, key) + if (hashtype == SIGHASH_SINGLE and i >= num_outputs): + used_sighash_single_out_of_bounds = True + tx.rehash() + for i in range(num_outputs): + temp_utxos.append(UTXO(tx.sha256, i, split_value)) + temp_utxos = temp_utxos[num_inputs:] + + block.vtx.append(tx) + + # Test the block periodically, if we're close to maxblocksize + if (get_virtual_size(block) > MAX_BLOCK_BASE_SIZE - 1000): + self.update_witness_block_with_transactions(block, []) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) + block = self.build_next_block() + + if (not used_sighash_single_out_of_bounds): + self.log.info("WARNING: this test run didn't attempt SIGHASH_SINGLE with out-of-bounds index value") + # Test the transactions we've added to the block + if (len(block.vtx) > 1): + self.update_witness_block_with_transactions(block, []) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) + + # Now test witness version 0 P2PKH transactions + pubkeyhash = hash160(pubkey) + scriptPKH = CScript([OP_0, pubkeyhash]) + tx = CTransaction() + tx.vin.append(CTxIn(COutPoint(temp_utxos[0].sha256, temp_utxos[0].n), b"")) + tx.vout.append(CTxOut(temp_utxos[0].nValue, scriptPKH)) + tx.wit.vtxinwit.append(CTxInWitness()) + sign_P2PK_witness_input(witness_program, tx, 0, SIGHASH_ALL, temp_utxos[0].nValue, key) + tx2 = CTransaction() + tx2.vin.append(CTxIn(COutPoint(tx.sha256, 0), b"")) + tx2.vout.append(CTxOut(tx.vout[0].nValue, CScript([OP_TRUE]))) + + script = GetP2PKHScript(pubkeyhash) + sig_hash = SegwitVersion1SignatureHash(script, tx2, 0, SIGHASH_ALL, tx.vout[0].nValue) + signature = key.sign(sig_hash) + b'\x01' # 0x1 is SIGHASH_ALL + + # Check that we can't have a scriptSig + tx2.vin[0].scriptSig = CScript([signature, pubkey]) + block = self.build_next_block() + self.update_witness_block_with_transactions(block, [tx, tx2]) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=False) + + # Move the signature to the witness. + block.vtx.pop() + tx2.wit.vtxinwit.append(CTxInWitness()) + tx2.wit.vtxinwit[0].scriptWitness.stack = [signature, pubkey] + tx2.vin[0].scriptSig = b"" + tx2.rehash() + + self.update_witness_block_with_transactions(block, [tx2]) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) + + temp_utxos.pop(0) + + # Update self.utxos for later tests by creating two outputs + # that consolidate all the coins in temp_utxos. + output_value = sum(i.nValue for i in temp_utxos) // 2 + + tx = CTransaction() + index = 0 + # Just spend to our usual anyone-can-spend output + tx.vout = [CTxOut(output_value, CScript([OP_TRUE]))] * 2 + for i in temp_utxos: + # Use SIGHASH_ALL|SIGHASH_ANYONECANPAY so we can build up + # the signatures as we go. + tx.vin.append(CTxIn(COutPoint(i.sha256, i.n), b"")) + tx.wit.vtxinwit.append(CTxInWitness()) + sign_P2PK_witness_input(witness_program, tx, index, SIGHASH_ALL|SIGHASH_ANYONECANPAY, i.nValue, key) + index += 1 + block = self.build_next_block() + self.update_witness_block_with_transactions(block, [tx]) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) + + for i in range(len(tx.vout)): + self.utxo.append(UTXO(tx.sha256, i, tx.vout[i].nValue)) + + + # Test P2SH wrapped witness programs. + def test_p2sh_witness(self, segwit_activated): + self.log.info("Testing P2SH witness transactions") + + assert(len(self.utxo)) + + # Prepare the p2sh-wrapped witness output + witness_program = CScript([OP_DROP, OP_TRUE]) + witness_hash = sha256(witness_program) + p2wsh_pubkey = CScript([OP_0, witness_hash]) + p2sh_witness_hash = hash160(p2wsh_pubkey) + scriptPubKey = CScript([OP_HASH160, p2sh_witness_hash, OP_EQUAL]) + scriptSig = CScript([p2wsh_pubkey]) # a push of the redeem script + + # Fund the P2SH output + tx = CTransaction() + tx.vin.append(CTxIn(COutPoint(self.utxo[0].sha256, self.utxo[0].n), b"")) + tx.vout.append(CTxOut(self.utxo[0].nValue-1000, scriptPubKey)) + tx.rehash() + + # Verify mempool acceptance and block validity + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx, with_witness=False, accepted=True) + block = self.build_next_block() + self.update_witness_block_with_transactions(block, [tx]) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True, with_witness=segwit_activated) + sync_blocks(self.nodes) + + # Now test attempts to spend the output. + spend_tx = CTransaction() + spend_tx.vin.append(CTxIn(COutPoint(tx.sha256, 0), scriptSig)) + spend_tx.vout.append(CTxOut(tx.vout[0].nValue-1000, CScript([OP_TRUE]))) + spend_tx.rehash() + + # This transaction should not be accepted into the mempool pre- or + # post-segwit. Mempool acceptance will use SCRIPT_VERIFY_WITNESS which + # will require a witness to spend a witness program regardless of + # segwit activation. Note that older bitcoind's that are not + # segwit-aware would also reject this for failing CLEANSTACK. + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, spend_tx, with_witness=False, accepted=False) + + # Try to put the witness script in the scriptSig, should also fail. + spend_tx.vin[0].scriptSig = CScript([p2wsh_pubkey, b'a']) + spend_tx.rehash() + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, spend_tx, with_witness=False, accepted=False) + + # Now put the witness script in the witness, should succeed after + # segwit activates. + spend_tx.vin[0].scriptSig = scriptSig + spend_tx.rehash() + spend_tx.wit.vtxinwit.append(CTxInWitness()) + spend_tx.wit.vtxinwit[0].scriptWitness.stack = [ b'a', witness_program ] + + # Verify mempool acceptance + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, spend_tx, with_witness=True, accepted=segwit_activated) + block = self.build_next_block() + self.update_witness_block_with_transactions(block, [spend_tx]) + + # If we're before activation, then sending this without witnesses + # should be valid. If we're after activation, then sending this with + # witnesses should be valid. + if segwit_activated: + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) + else: + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True, with_witness=False) + + # Update self.utxo + self.utxo.pop(0) + self.utxo.append(UTXO(spend_tx.sha256, 0, spend_tx.vout[0].nValue)) + + # Test the behavior of starting up a segwit-aware node after the softfork + # has activated. As segwit requires different block data than pre-segwit + # nodes would have stored, this requires special handling. + # To enable this test, pass --oldbinary= to + # the test. + def test_upgrade_after_activation(self, node_id): + self.log.info("Testing software upgrade after softfork activation") + + assert(node_id != 0) # node0 is assumed to be a segwit-active bitcoind + + # Make sure the nodes are all up + sync_blocks(self.nodes) + + # Restart with the new binary + self.stop_node(node_id) + self.start_node(node_id, extra_args=["-vbparams=segwit:0:999999999999"]) + connect_nodes(self.nodes[0], node_id) + + sync_blocks(self.nodes) + + # Make sure that this peer thinks segwit has activated. + assert(get_bip9_status(self.nodes[node_id], 'segwit')['status'] == "active") + + # Make sure this peers blocks match those of node0. + height = self.nodes[node_id].getblockcount() + while height >= 0: + block_hash = self.nodes[node_id].getblockhash(height) + assert_equal(block_hash, self.nodes[0].getblockhash(height)) + assert_equal(self.nodes[0].getblock(block_hash), self.nodes[node_id].getblock(block_hash)) + height -= 1 + + + def test_witness_sigops(self): + '''Ensure sigop counting is correct inside witnesses.''' + self.log.info("Testing sigops limit") + + assert(len(self.utxo)) + + # Keep this under MAX_OPS_PER_SCRIPT (201) + witness_program = CScript([OP_TRUE, OP_IF, OP_TRUE, OP_ELSE] + [OP_CHECKMULTISIG]*5 + [OP_CHECKSIG]*193 + [OP_ENDIF]) + witness_hash = sha256(witness_program) + scriptPubKey = CScript([OP_0, witness_hash]) + + sigops_per_script = 20*5 + 193*1 + # We'll produce 2 extra outputs, one with a program that would take us + # over max sig ops, and one with a program that would exactly reach max + # sig ops + outputs = (MAX_SIGOP_COST // sigops_per_script) + 2 + extra_sigops_available = MAX_SIGOP_COST % sigops_per_script + + # We chose the number of checkmultisigs/checksigs to make this work: + assert(extra_sigops_available < 100) # steer clear of MAX_OPS_PER_SCRIPT + + # This script, when spent with the first + # N(=MAX_SIGOP_COST//sigops_per_script) outputs of our transaction, + # would push us just over the block sigop limit. + witness_program_toomany = CScript([OP_TRUE, OP_IF, OP_TRUE, OP_ELSE] + [OP_CHECKSIG]*(extra_sigops_available + 1) + [OP_ENDIF]) + witness_hash_toomany = sha256(witness_program_toomany) + scriptPubKey_toomany = CScript([OP_0, witness_hash_toomany]) + + # If we spend this script instead, we would exactly reach our sigop + # limit (for witness sigops). + witness_program_justright = CScript([OP_TRUE, OP_IF, OP_TRUE, OP_ELSE] + [OP_CHECKSIG]*(extra_sigops_available) + [OP_ENDIF]) + witness_hash_justright = sha256(witness_program_justright) + scriptPubKey_justright = CScript([OP_0, witness_hash_justright]) + + # First split our available utxo into a bunch of outputs + split_value = self.utxo[0].nValue // outputs + tx = CTransaction() + tx.vin.append(CTxIn(COutPoint(self.utxo[0].sha256, self.utxo[0].n), b"")) + for i in range(outputs): + tx.vout.append(CTxOut(split_value, scriptPubKey)) + tx.vout[-2].scriptPubKey = scriptPubKey_toomany + tx.vout[-1].scriptPubKey = scriptPubKey_justright + tx.rehash() + + block_1 = self.build_next_block() + self.update_witness_block_with_transactions(block_1, [tx]) + test_witness_block(self.nodes[0].rpc, self.test_node, block_1, accepted=True) + + tx2 = CTransaction() + # If we try to spend the first n-1 outputs from tx, that should be + # too many sigops. + total_value = 0 + for i in range(outputs-1): + tx2.vin.append(CTxIn(COutPoint(tx.sha256, i), b"")) + tx2.wit.vtxinwit.append(CTxInWitness()) + tx2.wit.vtxinwit[-1].scriptWitness.stack = [ witness_program ] + total_value += tx.vout[i].nValue + tx2.wit.vtxinwit[-1].scriptWitness.stack = [ witness_program_toomany ] + tx2.vout.append(CTxOut(total_value, CScript([OP_TRUE]))) + tx2.rehash() + + block_2 = self.build_next_block() + self.update_witness_block_with_transactions(block_2, [tx2]) + test_witness_block(self.nodes[0].rpc, self.test_node, block_2, accepted=False) + + # Try dropping the last input in tx2, and add an output that has + # too many sigops (contributing to legacy sigop count). + checksig_count = (extra_sigops_available // 4) + 1 + scriptPubKey_checksigs = CScript([OP_CHECKSIG]*checksig_count) + tx2.vout.append(CTxOut(0, scriptPubKey_checksigs)) + tx2.vin.pop() + tx2.wit.vtxinwit.pop() + tx2.vout[0].nValue -= tx.vout[-2].nValue + tx2.rehash() + block_3 = self.build_next_block() + self.update_witness_block_with_transactions(block_3, [tx2]) + test_witness_block(self.nodes[0].rpc, self.test_node, block_3, accepted=False) + + # If we drop the last checksig in this output, the tx should succeed. + block_4 = self.build_next_block() + tx2.vout[-1].scriptPubKey = CScript([OP_CHECKSIG]*(checksig_count-1)) + tx2.rehash() + self.update_witness_block_with_transactions(block_4, [tx2]) + test_witness_block(self.nodes[0].rpc, self.test_node, block_4, accepted=True) + + # Reset the tip back down for the next test + sync_blocks(self.nodes) + for x in self.nodes: + x.invalidateblock(block_4.hash) + + # Try replacing the last input of tx2 to be spending the last + # output of tx + block_5 = self.build_next_block() + tx2.vout.pop() + tx2.vin.append(CTxIn(COutPoint(tx.sha256, outputs-1), b"")) + tx2.wit.vtxinwit.append(CTxInWitness()) + tx2.wit.vtxinwit[-1].scriptWitness.stack = [ witness_program_justright ] + tx2.rehash() + self.update_witness_block_with_transactions(block_5, [tx2]) + test_witness_block(self.nodes[0].rpc, self.test_node, block_5, accepted=True) + + # TODO: test p2sh sigop counting + + def test_getblocktemplate_before_lockin(self): + self.log.info("Testing getblocktemplate setting of segwit versionbit (before lockin)") + # Node0 is segwit aware, node2 is not. + for node in [self.nodes[0], self.nodes[2]]: + gbt_results = node.getblocktemplate() + block_version = gbt_results['version'] + # If we're not indicating segwit support, we will still be + # signalling for segwit activation. + assert_equal((block_version & (1 << VB_WITNESS_BIT) != 0), node == self.nodes[0]) + # If we don't specify the segwit rule, then we won't get a default + # commitment. + assert('default_witness_commitment' not in gbt_results) + + # Workaround: + # Can either change the tip, or change the mempool and wait 5 seconds + # to trigger a recomputation of getblocktemplate. + txid = int(self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 1), 16) + # Using mocktime lets us avoid sleep() + sync_mempools(self.nodes) + self.nodes[0].setmocktime(int(time.time())+10) + self.nodes[2].setmocktime(int(time.time())+10) + + for node in [self.nodes[0], self.nodes[2]]: + gbt_results = node.getblocktemplate({"rules" : ["segwit"]}) + block_version = gbt_results['version'] + if node == self.nodes[2]: + # If this is a non-segwit node, we should still not get a witness + # commitment, nor a version bit signalling segwit. + assert_equal(block_version & (1 << VB_WITNESS_BIT), 0) + assert('default_witness_commitment' not in gbt_results) + else: + # For segwit-aware nodes, check the version bit and the witness + # commitment are correct. + assert(block_version & (1 << VB_WITNESS_BIT) != 0) + assert('default_witness_commitment' in gbt_results) + witness_commitment = gbt_results['default_witness_commitment'] + + # Check that default_witness_commitment is present. + witness_root = CBlock.get_merkle_root([ser_uint256(0), + ser_uint256(txid)]) + script = get_witness_script(witness_root, 0) + assert_equal(witness_commitment, bytes_to_hex_str(script)) + + # undo mocktime + self.nodes[0].setmocktime(0) + self.nodes[2].setmocktime(0) + + # Uncompressed pubkeys are no longer supported in default relay policy, + # but (for now) are still valid in blocks. + def test_uncompressed_pubkey(self): + self.log.info("Testing uncompressed pubkeys") + # Segwit transactions using uncompressed pubkeys are not accepted + # under default policy, but should still pass consensus. + key = CECKey() + key.set_secretbytes(b"9") + key.set_compressed(False) + pubkey = CPubKey(key.get_pubkey()) + assert_equal(len(pubkey), 65) # This should be an uncompressed pubkey + + assert(len(self.utxo) > 0) + utxo = self.utxo.pop(0) + + # Test 1: P2WPKH + # First create a P2WPKH output that uses an uncompressed pubkey + pubkeyhash = hash160(pubkey) + scriptPKH = CScript([OP_0, pubkeyhash]) + tx = CTransaction() + tx.vin.append(CTxIn(COutPoint(utxo.sha256, utxo.n), b"")) + tx.vout.append(CTxOut(utxo.nValue-1000, scriptPKH)) + tx.rehash() + + # Confirm it in a block. + block = self.build_next_block() + self.update_witness_block_with_transactions(block, [tx]) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) + + # Now try to spend it. Send it to a P2WSH output, which we'll + # use in the next test. + witness_program = CScript([pubkey, CScriptOp(OP_CHECKSIG)]) + witness_hash = sha256(witness_program) + scriptWSH = CScript([OP_0, witness_hash]) + + tx2 = CTransaction() + tx2.vin.append(CTxIn(COutPoint(tx.sha256, 0), b"")) + tx2.vout.append(CTxOut(tx.vout[0].nValue-1000, scriptWSH)) + script = GetP2PKHScript(pubkeyhash) + sig_hash = SegwitVersion1SignatureHash(script, tx2, 0, SIGHASH_ALL, tx.vout[0].nValue) + signature = key.sign(sig_hash) + b'\x01' # 0x1 is SIGHASH_ALL + tx2.wit.vtxinwit.append(CTxInWitness()) + tx2.wit.vtxinwit[0].scriptWitness.stack = [ signature, pubkey ] + tx2.rehash() + + # Should fail policy test. + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx2, True, False, b'non-mandatory-script-verify-flag (Using non-compressed keys in segwit)') + # But passes consensus. + block = self.build_next_block() + self.update_witness_block_with_transactions(block, [tx2]) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) + + # Test 2: P2WSH + # Try to spend the P2WSH output created in last test. + # Send it to a P2SH(P2WSH) output, which we'll use in the next test. + p2sh_witness_hash = hash160(scriptWSH) + scriptP2SH = CScript([OP_HASH160, p2sh_witness_hash, OP_EQUAL]) + scriptSig = CScript([scriptWSH]) + + tx3 = CTransaction() + tx3.vin.append(CTxIn(COutPoint(tx2.sha256, 0), b"")) + tx3.vout.append(CTxOut(tx2.vout[0].nValue-1000, scriptP2SH)) + tx3.wit.vtxinwit.append(CTxInWitness()) + sign_P2PK_witness_input(witness_program, tx3, 0, SIGHASH_ALL, tx2.vout[0].nValue, key) + + # Should fail policy test. + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx3, True, False, b'non-mandatory-script-verify-flag (Using non-compressed keys in segwit)') + # But passes consensus. + block = self.build_next_block() + self.update_witness_block_with_transactions(block, [tx3]) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) + + # Test 3: P2SH(P2WSH) + # Try to spend the P2SH output created in the last test. + # Send it to a P2PKH output, which we'll use in the next test. + scriptPubKey = GetP2PKHScript(pubkeyhash) + tx4 = CTransaction() + tx4.vin.append(CTxIn(COutPoint(tx3.sha256, 0), scriptSig)) + tx4.vout.append(CTxOut(tx3.vout[0].nValue-1000, scriptPubKey)) + tx4.wit.vtxinwit.append(CTxInWitness()) + sign_P2PK_witness_input(witness_program, tx4, 0, SIGHASH_ALL, tx3.vout[0].nValue, key) + + # Should fail policy test. + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx4, True, False, b'non-mandatory-script-verify-flag (Using non-compressed keys in segwit)') + block = self.build_next_block() + self.update_witness_block_with_transactions(block, [tx4]) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) + + # Test 4: Uncompressed pubkeys should still be valid in non-segwit + # transactions. + tx5 = CTransaction() + tx5.vin.append(CTxIn(COutPoint(tx4.sha256, 0), b"")) + tx5.vout.append(CTxOut(tx4.vout[0].nValue-1000, CScript([OP_TRUE]))) + (sig_hash, err) = SignatureHash(scriptPubKey, tx5, 0, SIGHASH_ALL) + signature = key.sign(sig_hash) + b'\x01' # 0x1 is SIGHASH_ALL + tx5.vin[0].scriptSig = CScript([signature, pubkey]) + tx5.rehash() + # Should pass policy and consensus. + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx5, True, True) + block = self.build_next_block() + self.update_witness_block_with_transactions(block, [tx5]) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) + self.utxo.append(UTXO(tx5.sha256, 0, tx5.vout[0].nValue)) + + def test_non_standard_witness(self): + self.log.info("Testing detection of non-standard P2WSH witness") + pad = chr(1).encode('latin-1') + + # Create scripts for tests + scripts = [] + scripts.append(CScript([OP_DROP] * 100)) + scripts.append(CScript([OP_DROP] * 99)) + scripts.append(CScript([pad * 59] * 59 + [OP_DROP] * 60)) + scripts.append(CScript([pad * 59] * 59 + [OP_DROP] * 61)) + + p2wsh_scripts = [] + + assert(len(self.utxo)) + tx = CTransaction() + tx.vin.append(CTxIn(COutPoint(self.utxo[0].sha256, self.utxo[0].n), b"")) + + # For each script, generate a pair of P2WSH and P2SH-P2WSH output. + outputvalue = (self.utxo[0].nValue - 1000) // (len(scripts) * 2) + for i in scripts: + p2wsh = CScript([OP_0, sha256(i)]) + p2sh = hash160(p2wsh) + p2wsh_scripts.append(p2wsh) + tx.vout.append(CTxOut(outputvalue, p2wsh)) + tx.vout.append(CTxOut(outputvalue, CScript([OP_HASH160, p2sh, OP_EQUAL]))) + tx.rehash() + txid = tx.sha256 + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx, with_witness=False, accepted=True) + + self.nodes[0].generate(1) + sync_blocks(self.nodes) + + # Creating transactions for tests + p2wsh_txs = [] + p2sh_txs = [] + for i in range(len(scripts)): + p2wsh_tx = CTransaction() + p2wsh_tx.vin.append(CTxIn(COutPoint(txid,i*2))) + p2wsh_tx.vout.append(CTxOut(outputvalue - 5000, CScript([OP_0, hash160(hex_str_to_bytes(""))]))) + p2wsh_tx.wit.vtxinwit.append(CTxInWitness()) + p2wsh_tx.rehash() + p2wsh_txs.append(p2wsh_tx) + p2sh_tx = CTransaction() + p2sh_tx.vin.append(CTxIn(COutPoint(txid,i*2+1), CScript([p2wsh_scripts[i]]))) + p2sh_tx.vout.append(CTxOut(outputvalue - 5000, CScript([OP_0, hash160(hex_str_to_bytes(""))]))) + p2sh_tx.wit.vtxinwit.append(CTxInWitness()) + p2sh_tx.rehash() + p2sh_txs.append(p2sh_tx) + + # Testing native P2WSH + # Witness stack size, excluding witnessScript, over 100 is non-standard + p2wsh_txs[0].wit.vtxinwit[0].scriptWitness.stack = [pad] * 101 + [scripts[0]] + test_transaction_acceptance(self.nodes[1].rpc, self.std_node, p2wsh_txs[0], True, False, b'bad-witness-nonstandard') + # Non-standard nodes should accept + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, p2wsh_txs[0], True, True) + + # Stack element size over 80 bytes is non-standard + p2wsh_txs[1].wit.vtxinwit[0].scriptWitness.stack = [pad * 81] * 100 + [scripts[1]] + test_transaction_acceptance(self.nodes[1].rpc, self.std_node, p2wsh_txs[1], True, False, b'bad-witness-nonstandard') + # Non-standard nodes should accept + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, p2wsh_txs[1], True, True) + # Standard nodes should accept if element size is not over 80 bytes + p2wsh_txs[1].wit.vtxinwit[0].scriptWitness.stack = [pad * 80] * 100 + [scripts[1]] + test_transaction_acceptance(self.nodes[1].rpc, self.std_node, p2wsh_txs[1], True, True) + + # witnessScript size at 3600 bytes is standard + p2wsh_txs[2].wit.vtxinwit[0].scriptWitness.stack = [pad, pad, scripts[2]] + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, p2wsh_txs[2], True, True) + test_transaction_acceptance(self.nodes[1].rpc, self.std_node, p2wsh_txs[2], True, True) + + # witnessScript size at 3601 bytes is non-standard + p2wsh_txs[3].wit.vtxinwit[0].scriptWitness.stack = [pad, pad, pad, scripts[3]] + test_transaction_acceptance(self.nodes[1].rpc, self.std_node, p2wsh_txs[3], True, False, b'bad-witness-nonstandard') + # Non-standard nodes should accept + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, p2wsh_txs[3], True, True) + + # Repeating the same tests with P2SH-P2WSH + p2sh_txs[0].wit.vtxinwit[0].scriptWitness.stack = [pad] * 101 + [scripts[0]] + test_transaction_acceptance(self.nodes[1].rpc, self.std_node, p2sh_txs[0], True, False, b'bad-witness-nonstandard') + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, p2sh_txs[0], True, True) + p2sh_txs[1].wit.vtxinwit[0].scriptWitness.stack = [pad * 81] * 100 + [scripts[1]] + test_transaction_acceptance(self.nodes[1].rpc, self.std_node, p2sh_txs[1], True, False, b'bad-witness-nonstandard') + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, p2sh_txs[1], True, True) + p2sh_txs[1].wit.vtxinwit[0].scriptWitness.stack = [pad * 80] * 100 + [scripts[1]] + test_transaction_acceptance(self.nodes[1].rpc, self.std_node, p2sh_txs[1], True, True) + p2sh_txs[2].wit.vtxinwit[0].scriptWitness.stack = [pad, pad, scripts[2]] + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, p2sh_txs[2], True, True) + test_transaction_acceptance(self.nodes[1].rpc, self.std_node, p2sh_txs[2], True, True) + p2sh_txs[3].wit.vtxinwit[0].scriptWitness.stack = [pad, pad, pad, scripts[3]] + test_transaction_acceptance(self.nodes[1].rpc, self.std_node, p2sh_txs[3], True, False, b'bad-witness-nonstandard') + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, p2sh_txs[3], True, True) + + self.nodes[0].generate(1) # Mine and clean up the mempool of non-standard node + # Valid but non-standard transactions in a block should be accepted by standard node + sync_blocks(self.nodes) + assert_equal(len(self.nodes[0].getrawmempool()), 0) + assert_equal(len(self.nodes[1].getrawmempool()), 0) + + self.utxo.pop(0) + + + def run_test(self): + # Setup the p2p connections and start up the network thread. + # self.test_node sets NODE_WITNESS|NODE_NETWORK + self.test_node = self.nodes[0].add_p2p_connection(TestNode(), services=NODE_NETWORK|NODE_WITNESS) + # self.old_node sets only NODE_NETWORK + self.old_node = self.nodes[0].add_p2p_connection(TestNode(), services=NODE_NETWORK) + # self.std_node is for testing node1 (fRequireStandard=true) + self.std_node = self.nodes[1].add_p2p_connection(TestNode(), services=NODE_NETWORK|NODE_WITNESS) + + network_thread_start() + + # Keep a place to store utxo's that can be used in later tests + self.utxo = [] + + # Test logic begins here + self.test_node.wait_for_verack() + + self.log.info("Starting tests before segwit lock in:") + + self.test_witness_services() # Verifies NODE_WITNESS + self.test_non_witness_transaction() # non-witness tx's are accepted + self.test_unnecessary_witness_before_segwit_activation() + self.test_block_relay(segwit_activated=False) + + # Advance to segwit being 'started' + self.advance_to_segwit_started() + sync_blocks(self.nodes) + self.test_getblocktemplate_before_lockin() + + sync_blocks(self.nodes) + + # At lockin, nothing should change. + self.log.info("Testing behavior post lockin, pre-activation") + self.advance_to_segwit_lockin() + + # Retest unnecessary witnesses + self.test_unnecessary_witness_before_segwit_activation() + self.test_witness_tx_relay_before_segwit_activation() + self.test_block_relay(segwit_activated=False) + self.test_p2sh_witness(segwit_activated=False) + self.test_standardness_v0(segwit_activated=False) + + sync_blocks(self.nodes) + + # Now activate segwit + self.log.info("Testing behavior after segwit activation") + self.advance_to_segwit_active() + + sync_blocks(self.nodes) + + # Test P2SH witness handling again + self.test_p2sh_witness(segwit_activated=True) + self.test_witness_commitments() + self.test_block_malleability() + self.test_witness_block_size() + self.test_submit_block() + self.test_extra_witness_data() + self.test_max_witness_push_length() + self.test_max_witness_program_length() + self.test_witness_input_length() + self.test_block_relay(segwit_activated=True) + self.test_tx_relay_after_segwit_activation() + self.test_standardness_v0(segwit_activated=True) + self.test_segwit_versions() + self.test_premature_coinbase_witness_spend() + self.test_uncompressed_pubkey() + self.test_signature_version_1() + self.test_non_standard_witness() + sync_blocks(self.nodes) + self.test_upgrade_after_activation(node_id=2) + self.test_witness_sigops() + + +if __name__ == '__main__': + SegWitTest().main() diff --git a/test/functional/p2p_sendheaders.py b/test/functional/p2p_sendheaders.py new file mode 100755 index 0000000000..8869aeaaea --- /dev/null +++ b/test/functional/p2p_sendheaders.py @@ -0,0 +1,598 @@ +#!/usr/bin/env python3 +# Copyright (c) 2014-2017 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test behavior of headers messages to announce blocks. + +Setup: + +- Two nodes: + - node0 is the node-under-test. We create two p2p connections to it. The + first p2p connection is a control and should only ever receive inv's. The + second p2p connection tests the headers sending logic. + - node1 is used to create reorgs. + +test_null_locators +================== + +Sends two getheaders requests with null locator values. First request's hashstop +value refers to validated block, while second request's hashstop value refers to +a block which hasn't been validated. Verifies only the first request returns +headers. + +test_nonnull_locators +===================== + +Part 1: No headers announcements before "sendheaders" +a. node mines a block [expect: inv] + send getdata for the block [expect: block] +b. node mines another block [expect: inv] + send getheaders and getdata [expect: headers, then block] +c. node mines another block [expect: inv] + peer mines a block, announces with header [expect: getdata] +d. node mines another block [expect: inv] + +Part 2: After "sendheaders", headers announcements should generally work. +a. peer sends sendheaders [expect: no response] + peer sends getheaders with current tip [expect: no response] +b. node mines a block [expect: tip header] +c. for N in 1, ..., 10: + * for announce-type in {inv, header} + - peer mines N blocks, announces with announce-type + [ expect: getheaders/getdata or getdata, deliver block(s) ] + - node mines a block [ expect: 1 header ] + +Part 3: Headers announcements stop after large reorg and resume after getheaders or inv from peer. +- For response-type in {inv, getheaders} + * node mines a 7 block reorg [ expect: headers announcement of 8 blocks ] + * node mines an 8-block reorg [ expect: inv at tip ] + * peer responds with getblocks/getdata [expect: inv, blocks ] + * node mines another block [ expect: inv at tip, peer sends getdata, expect: block ] + * node mines another block at tip [ expect: inv ] + * peer responds with getheaders with an old hashstop more than 8 blocks back [expect: headers] + * peer requests block [ expect: block ] + * node mines another block at tip [ expect: inv, peer sends getdata, expect: block ] + * peer sends response-type [expect headers if getheaders, getheaders/getdata if mining new block] + * node mines 1 block [expect: 1 header, peer responds with getdata] + +Part 4: Test direct fetch behavior +a. Announce 2 old block headers. + Expect: no getdata requests. +b. Announce 3 new blocks via 1 headers message. + Expect: one getdata request for all 3 blocks. + (Send blocks.) +c. Announce 1 header that forks off the last two blocks. + Expect: no response. +d. Announce 1 more header that builds on that fork. + Expect: one getdata request for two blocks. +e. Announce 16 more headers that build on that fork. + Expect: getdata request for 14 more blocks. +f. Announce 1 more header that builds on that fork. + Expect: no response. + +Part 5: Test handling of headers that don't connect. +a. Repeat 10 times: + 1. Announce a header that doesn't connect. + Expect: getheaders message + 2. Send headers chain. + Expect: getdata for the missing blocks, tip update. +b. Then send 9 more headers that don't connect. + Expect: getheaders message each time. +c. Announce a header that does connect. + Expect: no response. +d. Announce 49 headers that don't connect. + Expect: getheaders message each time. +e. Announce one more that doesn't connect. + Expect: disconnect. +""" +from test_framework.blocktools import create_block, create_coinbase +from test_framework.mininode import ( + CBlockHeader, + CInv, + NODE_WITNESS, + network_thread_start, + P2PInterface, + mininode_lock, + msg_block, + msg_getblocks, + msg_getdata, + msg_getheaders, + msg_headers, + msg_inv, + msg_sendheaders, +) +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + sync_blocks, + wait_until, +) + +DIRECT_FETCH_RESPONSE_TIME = 0.05 + +class BaseNode(P2PInterface): + def __init__(self): + super().__init__() + + self.block_announced = False + self.last_blockhash_announced = None + + def send_get_data(self, block_hashes): + """Request data for a list of block hashes.""" + msg = msg_getdata() + for x in block_hashes: + msg.inv.append(CInv(2, x)) + self.send_message(msg) + + def send_get_headers(self, locator, hashstop): + msg = msg_getheaders() + msg.locator.vHave = locator + msg.hashstop = hashstop + self.send_message(msg) + + def send_block_inv(self, blockhash): + msg = msg_inv() + msg.inv = [CInv(2, blockhash)] + self.send_message(msg) + + def send_header_for_blocks(self, new_blocks): + headers_message = msg_headers() + headers_message.headers = [CBlockHeader(b) for b in new_blocks] + self.send_message(headers_message) + + def send_getblocks(self, locator): + getblocks_message = msg_getblocks() + getblocks_message.locator.vHave = locator + self.send_message(getblocks_message) + + def wait_for_getdata(self, hash_list, timeout=60): + if hash_list == []: + return + + test_function = lambda: "getdata" in self.last_message and [x.hash for x in self.last_message["getdata"].inv] == hash_list + wait_until(test_function, timeout=timeout, lock=mininode_lock) + + def wait_for_block_announcement(self, block_hash, timeout=60): + test_function = lambda: self.last_blockhash_announced == block_hash + wait_until(test_function, timeout=timeout, lock=mininode_lock) + + def on_inv(self, message): + self.block_announced = True + self.last_blockhash_announced = message.inv[-1].hash + + def on_headers(self, message): + if len(message.headers): + self.block_announced = True + message.headers[-1].calc_sha256() + self.last_blockhash_announced = message.headers[-1].sha256 + + def clear_last_announcement(self): + with mininode_lock: + self.block_announced = False + self.last_message.pop("inv", None) + self.last_message.pop("headers", None) + + def check_last_announcement(self, headers=None, inv=None): + """Test whether the last announcement received had the right header or the right inv. + + inv and headers should be lists of block hashes.""" + + test_function = lambda: self.block_announced + wait_until(test_function, timeout=60, lock=mininode_lock) + + with mininode_lock: + self.block_announced = False + + compare_inv = [] + if "inv" in self.last_message: + compare_inv = [x.hash for x in self.last_message["inv"].inv] + if inv is not None: + assert_equal(compare_inv, inv) + + compare_headers = [] + if "headers" in self.last_message: + compare_headers = [x.sha256 for x in self.last_message["headers"].headers] + if headers is not None: + assert_equal(compare_headers, headers) + + self.last_message.pop("inv", None) + self.last_message.pop("headers", None) + +class SendHeadersTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 2 + + def mine_blocks(self, count): + """Mine count blocks and return the new tip.""" + + # Clear out last block announcement from each p2p listener + [x.clear_last_announcement() for x in self.nodes[0].p2ps] + self.nodes[0].generate(count) + return int(self.nodes[0].getbestblockhash(), 16) + + def mine_reorg(self, length): + """Mine a reorg that invalidates length blocks (replacing them with # length+1 blocks). + + Note: we clear the state of our p2p connections after the + to-be-reorged-out blocks are mined, so that we don't break later tests. + return the list of block hashes newly mined.""" + + self.nodes[0].generate(length) # make sure all invalidated blocks are node0's + sync_blocks(self.nodes, wait=0.1) + for x in self.nodes[0].p2ps: + x.wait_for_block_announcement(int(self.nodes[0].getbestblockhash(), 16)) + x.clear_last_announcement() + + tip_height = self.nodes[1].getblockcount() + hash_to_invalidate = self.nodes[1].getblockhash(tip_height - (length - 1)) + self.nodes[1].invalidateblock(hash_to_invalidate) + all_hashes = self.nodes[1].generate(length + 1) # Must be longer than the orig chain + sync_blocks(self.nodes, wait=0.1) + return [int(x, 16) for x in all_hashes] + + def run_test(self): + # Setup the p2p connections and start up the network thread. + inv_node = self.nodes[0].add_p2p_connection(BaseNode()) + # Make sure NODE_NETWORK is not set for test_node, so no block download + # will occur outside of direct fetching + test_node = self.nodes[0].add_p2p_connection(BaseNode(), services=NODE_WITNESS) + + network_thread_start() + + # Test logic begins here + inv_node.wait_for_verack() + test_node.wait_for_verack() + + # Ensure verack's have been processed by our peer + inv_node.sync_with_ping() + test_node.sync_with_ping() + + self.test_null_locators(test_node, inv_node) + self.test_nonnull_locators(test_node, inv_node) + + def test_null_locators(self, test_node, inv_node): + tip = self.nodes[0].getblockheader(self.nodes[0].generate(1)[0]) + tip_hash = int(tip["hash"], 16) + + inv_node.check_last_announcement(inv=[tip_hash], headers=[]) + test_node.check_last_announcement(inv=[tip_hash], headers=[]) + + self.log.info("Verify getheaders with null locator and valid hashstop returns headers.") + test_node.clear_last_announcement() + test_node.send_get_headers(locator=[], hashstop=tip_hash) + test_node.check_last_announcement(headers=[tip_hash]) + + self.log.info("Verify getheaders with null locator and invalid hashstop does not return headers.") + block = create_block(int(tip["hash"], 16), create_coinbase(tip["height"] + 1), tip["mediantime"] + 1) + block.solve() + test_node.send_header_for_blocks([block]) + test_node.clear_last_announcement() + test_node.send_get_headers(locator=[], hashstop=int(block.hash, 16)) + test_node.sync_with_ping() + assert_equal(test_node.block_announced, False) + inv_node.clear_last_announcement() + test_node.send_message(msg_block(block)) + inv_node.check_last_announcement(inv=[int(block.hash, 16)], headers=[]) + + def test_nonnull_locators(self, test_node, inv_node): + tip = int(self.nodes[0].getbestblockhash(), 16) + + # PART 1 + # 1. Mine a block; expect inv announcements each time + self.log.info("Part 1: headers don't start before sendheaders message...") + for i in range(4): + old_tip = tip + tip = self.mine_blocks(1) + inv_node.check_last_announcement(inv=[tip], headers=[]) + test_node.check_last_announcement(inv=[tip], headers=[]) + # Try a few different responses; none should affect next announcement + if i == 0: + # first request the block + test_node.send_get_data([tip]) + test_node.wait_for_block(tip) + elif i == 1: + # next try requesting header and block + test_node.send_get_headers(locator=[old_tip], hashstop=tip) + test_node.send_get_data([tip]) + test_node.wait_for_block(tip) + test_node.clear_last_announcement() # since we requested headers... + elif i == 2: + # this time announce own block via headers + height = self.nodes[0].getblockcount() + last_time = self.nodes[0].getblock(self.nodes[0].getbestblockhash())['time'] + block_time = last_time + 1 + new_block = create_block(tip, create_coinbase(height + 1), block_time) + new_block.solve() + test_node.send_header_for_blocks([new_block]) + test_node.wait_for_getdata([new_block.sha256]) + test_node.send_message(msg_block(new_block)) + test_node.sync_with_ping() # make sure this block is processed + inv_node.clear_last_announcement() + test_node.clear_last_announcement() + + self.log.info("Part 1: success!") + self.log.info("Part 2: announce blocks with headers after sendheaders message...") + # PART 2 + # 2. Send a sendheaders message and test that headers announcements + # commence and keep working. + test_node.send_message(msg_sendheaders()) + prev_tip = int(self.nodes[0].getbestblockhash(), 16) + test_node.send_get_headers(locator=[prev_tip], hashstop=0) + test_node.sync_with_ping() + + # Now that we've synced headers, headers announcements should work + tip = self.mine_blocks(1) + inv_node.check_last_announcement(inv=[tip], headers=[]) + test_node.check_last_announcement(headers=[tip]) + + height = self.nodes[0].getblockcount() + 1 + block_time += 10 # Advance far enough ahead + for i in range(10): + # Mine i blocks, and alternate announcing either via + # inv (of tip) or via headers. After each, new blocks + # mined by the node should successfully be announced + # with block header, even though the blocks are never requested + for j in range(2): + blocks = [] + for b in range(i + 1): + blocks.append(create_block(tip, create_coinbase(height), block_time)) + blocks[-1].solve() + tip = blocks[-1].sha256 + block_time += 1 + height += 1 + if j == 0: + # Announce via inv + test_node.send_block_inv(tip) + test_node.wait_for_getheaders() + # Should have received a getheaders now + test_node.send_header_for_blocks(blocks) + # Test that duplicate inv's won't result in duplicate + # getdata requests, or duplicate headers announcements + [inv_node.send_block_inv(x.sha256) for x in blocks] + test_node.wait_for_getdata([x.sha256 for x in blocks]) + inv_node.sync_with_ping() + else: + # Announce via headers + test_node.send_header_for_blocks(blocks) + test_node.wait_for_getdata([x.sha256 for x in blocks]) + # Test that duplicate headers won't result in duplicate + # getdata requests (the check is further down) + inv_node.send_header_for_blocks(blocks) + inv_node.sync_with_ping() + [test_node.send_message(msg_block(x)) for x in blocks] + test_node.sync_with_ping() + inv_node.sync_with_ping() + # This block should not be announced to the inv node (since it also + # broadcast it) + assert "inv" not in inv_node.last_message + assert "headers" not in inv_node.last_message + tip = self.mine_blocks(1) + inv_node.check_last_announcement(inv=[tip], headers=[]) + test_node.check_last_announcement(headers=[tip]) + height += 1 + block_time += 1 + + self.log.info("Part 2: success!") + + self.log.info("Part 3: headers announcements can stop after large reorg, and resume after headers/inv from peer...") + + # PART 3. Headers announcements can stop after large reorg, and resume after + # getheaders or inv from peer. + for j in range(2): + # First try mining a reorg that can propagate with header announcement + new_block_hashes = self.mine_reorg(length=7) + tip = new_block_hashes[-1] + inv_node.check_last_announcement(inv=[tip], headers=[]) + test_node.check_last_announcement(headers=new_block_hashes) + + block_time += 8 + + # Mine a too-large reorg, which should be announced with a single inv + new_block_hashes = self.mine_reorg(length=8) + tip = new_block_hashes[-1] + inv_node.check_last_announcement(inv=[tip], headers=[]) + test_node.check_last_announcement(inv=[tip], headers=[]) + + block_time += 9 + + fork_point = self.nodes[0].getblock("%02x" % new_block_hashes[0])["previousblockhash"] + fork_point = int(fork_point, 16) + + # Use getblocks/getdata + test_node.send_getblocks(locator=[fork_point]) + test_node.check_last_announcement(inv=new_block_hashes, headers=[]) + test_node.send_get_data(new_block_hashes) + test_node.wait_for_block(new_block_hashes[-1]) + + for i in range(3): + # Mine another block, still should get only an inv + tip = self.mine_blocks(1) + inv_node.check_last_announcement(inv=[tip], headers=[]) + test_node.check_last_announcement(inv=[tip], headers=[]) + if i == 0: + # Just get the data -- shouldn't cause headers announcements to resume + test_node.send_get_data([tip]) + test_node.wait_for_block(tip) + elif i == 1: + # Send a getheaders message that shouldn't trigger headers announcements + # to resume (best header sent will be too old) + test_node.send_get_headers(locator=[fork_point], hashstop=new_block_hashes[1]) + test_node.send_get_data([tip]) + test_node.wait_for_block(tip) + elif i == 2: + test_node.send_get_data([tip]) + test_node.wait_for_block(tip) + # This time, try sending either a getheaders to trigger resumption + # of headers announcements, or mine a new block and inv it, also + # triggering resumption of headers announcements. + if j == 0: + test_node.send_get_headers(locator=[tip], hashstop=0) + test_node.sync_with_ping() + else: + test_node.send_block_inv(tip) + test_node.sync_with_ping() + # New blocks should now be announced with header + tip = self.mine_blocks(1) + inv_node.check_last_announcement(inv=[tip], headers=[]) + test_node.check_last_announcement(headers=[tip]) + + self.log.info("Part 3: success!") + + self.log.info("Part 4: Testing direct fetch behavior...") + tip = self.mine_blocks(1) + height = self.nodes[0].getblockcount() + 1 + last_time = self.nodes[0].getblock(self.nodes[0].getbestblockhash())['time'] + block_time = last_time + 1 + + # Create 2 blocks. Send the blocks, then send the headers. + blocks = [] + for b in range(2): + blocks.append(create_block(tip, create_coinbase(height), block_time)) + blocks[-1].solve() + tip = blocks[-1].sha256 + block_time += 1 + height += 1 + inv_node.send_message(msg_block(blocks[-1])) + + inv_node.sync_with_ping() # Make sure blocks are processed + test_node.last_message.pop("getdata", None) + test_node.send_header_for_blocks(blocks) + test_node.sync_with_ping() + # should not have received any getdata messages + with mininode_lock: + assert "getdata" not in test_node.last_message + + # This time, direct fetch should work + blocks = [] + for b in range(3): + blocks.append(create_block(tip, create_coinbase(height), block_time)) + blocks[-1].solve() + tip = blocks[-1].sha256 + block_time += 1 + height += 1 + + test_node.send_header_for_blocks(blocks) + test_node.sync_with_ping() + test_node.wait_for_getdata([x.sha256 for x in blocks], timeout=DIRECT_FETCH_RESPONSE_TIME) + + [test_node.send_message(msg_block(x)) for x in blocks] + + test_node.sync_with_ping() + + # Now announce a header that forks the last two blocks + tip = blocks[0].sha256 + height -= 1 + blocks = [] + + # Create extra blocks for later + for b in range(20): + blocks.append(create_block(tip, create_coinbase(height), block_time)) + blocks[-1].solve() + tip = blocks[-1].sha256 + block_time += 1 + height += 1 + + # Announcing one block on fork should not trigger direct fetch + # (less work than tip) + test_node.last_message.pop("getdata", None) + test_node.send_header_for_blocks(blocks[0:1]) + test_node.sync_with_ping() + with mininode_lock: + assert "getdata" not in test_node.last_message + + # Announcing one more block on fork should trigger direct fetch for + # both blocks (same work as tip) + test_node.send_header_for_blocks(blocks[1:2]) + test_node.sync_with_ping() + test_node.wait_for_getdata([x.sha256 for x in blocks[0:2]], timeout=DIRECT_FETCH_RESPONSE_TIME) + + # Announcing 16 more headers should trigger direct fetch for 14 more + # blocks + test_node.send_header_for_blocks(blocks[2:18]) + test_node.sync_with_ping() + test_node.wait_for_getdata([x.sha256 for x in blocks[2:16]], timeout=DIRECT_FETCH_RESPONSE_TIME) + + # Announcing 1 more header should not trigger any response + test_node.last_message.pop("getdata", None) + test_node.send_header_for_blocks(blocks[18:19]) + test_node.sync_with_ping() + with mininode_lock: + assert "getdata" not in test_node.last_message + + self.log.info("Part 4: success!") + + # Now deliver all those blocks we announced. + [test_node.send_message(msg_block(x)) for x in blocks] + + self.log.info("Part 5: Testing handling of unconnecting headers") + # First we test that receipt of an unconnecting header doesn't prevent + # chain sync. + for i in range(10): + test_node.last_message.pop("getdata", None) + blocks = [] + # Create two more blocks. + for j in range(2): + blocks.append(create_block(tip, create_coinbase(height), block_time)) + blocks[-1].solve() + tip = blocks[-1].sha256 + block_time += 1 + height += 1 + # Send the header of the second block -> this won't connect. + with mininode_lock: + test_node.last_message.pop("getheaders", None) + test_node.send_header_for_blocks([blocks[1]]) + test_node.wait_for_getheaders() + test_node.send_header_for_blocks(blocks) + test_node.wait_for_getdata([x.sha256 for x in blocks]) + [test_node.send_message(msg_block(x)) for x in blocks] + test_node.sync_with_ping() + assert_equal(int(self.nodes[0].getbestblockhash(), 16), blocks[1].sha256) + + blocks = [] + # Now we test that if we repeatedly don't send connecting headers, we + # don't go into an infinite loop trying to get them to connect. + MAX_UNCONNECTING_HEADERS = 10 + for j in range(MAX_UNCONNECTING_HEADERS + 1): + blocks.append(create_block(tip, create_coinbase(height), block_time)) + blocks[-1].solve() + tip = blocks[-1].sha256 + block_time += 1 + height += 1 + + for i in range(1, MAX_UNCONNECTING_HEADERS): + # Send a header that doesn't connect, check that we get a getheaders. + with mininode_lock: + test_node.last_message.pop("getheaders", None) + test_node.send_header_for_blocks([blocks[i]]) + test_node.wait_for_getheaders() + + # Next header will connect, should re-set our count: + test_node.send_header_for_blocks([blocks[0]]) + + # Remove the first two entries (blocks[1] would connect): + blocks = blocks[2:] + + # Now try to see how many unconnecting headers we can send + # before we get disconnected. Should be 5*MAX_UNCONNECTING_HEADERS + for i in range(5 * MAX_UNCONNECTING_HEADERS - 1): + # Send a header that doesn't connect, check that we get a getheaders. + with mininode_lock: + test_node.last_message.pop("getheaders", None) + test_node.send_header_for_blocks([blocks[i % len(blocks)]]) + test_node.wait_for_getheaders() + + # Eventually this stops working. + test_node.send_header_for_blocks([blocks[-1]]) + + # Should get disconnected + test_node.wait_for_disconnect() + + self.log.info("Part 5: success!") + + # Finally, check that the inv node never received a getdata request, + # throughout the test + assert "getdata" not in inv_node.last_message + +if __name__ == '__main__': + SendHeadersTest().main() diff --git a/test/functional/p2p_timeouts.py b/test/functional/p2p_timeouts.py new file mode 100755 index 0000000000..6d21095cc6 --- /dev/null +++ b/test/functional/p2p_timeouts.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +# Copyright (c) 2016-2017 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test various net timeouts. + +- Create three bitcoind nodes: + + no_verack_node - we never send a verack in response to their version + no_version_node - we never send a version (only a ping) + no_send_node - we never send any P2P message. + +- Start all three nodes +- Wait 1 second +- Assert that we're connected +- Send a ping to no_verack_node and no_version_node +- Wait 30 seconds +- Assert that we're still connected +- Send a ping to no_verack_node and no_version_node +- Wait 31 seconds +- Assert that we're no longer connected (timeout to receive version/verack is 60 seconds) +""" + +from time import sleep + +from test_framework.mininode import * +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import * + +class TestNode(P2PInterface): + def on_version(self, message): + # Don't send a verack in response + pass + +class TimeoutsTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 1 + + def run_test(self): + # Setup the p2p connections and start up the network thread. + no_verack_node = self.nodes[0].add_p2p_connection(TestNode()) + no_version_node = self.nodes[0].add_p2p_connection(TestNode(), send_version=False) + no_send_node = self.nodes[0].add_p2p_connection(TestNode(), send_version=False) + + network_thread_start() + + sleep(1) + + assert no_verack_node.connected + assert no_version_node.connected + assert no_send_node.connected + + no_verack_node.send_message(msg_ping()) + no_version_node.send_message(msg_ping()) + + sleep(30) + + assert "version" in no_verack_node.last_message + + assert no_verack_node.connected + assert no_version_node.connected + assert no_send_node.connected + + no_verack_node.send_message(msg_ping()) + no_version_node.send_message(msg_ping()) + + sleep(31) + + assert not no_verack_node.connected + assert not no_version_node.connected + assert not no_send_node.connected + +if __name__ == '__main__': + TimeoutsTest().main() diff --git a/test/functional/p2p_unrequested_blocks.py b/test/functional/p2p_unrequested_blocks.py new file mode 100755 index 0000000000..672626f15b --- /dev/null +++ b/test/functional/p2p_unrequested_blocks.py @@ -0,0 +1,323 @@ +#!/usr/bin/env python3 +# Copyright (c) 2015-2017 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test processing of unrequested blocks. + +Setup: two nodes, node0+node1, not connected to each other. Node1 will have +nMinimumChainWork set to 0x10, so it won't process low-work unrequested blocks. + +We have one P2PInterface connection to node0 called test_node, and one to node1 +called min_work_node. + +The test: +1. Generate one block on each node, to leave IBD. + +2. Mine a new block on each tip, and deliver to each node from node's peer. + The tip should advance for node0, but node1 should skip processing due to + nMinimumChainWork. + +Node1 is unused in tests 3-7: + +3. Mine a block that forks from the genesis block, and deliver to test_node. + Node0 should not process this block (just accept the header), because it + is unrequested and doesn't have more or equal work to the tip. + +4a,b. Send another two blocks that build on the forking block. + Node0 should process the second block but be stuck on the shorter chain, + because it's missing an intermediate block. + +4c.Send 288 more blocks on the longer chain (the number of blocks ahead + we currently store). + Node0 should process all but the last block (too far ahead in height). + +5. Send a duplicate of the block in #3 to Node0. + Node0 should not process the block because it is unrequested, and stay on + the shorter chain. + +6. Send Node0 an inv for the height 3 block produced in #4 above. + Node0 should figure out that Node0 has the missing height 2 block and send a + getdata. + +7. Send Node0 the missing block again. + Node0 should process and the tip should advance. + +8. Create a fork which is invalid at a height longer than the current chain + (ie to which the node will try to reorg) but which has headers built on top + of the invalid block. Check that we get disconnected if we send more headers + on the chain the node now knows to be invalid. + +9. Test Node1 is able to sync when connected to node0 (which should have sufficient + work on its chain). +""" + +from test_framework.mininode import * +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import * +import time +from test_framework.blocktools import create_block, create_coinbase, create_transaction + +class AcceptBlockTest(BitcoinTestFramework): + def add_options(self, parser): + parser.add_option("--testbinary", dest="testbinary", + default=os.getenv("BITCOIND", "bitcoind"), + help="bitcoind binary to test") + + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 2 + self.extra_args = [[], ["-minimumchainwork=0x10"]] + + def setup_network(self): + # Node0 will be used to test behavior of processing unrequested blocks + # from peers which are not whitelisted, while Node1 will be used for + # the whitelisted case. + # Node2 will be used for non-whitelisted peers to test the interaction + # with nMinimumChainWork. + self.setup_nodes() + + def run_test(self): + # Setup the p2p connections and start up the network thread. + # test_node connects to node0 (not whitelisted) + test_node = self.nodes[0].add_p2p_connection(P2PInterface()) + # min_work_node connects to node1 (whitelisted) + min_work_node = self.nodes[1].add_p2p_connection(P2PInterface()) + + network_thread_start() + + # Test logic begins here + test_node.wait_for_verack() + min_work_node.wait_for_verack() + + # 1. Have nodes mine a block (leave IBD) + [ n.generate(1) for n in self.nodes ] + tips = [ int("0x" + n.getbestblockhash(), 0) for n in self.nodes ] + + # 2. Send one block that builds on each tip. + # This should be accepted by node0 + blocks_h2 = [] # the height 2 blocks on each node's chain + block_time = int(time.time()) + 1 + for i in range(2): + blocks_h2.append(create_block(tips[i], create_coinbase(2), block_time)) + blocks_h2[i].solve() + block_time += 1 + test_node.send_message(msg_block(blocks_h2[0])) + min_work_node.send_message(msg_block(blocks_h2[1])) + + for x in [test_node, min_work_node]: + x.sync_with_ping() + assert_equal(self.nodes[0].getblockcount(), 2) + assert_equal(self.nodes[1].getblockcount(), 1) + self.log.info("First height 2 block accepted by node0; correctly rejected by node1") + + # 3. Send another block that builds on genesis. + block_h1f = create_block(int("0x" + self.nodes[0].getblockhash(0), 0), create_coinbase(1), block_time) + block_time += 1 + block_h1f.solve() + test_node.send_message(msg_block(block_h1f)) + + test_node.sync_with_ping() + tip_entry_found = False + for x in self.nodes[0].getchaintips(): + if x['hash'] == block_h1f.hash: + assert_equal(x['status'], "headers-only") + tip_entry_found = True + assert(tip_entry_found) + assert_raises_rpc_error(-1, "Block not found on disk", self.nodes[0].getblock, block_h1f.hash) + + # 4. Send another two block that build on the fork. + block_h2f = create_block(block_h1f.sha256, create_coinbase(2), block_time) + block_time += 1 + block_h2f.solve() + test_node.send_message(msg_block(block_h2f)) + + test_node.sync_with_ping() + # Since the earlier block was not processed by node, the new block + # can't be fully validated. + tip_entry_found = False + for x in self.nodes[0].getchaintips(): + if x['hash'] == block_h2f.hash: + assert_equal(x['status'], "headers-only") + tip_entry_found = True + assert(tip_entry_found) + + # But this block should be accepted by node since it has equal work. + self.nodes[0].getblock(block_h2f.hash) + self.log.info("Second height 2 block accepted, but not reorg'ed to") + + # 4b. Now send another block that builds on the forking chain. + block_h3 = create_block(block_h2f.sha256, create_coinbase(3), block_h2f.nTime+1) + block_h3.solve() + test_node.send_message(msg_block(block_h3)) + + test_node.sync_with_ping() + # Since the earlier block was not processed by node, the new block + # can't be fully validated. + tip_entry_found = False + for x in self.nodes[0].getchaintips(): + if x['hash'] == block_h3.hash: + assert_equal(x['status'], "headers-only") + tip_entry_found = True + assert(tip_entry_found) + self.nodes[0].getblock(block_h3.hash) + + # But this block should be accepted by node since it has more work. + self.nodes[0].getblock(block_h3.hash) + self.log.info("Unrequested more-work block accepted") + + # 4c. Now mine 288 more blocks and deliver; all should be processed but + # the last (height-too-high) on node (as long as its not missing any headers) + tip = block_h3 + all_blocks = [] + for i in range(288): + next_block = create_block(tip.sha256, create_coinbase(i + 4), tip.nTime+1) + next_block.solve() + all_blocks.append(next_block) + tip = next_block + + # Now send the block at height 5 and check that it wasn't accepted (missing header) + test_node.send_message(msg_block(all_blocks[1])) + test_node.sync_with_ping() + assert_raises_rpc_error(-5, "Block not found", self.nodes[0].getblock, all_blocks[1].hash) + assert_raises_rpc_error(-5, "Block not found", self.nodes[0].getblockheader, all_blocks[1].hash) + + # The block at height 5 should be accepted if we provide the missing header, though + headers_message = msg_headers() + headers_message.headers.append(CBlockHeader(all_blocks[0])) + test_node.send_message(headers_message) + test_node.send_message(msg_block(all_blocks[1])) + test_node.sync_with_ping() + self.nodes[0].getblock(all_blocks[1].hash) + + # Now send the blocks in all_blocks + for i in range(288): + test_node.send_message(msg_block(all_blocks[i])) + test_node.sync_with_ping() + + # Blocks 1-287 should be accepted, block 288 should be ignored because it's too far ahead + for x in all_blocks[:-1]: + self.nodes[0].getblock(x.hash) + assert_raises_rpc_error(-1, "Block not found on disk", self.nodes[0].getblock, all_blocks[-1].hash) + + # 5. Test handling of unrequested block on the node that didn't process + # Should still not be processed (even though it has a child that has more + # work). + + # The node should have requested the blocks at some point, so + # disconnect/reconnect first + + self.nodes[0].disconnect_p2ps() + self.nodes[1].disconnect_p2ps() + network_thread_join() + + test_node = self.nodes[0].add_p2p_connection(P2PInterface()) + network_thread_start() + test_node.wait_for_verack() + + test_node.send_message(msg_block(block_h1f)) + + test_node.sync_with_ping() + assert_equal(self.nodes[0].getblockcount(), 2) + self.log.info("Unrequested block that would complete more-work chain was ignored") + + # 6. Try to get node to request the missing block. + # Poke the node with an inv for block at height 3 and see if that + # triggers a getdata on block 2 (it should if block 2 is missing). + with mininode_lock: + # Clear state so we can check the getdata request + test_node.last_message.pop("getdata", None) + test_node.send_message(msg_inv([CInv(2, block_h3.sha256)])) + + test_node.sync_with_ping() + with mininode_lock: + getdata = test_node.last_message["getdata"] + + # Check that the getdata includes the right block + assert_equal(getdata.inv[0].hash, block_h1f.sha256) + self.log.info("Inv at tip triggered getdata for unprocessed block") + + # 7. Send the missing block for the third time (now it is requested) + test_node.send_message(msg_block(block_h1f)) + + test_node.sync_with_ping() + assert_equal(self.nodes[0].getblockcount(), 290) + self.nodes[0].getblock(all_blocks[286].hash) + assert_equal(self.nodes[0].getbestblockhash(), all_blocks[286].hash) + assert_raises_rpc_error(-1, "Block not found on disk", self.nodes[0].getblock, all_blocks[287].hash) + self.log.info("Successfully reorged to longer chain from non-whitelisted peer") + + # 8. Create a chain which is invalid at a height longer than the + # current chain, but which has more blocks on top of that + block_289f = create_block(all_blocks[284].sha256, create_coinbase(289), all_blocks[284].nTime+1) + block_289f.solve() + block_290f = create_block(block_289f.sha256, create_coinbase(290), block_289f.nTime+1) + block_290f.solve() + block_291 = create_block(block_290f.sha256, create_coinbase(291), block_290f.nTime+1) + # block_291 spends a coinbase below maturity! + block_291.vtx.append(create_transaction(block_290f.vtx[0], 0, b"42", 1)) + block_291.hashMerkleRoot = block_291.calc_merkle_root() + block_291.solve() + block_292 = create_block(block_291.sha256, create_coinbase(292), block_291.nTime+1) + block_292.solve() + + # Now send all the headers on the chain and enough blocks to trigger reorg + headers_message = msg_headers() + headers_message.headers.append(CBlockHeader(block_289f)) + headers_message.headers.append(CBlockHeader(block_290f)) + headers_message.headers.append(CBlockHeader(block_291)) + headers_message.headers.append(CBlockHeader(block_292)) + test_node.send_message(headers_message) + + test_node.sync_with_ping() + tip_entry_found = False + for x in self.nodes[0].getchaintips(): + if x['hash'] == block_292.hash: + assert_equal(x['status'], "headers-only") + tip_entry_found = True + assert(tip_entry_found) + assert_raises_rpc_error(-1, "Block not found on disk", self.nodes[0].getblock, block_292.hash) + + test_node.send_message(msg_block(block_289f)) + test_node.send_message(msg_block(block_290f)) + + test_node.sync_with_ping() + self.nodes[0].getblock(block_289f.hash) + self.nodes[0].getblock(block_290f.hash) + + test_node.send_message(msg_block(block_291)) + + # At this point we've sent an obviously-bogus block, wait for full processing + # without assuming whether we will be disconnected or not + try: + # Only wait a short while so the test doesn't take forever if we do get + # disconnected + test_node.sync_with_ping(timeout=1) + except AssertionError: + test_node.wait_for_disconnect() + + self.nodes[0].disconnect_p2ps() + test_node = self.nodes[0].add_p2p_connection(P2PInterface()) + + network_thread_start() + test_node.wait_for_verack() + + # We should have failed reorg and switched back to 290 (but have block 291) + assert_equal(self.nodes[0].getblockcount(), 290) + assert_equal(self.nodes[0].getbestblockhash(), all_blocks[286].hash) + assert_equal(self.nodes[0].getblock(block_291.hash)["confirmations"], -1) + + # Now send a new header on the invalid chain, indicating we're forked off, and expect to get disconnected + block_293 = create_block(block_292.sha256, create_coinbase(293), block_292.nTime+1) + block_293.solve() + headers_message = msg_headers() + headers_message.headers.append(CBlockHeader(block_293)) + test_node.send_message(headers_message) + test_node.wait_for_disconnect() + + # 9. Connect node1 to node0 and ensure it is able to sync + connect_nodes(self.nodes[0], 1) + sync_blocks([self.nodes[0], self.nodes[1]]) + self.log.info("Successfully synced nodes 1 and 0") + +if __name__ == '__main__': + AcceptBlockTest().main() diff --git a/test/functional/sendheaders.py b/test/functional/sendheaders.py deleted file mode 100755 index 8869aeaaea..0000000000 --- a/test/functional/sendheaders.py +++ /dev/null @@ -1,598 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2014-2017 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test behavior of headers messages to announce blocks. - -Setup: - -- Two nodes: - - node0 is the node-under-test. We create two p2p connections to it. The - first p2p connection is a control and should only ever receive inv's. The - second p2p connection tests the headers sending logic. - - node1 is used to create reorgs. - -test_null_locators -================== - -Sends two getheaders requests with null locator values. First request's hashstop -value refers to validated block, while second request's hashstop value refers to -a block which hasn't been validated. Verifies only the first request returns -headers. - -test_nonnull_locators -===================== - -Part 1: No headers announcements before "sendheaders" -a. node mines a block [expect: inv] - send getdata for the block [expect: block] -b. node mines another block [expect: inv] - send getheaders and getdata [expect: headers, then block] -c. node mines another block [expect: inv] - peer mines a block, announces with header [expect: getdata] -d. node mines another block [expect: inv] - -Part 2: After "sendheaders", headers announcements should generally work. -a. peer sends sendheaders [expect: no response] - peer sends getheaders with current tip [expect: no response] -b. node mines a block [expect: tip header] -c. for N in 1, ..., 10: - * for announce-type in {inv, header} - - peer mines N blocks, announces with announce-type - [ expect: getheaders/getdata or getdata, deliver block(s) ] - - node mines a block [ expect: 1 header ] - -Part 3: Headers announcements stop after large reorg and resume after getheaders or inv from peer. -- For response-type in {inv, getheaders} - * node mines a 7 block reorg [ expect: headers announcement of 8 blocks ] - * node mines an 8-block reorg [ expect: inv at tip ] - * peer responds with getblocks/getdata [expect: inv, blocks ] - * node mines another block [ expect: inv at tip, peer sends getdata, expect: block ] - * node mines another block at tip [ expect: inv ] - * peer responds with getheaders with an old hashstop more than 8 blocks back [expect: headers] - * peer requests block [ expect: block ] - * node mines another block at tip [ expect: inv, peer sends getdata, expect: block ] - * peer sends response-type [expect headers if getheaders, getheaders/getdata if mining new block] - * node mines 1 block [expect: 1 header, peer responds with getdata] - -Part 4: Test direct fetch behavior -a. Announce 2 old block headers. - Expect: no getdata requests. -b. Announce 3 new blocks via 1 headers message. - Expect: one getdata request for all 3 blocks. - (Send blocks.) -c. Announce 1 header that forks off the last two blocks. - Expect: no response. -d. Announce 1 more header that builds on that fork. - Expect: one getdata request for two blocks. -e. Announce 16 more headers that build on that fork. - Expect: getdata request for 14 more blocks. -f. Announce 1 more header that builds on that fork. - Expect: no response. - -Part 5: Test handling of headers that don't connect. -a. Repeat 10 times: - 1. Announce a header that doesn't connect. - Expect: getheaders message - 2. Send headers chain. - Expect: getdata for the missing blocks, tip update. -b. Then send 9 more headers that don't connect. - Expect: getheaders message each time. -c. Announce a header that does connect. - Expect: no response. -d. Announce 49 headers that don't connect. - Expect: getheaders message each time. -e. Announce one more that doesn't connect. - Expect: disconnect. -""" -from test_framework.blocktools import create_block, create_coinbase -from test_framework.mininode import ( - CBlockHeader, - CInv, - NODE_WITNESS, - network_thread_start, - P2PInterface, - mininode_lock, - msg_block, - msg_getblocks, - msg_getdata, - msg_getheaders, - msg_headers, - msg_inv, - msg_sendheaders, -) -from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import ( - assert_equal, - sync_blocks, - wait_until, -) - -DIRECT_FETCH_RESPONSE_TIME = 0.05 - -class BaseNode(P2PInterface): - def __init__(self): - super().__init__() - - self.block_announced = False - self.last_blockhash_announced = None - - def send_get_data(self, block_hashes): - """Request data for a list of block hashes.""" - msg = msg_getdata() - for x in block_hashes: - msg.inv.append(CInv(2, x)) - self.send_message(msg) - - def send_get_headers(self, locator, hashstop): - msg = msg_getheaders() - msg.locator.vHave = locator - msg.hashstop = hashstop - self.send_message(msg) - - def send_block_inv(self, blockhash): - msg = msg_inv() - msg.inv = [CInv(2, blockhash)] - self.send_message(msg) - - def send_header_for_blocks(self, new_blocks): - headers_message = msg_headers() - headers_message.headers = [CBlockHeader(b) for b in new_blocks] - self.send_message(headers_message) - - def send_getblocks(self, locator): - getblocks_message = msg_getblocks() - getblocks_message.locator.vHave = locator - self.send_message(getblocks_message) - - def wait_for_getdata(self, hash_list, timeout=60): - if hash_list == []: - return - - test_function = lambda: "getdata" in self.last_message and [x.hash for x in self.last_message["getdata"].inv] == hash_list - wait_until(test_function, timeout=timeout, lock=mininode_lock) - - def wait_for_block_announcement(self, block_hash, timeout=60): - test_function = lambda: self.last_blockhash_announced == block_hash - wait_until(test_function, timeout=timeout, lock=mininode_lock) - - def on_inv(self, message): - self.block_announced = True - self.last_blockhash_announced = message.inv[-1].hash - - def on_headers(self, message): - if len(message.headers): - self.block_announced = True - message.headers[-1].calc_sha256() - self.last_blockhash_announced = message.headers[-1].sha256 - - def clear_last_announcement(self): - with mininode_lock: - self.block_announced = False - self.last_message.pop("inv", None) - self.last_message.pop("headers", None) - - def check_last_announcement(self, headers=None, inv=None): - """Test whether the last announcement received had the right header or the right inv. - - inv and headers should be lists of block hashes.""" - - test_function = lambda: self.block_announced - wait_until(test_function, timeout=60, lock=mininode_lock) - - with mininode_lock: - self.block_announced = False - - compare_inv = [] - if "inv" in self.last_message: - compare_inv = [x.hash for x in self.last_message["inv"].inv] - if inv is not None: - assert_equal(compare_inv, inv) - - compare_headers = [] - if "headers" in self.last_message: - compare_headers = [x.sha256 for x in self.last_message["headers"].headers] - if headers is not None: - assert_equal(compare_headers, headers) - - self.last_message.pop("inv", None) - self.last_message.pop("headers", None) - -class SendHeadersTest(BitcoinTestFramework): - def set_test_params(self): - self.setup_clean_chain = True - self.num_nodes = 2 - - def mine_blocks(self, count): - """Mine count blocks and return the new tip.""" - - # Clear out last block announcement from each p2p listener - [x.clear_last_announcement() for x in self.nodes[0].p2ps] - self.nodes[0].generate(count) - return int(self.nodes[0].getbestblockhash(), 16) - - def mine_reorg(self, length): - """Mine a reorg that invalidates length blocks (replacing them with # length+1 blocks). - - Note: we clear the state of our p2p connections after the - to-be-reorged-out blocks are mined, so that we don't break later tests. - return the list of block hashes newly mined.""" - - self.nodes[0].generate(length) # make sure all invalidated blocks are node0's - sync_blocks(self.nodes, wait=0.1) - for x in self.nodes[0].p2ps: - x.wait_for_block_announcement(int(self.nodes[0].getbestblockhash(), 16)) - x.clear_last_announcement() - - tip_height = self.nodes[1].getblockcount() - hash_to_invalidate = self.nodes[1].getblockhash(tip_height - (length - 1)) - self.nodes[1].invalidateblock(hash_to_invalidate) - all_hashes = self.nodes[1].generate(length + 1) # Must be longer than the orig chain - sync_blocks(self.nodes, wait=0.1) - return [int(x, 16) for x in all_hashes] - - def run_test(self): - # Setup the p2p connections and start up the network thread. - inv_node = self.nodes[0].add_p2p_connection(BaseNode()) - # Make sure NODE_NETWORK is not set for test_node, so no block download - # will occur outside of direct fetching - test_node = self.nodes[0].add_p2p_connection(BaseNode(), services=NODE_WITNESS) - - network_thread_start() - - # Test logic begins here - inv_node.wait_for_verack() - test_node.wait_for_verack() - - # Ensure verack's have been processed by our peer - inv_node.sync_with_ping() - test_node.sync_with_ping() - - self.test_null_locators(test_node, inv_node) - self.test_nonnull_locators(test_node, inv_node) - - def test_null_locators(self, test_node, inv_node): - tip = self.nodes[0].getblockheader(self.nodes[0].generate(1)[0]) - tip_hash = int(tip["hash"], 16) - - inv_node.check_last_announcement(inv=[tip_hash], headers=[]) - test_node.check_last_announcement(inv=[tip_hash], headers=[]) - - self.log.info("Verify getheaders with null locator and valid hashstop returns headers.") - test_node.clear_last_announcement() - test_node.send_get_headers(locator=[], hashstop=tip_hash) - test_node.check_last_announcement(headers=[tip_hash]) - - self.log.info("Verify getheaders with null locator and invalid hashstop does not return headers.") - block = create_block(int(tip["hash"], 16), create_coinbase(tip["height"] + 1), tip["mediantime"] + 1) - block.solve() - test_node.send_header_for_blocks([block]) - test_node.clear_last_announcement() - test_node.send_get_headers(locator=[], hashstop=int(block.hash, 16)) - test_node.sync_with_ping() - assert_equal(test_node.block_announced, False) - inv_node.clear_last_announcement() - test_node.send_message(msg_block(block)) - inv_node.check_last_announcement(inv=[int(block.hash, 16)], headers=[]) - - def test_nonnull_locators(self, test_node, inv_node): - tip = int(self.nodes[0].getbestblockhash(), 16) - - # PART 1 - # 1. Mine a block; expect inv announcements each time - self.log.info("Part 1: headers don't start before sendheaders message...") - for i in range(4): - old_tip = tip - tip = self.mine_blocks(1) - inv_node.check_last_announcement(inv=[tip], headers=[]) - test_node.check_last_announcement(inv=[tip], headers=[]) - # Try a few different responses; none should affect next announcement - if i == 0: - # first request the block - test_node.send_get_data([tip]) - test_node.wait_for_block(tip) - elif i == 1: - # next try requesting header and block - test_node.send_get_headers(locator=[old_tip], hashstop=tip) - test_node.send_get_data([tip]) - test_node.wait_for_block(tip) - test_node.clear_last_announcement() # since we requested headers... - elif i == 2: - # this time announce own block via headers - height = self.nodes[0].getblockcount() - last_time = self.nodes[0].getblock(self.nodes[0].getbestblockhash())['time'] - block_time = last_time + 1 - new_block = create_block(tip, create_coinbase(height + 1), block_time) - new_block.solve() - test_node.send_header_for_blocks([new_block]) - test_node.wait_for_getdata([new_block.sha256]) - test_node.send_message(msg_block(new_block)) - test_node.sync_with_ping() # make sure this block is processed - inv_node.clear_last_announcement() - test_node.clear_last_announcement() - - self.log.info("Part 1: success!") - self.log.info("Part 2: announce blocks with headers after sendheaders message...") - # PART 2 - # 2. Send a sendheaders message and test that headers announcements - # commence and keep working. - test_node.send_message(msg_sendheaders()) - prev_tip = int(self.nodes[0].getbestblockhash(), 16) - test_node.send_get_headers(locator=[prev_tip], hashstop=0) - test_node.sync_with_ping() - - # Now that we've synced headers, headers announcements should work - tip = self.mine_blocks(1) - inv_node.check_last_announcement(inv=[tip], headers=[]) - test_node.check_last_announcement(headers=[tip]) - - height = self.nodes[0].getblockcount() + 1 - block_time += 10 # Advance far enough ahead - for i in range(10): - # Mine i blocks, and alternate announcing either via - # inv (of tip) or via headers. After each, new blocks - # mined by the node should successfully be announced - # with block header, even though the blocks are never requested - for j in range(2): - blocks = [] - for b in range(i + 1): - blocks.append(create_block(tip, create_coinbase(height), block_time)) - blocks[-1].solve() - tip = blocks[-1].sha256 - block_time += 1 - height += 1 - if j == 0: - # Announce via inv - test_node.send_block_inv(tip) - test_node.wait_for_getheaders() - # Should have received a getheaders now - test_node.send_header_for_blocks(blocks) - # Test that duplicate inv's won't result in duplicate - # getdata requests, or duplicate headers announcements - [inv_node.send_block_inv(x.sha256) for x in blocks] - test_node.wait_for_getdata([x.sha256 for x in blocks]) - inv_node.sync_with_ping() - else: - # Announce via headers - test_node.send_header_for_blocks(blocks) - test_node.wait_for_getdata([x.sha256 for x in blocks]) - # Test that duplicate headers won't result in duplicate - # getdata requests (the check is further down) - inv_node.send_header_for_blocks(blocks) - inv_node.sync_with_ping() - [test_node.send_message(msg_block(x)) for x in blocks] - test_node.sync_with_ping() - inv_node.sync_with_ping() - # This block should not be announced to the inv node (since it also - # broadcast it) - assert "inv" not in inv_node.last_message - assert "headers" not in inv_node.last_message - tip = self.mine_blocks(1) - inv_node.check_last_announcement(inv=[tip], headers=[]) - test_node.check_last_announcement(headers=[tip]) - height += 1 - block_time += 1 - - self.log.info("Part 2: success!") - - self.log.info("Part 3: headers announcements can stop after large reorg, and resume after headers/inv from peer...") - - # PART 3. Headers announcements can stop after large reorg, and resume after - # getheaders or inv from peer. - for j in range(2): - # First try mining a reorg that can propagate with header announcement - new_block_hashes = self.mine_reorg(length=7) - tip = new_block_hashes[-1] - inv_node.check_last_announcement(inv=[tip], headers=[]) - test_node.check_last_announcement(headers=new_block_hashes) - - block_time += 8 - - # Mine a too-large reorg, which should be announced with a single inv - new_block_hashes = self.mine_reorg(length=8) - tip = new_block_hashes[-1] - inv_node.check_last_announcement(inv=[tip], headers=[]) - test_node.check_last_announcement(inv=[tip], headers=[]) - - block_time += 9 - - fork_point = self.nodes[0].getblock("%02x" % new_block_hashes[0])["previousblockhash"] - fork_point = int(fork_point, 16) - - # Use getblocks/getdata - test_node.send_getblocks(locator=[fork_point]) - test_node.check_last_announcement(inv=new_block_hashes, headers=[]) - test_node.send_get_data(new_block_hashes) - test_node.wait_for_block(new_block_hashes[-1]) - - for i in range(3): - # Mine another block, still should get only an inv - tip = self.mine_blocks(1) - inv_node.check_last_announcement(inv=[tip], headers=[]) - test_node.check_last_announcement(inv=[tip], headers=[]) - if i == 0: - # Just get the data -- shouldn't cause headers announcements to resume - test_node.send_get_data([tip]) - test_node.wait_for_block(tip) - elif i == 1: - # Send a getheaders message that shouldn't trigger headers announcements - # to resume (best header sent will be too old) - test_node.send_get_headers(locator=[fork_point], hashstop=new_block_hashes[1]) - test_node.send_get_data([tip]) - test_node.wait_for_block(tip) - elif i == 2: - test_node.send_get_data([tip]) - test_node.wait_for_block(tip) - # This time, try sending either a getheaders to trigger resumption - # of headers announcements, or mine a new block and inv it, also - # triggering resumption of headers announcements. - if j == 0: - test_node.send_get_headers(locator=[tip], hashstop=0) - test_node.sync_with_ping() - else: - test_node.send_block_inv(tip) - test_node.sync_with_ping() - # New blocks should now be announced with header - tip = self.mine_blocks(1) - inv_node.check_last_announcement(inv=[tip], headers=[]) - test_node.check_last_announcement(headers=[tip]) - - self.log.info("Part 3: success!") - - self.log.info("Part 4: Testing direct fetch behavior...") - tip = self.mine_blocks(1) - height = self.nodes[0].getblockcount() + 1 - last_time = self.nodes[0].getblock(self.nodes[0].getbestblockhash())['time'] - block_time = last_time + 1 - - # Create 2 blocks. Send the blocks, then send the headers. - blocks = [] - for b in range(2): - blocks.append(create_block(tip, create_coinbase(height), block_time)) - blocks[-1].solve() - tip = blocks[-1].sha256 - block_time += 1 - height += 1 - inv_node.send_message(msg_block(blocks[-1])) - - inv_node.sync_with_ping() # Make sure blocks are processed - test_node.last_message.pop("getdata", None) - test_node.send_header_for_blocks(blocks) - test_node.sync_with_ping() - # should not have received any getdata messages - with mininode_lock: - assert "getdata" not in test_node.last_message - - # This time, direct fetch should work - blocks = [] - for b in range(3): - blocks.append(create_block(tip, create_coinbase(height), block_time)) - blocks[-1].solve() - tip = blocks[-1].sha256 - block_time += 1 - height += 1 - - test_node.send_header_for_blocks(blocks) - test_node.sync_with_ping() - test_node.wait_for_getdata([x.sha256 for x in blocks], timeout=DIRECT_FETCH_RESPONSE_TIME) - - [test_node.send_message(msg_block(x)) for x in blocks] - - test_node.sync_with_ping() - - # Now announce a header that forks the last two blocks - tip = blocks[0].sha256 - height -= 1 - blocks = [] - - # Create extra blocks for later - for b in range(20): - blocks.append(create_block(tip, create_coinbase(height), block_time)) - blocks[-1].solve() - tip = blocks[-1].sha256 - block_time += 1 - height += 1 - - # Announcing one block on fork should not trigger direct fetch - # (less work than tip) - test_node.last_message.pop("getdata", None) - test_node.send_header_for_blocks(blocks[0:1]) - test_node.sync_with_ping() - with mininode_lock: - assert "getdata" not in test_node.last_message - - # Announcing one more block on fork should trigger direct fetch for - # both blocks (same work as tip) - test_node.send_header_for_blocks(blocks[1:2]) - test_node.sync_with_ping() - test_node.wait_for_getdata([x.sha256 for x in blocks[0:2]], timeout=DIRECT_FETCH_RESPONSE_TIME) - - # Announcing 16 more headers should trigger direct fetch for 14 more - # blocks - test_node.send_header_for_blocks(blocks[2:18]) - test_node.sync_with_ping() - test_node.wait_for_getdata([x.sha256 for x in blocks[2:16]], timeout=DIRECT_FETCH_RESPONSE_TIME) - - # Announcing 1 more header should not trigger any response - test_node.last_message.pop("getdata", None) - test_node.send_header_for_blocks(blocks[18:19]) - test_node.sync_with_ping() - with mininode_lock: - assert "getdata" not in test_node.last_message - - self.log.info("Part 4: success!") - - # Now deliver all those blocks we announced. - [test_node.send_message(msg_block(x)) for x in blocks] - - self.log.info("Part 5: Testing handling of unconnecting headers") - # First we test that receipt of an unconnecting header doesn't prevent - # chain sync. - for i in range(10): - test_node.last_message.pop("getdata", None) - blocks = [] - # Create two more blocks. - for j in range(2): - blocks.append(create_block(tip, create_coinbase(height), block_time)) - blocks[-1].solve() - tip = blocks[-1].sha256 - block_time += 1 - height += 1 - # Send the header of the second block -> this won't connect. - with mininode_lock: - test_node.last_message.pop("getheaders", None) - test_node.send_header_for_blocks([blocks[1]]) - test_node.wait_for_getheaders() - test_node.send_header_for_blocks(blocks) - test_node.wait_for_getdata([x.sha256 for x in blocks]) - [test_node.send_message(msg_block(x)) for x in blocks] - test_node.sync_with_ping() - assert_equal(int(self.nodes[0].getbestblockhash(), 16), blocks[1].sha256) - - blocks = [] - # Now we test that if we repeatedly don't send connecting headers, we - # don't go into an infinite loop trying to get them to connect. - MAX_UNCONNECTING_HEADERS = 10 - for j in range(MAX_UNCONNECTING_HEADERS + 1): - blocks.append(create_block(tip, create_coinbase(height), block_time)) - blocks[-1].solve() - tip = blocks[-1].sha256 - block_time += 1 - height += 1 - - for i in range(1, MAX_UNCONNECTING_HEADERS): - # Send a header that doesn't connect, check that we get a getheaders. - with mininode_lock: - test_node.last_message.pop("getheaders", None) - test_node.send_header_for_blocks([blocks[i]]) - test_node.wait_for_getheaders() - - # Next header will connect, should re-set our count: - test_node.send_header_for_blocks([blocks[0]]) - - # Remove the first two entries (blocks[1] would connect): - blocks = blocks[2:] - - # Now try to see how many unconnecting headers we can send - # before we get disconnected. Should be 5*MAX_UNCONNECTING_HEADERS - for i in range(5 * MAX_UNCONNECTING_HEADERS - 1): - # Send a header that doesn't connect, check that we get a getheaders. - with mininode_lock: - test_node.last_message.pop("getheaders", None) - test_node.send_header_for_blocks([blocks[i % len(blocks)]]) - test_node.wait_for_getheaders() - - # Eventually this stops working. - test_node.send_header_for_blocks([blocks[-1]]) - - # Should get disconnected - test_node.wait_for_disconnect() - - self.log.info("Part 5: success!") - - # Finally, check that the inv node never received a getdata request, - # throughout the test - assert "getdata" not in inv_node.last_message - -if __name__ == '__main__': - SendHeadersTest().main() diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index edc1fed8cb..26beef5a52 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -60,16 +60,16 @@ BASE_SCRIPTS= [ # vv Tests less than 5m vv 'feature_block.py', 'fundrawtransaction.py', - 'p2p-compactblocks.py', + 'p2p_compactblocks.py', 'feature_segwit.py', # vv Tests less than 2m vv 'wallet_basic.py', 'wallet_accounts.py', - 'p2p-segwit.py', + 'p2p_segwit.py', 'wallet_dump.py', 'listtransactions.py', # vv Tests less than 60s vv - 'sendheaders.py', + 'p2p_sendheaders.py', 'wallet_zapwallettxes.py', 'wallet_importmulti.py', 'mempool_limit.py', @@ -99,17 +99,17 @@ BASE_SCRIPTS= [ 'multi_rpc.py', 'feature_proxy.py', 'signrawtransactions.py', - 'disconnect_ban.py', + 'p2p_disconnect_ban.py', 'decodescript.py', 'blockchain.py', 'deprecated_rpc.py', 'wallet_disable.py', 'net.py', 'wallet_keypool.py', - 'p2p-mempool.py', + 'p2p_mempool.py', 'prioritise_transaction.py', - 'invalidblockrequest.py', - 'invalidtxrequest.py', + 'p2p_invalid_block.py', + 'p2p_invalid_tx.py', 'feature_versionbits_warning.py', 'preciousblock.py', 'wallet_importprunedfunds.py', @@ -120,18 +120,18 @@ BASE_SCRIPTS= [ 'wallet_bumpfee.py', 'rpcnamedargs.py', 'wallet_listsinceblock.py', - 'p2p-leaktests.py', + 'p2p_leak.py', 'wallet_encryption.py', 'feature_dersig.py', 'feature_cltv.py', 'uptime.py', 'wallet_resendwallettransactions.py', 'feature_minchainwork.py', - 'p2p-fingerprint.py', + 'p2p_fingerprint.py', 'feature_uacomment.py', - 'p2p-acceptblock.py', + 'p2p_unrequested_blocks.py', 'feature_logging.py', - 'node_network_limited.py', + 'p2p_node_network_limited.py', 'feature_config_args.py', # Don't append tests at the end to avoid merge conflicts # Put them in a random line within the section that fits their approximate run-time @@ -150,10 +150,10 @@ EXTENDED_SCRIPTS = [ # vv Tests less than 2m vv 'feature_bip68_sequence.py', 'getblocktemplate_longpoll.py', - 'p2p-timeouts.py', + 'p2p_timeouts.py', # vv Tests less than 60s vv 'feature_bip9_softforks.py', - 'p2p-feefilter.py', + 'p2p_feefilter.py', 'rpcbind_test.py', # vv Tests less than 30s vv 'feature_assumevalid.py', @@ -474,7 +474,7 @@ class TestResult(): def check_script_prefixes(): """Check that no more than `EXPECTED_VIOLATION_COUNT` of the test scripts don't start with one of the allowed name prefixes.""" - EXPECTED_VIOLATION_COUNT = 37 + EXPECTED_VIOLATION_COUNT = 24 # LEEWAY is provided as a transition measure, so that pull-requests # that introduce new tests that don't conform with the naming -- cgit v1.2.3