diff options
Diffstat (limited to 'qa')
| -rwxr-xr-x | qa/pull-tester/rpc-tests.py | 1 | ||||
| -rwxr-xr-x | qa/rpc-tests/bip65-cltv-p2p.py | 40 | ||||
| -rwxr-xr-x | qa/rpc-tests/bip65-cltv.py | 3 | ||||
| -rwxr-xr-x | qa/rpc-tests/bipdersig-p2p.py | 42 | ||||
| -rwxr-xr-x | qa/rpc-tests/p2p-compactblocks.py | 608 | ||||
| -rwxr-xr-x | qa/rpc-tests/test_framework/mininode.py | 373 | ||||
| -rw-r--r-- | qa/rpc-tests/test_framework/siphash.py | 64 | ||||
| -rwxr-xr-x | qa/rpc-tests/wallet-dump.py | 144 |
8 files changed, 1077 insertions, 198 deletions
diff --git a/qa/pull-tester/rpc-tests.py b/qa/pull-tester/rpc-tests.py index e3921cfbed..f65a3eefc3 100755 --- a/qa/pull-tester/rpc-tests.py +++ b/qa/pull-tester/rpc-tests.py @@ -142,6 +142,7 @@ testScripts = [ 'segwit.py', 'importprunedfunds.py', 'signmessages.py', + 'p2p-compactblocks.py', ] if ENABLE_ZMQ: testScripts.append('zmq_test.py') diff --git a/qa/rpc-tests/bip65-cltv-p2p.py b/qa/rpc-tests/bip65-cltv-p2p.py index 754b6873b7..e903b2fbf0 100755 --- a/qa/rpc-tests/bip65-cltv-p2p.py +++ b/qa/rpc-tests/bip65-cltv-p2p.py @@ -71,9 +71,9 @@ class BIP65Test(ComparisonTestFramework): self.nodeaddress = self.nodes[0].getnewaddress() self.last_block_time = int(time.time()) - ''' 98 more version 3 blocks ''' + ''' 398 more version 3 blocks ''' test_blocks = [] - for i in range(98): + for i in range(398): block = create_block(self.tip, create_coinbase(height), self.last_block_time + 1) block.nVersion = 3 block.rehash() @@ -118,24 +118,6 @@ class BIP65Test(ComparisonTestFramework): height += 1 yield TestInstance([[block, True]]) - ''' - Check that the new CLTV rules are enforced in the 751st version 4 - block. - ''' - spendtx = self.create_transaction(self.nodes[0], - self.coinbase_blocks[1], self.nodeaddress, 1.0) - cltv_invalidate(spendtx) - spendtx.rehash() - - block = create_block(self.tip, create_coinbase(height), self.last_block_time + 1) - block.nVersion = 4 - block.vtx.append(spendtx) - block.hashMerkleRoot = block.calc_merkle_root() - block.rehash() - block.solve() - self.last_block_time += 1 - yield TestInstance([[block, False]]) - ''' Mine 199 new version blocks on last valid tip ''' test_blocks = [] for i in range(199): @@ -169,6 +151,24 @@ class BIP65Test(ComparisonTestFramework): height += 1 yield TestInstance([[block, True]]) + ''' + Check that the new CLTV rules are enforced in the 951st version 4 + block. + ''' + spendtx = self.create_transaction(self.nodes[0], + self.coinbase_blocks[1], self.nodeaddress, 1.0) + cltv_invalidate(spendtx) + spendtx.rehash() + + block = create_block(self.tip, create_coinbase(height), self.last_block_time + 1) + block.nVersion = 4 + block.vtx.append(spendtx) + block.hashMerkleRoot = block.calc_merkle_root() + block.rehash() + block.solve() + self.last_block_time += 1 + yield TestInstance([[block, False]]) + ''' Mine 1 old version block, should be invalid ''' block = create_block(self.tip, create_coinbase(height), self.last_block_time + 1) block.nVersion = 3 diff --git a/qa/rpc-tests/bip65-cltv.py b/qa/rpc-tests/bip65-cltv.py index abba7fc20e..baa77b92a0 100755 --- a/qa/rpc-tests/bip65-cltv.py +++ b/qa/rpc-tests/bip65-cltv.py @@ -30,7 +30,8 @@ class BIP65Test(BitcoinTestFramework): cnt = self.nodes[0].getblockcount() # Mine some old-version blocks - self.nodes[1].generate(100) + self.nodes[1].generate(200) + cnt += 100 self.sync_all() if (self.nodes[0].getblockcount() != cnt + 100): raise AssertionError("Failed to mine 100 version=3 blocks") diff --git a/qa/rpc-tests/bipdersig-p2p.py b/qa/rpc-tests/bipdersig-p2p.py index 4e4936a4ae..3bad5af5e6 100755 --- a/qa/rpc-tests/bipdersig-p2p.py +++ b/qa/rpc-tests/bipdersig-p2p.py @@ -79,9 +79,9 @@ class BIP66Test(ComparisonTestFramework): self.nodeaddress = self.nodes[0].getnewaddress() self.last_block_time = int(time.time()) - ''' 98 more version 2 blocks ''' + ''' 298 more version 2 blocks ''' test_blocks = [] - for i in range(98): + for i in range(298): block = create_block(self.tip, create_coinbase(height), self.last_block_time + 1) block.nVersion = 2 block.rehash() @@ -124,25 +124,7 @@ class BIP66Test(ComparisonTestFramework): self.last_block_time += 1 self.tip = block.sha256 height += 1 - yield TestInstance([[block, True]]) - - ''' - Check that the new DERSIG rules are enforced in the 751st version 3 - block. - ''' - spendtx = self.create_transaction(self.nodes[0], - self.coinbase_blocks[1], self.nodeaddress, 1.0) - unDERify(spendtx) - spendtx.rehash() - - block = create_block(self.tip, create_coinbase(height), self.last_block_time + 1) - block.nVersion = 3 - block.vtx.append(spendtx) - block.hashMerkleRoot = block.calc_merkle_root() - block.rehash() - block.solve() - self.last_block_time += 1 - yield TestInstance([[block, False]]) + yield TestInstance([[block, True]]) ''' Mine 199 new version blocks on last valid tip ''' test_blocks = [] @@ -177,6 +159,24 @@ class BIP66Test(ComparisonTestFramework): height += 1 yield TestInstance([[block, True]]) + ''' + Check that the new DERSIG rules are enforced in the 951st version 3 + block. + ''' + spendtx = self.create_transaction(self.nodes[0], + self.coinbase_blocks[1], self.nodeaddress, 1.0) + unDERify(spendtx) + spendtx.rehash() + + block = create_block(self.tip, create_coinbase(height), self.last_block_time + 1) + block.nVersion = 3 + block.vtx.append(spendtx) + block.hashMerkleRoot = block.calc_merkle_root() + block.rehash() + block.solve() + self.last_block_time += 1 + yield TestInstance([[block, False]]) + ''' Mine 1 old version block, should be invalid ''' block = create_block(self.tip, create_coinbase(height), self.last_block_time + 1) block.nVersion = 2 diff --git a/qa/rpc-tests/p2p-compactblocks.py b/qa/rpc-tests/p2p-compactblocks.py new file mode 100755 index 0000000000..7fe7ecc16c --- /dev/null +++ b/qa/rpc-tests/p2p-compactblocks.py @@ -0,0 +1,608 @@ +#!/usr/bin/env python3 +# Copyright (c) 2016 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +from test_framework.mininode import * +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import * +from test_framework.blocktools import create_block, create_coinbase +from test_framework.siphash import siphash256 +from test_framework.script import CScript, OP_TRUE + +''' +CompactBlocksTest -- test compact blocks (BIP 152) +''' + + +# TestNode: A peer we use to send messages to bitcoind, and store responses. +class TestNode(SingleNodeConnCB): + def __init__(self): + SingleNodeConnCB.__init__(self) + self.last_sendcmpct = None + self.last_headers = None + self.last_inv = None + self.last_cmpctblock = None + self.block_announced = False + self.last_getdata = None + self.last_getblocktxn = None + self.last_block = None + self.last_blocktxn = None + + def on_sendcmpct(self, conn, message): + self.last_sendcmpct = message + + def on_block(self, conn, message): + self.last_block = message + + def on_cmpctblock(self, conn, message): + self.last_cmpctblock = message + self.block_announced = True + + def on_headers(self, conn, message): + self.last_headers = message + self.block_announced = True + + def on_inv(self, conn, message): + self.last_inv = message + self.block_announced = True + + def on_getdata(self, conn, message): + self.last_getdata = message + + def on_getblocktxn(self, conn, message): + self.last_getblocktxn = message + + def on_blocktxn(self, conn, message): + self.last_blocktxn = message + + # Requires caller to hold mininode_lock + def received_block_announcement(self): + return self.block_announced + + def clear_block_announcement(self): + with mininode_lock: + self.block_announced = False + self.last_inv = None + self.last_headers = None + self.last_cmpctblock = None + + def get_headers(self, locator, hashstop): + msg = msg_getheaders() + msg.locator.vHave = locator + msg.hashstop = hashstop + self.connection.send_message(msg) + + def send_header_for_blocks(self, new_blocks): + headers_message = msg_headers() + headers_message.headers = [CBlockHeader(b) for b in new_blocks] + self.send_message(headers_message) + + +class CompactBlocksTest(BitcoinTestFramework): + def __init__(self): + super().__init__() + self.setup_clean_chain = True + self.num_nodes = 1 + self.utxos = [] + + def setup_network(self): + self.nodes = [] + + # Turn off segwit in this test, as compact blocks don't currently work + # with segwit. (After BIP 152 is updated to support segwit, we can + # test behavior with and without segwit enabled by adding a second node + # to the test.) + self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, [["-debug", "-logtimemicros=1", "-bip9params=segwit:0:0"]]) + + def build_block_on_tip(self): + height = self.nodes[0].getblockcount() + tip = self.nodes[0].getbestblockhash() + mtp = self.nodes[0].getblockheader(tip)['mediantime'] + block = create_block(int(tip, 16), create_coinbase(height + 1), mtp + 1) + block.solve() + return block + + # Create 10 more anyone-can-spend utxo's for testing. + def make_utxos(self): + block = self.build_block_on_tip() + self.test_node.send_and_ping(msg_block(block)) + assert(int(self.nodes[0].getbestblockhash(), 16) == block.sha256) + self.nodes[0].generate(100) + + total_value = block.vtx[0].vout[0].nValue + out_value = total_value // 10 + tx = CTransaction() + tx.vin.append(CTxIn(COutPoint(block.vtx[0].sha256, 0), b'')) + for i in range(10): + tx.vout.append(CTxOut(out_value, CScript([OP_TRUE]))) + tx.rehash() + + block2 = self.build_block_on_tip() + block2.vtx.append(tx) + block2.hashMerkleRoot = block2.calc_merkle_root() + block2.solve() + self.test_node.send_and_ping(msg_block(block2)) + assert_equal(int(self.nodes[0].getbestblockhash(), 16), block2.sha256) + self.utxos.extend([[tx.sha256, i, out_value] for i in range(10)]) + return + + # Test "sendcmpct": + # - No compact block announcements or getdata(MSG_CMPCT_BLOCK) unless + # sendcmpct is sent. + # - If sendcmpct is sent with version > 0, the message is ignored. + # - If sendcmpct is sent with boolean 0, then block announcements are not + # made with compact blocks. + # - If sendcmpct is then sent with boolean 1, then new block announcements + # are made with compact blocks. + def test_sendcmpct(self): + print("Testing SENDCMPCT p2p message... ") + + # Make sure we get a version 0 SENDCMPCT message from our peer + def received_sendcmpct(): + return (self.test_node.last_sendcmpct is not None) + got_message = wait_until(received_sendcmpct, timeout=30) + assert(got_message) + assert_equal(self.test_node.last_sendcmpct.version, 1) + + tip = int(self.nodes[0].getbestblockhash(), 16) + + def check_announcement_of_new_block(node, peer, predicate): + self.test_node.clear_block_announcement() + node.generate(1) + got_message = wait_until(peer.received_block_announcement, timeout=30) + assert(got_message) + with mininode_lock: + assert(predicate) + + # We shouldn't get any block announcements via cmpctblock yet. + check_announcement_of_new_block(self.nodes[0], self.test_node, lambda: self.test_node.last_cmpctblock is None) + + # Try one more time, this time after requesting headers. + self.test_node.clear_block_announcement() + self.test_node.get_headers(locator=[tip], hashstop=0) + wait_until(self.test_node.received_block_announcement, timeout=30) + self.test_node.clear_block_announcement() + + check_announcement_of_new_block(self.nodes[0], self.test_node, lambda: self.test_node.last_cmpctblock is None and self.test_node.last_inv is not None) + + # Now try a SENDCMPCT message with too-high version + sendcmpct = msg_sendcmpct() + sendcmpct.version = 2 + self.test_node.send_message(sendcmpct) + + check_announcement_of_new_block(self.nodes[0], self.test_node, lambda: self.test_node.last_cmpctblock is None) + + # Now try a SENDCMPCT message with valid version, but announce=False + self.test_node.send_message(msg_sendcmpct()) + check_announcement_of_new_block(self.nodes[0], self.test_node, lambda: self.test_node.last_cmpctblock is None) + + # Finally, try a SENDCMPCT message with announce=True + sendcmpct.version = 1 + sendcmpct.announce = True + self.test_node.send_message(sendcmpct) + check_announcement_of_new_block(self.nodes[0], self.test_node, lambda: self.test_node.last_cmpctblock is not None) + + # Try one more time + check_announcement_of_new_block(self.nodes[0], self.test_node, lambda: self.test_node.last_cmpctblock is not None) + + # Try one more time, after turning on sendheaders + self.test_node.send_message(msg_sendheaders()) + check_announcement_of_new_block(self.nodes[0], self.test_node, lambda: self.test_node.last_cmpctblock is not None) + + # Now turn off announcements + sendcmpct.announce = False + check_announcement_of_new_block(self.nodes[0], self.test_node, lambda: self.test_node.last_cmpctblock is None and self.test_node.last_headers is not None) + + # This test actually causes bitcoind to (reasonably!) disconnect us, so do this last. + def test_invalid_cmpctblock_message(self): + print("Testing invalid index in cmpctblock message...") + self.nodes[0].generate(101) + block = self.build_block_on_tip() + + cmpct_block = P2PHeaderAndShortIDs() + cmpct_block.header = CBlockHeader(block) + cmpct_block.prefilled_txn_length = 1 + # This index will be too high + prefilled_txn = PrefilledTransaction(1, block.vtx[0]) + cmpct_block.prefilled_txn = [prefilled_txn] + self.test_node.send_and_ping(msg_cmpctblock(cmpct_block)) + assert(int(self.nodes[0].getbestblockhash(), 16) == block.hashPrevBlock) + + # Compare the generated shortids to what we expect based on BIP 152, given + # bitcoind's choice of nonce. + def test_compactblock_construction(self): + print("Testing compactblock headers and shortIDs are correct...") + + # Generate a bunch of transactions. + self.nodes[0].generate(101) + num_transactions = 25 + address = self.nodes[0].getnewaddress() + for i in range(num_transactions): + self.nodes[0].sendtoaddress(address, 0.1) + + # Now mine a block, and look at the resulting compact block. + self.test_node.clear_block_announcement() + block_hash = int(self.nodes[0].generate(1)[0], 16) + + # Store the raw block in our internal format. + block = FromHex(CBlock(), self.nodes[0].getblock("%02x" % block_hash, False)) + [tx.calc_sha256() for tx in block.vtx] + block.rehash() + + # Don't care which type of announcement came back for this test; just + # request the compact block if we didn't get one yet. + wait_until(self.test_node.received_block_announcement, timeout=30) + + with mininode_lock: + if self.test_node.last_cmpctblock is None: + self.test_node.clear_block_announcement() + inv = CInv(4, block_hash) # 4 == "CompactBlock" + self.test_node.send_message(msg_getdata([inv])) + + wait_until(self.test_node.received_block_announcement, timeout=30) + + # Now we should have the compactblock + header_and_shortids = None + with mininode_lock: + assert(self.test_node.last_cmpctblock is not None) + # Convert the on-the-wire representation to absolute indexes + header_and_shortids = HeaderAndShortIDs(self.test_node.last_cmpctblock.header_and_shortids) + + # Check that we got the right block! + header_and_shortids.header.calc_sha256() + assert_equal(header_and_shortids.header.sha256, block_hash) + + # Make sure the prefilled_txn appears to have included the coinbase + assert(len(header_and_shortids.prefilled_txn) >= 1) + assert_equal(header_and_shortids.prefilled_txn[0].index, 0) + + # Check that all prefilled_txn entries match what's in the block. + for entry in header_and_shortids.prefilled_txn: + entry.tx.calc_sha256() + assert_equal(entry.tx.sha256, block.vtx[entry.index].sha256) + + # Check that the cmpctblock message announced all the transactions. + assert_equal(len(header_and_shortids.prefilled_txn) + len(header_and_shortids.shortids), len(block.vtx)) + + # And now check that all the shortids are as expected as well. + # Determine the siphash keys to use. + [k0, k1] = header_and_shortids.get_siphash_keys() + + index = 0 + while index < len(block.vtx): + if (len(header_and_shortids.prefilled_txn) > 0 and + header_and_shortids.prefilled_txn[0].index == index): + # Already checked prefilled transactions above + header_and_shortids.prefilled_txn.pop(0) + else: + shortid = calculate_shortid(k0, k1, block.vtx[index].sha256) + assert_equal(shortid, header_and_shortids.shortids[0]) + header_and_shortids.shortids.pop(0) + index += 1 + + # Test that bitcoind requests compact blocks when we announce new blocks + # via header or inv, and that responding to getblocktxn causes the block + # to be successfully reconstructed. + def test_compactblock_requests(self): + print("Testing compactblock requests... ") + + # Try announcing a block with an inv or header, expect a compactblock + # request + for announce in ["inv", "header"]: + block = self.build_block_on_tip() + with mininode_lock: + self.test_node.last_getdata = None + + if announce == "inv": + self.test_node.send_message(msg_inv([CInv(2, block.sha256)])) + else: + self.test_node.send_header_for_blocks([block]) + success = wait_until(lambda: self.test_node.last_getdata is not None, timeout=30) + assert(success) + assert_equal(len(self.test_node.last_getdata.inv), 1) + assert_equal(self.test_node.last_getdata.inv[0].type, 4) + assert_equal(self.test_node.last_getdata.inv[0].hash, block.sha256) + + # Send back a compactblock message that omits the coinbase + comp_block = HeaderAndShortIDs() + comp_block.header = CBlockHeader(block) + comp_block.nonce = 0 + comp_block.shortids = [1] # this is useless, and wrong + self.test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) + assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.hashPrevBlock) + # Expect a getblocktxn message. + with mininode_lock: + assert(self.test_node.last_getblocktxn is not None) + absolute_indexes = self.test_node.last_getblocktxn.block_txn_request.to_absolute() + assert_equal(absolute_indexes, [0]) # should be a coinbase request + + # Send the coinbase, and verify that the tip advances. + msg = msg_blocktxn() + msg.block_transactions.blockhash = block.sha256 + msg.block_transactions.transactions = [block.vtx[0]] + self.test_node.send_and_ping(msg) + assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.sha256) + + # Create a chain of transactions from given utxo, and add to a new block. + def build_block_with_transactions(self, utxo, num_transactions): + block = self.build_block_on_tip() + + for i in range(num_transactions): + tx = CTransaction() + tx.vin.append(CTxIn(COutPoint(utxo[0], utxo[1]), b'')) + tx.vout.append(CTxOut(utxo[2] - 1000, CScript([OP_TRUE]))) + tx.rehash() + utxo = [tx.sha256, 0, tx.vout[0].nValue] + block.vtx.append(tx) + + block.hashMerkleRoot = block.calc_merkle_root() + block.solve() + return block + + # Test that we only receive getblocktxn requests for transactions that the + # node needs, and that responding to them causes the block to be + # reconstructed. + def test_getblocktxn_requests(self): + print("Testing getblocktxn requests...") + + # First try announcing compactblocks that won't reconstruct, and verify + # that we receive getblocktxn messages back. + utxo = self.utxos.pop(0) + + block = self.build_block_with_transactions(utxo, 5) + self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue]) + + comp_block = HeaderAndShortIDs() + comp_block.initialize_from_block(block) + + self.test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) + with mininode_lock: + assert(self.test_node.last_getblocktxn is not None) + absolute_indexes = self.test_node.last_getblocktxn.block_txn_request.to_absolute() + assert_equal(absolute_indexes, [1, 2, 3, 4, 5]) + msg = msg_blocktxn() + msg.block_transactions = BlockTransactions(block.sha256, block.vtx[1:]) + self.test_node.send_and_ping(msg) + assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.sha256) + + utxo = self.utxos.pop(0) + block = self.build_block_with_transactions(utxo, 5) + self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue]) + + # Now try interspersing the prefilled transactions + comp_block.initialize_from_block(block, prefill_list=[0, 1, 5]) + self.test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) + with mininode_lock: + assert(self.test_node.last_getblocktxn is not None) + absolute_indexes = self.test_node.last_getblocktxn.block_txn_request.to_absolute() + assert_equal(absolute_indexes, [2, 3, 4]) + msg.block_transactions = BlockTransactions(block.sha256, block.vtx[2:5]) + self.test_node.send_and_ping(msg) + assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.sha256) + + # Now try giving one transaction ahead of time. + utxo = self.utxos.pop(0) + block = self.build_block_with_transactions(utxo, 5) + self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue]) + self.test_node.send_and_ping(msg_tx(block.vtx[1])) + assert(block.vtx[1].hash in self.nodes[0].getrawmempool()) + + # Prefill 4 out of the 6 transactions, and verify that only the one + # that was not in the mempool is requested. + comp_block.initialize_from_block(block, prefill_list=[0, 2, 3, 4]) + self.test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) + with mininode_lock: + assert(self.test_node.last_getblocktxn is not None) + absolute_indexes = self.test_node.last_getblocktxn.block_txn_request.to_absolute() + assert_equal(absolute_indexes, [5]) + + msg.block_transactions = BlockTransactions(block.sha256, [block.vtx[5]]) + self.test_node.send_and_ping(msg) + assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.sha256) + + # Now provide all transactions to the node before the block is + # announced and verify reconstruction happens immediately. + utxo = self.utxos.pop(0) + block = self.build_block_with_transactions(utxo, 10) + self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue]) + for tx in block.vtx[1:]: + self.test_node.send_message(msg_tx(tx)) + self.test_node.sync_with_ping() + # Make sure all transactions were accepted. + mempool = self.nodes[0].getrawmempool() + for tx in block.vtx[1:]: + assert(tx.hash in mempool) + + # Clear out last request. + with mininode_lock: + self.test_node.last_getblocktxn = None + + # Send compact block + comp_block.initialize_from_block(block, prefill_list=[0]) + self.test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) + with mininode_lock: + # Shouldn't have gotten a request for any transaction + assert(self.test_node.last_getblocktxn is None) + # Tip should have updated + assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.sha256) + + # Incorrectly responding to a getblocktxn shouldn't cause the block to be + # permanently failed. + def test_incorrect_blocktxn_response(self): + print("Testing handling of incorrect blocktxn responses...") + + if (len(self.utxos) == 0): + self.make_utxos() + utxo = self.utxos.pop(0) + + block = self.build_block_with_transactions(utxo, 10) + self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue]) + # Relay the first 5 transactions from the block in advance + for tx in block.vtx[1:6]: + self.test_node.send_message(msg_tx(tx)) + self.test_node.sync_with_ping() + # Make sure all transactions were accepted. + mempool = self.nodes[0].getrawmempool() + for tx in block.vtx[1:6]: + assert(tx.hash in mempool) + + # Send compact block + comp_block = HeaderAndShortIDs() + comp_block.initialize_from_block(block, prefill_list=[0]) + self.test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) + absolute_indexes = [] + with mininode_lock: + assert(self.test_node.last_getblocktxn is not None) + absolute_indexes = self.test_node.last_getblocktxn.block_txn_request.to_absolute() + assert_equal(absolute_indexes, [6, 7, 8, 9, 10]) + + # Now give an incorrect response. + # Note that it's possible for bitcoind to be smart enough to know we're + # lying, since it could check to see if the shortid matches what we're + # sending, and eg disconnect us for misbehavior. If that behavior + # change were made, we could just modify this test by having a + # different peer provide the block further down, so that we're still + # verifying that the block isn't marked bad permanently. This is good + # enough for now. + msg = msg_blocktxn() + msg.block_transactions = BlockTransactions(block.sha256, [block.vtx[5]] + block.vtx[7:]) + self.test_node.send_and_ping(msg) + + # Tip should not have updated + assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.hashPrevBlock) + + # We should receive a getdata request + success = wait_until(lambda: self.test_node.last_getdata is not None, timeout=10) + assert(success) + assert_equal(len(self.test_node.last_getdata.inv), 1) + assert_equal(self.test_node.last_getdata.inv[0].type, 2) + assert_equal(self.test_node.last_getdata.inv[0].hash, block.sha256) + + # Deliver the block + self.test_node.send_and_ping(msg_block(block)) + assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.sha256) + + def test_getblocktxn_handler(self): + print("Testing getblocktxn handler...") + + # bitcoind won't respond for blocks whose height is more than 15 blocks + # deep. + MAX_GETBLOCKTXN_DEPTH = 15 + chain_height = self.nodes[0].getblockcount() + current_height = chain_height + while (current_height >= chain_height - MAX_GETBLOCKTXN_DEPTH): + block_hash = self.nodes[0].getblockhash(current_height) + block = FromHex(CBlock(), self.nodes[0].getblock(block_hash, False)) + + msg = msg_getblocktxn() + msg.block_txn_request = BlockTransactionsRequest(int(block_hash, 16), []) + num_to_request = random.randint(1, len(block.vtx)) + msg.block_txn_request.from_absolute(sorted(random.sample(range(len(block.vtx)), num_to_request))) + self.test_node.send_message(msg) + success = wait_until(lambda: self.test_node.last_blocktxn is not None, timeout=10) + assert(success) + + [tx.calc_sha256() for tx in block.vtx] + with mininode_lock: + assert_equal(self.test_node.last_blocktxn.block_transactions.blockhash, int(block_hash, 16)) + all_indices = msg.block_txn_request.to_absolute() + for index in all_indices: + tx = self.test_node.last_blocktxn.block_transactions.transactions.pop(0) + tx.calc_sha256() + assert_equal(tx.sha256, block.vtx[index].sha256) + self.test_node.last_blocktxn = None + current_height -= 1 + + # Next request should be ignored, as we're past the allowed depth. + block_hash = self.nodes[0].getblockhash(current_height) + msg.block_txn_request = BlockTransactionsRequest(int(block_hash, 16), [0]) + self.test_node.send_and_ping(msg) + with mininode_lock: + assert_equal(self.test_node.last_blocktxn, None) + + def test_compactblocks_not_at_tip(self): + print("Testing compactblock requests/announcements not at chain tip...") + + # Test that requesting old compactblocks doesn't work. + MAX_CMPCTBLOCK_DEPTH = 11 + new_blocks = [] + for i in range(MAX_CMPCTBLOCK_DEPTH): + self.test_node.clear_block_announcement() + new_blocks.append(self.nodes[0].generate(1)[0]) + wait_until(self.test_node.received_block_announcement, timeout=30) + + self.test_node.clear_block_announcement() + self.test_node.send_message(msg_getdata([CInv(4, int(new_blocks[0], 16))])) + success = wait_until(lambda: self.test_node.last_cmpctblock is not None, timeout=30) + assert(success) + + self.test_node.clear_block_announcement() + self.nodes[0].generate(1) + wait_until(self.test_node.received_block_announcement, timeout=30) + self.test_node.clear_block_announcement() + self.test_node.send_message(msg_getdata([CInv(4, int(new_blocks[0], 16))])) + success = wait_until(lambda: self.test_node.last_block is not None, timeout=30) + assert(success) + with mininode_lock: + self.test_node.last_block.block.calc_sha256() + assert_equal(self.test_node.last_block.block.sha256, int(new_blocks[0], 16)) + + # Generate an old compactblock, and verify that it's not accepted. + cur_height = self.nodes[0].getblockcount() + hashPrevBlock = int(self.nodes[0].getblockhash(cur_height-5), 16) + block = self.build_block_on_tip() + block.hashPrevBlock = hashPrevBlock + block.solve() + + comp_block = HeaderAndShortIDs() + comp_block.initialize_from_block(block) + self.test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) + + tips = self.nodes[0].getchaintips() + found = False + for x in tips: + if x["hash"] == block.hash: + assert_equal(x["status"], "headers-only") + found = True + break + assert(found) + + # Requesting this block via getblocktxn should silently fail + # (to avoid fingerprinting attacks). + msg = msg_getblocktxn() + msg.block_txn_request = BlockTransactionsRequest(block.sha256, [0]) + with mininode_lock: + self.test_node.last_blocktxn = None + self.test_node.send_and_ping(msg) + with mininode_lock: + assert(self.test_node.last_blocktxn is None) + + def run_test(self): + # Setup the p2p connections and start up the network thread. + self.test_node = TestNode() + + connections = [] + connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], self.test_node)) + self.test_node.add_connection(connections[0]) + + NetworkThread().start() # Start up network handling in another thread + + # Test logic begins here + self.test_node.wait_for_verack() + + # We will need UTXOs to construct transactions in later tests. + self.make_utxos() + + self.test_sendcmpct() + self.test_compactblock_construction() + self.test_compactblock_requests() + self.test_getblocktxn_requests() + self.test_getblocktxn_handler() + self.test_compactblocks_not_at_tip() + self.test_incorrect_blocktxn_response() + self.test_invalid_cmpctblock_message() + + +if __name__ == '__main__': + CompactBlocksTest().main() diff --git a/qa/rpc-tests/test_framework/mininode.py b/qa/rpc-tests/test_framework/mininode.py index cdd5292cd6..caffab3535 100755 --- a/qa/rpc-tests/test_framework/mininode.py +++ b/qa/rpc-tests/test_framework/mininode.py @@ -36,9 +36,10 @@ from threading import RLock from threading import Thread import logging import copy +from test_framework.siphash import siphash256 BIP0031_VERSION = 60000 -MY_VERSION = 60001 # past bip-31 for ping/pong +MY_VERSION = 70014 # past bip-31 for ping/pong MY_SUBVERSION = b"/python-mininode-tester:0.0.3/" MAX_INV_SZ = 50000 @@ -52,7 +53,7 @@ NODE_BLOOM = (1 << 2) NODE_WITNESS = (1 << 3) # Keep our own socket map for asyncore, so that we can track disconnects -# ourselves (to workaround an issue with closing an asyncore socket when +# ourselves (to workaround an issue with closing an asyncore socket when # using select) mininode_socket_map = dict() @@ -74,8 +75,19 @@ def ripemd160(s): def hash256(s): return sha256(sha256(s)) +def ser_compact_size(l): + r = b"" + if l < 253: + r = struct.pack("B", l) + elif l < 0x10000: + r = struct.pack("<BH", 253, l) + elif l < 0x100000000: + r = struct.pack("<BI", 254, l) + else: + r = struct.pack("<BQ", 255, l) + return r -def deser_string(f): +def deser_compact_size(f): nit = struct.unpack("<B", f.read(1))[0] if nit == 253: nit = struct.unpack("<H", f.read(2))[0] @@ -83,16 +95,14 @@ def deser_string(f): nit = struct.unpack("<I", f.read(4))[0] elif nit == 255: nit = struct.unpack("<Q", f.read(8))[0] + return nit + +def deser_string(f): + nit = deser_compact_size(f) return f.read(nit) def ser_string(s): - if len(s) < 253: - return struct.pack("B", len(s)) + s - elif len(s) < 0x10000: - return struct.pack("<BH", 253, len(s)) + s - elif len(s) < 0x100000000: - return struct.pack("<BI", 254, len(s)) + s - return struct.pack("<BQ", 255, len(s)) + s + return ser_compact_size(len(s)) + s def deser_uint256(f): r = 0 @@ -125,13 +135,7 @@ def uint256_from_compact(c): def deser_vector(f, c): - nit = struct.unpack("<B", f.read(1))[0] - if nit == 253: - nit = struct.unpack("<H", f.read(2))[0] - elif nit == 254: - nit = struct.unpack("<I", f.read(4))[0] - elif nit == 255: - nit = struct.unpack("<Q", f.read(8))[0] + nit = deser_compact_size(f) r = [] for i in range(nit): t = c() @@ -144,15 +148,7 @@ def deser_vector(f, c): # entries in the vector (we use this for serializing the vector of transactions # for a witness block). def ser_vector(l, ser_function_name=None): - r = b"" - if len(l) < 253: - r = struct.pack("B", len(l)) - elif len(l) < 0x10000: - r = struct.pack("<BH", 253, len(l)) - elif len(l) < 0x100000000: - r = struct.pack("<BI", 254, len(l)) - else: - r = struct.pack("<BQ", 255, len(l)) + r = ser_compact_size(len(l)) for i in l: if ser_function_name: r += getattr(i, ser_function_name)() @@ -162,13 +158,7 @@ def ser_vector(l, ser_function_name=None): def deser_uint256_vector(f): - nit = struct.unpack("<B", f.read(1))[0] - if nit == 253: - nit = struct.unpack("<H", f.read(2))[0] - elif nit == 254: - nit = struct.unpack("<I", f.read(4))[0] - elif nit == 255: - nit = struct.unpack("<Q", f.read(8))[0] + nit = deser_compact_size(f) r = [] for i in range(nit): t = deser_uint256(f) @@ -177,28 +167,14 @@ def deser_uint256_vector(f): def ser_uint256_vector(l): - r = b"" - if len(l) < 253: - r = struct.pack("B", len(l)) - elif len(l) < 0x10000: - r = struct.pack("<BH", 253, len(l)) - elif len(l) < 0x100000000: - r = struct.pack("<BI", 254, len(l)) - else: - r = struct.pack("<BQ", 255, len(l)) + r = ser_compact_size(len(l)) for i in l: r += ser_uint256(i) return r def deser_string_vector(f): - nit = struct.unpack("<B", f.read(1))[0] - if nit == 253: - nit = struct.unpack("<H", f.read(2))[0] - elif nit == 254: - nit = struct.unpack("<I", f.read(4))[0] - elif nit == 255: - nit = struct.unpack("<Q", f.read(8))[0] + nit = deser_compact_size(f) r = [] for i in range(nit): t = deser_string(f) @@ -207,28 +183,14 @@ def deser_string_vector(f): def ser_string_vector(l): - r = b"" - if len(l) < 253: - r = struct.pack("B", len(l)) - elif len(l) < 0x10000: - r = struct.pack("<BH", 253, len(l)) - elif len(l) < 0x100000000: - r = struct.pack("<BI", 254, len(l)) - else: - r = struct.pack("<BQ", 255, len(l)) + r = ser_compact_size(len(l)) for sv in l: r += ser_string(sv) return r def deser_int_vector(f): - nit = struct.unpack("<B", f.read(1))[0] - if nit == 253: - nit = struct.unpack("<H", f.read(2))[0] - elif nit == 254: - nit = struct.unpack("<I", f.read(4))[0] - elif nit == 255: - nit = struct.unpack("<Q", f.read(8))[0] + nit = deser_compact_size(f) r = [] for i in range(nit): t = struct.unpack("<i", f.read(4))[0] @@ -237,15 +199,7 @@ def deser_int_vector(f): def ser_int_vector(l): - r = b"" - if len(l) < 253: - r = struct.pack("B", len(l)) - elif len(l) < 0x10000: - r = struct.pack("<BH", 253, len(l)) - elif len(l) < 0x100000000: - r = struct.pack("<BI", 254, len(l)) - else: - r = struct.pack("<BQ", 255, len(l)) + r = ser_compact_size(len(l)) for i in l: r += struct.pack("<i", i) return r @@ -294,7 +248,8 @@ class CInv(object): 1: "TX", 2: "Block", 1|MSG_WITNESS_FLAG: "WitnessTx", - 2|MSG_WITNESS_FLAG : "WitnessBlock" + 2|MSG_WITNESS_FLAG : "WitnessBlock", + 4: "CompactBlock" } def __init__(self, t=0, h=0): @@ -781,6 +736,187 @@ class CAlert(object): % (len(self.vchMsg), len(self.vchSig)) +class PrefilledTransaction(object): + def __init__(self, index=0, tx = None): + self.index = index + self.tx = tx + + def deserialize(self, f): + self.index = deser_compact_size(f) + self.tx = CTransaction() + self.tx.deserialize(f) + + def serialize(self, with_witness=False): + r = b"" + r += ser_compact_size(self.index) + if with_witness: + r += self.tx.serialize_with_witness() + else: + r += self.tx.serialize_without_witness() + return r + + def __repr__(self): + return "PrefilledTransaction(index=%d, tx=%s)" % (self.index, repr(self.tx)) + +# This is what we send on the wire, in a cmpctblock message. +class P2PHeaderAndShortIDs(object): + def __init__(self): + self.header = CBlockHeader() + self.nonce = 0 + self.shortids_length = 0 + self.shortids = [] + self.prefilled_txn_length = 0 + self.prefilled_txn = [] + + def deserialize(self, f): + self.header.deserialize(f) + self.nonce = struct.unpack("<Q", f.read(8))[0] + self.shortids_length = deser_compact_size(f) + for i in range(self.shortids_length): + # shortids are defined to be 6 bytes in the spec, so append + # two zero bytes and read it in as an 8-byte number + self.shortids.append(struct.unpack("<Q", f.read(6) + b'\x00\x00')[0]) + self.prefilled_txn = deser_vector(f, PrefilledTransaction) + self.prefilled_txn_length = len(self.prefilled_txn) + + def serialize(self, with_witness=False): + r = b"" + r += self.header.serialize() + r += struct.pack("<Q", self.nonce) + r += ser_compact_size(self.shortids_length) + for x in self.shortids: + # We only want the first 6 bytes + r += struct.pack("<Q", x)[0:6] + r += ser_vector(self.prefilled_txn) + return r + + def __repr__(self): + return "P2PHeaderAndShortIDs(header=%s, nonce=%d, shortids_length=%d, shortids=%s, prefilled_txn_length=%d, prefilledtxn=%s" % (repr(self.header), self.nonce, self.shortids_length, repr(self.shortids), self.prefilled_txn_length, repr(self.prefilled_txn)) + + +# Calculate the BIP 152-compact blocks shortid for a given transaction hash +def calculate_shortid(k0, k1, tx_hash): + expected_shortid = siphash256(k0, k1, tx_hash) + expected_shortid &= 0x0000ffffffffffff + return expected_shortid + +# This version gets rid of the array lengths, and reinterprets the differential +# encoding into indices that can be used for lookup. +class HeaderAndShortIDs(object): + def __init__(self, p2pheaders_and_shortids = None): + self.header = CBlockHeader() + self.nonce = 0 + self.shortids = [] + self.prefilled_txn = [] + + if p2pheaders_and_shortids != None: + self.header = p2pheaders_and_shortids.header + self.nonce = p2pheaders_and_shortids.nonce + self.shortids = p2pheaders_and_shortids.shortids + last_index = -1 + for x in p2pheaders_and_shortids.prefilled_txn: + self.prefilled_txn.append(PrefilledTransaction(x.index + last_index + 1, x.tx)) + last_index = self.prefilled_txn[-1].index + + def to_p2p(self): + ret = P2PHeaderAndShortIDs() + ret.header = self.header + ret.nonce = self.nonce + ret.shortids_length = len(self.shortids) + ret.shortids = self.shortids + ret.prefilled_txn_length = len(self.prefilled_txn) + ret.prefilled_txn = [] + last_index = -1 + for x in self.prefilled_txn: + ret.prefilled_txn.append(PrefilledTransaction(x.index - last_index - 1, x.tx)) + last_index = x.index + return ret + + def get_siphash_keys(self): + header_nonce = self.header.serialize() + header_nonce += struct.pack("<Q", self.nonce) + hash_header_nonce_as_str = sha256(header_nonce) + key0 = struct.unpack("<Q", hash_header_nonce_as_str[0:8])[0] + key1 = struct.unpack("<Q", hash_header_nonce_as_str[8:16])[0] + return [ key0, key1 ] + + def initialize_from_block(self, block, nonce=0, prefill_list = [0]): + self.header = CBlockHeader(block) + self.nonce = nonce + self.prefilled_txn = [ PrefilledTransaction(i, block.vtx[i]) for i in prefill_list ] + self.shortids = [] + [k0, k1] = self.get_siphash_keys() + for i in range(len(block.vtx)): + if i not in prefill_list: + self.shortids.append(calculate_shortid(k0, k1, block.vtx[i].sha256)) + + def __repr__(self): + return "HeaderAndShortIDs(header=%s, nonce=%d, shortids=%s, prefilledtxn=%s" % (repr(self.header), self.nonce, repr(self.shortids), repr(self.prefilled_txn)) + + +class BlockTransactionsRequest(object): + + def __init__(self, blockhash=0, indexes = None): + self.blockhash = blockhash + self.indexes = indexes if indexes != None else [] + + def deserialize(self, f): + self.blockhash = deser_uint256(f) + indexes_length = deser_compact_size(f) + for i in range(indexes_length): + self.indexes.append(deser_compact_size(f)) + + def serialize(self): + r = b"" + r += ser_uint256(self.blockhash) + r += ser_compact_size(len(self.indexes)) + for x in self.indexes: + r += ser_compact_size(x) + return r + + # helper to set the differentially encoded indexes from absolute ones + def from_absolute(self, absolute_indexes): + self.indexes = [] + last_index = -1 + for x in absolute_indexes: + self.indexes.append(x-last_index-1) + last_index = x + + def to_absolute(self): + absolute_indexes = [] + last_index = -1 + for x in self.indexes: + absolute_indexes.append(x+last_index+1) + last_index = absolute_indexes[-1] + return absolute_indexes + + def __repr__(self): + return "BlockTransactionsRequest(hash=%064x indexes=%s)" % (self.blockhash, repr(self.indexes)) + + +class BlockTransactions(object): + + def __init__(self, blockhash=0, transactions = None): + self.blockhash = blockhash + self.transactions = transactions if transactions != None else [] + + def deserialize(self, f): + self.blockhash = deser_uint256(f) + self.transactions = deser_vector(f, CTransaction) + + def serialize(self, with_witness=False): + r = b"" + r += ser_uint256(self.blockhash) + if with_witness: + r += ser_vector(self.transactions, "serialize_with_witness") + else: + r += ser_vector(self.transactions) + return r + + def __repr__(self): + return "BlockTransactions(hash=%064x transactions=%s)" % (self.blockhash, repr(self.transactions)) + + # Objects that correspond to messages on the wire class msg_version(object): command = b"version" @@ -1215,6 +1351,79 @@ class msg_feefilter(object): def __repr__(self): return "msg_feefilter(feerate=%08x)" % self.feerate +class msg_sendcmpct(object): + command = b"sendcmpct" + + def __init__(self): + self.announce = False + self.version = 1 + + def deserialize(self, f): + self.announce = struct.unpack("<?", f.read(1))[0] + self.version = struct.unpack("<Q", f.read(8))[0] + + def serialize(self): + r = b"" + r += struct.pack("<?", self.announce) + r += struct.pack("<Q", self.version) + return r + + def __repr__(self): + return "msg_sendcmpct(announce=%s, version=%lu)" % (self.announce, self.version) + +class msg_cmpctblock(object): + command = b"cmpctblock" + + def __init__(self, header_and_shortids = None): + self.header_and_shortids = header_and_shortids + + def deserialize(self, f): + self.header_and_shortids = P2PHeaderAndShortIDs() + self.header_and_shortids.deserialize(f) + + def serialize(self): + r = b"" + r += self.header_and_shortids.serialize() + return r + + def __repr__(self): + return "msg_cmpctblock(HeaderAndShortIDs=%s)" % repr(self.header_and_shortids) + +class msg_getblocktxn(object): + command = b"getblocktxn" + + def __init__(self): + self.block_txn_request = None + + def deserialize(self, f): + self.block_txn_request = BlockTransactionsRequest() + self.block_txn_request.deserialize(f) + + def serialize(self): + r = b"" + r += self.block_txn_request.serialize() + return r + + def __repr__(self): + return "msg_getblocktxn(block_txn_request=%s)" % (repr(self.block_txn_request)) + +class msg_blocktxn(object): + command = b"blocktxn" + + def __init__(self): + self.block_transactions = BlockTransactions() + + def deserialize(self, f): + self.block_transactions.deserialize(f) + + def serialize(self): + r = b"" + r += self.block_transactions.serialize() + return r + + def __repr__(self): + return "msg_blocktxn(block_transactions=%s)" % (repr(self.block_transactions)) + # This is what a callback should look like for NodeConn # Reimplement the on_* functions to provide handling for events class NodeConnCB(object): @@ -1295,6 +1504,10 @@ class NodeConnCB(object): def on_pong(self, conn, message): pass def on_feefilter(self, conn, message): pass def on_sendheaders(self, conn, message): pass + def on_sendcmpct(self, conn, message): pass + def on_cmpctblock(self, conn, message): pass + def on_getblocktxn(self, conn, message): pass + def on_blocktxn(self, conn, message): pass # More useful callbacks and functions for NodeConnCB's which have a single NodeConn class SingleNodeConnCB(NodeConnCB): @@ -1311,6 +1524,10 @@ class SingleNodeConnCB(NodeConnCB): def send_message(self, message): self.connection.send_message(message) + def send_and_ping(self, message): + self.send_message(message) + self.sync_with_ping() + def on_pong(self, conn, message): self.last_pong = message @@ -1344,7 +1561,11 @@ class NodeConn(asyncore.dispatcher): b"reject": msg_reject, b"mempool": msg_mempool, b"feefilter": msg_feefilter, - b"sendheaders": msg_sendheaders + b"sendheaders": msg_sendheaders, + b"sendcmpct": msg_sendcmpct, + b"cmpctblock": msg_cmpctblock, + b"getblocktxn": msg_getblocktxn, + b"blocktxn": msg_blocktxn } MAGIC_BYTES = { "mainnet": b"\xf9\xbe\xb4\xd9", # mainnet diff --git a/qa/rpc-tests/test_framework/siphash.py b/qa/rpc-tests/test_framework/siphash.py new file mode 100644 index 0000000000..9c0574bd93 --- /dev/null +++ b/qa/rpc-tests/test_framework/siphash.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +# Copyright (c) 2016 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +# +# siphash.py - Specialized SipHash-2-4 implementations +# +# This implements SipHash-2-4 for 256-bit integers. + +def rotl64(n, b): + return n >> (64 - b) | (n & ((1 << (64 - b)) - 1)) << b + +def siphash_round(v0, v1, v2, v3): + v0 = (v0 + v1) & ((1 << 64) - 1) + v1 = rotl64(v1, 13) + v1 ^= v0 + v0 = rotl64(v0, 32) + v2 = (v2 + v3) & ((1 << 64) - 1) + v3 = rotl64(v3, 16) + v3 ^= v2 + v0 = (v0 + v3) & ((1 << 64) - 1) + v3 = rotl64(v3, 21) + v3 ^= v0 + v2 = (v2 + v1) & ((1 << 64) - 1) + v1 = rotl64(v1, 17) + v1 ^= v2 + v2 = rotl64(v2, 32) + return (v0, v1, v2, v3) + +def siphash256(k0, k1, h): + n0 = h & ((1 << 64) - 1) + n1 = (h >> 64) & ((1 << 64) - 1) + n2 = (h >> 128) & ((1 << 64) - 1) + n3 = (h >> 192) & ((1 << 64) - 1) + v0 = 0x736f6d6570736575 ^ k0 + v1 = 0x646f72616e646f6d ^ k1 + v2 = 0x6c7967656e657261 ^ k0 + v3 = 0x7465646279746573 ^ k1 ^ n0 + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0 ^= n0 + v3 ^= n1 + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0 ^= n1 + v3 ^= n2 + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0 ^= n2 + v3 ^= n3 + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0 ^= n3 + v3 ^= 0x2000000000000000 + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0 ^= 0x2000000000000000 + v2 ^= 0xFF + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + return v0 ^ v1 ^ v2 ^ v3 diff --git a/qa/rpc-tests/wallet-dump.py b/qa/rpc-tests/wallet-dump.py index dd675f57fc..6028d2c20b 100755 --- a/qa/rpc-tests/wallet-dump.py +++ b/qa/rpc-tests/wallet-dump.py @@ -4,9 +4,52 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import * -import os -import shutil +from test_framework.util import (start_nodes, start_node, assert_equal, bitcoind_processes) + + +def read_dump(file_name, addrs, hd_master_addr_old): + """ + Read the given dump, count the addrs that match, count change and reserve. + Also check that the old hd_master is inactive + """ + with open(file_name) as inputfile: + found_addr = 0 + found_addr_chg = 0 + found_addr_rsv = 0 + hd_master_addr_ret = None + for line in inputfile: + # only read non comment lines + if line[0] != "#" and len(line) > 10: + # split out some data + key_label, comment = line.split("#") + # key = key_label.split(" ")[0] + keytype = key_label.split(" ")[2] + if len(comment) > 1: + addr_keypath = comment.split(" addr=")[1] + addr = addr_keypath.split(" ")[0] + keypath = None + if keytype == "inactivehdmaster=1": + # ensure the old master is still available + assert(hd_master_addr_old == addr) + elif keytype == "hdmaster=1": + # ensure we have generated a new hd master key + assert(hd_master_addr_old != addr) + hd_master_addr_ret = addr + else: + keypath = addr_keypath.rstrip().split("hdkeypath=")[1] + + # count key types + for addrObj in addrs: + if addrObj['address'] == addr and addrObj['hdkeypath'] == keypath and keytype == "label=": + found_addr += 1 + break + elif keytype == "change=1": + found_addr_chg += 1 + break + elif keytype == "reserve=1": + found_addr_rsv += 1 + break + return found_addr, found_addr_chg, found_addr_rsv, hd_master_addr_ret class WalletDumpTest(BitcoinTestFramework): @@ -15,106 +58,47 @@ class WalletDumpTest(BitcoinTestFramework): super().__init__() self.setup_clean_chain = False self.num_nodes = 1 + self.extra_args = [["-keypool=90"]] def setup_network(self, split=False): - extra_args = [["-keypool=100"]] - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, extra_args) + self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, self.extra_args) def run_test (self): tmpdir = self.options.tmpdir - #generate 20 addresses to compare against the dump + # generate 20 addresses to compare against the dump test_addr_count = 20 addrs = [] for i in range(0,test_addr_count): addr = self.nodes[0].getnewaddress() vaddr= self.nodes[0].validateaddress(addr) #required to get hd keypath addrs.append(vaddr) + # Should be a no-op: + self.nodes[0].keypoolrefill() # dump unencrypted wallet self.nodes[0].dumpwallet(tmpdir + "/node0/wallet.unencrypted.dump") - #open file - inputfile = open(tmpdir + "/node0/wallet.unencrypted.dump") - found_addr = 0 - found_addr_chg = 0 - found_addr_rsv = 0 - hdmasteraddr = "" - for line in inputfile: - #only read non comment lines - if line[0] != "#" and len(line) > 10: - #split out some data - keyLabel, comment = line.split("#") - key = keyLabel.split(" ")[0] - keytype = keyLabel.split(" ")[2] - if len(comment) > 1: - addrKeypath = comment.split(" addr=")[1] - addr = addrKeypath.split(" ")[0] - keypath = "" - if keytype != "hdmaster=1": - keypath = addrKeypath.rstrip().split("hdkeypath=")[1] - else: - #keep hd master for later comp. - hdmasteraddr = addr - - #count key types - for addrObj in addrs: - if (addrObj['address'] == addr and addrObj['hdkeypath'] == keypath and keytype == "label="): - found_addr+=1 - break - elif (keytype == "change=1"): - found_addr_chg+=1 - break - elif (keytype == "reserve=1"): - found_addr_rsv+=1 - break - assert(found_addr == test_addr_count) #all keys must be in the dump - assert(found_addr_chg == 50) #50 blocks where mined - assert(found_addr_rsv == 100) #100 reserve keys (keypool) + found_addr, found_addr_chg, found_addr_rsv, hd_master_addr_unenc = \ + read_dump(tmpdir + "/node0/wallet.unencrypted.dump", addrs, None) + assert_equal(found_addr, test_addr_count) # all keys must be in the dump + assert_equal(found_addr_chg, 50) # 50 blocks where mined + assert_equal(found_addr_rsv, 90 + 1) # keypool size (TODO: fix off-by-one) #encrypt wallet, restart, unlock and dump self.nodes[0].encryptwallet('test') bitcoind_processes[0].wait() - self.nodes[0] = start_node(0, self.options.tmpdir) + self.nodes[0] = start_node(0, self.options.tmpdir, self.extra_args[0]) self.nodes[0].walletpassphrase('test', 10) + # Should be a no-op: + self.nodes[0].keypoolrefill() self.nodes[0].dumpwallet(tmpdir + "/node0/wallet.encrypted.dump") - #open dump done with an encrypted wallet - inputfile = open(tmpdir + "/node0/wallet.encrypted.dump") - found_addr = 0 - found_addr_chg = 0 - found_addr_rsv = 0 - for line in inputfile: - if line[0] != "#" and len(line) > 10: - keyLabel, comment = line.split("#") - key = keyLabel.split(" ")[0] - keytype = keyLabel.split(" ")[2] - if len(comment) > 1: - addrKeypath = comment.split(" addr=")[1] - addr = addrKeypath.split(" ")[0] - keypath = "" - if keytype != "hdmaster=1": - keypath = addrKeypath.rstrip().split("hdkeypath=")[1] - else: - #ensure we have generated a new hd master key - assert(hdmasteraddr != addr) - if keytype == "inactivehdmaster=1": - #ensure the old master is still available - assert(hdmasteraddr == addr) - for addrObj in addrs: - if (addrObj['address'] == addr and addrObj['hdkeypath'] == keypath and keytype == "label="): - found_addr+=1 - break - elif (keytype == "change=1"): - found_addr_chg+=1 - break - elif (keytype == "reserve=1"): - found_addr_rsv+=1 - break - - assert(found_addr == test_addr_count) - assert(found_addr_chg == 150) #old reserve keys are marked as change now - assert(found_addr_rsv == 100) #keypool size + found_addr, found_addr_chg, found_addr_rsv, hd_master_addr_enc = \ + read_dump(tmpdir + "/node0/wallet.encrypted.dump", addrs, hd_master_addr_unenc) + assert_equal(found_addr, test_addr_count) + assert_equal(found_addr_chg, 90 + 1 + 50) # old reserve keys are marked as change now + assert_equal(found_addr_rsv, 90 + 1) # keypool size (TODO: fix off-by-one) if __name__ == '__main__': WalletDumpTest().main () |