diff options
Diffstat (limited to 'test/functional/feature_pruning.py')
| -rwxr-xr-x | test/functional/feature_pruning.py | 227 |
1 files changed, 111 insertions, 116 deletions
diff --git a/test/functional/feature_pruning.py b/test/functional/feature_pruning.py index 3e1ba88f0a..66c395d7a2 100755 --- a/test/functional/feature_pruning.py +++ b/test/functional/feature_pruning.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the pruning code. @@ -8,32 +8,80 @@ WARNING: This test uses 4GB of disk space. This test takes 30 mins or more (up to 2 hours) """ - -from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_greater_than, assert_raises_rpc_error, connect_nodes, mine_large_block, sync_blocks, wait_until - import os -MIN_BLOCKS_TO_KEEP = 288 +from test_framework.blocktools import create_coinbase +from test_framework.messages import CBlock, ToHex +from test_framework.script import CScript, OP_RETURN, OP_NOP +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + assert_greater_than, + assert_raises_rpc_error, + connect_nodes, + disconnect_nodes, + wait_until, +) # Rescans start at the earliest block up to 2 hours before a key timestamp, so # the manual prune RPC avoids pruning blocks in the same window to be # compatible with pruning based on key creation time. TIMESTAMP_WINDOW = 2 * 60 * 60 +def mine_large_blocks(node, n): + # Make a large scriptPubKey for the coinbase transaction. This is OP_RETURN + # followed by 950k of OP_NOP. This would be non-standard in a non-coinbase + # transaction but is consensus valid. + + # Set the nTime if this is the first time this function has been called. + # A static variable ensures that time is monotonicly increasing and is therefore + # different for each block created => blockhash is unique. + if "nTimes" not in mine_large_blocks.__dict__: + mine_large_blocks.nTime = 0 + + # Get the block parameters for the first block + big_script = CScript([OP_RETURN] + [OP_NOP] * 950000) + best_block = node.getblock(node.getbestblockhash()) + height = int(best_block["height"]) + 1 + mine_large_blocks.nTime = max(mine_large_blocks.nTime, int(best_block["time"])) + 1 + previousblockhash = int(best_block["hash"], 16) + + for _ in range(n): + # Build the coinbase transaction (with large scriptPubKey) + coinbase_tx = create_coinbase(height) + coinbase_tx.vin[0].nSequence = 2 ** 32 - 1 + coinbase_tx.vout[0].scriptPubKey = big_script + coinbase_tx.rehash() + + # Build the block + block = CBlock() + block.nVersion = best_block["version"] + block.hashPrevBlock = previousblockhash + block.nTime = mine_large_blocks.nTime + block.nBits = int('207fffff', 16) + block.nNonce = 0 + block.vtx = [coinbase_tx] + block.hashMerkleRoot = block.calc_merkle_root() + block.solve() + + # Submit to the node + node.submitblock(ToHex(block)) + + previousblockhash = block.sha256 + height += 1 + mine_large_blocks.nTime += 1 def calc_usage(blockdir): - return sum(os.path.getsize(blockdir+f) for f in os.listdir(blockdir) if os.path.isfile(os.path.join(blockdir, f))) / (1024. * 1024.) + return sum(os.path.getsize(blockdir + f) for f in os.listdir(blockdir) if os.path.isfile(os.path.join(blockdir, f))) / (1024. * 1024.) class PruneTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 6 - self.rpc_timeout = 900 # Create nodes 0 and 1 to mine. # Create node 2 to test pruning. - self.full_node_default_args = ["-maxreceivebuffer=20000", "-checkblocks=5", "-limitdescendantcount=100", "-limitdescendantsize=5000", "-limitancestorcount=100", "-limitancestorsize=5000"] + self.full_node_default_args = ["-maxreceivebuffer=20000", "-checkblocks=5"] # Create nodes 3 and 4 to test manual pruning (they will be re-started with manual pruning later) # Create nodes 5 to test wallet in prune mode, but do not connect self.extra_args = [ @@ -55,10 +103,10 @@ class PruneTest(BitcoinTestFramework): connect_nodes(self.nodes[0], 1) connect_nodes(self.nodes[1], 2) - connect_nodes(self.nodes[2], 0) + connect_nodes(self.nodes[0], 2) connect_nodes(self.nodes[0], 3) connect_nodes(self.nodes[0], 4) - sync_blocks(self.nodes[0:5]) + self.sync_blocks(self.nodes[0:5]) def setup_nodes(self): self.add_nodes(self.num_nodes, self.extra_args) @@ -69,23 +117,21 @@ class PruneTest(BitcoinTestFramework): def create_big_chain(self): # Start by creating some coinbases we can spend later self.nodes[1].generate(200) - sync_blocks(self.nodes[0:2]) + self.sync_blocks(self.nodes[0:2]) self.nodes[0].generate(150) + # Then mine enough full blocks to create more than 550MiB of data - for i in range(645): - mine_large_block(self.nodes[0], self.utxo_cache_0) + mine_large_blocks(self.nodes[0], 645) - sync_blocks(self.nodes[0:5]) + self.sync_blocks(self.nodes[0:5]) def test_height_min(self): - if not os.path.isfile(os.path.join(self.prunedir, "blk00000.dat")): - raise AssertionError("blk00000.dat is missing, pruning too early") + assert os.path.isfile(os.path.join(self.prunedir, "blk00000.dat")), "blk00000.dat is missing, pruning too early" self.log.info("Success") self.log.info("Though we're already using more than 550MiB, current usage: %d" % calc_usage(self.prunedir)) self.log.info("Mining 25 more blocks should cause the first block file to be pruned") # Pruning doesn't run until we're allocating another chunk, 20 full blocks past the height cutoff will ensure this - for i in range(25): - mine_large_block(self.nodes[0], self.utxo_cache_0) + mine_large_blocks(self.nodes[0], 25) # Wait for blk00000.dat to be pruned wait_until(lambda: not os.path.isfile(os.path.join(self.prunedir, "blk00000.dat")), timeout=30) @@ -93,8 +139,7 @@ class PruneTest(BitcoinTestFramework): self.log.info("Success") usage = calc_usage(self.prunedir) self.log.info("Usage should be below target: %d" % usage) - if (usage > 550): - raise AssertionError("Pruning target not being met") + assert_greater_than(550, usage) def create_chain_with_staleblocks(self): # Create stale blocks in manageable sized chunks @@ -103,90 +148,66 @@ class PruneTest(BitcoinTestFramework): for j in range(12): # Disconnect node 0 so it can mine a longer reorg chain without knowing about node 1's soon-to-be-stale chain # Node 2 stays connected, so it hears about the stale blocks and then reorg's when node0 reconnects - # Stopping node 0 also clears its mempool, so it doesn't have node1's transactions to accidentally mine - self.stop_node(0) - self.start_node(0, extra_args=self.full_node_default_args) + disconnect_nodes(self.nodes[0], 1) + disconnect_nodes(self.nodes[0], 2) # Mine 24 blocks in node 1 - for i in range(24): - if j == 0: - mine_large_block(self.nodes[1], self.utxo_cache_1) - else: - # Add node1's wallet transactions back to the mempool, to - # avoid the mined blocks from being too small. - self.nodes[1].resendwallettransactions() - self.nodes[1].generate(1) #tx's already in mempool from previous disconnects + mine_large_blocks(self.nodes[1], 24) # Reorg back with 25 block chain from node 0 - for i in range(25): - mine_large_block(self.nodes[0], self.utxo_cache_0) + mine_large_blocks(self.nodes[0], 25) # Create connections in the order so both nodes can see the reorg at the same time - connect_nodes(self.nodes[1], 0) - connect_nodes(self.nodes[2], 0) - sync_blocks(self.nodes[0:3]) + connect_nodes(self.nodes[0], 1) + connect_nodes(self.nodes[0], 2) + self.sync_blocks(self.nodes[0:3]) self.log.info("Usage can be over target because of high stale rate: %d" % calc_usage(self.prunedir)) def reorg_test(self): # Node 1 will mine a 300 block chain starting 287 blocks back from Node 0 and Node 2's tip # This will cause Node 2 to do a reorg requiring 288 blocks of undo data to the reorg_test chain - # Reboot node 1 to clear its mempool (hopefully make the invalidate faster) - # Lower the block max size so we don't keep mining all our big mempool transactions (from disconnected blocks) - self.stop_node(1) - self.start_node(1, extra_args=["-maxreceivebuffer=20000","-blockmaxweight=20000", "-checkblocks=5"]) height = self.nodes[1].getblockcount() self.log.info("Current block height: %d" % height) - invalidheight = height-287 - badhash = self.nodes[1].getblockhash(invalidheight) - self.log.info("Invalidating block %s at height %d" % (badhash,invalidheight)) - self.nodes[1].invalidateblock(badhash) + self.forkheight = height - 287 + self.forkhash = self.nodes[1].getblockhash(self.forkheight) + self.log.info("Invalidating block %s at height %d" % (self.forkhash, self.forkheight)) + self.nodes[1].invalidateblock(self.forkhash) # We've now switched to our previously mined-24 block fork on node 1, but that's not what we want # So invalidate that fork as well, until we're on the same chain as node 0/2 (but at an ancestor 288 blocks ago) - mainchainhash = self.nodes[0].getblockhash(invalidheight - 1) - curhash = self.nodes[1].getblockhash(invalidheight - 1) + mainchainhash = self.nodes[0].getblockhash(self.forkheight - 1) + curhash = self.nodes[1].getblockhash(self.forkheight - 1) while curhash != mainchainhash: self.nodes[1].invalidateblock(curhash) - curhash = self.nodes[1].getblockhash(invalidheight - 1) + curhash = self.nodes[1].getblockhash(self.forkheight - 1) - assert(self.nodes[1].getblockcount() == invalidheight - 1) + assert self.nodes[1].getblockcount() == self.forkheight - 1 self.log.info("New best height: %d" % self.nodes[1].getblockcount()) - # Reboot node1 to clear those giant tx's from mempool - self.stop_node(1) - self.start_node(1, extra_args=["-maxreceivebuffer=20000","-blockmaxweight=20000", "-checkblocks=5"]) + # Disconnect node1 and generate the new chain + disconnect_nodes(self.nodes[0], 1) + disconnect_nodes(self.nodes[1], 2) self.log.info("Generating new longer chain of 300 more blocks") self.nodes[1].generate(300) self.log.info("Reconnect nodes") connect_nodes(self.nodes[0], 1) - connect_nodes(self.nodes[2], 1) - sync_blocks(self.nodes[0:3], timeout=120) + connect_nodes(self.nodes[1], 2) + self.sync_blocks(self.nodes[0:3], timeout=120) self.log.info("Verify height on node 2: %d" % self.nodes[2].getblockcount()) - self.log.info("Usage possibly still high bc of stale blocks in block files: %d" % calc_usage(self.prunedir)) + self.log.info("Usage possibly still high because of stale blocks in block files: %d" % calc_usage(self.prunedir)) - self.log.info("Mine 220 more blocks so we have requisite history (some blocks will be big and cause pruning of previous chain)") + self.log.info("Mine 220 more large blocks so we have requisite history") - # Get node0's wallet transactions back in its mempool, to avoid the - # mined blocks from being too small. - self.nodes[0].resendwallettransactions() - - for i in range(22): - # This can be slow, so do this in multiple RPC calls to avoid - # RPC timeouts. - self.nodes[0].generate(10) #node 0 has many large tx's in its mempool from the disconnects - sync_blocks(self.nodes[0:3], timeout=300) + mine_large_blocks(self.nodes[0], 220) usage = calc_usage(self.prunedir) self.log.info("Usage should be below target: %d" % usage) - if (usage > 550): - raise AssertionError("Pruning target not being met") - - return invalidheight,badhash + assert_greater_than(550, usage) def reorg_back(self): # Verify that a block on the old main chain fork has been pruned away @@ -219,17 +240,17 @@ class PruneTest(BitcoinTestFramework): blocks_to_mine = first_reorg_height + 1 - self.mainchainheight self.log.info("Rewind node 0 to prev main chain to mine longer chain to trigger redownload. Blocks needed: %d" % blocks_to_mine) self.nodes[0].invalidateblock(curchainhash) - assert(self.nodes[0].getblockcount() == self.mainchainheight) - assert(self.nodes[0].getbestblockhash() == self.mainchainhash2) + assert_equal(self.nodes[0].getblockcount(), self.mainchainheight) + assert_equal(self.nodes[0].getbestblockhash(), self.mainchainhash2) goalbesthash = self.nodes[0].generate(blocks_to_mine)[-1] goalbestheight = first_reorg_height + 1 self.log.info("Verify node 2 reorged back to the main chain, some blocks of which it had to redownload") # Wait for Node 2 to reorg to proper height wait_until(lambda: self.nodes[2].getblockcount() >= goalbestheight, timeout=900) - assert(self.nodes[2].getbestblockhash() == goalbesthash) + assert_equal(self.nodes[2].getbestblockhash(), goalbesthash) # Verify we can now have the data for a block previously pruned - assert(self.nodes[2].getblock(self.forkhash)["height"] == self.forkheight) + assert_equal(self.nodes[2].getblock(self.forkhash)["height"], self.forkheight) def manual_test(self, node_number, use_timestamp): # at this point, node has 995 blocks and has not yet run in prune mode @@ -250,20 +271,9 @@ class PruneTest(BitcoinTestFramework): else: return index - def prune(index, expected_ret=None): + def prune(index): ret = node.pruneblockchain(height=height(index)) - # Check the return value. When use_timestamp is True, just check - # that the return value is less than or equal to the expected - # value, because when more than one block is generated per second, - # a timestamp will not be granular enough to uniquely identify an - # individual block. - if expected_ret is None: - expected_ret = index - if use_timestamp: - assert_greater_than(ret, 0) - assert_greater_than(expected_ret + 1, ret) - else: - assert_equal(ret, expected_ret) + assert_equal(ret, node.getblockchaininfo()['pruneheight']) def has_block(index): return os.path.isfile(os.path.join(self.nodes[node_number].datadir, "regtest", "blocks", "blk{:05}.dat".format(index))) @@ -287,38 +297,30 @@ class PruneTest(BitcoinTestFramework): # height=100 too low to prune first block file so this is a no-op prune(100) - if not has_block(0): - raise AssertionError("blk00000.dat is missing when should still be there") + assert has_block(0), "blk00000.dat is missing when should still be there" # Does nothing node.pruneblockchain(height(0)) - if not has_block(0): - raise AssertionError("blk00000.dat is missing when should still be there") + assert has_block(0), "blk00000.dat is missing when should still be there" # height=500 should prune first file prune(500) - if has_block(0): - raise AssertionError("blk00000.dat is still there, should be pruned by now") - if not has_block(1): - raise AssertionError("blk00001.dat is missing when should still be there") + assert not has_block(0), "blk00000.dat is still there, should be pruned by now" + assert has_block(1), "blk00001.dat is missing when should still be there" # height=650 should prune second file prune(650) - if has_block(1): - raise AssertionError("blk00001.dat is still there, should be pruned by now") + assert not has_block(1), "blk00001.dat is still there, should be pruned by now" # height=1000 should not prune anything more, because tip-288 is in blk00002.dat. - prune(1000, 1001 - MIN_BLOCKS_TO_KEEP) - if not has_block(2): - raise AssertionError("blk00002.dat is still there, should be pruned by now") + prune(1000) + assert has_block(2), "blk00002.dat is still there, should be pruned by now" # advance the tip so blk00002.dat and blk00003.dat can be pruned (the last 288 blocks should now be in blk00004.dat) node.generate(288) prune(1000) - if has_block(2): - raise AssertionError("blk00002.dat is still there, should be pruned by now") - if has_block(3): - raise AssertionError("blk00003.dat is still there, should be pruned by now") + assert not has_block(2), "blk00002.dat is still there, should be pruned by now" + assert not has_block(3), "blk00003.dat is still there, should be pruned by now" # stop node, start back up with auto-prune at 550 MiB, make sure still runs self.stop_node(node_number) @@ -338,22 +340,15 @@ class PruneTest(BitcoinTestFramework): self.log.info("Syncing node 5 to test wallet") connect_nodes(self.nodes[0], 5) nds = [self.nodes[0], self.nodes[5]] - sync_blocks(nds, wait=5, timeout=300) - self.stop_node(5) #stop and start to trigger rescan + self.sync_blocks(nds, wait=5, timeout=300) + self.stop_node(5) # stop and start to trigger rescan self.start_node(5, extra_args=["-prune=550"]) self.log.info("Success") def run_test(self): - self.log.info("Warning! This test requires 4GB of disk space and takes over 30 mins (up to 2 hours)") - self.log.info("Mining a big blockchain of 995 blocks") - - # Determine default relay fee - self.relayfee = self.nodes[0].getnetworkinfo()["relayfee"] - - # Cache for utxos, as the listunspent may take a long time later in the test - self.utxo_cache_0 = [] - self.utxo_cache_1 = [] + self.log.info("Warning! This test requires 4GB of disk space") + self.log.info("Mining a big blockchain of 995 blocks") self.create_big_chain() # Chain diagram key: # * blocks on main chain @@ -394,11 +389,11 @@ class PruneTest(BitcoinTestFramework): # +...+(1044) &.. $...$(1319) # Save some current chain state for later use - self.mainchainheight = self.nodes[2].getblockcount() #1320 + self.mainchainheight = self.nodes[2].getblockcount() # 1320 self.mainchainhash2 = self.nodes[2].getblockhash(self.mainchainheight) self.log.info("Check that we can survive a 288 block reorg still") - (self.forkheight,self.forkhash) = self.reorg_test() #(1033, ) + self.reorg_test() # (1033, ) # Now create a 288 block reorg by mining a longer chain on N1 # First disconnect N1 # Then invalidate 1033 on main chain and 1032 on fork so height is 1032 on main chain |