aboutsummaryrefslogtreecommitdiff
path: root/test/functional/feature_pruning.py
diff options
context:
space:
mode:
Diffstat (limited to 'test/functional/feature_pruning.py')
-rwxr-xr-xtest/functional/feature_pruning.py186
1 files changed, 93 insertions, 93 deletions
diff --git a/test/functional/feature_pruning.py b/test/functional/feature_pruning.py
index 12da2655ee..49951e24f3 100755
--- a/test/functional/feature_pruning.py
+++ b/test/functional/feature_pruning.py
@@ -8,11 +8,13 @@ WARNING:
This test uses 4GB of disk space.
This test takes 30 mins or more (up to 2 hours)
"""
+import os
+from test_framework.blocktools import create_coinbase
+from test_framework.messages import CBlock, ToHex
+from test_framework.script import CScript, OP_RETURN, OP_NOP
from test_framework.test_framework import BitcoinTestFramework
-from test_framework.util import assert_equal, assert_greater_than, assert_raises_rpc_error, connect_nodes, mine_large_block, sync_blocks, wait_until
-
-import os
+from test_framework.util import assert_equal, assert_greater_than, assert_raises_rpc_error, connect_nodes, disconnect_nodes, sync_blocks, wait_until
MIN_BLOCKS_TO_KEEP = 288
@@ -21,19 +23,59 @@ MIN_BLOCKS_TO_KEEP = 288
# compatible with pruning based on key creation time.
TIMESTAMP_WINDOW = 2 * 60 * 60
+def mine_large_blocks(node, n):
+ # Make a large scriptPubKey for the coinbase transaction. This is OP_RETURN
+ # followed by 950k of OP_NOP. This would be non-standard in a non-coinbase
+ # transaction but is consensus valid.
+
+ # Get the block parameters for the first block
+ big_script = CScript([OP_RETURN] + [OP_NOP] * 950000)
+ best_block = node.getblock(node.getbestblockhash())
+ height = int(best_block["height"]) + 1
+ try:
+ # Static variable ensures that time is monotonicly increasing and is therefore
+ # different for each block created => blockhash is unique.
+ mine_large_blocks.nTime = min(mine_large_blocks.nTime, int(best_block["time"])) + 1
+ except AttributeError:
+ mine_large_blocks.nTime = int(best_block["time"]) + 1
+ previousblockhash = int(best_block["hash"], 16)
+
+ for _ in range(n):
+ # Build the coinbase transaction (with large scriptPubKey)
+ coinbase_tx = create_coinbase(height)
+ coinbase_tx.vin[0].nSequence = 2 ** 32 - 1
+ coinbase_tx.vout[0].scriptPubKey = big_script
+ coinbase_tx.rehash()
+
+ # Build the block
+ block = CBlock()
+ block.nVersion = best_block["version"]
+ block.hashPrevBlock = previousblockhash
+ block.nTime = mine_large_blocks.nTime
+ block.nBits = int('207fffff', 16)
+ block.nNonce = 0
+ block.vtx = [coinbase_tx]
+ block.hashMerkleRoot = block.calc_merkle_root()
+ block.solve()
+
+ # Submit to the node
+ node.submitblock(ToHex(block))
+
+ previousblockhash = block.sha256
+ height += 1
+ mine_large_blocks.nTime += 1
def calc_usage(blockdir):
- return sum(os.path.getsize(blockdir+f) for f in os.listdir(blockdir) if os.path.isfile(os.path.join(blockdir, f))) / (1024. * 1024.)
+ return sum(os.path.getsize(blockdir + f) for f in os.listdir(blockdir) if os.path.isfile(os.path.join(blockdir, f))) / (1024. * 1024.)
class PruneTest(BitcoinTestFramework):
def set_test_params(self):
self.setup_clean_chain = True
self.num_nodes = 6
- self.rpc_timeout = 900
# Create nodes 0 and 1 to mine.
# Create node 2 to test pruning.
- self.full_node_default_args = ["-maxreceivebuffer=20000", "-checkblocks=5", "-limitdescendantcount=100", "-limitdescendantsize=5000", "-limitancestorcount=100", "-limitancestorsize=5000"]
+ self.full_node_default_args = ["-maxreceivebuffer=20000", "-checkblocks=5"]
# Create nodes 3 and 4 to test manual pruning (they will be re-started with manual pruning later)
# Create nodes 5 to test wallet in prune mode, but do not connect
self.extra_args = [
@@ -55,7 +97,7 @@ class PruneTest(BitcoinTestFramework):
connect_nodes(self.nodes[0], 1)
connect_nodes(self.nodes[1], 2)
- connect_nodes(self.nodes[2], 0)
+ connect_nodes(self.nodes[0], 2)
connect_nodes(self.nodes[0], 3)
connect_nodes(self.nodes[0], 4)
sync_blocks(self.nodes[0:5])
@@ -71,21 +113,19 @@ class PruneTest(BitcoinTestFramework):
self.nodes[1].generate(200)
sync_blocks(self.nodes[0:2])
self.nodes[0].generate(150)
+
# Then mine enough full blocks to create more than 550MiB of data
- for i in range(645):
- mine_large_block(self.nodes[0], self.utxo_cache_0)
+ mine_large_blocks(self.nodes[0], 645)
sync_blocks(self.nodes[0:5])
def test_height_min(self):
- if not os.path.isfile(os.path.join(self.prunedir, "blk00000.dat")):
- raise AssertionError("blk00000.dat is missing, pruning too early")
+ assert os.path.isfile(os.path.join(self.prunedir, "blk00000.dat")), "blk00000.dat is missing, pruning too early"
self.log.info("Success")
self.log.info("Though we're already using more than 550MiB, current usage: %d" % calc_usage(self.prunedir))
self.log.info("Mining 25 more blocks should cause the first block file to be pruned")
# Pruning doesn't run until we're allocating another chunk, 20 full blocks past the height cutoff will ensure this
- for i in range(25):
- mine_large_block(self.nodes[0], self.utxo_cache_0)
+ mine_large_blocks(self.nodes[0], 25)
# Wait for blk00000.dat to be pruned
wait_until(lambda: not os.path.isfile(os.path.join(self.prunedir, "blk00000.dat")), timeout=30)
@@ -93,8 +133,7 @@ class PruneTest(BitcoinTestFramework):
self.log.info("Success")
usage = calc_usage(self.prunedir)
self.log.info("Usage should be below target: %d" % usage)
- if (usage > 550):
- raise AssertionError("Pruning target not being met")
+ assert_greater_than(550, usage)
def create_chain_with_staleblocks(self):
# Create stale blocks in manageable sized chunks
@@ -103,26 +142,17 @@ class PruneTest(BitcoinTestFramework):
for j in range(12):
# Disconnect node 0 so it can mine a longer reorg chain without knowing about node 1's soon-to-be-stale chain
# Node 2 stays connected, so it hears about the stale blocks and then reorg's when node0 reconnects
- # Stopping node 0 also clears its mempool, so it doesn't have node1's transactions to accidentally mine
- self.stop_node(0)
- self.start_node(0, extra_args=self.full_node_default_args)
+ disconnect_nodes(self.nodes[0], 1)
+ disconnect_nodes(self.nodes[0], 2)
# Mine 24 blocks in node 1
- for i in range(24):
- if j == 0:
- mine_large_block(self.nodes[1], self.utxo_cache_1)
- else:
- # Add node1's wallet transactions back to the mempool, to
- # avoid the mined blocks from being too small.
- self.nodes[1].resendwallettransactions()
- self.nodes[1].generate(1) #tx's already in mempool from previous disconnects
+ mine_large_blocks(self.nodes[1], 24)
# Reorg back with 25 block chain from node 0
- for i in range(25):
- mine_large_block(self.nodes[0], self.utxo_cache_0)
+ mine_large_blocks(self.nodes[0], 25)
# Create connections in the order so both nodes can see the reorg at the same time
- connect_nodes(self.nodes[1], 0)
- connect_nodes(self.nodes[2], 0)
+ connect_nodes(self.nodes[0], 1)
+ connect_nodes(self.nodes[0], 2)
sync_blocks(self.nodes[0:3])
self.log.info("Usage can be over target because of high stale rate: %d" % calc_usage(self.prunedir))
@@ -130,63 +160,48 @@ class PruneTest(BitcoinTestFramework):
def reorg_test(self):
# Node 1 will mine a 300 block chain starting 287 blocks back from Node 0 and Node 2's tip
# This will cause Node 2 to do a reorg requiring 288 blocks of undo data to the reorg_test chain
- # Reboot node 1 to clear its mempool (hopefully make the invalidate faster)
- # Lower the block max size so we don't keep mining all our big mempool transactions (from disconnected blocks)
- self.stop_node(1)
- self.start_node(1, extra_args=["-maxreceivebuffer=20000","-blockmaxweight=20000", "-checkblocks=5"])
height = self.nodes[1].getblockcount()
self.log.info("Current block height: %d" % height)
- invalidheight = height-287
- badhash = self.nodes[1].getblockhash(invalidheight)
- self.log.info("Invalidating block %s at height %d" % (badhash,invalidheight))
- self.nodes[1].invalidateblock(badhash)
+ self.forkheight = height - 287
+ self.forkhash = self.nodes[1].getblockhash(self.forkheight)
+ self.log.info("Invalidating block %s at height %d" % (self.forkhash, self.forkheight))
+ self.nodes[1].invalidateblock(self.forkhash)
# We've now switched to our previously mined-24 block fork on node 1, but that's not what we want
# So invalidate that fork as well, until we're on the same chain as node 0/2 (but at an ancestor 288 blocks ago)
- mainchainhash = self.nodes[0].getblockhash(invalidheight - 1)
- curhash = self.nodes[1].getblockhash(invalidheight - 1)
+ mainchainhash = self.nodes[0].getblockhash(self.forkheight - 1)
+ curhash = self.nodes[1].getblockhash(self.forkheight - 1)
while curhash != mainchainhash:
self.nodes[1].invalidateblock(curhash)
- curhash = self.nodes[1].getblockhash(invalidheight - 1)
+ curhash = self.nodes[1].getblockhash(self.forkheight - 1)
- assert self.nodes[1].getblockcount() == invalidheight - 1
+ assert self.nodes[1].getblockcount() == self.forkheight - 1
self.log.info("New best height: %d" % self.nodes[1].getblockcount())
- # Reboot node1 to clear those giant tx's from mempool
- self.stop_node(1)
- self.start_node(1, extra_args=["-maxreceivebuffer=20000","-blockmaxweight=20000", "-checkblocks=5"])
+ # Disconnect node1 and generate the new chain
+ disconnect_nodes(self.nodes[0], 1)
+ disconnect_nodes(self.nodes[1], 2)
self.log.info("Generating new longer chain of 300 more blocks")
self.nodes[1].generate(300)
self.log.info("Reconnect nodes")
connect_nodes(self.nodes[0], 1)
- connect_nodes(self.nodes[2], 1)
+ connect_nodes(self.nodes[1], 2)
sync_blocks(self.nodes[0:3], timeout=120)
self.log.info("Verify height on node 2: %d" % self.nodes[2].getblockcount())
- self.log.info("Usage possibly still high bc of stale blocks in block files: %d" % calc_usage(self.prunedir))
-
- self.log.info("Mine 220 more blocks so we have requisite history (some blocks will be big and cause pruning of previous chain)")
+ self.log.info("Usage possibly still high because of stale blocks in block files: %d" % calc_usage(self.prunedir))
- # Get node0's wallet transactions back in its mempool, to avoid the
- # mined blocks from being too small.
- self.nodes[0].resendwallettransactions()
+ self.log.info("Mine 220 more large blocks so we have requisite history")
- for i in range(22):
- # This can be slow, so do this in multiple RPC calls to avoid
- # RPC timeouts.
- self.nodes[0].generate(10) #node 0 has many large tx's in its mempool from the disconnects
- sync_blocks(self.nodes[0:3], timeout=300)
+ mine_large_blocks(self.nodes[0], 220)
usage = calc_usage(self.prunedir)
self.log.info("Usage should be below target: %d" % usage)
- if (usage > 550):
- raise AssertionError("Pruning target not being met")
-
- return invalidheight,badhash
+ assert_greater_than(550, usage)
def reorg_back(self):
# Verify that a block on the old main chain fork has been pruned away
@@ -219,17 +234,17 @@ class PruneTest(BitcoinTestFramework):
blocks_to_mine = first_reorg_height + 1 - self.mainchainheight
self.log.info("Rewind node 0 to prev main chain to mine longer chain to trigger redownload. Blocks needed: %d" % blocks_to_mine)
self.nodes[0].invalidateblock(curchainhash)
- assert self.nodes[0].getblockcount() == self.mainchainheight
- assert self.nodes[0].getbestblockhash() == self.mainchainhash2
+ assert_equal(self.nodes[0].getblockcount(), self.mainchainheight)
+ assert_equal(self.nodes[0].getbestblockhash(), self.mainchainhash2)
goalbesthash = self.nodes[0].generate(blocks_to_mine)[-1]
goalbestheight = first_reorg_height + 1
self.log.info("Verify node 2 reorged back to the main chain, some blocks of which it had to redownload")
# Wait for Node 2 to reorg to proper height
wait_until(lambda: self.nodes[2].getblockcount() >= goalbestheight, timeout=900)
- assert self.nodes[2].getbestblockhash() == goalbesthash
+ assert_equal(self.nodes[2].getbestblockhash(), goalbesthash)
# Verify we can now have the data for a block previously pruned
- assert self.nodes[2].getblock(self.forkhash)["height"] == self.forkheight
+ assert_equal(self.nodes[2].getblock(self.forkhash)["height"], self.forkheight)
def manual_test(self, node_number, use_timestamp):
# at this point, node has 995 blocks and has not yet run in prune mode
@@ -287,38 +302,30 @@ class PruneTest(BitcoinTestFramework):
# height=100 too low to prune first block file so this is a no-op
prune(100)
- if not has_block(0):
- raise AssertionError("blk00000.dat is missing when should still be there")
+ assert has_block(0), "blk00000.dat is missing when should still be there"
# Does nothing
node.pruneblockchain(height(0))
- if not has_block(0):
- raise AssertionError("blk00000.dat is missing when should still be there")
+ assert has_block(0), "blk00000.dat is missing when should still be there"
# height=500 should prune first file
prune(500)
- if has_block(0):
- raise AssertionError("blk00000.dat is still there, should be pruned by now")
- if not has_block(1):
- raise AssertionError("blk00001.dat is missing when should still be there")
+ assert not has_block(0), "blk00000.dat is still there, should be pruned by now"
+ assert has_block(1), "blk00001.dat is missing when should still be there"
# height=650 should prune second file
prune(650)
- if has_block(1):
- raise AssertionError("blk00001.dat is still there, should be pruned by now")
+ assert not has_block(1), "blk00001.dat is still there, should be pruned by now"
# height=1000 should not prune anything more, because tip-288 is in blk00002.dat.
prune(1000, 1001 - MIN_BLOCKS_TO_KEEP)
- if not has_block(2):
- raise AssertionError("blk00002.dat is still there, should be pruned by now")
+ assert has_block(2), "blk00002.dat is still there, should be pruned by now"
# advance the tip so blk00002.dat and blk00003.dat can be pruned (the last 288 blocks should now be in blk00004.dat)
node.generate(288)
prune(1000)
- if has_block(2):
- raise AssertionError("blk00002.dat is still there, should be pruned by now")
- if has_block(3):
- raise AssertionError("blk00003.dat is still there, should be pruned by now")
+ assert not has_block(2), "blk00002.dat is still there, should be pruned by now"
+ assert not has_block(3), "blk00003.dat is still there, should be pruned by now"
# stop node, start back up with auto-prune at 550 MiB, make sure still runs
self.stop_node(node_number)
@@ -339,21 +346,14 @@ class PruneTest(BitcoinTestFramework):
connect_nodes(self.nodes[0], 5)
nds = [self.nodes[0], self.nodes[5]]
sync_blocks(nds, wait=5, timeout=300)
- self.stop_node(5) #stop and start to trigger rescan
+ self.stop_node(5) # stop and start to trigger rescan
self.start_node(5, extra_args=["-prune=550"])
self.log.info("Success")
def run_test(self):
- self.log.info("Warning! This test requires 4GB of disk space and takes over 30 mins (up to 2 hours)")
- self.log.info("Mining a big blockchain of 995 blocks")
-
- # Determine default relay fee
- self.relayfee = self.nodes[0].getnetworkinfo()["relayfee"]
-
- # Cache for utxos, as the listunspent may take a long time later in the test
- self.utxo_cache_0 = []
- self.utxo_cache_1 = []
+ self.log.info("Warning! This test requires 4GB of disk space")
+ self.log.info("Mining a big blockchain of 995 blocks")
self.create_big_chain()
# Chain diagram key:
# * blocks on main chain
@@ -394,11 +394,11 @@ class PruneTest(BitcoinTestFramework):
# +...+(1044) &.. $...$(1319)
# Save some current chain state for later use
- self.mainchainheight = self.nodes[2].getblockcount() #1320
+ self.mainchainheight = self.nodes[2].getblockcount() # 1320
self.mainchainhash2 = self.nodes[2].getblockhash(self.mainchainheight)
self.log.info("Check that we can survive a 288 block reorg still")
- (self.forkheight,self.forkhash) = self.reorg_test() #(1033, )
+ self.reorg_test() # (1033, )
# Now create a 288 block reorg by mining a longer chain on N1
# First disconnect N1
# Then invalidate 1033 on main chain and 1032 on fork so height is 1032 on main chain