Merge #7871: Manual block file pruning.

afffeea fixup! Add pruneblockchain RPC to enable manual block file pruning. (Russell Yanofsky)
1fc4ec7 Add pruneblockchain RPC to enable manual block file pruning. (mrbandrews)

1 files changed, 125 insertions, 3 deletions

diff --git a/qa/rpc-tests/pruning.py b/qa/rpc-tests/pruning.py
index 78b8938e4a..25f0779003 100755
--- a/qa/rpc-tests/pruning.py
+++ b/qa/rpc-tests/pruning.py
@@ -25,7 +25,7 @@ class PruneTest(BitcoinTestFramework):
     def __init__(self):
         super().__init__()
         self.setup_clean_chain = True
-        self.num_nodes = 3
+        self.num_nodes = 6
 
         # Cache for utxos, as the listunspent may take a long time later in the test
         self.utxo_cache_0 = []
@@ -43,10 +43,22 @@ class PruneTest(BitcoinTestFramework):
         self.nodes.append(start_node(2, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-prune=550"], timewait=900))
         self.prunedir = self.options.tmpdir+"/node2/regtest/blocks/"
 
+        # Create nodes 3 and 4 to test manual pruning (they will be re-started with manual pruning later)
+        self.nodes.append(start_node(3, self.options.tmpdir, ["-debug=0","-maxreceivebuffer=20000","-blockmaxsize=999000"], timewait=900))
+        self.nodes.append(start_node(4, self.options.tmpdir, ["-debug=0","-maxreceivebuffer=20000","-blockmaxsize=999000"], timewait=900))
+
+        # Create nodes 5 to test wallet in prune mode, but do not connect
+        self.nodes.append(start_node(5, self.options.tmpdir, ["-debug=0", "-prune=550"]))
+
+        # Determine default relay fee
+        self.relayfee = self.nodes[0].getnetworkinfo()["relayfee"]
+
         connect_nodes(self.nodes[0], 1)
         connect_nodes(self.nodes[1], 2)
         connect_nodes(self.nodes[2], 0)
-        sync_blocks(self.nodes[0:3])
+        connect_nodes(self.nodes[0], 3)
+        connect_nodes(self.nodes[0], 4)
+        sync_blocks(self.nodes[0:5])
 
     def create_big_chain(self):
         # Start by creating some coinbases we can spend later
@@ -57,7 +69,7 @@ class PruneTest(BitcoinTestFramework):
         for i in range(645):
             mine_large_block(self.nodes[0], self.utxo_cache_0)
 
-        sync_blocks(self.nodes[0:3])
+        sync_blocks(self.nodes[0:5])
 
     def test_height_min(self):
         if not os.path.isfile(self.prunedir+"blk00000.dat"):
@@ -212,6 +224,103 @@ class PruneTest(BitcoinTestFramework):
         # Verify we can now have the data for a block previously pruned
         assert(self.nodes[2].getblock(self.forkhash)["height"] == self.forkheight)
 
+    def manual_test(self, node_number, use_timestamp):
+        # at this point, node has 995 blocks and has not yet run in prune mode
+        node = self.nodes[node_number] = start_node(node_number, self.options.tmpdir, ["-debug=0"], timewait=900)
+        assert_equal(node.getblockcount(), 995)
+        assert_raises_message(JSONRPCException, "not in prune mode", node.pruneblockchain, 500)
+        stop_node(node, node_number)
+
+        # now re-start in manual pruning mode
+        node = self.nodes[node_number] = start_node(node_number, self.options.tmpdir, ["-debug=0","-prune=1"], timewait=900)
+        assert_equal(node.getblockcount(), 995)
+
+        def height(index):
+            if use_timestamp:
+                return node.getblockheader(node.getblockhash(index))["time"]
+            else:
+                return index
+
+        def has_block(index):
+            return os.path.isfile(self.options.tmpdir + "/node{}/regtest/blocks/blk{:05}.dat".format(node_number, index))
+
+        # should not prune because chain tip of node 3 (995) < PruneAfterHeight (1000)
+        assert_raises_message(JSONRPCException, "Blockchain is too short for pruning", node.pruneblockchain, height(500))
+
+        # mine 6 blocks so we are at height 1001 (i.e., above PruneAfterHeight)
+        node.generate(6)
+
+        # negative and zero inputs should raise an exception
+        try:
+            node.pruneblockchain(-10)
+            raise AssertionError("pruneblockchain(-10) should have failed.")
+        except:
+            pass
+
+        try:
+            node.pruneblockchain(0)
+            raise AssertionError("pruneblockchain(0) should have failed.")
+        except:
+            pass
+
+        # height=100 too low to prune first block file so this is a no-op
+        node.pruneblockchain(height(100))
+        if not has_block(0):
+            raise AssertionError("blk00000.dat is missing when should still be there")
+
+        # height=500 should prune first file
+        node.pruneblockchain(height(500))
+        if has_block(0):
+            raise AssertionError("blk00000.dat is still there, should be pruned by now")
+        if not has_block(1):
+            raise AssertionError("blk00001.dat is missing when should still be there")
+
+        # height=650 should prune second file
+        node.pruneblockchain(height(650))
+        if has_block(1):
+            raise AssertionError("blk00001.dat is still there, should be pruned by now")
+
+        # height=1000 should not prune anything more, because tip-288 is in blk00002.dat.
+        node.pruneblockchain(height(1000))
+        if not has_block(2):
+            raise AssertionError("blk00002.dat is still there, should be pruned by now")
+
+        # advance the tip so blk00002.dat and blk00003.dat can be pruned (the last 288 blocks should now be in blk00004.dat)
+        node.generate(288)
+        node.pruneblockchain(height(1000))
+        if has_block(2):
+            raise AssertionError("blk00002.dat is still there, should be pruned by now")
+        if has_block(3):
+            raise AssertionError("blk00003.dat is still there, should be pruned by now")
+
+        # stop node, start back up with auto-prune at 550MB, make sure still runs
+        stop_node(node, node_number)
+        self.nodes[node_number] = start_node(node_number, self.options.tmpdir, ["-debug=0","-prune=550"], timewait=900)
+
+        print("Success")
+
+    def wallet_test(self):
+        # check that the pruning node's wallet is still in good shape
+        print("Stop and start pruning node to trigger wallet rescan")
+        try:
+            stop_node(self.nodes[2], 2)
+            start_node(2, self.options.tmpdir, ["-debug=1","-prune=550"])
+            print("Success")
+        except Exception as detail:
+            raise AssertionError("Wallet test: unable to re-start the pruning node")
+
+        # check that wallet loads loads successfully when restarting a pruned node after IBD.
+        # this was reported to fail in #7494.
+        print ("Syncing node 5 to test wallet")
+        connect_nodes(self.nodes[0], 5)
+        nds = [self.nodes[0], self.nodes[5]]
+        sync_blocks(nds)
+        try:
+            stop_node(self.nodes[5],5) #stop and start to trigger rescan
+            start_node(5, self.options.tmpdir, ["-debug=1","-prune=550"])
+            print ("Success")
+        except Exception as detail:
+            raise AssertionError("Wallet test: unable to re-start node5")
 
     def run_test(self):
         print("Warning! This test requires 4GB of disk space and takes over 30 mins (up to 2 hours)")
@@ -226,6 +335,10 @@ class PruneTest(BitcoinTestFramework):
         # Start by mining a simple chain that all nodes have
         # N0=N1=N2 **...*(995)
 
+        # stop manual-pruning node with 995 blocks
+        stop_node(self.nodes[3],3)
+        stop_node(self.nodes[4],4)
+
         print("Check that we haven't started pruning yet because we're below PruneAfterHeight")
         self.test_height_min()
         # Extend this chain past the PruneAfterHeight
@@ -308,6 +421,15 @@ class PruneTest(BitcoinTestFramework):
         #
         # N1 doesn't change because 1033 on main chain (*) is invalid
 
+        print("Test manual pruning with block indices")
+        self.manual_test(3, use_timestamp=False)
+
+        print("Test manual pruning with timestamps")
+        self.manual_test(4, use_timestamp=True)
+
+        print("Test wallet re-scan")
+        self.wallet_test()
+
         print("Done")
 
 if __name__ == '__main__':