Merge bitcoin/bitcoin#24865: rpc: Enable wallet import on pruned nodes and add test

564b580bf07742483a140c7c095b896a6d5d6cad test: Introduce MIN_BLOCKS_TO_KEEP constant (Aurèle Oulès)
71d9a7c03b44236c2fea2b74f92a69234d29f717 test: Wallet imports on pruned nodes (Aurèle Oulès)
e6906fcf9e4d5692ead6c9bf5a2e11673315a1f5 rpc: Enable wallet import on pruned nodes (Aurèle Oulès)

Pull request description:

  Reopens #16037

  I have rebased the PR, addressed the comments of the original PR and added a functional test.

  > Before this change importwallet fails if any block is pruned. This PR makes it possible to importwallet if all required blocks aren't pruned. This is possible because the dump format includes key timestamps.

  For reviewers:
  `python test/functional/wallet_pruning.py --nocleanup` will generate a large blockchain (~700MB) that can be used to manually test wallet imports on a pruned node. Node0 is not pruned, while node1 is.

ACKs for top commit:
  kouloumos:
    ACK 564b580bf07742483a140c7c095b896a6d5d6cad
  achow101:
    reACK 564b580bf07742483a140c7c095b896a6d5d6cad
  furszy:
    ACK 564b580
  w0xlt:
    ACK https://github.com/bitcoin/bitcoin/pull/24865/commits/564b580bf07742483a140c7c095b896a6d5d6cad

Tree-SHA512: b345a6c455fcb6581cdaa5f7a55d79e763a55cb08c81d66be5b12794985d79cd51b9b39bdcd0f7ba0a2a2643e9b2ddc49310ff03d16b430df2f74e990800eabf

4 files changed, 170 insertions, 19 deletions

diff --git a/test/functional/feature_pruning.py b/test/functional/feature_pruning.py
index 58bc6ca67c..de1ea8a3a6 100755
--- a/test/functional/feature_pruning.py
+++ b/test/functional/feature_pruning.py
@@ -10,8 +10,11 @@ This test takes 30 mins or more (up to 2 hours)
 """
 import os
 
-from test_framework.blocktools import create_coinbase
-from test_framework.messages import CBlock
+from test_framework.blocktools import (
+    MIN_BLOCKS_TO_KEEP,
+    create_block,
+    create_coinbase,
+)
 from test_framework.script import (
     CScript,
     OP_NOP,
@@ -48,21 +51,7 @@ def mine_large_blocks(node, n):
     previousblockhash = int(best_block["hash"], 16)
 
     for _ in range(n):
-        # Build the coinbase transaction (with large scriptPubKey)
-        coinbase_tx = create_coinbase(height)
-        coinbase_tx.vin[0].nSequence = 2 ** 32 - 1
-        coinbase_tx.vout[0].scriptPubKey = big_script
-        coinbase_tx.rehash()
-
-        # Build the block
-        block = CBlock()
-        block.nVersion = best_block["version"]
-        block.hashPrevBlock = previousblockhash
-        block.nTime = mine_large_blocks.nTime
-        block.nBits = int('207fffff', 16)
-        block.nNonce = 0
-        block.vtx = [coinbase_tx]
-        block.hashMerkleRoot = block.calc_merkle_root()
+        block = create_block(hashprev=previousblockhash, ntime=mine_large_blocks.nTime, coinbase=create_coinbase(height, script_pubkey=big_script))
         block.solve()
 
         # Submit to the node
@@ -345,7 +334,7 @@ class PruneTest(BitcoinTestFramework):
         assert has_block(2), "blk00002.dat is still there, should be pruned by now"
 
         # advance the tip so blk00002.dat and blk00003.dat can be pruned (the last 288 blocks should now be in blk00004.dat)
-        self.generate(node, 288, sync_fun=self.no_op)
+        self.generate(node, MIN_BLOCKS_TO_KEEP, sync_fun=self.no_op)
         prune(1000)
         assert not has_block(2), "blk00002.dat is still there, should be pruned by now"
         assert not has_block(3), "blk00003.dat is still there, should be pruned by now"
diff --git a/test/functional/test_framework/blocktools.py b/test/functional/test_framework/blocktools.py
index 574ea10356..f8e94ca6ba 100644
--- a/test/functional/test_framework/blocktools.py
+++ b/test/functional/test_framework/blocktools.py
@@ -61,6 +61,7 @@ WITNESS_COMMITMENT_HEADER = b"\xaa\x21\xa9\xed"
 
 NORMAL_GBT_REQUEST_PARAMS = {"rules": ["segwit"]}
 VERSIONBITS_LAST_OLD_BLOCK_VERSION = 4
+MIN_BLOCKS_TO_KEEP = 288
 
 
 def create_block(hashprev=None, coinbase=None, ntime=None, *, version=None, tmpl=None, txlist=None):
@@ -120,7 +121,7 @@ def script_BIP34_coinbase_height(height):
     return CScript([CScriptNum(height)])
 
 
-def create_coinbase(height, pubkey=None, extra_output_script=None, fees=0, nValue=50):
+def create_coinbase(height, pubkey=None, *, script_pubkey=None, extra_output_script=None, fees=0, nValue=50):
     """Create a coinbase transaction.
 
     If pubkey is passed in, the coinbase output will be a P2PK output;
@@ -138,6 +139,8 @@ def create_coinbase(height, pubkey=None, extra_output_script=None, fees=0, nValu
         coinbaseoutput.nValue += fees
     if pubkey is not None:
         coinbaseoutput.scriptPubKey = key_to_p2pk_script(pubkey)
+    elif script_pubkey is not None:
+        coinbaseoutput.scriptPubKey = script_pubkey
     else:
         coinbaseoutput.scriptPubKey = CScript([OP_TRUE])
     coinbase.vout = [coinbaseoutput]
diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py
index 31b308546d..ed547b68a8 100755
--- a/test/functional/test_runner.py
+++ b/test/functional/test_runner.py
@@ -85,6 +85,7 @@ EXTENDED_SCRIPTS = [
     'feature_pruning.py',
     'feature_dbcrash.py',
     'feature_index_prune.py',
+    'wallet_pruning.py --legacy-wallet',
 ]
 
 BASE_SCRIPTS = [
diff --git a/test/functional/wallet_pruning.py b/test/functional/wallet_pruning.py
new file mode 100755
index 0000000000..6d8475ce8d
--- /dev/null
+++ b/test/functional/wallet_pruning.py
@@ -0,0 +1,158 @@
+#!/usr/bin/env python3
+# Copyright (c) 2022 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+"""Test wallet import on pruned node."""
+import os
+
+from test_framework.util import assert_equal, assert_raises_rpc_error
+from test_framework.blocktools import (
+    COINBASE_MATURITY,
+    create_block
+)
+from test_framework.blocktools import create_coinbase
+from test_framework.test_framework import BitcoinTestFramework
+
+from test_framework.script import (
+    CScript,
+    OP_RETURN,
+    OP_TRUE,
+)
+
+class WalletPruningTest(BitcoinTestFramework):
+    def add_options(self, parser):
+        self.add_wallet_options(parser, descriptors=False)
+
+    def set_test_params(self):
+        self.setup_clean_chain = True
+        self.num_nodes = 2
+        self.wallet_names = []
+        self.extra_args = [
+            [], # node dedicated to mining
+            ['-prune=550'], # node dedicated to testing pruning
+        ]
+
+    def skip_test_if_missing_module(self):
+        self.skip_if_no_wallet()
+        self.skip_if_no_bdb()
+
+    def mine_large_blocks(self, node, n):
+        # Get the block parameters for the first block
+        best_block = node.getblock(node.getbestblockhash())
+        height = int(best_block["height"]) + 1
+        self.nTime = max(self.nTime, int(best_block["time"])) + 1
+        previousblockhash = int(best_block["hash"], 16)
+        big_script = CScript([OP_RETURN] + [OP_TRUE] * 950000)
+        for _ in range(n):
+            block = create_block(hashprev=previousblockhash, ntime=self.nTime, coinbase=create_coinbase(height, script_pubkey=big_script))
+            block.solve()
+
+            # Submit to the node
+            node.submitblock(block.serialize().hex())
+
+            previousblockhash = block.sha256
+            height += 1
+
+            # Simulate 10 minutes of work time per block
+            # Important for matching a timestamp with a block +- some window
+            self.nTime += 600
+            for n in self.nodes:
+                if n.running:
+                    n.setmocktime(self.nTime) # Update node's time to accept future blocks
+        self.sync_all()
+
+    def test_wallet_import_pruned(self, wallet_name):
+        self.log.info("Make sure we can import wallet when pruned and required blocks are still available")
+
+        wallet_file = wallet_name + ".dat"
+        wallet_birthheight = self.get_birthheight(wallet_file)
+
+        # Verify that the block at wallet's birthheight is available at the pruned node
+        self.nodes[1].getblock(self.nodes[1].getblockhash(wallet_birthheight))
+
+        # Import wallet into pruned node
+        self.nodes[1].createwallet(wallet_name="wallet_pruned", descriptors=False, load_on_startup=True)
+        self.nodes[1].importwallet(os.path.join(self.nodes[0].datadir, wallet_file))
+
+        # Make sure that prune node's wallet correctly accounts for balances
+        assert_equal(self.nodes[1].getbalance(), self.nodes[0].getbalance())
+
+        self.log.info("- Done")
+
+    def test_wallet_import_pruned_with_missing_blocks(self, wallet_name):
+        self.log.info("Make sure we cannot import wallet when pruned and required blocks are not available")
+
+        wallet_file = wallet_name + ".dat"
+        wallet_birthheight = self.get_birthheight(wallet_file)
+
+        # Verify that the block at wallet's birthheight is not available at the pruned node
+        assert_raises_rpc_error(-1, "Block not available (pruned data)", self.nodes[1].getblock, self.nodes[1].getblockhash(wallet_birthheight))
+
+        # Make sure wallet cannot be imported because of missing blocks
+        # This will try to rescan blocks `TIMESTAMP_WINDOW` (2h) before the wallet birthheight.
+        # There are 6 blocks an hour, so 11 blocks (excluding birthheight).
+        assert_raises_rpc_error(-4, f"Pruned blocks from height {wallet_birthheight - 11} required to import keys. Use RPC call getblockchaininfo to determine your pruned height.", self.nodes[1].importwallet, os.path.join(self.nodes[0].datadir, wallet_file))
+        self.log.info("- Done")
+
+    def get_birthheight(self, wallet_file):
+        """Gets birthheight of a wallet on node0"""
+        with open(os.path.join(self.nodes[0].datadir, wallet_file), 'r', encoding="utf8") as f:
+            for line in f:
+                if line.startswith('# * Best block at time of backup'):
+                    wallet_birthheight = int(line.split(' ')[9])
+                    return wallet_birthheight
+
+    def has_block(self, block_index):
+        """Checks if the pruned node has the specific blk0000*.dat file"""
+        return os.path.isfile(os.path.join(self.nodes[1].datadir, self.chain, "blocks", f"blk{block_index:05}.dat"))
+
+    def create_wallet(self, wallet_name, *, unload=False):
+        """Creates and dumps a wallet on the non-pruned node0 to be later import by the pruned node"""
+        self.nodes[0].createwallet(wallet_name=wallet_name, descriptors=False, load_on_startup=True)
+        self.nodes[0].dumpwallet(os.path.join(self.nodes[0].datadir, wallet_name + ".dat"))
+        if (unload):
+            self.nodes[0].unloadwallet(wallet_name)
+
+    def run_test(self):
+        self.nTime = 0
+        self.log.info("Warning! This test requires ~1.3GB of disk space")
+
+        self.log.info("Generating a long chain of blocks...")
+
+        # A blk*.dat file is 128MB
+        # Generate 250 light blocks
+        self.generate(self.nodes[0], 250, sync_fun=self.no_op)
+        # Generate 50MB worth of large blocks in the blk00000.dat file
+        self.mine_large_blocks(self.nodes[0], 50)
+
+        # Create a wallet which birth's block is in the blk00000.dat file
+        wallet_birthheight_1 = "wallet_birthheight_1"
+        assert_equal(self.has_block(1), False)
+        self.create_wallet(wallet_birthheight_1, unload=True)
+
+        # Generate enough large blocks to reach pruning disk limit
+        # Not pruning yet because we are still below PruneAfterHeight
+        self.mine_large_blocks(self.nodes[0], 600)
+        self.log.info("- Long chain created")
+
+        # Create a wallet with birth height > wallet_birthheight_1
+        wallet_birthheight_2 = "wallet_birthheight_2"
+        self.create_wallet(wallet_birthheight_2)
+
+        # Fund wallet to later verify that importwallet correctly accounts for balances
+        self.generatetoaddress(self.nodes[0], COINBASE_MATURITY + 1, self.nodes[0].getnewaddress(), sync_fun=self.no_op)
+
+        # We've reached pruning storage & height limit but
+        # pruning doesn't run until another chunk (blk*.dat file) is allocated.
+        # That's why we are generating another 5 large blocks
+        self.mine_large_blocks(self.nodes[0], 5)
+
+        # blk00000.dat file is now pruned from node1
+        assert_equal(self.has_block(0), False)
+
+        self.test_wallet_import_pruned(wallet_birthheight_2)
+        self.test_wallet_import_pruned_with_missing_blocks(wallet_birthheight_1)
+
+if __name__ == '__main__':
+    WalletPruningTest().main()