Merge bitcoin/bitcoin#23304: wallet: Derive inactive HD chains in additional places

c4d76c6faa3adf06f192649e169ca860ce420d30 tests: Tests for inactive HD chains (Andrew Chow)
8077862c5e8a3ed501f0baabc33536eb16922ceb wallet: Refactor TopUp to be able to top up inactive chains too (Andrew Chow)
70134eb34f58f0c572e7c3775e292d408f03b5ab wallet: Properly set hd chain counters when loading (Andrew Chow)
961b9e4e40019a87eaa11c8a9c3305870f7a6d75 wallet: Parse hdKeypath if key_origin is not available (Andrew Chow)
0652ee73ec880a66ec88bde007ee03c0b9d1b074 Add size check on meta.key_origin.path (Rob Fielding)

Pull request description:

  Currently inactive HD chains are only derived from at the time a key in that chain is found to have been used. However, at that time, the wallet may not be able to derive keys (e.g. it is locked). Currently we would just move on and not derive any new keys, however this could result in missing funds.

  This PR resolves this problem by adding memory only variables to `CHDChain` which track the highest known index. `TopUp` is modified to always try to top up the inactive HD chains, and this process will use the new variables to determine how much to top up. In this way, after an encrypted wallet is unlocked, the inactive HD chains will be topped up and hopefully funds will not be missed.

  Note that because these variables are not persisted to disk (because `CHDChain`s for inactive HD chains are not written to disk), if an encrypted wallet is not unlocked in the same session as a key from an inactive chain is found to be used, then it will not be topped up later unless more keys are found.

  Additionally, wallets which do not have upgraded key metadata will not derive any keys from inactive HD chains. This is resolved by using the derivation path string in `CKeyMetadata.hdKeypath` to determine what indexes to derive.

ACKs for top commit:
  laanwj:
    Code review ACK c4d76c6faa3adf06f192649e169ca860ce420d30

Tree-SHA512: b2b572ad7f1b1b2847edece09f7583543d63997e18ae32764e5a27ad608dd64b9bdb2d84ea27137894e986a8e82f047a3dba9c8015b74f5f179961911f0c4095

3 files changed, 151 insertions, 0 deletions

diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py
index 289e83579b..e56d4aa492 100755
--- a/test/functional/test_framework/test_node.py
+++ b/test/functional/test_framework/test_node.py
@@ -743,6 +743,9 @@ class RPCOverloadWrapper():
     def __getattr__(self, name):
         return getattr(self.rpc, name)
 
+    def createwallet_passthrough(self, *args, **kwargs):
+        return self.__getattr__("createwallet")(*args, **kwargs)
+
     def createwallet(self, wallet_name, disable_private_keys=None, blank=None, passphrase='', avoid_reuse=None, descriptors=None, load_on_startup=None, external_signer=None):
         if descriptors is None:
             descriptors = self.descriptors
diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py
index 516e8be638..d15edcedd2 100755
--- a/test/functional/test_runner.py
+++ b/test/functional/test_runner.py
@@ -281,6 +281,7 @@ BASE_SCRIPTS = [
     'wallet_send.py --descriptors',
     'wallet_create_tx.py --descriptors',
     'wallet_taproot.py',
+    'wallet_inactive_hdchains.py',
     'p2p_fingerprint.py',
     'feature_uacomment.py',
     'feature_init.py',
diff --git a/test/functional/wallet_inactive_hdchains.py b/test/functional/wallet_inactive_hdchains.py
new file mode 100755
index 0000000000..e1dad00876
--- /dev/null
+++ b/test/functional/wallet_inactive_hdchains.py
@@ -0,0 +1,147 @@
+#!/usr/bin/env python3
+# Copyright (c) 2021 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+"""
+Test Inactive HD Chains.
+"""
+import os
+import shutil
+import time
+
+from test_framework.authproxy import JSONRPCException
+from test_framework.test_framework import BitcoinTestFramework
+from test_framework.wallet_util import (
+    get_generate_key,
+)
+
+
+class InactiveHDChainsTest(BitcoinTestFramework):
+    def set_test_params(self):
+        self.setup_clean_chain = True
+        self.num_nodes = 2
+        self.extra_args = [["-keypool=10"], ["-nowallet", "-keypool=10"]]
+
+    def skip_test_if_missing_module(self):
+        self.skip_if_no_wallet()
+        self.skip_if_no_bdb()
+        self.skip_if_no_previous_releases()
+
+    def setup_nodes(self):
+        self.add_nodes(self.num_nodes, extra_args=self.extra_args, versions=[
+            None,
+            170200, # 0.17.2 Does not have the key metadata upgrade
+        ])
+
+        self.start_nodes()
+        self.init_wallet(node=0)
+
+    def prepare_wallets(self, wallet_basename, encrypt=False):
+        self.nodes[0].createwallet(wallet_name=f"{wallet_basename}_base", descriptors=False, blank=True)
+        self.nodes[0].createwallet(wallet_name=f"{wallet_basename}_test", descriptors=False, blank=True)
+        base_wallet = self.nodes[0].get_wallet_rpc(f"{wallet_basename}_base")
+        test_wallet = self.nodes[0].get_wallet_rpc(f"{wallet_basename}_test")
+
+        # Setup both wallets with the same HD seed
+        seed = get_generate_key()
+        base_wallet.sethdseed(True, seed.privkey)
+        test_wallet.sethdseed(True, seed.privkey)
+
+        if encrypt:
+            # Encrypting will generate a new HD seed and flush the keypool
+            test_wallet.encryptwallet("pass")
+        else:
+            # Generate a new HD seed on the test wallet
+            test_wallet.sethdseed()
+
+        return base_wallet, test_wallet
+
+    def do_inactive_test(self, base_wallet, test_wallet, encrypt=False):
+        default = self.nodes[0].get_wallet_rpc(self.default_wallet_name)
+
+        # The first address should be known by both wallets.
+        addr1 = base_wallet.getnewaddress()
+        assert test_wallet.getaddressinfo(addr1)["ismine"]
+        # The address at index 9 is the first address that the test wallet will not know initially
+        for _ in range(0, 9):
+            base_wallet.getnewaddress()
+        addr2 = base_wallet.getnewaddress()
+        assert not test_wallet.getaddressinfo(addr2)["ismine"]
+
+        # Send to first address on the old seed
+        txid = default.sendtoaddress(addr1, 10)
+        self.generate(self.nodes[0], 1)
+
+        # Wait for the test wallet to see the transaction
+        while True:
+            try:
+                test_wallet.gettransaction(txid)
+                break
+            except JSONRPCException:
+                time.sleep(0.1)
+
+        if encrypt:
+            # The test wallet will not be able to generate the topped up keypool
+            # until it is unlocked. So it still should not know about the second address
+            assert not test_wallet.getaddressinfo(addr2)["ismine"]
+            test_wallet.walletpassphrase("pass", 1)
+
+        # The test wallet should now know about the second address as it
+        # should have generated it in the inactive chain's keypool
+        assert test_wallet.getaddressinfo(addr2)["ismine"]
+
+        # Send to second address on the old seed
+        txid = default.sendtoaddress(addr2, 10)
+        self.generate(self.nodes[0], 1)
+        test_wallet.gettransaction(txid)
+
+    def test_basic(self):
+        self.log.info("Test basic case for inactive HD chains")
+        self.do_inactive_test(*self.prepare_wallets("basic"))
+
+    def test_encrypted_wallet(self):
+        self.log.info("Test inactive HD chains when wallet is encrypted")
+        self.do_inactive_test(*self.prepare_wallets("enc", encrypt=True), encrypt=True)
+
+    def test_without_upgraded_keymeta(self):
+        # Test that it is possible to top up inactive hd chains even if there is no key origin
+        # in CKeyMetadata. This tests for the segfault reported in
+        # https://github.com/bitcoin/bitcoin/issues/21605
+        self.log.info("Test that topping up inactive HD chains does not need upgraded key origin")
+
+        self.nodes[0].createwallet(wallet_name="keymeta_base", descriptors=False, blank=True)
+        # Createwallet is overridden in the test framework so that the descriptor option can be filled
+        # depending on the test's cli args. However we don't want to do that when using old nodes that
+        # do not support descriptors. So we use the createwallet_passthrough function.
+        self.nodes[1].createwallet_passthrough(wallet_name="keymeta_test")
+        base_wallet = self.nodes[0].get_wallet_rpc("keymeta_base")
+        test_wallet = self.nodes[1].get_wallet_rpc("keymeta_test")
+
+        # Setup both wallets with the same HD seed
+        seed = get_generate_key()
+        base_wallet.sethdseed(True, seed.privkey)
+        test_wallet.sethdseed(True, seed.privkey)
+
+        # Encrypting will generate a new HD seed and flush the keypool
+        test_wallet.encryptwallet("pass")
+
+        # Copy test wallet to node 0
+        test_wallet.unloadwallet()
+        test_wallet_dir = os.path.join(self.nodes[1].datadir, "regtest/wallets/keymeta_test")
+        new_test_wallet_dir = os.path.join(self.nodes[0].datadir, "regtest/wallets/keymeta_test")
+        shutil.copytree(test_wallet_dir, new_test_wallet_dir)
+        self.nodes[0].loadwallet("keymeta_test")
+        test_wallet = self.nodes[0].get_wallet_rpc("keymeta_test")
+
+        self.do_inactive_test(base_wallet, test_wallet, encrypt=True)
+
+    def run_test(self):
+        self.generate(self.nodes[0], 101)
+
+        self.test_basic()
+        self.test_encrypted_wallet()
+        self.test_without_upgraded_keymeta()
+
+
+if __name__ == '__main__':
+    InactiveHDChainsTest().main()