Merge #19137: wallettool: Add dump and createfromdump commands

23cac24dd3f2aaf88aab978e7ef4905772815cd2 tests: Test bitcoin-wallet dump and createfromdump (Andrew Chow)
a88c320041bd1cd1786b2dfd9ab698a67c2a57c6 wallettool: Add createfromdump command (Andrew Chow)
e1e7a90d5f0616a46ffadd62a9f1c65406cca6b4 wallettool: Add dump command (Andrew Chow)

Pull request description:

  Adds two commands to the `bitcoin-wallet` tool: `dump` and `createfromdump`. These commands will be useful for a wallet storage migration in the future. It is also generally useful to have a storage agnostic dump like this. These commands are similar to BDB's `db_dump` and `db_load` tools. This can also be useful for manual construction of a wallet file for tests.

  `dump` outputs every key-value pair from the wallet as comma separated hex. Each key-value pair is on its own line with the key and value in hex separated by a comma. This is output to the file specified by the new `-dumpfile` option.

  `createfromdump` takes a file produced by `dump` and creates a new wallet file with exactly the records specified in that file.

  A new option `-dumpfile` is added to the wallet tool. When used with `dump`, the records will be written to the specified file. When used with `createfromdump`, the file is read and the key-value pairs constructed from it. `createfromdump` requires `-dumpfile`.

  A simple round-trip test is added to the `tool_wallet.py`.

  This PR is based on #19334,

ACKs for top commit:
  Sjors:
    re-utACK 23cac24
  MarcoFalke:
    re review ACK 23cac24dd3f2aaf88aab978e7ef4905772815cd2 only change is rebase and removing useless shared_ptr wrapper 🎼
  ryanofsky:
    Code review ACK 23cac24dd3f2aaf88aab978e7ef4905772815cd2. Only changes since last review rebase and changing a pointer to a reference

Tree-SHA512: 2d63cf62baca3d16495aa698dc02f7d889c81b41015e9c92c23c275bb4a690fc176d351c3fd7f310bd6b17f5a936cc9be694cbecd702af741b96c0f530e72fa2

1 files changed, 161 insertions, 0 deletions

diff --git a/test/functional/tool_wallet.py b/test/functional/tool_wallet.py
index 35576f00ea..1093982929 100755
--- a/test/functional/tool_wallet.py
+++ b/test/functional/tool_wallet.py
@@ -10,6 +10,8 @@ import stat
 import subprocess
 import textwrap
 
+from collections import OrderedDict
+
 from test_framework.test_framework import BitcoinTestFramework
 from test_framework.util import assert_equal
 
@@ -96,6 +98,89 @@ class ToolWalletTest(BitcoinTestFramework):
                 Address Book: %d
             ''' % (wallet_name, keypool, transactions, address * output_types))
 
+    def read_dump(self, filename):
+        dump = OrderedDict()
+        with open(filename, "r", encoding="utf8") as f:
+            for row in f:
+                row = row.strip()
+                key, value = row.split(',')
+                dump[key] = value
+        return dump
+
+    def assert_is_sqlite(self, filename):
+        with open(filename, 'rb') as f:
+            file_magic = f.read(16)
+            assert file_magic == b'SQLite format 3\x00'
+
+    def assert_is_bdb(self, filename):
+        with open(filename, 'rb') as f:
+            f.seek(12, 0)
+            file_magic = f.read(4)
+            assert file_magic == b'\x00\x05\x31\x62' or file_magic == b'\x62\x31\x05\x00'
+
+    def write_dump(self, dump, filename, magic=None, skip_checksum=False):
+        if magic is None:
+            magic = "BITCOIN_CORE_WALLET_DUMP"
+        with open(filename, "w", encoding="utf8") as f:
+            row = ",".join([magic, dump[magic]]) + "\n"
+            f.write(row)
+            for k, v in dump.items():
+                if k == magic or k == "checksum":
+                    continue
+                row = ",".join([k, v]) + "\n"
+                f.write(row)
+            if not skip_checksum:
+                row = ",".join(["checksum", dump["checksum"]]) + "\n"
+                f.write(row)
+
+    def assert_dump(self, expected, received):
+        e = expected.copy()
+        r = received.copy()
+
+        # BDB will add a "version" record that is not present in sqlite
+        # In that case, we should ignore this record in both
+        # But because this also effects the checksum, we also need to drop that.
+        v_key = "0776657273696f6e" # Version key
+        if v_key in e and v_key not in r:
+            del e[v_key]
+            del e["checksum"]
+            del r["checksum"]
+        if v_key not in e and v_key in r:
+            del r[v_key]
+            del e["checksum"]
+            del r["checksum"]
+
+        assert_equal(len(e), len(r))
+        for k, v in e.items():
+            assert_equal(v, r[k])
+
+    def do_tool_createfromdump(self, wallet_name, dumpfile, file_format=None):
+        dumppath = os.path.join(self.nodes[0].datadir, dumpfile)
+        rt_dumppath = os.path.join(self.nodes[0].datadir, "rt-{}.dump".format(wallet_name))
+
+        dump_data = self.read_dump(dumppath)
+
+        args = ["-wallet={}".format(wallet_name),
+                "-dumpfile={}".format(dumppath)]
+        if file_format is not None:
+            args.append("-format={}".format(file_format))
+        args.append("createfromdump")
+
+        load_output = ""
+        if file_format is not None and file_format != dump_data["format"]:
+            load_output += "Warning: Dumpfile wallet format \"{}\" does not match command line specified format \"{}\".\n".format(dump_data["format"], file_format)
+        self.assert_tool_output(load_output, *args)
+        assert os.path.isdir(os.path.join(self.nodes[0].datadir, "regtest/wallets", wallet_name))
+
+        self.assert_tool_output("The dumpfile may contain private keys. To ensure the safety of your Bitcoin, do not share the dumpfile.\n", '-wallet={}'.format(wallet_name), '-dumpfile={}'.format(rt_dumppath), 'dump')
+
+        rt_dump_data = self.read_dump(rt_dumppath)
+        wallet_dat = os.path.join(self.nodes[0].datadir, "regtest/wallets/", wallet_name, "wallet.dat")
+        if rt_dump_data["format"] == "bdb":
+            self.assert_is_bdb(wallet_dat)
+        else:
+            self.assert_is_sqlite(wallet_dat)
+
     def test_invalid_tool_commands_and_args(self):
         self.log.info('Testing that various invalid commands raise with specific error messages')
         self.assert_raises_tool_error('Invalid command: foo', 'foo')
@@ -228,6 +313,81 @@ class ToolWalletTest(BitcoinTestFramework):
 
         self.assert_tool_output('', '-wallet=salvage', 'salvage')
 
+    def test_dump_createfromdump(self):
+        self.start_node(0)
+        self.nodes[0].createwallet("todump")
+        file_format = self.nodes[0].get_wallet_rpc("todump").getwalletinfo()["format"]
+        self.nodes[0].createwallet("todump2")
+        self.stop_node(0)
+
+        self.log.info('Checking dump arguments')
+        self.assert_raises_tool_error('No dump file provided. To use dump, -dumpfile=<filename> must be provided.', '-wallet=todump', 'dump')
+
+        self.log.info('Checking basic dump')
+        wallet_dump = os.path.join(self.nodes[0].datadir, "wallet.dump")
+        self.assert_tool_output('The dumpfile may contain private keys. To ensure the safety of your Bitcoin, do not share the dumpfile.\n', '-wallet=todump', '-dumpfile={}'.format(wallet_dump), 'dump')
+
+        dump_data = self.read_dump(wallet_dump)
+        orig_dump = dump_data.copy()
+        # Check the dump magic
+        assert_equal(dump_data['BITCOIN_CORE_WALLET_DUMP'], '1')
+        # Check the file format
+        assert_equal(dump_data["format"], file_format)
+
+        self.log.info('Checking that a dumpfile cannot be overwritten')
+        self.assert_raises_tool_error('File {} already exists. If you are sure this is what you want, move it out of the way first.'.format(wallet_dump),  '-wallet=todump2', '-dumpfile={}'.format(wallet_dump), 'dump')
+
+        self.log.info('Checking createfromdump arguments')
+        self.assert_raises_tool_error('No dump file provided. To use createfromdump, -dumpfile=<filename> must be provided.', '-wallet=todump', 'createfromdump')
+        non_exist_dump = os.path.join(self.nodes[0].datadir, "wallet.nodump")
+        self.assert_raises_tool_error('Unknown wallet file format "notaformat" provided. Please provide one of "bdb" or "sqlite".', '-wallet=todump', '-format=notaformat', '-dumpfile={}'.format(wallet_dump), 'createfromdump')
+        self.assert_raises_tool_error('Dump file {} does not exist.'.format(non_exist_dump), '-wallet=todump', '-dumpfile={}'.format(non_exist_dump), 'createfromdump')
+        wallet_path = os.path.join(self.nodes[0].datadir, 'regtest/wallets/todump2')
+        self.assert_raises_tool_error('Failed to create database path \'{}\'. Database already exists.'.format(wallet_path), '-wallet=todump2', '-dumpfile={}'.format(wallet_dump), 'createfromdump')
+
+        self.log.info('Checking createfromdump')
+        self.do_tool_createfromdump("load", "wallet.dump")
+        self.do_tool_createfromdump("load-bdb", "wallet.dump", "bdb")
+        self.do_tool_createfromdump("load-sqlite", "wallet.dump", "sqlite")
+
+        self.log.info('Checking createfromdump handling of magic and versions')
+        bad_ver_wallet_dump = os.path.join(self.nodes[0].datadir, "wallet-bad_ver1.dump")
+        dump_data["BITCOIN_CORE_WALLET_DUMP"] = "0"
+        self.write_dump(dump_data, bad_ver_wallet_dump)
+        self.assert_raises_tool_error('Error: Dumpfile version is not supported. This version of bitcoin-wallet only supports version 1 dumpfiles. Got dumpfile with version 0', '-wallet=badload', '-dumpfile={}'.format(bad_ver_wallet_dump), 'createfromdump')
+        assert not os.path.isdir(os.path.join(self.nodes[0].datadir, "regtest/wallets", "badload"))
+        bad_ver_wallet_dump = os.path.join(self.nodes[0].datadir, "wallet-bad_ver2.dump")
+        dump_data["BITCOIN_CORE_WALLET_DUMP"] = "2"
+        self.write_dump(dump_data, bad_ver_wallet_dump)
+        self.assert_raises_tool_error('Error: Dumpfile version is not supported. This version of bitcoin-wallet only supports version 1 dumpfiles. Got dumpfile with version 2', '-wallet=badload', '-dumpfile={}'.format(bad_ver_wallet_dump), 'createfromdump')
+        assert not os.path.isdir(os.path.join(self.nodes[0].datadir, "regtest/wallets", "badload"))
+        bad_magic_wallet_dump = os.path.join(self.nodes[0].datadir, "wallet-bad_magic.dump")
+        del dump_data["BITCOIN_CORE_WALLET_DUMP"]
+        dump_data["not_the_right_magic"] = "1"
+        self.write_dump(dump_data, bad_magic_wallet_dump, "not_the_right_magic")
+        self.assert_raises_tool_error('Error: Dumpfile identifier record is incorrect. Got "not_the_right_magic", expected "BITCOIN_CORE_WALLET_DUMP".', '-wallet=badload', '-dumpfile={}'.format(bad_magic_wallet_dump), 'createfromdump')
+        assert not os.path.isdir(os.path.join(self.nodes[0].datadir, "regtest/wallets", "badload"))
+
+        self.log.info('Checking createfromdump handling of checksums')
+        bad_sum_wallet_dump = os.path.join(self.nodes[0].datadir, "wallet-bad_sum1.dump")
+        dump_data = orig_dump.copy()
+        checksum = dump_data["checksum"]
+        dump_data["checksum"] = "1" * 64
+        self.write_dump(dump_data, bad_sum_wallet_dump)
+        self.assert_raises_tool_error('Error: Dumpfile checksum does not match. Computed {}, expected {}'.format(checksum, "1" * 64), '-wallet=bad', '-dumpfile={}'.format(bad_sum_wallet_dump), 'createfromdump')
+        assert not os.path.isdir(os.path.join(self.nodes[0].datadir, "regtest/wallets", "badload"))
+        bad_sum_wallet_dump = os.path.join(self.nodes[0].datadir, "wallet-bad_sum2.dump")
+        del dump_data["checksum"]
+        self.write_dump(dump_data, bad_sum_wallet_dump, skip_checksum=True)
+        self.assert_raises_tool_error('Error: Missing checksum', '-wallet=badload', '-dumpfile={}'.format(bad_sum_wallet_dump), 'createfromdump')
+        assert not os.path.isdir(os.path.join(self.nodes[0].datadir, "regtest/wallets", "badload"))
+        bad_sum_wallet_dump = os.path.join(self.nodes[0].datadir, "wallet-bad_sum3.dump")
+        dump_data["checksum"] = "2" * 10
+        self.write_dump(dump_data, bad_sum_wallet_dump)
+        self.assert_raises_tool_error('Error: Dumpfile checksum does not match. Computed {}, expected {}{}'.format(checksum, "2" * 10, "0" * 54), '-wallet=badload', '-dumpfile={}'.format(bad_sum_wallet_dump), 'createfromdump')
+        assert not os.path.isdir(os.path.join(self.nodes[0].datadir, "regtest/wallets", "badload"))
+
+
     def run_test(self):
         self.wallet_path = os.path.join(self.nodes[0].datadir, self.chain, 'wallets', self.default_wallet_name, self.wallet_data_filename)
         self.test_invalid_tool_commands_and_args()
@@ -239,6 +399,7 @@ class ToolWalletTest(BitcoinTestFramework):
         if not self.options.descriptors:
             # Salvage is a legacy wallet only thing
             self.test_salvage()
+        self.test_dump_createfromdump()
 
 if __name__ == '__main__':
     ToolWalletTest().main()