From 70cfbfdadf16d3b115309c6938f07ef5b96c7cc1 Mon Sep 17 00:00:00 2001 From: Ava Chow Date: Tue, 2 Jan 2024 16:36:17 -0500 Subject: wallettool: Optionally use BERKELEY_RO as format when dumping BDB wallets In order to ease the transition to not having BDB, make the dump tool use DatabaseFormmat::BERKELEY_RO when -withinternalbdb is set. --- test/functional/test_runner.py | 1 + test/functional/tool_wallet.py | 3 +++ 2 files changed, 4 insertions(+) (limited to 'test') diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 3f6e47d410..89e4aa7055 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -192,6 +192,7 @@ BASE_SCRIPTS = [ 'mempool_resurrect.py', 'wallet_txn_doublespend.py --mineblock', 'tool_wallet.py --legacy-wallet', + 'tool_wallet.py --legacy-wallet --bdbro', 'tool_wallet.py --descriptors', 'tool_signet_miner.py --legacy-wallet', 'tool_signet_miner.py --descriptors', diff --git a/test/functional/tool_wallet.py b/test/functional/tool_wallet.py index fc042bca66..704f1fe9e8 100755 --- a/test/functional/tool_wallet.py +++ b/test/functional/tool_wallet.py @@ -21,6 +21,7 @@ from test_framework.util import ( class ToolWalletTest(BitcoinTestFramework): def add_options(self, parser): self.add_wallet_options(parser) + parser.add_argument("--bdbro", action="store_true", help="Use the BerkeleyRO internal parser when dumping a Berkeley DB wallet file") def set_test_params(self): self.num_nodes = 1 @@ -35,6 +36,8 @@ class ToolWalletTest(BitcoinTestFramework): default_args = ['-datadir={}'.format(self.nodes[0].datadir_path), '-chain=%s' % self.chain] if not self.options.descriptors and 'create' in args: default_args.append('-legacy') + if "dump" in args and self.options.bdbro: + default_args.append("-withinternalbdb") return subprocess.Popen([self.options.bitcoinwallet] + default_args + list(args), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) -- cgit v1.2.3 From fd7b16e391ed320e35255157a28be14c947ef30a Mon Sep 17 00:00:00 2001 From: Ava Chow Date: Mon, 22 Apr 2024 15:39:07 -0400 Subject: test: Test dumps of other endian BDB files --- test/functional/tool_wallet.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) (limited to 'test') diff --git a/test/functional/tool_wallet.py b/test/functional/tool_wallet.py index 704f1fe9e8..c08f96bbdc 100755 --- a/test/functional/tool_wallet.py +++ b/test/functional/tool_wallet.py @@ -454,6 +454,25 @@ class ToolWalletTest(BitcoinTestFramework): ''') self.assert_tool_output(expected_output, "-wallet=conflicts", "info") + def test_dump_endianness(self): + self.log.info("Testing dumps of the same contents with different BDB endianness") + + self.start_node(0) + self.nodes[0].createwallet("endian") + self.stop_node(0) + + wallet_dump = self.nodes[0].datadir_path / "endian.dump" + self.assert_tool_output("The dumpfile may contain private keys. To ensure the safety of your Bitcoin, do not share the dumpfile.\n", "-wallet=endian", f"-dumpfile={wallet_dump}", "dump") + expected_dump = self.read_dump(wallet_dump) + + self.do_tool_createfromdump("native_endian", "endian.dump", "bdb") + native_dump = self.read_dump(self.nodes[0].datadir_path / "rt-native_endian.dump") + self.assert_dump(expected_dump, native_dump) + + self.do_tool_createfromdump("other_endian", "endian.dump", "bdb_swap") + other_dump = self.read_dump(self.nodes[0].datadir_path / "rt-other_endian.dump") + self.assert_dump(expected_dump, other_dump) + def run_test(self): self.wallet_path = self.nodes[0].wallets_path / self.default_wallet_name / self.wallet_data_filename self.test_invalid_tool_commands_and_args() @@ -465,6 +484,7 @@ class ToolWalletTest(BitcoinTestFramework): if not self.options.descriptors: # Salvage is a legacy wallet only thing self.test_salvage() + self.test_dump_endianness() self.test_dump_createfromdump() self.test_chainless_conflicts() -- cgit v1.2.3 From c1984f128284589423b7e0cc06c9a3b23a242d95 Mon Sep 17 00:00:00 2001 From: Ava Chow Date: Mon, 22 Apr 2024 16:06:29 -0400 Subject: test: Test dumping dbs with overflow pages --- test/functional/tool_wallet.py | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) (limited to 'test') diff --git a/test/functional/tool_wallet.py b/test/functional/tool_wallet.py index c08f96bbdc..92da687b02 100755 --- a/test/functional/tool_wallet.py +++ b/test/functional/tool_wallet.py @@ -14,6 +14,7 @@ from collections import OrderedDict from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, + assert_greater_than, sha256sum_file, ) @@ -473,6 +474,42 @@ class ToolWalletTest(BitcoinTestFramework): other_dump = self.read_dump(self.nodes[0].datadir_path / "rt-other_endian.dump") self.assert_dump(expected_dump, other_dump) + def test_dump_very_large_records(self): + self.log.info("Test that wallets with large records are successfully dumped") + + self.start_node(0) + self.nodes[0].createwallet("bigrecords") + wallet = self.nodes[0].get_wallet_rpc("bigrecords") + + # Both BDB and sqlite have maximum page sizes of 65536 bytes, with defaults of 4096 + # When a record exceeds some size threshold, both BDB and SQLite will store the data + # in one or more overflow pages. We want to make sure that our tooling can dump such + # records, even when they span multiple pages. To make a large record, we just need + # to make a very big transaction. + self.generate(self.nodes[0], 101) + def_wallet = self.nodes[0].get_wallet_rpc(self.default_wallet_name) + outputs = {} + for i in range(500): + outputs[wallet.getnewaddress(address_type="p2sh-segwit")] = 0.01 + def_wallet.sendmany(amounts=outputs) + self.generate(self.nodes[0], 1) + send_res = wallet.sendall([def_wallet.getnewaddress()]) + self.generate(self.nodes[0], 1) + assert_equal(send_res["complete"], True) + tx = wallet.gettransaction(txid=send_res["txid"], verbose=True) + assert_greater_than(tx["decoded"]["size"], 70000) + + self.stop_node(0) + + wallet_dump = self.nodes[0].datadir_path / "bigrecords.dump" + self.assert_tool_output("The dumpfile may contain private keys. To ensure the safety of your Bitcoin, do not share the dumpfile.\n", "-wallet=bigrecords", f"-dumpfile={wallet_dump}", "dump") + dump = self.read_dump(wallet_dump) + for k,v in dump.items(): + if tx["hex"] in v: + break + else: + assert False, "Big transaction was not found in wallet dump" + def run_test(self): self.wallet_path = self.nodes[0].wallets_path / self.default_wallet_name / self.wallet_data_filename self.test_invalid_tool_commands_and_args() @@ -487,6 +524,7 @@ class ToolWalletTest(BitcoinTestFramework): self.test_dump_endianness() self.test_dump_createfromdump() self.test_chainless_conflicts() + self.test_dump_very_large_records() if __name__ == '__main__': ToolWalletTest().main() -- cgit v1.2.3 From 0b753156ce60c29efb2386954ba7555ad8f642f5 Mon Sep 17 00:00:00 2001 From: Ava Chow Date: Mon, 22 Apr 2024 16:42:44 -0400 Subject: test: Test bdb_ro dump of wallet without reset LSNs --- test/functional/test_framework/test_node.py | 5 ++-- test/functional/tool_wallet.py | 37 +++++++++++++++++++++++++++-- 2 files changed, 38 insertions(+), 4 deletions(-) (limited to 'test') diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index 67e0be5280..534f25e535 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -419,8 +419,9 @@ class TestNode(): return True def wait_until_stopped(self, *, timeout=BITCOIND_PROC_WAIT_TIMEOUT, expect_error=False, **kwargs): - expected_ret_code = 1 if expect_error else 0 # Whether node shutdown return EXIT_FAILURE or EXIT_SUCCESS - self.wait_until(lambda: self.is_node_stopped(expected_ret_code=expected_ret_code, **kwargs), timeout=timeout) + if "expected_ret_code" not in kwargs: + kwargs["expected_ret_code"] = 1 if expect_error else 0 # Whether node shutdown return EXIT_FAILURE or EXIT_SUCCESS + self.wait_until(lambda: self.is_node_stopped(**kwargs), timeout=timeout) def replace_in_config(self, replacements): """ diff --git a/test/functional/tool_wallet.py b/test/functional/tool_wallet.py index 92da687b02..51239831cc 100755 --- a/test/functional/tool_wallet.py +++ b/test/functional/tool_wallet.py @@ -5,6 +5,7 @@ """Test bitcoin-wallet.""" import os +import platform import stat import subprocess import textwrap @@ -45,9 +46,13 @@ class ToolWalletTest(BitcoinTestFramework): def assert_raises_tool_error(self, error, *args): p = self.bitcoin_wallet_process(*args) stdout, stderr = p.communicate() - assert_equal(p.poll(), 1) assert_equal(stdout, '') - assert_equal(stderr.strip(), error) + if isinstance(error, tuple): + assert_equal(p.poll(), error[0]) + assert error[1] in stderr.strip() + else: + assert_equal(p.poll(), 1) + assert error in stderr.strip() def assert_tool_output(self, output, *args): p = self.bitcoin_wallet_process(*args) @@ -510,6 +515,33 @@ class ToolWalletTest(BitcoinTestFramework): else: assert False, "Big transaction was not found in wallet dump" + def test_dump_unclean_lsns(self): + if not self.options.bdbro: + return + self.log.info("Test that a legacy wallet that has not been compacted is not dumped by bdbro") + + self.start_node(0, extra_args=["-flushwallet=0"]) + self.nodes[0].createwallet("unclean_lsn") + wallet = self.nodes[0].get_wallet_rpc("unclean_lsn") + # First unload and load normally to make sure everything is written + wallet.unloadwallet() + self.nodes[0].loadwallet("unclean_lsn") + # Next cause a bunch of writes by filling the keypool + wallet.keypoolrefill(wallet.getwalletinfo()["keypoolsize"] + 100) + # Lastly kill bitcoind so that the LSNs don't get reset + self.nodes[0].process.kill() + self.nodes[0].wait_until_stopped(expected_ret_code=1 if platform.system() == "Windows" else -9) + assert self.nodes[0].is_node_stopped() + + wallet_dump = self.nodes[0].datadir_path / "unclean_lsn.dump" + self.assert_raises_tool_error("LSNs are not reset, this database is not completely flushed. Please reopen then close the database with a version that has BDB support", "-wallet=unclean_lsn", f"-dumpfile={wallet_dump}", "dump") + + # File can be dumped after reload it normally + self.start_node(0) + self.nodes[0].loadwallet("unclean_lsn") + self.stop_node(0) + self.assert_tool_output("The dumpfile may contain private keys. To ensure the safety of your Bitcoin, do not share the dumpfile.\n", "-wallet=unclean_lsn", f"-dumpfile={wallet_dump}", "dump") + def run_test(self): self.wallet_path = self.nodes[0].wallets_path / self.default_wallet_name / self.wallet_data_filename self.test_invalid_tool_commands_and_args() @@ -522,6 +554,7 @@ class ToolWalletTest(BitcoinTestFramework): # Salvage is a legacy wallet only thing self.test_salvage() self.test_dump_endianness() + self.test_dump_unclean_lsns() self.test_dump_createfromdump() self.test_chainless_conflicts() self.test_dump_very_large_records() -- cgit v1.2.3 From d51fbab4b32d56765e8faab6ad01245fb259b0ca Mon Sep 17 00:00:00 2001 From: Ava Chow Date: Mon, 22 Apr 2024 17:08:18 -0400 Subject: wallet, test: Be able to always swap BDB endianness --- test/functional/test_runner.py | 1 + test/functional/tool_wallet.py | 3 +++ 2 files changed, 4 insertions(+) (limited to 'test') diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 89e4aa7055..0a28cc9394 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -193,6 +193,7 @@ BASE_SCRIPTS = [ 'wallet_txn_doublespend.py --mineblock', 'tool_wallet.py --legacy-wallet', 'tool_wallet.py --legacy-wallet --bdbro', + 'tool_wallet.py --legacy-wallet --bdbro --swap-bdb-endian', 'tool_wallet.py --descriptors', 'tool_signet_miner.py --legacy-wallet', 'tool_signet_miner.py --descriptors', diff --git a/test/functional/tool_wallet.py b/test/functional/tool_wallet.py index 51239831cc..dcf74f6075 100755 --- a/test/functional/tool_wallet.py +++ b/test/functional/tool_wallet.py @@ -24,11 +24,14 @@ class ToolWalletTest(BitcoinTestFramework): def add_options(self, parser): self.add_wallet_options(parser) parser.add_argument("--bdbro", action="store_true", help="Use the BerkeleyRO internal parser when dumping a Berkeley DB wallet file") + parser.add_argument("--swap-bdb-endian", action="store_true",help="When making Legacy BDB wallets, always make then byte swapped internally") def set_test_params(self): self.num_nodes = 1 self.setup_clean_chain = True self.rpc_timeout = 120 + if self.options.swap_bdb_endian: + self.extra_args = [["-swapbdbendian"]] def skip_test_if_missing_module(self): self.skip_if_no_wallet() -- cgit v1.2.3