diff options
Diffstat (limited to 'test')
| -rw-r--r-- | test/functional/data/wallets/high_minversion/.walletlock | 0 | ||||
| -rw-r--r-- | test/functional/data/wallets/high_minversion/GENERATE.md | 8 | ||||
| -rw-r--r-- | test/functional/data/wallets/high_minversion/db.log | 0 | ||||
| -rw-r--r-- | test/functional/data/wallets/high_minversion/wallet.dat | bin | 16384 -> 0 bytes | |||
| -rwxr-xr-x | test/functional/feature_proxy.py | 30 | ||||
| -rwxr-xr-x | test/functional/feature_taproot.py | 25 | ||||
| -rw-r--r-- | test/functional/test_framework/bdb.py | 152 | ||||
| -rw-r--r-- | test/functional/test_framework/blocktools.py | 9 | ||||
| -rw-r--r-- | test/functional/test_framework/key.py | 5 | ||||
| -rw-r--r-- | test/functional/test_framework/util.py | 9 | ||||
| -rwxr-xr-x | test/functional/wallet_importdescriptors.py | 21 | ||||
| -rwxr-xr-x | test/functional/wallet_listsinceblock.py | 1 | ||||
| -rwxr-xr-x | test/functional/wallet_multiwallet.py | 20 | ||||
| -rwxr-xr-x | test/functional/wallet_upgradewallet.py | 257 | ||||
| -rw-r--r-- | test/sanitizer_suppressions/tsan | 3 | ||||
| -rw-r--r-- | test/sanitizer_suppressions/ubsan | 5 |
16 files changed, 478 insertions, 67 deletions
diff --git a/test/functional/data/wallets/high_minversion/.walletlock b/test/functional/data/wallets/high_minversion/.walletlock deleted file mode 100644 index e69de29bb2..0000000000 --- a/test/functional/data/wallets/high_minversion/.walletlock +++ /dev/null diff --git a/test/functional/data/wallets/high_minversion/GENERATE.md b/test/functional/data/wallets/high_minversion/GENERATE.md deleted file mode 100644 index e55c4557ca..0000000000 --- a/test/functional/data/wallets/high_minversion/GENERATE.md +++ /dev/null @@ -1,8 +0,0 @@ -The wallet has been created by starting Bitcoin Core with the options -`-regtest -datadir=/tmp -nowallet -walletdir=$(pwd)/test/functional/data/wallets/`. - -In the source code, `WalletFeature::FEATURE_LATEST` has been modified to be large, so that the minversion is too high -for a current build of the wallet. - -The wallet has then been created with the RPC `createwallet high_minversion true true`, so that a blank wallet with -private keys disabled is created. diff --git a/test/functional/data/wallets/high_minversion/db.log b/test/functional/data/wallets/high_minversion/db.log deleted file mode 100644 index e69de29bb2..0000000000 --- a/test/functional/data/wallets/high_minversion/db.log +++ /dev/null diff --git a/test/functional/data/wallets/high_minversion/wallet.dat b/test/functional/data/wallets/high_minversion/wallet.dat Binary files differdeleted file mode 100644 index 99ab809263..0000000000 --- a/test/functional/data/wallets/high_minversion/wallet.dat +++ /dev/null diff --git a/test/functional/feature_proxy.py b/test/functional/feature_proxy.py index dfae58e860..05b658ed87 100755 --- a/test/functional/feature_proxy.py +++ b/test/functional/feature_proxy.py @@ -26,6 +26,8 @@ addnode connect to IPv4 addnode connect to IPv6 addnode connect to onion addnode connect to generic DNS name + +- Test getnetworkinfo for each node """ import socket @@ -41,12 +43,16 @@ from test_framework.util import ( from test_framework.netutil import test_ipv6_local RANGE_BEGIN = PORT_MIN + 2 * PORT_RANGE # Start after p2p and rpc ports -# From GetNetworkName() in netbase.cpp: -NET_UNROUTABLE = "" + +# Networks returned by RPC getpeerinfo, defined in src/netbase.cpp::GetNetworkName() +NET_UNROUTABLE = "unroutable" NET_IPV4 = "ipv4" NET_IPV6 = "ipv6" NET_ONION = "onion" +# Networks returned by RPC getnetworkinfo, defined in src/rpc/net.cpp::GetNetworksInfo() +NETWORKS = frozenset({NET_IPV4, NET_IPV6, NET_ONION}) + class ProxyTest(BitcoinTestFramework): def set_test_params(self): @@ -84,14 +90,14 @@ class ProxyTest(BitcoinTestFramework): self.serv3 = Socks5Server(self.conf3) self.serv3.start() - # Note: proxies are not used to connect to local nodes - # this is because the proxy to use is based on CService.GetNetwork(), which return NET_UNROUTABLE for localhost + # Note: proxies are not used to connect to local nodes. This is because the proxy to + # use is based on CService.GetNetwork(), which returns NET_UNROUTABLE for localhost. args = [ ['-listen', '-proxy=%s:%i' % (self.conf1.addr),'-proxyrandomize=1'], ['-listen', '-proxy=%s:%i' % (self.conf1.addr),'-onion=%s:%i' % (self.conf2.addr),'-proxyrandomize=0'], ['-listen', '-proxy=%s:%i' % (self.conf2.addr),'-proxyrandomize=1'], [] - ] + ] if self.have_ipv6: args[3] = ['-listen', '-proxy=[%s]:%i' % (self.conf3.addr),'-proxyrandomize=0', '-noonion'] self.add_nodes(self.num_nodes, extra_args=args) @@ -189,15 +195,17 @@ class ProxyTest(BitcoinTestFramework): r[x['name']] = x return r - # test RPC getnetworkinfo + self.log.info("Test RPC getnetworkinfo") n0 = networks_dict(self.nodes[0].getnetworkinfo()) - for net in ['ipv4','ipv6','onion']: + assert_equal(NETWORKS, n0.keys()) + for net in NETWORKS: assert_equal(n0[net]['proxy'], '%s:%i' % (self.conf1.addr)) assert_equal(n0[net]['proxy_randomize_credentials'], True) assert_equal(n0['onion']['reachable'], True) n1 = networks_dict(self.nodes[1].getnetworkinfo()) - for net in ['ipv4','ipv6']: + assert_equal(NETWORKS, n1.keys()) + for net in ['ipv4', 'ipv6']: assert_equal(n1[net]['proxy'], '%s:%i' % (self.conf1.addr)) assert_equal(n1[net]['proxy_randomize_credentials'], False) assert_equal(n1['onion']['proxy'], '%s:%i' % (self.conf2.addr)) @@ -205,14 +213,16 @@ class ProxyTest(BitcoinTestFramework): assert_equal(n1['onion']['reachable'], True) n2 = networks_dict(self.nodes[2].getnetworkinfo()) - for net in ['ipv4','ipv6','onion']: + assert_equal(NETWORKS, n2.keys()) + for net in NETWORKS: assert_equal(n2[net]['proxy'], '%s:%i' % (self.conf2.addr)) assert_equal(n2[net]['proxy_randomize_credentials'], True) assert_equal(n2['onion']['reachable'], True) if self.have_ipv6: n3 = networks_dict(self.nodes[3].getnetworkinfo()) - for net in ['ipv4','ipv6']: + assert_equal(NETWORKS, n3.keys()) + for net in NETWORKS: assert_equal(n3[net]['proxy'], '[%s]:%i' % (self.conf3.addr)) assert_equal(n3[net]['proxy_randomize_credentials'], False) assert_equal(n3['onion']['reachable'], False) diff --git a/test/functional/feature_taproot.py b/test/functional/feature_taproot.py index 3e47e24a3b..6e28cfb265 100755 --- a/test/functional/feature_taproot.py +++ b/test/functional/feature_taproot.py @@ -9,6 +9,7 @@ from test_framework.blocktools import ( create_block, add_witness_commitment, MAX_BLOCK_SIGOPS_WEIGHT, + NORMAL_GBT_REQUEST_PARAMS, WITNESS_SCALE_FACTOR, ) from test_framework.messages import ( @@ -1199,7 +1200,7 @@ class TaprootTest(BitcoinTestFramework): self.num_nodes = 2 self.setup_clean_chain = True # Node 0 has Taproot inactive, Node 1 active. - self.extra_args = [["-whitelist=127.0.0.1", "-par=1", "-vbparams=taproot:1:1"], ["-whitelist=127.0.0.1", "-par=1"]] + self.extra_args = [["-par=1", "-vbparams=taproot:1:1"], ["-par=1"]] def block_submit(self, node, txs, msg, err_msg, cb_pubkey=None, fees=0, sigops_weight=0, witness=False, accept=False): @@ -1218,7 +1219,7 @@ class TaprootTest(BitcoinTestFramework): witness and add_witness_commitment(block) block.rehash() block.solve() - block_response = node.submitblock(block.serialize(True).hex()) + block_response = node.submitblock(block.serialize().hex()) if err_msg is not None: assert block_response is not None and err_msg in block_response, "Missing error message '%s' from block response '%s': %s" % (err_msg, "(None)" if block_response is None else block_response, msg) if (accept): @@ -1436,17 +1437,27 @@ class TaprootTest(BitcoinTestFramework): self.log.info(" - Done") def run_test(self): - self.connect_nodes(0, 1) - # Post-taproot activation tests go first (pre-taproot tests' blocks are invalid post-taproot). self.log.info("Post-activation tests...") self.nodes[1].generate(101) self.test_spenders(self.nodes[1], spenders_taproot_active(), input_counts=[1, 2, 2, 2, 2, 3]) - # Transfer % of funds to pre-taproot node. + # Transfer funds to pre-taproot node. addr = self.nodes[0].getnewaddress() - self.nodes[1].sendtoaddress(address=addr, amount=int(self.nodes[1].getbalance() * 70000000) / 100000000) - self.nodes[1].generate(1) + rawtx = self.nodes[1].createrawtransaction( + inputs=[{ + 'txid': i['txid'], + 'vout': i['vout'] + } for i in self.nodes[1].listunspent()], + outputs={addr: self.nodes[1].getbalance()}, + ) + rawtx = self.nodes[1].signrawtransactionwithwallet(rawtx)['hex'] + # Transaction is too large to fit into the mempool, so put it into a block + block = create_block(tmpl=self.nodes[1].getblocktemplate(NORMAL_GBT_REQUEST_PARAMS), txlist=[rawtx]) + add_witness_commitment(block) + block.rehash() + block.solve() + assert_equal(None, self.nodes[1].submitblock(block.serialize().hex())) self.sync_blocks() # Pre-taproot activation tests. diff --git a/test/functional/test_framework/bdb.py b/test/functional/test_framework/bdb.py new file mode 100644 index 0000000000..9de358aa0a --- /dev/null +++ b/test/functional/test_framework/bdb.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +""" +Utilities for working directly with the wallet's BDB database file + +This is specific to the configuration of BDB used in this project: + - pagesize: 4096 bytes + - Outer database contains single subdatabase named 'main' + - btree + - btree leaf pages + +Each key-value pair is two entries in a btree leaf. The first is the key, the one that follows +is the value. And so on. Note that the entry data is itself not in the correct order. Instead +entry offsets are stored in the correct order and those offsets are needed to then retrieve +the data itself. + +Page format can be found in BDB source code dbinc/db_page.h +This only implements the deserialization of btree metadata pages and normal btree pages. Overflow +pages are not implemented but may be needed in the future if dealing with wallets with large +transactions. + +`db_dump -da wallet.dat` is useful to see the data in a wallet.dat BDB file +""" + +import binascii +import struct + +# Important constants +PAGESIZE = 4096 +OUTER_META_PAGE = 0 +INNER_META_PAGE = 2 + +# Page type values +BTREE_INTERNAL = 3 +BTREE_LEAF = 5 +BTREE_META = 9 + +# Some magic numbers for sanity checking +BTREE_MAGIC = 0x053162 +DB_VERSION = 9 + +# Deserializes a leaf page into a dict. +# Btree internal pages have the same header, for those, return None. +# For the btree leaf pages, deserialize them and put all the data into a dict +def dump_leaf_page(data): + page_info = {} + page_header = data[0:26] + _, pgno, prev_pgno, next_pgno, entries, hf_offset, level, pg_type = struct.unpack('QIIIHHBB', page_header) + page_info['pgno'] = pgno + page_info['prev_pgno'] = prev_pgno + page_info['next_pgno'] = next_pgno + page_info['entries'] = entries + page_info['hf_offset'] = hf_offset + page_info['level'] = level + page_info['pg_type'] = pg_type + page_info['entry_offsets'] = struct.unpack('{}H'.format(entries), data[26:26 + entries * 2]) + page_info['entries'] = [] + + if pg_type == BTREE_INTERNAL: + # Skip internal pages. These are the internal nodes of the btree and don't contain anything relevant to us + return None + + assert pg_type == BTREE_LEAF, 'A non-btree leaf page has been encountered while dumping leaves' + + for i in range(0, entries): + offset = page_info['entry_offsets'][i] + entry = {'offset': offset} + page_data_header = data[offset:offset + 3] + e_len, pg_type = struct.unpack('HB', page_data_header) + entry['len'] = e_len + entry['pg_type'] = pg_type + entry['data'] = data[offset + 3:offset + 3 + e_len] + page_info['entries'].append(entry) + + return page_info + +# Deserializes a btree metadata page into a dict. +# Does a simple sanity check on the magic value, type, and version +def dump_meta_page(page): + # metadata page + # general metadata + metadata = {} + meta_page = page[0:72] + _, pgno, magic, version, pagesize, encrypt_alg, pg_type, metaflags, _, free, last_pgno, nparts, key_count, record_count, flags, uid = struct.unpack('QIIIIBBBBIIIIII20s', meta_page) + metadata['pgno'] = pgno + metadata['magic'] = magic + metadata['version'] = version + metadata['pagesize'] = pagesize + metadata['encrypt_alg'] = encrypt_alg + metadata['pg_type'] = pg_type + metadata['metaflags'] = metaflags + metadata['free'] = free + metadata['last_pgno'] = last_pgno + metadata['nparts'] = nparts + metadata['key_count'] = key_count + metadata['record_count'] = record_count + metadata['flags'] = flags + metadata['uid'] = binascii.hexlify(uid) + + assert magic == BTREE_MAGIC, 'bdb magic does not match bdb btree magic' + assert pg_type == BTREE_META, 'Metadata page is not a btree metadata page' + assert version == DB_VERSION, 'Database too new' + + # btree metadata + btree_meta_page = page[72:512] + _, minkey, re_len, re_pad, root, _, crypto_magic, _, iv, chksum = struct.unpack('IIIII368sI12s16s20s', btree_meta_page) + metadata['minkey'] = minkey + metadata['re_len'] = re_len + metadata['re_pad'] = re_pad + metadata['root'] = root + metadata['crypto_magic'] = crypto_magic + metadata['iv'] = binascii.hexlify(iv) + metadata['chksum'] = binascii.hexlify(chksum) + return metadata + +# Given the dict from dump_leaf_page, get the key-value pairs and put them into a dict +def extract_kv_pairs(page_data): + out = {} + last_key = None + for i, entry in enumerate(page_data['entries']): + # By virtue of these all being pairs, even number entries are keys, and odd are values + if i % 2 == 0: + out[entry['data']] = b'' + last_key = entry['data'] + else: + out[last_key] = entry['data'] + return out + +# Extract the key-value pairs of the BDB file given in filename +def dump_bdb_kv(filename): + # Read in the BDB file and start deserializing it + pages = [] + with open(filename, 'rb') as f: + data = f.read(PAGESIZE) + while len(data) > 0: + pages.append(data) + data = f.read(PAGESIZE) + + # Sanity check the meta pages + dump_meta_page(pages[OUTER_META_PAGE]) + dump_meta_page(pages[INNER_META_PAGE]) + + # Fetch the kv pairs from the leaf pages + kv = {} + for i in range(3, len(pages)): + info = dump_leaf_page(pages[i]) + if info is not None: + info_kv = extract_kv_pairs(info) + kv = {**kv, **info_kv} + return kv diff --git a/test/functional/test_framework/blocktools.py b/test/functional/test_framework/blocktools.py index 64bc7e0485..6b7214f03a 100644 --- a/test/functional/test_framework/blocktools.py +++ b/test/functional/test_framework/blocktools.py @@ -5,7 +5,6 @@ """Utilities for manipulating blocks and transactions.""" from binascii import a2b_hex -import io import struct import time import unittest @@ -45,7 +44,6 @@ from .script import ( hash160, ) from .util import assert_equal -from io import BytesIO WITNESS_SCALE_FACTOR = 4 MAX_BLOCK_SIGOPS = 20000 @@ -78,9 +76,7 @@ def create_block(hashprev=None, coinbase=None, ntime=None, *, version=None, tmpl if txlist: for tx in txlist: if not hasattr(tx, 'calc_sha256'): - txo = CTransaction() - txo.deserialize(io.BytesIO(tx)) - tx = txo + tx = FromHex(CTransaction(), tx) block.vtx.append(tx) block.hashMerkleRoot = block.calc_merkle_root() block.calc_sha256() @@ -166,8 +162,7 @@ def create_transaction(node, txid, to_address, *, amount): sign for the output that is being spent. """ raw_tx = create_raw_transaction(node, txid, to_address, amount=amount) - tx = CTransaction() - tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx))) + tx = FromHex(CTransaction(), raw_tx) return tx def create_raw_transaction(node, txid, to_address, *, amount): diff --git a/test/functional/test_framework/key.py b/test/functional/test_framework/key.py index abf2507154..f3d13c049b 100644 --- a/test/functional/test_framework/key.py +++ b/test/functional/test_framework/key.py @@ -10,7 +10,6 @@ import csv import hashlib import os import random -import sys import unittest from .util import modinv @@ -22,6 +21,7 @@ def TaggedHash(tag, data): return hashlib.sha256(ss).digest() def xor_bytes(b0, b1): + assert len(b0) == len(b1) return bytes(x ^ y for (x, y) in zip(b0, b1)) def jacobi_symbol(n, k): @@ -523,7 +523,8 @@ class TestFrameworkKey(unittest.TestCase): def test_schnorr_testvectors(self): """Implement the BIP340 test vectors (read from bip340_test_vectors.csv).""" num_tests = 0 - with open(os.path.join(sys.path[0], 'test_framework', 'bip340_test_vectors.csv'), newline='', encoding='utf8') as csvfile: + vectors_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'bip340_test_vectors.csv') + with open(vectors_file, newline='', encoding='utf8') as csvfile: reader = csv.reader(csvfile) next(reader) for row in reader: diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index 3356f1ab10..62ff5c6e33 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -8,6 +8,7 @@ from base64 import b64encode from binascii import unhexlify from decimal import Decimal, ROUND_DOWN from subprocess import CalledProcessError +import hashlib import inspect import json import logging @@ -260,6 +261,14 @@ def wait_until_helper(predicate, *, attempts=float('inf'), timeout=float('inf'), raise AssertionError("Predicate {} not true after {} seconds".format(predicate_source, timeout)) raise RuntimeError('Unreachable') +def sha256sum_file(filename): + h = hashlib.sha256() + with open(filename, 'rb') as f: + d = f.read(4096) + while len(d) > 0: + h.update(d) + d = f.read(4096) + return h.digest() # RPC/P2P connection constants and functions ############################################ diff --git a/test/functional/wallet_importdescriptors.py b/test/functional/wallet_importdescriptors.py index 949adeb703..2903a84998 100755 --- a/test/functional/wallet_importdescriptors.py +++ b/test/functional/wallet_importdescriptors.py @@ -15,6 +15,7 @@ variants. - `test_address()` is called to call getaddressinfo for an address on node1 and test the values returned.""" +from test_framework.address import key_to_p2pkh from test_framework.test_framework import BitcoinTestFramework from test_framework.descriptors import descsum_create from test_framework.util import ( @@ -107,6 +108,17 @@ class ImportDescriptorsTest(BitcoinTestFramework): error_code=-8, error_message="Internal addresses should not have a label") + self.log.info("Internal addresses should be detected as such") + key = get_generate_key() + addr = key_to_p2pkh(key.pubkey) + self.test_importdesc({"desc": descsum_create("pkh(" + key.pubkey + ")"), + "timestamp": "now", + "internal": True}, + success=True) + info = w1.getaddressinfo(addr) + assert_equal(info["ismine"], True) + assert_equal(info["ischange"], True) + # # Test importing of a P2SH-P2WPKH descriptor key = get_generate_key() self.log.info("Should not import a p2sh-p2wpkh descriptor without checksum") @@ -209,6 +221,15 @@ class ImportDescriptorsTest(BitcoinTestFramework): success=False, error_code=-4, error_message='Cannot import private keys to a wallet with private keys disabled') + + self.log.info("Should not import a descriptor with hardened derivations when private keys are disabled") + self.test_importdesc({"desc": descsum_create("wpkh(" + xpub + "/1h/*)"), + "timestamp": "now", + "range": 1}, + success=False, + error_code=-4, + error_message='Cannot expand descriptor. Probably because of hardened derivations without private keys provided') + for address in addresses: test_address(w1, address, diff --git a/test/functional/wallet_listsinceblock.py b/test/functional/wallet_listsinceblock.py index 07c14db6b1..6a1b9097c5 100755 --- a/test/functional/wallet_listsinceblock.py +++ b/test/functional/wallet_listsinceblock.py @@ -191,6 +191,7 @@ class ListSinceBlockTest(BitcoinTestFramework): address = key_to_p2wpkh(eckey.get_pubkey().get_bytes()) self.nodes[2].sendtoaddress(address, 10) self.nodes[2].generate(6) + self.sync_all() self.nodes[2].importprivkey(privkey) utxos = self.nodes[2].listunspent() utxo = [u for u in utxos if u["address"] == address][0] diff --git a/test/functional/wallet_multiwallet.py b/test/functional/wallet_multiwallet.py index abdc279197..df16ec741f 100755 --- a/test/functional/wallet_multiwallet.py +++ b/test/functional/wallet_multiwallet.py @@ -10,6 +10,7 @@ from decimal import Decimal from threading import Thread import os import shutil +import stat import time from test_framework.authproxy import JSONRPCException @@ -78,6 +79,11 @@ class MultiWalletTest(BitcoinTestFramework): os.mkdir(wallet_dir('w7')) os.symlink('w7', wallet_dir('w7_symlink')) + os.symlink('..', wallet_dir('recursive_dir_symlink')) + + os.mkdir(wallet_dir('self_walletdat_symlink')) + os.symlink('wallet.dat', wallet_dir('self_walletdat_symlink/wallet.dat')) + # rename wallet.dat to make sure plain wallet file paths (as opposed to # directory paths) can be loaded # create another dummy wallet for use in testing backups later @@ -117,7 +123,16 @@ class MultiWalletTest(BitcoinTestFramework): self.nodes[0].createwallet(wallet_name) for wallet_name in to_load: self.nodes[0].loadwallet(wallet_name) - assert_equal(sorted(map(lambda w: w['name'], self.nodes[0].listwalletdir()['wallets'])), sorted(in_wallet_dir)) + + os.mkdir(wallet_dir('no_access')) + os.chmod(wallet_dir('no_access'), 0) + try: + with self.nodes[0].assert_debug_log(expected_msgs=['Too many levels of symbolic links', 'Error scanning']): + walletlist = self.nodes[0].listwalletdir()['wallets'] + finally: + # Need to ensure access is restored for cleanup + os.chmod(wallet_dir('no_access'), stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) + assert_equal(sorted(map(lambda w: w['name'], walletlist)), sorted(in_wallet_dir)) assert_equal(set(node.listwallets()), set(wallet_names)) @@ -156,6 +171,9 @@ class MultiWalletTest(BitcoinTestFramework): open(not_a_dir, 'a', encoding="utf8").close() self.nodes[0].assert_start_raises_init_error(['-walletdir=' + not_a_dir], 'Error: Specified -walletdir "' + not_a_dir + '" is not a directory') + self.log.info("Do not allow -upgradewallet with multiwallet") + self.nodes[0].assert_start_raises_init_error(['-upgradewallet'], "Error: Error parsing command line arguments: Invalid parameter -upgradewallet") + # if wallets/ doesn't exist, datadir should be the default wallet dir wallet_dir2 = data_dir('walletdir') os.rename(wallet_dir(), wallet_dir2) diff --git a/test/functional/wallet_upgradewallet.py b/test/functional/wallet_upgradewallet.py index 15d9b109c5..8ab4b3f76c 100755 --- a/test/functional/wallet_upgradewallet.py +++ b/test/functional/wallet_upgradewallet.py @@ -13,23 +13,47 @@ Only v0.15.2 and v0.16.3 are required by this test. The others are used in featu import os import shutil +import struct +from io import BytesIO + +from test_framework.bdb import dump_bdb_kv +from test_framework.messages import deser_compact_size, deser_string from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, assert_greater_than, assert_is_hex_string, + assert_raises_rpc_error, + sha256sum_file, ) +UPGRADED_KEYMETA_VERSION = 12 + +def deser_keymeta(f): + ver, create_time = struct.unpack('<Iq', f.read(12)) + kp_str = deser_string(f) + seed_id = f.read(20) + fpr = f.read(4) + path_len = 0 + path = [] + has_key_orig = False + if ver == UPGRADED_KEYMETA_VERSION: + path_len = deser_compact_size(f) + for i in range(0, path_len): + path.append(struct.unpack('<I', f.read(4))[0]) + has_key_orig = bool(f.read(1)) + return ver, create_time, kp_str, seed_id, fpr, path_len, path, has_key_orig + class UpgradeWalletTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 3 self.extra_args = [ - ["-addresstype=bech32"], # current wallet version - ["-usehd=1"], # v0.16.3 wallet - ["-usehd=0"] # v0.15.2 wallet + ["-addresstype=bech32", "-keypool=2"], # current wallet version + ["-usehd=1", "-keypool=2"], # v0.16.3 wallet + ["-usehd=0", "-keypool=2"] # v0.15.2 wallet ] self.wallet_names = [self.default_wallet_name, None, None] @@ -87,22 +111,53 @@ class UpgradeWalletTest(BitcoinTestFramework): self.log.info("Test upgradewallet RPC...") # Prepare for copying of the older wallet - node_master_wallet_dir = os.path.join(node_master.datadir, "regtest/wallets") + node_master_wallet_dir = os.path.join(node_master.datadir, "regtest/wallets", self.default_wallet_name) + node_master_wallet = os.path.join(node_master_wallet_dir, self.default_wallet_name, self.wallet_data_filename) v16_3_wallet = os.path.join(v16_3_node.datadir, "regtest/wallets/wallet.dat") v15_2_wallet = os.path.join(v15_2_node.datadir, "regtest/wallet.dat") + split_hd_wallet = os.path.join(v15_2_node.datadir, "regtest/splithd") self.stop_nodes() - # Copy the 0.16.3 wallet to the last Bitcoin Core version and open it: - shutil.rmtree(node_master_wallet_dir) - os.mkdir(node_master_wallet_dir) - shutil.copy( - v16_3_wallet, - node_master_wallet_dir - ) - self.restart_node(0, ['-nowallet']) - node_master.loadwallet('') + # Make split hd wallet + self.start_node(2, ['-usehd=1', '-keypool=2', '-wallet=splithd']) + self.stop_node(2) + + def copy_v16(): + node_master.get_wallet_rpc(self.default_wallet_name).unloadwallet() + # Copy the 0.16.3 wallet to the last Bitcoin Core version and open it: + shutil.rmtree(node_master_wallet_dir) + os.mkdir(node_master_wallet_dir) + shutil.copy( + v16_3_wallet, + node_master_wallet_dir + ) + node_master.loadwallet(self.default_wallet_name) + + def copy_non_hd(): + node_master.get_wallet_rpc(self.default_wallet_name).unloadwallet() + # Copy the 0.15.2 non hd wallet to the last Bitcoin Core version and open it: + shutil.rmtree(node_master_wallet_dir) + os.mkdir(node_master_wallet_dir) + shutil.copy( + v15_2_wallet, + node_master_wallet_dir + ) + node_master.loadwallet(self.default_wallet_name) - wallet = node_master.get_wallet_rpc('') + def copy_split_hd(): + node_master.get_wallet_rpc(self.default_wallet_name).unloadwallet() + # Copy the 0.15.2 split hd wallet to the last Bitcoin Core version and open it: + shutil.rmtree(node_master_wallet_dir) + os.mkdir(node_master_wallet_dir) + shutil.copy( + split_hd_wallet, + os.path.join(node_master_wallet_dir, 'wallet.dat') + ) + node_master.loadwallet(self.default_wallet_name) + + self.restart_node(0) + copy_v16() + wallet = node_master.get_wallet_rpc(self.default_wallet_name) old_version = wallet.getwalletinfo()["walletversion"] # calling upgradewallet without version arguments @@ -114,18 +169,8 @@ class UpgradeWalletTest(BitcoinTestFramework): # wallet should still contain the same balance assert_equal(wallet.getbalance(), v16_3_balance) - self.stop_node(0) - # Copy the 0.15.2 wallet to the last Bitcoin Core version and open it: - shutil.rmtree(node_master_wallet_dir) - os.mkdir(node_master_wallet_dir) - shutil.copy( - v15_2_wallet, - node_master_wallet_dir - ) - self.restart_node(0, ['-nowallet']) - node_master.loadwallet('') - - wallet = node_master.get_wallet_rpc('') + copy_non_hd() + wallet = node_master.get_wallet_rpc(self.default_wallet_name) # should have no master key hash before conversion assert_equal('hdseedid' in wallet.getwalletinfo(), False) # calling upgradewallet with explicit version number @@ -137,5 +182,165 @@ class UpgradeWalletTest(BitcoinTestFramework): # after conversion master key hash should be present assert_is_hex_string(wallet.getwalletinfo()['hdseedid']) + self.log.info('Intermediary versions don\'t effect anything') + copy_non_hd() + # Wallet starts with 60000 + assert_equal(60000, wallet.getwalletinfo()['walletversion']) + wallet.unloadwallet() + before_checksum = sha256sum_file(node_master_wallet) + node_master.loadwallet('') + # Can "upgrade" to 129999 which should have no effect on the wallet + wallet.upgradewallet(129999) + assert_equal(60000, wallet.getwalletinfo()['walletversion']) + wallet.unloadwallet() + assert_equal(before_checksum, sha256sum_file(node_master_wallet)) + node_master.loadwallet('') + + self.log.info('Wallets cannot be downgraded') + copy_non_hd() + assert_raises_rpc_error(-4, 'Cannot downgrade wallet', wallet.upgradewallet, 40000) + wallet.unloadwallet() + assert_equal(before_checksum, sha256sum_file(node_master_wallet)) + node_master.loadwallet('') + + self.log.info('Can upgrade to HD') + # Inspect the old wallet and make sure there is no hdchain + orig_kvs = dump_bdb_kv(node_master_wallet) + assert b'\x07hdchain' not in orig_kvs + # Upgrade to HD, no split + wallet.upgradewallet(130000) + assert_equal(130000, wallet.getwalletinfo()['walletversion']) + # Check that there is now a hd chain and it is version 1, no internal chain counter + new_kvs = dump_bdb_kv(node_master_wallet) + assert b'\x07hdchain' in new_kvs + hd_chain = new_kvs[b'\x07hdchain'] + assert_equal(28, len(hd_chain)) + hd_chain_version, external_counter, seed_id = struct.unpack('<iI20s', hd_chain) + assert_equal(1, hd_chain_version) + seed_id = bytearray(seed_id) + seed_id.reverse() + old_kvs = new_kvs + # First 2 keys should still be non-HD + for i in range(0, 2): + info = wallet.getaddressinfo(wallet.getnewaddress()) + assert 'hdkeypath' not in info + assert 'hdseedid' not in info + # Next key should be HD + info = wallet.getaddressinfo(wallet.getnewaddress()) + assert_equal(seed_id.hex(), info['hdseedid']) + assert_equal('m/0\'/0\'/0\'', info['hdkeypath']) + prev_seed_id = info['hdseedid'] + # Change key should be the same keypool + info = wallet.getaddressinfo(wallet.getrawchangeaddress()) + assert_equal(prev_seed_id, info['hdseedid']) + assert_equal('m/0\'/0\'/1\'', info['hdkeypath']) + + self.log.info('Cannot upgrade to HD Split, needs Pre Split Keypool') + assert_raises_rpc_error(-4, 'Cannot upgrade a non HD split wallet without upgrading to support pre split keypool', wallet.upgradewallet, 139900) + assert_equal(130000, wallet.getwalletinfo()['walletversion']) + assert_raises_rpc_error(-4, 'Cannot upgrade a non HD split wallet without upgrading to support pre split keypool', wallet.upgradewallet, 159900) + assert_equal(130000, wallet.getwalletinfo()['walletversion']) + assert_raises_rpc_error(-4, 'Cannot upgrade a non HD split wallet without upgrading to support pre split keypool', wallet.upgradewallet, 169899) + assert_equal(130000, wallet.getwalletinfo()['walletversion']) + + self.log.info('Upgrade HD to HD chain split') + wallet.upgradewallet(169900) + assert_equal(169900, wallet.getwalletinfo()['walletversion']) + # Check that the hdchain updated correctly + new_kvs = dump_bdb_kv(node_master_wallet) + hd_chain = new_kvs[b'\x07hdchain'] + assert_equal(32, len(hd_chain)) + hd_chain_version, external_counter, seed_id, internal_counter = struct.unpack('<iI20sI', hd_chain) + assert_equal(2, hd_chain_version) + assert_equal(0, internal_counter) + seed_id = bytearray(seed_id) + seed_id.reverse() + assert_equal(seed_id.hex(), prev_seed_id) + # Next change address is the same keypool + info = wallet.getaddressinfo(wallet.getrawchangeaddress()) + assert_equal(prev_seed_id, info['hdseedid']) + assert_equal('m/0\'/0\'/2\'', info['hdkeypath']) + # Next change address is the new keypool + info = wallet.getaddressinfo(wallet.getrawchangeaddress()) + assert_equal(prev_seed_id, info['hdseedid']) + assert_equal('m/0\'/1\'/0\'', info['hdkeypath']) + # External addresses use the same keypool + info = wallet.getaddressinfo(wallet.getnewaddress()) + assert_equal(prev_seed_id, info['hdseedid']) + assert_equal('m/0\'/0\'/3\'', info['hdkeypath']) + + self.log.info('Upgrade non-HD to HD chain split') + copy_non_hd() + wallet.upgradewallet(169900) + assert_equal(169900, wallet.getwalletinfo()['walletversion']) + # Check that the hdchain updated correctly + new_kvs = dump_bdb_kv(node_master_wallet) + hd_chain = new_kvs[b'\x07hdchain'] + assert_equal(32, len(hd_chain)) + hd_chain_version, external_counter, seed_id, internal_counter = struct.unpack('<iI20sI', hd_chain) + assert_equal(2, hd_chain_version) + assert_equal(2, internal_counter) + # Drain the keypool by fetching one external key and one change key. Should still be the same keypool + info = wallet.getaddressinfo(wallet.getnewaddress()) + assert 'hdseedid' not in info + assert 'hdkeypath' not in info + info = wallet.getaddressinfo(wallet.getrawchangeaddress()) + assert 'hdseedid' not in info + assert 'hdkeypath' not in info + # The next addresses are HD and should be on different HD chains + info = wallet.getaddressinfo(wallet.getnewaddress()) + ext_id = info['hdseedid'] + assert_equal('m/0\'/0\'/0\'', info['hdkeypath']) + info = wallet.getaddressinfo(wallet.getrawchangeaddress()) + assert_equal(ext_id, info['hdseedid']) + assert_equal('m/0\'/1\'/0\'', info['hdkeypath']) + + self.log.info('KeyMetadata should upgrade when loading into master') + copy_v16() + old_kvs = dump_bdb_kv(v16_3_wallet) + new_kvs = dump_bdb_kv(node_master_wallet) + for k, old_v in old_kvs.items(): + if k.startswith(b'\x07keymeta'): + new_ver, new_create_time, new_kp_str, new_seed_id, new_fpr, new_path_len, new_path, new_has_key_orig = deser_keymeta(BytesIO(new_kvs[k])) + old_ver, old_create_time, old_kp_str, old_seed_id, old_fpr, old_path_len, old_path, old_has_key_orig = deser_keymeta(BytesIO(old_v)) + assert_equal(10, old_ver) + if old_kp_str == b"": # imported things that don't have keymeta (i.e. imported coinbase privkeys) won't be upgraded + assert_equal(new_kvs[k], old_v) + continue + assert_equal(12, new_ver) + assert_equal(new_create_time, old_create_time) + assert_equal(new_kp_str, old_kp_str) + assert_equal(new_seed_id, old_seed_id) + assert_equal(0, old_path_len) + assert_equal(new_path_len, len(new_path)) + assert_equal([], old_path) + assert_equal(False, old_has_key_orig) + assert_equal(True, new_has_key_orig) + + # Check that the path is right + built_path = [] + for s in new_kp_str.decode().split('/')[1:]: + h = 0 + if s[-1] == '\'': + s = s[:-1] + h = 0x80000000 + p = int(s) | h + built_path.append(p) + assert_equal(new_path, built_path) + + self.log.info('Upgrading to NO_DEFAULT_KEY should not remove the defaultkey') + copy_split_hd() + # Check the wallet has a default key initially + old_kvs = dump_bdb_kv(node_master_wallet) + defaultkey = old_kvs[b'\x0adefaultkey'] + # Upgrade the wallet. Should still have the same default key + wallet.upgradewallet(159900) + new_kvs = dump_bdb_kv(node_master_wallet) + up_defaultkey = new_kvs[b'\x0adefaultkey'] + assert_equal(defaultkey, up_defaultkey) + # 0.16.3 doesn't have a default key + v16_3_kvs = dump_bdb_kv(v16_3_wallet) + assert b'\x0adefaultkey' not in v16_3_kvs + if __name__ == '__main__': UpgradeWalletTest().main() diff --git a/test/sanitizer_suppressions/tsan b/test/sanitizer_suppressions/tsan index 625085c55b..48f81f3dbf 100644 --- a/test/sanitizer_suppressions/tsan +++ b/test/sanitizer_suppressions/tsan @@ -32,7 +32,7 @@ deadlock:CConnman::ForNode deadlock:CConnman::GetNodeStats deadlock:CChainState::ConnectTip deadlock:UpdateTip -deadlock:wallet_tests::CreateWalletFromFile +deadlock:wallet_tests::CreateWallet # WalletBatch (unidentified deadlock) deadlock:WalletBatch @@ -47,3 +47,4 @@ deadlock:src/qt/test/* # External libraries deadlock:libdb race:libzmq +race:epoll_ctl # https://github.com/bitcoin/bitcoin/pull/20218 diff --git a/test/sanitizer_suppressions/ubsan b/test/sanitizer_suppressions/ubsan index 75257d886b..e2f3fec851 100644 --- a/test/sanitizer_suppressions/ubsan +++ b/test/sanitizer_suppressions/ubsan @@ -1,8 +1,3 @@ -# -fsanitize=undefined suppressions -# ================================= -float-divide-by-zero:validation.cpp -float-divide-by-zero:wallet/wallet.cpp - # -fsanitize=integer suppressions # =============================== # Unsigned integer overflow occurs when the result of an unsigned integer |