From 842ae3842df489f1b8d68e67a234788966218184 Mon Sep 17 00:00:00 2001 From: Andrew Chow Date: Wed, 29 Apr 2020 14:48:43 -0400 Subject: wallet: Add utility method for CanSupportFeature --- src/wallet/wallet.h | 2 +- src/wallet/walletutil.cpp | 5 +++++ src/wallet/walletutil.h | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 245144a1c9..fcff0c557b 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -801,7 +801,7 @@ public: bool IsTrusted(const CWalletTx& wtx, std::set& trusted_parents) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); //! check whether we are allowed to upgrade (or already support) to the named feature - bool CanSupportFeature(enum WalletFeature wf) const override EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) { AssertLockHeld(cs_wallet); return nWalletMaxVersion >= wf; } + bool CanSupportFeature(enum WalletFeature wf) const override EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) { AssertLockHeld(cs_wallet); return IsFeatureSupported(nWalletMaxVersion, wf); } /** * populate vCoins with vector of available COutputs. diff --git a/src/wallet/walletutil.cpp b/src/wallet/walletutil.cpp index a2a55f9751..9ac90a21ca 100644 --- a/src/wallet/walletutil.cpp +++ b/src/wallet/walletutil.cpp @@ -70,3 +70,8 @@ std::vector ListWalletDir() return paths; } + +bool IsFeatureSupported(int wallet_version, int feature_version) +{ + return wallet_version >= feature_version; +} diff --git a/src/wallet/walletutil.h b/src/wallet/walletutil.h index afdcb2e18a..b0ab76fc31 100644 --- a/src/wallet/walletutil.h +++ b/src/wallet/walletutil.h @@ -29,7 +29,7 @@ enum WalletFeature FEATURE_LATEST = FEATURE_PRE_SPLIT_KEYPOOL }; - +bool IsFeatureSupported(int wallet_version, int feature_version); enum WalletFlags : uint64_t { // wallet flags in the upper section (> 1 << 31) will lead to not opening the wallet if flag is unknown -- cgit v1.2.3 From 5f720544f34dedf75b063b962845fa8eca604514 Mon Sep 17 00:00:00 2001 From: Andrew Chow Date: Wed, 29 Apr 2020 16:58:12 -0400 Subject: wallet: Add GetClosestWalletFeature function Given a version number, get the closest supported WalletFeature for a version number. --- src/wallet/walletutil.cpp | 13 +++++++++++++ src/wallet/walletutil.h | 1 + 2 files changed, 14 insertions(+) diff --git a/src/wallet/walletutil.cpp b/src/wallet/walletutil.cpp index 9ac90a21ca..0b7cf692ba 100644 --- a/src/wallet/walletutil.cpp +++ b/src/wallet/walletutil.cpp @@ -75,3 +75,16 @@ bool IsFeatureSupported(int wallet_version, int feature_version) { return wallet_version >= feature_version; } + +WalletFeature GetClosestWalletFeature(int version) +{ + if (version >= FEATURE_LATEST) return FEATURE_LATEST; + if (version >= FEATURE_PRE_SPLIT_KEYPOOL) return FEATURE_PRE_SPLIT_KEYPOOL; + if (version >= FEATURE_NO_DEFAULT_KEY) return FEATURE_NO_DEFAULT_KEY; + if (version >= FEATURE_HD_SPLIT) return FEATURE_HD_SPLIT; + if (version >= FEATURE_HD) return FEATURE_HD; + if (version >= FEATURE_COMPRPUBKEY) return FEATURE_COMPRPUBKEY; + if (version >= FEATURE_WALLETCRYPT) return FEATURE_WALLETCRYPT; + if (version >= FEATURE_BASE) return FEATURE_BASE; + return static_cast(0); +} diff --git a/src/wallet/walletutil.h b/src/wallet/walletutil.h index b0ab76fc31..27521abd81 100644 --- a/src/wallet/walletutil.h +++ b/src/wallet/walletutil.h @@ -30,6 +30,7 @@ enum WalletFeature }; bool IsFeatureSupported(int wallet_version, int feature_version); +WalletFeature GetClosestWalletFeature(int version); enum WalletFlags : uint64_t { // wallet flags in the upper section (> 1 << 31) will lead to not opening the wallet if flag is unknown -- cgit v1.2.3 From bd7398cc6258c258e9f4411c50630ec4a552341b Mon Sep 17 00:00:00 2001 From: Andrew Chow Date: Wed, 29 Apr 2020 17:00:24 -0400 Subject: wallet: have ScriptPubKeyMan::Upgrade check against the new version Instead of using CanSupportFeature and relying on nWalletMaxVersion, take the new version we are upgrading to and use IsSupportedFeature with that and the previous wallet version. --- src/wallet/scriptpubkeyman.cpp | 6 +++--- src/wallet/scriptpubkeyman.h | 4 ++-- src/wallet/wallet.cpp | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/wallet/scriptpubkeyman.cpp b/src/wallet/scriptpubkeyman.cpp index 188289b010..149e8a4c9b 100644 --- a/src/wallet/scriptpubkeyman.cpp +++ b/src/wallet/scriptpubkeyman.cpp @@ -438,12 +438,12 @@ bool LegacyScriptPubKeyMan::CanGetAddresses(bool internal) const return keypool_has_keys; } -bool LegacyScriptPubKeyMan::Upgrade(int prev_version, bilingual_str& error) +bool LegacyScriptPubKeyMan::Upgrade(int prev_version, int new_version, bilingual_str& error) { LOCK(cs_KeyStore); bool hd_upgrade = false; bool split_upgrade = false; - if (m_storage.CanSupportFeature(FEATURE_HD) && !IsHDEnabled()) { + if (IsFeatureSupported(new_version, FEATURE_HD) && !IsHDEnabled()) { WalletLogPrintf("Upgrading wallet to HD\n"); m_storage.SetMinVersion(FEATURE_HD); @@ -453,7 +453,7 @@ bool LegacyScriptPubKeyMan::Upgrade(int prev_version, bilingual_str& error) hd_upgrade = true; } // Upgrade to HD chain split if necessary - if (m_storage.CanSupportFeature(FEATURE_HD_SPLIT)) { + if (IsFeatureSupported(new_version, FEATURE_HD_SPLIT)) { WalletLogPrintf("Upgrading wallet to use HD chain split\n"); m_storage.SetMinVersion(FEATURE_PRE_SPLIT_KEYPOOL); split_upgrade = FEATURE_HD_SPLIT > prev_version; diff --git a/src/wallet/scriptpubkeyman.h b/src/wallet/scriptpubkeyman.h index 14fb1fa89f..afbb1922c9 100644 --- a/src/wallet/scriptpubkeyman.h +++ b/src/wallet/scriptpubkeyman.h @@ -206,7 +206,7 @@ public: virtual bool CanGetAddresses(bool internal = false) const { return false; } /** Upgrades the wallet to the specified version */ - virtual bool Upgrade(int prev_version, bilingual_str& error) { return false; } + virtual bool Upgrade(int prev_version, int new_version, bilingual_str& error) { return false; } virtual bool HavePrivateKeys() const { return false; } @@ -371,7 +371,7 @@ public: bool SetupGeneration(bool force = false) override; - bool Upgrade(int prev_version, bilingual_str& error) override; + bool Upgrade(int prev_version, int new_version, bilingual_str& error) override; bool HavePrivateKeys() const override; diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 6b7d05fdf3..dc1365b0bb 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -4120,7 +4120,7 @@ const CAddressBookData* CWallet::FindAddressBookEntry(const CTxDestination& dest bool CWallet::UpgradeWallet(int version, bilingual_str& error, std::vector& warnings) { int prev_version = GetVersion(); - int nMaxVersion = version; + int& nMaxVersion = version; if (nMaxVersion == 0) // the -upgradewallet without argument case { WalletLogPrintf("Performing wallet upgrade to %i\n", FEATURE_LATEST); @@ -4146,7 +4146,7 @@ bool CWallet::UpgradeWallet(int version, bilingual_str& error, std::vectorUpgrade(prev_version, error)) { + if (!spk_man->Upgrade(prev_version, version, error)) { return false; } } -- cgit v1.2.3 From 8e32e1c41c995e832e643f605d35a7aa112837e6 Mon Sep 17 00:00:00 2001 From: Andrew Chow Date: Wed, 29 Apr 2020 17:07:52 -0400 Subject: wallet: remove nWalletMaxVersion nWalletMaxVersion was used to allow an upgrade to a version only when the new feature was used. This makes sense for the old -upgradewallet startup option. But because upgradewallet is now a RPC, putting off the version bump like this does not make sense. Instead, immediately upgrading to the given version number makes sense. --- src/wallet/scriptpubkeyman.h | 2 +- src/wallet/wallet.cpp | 41 ++++++++++------------------------------- src/wallet/wallet.h | 16 +++++----------- 3 files changed, 16 insertions(+), 43 deletions(-) diff --git a/src/wallet/scriptpubkeyman.h b/src/wallet/scriptpubkeyman.h index afbb1922c9..78bfeeabc5 100644 --- a/src/wallet/scriptpubkeyman.h +++ b/src/wallet/scriptpubkeyman.h @@ -37,7 +37,7 @@ public: virtual bool IsWalletFlagSet(uint64_t) const = 0; virtual void UnsetBlankWalletFlag(WalletBatch&) = 0; virtual bool CanSupportFeature(enum WalletFeature) const = 0; - virtual void SetMinVersion(enum WalletFeature, WalletBatch* = nullptr, bool = false) = 0; + virtual void SetMinVersion(enum WalletFeature, WalletBatch* = nullptr) = 0; virtual const CKeyingMaterial& GetEncryptionKey() const = 0; virtual bool HasEncryptionKeys() const = 0; virtual bool IsLocked() const = 0; diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index dc1365b0bb..be7b30009c 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -435,21 +435,13 @@ void CWallet::chainStateFlushed(const CBlockLocator& loc) batch.WriteBestBlock(loc); } -void CWallet::SetMinVersion(enum WalletFeature nVersion, WalletBatch* batch_in, bool fExplicit) +void CWallet::SetMinVersion(enum WalletFeature nVersion, WalletBatch* batch_in) { LOCK(cs_wallet); if (nWalletVersion >= nVersion) return; - - // when doing an explicit upgrade, if we pass the max version permitted, upgrade all the way - if (fExplicit && nVersion > nWalletMaxVersion) - nVersion = FEATURE_LATEST; - nWalletVersion = nVersion; - if (nVersion > nWalletMaxVersion) - nWalletMaxVersion = nVersion; - { WalletBatch* batch = batch_in ? batch_in : new WalletBatch(*database); if (nWalletVersion > 40000) @@ -459,18 +451,6 @@ void CWallet::SetMinVersion(enum WalletFeature nVersion, WalletBatch* batch_in, } } -bool CWallet::SetMaxVersion(int nVersion) -{ - LOCK(cs_wallet); - // cannot downgrade below current version - if (nWalletVersion > nVersion) - return false; - - nWalletMaxVersion = nVersion; - - return true; -} - std::set CWallet::GetConflicts(const uint256& txid) const { std::set result; @@ -655,7 +635,7 @@ bool CWallet::EncryptWallet(const SecureString& strWalletPassphrase) } // Encryption was introduced in version 0.4.0 - SetMinVersion(FEATURE_WALLETCRYPT, encrypted_batch, true); + SetMinVersion(FEATURE_WALLETCRYPT, encrypted_batch); if (!encrypted_batch->TxnCommit()) { delete encrypted_batch; @@ -4120,31 +4100,30 @@ const CAddressBookData* CWallet::FindAddressBookEntry(const CTxDestination& dest bool CWallet::UpgradeWallet(int version, bilingual_str& error, std::vector& warnings) { int prev_version = GetVersion(); - int& nMaxVersion = version; - if (nMaxVersion == 0) // the -upgradewallet without argument case + if (version == 0) // the -upgradewallet without argument case { WalletLogPrintf("Performing wallet upgrade to %i\n", FEATURE_LATEST); - nMaxVersion = FEATURE_LATEST; - SetMinVersion(FEATURE_LATEST); // permanently upgrade the wallet immediately + version = FEATURE_LATEST; } else { - WalletLogPrintf("Allowing wallet upgrade up to %i\n", nMaxVersion); + WalletLogPrintf("Allowing wallet upgrade up to %i\n", version); } - if (nMaxVersion < GetVersion()) + if (version < prev_version) { error = _("Cannot downgrade wallet"); return false; } - SetMaxVersion(nMaxVersion); LOCK(cs_wallet); // Do not upgrade versions to any version between HD_SPLIT and FEATURE_PRE_SPLIT_KEYPOOL unless already supporting HD_SPLIT - int max_version = GetVersion(); - if (!CanSupportFeature(FEATURE_HD_SPLIT) && max_version >= FEATURE_HD_SPLIT && max_version < FEATURE_PRE_SPLIT_KEYPOOL) { + if (!CanSupportFeature(FEATURE_HD_SPLIT) && version >= FEATURE_HD_SPLIT && version < FEATURE_PRE_SPLIT_KEYPOOL) { error = _("Cannot upgrade a non HD split wallet without upgrading to support pre split keypool. Please use version 169900 or no version specified."); return false; } + // Permanently upgrade to the version + SetMinVersion(GetClosestWalletFeature(version)); + for (auto spk_man : GetActiveScriptPubKeyMans()) { if (!spk_man->Upgrade(prev_version, version, error)) { return false; diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index fcff0c557b..5b4597df4c 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -636,9 +636,6 @@ private: //! the current wallet version: clients below this version are not able to load the wallet int nWalletVersion GUARDED_BY(cs_wallet){FEATURE_BASE}; - //! the maximum wallet format version: memory-only variable that specifies to what version this wallet may be upgraded - int nWalletMaxVersion GUARDED_BY(cs_wallet) = FEATURE_BASE; - int64_t nNextResend = 0; bool fBroadcastTransactions = false; // Local time that the tip block was received. Used to schedule wallet rebroadcasts. @@ -800,8 +797,8 @@ public: const CWalletTx* GetWalletTx(const uint256& hash) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); bool IsTrusted(const CWalletTx& wtx, std::set& trusted_parents) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - //! check whether we are allowed to upgrade (or already support) to the named feature - bool CanSupportFeature(enum WalletFeature wf) const override EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) { AssertLockHeld(cs_wallet); return IsFeatureSupported(nWalletMaxVersion, wf); } + //! check whether we support the named feature + bool CanSupportFeature(enum WalletFeature wf) const override EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) { AssertLockHeld(cs_wallet); return IsFeatureSupported(nWalletVersion, wf); } /** * populate vCoins with vector of available COutputs. @@ -853,7 +850,7 @@ public: //! Upgrade stored CKeyMetadata objects to store key origin info as KeyOriginInfo void UpgradeKeyMetadata() EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - bool LoadMinVersion(int nVersion) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) { AssertLockHeld(cs_wallet); nWalletVersion = nVersion; nWalletMaxVersion = std::max(nWalletMaxVersion, nVersion); return true; } + bool LoadMinVersion(int nVersion) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) { AssertLockHeld(cs_wallet); nWalletVersion = nVersion; return true; } /** * Adds a destination data tuple to the store, and saves it to disk @@ -1076,11 +1073,8 @@ public: unsigned int GetKeyPoolSize() const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - //! signify that a particular wallet feature is now used. this may change nWalletVersion and nWalletMaxVersion if those are lower - void SetMinVersion(enum WalletFeature, WalletBatch* batch_in = nullptr, bool fExplicit = false) override; - - //! change which version we're allowed to upgrade to (note that this does not immediately imply upgrading to that format) - bool SetMaxVersion(int nVersion); + //! signify that a particular wallet feature is now used. + void SetMinVersion(enum WalletFeature, WalletBatch* batch_in = nullptr) override; //! get the current wallet format (the oldest client version guaranteed to understand this wallet) int GetVersion() const { LOCK(cs_wallet); return nWalletVersion; } -- cgit v1.2.3 From 0bd995aa19be65b0dd23df1df571c71428c2bc32 Mon Sep 17 00:00:00 2001 From: Andrew Chow Date: Wed, 29 Apr 2020 17:11:19 -0400 Subject: wallet: upgrade the CHDChain version number when upgrading to split hd --- src/wallet/scriptpubkeyman.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/wallet/scriptpubkeyman.cpp b/src/wallet/scriptpubkeyman.cpp index 149e8a4c9b..d2e1be6402 100644 --- a/src/wallet/scriptpubkeyman.cpp +++ b/src/wallet/scriptpubkeyman.cpp @@ -457,6 +457,13 @@ bool LegacyScriptPubKeyMan::Upgrade(int prev_version, int new_version, bilingual WalletLogPrintf("Upgrading wallet to use HD chain split\n"); m_storage.SetMinVersion(FEATURE_PRE_SPLIT_KEYPOOL); split_upgrade = FEATURE_HD_SPLIT > prev_version; + // Upgrade the HDChain + if (m_hd_chain.nVersion < CHDChain::VERSION_HD_CHAIN_SPLIT) { + m_hd_chain.nVersion = CHDChain::VERSION_HD_CHAIN_SPLIT; + if (!WalletBatch(m_storage.GetDatabase()).WriteHDChain(m_hd_chain)) { + throw std::runtime_error(std::string(__func__) + ": writing chain failed"); + } + } } // Mark all keys currently in the keypool as pre-split if (split_upgrade) { -- cgit v1.2.3 From 092fc434854f881330771a93a1280ac67b1d3549 Mon Sep 17 00:00:00 2001 From: Andrew Chow Date: Thu, 30 Apr 2020 01:05:55 -0400 Subject: tests: Add a sha256sum_file function to util --- test/functional/test_framework/util.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index 7688febae7..0f4713d50e 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -8,6 +8,7 @@ from base64 import b64encode from binascii import unhexlify from decimal import Decimal, ROUND_DOWN from subprocess import CalledProcessError +import hashlib import inspect import json import logging @@ -260,6 +261,14 @@ def wait_until_helper(predicate, *, attempts=float('inf'), timeout=float('inf'), raise AssertionError("Predicate {} not true after {} seconds".format(predicate_source, timeout)) raise RuntimeError('Unreachable') +def sha256sum_file(filename): + h = hashlib.sha256() + with open(filename, 'rb') as f: + d = f.read(4096) + while len(d) > 0: + h.update(d) + d = f.read(4096) + return h.digest() # RPC/P2P connection constants and functions ############################################ -- cgit v1.2.3 From 4b418a9decc3e855ee4b0bbf9e61121c8e9904e5 Mon Sep 17 00:00:00 2001 From: Andrew Chow Date: Thu, 30 Apr 2020 01:06:22 -0400 Subject: test: Add test_framework/bdb.py module for inspecting bdb files For upgrade tests and possibly other tests, it is useful to inspect the bdb file for the wallet (i.e. the wallet.dat file). test_framework/bdb.py is an implementation of bdb file deserialization specific for Bitcoin Core's usage. --- test/functional/test_framework/bdb.py | 152 ++++++++++++++++++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100644 test/functional/test_framework/bdb.py diff --git a/test/functional/test_framework/bdb.py b/test/functional/test_framework/bdb.py new file mode 100644 index 0000000000..9de358aa0a --- /dev/null +++ b/test/functional/test_framework/bdb.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +""" +Utilities for working directly with the wallet's BDB database file + +This is specific to the configuration of BDB used in this project: + - pagesize: 4096 bytes + - Outer database contains single subdatabase named 'main' + - btree + - btree leaf pages + +Each key-value pair is two entries in a btree leaf. The first is the key, the one that follows +is the value. And so on. Note that the entry data is itself not in the correct order. Instead +entry offsets are stored in the correct order and those offsets are needed to then retrieve +the data itself. + +Page format can be found in BDB source code dbinc/db_page.h +This only implements the deserialization of btree metadata pages and normal btree pages. Overflow +pages are not implemented but may be needed in the future if dealing with wallets with large +transactions. + +`db_dump -da wallet.dat` is useful to see the data in a wallet.dat BDB file +""" + +import binascii +import struct + +# Important constants +PAGESIZE = 4096 +OUTER_META_PAGE = 0 +INNER_META_PAGE = 2 + +# Page type values +BTREE_INTERNAL = 3 +BTREE_LEAF = 5 +BTREE_META = 9 + +# Some magic numbers for sanity checking +BTREE_MAGIC = 0x053162 +DB_VERSION = 9 + +# Deserializes a leaf page into a dict. +# Btree internal pages have the same header, for those, return None. +# For the btree leaf pages, deserialize them and put all the data into a dict +def dump_leaf_page(data): + page_info = {} + page_header = data[0:26] + _, pgno, prev_pgno, next_pgno, entries, hf_offset, level, pg_type = struct.unpack('QIIIHHBB', page_header) + page_info['pgno'] = pgno + page_info['prev_pgno'] = prev_pgno + page_info['next_pgno'] = next_pgno + page_info['entries'] = entries + page_info['hf_offset'] = hf_offset + page_info['level'] = level + page_info['pg_type'] = pg_type + page_info['entry_offsets'] = struct.unpack('{}H'.format(entries), data[26:26 + entries * 2]) + page_info['entries'] = [] + + if pg_type == BTREE_INTERNAL: + # Skip internal pages. These are the internal nodes of the btree and don't contain anything relevant to us + return None + + assert pg_type == BTREE_LEAF, 'A non-btree leaf page has been encountered while dumping leaves' + + for i in range(0, entries): + offset = page_info['entry_offsets'][i] + entry = {'offset': offset} + page_data_header = data[offset:offset + 3] + e_len, pg_type = struct.unpack('HB', page_data_header) + entry['len'] = e_len + entry['pg_type'] = pg_type + entry['data'] = data[offset + 3:offset + 3 + e_len] + page_info['entries'].append(entry) + + return page_info + +# Deserializes a btree metadata page into a dict. +# Does a simple sanity check on the magic value, type, and version +def dump_meta_page(page): + # metadata page + # general metadata + metadata = {} + meta_page = page[0:72] + _, pgno, magic, version, pagesize, encrypt_alg, pg_type, metaflags, _, free, last_pgno, nparts, key_count, record_count, flags, uid = struct.unpack('QIIIIBBBBIIIIII20s', meta_page) + metadata['pgno'] = pgno + metadata['magic'] = magic + metadata['version'] = version + metadata['pagesize'] = pagesize + metadata['encrypt_alg'] = encrypt_alg + metadata['pg_type'] = pg_type + metadata['metaflags'] = metaflags + metadata['free'] = free + metadata['last_pgno'] = last_pgno + metadata['nparts'] = nparts + metadata['key_count'] = key_count + metadata['record_count'] = record_count + metadata['flags'] = flags + metadata['uid'] = binascii.hexlify(uid) + + assert magic == BTREE_MAGIC, 'bdb magic does not match bdb btree magic' + assert pg_type == BTREE_META, 'Metadata page is not a btree metadata page' + assert version == DB_VERSION, 'Database too new' + + # btree metadata + btree_meta_page = page[72:512] + _, minkey, re_len, re_pad, root, _, crypto_magic, _, iv, chksum = struct.unpack('IIIII368sI12s16s20s', btree_meta_page) + metadata['minkey'] = minkey + metadata['re_len'] = re_len + metadata['re_pad'] = re_pad + metadata['root'] = root + metadata['crypto_magic'] = crypto_magic + metadata['iv'] = binascii.hexlify(iv) + metadata['chksum'] = binascii.hexlify(chksum) + return metadata + +# Given the dict from dump_leaf_page, get the key-value pairs and put them into a dict +def extract_kv_pairs(page_data): + out = {} + last_key = None + for i, entry in enumerate(page_data['entries']): + # By virtue of these all being pairs, even number entries are keys, and odd are values + if i % 2 == 0: + out[entry['data']] = b'' + last_key = entry['data'] + else: + out[last_key] = entry['data'] + return out + +# Extract the key-value pairs of the BDB file given in filename +def dump_bdb_kv(filename): + # Read in the BDB file and start deserializing it + pages = [] + with open(filename, 'rb') as f: + data = f.read(PAGESIZE) + while len(data) > 0: + pages.append(data) + data = f.read(PAGESIZE) + + # Sanity check the meta pages + dump_meta_page(pages[OUTER_META_PAGE]) + dump_meta_page(pages[INNER_META_PAGE]) + + # Fetch the kv pairs from the leaf pages + kv = {} + for i in range(3, len(pages)): + info = dump_leaf_page(pages[i]) + if info is not None: + info_kv = extract_kv_pairs(info) + kv = {**kv, **info_kv} + return kv -- cgit v1.2.3 From bf7635963c03203e7189ddaa56c6b086a0108cbf Mon Sep 17 00:00:00 2001 From: Andrew Chow Date: Thu, 30 Apr 2020 16:52:52 -0400 Subject: tests: Test specific upgradewallet scenarios and that upgrades work --- test/functional/wallet_upgradewallet.py | 257 ++++++++++++++++++++++++++++---- 1 file changed, 231 insertions(+), 26 deletions(-) diff --git a/test/functional/wallet_upgradewallet.py b/test/functional/wallet_upgradewallet.py index 446a601aee..a860805185 100755 --- a/test/functional/wallet_upgradewallet.py +++ b/test/functional/wallet_upgradewallet.py @@ -13,23 +13,47 @@ Only v0.15.2 and v0.16.3 are required by this test. The others are used in featu import os import shutil +import struct +from io import BytesIO + +from test_framework.bdb import dump_bdb_kv +from test_framework.messages import deser_compact_size, deser_string from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, assert_greater_than, assert_is_hex_string, + assert_raises_rpc_error, + sha256sum_file, ) +UPGRADED_KEYMETA_VERSION = 12 + +def deser_keymeta(f): + ver, create_time = struct.unpack(' Date: Tue, 21 Apr 2020 18:57:51 -0400 Subject: test: Remove unused wallet.dat --- .../functional/data/wallets/high_minversion/.walletlock | 0 .../functional/data/wallets/high_minversion/GENERATE.md | 8 -------- test/functional/data/wallets/high_minversion/db.log | 0 test/functional/data/wallets/high_minversion/wallet.dat | Bin 16384 -> 0 bytes 4 files changed, 8 deletions(-) delete mode 100644 test/functional/data/wallets/high_minversion/.walletlock delete mode 100644 test/functional/data/wallets/high_minversion/GENERATE.md delete mode 100644 test/functional/data/wallets/high_minversion/db.log delete mode 100644 test/functional/data/wallets/high_minversion/wallet.dat diff --git a/test/functional/data/wallets/high_minversion/.walletlock b/test/functional/data/wallets/high_minversion/.walletlock deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/functional/data/wallets/high_minversion/GENERATE.md b/test/functional/data/wallets/high_minversion/GENERATE.md deleted file mode 100644 index e55c4557ca..0000000000 --- a/test/functional/data/wallets/high_minversion/GENERATE.md +++ /dev/null @@ -1,8 +0,0 @@ -The wallet has been created by starting Bitcoin Core with the options -`-regtest -datadir=/tmp -nowallet -walletdir=$(pwd)/test/functional/data/wallets/`. - -In the source code, `WalletFeature::FEATURE_LATEST` has been modified to be large, so that the minversion is too high -for a current build of the wallet. - -The wallet has then been created with the RPC `createwallet high_minversion true true`, so that a blank wallet with -private keys disabled is created. diff --git a/test/functional/data/wallets/high_minversion/db.log b/test/functional/data/wallets/high_minversion/db.log deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/functional/data/wallets/high_minversion/wallet.dat b/test/functional/data/wallets/high_minversion/wallet.dat deleted file mode 100644 index 99ab809263..0000000000 Binary files a/test/functional/data/wallets/high_minversion/wallet.dat and /dev/null differ -- cgit v1.2.3 From 5f9c0b6360215636cfa62a70d3a70f1feb3977ab Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Tue, 21 Apr 2020 18:55:20 -0400 Subject: wallet: Remove -upgradewallet from dummywallet --- src/dummywallet.cpp | 1 - src/wallet/wallet.cpp | 3 +-- test/functional/wallet_multiwallet.py | 3 +++ 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/dummywallet.cpp b/src/dummywallet.cpp index 8d2dcd0279..4543f098a1 100644 --- a/src/dummywallet.cpp +++ b/src/dummywallet.cpp @@ -40,7 +40,6 @@ void DummyWalletInit::AddWalletOptions(ArgsManager& argsman) const "-salvagewallet", "-spendzeroconfchange", "-txconfirmtarget=", - "-upgradewallet", "-wallet=", "-walletbroadcast", "-walletdir=", diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index be7b30009c..11ce0d0c05 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -4100,8 +4100,7 @@ const CAddressBookData* CWallet::FindAddressBookEntry(const CTxDestination& dest bool CWallet::UpgradeWallet(int version, bilingual_str& error, std::vector& warnings) { int prev_version = GetVersion(); - if (version == 0) // the -upgradewallet without argument case - { + if (version == 0) { WalletLogPrintf("Performing wallet upgrade to %i\n", FEATURE_LATEST); version = FEATURE_LATEST; } else { diff --git a/test/functional/wallet_multiwallet.py b/test/functional/wallet_multiwallet.py index 61791a756c..ab3edbd04c 100755 --- a/test/functional/wallet_multiwallet.py +++ b/test/functional/wallet_multiwallet.py @@ -139,6 +139,9 @@ class MultiWalletTest(BitcoinTestFramework): open(not_a_dir, 'a', encoding="utf8").close() self.nodes[0].assert_start_raises_init_error(['-walletdir=' + not_a_dir], 'Error: Specified -walletdir "' + not_a_dir + '" is not a directory') + self.log.info("Do not allow -upgradewallet with multiwallet") + self.nodes[0].assert_start_raises_init_error(['-upgradewallet'], "Error: Error parsing command line arguments: Invalid parameter -upgradewallet") + # if wallets/ doesn't exist, datadir should be the default wallet dir wallet_dir2 = data_dir('walletdir') os.rename(wallet_dir(), wallet_dir2) -- cgit v1.2.3